@kuralle-syrinx/realtime 4.1.0 → 4.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,1147 +0,0 @@
1
- // SPDX-License-Identifier: MIT
2
-
3
- import { afterEach, describe, expect, it } from "vitest";
4
-
5
- import {
6
- PipelineBusImpl,
7
- Route,
8
- VoiceAgentSession,
9
- type DelegateQueryPacket,
10
- type DelegateResultPacket,
11
- type EndOfSpeechPacket,
12
- type InterruptTtsPacket,
13
- type InterruptionDetectedPacket,
14
- type LlmDeltaPacket,
15
- type LlmResponseDonePacket,
16
- type LlmErrorPacket,
17
- type LlmToolResultPacket,
18
- type Reasoner,
19
- type ReasonerMessage,
20
- type RecordAssistantAudioPacket,
21
- type SttResultPacket,
22
- type TextToSpeechAudioPacket,
23
- type TextToSpeechEndPacket,
24
- type TurnChangePacket,
25
- type VoicePacket,
26
- } from "@kuralle-syrinx/core";
27
-
28
- import type { RealtimeAdapter, RealtimeEvent } from "./realtime-adapter.js";
29
- import { RealtimeBridge } from "./realtime-bridge.js";
30
-
31
- class FakeRealtimeAdapter implements RealtimeAdapter {
32
- readonly caps: RealtimeAdapter["caps"];
33
-
34
- constructor(caps?: Partial<RealtimeAdapter["caps"]>) {
35
- this.caps = {
36
- inputSampleRateHz: 24_000,
37
- outputSampleRateHz: 24_000,
38
- supportsConcurrentToolAudio: true,
39
- supportsTruncate: true,
40
- emitsServerSpeechStarted: true,
41
- ...caps,
42
- };
43
- }
44
-
45
- private readonly queued: RealtimeEvent[] = [];
46
- private readonly waiters: Array<(event: RealtimeEvent | null) => void> = [];
47
- private closed = false;
48
- readonly sentAudio: Uint8Array[] = [];
49
-
50
- readonly events: AsyncIterable<RealtimeEvent> = {
51
- [Symbol.asyncIterator]: () => ({
52
- next: async (): Promise<IteratorResult<RealtimeEvent>> => {
53
- if (this.queued.length > 0) {
54
- return { value: this.queued.shift()!, done: false };
55
- }
56
- if (this.closed) return { value: undefined, done: true };
57
- const event = await new Promise<RealtimeEvent | null>((resolve) => {
58
- this.waiters.push(resolve);
59
- });
60
- if (event === null) return { value: undefined, done: true };
61
- return { value: event, done: false };
62
- },
63
- }),
64
- };
65
-
66
- async open(_signal: AbortSignal): Promise<void> {}
67
-
68
- sendAudio(pcm16: Uint8Array): void {
69
- this.sentAudio.push(pcm16);
70
- }
71
-
72
- readonly sentText: string[] = [];
73
-
74
- sendText(text: string): void {
75
- this.sentText.push(text);
76
- }
77
-
78
- readonly cancelCalls: number[] = [];
79
-
80
- cancelResponse(audioEndMs: number): void {
81
- this.cancelCalls.push(audioEndMs);
82
- }
83
-
84
- readonly injectedToolResults: Array<{ toolId: string; text: string }> = [];
85
-
86
- injectToolResult(toolId: string, text: string): void {
87
- this.injectedToolResults.push({ toolId, text });
88
- }
89
-
90
- async close(): Promise<void> {
91
- this.closed = true;
92
- for (const resolve of this.waiters.splice(0)) resolve(null);
93
- }
94
-
95
- emit(event: RealtimeEvent): void {
96
- const waiter = this.waiters.shift();
97
- if (waiter) waiter(event);
98
- else this.queued.push(event);
99
- }
100
-
101
- }
102
-
103
- function pcmFromSamples(samples: readonly number[]): Uint8Array {
104
- const pcm = Int16Array.from(samples);
105
- return new Uint8Array(pcm.buffer, pcm.byteOffset, pcm.byteLength);
106
- }
107
-
108
- function frameSizedPcm24k(): Uint8Array {
109
- return pcmFromSamples(Array.from({ length: 960 }, (_, i) => i));
110
- }
111
-
112
- function frameDurationMs(frame: TextToSpeechAudioPacket): number {
113
- return (frame.audio.byteLength / 2 / frame.sampleRateHz) * 1000;
114
- }
115
-
116
- async function waitForCondition(predicate: () => boolean, timeoutMs = 2000): Promise<void> {
117
- const startedAt = Date.now();
118
- while (Date.now() - startedAt < timeoutMs) {
119
- if (predicate()) return;
120
- await new Promise((resolve) => setTimeout(resolve, 5));
121
- }
122
- throw new Error("Timed out waiting for condition");
123
- }
124
-
125
- describe("RealtimeBridge", () => {
126
- const buses: PipelineBusImpl[] = [];
127
-
128
- afterEach(() => {
129
- for (const bus of buses.splice(0)) bus.stop();
130
- });
131
-
132
- it("maps one turn to turn.change → tts.audio → eos.turn_complete → tts.end with one contextId", async () => {
133
- const adapter = new FakeRealtimeAdapter();
134
- const bridge = new RealtimeBridge(adapter);
135
- const bus = new PipelineBusImpl();
136
- buses.push(bus);
137
- const packets: VoicePacket[] = [];
138
- bus.on("turn.change", (pkt) => { packets.push(pkt as TurnChangePacket); });
139
- bus.on("tts.audio", (pkt) => { packets.push(pkt as TextToSpeechAudioPacket); });
140
- bus.on("eos.turn_complete", (pkt) => { packets.push(pkt as EndOfSpeechPacket); });
141
- bus.on("tts.end", (pkt) => { packets.push(pkt as TextToSpeechEndPacket); });
142
- bus.on("stt.result", (pkt) => { packets.push(pkt as SttResultPacket); });
143
-
144
- const started = bus.start();
145
- await bridge.initialize(bus, {});
146
-
147
- bus.push(Route.Main, {
148
- kind: "user.audio_received",
149
- contextId: "transport-turn",
150
- timestampMs: Date.now(),
151
- audio: pcmFromSamples([100, 200, 300, 400]),
152
- });
153
-
154
- adapter.emit({ type: "response_started" });
155
- adapter.emit({
156
- type: "audio",
157
- pcm16: pcmFromSamples(Array.from({ length: 960 }, (_, i) => i)),
158
- sampleRateHz: 24_000,
159
- });
160
- adapter.emit({ type: "transcript", role: "user", text: "hello there", final: true });
161
- adapter.emit({ type: "response_done" });
162
-
163
- await waitForCondition(() => packets.some((p) => p.kind === "tts.end"));
164
-
165
- const turnChanges = packets.filter((p): p is TurnChangePacket => p.kind === "turn.change");
166
- const audio = packets.filter((p): p is TextToSpeechAudioPacket => p.kind === "tts.audio");
167
- const turnComplete = packets.filter((p): p is EndOfSpeechPacket => p.kind === "eos.turn_complete");
168
- const ends = packets.filter((p): p is TextToSpeechEndPacket => p.kind === "tts.end");
169
-
170
- expect(turnChanges).toHaveLength(1);
171
- expect(audio.length).toBeGreaterThan(0);
172
- expect(turnComplete).toHaveLength(1);
173
- expect(ends).toHaveLength(1);
174
-
175
- const contextId = turnChanges[0]!.contextId;
176
- expect(contextId.length).toBeGreaterThan(0);
177
- expect(audio.every((frame) => frame.contextId === contextId)).toBe(true);
178
- expect(turnComplete[0]!.contextId).toBe(contextId);
179
- expect(ends[0]!.contextId).toBe(contextId);
180
- expect(audio.every((frame) => frame.sampleRateHz === 16_000)).toBe(true);
181
- expect(audio.every((frame) => frameDurationMs(frame) <= 20)).toBe(true);
182
- expect(adapter.sentAudio.length).toBeGreaterThan(0);
183
-
184
- await bridge.close();
185
- bus.stop();
186
- await started;
187
- });
188
-
189
- it("forwards a user.text_received turn to adapter.sendText (typed input), ignoring blank text", async () => {
190
- const adapter = new FakeRealtimeAdapter();
191
- const bridge = new RealtimeBridge(adapter);
192
- const bus = new PipelineBusImpl();
193
- buses.push(bus);
194
-
195
- const started = bus.start();
196
- await bridge.initialize(bus, {});
197
-
198
- bus.push(Route.Main, {
199
- kind: "user.text_received",
200
- contextId: "transport-turn",
201
- timestampMs: Date.now(),
202
- text: "when is the late-add deadline?",
203
- });
204
- bus.push(Route.Main, {
205
- kind: "user.text_received",
206
- contextId: "transport-turn",
207
- timestampMs: Date.now(),
208
- text: " ",
209
- });
210
-
211
- await waitForCondition(() => adapter.sentText.length > 0);
212
- expect(adapter.sentText).toEqual(["when is the late-add deadline?"]);
213
-
214
- await bridge.close();
215
- bus.stop();
216
- await started;
217
- });
218
-
219
- it("mints a fresh contextId for each response_started (R1)", async () => {
220
- const adapter = new FakeRealtimeAdapter();
221
- const bridge = new RealtimeBridge(adapter);
222
- const bus = new PipelineBusImpl();
223
- buses.push(bus);
224
- const turnChanges: TurnChangePacket[] = [];
225
- bus.on("turn.change", (pkt) => { turnChanges.push(pkt as TurnChangePacket); });
226
-
227
- const started = bus.start();
228
- await bridge.initialize(bus, {});
229
-
230
- adapter.emit({ type: "response_started" });
231
- adapter.emit({ type: "response_done" });
232
- adapter.emit({ type: "response_started" });
233
- adapter.emit({ type: "response_done" });
234
-
235
- await waitForCondition(() => turnChanges.length >= 2);
236
-
237
- expect(turnChanges[0]!.contextId).not.toBe(turnChanges[1]!.contextId);
238
- expect(turnChanges[1]!.previousContextId).toBe(turnChanges[0]!.contextId);
239
-
240
- await bridge.close();
241
- bus.stop();
242
- await started;
243
- });
244
-
245
- it("R-04: mints contextId via globalThis.crypto.randomUUID without node:crypto", async () => {
246
- const adapter = new FakeRealtimeAdapter();
247
- const bridge = new RealtimeBridge(adapter);
248
- const bus = new PipelineBusImpl();
249
- buses.push(bus);
250
- const turnChanges: TurnChangePacket[] = [];
251
- bus.on("turn.change", (pkt) => { turnChanges.push(pkt as TurnChangePacket); });
252
-
253
- const started = bus.start();
254
- await bridge.initialize(bus, {});
255
-
256
- adapter.emit({ type: "response_started" });
257
- await waitForCondition(() => turnChanges.length >= 1);
258
- expect(turnChanges[0]!.contextId).toMatch(
259
- /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/,
260
- );
261
-
262
- await bridge.close();
263
- bus.stop();
264
- await started;
265
- });
266
-
267
- it("runDelegate injects finish.text when the Reasoner yields no deltas (R-08)", async () => {
268
- const adapter = new FakeRealtimeAdapter();
269
- const answerText = "Submit the Late Add Petition via the Student Relations portal.";
270
- const reasoner: Reasoner = {
271
- stream: () => (async function* () {
272
- yield { type: "finish", reason: "stop", text: answerText };
273
- })(),
274
- };
275
- const bridge = new RealtimeBridge(adapter, reasoner);
276
- const bus = new PipelineBusImpl();
277
- buses.push(bus);
278
- const toolResults: LlmToolResultPacket[] = [];
279
- bus.on("llm.tool_result", (pkt) => { toolResults.push(pkt as LlmToolResultPacket); });
280
-
281
- const started = bus.start();
282
- await bridge.initialize(bus, {});
283
-
284
- adapter.emit({ type: "response_started" });
285
- adapter.emit({
286
- type: "tool_call",
287
- toolId: "call_delegate_1",
288
- toolName: "consult_knowledge",
289
- args: { query: "Can I still add Biology 101?" },
290
- });
291
-
292
- await waitForCondition(() => adapter.injectedToolResults.length === 1);
293
-
294
- // The bus packet keeps the raw answer; the front model receives the G1 envelope (default).
295
- expect(toolResults).toHaveLength(1);
296
- expect(toolResults[0]!.result).toBe(answerText);
297
- expect(adapter.injectedToolResults[0]!.toolId).toBe("call_delegate_1");
298
- expect(JSON.parse(adapter.injectedToolResults[0]!.text)).toEqual({
299
- response_text: answerText,
300
- require_repeat_verbatim: true,
301
- });
302
-
303
- await bridge.close();
304
- bus.stop();
305
- await started;
306
- });
307
-
308
- it("R-08: finish reason length uses accumulated text", async () => {
309
- const adapter = new FakeRealtimeAdapter();
310
- const reasoner: Reasoner = {
311
- stream: () => (async function* () {
312
- yield { type: "text-delta", text: "Partial " };
313
- yield { type: "finish", reason: "length", text: "Partial answer truncated" };
314
- })(),
315
- };
316
- const bridge = new RealtimeBridge(adapter, reasoner);
317
- const bus = new PipelineBusImpl();
318
- buses.push(bus);
319
-
320
- const started = bus.start();
321
- await bridge.initialize(bus, {});
322
-
323
- adapter.emit({ type: "response_started" });
324
- adapter.emit({
325
- type: "tool_call",
326
- toolId: "call_len",
327
- toolName: "consult_knowledge",
328
- args: { query: "long query" },
329
- });
330
-
331
- await waitForCondition(() => adapter.injectedToolResults.length === 1);
332
- expect(JSON.parse(adapter.injectedToolResults[0]!.text)).toMatchObject({ response_text: "Partial " });
333
-
334
- await bridge.close();
335
- bus.stop();
336
- await started;
337
- });
338
-
339
- it("G1/WBS-2: envelope carries the render directive when configured", async () => {
340
- const adapter = new FakeRealtimeAdapter();
341
- const reasoner: Reasoner = {
342
- stream: () => (async function* () {
343
- yield { type: "finish", reason: "stop", text: "The fee is 5000 rupees." };
344
- })(),
345
- };
346
- const bridge = new RealtimeBridge(adapter, reasoner, "consult_knowledge", {
347
- renderDirective: "translate_faithfully",
348
- });
349
- const bus = new PipelineBusImpl();
350
- buses.push(bus);
351
-
352
- const started = bus.start();
353
- await bridge.initialize(bus, {});
354
-
355
- adapter.emit({ type: "response_started" });
356
- adapter.emit({
357
- type: "tool_call",
358
- toolId: "call_env_1",
359
- toolName: "consult_knowledge",
360
- args: { query: "fees" },
361
- });
362
-
363
- await waitForCondition(() => adapter.injectedToolResults.length === 1);
364
- expect(JSON.parse(adapter.injectedToolResults[0]!.text)).toEqual({
365
- response_text: "The fee is 5000 rupees.",
366
- require_repeat_verbatim: true,
367
- render: "translate_faithfully",
368
- });
369
-
370
- await bridge.close();
371
- bus.stop();
372
- await started;
373
- });
374
-
375
- it('G1/WBS-2: toolResultFormat "string" opts out to the raw answer (pre-RFC behavior)', async () => {
376
- const adapter = new FakeRealtimeAdapter();
377
- const reasoner: Reasoner = {
378
- stream: () => (async function* () {
379
- yield { type: "finish", reason: "stop", text: "raw answer" };
380
- })(),
381
- };
382
- const bridge = new RealtimeBridge(adapter, reasoner, "consult_knowledge", {
383
- toolResultFormat: "string",
384
- });
385
- const bus = new PipelineBusImpl();
386
- buses.push(bus);
387
-
388
- const started = bus.start();
389
- await bridge.initialize(bus, {});
390
-
391
- adapter.emit({ type: "response_started" });
392
- adapter.emit({
393
- type: "tool_call",
394
- toolId: "call_env_2",
395
- toolName: "consult_knowledge",
396
- args: { query: "q" },
397
- });
398
-
399
- await waitForCondition(() => adapter.injectedToolResults.length === 1);
400
- expect(adapter.injectedToolResults[0]!.text).toBe("raw answer");
401
-
402
- await bridge.close();
403
- bus.stop();
404
- await started;
405
- });
406
-
407
- it("G2/WBS-1: delegate turn emits delegate.query then delegate.result on the Background route", async () => {
408
- const adapter = new FakeRealtimeAdapter();
409
- const reasoner: Reasoner = {
410
- stream: () => (async function* () {
411
- yield { type: "tool-call", toolId: "rag-1", toolName: "retrieve", args: { q: "deadline" } };
412
- yield { type: "tool-result", toolId: "rag-1", toolName: "retrieve", result: "chunk" };
413
- yield { type: "text-delta", text: "The deadline is March 31." };
414
- yield { type: "finish", reason: "stop", text: "The deadline is March 31." };
415
- })(),
416
- };
417
- const bridge = new RealtimeBridge(adapter, reasoner);
418
- const background: Array<{ route: Route; packet: VoicePacket }> = [];
419
- const bus = new PipelineBusImpl({
420
- onPacket: (route, packet) => {
421
- if (packet.kind.startsWith("delegate.")) background.push({ route, packet });
422
- },
423
- });
424
- buses.push(bus);
425
-
426
- const started = bus.start();
427
- await bridge.initialize(bus, {});
428
-
429
- adapter.emit({ type: "response_started" });
430
- adapter.emit({
431
- type: "tool_call",
432
- toolId: "call_obs_1",
433
- toolName: "consult_knowledge",
434
- args: { query: "When is the exam registration deadline?" },
435
- });
436
-
437
- await waitForCondition(() => adapter.injectedToolResults.length === 1);
438
- await waitForCondition(() => background.length === 2);
439
-
440
- expect(background.map((entry) => entry.route)).toEqual([Route.Background, Route.Background]);
441
- const [query, result] = background.map((entry) => entry.packet);
442
- expect(query).toMatchObject({
443
- kind: "delegate.query",
444
- query: "When is the exam registration deadline?",
445
- toolId: "call_obs_1",
446
- toolName: "consult_knowledge",
447
- });
448
- expect(query!.contextId).toBeTruthy();
449
- expect(result).toMatchObject({
450
- kind: "delegate.result",
451
- query: "When is the exam registration deadline?",
452
- answer: "The deadline is March 31.",
453
- grounded: true,
454
- toolId: "call_obs_1",
455
- toolName: "consult_knowledge",
456
- contextId: query!.contextId,
457
- });
458
- expect((result as DelegateResultPacket).durationMs).toBeGreaterThanOrEqual(0);
459
-
460
- await bridge.close();
461
- bus.stop();
462
- await started;
463
- });
464
-
465
- it("G2/WBS-1: delegate.result grounded=false when the reasoner used no tool", async () => {
466
- const adapter = new FakeRealtimeAdapter();
467
- const reasoner: Reasoner = {
468
- stream: () => (async function* () {
469
- yield { type: "text-delta", text: "From memory." };
470
- yield { type: "finish", reason: "stop", text: "From memory." };
471
- })(),
472
- };
473
- const bridge = new RealtimeBridge(adapter, reasoner);
474
- const results: DelegateResultPacket[] = [];
475
- const bus = new PipelineBusImpl();
476
- buses.push(bus);
477
- bus.on("delegate.result", (pkt) => { results.push(pkt as DelegateResultPacket); });
478
-
479
- const started = bus.start();
480
- await bridge.initialize(bus, {});
481
-
482
- adapter.emit({ type: "response_started" });
483
- adapter.emit({
484
- type: "tool_call",
485
- toolId: "call_obs_2",
486
- toolName: "consult_knowledge",
487
- args: { query: "ungrounded" },
488
- });
489
-
490
- await waitForCondition(() => results.length === 1);
491
- expect(results[0]!.grounded).toBe(false);
492
-
493
- await bridge.close();
494
- bus.stop();
495
- await started;
496
- });
497
-
498
- it("G2/WBS-1: failed delegate emits delegate.query but no delegate.result", async () => {
499
- const adapter = new FakeRealtimeAdapter();
500
- const reasoner: Reasoner = {
501
- stream: () => (async function* () {
502
- yield { type: "error", cause: new Error("brain down"), recoverable: false };
503
- })(),
504
- };
505
- const bridge = new RealtimeBridge(adapter, reasoner);
506
- const queries: DelegateQueryPacket[] = [];
507
- const results: DelegateResultPacket[] = [];
508
- const errors: LlmErrorPacket[] = [];
509
- const bus = new PipelineBusImpl();
510
- buses.push(bus);
511
- bus.on("delegate.query", (pkt) => { queries.push(pkt as DelegateQueryPacket); });
512
- bus.on("delegate.result", (pkt) => { results.push(pkt as DelegateResultPacket); });
513
- bus.on("llm.error", (pkt) => { errors.push(pkt as LlmErrorPacket); });
514
-
515
- const started = bus.start();
516
- await bridge.initialize(bus, {});
517
-
518
- adapter.emit({ type: "response_started" });
519
- adapter.emit({
520
- type: "tool_call",
521
- toolId: "call_obs_3",
522
- toolName: "consult_knowledge",
523
- args: { query: "will fail" },
524
- });
525
-
526
- await waitForCondition(() => errors.length === 1);
527
- expect(queries).toHaveLength(1);
528
- expect(results).toHaveLength(0);
529
-
530
- await bridge.close();
531
- bus.stop();
532
- await started;
533
- });
534
-
535
- it("R-08: recoverable error surfaces llm.error without injecting empty output", async () => {
536
- const adapter = new FakeRealtimeAdapter();
537
- const reasoner: Reasoner = {
538
- stream: () => (async function* () {
539
- yield { type: "error", cause: new Error("rate limited"), recoverable: true };
540
- })(),
541
- };
542
- const bridge = new RealtimeBridge(adapter, reasoner);
543
- const bus = new PipelineBusImpl();
544
- buses.push(bus);
545
- const errors: LlmErrorPacket[] = [];
546
- bus.on("llm.error", (pkt) => { errors.push(pkt as LlmErrorPacket); });
547
-
548
- const started = bus.start();
549
- await bridge.initialize(bus, {});
550
-
551
- adapter.emit({ type: "response_started" });
552
- adapter.emit({
553
- type: "tool_call",
554
- toolId: "call_recover",
555
- toolName: "consult_knowledge",
556
- args: { query: "test" },
557
- });
558
-
559
- await waitForCondition(() => errors.length === 1);
560
- expect(errors[0]!.isRecoverable).toBe(true);
561
- expect(adapter.injectedToolResults).toHaveLength(0);
562
-
563
- await bridge.close();
564
- bus.stop();
565
- await started;
566
- });
567
-
568
- it("R-08: nonrecoverable error emits llm.error without injecting output", async () => {
569
- const adapter = new FakeRealtimeAdapter();
570
- const reasoner: Reasoner = {
571
- stream: () => (async function* () {
572
- yield { type: "error", cause: new Error("auth failed"), recoverable: false };
573
- })(),
574
- };
575
- const bridge = new RealtimeBridge(adapter, reasoner);
576
- const bus = new PipelineBusImpl();
577
- buses.push(bus);
578
- const errors: LlmErrorPacket[] = [];
579
- bus.on("llm.error", (pkt) => { errors.push(pkt as LlmErrorPacket); });
580
-
581
- const started = bus.start();
582
- await bridge.initialize(bus, {});
583
-
584
- adapter.emit({ type: "response_started" });
585
- adapter.emit({
586
- type: "tool_call",
587
- toolId: "call_fatal",
588
- toolName: "consult_knowledge",
589
- args: { query: "test" },
590
- });
591
-
592
- await waitForCondition(() => errors.length === 1);
593
- expect(errors[0]!.isRecoverable).toBe(false);
594
- expect(adapter.injectedToolResults).toHaveLength(0);
595
-
596
- await bridge.close();
597
- bus.stop();
598
- await started;
599
- });
600
-
601
- it("runDelegate rejects suspended Reasoner without injecting an answer (R-08)", async () => {
602
- const adapter = new FakeRealtimeAdapter();
603
- const reasoner: Reasoner = {
604
- stream: () => (async function* () {
605
- yield { type: "suspended", runId: "run-suspended-1", payload: { step: "approval" } };
606
- })(),
607
- };
608
- const bridge = new RealtimeBridge(adapter, reasoner);
609
- const bus = new PipelineBusImpl();
610
- buses.push(bus);
611
- const toolResults: LlmToolResultPacket[] = [];
612
- const errors: LlmErrorPacket[] = [];
613
- bus.on("llm.tool_result", (pkt) => { toolResults.push(pkt as LlmToolResultPacket); });
614
- bus.on("llm.error", (pkt) => { errors.push(pkt as LlmErrorPacket); });
615
-
616
- const started = bus.start();
617
- await bridge.initialize(bus, {});
618
-
619
- adapter.emit({ type: "response_started" });
620
- adapter.emit({
621
- type: "tool_call",
622
- toolId: "call_delegate_2",
623
- toolName: "consult_knowledge",
624
- args: { query: "Need advisor approval" },
625
- });
626
-
627
- await waitForCondition(() => errors.length === 1);
628
-
629
- expect(adapter.injectedToolResults).toHaveLength(0);
630
- expect(toolResults).toHaveLength(0);
631
-
632
- await bridge.close();
633
- bus.stop();
634
- await started;
635
- });
636
-
637
- it("R-07: mismatched tool_call name emits recoverable llm.error instead of silent ignore", async () => {
638
- const adapter = new FakeRealtimeAdapter();
639
- const reasoner: Reasoner = {
640
- stream: () => (async function* () {
641
- yield { type: "finish", reason: "stop", text: "never called" };
642
- })(),
643
- };
644
- const bridge = new RealtimeBridge(adapter, reasoner, "consult_knowledge");
645
- const bus = new PipelineBusImpl();
646
- buses.push(bus);
647
- const errors: LlmErrorPacket[] = [];
648
- bus.on("llm.error", (pkt) => { errors.push(pkt as LlmErrorPacket); });
649
-
650
- const started = bus.start();
651
- await bridge.initialize(bus, {});
652
-
653
- adapter.emit({ type: "response_started" });
654
- adapter.emit({
655
- type: "tool_call",
656
- toolId: "call_wrong",
657
- toolName: "wrong_tool",
658
- args: { query: "test" },
659
- });
660
-
661
- await waitForCondition(() => errors.length === 1);
662
- expect(errors[0]!.isRecoverable).toBe(true);
663
- expect(errors[0]!.cause.message).toContain("wrong_tool");
664
- expect(adapter.injectedToolResults).toHaveLength(0);
665
-
666
- await bridge.close();
667
- bus.stop();
668
- await started;
669
- });
670
-
671
- it("R-08: delegate tool_call missing the query argument emits recoverable error, never calls reasoner", async () => {
672
- const adapter = new FakeRealtimeAdapter();
673
- let reasonerCalled = false;
674
- const reasoner: Reasoner = {
675
- stream: () => {
676
- reasonerCalled = true;
677
- return (async function* () {
678
- yield { type: "finish", reason: "stop", text: "should not run" };
679
- })();
680
- },
681
- };
682
- const bridge = new RealtimeBridge(adapter, reasoner, "consult_knowledge");
683
- const bus = new PipelineBusImpl();
684
- buses.push(bus);
685
- const errors: LlmErrorPacket[] = [];
686
- bus.on("llm.error", (pkt) => { errors.push(pkt as LlmErrorPacket); });
687
-
688
- const started = bus.start();
689
- await bridge.initialize(bus, {});
690
-
691
- adapter.emit({ type: "response_started" });
692
- adapter.emit({
693
- type: "tool_call",
694
- toolId: "call_noarg",
695
- toolName: "consult_knowledge",
696
- args: { question: "wrong arg name" },
697
- });
698
-
699
- await waitForCondition(() => errors.length === 1);
700
- expect(errors[0]!.isRecoverable).toBe(true);
701
- expect(errors[0]!.cause.message).toContain("query");
702
- expect(reasonerCalled).toBe(false);
703
- expect(adapter.injectedToolResults).toHaveLength(0);
704
-
705
- await bridge.close();
706
- bus.stop();
707
- await started;
708
- });
709
-
710
- it("R-09: contextProvider messages are passed to reasoner.stream", async () => {
711
- const adapter = new FakeRealtimeAdapter();
712
- const prior: ReasonerMessage[] = [
713
- { role: "user", content: "prior question" },
714
- { role: "assistant", content: "prior answer" },
715
- ];
716
- let receivedMessages: readonly ReasonerMessage[] | undefined;
717
- const reasoner: Reasoner = {
718
- stream: (turn) => {
719
- receivedMessages = turn.messages;
720
- return (async function* () {
721
- yield { type: "finish", reason: "stop", text: "with context" };
722
- })();
723
- },
724
- };
725
- const bridge = new RealtimeBridge(adapter, reasoner, "consult_knowledge", {
726
- contextProvider: () => prior,
727
- });
728
- const bus = new PipelineBusImpl();
729
- buses.push(bus);
730
-
731
- const started = bus.start();
732
- await bridge.initialize(bus, {});
733
-
734
- adapter.emit({ type: "response_started" });
735
- adapter.emit({
736
- type: "tool_call",
737
- toolId: "call_ctx",
738
- toolName: "consult_knowledge",
739
- args: { query: "follow up" },
740
- });
741
-
742
- await waitForCondition(() => adapter.injectedToolResults.length === 1);
743
- expect(receivedMessages).toEqual(prior);
744
-
745
- await bridge.close();
746
- bus.stop();
747
- await started;
748
- });
749
-
750
- it("R-10: routes a non-delegate tool call to onFrontToolCall and injects its result (no error)", async () => {
751
- const adapter = new FakeRealtimeAdapter();
752
- const reasoner: Reasoner = {
753
- stream: () => (async function* () {
754
- yield { type: "finish", reason: "stop", text: "delegate not called" };
755
- })(),
756
- };
757
- const frontCalls: Array<{ toolName: string; args: Record<string, unknown> }> = [];
758
- const bridge = new RealtimeBridge(adapter, reasoner, "consult_knowledge", {
759
- onFrontToolCall: (c) => {
760
- frontCalls.push({ toolName: c.toolName, args: c.args });
761
- return "acknowledged";
762
- },
763
- });
764
- const bus = new PipelineBusImpl();
765
- buses.push(bus);
766
- const errors: LlmErrorPacket[] = [];
767
- bus.on("llm.error", (pkt) => { errors.push(pkt as LlmErrorPacket); });
768
-
769
- const started = bus.start();
770
- await bridge.initialize(bus, {});
771
-
772
- adapter.emit({ type: "response_started" });
773
- adapter.emit({ type: "tool_call", toolId: "call_front", toolName: "wait_for_user", args: { reason: "hold music" } });
774
-
775
- await waitForCondition(() => adapter.injectedToolResults.length === 1);
776
- expect(frontCalls).toEqual([{ toolName: "wait_for_user", args: { reason: "hold music" } }]);
777
- expect(adapter.injectedToolResults[0]).toEqual({ toolId: "call_front", text: "acknowledged" });
778
- expect(errors).toEqual([]); // front tools must NOT abort the turn
779
-
780
- await bridge.close();
781
- bus.stop();
782
- await started;
783
- });
784
-
785
- it("R-11: forwards the full tool-call args to the Reasoner, not just the query", async () => {
786
- const adapter = new FakeRealtimeAdapter();
787
- let receivedArgs: Record<string, unknown> | undefined;
788
- const reasoner: Reasoner = {
789
- stream: (turn) => {
790
- receivedArgs = turn.toolArgs;
791
- return (async function* () {
792
- yield { type: "finish", reason: "stop", text: "ok" };
793
- })();
794
- },
795
- };
796
- const bridge = new RealtimeBridge(adapter, reasoner, "consult_knowledge");
797
- const bus = new PipelineBusImpl();
798
- buses.push(bus);
799
-
800
- const started = bus.start();
801
- await bridge.initialize(bus, {});
802
-
803
- adapter.emit({ type: "response_started" });
804
- adapter.emit({
805
- type: "tool_call",
806
- toolId: "call_args",
807
- toolName: "consult_knowledge",
808
- args: { query: "What's the deadline?", reply_language: "si" },
809
- });
810
-
811
- await waitForCondition(() => adapter.injectedToolResults.length === 1);
812
- expect(receivedArgs).toEqual({ query: "What's the deadline?", reply_language: "si" });
813
-
814
- await bridge.close();
815
- bus.stop();
816
- await started;
817
- });
818
-
819
- it("R-13: surfaces the assistant transcript as llm.delta/llm.done (client captions data)", async () => {
820
- // Regression guard for #6: a realtime turn's assistant words must reach the bus as
821
- // llm.delta/llm.done — VoiceAgentSession turns those into agent_text_delta/agent_finished,
822
- // which the edge protocol forwards to the client as agent_chunk/agent_end.
823
- const adapter = new FakeRealtimeAdapter();
824
- const bridge = new RealtimeBridge(adapter);
825
- const bus = new PipelineBusImpl();
826
- buses.push(bus);
827
- const deltas: LlmDeltaPacket[] = [];
828
- const dones: LlmResponseDonePacket[] = [];
829
- bus.on("llm.delta", (pkt) => { deltas.push(pkt as LlmDeltaPacket); });
830
- bus.on("llm.done", (pkt) => { dones.push(pkt as LlmResponseDonePacket); });
831
-
832
- const started = bus.start();
833
- await bridge.initialize(bus, {});
834
-
835
- adapter.emit({ type: "response_started" });
836
- adapter.emit({ type: "transcript", role: "assistant", text: "The deadline is March 31.", final: true });
837
- adapter.emit({ type: "response_done" });
838
-
839
- await waitForCondition(() => dones.length === 1);
840
- expect(deltas.map((d) => d.text).join("")).toContain("The deadline is March 31.");
841
- expect(dones[0]!.text).toContain("The deadline is March 31.");
842
-
843
- await bridge.close();
844
- bus.stop();
845
- await started;
846
- });
847
-
848
- it("surfaces a delta-only assistant transcript (Gemini Live: no final transcript event)", async () => {
849
- // Gemini Live streams the assistant transcript as non-final fragments and never emits a
850
- // final transcript. The bridge must still surface the concatenated words as llm.delta/llm.done
851
- // so client captions work — without double-counting providers (OpenAI) that DO send a final.
852
- const adapter = new FakeRealtimeAdapter();
853
- const bridge = new RealtimeBridge(adapter);
854
- const bus = new PipelineBusImpl();
855
- buses.push(bus);
856
- const dones: LlmResponseDonePacket[] = [];
857
- bus.on("llm.done", (pkt) => { dones.push(pkt as LlmResponseDonePacket); });
858
-
859
- const started = bus.start();
860
- await bridge.initialize(bus, {});
861
-
862
- adapter.emit({ type: "response_started" });
863
- adapter.emit({ type: "transcript", role: "assistant", text: "The capital", final: false });
864
- adapter.emit({ type: "transcript", role: "assistant", text: " of France", final: false });
865
- adapter.emit({ type: "transcript", role: "assistant", text: " is", final: false });
866
- adapter.emit({ type: "transcript", role: "assistant", text: " Paris.", final: false });
867
- adapter.emit({ type: "response_done" });
868
-
869
- await waitForCondition(() => dones.length === 1);
870
- expect(dones[0]!.text).toBe("The capital of France is Paris.");
871
-
872
- await bridge.close();
873
- bus.stop();
874
- await started;
875
- });
876
-
877
- it("prefers the final transcript over deltas (OpenAI: deltas + final, no double-count)", async () => {
878
- const adapter = new FakeRealtimeAdapter();
879
- const bridge = new RealtimeBridge(adapter);
880
- const bus = new PipelineBusImpl();
881
- buses.push(bus);
882
- const dones: LlmResponseDonePacket[] = [];
883
- bus.on("llm.done", (pkt) => { dones.push(pkt as LlmResponseDonePacket); });
884
-
885
- const started = bus.start();
886
- await bridge.initialize(bus, {});
887
-
888
- adapter.emit({ type: "response_started" });
889
- adapter.emit({ type: "transcript", role: "assistant", text: "Let me ", final: false });
890
- adapter.emit({ type: "transcript", role: "assistant", text: "check that.", final: false });
891
- adapter.emit({ type: "transcript", role: "assistant", text: "Let me check that.", final: true });
892
- adapter.emit({ type: "response_done" });
893
-
894
- await waitForCondition(() => dones.length === 1);
895
- expect(dones[0]!.text).toBe("Let me check that.");
896
-
897
- await bridge.close();
898
- bus.stop();
899
- await started;
900
- });
901
-
902
- it("R-12: coalesces tiny audio deltas and never emits odd-length tts.audio", async () => {
903
- const adapter = new FakeRealtimeAdapter();
904
- const bridge = new RealtimeBridge(adapter);
905
- const bus = new PipelineBusImpl();
906
- buses.push(bus);
907
- const audio: TextToSpeechAudioPacket[] = [];
908
- bus.on("tts.audio", (pkt) => { audio.push(pkt as TextToSpeechAudioPacket); });
909
-
910
- const started = bus.start();
911
- await bridge.initialize(bus, {});
912
-
913
- adapter.emit({ type: "response_started" });
914
- adapter.emit({ type: "audio", pcm16: new Uint8Array([1, 2]), sampleRateHz: 24_000 });
915
- adapter.emit({ type: "audio", pcm16: new Uint8Array([3, 4]), sampleRateHz: 24_000 });
916
- expect(audio).toHaveLength(0);
917
-
918
- adapter.emit({ type: "audio", pcm16: new Uint8Array([5]), sampleRateHz: 24_000 });
919
- expect(audio).toHaveLength(0);
920
-
921
- adapter.emit({ type: "response_done" });
922
- await waitForCondition(() => audio.length > 0);
923
-
924
- expect(audio.every((frame) => frame.audio.byteLength % 2 === 0)).toBe(true);
925
- expect(audio.every((frame) => frame.audio.byteLength <= 640 || frameDurationMs(frame) <= 20)).toBe(true);
926
-
927
- await bridge.close();
928
- bus.stop();
929
- await started;
930
- });
931
-
932
- it("R-13: adapter event handler throw emits llm.error; adapter close exits quietly", async () => {
933
- const adapter = new FakeRealtimeAdapter();
934
- const bridge = new RealtimeBridge(adapter);
935
- const bus = new PipelineBusImpl();
936
- buses.push(bus);
937
- const errors: LlmErrorPacket[] = [];
938
- bus.on("llm.error", (pkt) => { errors.push(pkt as LlmErrorPacket); });
939
-
940
- const started = bus.start();
941
- await bridge.initialize(bus, {});
942
-
943
- const originalRandomUUID = globalThis.crypto.randomUUID.bind(globalThis.crypto);
944
- globalThis.crypto.randomUUID = () => {
945
- throw new Error("handler boom");
946
- };
947
- try {
948
- adapter.emit({ type: "response_started" });
949
- await waitForCondition(() => errors.length === 1);
950
- expect(errors[0]!.isRecoverable).toBe(false);
951
- expect(errors[0]!.cause.message).toBe("handler boom");
952
- } finally {
953
- globalThis.crypto.randomUUID = originalRandomUUID;
954
- }
955
-
956
- const errorsBeforeClose = errors.length;
957
- await bridge.close();
958
- expect(errors.length).toBe(errorsBeforeClose);
959
-
960
- bus.stop();
961
- await started;
962
- });
963
-
964
- it("does not emit interrupt.detected when emitsServerSpeechStarted is false", async () => {
965
- const adapter = new FakeRealtimeAdapter({ emitsServerSpeechStarted: false });
966
- const bridge = new RealtimeBridge(adapter);
967
- const bus = new PipelineBusImpl();
968
- buses.push(bus);
969
- const interrupts: Array<{ kind: string }> = [];
970
- bus.on("interrupt.detected", (pkt) => { interrupts.push(pkt as { kind: string }); });
971
-
972
- const started = bus.start();
973
- await bridge.initialize(bus, {});
974
-
975
- adapter.emit({ type: "response_started" });
976
- adapter.emit({ type: "speech_started" });
977
-
978
- await new Promise((resolve) => setTimeout(resolve, 50));
979
- expect(interrupts).toHaveLength(0);
980
-
981
- await bridge.close();
982
- bus.stop();
983
- await started;
984
- });
985
-
986
- it("double-barge-in: second turn tts.audio is not dropped after barge-in (R1, R3)", async () => {
987
- const adapter = new FakeRealtimeAdapter();
988
- let delegateSignal: AbortSignal | undefined;
989
- const reasoner: Reasoner = {
990
- stream: ({ signal }) => {
991
- delegateSignal = signal;
992
- return (async function* () {
993
- await new Promise((resolve) => setTimeout(resolve, 60_000));
994
- yield { type: "finish", reason: "stop", text: "never" };
995
- })();
996
- },
997
- };
998
- const bridge = new RealtimeBridge(adapter, reasoner);
999
- const session = new VoiceAgentSession({
1000
- plugins: { realtime: {} },
1001
- endpointingOwner: "timer",
1002
- minInterruptionMs: 0,
1003
- });
1004
- session.registerPlugin("realtime", bridge);
1005
-
1006
- const turnChanges: TurnChangePacket[] = [];
1007
- const recorded: RecordAssistantAudioPacket[] = [];
1008
- const interruptTts: InterruptTtsPacket[] = [];
1009
-
1010
- session.bus.on("turn.change", (pkt) => { turnChanges.push(pkt as TurnChangePacket); });
1011
- session.bus.on("record.assistant_audio", (pkt) => {
1012
- const p = pkt as RecordAssistantAudioPacket;
1013
- if (!p.truncate) recorded.push(p);
1014
- });
1015
- session.bus.on("interrupt.tts", (pkt) => { interruptTts.push(pkt as InterruptTtsPacket); });
1016
-
1017
- await session.start();
1018
-
1019
- adapter.emit({ type: "response_started" });
1020
- await waitForCondition(() => turnChanges.length >= 1);
1021
- const contextA = turnChanges[0]!.contextId;
1022
-
1023
- adapter.emit({
1024
- type: "audio",
1025
- pcm16: pcmFromSamples([100, 200, 300, 400]),
1026
- sampleRateHz: 24_000,
1027
- });
1028
- adapter.emit({ type: "speech_started" });
1029
- await waitForCondition(() => interruptTts.length >= 1);
1030
- expect(adapter.cancelCalls.length).toBeGreaterThanOrEqual(1);
1031
- expect(interruptTts[0]!.contextId).toBe(contextA);
1032
-
1033
- adapter.emit({ type: "response_done" });
1034
-
1035
- adapter.emit({ type: "response_started" });
1036
- await waitForCondition(() => turnChanges.length >= 2);
1037
- const contextB = turnChanges[1]!.contextId;
1038
- expect(contextB).not.toBe(contextA);
1039
-
1040
- adapter.emit({
1041
- type: "audio",
1042
- pcm16: frameSizedPcm24k(),
1043
- sampleRateHz: 24_000,
1044
- });
1045
- await waitForCondition(() => recorded.some((p) => p.contextId === contextB));
1046
-
1047
- adapter.emit({ type: "speech_started" });
1048
- await waitForCondition(() => interruptTts.length >= 2);
1049
- expect(adapter.cancelCalls.length).toBeGreaterThanOrEqual(2);
1050
-
1051
- adapter.emit({ type: "response_done" });
1052
-
1053
- adapter.emit({ type: "response_started" });
1054
- await waitForCondition(() => turnChanges.length >= 3);
1055
- const contextC = turnChanges[2]!.contextId;
1056
-
1057
- adapter.emit({
1058
- type: "audio",
1059
- pcm16: frameSizedPcm24k(),
1060
- sampleRateHz: 24_000,
1061
- });
1062
- await waitForCondition(() => recorded.some((p) => p.contextId === contextC));
1063
-
1064
- expect(recorded.filter((p) => p.contextId === contextB).length).toBeGreaterThan(0);
1065
- expect(recorded.filter((p) => p.contextId === contextC).length).toBeGreaterThan(0);
1066
-
1067
- adapter.emit({ type: "response_started" });
1068
- await waitForCondition(() => turnChanges.length >= 4);
1069
- const contextDelegate = turnChanges[3]!.contextId;
1070
- adapter.emit({
1071
- type: "tool_call",
1072
- toolId: "call_barge_delegate",
1073
- toolName: "consult_knowledge",
1074
- args: { query: "late add policy" },
1075
- });
1076
- await waitForCondition(() => delegateSignal !== undefined);
1077
- session.bus.push(Route.Critical, {
1078
- kind: "interrupt.tts",
1079
- contextId: contextDelegate,
1080
- timestampMs: Date.now(),
1081
- });
1082
- await waitForCondition(() => delegateSignal!.aborted);
1083
-
1084
- await session.close();
1085
- });
1086
-
1087
- it("BUG1: a barge-in DURING the reasoner gap drains speech_started, fires interrupt.detected, and aborts the delegate before it completes", async () => {
1088
- // The barge-in is delivered on the adapter EVENT STREAM (not a direct interrupt.tts bus
1089
- // push). With the old serial pump, awaiting the delegate froze the pump for the whole
1090
- // reasoner gap, so this speech_started stayed queued and never became interrupt.detected —
1091
- // the unwanted answer was then voiced over the user. The delegate must run off the pump.
1092
- const adapter = new FakeRealtimeAdapter();
1093
- let delegateSignal: AbortSignal | undefined;
1094
- let reasonerCompleted = false;
1095
- const reasoner: Reasoner = {
1096
- stream: ({ signal }) => {
1097
- delegateSignal = signal;
1098
- return (async function* () {
1099
- // The ~1.5–3s reasoner "thinking gap", exaggerated: the barge-in must abort it, not
1100
- // wait it out.
1101
- await new Promise((resolve) => setTimeout(resolve, 60_000));
1102
- reasonerCompleted = true;
1103
- yield { type: "finish", reason: "stop", text: "unwanted late answer" };
1104
- })();
1105
- },
1106
- };
1107
- const bridge = new RealtimeBridge(adapter, reasoner);
1108
- const session = new VoiceAgentSession({
1109
- plugins: { realtime: {} },
1110
- endpointingOwner: "timer",
1111
- minInterruptionMs: 0,
1112
- });
1113
- session.registerPlugin("realtime", bridge);
1114
-
1115
- const interruptDetected: InterruptionDetectedPacket[] = [];
1116
- const turnChanges: TurnChangePacket[] = [];
1117
- session.bus.on("interrupt.detected", (pkt) => { interruptDetected.push(pkt as InterruptionDetectedPacket); });
1118
- session.bus.on("turn.change", (pkt) => { turnChanges.push(pkt as TurnChangePacket); });
1119
-
1120
- await session.start();
1121
-
1122
- // Turn A: front model responds, emits audio, then calls the delegate tool.
1123
- adapter.emit({ type: "response_started" });
1124
- await waitForCondition(() => turnChanges.length >= 1);
1125
- adapter.emit({ type: "audio", pcm16: frameSizedPcm24k(), sampleRateHz: 24_000 });
1126
- adapter.emit({
1127
- type: "tool_call",
1128
- toolId: "call_gap_delegate",
1129
- toolName: "consult_knowledge",
1130
- args: { query: "late add policy" },
1131
- });
1132
-
1133
- // Delegate is now in flight (reasoner sleeping) — signal wired, not yet aborted.
1134
- await waitForCondition(() => delegateSignal !== undefined);
1135
- expect(delegateSignal!.aborted).toBe(false);
1136
-
1137
- // User barges in mid-gap, on the adapter event stream.
1138
- adapter.emit({ type: "speech_started" });
1139
-
1140
- // The pump kept draining: speech_started → interrupt.detected → session → interrupt.tts → abort.
1141
- await waitForCondition(() => interruptDetected.length >= 1);
1142
- await waitForCondition(() => delegateSignal!.aborted);
1143
- expect(reasonerCompleted).toBe(false);
1144
-
1145
- await session.close();
1146
- });
1147
- });