@kuralle-syrinx/core 4.2.0 → 4.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/package.json +2 -2
  2. package/src/audio/alaw.ts +56 -0
  3. package/src/audio/g722.ts +329 -0
  4. package/src/audio/index.ts +12 -0
  5. package/src/audio/loudness.ts +87 -0
  6. package/src/idle-timeout.ts +1 -0
  7. package/src/index.ts +63 -3
  8. package/src/interaction-policy.ts +12 -0
  9. package/src/observability-observer.ts +8 -4
  10. package/src/observability.ts +30 -1
  11. package/src/packet-factories.ts +110 -2
  12. package/src/packets.ts +116 -1
  13. package/src/plugin-contract.ts +41 -0
  14. package/src/policies/rule-based.ts +17 -2
  15. package/src/pricing.ts +137 -0
  16. package/src/primary-speaker-gate.ts +23 -3
  17. package/src/reasoner.ts +28 -1
  18. package/src/spend-cap.ts +52 -0
  19. package/src/turn-arbiter.ts +34 -2
  20. package/src/voice-agent-session.ts +453 -34
  21. package/src/voice-text.ts +69 -0
  22. package/src/audio/audio.test.ts +0 -285
  23. package/src/audio-envelope.test.ts +0 -167
  24. package/src/confidence-to-wait.test.ts +0 -21
  25. package/src/error-handler.test.ts +0 -56
  26. package/src/hedge-throwing-backend.test.ts +0 -46
  27. package/src/interaction-coordinator.test.ts +0 -425
  28. package/src/iu-ledger.test.ts +0 -276
  29. package/src/latency-filler.test.ts +0 -62
  30. package/src/observability-observer.test.ts +0 -245
  31. package/src/observability.test.ts +0 -85
  32. package/src/packet-factories.test.ts +0 -53
  33. package/src/pipeline-bus.g10.test.ts +0 -145
  34. package/src/pipeline-bus.test.ts +0 -211
  35. package/src/policies/defer.test.ts +0 -86
  36. package/src/policies/rule-based.test.ts +0 -269
  37. package/src/primary-speaker-gate.test.ts +0 -150
  38. package/src/provider-fallback.test.ts +0 -87
  39. package/src/reasoner-hedge.test.ts +0 -361
  40. package/src/reasoner-route.test.ts +0 -248
  41. package/src/reasoner.test.ts +0 -69
  42. package/src/retry.test.ts +0 -83
  43. package/src/route-throwing-spec.test.ts +0 -44
  44. package/src/tts-playout-clock.test.ts +0 -125
  45. package/src/turn-arbiter.characterization.test.ts +0 -477
  46. package/src/turn-arbiter.test.ts +0 -567
  47. package/src/voice-agent-session.test.ts +0 -3316
  48. package/src/voice-text.test.ts +0 -94
  49. package/tsconfig.json +0 -21
@@ -1,3316 +0,0 @@
1
- // SPDX-License-Identifier: MIT
2
-
3
- import { describe, it, expect, vi } from "vitest";
4
- import { VoiceAgentSession } from "./voice-agent-session.js";
5
- import {
6
- Route,
7
- type InteractionDecision,
8
- type InteractionObservation,
9
- type InteractionPolicy,
10
- type PipelineBus,
11
- type PluginConfig,
12
- type VoicePlugin,
13
- } from "./index.js";
14
- import { InteractionCoordinator } from "./interaction-coordinator.js";
15
- import type { Scheduler, ScheduledCallback } from "./scheduler.js";
16
- import { ErrorCategory, type InteractionBackchannelPacket } from "./packets.js";
17
- import type {
18
- EndOfSpeechAudioPacket,
19
- RecordAssistantAudioPacket,
20
- RecordUserAudioPacket,
21
- SpeechToTextAudioPacket,
22
- SttInterimPacket,
23
- SttPartialPacket,
24
- SttResultPacket,
25
- EndOfSpeechPacket,
26
- LlmDeltaPacket,
27
- LlmResponseDonePacket,
28
- TextToSpeechDonePacket,
29
- TextToSpeechAudioPacket,
30
- TextToSpeechEndPacket,
31
- TextToSpeechPlayoutProgressPacket,
32
- TextToSpeechTextPacket,
33
- InterruptTtsPacket,
34
- InterruptLlmPacket,
35
- InterruptionDetectedPacket,
36
- VadSpeechEndedPacket,
37
- UserAudioReceivedPacket,
38
- UserInputPacket,
39
- VadAudioPacket,
40
- ModeSwitchCompletedPacket,
41
- VadSpeechStartedPacket,
42
- VadSpeechActivityPacket,
43
- LlmErrorPacket,
44
- TtsErrorPacket,
45
- PipelineErrorPacket,
46
- } from "./packets.js";
47
- import {
48
- BYSTANDER_SPEAKER_TONE_HZ,
49
- PRIMARY_SPEAKER_TONE_HZ,
50
- ASSISTANT_ECHO_TONE_HZ,
51
- synthesizeTonePcm16,
52
- } from "./primary-speaker-fixtures.js";
53
-
54
- class CapturingPlugin implements VoicePlugin {
55
- config: PluginConfig | null = null;
56
- forceFinalize = vi.fn();
57
-
58
- async initialize(_bus: PipelineBus, config: PluginConfig): Promise<void> {
59
- this.config = config;
60
- }
61
-
62
- async close(): Promise<void> {
63
- // no-op
64
- }
65
- }
66
-
67
- class OrderedClosePlugin implements VoicePlugin {
68
- constructor(
69
- private readonly name: string,
70
- private readonly closeOrder: string[],
71
- ) {}
72
-
73
- async initialize(): Promise<void> {
74
- // no-op
75
- }
76
-
77
- async close(): Promise<void> {
78
- this.closeOrder.push(this.name);
79
- }
80
- }
81
-
82
- class SlowClosePlugin implements VoicePlugin {
83
- closeCount = 0;
84
-
85
- async initialize(): Promise<void> {
86
- // no-op
87
- }
88
-
89
- async close(): Promise<void> {
90
- this.closeCount += 1;
91
- await new Promise((resolve) => setTimeout(resolve, 20));
92
- }
93
- }
94
-
95
- class FailingInitPlugin implements VoicePlugin {
96
- async initialize(): Promise<void> {
97
- throw new Error("init failed");
98
- }
99
-
100
- async close(): Promise<void> {
101
- // no-op
102
- }
103
- }
104
-
105
- class EndpointingPlugin extends CapturingPlugin {
106
- initializeCount = 0;
107
-
108
- constructor(readonly endpointingCapability: NonNullable<VoicePlugin["endpointingCapability"]>) {
109
- super();
110
- }
111
-
112
- override async initialize(bus: PipelineBus, config: PluginConfig): Promise<void> {
113
- this.initializeCount += 1;
114
- await super.initialize(bus, config);
115
- }
116
- }
117
-
118
- class CapturingInteractionPolicy implements InteractionPolicy {
119
- readonly observations: InteractionObservation[] = [];
120
- readonly resetContextIds: string[] = [];
121
- initializeCount = 0;
122
- closeCount = 0;
123
- initializedConfig: Record<string, unknown> | null = null;
124
-
125
- constructor(
126
- private readonly decide: (observation: InteractionObservation) => readonly InteractionDecision[] = () => [],
127
- ) {}
128
-
129
- async initialize(config: Record<string, unknown>): Promise<void> {
130
- this.initializeCount += 1;
131
- this.initializedConfig = config;
132
- }
133
-
134
- observe(observation: InteractionObservation): readonly InteractionDecision[] {
135
- this.observations.push(observation);
136
- return this.decide(observation);
137
- }
138
-
139
- reset(contextId: string): void {
140
- this.resetContextIds.push(contextId);
141
- }
142
-
143
- async close(): Promise<void> {
144
- this.closeCount += 1;
145
- }
146
- }
147
-
148
- /** Records scheduled/cancelled timers without firing them, for leak assertions. */
149
- class RecordingScheduler implements Scheduler {
150
- readonly timers = new Map<string, { delayMs: number; cb: ScheduledCallback }>();
151
- schedule(key: string, delayMs: number, cb: ScheduledCallback): void {
152
- this.timers.set(key, { delayMs, cb });
153
- }
154
- cancel(key: string): void {
155
- this.timers.delete(key);
156
- }
157
- pendingKeys(prefix: string): string[] {
158
- return [...this.timers.keys()].filter((k) => k.startsWith(prefix));
159
- }
160
- }
161
-
162
- class InterruptAwareStreamingTtsPlugin implements VoicePlugin {
163
- private bus: PipelineBus | null = null;
164
- private interval: ReturnType<typeof setInterval> | null = null;
165
- private contextId = "";
166
- emittedAudioCount = 0;
167
- interruptObservedAtMs = 0;
168
-
169
- async initialize(bus: PipelineBus): Promise<void> {
170
- this.bus = bus;
171
- bus.on("tts.text", (pkt) => {
172
- this.startStreaming((pkt as TextToSpeechTextPacket).contextId);
173
- });
174
- bus.on("interrupt.tts", (pkt) => {
175
- this.interruptObservedAtMs = performance.now();
176
- this.stopStreaming();
177
- this.bus?.push(Route.Main, {
178
- kind: "tts.end",
179
- contextId: (pkt as InterruptTtsPacket).contextId,
180
- timestampMs: Date.now(),
181
- });
182
- });
183
- }
184
-
185
- async close(): Promise<void> {
186
- this.stopStreaming();
187
- this.bus = null;
188
- }
189
-
190
- private startStreaming(contextId: string): void {
191
- this.contextId = contextId;
192
- if (this.interval) return;
193
- this.interval = setInterval(() => {
194
- this.emittedAudioCount++;
195
- this.bus?.push(Route.Main, {
196
- kind: "tts.audio",
197
- contextId: this.contextId,
198
- timestampMs: Date.now(),
199
- audio: new Uint8Array([1, 2, 3, 4]),
200
- sampleRateHz: 16000,
201
- } satisfies TextToSpeechAudioPacket);
202
- }, 5);
203
- }
204
-
205
- private stopStreaming(): void {
206
- if (!this.interval) return;
207
- clearInterval(this.interval);
208
- this.interval = null;
209
- }
210
- }
211
-
212
- async function closeSession(session: VoiceAgentSession): Promise<void> {
213
- if (session.state !== "closed") {
214
- await session.close();
215
- }
216
- }
217
-
218
- async function enrollPrimarySpeaker(
219
- session: VoiceAgentSession,
220
- contextId = "user-enroll",
221
- ): Promise<void> {
222
- const chunk = synthesizeTonePcm16({
223
- frequencyHz: PRIMARY_SPEAKER_TONE_HZ,
224
- durationMs: 32,
225
- });
226
- const t0 = Date.now();
227
- session.bus.push(Route.Main, {
228
- kind: "vad.speech_started",
229
- contextId,
230
- timestampMs: t0,
231
- confidence: 0.99,
232
- } satisfies VadSpeechStartedPacket);
233
- for (let i = 0; i < 12; i += 1) {
234
- session.bus.push(Route.Main, {
235
- kind: "user.audio_received",
236
- contextId,
237
- timestampMs: t0 + i * 20,
238
- audio: chunk,
239
- } satisfies UserAudioReceivedPacket);
240
- }
241
- session.bus.push(Route.Main, {
242
- kind: "vad.speech_ended",
243
- contextId,
244
- timestampMs: t0 + 300,
245
- } satisfies VadSpeechEndedPacket);
246
- await new Promise((resolve) => setTimeout(resolve, 10));
247
- }
248
-
249
- describe("VoiceAgentSession", () => {
250
- it("owns an injected policy lifecycle and feeds decoded audio plus playout observations", async () => {
251
- const policy = new CapturingInteractionPolicy();
252
- const session = new VoiceAgentSession({
253
- plugins: {},
254
- interactionPolicy: policy,
255
- interactionPolicyConfig: { model_path: "/tmp/policy.onnx" },
256
- });
257
- await session.start();
258
-
259
- session.bus.push(Route.Main, {
260
- kind: "user.audio_received",
261
- contextId: "turn-policy",
262
- timestampMs: 1000,
263
- audio: new Uint8Array([0x34, 0x12, 0xcc, 0xff]),
264
- } satisfies UserAudioReceivedPacket);
265
- session.bus.push(Route.Main, {
266
- kind: "tts.audio",
267
- contextId: "turn-policy",
268
- timestampMs: 1100,
269
- audio: new Uint8Array(640),
270
- sampleRateHz: 16000,
271
- } satisfies TextToSpeechAudioPacket);
272
- session.bus.push(Route.Main, {
273
- kind: "tts.playout_progress",
274
- contextId: "turn-policy",
275
- timestampMs: 1200,
276
- playedOutMs: 20,
277
- complete: true,
278
- } satisfies TextToSpeechPlayoutProgressPacket);
279
- await new Promise((resolve) => setTimeout(resolve, 20));
280
-
281
- expect(policy.initializeCount).toBe(1);
282
- expect(policy.initializedConfig).toEqual({ model_path: "/tmp/policy.onnx" });
283
- const audioFrame = policy.observations.find((observation) => observation.kind === "audio_frame");
284
- expect(audioFrame).toMatchObject({ contextId: "turn-policy", timestampMs: 1000, sampleRateHz: 16000 });
285
- expect(audioFrame?.kind === "audio_frame" ? [...(audioFrame.audio ?? [])] : []).toEqual([4660, -52]);
286
- expect(policy.observations.filter((observation) => observation.kind === "playout_tick")).toEqual([
287
- expect.objectContaining({
288
- contextId: "turn-policy",
289
- ttsActive: true,
290
- sampleRateHz: 16000,
291
- audio: expect.any(Int16Array),
292
- }),
293
- expect.objectContaining({ contextId: "turn-policy", playedOutMs: 20, ttsActive: false }),
294
- ]);
295
-
296
- await closeSession(session);
297
- expect(policy.closeCount).toBe(1);
298
- });
299
-
300
- it("threads the packet's true sample rate to the injected policy audio_frame (not a hardcoded 16k)", async () => {
301
- const policy = new CapturingInteractionPolicy();
302
- const session = new VoiceAgentSession({ plugins: {}, interactionPolicy: policy });
303
- await session.start();
304
-
305
- session.bus.push(Route.Main, {
306
- kind: "user.audio_received",
307
- contextId: "turn-rate",
308
- timestampMs: 2000,
309
- audio: new Uint8Array([0x34, 0x12, 0xcc, 0xff]),
310
- sampleRateHz: 24000,
311
- } satisfies UserAudioReceivedPacket);
312
- await new Promise((resolve) => setTimeout(resolve, 20));
313
-
314
- const audioFrame = policy.observations.find((o) => o.kind === "audio_frame");
315
- expect(audioFrame).toMatchObject({ contextId: "turn-rate", sampleRateHz: 24000 });
316
-
317
- await closeSession(session);
318
- });
319
-
320
- it("cancels a pending interaction playout timer on close even after its turn completed", async () => {
321
- const scheduler = new RecordingScheduler();
322
- const policy = new CapturingInteractionPolicy();
323
- const session = new VoiceAgentSession({ plugins: {}, interactionPolicy: policy, scheduler });
324
- await session.start();
325
-
326
- session.bus.push(Route.Main, {
327
- kind: "tts.audio",
328
- contextId: "turn-x",
329
- timestampMs: 1000,
330
- audio: new Uint8Array(640),
331
- sampleRateHz: 16000,
332
- } satisfies TextToSpeechAudioPacket);
333
- session.bus.push(Route.Main, {
334
- kind: "tts.end",
335
- contextId: "turn-x",
336
- timestampMs: 1100,
337
- } satisfies TextToSpeechEndPacket);
338
- await new Promise((resolve) => setTimeout(resolve, 10));
339
- expect(scheduler.pendingKeys("interaction.playout:")).toEqual(["interaction.playout:turn-x"]);
340
-
341
- // Turn completes (telephony reuses the contextId) — this deletes turn-x from
342
- // firstTtsAudioFired, orphaning the timer from the old stop() cancel loop.
343
- session.bus.push(Route.Main, {
344
- kind: "eos.turn_complete",
345
- contextId: "turn-x",
346
- timestampMs: 1200,
347
- text: "done",
348
- transcripts: [],
349
- } satisfies EndOfSpeechPacket);
350
- await new Promise((resolve) => setTimeout(resolve, 10));
351
- expect(scheduler.pendingKeys("interaction.playout:")).toEqual(["interaction.playout:turn-x"]);
352
-
353
- await session.close();
354
- expect(scheduler.pendingKeys("interaction.playout:")).toEqual([]);
355
- });
356
-
357
- it("does not initialize an injected policy when full-duplex defer mode owns interaction", async () => {
358
- const policy = new CapturingInteractionPolicy();
359
- const session = new VoiceAgentSession({ plugins: {}, interactionPolicy: policy, fullDuplex: true });
360
- await session.start();
361
- await closeSession(session);
362
-
363
- expect(policy.initializeCount).toBe(0);
364
- expect(policy.closeCount).toBe(0);
365
- expect(policy.observations).toEqual([]);
366
- });
367
-
368
- it("makes an injected policy the sole endpoint owner while keeping provider STT initialized", async () => {
369
- const policy = new CapturingInteractionPolicy();
370
- const provider = new EndpointingPlugin({
371
- owner: "provider_stt",
372
- disableConfig: { emit_eos_on_final: false },
373
- });
374
- const legacySmartTurn = new EndpointingPlugin({ owner: "smart_turn" });
375
- const session = new VoiceAgentSession({
376
- plugins: { stt: { emit_eos_on_final: true }, eos: {} },
377
- endpointingOwner: "smart_turn",
378
- interactionPolicy: policy,
379
- });
380
- session.registerPlugin("stt", provider);
381
- session.registerPlugin("eos", legacySmartTurn);
382
- await session.start();
383
-
384
- expect(provider.initializeCount).toBe(1);
385
- expect(provider.config).toEqual({ emit_eos_on_final: false });
386
- expect(legacySmartTurn.initializeCount).toBe(0);
387
- expect(policy.initializeCount).toBe(1);
388
- await closeSession(session);
389
- });
390
-
391
- it("passes configured plugin options to each plugin during initialization", async () => {
392
- const plugin = new CapturingPlugin();
393
- const session = new VoiceAgentSession({
394
- plugins: {
395
- stt: {
396
- api_key: "test-key",
397
- endpointing: 300,
398
- },
399
- },
400
- });
401
-
402
- session.registerPlugin("stt", plugin);
403
- await session.start();
404
-
405
- expect(plugin.config).toEqual({
406
- api_key: "test-key",
407
- endpointing: 300,
408
- });
409
-
410
- await closeSession(session);
411
- });
412
-
413
- it("force-finalizes STT when audio stops and no final result arrives", async () => {
414
- const plugin = new CapturingPlugin();
415
- const session = new VoiceAgentSession({
416
- plugins: { stt: {} },
417
- sttForceFinalizeTimeoutMs: 10,
418
- });
419
-
420
- session.registerPlugin("stt", plugin);
421
- await session.start();
422
-
423
- const audioPacket: SpeechToTextAudioPacket = {
424
- kind: "stt.audio",
425
- contextId: "turn-1",
426
- timestampMs: Date.now(),
427
- audio: new Uint8Array([1, 2, 3, 4]),
428
- };
429
- session.bus.push(Route.Main, audioPacket);
430
-
431
- await new Promise((resolve) => setTimeout(resolve, 30));
432
-
433
- expect(plugin.forceFinalize).toHaveBeenCalledTimes(1);
434
- expect(plugin.forceFinalize).toHaveBeenCalledWith("turn-1");
435
-
436
- await closeSession(session);
437
- });
438
-
439
- it("cancels pending STT force-finalization when a final result arrives", async () => {
440
- const plugin = new CapturingPlugin();
441
- const session = new VoiceAgentSession({
442
- plugins: { stt: {} },
443
- sttForceFinalizeTimeoutMs: 30,
444
- });
445
-
446
- session.registerPlugin("stt", plugin);
447
- await session.start();
448
-
449
- const audioPacket: SpeechToTextAudioPacket = {
450
- kind: "stt.audio",
451
- contextId: "turn-1",
452
- timestampMs: Date.now(),
453
- audio: new Uint8Array([1, 2, 3, 4]),
454
- };
455
- session.bus.push(Route.Main, audioPacket);
456
-
457
- await new Promise((resolve) => setTimeout(resolve, 5));
458
-
459
- const finalPacket: SttResultPacket = {
460
- kind: "stt.result",
461
- contextId: "turn-1",
462
- timestampMs: Date.now(),
463
- text: "hello",
464
- confidence: 0.99,
465
- };
466
- session.bus.push(Route.Main, finalPacket);
467
-
468
- await new Promise((resolve) => setTimeout(resolve, 45));
469
-
470
- expect(plugin.forceFinalize).not.toHaveBeenCalled();
471
-
472
- await closeSession(session);
473
- });
474
-
475
- it("fans user audio out to recorder, VAD, STT, and EOS routes", async () => {
476
- const session = new VoiceAgentSession({ plugins: {}, endpointingOwner: "smart_turn" });
477
- await session.start();
478
-
479
- const recordPackets: RecordUserAudioPacket[] = [];
480
- const vadPackets: VadAudioPacket[] = [];
481
- const sttPackets: SpeechToTextAudioPacket[] = [];
482
- const eosPackets: EndOfSpeechAudioPacket[] = [];
483
-
484
- session.bus.on("record.user_audio", (pkt) => {
485
- recordPackets.push(pkt as RecordUserAudioPacket);
486
- });
487
- session.bus.on("vad.audio", (pkt) => {
488
- vadPackets.push(pkt as VadAudioPacket);
489
- });
490
- session.bus.on("stt.audio", (pkt) => {
491
- sttPackets.push(pkt as SpeechToTextAudioPacket);
492
- });
493
- session.bus.on("eos.audio", (pkt) => {
494
- eosPackets.push(pkt as EndOfSpeechAudioPacket);
495
- });
496
-
497
- const audio = new Uint8Array([1, 2, 3, 4]);
498
- const userAudioPacket: UserAudioReceivedPacket = {
499
- kind: "user.audio_received",
500
- contextId: "turn-1",
501
- timestampMs: Date.now(),
502
- audio,
503
- };
504
- session.bus.push(Route.Main, userAudioPacket);
505
-
506
- await new Promise((resolve) => setTimeout(resolve, 20));
507
-
508
- expect(recordPackets).toHaveLength(1);
509
- expect(vadPackets).toHaveLength(1);
510
- expect(sttPackets).toHaveLength(1);
511
- expect(eosPackets).toHaveLength(1);
512
- expect(recordPackets[0]!.audio).toBe(audio);
513
- expect(vadPackets[0]!.audio).toBe(audio);
514
- expect(sttPackets[0]!.audio).toBe(audio);
515
- expect(eosPackets[0]!.audio).toBe(audio);
516
-
517
- await closeSession(session);
518
- });
519
-
520
- it("G2/WBS-1: surfaces delegate.query/delegate.result packets as delegate_query/delegate_result session events", async () => {
521
- const session = new VoiceAgentSession({ plugins: {} });
522
- const queries: Array<{ turnId: string; query: string; toolName?: string }> = [];
523
- const results: Array<{ turnId: string; query: string; answer: string; durationMs: number; grounded: boolean }> = [];
524
- session.on("delegate_query", (event) => queries.push(event));
525
- session.on("delegate_result", (event) => results.push(event));
526
- await session.start();
527
-
528
- session.bus.push(Route.Background, {
529
- kind: "delegate.query",
530
- contextId: "turn-1",
531
- timestampMs: Date.now(),
532
- query: "When is the deadline?",
533
- toolId: "call_1",
534
- toolName: "consult_knowledge",
535
- });
536
- session.bus.push(Route.Background, {
537
- kind: "delegate.result",
538
- contextId: "turn-1",
539
- timestampMs: Date.now(),
540
- query: "When is the deadline?",
541
- answer: "March 31.",
542
- durationMs: 42,
543
- grounded: true,
544
- toolId: "call_1",
545
- toolName: "consult_knowledge",
546
- });
547
-
548
- await new Promise((resolve) => setTimeout(resolve, 20));
549
-
550
- expect(queries).toEqual([
551
- { tsMs: expect.any(Number), turnId: "turn-1", query: "When is the deadline?", toolId: "call_1", toolName: "consult_knowledge" },
552
- ]);
553
- expect(results).toEqual([
554
- {
555
- tsMs: expect.any(Number),
556
- turnId: "turn-1",
557
- query: "When is the deadline?",
558
- answer: "March 31.",
559
- durationMs: 42,
560
- grounded: true,
561
- toolId: "call_1",
562
- toolName: "consult_knowledge",
563
- },
564
- ]);
565
-
566
- await closeSession(session);
567
- });
568
-
569
- it("emits a decomposed turn_latency session event on first TTS audio", async () => {
570
- const session = new VoiceAgentSession({ plugins: {} });
571
- const events: Array<Record<string, unknown>> = [];
572
- session.on("turn_latency", (event) => {
573
- events.push(event);
574
- });
575
- await session.start();
576
-
577
- const t0 = 100_000;
578
- session.bus.push(Route.Main, { kind: "vad.speech_ended", contextId: "turn-1", timestampMs: t0 });
579
- await new Promise((resolve) => setTimeout(resolve, 10));
580
- session.bus.push(Route.Main, {
581
- kind: "eos.turn_complete",
582
- contextId: "turn-1",
583
- timestampMs: t0 + 200,
584
- text: "what are the lab fees",
585
- transcripts: [],
586
- });
587
- await new Promise((resolve) => setTimeout(resolve, 10));
588
- session.bus.push(Route.Main, { kind: "llm.delta", contextId: "turn-1", timestampMs: t0 + 900, text: "The fee is ten dollars." });
589
- await new Promise((resolve) => setTimeout(resolve, 10));
590
- session.bus.push(Route.Main, {
591
- kind: "tts.audio",
592
- contextId: "turn-1",
593
- timestampMs: t0 + 1150,
594
- audio: new Uint8Array(320),
595
- sampleRateHz: 16000,
596
- });
597
- await new Promise((resolve) => setTimeout(resolve, 20));
598
-
599
- expect(events).toHaveLength(1);
600
- expect(events[0]).toEqual({
601
- tsMs: expect.any(Number),
602
- turnId: "turn-1",
603
- ttfaMs: 1150,
604
- eouDelayMs: 200,
605
- llmTtftMs: 700,
606
- ttsTtfbMs: 250,
607
- fillerUsed: false,
608
- backchannelUsed: false,
609
- });
610
-
611
- await closeSession(session);
612
- });
613
-
614
- it("turn_latency anchors TTFA to eos when no VAD speech-end was seen, and marks filler turns", async () => {
615
- const session = new VoiceAgentSession({ plugins: {}, latencyFillerEnabled: true });
616
- const events: Array<Record<string, unknown>> = [];
617
- session.on("turn_latency", (event) => {
618
- events.push(event);
619
- });
620
- await session.start();
621
-
622
- const t0 = 200_000;
623
- session.bus.push(Route.Main, {
624
- kind: "eos.turn_complete",
625
- contextId: "turn-2",
626
- timestampMs: t0,
627
- text: "tell me about registration holds",
628
- transcripts: [],
629
- });
630
- await new Promise((resolve) => setTimeout(resolve, 10));
631
- session.bus.push(Route.Main, {
632
- kind: "tts.audio",
633
- contextId: "turn-2",
634
- timestampMs: t0 + 400,
635
- audio: new Uint8Array(320),
636
- sampleRateHz: 16000,
637
- });
638
- await new Promise((resolve) => setTimeout(resolve, 20));
639
-
640
- expect(events).toHaveLength(1);
641
- expect(events[0]).toMatchObject({
642
- turnId: "turn-2",
643
- ttfaMs: 400,
644
- fillerUsed: true,
645
- backchannelUsed: false,
646
- });
647
- expect(events[0]!["eouDelayMs"]).toBeUndefined();
648
-
649
- await closeSession(session);
650
- });
651
-
652
- it("turn_latency is not emitted for an interrupted turn", async () => {
653
- const session = new VoiceAgentSession({ plugins: {} });
654
- const events: unknown[] = [];
655
- session.on("turn_latency", (event) => {
656
- events.push(event);
657
- });
658
- await session.start();
659
-
660
- const t0 = 300_000;
661
- session.bus.push(Route.Main, {
662
- kind: "eos.turn_complete",
663
- contextId: "turn-3",
664
- timestampMs: t0,
665
- text: "hello there",
666
- transcripts: [],
667
- });
668
- await new Promise((resolve) => setTimeout(resolve, 10));
669
- session.bus.push(Route.Critical, {
670
- kind: "interrupt.detected",
671
- contextId: "turn-3",
672
- timestampMs: t0 + 100,
673
- });
674
- await new Promise((resolve) => setTimeout(resolve, 10));
675
- session.bus.push(Route.Main, {
676
- kind: "tts.audio",
677
- contextId: "turn-3",
678
- timestampMs: t0 + 300,
679
- audio: new Uint8Array(320),
680
- sampleRateHz: 16000,
681
- });
682
- await new Promise((resolve) => setTimeout(resolve, 20));
683
-
684
- expect(events).toHaveLength(0);
685
-
686
- await closeSession(session);
687
- });
688
-
689
- it("IP-C3: tool_call_cue.delayed emits interaction.backchannel end-to-end", async () => {
690
- const session = new VoiceAgentSession({ plugins: {}, delayCueAfterMs: 30 });
691
- const backchannels: InteractionBackchannelPacket[] = [];
692
- session.bus.on("interaction.backchannel", (pkt) => {
693
- backchannels.push(pkt as InteractionBackchannelPacket);
694
- });
695
- await session.start();
696
-
697
- session.bus.push(Route.Main, {
698
- kind: "llm.tool_call",
699
- contextId: "turn-1",
700
- timestampMs: Date.now(),
701
- toolId: "t1",
702
- toolName: "consult_knowledge",
703
- toolArgs: { query: "fees" },
704
- });
705
-
706
- await new Promise((resolve) => setTimeout(resolve, 70));
707
-
708
- expect(backchannels).toHaveLength(1);
709
- expect(backchannels[0]).toMatchObject({ contextId: "turn-1", cue: "mm_hmm" });
710
-
711
- await closeSession(session);
712
- });
713
-
714
- it("G3/WBS-3: tool-call lifecycle cues fire started → delayed → complete", async () => {
715
- const session = new VoiceAgentSession({ plugins: {}, delayCueAfterMs: 30 });
716
- const cues: Array<{ phase: string; turnId: string; toolId: string; toolName: string; afterMs?: number }> = [];
717
- session.on("tool_call_cue", (event) => cues.push(event));
718
- await session.start();
719
-
720
- session.bus.push(Route.Main, {
721
- kind: "llm.tool_call",
722
- contextId: "turn-1",
723
- timestampMs: Date.now(),
724
- toolId: "t1",
725
- toolName: "consult_knowledge",
726
- toolArgs: { query: "fees" },
727
- });
728
-
729
- await new Promise((resolve) => setTimeout(resolve, 70));
730
-
731
- session.bus.push(Route.Main, {
732
- kind: "llm.tool_result",
733
- contextId: "turn-1",
734
- timestampMs: Date.now(),
735
- toolId: "t1",
736
- toolName: "consult_knowledge",
737
- result: "answer",
738
- });
739
-
740
- await new Promise((resolve) => setTimeout(resolve, 20));
741
-
742
- expect(cues.map((cue) => cue.phase)).toEqual(["started", "delayed", "complete"]);
743
- expect(cues[0]).toMatchObject({ turnId: "turn-1", toolId: "t1", toolName: "consult_knowledge" });
744
- expect(cues[1]!.afterMs).toBe(30);
745
-
746
- await closeSession(session);
747
- });
748
-
749
- it("G3/WBS-3: no delayed cue when the result beats the timer; timer 0 disables it", async () => {
750
- const session = new VoiceAgentSession({ plugins: {}, delayCueAfterMs: 5000 });
751
- const cues: Array<{ phase: string }> = [];
752
- session.on("tool_call_cue", (event) => cues.push(event));
753
- await session.start();
754
-
755
- session.bus.push(Route.Main, {
756
- kind: "llm.tool_call",
757
- contextId: "turn-1",
758
- timestampMs: Date.now(),
759
- toolId: "t1",
760
- toolName: "consult_knowledge",
761
- toolArgs: {},
762
- });
763
- session.bus.push(Route.Main, {
764
- kind: "llm.tool_result",
765
- contextId: "turn-1",
766
- timestampMs: Date.now(),
767
- toolId: "t1",
768
- toolName: "consult_knowledge",
769
- result: "fast answer",
770
- });
771
-
772
- await new Promise((resolve) => setTimeout(resolve, 30));
773
- expect(cues.map((cue) => cue.phase)).toEqual(["started", "complete"]);
774
-
775
- await closeSession(session);
776
-
777
- const disabled = new VoiceAgentSession({ plugins: {}, delayCueAfterMs: 0 });
778
- const disabledCues: Array<{ phase: string }> = [];
779
- disabled.on("tool_call_cue", (event) => disabledCues.push(event));
780
- await disabled.start();
781
- disabled.bus.push(Route.Main, {
782
- kind: "llm.tool_call",
783
- contextId: "turn-2",
784
- timestampMs: Date.now(),
785
- toolId: "t2",
786
- toolName: "consult_knowledge",
787
- toolArgs: {},
788
- });
789
- await new Promise((resolve) => setTimeout(resolve, 30));
790
- expect(disabledCues.map((cue) => cue.phase)).toEqual(["started"]);
791
- await closeSession(disabled);
792
- });
793
-
794
- it("G3/WBS-3: llm.error while a tool call is pending fires the failed cue", async () => {
795
- const session = new VoiceAgentSession({ plugins: {}, delayCueAfterMs: 0 });
796
- const cues: Array<{ phase: string; toolId: string }> = [];
797
- session.on("tool_call_cue", (event) => cues.push(event));
798
- await session.start();
799
-
800
- session.bus.push(Route.Main, {
801
- kind: "llm.tool_call",
802
- contextId: "turn-1",
803
- timestampMs: Date.now(),
804
- toolId: "t1",
805
- toolName: "consult_knowledge",
806
- toolArgs: {},
807
- });
808
- await new Promise((resolve) => setTimeout(resolve, 10)); // let Main drain before the Critical error
809
- session.bus.push(Route.Critical, {
810
- kind: "llm.error",
811
- contextId: "turn-1",
812
- timestampMs: Date.now(),
813
- component: "bridge",
814
- category: ErrorCategory.NetworkTimeout,
815
- cause: new Error("delegate failed"),
816
- isRecoverable: true,
817
- });
818
-
819
- await new Promise((resolve) => setTimeout(resolve, 30));
820
- expect(cues.map((cue) => cue.phase)).toEqual(["started", "failed"]);
821
- expect(cues[1]!.toolId).toBe("t1");
822
-
823
- await closeSession(session);
824
- });
825
-
826
- it("G3/WBS-3 (R5): barge-in fails the pending cue and the interrupt path is unaffected", async () => {
827
- const session = new VoiceAgentSession({ plugins: {}, delayCueAfterMs: 5000, minInterruptionMs: 0 });
828
- const cues: Array<{ phase: string }> = [];
829
- const interrupts: string[] = [];
830
- session.on("tool_call_cue", (event) => cues.push(event));
831
- session.bus.on("interrupt.tts", (pkt) => { interrupts.push((pkt as { contextId: string }).contextId); });
832
- await session.start();
833
-
834
- session.bus.push(Route.Main, {
835
- kind: "llm.tool_call",
836
- contextId: "turn-1",
837
- timestampMs: Date.now(),
838
- toolId: "t1",
839
- toolName: "consult_knowledge",
840
- toolArgs: {},
841
- });
842
- await new Promise((resolve) => setTimeout(resolve, 10)); // let Main drain before the Critical interrupt
843
- session.bus.push(Route.Critical, {
844
- kind: "interrupt.detected",
845
- contextId: "turn-1",
846
- timestampMs: Date.now(),
847
- source: "client",
848
- });
849
-
850
- await new Promise((resolve) => setTimeout(resolve, 30));
851
- expect(cues.map((cue) => cue.phase)).toEqual(["started", "failed"]);
852
- expect(interrupts).toContain("turn-1"); // barge-in cancel still flowed
853
-
854
- await closeSession(session);
855
- });
856
-
857
- it("emits normalized debug events for bus packets", async () => {
858
- const session = new VoiceAgentSession({ plugins: {} });
859
- const reader = session.debugEvents.getReader();
860
- await session.start();
861
-
862
- session.bus.push(Route.Main, {
863
- kind: "user.text_received",
864
- contextId: "turn-1",
865
- timestampMs: Date.now(),
866
- text: "hello",
867
- });
868
-
869
- let first = await reader.read();
870
- while (!first.done && first.value?.data.kind !== "user.text_received") {
871
- first = await reader.read();
872
- }
873
- reader.releaseLock();
874
-
875
- expect(first.value).toMatchObject({
876
- component: "bus",
877
- type: "packet",
878
- data: {
879
- context_id: "turn-1",
880
- route: "Main",
881
- kind: "user.text_received",
882
- },
883
- });
884
-
885
- await closeSession(session);
886
- });
887
-
888
- it("finalizes plugins in deterministic reverse stage order", async () => {
889
- const closeOrder: string[] = [];
890
- const session = new VoiceAgentSession({
891
- plugins: {
892
- recorder: {},
893
- tts: {},
894
- vad: {},
895
- stt: {},
896
- },
897
- });
898
-
899
- session.registerPlugin("recorder", new OrderedClosePlugin("recorder", closeOrder));
900
- session.registerPlugin("tts", new OrderedClosePlugin("tts", closeOrder));
901
- session.registerPlugin("vad", new OrderedClosePlugin("vad", closeOrder));
902
- session.registerPlugin("stt", new OrderedClosePlugin("stt", closeOrder));
903
-
904
- await session.start();
905
- await session.close();
906
-
907
- expect(closeOrder).toEqual(["vad", "tts", "stt", "recorder"]);
908
- });
909
-
910
- it("shares one in-flight close across concurrent callers", async () => {
911
- const plugin = new SlowClosePlugin();
912
- const session = new VoiceAgentSession({ plugins: { recorder: {} } });
913
-
914
- session.registerPlugin("recorder", plugin);
915
- await session.start();
916
- await Promise.all([session.close(), session.close(), session.close()]);
917
-
918
- expect(plugin.closeCount).toBe(1);
919
- expect(session.state).toBe("closed");
920
- });
921
-
922
- it("tears down initialized plugins in reverse order after init failure", async () => {
923
- const closeOrder: string[] = [];
924
- const errors: Array<{ stage: string; message: string }> = [];
925
- const session = new VoiceAgentSession({
926
- plugins: {
927
- recorder: {},
928
- stt: {},
929
- tts: {},
930
- },
931
- });
932
-
933
- session.registerPlugin("recorder", new OrderedClosePlugin("recorder", closeOrder));
934
- session.registerPlugin("stt", new OrderedClosePlugin("stt", closeOrder));
935
- session.registerPlugin("tts", new FailingInitPlugin());
936
- session.on("error", (event) => {
937
- errors.push({ stage: event.stage, message: event.message });
938
- });
939
-
940
- await expect(session.start()).rejects.toThrow("Initialization failed at tts/tts");
941
- await new Promise((resolve) => setTimeout(resolve, 20));
942
-
943
- expect(session.state).toBe("failed");
944
- expect(closeOrder).toEqual(["stt", "recorder"]);
945
- expect(errors).toEqual([
946
- expect.objectContaining({
947
- stage: "init.tts",
948
- message: expect.stringContaining("Initialization failed: tts/tts"),
949
- }),
950
- ]);
951
-
952
- await closeSession(session);
953
- });
954
-
955
- it("switches audio to text immediately and tears down audio plugins in background", async () => {
956
- const closeOrder: string[] = [];
957
- const session = new VoiceAgentSession({
958
- plugins: {
959
- stt: {},
960
- tts: {},
961
- vad: {},
962
- },
963
- });
964
- const completed: ModeSwitchCompletedPacket[] = [];
965
-
966
- session.registerPlugin("stt", new OrderedClosePlugin("stt", closeOrder));
967
- session.registerPlugin("tts", new OrderedClosePlugin("tts", closeOrder));
968
- session.registerPlugin("vad", new OrderedClosePlugin("vad", closeOrder));
969
-
970
- await session.start();
971
- session.bus.on("mode.switch_completed", (pkt) => {
972
- completed.push(pkt as ModeSwitchCompletedPacket);
973
- });
974
-
975
- await session.switchMode("text");
976
- await new Promise((resolve) => setTimeout(resolve, 30));
977
-
978
- expect(completed).toEqual([
979
- expect.objectContaining({
980
- kind: "mode.switch_completed",
981
- mode: "text",
982
- }),
983
- ]);
984
- expect(closeOrder).toEqual(["vad", "tts", "stt"]);
985
-
986
- await closeSession(session);
987
- });
988
-
989
- it("routes EOS completions to normalized user input", async () => {
990
- const session = new VoiceAgentSession({ plugins: {} });
991
- const userInputs: UserInputPacket[] = [];
992
-
993
- await session.start();
994
- session.bus.on("user.input", (pkt) => {
995
- userInputs.push(pkt as UserInputPacket);
996
- });
997
-
998
- const eosPacket: EndOfSpeechPacket = {
999
- kind: "eos.turn_complete",
1000
- contextId: "turn-1",
1001
- timestampMs: Date.now(),
1002
- text: "hello world",
1003
- transcripts: [
1004
- {
1005
- kind: "stt.result",
1006
- contextId: "turn-1",
1007
- timestampMs: Date.now(),
1008
- text: "hello world",
1009
- confidence: 0.9,
1010
- language: "en-US",
1011
- },
1012
- ],
1013
- };
1014
- session.bus.push(Route.Main, eosPacket);
1015
-
1016
- await new Promise((resolve) => setTimeout(resolve, 20));
1017
-
1018
- expect(userInputs).toEqual([
1019
- {
1020
- kind: "user.input",
1021
- contextId: "turn-1",
1022
- timestampMs: expect.any(Number),
1023
- text: "hello world",
1024
- language: "en-US",
1025
- },
1026
- ]);
1027
-
1028
- await closeSession(session);
1029
- });
1030
-
1031
- it("routes sentence-complete LLM output to TTS text and done packets", async () => {
1032
- const session = new VoiceAgentSession({ plugins: {} });
1033
- const ttsText: TextToSpeechTextPacket[] = [];
1034
- const ttsDone: TextToSpeechDonePacket[] = [];
1035
-
1036
- await session.start();
1037
- session.bus.on("tts.text", (pkt) => {
1038
- ttsText.push(pkt as TextToSpeechTextPacket);
1039
- });
1040
- session.bus.on("tts.done", (pkt) => {
1041
- ttsDone.push(pkt as TextToSpeechDonePacket);
1042
- });
1043
-
1044
- const deltaPacket: LlmDeltaPacket = {
1045
- kind: "llm.delta",
1046
- contextId: "turn-1",
1047
- timestampMs: Date.now(),
1048
- text: "Hello ",
1049
- };
1050
- const deltaPacket2: LlmDeltaPacket = {
1051
- kind: "llm.delta",
1052
- contextId: "turn-1",
1053
- timestampMs: Date.now(),
1054
- text: "there. How can I help",
1055
- };
1056
- const donePacket: LlmResponseDonePacket = {
1057
- kind: "llm.done",
1058
- contextId: "turn-1",
1059
- timestampMs: Date.now(),
1060
- text: "Hello there. How can I help",
1061
- };
1062
- session.bus.push(Route.Main, deltaPacket);
1063
- session.bus.push(Route.Main, deltaPacket2);
1064
- session.bus.push(Route.Main, donePacket);
1065
-
1066
- await new Promise((resolve) => setTimeout(resolve, 20));
1067
-
1068
- expect(ttsText).toEqual([
1069
- {
1070
- kind: "tts.text",
1071
- contextId: "turn-1",
1072
- timestampMs: expect.any(Number),
1073
- text: "Hello there.",
1074
- },
1075
- {
1076
- kind: "tts.text",
1077
- contextId: "turn-1",
1078
- timestampMs: expect.any(Number),
1079
- text: "How can I help",
1080
- },
1081
- ]);
1082
- expect(ttsDone).toEqual([
1083
- {
1084
- kind: "tts.done",
1085
- contextId: "turn-1",
1086
- timestampMs: expect.any(Number),
1087
- text: "Hello there. How can I help",
1088
- },
1089
- ]);
1090
-
1091
- await closeSession(session);
1092
- });
1093
-
1094
- it("flushes final LLM tails to TTS when the provider completes", async () => {
1095
- const session = new VoiceAgentSession({ plugins: {} });
1096
- const ttsText: TextToSpeechTextPacket[] = [];
1097
- const ttsDone: TextToSpeechDonePacket[] = [];
1098
- const flushed: string[] = [];
1099
-
1100
- await session.start();
1101
- session.bus.on("tts.text", (pkt) => {
1102
- ttsText.push(pkt as TextToSpeechTextPacket);
1103
- });
1104
- session.bus.on("tts.done", (pkt) => {
1105
- ttsDone.push(pkt as TextToSpeechDonePacket);
1106
- });
1107
- session.bus.on("metric.conversation", (pkt) => {
1108
- const metric = pkt as unknown as { name: string; value: string };
1109
- if (metric.name === "tts.final_tail_flushed") flushed.push(metric.value);
1110
- });
1111
-
1112
- session.bus.push(Route.Main, {
1113
- kind: "llm.delta",
1114
- contextId: "turn-1",
1115
- timestampMs: Date.now(),
1116
- text: "You should contact your instructor and upload their email",
1117
- } satisfies LlmDeltaPacket);
1118
- session.bus.push(Route.Main, {
1119
- kind: "llm.done",
1120
- contextId: "turn-1",
1121
- timestampMs: Date.now(),
1122
- text: "You should contact your instructor and upload their email",
1123
- } satisfies LlmResponseDonePacket);
1124
-
1125
- await new Promise((resolve) => setTimeout(resolve, 40));
1126
-
1127
- expect(ttsText).toEqual([
1128
- {
1129
- kind: "tts.text",
1130
- contextId: "turn-1",
1131
- timestampMs: expect.any(Number),
1132
- text: "You should contact your instructor and upload their email",
1133
- },
1134
- ]);
1135
- expect(ttsDone).toEqual([
1136
- {
1137
- kind: "tts.done",
1138
- contextId: "turn-1",
1139
- timestampMs: expect.any(Number),
1140
- text: "You should contact your instructor and upload their email",
1141
- },
1142
- ]);
1143
- expect(flushed).toEqual(["You should contact your instructor and upload their email"]);
1144
-
1145
- await closeSession(session);
1146
- });
1147
-
1148
- it("streams non-English terminal punctuation as complete TTS text", async () => {
1149
- const session = new VoiceAgentSession({ plugins: {} });
1150
- const ttsText: TextToSpeechTextPacket[] = [];
1151
-
1152
- await session.start();
1153
- session.bus.on("tts.text", (pkt) => {
1154
- ttsText.push(pkt as TextToSpeechTextPacket);
1155
- });
1156
-
1157
- session.bus.push(Route.Main, {
1158
- kind: "llm.delta",
1159
- contextId: "turn-1",
1160
- timestampMs: Date.now(),
1161
- text: "手続きできます。次の文はまだ",
1162
- } satisfies LlmDeltaPacket);
1163
- session.bus.push(Route.Main, {
1164
- kind: "llm.done",
1165
- contextId: "turn-1",
1166
- timestampMs: Date.now(),
1167
- text: "手続きできます。次の文はまだ",
1168
- } satisfies LlmResponseDonePacket);
1169
-
1170
- await new Promise((resolve) => setTimeout(resolve, 40));
1171
-
1172
- expect(ttsText).toEqual([
1173
- {
1174
- kind: "tts.text",
1175
- contextId: "turn-1",
1176
- timestampMs: expect.any(Number),
1177
- text: "手続きできます。",
1178
- },
1179
- {
1180
- kind: "tts.text",
1181
- contextId: "turn-1",
1182
- timestampMs: expect.any(Number),
1183
- text: "次の文はまだ",
1184
- },
1185
- ]);
1186
-
1187
- await closeSession(session);
1188
- });
1189
-
1190
- it("routes TTS audio to assistant recording", async () => {
1191
- const session = new VoiceAgentSession({ plugins: {} });
1192
- const recorded: RecordAssistantAudioPacket[] = [];
1193
-
1194
- await session.start();
1195
- session.bus.on("record.assistant_audio", (pkt) => {
1196
- recorded.push(pkt as RecordAssistantAudioPacket);
1197
- });
1198
-
1199
- const audio = new Uint8Array([1, 2, 3, 4]);
1200
- const ttsAudioPacket: TextToSpeechAudioPacket = {
1201
- kind: "tts.audio",
1202
- contextId: "turn-1",
1203
- timestampMs: Date.now(),
1204
- audio,
1205
- sampleRateHz: 16000,
1206
- };
1207
- session.bus.push(Route.Main, ttsAudioPacket);
1208
-
1209
- await new Promise((resolve) => setTimeout(resolve, 20));
1210
-
1211
- expect(recorded).toEqual([
1212
- {
1213
- kind: "record.assistant_audio",
1214
- contextId: "turn-1",
1215
- timestampMs: expect.any(Number),
1216
- audio,
1217
- sampleRateHz: 16000,
1218
- truncate: false,
1219
- },
1220
- ]);
1221
-
1222
- await closeSession(session);
1223
- });
1224
-
1225
- it("rejects TTS audio without sample-rate metadata before recording it", async () => {
1226
- const session = new VoiceAgentSession({ plugins: {} });
1227
- const recorded: RecordAssistantAudioPacket[] = [];
1228
- const errors: Array<{ stage: string; message: string }> = [];
1229
-
1230
- await session.start();
1231
- session.bus.on("record.assistant_audio", (pkt) => {
1232
- recorded.push(pkt as RecordAssistantAudioPacket);
1233
- });
1234
- session.on("error", (event) => {
1235
- errors.push({ stage: event.stage, message: event.message });
1236
- });
1237
-
1238
- session.bus.push(Route.Main, {
1239
- kind: "tts.audio",
1240
- contextId: "turn-missing-rate",
1241
- timestampMs: Date.now(),
1242
- audio: new Uint8Array([1, 2, 3, 4]),
1243
- } as unknown as TextToSpeechAudioPacket);
1244
-
1245
- await new Promise((resolve) => setTimeout(resolve, 20));
1246
-
1247
- expect(recorded).toEqual([]);
1248
- expect(errors).toEqual([
1249
- expect.objectContaining({
1250
- stage: "pipeline.error",
1251
- message: "tts.audio sampleRateHz must be a positive integer",
1252
- }),
1253
- ]);
1254
-
1255
- await closeSession(session);
1256
- });
1257
-
1258
- it("uses TTS audio sample-rate metadata for idle playback timing", async () => {
1259
- vi.useFakeTimers();
1260
- vi.setSystemTime(0);
1261
-
1262
- const session = new VoiceAgentSession({
1263
- plugins: {},
1264
- idleTimeout: {
1265
- durationMs: 100,
1266
- maxConsecutive: 0,
1267
- escalationMessages: ["still there?"],
1268
- disconnectAfterMax: false,
1269
- },
1270
- });
1271
- const injected: string[] = [];
1272
-
1273
- await session.start();
1274
- session.bus.on("inject.message", (pkt) => {
1275
- injected.push((pkt as unknown as { text: string }).text);
1276
- });
1277
-
1278
- try {
1279
- session.bus.push(Route.Main, {
1280
- kind: "behavior.idle_timeout_start",
1281
- contextId: "turn-1",
1282
- timestampMs: Date.now(),
1283
- });
1284
- await vi.advanceTimersByTimeAsync(0);
1285
-
1286
- session.bus.push(Route.Main, {
1287
- kind: "tts.audio",
1288
- contextId: "turn-1",
1289
- timestampMs: Date.now(),
1290
- audio: new Uint8Array(3200),
1291
- sampleRateHz: 16000,
1292
- } satisfies TextToSpeechAudioPacket);
1293
- await vi.advanceTimersByTimeAsync(0);
1294
-
1295
- await vi.advanceTimersByTimeAsync(199);
1296
- expect(injected).toEqual([]);
1297
-
1298
- await vi.advanceTimersByTimeAsync(1);
1299
- await vi.advanceTimersByTimeAsync(0);
1300
- expect(injected).toEqual(["still there?"]);
1301
- } finally {
1302
- vi.useRealTimers();
1303
- await closeSession(session);
1304
- }
1305
- });
1306
-
1307
- it("stops assistant audio output within 50ms of VAD barge-in", async () => {
1308
- const tts = new InterruptAwareStreamingTtsPlugin();
1309
- // Gate disabled: this test exercises the immediate-cut path latency.
1310
- const session = new VoiceAgentSession({ plugins: { tts: {} }, minInterruptionMs: 0 });
1311
- const recordedAtMs: number[] = [];
1312
- const interrupts: InterruptTtsPacket[] = [];
1313
-
1314
- session.registerPlugin("tts", tts);
1315
- await session.start();
1316
- session.bus.on("record.assistant_audio", () => {
1317
- recordedAtMs.push(performance.now());
1318
- });
1319
- session.bus.on("interrupt.tts", (pkt) => {
1320
- interrupts.push(pkt as InterruptTtsPacket);
1321
- });
1322
-
1323
- session.bus.push(Route.Main, {
1324
- kind: "tts.text",
1325
- contextId: "assistant-turn",
1326
- timestampMs: Date.now(),
1327
- text: "stream until interrupted",
1328
- } satisfies TextToSpeechTextPacket);
1329
-
1330
- while (recordedAtMs.length < 2) {
1331
- await new Promise((resolve) => setTimeout(resolve, 1));
1332
- }
1333
-
1334
- const vadDetectedAtMs = performance.now();
1335
- session.bus.push(Route.Main, {
1336
- kind: "vad.speech_started",
1337
- contextId: "user-barge-in",
1338
- timestampMs: Date.now(),
1339
- confidence: 0.99,
1340
- } satisfies VadSpeechStartedPacket);
1341
-
1342
- await new Promise((resolve) => setTimeout(resolve, 80));
1343
-
1344
- const lastAudioAfterVadMs = Math.max(
1345
- 0,
1346
- ...recordedAtMs.filter((timestampMs) => timestampMs >= vadDetectedAtMs)
1347
- .map((timestampMs) => timestampMs - vadDetectedAtMs),
1348
- );
1349
-
1350
- expect(interrupts).toEqual([
1351
- expect.objectContaining({
1352
- kind: "interrupt.tts",
1353
- contextId: "assistant-turn",
1354
- }),
1355
- ]);
1356
- expect(tts.interruptObservedAtMs - vadDetectedAtMs).toBeLessThan(50);
1357
- expect(lastAudioAfterVadMs).toBeLessThan(50);
1358
-
1359
- await closeSession(session);
1360
- });
1361
-
1362
- it("commits a barge-in only after user speech is sustained past minInterruptionMs", async () => {
1363
- const session = new VoiceAgentSession({ plugins: {}, minInterruptionMs: 280 });
1364
- const interrupts: InterruptTtsPacket[] = [];
1365
- const metrics: string[] = [];
1366
-
1367
- await session.start();
1368
- session.bus.on("interrupt.tts", (pkt) => {
1369
- interrupts.push(pkt as InterruptTtsPacket);
1370
- });
1371
- session.bus.on("metric.conversation", (pkt) => {
1372
- metrics.push((pkt as unknown as { name: string }).name);
1373
- });
1374
-
1375
- // Assistant is speaking.
1376
- session.bus.push(Route.Main, {
1377
- kind: "tts.audio",
1378
- contextId: "assistant-turn",
1379
- timestampMs: Date.now(),
1380
- audio: new Uint8Array([1, 2, 3, 4]),
1381
- sampleRateHz: 16000,
1382
- } satisfies TextToSpeechAudioPacket);
1383
- await new Promise((resolve) => setTimeout(resolve, 10));
1384
-
1385
- const t0 = Date.now();
1386
- session.bus.push(Route.Main, {
1387
- kind: "vad.speech_started",
1388
- contextId: "user",
1389
- timestampMs: t0,
1390
- confidence: 0.99,
1391
- } satisfies VadSpeechStartedPacket);
1392
- // Activity below threshold — no interrupt yet.
1393
- session.bus.push(Route.Main, {
1394
- kind: "vad.speech_activity",
1395
- contextId: "user",
1396
- timestampMs: t0 + 100,
1397
- isAsync: true,
1398
- } satisfies VadSpeechActivityPacket);
1399
- await new Promise((resolve) => setTimeout(resolve, 20));
1400
- expect(interrupts).toEqual([]);
1401
-
1402
- // Activity past threshold — interrupt commits.
1403
- session.bus.push(Route.Main, {
1404
- kind: "vad.speech_activity",
1405
- contextId: "user",
1406
- timestampMs: t0 + 300,
1407
- isAsync: true,
1408
- } satisfies VadSpeechActivityPacket);
1409
- await new Promise((resolve) => setTimeout(resolve, 20));
1410
-
1411
- expect(interrupts).toEqual([
1412
- expect.objectContaining({ kind: "interrupt.tts", contextId: "assistant-turn" }),
1413
- ]);
1414
- expect(metrics).toContain("interrupt.committed_after_ms");
1415
-
1416
- await closeSession(session);
1417
- });
1418
-
1419
- it("fullDuplex:true runs the interaction policy observe-only (no VAD-driven interrupt)", async () => {
1420
- const session = new VoiceAgentSession({ plugins: {}, minInterruptionMs: 280, fullDuplex: true });
1421
- const interrupts: InterruptTtsPacket[] = [];
1422
-
1423
- await session.start();
1424
- session.bus.on("interrupt.tts", (pkt) => {
1425
- interrupts.push(pkt as InterruptTtsPacket);
1426
- });
1427
-
1428
- session.bus.push(Route.Main, {
1429
- kind: "tts.audio",
1430
- contextId: "assistant-turn",
1431
- timestampMs: Date.now(),
1432
- audio: new Uint8Array([1, 2, 3, 4]),
1433
- sampleRateHz: 16000,
1434
- } satisfies TextToSpeechAudioPacket);
1435
- await new Promise((resolve) => setTimeout(resolve, 10));
1436
-
1437
- const t0 = Date.now();
1438
- session.bus.push(Route.Main, {
1439
- kind: "vad.speech_started",
1440
- contextId: "user",
1441
- timestampMs: t0,
1442
- confidence: 0.99,
1443
- } satisfies VadSpeechStartedPacket);
1444
- session.bus.push(Route.Main, {
1445
- kind: "vad.speech_activity",
1446
- contextId: "user",
1447
- timestampMs: t0 + 100,
1448
- isAsync: true,
1449
- } satisfies VadSpeechActivityPacket);
1450
- await new Promise((resolve) => setTimeout(resolve, 20));
1451
- expect(interrupts).toEqual([]);
1452
-
1453
- session.bus.push(Route.Main, {
1454
- kind: "vad.speech_activity",
1455
- contextId: "user",
1456
- timestampMs: t0 + 300,
1457
- isAsync: true,
1458
- } satisfies VadSpeechActivityPacket);
1459
- await new Promise((resolve) => setTimeout(resolve, 20));
1460
-
1461
- expect(interrupts).toEqual([]);
1462
-
1463
- await closeSession(session);
1464
- });
1465
-
1466
- it("fullDuplex:true still honors a direct client interrupt (executor survives defer mode) (IP-C2 regression)", async () => {
1467
- // Defer mode swaps only the coordinator's DRIVE policy to observe-only; the executor stays the
1468
- // rule policy's arbiter, so a client-initiated "stop" (requestClientInterrupt) must still fire —
1469
- // the front owning turn-taking does not disable the user's explicit interrupt.
1470
- const session = new VoiceAgentSession({ plugins: {}, minInterruptionMs: 280, fullDuplex: true });
1471
- const interrupts: InterruptTtsPacket[] = [];
1472
-
1473
- await session.start();
1474
- session.bus.on("interrupt.tts", (pkt) => {
1475
- interrupts.push(pkt as InterruptTtsPacket);
1476
- });
1477
-
1478
- session.bus.push(Route.Main, {
1479
- kind: "tts.audio",
1480
- contextId: "assistant-turn",
1481
- timestampMs: Date.now(),
1482
- audio: new Uint8Array([1, 2, 3, 4]),
1483
- sampleRateHz: 16000,
1484
- } satisfies TextToSpeechAudioPacket);
1485
- await new Promise((resolve) => setTimeout(resolve, 10));
1486
-
1487
- session.requestClientInterrupt("assistant-turn");
1488
- await new Promise((resolve) => setTimeout(resolve, 20));
1489
-
1490
- expect(interrupts).toEqual([
1491
- expect.objectContaining({ kind: "interrupt.tts", contextId: "assistant-turn" }),
1492
- ]);
1493
-
1494
- await closeSession(session);
1495
- });
1496
-
1497
- it("stt.partial carries wordTimings but does NOT itself drive barge-in (IP-C4 no-double-drive guard)", async () => {
1498
- // IP-C4: stt.partial is the rich-seam carrier; the session caches its wordTimings for the
1499
- // observation but barge-in stays driven ONLY by stt.interim/stt.result. Pushing sustained
1500
- // stt.partial (no interim) during active TTS must NOT interrupt — proving no second barge-in driver.
1501
- const session = new VoiceAgentSession({ plugins: {}, minInterruptionMs: 280 });
1502
- const interrupts: InterruptTtsPacket[] = [];
1503
-
1504
- await session.start();
1505
- session.bus.on("interrupt.tts", (pkt) => {
1506
- interrupts.push(pkt as InterruptTtsPacket);
1507
- });
1508
- session.bus.push(Route.Main, {
1509
- kind: "tts.audio",
1510
- contextId: "assistant-turn",
1511
- timestampMs: Date.now(),
1512
- audio: new Uint8Array([1, 2, 3, 4]),
1513
- sampleRateHz: 16000,
1514
- } satisfies TextToSpeechAudioPacket);
1515
- await new Promise((resolve) => setTimeout(resolve, 10));
1516
-
1517
- const t0 = Date.now();
1518
- session.bus.push(Route.Main, {
1519
- kind: "stt.partial",
1520
- contextId: "user",
1521
- timestampMs: t0,
1522
- text: "wait actually I need",
1523
- wordTimings: [{ word: "wait", startMs: 0, endMs: 200, confidence: 0.9 }],
1524
- } satisfies SttPartialPacket);
1525
- session.bus.push(Route.Main, {
1526
- kind: "stt.partial",
1527
- contextId: "user",
1528
- timestampMs: t0 + 400,
1529
- text: "wait actually I need something else",
1530
- wordTimings: [{ word: "wait", startMs: 0, endMs: 200, confidence: 0.9 }],
1531
- } satisfies SttPartialPacket);
1532
- await new Promise((resolve) => setTimeout(resolve, 20));
1533
-
1534
- expect(interrupts).toEqual([]);
1535
-
1536
- await closeSession(session);
1537
- });
1538
-
1539
- it("commits a barge-in from provider STT interim transcripts when no VAD plugin is registered", async () => {
1540
- // Cascade deployments with endpointingOwner "provider_stt" (the default) have
1541
- // no vad.speech_started producer — interim transcripts during TTS playout are
1542
- // the barge-in evidence.
1543
- const session = new VoiceAgentSession({ plugins: {}, minInterruptionMs: 280 });
1544
- const interrupts: InterruptTtsPacket[] = [];
1545
- const metrics: string[] = [];
1546
-
1547
- await session.start();
1548
- session.bus.on("interrupt.tts", (pkt) => {
1549
- interrupts.push(pkt as InterruptTtsPacket);
1550
- });
1551
- session.bus.on("metric.conversation", (pkt) => {
1552
- metrics.push((pkt as unknown as { name: string }).name);
1553
- });
1554
-
1555
- // Assistant is speaking.
1556
- session.bus.push(Route.Main, {
1557
- kind: "tts.audio",
1558
- contextId: "assistant-turn",
1559
- timestampMs: Date.now(),
1560
- audio: new Uint8Array([1, 2, 3, 4]),
1561
- sampleRateHz: 16000,
1562
- } satisfies TextToSpeechAudioPacket);
1563
- await new Promise((resolve) => setTimeout(resolve, 10));
1564
-
1565
- const t0 = Date.now();
1566
- session.bus.push(Route.Main, {
1567
- kind: "stt.interim",
1568
- contextId: "user",
1569
- timestampMs: t0,
1570
- text: "wait actually",
1571
- } satisfies SttInterimPacket);
1572
- await new Promise((resolve) => setTimeout(resolve, 20));
1573
- expect(interrupts).toEqual([]);
1574
-
1575
- session.bus.push(Route.Main, {
1576
- kind: "stt.interim",
1577
- contextId: "user",
1578
- timestampMs: t0 + 300,
1579
- text: "wait actually I need something else",
1580
- } satisfies SttInterimPacket);
1581
- await new Promise((resolve) => setTimeout(resolve, 20));
1582
-
1583
- expect(interrupts).toEqual([
1584
- expect.objectContaining({ kind: "interrupt.tts", contextId: "assistant-turn" }),
1585
- ]);
1586
- expect(metrics).toContain("interrupt.committed_after_ms");
1587
-
1588
- await closeSession(session);
1589
- });
1590
-
1591
- it("attaches cached stt.partial wordTimings to the next barge-in observation (IP-C4)", async () => {
1592
- const session = new VoiceAgentSession({ plugins: {}, minInterruptionMs: 280 });
1593
- const observations: InteractionObservation[] = [];
1594
- await session.start();
1595
-
1596
- const interaction = (session as unknown as { interaction: InteractionCoordinator }).interaction;
1597
- const originalObserve = interaction.observe.bind(interaction);
1598
- interaction.observe = (obs) => {
1599
- observations.push(obs);
1600
- originalObserve(obs);
1601
- };
1602
-
1603
- session.bus.push(Route.Main, {
1604
- kind: "stt.partial",
1605
- contextId: "user",
1606
- timestampMs: Date.now(),
1607
- text: "wait actually",
1608
- wordTimings: [{ word: "wait", startMs: 100, endMs: 300, confidence: 0.9 }],
1609
- } satisfies SttPartialPacket);
1610
-
1611
- session.bus.push(Route.Main, {
1612
- kind: "stt.interim",
1613
- contextId: "user",
1614
- timestampMs: Date.now(),
1615
- text: "wait actually",
1616
- } satisfies SttInterimPacket);
1617
- await new Promise((resolve) => setTimeout(resolve, 20));
1618
-
1619
- expect(observations).toEqual(
1620
- expect.arrayContaining([
1621
- expect.objectContaining({
1622
- kind: "stt_partial",
1623
- text: "wait actually",
1624
- wordTimings: [{ word: "wait", startMs: 100, endMs: 300, confidence: 0.9 }],
1625
- }),
1626
- ]),
1627
- );
1628
-
1629
- await closeSession(session);
1630
- });
1631
-
1632
- it("suppresses a backchannel provider-STT interim through the interaction seam (IP-C1 regression)", async () => {
1633
- // IP-C1 routed provider-STT barge-in through InteractionCoordinator ->
1634
- // RuleBasedInteractionPolicy -> TurnArbiter. This pins that the reshaped
1635
- // chain still SUPPRESSES a sustained backchannel ("okay") rather than cutting
1636
- // the assistant — the session-level suppression case the policy-unit test
1637
- // (rule-based.test.ts, vad-driven) and the commit test above do not cover.
1638
- const session = new VoiceAgentSession({ plugins: {}, minInterruptionMs: 280 });
1639
- const interrupts: InterruptTtsPacket[] = [];
1640
- const metrics: string[] = [];
1641
-
1642
- await session.start();
1643
- session.bus.on("interrupt.tts", (pkt) => {
1644
- interrupts.push(pkt as InterruptTtsPacket);
1645
- });
1646
- session.bus.on("metric.conversation", (pkt) => {
1647
- metrics.push((pkt as unknown as { name: string }).name);
1648
- });
1649
-
1650
- // Assistant is speaking.
1651
- session.bus.push(Route.Main, {
1652
- kind: "tts.audio",
1653
- contextId: "assistant-turn",
1654
- timestampMs: Date.now(),
1655
- audio: new Uint8Array([1, 2, 3, 4]),
1656
- sampleRateHz: 16000,
1657
- } satisfies TextToSpeechAudioPacket);
1658
- await new Promise((resolve) => setTimeout(resolve, 10));
1659
-
1660
- // Sustained-past-minInterruptionMs backchannel evidence: opens the pending
1661
- // window, then the second interim (past 280ms) reaches tryCommit -> the
1662
- // arbiter's backchannel suppression fires instead of an interrupt decision.
1663
- const t0 = Date.now();
1664
- session.bus.push(Route.Main, {
1665
- kind: "stt.interim",
1666
- contextId: "user",
1667
- timestampMs: t0,
1668
- text: "okay",
1669
- } satisfies SttInterimPacket);
1670
- await new Promise((resolve) => setTimeout(resolve, 20));
1671
- session.bus.push(Route.Main, {
1672
- kind: "stt.interim",
1673
- contextId: "user",
1674
- timestampMs: t0 + 300,
1675
- text: "okay",
1676
- } satisfies SttInterimPacket);
1677
- await new Promise((resolve) => setTimeout(resolve, 20));
1678
-
1679
- expect(interrupts).toEqual([]);
1680
- expect(metrics).toContain("interrupt.suppressed_backchannel");
1681
-
1682
- await closeSession(session);
1683
- });
1684
-
1685
- it("emits interrupt.onset_to_logic_cancel_ms and stamps interrupt.tts/llm with detected onset", async () => {
1686
- const session = new VoiceAgentSession({ plugins: {} });
1687
- const onsetMetrics: Array<{ name: string; value: string }> = [];
1688
- const ttsInterrupts: InterruptTtsPacket[] = [];
1689
- const llmInterrupts: InterruptLlmPacket[] = [];
1690
-
1691
- await session.start();
1692
- session.bus.on("metric.conversation", (pkt) => {
1693
- const metric = pkt as unknown as { name: string; value: string };
1694
- if (metric.name === "interrupt.onset_to_logic_cancel_ms") onsetMetrics.push(metric);
1695
- });
1696
- session.bus.on("interrupt.tts", (pkt) => {
1697
- ttsInterrupts.push(pkt as InterruptTtsPacket);
1698
- });
1699
- session.bus.on("interrupt.llm", (pkt) => {
1700
- llmInterrupts.push(pkt as InterruptLlmPacket);
1701
- });
1702
-
1703
- const onset = 1_700_000_000_000;
1704
- session.bus.push(Route.Critical, {
1705
- kind: "interrupt.detected",
1706
- contextId: "assistant-turn",
1707
- timestampMs: onset,
1708
- source: "vad",
1709
- } satisfies InterruptionDetectedPacket);
1710
- await new Promise((resolve) => setTimeout(resolve, 10));
1711
-
1712
- expect(onsetMetrics).toEqual([
1713
- expect.objectContaining({
1714
- name: "interrupt.onset_to_logic_cancel_ms",
1715
- value: expect.stringMatching(/^\d+$/),
1716
- }),
1717
- ]);
1718
- expect(Number(onsetMetrics[0]!.value)).toBeGreaterThanOrEqual(0);
1719
- expect(ttsInterrupts).toEqual([
1720
- expect.objectContaining({ kind: "interrupt.tts", contextId: "assistant-turn", timestampMs: onset }),
1721
- ]);
1722
- expect(llmInterrupts).toEqual([
1723
- expect.objectContaining({ kind: "interrupt.llm", contextId: "assistant-turn", timestampMs: onset }),
1724
- ]);
1725
-
1726
- await closeSession(session);
1727
- });
1728
-
1729
- it("keeps the assistant interruptible after tts.end until its audio finishes playing out", async () => {
1730
- // TTS streams faster than realtime: a chunk representing ~800ms of audio can
1731
- // arrive (and tts.end fire) within a few ms. The assistant is still audibly
1732
- // playing for the remaining ~800ms, so a barge-in in that window must still
1733
- // interrupt it — the speaking state is keyed on playout, not generation.
1734
- const session = new VoiceAgentSession({ plugins: {}, minInterruptionMs: 0 });
1735
- const interrupts: InterruptTtsPacket[] = [];
1736
-
1737
- await session.start();
1738
- session.bus.on("interrupt.tts", (pkt) => {
1739
- interrupts.push(pkt as InterruptTtsPacket);
1740
- });
1741
-
1742
- // 25600 bytes @ 16 kHz s16 = 800ms of playout, delivered as one burst.
1743
- session.bus.push(Route.Main, {
1744
- kind: "tts.audio",
1745
- contextId: "assistant-turn",
1746
- timestampMs: Date.now(),
1747
- audio: new Uint8Array(25600),
1748
- sampleRateHz: 16000,
1749
- } satisfies TextToSpeechAudioPacket);
1750
- session.bus.push(Route.Main, {
1751
- kind: "tts.end",
1752
- contextId: "assistant-turn",
1753
- timestampMs: Date.now(),
1754
- } satisfies TextToSpeechEndPacket);
1755
- // Well inside the 800ms playout window — the assistant is still talking.
1756
- await new Promise((resolve) => setTimeout(resolve, 60));
1757
-
1758
- session.bus.push(Route.Main, {
1759
- kind: "vad.speech_started",
1760
- contextId: "user",
1761
- timestampMs: Date.now(),
1762
- confidence: 0.99,
1763
- } satisfies VadSpeechStartedPacket);
1764
- await new Promise((resolve) => setTimeout(resolve, 20));
1765
-
1766
- expect(interrupts).toEqual([
1767
- expect.objectContaining({ kind: "interrupt.tts", contextId: "assistant-turn" }),
1768
- ]);
1769
-
1770
- await closeSession(session);
1771
- });
1772
-
1773
- it("releases the assistant context once its playout estimate elapses", async () => {
1774
- const session = new VoiceAgentSession({ plugins: {}, minInterruptionMs: 0 });
1775
- const interrupts: InterruptTtsPacket[] = [];
1776
-
1777
- await session.start();
1778
- session.bus.on("interrupt.tts", (pkt) => {
1779
- interrupts.push(pkt as InterruptTtsPacket);
1780
- });
1781
-
1782
- // 3200 bytes @ 16 kHz s16 = 100ms of playout.
1783
- session.bus.push(Route.Main, {
1784
- kind: "tts.audio",
1785
- contextId: "assistant-turn",
1786
- timestampMs: Date.now(),
1787
- audio: new Uint8Array(3200),
1788
- sampleRateHz: 16000,
1789
- } satisfies TextToSpeechAudioPacket);
1790
- session.bus.push(Route.Main, {
1791
- kind: "tts.end",
1792
- contextId: "assistant-turn",
1793
- timestampMs: Date.now(),
1794
- } satisfies TextToSpeechEndPacket);
1795
- // Past the 100ms playout window — the assistant has finished speaking.
1796
- await new Promise((resolve) => setTimeout(resolve, 250));
1797
-
1798
- session.bus.push(Route.Main, {
1799
- kind: "vad.speech_started",
1800
- contextId: "user",
1801
- timestampMs: Date.now(),
1802
- confidence: 0.99,
1803
- } satisfies VadSpeechStartedPacket);
1804
- await new Promise((resolve) => setTimeout(resolve, 20));
1805
-
1806
- expect(interrupts).toEqual([]);
1807
-
1808
- await closeSession(session);
1809
- });
1810
-
1811
- it("keeps the context interruptible past the duration estimate while the transport reports active playout", async () => {
1812
- // A paced transport reports real playout; under send-buffer backpressure the
1813
- // audio plays longer than its sample-duration. The estimate must defer to the
1814
- // transport so barge-in stays armed for the real playout window.
1815
- const session = new VoiceAgentSession({ plugins: {}, minInterruptionMs: 0 });
1816
- const interrupts: InterruptTtsPacket[] = [];
1817
-
1818
- await session.start();
1819
- session.bus.on("interrupt.tts", (pkt) => {
1820
- interrupts.push(pkt as InterruptTtsPacket);
1821
- });
1822
-
1823
- // 3200 bytes @ 16 kHz s16 = 100ms estimate.
1824
- session.bus.push(Route.Main, {
1825
- kind: "tts.audio",
1826
- contextId: "assistant-turn",
1827
- timestampMs: Date.now(),
1828
- audio: new Uint8Array(3200),
1829
- sampleRateHz: 16000,
1830
- } satisfies TextToSpeechAudioPacket);
1831
- session.bus.push(Route.Main, {
1832
- kind: "tts.end",
1833
- contextId: "assistant-turn",
1834
- timestampMs: Date.now(),
1835
- } satisfies TextToSpeechEndPacket);
1836
- // Transport is still pacing this context (not complete) — real playout ongoing.
1837
- session.bus.push(Route.Main, {
1838
- kind: "tts.playout_progress",
1839
- contextId: "assistant-turn",
1840
- timestampMs: Date.now(),
1841
- playedOutMs: 40,
1842
- complete: false,
1843
- } satisfies TextToSpeechPlayoutProgressPacket);
1844
- // Past the 100ms estimate, but the transport has not reported completion.
1845
- await new Promise((resolve) => setTimeout(resolve, 200));
1846
-
1847
- session.bus.push(Route.Main, {
1848
- kind: "vad.speech_started",
1849
- contextId: "user",
1850
- timestampMs: Date.now(),
1851
- confidence: 0.99,
1852
- } satisfies VadSpeechStartedPacket);
1853
- await new Promise((resolve) => setTimeout(resolve, 20));
1854
-
1855
- expect(interrupts).toEqual([
1856
- expect.objectContaining({ kind: "interrupt.tts", contextId: "assistant-turn" }),
1857
- ]);
1858
-
1859
- await closeSession(session);
1860
- });
1861
-
1862
- it("releases the assistant context when the transport reports playout complete", async () => {
1863
- const session = new VoiceAgentSession({ plugins: {}, minInterruptionMs: 0 });
1864
- const interrupts: InterruptTtsPacket[] = [];
1865
-
1866
- await session.start();
1867
- session.bus.on("interrupt.tts", (pkt) => {
1868
- interrupts.push(pkt as InterruptTtsPacket);
1869
- });
1870
-
1871
- session.bus.push(Route.Main, {
1872
- kind: "tts.audio",
1873
- contextId: "assistant-turn",
1874
- timestampMs: Date.now(),
1875
- audio: new Uint8Array(25600),
1876
- sampleRateHz: 16000,
1877
- } satisfies TextToSpeechAudioPacket);
1878
- session.bus.push(Route.Main, {
1879
- kind: "tts.end",
1880
- contextId: "assistant-turn",
1881
- timestampMs: Date.now(),
1882
- } satisfies TextToSpeechEndPacket);
1883
- // Transport confirms the audio finished playing out (authoritative).
1884
- session.bus.push(Route.Main, {
1885
- kind: "tts.playout_progress",
1886
- contextId: "assistant-turn",
1887
- timestampMs: Date.now(),
1888
- playedOutMs: 800,
1889
- complete: true,
1890
- } satisfies TextToSpeechPlayoutProgressPacket);
1891
- await new Promise((resolve) => setTimeout(resolve, 20));
1892
-
1893
- session.bus.push(Route.Main, {
1894
- kind: "vad.speech_started",
1895
- contextId: "user",
1896
- timestampMs: Date.now(),
1897
- confidence: 0.99,
1898
- } satisfies VadSpeechStartedPacket);
1899
- await new Promise((resolve) => setTimeout(resolve, 20));
1900
-
1901
- expect(interrupts).toEqual([]);
1902
-
1903
- await closeSession(session);
1904
- });
1905
-
1906
- it("suppresses a short speech blip during playback without interrupting the agent", async () => {
1907
- const session = new VoiceAgentSession({ plugins: {}, minInterruptionMs: 280 });
1908
- const interrupts: InterruptTtsPacket[] = [];
1909
- const metrics: string[] = [];
1910
-
1911
- await session.start();
1912
- session.bus.on("interrupt.tts", (pkt) => {
1913
- interrupts.push(pkt as InterruptTtsPacket);
1914
- });
1915
- session.bus.on("metric.conversation", (pkt) => {
1916
- metrics.push((pkt as unknown as { name: string }).name);
1917
- });
1918
-
1919
- session.bus.push(Route.Main, {
1920
- kind: "tts.audio",
1921
- contextId: "assistant-turn",
1922
- timestampMs: Date.now(),
1923
- audio: new Uint8Array([1, 2, 3, 4]),
1924
- sampleRateHz: 16000,
1925
- } satisfies TextToSpeechAudioPacket);
1926
- await new Promise((resolve) => setTimeout(resolve, 10));
1927
-
1928
- const t0 = Date.now();
1929
- session.bus.push(Route.Main, {
1930
- kind: "vad.speech_started",
1931
- contextId: "user",
1932
- timestampMs: t0,
1933
- confidence: 0.99,
1934
- } satisfies VadSpeechStartedPacket);
1935
- session.bus.push(Route.Main, {
1936
- kind: "vad.speech_activity",
1937
- contextId: "user",
1938
- timestampMs: t0 + 90,
1939
- isAsync: true,
1940
- } satisfies VadSpeechActivityPacket);
1941
- // Speech ends before sustaining past the gate — a blip (cough / click / "mhm").
1942
- session.bus.push(Route.Main, {
1943
- kind: "vad.speech_ended",
1944
- contextId: "user",
1945
- timestampMs: t0 + 130,
1946
- } satisfies VadSpeechEndedPacket);
1947
- await new Promise((resolve) => setTimeout(resolve, 30));
1948
-
1949
- expect(interrupts).toEqual([]);
1950
- expect(metrics).toContain("interrupt.suppressed_short_speech");
1951
-
1952
- await closeSession(session);
1953
- });
1954
-
1955
- it("emits vaqi.latency_ms once per turn from user stop to first assistant audio", async () => {
1956
- const session = new VoiceAgentSession({ plugins: {} });
1957
- const metrics: Array<{ name: string; value: string }> = [];
1958
-
1959
- await session.start();
1960
- session.bus.on("metric.conversation", (pkt) => {
1961
- const m = pkt as unknown as { name: string; value: string };
1962
- if (m.name === "vaqi.latency_ms") metrics.push({ name: m.name, value: m.value });
1963
- });
1964
-
1965
- const userStoppedMs = 1000;
1966
- session.bus.push(Route.Main, {
1967
- kind: "vad.speech_ended",
1968
- contextId: "turn-1",
1969
- timestampMs: userStoppedMs,
1970
- } satisfies VadSpeechEndedPacket);
1971
-
1972
- const firstAudioMs = 1350;
1973
- session.bus.push(Route.Main, {
1974
- kind: "tts.audio",
1975
- contextId: "turn-1",
1976
- timestampMs: firstAudioMs,
1977
- audio: new Uint8Array([1, 2, 3, 4]),
1978
- sampleRateHz: 16000,
1979
- } satisfies TextToSpeechAudioPacket);
1980
-
1981
- // Second audio packet for same turn — must NOT emit a second latency metric.
1982
- session.bus.push(Route.Main, {
1983
- kind: "tts.audio",
1984
- contextId: "turn-1",
1985
- timestampMs: firstAudioMs + 50,
1986
- audio: new Uint8Array([5, 6, 7, 8]),
1987
- sampleRateHz: 16000,
1988
- } satisfies TextToSpeechAudioPacket);
1989
-
1990
- await new Promise((resolve) => setTimeout(resolve, 20));
1991
-
1992
- expect(metrics).toEqual([{ name: "vaqi.latency_ms", value: "350" }]);
1993
-
1994
- await closeSession(session);
1995
- });
1996
-
1997
- it("emits vaqi.interruption and interrupt.latency_ms when a barge-in is committed via the gate", async () => {
1998
- const session = new VoiceAgentSession({ plugins: {}, minInterruptionMs: 200 });
1999
- const metrics: Array<{ name: string; value: string }> = [];
2000
-
2001
- await session.start();
2002
- session.bus.on("metric.conversation", (pkt) => {
2003
- const m = pkt as unknown as { name: string; value: string };
2004
- if (m.name === "vaqi.interruption" || m.name === "interrupt.latency_ms") {
2005
- metrics.push({ name: m.name, value: m.value });
2006
- }
2007
- });
2008
-
2009
- session.bus.push(Route.Main, {
2010
- kind: "tts.audio",
2011
- contextId: "assistant-turn",
2012
- timestampMs: Date.now(),
2013
- audio: new Uint8Array([1, 2, 3, 4]),
2014
- sampleRateHz: 16000,
2015
- } satisfies TextToSpeechAudioPacket);
2016
- await new Promise((resolve) => setTimeout(resolve, 10));
2017
-
2018
- const t0 = 5000;
2019
- session.bus.push(Route.Main, {
2020
- kind: "vad.speech_started",
2021
- contextId: "user",
2022
- timestampMs: t0,
2023
- confidence: 0.99,
2024
- } satisfies VadSpeechStartedPacket);
2025
- session.bus.push(Route.Main, {
2026
- kind: "vad.speech_activity",
2027
- contextId: "user",
2028
- timestampMs: t0 + 300,
2029
- isAsync: true,
2030
- } satisfies VadSpeechActivityPacket);
2031
- await new Promise((resolve) => setTimeout(resolve, 20));
2032
-
2033
- expect(metrics).toContainEqual({ name: "vaqi.interruption", value: "1" });
2034
- expect(metrics).toContainEqual({ name: "interrupt.latency_ms", value: "300" });
2035
-
2036
- await closeSession(session);
2037
- });
2038
-
2039
- it("emits vaqi.interruption immediately when the interruption gate is disabled", async () => {
2040
- const session = new VoiceAgentSession({ plugins: {}, minInterruptionMs: 0 });
2041
- const metrics: Array<{ name: string; value: string }> = [];
2042
-
2043
- await session.start();
2044
- session.bus.on("metric.conversation", (pkt) => {
2045
- const m = pkt as unknown as { name: string; value: string };
2046
- if (m.name === "vaqi.interruption") metrics.push({ name: m.name, value: m.value });
2047
- });
2048
-
2049
- session.bus.push(Route.Main, {
2050
- kind: "tts.audio",
2051
- contextId: "assistant-turn",
2052
- timestampMs: Date.now(),
2053
- audio: new Uint8Array([1, 2, 3, 4]),
2054
- sampleRateHz: 16000,
2055
- } satisfies TextToSpeechAudioPacket);
2056
- await new Promise((resolve) => setTimeout(resolve, 10));
2057
-
2058
- session.bus.push(Route.Main, {
2059
- kind: "vad.speech_started",
2060
- contextId: "user-barge-in",
2061
- timestampMs: Date.now(),
2062
- confidence: 0.99,
2063
- } satisfies VadSpeechStartedPacket);
2064
- await new Promise((resolve) => setTimeout(resolve, 20));
2065
-
2066
- expect(metrics).toEqual([{ name: "vaqi.interruption", value: "1" }]);
2067
-
2068
- await closeSession(session);
2069
- });
2070
-
2071
- it("emits vaqi.missed_response when no assistant audio arrives within the window", async () => {
2072
- vi.useFakeTimers();
2073
- vi.setSystemTime(1000);
2074
-
2075
- const session = new VoiceAgentSession({
2076
- plugins: {},
2077
- vaqiMissedResponseMs: 2000,
2078
- });
2079
- const metrics: Array<{ name: string; value: string }> = [];
2080
-
2081
- await session.start();
2082
- session.bus.on("metric.conversation", (pkt) => {
2083
- const m = pkt as unknown as { name: string; value: string };
2084
- if (m.name === "vaqi.missed_response") metrics.push({ name: m.name, value: m.value });
2085
- });
2086
-
2087
- try {
2088
- session.bus.push(Route.Main, {
2089
- kind: "vad.speech_ended",
2090
- contextId: "turn-1",
2091
- timestampMs: Date.now(),
2092
- } satisfies VadSpeechEndedPacket);
2093
- await vi.advanceTimersByTimeAsync(0);
2094
-
2095
- await vi.advanceTimersByTimeAsync(1999);
2096
- expect(metrics).toEqual([]);
2097
-
2098
- await vi.advanceTimersByTimeAsync(1);
2099
- expect(metrics).toHaveLength(1);
2100
- expect(metrics[0]!.name).toBe("vaqi.missed_response");
2101
- expect(Number(metrics[0]!.value)).toBeGreaterThanOrEqual(2000);
2102
- } finally {
2103
- vi.useRealTimers();
2104
- await closeSession(session);
2105
- }
2106
- });
2107
-
2108
- it("cancels vaqi.missed_response timer on session close to avoid leaks", async () => {
2109
- const session = new VoiceAgentSession({
2110
- plugins: {},
2111
- vaqiMissedResponseMs: 30,
2112
- });
2113
- const metrics: Array<{ name: string }> = [];
2114
-
2115
- await session.start();
2116
- session.bus.on("metric.conversation", (pkt) => {
2117
- const m = pkt as unknown as { name: string };
2118
- if (m.name === "vaqi.missed_response") metrics.push({ name: m.name });
2119
- });
2120
-
2121
- session.bus.push(Route.Main, {
2122
- kind: "vad.speech_ended",
2123
- contextId: "turn-1",
2124
- timestampMs: Date.now(),
2125
- } satisfies VadSpeechEndedPacket);
2126
-
2127
- await new Promise((resolve) => setTimeout(resolve, 10));
2128
- await session.close();
2129
- await new Promise((resolve) => setTimeout(resolve, 60));
2130
-
2131
- expect(metrics).toEqual([]);
2132
-
2133
- await closeSession(session);
2134
- });
2135
-
2136
- it("does not fire a stale barge-in if the assistant finishes during the gate window", async () => {
2137
- const session = new VoiceAgentSession({ plugins: {}, minInterruptionMs: 280 });
2138
- const interrupts: InterruptTtsPacket[] = [];
2139
- const metrics: string[] = [];
2140
-
2141
- await session.start();
2142
- session.bus.on("interrupt.tts", (pkt) => {
2143
- interrupts.push(pkt as InterruptTtsPacket);
2144
- });
2145
- session.bus.on("metric.conversation", (pkt) => {
2146
- metrics.push((pkt as unknown as { name: string }).name);
2147
- });
2148
-
2149
- session.bus.push(Route.Main, {
2150
- kind: "tts.audio",
2151
- contextId: "assistant-turn",
2152
- timestampMs: Date.now(),
2153
- audio: new Uint8Array([1, 2, 3, 4]),
2154
- sampleRateHz: 16000,
2155
- } satisfies TextToSpeechAudioPacket);
2156
- await new Promise((resolve) => setTimeout(resolve, 10));
2157
-
2158
- const t0 = Date.now();
2159
- session.bus.push(Route.Main, {
2160
- kind: "vad.speech_started",
2161
- contextId: "user",
2162
- timestampMs: t0,
2163
- confidence: 0.99,
2164
- } satisfies VadSpeechStartedPacket);
2165
- // Assistant finishes speaking before the user's speech sustains past the gate.
2166
- session.bus.push(Route.Main, {
2167
- kind: "tts.end",
2168
- contextId: "assistant-turn",
2169
- timestampMs: t0 + 50,
2170
- } satisfies TextToSpeechEndPacket);
2171
- await new Promise((resolve) => setTimeout(resolve, 10));
2172
- session.bus.push(Route.Main, {
2173
- kind: "vad.speech_activity",
2174
- contextId: "user",
2175
- timestampMs: t0 + 300,
2176
- isAsync: true,
2177
- } satisfies VadSpeechActivityPacket);
2178
- await new Promise((resolve) => setTimeout(resolve, 20));
2179
-
2180
- expect(interrupts).toEqual([]);
2181
- expect(metrics).toContain("interrupt.gate_resolved_after_tts_end");
2182
-
2183
- await closeSession(session);
2184
- });
2185
-
2186
- it("speaks a graceful fallback when the LLM fails a turn (never fail silently)", async () => {
2187
- const session = new VoiceAgentSession({ plugins: {}, errorFallbackText: "One moment please." });
2188
- const ttsTexts: string[] = [];
2189
- const metrics: string[] = [];
2190
- await session.start();
2191
- session.bus.on("tts.text", (pkt) => {
2192
- ttsTexts.push((pkt as unknown as { text: string }).text);
2193
- });
2194
- session.bus.on("metric.conversation", (pkt) => {
2195
- metrics.push((pkt as unknown as { name: string }).name);
2196
- });
2197
-
2198
- session.bus.push(Route.Critical, {
2199
- kind: "llm.error",
2200
- contextId: "turn-1",
2201
- timestampMs: Date.now(),
2202
- component: "llm",
2203
- category: ErrorCategory.NetworkTimeout,
2204
- cause: new Error("provider timeout"),
2205
- isRecoverable: true,
2206
- } satisfies LlmErrorPacket);
2207
-
2208
- await new Promise((resolve) => setTimeout(resolve, 50));
2209
-
2210
- expect(metrics).toContain("error.fallback_spoken");
2211
- expect(ttsTexts.join(" ")).toContain("One moment please.");
2212
-
2213
- await closeSession(session);
2214
- });
2215
-
2216
- it("does not speak an LLM fallback for a TTS failure (needs canned audio, not the broken TTS)", async () => {
2217
- const session = new VoiceAgentSession({ plugins: {}, errorFallbackText: "One moment please." });
2218
- const metrics: string[] = [];
2219
- await session.start();
2220
- session.bus.on("metric.conversation", (pkt) => {
2221
- metrics.push((pkt as unknown as { name: string }).name);
2222
- });
2223
-
2224
- session.bus.push(Route.Critical, {
2225
- kind: "tts.error",
2226
- contextId: "turn-1",
2227
- timestampMs: Date.now(),
2228
- component: "tts",
2229
- category: ErrorCategory.NetworkTimeout,
2230
- cause: new Error("tts down"),
2231
- isRecoverable: true,
2232
- } satisfies TtsErrorPacket);
2233
-
2234
- await new Promise((resolve) => setTimeout(resolve, 30));
2235
-
2236
- expect(metrics).not.toContain("error.fallback_spoken");
2237
-
2238
- await closeSession(session);
2239
- });
2240
-
2241
- it("fires the TTS stall watchdog when output goes silent mid-utterance", async () => {
2242
- const session = new VoiceAgentSession({ plugins: {}, ttsStallMs: 30 });
2243
- const metrics: string[] = [];
2244
- const ttsErrors: Array<{ category: string }> = [];
2245
- await session.start();
2246
- session.bus.on("metric.conversation", (pkt) => {
2247
- metrics.push((pkt as unknown as { name: string }).name);
2248
- });
2249
- session.bus.on("tts.error", (pkt) => {
2250
- ttsErrors.push({ category: (pkt as unknown as { category: string }).category });
2251
- });
2252
-
2253
- // TTS produces one chunk then goes silent — no further audio, no tts.end.
2254
- session.bus.push(Route.Main, {
2255
- kind: "tts.audio",
2256
- contextId: "turn-1",
2257
- timestampMs: Date.now(),
2258
- audio: new Uint8Array([1, 2, 3, 4]),
2259
- sampleRateHz: 16000,
2260
- } satisfies TextToSpeechAudioPacket);
2261
-
2262
- await new Promise((resolve) => setTimeout(resolve, 90)); // > ttsStallMs
2263
-
2264
- expect(metrics).toContain("tts.stall_detected");
2265
- expect(ttsErrors.some((e) => e.category === "network_timeout")).toBe(true);
2266
-
2267
- await closeSession(session);
2268
- });
2269
-
2270
- it("does not fire the TTS stall watchdog when tts.end arrives", async () => {
2271
- const session = new VoiceAgentSession({ plugins: {}, ttsStallMs: 30 });
2272
- const metrics: string[] = [];
2273
- await session.start();
2274
- session.bus.on("metric.conversation", (pkt) => {
2275
- metrics.push((pkt as unknown as { name: string }).name);
2276
- });
2277
-
2278
- session.bus.push(Route.Main, {
2279
- kind: "tts.audio",
2280
- contextId: "turn-1",
2281
- timestampMs: Date.now(),
2282
- audio: new Uint8Array([1, 2, 3, 4]),
2283
- sampleRateHz: 16000,
2284
- } satisfies TextToSpeechAudioPacket);
2285
- await new Promise((resolve) => setTimeout(resolve, 10));
2286
- session.bus.push(Route.Main, {
2287
- kind: "tts.end",
2288
- contextId: "turn-1",
2289
- timestampMs: Date.now(),
2290
- } satisfies TextToSpeechEndPacket);
2291
- await new Promise((resolve) => setTimeout(resolve, 60)); // past ttsStallMs
2292
-
2293
- expect(metrics).not.toContain("tts.stall_detected");
2294
-
2295
- await closeSession(session);
2296
- });
2297
-
2298
- it("test:input_stall_emits_recovery — fires recoverable pipeline.error and metric when inbound audio stalls", async () => {
2299
- vi.useFakeTimers();
2300
- vi.setSystemTime(1000);
2301
-
2302
- const session = new VoiceAgentSession({ plugins: {}, inputCadenceTimeoutMs: 2000 });
2303
- const metrics: string[] = [];
2304
- const pipelineErrors: Array<{ category: string; isRecoverable: boolean; message: string }> = [];
2305
-
2306
- await session.start();
2307
- session.bus.on("metric.conversation", (pkt) => {
2308
- metrics.push((pkt as unknown as { name: string }).name);
2309
- });
2310
- session.bus.on("pipeline.error", (pkt) => {
2311
- const err = pkt as PipelineErrorPacket;
2312
- pipelineErrors.push({
2313
- category: err.category,
2314
- isRecoverable: err.isRecoverable,
2315
- message: err.cause.message,
2316
- });
2317
- });
2318
-
2319
- try {
2320
- session.bus.push(Route.Main, {
2321
- kind: "user.audio_received",
2322
- contextId: "turn-1",
2323
- timestampMs: Date.now(),
2324
- audio: new Uint8Array([1, 2, 3, 4]),
2325
- } satisfies UserAudioReceivedPacket);
2326
- await vi.advanceTimersByTimeAsync(0);
2327
-
2328
- await vi.advanceTimersByTimeAsync(1999);
2329
- expect(metrics).not.toContain("input.cadence_stall_ms");
2330
- expect(pipelineErrors).toEqual([]);
2331
-
2332
- await vi.advanceTimersByTimeAsync(1);
2333
- expect(metrics).toContain("input.cadence_stall_ms");
2334
- expect(pipelineErrors).toEqual([
2335
- expect.objectContaining({
2336
- category: "network_timeout",
2337
- isRecoverable: true,
2338
- message: "inbound audio stalled",
2339
- }),
2340
- ]);
2341
- } finally {
2342
- vi.useRealTimers();
2343
- await closeSession(session);
2344
- }
2345
- });
2346
-
2347
- it("test:cadence_reset_on_audio — inbound audio before the window resets the cadence watchdog", async () => {
2348
- vi.useFakeTimers();
2349
- vi.setSystemTime(1000);
2350
-
2351
- const session = new VoiceAgentSession({ plugins: {}, inputCadenceTimeoutMs: 2000 });
2352
- const metrics: string[] = [];
2353
- const pipelineErrors: PipelineErrorPacket[] = [];
2354
-
2355
- await session.start();
2356
- session.bus.on("metric.conversation", (pkt) => {
2357
- metrics.push((pkt as unknown as { name: string }).name);
2358
- });
2359
- session.bus.on("pipeline.error", (pkt) => {
2360
- pipelineErrors.push(pkt as PipelineErrorPacket);
2361
- });
2362
-
2363
- try {
2364
- session.bus.push(Route.Main, {
2365
- kind: "user.audio_received",
2366
- contextId: "turn-1",
2367
- timestampMs: Date.now(),
2368
- audio: new Uint8Array([1, 2, 3, 4]),
2369
- } satisfies UserAudioReceivedPacket);
2370
- await vi.advanceTimersByTimeAsync(0);
2371
-
2372
- await vi.advanceTimersByTimeAsync(1500);
2373
- session.bus.push(Route.Main, {
2374
- kind: "user.audio_received",
2375
- contextId: "turn-1",
2376
- timestampMs: Date.now(),
2377
- audio: new Uint8Array([5, 6, 7, 8]),
2378
- } satisfies UserAudioReceivedPacket);
2379
- await vi.advanceTimersByTimeAsync(0);
2380
-
2381
- await vi.advanceTimersByTimeAsync(1999);
2382
- expect(metrics).not.toContain("input.cadence_stall_ms");
2383
- expect(pipelineErrors).toEqual([]);
2384
- } finally {
2385
- vi.useRealTimers();
2386
- await closeSession(session);
2387
- }
2388
- });
2389
-
2390
- it("does not arm the input cadence watchdog when inputCadenceTimeoutMs is 0", async () => {
2391
- vi.useFakeTimers();
2392
- vi.setSystemTime(1000);
2393
-
2394
- const session = new VoiceAgentSession({ plugins: {} });
2395
- const metrics: string[] = [];
2396
- const pipelineErrors: PipelineErrorPacket[] = [];
2397
-
2398
- await session.start();
2399
- session.bus.on("metric.conversation", (pkt) => {
2400
- metrics.push((pkt as unknown as { name: string }).name);
2401
- });
2402
- session.bus.on("pipeline.error", (pkt) => {
2403
- pipelineErrors.push(pkt as PipelineErrorPacket);
2404
- });
2405
-
2406
- try {
2407
- session.bus.push(Route.Main, {
2408
- kind: "user.audio_received",
2409
- contextId: "turn-1",
2410
- timestampMs: Date.now(),
2411
- audio: new Uint8Array([1, 2, 3, 4]),
2412
- } satisfies UserAudioReceivedPacket);
2413
- await vi.advanceTimersByTimeAsync(0);
2414
- await vi.advanceTimersByTimeAsync(60_000);
2415
-
2416
- expect(metrics).not.toContain("input.cadence_stall_ms");
2417
- expect(pipelineErrors).toEqual([]);
2418
- } finally {
2419
- vi.useRealTimers();
2420
- await closeSession(session);
2421
- }
2422
- });
2423
-
2424
- it("tells the recorder to truncate queued assistant audio on barge-in", async () => {
2425
- const session = new VoiceAgentSession({ plugins: {}, minInterruptionMs: 0 });
2426
- const recorded: RecordAssistantAudioPacket[] = [];
2427
-
2428
- await session.start();
2429
- session.bus.on("record.assistant_audio", (pkt) => {
2430
- recorded.push(pkt as RecordAssistantAudioPacket);
2431
- });
2432
-
2433
- session.bus.push(Route.Main, {
2434
- kind: "tts.audio",
2435
- contextId: "assistant-turn",
2436
- timestampMs: Date.now(),
2437
- audio: new Uint8Array([1, 2, 3, 4]),
2438
- sampleRateHz: 16000,
2439
- } satisfies TextToSpeechAudioPacket);
2440
- await new Promise((resolve) => setTimeout(resolve, 20));
2441
-
2442
- session.bus.push(Route.Main, {
2443
- kind: "vad.speech_started",
2444
- contextId: "user-barge-in",
2445
- timestampMs: Date.now(),
2446
- confidence: 0.99,
2447
- } satisfies VadSpeechStartedPacket);
2448
- await new Promise((resolve) => setTimeout(resolve, 20));
2449
-
2450
- expect(recorded).toEqual([
2451
- expect.objectContaining({
2452
- kind: "record.assistant_audio",
2453
- contextId: "assistant-turn",
2454
- truncate: false,
2455
- }),
2456
- expect.objectContaining({
2457
- kind: "record.assistant_audio",
2458
- contextId: "assistant-turn",
2459
- truncate: true,
2460
- audio: new Uint8Array(0),
2461
- }),
2462
- ]);
2463
-
2464
- await closeSession(session);
2465
- });
2466
-
2467
- it("does not reopen TTS from late LLM or TTS packets after barge-in", async () => {
2468
- const session = new VoiceAgentSession({ plugins: {}, minInterruptionMs: 0 });
2469
- const ttsText: TextToSpeechTextPacket[] = [];
2470
- const ttsDone: TextToSpeechDonePacket[] = [];
2471
- const recorded: RecordAssistantAudioPacket[] = [];
2472
- const metrics: string[] = [];
2473
-
2474
- await session.start();
2475
- session.bus.on("tts.text", (pkt) => {
2476
- ttsText.push(pkt as TextToSpeechTextPacket);
2477
- });
2478
- session.bus.on("tts.done", (pkt) => {
2479
- ttsDone.push(pkt as TextToSpeechDonePacket);
2480
- });
2481
- session.bus.on("record.assistant_audio", (pkt) => {
2482
- recorded.push(pkt as RecordAssistantAudioPacket);
2483
- });
2484
- session.bus.on("metric.conversation", (pkt) => {
2485
- const metric = pkt as unknown as { name: string };
2486
- metrics.push(metric.name);
2487
- });
2488
-
2489
- session.bus.push(Route.Main, {
2490
- kind: "tts.audio",
2491
- contextId: "assistant-turn",
2492
- timestampMs: Date.now(),
2493
- audio: new Uint8Array([1, 2, 3, 4]),
2494
- sampleRateHz: 16000,
2495
- } satisfies TextToSpeechAudioPacket);
2496
- await new Promise((resolve) => setTimeout(resolve, 20));
2497
-
2498
- session.bus.push(Route.Main, {
2499
- kind: "vad.speech_started",
2500
- contextId: "user-barge-in",
2501
- timestampMs: Date.now(),
2502
- confidence: 0.99,
2503
- } satisfies VadSpeechStartedPacket);
2504
- await new Promise((resolve) => setTimeout(resolve, 20));
2505
-
2506
- session.bus.push(Route.Main, {
2507
- kind: "llm.delta",
2508
- contextId: "assistant-turn",
2509
- timestampMs: Date.now(),
2510
- text: " This late text must not be spoken.",
2511
- } satisfies LlmDeltaPacket);
2512
- session.bus.push(Route.Main, {
2513
- kind: "llm.done",
2514
- contextId: "assistant-turn",
2515
- timestampMs: Date.now(),
2516
- text: "This late done must not flush TTS.",
2517
- } satisfies LlmResponseDonePacket);
2518
- session.bus.push(Route.Main, {
2519
- kind: "tts.audio",
2520
- contextId: "assistant-turn",
2521
- timestampMs: Date.now(),
2522
- audio: new Uint8Array([5, 6, 7, 8]),
2523
- sampleRateHz: 16000,
2524
- } satisfies TextToSpeechAudioPacket);
2525
- await new Promise((resolve) => setTimeout(resolve, 30));
2526
-
2527
- expect(ttsText).toEqual([]);
2528
- expect(ttsDone).toEqual([]);
2529
- expect(recorded).toEqual([
2530
- expect.objectContaining({
2531
- contextId: "assistant-turn",
2532
- truncate: false,
2533
- audio: new Uint8Array([1, 2, 3, 4]),
2534
- }),
2535
- expect.objectContaining({
2536
- contextId: "assistant-turn",
2537
- truncate: true,
2538
- audio: new Uint8Array(0),
2539
- }),
2540
- ]);
2541
- expect(metrics).toContain("llm.delta_ignored_after_interrupt");
2542
- expect(metrics).toContain("llm.done_ignored_after_interrupt");
2543
- expect(metrics).toContain("tts.audio_ignored_after_interrupt");
2544
-
2545
- await closeSession(session);
2546
- });
2547
-
2548
- it("suppresses sustained bystander barge-in when a primary speaker profile is enrolled", async () => {
2549
- const session = new VoiceAgentSession({ plugins: {}, minInterruptionMs: 280 });
2550
- const interrupts: InterruptTtsPacket[] = [];
2551
- const metrics: string[] = [];
2552
-
2553
- await session.start();
2554
- session.bus.on("interrupt.tts", (pkt) => {
2555
- interrupts.push(pkt as InterruptTtsPacket);
2556
- });
2557
- session.bus.on("metric.conversation", (pkt) => {
2558
- metrics.push((pkt as unknown as { name: string }).name);
2559
- });
2560
-
2561
- await enrollPrimarySpeaker(session);
2562
-
2563
- session.bus.push(Route.Main, {
2564
- kind: "tts.audio",
2565
- contextId: "assistant-turn",
2566
- timestampMs: Date.now(),
2567
- audio: synthesizeTonePcm16({
2568
- frequencyHz: ASSISTANT_ECHO_TONE_HZ,
2569
- durationMs: 32,
2570
- }),
2571
- sampleRateHz: 16000,
2572
- } satisfies TextToSpeechAudioPacket);
2573
- await new Promise((resolve) => setTimeout(resolve, 10));
2574
-
2575
- const bystander = synthesizeTonePcm16({
2576
- frequencyHz: BYSTANDER_SPEAKER_TONE_HZ,
2577
- durationMs: 32,
2578
- });
2579
- const t0 = Date.now();
2580
- session.bus.push(Route.Main, {
2581
- kind: "vad.speech_started",
2582
- contextId: "user-barge",
2583
- timestampMs: t0,
2584
- confidence: 0.99,
2585
- } satisfies VadSpeechStartedPacket);
2586
- for (let i = 0; i < 10; i += 1) {
2587
- session.bus.push(Route.Main, {
2588
- kind: "vad.audio",
2589
- contextId: "user-barge",
2590
- timestampMs: t0 + 20 + i * 30,
2591
- audio: bystander,
2592
- } satisfies VadAudioPacket);
2593
- }
2594
- session.bus.push(Route.Main, {
2595
- kind: "vad.speech_activity",
2596
- contextId: "user-barge",
2597
- timestampMs: t0 + 320,
2598
- isAsync: true,
2599
- } satisfies VadSpeechActivityPacket);
2600
- await new Promise((resolve) => setTimeout(resolve, 20));
2601
-
2602
- expect(interrupts).toEqual([]);
2603
- expect(metrics).toContain("interrupt.suppressed_non_primary");
2604
-
2605
- await closeSession(session);
2606
- });
2607
-
2608
- it("commits a primary-speaker barge-in composed with the G1 time gate", async () => {
2609
- const session = new VoiceAgentSession({ plugins: {}, minInterruptionMs: 280 });
2610
- const interrupts: InterruptTtsPacket[] = [];
2611
-
2612
- await session.start();
2613
- session.bus.on("interrupt.tts", (pkt) => {
2614
- interrupts.push(pkt as InterruptTtsPacket);
2615
- });
2616
-
2617
- await enrollPrimarySpeaker(session);
2618
-
2619
- session.bus.push(Route.Main, {
2620
- kind: "tts.audio",
2621
- contextId: "assistant-turn",
2622
- timestampMs: Date.now(),
2623
- audio: new Uint8Array([1, 2, 3, 4]),
2624
- sampleRateHz: 16000,
2625
- } satisfies TextToSpeechAudioPacket);
2626
- await new Promise((resolve) => setTimeout(resolve, 10));
2627
-
2628
- const primary = synthesizeTonePcm16({
2629
- frequencyHz: PRIMARY_SPEAKER_TONE_HZ,
2630
- durationMs: 32,
2631
- });
2632
- const t0 = Date.now();
2633
- session.bus.push(Route.Main, {
2634
- kind: "vad.speech_started",
2635
- contextId: "user-barge",
2636
- timestampMs: t0,
2637
- confidence: 0.99,
2638
- } satisfies VadSpeechStartedPacket);
2639
- for (let i = 0; i < 10; i += 1) {
2640
- session.bus.push(Route.Main, {
2641
- kind: "vad.audio",
2642
- contextId: "user-barge",
2643
- timestampMs: t0 + 20 + i * 30,
2644
- audio: primary,
2645
- } satisfies VadAudioPacket);
2646
- }
2647
- session.bus.push(Route.Main, {
2648
- kind: "vad.speech_activity",
2649
- contextId: "user-barge",
2650
- timestampMs: t0 + 320,
2651
- isAsync: true,
2652
- } satisfies VadSpeechActivityPacket);
2653
- await new Promise((resolve) => setTimeout(resolve, 20));
2654
-
2655
- expect(interrupts).toEqual([
2656
- expect.objectContaining({ kind: "interrupt.tts", contextId: "assistant-turn" }),
2657
- ]);
2658
-
2659
- await closeSession(session);
2660
- });
2661
-
2662
- it("preserves G1-only barge-in when no primary speaker profile is enrolled", async () => {
2663
- const session = new VoiceAgentSession({ plugins: {}, minInterruptionMs: 280 });
2664
- const interrupts: InterruptTtsPacket[] = [];
2665
- const metrics: string[] = [];
2666
-
2667
- await session.start();
2668
- session.bus.on("interrupt.tts", (pkt) => {
2669
- interrupts.push(pkt as InterruptTtsPacket);
2670
- });
2671
- session.bus.on("metric.conversation", (pkt) => {
2672
- metrics.push((pkt as unknown as { name: string }).name);
2673
- });
2674
-
2675
- session.bus.push(Route.Main, {
2676
- kind: "tts.audio",
2677
- contextId: "assistant-turn",
2678
- timestampMs: Date.now(),
2679
- audio: new Uint8Array([1, 2, 3, 4]),
2680
- sampleRateHz: 16000,
2681
- } satisfies TextToSpeechAudioPacket);
2682
- await new Promise((resolve) => setTimeout(resolve, 10));
2683
-
2684
- const t0 = Date.now();
2685
- session.bus.push(Route.Main, {
2686
- kind: "vad.speech_started",
2687
- contextId: "user",
2688
- timestampMs: t0,
2689
- confidence: 0.99,
2690
- } satisfies VadSpeechStartedPacket);
2691
- session.bus.push(Route.Main, {
2692
- kind: "vad.speech_activity",
2693
- contextId: "user",
2694
- timestampMs: t0 + 300,
2695
- isAsync: true,
2696
- } satisfies VadSpeechActivityPacket);
2697
- await new Promise((resolve) => setTimeout(resolve, 20));
2698
-
2699
- expect(interrupts).toEqual([
2700
- expect.objectContaining({ kind: "interrupt.tts", contextId: "assistant-turn" }),
2701
- ]);
2702
- expect(metrics).not.toContain("interrupt.suppressed_non_primary");
2703
-
2704
- await closeSession(session);
2705
- });
2706
-
2707
- it("enqueues filler TTS at endpoint before the first LLM token when enabled", async () => {
2708
- const session = new VoiceAgentSession({
2709
- plugins: {},
2710
- latencyFillerEnabled: true,
2711
- });
2712
- const ttsText: TextToSpeechTextPacket[] = [];
2713
- const metrics: string[] = [];
2714
-
2715
- await session.start();
2716
- session.bus.on("tts.text", (pkt) => {
2717
- ttsText.push(pkt as TextToSpeechTextPacket);
2718
- });
2719
- session.bus.on("metric.conversation", (pkt) => {
2720
- metrics.push((pkt as unknown as { name: string }).name);
2721
- });
2722
-
2723
- session.bus.push(Route.Main, {
2724
- kind: "eos.turn_complete",
2725
- contextId: "turn-1",
2726
- timestampMs: 1000,
2727
- text: "Can I add Biology 101?",
2728
- transcripts: [],
2729
- } satisfies EndOfSpeechPacket);
2730
-
2731
- await new Promise((resolve) => setTimeout(resolve, 10));
2732
-
2733
- expect(ttsText).toEqual([
2734
- expect.objectContaining({
2735
- kind: "tts.text",
2736
- contextId: "turn-1",
2737
- text: "Well,",
2738
- }),
2739
- ]);
2740
- expect(metrics).toContain("filler.started");
2741
-
2742
- session.bus.push(Route.Main, {
2743
- kind: "llm.delta",
2744
- contextId: "turn-1",
2745
- timestampMs: 1500,
2746
- text: "You can still submit a late add petition.",
2747
- } satisfies LlmDeltaPacket);
2748
-
2749
- await new Promise((resolve) => setTimeout(resolve, 10));
2750
-
2751
- expect(ttsText).toEqual([
2752
- expect.objectContaining({ text: "Well," }),
2753
- expect.objectContaining({ text: "You can still submit a late add petition." }),
2754
- ]);
2755
- expect(metrics).toContain("filler.spliced");
2756
-
2757
- await closeSession(session);
2758
- });
2759
-
2760
- it("cancels filler when the user keeps talking after endpoint", async () => {
2761
- const session = new VoiceAgentSession({
2762
- plugins: {},
2763
- latencyFillerEnabled: true,
2764
- });
2765
- const interrupts: InterruptTtsPacket[] = [];
2766
- const metrics: string[] = [];
2767
-
2768
- await session.start();
2769
- session.bus.on("interrupt.tts", (pkt) => {
2770
- interrupts.push(pkt as InterruptTtsPacket);
2771
- });
2772
- session.bus.on("metric.conversation", (pkt) => {
2773
- metrics.push((pkt as unknown as { name: string }).name);
2774
- });
2775
-
2776
- session.bus.push(Route.Main, {
2777
- kind: "eos.turn_complete",
2778
- contextId: "turn-1",
2779
- timestampMs: 1000,
2780
- text: "I need help with",
2781
- transcripts: [],
2782
- } satisfies EndOfSpeechPacket);
2783
- await new Promise((resolve) => setTimeout(resolve, 10));
2784
-
2785
- session.bus.push(Route.Main, {
2786
- kind: "vad.speech_started",
2787
- contextId: "turn-1",
2788
- timestampMs: 1100,
2789
- confidence: 0.99,
2790
- } satisfies VadSpeechStartedPacket);
2791
- await new Promise((resolve) => setTimeout(resolve, 10));
2792
-
2793
- expect(interrupts).toEqual([
2794
- expect.objectContaining({ kind: "interrupt.tts", contextId: "turn-1" }),
2795
- ]);
2796
- expect(metrics).toContain("filler.cancelled");
2797
-
2798
- await closeSession(session);
2799
- });
2800
-
2801
- it("clears latency filler state on recoverable component errors", async () => {
2802
- const session = new VoiceAgentSession({
2803
- plugins: {},
2804
- latencyFillerEnabled: true,
2805
- });
2806
- await session.start();
2807
-
2808
- session.bus.push(Route.Main, {
2809
- kind: "eos.turn_complete",
2810
- contextId: "turn-error-clear",
2811
- timestampMs: 1000,
2812
- text: "hello",
2813
- transcripts: [],
2814
- } satisfies EndOfSpeechPacket);
2815
- await new Promise((resolve) => setTimeout(resolve, 10));
2816
- expect(session["latencyFiller"].getState("turn-error-clear")).toBeDefined();
2817
-
2818
- session.bus.push(Route.Main, {
2819
- kind: "llm.error",
2820
- contextId: "turn-error-clear",
2821
- timestampMs: 1100,
2822
- component: "llm",
2823
- category: ErrorCategory.NetworkTimeout,
2824
- cause: new Error("provider down"),
2825
- isRecoverable: true,
2826
- } satisfies LlmErrorPacket);
2827
- await new Promise((resolve) => setTimeout(resolve, 10));
2828
-
2829
- expect(session["latencyFiller"].getState("turn-error-clear")).toBeUndefined();
2830
-
2831
- await closeSession(session);
2832
- });
2833
-
2834
- it("splices filler into the real response without duplicating connectives", async () => {
2835
- const session = new VoiceAgentSession({
2836
- plugins: {},
2837
- latencyFillerEnabled: true,
2838
- });
2839
- const ttsText: TextToSpeechTextPacket[] = [];
2840
-
2841
- await session.start();
2842
- session.bus.on("tts.text", (pkt) => {
2843
- ttsText.push(pkt as TextToSpeechTextPacket);
2844
- });
2845
-
2846
- session.bus.push(Route.Main, {
2847
- kind: "eos.turn_complete",
2848
- contextId: "turn-1",
2849
- timestampMs: 1000,
2850
- text: "hello",
2851
- transcripts: [],
2852
- } satisfies EndOfSpeechPacket);
2853
-
2854
- session.bus.push(Route.Main, {
2855
- kind: "llm.delta",
2856
- contextId: "turn-1",
2857
- timestampMs: 1400,
2858
- text: "So the petition is still open.",
2859
- } satisfies LlmDeltaPacket);
2860
- session.bus.push(Route.Main, {
2861
- kind: "llm.done",
2862
- contextId: "turn-1",
2863
- timestampMs: 1401,
2864
- text: "So the petition is still open.",
2865
- } satisfies LlmResponseDonePacket);
2866
-
2867
- await new Promise((resolve) => setTimeout(resolve, 20));
2868
-
2869
- expect(ttsText.map((pkt) => pkt.text)).toEqual([
2870
- "So,",
2871
- "the petition is still open.",
2872
- ]);
2873
-
2874
- await closeSession(session);
2875
- });
2876
-
2877
- describe("endpointingOwner invariant (VE-02)", () => {
2878
- it("throws for unsupported endpointingOwner in constructor", () => {
2879
- expect(
2880
- () =>
2881
- new VoiceAgentSession({
2882
- plugins: {},
2883
- endpointingOwner: "bogus" as "provider_stt",
2884
- }),
2885
- ).toThrow("Unsupported endpointingOwner: bogus");
2886
- });
2887
-
2888
- it("with owner unset defaults to provider STT and does not fan user audio to eos.audio", async () => {
2889
- const session = new VoiceAgentSession({ plugins: {} });
2890
- await session.start();
2891
-
2892
- const eosPackets: EndOfSpeechAudioPacket[] = [];
2893
- const sttPackets: SpeechToTextAudioPacket[] = [];
2894
- session.bus.on("eos.audio", (pkt) => {
2895
- eosPackets.push(pkt as EndOfSpeechAudioPacket);
2896
- });
2897
- session.bus.on("stt.audio", (pkt) => {
2898
- sttPackets.push(pkt as SpeechToTextAudioPacket);
2899
- });
2900
-
2901
- session.bus.push(Route.Main, {
2902
- kind: "user.audio_received",
2903
- contextId: "turn-default-owner",
2904
- timestampMs: Date.now(),
2905
- audio: new Uint8Array([1, 2, 3]),
2906
- } satisfies UserAudioReceivedPacket);
2907
-
2908
- await new Promise((resolve) => setTimeout(resolve, 20));
2909
- expect(sttPackets).toHaveLength(1);
2910
- expect(eosPackets).toHaveLength(0);
2911
-
2912
- await closeSession(session);
2913
- });
2914
-
2915
- it("with provider_stt owner does not fan eos.audio but routes EOS to one user.input", async () => {
2916
- const session = new VoiceAgentSession({
2917
- plugins: {},
2918
- endpointingOwner: "provider_stt",
2919
- });
2920
- const eosPackets: EndOfSpeechAudioPacket[] = [];
2921
- const userInputs: UserInputPacket[] = [];
2922
-
2923
- await session.start();
2924
- session.bus.on("eos.audio", (pkt) => {
2925
- eosPackets.push(pkt as EndOfSpeechAudioPacket);
2926
- });
2927
- session.bus.on("user.input", (pkt) => {
2928
- userInputs.push(pkt as UserInputPacket);
2929
- });
2930
-
2931
- session.bus.push(Route.Main, {
2932
- kind: "user.audio_received",
2933
- contextId: "turn-stt",
2934
- timestampMs: Date.now(),
2935
- audio: new Uint8Array([9, 8, 7]),
2936
- } satisfies UserAudioReceivedPacket);
2937
-
2938
- await new Promise((resolve) => setTimeout(resolve, 20));
2939
- expect(eosPackets).toHaveLength(0);
2940
-
2941
- session.bus.push(Route.Main, {
2942
- kind: "eos.turn_complete",
2943
- contextId: "turn-stt",
2944
- timestampMs: Date.now(),
2945
- text: "from provider",
2946
- transcripts: [],
2947
- } satisfies EndOfSpeechPacket);
2948
-
2949
- await new Promise((resolve) => setTimeout(resolve, 20));
2950
- expect(userInputs).toHaveLength(1);
2951
- expect(userInputs[0]!.text).toBe("from provider");
2952
-
2953
- await closeSession(session);
2954
- });
2955
-
2956
- it("allows multiple user turns with the same stable transport contextId", async () => {
2957
- const session = new VoiceAgentSession({
2958
- plugins: {},
2959
- endpointingOwner: "provider_stt",
2960
- });
2961
- const userInputs: UserInputPacket[] = [];
2962
- const finals: Array<{ turnId: string; text: string }> = [];
2963
-
2964
- await session.start();
2965
- session.bus.on("user.input", (pkt) => {
2966
- userInputs.push(pkt as UserInputPacket);
2967
- });
2968
- session.on("user_input_final", (event) => {
2969
- finals.push({ turnId: event.turnId, text: event.text });
2970
- });
2971
-
2972
- session.bus.push(Route.Main, {
2973
- kind: "vad.speech_started",
2974
- contextId: "call-stable",
2975
- timestampMs: 1000,
2976
- confidence: 0.99,
2977
- } satisfies VadSpeechStartedPacket);
2978
- session.bus.push(Route.Main, {
2979
- kind: "eos.turn_complete",
2980
- contextId: "call-stable",
2981
- timestampMs: 1100,
2982
- text: "first turn",
2983
- transcripts: [],
2984
- } satisfies EndOfSpeechPacket);
2985
- await new Promise((resolve) => setTimeout(resolve, 20));
2986
-
2987
- session.bus.push(Route.Main, {
2988
- kind: "vad.speech_started",
2989
- contextId: "call-stable",
2990
- timestampMs: 2000,
2991
- confidence: 0.99,
2992
- } satisfies VadSpeechStartedPacket);
2993
- session.bus.push(Route.Main, {
2994
- kind: "eos.turn_complete",
2995
- contextId: "call-stable",
2996
- timestampMs: 2100,
2997
- text: "second turn",
2998
- transcripts: [],
2999
- } satisfies EndOfSpeechPacket);
3000
- await new Promise((resolve) => setTimeout(resolve, 20));
3001
-
3002
- expect(userInputs.map((pkt) => pkt.text)).toEqual(["first turn", "second turn"]);
3003
- expect(finals).toEqual([
3004
- { turnId: "call-stable", text: "first turn" },
3005
- { turnId: "call-stable", text: "second turn" },
3006
- ]);
3007
-
3008
- await closeSession(session);
3009
- });
3010
-
3011
- it("re-arms per-turn guard state across a barge-in on a stable contextId (no stale interrupt-drop; emits interrupt.stt)", async () => {
3012
- const session = new VoiceAgentSession({
3013
- plugins: {},
3014
- endpointingOwner: "provider_stt",
3015
- minInterruptionMs: 0,
3016
- });
3017
- const ignoredAfterInterrupt: string[] = [];
3018
- const sttInterrupts: string[] = [];
3019
-
3020
- await session.start();
3021
- session.bus.on("metric.conversation", (pkt) => {
3022
- const m = pkt as { name?: string };
3023
- if (m.name === "llm.delta_ignored_after_interrupt") ignoredAfterInterrupt.push(m.name);
3024
- });
3025
- session.bus.on("interrupt.stt", (pkt) => {
3026
- sttInterrupts.push((pkt as { contextId: string }).contextId);
3027
- });
3028
-
3029
- // Turn 1 on a stable (telephony-style) contextId, then a barge-in during its response.
3030
- session.bus.push(Route.Main, {
3031
- kind: "vad.speech_started",
3032
- contextId: "call-stable",
3033
- timestampMs: 1000,
3034
- confidence: 0.99,
3035
- } satisfies VadSpeechStartedPacket);
3036
- session.bus.push(Route.Main, {
3037
- kind: "eos.turn_complete",
3038
- contextId: "call-stable",
3039
- timestampMs: 1100,
3040
- text: "first turn",
3041
- transcripts: [],
3042
- } satisfies EndOfSpeechPacket);
3043
- await new Promise((resolve) => setTimeout(resolve, 20));
3044
- session.bus.push(Route.Critical, {
3045
- kind: "interrupt.detected",
3046
- contextId: "call-stable",
3047
- timestampMs: 1200,
3048
- source: "vad",
3049
- } satisfies InterruptionDetectedPacket);
3050
- await new Promise((resolve) => setTimeout(resolve, 20));
3051
-
3052
- // interrupt.stt is now emitted on barge-in so provider STT resets transcript state.
3053
- expect(sttInterrupts).toContain("call-stable");
3054
-
3055
- // Turn 2 reuses the SAME contextId. Before the fix, the stale interrupted-generation
3056
- // flag from turn 1 dropped turn 2's llm.delta as "llm.delta_ignored_after_interrupt".
3057
- session.bus.push(Route.Main, {
3058
- kind: "vad.speech_started",
3059
- contextId: "call-stable",
3060
- timestampMs: 2000,
3061
- confidence: 0.99,
3062
- } satisfies VadSpeechStartedPacket);
3063
- session.bus.push(Route.Main, {
3064
- kind: "eos.turn_complete",
3065
- contextId: "call-stable",
3066
- timestampMs: 2100,
3067
- text: "second turn",
3068
- transcripts: [],
3069
- } satisfies EndOfSpeechPacket);
3070
- await new Promise((resolve) => setTimeout(resolve, 20));
3071
- session.bus.push(Route.Main, {
3072
- kind: "llm.delta",
3073
- contextId: "call-stable",
3074
- timestampMs: 2200,
3075
- text: "second turn reply",
3076
- } satisfies LlmDeltaPacket);
3077
- await new Promise((resolve) => setTimeout(resolve, 20));
3078
-
3079
- // Turn 2's response is NOT suppressed by turn 1's barge-in.
3080
- expect(ignoredAfterInterrupt).toEqual([]);
3081
-
3082
- await closeSession(session);
3083
- });
3084
-
3085
- it("initializes only the provider finalizer when provider_stt owns endpointing", async () => {
3086
- const provider = new EndpointingPlugin({
3087
- owner: "provider_stt",
3088
- disableConfig: { emit_eos_on_final: false },
3089
- });
3090
- const smartTurn = new EndpointingPlugin({ owner: "smart_turn" });
3091
- const session = new VoiceAgentSession({
3092
- plugins: {
3093
- stt: { emit_eos_on_final: true },
3094
- eos: {},
3095
- },
3096
- endpointingOwner: "provider_stt",
3097
- });
3098
- session.registerPlugin("stt", provider);
3099
- session.registerPlugin("eos", smartTurn);
3100
-
3101
- await session.start();
3102
-
3103
- expect(provider.initializeCount).toBe(1);
3104
- expect(provider.config).toEqual({ emit_eos_on_final: true });
3105
- expect(smartTurn.initializeCount).toBe(0);
3106
-
3107
- await closeSession(session);
3108
- });
3109
-
3110
- it("forces provider EOS off while keeping STT initialized when smart_turn owns endpointing", async () => {
3111
- const provider = new EndpointingPlugin({
3112
- owner: "provider_stt",
3113
- disableConfig: {
3114
- emit_eos_on_final: false,
3115
- finalize_on_speech_final: false,
3116
- },
3117
- });
3118
- const smartTurn = new EndpointingPlugin({ owner: "smart_turn" });
3119
- const session = new VoiceAgentSession({
3120
- plugins: {
3121
- stt: { emit_eos_on_final: true, finalize_on_speech_final: true },
3122
- eos: {},
3123
- },
3124
- endpointingOwner: "smart_turn",
3125
- });
3126
- session.registerPlugin("stt", provider);
3127
- session.registerPlugin("eos", smartTurn);
3128
-
3129
- await session.start();
3130
-
3131
- expect(provider.initializeCount).toBe(1);
3132
- expect(provider.config).toEqual({
3133
- emit_eos_on_final: false,
3134
- finalize_on_speech_final: false,
3135
- });
3136
- expect(smartTurn.initializeCount).toBe(1);
3137
-
3138
- await closeSession(session);
3139
- });
3140
-
3141
- it("throws at startup when the selected endpointing owner has multiple finalizers", async () => {
3142
- const session = new VoiceAgentSession({
3143
- plugins: { sttA: {}, sttB: {} },
3144
- endpointingOwner: "provider_stt",
3145
- });
3146
- session.registerPlugin("sttA", new EndpointingPlugin({ owner: "provider_stt" }));
3147
- session.registerPlugin("sttB", new EndpointingPlugin({ owner: "provider_stt" }));
3148
-
3149
- await expect(session.start()).rejects.toThrow(
3150
- "endpointingOwner=provider_stt requires exactly one registered provider_stt EOS finalizer; found 2",
3151
- );
3152
- await closeSession(session);
3153
- });
3154
-
3155
- it("drops duplicate eos.turn_complete for same contextId with eos.duplicate_dropped metric", async () => {
3156
- const session = new VoiceAgentSession({ plugins: {} });
3157
- const userInputs: UserInputPacket[] = [];
3158
- const duplicateMetrics: Array<{ name: string; value: string }> = [];
3159
-
3160
- await session.start();
3161
- session.bus.on("user.input", (pkt) => {
3162
- userInputs.push(pkt as UserInputPacket);
3163
- });
3164
- session.bus.on("metric.conversation", (pkt) => {
3165
- const m = pkt as unknown as { name: string; value: string };
3166
- if (m.name === "eos.duplicate_dropped") {
3167
- duplicateMetrics.push({ name: m.name, value: m.value });
3168
- }
3169
- });
3170
-
3171
- const eosPacket: EndOfSpeechPacket = {
3172
- kind: "eos.turn_complete",
3173
- contextId: "turn-dup",
3174
- timestampMs: Date.now(),
3175
- text: "once",
3176
- transcripts: [],
3177
- };
3178
- session.bus.push(Route.Main, eosPacket);
3179
- session.bus.push(Route.Main, eosPacket);
3180
-
3181
- await new Promise((resolve) => setTimeout(resolve, 20));
3182
-
3183
- expect(userInputs).toHaveLength(1);
3184
- expect(userInputs[0]!.text).toBe("once");
3185
- expect(duplicateMetrics).toEqual([{ name: "eos.duplicate_dropped", value: "1" }]);
3186
-
3187
- await closeSession(session);
3188
- });
3189
- });
3190
- });
3191
-
3192
- describe("VoiceAgentSession — handler errors must not kill the call", () => {
3193
- it("a throwing bus handler surfaces a recoverable pipeline.error and the session stays alive", async () => {
3194
- const session = new VoiceAgentSession({ plugins: {} });
3195
- const errors: Array<{ stage: string }> = [];
3196
- const partials: Array<{ text: string }> = [];
3197
- await session.start();
3198
- session.on("error", (event) => {
3199
- errors.push(event);
3200
- });
3201
- session.on("user_input_partial", (event) => {
3202
- partials.push(event);
3203
- });
3204
- session.bus.on("tts.text", () => {
3205
- throw new Error("plugin bug: scripted batches missing");
3206
- });
3207
-
3208
- session.bus.push(Route.Main, {
3209
- kind: "tts.text",
3210
- contextId: "turn-boom",
3211
- timestampMs: Date.now(),
3212
- text: "speak this",
3213
- } satisfies TextToSpeechTextPacket);
3214
- await new Promise((resolve) => setTimeout(resolve, 20));
3215
-
3216
- expect(errors).toEqual([
3217
- expect.objectContaining({ stage: "pipeline.error" }),
3218
- ]);
3219
-
3220
- // The call must continue: later packets still flow end to end.
3221
- session.bus.push(Route.Main, {
3222
- kind: "stt.interim",
3223
- contextId: "turn-after",
3224
- timestampMs: Date.now(),
3225
- text: "still alive",
3226
- } satisfies SttInterimPacket);
3227
- await new Promise((resolve) => setTimeout(resolve, 20));
3228
- expect(partials).toEqual([
3229
- expect.objectContaining({ text: "still alive" }),
3230
- ]);
3231
-
3232
- await closeSession(session);
3233
- });
3234
- });
3235
-
3236
- describe("VoiceAgentSession supersede + thinking-phase barge-in", () => {
3237
- it("cancels a still-playing prior turn's TTS when a new turn completes (L1)", async () => {
3238
- const session = new VoiceAgentSession({ plugins: {} });
3239
- const interruptTts: InterruptTtsPacket[] = [];
3240
- await session.start();
3241
- session.bus.on("interrupt.tts", (pkt) => { interruptTts.push(pkt as InterruptTtsPacket); });
3242
-
3243
- // Turn 1 is generating + has streamed audio (still playing out).
3244
- session.bus.push(Route.Main, {
3245
- kind: "eos.turn_complete", contextId: "turn-1", timestampMs: Date.now(), text: "one", transcripts: [],
3246
- } satisfies EndOfSpeechPacket);
3247
- await new Promise((r) => setTimeout(r, 5));
3248
- session.bus.push(Route.Main, {
3249
- kind: "tts.audio", contextId: "turn-1", timestampMs: Date.now(),
3250
- audio: new Uint8Array(16000), sampleRateHz: 16000, // ~0.5s of audio still playing
3251
- } satisfies TextToSpeechAudioPacket);
3252
- await new Promise((r) => setTimeout(r, 5));
3253
-
3254
- // Turn 2 completes while turn 1 is still playing → turn 1 must be cancelled.
3255
- session.bus.push(Route.Main, {
3256
- kind: "eos.turn_complete", contextId: "turn-2", timestampMs: Date.now(), text: "two", transcripts: [],
3257
- } satisfies EndOfSpeechPacket);
3258
- await new Promise((r) => setTimeout(r, 10));
3259
-
3260
- expect(interruptTts.some((p) => p.contextId === "turn-1")).toBe(true);
3261
- expect(interruptTts.some((p) => p.contextId === "turn-2")).toBe(false);
3262
-
3263
- await closeSession(session);
3264
- });
3265
-
3266
- it("honors a client interrupt during the reasoner TTFT gap, before any audio (B3)", async () => {
3267
- const session = new VoiceAgentSession({ plugins: {} });
3268
- const interruptLlm: InterruptLlmPacket[] = [];
3269
- await session.start();
3270
- session.bus.on("interrupt.llm", (pkt) => { interruptLlm.push(pkt as InterruptLlmPacket); });
3271
-
3272
- // Turn completes; generation is in-flight but NO tts.audio has played yet.
3273
- session.bus.push(Route.Main, {
3274
- kind: "eos.turn_complete", contextId: "turn-think", timestampMs: Date.now(), text: "q", transcripts: [],
3275
- } satisfies EndOfSpeechPacket);
3276
- await new Promise((r) => setTimeout(r, 5));
3277
-
3278
- session.requestClientInterrupt("turn-think");
3279
- await new Promise((r) => setTimeout(r, 10));
3280
-
3281
- expect(interruptLlm.some((p) => p.contextId === "turn-think")).toBe(true);
3282
-
3283
- await closeSession(session);
3284
- });
3285
-
3286
- it("drops a backchannel during the assistant's turn: no cancel, no second response (B4)", async () => {
3287
- const session = new VoiceAgentSession({ plugins: {} });
3288
- const interruptTts: InterruptTtsPacket[] = [];
3289
- const userInputs: UserInputPacket[] = [];
3290
- await session.start();
3291
- session.bus.on("interrupt.tts", (pkt) => { interruptTts.push(pkt as InterruptTtsPacket); });
3292
- session.bus.on("user.input", (pkt) => { userInputs.push(pkt as UserInputPacket); });
3293
-
3294
- // Assistant is speaking turn-1 (audio actively playing out).
3295
- session.bus.push(Route.Main, {
3296
- kind: "eos.turn_complete", contextId: "turn-1", timestampMs: Date.now(), text: "here is the answer", transcripts: [],
3297
- } satisfies EndOfSpeechPacket);
3298
- await new Promise((r) => setTimeout(r, 5));
3299
- session.bus.push(Route.Main, {
3300
- kind: "tts.audio", contextId: "turn-1", timestampMs: Date.now(), audio: new Uint8Array(16000), sampleRateHz: 16000,
3301
- } satisfies TextToSpeechAudioPacket);
3302
- await new Promise((r) => setTimeout(r, 5));
3303
-
3304
- // User says "uh-huh" (a rotated context) mid-answer — a backchannel, not a turn.
3305
- session.bus.push(Route.Main, {
3306
- kind: "eos.turn_complete", contextId: "turn-2", timestampMs: Date.now(), text: "uh-huh", transcripts: [],
3307
- } satisfies EndOfSpeechPacket);
3308
- await new Promise((r) => setTimeout(r, 10));
3309
-
3310
- // The assistant's turn is NOT cancelled and the backchannel does NOT drive the LLM.
3311
- expect(interruptTts.some((p) => p.contextId === "turn-1")).toBe(false);
3312
- expect(userInputs.some((p) => p.contextId === "turn-2")).toBe(false);
3313
-
3314
- await closeSession(session);
3315
- });
3316
- });