@kuralle-syrinx/core 3.1.0 → 4.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@kuralle-syrinx/core",
3
- "version": "3.1.0",
3
+ "version": "4.1.0",
4
4
  "private": false,
5
5
  "type": "module",
6
6
  "license": "MIT",
@@ -49,7 +49,11 @@ export function createConversationEventStream(): [
49
49
  });
50
50
 
51
51
  const push = (event: ConversationEvent): void => {
52
- if (controller) {
52
+ // Only retain events when a reader is attached. A ReadableStream buffers
53
+ // enqueue() without bound; with no debug consumer (nothing in-tree reads this
54
+ // by default) every packet's event would be retained for the whole call and
55
+ // grow memory unbounded. `locked` is true once getReader() is called.
56
+ if (controller && stream.locked) {
53
57
  try {
54
58
  controller.enqueue(event);
55
59
  } catch {
package/src/index.ts CHANGED
@@ -36,6 +36,7 @@ export {
36
36
  type EndOfSpeechAudioPacket,
37
37
  type EndOfSpeechPacket,
38
38
  type InterimEndOfSpeechPacket,
39
+ type EndOfSpeechRetractedPacket,
39
40
  type UserInputPacket,
40
41
  } from "./packets.js";
41
42
 
@@ -63,6 +64,20 @@ export {
63
64
  type ReasoningResumePacket,
64
65
  } from "./packets.js";
65
66
 
67
+ // Pipeline packets — delegate (Responder-Thinker) observability
68
+ export {
69
+ type DelegateQueryPacket,
70
+ type DelegateResultPacket,
71
+ type DelegatePacket,
72
+ type RealtimeResumptionHandlePacket,
73
+ } from "./packets.js";
74
+
75
+ // Durable reasoner session state (G4)
76
+ export {
77
+ InMemoryReasonerSessionStore,
78
+ type ReasonerSessionStore,
79
+ } from "./reasoner-session-store.js";
80
+
66
81
  // Pipeline packets — TTS
67
82
  export {
68
83
  type TextToSpeechTextPacket,
package/src/packets.ts CHANGED
@@ -211,6 +211,16 @@ export interface InterimEndOfSpeechPacket extends VoicePacket {
211
211
  readonly text: string;
212
212
  }
213
213
 
214
+ /**
215
+ * Retraction of a prior `eos.interim` for the same context: the endpoint model
216
+ * signalled a likely turn end, then the user kept speaking (Deepgram Flux
217
+ * `TurnResumed` semantics). Consumers doing speculative work off `eos.interim`
218
+ * must cancel it.
219
+ */
220
+ export interface EndOfSpeechRetractedPacket extends VoicePacket {
221
+ readonly kind: "eos.retracted";
222
+ }
223
+
214
224
  // =============================================================================
215
225
  // User Input (processed — feeds LLM)
216
226
  // =============================================================================
@@ -300,12 +310,58 @@ export interface ReasoningSuspendedPacket extends VoicePacket {
300
310
  readonly payload: unknown;
301
311
  }
302
312
 
313
+ // =============================================================================
314
+ // Delegate (Responder-Thinker) observability packets (Background route)
315
+ // =============================================================================
316
+
317
+ /**
318
+ * The bridge handed a query to the Reasoner. Emitted the moment the delegate
319
+ * turn starts (RealtimeBridge `runDelegate`) or the cascade turn reaches the
320
+ * reasoner (ReasoningBridge), BEFORE the stream runs. Background route:
321
+ * droppable, never blocks the hot path (RFC bimodel-delegate-seam R4).
322
+ * `toolId`/`toolName` are present on realtime delegate turns (the front-model
323
+ * tool call that triggered the delegate); absent on cascade turns.
324
+ */
325
+ export interface DelegateQueryPacket extends VoicePacket {
326
+ readonly kind: "delegate.query";
327
+ readonly query: string;
328
+ readonly toolId?: string;
329
+ readonly toolName?: string;
330
+ }
331
+
332
+ /**
333
+ * The Reasoner produced the turn's final answer. Self-contained: carries the
334
+ * `query` again so a consumer can log/persist the Q&A pair from this one packet
335
+ * without correlating against the earlier `delegate.query`. `durationMs` spans
336
+ * query → answer; `grounded` is true when the reasoner stream surfaced at least
337
+ * one `tool-result` part (e.g. a retrieval hit) while producing the answer.
338
+ */
339
+ export interface DelegateResultPacket extends VoicePacket {
340
+ readonly kind: "delegate.result";
341
+ readonly query: string;
342
+ readonly answer: string;
343
+ readonly durationMs: number;
344
+ readonly grounded: boolean;
345
+ readonly toolId?: string;
346
+ readonly toolName?: string;
347
+ }
348
+
303
349
  export interface ReasoningResumePacket extends VoicePacket {
304
350
  readonly kind: "reasoning.resume";
305
351
  readonly runId: string;
306
352
  readonly data: unknown;
307
353
  }
308
354
 
355
+ /**
356
+ * G4: a realtime provider with native session resume (Gemini Live) issued a new
357
+ * resumption handle. Background route; a durable host (e.g. cf-agents) persists
358
+ * the latest handle and passes it back on reconnect instead of replaying history.
359
+ */
360
+ export interface RealtimeResumptionHandlePacket extends VoicePacket {
361
+ readonly kind: "realtime.resumption_handle";
362
+ readonly handle: string;
363
+ }
364
+
309
365
  // =============================================================================
310
366
  // Output Pipeline Packets (LLM text → TTS audio)
311
367
  // =============================================================================
@@ -510,6 +566,7 @@ export type InputPacket =
510
566
  | EndOfSpeechAudioPacket
511
567
  | EndOfSpeechPacket
512
568
  | InterimEndOfSpeechPacket
569
+ | EndOfSpeechRetractedPacket
513
570
  | UserInputPacket;
514
571
 
515
572
  /** All interruption packets (Critical route). */
@@ -551,3 +608,6 @@ export type AnyErrorPacket =
551
608
 
552
609
  /** Observability packets (Background route). */
553
610
  export type ObservabilityPacket = ConversationMetricPacket | TurnBoundaryEventPacket;
611
+
612
+ /** Delegate (Responder-Thinker) lifecycle packets (Background route). */
613
+ export type DelegatePacket = DelegateQueryPacket | DelegateResultPacket;
@@ -171,6 +171,20 @@ describe("PipelineBusImpl", () => {
171
171
  packet: { kind: "critical.event", contextId: "critical-1" },
172
172
  });
173
173
  });
174
+
175
+ it("does not retain packets when no reader is attached (drop-on-unread)", async () => {
176
+ const bus = createBus();
177
+ // No getReader() → stream unlocked → nothing retained (else a call with no
178
+ // recorder would buffer every audio packet and OOM).
179
+ for (let i = 0; i < 1000; i++) bus.push(Route.Main, pkt("main.event", `n-${i}`));
180
+
181
+ // A reader that attaches later sees only packets pushed AFTER it attaches.
182
+ const reader = bus.allPackets.getReader();
183
+ bus.push(Route.Main, pkt("main.event", "after-reader"));
184
+ const next = await reader.read();
185
+ reader.releaseLock();
186
+ expect(next.value).toMatchObject({ packet: { contextId: "after-reader" } });
187
+ });
174
188
  });
175
189
 
176
190
  describe("error handling", () => {
@@ -239,6 +239,11 @@ export class PipelineBusImpl implements PipelineBus {
239
239
  private publishAllPackets(route: Route, packet: VoicePacket): void {
240
240
  this.onPacket?.(route, packet);
241
241
  if (!this.allPacketsController) return;
242
+ // Only retain packets when a reader is actually attached. A ReadableStream
243
+ // buffers enqueue() without bound until read, so with no consumer (the default
244
+ // deployment — recorder is optional) this would retain every audio buffer of
245
+ // the whole call and OOM. `locked` is true once getReader() is called.
246
+ if (!this.allPackets.locked) return;
242
247
  try {
243
248
  this.allPacketsController.enqueue({ route, packet });
244
249
  } catch {
@@ -0,0 +1,37 @@
1
+ // SPDX-License-Identifier: MIT
2
+ //
3
+ // Syrinx Kernel v2 — durable reasoner session state (G4, RFC bimodel-delegate-seam)
4
+ //
5
+ // The bridge owns conversation history (reasoner.ts §4.5); this seam makes that
6
+ // history durable so a reasoner resumes with the same context after a host
7
+ // eviction/reconnect instead of restarting amnesiac. Concrete backends: the
8
+ // in-memory impl below (Node/tests), a DO-SQLite impl in @kuralle-syrinx/cf-agents.
9
+
10
+ import type { ReasonerMessage } from "./reasoner.js";
11
+
12
+ /**
13
+ * Durable per-session reasoner conversation history. `save` persists the full
14
+ * (already bounded) history snapshot — snapshot semantics, not append, because
15
+ * barge-in truncation rewrites earlier messages to the heard prefix (G25).
16
+ */
17
+ export interface ReasonerSessionStore {
18
+ load(sessionId: string): Promise<readonly ReasonerMessage[]> | readonly ReasonerMessage[];
19
+ save(sessionId: string, messages: readonly ReasonerMessage[]): Promise<void> | void;
20
+ clear(sessionId: string): Promise<void> | void;
21
+ }
22
+
23
+ export class InMemoryReasonerSessionStore implements ReasonerSessionStore {
24
+ private readonly sessions = new Map<string, readonly ReasonerMessage[]>();
25
+
26
+ load(sessionId: string): readonly ReasonerMessage[] {
27
+ return this.sessions.get(sessionId) ?? [];
28
+ }
29
+
30
+ save(sessionId: string, messages: readonly ReasonerMessage[]): void {
31
+ this.sessions.set(sessionId, [...messages]);
32
+ }
33
+
34
+ clear(sessionId: string): void {
35
+ this.sessions.delete(sessionId);
36
+ }
37
+ }
@@ -89,6 +89,20 @@ export class TtsPlayoutClock {
89
89
  return this.active.has(contextId);
90
90
  }
91
91
 
92
+ /** All still-active contexts, insertion order (oldest first). */
93
+ activeContexts(): string[] {
94
+ return [...this.active];
95
+ }
96
+
97
+ /**
98
+ * Wall-clock estimate of when this context's audio finishes playing out, or
99
+ * undefined if unknown. Anchors the idle timer to real playout end rather than
100
+ * chunk-arrival (TTS streams faster than realtime).
101
+ */
102
+ playoutEnd(contextId: string): number | undefined {
103
+ return this.playoutEndMs.get(contextId);
104
+ }
105
+
92
106
  /** The most-recently-added still-active context (insertion order), or "" if none. */
93
107
  latestActive(): string {
94
108
  let latest = "";
@@ -49,7 +49,7 @@ const BACKCHANNELS = new Set([
49
49
  "oh",
50
50
  ]);
51
51
 
52
- function isBackchannel(text: string): boolean {
52
+ export function isBackchannel(text: string): boolean {
53
53
  const norm = text
54
54
  .toLowerCase()
55
55
  .replace(/[^a-z\s'-]/g, "")
@@ -321,6 +321,316 @@ describe("VoiceAgentSession", () => {
321
321
  await closeSession(session);
322
322
  });
323
323
 
324
+ it("G2/WBS-1: surfaces delegate.query/delegate.result packets as delegate_query/delegate_result session events", async () => {
325
+ const session = new VoiceAgentSession({ plugins: {} });
326
+ const queries: Array<{ turnId: string; query: string; toolName?: string }> = [];
327
+ const results: Array<{ turnId: string; query: string; answer: string; durationMs: number; grounded: boolean }> = [];
328
+ session.on("delegate_query", (event) => queries.push(event));
329
+ session.on("delegate_result", (event) => results.push(event));
330
+ await session.start();
331
+
332
+ session.bus.push(Route.Background, {
333
+ kind: "delegate.query",
334
+ contextId: "turn-1",
335
+ timestampMs: Date.now(),
336
+ query: "When is the deadline?",
337
+ toolId: "call_1",
338
+ toolName: "consult_knowledge",
339
+ });
340
+ session.bus.push(Route.Background, {
341
+ kind: "delegate.result",
342
+ contextId: "turn-1",
343
+ timestampMs: Date.now(),
344
+ query: "When is the deadline?",
345
+ answer: "March 31.",
346
+ durationMs: 42,
347
+ grounded: true,
348
+ toolId: "call_1",
349
+ toolName: "consult_knowledge",
350
+ });
351
+
352
+ await new Promise((resolve) => setTimeout(resolve, 20));
353
+
354
+ expect(queries).toEqual([
355
+ { tsMs: expect.any(Number), turnId: "turn-1", query: "When is the deadline?", toolId: "call_1", toolName: "consult_knowledge" },
356
+ ]);
357
+ expect(results).toEqual([
358
+ {
359
+ tsMs: expect.any(Number),
360
+ turnId: "turn-1",
361
+ query: "When is the deadline?",
362
+ answer: "March 31.",
363
+ durationMs: 42,
364
+ grounded: true,
365
+ toolId: "call_1",
366
+ toolName: "consult_knowledge",
367
+ },
368
+ ]);
369
+
370
+ await closeSession(session);
371
+ });
372
+
373
+ it("emits a decomposed turn_latency session event on first TTS audio", async () => {
374
+ const session = new VoiceAgentSession({ plugins: {} });
375
+ const events: Array<Record<string, unknown>> = [];
376
+ session.on("turn_latency", (event) => {
377
+ events.push(event);
378
+ });
379
+ await session.start();
380
+
381
+ const t0 = 100_000;
382
+ session.bus.push(Route.Main, { kind: "vad.speech_ended", contextId: "turn-1", timestampMs: t0 });
383
+ await new Promise((resolve) => setTimeout(resolve, 10));
384
+ session.bus.push(Route.Main, {
385
+ kind: "eos.turn_complete",
386
+ contextId: "turn-1",
387
+ timestampMs: t0 + 200,
388
+ text: "what are the lab fees",
389
+ transcripts: [],
390
+ });
391
+ await new Promise((resolve) => setTimeout(resolve, 10));
392
+ session.bus.push(Route.Main, { kind: "llm.delta", contextId: "turn-1", timestampMs: t0 + 900, text: "The fee is ten dollars." });
393
+ await new Promise((resolve) => setTimeout(resolve, 10));
394
+ session.bus.push(Route.Main, {
395
+ kind: "tts.audio",
396
+ contextId: "turn-1",
397
+ timestampMs: t0 + 1150,
398
+ audio: new Uint8Array(320),
399
+ sampleRateHz: 16000,
400
+ });
401
+ await new Promise((resolve) => setTimeout(resolve, 20));
402
+
403
+ expect(events).toHaveLength(1);
404
+ expect(events[0]).toEqual({
405
+ tsMs: expect.any(Number),
406
+ turnId: "turn-1",
407
+ ttfaMs: 1150,
408
+ eouDelayMs: 200,
409
+ llmTtftMs: 700,
410
+ ttsTtfbMs: 250,
411
+ fillerUsed: false,
412
+ });
413
+
414
+ await closeSession(session);
415
+ });
416
+
417
+ it("turn_latency anchors TTFA to eos when no VAD speech-end was seen, and marks filler turns", async () => {
418
+ const session = new VoiceAgentSession({ plugins: {}, latencyFillerEnabled: true });
419
+ const events: Array<Record<string, unknown>> = [];
420
+ session.on("turn_latency", (event) => {
421
+ events.push(event);
422
+ });
423
+ await session.start();
424
+
425
+ const t0 = 200_000;
426
+ session.bus.push(Route.Main, {
427
+ kind: "eos.turn_complete",
428
+ contextId: "turn-2",
429
+ timestampMs: t0,
430
+ text: "tell me about registration holds",
431
+ transcripts: [],
432
+ });
433
+ await new Promise((resolve) => setTimeout(resolve, 10));
434
+ session.bus.push(Route.Main, {
435
+ kind: "tts.audio",
436
+ contextId: "turn-2",
437
+ timestampMs: t0 + 400,
438
+ audio: new Uint8Array(320),
439
+ sampleRateHz: 16000,
440
+ });
441
+ await new Promise((resolve) => setTimeout(resolve, 20));
442
+
443
+ expect(events).toHaveLength(1);
444
+ expect(events[0]).toMatchObject({
445
+ turnId: "turn-2",
446
+ ttfaMs: 400,
447
+ fillerUsed: true,
448
+ });
449
+ expect(events[0]!["eouDelayMs"]).toBeUndefined();
450
+
451
+ await closeSession(session);
452
+ });
453
+
454
+ it("turn_latency is not emitted for an interrupted turn", async () => {
455
+ const session = new VoiceAgentSession({ plugins: {} });
456
+ const events: unknown[] = [];
457
+ session.on("turn_latency", (event) => {
458
+ events.push(event);
459
+ });
460
+ await session.start();
461
+
462
+ const t0 = 300_000;
463
+ session.bus.push(Route.Main, {
464
+ kind: "eos.turn_complete",
465
+ contextId: "turn-3",
466
+ timestampMs: t0,
467
+ text: "hello there",
468
+ transcripts: [],
469
+ });
470
+ await new Promise((resolve) => setTimeout(resolve, 10));
471
+ session.bus.push(Route.Critical, {
472
+ kind: "interrupt.detected",
473
+ contextId: "turn-3",
474
+ timestampMs: t0 + 100,
475
+ });
476
+ await new Promise((resolve) => setTimeout(resolve, 10));
477
+ session.bus.push(Route.Main, {
478
+ kind: "tts.audio",
479
+ contextId: "turn-3",
480
+ timestampMs: t0 + 300,
481
+ audio: new Uint8Array(320),
482
+ sampleRateHz: 16000,
483
+ });
484
+ await new Promise((resolve) => setTimeout(resolve, 20));
485
+
486
+ expect(events).toHaveLength(0);
487
+
488
+ await closeSession(session);
489
+ });
490
+
491
+ it("G3/WBS-3: tool-call lifecycle cues fire started → delayed → complete", async () => {
492
+ const session = new VoiceAgentSession({ plugins: {}, delayCueAfterMs: 30 });
493
+ const cues: Array<{ phase: string; turnId: string; toolId: string; toolName: string; afterMs?: number }> = [];
494
+ session.on("tool_call_cue", (event) => cues.push(event));
495
+ await session.start();
496
+
497
+ session.bus.push(Route.Main, {
498
+ kind: "llm.tool_call",
499
+ contextId: "turn-1",
500
+ timestampMs: Date.now(),
501
+ toolId: "t1",
502
+ toolName: "consult_knowledge",
503
+ toolArgs: { query: "fees" },
504
+ });
505
+
506
+ await new Promise((resolve) => setTimeout(resolve, 70));
507
+
508
+ session.bus.push(Route.Main, {
509
+ kind: "llm.tool_result",
510
+ contextId: "turn-1",
511
+ timestampMs: Date.now(),
512
+ toolId: "t1",
513
+ toolName: "consult_knowledge",
514
+ result: "answer",
515
+ });
516
+
517
+ await new Promise((resolve) => setTimeout(resolve, 20));
518
+
519
+ expect(cues.map((cue) => cue.phase)).toEqual(["started", "delayed", "complete"]);
520
+ expect(cues[0]).toMatchObject({ turnId: "turn-1", toolId: "t1", toolName: "consult_knowledge" });
521
+ expect(cues[1]!.afterMs).toBe(30);
522
+
523
+ await closeSession(session);
524
+ });
525
+
526
+ it("G3/WBS-3: no delayed cue when the result beats the timer; timer 0 disables it", async () => {
527
+ const session = new VoiceAgentSession({ plugins: {}, delayCueAfterMs: 5000 });
528
+ const cues: Array<{ phase: string }> = [];
529
+ session.on("tool_call_cue", (event) => cues.push(event));
530
+ await session.start();
531
+
532
+ session.bus.push(Route.Main, {
533
+ kind: "llm.tool_call",
534
+ contextId: "turn-1",
535
+ timestampMs: Date.now(),
536
+ toolId: "t1",
537
+ toolName: "consult_knowledge",
538
+ toolArgs: {},
539
+ });
540
+ session.bus.push(Route.Main, {
541
+ kind: "llm.tool_result",
542
+ contextId: "turn-1",
543
+ timestampMs: Date.now(),
544
+ toolId: "t1",
545
+ toolName: "consult_knowledge",
546
+ result: "fast answer",
547
+ });
548
+
549
+ await new Promise((resolve) => setTimeout(resolve, 30));
550
+ expect(cues.map((cue) => cue.phase)).toEqual(["started", "complete"]);
551
+
552
+ await closeSession(session);
553
+
554
+ const disabled = new VoiceAgentSession({ plugins: {}, delayCueAfterMs: 0 });
555
+ const disabledCues: Array<{ phase: string }> = [];
556
+ disabled.on("tool_call_cue", (event) => disabledCues.push(event));
557
+ await disabled.start();
558
+ disabled.bus.push(Route.Main, {
559
+ kind: "llm.tool_call",
560
+ contextId: "turn-2",
561
+ timestampMs: Date.now(),
562
+ toolId: "t2",
563
+ toolName: "consult_knowledge",
564
+ toolArgs: {},
565
+ });
566
+ await new Promise((resolve) => setTimeout(resolve, 30));
567
+ expect(disabledCues.map((cue) => cue.phase)).toEqual(["started"]);
568
+ await closeSession(disabled);
569
+ });
570
+
571
+ it("G3/WBS-3: llm.error while a tool call is pending fires the failed cue", async () => {
572
+ const session = new VoiceAgentSession({ plugins: {}, delayCueAfterMs: 0 });
573
+ const cues: Array<{ phase: string; toolId: string }> = [];
574
+ session.on("tool_call_cue", (event) => cues.push(event));
575
+ await session.start();
576
+
577
+ session.bus.push(Route.Main, {
578
+ kind: "llm.tool_call",
579
+ contextId: "turn-1",
580
+ timestampMs: Date.now(),
581
+ toolId: "t1",
582
+ toolName: "consult_knowledge",
583
+ toolArgs: {},
584
+ });
585
+ await new Promise((resolve) => setTimeout(resolve, 10)); // let Main drain before the Critical error
586
+ session.bus.push(Route.Critical, {
587
+ kind: "llm.error",
588
+ contextId: "turn-1",
589
+ timestampMs: Date.now(),
590
+ component: "bridge",
591
+ category: ErrorCategory.NetworkTimeout,
592
+ cause: new Error("delegate failed"),
593
+ isRecoverable: true,
594
+ });
595
+
596
+ await new Promise((resolve) => setTimeout(resolve, 30));
597
+ expect(cues.map((cue) => cue.phase)).toEqual(["started", "failed"]);
598
+ expect(cues[1]!.toolId).toBe("t1");
599
+
600
+ await closeSession(session);
601
+ });
602
+
603
+ it("G3/WBS-3 (R5): barge-in fails the pending cue and the interrupt path is unaffected", async () => {
604
+ const session = new VoiceAgentSession({ plugins: {}, delayCueAfterMs: 5000, minInterruptionMs: 0 });
605
+ const cues: Array<{ phase: string }> = [];
606
+ const interrupts: string[] = [];
607
+ session.on("tool_call_cue", (event) => cues.push(event));
608
+ session.bus.on("interrupt.tts", (pkt) => { interrupts.push((pkt as { contextId: string }).contextId); });
609
+ await session.start();
610
+
611
+ session.bus.push(Route.Main, {
612
+ kind: "llm.tool_call",
613
+ contextId: "turn-1",
614
+ timestampMs: Date.now(),
615
+ toolId: "t1",
616
+ toolName: "consult_knowledge",
617
+ toolArgs: {},
618
+ });
619
+ await new Promise((resolve) => setTimeout(resolve, 10)); // let Main drain before the Critical interrupt
620
+ session.bus.push(Route.Critical, {
621
+ kind: "interrupt.detected",
622
+ contextId: "turn-1",
623
+ timestampMs: Date.now(),
624
+ source: "client",
625
+ });
626
+
627
+ await new Promise((resolve) => setTimeout(resolve, 30));
628
+ expect(cues.map((cue) => cue.phase)).toEqual(["started", "failed"]);
629
+ expect(interrupts).toContain("turn-1"); // barge-in cancel still flowed
630
+
631
+ await closeSession(session);
632
+ });
633
+
324
634
  it("emits normalized debug events for bus packets", async () => {
325
635
  const session = new VoiceAgentSession({ plugins: {} });
326
636
  const reader = session.debugEvents.getReader();
@@ -2485,3 +2795,85 @@ describe("VoiceAgentSession — handler errors must not kill the call", () => {
2485
2795
  await closeSession(session);
2486
2796
  });
2487
2797
  });
2798
+
2799
+ describe("VoiceAgentSession supersede + thinking-phase barge-in", () => {
2800
+ it("cancels a still-playing prior turn's TTS when a new turn completes (L1)", async () => {
2801
+ const session = new VoiceAgentSession({ plugins: {} });
2802
+ const interruptTts: InterruptTtsPacket[] = [];
2803
+ await session.start();
2804
+ session.bus.on("interrupt.tts", (pkt) => { interruptTts.push(pkt as InterruptTtsPacket); });
2805
+
2806
+ // Turn 1 is generating + has streamed audio (still playing out).
2807
+ session.bus.push(Route.Main, {
2808
+ kind: "eos.turn_complete", contextId: "turn-1", timestampMs: Date.now(), text: "one", transcripts: [],
2809
+ } satisfies EndOfSpeechPacket);
2810
+ await new Promise((r) => setTimeout(r, 5));
2811
+ session.bus.push(Route.Main, {
2812
+ kind: "tts.audio", contextId: "turn-1", timestampMs: Date.now(),
2813
+ audio: new Uint8Array(16000), sampleRateHz: 16000, // ~0.5s of audio still playing
2814
+ } satisfies TextToSpeechAudioPacket);
2815
+ await new Promise((r) => setTimeout(r, 5));
2816
+
2817
+ // Turn 2 completes while turn 1 is still playing → turn 1 must be cancelled.
2818
+ session.bus.push(Route.Main, {
2819
+ kind: "eos.turn_complete", contextId: "turn-2", timestampMs: Date.now(), text: "two", transcripts: [],
2820
+ } satisfies EndOfSpeechPacket);
2821
+ await new Promise((r) => setTimeout(r, 10));
2822
+
2823
+ expect(interruptTts.some((p) => p.contextId === "turn-1")).toBe(true);
2824
+ expect(interruptTts.some((p) => p.contextId === "turn-2")).toBe(false);
2825
+
2826
+ await closeSession(session);
2827
+ });
2828
+
2829
+ it("honors a client interrupt during the reasoner TTFT gap, before any audio (B3)", async () => {
2830
+ const session = new VoiceAgentSession({ plugins: {} });
2831
+ const interruptLlm: InterruptLlmPacket[] = [];
2832
+ await session.start();
2833
+ session.bus.on("interrupt.llm", (pkt) => { interruptLlm.push(pkt as InterruptLlmPacket); });
2834
+
2835
+ // Turn completes; generation is in-flight but NO tts.audio has played yet.
2836
+ session.bus.push(Route.Main, {
2837
+ kind: "eos.turn_complete", contextId: "turn-think", timestampMs: Date.now(), text: "q", transcripts: [],
2838
+ } satisfies EndOfSpeechPacket);
2839
+ await new Promise((r) => setTimeout(r, 5));
2840
+
2841
+ session.requestClientInterrupt("turn-think");
2842
+ await new Promise((r) => setTimeout(r, 10));
2843
+
2844
+ expect(interruptLlm.some((p) => p.contextId === "turn-think")).toBe(true);
2845
+
2846
+ await closeSession(session);
2847
+ });
2848
+
2849
+ it("drops a backchannel during the assistant's turn: no cancel, no second response (B4)", async () => {
2850
+ const session = new VoiceAgentSession({ plugins: {} });
2851
+ const interruptTts: InterruptTtsPacket[] = [];
2852
+ const userInputs: UserInputPacket[] = [];
2853
+ await session.start();
2854
+ session.bus.on("interrupt.tts", (pkt) => { interruptTts.push(pkt as InterruptTtsPacket); });
2855
+ session.bus.on("user.input", (pkt) => { userInputs.push(pkt as UserInputPacket); });
2856
+
2857
+ // Assistant is speaking turn-1 (audio actively playing out).
2858
+ session.bus.push(Route.Main, {
2859
+ kind: "eos.turn_complete", contextId: "turn-1", timestampMs: Date.now(), text: "here is the answer", transcripts: [],
2860
+ } satisfies EndOfSpeechPacket);
2861
+ await new Promise((r) => setTimeout(r, 5));
2862
+ session.bus.push(Route.Main, {
2863
+ kind: "tts.audio", contextId: "turn-1", timestampMs: Date.now(), audio: new Uint8Array(16000), sampleRateHz: 16000,
2864
+ } satisfies TextToSpeechAudioPacket);
2865
+ await new Promise((r) => setTimeout(r, 5));
2866
+
2867
+ // User says "uh-huh" (a rotated context) mid-answer — a backchannel, not a turn.
2868
+ session.bus.push(Route.Main, {
2869
+ kind: "eos.turn_complete", contextId: "turn-2", timestampMs: Date.now(), text: "uh-huh", transcripts: [],
2870
+ } satisfies EndOfSpeechPacket);
2871
+ await new Promise((r) => setTimeout(r, 10));
2872
+
2873
+ // The assistant's turn is NOT cancelled and the backchannel does NOT drive the LLM.
2874
+ expect(interruptTts.some((p) => p.contextId === "turn-1")).toBe(false);
2875
+ expect(userInputs.some((p) => p.contextId === "turn-2")).toBe(false);
2876
+
2877
+ await closeSession(session);
2878
+ });
2879
+ });
@@ -29,6 +29,8 @@ import type {
29
29
  UserAudioReceivedPacket,
30
30
  UserTextReceivedPacket,
31
31
  InterruptionDetectedPacket,
32
+ DelegateQueryPacket,
33
+ DelegateResultPacket,
32
34
  LlmDeltaPacket,
33
35
  LlmResponseDonePacket,
34
36
  LlmToolCallPacket,
@@ -61,7 +63,7 @@ import { LatencyFillerController } from "./latency-filler.js";
61
63
  import { PrimarySpeakerGate } from "./primary-speaker-gate.js";
62
64
  import { takeCompleteVoiceText, isCompleteVoiceText, appendVoiceText } from "./voice-text.js";
63
65
  import { TtsPlayoutClock } from "./tts-playout-clock.js";
64
- import { TurnArbiter } from "./turn-arbiter.js";
66
+ import { TurnArbiter, isBackchannel } from "./turn-arbiter.js";
65
67
  import * as make from "./packet-factories.js";
66
68
  import { pluginStage, stageOrder, isAudioStage } from "./init-stage-order.js";
67
69
  import {
@@ -147,6 +149,14 @@ export interface VoiceAgentSessionConfig {
147
149
  * (TTS/STT-failure fallback needs canned audio / a clarification prompt — out of scope.)
148
150
  */
149
151
  errorFallbackText?: string;
152
+ /**
153
+ * G3 (RFC bimodel-delegate-seam): ms a tool call may stay pending before the
154
+ * `tool_call_cue` session event fires its time-triggered `"delayed"` phase — the
155
+ * "still working" cue clients render during a long reasoner wait (cf. Vapi's
156
+ * `request-response-delayed` + `timingMilliseconds`). 0 disables the delayed
157
+ * phase; started/complete/failed always fire. Default: 2000.
158
+ */
159
+ delayCueAfterMs?: number;
150
160
  /**
151
161
  * Which component owns turn boundary (EOS) for this session. Defaults to
152
162
  * provider STT ownership; Smart Turn sessions must opt in explicitly.
@@ -170,6 +180,33 @@ export interface VoiceAgentSessionEvents {
170
180
  agent_text_delta: (event: { tsMs: number; turnId: string; delta: string }) => void;
171
181
  agent_tool_call: (event: { tsMs: number; turnId: string; id: string; name: string; args: Record<string, unknown> }) => void;
172
182
  agent_tool_result: (event: { tsMs: number; turnId: string; id: string; result: string; durationMs: number }) => void;
183
+ delegate_query: (event: { tsMs: number; turnId: string; query: string; toolId?: string; toolName?: string }) => void;
184
+ delegate_result: (event: { tsMs: number; turnId: string; query: string; answer: string; durationMs: number; grounded: boolean; toolId?: string; toolName?: string }) => void;
185
+ /**
186
+ * G3: typed preamble/filler lifecycle for a pending tool call (Vapi-shaped:
187
+ * started / delayed / complete / failed). `delayed` is time-triggered by
188
+ * `delayCueAfterMs` while the call is still pending; `failed` fires on an LLM/bridge
189
+ * error, a barge-in, or a superseding turn while pending (R5). Transports surface
190
+ * these as `tool_call_*` wire messages — the standard "thinking" cue.
191
+ */
192
+ tool_call_cue: (event: { tsMs: number; turnId: string; phase: "started" | "delayed" | "complete" | "failed"; toolId: string; toolName: string; afterMs?: number }) => void;
193
+ /**
194
+ * Per-turn latency decomposition, emitted once at the turn's first TTS audio.
195
+ * `ttfaMs` is anchored to the real end of user speech (VAD speech-end, falling
196
+ * back to the endpoint decision) — and `fillerUsed` flags turns where a latency
197
+ * filler spoke first, so TTFA on those turns is not mistaken for reasoning speed.
198
+ * Decomposition: eouDelayMs (speech end → endpoint) + llmTtftMs (endpoint →
199
+ * first LLM delta) + ttsTtfbMs (first TTS text dispatched → first audio).
200
+ */
201
+ turn_latency: (event: {
202
+ tsMs: number;
203
+ turnId: string;
204
+ ttfaMs: number;
205
+ eouDelayMs?: number;
206
+ llmTtftMs?: number;
207
+ ttsTtfbMs?: number;
208
+ fillerUsed: boolean;
209
+ }) => void;
173
210
  agent_first_audio: (event: { tsMs: number; turnId: string }) => void;
174
211
  agent_finished: (event: { tsMs: number; turnId: string } & Record<string, unknown>) => void;
175
212
  error: (event: { tsMs: number; stage: string; category: string; message: string }) => void;
@@ -184,9 +221,22 @@ interface TtsTextBuffer {
184
221
  emitted: string;
185
222
  }
186
223
 
224
+ interface TurnTiming {
225
+ speechEndedMs?: number;
226
+ eosMs?: number;
227
+ firstLlmDeltaMs?: number;
228
+ firstTtsTextMs?: number;
229
+ fillerUsed?: boolean;
230
+ }
231
+
187
232
  /** Suffix marking a context created to speak an error fallback, so it never recurses. */
188
233
  const FALLBACK_CONTEXT_SUFFIX = ":error-fallback";
189
234
 
235
+ /** Scheduler key for a pending tool call's G3 delayed-cue timer. */
236
+ function toolCueTimerKey(contextId: string, toolId: string): string {
237
+ return `tool_cue:${contextId}:${toolId}`;
238
+ }
239
+
190
240
  // =============================================================================
191
241
  // Session Implementation
192
242
  // =============================================================================
@@ -211,6 +261,10 @@ export class VoiceAgentSession {
211
261
  private readonly scheduler: Scheduler;
212
262
  private readonly ttsPlayout: TtsPlayoutClock;
213
263
  private interruptedGenerationContextIds = new Set<string>();
264
+ // Turns whose generation is in-flight (eos.turn_complete emitted, not yet
265
+ // finished/interrupted). Lets a client "stop" during the reasoner TTFT gap —
266
+ // before any audio plays — still abort the turn (thinking-phase barge-in, B3).
267
+ private generatingContextIds = new Set<string>();
214
268
  private ttsTextBuffers = new Map<string, TtsTextBuffer>();
215
269
  private readonly minInterruptionMs: number;
216
270
  private readonly primarySpeakerGate: PrimarySpeakerGate;
@@ -223,10 +277,14 @@ export class VoiceAgentSession {
223
277
  private readonly watchdogs!: VoiceSessionWatchdogs;
224
278
  private readonly observabilityObserver: ObservabilityObserver;
225
279
  private turnUserStoppedAtMs = new Map<string, number>();
280
+ private turnTimings = new Map<string, TurnTiming>();
226
281
  private speakerEnrollmentContextId: string | null = null;
227
282
  private firstTtsAudioFired = new Set<string>();
228
283
  private readonly errorFallbackText: string;
229
284
  private fallbackInjectedContexts = new Set<string>();
285
+ // G3: pending tool calls per context (toolId → toolName) driving the tool_call_cue lifecycle.
286
+ private readonly delayCueAfterMs: number;
287
+ private pendingToolCues = new Map<string, Map<string, string>>();
230
288
  private readonly endpointingOwner: "provider_stt" | "smart_turn" | "timer";
231
289
  private lastFinalizedContextId = "";
232
290
 
@@ -241,6 +299,7 @@ export class VoiceAgentSession {
241
299
  this.ttsPlayout = new TtsPlayoutClock(this.scheduler);
242
300
  this.sttForceFinalizeTimeoutMs = config.sttForceFinalizeTimeoutMs ?? 7000;
243
301
  this.minInterruptionMs = config.minInterruptionMs ?? 280;
302
+ this.delayCueAfterMs = config.delayCueAfterMs ?? 2000;
244
303
  this.primarySpeakerGate = new PrimarySpeakerGate({
245
304
  enabled: config.primarySpeakerBargeInEnabled !== false,
246
305
  });
@@ -389,11 +448,16 @@ export class VoiceAgentSession {
389
448
  this.ttsPlayout.clear();
390
449
  this.turnArbiter.clear();
391
450
  this.turnUserStoppedAtMs.clear();
451
+ this.turnTimings.clear();
392
452
  this.firstTtsAudioFired.clear();
393
453
  this.fallbackInjectedContexts.clear();
394
454
  this.ttsTextBuffers.clear();
395
455
  this.interruptedGenerationContextIds.clear();
396
456
  this.firstLlmDeltaReceived.clear();
457
+ for (const [contextId, pending] of this.pendingToolCues) {
458
+ for (const toolId of pending.keys()) this.scheduler.cancel(toolCueTimerKey(contextId, toolId));
459
+ }
460
+ this.pendingToolCues.clear();
397
461
 
398
462
  // 2. Run finalize chain (reverse order)
399
463
  await runFinalizeChain(this.initSteps);
@@ -414,7 +478,16 @@ export class VoiceAgentSession {
414
478
  }
415
479
 
416
480
  requestClientInterrupt(contextId: string): void {
417
- this.turnArbiter.commitClientInterrupt(contextId);
481
+ // Playing out → the arbiter owns the barge-in (primary-speaker reset + metrics).
482
+ if (this.ttsPlayout.isActive(contextId)) {
483
+ this.turnArbiter.commitClientInterrupt(contextId);
484
+ return;
485
+ }
486
+ // Thinking-phase barge-in (B3): no audio yet, but a generation is in-flight —
487
+ // abort it so "stop" during the reasoner TTFT gap is honored, not dropped.
488
+ if (this.generatingContextIds.has(contextId)) {
489
+ this.bus.push(Route.Critical, make.interruptDetected(contextId, Date.now(), "client"));
490
+ }
418
491
  }
419
492
 
420
493
  // =========================================================================
@@ -486,6 +559,10 @@ export class VoiceAgentSession {
486
559
  this.bus.on("llm.tool_call", this.handleLlmToolCall.bind(this));
487
560
  this.bus.on("llm.tool_result", this.handleLlmToolResult.bind(this));
488
561
 
562
+ // Delegate (Responder-Thinker) observability — G2, RFC bimodel-delegate-seam
563
+ this.bus.on("delegate.query", this.handleDelegateQuery.bind(this));
564
+ this.bus.on("delegate.result", this.handleDelegateResult.bind(this));
565
+
489
566
  // TTS
490
567
  this.bus.on("tts.audio", this.handleTtsAudio.bind(this));
491
568
  this.bus.on("tts.end", this.handleTtsEnd.bind(this));
@@ -691,14 +768,61 @@ export class VoiceAgentSession {
691
768
  this.turnArbiter.onSpeechEnded(pkt, Boolean(this.latestActiveTtsContextId()));
692
769
 
693
770
  this.turnUserStoppedAtMs.set(pkt.contextId, pkt.timestampMs);
771
+ this.timingFor(pkt.contextId).speechEndedMs = pkt.timestampMs;
694
772
  this.watchdogs.startVaqiMissedResponseTimer(pkt.contextId, pkt.timestampMs);
695
773
  }
696
774
 
775
+ private timingFor(contextId: string): TurnTiming {
776
+ let timing = this.turnTimings.get(contextId);
777
+ if (!timing) {
778
+ timing = {};
779
+ this.turnTimings.set(contextId, timing);
780
+ }
781
+ return timing;
782
+ }
783
+
784
+ private emitTurnLatency(contextId: string, firstAudioMs: number): void {
785
+ const timing = this.turnTimings.get(contextId);
786
+ this.turnTimings.delete(contextId);
787
+ if (!timing) return;
788
+ const anchorMs = timing.speechEndedMs ?? timing.eosMs;
789
+ if (anchorMs === undefined) return; // text-injected or fallback turn — not a voice TTFA
790
+ this.emit("turn_latency", {
791
+ tsMs: firstAudioMs,
792
+ turnId: contextId,
793
+ ttfaMs: firstAudioMs - anchorMs,
794
+ ...(timing.speechEndedMs !== undefined && timing.eosMs !== undefined
795
+ ? { eouDelayMs: timing.eosMs - timing.speechEndedMs }
796
+ : {}),
797
+ ...(timing.eosMs !== undefined && timing.firstLlmDeltaMs !== undefined
798
+ ? { llmTtftMs: timing.firstLlmDeltaMs - timing.eosMs }
799
+ : {}),
800
+ ...(timing.firstTtsTextMs !== undefined
801
+ ? { ttsTtfbMs: firstAudioMs - timing.firstTtsTextMs }
802
+ : {}),
803
+ fillerUsed: timing.fillerUsed === true,
804
+ });
805
+ }
806
+
697
807
  private handleTurnComplete(pkt: EndOfSpeechPacket): void {
698
808
  if (this.lastFinalizedContextId === pkt.contextId) {
699
809
  this.bus.push(Route.Background, make.metric(pkt.contextId, "eos.duplicate_dropped", "1"));
700
810
  return;
701
811
  }
812
+
813
+ // A backchannel ("uh-huh", "okay") uttered WHILE the assistant is still speaking
814
+ // is not a turn (B4). Dropping it here does two things: it does NOT cancel the
815
+ // assistant's in-flight answer (the supersede path below would otherwise kill it),
816
+ // and it does NOT spawn a second LLM response to the backchannel (the double-reply
817
+ // bug). A backchannel with no assistant currently speaking IS a real turn — only
818
+ // suppress when another context's TTS is actively playing. (English-only classifier
819
+ // today — locale-aware backchannels are a separate improvement.)
820
+ const otherTtsActive = this.ttsPlayout.activeContexts().some((c) => c !== pkt.contextId);
821
+ if (otherTtsActive && isBackchannel(pkt.text)) {
822
+ this.bus.push(Route.Background, make.metric(pkt.contextId, "turn.backchannel_dropped", pkt.text));
823
+ return;
824
+ }
825
+
702
826
  this.lastFinalizedContextId = pkt.contextId;
703
827
 
704
828
  // Re-arm per-turn guard state for the next turn. Transports with a stable
@@ -711,6 +835,16 @@ export class VoiceAgentSession {
711
835
  this.interruptedGenerationContextIds.delete(pkt.contextId);
712
836
  this.fallbackInjectedContexts.delete(pkt.contextId);
713
837
 
838
+ // Supersede (L1): a new turn must cancel any still-active prior-turn TTS or
839
+ // generation. Without this, a false-EOS (early endpoint on a mid-sentence
840
+ // pause) starts turn N, the user resumes, turn N's already-emitted audio
841
+ // keeps synthesizing, and it plays over the user while turn N+1 is answered.
842
+ // The bridge supersedes the *LLM*; only the session can stop the *TTS*.
843
+ for (const activeCtx of this.ttsPlayout.activeContexts()) {
844
+ if (activeCtx !== pkt.contextId) this.cancelStaleGeneration(activeCtx, pkt.timestampMs);
845
+ }
846
+
847
+ this.generatingContextIds.add(pkt.contextId);
714
848
  this.currentTurnId = pkt.contextId;
715
849
  this.idleTimeout.setContextId(pkt.contextId);
716
850
 
@@ -727,11 +861,19 @@ export class VoiceAgentSession {
727
861
  timestampMs: pkt.timestampMs,
728
862
  });
729
863
 
730
- // Stop idle timeout while LLM is processing
731
- this.bus.push(Route.Main, make.stopIdleTimeout(pkt.contextId, Date.now(), false));
864
+ // Stop idle timeout while the LLM processes. The user just spoke — that is
865
+ // genuine engagement, so reset the idle *escalation* count (P2): a user who
866
+ // answers the first "are you there?" must not be escalated straight to the
867
+ // disconnect prompt later in the call.
868
+ this.bus.push(Route.Main, make.stopIdleTimeout(pkt.contextId, Date.now(), true));
869
+
870
+ const timing = this.timingFor(pkt.contextId);
871
+ timing.eosMs = pkt.timestampMs;
732
872
 
733
873
  const fillerText = this.latencyFiller.start(pkt.contextId, pkt.text, pkt.timestampMs);
734
874
  if (fillerText) {
875
+ timing.fillerUsed = true;
876
+ timing.firstTtsTextMs ??= pkt.timestampMs;
735
877
  this.bus.push(Route.Main, make.ttsText(pkt.contextId, Date.now(), fillerText));
736
878
  this.bus.push(Route.Background, make.metric(pkt.contextId, "filler.started", fillerText));
737
879
  }
@@ -763,6 +905,8 @@ export class VoiceAgentSession {
763
905
  let deltaText = pkt.text;
764
906
  if (!this.firstLlmDeltaReceived.has(pkt.contextId)) {
765
907
  this.firstLlmDeltaReceived.add(pkt.contextId);
908
+ const timing = this.turnTimings.get(pkt.contextId);
909
+ if (timing) timing.firstLlmDeltaMs ??= pkt.timestampMs;
766
910
  if (this.latencyFiller.isActive(pkt.contextId)) {
767
911
  deltaText = this.latencyFiller.spliceLlmDelta(pkt.contextId, deltaText);
768
912
  this.bus.push(Route.Background, make.metric(pkt.contextId, "filler.spliced", "1"));
@@ -781,17 +925,18 @@ export class VoiceAgentSession {
781
925
  timestampMs: pkt.timestampMs,
782
926
  });
783
927
 
784
- this.bufferTtsText(pkt.contextId, deltaText);
928
+ this.bufferTtsText(pkt.contextId, deltaText, pkt.timestampMs);
785
929
  }
786
930
 
787
931
  private handleLlmDone(pkt: LlmResponseDonePacket): void {
932
+ this.generatingContextIds.delete(pkt.contextId);
788
933
  if (this.interruptedGenerationContextIds.has(pkt.contextId)) {
789
934
  this.ttsTextBuffers.delete(pkt.contextId);
790
935
  this.bus.push(Route.Background, make.metric(pkt.contextId, "llm.done_ignored_after_interrupt", "1"));
791
936
  return;
792
937
  }
793
938
 
794
- const spokenText = this.flushTtsText(pkt.contextId);
939
+ const spokenText = this.flushTtsText(pkt.contextId, pkt.timestampMs);
795
940
  this.emit("agent_finished", {
796
941
  tsMs: pkt.timestampMs,
797
942
  turnId: pkt.contextId,
@@ -809,11 +954,13 @@ export class VoiceAgentSession {
809
954
  this.bus.push(Route.Main, make.ttsDone(pkt.contextId, Date.now(), spokenText));
810
955
  }
811
956
 
812
- private bufferTtsText(contextId: string, text: string): void {
957
+ private bufferTtsText(contextId: string, text: string, tsMs?: number): void {
813
958
  const buffer = this.ttsTextBuffers.get(contextId) ?? { pending: "", emitted: "" };
814
959
  buffer.pending += text;
815
960
  const complete = takeCompleteVoiceText(buffer.pending);
816
961
  if (complete.text) {
962
+ const timing = this.turnTimings.get(contextId);
963
+ if (timing) timing.firstTtsTextMs ??= tsMs ?? Date.now();
817
964
  this.bus.push(Route.Main, make.ttsText(contextId, Date.now(), complete.text));
818
965
  buffer.emitted = appendVoiceText(buffer.emitted, complete.text);
819
966
  }
@@ -821,11 +968,13 @@ export class VoiceAgentSession {
821
968
  this.ttsTextBuffers.set(contextId, buffer);
822
969
  }
823
970
 
824
- private flushTtsText(contextId: string): string {
971
+ private flushTtsText(contextId: string, tsMs?: number): string {
825
972
  const buffer = this.ttsTextBuffers.get(contextId);
826
973
  if (!buffer) return "";
827
974
  const tail = buffer.pending.trim();
828
975
  if (tail) {
976
+ const timing = this.turnTimings.get(contextId);
977
+ if (timing) timing.firstTtsTextMs ??= tsMs ?? Date.now();
829
978
  this.bus.push(Route.Main, make.ttsText(contextId, Date.now(), tail));
830
979
  buffer.emitted = appendVoiceText(buffer.emitted, tail);
831
980
  buffer.pending = "";
@@ -865,9 +1014,101 @@ export class VoiceAgentSession {
865
1014
  },
866
1015
  timestampMs: pkt.timestampMs,
867
1016
  });
1017
+
1018
+ // G3: arm the typed preamble/filler lifecycle for this pending tool call.
1019
+ const pending = this.pendingToolCues.get(pkt.contextId) ?? new Map<string, string>();
1020
+ pending.set(pkt.toolId, pkt.toolName);
1021
+ this.pendingToolCues.set(pkt.contextId, pending);
1022
+ this.emitToolCallCue(pkt.contextId, "started", pkt.toolId, pkt.toolName);
1023
+ if (this.delayCueAfterMs > 0) {
1024
+ const afterMs = this.delayCueAfterMs;
1025
+ this.scheduler.schedule(toolCueTimerKey(pkt.contextId, pkt.toolId), afterMs, () => {
1026
+ if (!this.pendingToolCues.get(pkt.contextId)?.has(pkt.toolId)) return;
1027
+ this.emitToolCallCue(pkt.contextId, "delayed", pkt.toolId, pkt.toolName, afterMs);
1028
+ });
1029
+ }
1030
+ }
1031
+
1032
+ private emitToolCallCue(
1033
+ contextId: string,
1034
+ phase: "started" | "delayed" | "complete" | "failed",
1035
+ toolId: string,
1036
+ toolName: string,
1037
+ afterMs?: number,
1038
+ ): void {
1039
+ this.emit("tool_call_cue", {
1040
+ tsMs: Date.now(),
1041
+ turnId: contextId,
1042
+ phase,
1043
+ toolId,
1044
+ toolName,
1045
+ ...(afterMs !== undefined ? { afterMs } : {}),
1046
+ });
1047
+ }
1048
+
1049
+ /** G3: resolve one pending tool call with a terminal cue phase. */
1050
+ private resolveToolCue(contextId: string, toolId: string, phase: "complete" | "failed"): void {
1051
+ const pending = this.pendingToolCues.get(contextId);
1052
+ const toolName = pending?.get(toolId);
1053
+ if (toolName === undefined) return;
1054
+ pending!.delete(toolId);
1055
+ if (pending!.size === 0) this.pendingToolCues.delete(contextId);
1056
+ this.scheduler.cancel(toolCueTimerKey(contextId, toolId));
1057
+ this.emitToolCallCue(contextId, phase, toolId, toolName);
1058
+ }
1059
+
1060
+ /** G3: fail every pending tool call for a context (error / barge-in / supersede). */
1061
+ private failPendingToolCues(contextId: string): void {
1062
+ const pending = this.pendingToolCues.get(contextId);
1063
+ if (!pending) return;
1064
+ for (const toolId of [...pending.keys()]) this.resolveToolCue(contextId, toolId, "failed");
1065
+ }
1066
+
1067
+ private handleDelegateQuery(pkt: DelegateQueryPacket): void {
1068
+ this.emit("delegate_query", {
1069
+ tsMs: pkt.timestampMs,
1070
+ turnId: pkt.contextId,
1071
+ query: pkt.query,
1072
+ toolId: pkt.toolId,
1073
+ toolName: pkt.toolName,
1074
+ });
1075
+ this.debugPush({
1076
+ component: "delegate",
1077
+ type: "query",
1078
+ data: {
1079
+ context_id: pkt.contextId,
1080
+ query: pkt.query,
1081
+ ...(pkt.toolName ? { tool_name: pkt.toolName } : {}),
1082
+ },
1083
+ timestampMs: pkt.timestampMs,
1084
+ });
1085
+ }
1086
+
1087
+ private handleDelegateResult(pkt: DelegateResultPacket): void {
1088
+ this.emit("delegate_result", {
1089
+ tsMs: pkt.timestampMs,
1090
+ turnId: pkt.contextId,
1091
+ query: pkt.query,
1092
+ answer: pkt.answer,
1093
+ durationMs: pkt.durationMs,
1094
+ grounded: pkt.grounded,
1095
+ toolId: pkt.toolId,
1096
+ toolName: pkt.toolName,
1097
+ });
1098
+ this.debugPush({
1099
+ component: "delegate",
1100
+ type: "result",
1101
+ data: {
1102
+ context_id: pkt.contextId,
1103
+ duration_ms: String(pkt.durationMs),
1104
+ grounded: String(pkt.grounded),
1105
+ },
1106
+ timestampMs: pkt.timestampMs,
1107
+ });
868
1108
  }
869
1109
 
870
1110
  private handleLlmToolResult(pkt: LlmToolResultPacket): void {
1111
+ this.resolveToolCue(pkt.contextId, pkt.toolId, "complete");
871
1112
  this.emit("agent_tool_result", {
872
1113
  tsMs: pkt.timestampMs,
873
1114
  turnId: pkt.contextId,
@@ -907,20 +1148,27 @@ export class VoiceAgentSession {
907
1148
  );
908
1149
  this.turnUserStoppedAtMs.delete(pkt.contextId);
909
1150
  }
1151
+ this.emitTurnLatency(pkt.contextId, pkt.timestampMs);
910
1152
  }
911
1153
 
912
1154
  this.primarySpeakerGate.observeAssistantPlayout(pkt.audio);
913
1155
  // Audio just arrived — (re)arm the stall watchdog for this turn's TTS output.
914
1156
  this.watchdogs.armTtsStallTimer(pkt.contextId);
915
1157
 
916
- // Extend idle timeout by audio duration to prevent timeout during playback.
1158
+ // Mark active and advance this context's playout cursor by the chunk's
1159
+ // realtime duration.
917
1160
  const sampleRateHz = requireTtsAudioSampleRate(pkt.sampleRateHz);
918
1161
  const audioDurationMs = estimatePcm16Duration(pkt.audio, sampleRateHz);
919
- this.idleTimeout.extend(audioDurationMs);
1162
+ const now = Date.now();
1163
+ this.ttsPlayout.noteAudio(pkt.contextId, audioDurationMs, now);
920
1164
 
921
- // Mark active and advance this context's playout cursor by the chunk's
922
- // realtime duration.
923
- this.ttsPlayout.noteAudio(pkt.contextId, audioDurationMs, Date.now());
1165
+ // Anchor the idle timer to when playout actually *ends* (P2), not to chunk
1166
+ // arrival. TTS streams faster than realtime, so extending by each chunk's
1167
+ // duration from arrival lets the timer fire mid-speech on a long answer.
1168
+ // playoutEnd() is cumulative across chunks, so this re-arms to durationMs
1169
+ // after the audio delivered so far finishes playing.
1170
+ const playoutEndMs = this.ttsPlayout.playoutEnd(pkt.contextId);
1171
+ this.idleTimeout.extend(playoutEndMs !== undefined ? Math.max(0, playoutEndMs - now) : audioDurationMs);
924
1172
 
925
1173
  this.debugPush({
926
1174
  component: "tts",
@@ -938,6 +1186,7 @@ export class VoiceAgentSession {
938
1186
  private handleTtsEnd(pkt: TextToSpeechEndPacket): void {
939
1187
  // Generation finished, but the streamed audio is still playing out. Keep the
940
1188
  // context interruptible until its playout estimate elapses, then release it.
1189
+ this.generatingContextIds.delete(pkt.contextId);
941
1190
  this.ttsPlayout.scheduleRelease(pkt.contextId, Date.now());
942
1191
  this.watchdogs.clearTtsStallTimerFor(pkt.contextId);
943
1192
  this.debugPush({
@@ -954,7 +1203,9 @@ export class VoiceAgentSession {
954
1203
 
955
1204
  private handleInterruptDetected(pkt: InterruptionDetectedPacket): void {
956
1205
  this.interruptedGenerationContextIds.add(pkt.contextId);
1206
+ this.failPendingToolCues(pkt.contextId); // G3: the aborted delegate's cue fails (R5)
957
1207
  this.latencyFiller.cancel(pkt.contextId);
1208
+ this.turnTimings.delete(pkt.contextId);
958
1209
  this.firstLlmDeltaReceived.delete(pkt.contextId);
959
1210
  this.ttsTextBuffers.delete(pkt.contextId);
960
1211
  this.ttsPlayout.release(pkt.contextId);
@@ -990,6 +1241,29 @@ export class VoiceAgentSession {
990
1241
  // segments into the next turn when a client reuses the same contextId
991
1242
  // (the provider STT plugins listen for interrupt.stt; previously unfired).
992
1243
  this.bus.push(Route.Critical, make.interruptStt(pkt.contextId, pkt.timestampMs));
1244
+ this.generatingContextIds.delete(pkt.contextId);
1245
+ }
1246
+
1247
+ /**
1248
+ * Cancel a stale prior-turn generation/playout when a new turn supersedes it
1249
+ * (L1). Mirrors the interrupt teardown but without the barge-in metrics — this
1250
+ * is a turn boundary, not a user interruption. Stops leftover TTS audio, aborts
1251
+ * the LLM, and drops late deltas/audio for the stale context.
1252
+ */
1253
+ private cancelStaleGeneration(contextId: string, timestampMs: number): void {
1254
+ this.interruptedGenerationContextIds.add(contextId);
1255
+ this.failPendingToolCues(contextId); // G3: a superseded turn's pending cue fails
1256
+ this.generatingContextIds.delete(contextId);
1257
+ this.latencyFiller.cancel(contextId);
1258
+ this.turnTimings.delete(contextId);
1259
+ this.firstLlmDeltaReceived.delete(contextId);
1260
+ this.ttsTextBuffers.delete(contextId);
1261
+ this.ttsPlayout.release(contextId);
1262
+ this.watchdogs.clearTtsStallTimerFor(contextId);
1263
+ this.bus.push(Route.Critical, make.recordAssistantTruncate(contextId, Date.now()));
1264
+ this.bus.push(Route.Critical, make.interruptTts(contextId, timestampMs));
1265
+ this.bus.push(Route.Critical, make.interruptLlm(contextId, timestampMs));
1266
+ this.bus.push(Route.Background, make.metric(contextId, "supersede.cancelled_stale_generation", "1"));
993
1267
  }
994
1268
 
995
1269
  private handleComponentError(pkt: VoiceErrorPacket): void {
@@ -1011,7 +1285,12 @@ export class VoiceAgentSession {
1011
1285
  timestampMs: pkt.timestampMs,
1012
1286
  });
1013
1287
 
1288
+ // G3: an LLM/bridge error while a tool call is pending means no result is coming.
1289
+ if (pkt.component === "llm" || pkt.component === "bridge") {
1290
+ this.failPendingToolCues(pkt.contextId);
1291
+ }
1014
1292
  this.latencyFiller.clear(pkt.contextId);
1293
+ this.generatingContextIds.delete(pkt.contextId);
1015
1294
  // The packet's own recoverability verdict is authoritative when present: the
1016
1295
  // bus marks handler exceptions (pipeline.error) recoverable by design — one
1017
1296
  // misbehaving handler must degrade the turn, not kill the whole call.
@@ -25,6 +25,19 @@ describe("isCompleteVoiceText", () => {
25
25
  expect(isCompleteVoiceText("مرحبا؟")).toBe(true); // Arabic question mark
26
26
  expect(isCompleteVoiceText("नमस्ते।")).toBe(true); // Devanagari danda
27
27
  });
28
+
29
+ it("does not treat abbreviation or decimal dots as sentence ends", () => {
30
+ // These would otherwise be voiced with a falling intonation and split from
31
+ // their continuation ("Dr." | "Smith", "twelve." | "fifty").
32
+ expect(isCompleteVoiceText("Dr.")).toBe(false);
33
+ expect(isCompleteVoiceText("e.g.")).toBe(false);
34
+ expect(isCompleteVoiceText("The total is $12.")).toBe(false);
35
+ expect(isCompleteVoiceText("Meet at 3 p.m.")).toBe(false);
36
+ expect(isCompleteVoiceText("His name is J.")).toBe(false);
37
+ // But a real sentence end after an abbreviation earlier in the text is fine.
38
+ expect(isCompleteVoiceText("Dr. Smith will see you now.")).toBe(true);
39
+ expect(isCompleteVoiceText("The total is $12.50 today.")).toBe(true);
40
+ });
28
41
  });
29
42
 
30
43
  describe("takeCompleteVoiceText", () => {
package/src/voice-text.ts CHANGED
@@ -33,12 +33,41 @@ export function takeCompleteVoiceText(text: string): { text: string; remaining:
33
33
  return { text: emitted.trimEnd(), remaining };
34
34
  }
35
35
 
36
+ // Common abbreviations whose trailing "." is not a sentence end. Lowercased,
37
+ // dots stripped, so "e.g." matches "eg". Kept small and English-centric — the
38
+ // turn-end flush handles anything that legitimately ends here.
39
+ const ABBREVIATIONS = new Set([
40
+ "mr", "mrs", "ms", "dr", "prof", "sr", "jr", "st", "vs", "etc", "eg", "ie",
41
+ "am", "pm", "no", "vol", "inc", "ltd", "co", "gen", "gov", "sen", "rep",
42
+ "apt", "dept", "approx", "est", "min", "max",
43
+ ]);
44
+
45
+ /**
46
+ * A segment ending in "." is not necessarily a finished sentence: it may be a
47
+ * decimal point ("$12." before "50") or an abbreviation dot ("Dr." before a
48
+ * name, "e.g."). Voicing those as sentence ends produces "twelve." (falling
49
+ * intonation) … "fifty", or splits "Dr." from the name. Defer them — if nothing
50
+ * follows, the turn-end flush still speaks the tail.
51
+ */
52
+ function isFalseTerminalDot(endsWithDot: string): boolean {
53
+ const beforeDot = endsWithDot.slice(0, -1);
54
+ if (/\d$/.test(beforeDot)) return true; // decimal / ordinal: "12." , "$3."
55
+ const word = beforeDot.match(/([A-Za-z][A-Za-z.]*)$/);
56
+ if (!word) return false;
57
+ const normalized = word[1]!.replace(/\./g, "").toLowerCase();
58
+ if (ABBREVIATIONS.has(normalized)) return true;
59
+ if (normalized.length === 1) return true; // single initial: "J."
60
+ return false;
61
+ }
62
+
36
63
  export function isCompleteVoiceText(text: string): boolean {
37
64
  const trimmed = text.trim();
38
65
  for (let index = trimmed.length - 1; index >= 0; index -= 1) {
39
66
  const char = trimmed[index]!;
40
67
  if (isClosingPunctuation(char)) continue;
41
- return isTerminalPunctuation(char);
68
+ if (!isTerminalPunctuation(char)) return false;
69
+ if (char === "." && isFalseTerminalDot(trimmed.slice(0, index + 1))) return false;
70
+ return true;
42
71
  }
43
72
  return false;
44
73
  }
@@ -68,8 +97,19 @@ function isTerminalPunctuation(char: string): boolean {
68
97
  char === "॥";
69
98
  }
70
99
 
100
+ // The ICU sentence segmenter is one of the more expensive Intl allocations;
101
+ // building one per LLM delta (50–200/turn) is avoidable CPU/GC churn on the
102
+ // token→TTS latency path. It is stateless, so cache one per process. `undefined`
103
+ // = not yet computed; `null` = unavailable (fall back to the regex splitter).
104
+ let cachedSegmenter: { segment(text: string): Iterable<SentenceSegment> } | null | undefined;
105
+
106
+ function getSentenceSegmenter(): { segment(text: string): Iterable<SentenceSegment> } | null {
107
+ if (cachedSegmenter === undefined) cachedSegmenter = createSentenceSegmenter();
108
+ return cachedSegmenter;
109
+ }
110
+
71
111
  function segmentSentences(text: string): string[] {
72
- const segmenter = createSentenceSegmenter();
112
+ const segmenter = getSentenceSegmenter();
73
113
  if (segmenter) {
74
114
  return Array.from(segmenter.segment(text), (part) => part.segment);
75
115
  }