@kuralle-syrinx/core 3.1.0 → 4.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/conversation-event.ts +5 -1
- package/src/index.ts +15 -0
- package/src/packets.ts +60 -0
- package/src/pipeline-bus.test.ts +14 -0
- package/src/pipeline-bus.ts +5 -0
- package/src/reasoner-session-store.ts +37 -0
- package/src/tts-playout-clock.ts +14 -0
- package/src/turn-arbiter.ts +1 -1
- package/src/voice-agent-session.test.ts +392 -0
- package/src/voice-agent-session.ts +292 -13
- package/src/voice-text.test.ts +13 -0
- package/src/voice-text.ts +42 -2
package/package.json
CHANGED
|
@@ -49,7 +49,11 @@ export function createConversationEventStream(): [
|
|
|
49
49
|
});
|
|
50
50
|
|
|
51
51
|
const push = (event: ConversationEvent): void => {
|
|
52
|
-
|
|
52
|
+
// Only retain events when a reader is attached. A ReadableStream buffers
|
|
53
|
+
// enqueue() without bound; with no debug consumer (nothing in-tree reads this
|
|
54
|
+
// by default) every packet's event would be retained for the whole call and
|
|
55
|
+
// grow memory unbounded. `locked` is true once getReader() is called.
|
|
56
|
+
if (controller && stream.locked) {
|
|
53
57
|
try {
|
|
54
58
|
controller.enqueue(event);
|
|
55
59
|
} catch {
|
package/src/index.ts
CHANGED
|
@@ -36,6 +36,7 @@ export {
|
|
|
36
36
|
type EndOfSpeechAudioPacket,
|
|
37
37
|
type EndOfSpeechPacket,
|
|
38
38
|
type InterimEndOfSpeechPacket,
|
|
39
|
+
type EndOfSpeechRetractedPacket,
|
|
39
40
|
type UserInputPacket,
|
|
40
41
|
} from "./packets.js";
|
|
41
42
|
|
|
@@ -63,6 +64,20 @@ export {
|
|
|
63
64
|
type ReasoningResumePacket,
|
|
64
65
|
} from "./packets.js";
|
|
65
66
|
|
|
67
|
+
// Pipeline packets — delegate (Responder-Thinker) observability
|
|
68
|
+
export {
|
|
69
|
+
type DelegateQueryPacket,
|
|
70
|
+
type DelegateResultPacket,
|
|
71
|
+
type DelegatePacket,
|
|
72
|
+
type RealtimeResumptionHandlePacket,
|
|
73
|
+
} from "./packets.js";
|
|
74
|
+
|
|
75
|
+
// Durable reasoner session state (G4)
|
|
76
|
+
export {
|
|
77
|
+
InMemoryReasonerSessionStore,
|
|
78
|
+
type ReasonerSessionStore,
|
|
79
|
+
} from "./reasoner-session-store.js";
|
|
80
|
+
|
|
66
81
|
// Pipeline packets — TTS
|
|
67
82
|
export {
|
|
68
83
|
type TextToSpeechTextPacket,
|
package/src/packets.ts
CHANGED
|
@@ -211,6 +211,16 @@ export interface InterimEndOfSpeechPacket extends VoicePacket {
|
|
|
211
211
|
readonly text: string;
|
|
212
212
|
}
|
|
213
213
|
|
|
214
|
+
/**
|
|
215
|
+
* Retraction of a prior `eos.interim` for the same context: the endpoint model
|
|
216
|
+
* signalled a likely turn end, then the user kept speaking (Deepgram Flux
|
|
217
|
+
* `TurnResumed` semantics). Consumers doing speculative work off `eos.interim`
|
|
218
|
+
* must cancel it.
|
|
219
|
+
*/
|
|
220
|
+
export interface EndOfSpeechRetractedPacket extends VoicePacket {
|
|
221
|
+
readonly kind: "eos.retracted";
|
|
222
|
+
}
|
|
223
|
+
|
|
214
224
|
// =============================================================================
|
|
215
225
|
// User Input (processed — feeds LLM)
|
|
216
226
|
// =============================================================================
|
|
@@ -300,12 +310,58 @@ export interface ReasoningSuspendedPacket extends VoicePacket {
|
|
|
300
310
|
readonly payload: unknown;
|
|
301
311
|
}
|
|
302
312
|
|
|
313
|
+
// =============================================================================
|
|
314
|
+
// Delegate (Responder-Thinker) observability packets (Background route)
|
|
315
|
+
// =============================================================================
|
|
316
|
+
|
|
317
|
+
/**
|
|
318
|
+
* The bridge handed a query to the Reasoner. Emitted the moment the delegate
|
|
319
|
+
* turn starts (RealtimeBridge `runDelegate`) or the cascade turn reaches the
|
|
320
|
+
* reasoner (ReasoningBridge), BEFORE the stream runs. Background route:
|
|
321
|
+
* droppable, never blocks the hot path (RFC bimodel-delegate-seam R4).
|
|
322
|
+
* `toolId`/`toolName` are present on realtime delegate turns (the front-model
|
|
323
|
+
* tool call that triggered the delegate); absent on cascade turns.
|
|
324
|
+
*/
|
|
325
|
+
export interface DelegateQueryPacket extends VoicePacket {
|
|
326
|
+
readonly kind: "delegate.query";
|
|
327
|
+
readonly query: string;
|
|
328
|
+
readonly toolId?: string;
|
|
329
|
+
readonly toolName?: string;
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
/**
|
|
333
|
+
* The Reasoner produced the turn's final answer. Self-contained: carries the
|
|
334
|
+
* `query` again so a consumer can log/persist the Q&A pair from this one packet
|
|
335
|
+
* without correlating against the earlier `delegate.query`. `durationMs` spans
|
|
336
|
+
* query → answer; `grounded` is true when the reasoner stream surfaced at least
|
|
337
|
+
* one `tool-result` part (e.g. a retrieval hit) while producing the answer.
|
|
338
|
+
*/
|
|
339
|
+
export interface DelegateResultPacket extends VoicePacket {
|
|
340
|
+
readonly kind: "delegate.result";
|
|
341
|
+
readonly query: string;
|
|
342
|
+
readonly answer: string;
|
|
343
|
+
readonly durationMs: number;
|
|
344
|
+
readonly grounded: boolean;
|
|
345
|
+
readonly toolId?: string;
|
|
346
|
+
readonly toolName?: string;
|
|
347
|
+
}
|
|
348
|
+
|
|
303
349
|
export interface ReasoningResumePacket extends VoicePacket {
|
|
304
350
|
readonly kind: "reasoning.resume";
|
|
305
351
|
readonly runId: string;
|
|
306
352
|
readonly data: unknown;
|
|
307
353
|
}
|
|
308
354
|
|
|
355
|
+
/**
|
|
356
|
+
* G4: a realtime provider with native session resume (Gemini Live) issued a new
|
|
357
|
+
* resumption handle. Background route; a durable host (e.g. cf-agents) persists
|
|
358
|
+
* the latest handle and passes it back on reconnect instead of replaying history.
|
|
359
|
+
*/
|
|
360
|
+
export interface RealtimeResumptionHandlePacket extends VoicePacket {
|
|
361
|
+
readonly kind: "realtime.resumption_handle";
|
|
362
|
+
readonly handle: string;
|
|
363
|
+
}
|
|
364
|
+
|
|
309
365
|
// =============================================================================
|
|
310
366
|
// Output Pipeline Packets (LLM text → TTS audio)
|
|
311
367
|
// =============================================================================
|
|
@@ -510,6 +566,7 @@ export type InputPacket =
|
|
|
510
566
|
| EndOfSpeechAudioPacket
|
|
511
567
|
| EndOfSpeechPacket
|
|
512
568
|
| InterimEndOfSpeechPacket
|
|
569
|
+
| EndOfSpeechRetractedPacket
|
|
513
570
|
| UserInputPacket;
|
|
514
571
|
|
|
515
572
|
/** All interruption packets (Critical route). */
|
|
@@ -551,3 +608,6 @@ export type AnyErrorPacket =
|
|
|
551
608
|
|
|
552
609
|
/** Observability packets (Background route). */
|
|
553
610
|
export type ObservabilityPacket = ConversationMetricPacket | TurnBoundaryEventPacket;
|
|
611
|
+
|
|
612
|
+
/** Delegate (Responder-Thinker) lifecycle packets (Background route). */
|
|
613
|
+
export type DelegatePacket = DelegateQueryPacket | DelegateResultPacket;
|
package/src/pipeline-bus.test.ts
CHANGED
|
@@ -171,6 +171,20 @@ describe("PipelineBusImpl", () => {
|
|
|
171
171
|
packet: { kind: "critical.event", contextId: "critical-1" },
|
|
172
172
|
});
|
|
173
173
|
});
|
|
174
|
+
|
|
175
|
+
it("does not retain packets when no reader is attached (drop-on-unread)", async () => {
|
|
176
|
+
const bus = createBus();
|
|
177
|
+
// No getReader() → stream unlocked → nothing retained (else a call with no
|
|
178
|
+
// recorder would buffer every audio packet and OOM).
|
|
179
|
+
for (let i = 0; i < 1000; i++) bus.push(Route.Main, pkt("main.event", `n-${i}`));
|
|
180
|
+
|
|
181
|
+
// A reader that attaches later sees only packets pushed AFTER it attaches.
|
|
182
|
+
const reader = bus.allPackets.getReader();
|
|
183
|
+
bus.push(Route.Main, pkt("main.event", "after-reader"));
|
|
184
|
+
const next = await reader.read();
|
|
185
|
+
reader.releaseLock();
|
|
186
|
+
expect(next.value).toMatchObject({ packet: { contextId: "after-reader" } });
|
|
187
|
+
});
|
|
174
188
|
});
|
|
175
189
|
|
|
176
190
|
describe("error handling", () => {
|
package/src/pipeline-bus.ts
CHANGED
|
@@ -239,6 +239,11 @@ export class PipelineBusImpl implements PipelineBus {
|
|
|
239
239
|
private publishAllPackets(route: Route, packet: VoicePacket): void {
|
|
240
240
|
this.onPacket?.(route, packet);
|
|
241
241
|
if (!this.allPacketsController) return;
|
|
242
|
+
// Only retain packets when a reader is actually attached. A ReadableStream
|
|
243
|
+
// buffers enqueue() without bound until read, so with no consumer (the default
|
|
244
|
+
// deployment — recorder is optional) this would retain every audio buffer of
|
|
245
|
+
// the whole call and OOM. `locked` is true once getReader() is called.
|
|
246
|
+
if (!this.allPackets.locked) return;
|
|
242
247
|
try {
|
|
243
248
|
this.allPacketsController.enqueue({ route, packet });
|
|
244
249
|
} catch {
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
// SPDX-License-Identifier: MIT
|
|
2
|
+
//
|
|
3
|
+
// Syrinx Kernel v2 — durable reasoner session state (G4, RFC bimodel-delegate-seam)
|
|
4
|
+
//
|
|
5
|
+
// The bridge owns conversation history (reasoner.ts §4.5); this seam makes that
|
|
6
|
+
// history durable so a reasoner resumes with the same context after a host
|
|
7
|
+
// eviction/reconnect instead of restarting amnesiac. Concrete backends: the
|
|
8
|
+
// in-memory impl below (Node/tests), a DO-SQLite impl in @kuralle-syrinx/cf-agents.
|
|
9
|
+
|
|
10
|
+
import type { ReasonerMessage } from "./reasoner.js";
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Durable per-session reasoner conversation history. `save` persists the full
|
|
14
|
+
* (already bounded) history snapshot — snapshot semantics, not append, because
|
|
15
|
+
* barge-in truncation rewrites earlier messages to the heard prefix (G25).
|
|
16
|
+
*/
|
|
17
|
+
export interface ReasonerSessionStore {
|
|
18
|
+
load(sessionId: string): Promise<readonly ReasonerMessage[]> | readonly ReasonerMessage[];
|
|
19
|
+
save(sessionId: string, messages: readonly ReasonerMessage[]): Promise<void> | void;
|
|
20
|
+
clear(sessionId: string): Promise<void> | void;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export class InMemoryReasonerSessionStore implements ReasonerSessionStore {
|
|
24
|
+
private readonly sessions = new Map<string, readonly ReasonerMessage[]>();
|
|
25
|
+
|
|
26
|
+
load(sessionId: string): readonly ReasonerMessage[] {
|
|
27
|
+
return this.sessions.get(sessionId) ?? [];
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
save(sessionId: string, messages: readonly ReasonerMessage[]): void {
|
|
31
|
+
this.sessions.set(sessionId, [...messages]);
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
clear(sessionId: string): void {
|
|
35
|
+
this.sessions.delete(sessionId);
|
|
36
|
+
}
|
|
37
|
+
}
|
package/src/tts-playout-clock.ts
CHANGED
|
@@ -89,6 +89,20 @@ export class TtsPlayoutClock {
|
|
|
89
89
|
return this.active.has(contextId);
|
|
90
90
|
}
|
|
91
91
|
|
|
92
|
+
/** All still-active contexts, insertion order (oldest first). */
|
|
93
|
+
activeContexts(): string[] {
|
|
94
|
+
return [...this.active];
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* Wall-clock estimate of when this context's audio finishes playing out, or
|
|
99
|
+
* undefined if unknown. Anchors the idle timer to real playout end rather than
|
|
100
|
+
* chunk-arrival (TTS streams faster than realtime).
|
|
101
|
+
*/
|
|
102
|
+
playoutEnd(contextId: string): number | undefined {
|
|
103
|
+
return this.playoutEndMs.get(contextId);
|
|
104
|
+
}
|
|
105
|
+
|
|
92
106
|
/** The most-recently-added still-active context (insertion order), or "" if none. */
|
|
93
107
|
latestActive(): string {
|
|
94
108
|
let latest = "";
|
package/src/turn-arbiter.ts
CHANGED
|
@@ -321,6 +321,316 @@ describe("VoiceAgentSession", () => {
|
|
|
321
321
|
await closeSession(session);
|
|
322
322
|
});
|
|
323
323
|
|
|
324
|
+
it("G2/WBS-1: surfaces delegate.query/delegate.result packets as delegate_query/delegate_result session events", async () => {
|
|
325
|
+
const session = new VoiceAgentSession({ plugins: {} });
|
|
326
|
+
const queries: Array<{ turnId: string; query: string; toolName?: string }> = [];
|
|
327
|
+
const results: Array<{ turnId: string; query: string; answer: string; durationMs: number; grounded: boolean }> = [];
|
|
328
|
+
session.on("delegate_query", (event) => queries.push(event));
|
|
329
|
+
session.on("delegate_result", (event) => results.push(event));
|
|
330
|
+
await session.start();
|
|
331
|
+
|
|
332
|
+
session.bus.push(Route.Background, {
|
|
333
|
+
kind: "delegate.query",
|
|
334
|
+
contextId: "turn-1",
|
|
335
|
+
timestampMs: Date.now(),
|
|
336
|
+
query: "When is the deadline?",
|
|
337
|
+
toolId: "call_1",
|
|
338
|
+
toolName: "consult_knowledge",
|
|
339
|
+
});
|
|
340
|
+
session.bus.push(Route.Background, {
|
|
341
|
+
kind: "delegate.result",
|
|
342
|
+
contextId: "turn-1",
|
|
343
|
+
timestampMs: Date.now(),
|
|
344
|
+
query: "When is the deadline?",
|
|
345
|
+
answer: "March 31.",
|
|
346
|
+
durationMs: 42,
|
|
347
|
+
grounded: true,
|
|
348
|
+
toolId: "call_1",
|
|
349
|
+
toolName: "consult_knowledge",
|
|
350
|
+
});
|
|
351
|
+
|
|
352
|
+
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
353
|
+
|
|
354
|
+
expect(queries).toEqual([
|
|
355
|
+
{ tsMs: expect.any(Number), turnId: "turn-1", query: "When is the deadline?", toolId: "call_1", toolName: "consult_knowledge" },
|
|
356
|
+
]);
|
|
357
|
+
expect(results).toEqual([
|
|
358
|
+
{
|
|
359
|
+
tsMs: expect.any(Number),
|
|
360
|
+
turnId: "turn-1",
|
|
361
|
+
query: "When is the deadline?",
|
|
362
|
+
answer: "March 31.",
|
|
363
|
+
durationMs: 42,
|
|
364
|
+
grounded: true,
|
|
365
|
+
toolId: "call_1",
|
|
366
|
+
toolName: "consult_knowledge",
|
|
367
|
+
},
|
|
368
|
+
]);
|
|
369
|
+
|
|
370
|
+
await closeSession(session);
|
|
371
|
+
});
|
|
372
|
+
|
|
373
|
+
it("emits a decomposed turn_latency session event on first TTS audio", async () => {
|
|
374
|
+
const session = new VoiceAgentSession({ plugins: {} });
|
|
375
|
+
const events: Array<Record<string, unknown>> = [];
|
|
376
|
+
session.on("turn_latency", (event) => {
|
|
377
|
+
events.push(event);
|
|
378
|
+
});
|
|
379
|
+
await session.start();
|
|
380
|
+
|
|
381
|
+
const t0 = 100_000;
|
|
382
|
+
session.bus.push(Route.Main, { kind: "vad.speech_ended", contextId: "turn-1", timestampMs: t0 });
|
|
383
|
+
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
384
|
+
session.bus.push(Route.Main, {
|
|
385
|
+
kind: "eos.turn_complete",
|
|
386
|
+
contextId: "turn-1",
|
|
387
|
+
timestampMs: t0 + 200,
|
|
388
|
+
text: "what are the lab fees",
|
|
389
|
+
transcripts: [],
|
|
390
|
+
});
|
|
391
|
+
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
392
|
+
session.bus.push(Route.Main, { kind: "llm.delta", contextId: "turn-1", timestampMs: t0 + 900, text: "The fee is ten dollars." });
|
|
393
|
+
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
394
|
+
session.bus.push(Route.Main, {
|
|
395
|
+
kind: "tts.audio",
|
|
396
|
+
contextId: "turn-1",
|
|
397
|
+
timestampMs: t0 + 1150,
|
|
398
|
+
audio: new Uint8Array(320),
|
|
399
|
+
sampleRateHz: 16000,
|
|
400
|
+
});
|
|
401
|
+
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
402
|
+
|
|
403
|
+
expect(events).toHaveLength(1);
|
|
404
|
+
expect(events[0]).toEqual({
|
|
405
|
+
tsMs: expect.any(Number),
|
|
406
|
+
turnId: "turn-1",
|
|
407
|
+
ttfaMs: 1150,
|
|
408
|
+
eouDelayMs: 200,
|
|
409
|
+
llmTtftMs: 700,
|
|
410
|
+
ttsTtfbMs: 250,
|
|
411
|
+
fillerUsed: false,
|
|
412
|
+
});
|
|
413
|
+
|
|
414
|
+
await closeSession(session);
|
|
415
|
+
});
|
|
416
|
+
|
|
417
|
+
it("turn_latency anchors TTFA to eos when no VAD speech-end was seen, and marks filler turns", async () => {
|
|
418
|
+
const session = new VoiceAgentSession({ plugins: {}, latencyFillerEnabled: true });
|
|
419
|
+
const events: Array<Record<string, unknown>> = [];
|
|
420
|
+
session.on("turn_latency", (event) => {
|
|
421
|
+
events.push(event);
|
|
422
|
+
});
|
|
423
|
+
await session.start();
|
|
424
|
+
|
|
425
|
+
const t0 = 200_000;
|
|
426
|
+
session.bus.push(Route.Main, {
|
|
427
|
+
kind: "eos.turn_complete",
|
|
428
|
+
contextId: "turn-2",
|
|
429
|
+
timestampMs: t0,
|
|
430
|
+
text: "tell me about registration holds",
|
|
431
|
+
transcripts: [],
|
|
432
|
+
});
|
|
433
|
+
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
434
|
+
session.bus.push(Route.Main, {
|
|
435
|
+
kind: "tts.audio",
|
|
436
|
+
contextId: "turn-2",
|
|
437
|
+
timestampMs: t0 + 400,
|
|
438
|
+
audio: new Uint8Array(320),
|
|
439
|
+
sampleRateHz: 16000,
|
|
440
|
+
});
|
|
441
|
+
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
442
|
+
|
|
443
|
+
expect(events).toHaveLength(1);
|
|
444
|
+
expect(events[0]).toMatchObject({
|
|
445
|
+
turnId: "turn-2",
|
|
446
|
+
ttfaMs: 400,
|
|
447
|
+
fillerUsed: true,
|
|
448
|
+
});
|
|
449
|
+
expect(events[0]!["eouDelayMs"]).toBeUndefined();
|
|
450
|
+
|
|
451
|
+
await closeSession(session);
|
|
452
|
+
});
|
|
453
|
+
|
|
454
|
+
it("turn_latency is not emitted for an interrupted turn", async () => {
|
|
455
|
+
const session = new VoiceAgentSession({ plugins: {} });
|
|
456
|
+
const events: unknown[] = [];
|
|
457
|
+
session.on("turn_latency", (event) => {
|
|
458
|
+
events.push(event);
|
|
459
|
+
});
|
|
460
|
+
await session.start();
|
|
461
|
+
|
|
462
|
+
const t0 = 300_000;
|
|
463
|
+
session.bus.push(Route.Main, {
|
|
464
|
+
kind: "eos.turn_complete",
|
|
465
|
+
contextId: "turn-3",
|
|
466
|
+
timestampMs: t0,
|
|
467
|
+
text: "hello there",
|
|
468
|
+
transcripts: [],
|
|
469
|
+
});
|
|
470
|
+
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
471
|
+
session.bus.push(Route.Critical, {
|
|
472
|
+
kind: "interrupt.detected",
|
|
473
|
+
contextId: "turn-3",
|
|
474
|
+
timestampMs: t0 + 100,
|
|
475
|
+
});
|
|
476
|
+
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
477
|
+
session.bus.push(Route.Main, {
|
|
478
|
+
kind: "tts.audio",
|
|
479
|
+
contextId: "turn-3",
|
|
480
|
+
timestampMs: t0 + 300,
|
|
481
|
+
audio: new Uint8Array(320),
|
|
482
|
+
sampleRateHz: 16000,
|
|
483
|
+
});
|
|
484
|
+
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
485
|
+
|
|
486
|
+
expect(events).toHaveLength(0);
|
|
487
|
+
|
|
488
|
+
await closeSession(session);
|
|
489
|
+
});
|
|
490
|
+
|
|
491
|
+
it("G3/WBS-3: tool-call lifecycle cues fire started → delayed → complete", async () => {
|
|
492
|
+
const session = new VoiceAgentSession({ plugins: {}, delayCueAfterMs: 30 });
|
|
493
|
+
const cues: Array<{ phase: string; turnId: string; toolId: string; toolName: string; afterMs?: number }> = [];
|
|
494
|
+
session.on("tool_call_cue", (event) => cues.push(event));
|
|
495
|
+
await session.start();
|
|
496
|
+
|
|
497
|
+
session.bus.push(Route.Main, {
|
|
498
|
+
kind: "llm.tool_call",
|
|
499
|
+
contextId: "turn-1",
|
|
500
|
+
timestampMs: Date.now(),
|
|
501
|
+
toolId: "t1",
|
|
502
|
+
toolName: "consult_knowledge",
|
|
503
|
+
toolArgs: { query: "fees" },
|
|
504
|
+
});
|
|
505
|
+
|
|
506
|
+
await new Promise((resolve) => setTimeout(resolve, 70));
|
|
507
|
+
|
|
508
|
+
session.bus.push(Route.Main, {
|
|
509
|
+
kind: "llm.tool_result",
|
|
510
|
+
contextId: "turn-1",
|
|
511
|
+
timestampMs: Date.now(),
|
|
512
|
+
toolId: "t1",
|
|
513
|
+
toolName: "consult_knowledge",
|
|
514
|
+
result: "answer",
|
|
515
|
+
});
|
|
516
|
+
|
|
517
|
+
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
518
|
+
|
|
519
|
+
expect(cues.map((cue) => cue.phase)).toEqual(["started", "delayed", "complete"]);
|
|
520
|
+
expect(cues[0]).toMatchObject({ turnId: "turn-1", toolId: "t1", toolName: "consult_knowledge" });
|
|
521
|
+
expect(cues[1]!.afterMs).toBe(30);
|
|
522
|
+
|
|
523
|
+
await closeSession(session);
|
|
524
|
+
});
|
|
525
|
+
|
|
526
|
+
it("G3/WBS-3: no delayed cue when the result beats the timer; timer 0 disables it", async () => {
|
|
527
|
+
const session = new VoiceAgentSession({ plugins: {}, delayCueAfterMs: 5000 });
|
|
528
|
+
const cues: Array<{ phase: string }> = [];
|
|
529
|
+
session.on("tool_call_cue", (event) => cues.push(event));
|
|
530
|
+
await session.start();
|
|
531
|
+
|
|
532
|
+
session.bus.push(Route.Main, {
|
|
533
|
+
kind: "llm.tool_call",
|
|
534
|
+
contextId: "turn-1",
|
|
535
|
+
timestampMs: Date.now(),
|
|
536
|
+
toolId: "t1",
|
|
537
|
+
toolName: "consult_knowledge",
|
|
538
|
+
toolArgs: {},
|
|
539
|
+
});
|
|
540
|
+
session.bus.push(Route.Main, {
|
|
541
|
+
kind: "llm.tool_result",
|
|
542
|
+
contextId: "turn-1",
|
|
543
|
+
timestampMs: Date.now(),
|
|
544
|
+
toolId: "t1",
|
|
545
|
+
toolName: "consult_knowledge",
|
|
546
|
+
result: "fast answer",
|
|
547
|
+
});
|
|
548
|
+
|
|
549
|
+
await new Promise((resolve) => setTimeout(resolve, 30));
|
|
550
|
+
expect(cues.map((cue) => cue.phase)).toEqual(["started", "complete"]);
|
|
551
|
+
|
|
552
|
+
await closeSession(session);
|
|
553
|
+
|
|
554
|
+
const disabled = new VoiceAgentSession({ plugins: {}, delayCueAfterMs: 0 });
|
|
555
|
+
const disabledCues: Array<{ phase: string }> = [];
|
|
556
|
+
disabled.on("tool_call_cue", (event) => disabledCues.push(event));
|
|
557
|
+
await disabled.start();
|
|
558
|
+
disabled.bus.push(Route.Main, {
|
|
559
|
+
kind: "llm.tool_call",
|
|
560
|
+
contextId: "turn-2",
|
|
561
|
+
timestampMs: Date.now(),
|
|
562
|
+
toolId: "t2",
|
|
563
|
+
toolName: "consult_knowledge",
|
|
564
|
+
toolArgs: {},
|
|
565
|
+
});
|
|
566
|
+
await new Promise((resolve) => setTimeout(resolve, 30));
|
|
567
|
+
expect(disabledCues.map((cue) => cue.phase)).toEqual(["started"]);
|
|
568
|
+
await closeSession(disabled);
|
|
569
|
+
});
|
|
570
|
+
|
|
571
|
+
it("G3/WBS-3: llm.error while a tool call is pending fires the failed cue", async () => {
|
|
572
|
+
const session = new VoiceAgentSession({ plugins: {}, delayCueAfterMs: 0 });
|
|
573
|
+
const cues: Array<{ phase: string; toolId: string }> = [];
|
|
574
|
+
session.on("tool_call_cue", (event) => cues.push(event));
|
|
575
|
+
await session.start();
|
|
576
|
+
|
|
577
|
+
session.bus.push(Route.Main, {
|
|
578
|
+
kind: "llm.tool_call",
|
|
579
|
+
contextId: "turn-1",
|
|
580
|
+
timestampMs: Date.now(),
|
|
581
|
+
toolId: "t1",
|
|
582
|
+
toolName: "consult_knowledge",
|
|
583
|
+
toolArgs: {},
|
|
584
|
+
});
|
|
585
|
+
await new Promise((resolve) => setTimeout(resolve, 10)); // let Main drain before the Critical error
|
|
586
|
+
session.bus.push(Route.Critical, {
|
|
587
|
+
kind: "llm.error",
|
|
588
|
+
contextId: "turn-1",
|
|
589
|
+
timestampMs: Date.now(),
|
|
590
|
+
component: "bridge",
|
|
591
|
+
category: ErrorCategory.NetworkTimeout,
|
|
592
|
+
cause: new Error("delegate failed"),
|
|
593
|
+
isRecoverable: true,
|
|
594
|
+
});
|
|
595
|
+
|
|
596
|
+
await new Promise((resolve) => setTimeout(resolve, 30));
|
|
597
|
+
expect(cues.map((cue) => cue.phase)).toEqual(["started", "failed"]);
|
|
598
|
+
expect(cues[1]!.toolId).toBe("t1");
|
|
599
|
+
|
|
600
|
+
await closeSession(session);
|
|
601
|
+
});
|
|
602
|
+
|
|
603
|
+
it("G3/WBS-3 (R5): barge-in fails the pending cue and the interrupt path is unaffected", async () => {
|
|
604
|
+
const session = new VoiceAgentSession({ plugins: {}, delayCueAfterMs: 5000, minInterruptionMs: 0 });
|
|
605
|
+
const cues: Array<{ phase: string }> = [];
|
|
606
|
+
const interrupts: string[] = [];
|
|
607
|
+
session.on("tool_call_cue", (event) => cues.push(event));
|
|
608
|
+
session.bus.on("interrupt.tts", (pkt) => { interrupts.push((pkt as { contextId: string }).contextId); });
|
|
609
|
+
await session.start();
|
|
610
|
+
|
|
611
|
+
session.bus.push(Route.Main, {
|
|
612
|
+
kind: "llm.tool_call",
|
|
613
|
+
contextId: "turn-1",
|
|
614
|
+
timestampMs: Date.now(),
|
|
615
|
+
toolId: "t1",
|
|
616
|
+
toolName: "consult_knowledge",
|
|
617
|
+
toolArgs: {},
|
|
618
|
+
});
|
|
619
|
+
await new Promise((resolve) => setTimeout(resolve, 10)); // let Main drain before the Critical interrupt
|
|
620
|
+
session.bus.push(Route.Critical, {
|
|
621
|
+
kind: "interrupt.detected",
|
|
622
|
+
contextId: "turn-1",
|
|
623
|
+
timestampMs: Date.now(),
|
|
624
|
+
source: "client",
|
|
625
|
+
});
|
|
626
|
+
|
|
627
|
+
await new Promise((resolve) => setTimeout(resolve, 30));
|
|
628
|
+
expect(cues.map((cue) => cue.phase)).toEqual(["started", "failed"]);
|
|
629
|
+
expect(interrupts).toContain("turn-1"); // barge-in cancel still flowed
|
|
630
|
+
|
|
631
|
+
await closeSession(session);
|
|
632
|
+
});
|
|
633
|
+
|
|
324
634
|
it("emits normalized debug events for bus packets", async () => {
|
|
325
635
|
const session = new VoiceAgentSession({ plugins: {} });
|
|
326
636
|
const reader = session.debugEvents.getReader();
|
|
@@ -2485,3 +2795,85 @@ describe("VoiceAgentSession — handler errors must not kill the call", () => {
|
|
|
2485
2795
|
await closeSession(session);
|
|
2486
2796
|
});
|
|
2487
2797
|
});
|
|
2798
|
+
|
|
2799
|
+
describe("VoiceAgentSession supersede + thinking-phase barge-in", () => {
|
|
2800
|
+
it("cancels a still-playing prior turn's TTS when a new turn completes (L1)", async () => {
|
|
2801
|
+
const session = new VoiceAgentSession({ plugins: {} });
|
|
2802
|
+
const interruptTts: InterruptTtsPacket[] = [];
|
|
2803
|
+
await session.start();
|
|
2804
|
+
session.bus.on("interrupt.tts", (pkt) => { interruptTts.push(pkt as InterruptTtsPacket); });
|
|
2805
|
+
|
|
2806
|
+
// Turn 1 is generating + has streamed audio (still playing out).
|
|
2807
|
+
session.bus.push(Route.Main, {
|
|
2808
|
+
kind: "eos.turn_complete", contextId: "turn-1", timestampMs: Date.now(), text: "one", transcripts: [],
|
|
2809
|
+
} satisfies EndOfSpeechPacket);
|
|
2810
|
+
await new Promise((r) => setTimeout(r, 5));
|
|
2811
|
+
session.bus.push(Route.Main, {
|
|
2812
|
+
kind: "tts.audio", contextId: "turn-1", timestampMs: Date.now(),
|
|
2813
|
+
audio: new Uint8Array(16000), sampleRateHz: 16000, // ~0.5s of audio still playing
|
|
2814
|
+
} satisfies TextToSpeechAudioPacket);
|
|
2815
|
+
await new Promise((r) => setTimeout(r, 5));
|
|
2816
|
+
|
|
2817
|
+
// Turn 2 completes while turn 1 is still playing → turn 1 must be cancelled.
|
|
2818
|
+
session.bus.push(Route.Main, {
|
|
2819
|
+
kind: "eos.turn_complete", contextId: "turn-2", timestampMs: Date.now(), text: "two", transcripts: [],
|
|
2820
|
+
} satisfies EndOfSpeechPacket);
|
|
2821
|
+
await new Promise((r) => setTimeout(r, 10));
|
|
2822
|
+
|
|
2823
|
+
expect(interruptTts.some((p) => p.contextId === "turn-1")).toBe(true);
|
|
2824
|
+
expect(interruptTts.some((p) => p.contextId === "turn-2")).toBe(false);
|
|
2825
|
+
|
|
2826
|
+
await closeSession(session);
|
|
2827
|
+
});
|
|
2828
|
+
|
|
2829
|
+
it("honors a client interrupt during the reasoner TTFT gap, before any audio (B3)", async () => {
|
|
2830
|
+
const session = new VoiceAgentSession({ plugins: {} });
|
|
2831
|
+
const interruptLlm: InterruptLlmPacket[] = [];
|
|
2832
|
+
await session.start();
|
|
2833
|
+
session.bus.on("interrupt.llm", (pkt) => { interruptLlm.push(pkt as InterruptLlmPacket); });
|
|
2834
|
+
|
|
2835
|
+
// Turn completes; generation is in-flight but NO tts.audio has played yet.
|
|
2836
|
+
session.bus.push(Route.Main, {
|
|
2837
|
+
kind: "eos.turn_complete", contextId: "turn-think", timestampMs: Date.now(), text: "q", transcripts: [],
|
|
2838
|
+
} satisfies EndOfSpeechPacket);
|
|
2839
|
+
await new Promise((r) => setTimeout(r, 5));
|
|
2840
|
+
|
|
2841
|
+
session.requestClientInterrupt("turn-think");
|
|
2842
|
+
await new Promise((r) => setTimeout(r, 10));
|
|
2843
|
+
|
|
2844
|
+
expect(interruptLlm.some((p) => p.contextId === "turn-think")).toBe(true);
|
|
2845
|
+
|
|
2846
|
+
await closeSession(session);
|
|
2847
|
+
});
|
|
2848
|
+
|
|
2849
|
+
it("drops a backchannel during the assistant's turn: no cancel, no second response (B4)", async () => {
|
|
2850
|
+
const session = new VoiceAgentSession({ plugins: {} });
|
|
2851
|
+
const interruptTts: InterruptTtsPacket[] = [];
|
|
2852
|
+
const userInputs: UserInputPacket[] = [];
|
|
2853
|
+
await session.start();
|
|
2854
|
+
session.bus.on("interrupt.tts", (pkt) => { interruptTts.push(pkt as InterruptTtsPacket); });
|
|
2855
|
+
session.bus.on("user.input", (pkt) => { userInputs.push(pkt as UserInputPacket); });
|
|
2856
|
+
|
|
2857
|
+
// Assistant is speaking turn-1 (audio actively playing out).
|
|
2858
|
+
session.bus.push(Route.Main, {
|
|
2859
|
+
kind: "eos.turn_complete", contextId: "turn-1", timestampMs: Date.now(), text: "here is the answer", transcripts: [],
|
|
2860
|
+
} satisfies EndOfSpeechPacket);
|
|
2861
|
+
await new Promise((r) => setTimeout(r, 5));
|
|
2862
|
+
session.bus.push(Route.Main, {
|
|
2863
|
+
kind: "tts.audio", contextId: "turn-1", timestampMs: Date.now(), audio: new Uint8Array(16000), sampleRateHz: 16000,
|
|
2864
|
+
} satisfies TextToSpeechAudioPacket);
|
|
2865
|
+
await new Promise((r) => setTimeout(r, 5));
|
|
2866
|
+
|
|
2867
|
+
// User says "uh-huh" (a rotated context) mid-answer — a backchannel, not a turn.
|
|
2868
|
+
session.bus.push(Route.Main, {
|
|
2869
|
+
kind: "eos.turn_complete", contextId: "turn-2", timestampMs: Date.now(), text: "uh-huh", transcripts: [],
|
|
2870
|
+
} satisfies EndOfSpeechPacket);
|
|
2871
|
+
await new Promise((r) => setTimeout(r, 10));
|
|
2872
|
+
|
|
2873
|
+
// The assistant's turn is NOT cancelled and the backchannel does NOT drive the LLM.
|
|
2874
|
+
expect(interruptTts.some((p) => p.contextId === "turn-1")).toBe(false);
|
|
2875
|
+
expect(userInputs.some((p) => p.contextId === "turn-2")).toBe(false);
|
|
2876
|
+
|
|
2877
|
+
await closeSession(session);
|
|
2878
|
+
});
|
|
2879
|
+
});
|
|
@@ -29,6 +29,8 @@ import type {
|
|
|
29
29
|
UserAudioReceivedPacket,
|
|
30
30
|
UserTextReceivedPacket,
|
|
31
31
|
InterruptionDetectedPacket,
|
|
32
|
+
DelegateQueryPacket,
|
|
33
|
+
DelegateResultPacket,
|
|
32
34
|
LlmDeltaPacket,
|
|
33
35
|
LlmResponseDonePacket,
|
|
34
36
|
LlmToolCallPacket,
|
|
@@ -61,7 +63,7 @@ import { LatencyFillerController } from "./latency-filler.js";
|
|
|
61
63
|
import { PrimarySpeakerGate } from "./primary-speaker-gate.js";
|
|
62
64
|
import { takeCompleteVoiceText, isCompleteVoiceText, appendVoiceText } from "./voice-text.js";
|
|
63
65
|
import { TtsPlayoutClock } from "./tts-playout-clock.js";
|
|
64
|
-
import { TurnArbiter } from "./turn-arbiter.js";
|
|
66
|
+
import { TurnArbiter, isBackchannel } from "./turn-arbiter.js";
|
|
65
67
|
import * as make from "./packet-factories.js";
|
|
66
68
|
import { pluginStage, stageOrder, isAudioStage } from "./init-stage-order.js";
|
|
67
69
|
import {
|
|
@@ -147,6 +149,14 @@ export interface VoiceAgentSessionConfig {
|
|
|
147
149
|
* (TTS/STT-failure fallback needs canned audio / a clarification prompt — out of scope.)
|
|
148
150
|
*/
|
|
149
151
|
errorFallbackText?: string;
|
|
152
|
+
/**
|
|
153
|
+
* G3 (RFC bimodel-delegate-seam): ms a tool call may stay pending before the
|
|
154
|
+
* `tool_call_cue` session event fires its time-triggered `"delayed"` phase — the
|
|
155
|
+
* "still working" cue clients render during a long reasoner wait (cf. Vapi's
|
|
156
|
+
* `request-response-delayed` + `timingMilliseconds`). 0 disables the delayed
|
|
157
|
+
* phase; started/complete/failed always fire. Default: 2000.
|
|
158
|
+
*/
|
|
159
|
+
delayCueAfterMs?: number;
|
|
150
160
|
/**
|
|
151
161
|
* Which component owns turn boundary (EOS) for this session. Defaults to
|
|
152
162
|
* provider STT ownership; Smart Turn sessions must opt in explicitly.
|
|
@@ -170,6 +180,33 @@ export interface VoiceAgentSessionEvents {
|
|
|
170
180
|
agent_text_delta: (event: { tsMs: number; turnId: string; delta: string }) => void;
|
|
171
181
|
agent_tool_call: (event: { tsMs: number; turnId: string; id: string; name: string; args: Record<string, unknown> }) => void;
|
|
172
182
|
agent_tool_result: (event: { tsMs: number; turnId: string; id: string; result: string; durationMs: number }) => void;
|
|
183
|
+
delegate_query: (event: { tsMs: number; turnId: string; query: string; toolId?: string; toolName?: string }) => void;
|
|
184
|
+
delegate_result: (event: { tsMs: number; turnId: string; query: string; answer: string; durationMs: number; grounded: boolean; toolId?: string; toolName?: string }) => void;
|
|
185
|
+
/**
|
|
186
|
+
* G3: typed preamble/filler lifecycle for a pending tool call (Vapi-shaped:
|
|
187
|
+
* started / delayed / complete / failed). `delayed` is time-triggered by
|
|
188
|
+
* `delayCueAfterMs` while the call is still pending; `failed` fires on an LLM/bridge
|
|
189
|
+
* error, a barge-in, or a superseding turn while pending (R5). Transports surface
|
|
190
|
+
* these as `tool_call_*` wire messages — the standard "thinking" cue.
|
|
191
|
+
*/
|
|
192
|
+
tool_call_cue: (event: { tsMs: number; turnId: string; phase: "started" | "delayed" | "complete" | "failed"; toolId: string; toolName: string; afterMs?: number }) => void;
|
|
193
|
+
/**
|
|
194
|
+
* Per-turn latency decomposition, emitted once at the turn's first TTS audio.
|
|
195
|
+
* `ttfaMs` is anchored to the real end of user speech (VAD speech-end, falling
|
|
196
|
+
* back to the endpoint decision) — and `fillerUsed` flags turns where a latency
|
|
197
|
+
* filler spoke first, so TTFA on those turns is not mistaken for reasoning speed.
|
|
198
|
+
* Decomposition: eouDelayMs (speech end → endpoint) + llmTtftMs (endpoint →
|
|
199
|
+
* first LLM delta) + ttsTtfbMs (first TTS text dispatched → first audio).
|
|
200
|
+
*/
|
|
201
|
+
turn_latency: (event: {
|
|
202
|
+
tsMs: number;
|
|
203
|
+
turnId: string;
|
|
204
|
+
ttfaMs: number;
|
|
205
|
+
eouDelayMs?: number;
|
|
206
|
+
llmTtftMs?: number;
|
|
207
|
+
ttsTtfbMs?: number;
|
|
208
|
+
fillerUsed: boolean;
|
|
209
|
+
}) => void;
|
|
173
210
|
agent_first_audio: (event: { tsMs: number; turnId: string }) => void;
|
|
174
211
|
agent_finished: (event: { tsMs: number; turnId: string } & Record<string, unknown>) => void;
|
|
175
212
|
error: (event: { tsMs: number; stage: string; category: string; message: string }) => void;
|
|
@@ -184,9 +221,22 @@ interface TtsTextBuffer {
|
|
|
184
221
|
emitted: string;
|
|
185
222
|
}
|
|
186
223
|
|
|
224
|
+
interface TurnTiming {
|
|
225
|
+
speechEndedMs?: number;
|
|
226
|
+
eosMs?: number;
|
|
227
|
+
firstLlmDeltaMs?: number;
|
|
228
|
+
firstTtsTextMs?: number;
|
|
229
|
+
fillerUsed?: boolean;
|
|
230
|
+
}
|
|
231
|
+
|
|
187
232
|
/** Suffix marking a context created to speak an error fallback, so it never recurses. */
|
|
188
233
|
const FALLBACK_CONTEXT_SUFFIX = ":error-fallback";
|
|
189
234
|
|
|
235
|
+
/** Scheduler key for a pending tool call's G3 delayed-cue timer. */
|
|
236
|
+
function toolCueTimerKey(contextId: string, toolId: string): string {
|
|
237
|
+
return `tool_cue:${contextId}:${toolId}`;
|
|
238
|
+
}
|
|
239
|
+
|
|
190
240
|
// =============================================================================
|
|
191
241
|
// Session Implementation
|
|
192
242
|
// =============================================================================
|
|
@@ -211,6 +261,10 @@ export class VoiceAgentSession {
|
|
|
211
261
|
private readonly scheduler: Scheduler;
|
|
212
262
|
private readonly ttsPlayout: TtsPlayoutClock;
|
|
213
263
|
private interruptedGenerationContextIds = new Set<string>();
|
|
264
|
+
// Turns whose generation is in-flight (eos.turn_complete emitted, not yet
|
|
265
|
+
// finished/interrupted). Lets a client "stop" during the reasoner TTFT gap —
|
|
266
|
+
// before any audio plays — still abort the turn (thinking-phase barge-in, B3).
|
|
267
|
+
private generatingContextIds = new Set<string>();
|
|
214
268
|
private ttsTextBuffers = new Map<string, TtsTextBuffer>();
|
|
215
269
|
private readonly minInterruptionMs: number;
|
|
216
270
|
private readonly primarySpeakerGate: PrimarySpeakerGate;
|
|
@@ -223,10 +277,14 @@ export class VoiceAgentSession {
|
|
|
223
277
|
private readonly watchdogs!: VoiceSessionWatchdogs;
|
|
224
278
|
private readonly observabilityObserver: ObservabilityObserver;
|
|
225
279
|
private turnUserStoppedAtMs = new Map<string, number>();
|
|
280
|
+
private turnTimings = new Map<string, TurnTiming>();
|
|
226
281
|
private speakerEnrollmentContextId: string | null = null;
|
|
227
282
|
private firstTtsAudioFired = new Set<string>();
|
|
228
283
|
private readonly errorFallbackText: string;
|
|
229
284
|
private fallbackInjectedContexts = new Set<string>();
|
|
285
|
+
// G3: pending tool calls per context (toolId → toolName) driving the tool_call_cue lifecycle.
|
|
286
|
+
private readonly delayCueAfterMs: number;
|
|
287
|
+
private pendingToolCues = new Map<string, Map<string, string>>();
|
|
230
288
|
private readonly endpointingOwner: "provider_stt" | "smart_turn" | "timer";
|
|
231
289
|
private lastFinalizedContextId = "";
|
|
232
290
|
|
|
@@ -241,6 +299,7 @@ export class VoiceAgentSession {
|
|
|
241
299
|
this.ttsPlayout = new TtsPlayoutClock(this.scheduler);
|
|
242
300
|
this.sttForceFinalizeTimeoutMs = config.sttForceFinalizeTimeoutMs ?? 7000;
|
|
243
301
|
this.minInterruptionMs = config.minInterruptionMs ?? 280;
|
|
302
|
+
this.delayCueAfterMs = config.delayCueAfterMs ?? 2000;
|
|
244
303
|
this.primarySpeakerGate = new PrimarySpeakerGate({
|
|
245
304
|
enabled: config.primarySpeakerBargeInEnabled !== false,
|
|
246
305
|
});
|
|
@@ -389,11 +448,16 @@ export class VoiceAgentSession {
|
|
|
389
448
|
this.ttsPlayout.clear();
|
|
390
449
|
this.turnArbiter.clear();
|
|
391
450
|
this.turnUserStoppedAtMs.clear();
|
|
451
|
+
this.turnTimings.clear();
|
|
392
452
|
this.firstTtsAudioFired.clear();
|
|
393
453
|
this.fallbackInjectedContexts.clear();
|
|
394
454
|
this.ttsTextBuffers.clear();
|
|
395
455
|
this.interruptedGenerationContextIds.clear();
|
|
396
456
|
this.firstLlmDeltaReceived.clear();
|
|
457
|
+
for (const [contextId, pending] of this.pendingToolCues) {
|
|
458
|
+
for (const toolId of pending.keys()) this.scheduler.cancel(toolCueTimerKey(contextId, toolId));
|
|
459
|
+
}
|
|
460
|
+
this.pendingToolCues.clear();
|
|
397
461
|
|
|
398
462
|
// 2. Run finalize chain (reverse order)
|
|
399
463
|
await runFinalizeChain(this.initSteps);
|
|
@@ -414,7 +478,16 @@ export class VoiceAgentSession {
|
|
|
414
478
|
}
|
|
415
479
|
|
|
416
480
|
requestClientInterrupt(contextId: string): void {
|
|
417
|
-
|
|
481
|
+
// Playing out → the arbiter owns the barge-in (primary-speaker reset + metrics).
|
|
482
|
+
if (this.ttsPlayout.isActive(contextId)) {
|
|
483
|
+
this.turnArbiter.commitClientInterrupt(contextId);
|
|
484
|
+
return;
|
|
485
|
+
}
|
|
486
|
+
// Thinking-phase barge-in (B3): no audio yet, but a generation is in-flight —
|
|
487
|
+
// abort it so "stop" during the reasoner TTFT gap is honored, not dropped.
|
|
488
|
+
if (this.generatingContextIds.has(contextId)) {
|
|
489
|
+
this.bus.push(Route.Critical, make.interruptDetected(contextId, Date.now(), "client"));
|
|
490
|
+
}
|
|
418
491
|
}
|
|
419
492
|
|
|
420
493
|
// =========================================================================
|
|
@@ -486,6 +559,10 @@ export class VoiceAgentSession {
|
|
|
486
559
|
this.bus.on("llm.tool_call", this.handleLlmToolCall.bind(this));
|
|
487
560
|
this.bus.on("llm.tool_result", this.handleLlmToolResult.bind(this));
|
|
488
561
|
|
|
562
|
+
// Delegate (Responder-Thinker) observability — G2, RFC bimodel-delegate-seam
|
|
563
|
+
this.bus.on("delegate.query", this.handleDelegateQuery.bind(this));
|
|
564
|
+
this.bus.on("delegate.result", this.handleDelegateResult.bind(this));
|
|
565
|
+
|
|
489
566
|
// TTS
|
|
490
567
|
this.bus.on("tts.audio", this.handleTtsAudio.bind(this));
|
|
491
568
|
this.bus.on("tts.end", this.handleTtsEnd.bind(this));
|
|
@@ -691,14 +768,61 @@ export class VoiceAgentSession {
|
|
|
691
768
|
this.turnArbiter.onSpeechEnded(pkt, Boolean(this.latestActiveTtsContextId()));
|
|
692
769
|
|
|
693
770
|
this.turnUserStoppedAtMs.set(pkt.contextId, pkt.timestampMs);
|
|
771
|
+
this.timingFor(pkt.contextId).speechEndedMs = pkt.timestampMs;
|
|
694
772
|
this.watchdogs.startVaqiMissedResponseTimer(pkt.contextId, pkt.timestampMs);
|
|
695
773
|
}
|
|
696
774
|
|
|
775
|
+
private timingFor(contextId: string): TurnTiming {
|
|
776
|
+
let timing = this.turnTimings.get(contextId);
|
|
777
|
+
if (!timing) {
|
|
778
|
+
timing = {};
|
|
779
|
+
this.turnTimings.set(contextId, timing);
|
|
780
|
+
}
|
|
781
|
+
return timing;
|
|
782
|
+
}
|
|
783
|
+
|
|
784
|
+
private emitTurnLatency(contextId: string, firstAudioMs: number): void {
|
|
785
|
+
const timing = this.turnTimings.get(contextId);
|
|
786
|
+
this.turnTimings.delete(contextId);
|
|
787
|
+
if (!timing) return;
|
|
788
|
+
const anchorMs = timing.speechEndedMs ?? timing.eosMs;
|
|
789
|
+
if (anchorMs === undefined) return; // text-injected or fallback turn — not a voice TTFA
|
|
790
|
+
this.emit("turn_latency", {
|
|
791
|
+
tsMs: firstAudioMs,
|
|
792
|
+
turnId: contextId,
|
|
793
|
+
ttfaMs: firstAudioMs - anchorMs,
|
|
794
|
+
...(timing.speechEndedMs !== undefined && timing.eosMs !== undefined
|
|
795
|
+
? { eouDelayMs: timing.eosMs - timing.speechEndedMs }
|
|
796
|
+
: {}),
|
|
797
|
+
...(timing.eosMs !== undefined && timing.firstLlmDeltaMs !== undefined
|
|
798
|
+
? { llmTtftMs: timing.firstLlmDeltaMs - timing.eosMs }
|
|
799
|
+
: {}),
|
|
800
|
+
...(timing.firstTtsTextMs !== undefined
|
|
801
|
+
? { ttsTtfbMs: firstAudioMs - timing.firstTtsTextMs }
|
|
802
|
+
: {}),
|
|
803
|
+
fillerUsed: timing.fillerUsed === true,
|
|
804
|
+
});
|
|
805
|
+
}
|
|
806
|
+
|
|
697
807
|
private handleTurnComplete(pkt: EndOfSpeechPacket): void {
|
|
698
808
|
if (this.lastFinalizedContextId === pkt.contextId) {
|
|
699
809
|
this.bus.push(Route.Background, make.metric(pkt.contextId, "eos.duplicate_dropped", "1"));
|
|
700
810
|
return;
|
|
701
811
|
}
|
|
812
|
+
|
|
813
|
+
// A backchannel ("uh-huh", "okay") uttered WHILE the assistant is still speaking
|
|
814
|
+
// is not a turn (B4). Dropping it here does two things: it does NOT cancel the
|
|
815
|
+
// assistant's in-flight answer (the supersede path below would otherwise kill it),
|
|
816
|
+
// and it does NOT spawn a second LLM response to the backchannel (the double-reply
|
|
817
|
+
// bug). A backchannel with no assistant currently speaking IS a real turn — only
|
|
818
|
+
// suppress when another context's TTS is actively playing. (English-only classifier
|
|
819
|
+
// today — locale-aware backchannels are a separate improvement.)
|
|
820
|
+
const otherTtsActive = this.ttsPlayout.activeContexts().some((c) => c !== pkt.contextId);
|
|
821
|
+
if (otherTtsActive && isBackchannel(pkt.text)) {
|
|
822
|
+
this.bus.push(Route.Background, make.metric(pkt.contextId, "turn.backchannel_dropped", pkt.text));
|
|
823
|
+
return;
|
|
824
|
+
}
|
|
825
|
+
|
|
702
826
|
this.lastFinalizedContextId = pkt.contextId;
|
|
703
827
|
|
|
704
828
|
// Re-arm per-turn guard state for the next turn. Transports with a stable
|
|
@@ -711,6 +835,16 @@ export class VoiceAgentSession {
|
|
|
711
835
|
this.interruptedGenerationContextIds.delete(pkt.contextId);
|
|
712
836
|
this.fallbackInjectedContexts.delete(pkt.contextId);
|
|
713
837
|
|
|
838
|
+
// Supersede (L1): a new turn must cancel any still-active prior-turn TTS or
|
|
839
|
+
// generation. Without this, a false-EOS (early endpoint on a mid-sentence
|
|
840
|
+
// pause) starts turn N, the user resumes, turn N's already-emitted audio
|
|
841
|
+
// keeps synthesizing, and it plays over the user while turn N+1 is answered.
|
|
842
|
+
// The bridge supersedes the *LLM*; only the session can stop the *TTS*.
|
|
843
|
+
for (const activeCtx of this.ttsPlayout.activeContexts()) {
|
|
844
|
+
if (activeCtx !== pkt.contextId) this.cancelStaleGeneration(activeCtx, pkt.timestampMs);
|
|
845
|
+
}
|
|
846
|
+
|
|
847
|
+
this.generatingContextIds.add(pkt.contextId);
|
|
714
848
|
this.currentTurnId = pkt.contextId;
|
|
715
849
|
this.idleTimeout.setContextId(pkt.contextId);
|
|
716
850
|
|
|
@@ -727,11 +861,19 @@ export class VoiceAgentSession {
|
|
|
727
861
|
timestampMs: pkt.timestampMs,
|
|
728
862
|
});
|
|
729
863
|
|
|
730
|
-
// Stop idle timeout while LLM is
|
|
731
|
-
|
|
864
|
+
// Stop idle timeout while the LLM processes. The user just spoke — that is
|
|
865
|
+
// genuine engagement, so reset the idle *escalation* count (P2): a user who
|
|
866
|
+
// answers the first "are you there?" must not be escalated straight to the
|
|
867
|
+
// disconnect prompt later in the call.
|
|
868
|
+
this.bus.push(Route.Main, make.stopIdleTimeout(pkt.contextId, Date.now(), true));
|
|
869
|
+
|
|
870
|
+
const timing = this.timingFor(pkt.contextId);
|
|
871
|
+
timing.eosMs = pkt.timestampMs;
|
|
732
872
|
|
|
733
873
|
const fillerText = this.latencyFiller.start(pkt.contextId, pkt.text, pkt.timestampMs);
|
|
734
874
|
if (fillerText) {
|
|
875
|
+
timing.fillerUsed = true;
|
|
876
|
+
timing.firstTtsTextMs ??= pkt.timestampMs;
|
|
735
877
|
this.bus.push(Route.Main, make.ttsText(pkt.contextId, Date.now(), fillerText));
|
|
736
878
|
this.bus.push(Route.Background, make.metric(pkt.contextId, "filler.started", fillerText));
|
|
737
879
|
}
|
|
@@ -763,6 +905,8 @@ export class VoiceAgentSession {
|
|
|
763
905
|
let deltaText = pkt.text;
|
|
764
906
|
if (!this.firstLlmDeltaReceived.has(pkt.contextId)) {
|
|
765
907
|
this.firstLlmDeltaReceived.add(pkt.contextId);
|
|
908
|
+
const timing = this.turnTimings.get(pkt.contextId);
|
|
909
|
+
if (timing) timing.firstLlmDeltaMs ??= pkt.timestampMs;
|
|
766
910
|
if (this.latencyFiller.isActive(pkt.contextId)) {
|
|
767
911
|
deltaText = this.latencyFiller.spliceLlmDelta(pkt.contextId, deltaText);
|
|
768
912
|
this.bus.push(Route.Background, make.metric(pkt.contextId, "filler.spliced", "1"));
|
|
@@ -781,17 +925,18 @@ export class VoiceAgentSession {
|
|
|
781
925
|
timestampMs: pkt.timestampMs,
|
|
782
926
|
});
|
|
783
927
|
|
|
784
|
-
this.bufferTtsText(pkt.contextId, deltaText);
|
|
928
|
+
this.bufferTtsText(pkt.contextId, deltaText, pkt.timestampMs);
|
|
785
929
|
}
|
|
786
930
|
|
|
787
931
|
private handleLlmDone(pkt: LlmResponseDonePacket): void {
|
|
932
|
+
this.generatingContextIds.delete(pkt.contextId);
|
|
788
933
|
if (this.interruptedGenerationContextIds.has(pkt.contextId)) {
|
|
789
934
|
this.ttsTextBuffers.delete(pkt.contextId);
|
|
790
935
|
this.bus.push(Route.Background, make.metric(pkt.contextId, "llm.done_ignored_after_interrupt", "1"));
|
|
791
936
|
return;
|
|
792
937
|
}
|
|
793
938
|
|
|
794
|
-
const spokenText = this.flushTtsText(pkt.contextId);
|
|
939
|
+
const spokenText = this.flushTtsText(pkt.contextId, pkt.timestampMs);
|
|
795
940
|
this.emit("agent_finished", {
|
|
796
941
|
tsMs: pkt.timestampMs,
|
|
797
942
|
turnId: pkt.contextId,
|
|
@@ -809,11 +954,13 @@ export class VoiceAgentSession {
|
|
|
809
954
|
this.bus.push(Route.Main, make.ttsDone(pkt.contextId, Date.now(), spokenText));
|
|
810
955
|
}
|
|
811
956
|
|
|
812
|
-
private bufferTtsText(contextId: string, text: string): void {
|
|
957
|
+
private bufferTtsText(contextId: string, text: string, tsMs?: number): void {
|
|
813
958
|
const buffer = this.ttsTextBuffers.get(contextId) ?? { pending: "", emitted: "" };
|
|
814
959
|
buffer.pending += text;
|
|
815
960
|
const complete = takeCompleteVoiceText(buffer.pending);
|
|
816
961
|
if (complete.text) {
|
|
962
|
+
const timing = this.turnTimings.get(contextId);
|
|
963
|
+
if (timing) timing.firstTtsTextMs ??= tsMs ?? Date.now();
|
|
817
964
|
this.bus.push(Route.Main, make.ttsText(contextId, Date.now(), complete.text));
|
|
818
965
|
buffer.emitted = appendVoiceText(buffer.emitted, complete.text);
|
|
819
966
|
}
|
|
@@ -821,11 +968,13 @@ export class VoiceAgentSession {
|
|
|
821
968
|
this.ttsTextBuffers.set(contextId, buffer);
|
|
822
969
|
}
|
|
823
970
|
|
|
824
|
-
private flushTtsText(contextId: string): string {
|
|
971
|
+
private flushTtsText(contextId: string, tsMs?: number): string {
|
|
825
972
|
const buffer = this.ttsTextBuffers.get(contextId);
|
|
826
973
|
if (!buffer) return "";
|
|
827
974
|
const tail = buffer.pending.trim();
|
|
828
975
|
if (tail) {
|
|
976
|
+
const timing = this.turnTimings.get(contextId);
|
|
977
|
+
if (timing) timing.firstTtsTextMs ??= tsMs ?? Date.now();
|
|
829
978
|
this.bus.push(Route.Main, make.ttsText(contextId, Date.now(), tail));
|
|
830
979
|
buffer.emitted = appendVoiceText(buffer.emitted, tail);
|
|
831
980
|
buffer.pending = "";
|
|
@@ -865,9 +1014,101 @@ export class VoiceAgentSession {
|
|
|
865
1014
|
},
|
|
866
1015
|
timestampMs: pkt.timestampMs,
|
|
867
1016
|
});
|
|
1017
|
+
|
|
1018
|
+
// G3: arm the typed preamble/filler lifecycle for this pending tool call.
|
|
1019
|
+
const pending = this.pendingToolCues.get(pkt.contextId) ?? new Map<string, string>();
|
|
1020
|
+
pending.set(pkt.toolId, pkt.toolName);
|
|
1021
|
+
this.pendingToolCues.set(pkt.contextId, pending);
|
|
1022
|
+
this.emitToolCallCue(pkt.contextId, "started", pkt.toolId, pkt.toolName);
|
|
1023
|
+
if (this.delayCueAfterMs > 0) {
|
|
1024
|
+
const afterMs = this.delayCueAfterMs;
|
|
1025
|
+
this.scheduler.schedule(toolCueTimerKey(pkt.contextId, pkt.toolId), afterMs, () => {
|
|
1026
|
+
if (!this.pendingToolCues.get(pkt.contextId)?.has(pkt.toolId)) return;
|
|
1027
|
+
this.emitToolCallCue(pkt.contextId, "delayed", pkt.toolId, pkt.toolName, afterMs);
|
|
1028
|
+
});
|
|
1029
|
+
}
|
|
1030
|
+
}
|
|
1031
|
+
|
|
1032
|
+
private emitToolCallCue(
|
|
1033
|
+
contextId: string,
|
|
1034
|
+
phase: "started" | "delayed" | "complete" | "failed",
|
|
1035
|
+
toolId: string,
|
|
1036
|
+
toolName: string,
|
|
1037
|
+
afterMs?: number,
|
|
1038
|
+
): void {
|
|
1039
|
+
this.emit("tool_call_cue", {
|
|
1040
|
+
tsMs: Date.now(),
|
|
1041
|
+
turnId: contextId,
|
|
1042
|
+
phase,
|
|
1043
|
+
toolId,
|
|
1044
|
+
toolName,
|
|
1045
|
+
...(afterMs !== undefined ? { afterMs } : {}),
|
|
1046
|
+
});
|
|
1047
|
+
}
|
|
1048
|
+
|
|
1049
|
+
/** G3: resolve one pending tool call with a terminal cue phase. */
|
|
1050
|
+
private resolveToolCue(contextId: string, toolId: string, phase: "complete" | "failed"): void {
|
|
1051
|
+
const pending = this.pendingToolCues.get(contextId);
|
|
1052
|
+
const toolName = pending?.get(toolId);
|
|
1053
|
+
if (toolName === undefined) return;
|
|
1054
|
+
pending!.delete(toolId);
|
|
1055
|
+
if (pending!.size === 0) this.pendingToolCues.delete(contextId);
|
|
1056
|
+
this.scheduler.cancel(toolCueTimerKey(contextId, toolId));
|
|
1057
|
+
this.emitToolCallCue(contextId, phase, toolId, toolName);
|
|
1058
|
+
}
|
|
1059
|
+
|
|
1060
|
+
/** G3: fail every pending tool call for a context (error / barge-in / supersede). */
|
|
1061
|
+
private failPendingToolCues(contextId: string): void {
|
|
1062
|
+
const pending = this.pendingToolCues.get(contextId);
|
|
1063
|
+
if (!pending) return;
|
|
1064
|
+
for (const toolId of [...pending.keys()]) this.resolveToolCue(contextId, toolId, "failed");
|
|
1065
|
+
}
|
|
1066
|
+
|
|
1067
|
+
private handleDelegateQuery(pkt: DelegateQueryPacket): void {
|
|
1068
|
+
this.emit("delegate_query", {
|
|
1069
|
+
tsMs: pkt.timestampMs,
|
|
1070
|
+
turnId: pkt.contextId,
|
|
1071
|
+
query: pkt.query,
|
|
1072
|
+
toolId: pkt.toolId,
|
|
1073
|
+
toolName: pkt.toolName,
|
|
1074
|
+
});
|
|
1075
|
+
this.debugPush({
|
|
1076
|
+
component: "delegate",
|
|
1077
|
+
type: "query",
|
|
1078
|
+
data: {
|
|
1079
|
+
context_id: pkt.contextId,
|
|
1080
|
+
query: pkt.query,
|
|
1081
|
+
...(pkt.toolName ? { tool_name: pkt.toolName } : {}),
|
|
1082
|
+
},
|
|
1083
|
+
timestampMs: pkt.timestampMs,
|
|
1084
|
+
});
|
|
1085
|
+
}
|
|
1086
|
+
|
|
1087
|
+
private handleDelegateResult(pkt: DelegateResultPacket): void {
|
|
1088
|
+
this.emit("delegate_result", {
|
|
1089
|
+
tsMs: pkt.timestampMs,
|
|
1090
|
+
turnId: pkt.contextId,
|
|
1091
|
+
query: pkt.query,
|
|
1092
|
+
answer: pkt.answer,
|
|
1093
|
+
durationMs: pkt.durationMs,
|
|
1094
|
+
grounded: pkt.grounded,
|
|
1095
|
+
toolId: pkt.toolId,
|
|
1096
|
+
toolName: pkt.toolName,
|
|
1097
|
+
});
|
|
1098
|
+
this.debugPush({
|
|
1099
|
+
component: "delegate",
|
|
1100
|
+
type: "result",
|
|
1101
|
+
data: {
|
|
1102
|
+
context_id: pkt.contextId,
|
|
1103
|
+
duration_ms: String(pkt.durationMs),
|
|
1104
|
+
grounded: String(pkt.grounded),
|
|
1105
|
+
},
|
|
1106
|
+
timestampMs: pkt.timestampMs,
|
|
1107
|
+
});
|
|
868
1108
|
}
|
|
869
1109
|
|
|
870
1110
|
private handleLlmToolResult(pkt: LlmToolResultPacket): void {
|
|
1111
|
+
this.resolveToolCue(pkt.contextId, pkt.toolId, "complete");
|
|
871
1112
|
this.emit("agent_tool_result", {
|
|
872
1113
|
tsMs: pkt.timestampMs,
|
|
873
1114
|
turnId: pkt.contextId,
|
|
@@ -907,20 +1148,27 @@ export class VoiceAgentSession {
|
|
|
907
1148
|
);
|
|
908
1149
|
this.turnUserStoppedAtMs.delete(pkt.contextId);
|
|
909
1150
|
}
|
|
1151
|
+
this.emitTurnLatency(pkt.contextId, pkt.timestampMs);
|
|
910
1152
|
}
|
|
911
1153
|
|
|
912
1154
|
this.primarySpeakerGate.observeAssistantPlayout(pkt.audio);
|
|
913
1155
|
// Audio just arrived — (re)arm the stall watchdog for this turn's TTS output.
|
|
914
1156
|
this.watchdogs.armTtsStallTimer(pkt.contextId);
|
|
915
1157
|
|
|
916
|
-
//
|
|
1158
|
+
// Mark active and advance this context's playout cursor by the chunk's
|
|
1159
|
+
// realtime duration.
|
|
917
1160
|
const sampleRateHz = requireTtsAudioSampleRate(pkt.sampleRateHz);
|
|
918
1161
|
const audioDurationMs = estimatePcm16Duration(pkt.audio, sampleRateHz);
|
|
919
|
-
|
|
1162
|
+
const now = Date.now();
|
|
1163
|
+
this.ttsPlayout.noteAudio(pkt.contextId, audioDurationMs, now);
|
|
920
1164
|
|
|
921
|
-
//
|
|
922
|
-
// realtime
|
|
923
|
-
|
|
1165
|
+
// Anchor the idle timer to when playout actually *ends* (P2), not to chunk
|
|
1166
|
+
// arrival. TTS streams faster than realtime, so extending by each chunk's
|
|
1167
|
+
// duration from arrival lets the timer fire mid-speech on a long answer.
|
|
1168
|
+
// playoutEnd() is cumulative across chunks, so this re-arms to durationMs
|
|
1169
|
+
// after the audio delivered so far finishes playing.
|
|
1170
|
+
const playoutEndMs = this.ttsPlayout.playoutEnd(pkt.contextId);
|
|
1171
|
+
this.idleTimeout.extend(playoutEndMs !== undefined ? Math.max(0, playoutEndMs - now) : audioDurationMs);
|
|
924
1172
|
|
|
925
1173
|
this.debugPush({
|
|
926
1174
|
component: "tts",
|
|
@@ -938,6 +1186,7 @@ export class VoiceAgentSession {
|
|
|
938
1186
|
private handleTtsEnd(pkt: TextToSpeechEndPacket): void {
|
|
939
1187
|
// Generation finished, but the streamed audio is still playing out. Keep the
|
|
940
1188
|
// context interruptible until its playout estimate elapses, then release it.
|
|
1189
|
+
this.generatingContextIds.delete(pkt.contextId);
|
|
941
1190
|
this.ttsPlayout.scheduleRelease(pkt.contextId, Date.now());
|
|
942
1191
|
this.watchdogs.clearTtsStallTimerFor(pkt.contextId);
|
|
943
1192
|
this.debugPush({
|
|
@@ -954,7 +1203,9 @@ export class VoiceAgentSession {
|
|
|
954
1203
|
|
|
955
1204
|
private handleInterruptDetected(pkt: InterruptionDetectedPacket): void {
|
|
956
1205
|
this.interruptedGenerationContextIds.add(pkt.contextId);
|
|
1206
|
+
this.failPendingToolCues(pkt.contextId); // G3: the aborted delegate's cue fails (R5)
|
|
957
1207
|
this.latencyFiller.cancel(pkt.contextId);
|
|
1208
|
+
this.turnTimings.delete(pkt.contextId);
|
|
958
1209
|
this.firstLlmDeltaReceived.delete(pkt.contextId);
|
|
959
1210
|
this.ttsTextBuffers.delete(pkt.contextId);
|
|
960
1211
|
this.ttsPlayout.release(pkt.contextId);
|
|
@@ -990,6 +1241,29 @@ export class VoiceAgentSession {
|
|
|
990
1241
|
// segments into the next turn when a client reuses the same contextId
|
|
991
1242
|
// (the provider STT plugins listen for interrupt.stt; previously unfired).
|
|
992
1243
|
this.bus.push(Route.Critical, make.interruptStt(pkt.contextId, pkt.timestampMs));
|
|
1244
|
+
this.generatingContextIds.delete(pkt.contextId);
|
|
1245
|
+
}
|
|
1246
|
+
|
|
1247
|
+
/**
|
|
1248
|
+
* Cancel a stale prior-turn generation/playout when a new turn supersedes it
|
|
1249
|
+
* (L1). Mirrors the interrupt teardown but without the barge-in metrics — this
|
|
1250
|
+
* is a turn boundary, not a user interruption. Stops leftover TTS audio, aborts
|
|
1251
|
+
* the LLM, and drops late deltas/audio for the stale context.
|
|
1252
|
+
*/
|
|
1253
|
+
private cancelStaleGeneration(contextId: string, timestampMs: number): void {
|
|
1254
|
+
this.interruptedGenerationContextIds.add(contextId);
|
|
1255
|
+
this.failPendingToolCues(contextId); // G3: a superseded turn's pending cue fails
|
|
1256
|
+
this.generatingContextIds.delete(contextId);
|
|
1257
|
+
this.latencyFiller.cancel(contextId);
|
|
1258
|
+
this.turnTimings.delete(contextId);
|
|
1259
|
+
this.firstLlmDeltaReceived.delete(contextId);
|
|
1260
|
+
this.ttsTextBuffers.delete(contextId);
|
|
1261
|
+
this.ttsPlayout.release(contextId);
|
|
1262
|
+
this.watchdogs.clearTtsStallTimerFor(contextId);
|
|
1263
|
+
this.bus.push(Route.Critical, make.recordAssistantTruncate(contextId, Date.now()));
|
|
1264
|
+
this.bus.push(Route.Critical, make.interruptTts(contextId, timestampMs));
|
|
1265
|
+
this.bus.push(Route.Critical, make.interruptLlm(contextId, timestampMs));
|
|
1266
|
+
this.bus.push(Route.Background, make.metric(contextId, "supersede.cancelled_stale_generation", "1"));
|
|
993
1267
|
}
|
|
994
1268
|
|
|
995
1269
|
private handleComponentError(pkt: VoiceErrorPacket): void {
|
|
@@ -1011,7 +1285,12 @@ export class VoiceAgentSession {
|
|
|
1011
1285
|
timestampMs: pkt.timestampMs,
|
|
1012
1286
|
});
|
|
1013
1287
|
|
|
1288
|
+
// G3: an LLM/bridge error while a tool call is pending means no result is coming.
|
|
1289
|
+
if (pkt.component === "llm" || pkt.component === "bridge") {
|
|
1290
|
+
this.failPendingToolCues(pkt.contextId);
|
|
1291
|
+
}
|
|
1014
1292
|
this.latencyFiller.clear(pkt.contextId);
|
|
1293
|
+
this.generatingContextIds.delete(pkt.contextId);
|
|
1015
1294
|
// The packet's own recoverability verdict is authoritative when present: the
|
|
1016
1295
|
// bus marks handler exceptions (pipeline.error) recoverable by design — one
|
|
1017
1296
|
// misbehaving handler must degrade the turn, not kill the whole call.
|
package/src/voice-text.test.ts
CHANGED
|
@@ -25,6 +25,19 @@ describe("isCompleteVoiceText", () => {
|
|
|
25
25
|
expect(isCompleteVoiceText("مرحبا؟")).toBe(true); // Arabic question mark
|
|
26
26
|
expect(isCompleteVoiceText("नमस्ते।")).toBe(true); // Devanagari danda
|
|
27
27
|
});
|
|
28
|
+
|
|
29
|
+
it("does not treat abbreviation or decimal dots as sentence ends", () => {
|
|
30
|
+
// These would otherwise be voiced with a falling intonation and split from
|
|
31
|
+
// their continuation ("Dr." | "Smith", "twelve." | "fifty").
|
|
32
|
+
expect(isCompleteVoiceText("Dr.")).toBe(false);
|
|
33
|
+
expect(isCompleteVoiceText("e.g.")).toBe(false);
|
|
34
|
+
expect(isCompleteVoiceText("The total is $12.")).toBe(false);
|
|
35
|
+
expect(isCompleteVoiceText("Meet at 3 p.m.")).toBe(false);
|
|
36
|
+
expect(isCompleteVoiceText("His name is J.")).toBe(false);
|
|
37
|
+
// But a real sentence end after an abbreviation earlier in the text is fine.
|
|
38
|
+
expect(isCompleteVoiceText("Dr. Smith will see you now.")).toBe(true);
|
|
39
|
+
expect(isCompleteVoiceText("The total is $12.50 today.")).toBe(true);
|
|
40
|
+
});
|
|
28
41
|
});
|
|
29
42
|
|
|
30
43
|
describe("takeCompleteVoiceText", () => {
|
package/src/voice-text.ts
CHANGED
|
@@ -33,12 +33,41 @@ export function takeCompleteVoiceText(text: string): { text: string; remaining:
|
|
|
33
33
|
return { text: emitted.trimEnd(), remaining };
|
|
34
34
|
}
|
|
35
35
|
|
|
36
|
+
// Common abbreviations whose trailing "." is not a sentence end. Lowercased,
|
|
37
|
+
// dots stripped, so "e.g." matches "eg". Kept small and English-centric — the
|
|
38
|
+
// turn-end flush handles anything that legitimately ends here.
|
|
39
|
+
const ABBREVIATIONS = new Set([
|
|
40
|
+
"mr", "mrs", "ms", "dr", "prof", "sr", "jr", "st", "vs", "etc", "eg", "ie",
|
|
41
|
+
"am", "pm", "no", "vol", "inc", "ltd", "co", "gen", "gov", "sen", "rep",
|
|
42
|
+
"apt", "dept", "approx", "est", "min", "max",
|
|
43
|
+
]);
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* A segment ending in "." is not necessarily a finished sentence: it may be a
|
|
47
|
+
* decimal point ("$12." before "50") or an abbreviation dot ("Dr." before a
|
|
48
|
+
* name, "e.g."). Voicing those as sentence ends produces "twelve." (falling
|
|
49
|
+
* intonation) … "fifty", or splits "Dr." from the name. Defer them — if nothing
|
|
50
|
+
* follows, the turn-end flush still speaks the tail.
|
|
51
|
+
*/
|
|
52
|
+
function isFalseTerminalDot(endsWithDot: string): boolean {
|
|
53
|
+
const beforeDot = endsWithDot.slice(0, -1);
|
|
54
|
+
if (/\d$/.test(beforeDot)) return true; // decimal / ordinal: "12." , "$3."
|
|
55
|
+
const word = beforeDot.match(/([A-Za-z][A-Za-z.]*)$/);
|
|
56
|
+
if (!word) return false;
|
|
57
|
+
const normalized = word[1]!.replace(/\./g, "").toLowerCase();
|
|
58
|
+
if (ABBREVIATIONS.has(normalized)) return true;
|
|
59
|
+
if (normalized.length === 1) return true; // single initial: "J."
|
|
60
|
+
return false;
|
|
61
|
+
}
|
|
62
|
+
|
|
36
63
|
export function isCompleteVoiceText(text: string): boolean {
|
|
37
64
|
const trimmed = text.trim();
|
|
38
65
|
for (let index = trimmed.length - 1; index >= 0; index -= 1) {
|
|
39
66
|
const char = trimmed[index]!;
|
|
40
67
|
if (isClosingPunctuation(char)) continue;
|
|
41
|
-
|
|
68
|
+
if (!isTerminalPunctuation(char)) return false;
|
|
69
|
+
if (char === "." && isFalseTerminalDot(trimmed.slice(0, index + 1))) return false;
|
|
70
|
+
return true;
|
|
42
71
|
}
|
|
43
72
|
return false;
|
|
44
73
|
}
|
|
@@ -68,8 +97,19 @@ function isTerminalPunctuation(char: string): boolean {
|
|
|
68
97
|
char === "॥";
|
|
69
98
|
}
|
|
70
99
|
|
|
100
|
+
// The ICU sentence segmenter is one of the more expensive Intl allocations;
|
|
101
|
+
// building one per LLM delta (50–200/turn) is avoidable CPU/GC churn on the
|
|
102
|
+
// token→TTS latency path. It is stateless, so cache one per process. `undefined`
|
|
103
|
+
// = not yet computed; `null` = unavailable (fall back to the regex splitter).
|
|
104
|
+
let cachedSegmenter: { segment(text: string): Iterable<SentenceSegment> } | null | undefined;
|
|
105
|
+
|
|
106
|
+
function getSentenceSegmenter(): { segment(text: string): Iterable<SentenceSegment> } | null {
|
|
107
|
+
if (cachedSegmenter === undefined) cachedSegmenter = createSentenceSegmenter();
|
|
108
|
+
return cachedSegmenter;
|
|
109
|
+
}
|
|
110
|
+
|
|
71
111
|
function segmentSentences(text: string): string[] {
|
|
72
|
-
const segmenter =
|
|
112
|
+
const segmenter = getSentenceSegmenter();
|
|
73
113
|
if (segmenter) {
|
|
74
114
|
return Array.from(segmenter.segment(text), (part) => part.segment);
|
|
75
115
|
}
|