@kuralle-syrinx/core 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +41 -0
- package/package.json +22 -0
- package/src/audio/audio.test.ts +285 -0
- package/src/audio/index.ts +5 -0
- package/src/audio/mulaw.ts +42 -0
- package/src/audio/pcm.ts +43 -0
- package/src/audio/resample.ts +177 -0
- package/src/audio-envelope.test.ts +167 -0
- package/src/audio-envelope.ts +143 -0
- package/src/conversation-event.ts +62 -0
- package/src/error-handler.test.ts +56 -0
- package/src/error-handler.ts +149 -0
- package/src/idle-timeout.ts +210 -0
- package/src/index.ts +215 -0
- package/src/init-chain.ts +137 -0
- package/src/init-stage-order.ts +79 -0
- package/src/latency-filler-fixtures.ts +16 -0
- package/src/latency-filler.test.ts +62 -0
- package/src/latency-filler.ts +125 -0
- package/src/mode-switcher.ts +110 -0
- package/src/observability-observer.test.ts +245 -0
- package/src/observability-observer.ts +195 -0
- package/src/observability.test.ts +85 -0
- package/src/observability.ts +93 -0
- package/src/packet-factories.test.ts +34 -0
- package/src/packet-factories.ts +243 -0
- package/src/packets.ts +553 -0
- package/src/pipeline-bus.g10.test.ts +145 -0
- package/src/pipeline-bus.test.ts +197 -0
- package/src/pipeline-bus.ts +369 -0
- package/src/plugin-contract.ts +79 -0
- package/src/primary-speaker-fixtures.ts +45 -0
- package/src/primary-speaker-gate.test.ts +150 -0
- package/src/primary-speaker-gate.ts +186 -0
- package/src/provider-fallback.test.ts +87 -0
- package/src/provider-fallback.ts +88 -0
- package/src/reasoner.test.ts +69 -0
- package/src/reasoner.ts +54 -0
- package/src/retry.test.ts +83 -0
- package/src/retry.ts +106 -0
- package/src/scheduler.ts +28 -0
- package/src/tts-playout-clock.test.ts +125 -0
- package/src/tts-playout-clock.ts +116 -0
- package/src/turn-arbiter.characterization.test.ts +477 -0
- package/src/turn-arbiter.test.ts +567 -0
- package/src/turn-arbiter.ts +283 -0
- package/src/voice-agent-session-util.ts +240 -0
- package/src/voice-agent-session.test.ts +2487 -0
- package/src/voice-agent-session.ts +1175 -0
- package/src/voice-text.test.ts +81 -0
- package/src/voice-text.ts +102 -0
- package/tsconfig.json +21 -0
package/src/packets.ts
ADDED
|
@@ -0,0 +1,553 @@
|
|
|
1
|
+
// SPDX-License-Identifier: MIT
|
|
2
|
+
//
|
|
3
|
+
// Syrinx Kernel v2 — Packet Type Definitions
|
|
4
|
+
//
|
|
5
|
+
// Every packet flowing through the PipelineBus uses these types.
|
|
6
|
+
// Naming convention:
|
|
7
|
+
// Commands (verb-first): InterruptTts, DenoiseAudio, ExecuteLlm
|
|
8
|
+
// Events (past-tense): VadSpeechStarted, SttResult, LlmResponseDelta
|
|
9
|
+
// Errors: SttError, TtsError, LlmError
|
|
10
|
+
// Lifecycle: InitStepCompleted, InitFailed, InitCompleted
|
|
11
|
+
|
|
12
|
+
// =============================================================================
|
|
13
|
+
// Base Types
|
|
14
|
+
// =============================================================================
|
|
15
|
+
|
|
16
|
+
/** Every packet flowing through the bus has these fields. */
|
|
17
|
+
export interface VoicePacket {
|
|
18
|
+
/** Discriminator. Examples: "stt.result", "vad.speech_started", "init.failed" */
|
|
19
|
+
readonly kind: string;
|
|
20
|
+
/** Turn or session identifier. Empty string for session-scoped packets. */
|
|
21
|
+
readonly contextId: string;
|
|
22
|
+
/** Wall-clock creation time in ms since epoch. */
|
|
23
|
+
readonly timestampMs: number;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/** Marker: this packet's handler runs fire-and-forget (not awaited). */
|
|
27
|
+
export interface AsyncPacket extends VoicePacket {
|
|
28
|
+
readonly isAsync: true;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
// =============================================================================
|
|
32
|
+
// Error Types
|
|
33
|
+
// =============================================================================
|
|
34
|
+
|
|
35
|
+
/** Categorized error types across all external-service components. */
|
|
36
|
+
export enum ErrorCategory {
|
|
37
|
+
RateLimit = "rate_limit", // HTTP 429 / quota exceeded — recoverable
|
|
38
|
+
NetworkTimeout = "network_timeout", // connection timeout / ECONNRESET — recoverable
|
|
39
|
+
Authentication = "authentication", // HTTP 401/403 — fatal
|
|
40
|
+
InvalidInput = "invalid_input", // HTTP 400 / invalid audio format — fatal
|
|
41
|
+
InternalFault = "internal_fault", // unexpected provider error — fatal
|
|
42
|
+
ResourceExhausted = "resource_exhausted", // credits depleted — fatal
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export interface VoiceErrorPacket extends VoicePacket {
|
|
46
|
+
/** Which component emitted the error. */
|
|
47
|
+
readonly component: "stt" | "tts" | "vad" | "eos" | "denoiser" | "llm" | "bridge" | "pipeline";
|
|
48
|
+
/** Machine-readable error category. */
|
|
49
|
+
readonly category: ErrorCategory;
|
|
50
|
+
/** Original error. May contain provider-specific details. */
|
|
51
|
+
readonly cause: Error;
|
|
52
|
+
/** Whether the session manager should retry (true) or terminate (false). */
|
|
53
|
+
readonly isRecoverable: boolean;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
// =============================================================================
|
|
57
|
+
// Lifecycle Types
|
|
58
|
+
// =============================================================================
|
|
59
|
+
|
|
60
|
+
export enum SessionState {
|
|
61
|
+
Uninitialized = "uninitialized",
|
|
62
|
+
Initializing = "initializing",
|
|
63
|
+
Ready = "ready",
|
|
64
|
+
Finalizing = "finalizing",
|
|
65
|
+
Closed = "closed",
|
|
66
|
+
Failed = "failed",
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
export enum InitStage {
|
|
70
|
+
Assistant = "assistant",
|
|
71
|
+
Conversation = "conversation",
|
|
72
|
+
Recorder = "recorder",
|
|
73
|
+
Normalizer = "normalizer",
|
|
74
|
+
Auth = "auth",
|
|
75
|
+
STT = "stt",
|
|
76
|
+
TTS = "tts",
|
|
77
|
+
VAD = "vad",
|
|
78
|
+
EOS = "eos",
|
|
79
|
+
Denoiser = "denoiser",
|
|
80
|
+
Behavior = "behavior",
|
|
81
|
+
Telemetry = "telemetry",
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
export interface InitStepCompletedPacket extends VoicePacket {
|
|
85
|
+
readonly kind: "init.step_completed";
|
|
86
|
+
readonly stage: InitStage;
|
|
87
|
+
readonly component: string;
|
|
88
|
+
/** Milliseconds taken to initialize this component. */
|
|
89
|
+
readonly initMs: number;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
export interface InitializationFailedPacket extends VoicePacket {
|
|
93
|
+
readonly kind: "init.failed";
|
|
94
|
+
readonly stage: InitStage;
|
|
95
|
+
readonly component: string;
|
|
96
|
+
readonly category: ErrorCategory;
|
|
97
|
+
readonly cause: Error;
|
|
98
|
+
readonly isRecoverable: false;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
export interface InitializationCompletedPacket extends VoicePacket {
|
|
102
|
+
readonly kind: "init.completed";
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
// =============================================================================
|
|
106
|
+
// Audio format contract
|
|
107
|
+
// =============================================================================
|
|
108
|
+
|
|
109
|
+
export interface AudioFormat {
|
|
110
|
+
readonly encoding: "pcm_s16le" | "mulaw" | "opus";
|
|
111
|
+
readonly sampleRateHz: number;
|
|
112
|
+
readonly channels: 1;
|
|
113
|
+
/** Target frame duration for paced output, when known. */
|
|
114
|
+
readonly frameDurationMs?: number;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
// =============================================================================
|
|
118
|
+
// Input Pipeline Packets (user audio → transcript)
|
|
119
|
+
// =============================================================================
|
|
120
|
+
|
|
121
|
+
export interface UserAudioReceivedPacket extends VoicePacket {
|
|
122
|
+
readonly kind: "user.audio_received";
|
|
123
|
+
/** Raw PCM audio (16-bit, mono, 16kHz). */
|
|
124
|
+
readonly audio: Uint8Array;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
export interface UserTextReceivedPacket extends VoicePacket {
|
|
128
|
+
readonly kind: "user.text_received";
|
|
129
|
+
readonly text: string;
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
export interface DenoiseAudioPacket extends VoicePacket {
|
|
133
|
+
readonly kind: "denoise.audio";
|
|
134
|
+
readonly audio: Uint8Array;
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
export interface DenoisedAudioPacket extends VoicePacket {
|
|
138
|
+
readonly kind: "denoise.result";
|
|
139
|
+
readonly audio: Uint8Array;
|
|
140
|
+
readonly noiseReduced: boolean;
|
|
141
|
+
readonly confidence: number;
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
export interface VadAudioPacket extends VoicePacket {
|
|
145
|
+
readonly kind: "vad.audio";
|
|
146
|
+
readonly audio: Uint8Array;
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
// Producers: local VAD plugins (silero-vad, pipecat-smart-turn) AND provider STT
|
|
150
|
+
// plugins that surface a native speech-start signal (e.g. Deepgram vad_events
|
|
151
|
+
// SpeechStarted, Deepgram Flux StartOfTurn). Provider-agnostic by design: any STT
|
|
152
|
+
// plugin with an equivalent event should emit this packet so barge-in works on
|
|
153
|
+
// VAD-less deployments. The TurnArbiter treats all producers identically.
|
|
154
|
+
export interface VadSpeechStartedPacket extends VoicePacket {
|
|
155
|
+
readonly kind: "vad.speech_started";
|
|
156
|
+
readonly confidence: number;
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
export interface VadSpeechEndedPacket extends VoicePacket {
|
|
160
|
+
readonly kind: "vad.speech_ended";
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
/** Heartbeat emitted on every audio chunk during active speech. EOS uses this to extend its timer. */
|
|
164
|
+
export interface VadSpeechActivityPacket extends VoicePacket, AsyncPacket {
|
|
165
|
+
readonly kind: "vad.speech_activity";
|
|
166
|
+
readonly isAsync: true;
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
export interface SpeechToTextAudioPacket extends VoicePacket {
|
|
170
|
+
readonly kind: "stt.audio";
|
|
171
|
+
readonly audio: Uint8Array;
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
export interface SttInterimPacket extends VoicePacket {
|
|
175
|
+
readonly kind: "stt.interim";
|
|
176
|
+
readonly text: string;
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
export interface SttResultPacket extends VoicePacket {
|
|
180
|
+
readonly kind: "stt.result";
|
|
181
|
+
readonly text: string;
|
|
182
|
+
readonly confidence: number;
|
|
183
|
+
readonly language?: string;
|
|
184
|
+
readonly provider?: Record<string, unknown>;
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
/** Requests that a streaming STT plugin publish its accumulated final transcript. */
|
|
188
|
+
export interface FinalizeSttPacket extends VoicePacket {
|
|
189
|
+
readonly kind: "stt.finalize";
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
export interface SttErrorPacket extends VoicePacket, VoiceErrorPacket {
|
|
193
|
+
readonly kind: "stt.error";
|
|
194
|
+
readonly component: "stt";
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
export interface EndOfSpeechAudioPacket extends VoicePacket {
|
|
198
|
+
readonly kind: "eos.audio";
|
|
199
|
+
readonly audio: Uint8Array;
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
export interface EndOfSpeechPacket extends VoicePacket {
|
|
203
|
+
readonly kind: "eos.turn_complete";
|
|
204
|
+
readonly text: string;
|
|
205
|
+
/** All accumulated STT transcripts for this turn. */
|
|
206
|
+
readonly transcripts: readonly SttResultPacket[];
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
export interface InterimEndOfSpeechPacket extends VoicePacket {
|
|
210
|
+
readonly kind: "eos.interim";
|
|
211
|
+
readonly text: string;
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
// =============================================================================
|
|
215
|
+
// User Input (processed — feeds LLM)
|
|
216
|
+
// =============================================================================
|
|
217
|
+
|
|
218
|
+
export interface UserInputPacket extends VoicePacket {
|
|
219
|
+
readonly kind: "user.input";
|
|
220
|
+
readonly text: string;
|
|
221
|
+
readonly language: string;
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
// =============================================================================
|
|
225
|
+
// Interruption Packets (flow through Critical route)
|
|
226
|
+
// =============================================================================
|
|
227
|
+
|
|
228
|
+
export type InterruptionSource = "vad" | "word" | "client";
|
|
229
|
+
|
|
230
|
+
export interface InterruptionDetectedPacket extends VoicePacket {
|
|
231
|
+
readonly kind: "interrupt.detected";
|
|
232
|
+
readonly source: InterruptionSource;
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
export interface InterruptTtsPacket extends VoicePacket {
|
|
236
|
+
readonly kind: "interrupt.tts";
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
export interface InterruptLlmPacket extends VoicePacket {
|
|
240
|
+
readonly kind: "interrupt.llm";
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
export interface InterruptSttPacket extends VoicePacket {
|
|
244
|
+
readonly kind: "interrupt.stt";
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
export interface TurnChangePacket extends VoicePacket {
|
|
248
|
+
readonly kind: "turn.change";
|
|
249
|
+
readonly previousContextId: string;
|
|
250
|
+
readonly reason: string;
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
export type DtmfDigit = "0" | "1" | "2" | "3" | "4" | "5" | "6" | "7" | "8" | "9" | "*" | "#";
|
|
254
|
+
|
|
255
|
+
export interface DtmfReceivedPacket extends VoicePacket {
|
|
256
|
+
readonly kind: "dtmf.received";
|
|
257
|
+
readonly digit: DtmfDigit;
|
|
258
|
+
readonly provider: "twilio" | "telnyx" | "smartpbx";
|
|
259
|
+
/** Raw carrier-reported digit string, for diagnostics. */
|
|
260
|
+
readonly rawDigit: string;
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
// =============================================================================
|
|
264
|
+
// LLM Pipeline Packets
|
|
265
|
+
// =============================================================================
|
|
266
|
+
|
|
267
|
+
export interface LlmDeltaPacket extends VoicePacket {
|
|
268
|
+
readonly kind: "llm.delta";
|
|
269
|
+
readonly text: string;
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
export interface LlmResponseDonePacket extends VoicePacket {
|
|
273
|
+
readonly kind: "llm.done";
|
|
274
|
+
readonly text: string;
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
export interface LlmErrorPacket extends VoicePacket, VoiceErrorPacket {
|
|
278
|
+
readonly kind: "llm.error";
|
|
279
|
+
readonly component: "llm" | "bridge";
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
export interface LlmToolCallPacket extends VoicePacket {
|
|
283
|
+
readonly kind: "llm.tool_call";
|
|
284
|
+
readonly toolId: string;
|
|
285
|
+
readonly toolName: string;
|
|
286
|
+
readonly toolArgs: Record<string, unknown>;
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
export interface LlmToolResultPacket extends VoicePacket {
|
|
290
|
+
readonly kind: "llm.tool_result";
|
|
291
|
+
readonly toolId: string;
|
|
292
|
+
readonly toolName: string;
|
|
293
|
+
readonly result: string;
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
export interface ReasoningSuspendedPacket extends VoicePacket {
|
|
297
|
+
readonly kind: "reasoning.suspended";
|
|
298
|
+
readonly runId: string;
|
|
299
|
+
readonly prompt?: string;
|
|
300
|
+
readonly payload: unknown;
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
export interface ReasoningResumePacket extends VoicePacket {
|
|
304
|
+
readonly kind: "reasoning.resume";
|
|
305
|
+
readonly runId: string;
|
|
306
|
+
readonly data: unknown;
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
// =============================================================================
|
|
310
|
+
// Output Pipeline Packets (LLM text → TTS audio)
|
|
311
|
+
// =============================================================================
|
|
312
|
+
|
|
313
|
+
export interface TextToSpeechTextPacket extends VoicePacket {
|
|
314
|
+
readonly kind: "tts.text";
|
|
315
|
+
readonly text: string;
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
export interface TextToSpeechDonePacket extends VoicePacket {
|
|
319
|
+
readonly kind: "tts.done";
|
|
320
|
+
readonly text: string;
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
export interface TextToSpeechAudioPacket extends VoicePacket {
|
|
324
|
+
readonly kind: "tts.audio";
|
|
325
|
+
/** PCM audio bytes (16-bit, mono). */
|
|
326
|
+
readonly audio: Uint8Array;
|
|
327
|
+
/** Source sample rate for the PCM payload. */
|
|
328
|
+
readonly sampleRateHz: number;
|
|
329
|
+
readonly provider?: Record<string, unknown>;
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
export interface TextToSpeechEndPacket extends VoicePacket {
|
|
333
|
+
readonly kind: "tts.end";
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
export interface TtsWordTimestamp {
|
|
337
|
+
readonly word: string;
|
|
338
|
+
/** Milliseconds from the start of audio for this TTS context. */
|
|
339
|
+
readonly startMs: number;
|
|
340
|
+
/** Milliseconds from the start of audio for this TTS context. */
|
|
341
|
+
readonly endMs: number;
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
/**
|
|
345
|
+
* Word-level timestamps for a TTS audio chunk, emitted by TTS plugins that
|
|
346
|
+
* support them (Cartesia, ElevenLabs). Enables the bridge to compute the spoken
|
|
347
|
+
* prefix (G2/G25): the subset of assistant text the user actually heard, used to
|
|
348
|
+
* rewrite history on barge-in at word granularity instead of text granularity.
|
|
349
|
+
* Times are cumulative from the start of the context's audio stream.
|
|
350
|
+
*/
|
|
351
|
+
export interface TextToSpeechWordTimestampsPacket extends VoicePacket {
|
|
352
|
+
readonly kind: "tts.word_timestamps";
|
|
353
|
+
readonly words: readonly TtsWordTimestamp[];
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
export interface TtsErrorPacket extends VoicePacket, VoiceErrorPacket {
|
|
357
|
+
readonly kind: "tts.error";
|
|
358
|
+
readonly component: "tts";
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
/**
|
|
362
|
+
* Realtime playout position for a context, emitted by the output transport's
|
|
363
|
+
* paced-playout layer as audio actually reaches the wire. This is the
|
|
364
|
+
* authoritative playout clock; turn-taking and recording consume it instead of
|
|
365
|
+
* reconstructing timing from generation arrival. Absent when no paced transport
|
|
366
|
+
* is wired (e.g. headless), in which case consumers fall back to a
|
|
367
|
+
* sample-duration estimate.
|
|
368
|
+
*/
|
|
369
|
+
/** First paced audio frame reached the wire for a context (unthrottled). */
|
|
370
|
+
export interface TextToSpeechPlayoutStartedPacket extends VoicePacket {
|
|
371
|
+
readonly kind: "tts.playout_started";
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
export interface TextToSpeechPlayoutProgressPacket extends VoicePacket {
|
|
375
|
+
readonly kind: "tts.playout_progress";
|
|
376
|
+
/** Cumulative realtime audio (ms) paced out to the wire for this context. */
|
|
377
|
+
readonly playedOutMs: number;
|
|
378
|
+
/** True on the final progress for the context — all generated audio has played out. */
|
|
379
|
+
readonly complete: boolean;
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
// =============================================================================
|
|
383
|
+
// Recording Packets
|
|
384
|
+
// =============================================================================
|
|
385
|
+
|
|
386
|
+
export interface RecordUserAudioPacket extends VoicePacket {
|
|
387
|
+
readonly kind: "record.user_audio";
|
|
388
|
+
readonly audio: Uint8Array;
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
export interface RecordAssistantAudioDataPacket extends VoicePacket {
|
|
392
|
+
readonly kind: "record.assistant_audio";
|
|
393
|
+
readonly audio: Uint8Array;
|
|
394
|
+
/** Source sample rate for assistant PCM. */
|
|
395
|
+
readonly sampleRateHz: number;
|
|
396
|
+
readonly truncate: false;
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
export interface RecordAssistantAudioTruncatePacket extends VoicePacket {
|
|
400
|
+
readonly kind: "record.assistant_audio";
|
|
401
|
+
readonly audio: Uint8Array;
|
|
402
|
+
readonly truncate: true;
|
|
403
|
+
}
|
|
404
|
+
|
|
405
|
+
export type RecordAssistantAudioPacket = RecordAssistantAudioDataPacket | RecordAssistantAudioTruncatePacket;
|
|
406
|
+
|
|
407
|
+
// =============================================================================
|
|
408
|
+
// Behavior Packets
|
|
409
|
+
// =============================================================================
|
|
410
|
+
|
|
411
|
+
export interface StartIdleTimeoutPacket extends VoicePacket {
|
|
412
|
+
readonly kind: "behavior.idle_timeout_start";
|
|
413
|
+
}
|
|
414
|
+
|
|
415
|
+
export interface StopIdleTimeoutPacket extends VoicePacket {
|
|
416
|
+
readonly kind: "behavior.idle_timeout_stop";
|
|
417
|
+
readonly resetCount: boolean;
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
export interface InjectMessagePacket extends VoicePacket {
|
|
421
|
+
readonly kind: "inject.message";
|
|
422
|
+
readonly text: string;
|
|
423
|
+
}
|
|
424
|
+
|
|
425
|
+
export interface DisconnectRequestedPacket extends VoicePacket {
|
|
426
|
+
readonly kind: "session.disconnect";
|
|
427
|
+
readonly reason: string;
|
|
428
|
+
}
|
|
429
|
+
|
|
430
|
+
// =============================================================================
|
|
431
|
+
// Mode Switching Packets
|
|
432
|
+
// =============================================================================
|
|
433
|
+
|
|
434
|
+
export interface ModeSwitchRequestedPacket extends VoicePacket {
|
|
435
|
+
readonly kind: "mode.switch_requested";
|
|
436
|
+
readonly mode: "text" | "audio";
|
|
437
|
+
}
|
|
438
|
+
|
|
439
|
+
export interface ModeSwitchCompletedPacket extends VoicePacket {
|
|
440
|
+
readonly kind: "mode.switch_completed";
|
|
441
|
+
readonly mode: "text" | "audio";
|
|
442
|
+
}
|
|
443
|
+
|
|
444
|
+
// =============================================================================
|
|
445
|
+
// Persistence Packets
|
|
446
|
+
// =============================================================================
|
|
447
|
+
|
|
448
|
+
export interface MessageCreatePacket extends VoicePacket {
|
|
449
|
+
readonly kind: "message.create";
|
|
450
|
+
readonly role: "user" | "assistant" | "system";
|
|
451
|
+
readonly text: string;
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
// =============================================================================
|
|
455
|
+
// Metric / Metadata Packets (Background route)
|
|
456
|
+
// =============================================================================
|
|
457
|
+
|
|
458
|
+
export interface ConversationMetricPacket extends VoicePacket {
|
|
459
|
+
readonly kind: "metric.conversation";
|
|
460
|
+
readonly name: string;
|
|
461
|
+
readonly value: string;
|
|
462
|
+
}
|
|
463
|
+
|
|
464
|
+
export type TurnBoundaryKind =
|
|
465
|
+
| "user_started_speaking"
|
|
466
|
+
| "user_stopped_speaking"
|
|
467
|
+
| "agent_thinking"
|
|
468
|
+
| "agent_started_speaking"
|
|
469
|
+
| "agent_audio_done"
|
|
470
|
+
| "interruption";
|
|
471
|
+
|
|
472
|
+
export interface TurnBoundaryEventPacket extends VoicePacket {
|
|
473
|
+
readonly kind: "obs.turn_boundary";
|
|
474
|
+
readonly boundary: TurnBoundaryKind;
|
|
475
|
+
readonly sessionId: string;
|
|
476
|
+
readonly speechId: string;
|
|
477
|
+
readonly requestId?: string;
|
|
478
|
+
/** Monotonic ms (performance.timeOrigin + performance.now()) — immune to wall-clock jumps. */
|
|
479
|
+
readonly monotonicMs: number;
|
|
480
|
+
readonly provider?: string;
|
|
481
|
+
readonly model?: string;
|
|
482
|
+
readonly region?: string;
|
|
483
|
+
readonly cancelled?: boolean;
|
|
484
|
+
}
|
|
485
|
+
|
|
486
|
+
export interface PipelineErrorPacket extends VoicePacket, VoiceErrorPacket {
|
|
487
|
+
readonly kind: "pipeline.error";
|
|
488
|
+
readonly component: "pipeline";
|
|
489
|
+
}
|
|
490
|
+
|
|
491
|
+
// =============================================================================
|
|
492
|
+
// Convenience union types
|
|
493
|
+
// =============================================================================
|
|
494
|
+
|
|
495
|
+
/** All input pipeline packets. */
|
|
496
|
+
export type InputPacket =
|
|
497
|
+
| UserAudioReceivedPacket
|
|
498
|
+
| UserTextReceivedPacket
|
|
499
|
+
| DenoiseAudioPacket
|
|
500
|
+
| DenoisedAudioPacket
|
|
501
|
+
| VadAudioPacket
|
|
502
|
+
| VadSpeechStartedPacket
|
|
503
|
+
| VadSpeechEndedPacket
|
|
504
|
+
| VadSpeechActivityPacket
|
|
505
|
+
| SpeechToTextAudioPacket
|
|
506
|
+
| SttInterimPacket
|
|
507
|
+
| SttResultPacket
|
|
508
|
+
| FinalizeSttPacket
|
|
509
|
+
| SttErrorPacket
|
|
510
|
+
| EndOfSpeechAudioPacket
|
|
511
|
+
| EndOfSpeechPacket
|
|
512
|
+
| InterimEndOfSpeechPacket
|
|
513
|
+
| UserInputPacket;
|
|
514
|
+
|
|
515
|
+
/** All interruption packets (Critical route). */
|
|
516
|
+
export type InterruptPacket =
|
|
517
|
+
| InterruptionDetectedPacket
|
|
518
|
+
| InterruptTtsPacket
|
|
519
|
+
| InterruptLlmPacket
|
|
520
|
+
| InterruptSttPacket
|
|
521
|
+
| TurnChangePacket;
|
|
522
|
+
|
|
523
|
+
/** All LLM output packets. */
|
|
524
|
+
export type LlmPacket =
|
|
525
|
+
| LlmDeltaPacket
|
|
526
|
+
| LlmResponseDonePacket
|
|
527
|
+
| LlmErrorPacket
|
|
528
|
+
| LlmToolCallPacket
|
|
529
|
+
| LlmToolResultPacket
|
|
530
|
+
| ReasoningSuspendedPacket
|
|
531
|
+
| ReasoningResumePacket;
|
|
532
|
+
|
|
533
|
+
/** All TTS output packets. */
|
|
534
|
+
export type TtsPacket =
|
|
535
|
+
| TextToSpeechTextPacket
|
|
536
|
+
| TextToSpeechDonePacket
|
|
537
|
+
| TextToSpeechAudioPacket
|
|
538
|
+
| TextToSpeechEndPacket
|
|
539
|
+
| TextToSpeechPlayoutStartedPacket
|
|
540
|
+
| TextToSpeechPlayoutProgressPacket
|
|
541
|
+
| TextToSpeechWordTimestampsPacket
|
|
542
|
+
| TtsErrorPacket;
|
|
543
|
+
|
|
544
|
+
/** All error packets (any component). */
|
|
545
|
+
export type AnyErrorPacket =
|
|
546
|
+
| SttErrorPacket
|
|
547
|
+
| TtsErrorPacket
|
|
548
|
+
| LlmErrorPacket
|
|
549
|
+
| PipelineErrorPacket
|
|
550
|
+
| InitializationFailedPacket;
|
|
551
|
+
|
|
552
|
+
/** Observability packets (Background route). */
|
|
553
|
+
export type ObservabilityPacket = ConversationMetricPacket | TurnBoundaryEventPacket;
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
// SPDX-License-Identifier: MIT
|
|
2
|
+
//
|
|
3
|
+
// G10 repro — bus head-of-line blocking.
|
|
4
|
+
//
|
|
5
|
+
// The drain loop awaits each sync handler before dequeuing the next batch
|
|
6
|
+
// (pipeline-bus.ts: `await this.dispatch(entry.packet)`), and dispatch awaits
|
|
7
|
+
// sync handlers serially. So a long-running sync Main handler (in production: the
|
|
8
|
+
// AI-SDK bridge running a multi-second LLM generation on `eos.turn_complete`)
|
|
9
|
+
// parks the loop — and a Critical `interrupt.detected` pushed during that window
|
|
10
|
+
// is NOT dispatched until the slow handler returns. That delays barge-in (and
|
|
11
|
+
// defers the llm.delta -> tts.text streaming the slow handler itself produces).
|
|
12
|
+
//
|
|
13
|
+
// These tests assert the DESIRED behavior: Critical packets are dispatched
|
|
14
|
+
// promptly even while a slow Main handler is in flight. They are RED against the
|
|
15
|
+
// current bus and turn GREEN once generation no longer parks the drain loop.
|
|
16
|
+
|
|
17
|
+
import { describe, it, expect } from "vitest";
|
|
18
|
+
import { PipelineBusImpl, Route } from "./pipeline-bus.js";
|
|
19
|
+
import { VoiceAgentSession } from "./voice-agent-session.js";
|
|
20
|
+
import type { PipelineBus } from "./pipeline-bus.js";
|
|
21
|
+
import type { VoicePlugin, PluginConfig } from "./plugin-contract.js";
|
|
22
|
+
import type { VoicePacket } from "./packets.js";
|
|
23
|
+
|
|
24
|
+
function pkt(kind: string, contextId = "t1"): VoicePacket {
|
|
25
|
+
return { kind, contextId, timestampMs: Date.now() } as unknown as VoicePacket;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
describe("G10 — bus head-of-line blocking", () => {
|
|
29
|
+
it("dispatches a Critical interrupt while a slow Main handler is still running", async () => {
|
|
30
|
+
const bus = new PipelineBusImpl();
|
|
31
|
+
const drain = bus.start();
|
|
32
|
+
|
|
33
|
+
let releaseSlow!: () => void;
|
|
34
|
+
const slowGate = new Promise<void>((resolve) => {
|
|
35
|
+
releaseSlow = resolve;
|
|
36
|
+
});
|
|
37
|
+
const timeline: Array<{ name: string; atMs: number }> = [];
|
|
38
|
+
const t0 = Date.now();
|
|
39
|
+
const mark = (name: string): void => {
|
|
40
|
+
timeline.push({ name, atMs: Date.now() - t0 });
|
|
41
|
+
};
|
|
42
|
+
|
|
43
|
+
// Slow Main handler — stands in for the bridge's long LLM generation. Registered
|
|
44
|
+
// as a concurrent producer so it does not park the drain loop.
|
|
45
|
+
bus.on("eos.turn_complete", async () => {
|
|
46
|
+
mark("slow-start");
|
|
47
|
+
await slowGate;
|
|
48
|
+
mark("slow-end");
|
|
49
|
+
}, { concurrent: true });
|
|
50
|
+
// Critical interrupt handler — stands in for barge-in handling.
|
|
51
|
+
bus.on("interrupt.detected", () => {
|
|
52
|
+
mark("interrupt-dispatched");
|
|
53
|
+
});
|
|
54
|
+
|
|
55
|
+
bus.push(Route.Main, pkt("eos.turn_complete"));
|
|
56
|
+
await new Promise((r) => setTimeout(r, 20)); // let the slow handler start
|
|
57
|
+
bus.push(Route.Critical, pkt("interrupt.detected"));
|
|
58
|
+
await new Promise((r) => setTimeout(r, 80)); // window in which a healthy bus dispatches Critical
|
|
59
|
+
|
|
60
|
+
const interruptedWhileSlowRunning = timeline.some((e) => e.name === "interrupt-dispatched");
|
|
61
|
+
|
|
62
|
+
releaseSlow();
|
|
63
|
+
await new Promise((r) => setTimeout(r, 20));
|
|
64
|
+
bus.stop();
|
|
65
|
+
await drain;
|
|
66
|
+
|
|
67
|
+
// The interrupt must have been dispatched BEFORE the slow handler finished.
|
|
68
|
+
const interrupt = timeline.find((e) => e.name === "interrupt-dispatched");
|
|
69
|
+
const slowEnd = timeline.find((e) => e.name === "slow-end");
|
|
70
|
+
expect(interrupt, `timeline: ${JSON.stringify(timeline)}`).toBeDefined();
|
|
71
|
+
expect(slowEnd).toBeDefined();
|
|
72
|
+
expect(interruptedWhileSlowRunning).toBe(true);
|
|
73
|
+
expect(interrupt!.atMs).toBeLessThan(slowEnd!.atMs);
|
|
74
|
+
});
|
|
75
|
+
});
|
|
76
|
+
|
|
77
|
+
// Production simulation: a realistic streaming LLM bridge emits tokens over time on
|
|
78
|
+
// `eos.turn_complete`. The session sentence-buffers them into `tts.text`. With the
|
|
79
|
+
// bus parked on the bridge's (awaited) handler, the deltas it pushes are not
|
|
80
|
+
// dispatched until the handler returns — so the FIRST sentence does not reach TTS
|
|
81
|
+
// until generation completes (streaming is defeated). Desired: the first sentence
|
|
82
|
+
// reaches TTS WHILE generation is still in flight.
|
|
83
|
+
class StreamingBridgePlugin implements VoicePlugin {
|
|
84
|
+
generationEndAtMs = -1;
|
|
85
|
+
constructor(private readonly t0Ms: number) {}
|
|
86
|
+
async initialize(bus: PipelineBus, _config: PluginConfig): Promise<void> {
|
|
87
|
+
bus.on("eos.turn_complete", async (pkt) => {
|
|
88
|
+
const contextId = (pkt as { contextId: string }).contextId;
|
|
89
|
+
// Two complete sentences streamed as tokens, 60 ms apart (~360 ms total).
|
|
90
|
+
const tokens = ["Hello", " there.", " How", " are", " you?", " done."];
|
|
91
|
+
for (const token of tokens) {
|
|
92
|
+
await new Promise((r) => setTimeout(r, 60));
|
|
93
|
+
bus.push(Route.Main, { kind: "llm.delta", contextId, timestampMs: Date.now(), text: token } as unknown as VoicePacket);
|
|
94
|
+
}
|
|
95
|
+
bus.push(Route.Main, { kind: "llm.done", contextId, timestampMs: Date.now(), text: tokens.join("") } as unknown as VoicePacket);
|
|
96
|
+
this.generationEndAtMs = Date.now() - this.t0Ms;
|
|
97
|
+
}, { concurrent: true });
|
|
98
|
+
}
|
|
99
|
+
async close(): Promise<void> {}
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
class RecordingTtsPlugin implements VoicePlugin {
|
|
103
|
+
ttsTextAtMs: number[] = [];
|
|
104
|
+
constructor(private readonly t0Ms: number) {}
|
|
105
|
+
async initialize(bus: PipelineBus, _config: PluginConfig): Promise<void> {
|
|
106
|
+
bus.on("tts.text", () => {
|
|
107
|
+
this.ttsTextAtMs.push(Date.now() - this.t0Ms);
|
|
108
|
+
});
|
|
109
|
+
}
|
|
110
|
+
async close(): Promise<void> {}
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
describe("G10 — production simulation: streaming LLM -> TTS not deferred by the bus", () => {
|
|
114
|
+
it("delivers the first sentence to TTS while generation is still in flight", async () => {
|
|
115
|
+
const t0 = Date.now();
|
|
116
|
+
const bridge = new StreamingBridgePlugin(t0);
|
|
117
|
+
const tts = new RecordingTtsPlugin(t0);
|
|
118
|
+
const session = new VoiceAgentSession({ plugins: { bridge: {}, tts: {} } });
|
|
119
|
+
session.registerPlugin("bridge", bridge);
|
|
120
|
+
session.registerPlugin("tts", tts);
|
|
121
|
+
await session.start();
|
|
122
|
+
|
|
123
|
+
session.bus.push(Route.Main, {
|
|
124
|
+
kind: "eos.turn_complete",
|
|
125
|
+
contextId: "turn-1",
|
|
126
|
+
timestampMs: Date.now(),
|
|
127
|
+
text: "what time is it",
|
|
128
|
+
transcripts: [],
|
|
129
|
+
} as unknown as VoicePacket);
|
|
130
|
+
|
|
131
|
+
// Wait past full generation + drain.
|
|
132
|
+
await new Promise((r) => setTimeout(r, 700));
|
|
133
|
+
if (session.state !== "closed") await session.close();
|
|
134
|
+
|
|
135
|
+
const firstTtsTextAtMs = tts.ttsTextAtMs[0] ?? Number.POSITIVE_INFINITY;
|
|
136
|
+
// The first sentence ("Hello there.") completes ~120 ms into a ~360 ms generation.
|
|
137
|
+
// Bug: it is not delivered until generation ends (>= generationEndAtMs).
|
|
138
|
+
// Desired: delivered while generation is still streaming.
|
|
139
|
+
expect(tts.ttsTextAtMs.length, "expected at least one tts.text").toBeGreaterThan(0);
|
|
140
|
+
expect(
|
|
141
|
+
firstTtsTextAtMs,
|
|
142
|
+
`firstTtsText=${firstTtsTextAtMs}ms generationEnd=${bridge.generationEndAtMs}ms all=${JSON.stringify(tts.ttsTextAtMs)}`,
|
|
143
|
+
).toBeLessThan(bridge.generationEndAtMs);
|
|
144
|
+
});
|
|
145
|
+
});
|