@kuralle-syrinx/deepgram 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/stt.ts ADDED
@@ -0,0 +1,721 @@
1
+ // SPDX-License-Identifier: MIT
2
+ //
3
+ // Syrinx Kernel v2 — Deepgram STT Plugin
4
+ //
5
+ // Follows Rapida's session-long connection pattern:
6
+ // - One WebSocket per session (not per turn)
7
+ // - Audio streams continuously, VAD/EOS requests Deepgram Finalize
8
+ // - KeepAlive holds the session open during post-turn silence and playout
9
+ // - CloseStream is only used when the session is shutting down, never per turn
10
+ // - Interim transcripts for real-time display
11
+ // - Final transcripts can trigger eos.turn_complete, or Pipecat EOS can own finalization
12
+ //
13
+ // Guards (Syrinx additions beyond Rapida):
14
+ // - forceFinalize() sends Deepgram Finalize and waits for provider confirmation
15
+ // - Never promotes interim/cached text to final without speech_final/from_finalize
16
+ // - Emits provider-boundary metrics for audio byte/duration and finalize provenance
17
+ //
18
+ // Reference: Rapida transformer/deepgram/stt.go + internal/stt_callback.go
19
+
20
+ import type { PipelineBus } from "@kuralle-syrinx/core";
21
+ import {
22
+ Route,
23
+ type AudioFormat,
24
+ type VoicePlugin,
25
+ type PluginConfig,
26
+ type SttErrorPacket,
27
+ type VadSpeechStartedPacket,
28
+ assertAudioFormat,
29
+ assertAudioPayload,
30
+ requireStringConfig,
31
+ optionalStringConfig,
32
+ readProviderRetryConfig,
33
+ categorizeSttError,
34
+ isRecoverable,
35
+ } from "@kuralle-syrinx/core";
36
+ import { WebSocketConnection, type SocketData, type SocketFactory } from "@kuralle-syrinx/ws";
37
+
38
+ interface ProviderTranscriptState {
39
+ lastInterimTranscript: string;
40
+ lastInterimConfidence: number;
41
+ finalTranscriptParts: string[];
42
+ finalConfidence: number;
43
+ }
44
+
45
+ export class DeepgramSTTPlugin implements VoicePlugin {
46
+ readonly endpointingCapability = {
47
+ owner: "provider_stt" as const,
48
+ disableConfig: {
49
+ emit_eos_on_final: false,
50
+ finalize_on_speech_final: false,
51
+ },
52
+ };
53
+
54
+ private bus: PipelineBus | null = null;
55
+ private apiKey: string = "";
56
+ private sampleRate: number = 16000;
57
+ private model: string = "nova-3";
58
+ private language: string = "en-US";
59
+ private endpointing: number = 300;
60
+ private endpointUrl: string = "wss://api.deepgram.com/v1/listen";
61
+ private smartFormat: boolean = true;
62
+ private interimResults: boolean = true;
63
+ private vadEvents: boolean = true;
64
+ private confidenceThreshold: number = 0;
65
+ private finalizeOnSpeechFinal: boolean = true;
66
+ private emitEosOnFinal: boolean = true;
67
+ private providerFinalizeTimeoutMs: number = 1200;
68
+ // Consecutive unconfirmed Finalize timeouts that force a connection reset. A single
69
+ // slow finalize discards its turn but keeps the (healthy) socket; only repeated
70
+ // failures with no confirmed final between them look like a wedged stream worth
71
+ // reconnecting. Prevents one slow finalize from cascading into the next turns.
72
+ private finalizeResetThreshold: number = 2;
73
+ private consecutiveFinalizeTimeouts = 0;
74
+ // When the provider never confirms a Finalize, complete the turn with the best transcript
75
+ // already buffered instead of dropping it. For live conversation a reply on slightly
76
+ // imperfect text beats silently losing the user's turn. Opt-in (off preserves the strict
77
+ // "never promote unconfirmed" behavior for callers that need it).
78
+ private finalizeTimeoutFallback: boolean = false;
79
+ private keepAliveIntervalMs: number = 3000;
80
+
81
+ // Session-long WebSocket, managed by the shared connection (reconnect, keepalive).
82
+ private conn: WebSocketConnection | null = null;
83
+ private currentContextId = "";
84
+ private streamStartTime = 0;
85
+ private disposers: Array<() => void> = [];
86
+
87
+ constructor(private readonly socketFactory?: SocketFactory) {}
88
+
89
+ private transcriptStateByContextId = new Map<string, ProviderTranscriptState>();
90
+ private finalizeRequestedContextIds = new Set<string>();
91
+ private finalizedContextIds = new Set<string>();
92
+ private speechFinalContextIds = new Set<string>();
93
+ private ignoreNextProviderFinalContextIds = new Set<string>();
94
+ private providerFinalizeTimers = new Map<string, ReturnType<typeof setTimeout>>();
95
+ private providerFinalizeCorrelationTimers = new Map<string, ReturnType<typeof setTimeout>>();
96
+ private pendingProviderFinalizeContextIds: string[] = [];
97
+ private audioStatsByContextId = new Map<string, {
98
+ bytes: number;
99
+ chunks: number;
100
+ firstSentAtMs: number;
101
+ lastSentAtMs: number;
102
+ }>();
103
+ private audioFormat: AudioFormat = { encoding: "pcm_s16le", sampleRateHz: 16000, channels: 1 };
104
+
105
+ async initialize(bus: PipelineBus, config: PluginConfig): Promise<void> {
106
+ this.bus = bus;
107
+ this.apiKey = requireStringConfig(config, "api_key");
108
+ this.sampleRate = (config["sample_rate"] as number) ?? 16000;
109
+ this.model = optionalStringConfig(config, "model") ?? "nova-3";
110
+ this.language = optionalStringConfig(config, "language") ?? "en-US";
111
+ this.endpointing = (config["endpointing"] as number) ?? 300;
112
+ this.endpointUrl = optionalStringConfig(config, "endpoint_url") ?? "wss://api.deepgram.com/v1/listen";
113
+ this.smartFormat = (config["smart_format"] as boolean) ?? true;
114
+ this.interimResults = (config["interim_results"] as boolean) ?? true;
115
+ // Opt-in: provider SpeechStarted → vad.speech_started is for VAD-less
116
+ // deployments (edge cascade). On sessions with a local VAD the duplicate,
117
+ // laggier speech-start signal corrupts VAD/EOS-owned turn-taking (the turn
118
+ // never completes) — proven by the Fly telephony spike.
119
+ this.vadEvents = (config["vad_events"] as boolean) ?? false;
120
+ this.confidenceThreshold =
121
+ (config["confidence_threshold"] as number) ?? 0;
122
+ this.finalizeOnSpeechFinal = (config["finalize_on_speech_final"] as boolean) ?? true;
123
+ this.emitEosOnFinal = (config["emit_eos_on_final"] as boolean) ?? true;
124
+ this.providerFinalizeTimeoutMs = (config["provider_finalize_timeout_ms"] as number) ?? 1200;
125
+ this.finalizeResetThreshold = (config["finalize_reset_threshold"] as number) ?? 2;
126
+ this.finalizeTimeoutFallback = (config["finalize_timeout_fallback"] as boolean) ?? false;
127
+ this.keepAliveIntervalMs = (config["keep_alive_interval_ms"] as number) ?? 3000;
128
+ this.audioFormat = { encoding: "pcm_s16le", sampleRateHz: this.sampleRate, channels: 1 };
129
+ assertAudioFormat(this.audioFormat);
130
+
131
+ // One session-long socket, managed (reconnect + KeepAlive) by WebSocketConnection.
132
+ this.conn = new WebSocketConnection({
133
+ url: () => {
134
+ const params = new URLSearchParams({
135
+ encoding: "linear16",
136
+ sample_rate: String(this.sampleRate),
137
+ interim_results: String(this.interimResults),
138
+ endpointing: String(this.endpointing),
139
+ smart_format: String(this.smartFormat),
140
+ model: this.model,
141
+ language: this.language,
142
+ channels: "1",
143
+ no_delay: "true",
144
+ vad_events: String(this.vadEvents),
145
+ });
146
+ const separator = this.endpointUrl.includes("?") ? "&" : "?";
147
+ return `${this.endpointUrl}${separator}${params.toString()}`;
148
+ },
149
+ headers: { Authorization: `Token ${this.apiKey}` },
150
+ socketFactory: this.socketFactory ?? await defaultSocketFactory(),
151
+ retry: readProviderRetryConfig(config),
152
+ replayBufferSize: (config["replay_buffer_size"] as number) ?? 64,
153
+ onReplay: (event, count) => {
154
+ this.pushMetric(this.currentContextId, `stt.deepgram.reconnect_replay_${event}`, String(count));
155
+ },
156
+ keepAliveIntervalMs: this.keepAliveIntervalMs,
157
+ keepAliveMessage: () => JSON.stringify({ type: "KeepAlive" }),
158
+ onMessage: (data) => {
159
+ if (typeof data === "string") this.handleProviderMessage(data);
160
+ },
161
+ onConnectionLost: (err) => {
162
+ this.discardProviderStateForReconnect();
163
+ this.emitError(this.currentContextId, err);
164
+ },
165
+ });
166
+ await this.conn.connect();
167
+
168
+ // Listen for audio packets — stream immediately
169
+ this.disposers.push(
170
+ bus.on("stt.audio", async (pkt: unknown) => {
171
+ const audioPkt = pkt as { audio: Uint8Array; contextId?: string };
172
+ if (audioPkt.contextId) {
173
+ this.currentContextId = audioPkt.contextId;
174
+ }
175
+ const sent = await this.sendAudio(audioPkt.audio, this.currentContextId);
176
+ if (sent) {
177
+ this.recordAudioSent(this.currentContextId, audioPkt.audio.byteLength);
178
+ }
179
+ if (this.streamStartTime === 0) {
180
+ this.streamStartTime = Date.now();
181
+ }
182
+ }),
183
+
184
+ // Turn change handler
185
+ bus.on("turn.change", (pkt: unknown) => {
186
+ const tc = pkt as { contextId: string };
187
+ this.currentContextId = tc.contextId;
188
+ this.streamStartTime = Date.now();
189
+ }),
190
+
191
+ // STT interrupt handler
192
+ bus.on("interrupt.stt", () => {
193
+ this.streamStartTime = Date.now();
194
+ this.resetTurnTranscriptState();
195
+ }),
196
+ bus.on("stt.finalize", (pkt: unknown) => {
197
+ const request = pkt as { contextId: string };
198
+ this.forceFinalize(request.contextId);
199
+ }),
200
+ );
201
+ }
202
+
203
+ private handleProviderMessage(data: string): void {
204
+ let msg: Record<string, unknown>;
205
+ try {
206
+ msg = JSON.parse(data) as Record<string, unknown>;
207
+ } catch (err) {
208
+ this.emitError(
209
+ this.currentContextId,
210
+ new Error(`Deepgram STT provider sent malformed JSON: ${err instanceof Error ? err.message : String(err)}`),
211
+ );
212
+ return;
213
+ }
214
+
215
+ if (isDeepgramProviderError(msg)) {
216
+ this.emitError(this.currentContextId, deepgramProviderError(msg));
217
+ return;
218
+ }
219
+
220
+ // Provider speech-start (vad_events=true): the generic speech-start signal —
221
+ // the same packet a local VAD plugin emits — so barge-in works on VAD-less
222
+ // deployments. The session/TurnArbiter owns what to do with it.
223
+ if (msg["type"] === "SpeechStarted") {
224
+ if (!this.vadEvents) return;
225
+ this.bus?.push(Route.Main, {
226
+ kind: "vad.speech_started",
227
+ contextId: this.currentContextId,
228
+ timestampMs: Date.now(),
229
+ confidence: 1,
230
+ } satisfies VadSpeechStartedPacket);
231
+ return;
232
+ }
233
+
234
+ const alt = providerAlternative(msg);
235
+ if (!alt || typeof alt["transcript"] !== "string") return;
236
+
237
+ const transcript = alt["transcript"].trim();
238
+ const confidence = typeof alt["confidence"] === "number" ? alt["confidence"] : 0;
239
+ const fromFinalize = msg["from_finalize"] === true;
240
+ const speechFinal = msg["speech_final"] === true;
241
+ const providerContextId = msg["is_final"] === true
242
+ ? this.contextIdForProviderFinal({ speechFinal, fromFinalize })
243
+ : this.currentContextId;
244
+ if (!transcript || this.finalizedContextIds.has(providerContextId)) return;
245
+
246
+ const state = this.transcriptState(providerContextId);
247
+ state.lastInterimTranscript = transcript;
248
+ state.lastInterimConfidence = confidence;
249
+
250
+ // Confidence threshold filter (Rapida pattern)
251
+ if (
252
+ this.confidenceThreshold > 0 &&
253
+ confidence < this.confidenceThreshold
254
+ ) {
255
+ this.bus?.push(Route.Background, {
256
+ kind: "metric.conversation",
257
+ contextId: providerContextId,
258
+ timestampMs: Date.now(),
259
+ name: "stt_low_confidence",
260
+ value: String(confidence),
261
+ });
262
+ return;
263
+ }
264
+
265
+ if (msg["is_final"] === true) {
266
+ if (this.ignoreNextProviderFinalContextIds.delete(providerContextId)) {
267
+ this.resetPendingTranscript(providerContextId);
268
+ return;
269
+ }
270
+ this.appendFinalSegment(providerContextId, transcript, confidence);
271
+ if (speechFinal) this.speechFinalContextIds.add(providerContextId);
272
+ const finalizeRequested = this.finalizeRequestedContextIds.has(providerContextId);
273
+ this.pushProviderFinalMetric(providerContextId, transcript, {
274
+ confidence,
275
+ speechFinal,
276
+ fromFinalize,
277
+ finalizeRequested,
278
+ });
279
+ this.pushResult(transcript, confidence, providerContextId, {
280
+ name: "deepgram",
281
+ model: this.model,
282
+ region: "global",
283
+ speechFinal,
284
+ fromFinalize,
285
+ finalizeRequested,
286
+ });
287
+ if (speechFinal || fromFinalize) {
288
+ this.resolveProviderFinalize(providerContextId);
289
+ }
290
+ if (this.emitEosOnFinal && ((this.finalizeOnSpeechFinal && speechFinal) || (finalizeRequested && fromFinalize))) {
291
+ this.pushTurnComplete(providerContextId);
292
+ }
293
+ } else {
294
+ this.pushInterim(transcript, providerContextId);
295
+ }
296
+ }
297
+
298
+ /** Stream audio to Deepgram immediately. No batching, no CloseStream. */
299
+ async sendAudio(audio: Uint8Array, contextId = this.currentContextId): Promise<boolean> {
300
+ if (audio.byteLength === 0) return true;
301
+ try {
302
+ assertAudioPayload(this.audioFormat, audio);
303
+ if (!this.conn) throw new Error("Deepgram STT is not connected");
304
+ await this.conn.ensureReady();
305
+ this.conn.send(audio);
306
+ return true;
307
+ } catch (err) {
308
+ this.emitError(contextId, err instanceof Error ? err : new Error(String(err)));
309
+ return false;
310
+ }
311
+ }
312
+
313
+ /** Request that Deepgram flush buffered audio and return provider-final text. */
314
+ forceFinalize(contextId?: string): void {
315
+ const ctxId = contextId ?? this.currentContextId;
316
+ if (!ctxId || !this.bus) return;
317
+ this.requestProviderFinalize(ctxId);
318
+ }
319
+
320
+ private requestProviderFinalize(contextId: string): void {
321
+ if (this.finalizedContextIds.has(contextId)) return;
322
+ this.finalizeRequestedContextIds.add(contextId);
323
+ this.trackPendingProviderFinalize(contextId);
324
+ this.pushMetric(contextId, "stt_provider_finalize_requested", this.audioStats(contextId));
325
+ if (this.conn?.isReady) {
326
+ this.conn.send(JSON.stringify({ type: "Finalize" }));
327
+ }
328
+ if (!this.emitEosOnFinal && this.hasFinalTranscript(contextId)) {
329
+ this.scheduleProviderFinalizeCorrelationExpiry(contextId);
330
+ return;
331
+ }
332
+
333
+ this.clearProviderFinalizeTimer(contextId);
334
+ if (this.providerFinalizeTimeoutMs <= 0) return;
335
+ const timer = setTimeout(() => {
336
+ this.providerFinalizeTimers.delete(contextId);
337
+ this.handleProviderFinalizeTimeout(contextId);
338
+ }, this.providerFinalizeTimeoutMs);
339
+ this.providerFinalizeTimers.set(contextId, timer);
340
+ }
341
+
342
+ private handleProviderFinalizeTimeout(contextId: string): void {
343
+ if (!this.finalizeRequestedContextIds.has(contextId) || this.finalizedContextIds.has(contextId)) return;
344
+ this.pushMetric(contextId, "stt_provider_finalize_timeout", this.audioStats(contextId));
345
+
346
+ // Graceful degradation (opt-in): rather than dropping the user's turn when the provider
347
+ // never confirms the Finalize, complete it with the best buffered text — confirmed
348
+ // is_final segments first, then the latest interim — so the turn still reaches the LLM.
349
+ if (this.finalizeTimeoutFallback) {
350
+ const state = this.transcriptState(contextId);
351
+ const fallbackText = this.combinedFinalTranscript(contextId) || state.lastInterimTranscript;
352
+ if (fallbackText) {
353
+ this.pushMetric(contextId, "stt_provider_finalize_timeout_fallback", this.audioStats(contextId));
354
+ // A late provider-final for this turn must not double-emit after we promote here.
355
+ this.ignoreNextProviderFinalContextIds.add(contextId);
356
+ this.pushFinal(fallbackText, state.finalConfidence || state.lastInterimConfidence, contextId);
357
+ this.resetPendingTranscript(contextId);
358
+ return;
359
+ }
360
+ // No buffered text — a silence-only/orphan turn (e.g. an always-on client that rotated its
361
+ // contextId onto trailing silence). Discard it silently; a finalize timeout on a turn that
362
+ // never had speech is not an error worth surfacing.
363
+ this.pushMetric(contextId, "stt_provider_finalize_timeout_empty_discard", this.audioStats(contextId));
364
+ this.discardUnconfirmedTurn(contextId);
365
+ return;
366
+ }
367
+
368
+ this.discardUnconfirmedTurn(contextId);
369
+ this.emitError(
370
+ contextId,
371
+ new Error("Deepgram STT Finalize timed out before speech_final/from_finalize confirmation"),
372
+ );
373
+ // A single slow finalize is not a wedged stream: keep the healthy socket so the
374
+ // next turn streams normally instead of stalling on a reconnect (which loses
375
+ // Deepgram-side context and cascades into more timeouts). Only reconnect once the
376
+ // failures repeat without a confirmed final between them (counter cleared in pushFinal).
377
+ this.consecutiveFinalizeTimeouts += 1;
378
+ if (this.consecutiveFinalizeTimeouts >= this.finalizeResetThreshold) {
379
+ this.consecutiveFinalizeTimeouts = 0;
380
+ this.conn?.reset();
381
+ }
382
+ }
383
+
384
+ private discardUnconfirmedTurn(contextId: string): void {
385
+ this.clearProviderFinalizeTimer(contextId);
386
+ this.finalizeRequestedContextIds.delete(contextId);
387
+ this.removePendingProviderFinalize(contextId);
388
+ this.speechFinalContextIds.delete(contextId);
389
+ this.ignoreNextProviderFinalContextIds.add(contextId);
390
+ this.audioStatsByContextId.delete(contextId);
391
+ this.resetPendingTranscript(contextId);
392
+ }
393
+
394
+ /** Emit final transcript + EOS turn complete. */
395
+ private pushFinal(transcript: string, confidence: number, contextId = this.currentContextId): void {
396
+ const ctxId = contextId;
397
+ this.resolveProviderFinalize(ctxId);
398
+ this.finalizedContextIds.add(ctxId);
399
+ this.consecutiveFinalizeTimeouts = 0;
400
+ this.pushMetric(ctxId, "stt_audio_sent", this.audioStats(ctxId));
401
+ this.pushResult(transcript, confidence, ctxId);
402
+
403
+ if (this.emitEosOnFinal) {
404
+ this.bus?.push(Route.Main, {
405
+ kind: "eos.turn_complete",
406
+ contextId: ctxId,
407
+ timestampMs: Date.now(),
408
+ text: transcript,
409
+ transcripts: [],
410
+ });
411
+ }
412
+ this.audioStatsByContextId.delete(ctxId);
413
+ }
414
+
415
+ private pushTurnComplete(contextId: string): void {
416
+ const transcript = this.combinedFinalTranscript(contextId);
417
+ if (!transcript || !this.bus) return;
418
+ this.resolveProviderFinalize(contextId);
419
+ this.finalizedContextIds.add(contextId);
420
+ this.consecutiveFinalizeTimeouts = 0;
421
+ this.pushMetric(contextId, "stt_audio_sent", this.audioStats(contextId));
422
+ this.bus.push(Route.Main, {
423
+ kind: "eos.turn_complete",
424
+ contextId,
425
+ timestampMs: Date.now(),
426
+ text: transcript,
427
+ transcripts: [],
428
+ });
429
+ this.audioStatsByContextId.delete(contextId);
430
+ this.resetPendingTranscript(contextId);
431
+ }
432
+
433
+ private pushResult(
434
+ transcript: string,
435
+ confidence: number,
436
+ contextId = this.currentContextId,
437
+ provider: Record<string, unknown> = { name: "deepgram", model: this.model, region: "global" },
438
+ ): void {
439
+ this.bus?.push(Route.Main, {
440
+ kind: "stt.result",
441
+ contextId,
442
+ timestampMs: Date.now(),
443
+ text: transcript,
444
+ confidence,
445
+ language: this.language,
446
+ provider,
447
+ });
448
+ }
449
+
450
+ /** Emit interim transcript for real-time display. */
451
+ private pushInterim(transcript: string, contextId = this.currentContextId): void {
452
+ this.bus?.push(Route.Main, {
453
+ kind: "stt.interim",
454
+ contextId,
455
+ timestampMs: Date.now(),
456
+ text: transcript,
457
+ });
458
+ }
459
+
460
+ private appendFinalSegment(contextId: string, transcript: string, confidence: number): void {
461
+ if (transcript.length === 0) return;
462
+ const state = this.transcriptState(contextId);
463
+ const last = state.finalTranscriptParts.at(-1);
464
+ if (last !== transcript) {
465
+ state.finalTranscriptParts.push(transcript);
466
+ }
467
+ state.finalConfidence = Math.max(state.finalConfidence, confidence);
468
+ }
469
+
470
+ private combinedFinalTranscript(contextId: string): string {
471
+ return this.transcriptState(contextId).finalTranscriptParts.join(" ").replace(/\s+/g, " ").trim();
472
+ }
473
+
474
+ private resetPendingTranscript(contextId: string): void {
475
+ this.transcriptStateByContextId.delete(contextId);
476
+ }
477
+
478
+ private resetTurnTranscriptState(): void {
479
+ this.resetPendingTranscript(this.currentContextId);
480
+ }
481
+
482
+ private transcriptState(contextId: string): ProviderTranscriptState {
483
+ const existing = this.transcriptStateByContextId.get(contextId);
484
+ if (existing) return existing;
485
+ const next: ProviderTranscriptState = {
486
+ lastInterimTranscript: "",
487
+ lastInterimConfidence: 0,
488
+ finalTranscriptParts: [],
489
+ finalConfidence: 0,
490
+ };
491
+ this.transcriptStateByContextId.set(contextId, next);
492
+ return next;
493
+ }
494
+
495
+ private hasFinalTranscript(contextId: string): boolean {
496
+ const state = this.transcriptStateByContextId.get(contextId);
497
+ return Boolean(state && state.finalTranscriptParts.length > 0);
498
+ }
499
+
500
+ private contextIdForProviderFinal(flags: { readonly speechFinal: boolean; readonly fromFinalize: boolean }): string {
501
+ const pending = this.pendingProviderFinalizeContextIds[0];
502
+ if (pending && (flags.speechFinal || flags.fromFinalize)) return pending;
503
+ return this.currentContextId;
504
+ }
505
+
506
+ private trackPendingProviderFinalize(contextId: string): void {
507
+ if (!this.pendingProviderFinalizeContextIds.includes(contextId)) {
508
+ this.pendingProviderFinalizeContextIds.push(contextId);
509
+ }
510
+ }
511
+
512
+ private removePendingProviderFinalize(contextId: string): void {
513
+ this.pendingProviderFinalizeContextIds = this.pendingProviderFinalizeContextIds.filter((ctxId) => ctxId !== contextId);
514
+ }
515
+
516
+ private scheduleProviderFinalizeCorrelationExpiry(contextId: string): void {
517
+ this.clearProviderFinalizeCorrelationTimer(contextId);
518
+ if (this.providerFinalizeTimeoutMs <= 0) return;
519
+ const timer = setTimeout(() => {
520
+ this.providerFinalizeCorrelationTimers.delete(contextId);
521
+ this.finalizeRequestedContextIds.delete(contextId);
522
+ this.removePendingProviderFinalize(contextId);
523
+ }, this.providerFinalizeTimeoutMs);
524
+ this.providerFinalizeCorrelationTimers.set(contextId, timer);
525
+ }
526
+
527
+ private resolveProviderFinalize(contextId: string): void {
528
+ this.finalizeRequestedContextIds.delete(contextId);
529
+ this.clearProviderFinalizeTimer(contextId);
530
+ this.clearProviderFinalizeCorrelationTimer(contextId);
531
+ this.removePendingProviderFinalize(contextId);
532
+ }
533
+
534
+ private recordAudioSent(contextId: string, byteLength: number): void {
535
+ if (!contextId) return;
536
+ const now = Date.now();
537
+ const current = this.audioStatsByContextId.get(contextId) ?? {
538
+ bytes: 0,
539
+ chunks: 0,
540
+ firstSentAtMs: now,
541
+ lastSentAtMs: now,
542
+ };
543
+ current.bytes += byteLength;
544
+ current.chunks += 1;
545
+ current.lastSentAtMs = now;
546
+ this.audioStatsByContextId.set(contextId, current);
547
+ }
548
+
549
+ private pushProviderFinalMetric(
550
+ contextId: string,
551
+ transcript: string,
552
+ flags: {
553
+ readonly confidence: number;
554
+ readonly speechFinal: boolean;
555
+ readonly fromFinalize: boolean;
556
+ readonly finalizeRequested: boolean;
557
+ },
558
+ ): void {
559
+ this.pushMetric(contextId, "stt_provider_final_segment", {
560
+ ...this.audioStats(contextId),
561
+ transcriptChars: transcript.length,
562
+ confidence: flags.confidence,
563
+ speechFinal: flags.speechFinal,
564
+ fromFinalize: flags.fromFinalize,
565
+ finalizeRequested: flags.finalizeRequested,
566
+ });
567
+ }
568
+
569
+ private audioStats(contextId: string): Record<string, number> {
570
+ const stats = this.audioStatsByContextId.get(contextId);
571
+ if (!stats) {
572
+ return {
573
+ bytes: 0,
574
+ chunks: 0,
575
+ durationMs: 0,
576
+ wallClockMs: 0,
577
+ };
578
+ }
579
+ return {
580
+ bytes: stats.bytes,
581
+ chunks: stats.chunks,
582
+ durationMs: Math.round((stats.bytes / 2 / this.sampleRate) * 1000),
583
+ wallClockMs: stats.lastSentAtMs - stats.firstSentAtMs,
584
+ };
585
+ }
586
+
587
+ private pushMetric(contextId: string, name: string, value: unknown): void {
588
+ this.bus?.push(Route.Background, {
589
+ kind: "metric.conversation",
590
+ contextId,
591
+ timestampMs: Date.now(),
592
+ name,
593
+ value: typeof value === "string" ? value : JSON.stringify(value),
594
+ });
595
+ }
596
+
597
+ private emitError(contextId: string, err: Error): void {
598
+ const category = categorizeSttError(err);
599
+ const packet: SttErrorPacket = {
600
+ kind: "stt.error",
601
+ contextId,
602
+ timestampMs: Date.now(),
603
+ component: "stt" as const,
604
+ category,
605
+ cause: err,
606
+ isRecoverable: isRecoverable(category),
607
+ };
608
+ this.bus?.push(Route.Critical, packet);
609
+ }
610
+
611
+ async close(): Promise<void> {
612
+ for (const dispose of this.disposers.splice(0)) dispose();
613
+ if (this.transcriptStateByContextId.size > 0) {
614
+ this.pushMetric(this.currentContextId, "stt_pending_transcript_discarded_on_close", this.audioStats(this.currentContextId));
615
+ }
616
+ for (const timer of this.providerFinalizeTimers.values()) clearTimeout(timer);
617
+ this.providerFinalizeTimers.clear();
618
+ for (const timer of this.providerFinalizeCorrelationTimers.values()) clearTimeout(timer);
619
+ this.providerFinalizeCorrelationTimers.clear();
620
+ this.pendingProviderFinalizeContextIds = [];
621
+ this.finalizeRequestedContextIds.clear();
622
+ this.finalizedContextIds.clear();
623
+ this.speechFinalContextIds.clear();
624
+ this.ignoreNextProviderFinalContextIds.clear();
625
+ this.transcriptStateByContextId.clear();
626
+ this.audioStatsByContextId.clear();
627
+
628
+ if (this.conn) {
629
+ // Graceful end-of-stream so Deepgram flushes and closes cleanly.
630
+ if (this.conn.isReady) {
631
+ try {
632
+ this.conn.send(JSON.stringify({ type: "CloseStream" }));
633
+ } catch {
634
+ // best effort
635
+ }
636
+ }
637
+ await this.conn.close();
638
+ this.conn = null;
639
+ }
640
+ this.bus = null;
641
+ }
642
+
643
+ private clearProviderFinalizeTimer(contextId: string): void {
644
+ const timer = this.providerFinalizeTimers.get(contextId);
645
+ if (!timer) return;
646
+ clearTimeout(timer);
647
+ this.providerFinalizeTimers.delete(contextId);
648
+ }
649
+
650
+ private clearProviderFinalizeCorrelationTimer(contextId: string): void {
651
+ const timer = this.providerFinalizeCorrelationTimers.get(contextId);
652
+ if (!timer) return;
653
+ clearTimeout(timer);
654
+ this.providerFinalizeCorrelationTimers.delete(contextId);
655
+ }
656
+
657
+ private discardProviderStateForReconnect(): void {
658
+ const contextId = this.currentContextId;
659
+ const discarded = this.transcriptStateByContextId.size > 0 ||
660
+ this.finalizeRequestedContextIds.size > 0 ||
661
+ this.audioStatsByContextId.size > 0 ||
662
+ this.providerFinalizeTimers.size > 0 ||
663
+ this.providerFinalizeCorrelationTimers.size > 0;
664
+ for (const timer of this.providerFinalizeTimers.values()) clearTimeout(timer);
665
+ this.providerFinalizeTimers.clear();
666
+ for (const timer of this.providerFinalizeCorrelationTimers.values()) clearTimeout(timer);
667
+ this.providerFinalizeCorrelationTimers.clear();
668
+ this.pendingProviderFinalizeContextIds = [];
669
+ this.finalizeRequestedContextIds.clear();
670
+ this.speechFinalContextIds.clear();
671
+ this.ignoreNextProviderFinalContextIds.clear();
672
+ this.audioStatsByContextId.clear();
673
+ // A reconnect (from any cause) starts a fresh provider stream, so the wedged-stream
674
+ // signal resets too — otherwise a stale count could force an avoidable reset on the
675
+ // first timeout after reconnecting.
676
+ this.consecutiveFinalizeTimeouts = 0;
677
+ this.transcriptStateByContextId.clear();
678
+ if (discarded && contextId) {
679
+ this.pushMetric(contextId, "stt_provider_reconnect_discarded_state", {});
680
+ }
681
+ }
682
+
683
+ }
684
+
685
+ async function defaultSocketFactory(): Promise<SocketFactory> {
686
+ const mod = await import("@kuralle-syrinx/ws/node");
687
+ return mod.createNodeWsSocket;
688
+ }
689
+
690
+ function providerAlternative(msg: Record<string, unknown>): Record<string, unknown> | null {
691
+ const channel = msg["channel"];
692
+ if (!channel || typeof channel !== "object") return null;
693
+ const alternatives = (channel as { alternatives?: unknown }).alternatives;
694
+ if (!Array.isArray(alternatives)) return null;
695
+ const first = alternatives[0];
696
+ return first && typeof first === "object" ? first as Record<string, unknown> : null;
697
+ }
698
+
699
+ function isDeepgramProviderError(msg: Record<string, unknown>): boolean {
700
+ const type = typeof msg["type"] === "string" ? msg["type"].toLowerCase() : "";
701
+ return type === "error" || typeof msg["err_code"] === "string" || typeof msg["err_msg"] === "string";
702
+ }
703
+
704
+ function deepgramProviderError(msg: Record<string, unknown>): Error {
705
+ const code = firstString(msg["code"], msg["err_code"]);
706
+ const description = firstString(msg["description"], msg["message"], msg["err_msg"], msg["details"]);
707
+ const requestId = firstString(msg["request_id"]);
708
+ const details = [
709
+ code ? `code=${code}` : "",
710
+ description,
711
+ requestId ? `request_id=${requestId}` : "",
712
+ ].filter((part) => part.length > 0).join(" ");
713
+ return new Error(details ? `Deepgram STT provider error: ${details}` : "Deepgram STT provider error");
714
+ }
715
+
716
+ function firstString(...values: unknown[]): string {
717
+ for (const value of values) {
718
+ if (typeof value === "string" && value.length > 0) return value;
719
+ }
720
+ return "";
721
+ }