@kuralle-syrinx/deepgram 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.handoff/finalize-contract-probe.test.ts +193 -0
- package/LICENSE +22 -0
- package/package.json +28 -0
- package/src/index.ts +7 -0
- package/src/stt.test.ts +1153 -0
- package/src/stt.ts +721 -0
- package/src/tts.test.ts +359 -0
- package/src/tts.ts +359 -0
- package/tsconfig.json +21 -0
package/src/stt.ts
ADDED
|
@@ -0,0 +1,721 @@
|
|
|
1
|
+
// SPDX-License-Identifier: MIT
|
|
2
|
+
//
|
|
3
|
+
// Syrinx Kernel v2 — Deepgram STT Plugin
|
|
4
|
+
//
|
|
5
|
+
// Follows Rapida's session-long connection pattern:
|
|
6
|
+
// - One WebSocket per session (not per turn)
|
|
7
|
+
// - Audio streams continuously, VAD/EOS requests Deepgram Finalize
|
|
8
|
+
// - KeepAlive holds the session open during post-turn silence and playout
|
|
9
|
+
// - CloseStream is only used when the session is shutting down, never per turn
|
|
10
|
+
// - Interim transcripts for real-time display
|
|
11
|
+
// - Final transcripts can trigger eos.turn_complete, or Pipecat EOS can own finalization
|
|
12
|
+
//
|
|
13
|
+
// Guards (Syrinx additions beyond Rapida):
|
|
14
|
+
// - forceFinalize() sends Deepgram Finalize and waits for provider confirmation
|
|
15
|
+
// - Never promotes interim/cached text to final without speech_final/from_finalize
|
|
16
|
+
// - Emits provider-boundary metrics for audio byte/duration and finalize provenance
|
|
17
|
+
//
|
|
18
|
+
// Reference: Rapida transformer/deepgram/stt.go + internal/stt_callback.go
|
|
19
|
+
|
|
20
|
+
import type { PipelineBus } from "@kuralle-syrinx/core";
|
|
21
|
+
import {
|
|
22
|
+
Route,
|
|
23
|
+
type AudioFormat,
|
|
24
|
+
type VoicePlugin,
|
|
25
|
+
type PluginConfig,
|
|
26
|
+
type SttErrorPacket,
|
|
27
|
+
type VadSpeechStartedPacket,
|
|
28
|
+
assertAudioFormat,
|
|
29
|
+
assertAudioPayload,
|
|
30
|
+
requireStringConfig,
|
|
31
|
+
optionalStringConfig,
|
|
32
|
+
readProviderRetryConfig,
|
|
33
|
+
categorizeSttError,
|
|
34
|
+
isRecoverable,
|
|
35
|
+
} from "@kuralle-syrinx/core";
|
|
36
|
+
import { WebSocketConnection, type SocketData, type SocketFactory } from "@kuralle-syrinx/ws";
|
|
37
|
+
|
|
38
|
+
interface ProviderTranscriptState {
|
|
39
|
+
lastInterimTranscript: string;
|
|
40
|
+
lastInterimConfidence: number;
|
|
41
|
+
finalTranscriptParts: string[];
|
|
42
|
+
finalConfidence: number;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export class DeepgramSTTPlugin implements VoicePlugin {
|
|
46
|
+
readonly endpointingCapability = {
|
|
47
|
+
owner: "provider_stt" as const,
|
|
48
|
+
disableConfig: {
|
|
49
|
+
emit_eos_on_final: false,
|
|
50
|
+
finalize_on_speech_final: false,
|
|
51
|
+
},
|
|
52
|
+
};
|
|
53
|
+
|
|
54
|
+
private bus: PipelineBus | null = null;
|
|
55
|
+
private apiKey: string = "";
|
|
56
|
+
private sampleRate: number = 16000;
|
|
57
|
+
private model: string = "nova-3";
|
|
58
|
+
private language: string = "en-US";
|
|
59
|
+
private endpointing: number = 300;
|
|
60
|
+
private endpointUrl: string = "wss://api.deepgram.com/v1/listen";
|
|
61
|
+
private smartFormat: boolean = true;
|
|
62
|
+
private interimResults: boolean = true;
|
|
63
|
+
private vadEvents: boolean = true;
|
|
64
|
+
private confidenceThreshold: number = 0;
|
|
65
|
+
private finalizeOnSpeechFinal: boolean = true;
|
|
66
|
+
private emitEosOnFinal: boolean = true;
|
|
67
|
+
private providerFinalizeTimeoutMs: number = 1200;
|
|
68
|
+
// Consecutive unconfirmed Finalize timeouts that force a connection reset. A single
|
|
69
|
+
// slow finalize discards its turn but keeps the (healthy) socket; only repeated
|
|
70
|
+
// failures with no confirmed final between them look like a wedged stream worth
|
|
71
|
+
// reconnecting. Prevents one slow finalize from cascading into the next turns.
|
|
72
|
+
private finalizeResetThreshold: number = 2;
|
|
73
|
+
private consecutiveFinalizeTimeouts = 0;
|
|
74
|
+
// When the provider never confirms a Finalize, complete the turn with the best transcript
|
|
75
|
+
// already buffered instead of dropping it. For live conversation a reply on slightly
|
|
76
|
+
// imperfect text beats silently losing the user's turn. Opt-in (off preserves the strict
|
|
77
|
+
// "never promote unconfirmed" behavior for callers that need it).
|
|
78
|
+
private finalizeTimeoutFallback: boolean = false;
|
|
79
|
+
private keepAliveIntervalMs: number = 3000;
|
|
80
|
+
|
|
81
|
+
// Session-long WebSocket, managed by the shared connection (reconnect, keepalive).
|
|
82
|
+
private conn: WebSocketConnection | null = null;
|
|
83
|
+
private currentContextId = "";
|
|
84
|
+
private streamStartTime = 0;
|
|
85
|
+
private disposers: Array<() => void> = [];
|
|
86
|
+
|
|
87
|
+
constructor(private readonly socketFactory?: SocketFactory) {}
|
|
88
|
+
|
|
89
|
+
private transcriptStateByContextId = new Map<string, ProviderTranscriptState>();
|
|
90
|
+
private finalizeRequestedContextIds = new Set<string>();
|
|
91
|
+
private finalizedContextIds = new Set<string>();
|
|
92
|
+
private speechFinalContextIds = new Set<string>();
|
|
93
|
+
private ignoreNextProviderFinalContextIds = new Set<string>();
|
|
94
|
+
private providerFinalizeTimers = new Map<string, ReturnType<typeof setTimeout>>();
|
|
95
|
+
private providerFinalizeCorrelationTimers = new Map<string, ReturnType<typeof setTimeout>>();
|
|
96
|
+
private pendingProviderFinalizeContextIds: string[] = [];
|
|
97
|
+
private audioStatsByContextId = new Map<string, {
|
|
98
|
+
bytes: number;
|
|
99
|
+
chunks: number;
|
|
100
|
+
firstSentAtMs: number;
|
|
101
|
+
lastSentAtMs: number;
|
|
102
|
+
}>();
|
|
103
|
+
private audioFormat: AudioFormat = { encoding: "pcm_s16le", sampleRateHz: 16000, channels: 1 };
|
|
104
|
+
|
|
105
|
+
async initialize(bus: PipelineBus, config: PluginConfig): Promise<void> {
|
|
106
|
+
this.bus = bus;
|
|
107
|
+
this.apiKey = requireStringConfig(config, "api_key");
|
|
108
|
+
this.sampleRate = (config["sample_rate"] as number) ?? 16000;
|
|
109
|
+
this.model = optionalStringConfig(config, "model") ?? "nova-3";
|
|
110
|
+
this.language = optionalStringConfig(config, "language") ?? "en-US";
|
|
111
|
+
this.endpointing = (config["endpointing"] as number) ?? 300;
|
|
112
|
+
this.endpointUrl = optionalStringConfig(config, "endpoint_url") ?? "wss://api.deepgram.com/v1/listen";
|
|
113
|
+
this.smartFormat = (config["smart_format"] as boolean) ?? true;
|
|
114
|
+
this.interimResults = (config["interim_results"] as boolean) ?? true;
|
|
115
|
+
// Opt-in: provider SpeechStarted → vad.speech_started is for VAD-less
|
|
116
|
+
// deployments (edge cascade). On sessions with a local VAD the duplicate,
|
|
117
|
+
// laggier speech-start signal corrupts VAD/EOS-owned turn-taking (the turn
|
|
118
|
+
// never completes) — proven by the Fly telephony spike.
|
|
119
|
+
this.vadEvents = (config["vad_events"] as boolean) ?? false;
|
|
120
|
+
this.confidenceThreshold =
|
|
121
|
+
(config["confidence_threshold"] as number) ?? 0;
|
|
122
|
+
this.finalizeOnSpeechFinal = (config["finalize_on_speech_final"] as boolean) ?? true;
|
|
123
|
+
this.emitEosOnFinal = (config["emit_eos_on_final"] as boolean) ?? true;
|
|
124
|
+
this.providerFinalizeTimeoutMs = (config["provider_finalize_timeout_ms"] as number) ?? 1200;
|
|
125
|
+
this.finalizeResetThreshold = (config["finalize_reset_threshold"] as number) ?? 2;
|
|
126
|
+
this.finalizeTimeoutFallback = (config["finalize_timeout_fallback"] as boolean) ?? false;
|
|
127
|
+
this.keepAliveIntervalMs = (config["keep_alive_interval_ms"] as number) ?? 3000;
|
|
128
|
+
this.audioFormat = { encoding: "pcm_s16le", sampleRateHz: this.sampleRate, channels: 1 };
|
|
129
|
+
assertAudioFormat(this.audioFormat);
|
|
130
|
+
|
|
131
|
+
// One session-long socket, managed (reconnect + KeepAlive) by WebSocketConnection.
|
|
132
|
+
this.conn = new WebSocketConnection({
|
|
133
|
+
url: () => {
|
|
134
|
+
const params = new URLSearchParams({
|
|
135
|
+
encoding: "linear16",
|
|
136
|
+
sample_rate: String(this.sampleRate),
|
|
137
|
+
interim_results: String(this.interimResults),
|
|
138
|
+
endpointing: String(this.endpointing),
|
|
139
|
+
smart_format: String(this.smartFormat),
|
|
140
|
+
model: this.model,
|
|
141
|
+
language: this.language,
|
|
142
|
+
channels: "1",
|
|
143
|
+
no_delay: "true",
|
|
144
|
+
vad_events: String(this.vadEvents),
|
|
145
|
+
});
|
|
146
|
+
const separator = this.endpointUrl.includes("?") ? "&" : "?";
|
|
147
|
+
return `${this.endpointUrl}${separator}${params.toString()}`;
|
|
148
|
+
},
|
|
149
|
+
headers: { Authorization: `Token ${this.apiKey}` },
|
|
150
|
+
socketFactory: this.socketFactory ?? await defaultSocketFactory(),
|
|
151
|
+
retry: readProviderRetryConfig(config),
|
|
152
|
+
replayBufferSize: (config["replay_buffer_size"] as number) ?? 64,
|
|
153
|
+
onReplay: (event, count) => {
|
|
154
|
+
this.pushMetric(this.currentContextId, `stt.deepgram.reconnect_replay_${event}`, String(count));
|
|
155
|
+
},
|
|
156
|
+
keepAliveIntervalMs: this.keepAliveIntervalMs,
|
|
157
|
+
keepAliveMessage: () => JSON.stringify({ type: "KeepAlive" }),
|
|
158
|
+
onMessage: (data) => {
|
|
159
|
+
if (typeof data === "string") this.handleProviderMessage(data);
|
|
160
|
+
},
|
|
161
|
+
onConnectionLost: (err) => {
|
|
162
|
+
this.discardProviderStateForReconnect();
|
|
163
|
+
this.emitError(this.currentContextId, err);
|
|
164
|
+
},
|
|
165
|
+
});
|
|
166
|
+
await this.conn.connect();
|
|
167
|
+
|
|
168
|
+
// Listen for audio packets — stream immediately
|
|
169
|
+
this.disposers.push(
|
|
170
|
+
bus.on("stt.audio", async (pkt: unknown) => {
|
|
171
|
+
const audioPkt = pkt as { audio: Uint8Array; contextId?: string };
|
|
172
|
+
if (audioPkt.contextId) {
|
|
173
|
+
this.currentContextId = audioPkt.contextId;
|
|
174
|
+
}
|
|
175
|
+
const sent = await this.sendAudio(audioPkt.audio, this.currentContextId);
|
|
176
|
+
if (sent) {
|
|
177
|
+
this.recordAudioSent(this.currentContextId, audioPkt.audio.byteLength);
|
|
178
|
+
}
|
|
179
|
+
if (this.streamStartTime === 0) {
|
|
180
|
+
this.streamStartTime = Date.now();
|
|
181
|
+
}
|
|
182
|
+
}),
|
|
183
|
+
|
|
184
|
+
// Turn change handler
|
|
185
|
+
bus.on("turn.change", (pkt: unknown) => {
|
|
186
|
+
const tc = pkt as { contextId: string };
|
|
187
|
+
this.currentContextId = tc.contextId;
|
|
188
|
+
this.streamStartTime = Date.now();
|
|
189
|
+
}),
|
|
190
|
+
|
|
191
|
+
// STT interrupt handler
|
|
192
|
+
bus.on("interrupt.stt", () => {
|
|
193
|
+
this.streamStartTime = Date.now();
|
|
194
|
+
this.resetTurnTranscriptState();
|
|
195
|
+
}),
|
|
196
|
+
bus.on("stt.finalize", (pkt: unknown) => {
|
|
197
|
+
const request = pkt as { contextId: string };
|
|
198
|
+
this.forceFinalize(request.contextId);
|
|
199
|
+
}),
|
|
200
|
+
);
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
private handleProviderMessage(data: string): void {
|
|
204
|
+
let msg: Record<string, unknown>;
|
|
205
|
+
try {
|
|
206
|
+
msg = JSON.parse(data) as Record<string, unknown>;
|
|
207
|
+
} catch (err) {
|
|
208
|
+
this.emitError(
|
|
209
|
+
this.currentContextId,
|
|
210
|
+
new Error(`Deepgram STT provider sent malformed JSON: ${err instanceof Error ? err.message : String(err)}`),
|
|
211
|
+
);
|
|
212
|
+
return;
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
if (isDeepgramProviderError(msg)) {
|
|
216
|
+
this.emitError(this.currentContextId, deepgramProviderError(msg));
|
|
217
|
+
return;
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
// Provider speech-start (vad_events=true): the generic speech-start signal —
|
|
221
|
+
// the same packet a local VAD plugin emits — so barge-in works on VAD-less
|
|
222
|
+
// deployments. The session/TurnArbiter owns what to do with it.
|
|
223
|
+
if (msg["type"] === "SpeechStarted") {
|
|
224
|
+
if (!this.vadEvents) return;
|
|
225
|
+
this.bus?.push(Route.Main, {
|
|
226
|
+
kind: "vad.speech_started",
|
|
227
|
+
contextId: this.currentContextId,
|
|
228
|
+
timestampMs: Date.now(),
|
|
229
|
+
confidence: 1,
|
|
230
|
+
} satisfies VadSpeechStartedPacket);
|
|
231
|
+
return;
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
const alt = providerAlternative(msg);
|
|
235
|
+
if (!alt || typeof alt["transcript"] !== "string") return;
|
|
236
|
+
|
|
237
|
+
const transcript = alt["transcript"].trim();
|
|
238
|
+
const confidence = typeof alt["confidence"] === "number" ? alt["confidence"] : 0;
|
|
239
|
+
const fromFinalize = msg["from_finalize"] === true;
|
|
240
|
+
const speechFinal = msg["speech_final"] === true;
|
|
241
|
+
const providerContextId = msg["is_final"] === true
|
|
242
|
+
? this.contextIdForProviderFinal({ speechFinal, fromFinalize })
|
|
243
|
+
: this.currentContextId;
|
|
244
|
+
if (!transcript || this.finalizedContextIds.has(providerContextId)) return;
|
|
245
|
+
|
|
246
|
+
const state = this.transcriptState(providerContextId);
|
|
247
|
+
state.lastInterimTranscript = transcript;
|
|
248
|
+
state.lastInterimConfidence = confidence;
|
|
249
|
+
|
|
250
|
+
// Confidence threshold filter (Rapida pattern)
|
|
251
|
+
if (
|
|
252
|
+
this.confidenceThreshold > 0 &&
|
|
253
|
+
confidence < this.confidenceThreshold
|
|
254
|
+
) {
|
|
255
|
+
this.bus?.push(Route.Background, {
|
|
256
|
+
kind: "metric.conversation",
|
|
257
|
+
contextId: providerContextId,
|
|
258
|
+
timestampMs: Date.now(),
|
|
259
|
+
name: "stt_low_confidence",
|
|
260
|
+
value: String(confidence),
|
|
261
|
+
});
|
|
262
|
+
return;
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
if (msg["is_final"] === true) {
|
|
266
|
+
if (this.ignoreNextProviderFinalContextIds.delete(providerContextId)) {
|
|
267
|
+
this.resetPendingTranscript(providerContextId);
|
|
268
|
+
return;
|
|
269
|
+
}
|
|
270
|
+
this.appendFinalSegment(providerContextId, transcript, confidence);
|
|
271
|
+
if (speechFinal) this.speechFinalContextIds.add(providerContextId);
|
|
272
|
+
const finalizeRequested = this.finalizeRequestedContextIds.has(providerContextId);
|
|
273
|
+
this.pushProviderFinalMetric(providerContextId, transcript, {
|
|
274
|
+
confidence,
|
|
275
|
+
speechFinal,
|
|
276
|
+
fromFinalize,
|
|
277
|
+
finalizeRequested,
|
|
278
|
+
});
|
|
279
|
+
this.pushResult(transcript, confidence, providerContextId, {
|
|
280
|
+
name: "deepgram",
|
|
281
|
+
model: this.model,
|
|
282
|
+
region: "global",
|
|
283
|
+
speechFinal,
|
|
284
|
+
fromFinalize,
|
|
285
|
+
finalizeRequested,
|
|
286
|
+
});
|
|
287
|
+
if (speechFinal || fromFinalize) {
|
|
288
|
+
this.resolveProviderFinalize(providerContextId);
|
|
289
|
+
}
|
|
290
|
+
if (this.emitEosOnFinal && ((this.finalizeOnSpeechFinal && speechFinal) || (finalizeRequested && fromFinalize))) {
|
|
291
|
+
this.pushTurnComplete(providerContextId);
|
|
292
|
+
}
|
|
293
|
+
} else {
|
|
294
|
+
this.pushInterim(transcript, providerContextId);
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
/** Stream audio to Deepgram immediately. No batching, no CloseStream. */
|
|
299
|
+
async sendAudio(audio: Uint8Array, contextId = this.currentContextId): Promise<boolean> {
|
|
300
|
+
if (audio.byteLength === 0) return true;
|
|
301
|
+
try {
|
|
302
|
+
assertAudioPayload(this.audioFormat, audio);
|
|
303
|
+
if (!this.conn) throw new Error("Deepgram STT is not connected");
|
|
304
|
+
await this.conn.ensureReady();
|
|
305
|
+
this.conn.send(audio);
|
|
306
|
+
return true;
|
|
307
|
+
} catch (err) {
|
|
308
|
+
this.emitError(contextId, err instanceof Error ? err : new Error(String(err)));
|
|
309
|
+
return false;
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
/** Request that Deepgram flush buffered audio and return provider-final text. */
|
|
314
|
+
forceFinalize(contextId?: string): void {
|
|
315
|
+
const ctxId = contextId ?? this.currentContextId;
|
|
316
|
+
if (!ctxId || !this.bus) return;
|
|
317
|
+
this.requestProviderFinalize(ctxId);
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
private requestProviderFinalize(contextId: string): void {
|
|
321
|
+
if (this.finalizedContextIds.has(contextId)) return;
|
|
322
|
+
this.finalizeRequestedContextIds.add(contextId);
|
|
323
|
+
this.trackPendingProviderFinalize(contextId);
|
|
324
|
+
this.pushMetric(contextId, "stt_provider_finalize_requested", this.audioStats(contextId));
|
|
325
|
+
if (this.conn?.isReady) {
|
|
326
|
+
this.conn.send(JSON.stringify({ type: "Finalize" }));
|
|
327
|
+
}
|
|
328
|
+
if (!this.emitEosOnFinal && this.hasFinalTranscript(contextId)) {
|
|
329
|
+
this.scheduleProviderFinalizeCorrelationExpiry(contextId);
|
|
330
|
+
return;
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
this.clearProviderFinalizeTimer(contextId);
|
|
334
|
+
if (this.providerFinalizeTimeoutMs <= 0) return;
|
|
335
|
+
const timer = setTimeout(() => {
|
|
336
|
+
this.providerFinalizeTimers.delete(contextId);
|
|
337
|
+
this.handleProviderFinalizeTimeout(contextId);
|
|
338
|
+
}, this.providerFinalizeTimeoutMs);
|
|
339
|
+
this.providerFinalizeTimers.set(contextId, timer);
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
private handleProviderFinalizeTimeout(contextId: string): void {
|
|
343
|
+
if (!this.finalizeRequestedContextIds.has(contextId) || this.finalizedContextIds.has(contextId)) return;
|
|
344
|
+
this.pushMetric(contextId, "stt_provider_finalize_timeout", this.audioStats(contextId));
|
|
345
|
+
|
|
346
|
+
// Graceful degradation (opt-in): rather than dropping the user's turn when the provider
|
|
347
|
+
// never confirms the Finalize, complete it with the best buffered text — confirmed
|
|
348
|
+
// is_final segments first, then the latest interim — so the turn still reaches the LLM.
|
|
349
|
+
if (this.finalizeTimeoutFallback) {
|
|
350
|
+
const state = this.transcriptState(contextId);
|
|
351
|
+
const fallbackText = this.combinedFinalTranscript(contextId) || state.lastInterimTranscript;
|
|
352
|
+
if (fallbackText) {
|
|
353
|
+
this.pushMetric(contextId, "stt_provider_finalize_timeout_fallback", this.audioStats(contextId));
|
|
354
|
+
// A late provider-final for this turn must not double-emit after we promote here.
|
|
355
|
+
this.ignoreNextProviderFinalContextIds.add(contextId);
|
|
356
|
+
this.pushFinal(fallbackText, state.finalConfidence || state.lastInterimConfidence, contextId);
|
|
357
|
+
this.resetPendingTranscript(contextId);
|
|
358
|
+
return;
|
|
359
|
+
}
|
|
360
|
+
// No buffered text — a silence-only/orphan turn (e.g. an always-on client that rotated its
|
|
361
|
+
// contextId onto trailing silence). Discard it silently; a finalize timeout on a turn that
|
|
362
|
+
// never had speech is not an error worth surfacing.
|
|
363
|
+
this.pushMetric(contextId, "stt_provider_finalize_timeout_empty_discard", this.audioStats(contextId));
|
|
364
|
+
this.discardUnconfirmedTurn(contextId);
|
|
365
|
+
return;
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
this.discardUnconfirmedTurn(contextId);
|
|
369
|
+
this.emitError(
|
|
370
|
+
contextId,
|
|
371
|
+
new Error("Deepgram STT Finalize timed out before speech_final/from_finalize confirmation"),
|
|
372
|
+
);
|
|
373
|
+
// A single slow finalize is not a wedged stream: keep the healthy socket so the
|
|
374
|
+
// next turn streams normally instead of stalling on a reconnect (which loses
|
|
375
|
+
// Deepgram-side context and cascades into more timeouts). Only reconnect once the
|
|
376
|
+
// failures repeat without a confirmed final between them (counter cleared in pushFinal).
|
|
377
|
+
this.consecutiveFinalizeTimeouts += 1;
|
|
378
|
+
if (this.consecutiveFinalizeTimeouts >= this.finalizeResetThreshold) {
|
|
379
|
+
this.consecutiveFinalizeTimeouts = 0;
|
|
380
|
+
this.conn?.reset();
|
|
381
|
+
}
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
private discardUnconfirmedTurn(contextId: string): void {
|
|
385
|
+
this.clearProviderFinalizeTimer(contextId);
|
|
386
|
+
this.finalizeRequestedContextIds.delete(contextId);
|
|
387
|
+
this.removePendingProviderFinalize(contextId);
|
|
388
|
+
this.speechFinalContextIds.delete(contextId);
|
|
389
|
+
this.ignoreNextProviderFinalContextIds.add(contextId);
|
|
390
|
+
this.audioStatsByContextId.delete(contextId);
|
|
391
|
+
this.resetPendingTranscript(contextId);
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
/** Emit final transcript + EOS turn complete. */
|
|
395
|
+
private pushFinal(transcript: string, confidence: number, contextId = this.currentContextId): void {
|
|
396
|
+
const ctxId = contextId;
|
|
397
|
+
this.resolveProviderFinalize(ctxId);
|
|
398
|
+
this.finalizedContextIds.add(ctxId);
|
|
399
|
+
this.consecutiveFinalizeTimeouts = 0;
|
|
400
|
+
this.pushMetric(ctxId, "stt_audio_sent", this.audioStats(ctxId));
|
|
401
|
+
this.pushResult(transcript, confidence, ctxId);
|
|
402
|
+
|
|
403
|
+
if (this.emitEosOnFinal) {
|
|
404
|
+
this.bus?.push(Route.Main, {
|
|
405
|
+
kind: "eos.turn_complete",
|
|
406
|
+
contextId: ctxId,
|
|
407
|
+
timestampMs: Date.now(),
|
|
408
|
+
text: transcript,
|
|
409
|
+
transcripts: [],
|
|
410
|
+
});
|
|
411
|
+
}
|
|
412
|
+
this.audioStatsByContextId.delete(ctxId);
|
|
413
|
+
}
|
|
414
|
+
|
|
415
|
+
private pushTurnComplete(contextId: string): void {
|
|
416
|
+
const transcript = this.combinedFinalTranscript(contextId);
|
|
417
|
+
if (!transcript || !this.bus) return;
|
|
418
|
+
this.resolveProviderFinalize(contextId);
|
|
419
|
+
this.finalizedContextIds.add(contextId);
|
|
420
|
+
this.consecutiveFinalizeTimeouts = 0;
|
|
421
|
+
this.pushMetric(contextId, "stt_audio_sent", this.audioStats(contextId));
|
|
422
|
+
this.bus.push(Route.Main, {
|
|
423
|
+
kind: "eos.turn_complete",
|
|
424
|
+
contextId,
|
|
425
|
+
timestampMs: Date.now(),
|
|
426
|
+
text: transcript,
|
|
427
|
+
transcripts: [],
|
|
428
|
+
});
|
|
429
|
+
this.audioStatsByContextId.delete(contextId);
|
|
430
|
+
this.resetPendingTranscript(contextId);
|
|
431
|
+
}
|
|
432
|
+
|
|
433
|
+
private pushResult(
|
|
434
|
+
transcript: string,
|
|
435
|
+
confidence: number,
|
|
436
|
+
contextId = this.currentContextId,
|
|
437
|
+
provider: Record<string, unknown> = { name: "deepgram", model: this.model, region: "global" },
|
|
438
|
+
): void {
|
|
439
|
+
this.bus?.push(Route.Main, {
|
|
440
|
+
kind: "stt.result",
|
|
441
|
+
contextId,
|
|
442
|
+
timestampMs: Date.now(),
|
|
443
|
+
text: transcript,
|
|
444
|
+
confidence,
|
|
445
|
+
language: this.language,
|
|
446
|
+
provider,
|
|
447
|
+
});
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
/** Emit interim transcript for real-time display. */
|
|
451
|
+
private pushInterim(transcript: string, contextId = this.currentContextId): void {
|
|
452
|
+
this.bus?.push(Route.Main, {
|
|
453
|
+
kind: "stt.interim",
|
|
454
|
+
contextId,
|
|
455
|
+
timestampMs: Date.now(),
|
|
456
|
+
text: transcript,
|
|
457
|
+
});
|
|
458
|
+
}
|
|
459
|
+
|
|
460
|
+
private appendFinalSegment(contextId: string, transcript: string, confidence: number): void {
|
|
461
|
+
if (transcript.length === 0) return;
|
|
462
|
+
const state = this.transcriptState(contextId);
|
|
463
|
+
const last = state.finalTranscriptParts.at(-1);
|
|
464
|
+
if (last !== transcript) {
|
|
465
|
+
state.finalTranscriptParts.push(transcript);
|
|
466
|
+
}
|
|
467
|
+
state.finalConfidence = Math.max(state.finalConfidence, confidence);
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
private combinedFinalTranscript(contextId: string): string {
|
|
471
|
+
return this.transcriptState(contextId).finalTranscriptParts.join(" ").replace(/\s+/g, " ").trim();
|
|
472
|
+
}
|
|
473
|
+
|
|
474
|
+
private resetPendingTranscript(contextId: string): void {
|
|
475
|
+
this.transcriptStateByContextId.delete(contextId);
|
|
476
|
+
}
|
|
477
|
+
|
|
478
|
+
private resetTurnTranscriptState(): void {
|
|
479
|
+
this.resetPendingTranscript(this.currentContextId);
|
|
480
|
+
}
|
|
481
|
+
|
|
482
|
+
private transcriptState(contextId: string): ProviderTranscriptState {
|
|
483
|
+
const existing = this.transcriptStateByContextId.get(contextId);
|
|
484
|
+
if (existing) return existing;
|
|
485
|
+
const next: ProviderTranscriptState = {
|
|
486
|
+
lastInterimTranscript: "",
|
|
487
|
+
lastInterimConfidence: 0,
|
|
488
|
+
finalTranscriptParts: [],
|
|
489
|
+
finalConfidence: 0,
|
|
490
|
+
};
|
|
491
|
+
this.transcriptStateByContextId.set(contextId, next);
|
|
492
|
+
return next;
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
private hasFinalTranscript(contextId: string): boolean {
|
|
496
|
+
const state = this.transcriptStateByContextId.get(contextId);
|
|
497
|
+
return Boolean(state && state.finalTranscriptParts.length > 0);
|
|
498
|
+
}
|
|
499
|
+
|
|
500
|
+
private contextIdForProviderFinal(flags: { readonly speechFinal: boolean; readonly fromFinalize: boolean }): string {
|
|
501
|
+
const pending = this.pendingProviderFinalizeContextIds[0];
|
|
502
|
+
if (pending && (flags.speechFinal || flags.fromFinalize)) return pending;
|
|
503
|
+
return this.currentContextId;
|
|
504
|
+
}
|
|
505
|
+
|
|
506
|
+
private trackPendingProviderFinalize(contextId: string): void {
|
|
507
|
+
if (!this.pendingProviderFinalizeContextIds.includes(contextId)) {
|
|
508
|
+
this.pendingProviderFinalizeContextIds.push(contextId);
|
|
509
|
+
}
|
|
510
|
+
}
|
|
511
|
+
|
|
512
|
+
private removePendingProviderFinalize(contextId: string): void {
|
|
513
|
+
this.pendingProviderFinalizeContextIds = this.pendingProviderFinalizeContextIds.filter((ctxId) => ctxId !== contextId);
|
|
514
|
+
}
|
|
515
|
+
|
|
516
|
+
private scheduleProviderFinalizeCorrelationExpiry(contextId: string): void {
|
|
517
|
+
this.clearProviderFinalizeCorrelationTimer(contextId);
|
|
518
|
+
if (this.providerFinalizeTimeoutMs <= 0) return;
|
|
519
|
+
const timer = setTimeout(() => {
|
|
520
|
+
this.providerFinalizeCorrelationTimers.delete(contextId);
|
|
521
|
+
this.finalizeRequestedContextIds.delete(contextId);
|
|
522
|
+
this.removePendingProviderFinalize(contextId);
|
|
523
|
+
}, this.providerFinalizeTimeoutMs);
|
|
524
|
+
this.providerFinalizeCorrelationTimers.set(contextId, timer);
|
|
525
|
+
}
|
|
526
|
+
|
|
527
|
+
private resolveProviderFinalize(contextId: string): void {
|
|
528
|
+
this.finalizeRequestedContextIds.delete(contextId);
|
|
529
|
+
this.clearProviderFinalizeTimer(contextId);
|
|
530
|
+
this.clearProviderFinalizeCorrelationTimer(contextId);
|
|
531
|
+
this.removePendingProviderFinalize(contextId);
|
|
532
|
+
}
|
|
533
|
+
|
|
534
|
+
private recordAudioSent(contextId: string, byteLength: number): void {
|
|
535
|
+
if (!contextId) return;
|
|
536
|
+
const now = Date.now();
|
|
537
|
+
const current = this.audioStatsByContextId.get(contextId) ?? {
|
|
538
|
+
bytes: 0,
|
|
539
|
+
chunks: 0,
|
|
540
|
+
firstSentAtMs: now,
|
|
541
|
+
lastSentAtMs: now,
|
|
542
|
+
};
|
|
543
|
+
current.bytes += byteLength;
|
|
544
|
+
current.chunks += 1;
|
|
545
|
+
current.lastSentAtMs = now;
|
|
546
|
+
this.audioStatsByContextId.set(contextId, current);
|
|
547
|
+
}
|
|
548
|
+
|
|
549
|
+
private pushProviderFinalMetric(
|
|
550
|
+
contextId: string,
|
|
551
|
+
transcript: string,
|
|
552
|
+
flags: {
|
|
553
|
+
readonly confidence: number;
|
|
554
|
+
readonly speechFinal: boolean;
|
|
555
|
+
readonly fromFinalize: boolean;
|
|
556
|
+
readonly finalizeRequested: boolean;
|
|
557
|
+
},
|
|
558
|
+
): void {
|
|
559
|
+
this.pushMetric(contextId, "stt_provider_final_segment", {
|
|
560
|
+
...this.audioStats(contextId),
|
|
561
|
+
transcriptChars: transcript.length,
|
|
562
|
+
confidence: flags.confidence,
|
|
563
|
+
speechFinal: flags.speechFinal,
|
|
564
|
+
fromFinalize: flags.fromFinalize,
|
|
565
|
+
finalizeRequested: flags.finalizeRequested,
|
|
566
|
+
});
|
|
567
|
+
}
|
|
568
|
+
|
|
569
|
+
private audioStats(contextId: string): Record<string, number> {
|
|
570
|
+
const stats = this.audioStatsByContextId.get(contextId);
|
|
571
|
+
if (!stats) {
|
|
572
|
+
return {
|
|
573
|
+
bytes: 0,
|
|
574
|
+
chunks: 0,
|
|
575
|
+
durationMs: 0,
|
|
576
|
+
wallClockMs: 0,
|
|
577
|
+
};
|
|
578
|
+
}
|
|
579
|
+
return {
|
|
580
|
+
bytes: stats.bytes,
|
|
581
|
+
chunks: stats.chunks,
|
|
582
|
+
durationMs: Math.round((stats.bytes / 2 / this.sampleRate) * 1000),
|
|
583
|
+
wallClockMs: stats.lastSentAtMs - stats.firstSentAtMs,
|
|
584
|
+
};
|
|
585
|
+
}
|
|
586
|
+
|
|
587
|
+
private pushMetric(contextId: string, name: string, value: unknown): void {
|
|
588
|
+
this.bus?.push(Route.Background, {
|
|
589
|
+
kind: "metric.conversation",
|
|
590
|
+
contextId,
|
|
591
|
+
timestampMs: Date.now(),
|
|
592
|
+
name,
|
|
593
|
+
value: typeof value === "string" ? value : JSON.stringify(value),
|
|
594
|
+
});
|
|
595
|
+
}
|
|
596
|
+
|
|
597
|
+
private emitError(contextId: string, err: Error): void {
|
|
598
|
+
const category = categorizeSttError(err);
|
|
599
|
+
const packet: SttErrorPacket = {
|
|
600
|
+
kind: "stt.error",
|
|
601
|
+
contextId,
|
|
602
|
+
timestampMs: Date.now(),
|
|
603
|
+
component: "stt" as const,
|
|
604
|
+
category,
|
|
605
|
+
cause: err,
|
|
606
|
+
isRecoverable: isRecoverable(category),
|
|
607
|
+
};
|
|
608
|
+
this.bus?.push(Route.Critical, packet);
|
|
609
|
+
}
|
|
610
|
+
|
|
611
|
+
async close(): Promise<void> {
|
|
612
|
+
for (const dispose of this.disposers.splice(0)) dispose();
|
|
613
|
+
if (this.transcriptStateByContextId.size > 0) {
|
|
614
|
+
this.pushMetric(this.currentContextId, "stt_pending_transcript_discarded_on_close", this.audioStats(this.currentContextId));
|
|
615
|
+
}
|
|
616
|
+
for (const timer of this.providerFinalizeTimers.values()) clearTimeout(timer);
|
|
617
|
+
this.providerFinalizeTimers.clear();
|
|
618
|
+
for (const timer of this.providerFinalizeCorrelationTimers.values()) clearTimeout(timer);
|
|
619
|
+
this.providerFinalizeCorrelationTimers.clear();
|
|
620
|
+
this.pendingProviderFinalizeContextIds = [];
|
|
621
|
+
this.finalizeRequestedContextIds.clear();
|
|
622
|
+
this.finalizedContextIds.clear();
|
|
623
|
+
this.speechFinalContextIds.clear();
|
|
624
|
+
this.ignoreNextProviderFinalContextIds.clear();
|
|
625
|
+
this.transcriptStateByContextId.clear();
|
|
626
|
+
this.audioStatsByContextId.clear();
|
|
627
|
+
|
|
628
|
+
if (this.conn) {
|
|
629
|
+
// Graceful end-of-stream so Deepgram flushes and closes cleanly.
|
|
630
|
+
if (this.conn.isReady) {
|
|
631
|
+
try {
|
|
632
|
+
this.conn.send(JSON.stringify({ type: "CloseStream" }));
|
|
633
|
+
} catch {
|
|
634
|
+
// best effort
|
|
635
|
+
}
|
|
636
|
+
}
|
|
637
|
+
await this.conn.close();
|
|
638
|
+
this.conn = null;
|
|
639
|
+
}
|
|
640
|
+
this.bus = null;
|
|
641
|
+
}
|
|
642
|
+
|
|
643
|
+
private clearProviderFinalizeTimer(contextId: string): void {
|
|
644
|
+
const timer = this.providerFinalizeTimers.get(contextId);
|
|
645
|
+
if (!timer) return;
|
|
646
|
+
clearTimeout(timer);
|
|
647
|
+
this.providerFinalizeTimers.delete(contextId);
|
|
648
|
+
}
|
|
649
|
+
|
|
650
|
+
private clearProviderFinalizeCorrelationTimer(contextId: string): void {
|
|
651
|
+
const timer = this.providerFinalizeCorrelationTimers.get(contextId);
|
|
652
|
+
if (!timer) return;
|
|
653
|
+
clearTimeout(timer);
|
|
654
|
+
this.providerFinalizeCorrelationTimers.delete(contextId);
|
|
655
|
+
}
|
|
656
|
+
|
|
657
|
+
private discardProviderStateForReconnect(): void {
|
|
658
|
+
const contextId = this.currentContextId;
|
|
659
|
+
const discarded = this.transcriptStateByContextId.size > 0 ||
|
|
660
|
+
this.finalizeRequestedContextIds.size > 0 ||
|
|
661
|
+
this.audioStatsByContextId.size > 0 ||
|
|
662
|
+
this.providerFinalizeTimers.size > 0 ||
|
|
663
|
+
this.providerFinalizeCorrelationTimers.size > 0;
|
|
664
|
+
for (const timer of this.providerFinalizeTimers.values()) clearTimeout(timer);
|
|
665
|
+
this.providerFinalizeTimers.clear();
|
|
666
|
+
for (const timer of this.providerFinalizeCorrelationTimers.values()) clearTimeout(timer);
|
|
667
|
+
this.providerFinalizeCorrelationTimers.clear();
|
|
668
|
+
this.pendingProviderFinalizeContextIds = [];
|
|
669
|
+
this.finalizeRequestedContextIds.clear();
|
|
670
|
+
this.speechFinalContextIds.clear();
|
|
671
|
+
this.ignoreNextProviderFinalContextIds.clear();
|
|
672
|
+
this.audioStatsByContextId.clear();
|
|
673
|
+
// A reconnect (from any cause) starts a fresh provider stream, so the wedged-stream
|
|
674
|
+
// signal resets too — otherwise a stale count could force an avoidable reset on the
|
|
675
|
+
// first timeout after reconnecting.
|
|
676
|
+
this.consecutiveFinalizeTimeouts = 0;
|
|
677
|
+
this.transcriptStateByContextId.clear();
|
|
678
|
+
if (discarded && contextId) {
|
|
679
|
+
this.pushMetric(contextId, "stt_provider_reconnect_discarded_state", {});
|
|
680
|
+
}
|
|
681
|
+
}
|
|
682
|
+
|
|
683
|
+
}
|
|
684
|
+
|
|
685
|
+
async function defaultSocketFactory(): Promise<SocketFactory> {
|
|
686
|
+
const mod = await import("@kuralle-syrinx/ws/node");
|
|
687
|
+
return mod.createNodeWsSocket;
|
|
688
|
+
}
|
|
689
|
+
|
|
690
|
+
function providerAlternative(msg: Record<string, unknown>): Record<string, unknown> | null {
|
|
691
|
+
const channel = msg["channel"];
|
|
692
|
+
if (!channel || typeof channel !== "object") return null;
|
|
693
|
+
const alternatives = (channel as { alternatives?: unknown }).alternatives;
|
|
694
|
+
if (!Array.isArray(alternatives)) return null;
|
|
695
|
+
const first = alternatives[0];
|
|
696
|
+
return first && typeof first === "object" ? first as Record<string, unknown> : null;
|
|
697
|
+
}
|
|
698
|
+
|
|
699
|
+
function isDeepgramProviderError(msg: Record<string, unknown>): boolean {
|
|
700
|
+
const type = typeof msg["type"] === "string" ? msg["type"].toLowerCase() : "";
|
|
701
|
+
return type === "error" || typeof msg["err_code"] === "string" || typeof msg["err_msg"] === "string";
|
|
702
|
+
}
|
|
703
|
+
|
|
704
|
+
function deepgramProviderError(msg: Record<string, unknown>): Error {
|
|
705
|
+
const code = firstString(msg["code"], msg["err_code"]);
|
|
706
|
+
const description = firstString(msg["description"], msg["message"], msg["err_msg"], msg["details"]);
|
|
707
|
+
const requestId = firstString(msg["request_id"]);
|
|
708
|
+
const details = [
|
|
709
|
+
code ? `code=${code}` : "",
|
|
710
|
+
description,
|
|
711
|
+
requestId ? `request_id=${requestId}` : "",
|
|
712
|
+
].filter((part) => part.length > 0).join(" ");
|
|
713
|
+
return new Error(details ? `Deepgram STT provider error: ${details}` : "Deepgram STT provider error");
|
|
714
|
+
}
|
|
715
|
+
|
|
716
|
+
function firstString(...values: unknown[]): string {
|
|
717
|
+
for (const value of values) {
|
|
718
|
+
if (typeof value === "string" && value.length > 0) return value;
|
|
719
|
+
}
|
|
720
|
+
return "";
|
|
721
|
+
}
|