@kuralle-syrinx/deepgram 4.4.1 → 4.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/flux.d.ts +43 -0
- package/dist/flux.d.ts.map +1 -0
- package/dist/flux.js +308 -0
- package/dist/flux.js.map +1 -0
- package/dist/index.d.ts +4 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +8 -0
- package/dist/index.js.map +1 -0
- package/dist/stt.d.ts +49 -0
- package/dist/stt.d.ts.map +1 -0
- package/dist/stt.js +733 -0
- package/dist/stt.js.map +1 -0
- package/dist/tts.d.ts +44 -0
- package/dist/tts.d.ts.map +1 -0
- package/dist/tts.js +375 -0
- package/dist/tts.js.map +1 -0
- package/package.json +26 -10
- package/src/flux.test.ts +355 -0
- package/src/stt.test.ts +1784 -0
- package/src/tts.test.ts +454 -0
package/dist/stt.js
ADDED
|
@@ -0,0 +1,733 @@
|
|
|
1
|
+
// SPDX-License-Identifier: MIT
|
|
2
|
+
//
|
|
3
|
+
// Syrinx Kernel v2 — Deepgram Nova STT Plugin
|
|
4
|
+
//
|
|
5
|
+
// Session lifecycle (socket, reconnect, keepalive, billing funnel) lives in
|
|
6
|
+
// @kuralle-syrinx/stt-core. This file is the Nova wire protocol: listen URL +
|
|
7
|
+
// query knobs, Finalize + CloseStream frames, and the provider-policy state
|
|
8
|
+
// machine (multi-segment accumulation, speech_final/from_finalize gating,
|
|
9
|
+
// Finalize timeout/fallback/reset, UtteranceEnd backstop).
|
|
10
|
+
import { Route, assertAudioFormat, optionalStringConfig, readProviderRetryConfig, requireStringConfig, } from "@kuralle-syrinx/core";
|
|
11
|
+
import { defaultNodeSocketFactory, startStreamingSttSession, } from "@kuralle-syrinx/stt-core";
|
|
12
|
+
/**
|
|
13
|
+
* A retired-contextId set (finalized turns) must stay bounded: contextIds are
|
|
14
|
+
* per-turn on every transport (telephony rotates `<callSid>-t<n>`), so an
|
|
15
|
+
* unbounded set would grow one entry per turn for the life of a call. Keeping a
|
|
16
|
+
* recent-turns window is enough to reject the late/duplicate provider finals the
|
|
17
|
+
* set exists to guard against, without leaking memory over a long conversation.
|
|
18
|
+
*/
|
|
19
|
+
const MAX_RETIRED_CONTEXTS = 256;
|
|
20
|
+
function boundedAdd(set, value, cap) {
|
|
21
|
+
set.add(value);
|
|
22
|
+
while (set.size > cap) {
|
|
23
|
+
const oldest = set.values().next().value;
|
|
24
|
+
if (oldest === undefined)
|
|
25
|
+
break;
|
|
26
|
+
set.delete(oldest);
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
class DeepgramSttWireProtocol {
|
|
30
|
+
cfg;
|
|
31
|
+
host = null;
|
|
32
|
+
transcriptStateByContextId = new Map();
|
|
33
|
+
finalizeRequestedContextIds = new Set();
|
|
34
|
+
finalizedContextIds = new Set();
|
|
35
|
+
speechFinalContextIds = new Set();
|
|
36
|
+
ignoreNextProviderFinalContextIds = new Set();
|
|
37
|
+
providerFinalizeTimers = new Map();
|
|
38
|
+
providerFinalizeCorrelationTimers = new Map();
|
|
39
|
+
pendingProviderFinalizeContextIds = [];
|
|
40
|
+
audioStatsByContextId = new Map();
|
|
41
|
+
consecutiveFinalizeTimeouts = 0;
|
|
42
|
+
/** Mutable so a hard-language reconfigure re-stamps stt.result to the new language. */
|
|
43
|
+
currentLanguage;
|
|
44
|
+
constructor(cfg) {
|
|
45
|
+
this.cfg = cfg;
|
|
46
|
+
this.currentLanguage = cfg.language;
|
|
47
|
+
}
|
|
48
|
+
setLanguage(language) {
|
|
49
|
+
this.currentLanguage = language;
|
|
50
|
+
}
|
|
51
|
+
attach(host) {
|
|
52
|
+
this.host = host;
|
|
53
|
+
}
|
|
54
|
+
encodeAudio(audio) {
|
|
55
|
+
// Record after the engine has committed to sending (encodeAudio is only invoked
|
|
56
|
+
// on the send path). Bytes are used for provider-boundary metrics, not billing
|
|
57
|
+
// (billing is the base's sent-bytes funnel).
|
|
58
|
+
this.recordAudioSent(this.cfg.getContextId(), audio.byteLength);
|
|
59
|
+
return [audio];
|
|
60
|
+
}
|
|
61
|
+
encodeFinalize(contextId) {
|
|
62
|
+
if (!contextId || this.finalizedContextIds.has(contextId))
|
|
63
|
+
return [];
|
|
64
|
+
return [JSON.stringify({ type: "Finalize" })];
|
|
65
|
+
}
|
|
66
|
+
onFinalizeSent(contextId) {
|
|
67
|
+
if (!contextId || this.finalizedContextIds.has(contextId))
|
|
68
|
+
return;
|
|
69
|
+
this.finalizeRequestedContextIds.add(contextId);
|
|
70
|
+
this.trackPendingProviderFinalize(contextId);
|
|
71
|
+
this.pushMetric(contextId, "stt_provider_finalize_requested", this.audioStats(contextId));
|
|
72
|
+
if (!this.cfg.emitEosOnFinal && this.hasFinalTranscript(contextId)) {
|
|
73
|
+
this.scheduleProviderFinalizeCorrelationExpiry(contextId);
|
|
74
|
+
return;
|
|
75
|
+
}
|
|
76
|
+
this.clearProviderFinalizeTimer(contextId);
|
|
77
|
+
if (this.cfg.providerFinalizeTimeoutMs <= 0)
|
|
78
|
+
return;
|
|
79
|
+
const timer = setTimeout(() => {
|
|
80
|
+
this.providerFinalizeTimers.delete(contextId);
|
|
81
|
+
this.handleProviderFinalizeTimeout(contextId);
|
|
82
|
+
}, this.cfg.providerFinalizeTimeoutMs);
|
|
83
|
+
this.providerFinalizeTimers.set(contextId, timer);
|
|
84
|
+
}
|
|
85
|
+
encodeClose() {
|
|
86
|
+
if (this.transcriptStateByContextId.size > 0) {
|
|
87
|
+
this.pushMetric(this.cfg.getContextId(), "stt_pending_transcript_discarded_on_close", this.audioStats(this.cfg.getContextId()));
|
|
88
|
+
}
|
|
89
|
+
this.clearAllProviderState({ keepFinalized: false });
|
|
90
|
+
return [JSON.stringify({ type: "CloseStream" })];
|
|
91
|
+
}
|
|
92
|
+
onConnectionLost() {
|
|
93
|
+
this.discardProviderStateForReconnect();
|
|
94
|
+
}
|
|
95
|
+
decode(data, _isBinary) {
|
|
96
|
+
if (typeof data !== "string")
|
|
97
|
+
return [];
|
|
98
|
+
let msg;
|
|
99
|
+
try {
|
|
100
|
+
msg = JSON.parse(data);
|
|
101
|
+
}
|
|
102
|
+
catch (err) {
|
|
103
|
+
return [
|
|
104
|
+
{
|
|
105
|
+
type: "error",
|
|
106
|
+
contextId: this.cfg.getContextId(),
|
|
107
|
+
error: new Error(`Deepgram STT provider sent malformed JSON: ${err instanceof Error ? err.message : String(err)}`),
|
|
108
|
+
},
|
|
109
|
+
];
|
|
110
|
+
}
|
|
111
|
+
if (isDeepgramProviderError(msg)) {
|
|
112
|
+
return [
|
|
113
|
+
{
|
|
114
|
+
type: "error",
|
|
115
|
+
contextId: this.cfg.getContextId(),
|
|
116
|
+
error: deepgramProviderError(msg),
|
|
117
|
+
},
|
|
118
|
+
];
|
|
119
|
+
}
|
|
120
|
+
if (msg["type"] === "SpeechStarted") {
|
|
121
|
+
if (!this.cfg.vadEvents)
|
|
122
|
+
return [];
|
|
123
|
+
return [{ type: "speech_started", contextId: this.cfg.getContextId() }];
|
|
124
|
+
}
|
|
125
|
+
if (msg["type"] === "UtteranceEnd") {
|
|
126
|
+
return this.handleUtteranceEnd();
|
|
127
|
+
}
|
|
128
|
+
const alt = providerAlternative(msg);
|
|
129
|
+
if (!alt || typeof alt["transcript"] !== "string")
|
|
130
|
+
return [];
|
|
131
|
+
const transcript = alt["transcript"].trim();
|
|
132
|
+
const confidence = typeof alt["confidence"] === "number" ? alt["confidence"] : 0;
|
|
133
|
+
const fromFinalize = msg["from_finalize"] === true;
|
|
134
|
+
const speechFinal = msg["speech_final"] === true;
|
|
135
|
+
const providerContextId = msg["is_final"] === true
|
|
136
|
+
? this.contextIdForProviderFinal({ speechFinal, fromFinalize })
|
|
137
|
+
: this.cfg.getContextId();
|
|
138
|
+
if (!transcript || this.finalizedContextIds.has(providerContextId))
|
|
139
|
+
return [];
|
|
140
|
+
const state = this.transcriptState(providerContextId);
|
|
141
|
+
state.lastInterimTranscript = transcript;
|
|
142
|
+
state.lastInterimConfidence = confidence;
|
|
143
|
+
if (this.cfg.confidenceThreshold > 0 && confidence < this.cfg.confidenceThreshold) {
|
|
144
|
+
this.pushMetric(providerContextId, "stt_low_confidence", String(confidence));
|
|
145
|
+
return [];
|
|
146
|
+
}
|
|
147
|
+
if (msg["is_final"] === true) {
|
|
148
|
+
return this.handleIsFinal(providerContextId, transcript, confidence, {
|
|
149
|
+
speechFinal,
|
|
150
|
+
fromFinalize,
|
|
151
|
+
});
|
|
152
|
+
}
|
|
153
|
+
return this.interimEvents(transcript, providerContextId, alt);
|
|
154
|
+
}
|
|
155
|
+
/** Clear per-turn transcript buffers (interrupt.stt). */
|
|
156
|
+
resetTurnTranscriptState() {
|
|
157
|
+
this.resetPendingTranscript(this.cfg.getContextId());
|
|
158
|
+
}
|
|
159
|
+
handleUtteranceEnd() {
|
|
160
|
+
const ctxId = this.cfg.getContextId();
|
|
161
|
+
if (!this.cfg.emitEosOnFinal ||
|
|
162
|
+
!ctxId ||
|
|
163
|
+
this.finalizedContextIds.has(ctxId) ||
|
|
164
|
+
!this.combinedFinalTranscript(ctxId)) {
|
|
165
|
+
return [];
|
|
166
|
+
}
|
|
167
|
+
this.pushMetric(ctxId, "deepgram.utterance_end_backstop", "1");
|
|
168
|
+
return this.commitTurnComplete(ctxId);
|
|
169
|
+
}
|
|
170
|
+
handleIsFinal(providerContextId, transcript, confidence, flags) {
|
|
171
|
+
if (this.ignoreNextProviderFinalContextIds.delete(providerContextId)) {
|
|
172
|
+
this.resetPendingTranscript(providerContextId);
|
|
173
|
+
return [];
|
|
174
|
+
}
|
|
175
|
+
this.appendFinalSegment(providerContextId, transcript, confidence);
|
|
176
|
+
if (flags.speechFinal)
|
|
177
|
+
this.speechFinalContextIds.add(providerContextId);
|
|
178
|
+
const finalizeRequested = this.finalizeRequestedContextIds.has(providerContextId);
|
|
179
|
+
this.pushProviderFinalMetric(providerContextId, transcript, {
|
|
180
|
+
confidence,
|
|
181
|
+
speechFinal: flags.speechFinal,
|
|
182
|
+
fromFinalize: flags.fromFinalize,
|
|
183
|
+
finalizeRequested,
|
|
184
|
+
});
|
|
185
|
+
const events = [
|
|
186
|
+
{
|
|
187
|
+
type: "final",
|
|
188
|
+
contextId: providerContextId,
|
|
189
|
+
text: transcript,
|
|
190
|
+
confidence,
|
|
191
|
+
language: this.currentLanguage,
|
|
192
|
+
// No speechFinal: base emits stt.result + bills, but NOT eos (eos comes via turn_complete).
|
|
193
|
+
provider: {
|
|
194
|
+
name: "deepgram",
|
|
195
|
+
model: this.cfg.model,
|
|
196
|
+
region: "global",
|
|
197
|
+
speechFinal: flags.speechFinal,
|
|
198
|
+
fromFinalize: flags.fromFinalize,
|
|
199
|
+
finalizeRequested,
|
|
200
|
+
},
|
|
201
|
+
},
|
|
202
|
+
];
|
|
203
|
+
if (flags.speechFinal || flags.fromFinalize) {
|
|
204
|
+
this.resolveProviderFinalize(providerContextId);
|
|
205
|
+
}
|
|
206
|
+
if (this.cfg.emitEosOnFinal &&
|
|
207
|
+
((this.cfg.finalizeOnSpeechFinal && flags.speechFinal) ||
|
|
208
|
+
(finalizeRequested && flags.fromFinalize))) {
|
|
209
|
+
events.push(...this.commitTurnComplete(providerContextId));
|
|
210
|
+
}
|
|
211
|
+
return events;
|
|
212
|
+
}
|
|
213
|
+
interimEvents(transcript, contextId, alt) {
|
|
214
|
+
const wordTimings = mapProviderWordTimings(alt);
|
|
215
|
+
const events = [{ type: "interim", contextId, text: transcript }];
|
|
216
|
+
events.push({
|
|
217
|
+
type: "partial",
|
|
218
|
+
contextId,
|
|
219
|
+
text: transcript,
|
|
220
|
+
...(wordTimings ? { wordTimings } : {}),
|
|
221
|
+
});
|
|
222
|
+
return events;
|
|
223
|
+
}
|
|
224
|
+
commitTurnComplete(contextId) {
|
|
225
|
+
const transcript = this.combinedFinalTranscript(contextId);
|
|
226
|
+
if (!transcript)
|
|
227
|
+
return [];
|
|
228
|
+
this.resolveProviderFinalize(contextId);
|
|
229
|
+
boundedAdd(this.finalizedContextIds, contextId, MAX_RETIRED_CONTEXTS);
|
|
230
|
+
this.consecutiveFinalizeTimeouts = 0;
|
|
231
|
+
this.pushMetric(contextId, "stt_audio_sent", this.audioStats(contextId));
|
|
232
|
+
this.audioStatsByContextId.delete(contextId);
|
|
233
|
+
this.resetPendingTranscript(contextId);
|
|
234
|
+
return [{ type: "turn_complete", contextId, text: transcript }];
|
|
235
|
+
}
|
|
236
|
+
handleProviderFinalizeTimeout(contextId) {
|
|
237
|
+
if (!this.finalizeRequestedContextIds.has(contextId) || this.finalizedContextIds.has(contextId)) {
|
|
238
|
+
return;
|
|
239
|
+
}
|
|
240
|
+
this.pushMetric(contextId, "stt_provider_finalize_timeout", this.audioStats(contextId));
|
|
241
|
+
if (this.cfg.finalizeTimeoutFallback) {
|
|
242
|
+
const state = this.transcriptState(contextId);
|
|
243
|
+
const fallbackText = this.combinedFinalTranscript(contextId) || state.lastInterimTranscript;
|
|
244
|
+
if (fallbackText) {
|
|
245
|
+
this.pushMetric(contextId, "stt_provider_finalize_timeout_fallback", this.audioStats(contextId));
|
|
246
|
+
this.ignoreNextProviderFinalContextIds.add(contextId);
|
|
247
|
+
this.promoteFallbackFinal(fallbackText, state.finalConfidence || state.lastInterimConfidence, contextId);
|
|
248
|
+
this.resetPendingTranscript(contextId);
|
|
249
|
+
return;
|
|
250
|
+
}
|
|
251
|
+
this.pushMetric(contextId, "stt_provider_finalize_timeout_empty_discard", this.audioStats(contextId));
|
|
252
|
+
this.discardUnconfirmedTurn(contextId);
|
|
253
|
+
return;
|
|
254
|
+
}
|
|
255
|
+
this.discardUnconfirmedTurn(contextId);
|
|
256
|
+
this.host?.emit({
|
|
257
|
+
type: "error",
|
|
258
|
+
contextId,
|
|
259
|
+
error: new Error("Deepgram STT Finalize timed out before speech_final/from_finalize confirmation"),
|
|
260
|
+
});
|
|
261
|
+
this.consecutiveFinalizeTimeouts += 1;
|
|
262
|
+
if (this.consecutiveFinalizeTimeouts >= this.cfg.finalizeResetThreshold) {
|
|
263
|
+
this.consecutiveFinalizeTimeouts = 0;
|
|
264
|
+
this.host?.reset();
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
/** Fallback promote: result + optional eos via base (speechFinal: true). */
|
|
268
|
+
promoteFallbackFinal(transcript, confidence, contextId) {
|
|
269
|
+
this.resolveProviderFinalize(contextId);
|
|
270
|
+
boundedAdd(this.finalizedContextIds, contextId, MAX_RETIRED_CONTEXTS);
|
|
271
|
+
this.consecutiveFinalizeTimeouts = 0;
|
|
272
|
+
this.pushMetric(contextId, "stt_audio_sent", this.audioStats(contextId));
|
|
273
|
+
this.host?.emit({
|
|
274
|
+
type: "final",
|
|
275
|
+
contextId,
|
|
276
|
+
text: transcript,
|
|
277
|
+
confidence,
|
|
278
|
+
language: this.currentLanguage,
|
|
279
|
+
speechFinal: true,
|
|
280
|
+
provider: { name: "deepgram", model: this.cfg.model, region: "global" },
|
|
281
|
+
});
|
|
282
|
+
this.audioStatsByContextId.delete(contextId);
|
|
283
|
+
}
|
|
284
|
+
discardUnconfirmedTurn(contextId) {
|
|
285
|
+
this.clearProviderFinalizeTimer(contextId);
|
|
286
|
+
this.finalizeRequestedContextIds.delete(contextId);
|
|
287
|
+
this.removePendingProviderFinalize(contextId);
|
|
288
|
+
this.speechFinalContextIds.delete(contextId);
|
|
289
|
+
this.ignoreNextProviderFinalContextIds.add(contextId);
|
|
290
|
+
this.audioStatsByContextId.delete(contextId);
|
|
291
|
+
this.resetPendingTranscript(contextId);
|
|
292
|
+
}
|
|
293
|
+
discardProviderStateForReconnect() {
|
|
294
|
+
const contextId = this.cfg.getContextId();
|
|
295
|
+
const discarded = this.transcriptStateByContextId.size > 0 ||
|
|
296
|
+
this.finalizeRequestedContextIds.size > 0 ||
|
|
297
|
+
this.audioStatsByContextId.size > 0 ||
|
|
298
|
+
this.providerFinalizeTimers.size > 0 ||
|
|
299
|
+
this.providerFinalizeCorrelationTimers.size > 0;
|
|
300
|
+
this.clearAllProviderState({ keepFinalized: true });
|
|
301
|
+
// A reconnect (from any cause) starts a fresh provider stream, so the wedged-stream
|
|
302
|
+
// signal resets too — otherwise a stale count could force an avoidable reset on the
|
|
303
|
+
// first timeout after reconnecting.
|
|
304
|
+
this.consecutiveFinalizeTimeouts = 0;
|
|
305
|
+
if (discarded && contextId) {
|
|
306
|
+
this.pushMetric(contextId, "stt_provider_reconnect_discarded_state", {});
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
clearAllProviderState(opts) {
|
|
310
|
+
for (const timer of this.providerFinalizeTimers.values())
|
|
311
|
+
clearTimeout(timer);
|
|
312
|
+
this.providerFinalizeTimers.clear();
|
|
313
|
+
for (const timer of this.providerFinalizeCorrelationTimers.values())
|
|
314
|
+
clearTimeout(timer);
|
|
315
|
+
this.providerFinalizeCorrelationTimers.clear();
|
|
316
|
+
this.pendingProviderFinalizeContextIds = [];
|
|
317
|
+
this.finalizeRequestedContextIds.clear();
|
|
318
|
+
if (!opts.keepFinalized)
|
|
319
|
+
this.finalizedContextIds.clear();
|
|
320
|
+
this.speechFinalContextIds.clear();
|
|
321
|
+
this.ignoreNextProviderFinalContextIds.clear();
|
|
322
|
+
this.transcriptStateByContextId.clear();
|
|
323
|
+
this.audioStatsByContextId.clear();
|
|
324
|
+
}
|
|
325
|
+
appendFinalSegment(contextId, transcript, confidence) {
|
|
326
|
+
if (transcript.length === 0)
|
|
327
|
+
return;
|
|
328
|
+
const state = this.transcriptState(contextId);
|
|
329
|
+
const last = state.finalTranscriptParts.at(-1);
|
|
330
|
+
if (last !== transcript) {
|
|
331
|
+
state.finalTranscriptParts.push(transcript);
|
|
332
|
+
}
|
|
333
|
+
state.finalConfidence = Math.max(state.finalConfidence, confidence);
|
|
334
|
+
}
|
|
335
|
+
combinedFinalTranscript(contextId) {
|
|
336
|
+
return this.transcriptState(contextId)
|
|
337
|
+
.finalTranscriptParts.join(" ")
|
|
338
|
+
.replace(/\s+/g, " ")
|
|
339
|
+
.trim();
|
|
340
|
+
}
|
|
341
|
+
resetPendingTranscript(contextId) {
|
|
342
|
+
this.transcriptStateByContextId.delete(contextId);
|
|
343
|
+
}
|
|
344
|
+
transcriptState(contextId) {
|
|
345
|
+
const existing = this.transcriptStateByContextId.get(contextId);
|
|
346
|
+
if (existing)
|
|
347
|
+
return existing;
|
|
348
|
+
const next = {
|
|
349
|
+
lastInterimTranscript: "",
|
|
350
|
+
lastInterimConfidence: 0,
|
|
351
|
+
finalTranscriptParts: [],
|
|
352
|
+
finalConfidence: 0,
|
|
353
|
+
};
|
|
354
|
+
this.transcriptStateByContextId.set(contextId, next);
|
|
355
|
+
return next;
|
|
356
|
+
}
|
|
357
|
+
hasFinalTranscript(contextId) {
|
|
358
|
+
const state = this.transcriptStateByContextId.get(contextId);
|
|
359
|
+
return Boolean(state && state.finalTranscriptParts.length > 0);
|
|
360
|
+
}
|
|
361
|
+
contextIdForProviderFinal(flags) {
|
|
362
|
+
const pending = this.pendingProviderFinalizeContextIds[0];
|
|
363
|
+
if (pending && (flags.speechFinal || flags.fromFinalize))
|
|
364
|
+
return pending;
|
|
365
|
+
return this.cfg.getContextId();
|
|
366
|
+
}
|
|
367
|
+
trackPendingProviderFinalize(contextId) {
|
|
368
|
+
if (!this.pendingProviderFinalizeContextIds.includes(contextId)) {
|
|
369
|
+
this.pendingProviderFinalizeContextIds.push(contextId);
|
|
370
|
+
}
|
|
371
|
+
}
|
|
372
|
+
removePendingProviderFinalize(contextId) {
|
|
373
|
+
this.pendingProviderFinalizeContextIds = this.pendingProviderFinalizeContextIds.filter((ctxId) => ctxId !== contextId);
|
|
374
|
+
}
|
|
375
|
+
scheduleProviderFinalizeCorrelationExpiry(contextId) {
|
|
376
|
+
this.clearProviderFinalizeCorrelationTimer(contextId);
|
|
377
|
+
if (this.cfg.providerFinalizeTimeoutMs <= 0)
|
|
378
|
+
return;
|
|
379
|
+
const timer = setTimeout(() => {
|
|
380
|
+
this.providerFinalizeCorrelationTimers.delete(contextId);
|
|
381
|
+
this.finalizeRequestedContextIds.delete(contextId);
|
|
382
|
+
this.removePendingProviderFinalize(contextId);
|
|
383
|
+
}, this.cfg.providerFinalizeTimeoutMs);
|
|
384
|
+
this.providerFinalizeCorrelationTimers.set(contextId, timer);
|
|
385
|
+
}
|
|
386
|
+
resolveProviderFinalize(contextId) {
|
|
387
|
+
this.finalizeRequestedContextIds.delete(contextId);
|
|
388
|
+
this.clearProviderFinalizeTimer(contextId);
|
|
389
|
+
this.clearProviderFinalizeCorrelationTimer(contextId);
|
|
390
|
+
this.removePendingProviderFinalize(contextId);
|
|
391
|
+
}
|
|
392
|
+
recordAudioSent(contextId, byteLength) {
|
|
393
|
+
if (!contextId)
|
|
394
|
+
return;
|
|
395
|
+
const now = Date.now();
|
|
396
|
+
const current = this.audioStatsByContextId.get(contextId) ?? {
|
|
397
|
+
bytes: 0,
|
|
398
|
+
chunks: 0,
|
|
399
|
+
firstSentAtMs: now,
|
|
400
|
+
lastSentAtMs: now,
|
|
401
|
+
};
|
|
402
|
+
current.bytes += byteLength;
|
|
403
|
+
current.chunks += 1;
|
|
404
|
+
current.lastSentAtMs = now;
|
|
405
|
+
this.audioStatsByContextId.set(contextId, current);
|
|
406
|
+
}
|
|
407
|
+
pushProviderFinalMetric(contextId, transcript, flags) {
|
|
408
|
+
this.pushMetric(contextId, "stt_provider_final_segment", {
|
|
409
|
+
...this.audioStats(contextId),
|
|
410
|
+
transcriptChars: transcript.length,
|
|
411
|
+
confidence: flags.confidence,
|
|
412
|
+
speechFinal: flags.speechFinal,
|
|
413
|
+
fromFinalize: flags.fromFinalize,
|
|
414
|
+
finalizeRequested: flags.finalizeRequested,
|
|
415
|
+
});
|
|
416
|
+
}
|
|
417
|
+
audioStats(contextId) {
|
|
418
|
+
const stats = this.audioStatsByContextId.get(contextId);
|
|
419
|
+
if (!stats) {
|
|
420
|
+
return {
|
|
421
|
+
bytes: 0,
|
|
422
|
+
chunks: 0,
|
|
423
|
+
durationMs: 0,
|
|
424
|
+
wallClockMs: 0,
|
|
425
|
+
};
|
|
426
|
+
}
|
|
427
|
+
return {
|
|
428
|
+
bytes: stats.bytes,
|
|
429
|
+
chunks: stats.chunks,
|
|
430
|
+
durationMs: Math.round((stats.bytes / 2 / this.cfg.sampleRate) * 1000),
|
|
431
|
+
wallClockMs: stats.lastSentAtMs - stats.firstSentAtMs,
|
|
432
|
+
};
|
|
433
|
+
}
|
|
434
|
+
pushMetric(contextId, name, value) {
|
|
435
|
+
this.cfg.onMetric(contextId, name, value);
|
|
436
|
+
}
|
|
437
|
+
clearProviderFinalizeTimer(contextId) {
|
|
438
|
+
const timer = this.providerFinalizeTimers.get(contextId);
|
|
439
|
+
if (!timer)
|
|
440
|
+
return;
|
|
441
|
+
clearTimeout(timer);
|
|
442
|
+
this.providerFinalizeTimers.delete(contextId);
|
|
443
|
+
}
|
|
444
|
+
clearProviderFinalizeCorrelationTimer(contextId) {
|
|
445
|
+
const timer = this.providerFinalizeCorrelationTimers.get(contextId);
|
|
446
|
+
if (!timer)
|
|
447
|
+
return;
|
|
448
|
+
clearTimeout(timer);
|
|
449
|
+
this.providerFinalizeCorrelationTimers.delete(contextId);
|
|
450
|
+
}
|
|
451
|
+
}
|
|
452
|
+
export class DeepgramSTTPlugin {
|
|
453
|
+
socketFactory;
|
|
454
|
+
endpointingCapability = {
|
|
455
|
+
owner: "provider_stt",
|
|
456
|
+
disableConfig: {
|
|
457
|
+
emit_eos_on_final: false,
|
|
458
|
+
finalize_on_speech_final: false,
|
|
459
|
+
},
|
|
460
|
+
};
|
|
461
|
+
bus = null;
|
|
462
|
+
session = null;
|
|
463
|
+
protocol = null;
|
|
464
|
+
currentContextId = "";
|
|
465
|
+
model = "nova-3";
|
|
466
|
+
language = "en-US";
|
|
467
|
+
sampleRate = 16000;
|
|
468
|
+
endpointing = 300;
|
|
469
|
+
endpointUrl = "wss://api.deepgram.com/v1/listen";
|
|
470
|
+
smartFormat = true;
|
|
471
|
+
interimResults = true;
|
|
472
|
+
vadEvents = false;
|
|
473
|
+
utteranceEndMs = 0;
|
|
474
|
+
keyterms = [];
|
|
475
|
+
encoding = "linear16";
|
|
476
|
+
channels = 1;
|
|
477
|
+
noDelay = true;
|
|
478
|
+
queryParams;
|
|
479
|
+
disposers = [];
|
|
480
|
+
constructor(socketFactory) {
|
|
481
|
+
this.socketFactory = socketFactory;
|
|
482
|
+
}
|
|
483
|
+
async initialize(bus, config) {
|
|
484
|
+
this.bus = bus;
|
|
485
|
+
const apiKey = requireStringConfig(config, "api_key");
|
|
486
|
+
this.sampleRate = config["sample_rate"] ?? 16000;
|
|
487
|
+
this.model = optionalStringConfig(config, "model") ?? "nova-3";
|
|
488
|
+
this.language = optionalStringConfig(config, "language") ?? "en-US";
|
|
489
|
+
this.endpointing = config["endpointing"] ?? 300;
|
|
490
|
+
this.endpointUrl =
|
|
491
|
+
optionalStringConfig(config, "endpoint_url") ?? "wss://api.deepgram.com/v1/listen";
|
|
492
|
+
this.smartFormat = config["smart_format"] ?? true;
|
|
493
|
+
this.interimResults = config["interim_results"] ?? true;
|
|
494
|
+
// Opt-in: provider SpeechStarted → vad.speech_started is for VAD-less
|
|
495
|
+
// deployments (edge cascade). On sessions with a local VAD the duplicate,
|
|
496
|
+
// laggier speech-start signal corrupts VAD/EOS-owned turn-taking (the turn
|
|
497
|
+
// never completes) — proven by the Fly telephony spike.
|
|
498
|
+
this.vadEvents = config["vad_events"] ?? false;
|
|
499
|
+
{
|
|
500
|
+
const raw = config["utterance_end_ms"] ?? 0;
|
|
501
|
+
this.utteranceEndMs = raw > 0 ? Math.max(1000, raw) : 0;
|
|
502
|
+
}
|
|
503
|
+
const confidenceThreshold = config["confidence_threshold"] ?? 0;
|
|
504
|
+
const finalizeOnSpeechFinal = config["finalize_on_speech_final"] ?? true;
|
|
505
|
+
const emitEosOnFinal = config["emit_eos_on_final"] ?? true;
|
|
506
|
+
const providerFinalizeTimeoutMs = config["provider_finalize_timeout_ms"] ?? 1200;
|
|
507
|
+
const finalizeResetThreshold = config["finalize_reset_threshold"] ?? 2;
|
|
508
|
+
const finalizeTimeoutFallback = config["finalize_timeout_fallback"] ?? false;
|
|
509
|
+
const keepAliveIntervalMs = config["keep_alive_interval_ms"] ?? 3000;
|
|
510
|
+
{
|
|
511
|
+
const raw = config["keyterm"];
|
|
512
|
+
this.keyterms = Array.isArray(raw)
|
|
513
|
+
? raw.filter((t) => typeof t === "string" && t.length > 0)
|
|
514
|
+
: typeof raw === "string" && raw.length > 0
|
|
515
|
+
? [raw]
|
|
516
|
+
: [];
|
|
517
|
+
}
|
|
518
|
+
this.encoding = optionalStringConfig(config, "encoding") ?? "linear16";
|
|
519
|
+
this.channels =
|
|
520
|
+
typeof config["channels"] === "number" && Number.isFinite(config["channels"])
|
|
521
|
+
? Math.max(1, Math.floor(config["channels"]))
|
|
522
|
+
: 1;
|
|
523
|
+
this.noDelay = config["no_delay"] ?? true;
|
|
524
|
+
this.queryParams = readPlainObject(config["query_params"]);
|
|
525
|
+
const audioFormat = {
|
|
526
|
+
encoding: "pcm_s16le",
|
|
527
|
+
sampleRateHz: this.sampleRate,
|
|
528
|
+
channels: 1,
|
|
529
|
+
};
|
|
530
|
+
assertAudioFormat(audioFormat);
|
|
531
|
+
// Track context before the session handlers so encodeAudio sees the live turn id.
|
|
532
|
+
this.disposers.push(bus.on("stt.audio", (pkt) => {
|
|
533
|
+
const audioPkt = pkt;
|
|
534
|
+
if (audioPkt.contextId)
|
|
535
|
+
this.currentContextId = audioPkt.contextId;
|
|
536
|
+
}), bus.on("turn.change", (pkt) => {
|
|
537
|
+
this.currentContextId = pkt.contextId;
|
|
538
|
+
}), bus.on("interrupt.stt", () => {
|
|
539
|
+
this.protocol?.resetTurnTranscriptState();
|
|
540
|
+
}));
|
|
541
|
+
this.protocol = new DeepgramSttWireProtocol({
|
|
542
|
+
model: this.model,
|
|
543
|
+
language: this.language,
|
|
544
|
+
sampleRate: this.sampleRate,
|
|
545
|
+
vadEvents: this.vadEvents,
|
|
546
|
+
confidenceThreshold,
|
|
547
|
+
finalizeOnSpeechFinal,
|
|
548
|
+
emitEosOnFinal,
|
|
549
|
+
providerFinalizeTimeoutMs,
|
|
550
|
+
finalizeResetThreshold,
|
|
551
|
+
finalizeTimeoutFallback,
|
|
552
|
+
getContextId: () => this.currentContextId,
|
|
553
|
+
onMetric: (contextId, name, value) => {
|
|
554
|
+
bus.push(Route.Background, {
|
|
555
|
+
kind: "metric.conversation",
|
|
556
|
+
contextId,
|
|
557
|
+
timestampMs: Date.now(),
|
|
558
|
+
name,
|
|
559
|
+
value: typeof value === "string" ? value : JSON.stringify(value),
|
|
560
|
+
});
|
|
561
|
+
},
|
|
562
|
+
});
|
|
563
|
+
this.session = await startStreamingSttSession(bus, {
|
|
564
|
+
protocol: this.protocol,
|
|
565
|
+
provider: { name: "deepgram", model: this.model, region: "global" },
|
|
566
|
+
format: audioFormat,
|
|
567
|
+
language: this.language,
|
|
568
|
+
emitEosOnFinal,
|
|
569
|
+
url: () => {
|
|
570
|
+
const params = new URLSearchParams({
|
|
571
|
+
encoding: this.encoding,
|
|
572
|
+
sample_rate: String(this.sampleRate),
|
|
573
|
+
interim_results: String(this.interimResults),
|
|
574
|
+
endpointing: String(this.endpointing),
|
|
575
|
+
smart_format: String(this.smartFormat),
|
|
576
|
+
model: this.model,
|
|
577
|
+
language: this.language,
|
|
578
|
+
channels: String(this.channels),
|
|
579
|
+
no_delay: String(this.noDelay),
|
|
580
|
+
vad_events: String(this.vadEvents),
|
|
581
|
+
...(this.utteranceEndMs > 0 ? { utterance_end_ms: String(this.utteranceEndMs) } : {}),
|
|
582
|
+
});
|
|
583
|
+
for (const term of this.keyterms)
|
|
584
|
+
params.append("keyterm", term);
|
|
585
|
+
applyQueryParams(params, this.queryParams);
|
|
586
|
+
const separator = this.endpointUrl.includes("?") ? "&" : "?";
|
|
587
|
+
return `${this.endpointUrl}${separator}${params.toString()}`;
|
|
588
|
+
},
|
|
589
|
+
headers: { Authorization: `Token ${apiKey}` },
|
|
590
|
+
socketFactory: this.socketFactory ?? (await defaultNodeSocketFactory()),
|
|
591
|
+
retry: readProviderRetryConfig(config),
|
|
592
|
+
replayBufferSize: config["replay_buffer_size"] ?? 64,
|
|
593
|
+
keepAliveIntervalMs,
|
|
594
|
+
keepAliveMessage: () => JSON.stringify({ type: "KeepAlive" }),
|
|
595
|
+
metricPrefix: "stt.deepgram",
|
|
596
|
+
});
|
|
597
|
+
}
|
|
598
|
+
/**
|
|
599
|
+
* Per-turn reconfigure of keyterms / silence endpointing. Nova has no in-band
|
|
600
|
+
* Configure message — updates instance state then reconnects via `session.reset()`
|
|
601
|
+
* so the re-evaluated `url()` carries the new params (LiveKit Nova-3 pattern).
|
|
602
|
+
* Call only at a turn boundary (between utterances); reconnect is not free and
|
|
603
|
+
* would drop mid-utterance audio. keyterms REPLACE the list (Flux semantics).
|
|
604
|
+
* `language` hard-switches recognition (e.g. "es-ES" or Nova-3 "multi") — applied on
|
|
605
|
+
* reconnect via the rebuilt `url()`, and re-stamped onto stt.result. Flux-only fields
|
|
606
|
+
* (eotThreshold, eagerEotThreshold, eotTimeoutMs, contextText) are ignored.
|
|
607
|
+
*/
|
|
608
|
+
get sttReconfigure() {
|
|
609
|
+
return this;
|
|
610
|
+
}
|
|
611
|
+
reconfigure(partial) {
|
|
612
|
+
let changed = false;
|
|
613
|
+
if (partial.keyterms !== undefined) {
|
|
614
|
+
this.keyterms = partial.keyterms;
|
|
615
|
+
changed = true;
|
|
616
|
+
}
|
|
617
|
+
if (partial.endpointingMs !== undefined) {
|
|
618
|
+
this.endpointing = partial.endpointingMs;
|
|
619
|
+
changed = true;
|
|
620
|
+
}
|
|
621
|
+
if (partial.language !== undefined) {
|
|
622
|
+
this.language = partial.language;
|
|
623
|
+
this.protocol?.setLanguage(partial.language);
|
|
624
|
+
changed = true;
|
|
625
|
+
}
|
|
626
|
+
if (changed)
|
|
627
|
+
this.session?.reset();
|
|
628
|
+
}
|
|
629
|
+
/** Request that Deepgram flush buffered audio and return provider-final text. */
|
|
630
|
+
forceFinalize(contextId) {
|
|
631
|
+
const ctxId = contextId ?? this.currentContextId;
|
|
632
|
+
if (!ctxId || !this.bus)
|
|
633
|
+
return;
|
|
634
|
+
this.bus.push(Route.Main, {
|
|
635
|
+
kind: "stt.finalize",
|
|
636
|
+
contextId: ctxId,
|
|
637
|
+
timestampMs: Date.now(),
|
|
638
|
+
});
|
|
639
|
+
}
|
|
640
|
+
async close() {
|
|
641
|
+
for (const dispose of this.disposers.splice(0))
|
|
642
|
+
dispose();
|
|
643
|
+
await this.session?.dispose();
|
|
644
|
+
this.session = null;
|
|
645
|
+
this.protocol = null;
|
|
646
|
+
this.bus = null;
|
|
647
|
+
}
|
|
648
|
+
}
|
|
649
|
+
function mapProviderWordTimings(alt) {
|
|
650
|
+
if (!alt)
|
|
651
|
+
return undefined;
|
|
652
|
+
const words = alt["words"];
|
|
653
|
+
if (!Array.isArray(words))
|
|
654
|
+
return undefined;
|
|
655
|
+
const timings = [];
|
|
656
|
+
for (const entry of words) {
|
|
657
|
+
if (!entry || typeof entry !== "object")
|
|
658
|
+
continue;
|
|
659
|
+
const row = entry;
|
|
660
|
+
const word = row["word"];
|
|
661
|
+
const start = row["start"];
|
|
662
|
+
const end = row["end"];
|
|
663
|
+
const confidence = row["confidence"];
|
|
664
|
+
if (typeof word !== "string" || typeof start !== "number" || typeof end !== "number")
|
|
665
|
+
continue;
|
|
666
|
+
timings.push({
|
|
667
|
+
word,
|
|
668
|
+
startMs: start * 1000,
|
|
669
|
+
endMs: end * 1000,
|
|
670
|
+
confidence: typeof confidence === "number" ? confidence : 0,
|
|
671
|
+
});
|
|
672
|
+
}
|
|
673
|
+
return timings.length > 0 ? timings : undefined;
|
|
674
|
+
}
|
|
675
|
+
function providerAlternative(msg) {
|
|
676
|
+
const channel = msg["channel"];
|
|
677
|
+
if (!channel || typeof channel !== "object")
|
|
678
|
+
return null;
|
|
679
|
+
const alternatives = channel.alternatives;
|
|
680
|
+
if (!Array.isArray(alternatives))
|
|
681
|
+
return null;
|
|
682
|
+
const first = alternatives[0];
|
|
683
|
+
return first && typeof first === "object" ? first : null;
|
|
684
|
+
}
|
|
685
|
+
function isDeepgramProviderError(msg) {
|
|
686
|
+
const type = typeof msg["type"] === "string" ? msg["type"].toLowerCase() : "";
|
|
687
|
+
return type === "error" || typeof msg["err_code"] === "string" || typeof msg["err_msg"] === "string";
|
|
688
|
+
}
|
|
689
|
+
function deepgramProviderError(msg) {
|
|
690
|
+
const code = firstString(msg["code"], msg["err_code"]);
|
|
691
|
+
const description = firstString(msg["description"], msg["message"], msg["err_msg"], msg["details"]);
|
|
692
|
+
const requestId = firstString(msg["request_id"]);
|
|
693
|
+
const details = [
|
|
694
|
+
code ? `code=${code}` : "",
|
|
695
|
+
description,
|
|
696
|
+
requestId ? `request_id=${requestId}` : "",
|
|
697
|
+
]
|
|
698
|
+
.filter((part) => part.length > 0)
|
|
699
|
+
.join(" ");
|
|
700
|
+
return new Error(details ? `Deepgram STT provider error: ${details}` : "Deepgram STT provider error");
|
|
701
|
+
}
|
|
702
|
+
function firstString(...values) {
|
|
703
|
+
for (const value of values) {
|
|
704
|
+
if (typeof value === "string" && value.length > 0)
|
|
705
|
+
return value;
|
|
706
|
+
}
|
|
707
|
+
return "";
|
|
708
|
+
}
|
|
709
|
+
function readPlainObject(value) {
|
|
710
|
+
if (value !== null && typeof value === "object" && !Array.isArray(value)) {
|
|
711
|
+
return value;
|
|
712
|
+
}
|
|
713
|
+
return undefined;
|
|
714
|
+
}
|
|
715
|
+
/** Merge open-ended provider query knobs. Arrays append; scalars set (override). */
|
|
716
|
+
function applyQueryParams(params, extra) {
|
|
717
|
+
if (!extra)
|
|
718
|
+
return;
|
|
719
|
+
for (const [key, value] of Object.entries(extra)) {
|
|
720
|
+
if (value === undefined || value === null)
|
|
721
|
+
continue;
|
|
722
|
+
if (Array.isArray(value)) {
|
|
723
|
+
for (const item of value) {
|
|
724
|
+
if (item === undefined || item === null)
|
|
725
|
+
continue;
|
|
726
|
+
params.append(key, String(item));
|
|
727
|
+
}
|
|
728
|
+
continue;
|
|
729
|
+
}
|
|
730
|
+
params.set(key, String(value));
|
|
731
|
+
}
|
|
732
|
+
}
|
|
733
|
+
//# sourceMappingURL=stt.js.map
|