@kuralle-syrinx/deepgram 4.2.0 → 4.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +6 -5
- package/src/flux.ts +198 -155
- package/src/stt.ts +544 -495
- package/src/tts.ts +35 -5
- package/.handoff/finalize-contract-probe.test.ts +0 -193
- package/src/flux.test.ts +0 -227
- package/src/stt.test.ts +0 -1323
- package/src/tts.test.ts +0 -359
- package/tsconfig.json +0 -21
package/src/stt.ts
CHANGED
|
@@ -1,39 +1,35 @@
|
|
|
1
1
|
// SPDX-License-Identifier: MIT
|
|
2
2
|
//
|
|
3
|
-
// Syrinx Kernel v2 — Deepgram STT Plugin
|
|
3
|
+
// Syrinx Kernel v2 — Deepgram Nova STT Plugin
|
|
4
4
|
//
|
|
5
|
-
//
|
|
6
|
-
//
|
|
7
|
-
//
|
|
8
|
-
//
|
|
9
|
-
//
|
|
10
|
-
// - Interim transcripts for real-time display
|
|
11
|
-
// - Final transcripts can trigger eos.turn_complete, or Pipecat EOS can own finalization
|
|
12
|
-
//
|
|
13
|
-
// Guards (Syrinx additions beyond Rapida):
|
|
14
|
-
// - forceFinalize() sends Deepgram Finalize and waits for provider confirmation
|
|
15
|
-
// - Never promotes interim/cached text to final without speech_final/from_finalize
|
|
16
|
-
// - Emits provider-boundary metrics for audio byte/duration and finalize provenance
|
|
17
|
-
//
|
|
18
|
-
// Reference: Rapida transformer/deepgram/stt.go + internal/stt_callback.go
|
|
5
|
+
// Session lifecycle (socket, reconnect, keepalive, billing funnel) lives in
|
|
6
|
+
// @kuralle-syrinx/stt-core. This file is the Nova wire protocol: listen URL +
|
|
7
|
+
// query knobs, Finalize + CloseStream frames, and the provider-policy state
|
|
8
|
+
// machine (multi-segment accumulation, speech_final/from_finalize gating,
|
|
9
|
+
// Finalize timeout/fallback/reset, UtteranceEnd backstop).
|
|
19
10
|
|
|
20
|
-
import type { PipelineBus } from "@kuralle-syrinx/core";
|
|
21
11
|
import {
|
|
22
12
|
Route,
|
|
23
|
-
type AudioFormat,
|
|
24
|
-
type VoicePlugin,
|
|
25
|
-
type PluginConfig,
|
|
26
|
-
type SttErrorPacket,
|
|
27
|
-
type VadSpeechStartedPacket,
|
|
28
13
|
assertAudioFormat,
|
|
29
|
-
assertAudioPayload,
|
|
30
|
-
requireStringConfig,
|
|
31
14
|
optionalStringConfig,
|
|
32
15
|
readProviderRetryConfig,
|
|
33
|
-
|
|
34
|
-
|
|
16
|
+
requireStringConfig,
|
|
17
|
+
type AudioFormat,
|
|
18
|
+
type PipelineBus,
|
|
19
|
+
type PluginConfig,
|
|
20
|
+
type SttReconfigure,
|
|
21
|
+
type SttReconfigurePartial,
|
|
22
|
+
type VoicePlugin,
|
|
35
23
|
} from "@kuralle-syrinx/core";
|
|
36
|
-
import {
|
|
24
|
+
import {
|
|
25
|
+
defaultNodeSocketFactory,
|
|
26
|
+
startStreamingSttSession,
|
|
27
|
+
type SttEvent,
|
|
28
|
+
type SttProtocolHost,
|
|
29
|
+
type SttWireProtocol,
|
|
30
|
+
type StreamingSttSession,
|
|
31
|
+
} from "@kuralle-syrinx/stt-core";
|
|
32
|
+
import type { SocketData, SocketFactory } from "@kuralle-syrinx/ws";
|
|
37
33
|
|
|
38
34
|
interface ProviderTranscriptState {
|
|
39
35
|
lastInterimTranscript: string;
|
|
@@ -60,57 +56,25 @@ function boundedAdd(set: Set<string>, value: string, cap: number): void {
|
|
|
60
56
|
}
|
|
61
57
|
}
|
|
62
58
|
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
private smartFormat: boolean = true;
|
|
80
|
-
private interimResults: boolean = true;
|
|
81
|
-
private vadEvents: boolean = true;
|
|
82
|
-
// Deepgram gap-based utterance boundary (ms). When > 0 (requires interim_results),
|
|
83
|
-
// Deepgram emits UtteranceEnd even when `speech_final` never fires — the documented
|
|
84
|
-
// backstop for noisy/continuous lines (telephony) where a turn would otherwise wedge.
|
|
85
|
-
private utteranceEndMs: number = 0;
|
|
86
|
-
private confidenceThreshold: number = 0;
|
|
87
|
-
private finalizeOnSpeechFinal: boolean = true;
|
|
88
|
-
private emitEosOnFinal: boolean = true;
|
|
89
|
-
private providerFinalizeTimeoutMs: number = 1200;
|
|
90
|
-
// Consecutive unconfirmed Finalize timeouts that force a connection reset. A single
|
|
91
|
-
// slow finalize discards its turn but keeps the (healthy) socket; only repeated
|
|
92
|
-
// failures with no confirmed final between them look like a wedged stream worth
|
|
93
|
-
// reconnecting. Prevents one slow finalize from cascading into the next turns.
|
|
94
|
-
private finalizeResetThreshold: number = 2;
|
|
95
|
-
private consecutiveFinalizeTimeouts = 0;
|
|
96
|
-
// When the provider never confirms a Finalize, complete the turn with the best transcript
|
|
97
|
-
// already buffered instead of dropping it. For live conversation a reply on slightly
|
|
98
|
-
// imperfect text beats silently losing the user's turn. Opt-in (off preserves the strict
|
|
99
|
-
// "never promote unconfirmed" behavior for callers that need it).
|
|
100
|
-
private finalizeTimeoutFallback: boolean = false;
|
|
101
|
-
private keepAliveIntervalMs: number = 3000;
|
|
102
|
-
// nova-3 keyterm prompting: bias recognition toward domain terms (names, products,
|
|
103
|
-
// codes) — the #1 production voice failure is mishearing exactly these.
|
|
104
|
-
private keyterms: readonly string[] = [];
|
|
105
|
-
|
|
106
|
-
// Session-long WebSocket, managed by the shared connection (reconnect, keepalive).
|
|
107
|
-
private conn: WebSocketConnection | null = null;
|
|
108
|
-
private currentContextId = "";
|
|
109
|
-
private streamStartTime = 0;
|
|
110
|
-
private disposers: Array<() => void> = [];
|
|
111
|
-
|
|
112
|
-
constructor(private readonly socketFactory?: SocketFactory) {}
|
|
59
|
+
type MetricSink = (contextId: string, name: string, value: unknown) => void;
|
|
60
|
+
|
|
61
|
+
interface DeepgramNovaProtocolConfig {
|
|
62
|
+
readonly model: string;
|
|
63
|
+
readonly language: string;
|
|
64
|
+
readonly sampleRate: number;
|
|
65
|
+
readonly vadEvents: boolean;
|
|
66
|
+
readonly confidenceThreshold: number;
|
|
67
|
+
readonly finalizeOnSpeechFinal: boolean;
|
|
68
|
+
readonly emitEosOnFinal: boolean;
|
|
69
|
+
readonly providerFinalizeTimeoutMs: number;
|
|
70
|
+
readonly finalizeResetThreshold: number;
|
|
71
|
+
readonly finalizeTimeoutFallback: boolean;
|
|
72
|
+
readonly getContextId: () => string;
|
|
73
|
+
readonly onMetric: MetricSink;
|
|
74
|
+
}
|
|
113
75
|
|
|
76
|
+
class DeepgramSttWireProtocol implements SttWireProtocol {
|
|
77
|
+
private host: SttProtocolHost | null = null;
|
|
114
78
|
private transcriptStateByContextId = new Map<string, ProviderTranscriptState>();
|
|
115
79
|
private finalizeRequestedContextIds = new Set<string>();
|
|
116
80
|
private finalizedContextIds = new Set<string>();
|
|
@@ -119,332 +83,314 @@ export class DeepgramSTTPlugin implements VoicePlugin {
|
|
|
119
83
|
private providerFinalizeTimers = new Map<string, ReturnType<typeof setTimeout>>();
|
|
120
84
|
private providerFinalizeCorrelationTimers = new Map<string, ReturnType<typeof setTimeout>>();
|
|
121
85
|
private pendingProviderFinalizeContextIds: string[] = [];
|
|
122
|
-
private audioStatsByContextId = new Map<
|
|
123
|
-
|
|
124
|
-
chunks: number;
|
|
125
|
-
firstSentAtMs: number;
|
|
126
|
-
lastSentAtMs: number;
|
|
127
|
-
}>();
|
|
128
|
-
private audioFormat: AudioFormat = { encoding: "pcm_s16le", sampleRateHz: 16000, channels: 1 };
|
|
129
|
-
|
|
130
|
-
async initialize(bus: PipelineBus, config: PluginConfig): Promise<void> {
|
|
131
|
-
this.bus = bus;
|
|
132
|
-
this.apiKey = requireStringConfig(config, "api_key");
|
|
133
|
-
this.sampleRate = (config["sample_rate"] as number) ?? 16000;
|
|
134
|
-
this.model = optionalStringConfig(config, "model") ?? "nova-3";
|
|
135
|
-
this.language = optionalStringConfig(config, "language") ?? "en-US";
|
|
136
|
-
this.endpointing = (config["endpointing"] as number) ?? 300;
|
|
137
|
-
this.endpointUrl = optionalStringConfig(config, "endpoint_url") ?? "wss://api.deepgram.com/v1/listen";
|
|
138
|
-
this.smartFormat = (config["smart_format"] as boolean) ?? true;
|
|
139
|
-
this.interimResults = (config["interim_results"] as boolean) ?? true;
|
|
140
|
-
// Opt-in: provider SpeechStarted → vad.speech_started is for VAD-less
|
|
141
|
-
// deployments (edge cascade). On sessions with a local VAD the duplicate,
|
|
142
|
-
// laggier speech-start signal corrupts VAD/EOS-owned turn-taking (the turn
|
|
143
|
-
// never completes) — proven by the Fly telephony spike.
|
|
144
|
-
this.vadEvents = (config["vad_events"] as boolean) ?? false;
|
|
145
|
-
// Deepgram requires utterance_end_ms >= 1000 and interim_results; clamp to that.
|
|
86
|
+
private audioStatsByContextId = new Map<
|
|
87
|
+
string,
|
|
146
88
|
{
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
this.finalizeOnSpeechFinal = (config["finalize_on_speech_final"] as boolean) ?? true;
|
|
153
|
-
this.emitEosOnFinal = (config["emit_eos_on_final"] as boolean) ?? true;
|
|
154
|
-
this.providerFinalizeTimeoutMs = (config["provider_finalize_timeout_ms"] as number) ?? 1200;
|
|
155
|
-
this.finalizeResetThreshold = (config["finalize_reset_threshold"] as number) ?? 2;
|
|
156
|
-
this.finalizeTimeoutFallback = (config["finalize_timeout_fallback"] as boolean) ?? false;
|
|
157
|
-
this.keepAliveIntervalMs = (config["keep_alive_interval_ms"] as number) ?? 3000;
|
|
158
|
-
{
|
|
159
|
-
const raw = config["keyterm"];
|
|
160
|
-
this.keyterms = Array.isArray(raw)
|
|
161
|
-
? raw.filter((t): t is string => typeof t === "string" && t.length > 0)
|
|
162
|
-
: typeof raw === "string" && raw.length > 0
|
|
163
|
-
? [raw]
|
|
164
|
-
: [];
|
|
89
|
+
bytes: number;
|
|
90
|
+
chunks: number;
|
|
91
|
+
firstSentAtMs: number;
|
|
92
|
+
lastSentAtMs: number;
|
|
93
|
+
billedBytes?: number;
|
|
165
94
|
}
|
|
166
|
-
|
|
167
|
-
|
|
95
|
+
>();
|
|
96
|
+
private consecutiveFinalizeTimeouts = 0;
|
|
97
|
+
/** Mutable so a hard-language reconfigure re-stamps stt.result to the new language. */
|
|
98
|
+
private currentLanguage: string;
|
|
168
99
|
|
|
169
|
-
|
|
170
|
-
this.
|
|
171
|
-
|
|
172
|
-
const params = new URLSearchParams({
|
|
173
|
-
encoding: "linear16",
|
|
174
|
-
sample_rate: String(this.sampleRate),
|
|
175
|
-
interim_results: String(this.interimResults),
|
|
176
|
-
endpointing: String(this.endpointing),
|
|
177
|
-
smart_format: String(this.smartFormat),
|
|
178
|
-
model: this.model,
|
|
179
|
-
language: this.language,
|
|
180
|
-
channels: "1",
|
|
181
|
-
no_delay: "true",
|
|
182
|
-
vad_events: String(this.vadEvents),
|
|
183
|
-
...(this.utteranceEndMs > 0 ? { utterance_end_ms: String(this.utteranceEndMs) } : {}),
|
|
184
|
-
});
|
|
185
|
-
for (const term of this.keyterms) params.append("keyterm", term);
|
|
186
|
-
const separator = this.endpointUrl.includes("?") ? "&" : "?";
|
|
187
|
-
return `${this.endpointUrl}${separator}${params.toString()}`;
|
|
188
|
-
},
|
|
189
|
-
headers: { Authorization: `Token ${this.apiKey}` },
|
|
190
|
-
socketFactory: this.socketFactory ?? await defaultSocketFactory(),
|
|
191
|
-
retry: readProviderRetryConfig(config),
|
|
192
|
-
replayBufferSize: (config["replay_buffer_size"] as number) ?? 64,
|
|
193
|
-
onReplay: (event, count) => {
|
|
194
|
-
this.pushMetric(this.currentContextId, `stt.deepgram.reconnect_replay_${event}`, String(count));
|
|
195
|
-
},
|
|
196
|
-
keepAliveIntervalMs: this.keepAliveIntervalMs,
|
|
197
|
-
keepAliveMessage: () => JSON.stringify({ type: "KeepAlive" }),
|
|
198
|
-
onMessage: (data) => {
|
|
199
|
-
if (typeof data === "string") this.handleProviderMessage(data);
|
|
200
|
-
},
|
|
201
|
-
onConnectionLost: (err) => {
|
|
202
|
-
this.discardProviderStateForReconnect();
|
|
203
|
-
this.emitError(this.currentContextId, err);
|
|
204
|
-
},
|
|
205
|
-
});
|
|
206
|
-
await this.conn.connect();
|
|
100
|
+
constructor(private readonly cfg: DeepgramNovaProtocolConfig) {
|
|
101
|
+
this.currentLanguage = cfg.language;
|
|
102
|
+
}
|
|
207
103
|
|
|
208
|
-
|
|
209
|
-
this.
|
|
210
|
-
|
|
211
|
-
const audioPkt = pkt as { audio: Uint8Array; contextId?: string };
|
|
212
|
-
if (audioPkt.contextId) {
|
|
213
|
-
this.currentContextId = audioPkt.contextId;
|
|
214
|
-
}
|
|
215
|
-
const sent = await this.sendAudio(audioPkt.audio, this.currentContextId);
|
|
216
|
-
if (sent) {
|
|
217
|
-
this.recordAudioSent(this.currentContextId, audioPkt.audio.byteLength);
|
|
218
|
-
}
|
|
219
|
-
if (this.streamStartTime === 0) {
|
|
220
|
-
this.streamStartTime = Date.now();
|
|
221
|
-
}
|
|
222
|
-
}),
|
|
104
|
+
setLanguage(language: string): void {
|
|
105
|
+
this.currentLanguage = language;
|
|
106
|
+
}
|
|
223
107
|
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
this.currentContextId = tc.contextId;
|
|
228
|
-
this.streamStartTime = Date.now();
|
|
229
|
-
}),
|
|
108
|
+
attach(host: SttProtocolHost): void {
|
|
109
|
+
this.host = host;
|
|
110
|
+
}
|
|
230
111
|
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
);
|
|
112
|
+
encodeAudio(audio: Uint8Array): readonly SocketData[] {
|
|
113
|
+
// Record after the engine has committed to sending (encodeAudio is only invoked
|
|
114
|
+
// on the send path). Bytes are used for provider-boundary metrics, not billing
|
|
115
|
+
// (billing is the base's sent-bytes funnel).
|
|
116
|
+
this.recordAudioSent(this.cfg.getContextId(), audio.byteLength);
|
|
117
|
+
return [audio];
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
encodeFinalize(contextId: string): readonly SocketData[] {
|
|
121
|
+
if (!contextId || this.finalizedContextIds.has(contextId)) return [];
|
|
122
|
+
return [JSON.stringify({ type: "Finalize" })];
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
onFinalizeSent(contextId: string): void {
|
|
126
|
+
if (!contextId || this.finalizedContextIds.has(contextId)) return;
|
|
127
|
+
this.finalizeRequestedContextIds.add(contextId);
|
|
128
|
+
this.trackPendingProviderFinalize(contextId);
|
|
129
|
+
this.pushMetric(contextId, "stt_provider_finalize_requested", this.audioStats(contextId));
|
|
130
|
+
|
|
131
|
+
if (!this.cfg.emitEosOnFinal && this.hasFinalTranscript(contextId)) {
|
|
132
|
+
this.scheduleProviderFinalizeCorrelationExpiry(contextId);
|
|
133
|
+
return;
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
this.clearProviderFinalizeTimer(contextId);
|
|
137
|
+
if (this.cfg.providerFinalizeTimeoutMs <= 0) return;
|
|
138
|
+
const timer = setTimeout(() => {
|
|
139
|
+
this.providerFinalizeTimers.delete(contextId);
|
|
140
|
+
this.handleProviderFinalizeTimeout(contextId);
|
|
141
|
+
}, this.cfg.providerFinalizeTimeoutMs);
|
|
142
|
+
this.providerFinalizeTimers.set(contextId, timer);
|
|
241
143
|
}
|
|
242
144
|
|
|
243
|
-
|
|
145
|
+
encodeClose(): readonly SocketData[] {
|
|
146
|
+
if (this.transcriptStateByContextId.size > 0) {
|
|
147
|
+
this.pushMetric(
|
|
148
|
+
this.cfg.getContextId(),
|
|
149
|
+
"stt_pending_transcript_discarded_on_close",
|
|
150
|
+
this.audioStats(this.cfg.getContextId()),
|
|
151
|
+
);
|
|
152
|
+
}
|
|
153
|
+
this.clearAllProviderState({ keepFinalized: false });
|
|
154
|
+
return [JSON.stringify({ type: "CloseStream" })];
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
onConnectionLost(): void {
|
|
158
|
+
this.discardProviderStateForReconnect();
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
decode(data: SocketData, _isBinary: boolean): readonly SttEvent[] {
|
|
162
|
+
if (typeof data !== "string") return [];
|
|
244
163
|
let msg: Record<string, unknown>;
|
|
245
164
|
try {
|
|
246
165
|
msg = JSON.parse(data) as Record<string, unknown>;
|
|
247
166
|
} catch (err) {
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
167
|
+
return [
|
|
168
|
+
{
|
|
169
|
+
type: "error",
|
|
170
|
+
contextId: this.cfg.getContextId(),
|
|
171
|
+
error: new Error(
|
|
172
|
+
`Deepgram STT provider sent malformed JSON: ${err instanceof Error ? err.message : String(err)}`,
|
|
173
|
+
),
|
|
174
|
+
},
|
|
175
|
+
];
|
|
253
176
|
}
|
|
254
177
|
|
|
255
178
|
if (isDeepgramProviderError(msg)) {
|
|
256
|
-
|
|
257
|
-
|
|
179
|
+
return [
|
|
180
|
+
{
|
|
181
|
+
type: "error",
|
|
182
|
+
contextId: this.cfg.getContextId(),
|
|
183
|
+
error: deepgramProviderError(msg),
|
|
184
|
+
},
|
|
185
|
+
];
|
|
258
186
|
}
|
|
259
187
|
|
|
260
|
-
// Provider speech-start (vad_events=true): the generic speech-start signal —
|
|
261
|
-
// the same packet a local VAD plugin emits — so barge-in works on VAD-less
|
|
262
|
-
// deployments. The session/TurnArbiter owns what to do with it.
|
|
263
188
|
if (msg["type"] === "SpeechStarted") {
|
|
264
|
-
if (!this.vadEvents) return;
|
|
265
|
-
this.
|
|
266
|
-
kind: "vad.speech_started",
|
|
267
|
-
contextId: this.currentContextId,
|
|
268
|
-
timestampMs: Date.now(),
|
|
269
|
-
confidence: 1,
|
|
270
|
-
} satisfies VadSpeechStartedPacket);
|
|
271
|
-
return;
|
|
189
|
+
if (!this.cfg.vadEvents) return [];
|
|
190
|
+
return [{ type: "speech_started", contextId: this.cfg.getContextId() }];
|
|
272
191
|
}
|
|
273
192
|
|
|
274
|
-
// Deepgram's gap-based utterance boundary (utterance_end_ms). This fires even
|
|
275
|
-
// when `speech_final` is missed on a noisy/continuous line, so it's the backstop
|
|
276
|
-
// that stops a turn from wedging forever. If we hold un-finalized transcript for
|
|
277
|
-
// the current turn, complete it.
|
|
278
193
|
if (msg["type"] === "UtteranceEnd") {
|
|
279
|
-
|
|
280
|
-
if (
|
|
281
|
-
this.emitEosOnFinal &&
|
|
282
|
-
ctxId &&
|
|
283
|
-
!this.finalizedContextIds.has(ctxId) &&
|
|
284
|
-
this.combinedFinalTranscript(ctxId)
|
|
285
|
-
) {
|
|
286
|
-
this.bus?.push(Route.Background, {
|
|
287
|
-
kind: "metric.conversation",
|
|
288
|
-
contextId: ctxId,
|
|
289
|
-
timestampMs: Date.now(),
|
|
290
|
-
name: "deepgram.utterance_end_backstop",
|
|
291
|
-
value: "1",
|
|
292
|
-
});
|
|
293
|
-
this.pushTurnComplete(ctxId);
|
|
294
|
-
}
|
|
295
|
-
return;
|
|
194
|
+
return this.handleUtteranceEnd();
|
|
296
195
|
}
|
|
297
196
|
|
|
298
197
|
const alt = providerAlternative(msg);
|
|
299
|
-
if (!alt || typeof alt["transcript"] !== "string") return;
|
|
198
|
+
if (!alt || typeof alt["transcript"] !== "string") return [];
|
|
300
199
|
|
|
301
200
|
const transcript = alt["transcript"].trim();
|
|
302
201
|
const confidence = typeof alt["confidence"] === "number" ? alt["confidence"] : 0;
|
|
303
202
|
const fromFinalize = msg["from_finalize"] === true;
|
|
304
203
|
const speechFinal = msg["speech_final"] === true;
|
|
305
|
-
const providerContextId =
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
204
|
+
const providerContextId =
|
|
205
|
+
msg["is_final"] === true
|
|
206
|
+
? this.contextIdForProviderFinal({ speechFinal, fromFinalize })
|
|
207
|
+
: this.cfg.getContextId();
|
|
208
|
+
if (!transcript || this.finalizedContextIds.has(providerContextId)) return [];
|
|
309
209
|
|
|
310
210
|
const state = this.transcriptState(providerContextId);
|
|
311
211
|
state.lastInterimTranscript = transcript;
|
|
312
212
|
state.lastInterimConfidence = confidence;
|
|
313
213
|
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
confidence < this.confidenceThreshold
|
|
318
|
-
) {
|
|
319
|
-
this.bus?.push(Route.Background, {
|
|
320
|
-
kind: "metric.conversation",
|
|
321
|
-
contextId: providerContextId,
|
|
322
|
-
timestampMs: Date.now(),
|
|
323
|
-
name: "stt_low_confidence",
|
|
324
|
-
value: String(confidence),
|
|
325
|
-
});
|
|
326
|
-
return;
|
|
214
|
+
if (this.cfg.confidenceThreshold > 0 && confidence < this.cfg.confidenceThreshold) {
|
|
215
|
+
this.pushMetric(providerContextId, "stt_low_confidence", String(confidence));
|
|
216
|
+
return [];
|
|
327
217
|
}
|
|
328
218
|
|
|
329
219
|
if (msg["is_final"] === true) {
|
|
330
|
-
|
|
331
|
-
this.resetPendingTranscript(providerContextId);
|
|
332
|
-
return;
|
|
333
|
-
}
|
|
334
|
-
this.appendFinalSegment(providerContextId, transcript, confidence);
|
|
335
|
-
if (speechFinal) this.speechFinalContextIds.add(providerContextId);
|
|
336
|
-
const finalizeRequested = this.finalizeRequestedContextIds.has(providerContextId);
|
|
337
|
-
this.pushProviderFinalMetric(providerContextId, transcript, {
|
|
338
|
-
confidence,
|
|
339
|
-
speechFinal,
|
|
340
|
-
fromFinalize,
|
|
341
|
-
finalizeRequested,
|
|
342
|
-
});
|
|
343
|
-
this.pushResult(transcript, confidence, providerContextId, {
|
|
344
|
-
name: "deepgram",
|
|
345
|
-
model: this.model,
|
|
346
|
-
region: "global",
|
|
220
|
+
return this.handleIsFinal(providerContextId, transcript, confidence, {
|
|
347
221
|
speechFinal,
|
|
348
222
|
fromFinalize,
|
|
349
|
-
finalizeRequested,
|
|
350
223
|
});
|
|
351
|
-
if (speechFinal || fromFinalize) {
|
|
352
|
-
this.resolveProviderFinalize(providerContextId);
|
|
353
|
-
}
|
|
354
|
-
if (this.emitEosOnFinal && ((this.finalizeOnSpeechFinal && speechFinal) || (finalizeRequested && fromFinalize))) {
|
|
355
|
-
this.pushTurnComplete(providerContextId);
|
|
356
|
-
}
|
|
357
|
-
} else {
|
|
358
|
-
this.pushInterim(transcript, providerContextId, alt);
|
|
359
224
|
}
|
|
225
|
+
|
|
226
|
+
return this.interimEvents(transcript, providerContextId, alt);
|
|
360
227
|
}
|
|
361
228
|
|
|
362
|
-
/**
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
try {
|
|
366
|
-
assertAudioPayload(this.audioFormat, audio);
|
|
367
|
-
if (!this.conn) throw new Error("Deepgram STT is not connected");
|
|
368
|
-
await this.conn.ensureReady();
|
|
369
|
-
this.conn.send(audio);
|
|
370
|
-
return true;
|
|
371
|
-
} catch (err) {
|
|
372
|
-
this.emitError(contextId, err instanceof Error ? err : new Error(String(err)));
|
|
373
|
-
return false;
|
|
374
|
-
}
|
|
229
|
+
/** Clear per-turn transcript buffers (interrupt.stt). */
|
|
230
|
+
resetTurnTranscriptState(): void {
|
|
231
|
+
this.resetPendingTranscript(this.cfg.getContextId());
|
|
375
232
|
}
|
|
376
233
|
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
234
|
+
private handleUtteranceEnd(): readonly SttEvent[] {
|
|
235
|
+
const ctxId = this.cfg.getContextId();
|
|
236
|
+
if (
|
|
237
|
+
!this.cfg.emitEosOnFinal ||
|
|
238
|
+
!ctxId ||
|
|
239
|
+
this.finalizedContextIds.has(ctxId) ||
|
|
240
|
+
!this.combinedFinalTranscript(ctxId)
|
|
241
|
+
) {
|
|
242
|
+
return [];
|
|
243
|
+
}
|
|
244
|
+
this.pushMetric(ctxId, "deepgram.utterance_end_backstop", "1");
|
|
245
|
+
return this.commitTurnComplete(ctxId);
|
|
382
246
|
}
|
|
383
247
|
|
|
384
|
-
private
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
248
|
+
private handleIsFinal(
|
|
249
|
+
providerContextId: string,
|
|
250
|
+
transcript: string,
|
|
251
|
+
confidence: number,
|
|
252
|
+
flags: { readonly speechFinal: boolean; readonly fromFinalize: boolean },
|
|
253
|
+
): readonly SttEvent[] {
|
|
254
|
+
if (this.ignoreNextProviderFinalContextIds.delete(providerContextId)) {
|
|
255
|
+
this.resetPendingTranscript(providerContextId);
|
|
256
|
+
return [];
|
|
391
257
|
}
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
258
|
+
this.appendFinalSegment(providerContextId, transcript, confidence);
|
|
259
|
+
if (flags.speechFinal) this.speechFinalContextIds.add(providerContextId);
|
|
260
|
+
const finalizeRequested = this.finalizeRequestedContextIds.has(providerContextId);
|
|
261
|
+
this.pushProviderFinalMetric(providerContextId, transcript, {
|
|
262
|
+
confidence,
|
|
263
|
+
speechFinal: flags.speechFinal,
|
|
264
|
+
fromFinalize: flags.fromFinalize,
|
|
265
|
+
finalizeRequested,
|
|
266
|
+
});
|
|
267
|
+
|
|
268
|
+
const events: SttEvent[] = [
|
|
269
|
+
{
|
|
270
|
+
type: "final",
|
|
271
|
+
contextId: providerContextId,
|
|
272
|
+
text: transcript,
|
|
273
|
+
confidence,
|
|
274
|
+
language: this.currentLanguage,
|
|
275
|
+
// No speechFinal: base emits stt.result + bills, but NOT eos (eos comes via turn_complete).
|
|
276
|
+
provider: {
|
|
277
|
+
name: "deepgram",
|
|
278
|
+
model: this.cfg.model,
|
|
279
|
+
region: "global",
|
|
280
|
+
speechFinal: flags.speechFinal,
|
|
281
|
+
fromFinalize: flags.fromFinalize,
|
|
282
|
+
finalizeRequested,
|
|
283
|
+
},
|
|
284
|
+
},
|
|
285
|
+
];
|
|
286
|
+
|
|
287
|
+
if (flags.speechFinal || flags.fromFinalize) {
|
|
288
|
+
this.resolveProviderFinalize(providerContextId);
|
|
395
289
|
}
|
|
290
|
+
if (
|
|
291
|
+
this.cfg.emitEosOnFinal &&
|
|
292
|
+
((this.cfg.finalizeOnSpeechFinal && flags.speechFinal) ||
|
|
293
|
+
(finalizeRequested && flags.fromFinalize))
|
|
294
|
+
) {
|
|
295
|
+
events.push(...this.commitTurnComplete(providerContextId));
|
|
296
|
+
}
|
|
297
|
+
return events;
|
|
298
|
+
}
|
|
396
299
|
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
300
|
+
private interimEvents(
|
|
301
|
+
transcript: string,
|
|
302
|
+
contextId: string,
|
|
303
|
+
alt: Record<string, unknown>,
|
|
304
|
+
): readonly SttEvent[] {
|
|
305
|
+
const wordTimings = mapProviderWordTimings(alt);
|
|
306
|
+
const events: SttEvent[] = [{ type: "interim", contextId, text: transcript }];
|
|
307
|
+
events.push({
|
|
308
|
+
type: "partial",
|
|
309
|
+
contextId,
|
|
310
|
+
text: transcript,
|
|
311
|
+
...(wordTimings ? { wordTimings } : {}),
|
|
312
|
+
});
|
|
313
|
+
return events;
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
private commitTurnComplete(contextId: string): readonly SttEvent[] {
|
|
317
|
+
const transcript = this.combinedFinalTranscript(contextId);
|
|
318
|
+
if (!transcript) return [];
|
|
319
|
+
this.resolveProviderFinalize(contextId);
|
|
320
|
+
boundedAdd(this.finalizedContextIds, contextId, MAX_RETIRED_CONTEXTS);
|
|
321
|
+
this.consecutiveFinalizeTimeouts = 0;
|
|
322
|
+
this.pushMetric(contextId, "stt_audio_sent", this.audioStats(contextId));
|
|
323
|
+
this.audioStatsByContextId.delete(contextId);
|
|
324
|
+
this.resetPendingTranscript(contextId);
|
|
325
|
+
return [{ type: "turn_complete", contextId, text: transcript }];
|
|
404
326
|
}
|
|
405
327
|
|
|
406
328
|
private handleProviderFinalizeTimeout(contextId: string): void {
|
|
407
|
-
if (!this.finalizeRequestedContextIds.has(contextId) || this.finalizedContextIds.has(contextId))
|
|
329
|
+
if (!this.finalizeRequestedContextIds.has(contextId) || this.finalizedContextIds.has(contextId)) {
|
|
330
|
+
return;
|
|
331
|
+
}
|
|
408
332
|
this.pushMetric(contextId, "stt_provider_finalize_timeout", this.audioStats(contextId));
|
|
409
333
|
|
|
410
|
-
|
|
411
|
-
// never confirms the Finalize, complete it with the best buffered text — confirmed
|
|
412
|
-
// is_final segments first, then the latest interim — so the turn still reaches the LLM.
|
|
413
|
-
if (this.finalizeTimeoutFallback) {
|
|
334
|
+
if (this.cfg.finalizeTimeoutFallback) {
|
|
414
335
|
const state = this.transcriptState(contextId);
|
|
415
336
|
const fallbackText = this.combinedFinalTranscript(contextId) || state.lastInterimTranscript;
|
|
416
337
|
if (fallbackText) {
|
|
417
|
-
this.pushMetric(
|
|
418
|
-
|
|
338
|
+
this.pushMetric(
|
|
339
|
+
contextId,
|
|
340
|
+
"stt_provider_finalize_timeout_fallback",
|
|
341
|
+
this.audioStats(contextId),
|
|
342
|
+
);
|
|
419
343
|
this.ignoreNextProviderFinalContextIds.add(contextId);
|
|
420
|
-
this.
|
|
344
|
+
this.promoteFallbackFinal(
|
|
345
|
+
fallbackText,
|
|
346
|
+
state.finalConfidence || state.lastInterimConfidence,
|
|
347
|
+
contextId,
|
|
348
|
+
);
|
|
421
349
|
this.resetPendingTranscript(contextId);
|
|
422
350
|
return;
|
|
423
351
|
}
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
352
|
+
this.pushMetric(
|
|
353
|
+
contextId,
|
|
354
|
+
"stt_provider_finalize_timeout_empty_discard",
|
|
355
|
+
this.audioStats(contextId),
|
|
356
|
+
);
|
|
428
357
|
this.discardUnconfirmedTurn(contextId);
|
|
429
358
|
return;
|
|
430
359
|
}
|
|
431
360
|
|
|
432
361
|
this.discardUnconfirmedTurn(contextId);
|
|
433
|
-
this.
|
|
362
|
+
this.host?.emit({
|
|
363
|
+
type: "error",
|
|
434
364
|
contextId,
|
|
435
|
-
new Error(
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
// Deepgram-side context and cascades into more timeouts). Only reconnect once the
|
|
440
|
-
// failures repeat without a confirmed final between them (counter cleared in pushFinal).
|
|
365
|
+
error: new Error(
|
|
366
|
+
"Deepgram STT Finalize timed out before speech_final/from_finalize confirmation",
|
|
367
|
+
),
|
|
368
|
+
});
|
|
441
369
|
this.consecutiveFinalizeTimeouts += 1;
|
|
442
|
-
if (this.consecutiveFinalizeTimeouts >= this.finalizeResetThreshold) {
|
|
370
|
+
if (this.consecutiveFinalizeTimeouts >= this.cfg.finalizeResetThreshold) {
|
|
443
371
|
this.consecutiveFinalizeTimeouts = 0;
|
|
444
|
-
this.
|
|
372
|
+
this.host?.reset();
|
|
445
373
|
}
|
|
446
374
|
}
|
|
447
375
|
|
|
376
|
+
/** Fallback promote: result + optional eos via base (speechFinal: true). */
|
|
377
|
+
private promoteFallbackFinal(transcript: string, confidence: number, contextId: string): void {
|
|
378
|
+
this.resolveProviderFinalize(contextId);
|
|
379
|
+
boundedAdd(this.finalizedContextIds, contextId, MAX_RETIRED_CONTEXTS);
|
|
380
|
+
this.consecutiveFinalizeTimeouts = 0;
|
|
381
|
+
this.pushMetric(contextId, "stt_audio_sent", this.audioStats(contextId));
|
|
382
|
+
this.host?.emit({
|
|
383
|
+
type: "final",
|
|
384
|
+
contextId,
|
|
385
|
+
text: transcript,
|
|
386
|
+
confidence,
|
|
387
|
+
language: this.currentLanguage,
|
|
388
|
+
speechFinal: true,
|
|
389
|
+
provider: { name: "deepgram", model: this.cfg.model, region: "global" },
|
|
390
|
+
});
|
|
391
|
+
this.audioStatsByContextId.delete(contextId);
|
|
392
|
+
}
|
|
393
|
+
|
|
448
394
|
private discardUnconfirmedTurn(contextId: string): void {
|
|
449
395
|
this.clearProviderFinalizeTimer(contextId);
|
|
450
396
|
this.finalizeRequestedContextIds.delete(contextId);
|
|
@@ -455,83 +401,36 @@ export class DeepgramSTTPlugin implements VoicePlugin {
|
|
|
455
401
|
this.resetPendingTranscript(contextId);
|
|
456
402
|
}
|
|
457
403
|
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
const
|
|
461
|
-
|
|
462
|
-
|
|
404
|
+
private discardProviderStateForReconnect(): void {
|
|
405
|
+
const contextId = this.cfg.getContextId();
|
|
406
|
+
const discarded =
|
|
407
|
+
this.transcriptStateByContextId.size > 0 ||
|
|
408
|
+
this.finalizeRequestedContextIds.size > 0 ||
|
|
409
|
+
this.audioStatsByContextId.size > 0 ||
|
|
410
|
+
this.providerFinalizeTimers.size > 0 ||
|
|
411
|
+
this.providerFinalizeCorrelationTimers.size > 0;
|
|
412
|
+
this.clearAllProviderState({ keepFinalized: true });
|
|
413
|
+
// A reconnect (from any cause) starts a fresh provider stream, so the wedged-stream
|
|
414
|
+
// signal resets too — otherwise a stale count could force an avoidable reset on the
|
|
415
|
+
// first timeout after reconnecting.
|
|
463
416
|
this.consecutiveFinalizeTimeouts = 0;
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
if (this.emitEosOnFinal) {
|
|
468
|
-
this.bus?.push(Route.Main, {
|
|
469
|
-
kind: "eos.turn_complete",
|
|
470
|
-
contextId: ctxId,
|
|
471
|
-
timestampMs: Date.now(),
|
|
472
|
-
text: transcript,
|
|
473
|
-
transcripts: [],
|
|
474
|
-
});
|
|
417
|
+
if (discarded && contextId) {
|
|
418
|
+
this.pushMetric(contextId, "stt_provider_reconnect_discarded_state", {});
|
|
475
419
|
}
|
|
476
|
-
this.audioStatsByContextId.delete(ctxId);
|
|
477
|
-
}
|
|
478
|
-
|
|
479
|
-
private pushTurnComplete(contextId: string): void {
|
|
480
|
-
const transcript = this.combinedFinalTranscript(contextId);
|
|
481
|
-
if (!transcript || !this.bus) return;
|
|
482
|
-
this.resolveProviderFinalize(contextId);
|
|
483
|
-
boundedAdd(this.finalizedContextIds, contextId, MAX_RETIRED_CONTEXTS);
|
|
484
|
-
this.consecutiveFinalizeTimeouts = 0;
|
|
485
|
-
this.pushMetric(contextId, "stt_audio_sent", this.audioStats(contextId));
|
|
486
|
-
this.bus.push(Route.Main, {
|
|
487
|
-
kind: "eos.turn_complete",
|
|
488
|
-
contextId,
|
|
489
|
-
timestampMs: Date.now(),
|
|
490
|
-
text: transcript,
|
|
491
|
-
transcripts: [],
|
|
492
|
-
});
|
|
493
|
-
this.audioStatsByContextId.delete(contextId);
|
|
494
|
-
this.resetPendingTranscript(contextId);
|
|
495
|
-
}
|
|
496
|
-
|
|
497
|
-
private pushResult(
|
|
498
|
-
transcript: string,
|
|
499
|
-
confidence: number,
|
|
500
|
-
contextId = this.currentContextId,
|
|
501
|
-
provider: Record<string, unknown> = { name: "deepgram", model: this.model, region: "global" },
|
|
502
|
-
): void {
|
|
503
|
-
this.bus?.push(Route.Main, {
|
|
504
|
-
kind: "stt.result",
|
|
505
|
-
contextId,
|
|
506
|
-
timestampMs: Date.now(),
|
|
507
|
-
text: transcript,
|
|
508
|
-
confidence,
|
|
509
|
-
language: this.language,
|
|
510
|
-
provider,
|
|
511
|
-
});
|
|
512
420
|
}
|
|
513
421
|
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
this.
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
});
|
|
527
|
-
const wordTimings = mapProviderWordTimings(alt);
|
|
528
|
-
this.bus?.push(Route.Main, {
|
|
529
|
-
kind: "stt.partial",
|
|
530
|
-
contextId,
|
|
531
|
-
timestampMs,
|
|
532
|
-
text: transcript,
|
|
533
|
-
...(wordTimings ? { wordTimings } : {}),
|
|
534
|
-
});
|
|
422
|
+
private clearAllProviderState(opts: { keepFinalized: boolean }): void {
|
|
423
|
+
for (const timer of this.providerFinalizeTimers.values()) clearTimeout(timer);
|
|
424
|
+
this.providerFinalizeTimers.clear();
|
|
425
|
+
for (const timer of this.providerFinalizeCorrelationTimers.values()) clearTimeout(timer);
|
|
426
|
+
this.providerFinalizeCorrelationTimers.clear();
|
|
427
|
+
this.pendingProviderFinalizeContextIds = [];
|
|
428
|
+
this.finalizeRequestedContextIds.clear();
|
|
429
|
+
if (!opts.keepFinalized) this.finalizedContextIds.clear();
|
|
430
|
+
this.speechFinalContextIds.clear();
|
|
431
|
+
this.ignoreNextProviderFinalContextIds.clear();
|
|
432
|
+
this.transcriptStateByContextId.clear();
|
|
433
|
+
this.audioStatsByContextId.clear();
|
|
535
434
|
}
|
|
536
435
|
|
|
537
436
|
private appendFinalSegment(contextId: string, transcript: string, confidence: number): void {
|
|
@@ -545,17 +444,16 @@ export class DeepgramSTTPlugin implements VoicePlugin {
|
|
|
545
444
|
}
|
|
546
445
|
|
|
547
446
|
private combinedFinalTranscript(contextId: string): string {
|
|
548
|
-
return this.transcriptState(contextId)
|
|
447
|
+
return this.transcriptState(contextId)
|
|
448
|
+
.finalTranscriptParts.join(" ")
|
|
449
|
+
.replace(/\s+/g, " ")
|
|
450
|
+
.trim();
|
|
549
451
|
}
|
|
550
452
|
|
|
551
453
|
private resetPendingTranscript(contextId: string): void {
|
|
552
454
|
this.transcriptStateByContextId.delete(contextId);
|
|
553
455
|
}
|
|
554
456
|
|
|
555
|
-
private resetTurnTranscriptState(): void {
|
|
556
|
-
this.resetPendingTranscript(this.currentContextId);
|
|
557
|
-
}
|
|
558
|
-
|
|
559
457
|
private transcriptState(contextId: string): ProviderTranscriptState {
|
|
560
458
|
const existing = this.transcriptStateByContextId.get(contextId);
|
|
561
459
|
if (existing) return existing;
|
|
@@ -574,10 +472,13 @@ export class DeepgramSTTPlugin implements VoicePlugin {
|
|
|
574
472
|
return Boolean(state && state.finalTranscriptParts.length > 0);
|
|
575
473
|
}
|
|
576
474
|
|
|
577
|
-
private contextIdForProviderFinal(flags: {
|
|
475
|
+
private contextIdForProviderFinal(flags: {
|
|
476
|
+
readonly speechFinal: boolean;
|
|
477
|
+
readonly fromFinalize: boolean;
|
|
478
|
+
}): string {
|
|
578
479
|
const pending = this.pendingProviderFinalizeContextIds[0];
|
|
579
480
|
if (pending && (flags.speechFinal || flags.fromFinalize)) return pending;
|
|
580
|
-
return this.
|
|
481
|
+
return this.cfg.getContextId();
|
|
581
482
|
}
|
|
582
483
|
|
|
583
484
|
private trackPendingProviderFinalize(contextId: string): void {
|
|
@@ -587,17 +488,19 @@ export class DeepgramSTTPlugin implements VoicePlugin {
|
|
|
587
488
|
}
|
|
588
489
|
|
|
589
490
|
private removePendingProviderFinalize(contextId: string): void {
|
|
590
|
-
this.pendingProviderFinalizeContextIds = this.pendingProviderFinalizeContextIds.filter(
|
|
491
|
+
this.pendingProviderFinalizeContextIds = this.pendingProviderFinalizeContextIds.filter(
|
|
492
|
+
(ctxId) => ctxId !== contextId,
|
|
493
|
+
);
|
|
591
494
|
}
|
|
592
495
|
|
|
593
496
|
private scheduleProviderFinalizeCorrelationExpiry(contextId: string): void {
|
|
594
497
|
this.clearProviderFinalizeCorrelationTimer(contextId);
|
|
595
|
-
if (this.providerFinalizeTimeoutMs <= 0) return;
|
|
498
|
+
if (this.cfg.providerFinalizeTimeoutMs <= 0) return;
|
|
596
499
|
const timer = setTimeout(() => {
|
|
597
500
|
this.providerFinalizeCorrelationTimers.delete(contextId);
|
|
598
501
|
this.finalizeRequestedContextIds.delete(contextId);
|
|
599
502
|
this.removePendingProviderFinalize(contextId);
|
|
600
|
-
}, this.providerFinalizeTimeoutMs);
|
|
503
|
+
}, this.cfg.providerFinalizeTimeoutMs);
|
|
601
504
|
this.providerFinalizeCorrelationTimers.set(contextId, timer);
|
|
602
505
|
}
|
|
603
506
|
|
|
@@ -656,65 +559,13 @@ export class DeepgramSTTPlugin implements VoicePlugin {
|
|
|
656
559
|
return {
|
|
657
560
|
bytes: stats.bytes,
|
|
658
561
|
chunks: stats.chunks,
|
|
659
|
-
durationMs: Math.round((stats.bytes / 2 / this.sampleRate) * 1000),
|
|
562
|
+
durationMs: Math.round((stats.bytes / 2 / this.cfg.sampleRate) * 1000),
|
|
660
563
|
wallClockMs: stats.lastSentAtMs - stats.firstSentAtMs,
|
|
661
564
|
};
|
|
662
565
|
}
|
|
663
566
|
|
|
664
567
|
private pushMetric(contextId: string, name: string, value: unknown): void {
|
|
665
|
-
this.
|
|
666
|
-
kind: "metric.conversation",
|
|
667
|
-
contextId,
|
|
668
|
-
timestampMs: Date.now(),
|
|
669
|
-
name,
|
|
670
|
-
value: typeof value === "string" ? value : JSON.stringify(value),
|
|
671
|
-
});
|
|
672
|
-
}
|
|
673
|
-
|
|
674
|
-
private emitError(contextId: string, err: Error): void {
|
|
675
|
-
const category = categorizeSttError(err);
|
|
676
|
-
const packet: SttErrorPacket = {
|
|
677
|
-
kind: "stt.error",
|
|
678
|
-
contextId,
|
|
679
|
-
timestampMs: Date.now(),
|
|
680
|
-
component: "stt" as const,
|
|
681
|
-
category,
|
|
682
|
-
cause: err,
|
|
683
|
-
isRecoverable: isRecoverable(category),
|
|
684
|
-
};
|
|
685
|
-
this.bus?.push(Route.Critical, packet);
|
|
686
|
-
}
|
|
687
|
-
|
|
688
|
-
async close(): Promise<void> {
|
|
689
|
-
for (const dispose of this.disposers.splice(0)) dispose();
|
|
690
|
-
if (this.transcriptStateByContextId.size > 0) {
|
|
691
|
-
this.pushMetric(this.currentContextId, "stt_pending_transcript_discarded_on_close", this.audioStats(this.currentContextId));
|
|
692
|
-
}
|
|
693
|
-
for (const timer of this.providerFinalizeTimers.values()) clearTimeout(timer);
|
|
694
|
-
this.providerFinalizeTimers.clear();
|
|
695
|
-
for (const timer of this.providerFinalizeCorrelationTimers.values()) clearTimeout(timer);
|
|
696
|
-
this.providerFinalizeCorrelationTimers.clear();
|
|
697
|
-
this.pendingProviderFinalizeContextIds = [];
|
|
698
|
-
this.finalizeRequestedContextIds.clear();
|
|
699
|
-
this.finalizedContextIds.clear();
|
|
700
|
-
this.speechFinalContextIds.clear();
|
|
701
|
-
this.ignoreNextProviderFinalContextIds.clear();
|
|
702
|
-
this.transcriptStateByContextId.clear();
|
|
703
|
-
this.audioStatsByContextId.clear();
|
|
704
|
-
|
|
705
|
-
if (this.conn) {
|
|
706
|
-
// Graceful end-of-stream so Deepgram flushes and closes cleanly.
|
|
707
|
-
if (this.conn.isReady) {
|
|
708
|
-
try {
|
|
709
|
-
this.conn.send(JSON.stringify({ type: "CloseStream" }));
|
|
710
|
-
} catch {
|
|
711
|
-
// best effort
|
|
712
|
-
}
|
|
713
|
-
}
|
|
714
|
-
await this.conn.close();
|
|
715
|
-
this.conn = null;
|
|
716
|
-
}
|
|
717
|
-
this.bus = null;
|
|
568
|
+
this.cfg.onMetric(contextId, name, value);
|
|
718
569
|
}
|
|
719
570
|
|
|
720
571
|
private clearProviderFinalizeTimer(contextId: string): void {
|
|
@@ -730,38 +581,211 @@ export class DeepgramSTTPlugin implements VoicePlugin {
|
|
|
730
581
|
clearTimeout(timer);
|
|
731
582
|
this.providerFinalizeCorrelationTimers.delete(contextId);
|
|
732
583
|
}
|
|
584
|
+
}
|
|
733
585
|
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
586
|
+
export class DeepgramSTTPlugin implements VoicePlugin {
|
|
587
|
+
readonly endpointingCapability = {
|
|
588
|
+
owner: "provider_stt" as const,
|
|
589
|
+
disableConfig: {
|
|
590
|
+
emit_eos_on_final: false,
|
|
591
|
+
finalize_on_speech_final: false,
|
|
592
|
+
},
|
|
593
|
+
};
|
|
594
|
+
|
|
595
|
+
private bus: PipelineBus | null = null;
|
|
596
|
+
private session: StreamingSttSession | null = null;
|
|
597
|
+
private protocol: DeepgramSttWireProtocol | null = null;
|
|
598
|
+
private currentContextId = "";
|
|
599
|
+
private model = "nova-3";
|
|
600
|
+
private language = "en-US";
|
|
601
|
+
private sampleRate = 16000;
|
|
602
|
+
private endpointing = 300;
|
|
603
|
+
private endpointUrl = "wss://api.deepgram.com/v1/listen";
|
|
604
|
+
private smartFormat = true;
|
|
605
|
+
private interimResults = true;
|
|
606
|
+
private vadEvents = false;
|
|
607
|
+
private utteranceEndMs = 0;
|
|
608
|
+
private keyterms: readonly string[] = [];
|
|
609
|
+
private encoding = "linear16";
|
|
610
|
+
private channels = 1;
|
|
611
|
+
private noDelay = true;
|
|
612
|
+
private queryParams: Record<string, unknown> | undefined;
|
|
613
|
+
private disposers: Array<() => void> = [];
|
|
614
|
+
|
|
615
|
+
constructor(private readonly socketFactory?: SocketFactory) {}
|
|
616
|
+
|
|
617
|
+
async initialize(bus: PipelineBus, config: PluginConfig): Promise<void> {
|
|
618
|
+
this.bus = bus;
|
|
619
|
+
const apiKey = requireStringConfig(config, "api_key");
|
|
620
|
+
this.sampleRate = (config["sample_rate"] as number) ?? 16000;
|
|
621
|
+
this.model = optionalStringConfig(config, "model") ?? "nova-3";
|
|
622
|
+
this.language = optionalStringConfig(config, "language") ?? "en-US";
|
|
623
|
+
this.endpointing = (config["endpointing"] as number) ?? 300;
|
|
624
|
+
this.endpointUrl =
|
|
625
|
+
optionalStringConfig(config, "endpoint_url") ?? "wss://api.deepgram.com/v1/listen";
|
|
626
|
+
this.smartFormat = (config["smart_format"] as boolean) ?? true;
|
|
627
|
+
this.interimResults = (config["interim_results"] as boolean) ?? true;
|
|
628
|
+
// Opt-in: provider SpeechStarted → vad.speech_started is for VAD-less
|
|
629
|
+
// deployments (edge cascade). On sessions with a local VAD the duplicate,
|
|
630
|
+
// laggier speech-start signal corrupts VAD/EOS-owned turn-taking (the turn
|
|
631
|
+
// never completes) — proven by the Fly telephony spike.
|
|
632
|
+
this.vadEvents = (config["vad_events"] as boolean) ?? false;
|
|
633
|
+
{
|
|
634
|
+
const raw = (config["utterance_end_ms"] as number) ?? 0;
|
|
635
|
+
this.utteranceEndMs = raw > 0 ? Math.max(1000, raw) : 0;
|
|
636
|
+
}
|
|
637
|
+
const confidenceThreshold = (config["confidence_threshold"] as number) ?? 0;
|
|
638
|
+
const finalizeOnSpeechFinal = (config["finalize_on_speech_final"] as boolean) ?? true;
|
|
639
|
+
const emitEosOnFinal = (config["emit_eos_on_final"] as boolean) ?? true;
|
|
640
|
+
const providerFinalizeTimeoutMs = (config["provider_finalize_timeout_ms"] as number) ?? 1200;
|
|
641
|
+
const finalizeResetThreshold = (config["finalize_reset_threshold"] as number) ?? 2;
|
|
642
|
+
const finalizeTimeoutFallback = (config["finalize_timeout_fallback"] as boolean) ?? false;
|
|
643
|
+
const keepAliveIntervalMs = (config["keep_alive_interval_ms"] as number) ?? 3000;
|
|
644
|
+
{
|
|
645
|
+
const raw = config["keyterm"];
|
|
646
|
+
this.keyterms = Array.isArray(raw)
|
|
647
|
+
? raw.filter((t): t is string => typeof t === "string" && t.length > 0)
|
|
648
|
+
: typeof raw === "string" && raw.length > 0
|
|
649
|
+
? [raw]
|
|
650
|
+
: [];
|
|
757
651
|
}
|
|
652
|
+
this.encoding = optionalStringConfig(config, "encoding") ?? "linear16";
|
|
653
|
+
this.channels =
|
|
654
|
+
typeof config["channels"] === "number" && Number.isFinite(config["channels"])
|
|
655
|
+
? Math.max(1, Math.floor(config["channels"] as number))
|
|
656
|
+
: 1;
|
|
657
|
+
this.noDelay = (config["no_delay"] as boolean) ?? true;
|
|
658
|
+
this.queryParams = readPlainObject(config["query_params"]);
|
|
659
|
+
|
|
660
|
+
const audioFormat: AudioFormat = {
|
|
661
|
+
encoding: "pcm_s16le",
|
|
662
|
+
sampleRateHz: this.sampleRate,
|
|
663
|
+
channels: 1,
|
|
664
|
+
};
|
|
665
|
+
assertAudioFormat(audioFormat);
|
|
666
|
+
|
|
667
|
+
// Track context before the session handlers so encodeAudio sees the live turn id.
|
|
668
|
+
this.disposers.push(
|
|
669
|
+
bus.on("stt.audio", (pkt: unknown) => {
|
|
670
|
+
const audioPkt = pkt as { contextId?: string };
|
|
671
|
+
if (audioPkt.contextId) this.currentContextId = audioPkt.contextId;
|
|
672
|
+
}),
|
|
673
|
+
bus.on("turn.change", (pkt: unknown) => {
|
|
674
|
+
this.currentContextId = (pkt as { contextId: string }).contextId;
|
|
675
|
+
}),
|
|
676
|
+
bus.on("interrupt.stt", () => {
|
|
677
|
+
this.protocol?.resetTurnTranscriptState();
|
|
678
|
+
}),
|
|
679
|
+
);
|
|
680
|
+
|
|
681
|
+
this.protocol = new DeepgramSttWireProtocol({
|
|
682
|
+
model: this.model,
|
|
683
|
+
language: this.language,
|
|
684
|
+
sampleRate: this.sampleRate,
|
|
685
|
+
vadEvents: this.vadEvents,
|
|
686
|
+
confidenceThreshold,
|
|
687
|
+
finalizeOnSpeechFinal,
|
|
688
|
+
emitEosOnFinal,
|
|
689
|
+
providerFinalizeTimeoutMs,
|
|
690
|
+
finalizeResetThreshold,
|
|
691
|
+
finalizeTimeoutFallback,
|
|
692
|
+
getContextId: () => this.currentContextId,
|
|
693
|
+
onMetric: (contextId, name, value) => {
|
|
694
|
+
bus.push(Route.Background, {
|
|
695
|
+
kind: "metric.conversation",
|
|
696
|
+
contextId,
|
|
697
|
+
timestampMs: Date.now(),
|
|
698
|
+
name,
|
|
699
|
+
value: typeof value === "string" ? value : JSON.stringify(value),
|
|
700
|
+
});
|
|
701
|
+
},
|
|
702
|
+
});
|
|
703
|
+
|
|
704
|
+
this.session = await startStreamingSttSession(bus, {
|
|
705
|
+
protocol: this.protocol,
|
|
706
|
+
provider: { name: "deepgram", model: this.model, region: "global" },
|
|
707
|
+
format: audioFormat,
|
|
708
|
+
language: this.language,
|
|
709
|
+
emitEosOnFinal,
|
|
710
|
+
url: () => {
|
|
711
|
+
const params = new URLSearchParams({
|
|
712
|
+
encoding: this.encoding,
|
|
713
|
+
sample_rate: String(this.sampleRate),
|
|
714
|
+
interim_results: String(this.interimResults),
|
|
715
|
+
endpointing: String(this.endpointing),
|
|
716
|
+
smart_format: String(this.smartFormat),
|
|
717
|
+
model: this.model,
|
|
718
|
+
language: this.language,
|
|
719
|
+
channels: String(this.channels),
|
|
720
|
+
no_delay: String(this.noDelay),
|
|
721
|
+
vad_events: String(this.vadEvents),
|
|
722
|
+
...(this.utteranceEndMs > 0 ? { utterance_end_ms: String(this.utteranceEndMs) } : {}),
|
|
723
|
+
});
|
|
724
|
+
for (const term of this.keyterms) params.append("keyterm", term);
|
|
725
|
+
applyQueryParams(params, this.queryParams);
|
|
726
|
+
const separator = this.endpointUrl.includes("?") ? "&" : "?";
|
|
727
|
+
return `${this.endpointUrl}${separator}${params.toString()}`;
|
|
728
|
+
},
|
|
729
|
+
headers: { Authorization: `Token ${apiKey}` },
|
|
730
|
+
socketFactory: this.socketFactory ?? (await defaultNodeSocketFactory()),
|
|
731
|
+
retry: readProviderRetryConfig(config),
|
|
732
|
+
replayBufferSize: (config["replay_buffer_size"] as number) ?? 64,
|
|
733
|
+
keepAliveIntervalMs,
|
|
734
|
+
keepAliveMessage: () => JSON.stringify({ type: "KeepAlive" }),
|
|
735
|
+
metricPrefix: "stt.deepgram",
|
|
736
|
+
});
|
|
758
737
|
}
|
|
759
738
|
|
|
760
|
-
|
|
739
|
+
/**
|
|
740
|
+
* Per-turn reconfigure of keyterms / silence endpointing. Nova has no in-band
|
|
741
|
+
* Configure message — updates instance state then reconnects via `session.reset()`
|
|
742
|
+
* so the re-evaluated `url()` carries the new params (LiveKit Nova-3 pattern).
|
|
743
|
+
* Call only at a turn boundary (between utterances); reconnect is not free and
|
|
744
|
+
* would drop mid-utterance audio. keyterms REPLACE the list (Flux semantics).
|
|
745
|
+
* `language` hard-switches recognition (e.g. "es-ES" or Nova-3 "multi") — applied on
|
|
746
|
+
* reconnect via the rebuilt `url()`, and re-stamped onto stt.result. Flux-only fields
|
|
747
|
+
* (eotThreshold, eagerEotThreshold, eotTimeoutMs, contextText) are ignored.
|
|
748
|
+
*/
|
|
749
|
+
get sttReconfigure(): SttReconfigure {
|
|
750
|
+
return this;
|
|
751
|
+
}
|
|
752
|
+
|
|
753
|
+
reconfigure(partial: SttReconfigurePartial): void {
|
|
754
|
+
let changed = false;
|
|
755
|
+
if (partial.keyterms !== undefined) {
|
|
756
|
+
this.keyterms = partial.keyterms;
|
|
757
|
+
changed = true;
|
|
758
|
+
}
|
|
759
|
+
if (partial.endpointingMs !== undefined) {
|
|
760
|
+
this.endpointing = partial.endpointingMs;
|
|
761
|
+
changed = true;
|
|
762
|
+
}
|
|
763
|
+
if (partial.language !== undefined) {
|
|
764
|
+
this.language = partial.language;
|
|
765
|
+
this.protocol?.setLanguage(partial.language);
|
|
766
|
+
changed = true;
|
|
767
|
+
}
|
|
768
|
+
if (changed) this.session?.reset();
|
|
769
|
+
}
|
|
770
|
+
|
|
771
|
+
/** Request that Deepgram flush buffered audio and return provider-final text. */
|
|
772
|
+
forceFinalize(contextId?: string): void {
|
|
773
|
+
const ctxId = contextId ?? this.currentContextId;
|
|
774
|
+
if (!ctxId || !this.bus) return;
|
|
775
|
+
this.bus.push(Route.Main, {
|
|
776
|
+
kind: "stt.finalize",
|
|
777
|
+
contextId: ctxId,
|
|
778
|
+
timestampMs: Date.now(),
|
|
779
|
+
});
|
|
780
|
+
}
|
|
761
781
|
|
|
762
|
-
async
|
|
763
|
-
|
|
764
|
-
|
|
782
|
+
async close(): Promise<void> {
|
|
783
|
+
for (const dispose of this.disposers.splice(0)) dispose();
|
|
784
|
+
await this.session?.dispose();
|
|
785
|
+
this.session = null;
|
|
786
|
+
this.protocol = null;
|
|
787
|
+
this.bus = null;
|
|
788
|
+
}
|
|
765
789
|
}
|
|
766
790
|
|
|
767
791
|
function mapProviderWordTimings(
|
|
@@ -795,7 +819,7 @@ function providerAlternative(msg: Record<string, unknown>): Record<string, unkno
|
|
|
795
819
|
const alternatives = (channel as { alternatives?: unknown }).alternatives;
|
|
796
820
|
if (!Array.isArray(alternatives)) return null;
|
|
797
821
|
const first = alternatives[0];
|
|
798
|
-
return first && typeof first === "object" ? first as Record<string, unknown> : null;
|
|
822
|
+
return first && typeof first === "object" ? (first as Record<string, unknown>) : null;
|
|
799
823
|
}
|
|
800
824
|
|
|
801
825
|
function isDeepgramProviderError(msg: Record<string, unknown>): boolean {
|
|
@@ -811,7 +835,9 @@ function deepgramProviderError(msg: Record<string, unknown>): Error {
|
|
|
811
835
|
code ? `code=${code}` : "",
|
|
812
836
|
description,
|
|
813
837
|
requestId ? `request_id=${requestId}` : "",
|
|
814
|
-
]
|
|
838
|
+
]
|
|
839
|
+
.filter((part) => part.length > 0)
|
|
840
|
+
.join(" ");
|
|
815
841
|
return new Error(details ? `Deepgram STT provider error: ${details}` : "Deepgram STT provider error");
|
|
816
842
|
}
|
|
817
843
|
|
|
@@ -821,3 +847,26 @@ function firstString(...values: unknown[]): string {
|
|
|
821
847
|
}
|
|
822
848
|
return "";
|
|
823
849
|
}
|
|
850
|
+
|
|
851
|
+
function readPlainObject(value: unknown): Record<string, unknown> | undefined {
|
|
852
|
+
if (value !== null && typeof value === "object" && !Array.isArray(value)) {
|
|
853
|
+
return value as Record<string, unknown>;
|
|
854
|
+
}
|
|
855
|
+
return undefined;
|
|
856
|
+
}
|
|
857
|
+
|
|
858
|
+
/** Merge open-ended provider query knobs. Arrays append; scalars set (override). */
|
|
859
|
+
function applyQueryParams(params: URLSearchParams, extra: Record<string, unknown> | undefined): void {
|
|
860
|
+
if (!extra) return;
|
|
861
|
+
for (const [key, value] of Object.entries(extra)) {
|
|
862
|
+
if (value === undefined || value === null) continue;
|
|
863
|
+
if (Array.isArray(value)) {
|
|
864
|
+
for (const item of value) {
|
|
865
|
+
if (item === undefined || item === null) continue;
|
|
866
|
+
params.append(key, String(item));
|
|
867
|
+
}
|
|
868
|
+
continue;
|
|
869
|
+
}
|
|
870
|
+
params.set(key, String(value));
|
|
871
|
+
}
|
|
872
|
+
}
|