@kuralle-syrinx/deepgram 3.0.0 → 4.0.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@kuralle-syrinx/deepgram",
-  "version": "3.0.0",
+  "version": "4.0.0",
   "private": false,
   "type": "module",
   "license": "MIT",
@@ -12,8 +12,8 @@
     "./tts": "./src/tts.ts"
   },
   "dependencies": {
-    "@kuralle-syrinx/core": "3.0.0",
-    "@kuralle-syrinx/ws": "3.0.0"
+    "@kuralle-syrinx/core": "4.0.0",
+    "@kuralle-syrinx/ws": "4.0.0"
   },
   "devDependencies": {
     "@types/ws": "^8.5.0",

package/src/stt.test.ts

@@ -175,6 +175,47 @@ describe("DeepgramSTTPlugin", () => {
     await started;
   });
 
+  it("completes a wedged turn via UtteranceEnd when speech_final never fires (noisy-line backstop)", async () => {
+    // A noisy/continuous line: Deepgram sends an is_final segment but never
+    // speech_final. UtteranceEnd (gap-based) must complete the turn so it can't wedge.
+    const endpointUrl = await createLocalServer((socket) => {
+      socket.on("message", (data, isBinary) => {
+        if (isBinary) {
+          socket.send(JSON.stringify({
+            is_final: true,
+            speech_final: false, // never speech_final on this noisy line
+            channel: { alternatives: [{ transcript: "book a room", confidence: 0.9 }] },
+          }));
+          setTimeout(() => socket.send(JSON.stringify({ type: "UtteranceEnd", last_word_end: 1.2 })), 5);
+          return;
+        }
+      });
+    });
+    const bus = new PipelineBusImpl();
+    const started = startBus(bus);
+    const plugin = new DeepgramSTTPlugin();
+    const turnCompletes: Array<{ text: string; contextId: string }> = [];
+    bus.on("eos.turn_complete", (pkt) => {
+      const p = pkt as unknown as { text: string; contextId: string };
+      turnCompletes.push({ text: p.text, contextId: p.contextId });
+    });
+
+    await plugin.initialize(bus, {
+      api_key: "test",
+      endpoint_url: endpointUrl,
+      sample_rate: 16000,
+      utterance_end_ms: 1000,
+    });
+    bus.push(Route.Main, { kind: "stt.audio", contextId: "turn-1", timestampMs: Date.now(), audio: new Uint8Array(640) });
+    await new Promise((resolve) => setTimeout(resolve, 40));
+
+    expect(turnCompletes).toEqual([{ text: "book a room", contextId: "turn-1" }]);
+
+    await plugin.close();
+    bus.stop();
+    await started;
+  });
+
   it("emits trailing provider-final segments after Pipecat requests finalize", async () => {
     const endpointUrl = await createLocalServer((socket) => {
       socket.on("message", (data, isBinary) => {

package/src/stt.ts

@@ -42,6 +42,24 @@ interface ProviderTranscriptState {
   finalConfidence: number;
 }
 
+/**
+ * A retired-contextId set (finalized turns) must stay bounded: contextIds are
+ * per-turn on every transport (telephony rotates `<callSid>-t<n>`), so an
+ * unbounded set would grow one entry per turn for the life of a call. Keeping a
+ * recent-turns window is enough to reject the late/duplicate provider finals the
+ * set exists to guard against, without leaking memory over a long conversation.
+ */
+const MAX_RETIRED_CONTEXTS = 256;
+
+function boundedAdd(set: Set<string>, value: string, cap: number): void {
+  set.add(value);
+  while (set.size > cap) {
+    const oldest = set.values().next().value;
+    if (oldest === undefined) break;
+    set.delete(oldest);
+  }
+}
+
 export class DeepgramSTTPlugin implements VoicePlugin {
   readonly endpointingCapability = {
     owner: "provider_stt" as const,
@@ -61,6 +79,10 @@ export class DeepgramSTTPlugin implements VoicePlugin {
   private smartFormat: boolean = true;
   private interimResults: boolean = true;
   private vadEvents: boolean = true;
+  // Deepgram gap-based utterance boundary (ms). When > 0 (requires interim_results),
+  // Deepgram emits UtteranceEnd even when `speech_final` never fires — the documented
+  // backstop for noisy/continuous lines (telephony) where a turn would otherwise wedge.
+  private utteranceEndMs: number = 0;
   private confidenceThreshold: number = 0;
   private finalizeOnSpeechFinal: boolean = true;
   private emitEosOnFinal: boolean = true;
@@ -117,6 +139,11 @@ export class DeepgramSTTPlugin implements VoicePlugin {
     // laggier speech-start signal corrupts VAD/EOS-owned turn-taking (the turn
     // never completes) — proven by the Fly telephony spike.
     this.vadEvents = (config["vad_events"] as boolean) ?? false;
+    // Deepgram requires utterance_end_ms >= 1000 and interim_results; clamp to that.
+    {
+      const raw = (config["utterance_end_ms"] as number) ?? 0;
+      this.utteranceEndMs = raw > 0 ? Math.max(1000, raw) : 0;
+    }
     this.confidenceThreshold =
       (config["confidence_threshold"] as number) ?? 0;
     this.finalizeOnSpeechFinal = (config["finalize_on_speech_final"] as boolean) ?? true;
@@ -142,6 +169,7 @@ export class DeepgramSTTPlugin implements VoicePlugin {
           channels: "1",
           no_delay: "true",
           vad_events: String(this.vadEvents),
+          ...(this.utteranceEndMs > 0 ? { utterance_end_ms: String(this.utteranceEndMs) } : {}),
         });
         const separator = this.endpointUrl.includes("?") ? "&" : "?";
         return `${this.endpointUrl}${separator}${params.toString()}`;
@@ -231,6 +259,30 @@ export class DeepgramSTTPlugin implements VoicePlugin {
       return;
     }
 
+    // Deepgram's gap-based utterance boundary (utterance_end_ms). This fires even
+    // when `speech_final` is missed on a noisy/continuous line, so it's the backstop
+    // that stops a turn from wedging forever. If we hold un-finalized transcript for
+    // the current turn, complete it.
+    if (msg["type"] === "UtteranceEnd") {
+      const ctxId = this.currentContextId;
+      if (
+        this.emitEosOnFinal &&
+        ctxId &&
+        !this.finalizedContextIds.has(ctxId) &&
+        this.combinedFinalTranscript(ctxId)
+      ) {
+        this.bus?.push(Route.Background, {
+          kind: "metric.conversation",
+          contextId: ctxId,
+          timestampMs: Date.now(),
+          name: "deepgram.utterance_end_backstop",
+          value: "1",
+        });
+        this.pushTurnComplete(ctxId);
+      }
+      return;
+    }
+
     const alt = providerAlternative(msg);
     if (!alt || typeof alt["transcript"] !== "string") return;
 
@@ -395,7 +447,7 @@ export class DeepgramSTTPlugin implements VoicePlugin {
   private pushFinal(transcript: string, confidence: number, contextId = this.currentContextId): void {
     const ctxId = contextId;
     this.resolveProviderFinalize(ctxId);
-    this.finalizedContextIds.add(ctxId);
+    boundedAdd(this.finalizedContextIds, ctxId, MAX_RETIRED_CONTEXTS);
     this.consecutiveFinalizeTimeouts = 0;
     this.pushMetric(ctxId, "stt_audio_sent", this.audioStats(ctxId));
     this.pushResult(transcript, confidence, ctxId);
@@ -416,7 +468,7 @@ export class DeepgramSTTPlugin implements VoicePlugin {
     const transcript = this.combinedFinalTranscript(contextId);
     if (!transcript || !this.bus) return;
     this.resolveProviderFinalize(contextId);
-    this.finalizedContextIds.add(contextId);
+    boundedAdd(this.finalizedContextIds, contextId, MAX_RETIRED_CONTEXTS);
     this.consecutiveFinalizeTimeouts = 0;
     this.pushMetric(contextId, "stt_audio_sent", this.audioStats(contextId));
     this.bus.push(Route.Main, {

package/src/tts.ts

@@ -40,6 +40,19 @@ const FLUSH_MSG = JSON.stringify({ type: "Flush" });
 const CLEAR_MSG = JSON.stringify({ type: "Clear" });
 const EMPTY = new Uint8Array(0);
 const KEEP_ALIVE_INTERVAL_MS = 10_000;
+// Retired-turn guard: contextIds are per-turn (telephony rotates), so cancelled
+// ids accumulate one per barge-in. Keep a recent-turns window to reject late
+// post-cancel audio without leaking over a long call.
+const MAX_CANCELLED_CONTEXTS = 256;
+
+function boundedAdd(set: Set<string>, value: string, cap: number): void {
+  set.add(value);
+  while (set.size > cap) {
+    const oldest = set.values().next().value;
+    if (oldest === undefined) break;
+    set.delete(oldest);
+  }
+}
 
 export class DeepgramTTSPlugin implements VoicePlugin {
   // socketFactory is injectable so the same plugin runs on Node (default) or
@@ -165,7 +178,7 @@ export class DeepgramTTSPlugin implements VoicePlugin {
 
   private async cancelActiveContexts(): Promise<void> {
     const contextIds = [...this.activeContexts];
-    for (const contextId of contextIds) this.cancelledContexts.add(contextId);
+    for (const contextId of contextIds) boundedAdd(this.cancelledContexts, contextId, MAX_CANCELLED_CONTEXTS);
     this.activeContexts.clear();
     for (const contextId of contextIds) this.clearFinishTimeout(contextId);
     this.currentContextId = "";