@kuralle-syrinx/deepgram 4.2.0 → 4.4.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@kuralle-syrinx/deepgram",
-  "version": "4.2.0",
+  "version": "4.4.0",
   "private": false,
   "description": "Deepgram adapters for Syrinx — nova-3 STT, Flux turn-aware STT (semantic end-of-turn), and Aura TTS",
   "keywords": [
@@ -31,14 +31,15 @@
     "./tts": "./src/tts.ts"
   },
   "dependencies": {
-    "@kuralle-syrinx/core": "4.2.0",
-    "@kuralle-syrinx/ws": "4.2.0"
+    "@kuralle-syrinx/core": "4.4.0",
+    "@kuralle-syrinx/stt-core": "4.4.0",
+    "@kuralle-syrinx/ws": "4.4.0"
   },
   "devDependencies": {
     "@types/ws": "^8.5.0",
     "typescript": "^5.7.0",
-    "vitest": "^2.1.0",
-    "ws": "^8.18.0"
+    "vitest": "^3.2.6",
+    "ws": "^8.21.0"
   },
   "scripts": {
     "typecheck": "tsc --noEmit",

package/src/flux.ts

@@ -7,7 +7,7 @@
 // is the semantic end-of-turn path for the edge cascade, where local ONNX
 // endpointers (smart-turn) cannot run.
 //
-// TurnInfo state machine → bus mapping:
+// TurnInfo state machine → bus mapping (via stt-core SttEvent):
 //   StartOfTurn     → vad.speech_started   (barge-in signal; Flux recommends it)
 //   Update          → stt.interim
 //   EagerEndOfTurn  → eos.interim          (speculative-generation trigger)
@@ -20,19 +20,29 @@
 // when no TurnResumed intervened, so a speculative result keyed on the eager
 // transcript can be committed as-is.
 
-import type { PipelineBus } from "@kuralle-syrinx/core";
 import {
   Route,
+  categorizeSttError,
+  isRecoverable,
+  type AudioFormat,
+  type PipelineBus,
   type PluginConfig,
   type SttErrorPacket,
+  type SttReconfigure,
+  type SttReconfigurePartial,
   type VoicePlugin,
-  categorizeSttError,
-  isRecoverable,
   optionalStringConfig,
   readProviderRetryConfig,
   requireStringConfig,
 } from "@kuralle-syrinx/core";
-import { WebSocketConnection, type SocketFactory } from "@kuralle-syrinx/ws";
+import {
+  defaultNodeSocketFactory,
+  startStreamingSttSession,
+  type SttEvent,
+  type SttWireProtocol,
+  type StreamingSttSession,
+} from "@kuralle-syrinx/stt-core";
+import type { SocketData, SocketFactory } from "@kuralle-syrinx/ws";
 
 interface TurnInfoMessage {
   readonly type: "TurnInfo";
@@ -49,6 +59,111 @@ function meanWordConfidence(words: TurnInfoMessage["words"]): number {
   return sum / words.length;
 }
 
+type MetricSink = (name: string, value: string) => void;
+
+class DeepgramFluxSttWireProtocol implements SttWireProtocol {
+  constructor(
+    private readonly speechStartedEvents: boolean,
+    private readonly onMetric: MetricSink,
+  ) {}
+
+  encodeFinalize(_contextId: string): readonly SocketData[] {
+    return [];
+  }
+
+  encodeClose(): readonly SocketData[] {
+    return [JSON.stringify({ type: "CloseStream" })];
+  }
+
+  encodeReconfigure(partial: SttReconfigurePartial): readonly SocketData[] {
+    const thresholds: Record<string, number> = {};
+    if (partial.eotThreshold !== undefined) thresholds["eot_threshold"] = partial.eotThreshold;
+    if (partial.eagerEotThreshold !== undefined) {
+      thresholds["eager_eot_threshold"] = partial.eagerEotThreshold;
+    }
+    if (partial.eotTimeoutMs !== undefined) thresholds["eot_timeout_ms"] = partial.eotTimeoutMs;
+
+    const configure: Record<string, unknown> = { type: "Configure" };
+    if (Object.keys(thresholds).length > 0) configure["thresholds"] = thresholds;
+    if (partial.keyterms !== undefined) configure["keyterms"] = partial.keyterms;
+    if (partial.languageHints !== undefined) configure["language_hints"] = partial.languageHints;
+
+    return [JSON.stringify(configure)];
+  }
+
+  decode(data: SocketData, _isBinary: boolean): readonly SttEvent[] {
+    if (typeof data !== "string") return [];
+    let msg: Record<string, unknown>;
+    try {
+      msg = JSON.parse(data) as Record<string, unknown>;
+    } catch (err) {
+      return [
+        {
+          type: "error",
+          error: new Error(
+            `Deepgram Flux sent malformed JSON: ${err instanceof Error ? err.message : String(err)}`,
+          ),
+        },
+      ];
+    }
+
+    if (msg["type"] === "Error") {
+      const description =
+        typeof msg["description"] === "string" ? msg["description"] : JSON.stringify(msg);
+      return [
+        {
+          type: "error",
+          error: new Error(`Deepgram Flux provider error: ${description}`),
+        },
+      ];
+    }
+    // Ack/nack for a mid-stream Configure (dynamic reconfigure). Surface as metrics so the
+    // conversation-state biasing that actuates reconfigure can observe that it took effect.
+    if (msg["type"] === "ConfigureSuccess") {
+      this.onMetric("stt.flux.configure_success", "1");
+      return [];
+    }
+    if (msg["type"] === "ConfigureFailure") {
+      const description =
+        typeof msg["description"] === "string" ? msg["description"] : JSON.stringify(msg);
+      this.onMetric("stt.flux.configure_failure", description);
+      return [];
+    }
+    if (msg["type"] !== "TurnInfo") return [];
+
+    const info = msg as unknown as TurnInfoMessage;
+    const transcript = (info.transcript ?? "").trim();
+
+    switch (info.event) {
+      case "StartOfTurn":
+        if (!this.speechStartedEvents) return [];
+        return [{ type: "speech_started" }];
+      case "Update":
+        if (!transcript) return [];
+        return [{ type: "interim", contextId: "", text: transcript }];
+      case "EagerEndOfTurn":
+        if (!transcript) return [];
+        return [{ type: "eos_interim", text: transcript }];
+      case "TurnResumed":
+        return [{ type: "eos_retracted" }];
+      case "EndOfTurn":
+        if (!transcript) return [];
+        return [
+          {
+            type: "final",
+            contextId: "",
+            text: transcript,
+            confidence: meanWordConfidence(info.words),
+            language: "en",
+            speechFinal: true,
+          },
+        ];
+      default:
+        return [];
+    }
+  }
+}
+
 export class DeepgramFluxSTTPlugin implements VoicePlugin {
   readonly endpointingCapability = {
     owner: "provider_stt" as const,
@@ -58,7 +173,7 @@ export class DeepgramFluxSTTPlugin implements VoicePlugin {
   };
 
   private bus: PipelineBus | null = null;
-  private apiKey = "";
+  private session: StreamingSttSession | null = null;
   private model = "flux-general-en";
   private endpointUrl = "wss://api.deepgram.com/v2/listen";
   private sampleRate = 16000;
@@ -68,11 +183,11 @@ export class DeepgramFluxSTTPlugin implements VoicePlugin {
   private keyterms: readonly string[] = [];
   private languageHints: readonly string[] = [];
   private speechStartedEvents = true;
-  private emitEosOnFinal = true;
-
-  private conn: WebSocketConnection | null = null;
+  private encoding = "linear16";
+  /** Open-ended Flux listen query knobs (profanity_filter, tag, mip_opt_out, …). */
+  private queryParams: Record<string, unknown> | undefined;
+  private apiKey = "";
   private currentContextId = "";
-  private disposers: Array<() => void> = [];
 
   constructor(private readonly socketFactory?: SocketFactory) {}
 
@@ -86,7 +201,7 @@ export class DeepgramFluxSTTPlugin implements VoicePlugin {
     this.eagerEotThreshold = config["eager_eot_threshold"] as number | undefined;
     this.eotTimeoutMs = (config["eot_timeout_ms"] as number) ?? 5000;
     this.speechStartedEvents = (config["speech_started_events"] as boolean) ?? true;
-    this.emitEosOnFinal = (config["emit_eos_on_final"] as boolean) ?? true;
+    const emitEosOnFinal = (config["emit_eos_on_final"] as boolean) ?? true;
     {
       const raw = config["keyterm"];
       this.keyterms = Array.isArray(raw)
@@ -103,13 +218,33 @@ export class DeepgramFluxSTTPlugin implements VoicePlugin {
           ? [raw]
           : [];
     }
+    this.encoding = optionalStringConfig(config, "encoding") ?? "linear16";
+    this.queryParams = readPlainObject(config["query_params"]);
 
-    const socketFactory = this.socketFactory ?? (await defaultSocketFactory());
-    this.conn = new WebSocketConnection({
+    const audioFormat: AudioFormat = {
+      encoding: "pcm_s16le",
+      sampleRateHz: this.sampleRate,
+      channels: 1,
+    };
+
+    this.session = await startStreamingSttSession(bus, {
+      protocol: new DeepgramFluxSttWireProtocol(this.speechStartedEvents, (name, value) => {
+        bus.push(Route.Background, {
+          kind: "metric.conversation",
+          contextId: "",
+          timestampMs: Date.now(),
+          name,
+          value,
+        });
+      }),
+      provider: { name: "deepgram", model: this.model, region: "global" },
+      format: audioFormat,
+      language: "en",
+      emitEosOnFinal,
       url: () => {
         const params = new URLSearchParams({
           model: this.model,
-          encoding: "linear16",
+          encoding: this.encoding,
           sample_rate: String(this.sampleRate),
           eot_threshold: String(this.eotThreshold),
           eot_timeout_ms: String(this.eotTimeoutMs),
@@ -119,151 +254,52 @@ export class DeepgramFluxSTTPlugin implements VoicePlugin {
         });
         for (const term of this.keyterms) params.append("keyterm", term);
         for (const hint of this.languageHints) params.append("language_hint", hint);
+        applyQueryParams(params, this.queryParams);
         const separator = this.endpointUrl.includes("?") ? "&" : "?";
         return `${this.endpointUrl}${separator}${params.toString()}`;
       },
       headers: { Authorization: `Token ${this.apiKey}` },
-      socketFactory,
+      socketFactory: this.socketFactory ?? (await defaultNodeSocketFactory()),
       retry: readProviderRetryConfig(config),
       replayBufferSize: (config["replay_buffer_size"] as number) ?? 64,
-      onReplay: (event, count) => {
-        this.pushMetric(this.currentContextId, `stt.flux.reconnect_replay_${event}`, String(count));
-      },
-      // No provider keepalive text message: the Flux v2 protocol keeps the turn
-      // model fed by continuous audio; transports stream silence frames between
-      // utterances, so an idle socket means the call itself has gone quiet.
-      onMessage: (data) => {
-        if (typeof data === "string") this.handleProviderMessage(data);
-      },
-      onConnectionLost: (err) => {
-        this.emitError(this.currentContextId, err);
-      },
+      metricPrefix: "stt.flux",
     });
-    await this.conn.connect();
+  }
 
-    this.disposers.push(
-      bus.on("stt.audio", async (pkt: unknown) => {
-        const audioPkt = pkt as { audio: Uint8Array; contextId?: string };
-        if (audioPkt.contextId) this.currentContextId = audioPkt.contextId;
-        if (!this.conn) return;
-        try {
-          await this.conn.ensureReady();
-          this.conn.send(audioPkt.audio);
-        } catch (err) {
-          this.emitError(this.currentContextId, err instanceof Error ? err : new Error(String(err)));
-        }
-      }),
-      bus.on("turn.change", (pkt: unknown) => {
-        const tc = pkt as { contextId: string };
-        this.currentContextId = tc.contextId;
-      }),
-    );
+  /**
+   * Mid-stream reconfigure of keyterms / end-of-turn thresholds via the Flux v2 `Configure`
+   * control message — no socket restart (Deepgram "on-the-fly configuration"). Instance state is
+   * updated too, so a reconnect replays the current config (single source of truth with `url()`).
+   * `keyterms` REPLACES the list (Flux semantics), not merges. The provider acks with
+   * ConfigureSuccess / ConfigureFailure (surfaced as metrics in the wire protocol).
+   * `language` (hard switch) is ignored: Flux's model is language-specific (flux-general-en) —
+   * use `languageHints` to bias.
+   */
+  get sttReconfigure(): SttReconfigure {
+    return this;
   }
 
-  private handleProviderMessage(data: string): void {
-    let msg: Record<string, unknown>;
+  reconfigure(partial: SttReconfigurePartial): void {
+    if (partial.keyterms !== undefined) this.keyterms = partial.keyterms;
+    if (partial.eotThreshold !== undefined) this.eotThreshold = partial.eotThreshold;
+    if (partial.eagerEotThreshold !== undefined) this.eagerEotThreshold = partial.eagerEotThreshold;
+    if (partial.eotTimeoutMs !== undefined) this.eotTimeoutMs = partial.eotTimeoutMs;
+    if (partial.languageHints !== undefined) this.languageHints = partial.languageHints;
+
     try {
-      msg = JSON.parse(data) as Record<string, unknown>;
+      this.session?.reconfigure(partial);
     } catch (err) {
-      this.emitError(
-        this.currentContextId,
-        new Error(`Deepgram Flux sent malformed JSON: ${err instanceof Error ? err.message : String(err)}`),
-      );
-      return;
-    }
-
-    if (msg["type"] === "Error") {
-      const description = typeof msg["description"] === "string" ? msg["description"] : JSON.stringify(msg);
-      this.emitError(this.currentContextId, new Error(`Deepgram Flux provider error: ${description}`));
-      return;
+      this.emitError(err instanceof Error ? err : new Error(String(err)));
     }
-    if (msg["type"] !== "TurnInfo") return;
-
-    const info = msg as unknown as TurnInfoMessage;
-    const contextId = this.currentContextId;
-    const transcript = (info.transcript ?? "").trim();
-
-    switch (info.event) {
-      case "StartOfTurn": {
-        if (!this.speechStartedEvents) return;
-        this.bus?.push(Route.Main, {
-          kind: "vad.speech_started",
-          contextId,
-          timestampMs: Date.now(),
-          confidence: 1,
-        });
-        return;
-      }
-      case "Update": {
-        if (!transcript) return;
-        this.bus?.push(Route.Main, {
-          kind: "stt.interim",
-          contextId,
-          timestampMs: Date.now(),
-          text: transcript,
-        });
-        return;
-      }
-      case "EagerEndOfTurn": {
-        if (!transcript) return;
-        this.bus?.push(Route.Main, {
-          kind: "eos.interim",
-          contextId,
-          timestampMs: Date.now(),
-          text: transcript,
-        });
-        return;
-      }
-      case "TurnResumed": {
-        this.bus?.push(Route.Main, {
-          kind: "eos.retracted",
-          contextId,
-          timestampMs: Date.now(),
-        });
-        return;
-      }
-      case "EndOfTurn": {
-        if (!transcript) return;
-        this.bus?.push(Route.Main, {
-          kind: "stt.result",
-          contextId,
-          timestampMs: Date.now(),
-          text: transcript,
-          confidence: meanWordConfidence(info.words),
-          language: "en",
-          provider: { name: "deepgram", model: this.model, region: "global" },
-        });
-        if (this.emitEosOnFinal) {
-          this.bus?.push(Route.Main, {
-            kind: "eos.turn_complete",
-            contextId,
-            timestampMs: Date.now(),
-            text: transcript,
-            transcripts: [],
-          });
-        }
-        return;
-      }
-    }
-  }
-
-  private pushMetric(contextId: string, name: string, value: string): void {
-    this.bus?.push(Route.Background, {
-      kind: "metric.conversation",
-      contextId,
-      timestampMs: Date.now(),
-      name,
-      value,
-    });
   }
 
-  private emitError(contextId: string, err: Error): void {
+  private emitError(err: Error): void {
     const category = categorizeSttError(err);
     const packet: SttErrorPacket = {
       kind: "stt.error",
-      contextId,
+      contextId: this.currentContextId,
       timestampMs: Date.now(),
-      component: "stt" as const,
+      component: "stt",
       category,
       cause: err,
       isRecoverable: isRecoverable(category),
@@ -272,23 +308,30 @@ export class DeepgramFluxSTTPlugin implements VoicePlugin {
   }
 
   async close(): Promise<void> {
-    for (const dispose of this.disposers.splice(0)) dispose();
-    if (this.conn) {
-      if (this.conn.isReady) {
-        try {
-          this.conn.send(JSON.stringify({ type: "CloseStream" }));
-        } catch {
-          // Socket already going away — CloseStream is best-effort.
-        }
-      }
-      await this.conn.close();
-      this.conn = null;
-    }
+    await this.session?.dispose();
+    this.session = null;
     this.bus = null;
   }
 }
 
-async function defaultSocketFactory(): Promise<SocketFactory> {
-  const mod = await import("@kuralle-syrinx/ws/node");
-  return mod.createNodeWsSocket;
+function readPlainObject(value: unknown): Record<string, unknown> | undefined {
+  if (value !== null && typeof value === "object" && !Array.isArray(value)) {
+    return value as Record<string, unknown>;
+  }
+  return undefined;
+}
+
+function applyQueryParams(params: URLSearchParams, extra: Record<string, unknown> | undefined): void {
+  if (!extra) return;
+  for (const [key, value] of Object.entries(extra)) {
+    if (value === undefined || value === null) continue;
+    if (Array.isArray(value)) {
+      for (const item of value) {
+        if (item === undefined || item === null) continue;
+        params.append(key, String(item));
+      }
+      continue;
+    }
+    params.set(key, String(value));
+  }
 }