@kuralle-syrinx/deepgram 4.0.0 → 4.1.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@kuralle-syrinx/deepgram",
-  "version": "4.0.0",
+  "version": "4.1.0",
   "private": false,
   "type": "module",
   "license": "MIT",
@@ -12,8 +12,8 @@
     "./tts": "./src/tts.ts"
   },
   "dependencies": {
-    "@kuralle-syrinx/core": "4.0.0",
-    "@kuralle-syrinx/ws": "4.0.0"
+    "@kuralle-syrinx/core": "4.1.0",
+    "@kuralle-syrinx/ws": "4.1.0"
   },
   "devDependencies": {
     "@types/ws": "^8.5.0",

package/src/flux.test.ts

@@ -0,0 +1,227 @@
+// SPDX-License-Identifier: MIT
+
+import { afterEach, describe, expect, it } from "vitest";
+import { WebSocketServer, type WebSocket } from "ws";
+import {
+  PipelineBusImpl,
+  Route,
+  type EndOfSpeechPacket,
+  type EndOfSpeechRetractedPacket,
+  type InterimEndOfSpeechPacket,
+  type SttInterimPacket,
+  type SttResultPacket,
+} from "@kuralle-syrinx/core";
+
+import { DeepgramFluxSTTPlugin } from "./flux.js";
+
+let servers: WebSocketServer[] = [];
+
+afterEach(async () => {
+  await Promise.all(
+    servers.splice(0).map(
+      (server) =>
+        new Promise<void>((resolve) => {
+          for (const client of server.clients) client.terminate();
+          server.close(() => resolve());
+        }),
+    ),
+  );
+});
+
+interface LocalServer {
+  endpointUrl: string;
+  connectionUrls: string[];
+  sockets: WebSocket[];
+}
+
+async function createLocalServer(): Promise<LocalServer> {
+  const server = await new Promise<WebSocketServer>((resolve) => {
+    let nextServer: WebSocketServer;
+    nextServer = new WebSocketServer({ port: 0 }, () => resolve(nextServer));
+  });
+  servers.push(server);
+  const state: LocalServer = { endpointUrl: "", connectionUrls: [], sockets: [] };
+  server.on("connection", (socket, req) => {
+    state.connectionUrls.push(req.url ?? "");
+    state.sockets.push(socket);
+  });
+  const address = server.address();
+  if (!address || typeof address === "string") throw new Error("Expected TCP server address");
+  state.endpointUrl = `ws://127.0.0.1:${address.port}/v2/listen`;
+  return state;
+}
+
+async function waitFor<T>(items: T[], count = 1): Promise<void> {
+  for (let i = 0; i < 100; i += 1) {
+    if (items.length >= count) return;
+    await new Promise((resolve) => setTimeout(resolve, 10));
+  }
+}
+
+function turnInfo(event: string, extra: Record<string, unknown> = {}): string {
+  return JSON.stringify({
+    type: "TurnInfo",
+    event,
+    turn_index: 0,
+    audio_window_start: 0,
+    audio_window_end: 0.6,
+    transcript: "",
+    words: [],
+    end_of_turn_confidence: 0.5,
+    ...extra,
+  });
+}
+
+async function startPlugin(
+  local: LocalServer,
+  config: Record<string, unknown> = {},
+): Promise<{ bus: PipelineBusImpl; plugin: DeepgramFluxSTTPlugin; started: Promise<void> }> {
+  const bus = new PipelineBusImpl();
+  const started = bus.start();
+  const plugin = new DeepgramFluxSTTPlugin();
+  await plugin.initialize(bus, {
+    api_key: "test",
+    endpoint_url: local.endpointUrl,
+    sample_rate: 16000,
+    ...config,
+  });
+  await waitFor(local.sockets);
+  // Feed one audio packet so the plugin learns the current contextId.
+  bus.push(Route.Main, {
+    kind: "stt.audio",
+    contextId: "turn-1",
+    timestampMs: Date.now(),
+    audio: new Uint8Array(320),
+  });
+  await new Promise((resolve) => setTimeout(resolve, 20));
+  return { bus, plugin, started };
+}
+
+describe("DeepgramFluxSTTPlugin", () => {
+  it("connects with Flux params and forwards keyterm + eager threshold", async () => {
+    const local = await createLocalServer();
+    const { bus, plugin, started } = await startPlugin(local, {
+      eot_threshold: 0.85,
+      eager_eot_threshold: 0.4,
+      eot_timeout_ms: 7000,
+      keyterm: ["Syrinx"],
+    });
+
+    await plugin.close();
+    bus.stop();
+    await started;
+
+    const url = local.connectionUrls[0]!;
+    expect(url).toContain("model=flux-general-en");
+    expect(url).toContain("encoding=linear16");
+    expect(url).toContain("sample_rate=16000");
+    expect(url).toContain("eot_threshold=0.85");
+    expect(url).toContain("eager_eot_threshold=0.4");
+    expect(url).toContain("eot_timeout_ms=7000");
+    expect(url).toContain("keyterm=Syrinx");
+  });
+
+  it("omits eager_eot_threshold by default (eager mode is opt-in)", async () => {
+    const local = await createLocalServer();
+    const { bus, plugin, started } = await startPlugin(local);
+
+    await plugin.close();
+    bus.stop();
+    await started;
+
+    const url = local.connectionUrls[0]!;
+    expect(url).toContain("eot_threshold=0.7");
+    expect(url).not.toContain("eager_eot_threshold=");
+  });
+
+  it("maps TurnInfo events onto the bus: Update→stt.interim, EndOfTurn→stt.result+eos.turn_complete", async () => {
+    const local = await createLocalServer();
+    const { bus, plugin, started } = await startPlugin(local);
+
+    const interims: SttInterimPacket[] = [];
+    const results: SttResultPacket[] = [];
+    const turnCompletes: EndOfSpeechPacket[] = [];
+    bus.on("stt.interim", (pkt) => {
+      interims.push(pkt as SttInterimPacket);
+    });
+    bus.on("stt.result", (pkt) => {
+      results.push(pkt as SttResultPacket);
+    });
+    bus.on("eos.turn_complete", (pkt) => {
+      turnCompletes.push(pkt as EndOfSpeechPacket);
+    });
+
+    const socket = local.sockets[0]!;
+    socket.send(turnInfo("Update", { transcript: "what are" }));
+    socket.send(
+      turnInfo("EndOfTurn", {
+        transcript: "what are the lab fees",
+        words: [
+          { word: "what", confidence: 0.99 },
+          { word: "are", confidence: 0.97 },
+        ],
+        end_of_turn_confidence: 0.91,
+      }),
+    );
+    await waitFor(turnCompletes);
+
+    await plugin.close();
+    bus.stop();
+    await started;
+
+    expect(interims.map((p) => p.text)).toContain("what are");
+    expect(results).toHaveLength(1);
+    expect(results[0]!.text).toBe("what are the lab fees");
+    expect(results[0]!.confidence).toBeCloseTo(0.98, 2);
+    expect(turnCompletes).toEqual([
+      expect.objectContaining({ contextId: "turn-1", text: "what are the lab fees" }),
+    ]);
+  });
+
+  it("emits eos.interim on EagerEndOfTurn and eos.retracted on TurnResumed", async () => {
+    const local = await createLocalServer();
+    const { bus, plugin, started } = await startPlugin(local, { eager_eot_threshold: 0.4 });
+
+    const eagers: InterimEndOfSpeechPacket[] = [];
+    const retractions: EndOfSpeechRetractedPacket[] = [];
+    bus.on("eos.interim", (pkt) => {
+      eagers.push(pkt as InterimEndOfSpeechPacket);
+    });
+    bus.on("eos.retracted", (pkt) => {
+      retractions.push(pkt as EndOfSpeechRetractedPacket);
+    });
+
+    const socket = local.sockets[0]!;
+    socket.send(turnInfo("EagerEndOfTurn", { transcript: "book a room", end_of_turn_confidence: 0.55 }));
+    await waitFor(eagers);
+    socket.send(turnInfo("TurnResumed"));
+    await waitFor(retractions);
+
+    await plugin.close();
+    bus.stop();
+    await started;
+
+    expect(eagers).toEqual([expect.objectContaining({ contextId: "turn-1", text: "book a room" })]);
+    expect(retractions).toEqual([expect.objectContaining({ contextId: "turn-1" })]);
+  });
+
+  it("emits vad.speech_started on StartOfTurn (Flux owns barge-in signalling)", async () => {
+    const local = await createLocalServer();
+    const { bus, plugin, started } = await startPlugin(local);
+
+    const speechStarts: unknown[] = [];
+    bus.on("vad.speech_started", (pkt) => {
+      speechStarts.push(pkt);
+    });
+
+    const socket = local.sockets[0]!;
+    socket.send(turnInfo("StartOfTurn", { transcript: "hello" }));
+    await waitFor(speechStarts);
+
+    await plugin.close();
+    bus.stop();
+    await started;
+
+    expect(speechStarts.length).toBe(1);
+  });
+});

package/src/flux.ts

@@ -0,0 +1,294 @@
+// SPDX-License-Identifier: MIT
+//
+// Deepgram Flux — turn-aware conversational STT (v2 listen API).
+//
+// One model produces transcripts AND owns turn detection, replacing the
+// VAD + silence-endpointing stack. Runs on Workers (plain WebSocket), so this
+// is the semantic end-of-turn path for the edge cascade, where local ONNX
+// endpointers (smart-turn) cannot run.
+//
+// TurnInfo state machine → bus mapping:
+//   StartOfTurn     → vad.speech_started   (barge-in signal; Flux recommends it)
+//   Update          → stt.interim
+//   EagerEndOfTurn  → eos.interim          (speculative-generation trigger)
+//   TurnResumed     → eos.retracted        (cancel speculative work)
+//   EndOfTurn       → stt.result + eos.turn_complete
+//
+// Eager mode is enabled by setting `eager_eot_threshold` (Deepgram: fires
+// 150–250ms before EndOfTurn at the cost of extra speculative LLM calls). The
+// EndOfTurn transcript exactly matches the preceding EagerEndOfTurn transcript
+// when no TurnResumed intervened, so a speculative result keyed on the eager
+// transcript can be committed as-is.
+
+import type { PipelineBus } from "@kuralle-syrinx/core";
+import {
+  Route,
+  type PluginConfig,
+  type SttErrorPacket,
+  type VoicePlugin,
+  categorizeSttError,
+  isRecoverable,
+  optionalStringConfig,
+  readProviderRetryConfig,
+  requireStringConfig,
+} from "@kuralle-syrinx/core";
+import { WebSocketConnection, type SocketFactory } from "@kuralle-syrinx/ws";
+
+interface TurnInfoMessage {
+  readonly type: "TurnInfo";
+  readonly event: "StartOfTurn" | "Update" | "EagerEndOfTurn" | "TurnResumed" | "EndOfTurn";
+  readonly transcript?: string;
+  readonly words?: ReadonlyArray<{ readonly word: string; readonly confidence: number }>;
+  readonly end_of_turn_confidence?: number;
+}
+
+function meanWordConfidence(words: TurnInfoMessage["words"]): number {
+  if (!words || words.length === 0) return 1;
+  let sum = 0;
+  for (const w of words) sum += w.confidence;
+  return sum / words.length;
+}
+
+export class DeepgramFluxSTTPlugin implements VoicePlugin {
+  readonly endpointingCapability = {
+    owner: "provider_stt" as const,
+    disableConfig: {
+      emit_eos_on_final: false,
+    },
+  };
+
+  private bus: PipelineBus | null = null;
+  private apiKey = "";
+  private model = "flux-general-en";
+  private endpointUrl = "wss://api.deepgram.com/v2/listen";
+  private sampleRate = 16000;
+  private eotThreshold = 0.7;
+  private eagerEotThreshold: number | undefined;
+  private eotTimeoutMs = 5000;
+  private keyterms: readonly string[] = [];
+  private languageHints: readonly string[] = [];
+  private speechStartedEvents = true;
+  private emitEosOnFinal = true;
+
+  private conn: WebSocketConnection | null = null;
+  private currentContextId = "";
+  private disposers: Array<() => void> = [];
+
+  constructor(private readonly socketFactory?: SocketFactory) {}
+
+  async initialize(bus: PipelineBus, config: PluginConfig): Promise<void> {
+    this.bus = bus;
+    this.apiKey = requireStringConfig(config, "api_key");
+    this.model = optionalStringConfig(config, "model") ?? "flux-general-en";
+    this.endpointUrl = optionalStringConfig(config, "endpoint_url") ?? "wss://api.deepgram.com/v2/listen";
+    this.sampleRate = (config["sample_rate"] as number) ?? 16000;
+    this.eotThreshold = (config["eot_threshold"] as number) ?? 0.7;
+    this.eagerEotThreshold = config["eager_eot_threshold"] as number | undefined;
+    this.eotTimeoutMs = (config["eot_timeout_ms"] as number) ?? 5000;
+    this.speechStartedEvents = (config["speech_started_events"] as boolean) ?? true;
+    this.emitEosOnFinal = (config["emit_eos_on_final"] as boolean) ?? true;
+    {
+      const raw = config["keyterm"];
+      this.keyterms = Array.isArray(raw)
+        ? raw.filter((t): t is string => typeof t === "string" && t.length > 0)
+        : typeof raw === "string" && raw.length > 0
+          ? [raw]
+          : [];
+    }
+    {
+      const raw = config["language_hint"];
+      this.languageHints = Array.isArray(raw)
+        ? raw.filter((t): t is string => typeof t === "string" && t.length > 0)
+        : typeof raw === "string" && raw.length > 0
+          ? [raw]
+          : [];
+    }
+
+    const socketFactory = this.socketFactory ?? (await defaultSocketFactory());
+    this.conn = new WebSocketConnection({
+      url: () => {
+        const params = new URLSearchParams({
+          model: this.model,
+          encoding: "linear16",
+          sample_rate: String(this.sampleRate),
+          eot_threshold: String(this.eotThreshold),
+          eot_timeout_ms: String(this.eotTimeoutMs),
+          ...(this.eagerEotThreshold !== undefined
+            ? { eager_eot_threshold: String(this.eagerEotThreshold) }
+            : {}),
+        });
+        for (const term of this.keyterms) params.append("keyterm", term);
+        for (const hint of this.languageHints) params.append("language_hint", hint);
+        const separator = this.endpointUrl.includes("?") ? "&" : "?";
+        return `${this.endpointUrl}${separator}${params.toString()}`;
+      },
+      headers: { Authorization: `Token ${this.apiKey}` },
+      socketFactory,
+      retry: readProviderRetryConfig(config),
+      replayBufferSize: (config["replay_buffer_size"] as number) ?? 64,
+      onReplay: (event, count) => {
+        this.pushMetric(this.currentContextId, `stt.flux.reconnect_replay_${event}`, String(count));
+      },
+      // No provider keepalive text message: the Flux v2 protocol keeps the turn
+      // model fed by continuous audio; transports stream silence frames between
+      // utterances, so an idle socket means the call itself has gone quiet.
+      onMessage: (data) => {
+        if (typeof data === "string") this.handleProviderMessage(data);
+      },
+      onConnectionLost: (err) => {
+        this.emitError(this.currentContextId, err);
+      },
+    });
+    await this.conn.connect();
+
+    this.disposers.push(
+      bus.on("stt.audio", async (pkt: unknown) => {
+        const audioPkt = pkt as { audio: Uint8Array; contextId?: string };
+        if (audioPkt.contextId) this.currentContextId = audioPkt.contextId;
+        if (!this.conn) return;
+        try {
+          await this.conn.ensureReady();
+          this.conn.send(audioPkt.audio);
+        } catch (err) {
+          this.emitError(this.currentContextId, err instanceof Error ? err : new Error(String(err)));
+        }
+      }),
+      bus.on("turn.change", (pkt: unknown) => {
+        const tc = pkt as { contextId: string };
+        this.currentContextId = tc.contextId;
+      }),
+    );
+  }
+
+  private handleProviderMessage(data: string): void {
+    let msg: Record<string, unknown>;
+    try {
+      msg = JSON.parse(data) as Record<string, unknown>;
+    } catch (err) {
+      this.emitError(
+        this.currentContextId,
+        new Error(`Deepgram Flux sent malformed JSON: ${err instanceof Error ? err.message : String(err)}`),
+      );
+      return;
+    }
+
+    if (msg["type"] === "Error") {
+      const description = typeof msg["description"] === "string" ? msg["description"] : JSON.stringify(msg);
+      this.emitError(this.currentContextId, new Error(`Deepgram Flux provider error: ${description}`));
+      return;
+    }
+    if (msg["type"] !== "TurnInfo") return;
+
+    const info = msg as unknown as TurnInfoMessage;
+    const contextId = this.currentContextId;
+    const transcript = (info.transcript ?? "").trim();
+
+    switch (info.event) {
+      case "StartOfTurn": {
+        if (!this.speechStartedEvents) return;
+        this.bus?.push(Route.Main, {
+          kind: "vad.speech_started",
+          contextId,
+          timestampMs: Date.now(),
+          confidence: 1,
+        });
+        return;
+      }
+      case "Update": {
+        if (!transcript) return;
+        this.bus?.push(Route.Main, {
+          kind: "stt.interim",
+          contextId,
+          timestampMs: Date.now(),
+          text: transcript,
+        });
+        return;
+      }
+      case "EagerEndOfTurn": {
+        if (!transcript) return;
+        this.bus?.push(Route.Main, {
+          kind: "eos.interim",
+          contextId,
+          timestampMs: Date.now(),
+          text: transcript,
+        });
+        return;
+      }
+      case "TurnResumed": {
+        this.bus?.push(Route.Main, {
+          kind: "eos.retracted",
+          contextId,
+          timestampMs: Date.now(),
+        });
+        return;
+      }
+      case "EndOfTurn": {
+        if (!transcript) return;
+        this.bus?.push(Route.Main, {
+          kind: "stt.result",
+          contextId,
+          timestampMs: Date.now(),
+          text: transcript,
+          confidence: meanWordConfidence(info.words),
+          language: "en",
+          provider: { name: "deepgram", model: this.model, region: "global" },
+        });
+        if (this.emitEosOnFinal) {
+          this.bus?.push(Route.Main, {
+            kind: "eos.turn_complete",
+            contextId,
+            timestampMs: Date.now(),
+            text: transcript,
+            transcripts: [],
+          });
+        }
+        return;
+      }
+    }
+  }
+
+  private pushMetric(contextId: string, name: string, value: string): void {
+    this.bus?.push(Route.Background, {
+      kind: "metric.conversation",
+      contextId,
+      timestampMs: Date.now(),
+      name,
+      value,
+    });
+  }
+
+  private emitError(contextId: string, err: Error): void {
+    const category = categorizeSttError(err);
+    const packet: SttErrorPacket = {
+      kind: "stt.error",
+      contextId,
+      timestampMs: Date.now(),
+      component: "stt" as const,
+      category,
+      cause: err,
+      isRecoverable: isRecoverable(category),
+    };
+    this.bus?.push(Route.Critical, packet);
+  }
+
+  async close(): Promise<void> {
+    for (const dispose of this.disposers.splice(0)) dispose();
+    if (this.conn) {
+      if (this.conn.isReady) {
+        try {
+          this.conn.send(JSON.stringify({ type: "CloseStream" }));
+        } catch {
+          // Socket already going away — CloseStream is best-effort.
+        }
+      }
+      await this.conn.close();
+      this.conn = null;
+    }
+    this.bus = null;
+  }
+}
+
+async function defaultSocketFactory(): Promise<SocketFactory> {
+  const mod = await import("@kuralle-syrinx/ws/node");
+  return mod.createNodeWsSocket;
+}

package/src/index.ts

@@ -4,4 +4,5 @@
 // Import from the barrel for both, or the `./stt` / `./tts` subpaths for one.
 
 export * from "./stt.js";
+export * from "./flux.js";
 export * from "./tts.js";

package/src/stt.test.ts

@@ -1191,4 +1191,67 @@ describe("DeepgramSTTPlugin provider speech-start (vad_events)", () => {
 
     expect(speechStarts).toHaveLength(0);
   });
+
+  it("passes keyterm config as repeatable query params (nova-3 keyterm prompting)", async () => {
+    const connectionUrls: string[] = [];
+    const server = await new Promise<WebSocketServer>((resolve) => {
+      let nextServer: WebSocketServer;
+      nextServer = new WebSocketServer({ port: 0 }, () => resolve(nextServer));
+    });
+    servers.push(server);
+    server.on("connection", (_socket, req) => {
+      connectionUrls.push(req.url ?? "");
+    });
+    const address = server.address();
+    if (!address || typeof address === "string") throw new Error("Expected TCP server address");
+    const endpointUrl = `ws://127.0.0.1:${address.port}/listen`;
+
+    const bus = new PipelineBusImpl();
+    const started = startBus(bus);
+    const plugin = new DeepgramSTTPlugin();
+    await plugin.initialize(bus, {
+      api_key: "test",
+      endpoint_url: endpointUrl,
+      sample_rate: 16000,
+      keyterm: ["Syrinx", "Kuralle Suite"],
+    });
+    await waitFor(connectionUrls);
+    await plugin.close();
+    bus.stop();
+    await started;
+
+    const url = connectionUrls[0]!;
+    expect(url).toContain("keyterm=Syrinx");
+    expect(url).toContain("keyterm=Kuralle+Suite");
+  });
+
+  it("omits keyterm from the URL when not configured", async () => {
+    const connectionUrls: string[] = [];
+    const server = await new Promise<WebSocketServer>((resolve) => {
+      let nextServer: WebSocketServer;
+      nextServer = new WebSocketServer({ port: 0 }, () => resolve(nextServer));
+    });
+    servers.push(server);
+    server.on("connection", (_socket, req) => {
+      connectionUrls.push(req.url ?? "");
+    });
+    const address = server.address();
+    if (!address || typeof address === "string") throw new Error("Expected TCP server address");
+    const endpointUrl = `ws://127.0.0.1:${address.port}/listen`;
+
+    const bus = new PipelineBusImpl();
+    const started = startBus(bus);
+    const plugin = new DeepgramSTTPlugin();
+    await plugin.initialize(bus, {
+      api_key: "test",
+      endpoint_url: endpointUrl,
+      sample_rate: 16000,
+    });
+    await waitFor(connectionUrls);
+    await plugin.close();
+    bus.stop();
+    await started;
+
+    expect(connectionUrls[0]!).not.toContain("keyterm=");
+  });
 });

package/src/stt.ts

@@ -99,6 +99,9 @@ export class DeepgramSTTPlugin implements VoicePlugin {
   // "never promote unconfirmed" behavior for callers that need it).
   private finalizeTimeoutFallback: boolean = false;
   private keepAliveIntervalMs: number = 3000;
+  // nova-3 keyterm prompting: bias recognition toward domain terms (names, products,
+  // codes) — the #1 production voice failure is mishearing exactly these.
+  private keyterms: readonly string[] = [];
 
   // Session-long WebSocket, managed by the shared connection (reconnect, keepalive).
   private conn: WebSocketConnection | null = null;
@@ -152,6 +155,14 @@ export class DeepgramSTTPlugin implements VoicePlugin {
     this.finalizeResetThreshold = (config["finalize_reset_threshold"] as number) ?? 2;
     this.finalizeTimeoutFallback = (config["finalize_timeout_fallback"] as boolean) ?? false;
     this.keepAliveIntervalMs = (config["keep_alive_interval_ms"] as number) ?? 3000;
+    {
+      const raw = config["keyterm"];
+      this.keyterms = Array.isArray(raw)
+        ? raw.filter((t): t is string => typeof t === "string" && t.length > 0)
+        : typeof raw === "string" && raw.length > 0
+          ? [raw]
+          : [];
+    }
     this.audioFormat = { encoding: "pcm_s16le", sampleRateHz: this.sampleRate, channels: 1 };
     assertAudioFormat(this.audioFormat);
 
@@ -171,6 +182,7 @@ export class DeepgramSTTPlugin implements VoicePlugin {
           vad_events: String(this.vadEvents),
           ...(this.utteranceEndMs > 0 ? { utterance_end_ms: String(this.utteranceEndMs) } : {}),
         });
+        for (const term of this.keyterms) params.append("keyterm", term);
         const separator = this.endpointUrl.includes("?") ? "&" : "?";
         return `${this.endpointUrl}${separator}${params.toString()}`;
       },