@kuralle-syrinx/deepgram 4.2.0 → 4.4.0

package/src/tts.ts

@@ -65,6 +65,8 @@ export class DeepgramTTSPlugin implements VoicePlugin {
   private model = "aura-2-thalia-en";
   private endpointUrl = "wss://api.deepgram.com/v1/speak";
   private sampleRate = 24000;
+  private encoding = "linear16";
+  private container = "none";
   private retryConfig: RetryConfig = readProviderRetryConfig({});
   // Deepgram's speak socket has no per-message context id, but the engine
   // synthesizes one turn at a time, so the audio streaming back belongs to the
@@ -81,6 +83,8 @@ export class DeepgramTTSPlugin implements VoicePlugin {
   private finishTimers = new Map<string, ReturnType<typeof setTimeout>>();
   private disposers: Array<() => void> = [];
   private audioFormat: AudioFormat = { encoding: "pcm_s16le", sampleRateHz: 24000, channels: 1 };
+  /** Synthesized character counts per contextId — billed on successful tts.end. */
+  private charactersByContextId = new Map<string, number>();
 
   async initialize(bus: PipelineBus, config: PluginConfig): Promise<void> {
     this.bus = bus;
@@ -88,6 +92,8 @@ export class DeepgramTTSPlugin implements VoicePlugin {
     this.model = optionalStringConfig(config, "model") ?? this.model;
     this.endpointUrl = optionalStringConfig(config, "endpoint_url") ?? this.endpointUrl;
     this.sampleRate = readPositiveInteger(config["sample_rate"], this.sampleRate);
+    this.encoding = optionalStringConfig(config, "encoding") ?? this.encoding;
+    this.container = optionalStringConfig(config, "container") ?? this.container;
     this.retryConfig = readProviderRetryConfig(config);
     const finishTimeoutMs = readNonNegativeInteger(config["finish_timeout_ms"], 2000);
     this.audioFormat = { encoding: "pcm_s16le", sampleRateHz: this.sampleRate, channels: 1 };
@@ -97,9 +103,9 @@ export class DeepgramTTSPlugin implements VoicePlugin {
       url: () => {
         const params = new URLSearchParams({
           model: this.model,
-          encoding: "linear16",
+          encoding: this.encoding,
           sample_rate: String(this.sampleRate),
-          container: "none",
+          container: this.container,
         });
         const separator = this.endpointUrl.includes("?") ? "&" : "?";
         return `${this.endpointUrl}${separator}${params.toString()}`;
@@ -143,7 +149,11 @@ export class DeepgramTTSPlugin implements VoicePlugin {
     if (this.cancelledContexts.has(contextId)) return;
     this.activeContexts.add(contextId);
     this.currentContextId = contextId;
-    if (!(await this.trySend(SPEAK(text), contextId))) this.activeContexts.delete(contextId);
+    if (!(await this.trySend(SPEAK(text), contextId))) {
+      this.activeContexts.delete(contextId);
+      return;
+    }
+    this.charactersByContextId.set(contextId, (this.charactersByContextId.get(contextId) ?? 0) + text.length);
   }
 
   private async finishContext(contextId: string): Promise<void> {
@@ -166,6 +176,7 @@ export class DeepgramTTSPlugin implements VoicePlugin {
     for (const dispose of this.disposers.splice(0)) dispose();
     this.activeContexts.clear();
     this.cancelledContexts.clear();
+    this.charactersByContextId.clear();
     for (const timer of this.finishTimers.values()) clearTimeout(timer);
     this.finishTimers.clear();
     this.currentContextId = "";
@@ -178,7 +189,10 @@ export class DeepgramTTSPlugin implements VoicePlugin {
 
   private async cancelActiveContexts(): Promise<void> {
     const contextIds = [...this.activeContexts];
-    for (const contextId of contextIds) boundedAdd(this.cancelledContexts, contextId, MAX_CANCELLED_CONTEXTS);
+    for (const contextId of contextIds) {
+      boundedAdd(this.cancelledContexts, contextId, MAX_CANCELLED_CONTEXTS);
+      this.charactersByContextId.delete(contextId);
+    }
     this.activeContexts.clear();
     for (const contextId of contextIds) this.clearFinishTimeout(contextId);
     this.currentContextId = "";
@@ -298,7 +312,10 @@ export class DeepgramTTSPlugin implements VoicePlugin {
   private failActiveContexts(err: Error): void {
     const contextIds = [...this.activeContexts];
     this.activeContexts.clear();
-    for (const contextId of contextIds) this.clearFinishTimeout(contextId);
+    for (const contextId of contextIds) {
+      this.clearFinishTimeout(contextId);
+      this.charactersByContextId.delete(contextId);
+    }
     this.currentContextId = "";
     this.carry = EMPTY;
     // A dropped socket voids any in-flight Clear/Cleared handshake: the server
@@ -330,6 +347,19 @@ export class DeepgramTTSPlugin implements VoicePlugin {
   }
 
   private emitEnd(contextId: string): void {
+    const characters = this.charactersByContextId.get(contextId) ?? 0;
+    this.charactersByContextId.delete(contextId);
+    if (characters > 0) {
+      this.bus?.push(Route.Background, {
+        kind: "usage.recorded",
+        contextId,
+        timestampMs: Date.now(),
+        stage: "tts",
+        provider: "deepgram",
+        model: this.model,
+        characters,
+      });
+    }
     this.bus?.push(Route.Main, {
       kind: "tts.end",
       contextId,

package/.handoff/finalize-contract-probe.test.ts

@@ -1,193 +0,0 @@
-import { describe, expect, it } from "vitest";
-import { WebSocketServer, type WebSocket } from "ws";
-import {
-  PipelineBusImpl,
-  Route,
-  type ConversationMetricPacket,
-  type SttErrorPacket,
-  type SttResultPacket,
-} from "@kuralle-syrinx/core";
-import { DeepgramSTTPlugin } from "../src/index.js";
-
-async function createServer(onConnection: (socket: WebSocket) => void): Promise<{
-  url: string;
-  close: () => Promise<void>;
-}> {
-  const server = await new Promise<WebSocketServer>((resolve) => {
-    let nextServer: WebSocketServer;
-    nextServer = new WebSocketServer({ port: 0 }, () => resolve(nextServer));
-  });
-  server.on("connection", onConnection);
-  const address = server.address();
-  if (!address || typeof address === "string") throw new Error("expected tcp address");
-  return {
-    url: `ws://127.0.0.1:${address.port}/listen`,
-    close: () => new Promise((resolve) => {
-      for (const client of server.clients) client.terminate();
-      server.close(() => resolve());
-    }),
-  };
-}
-
-async function waitUntil(predicate: () => boolean): Promise<void> {
-  for (let i = 0; i < 100; i += 1) {
-    if (predicate()) return;
-    await new Promise((resolve) => setTimeout(resolve, 10));
-  }
-  throw new Error("timed out waiting for probe condition");
-}
-
-describe("Deepgram finalize contract probe", () => {
-  it("keeps a delayed Finalize response correlated to the requested context after currentContextId moves", async () => {
-    const controlMessages: string[] = [];
-    const server = await createServer((socket) => {
-      socket.on("message", (data, isBinary) => {
-        if (isBinary) return;
-        const msg = JSON.parse(data.toString()) as { type?: string };
-        if (msg.type) controlMessages.push(msg.type);
-        if (msg.type !== "Finalize") return;
-        setTimeout(() => {
-          socket.send(JSON.stringify({
-            is_final: true,
-            speech_final: false,
-            from_finalize: true,
-            channel: { alternatives: [{ transcript: "old turn confirmed", confidence: 0.91 }] },
-          }));
-        }, 10);
-      });
-    });
-    const bus = new PipelineBusImpl();
-    const started = bus.start();
-    const plugin = new DeepgramSTTPlugin();
-    const finals: SttResultPacket[] = [];
-    const errors: SttErrorPacket[] = [];
-    const metrics: ConversationMetricPacket[] = [];
-    bus.on("stt.result", (pkt) => finals.push(pkt as SttResultPacket));
-    bus.on("stt.error", (pkt) => errors.push(pkt as SttErrorPacket));
-    bus.on("metric.conversation", (pkt) => metrics.push(pkt as ConversationMetricPacket));
-
-    await plugin.initialize(bus, {
-      api_key: "test",
-      endpoint_url: server.url,
-      sample_rate: 16000,
-      emit_eos_on_final: false,
-      finalize_on_speech_final: false,
-      provider_finalize_timeout_ms: 40,
-      finalize_timeout_fallback: false,
-    });
-    bus.push(Route.Main, {
-      kind: "stt.audio",
-      contextId: "old-vad-fragment",
-      timestampMs: Date.now(),
-      audio: new Uint8Array(640),
-    });
-    await new Promise((resolve) => setTimeout(resolve, 5));
-
-    bus.push(Route.Main, {
-      kind: "turn.change",
-      contextId: "new-vad-fragment",
-      previousContextId: "old-vad-fragment",
-      reason: "probe_vad_fragment_split",
-      timestampMs: Date.now(),
-    });
-    await new Promise((resolve) => setTimeout(resolve, 5));
-
-    plugin.forceFinalize("old-vad-fragment");
-    await waitUntil(() => finals.length > 0);
-    await new Promise((resolve) => setTimeout(resolve, 60));
-
-    expect(controlMessages).toContain("Finalize");
-    expect(finals).toEqual([
-      expect.objectContaining({
-        contextId: "old-vad-fragment",
-        text: "old turn confirmed",
-      }),
-    ]);
-    expect(errors).toHaveLength(0);
-    expect(metrics).toEqual(expect.arrayContaining([
-      expect.objectContaining({
-        name: "stt_provider_final_segment",
-        contextId: "old-vad-fragment",
-        value: expect.stringContaining("\"fromFinalize\":true"),
-      }),
-    ]));
-    expect(metrics).not.toEqual(expect.arrayContaining([
-      expect.objectContaining({
-        name: "stt_provider_finalize_timeout",
-        contextId: "old-vad-fragment",
-      }),
-    ]));
-
-    await plugin.close();
-    bus.stop();
-    await started;
-    await server.close();
-  });
-
-  it("releases provider-final text without speech_final/from_finalize and does not need timeout fallback", async () => {
-    const controlMessages: string[] = [];
-    const server = await createServer((socket) => {
-      socket.on("message", (data, isBinary) => {
-        if (isBinary) {
-          socket.send(JSON.stringify({
-            is_final: true,
-            speech_final: false,
-            channel: { alternatives: [{ transcript: "just want to complain again so", confidence: 0.84 }] },
-          }));
-          return;
-        }
-        const msg = JSON.parse(data.toString()) as { type?: string };
-        if (msg.type) controlMessages.push(msg.type);
-      });
-    });
-    const bus = new PipelineBusImpl();
-    const started = bus.start();
-    const plugin = new DeepgramSTTPlugin();
-    const finals: SttResultPacket[] = [];
-    const errors: SttErrorPacket[] = [];
-    const metrics: ConversationMetricPacket[] = [];
-    bus.on("stt.result", (pkt) => finals.push(pkt as SttResultPacket));
-    bus.on("stt.error", (pkt) => errors.push(pkt as SttErrorPacket));
-    bus.on("metric.conversation", (pkt) => metrics.push(pkt as ConversationMetricPacket));
-
-    await plugin.initialize(bus, {
-      api_key: "test",
-      endpoint_url: server.url,
-      sample_rate: 16000,
-      emit_eos_on_final: false,
-      finalize_on_speech_final: false,
-      provider_finalize_timeout_ms: 20,
-      finalize_timeout_fallback: false,
-    });
-    bus.push(Route.Main, {
-      kind: "stt.audio",
-      contextId: "messy-live-turn",
-      timestampMs: Date.now(),
-      audio: new Uint8Array(640),
-    });
-    await waitUntil(() => metrics.some((m) => m.name === "stt_provider_final_segment"));
-    await waitUntil(() => finals.length > 0);
-    plugin.forceFinalize("messy-live-turn");
-    await new Promise((resolve) => setTimeout(resolve, 40));
-
-    expect(controlMessages).toContain("Finalize");
-    expect(finals).toEqual([
-      expect.objectContaining({
-        contextId: "messy-live-turn",
-        text: "just want to complain again so",
-      }),
-    ]);
-    expect(errors).toHaveLength(0);
-    expect(metrics).toEqual(expect.arrayContaining([
-      expect.objectContaining({ name: "stt_provider_final_segment" }),
-    ]));
-    expect(metrics).not.toEqual(expect.arrayContaining([
-      expect.objectContaining({ name: "stt_provider_finalize_timeout" }),
-    ]));
-
-    await plugin.close();
-    bus.stop();
-    await started;
-    await server.close();
-  });
-});

package/src/flux.test.ts

@@ -1,227 +0,0 @@
-// SPDX-License-Identifier: MIT
-
-import { afterEach, describe, expect, it } from "vitest";
-import { WebSocketServer, type WebSocket } from "ws";
-import {
-  PipelineBusImpl,
-  Route,
-  type EndOfSpeechPacket,
-  type EndOfSpeechRetractedPacket,
-  type InterimEndOfSpeechPacket,
-  type SttInterimPacket,
-  type SttResultPacket,
-} from "@kuralle-syrinx/core";
-
-import { DeepgramFluxSTTPlugin } from "./flux.js";
-
-let servers: WebSocketServer[] = [];
-
-afterEach(async () => {
-  await Promise.all(
-    servers.splice(0).map(
-      (server) =>
-        new Promise<void>((resolve) => {
-          for (const client of server.clients) client.terminate();
-          server.close(() => resolve());
-        }),
-    ),
-  );
-});
-
-interface LocalServer {
-  endpointUrl: string;
-  connectionUrls: string[];
-  sockets: WebSocket[];
-}
-
-async function createLocalServer(): Promise<LocalServer> {
-  const server = await new Promise<WebSocketServer>((resolve) => {
-    let nextServer: WebSocketServer;
-    nextServer = new WebSocketServer({ port: 0 }, () => resolve(nextServer));
-  });
-  servers.push(server);
-  const state: LocalServer = { endpointUrl: "", connectionUrls: [], sockets: [] };
-  server.on("connection", (socket, req) => {
-    state.connectionUrls.push(req.url ?? "");
-    state.sockets.push(socket);
-  });
-  const address = server.address();
-  if (!address || typeof address === "string") throw new Error("Expected TCP server address");
-  state.endpointUrl = `ws://127.0.0.1:${address.port}/v2/listen`;
-  return state;
-}
-
-async function waitFor<T>(items: T[], count = 1): Promise<void> {
-  for (let i = 0; i < 100; i += 1) {
-    if (items.length >= count) return;
-    await new Promise((resolve) => setTimeout(resolve, 10));
-  }
-}
-
-function turnInfo(event: string, extra: Record<string, unknown> = {}): string {
-  return JSON.stringify({
-    type: "TurnInfo",
-    event,
-    turn_index: 0,
-    audio_window_start: 0,
-    audio_window_end: 0.6,
-    transcript: "",
-    words: [],
-    end_of_turn_confidence: 0.5,
-    ...extra,
-  });
-}
-
-async function startPlugin(
-  local: LocalServer,
-  config: Record<string, unknown> = {},
-): Promise<{ bus: PipelineBusImpl; plugin: DeepgramFluxSTTPlugin; started: Promise<void> }> {
-  const bus = new PipelineBusImpl();
-  const started = bus.start();
-  const plugin = new DeepgramFluxSTTPlugin();
-  await plugin.initialize(bus, {
-    api_key: "test",
-    endpoint_url: local.endpointUrl,
-    sample_rate: 16000,
-    ...config,
-  });
-  await waitFor(local.sockets);
-  // Feed one audio packet so the plugin learns the current contextId.
-  bus.push(Route.Main, {
-    kind: "stt.audio",
-    contextId: "turn-1",
-    timestampMs: Date.now(),
-    audio: new Uint8Array(320),
-  });
-  await new Promise((resolve) => setTimeout(resolve, 20));
-  return { bus, plugin, started };
-}
-
-describe("DeepgramFluxSTTPlugin", () => {
-  it("connects with Flux params and forwards keyterm + eager threshold", async () => {
-    const local = await createLocalServer();
-    const { bus, plugin, started } = await startPlugin(local, {
-      eot_threshold: 0.85,
-      eager_eot_threshold: 0.4,
-      eot_timeout_ms: 7000,
-      keyterm: ["Syrinx"],
-    });
-
-    await plugin.close();
-    bus.stop();
-    await started;
-
-    const url = local.connectionUrls[0]!;
-    expect(url).toContain("model=flux-general-en");
-    expect(url).toContain("encoding=linear16");
-    expect(url).toContain("sample_rate=16000");
-    expect(url).toContain("eot_threshold=0.85");
-    expect(url).toContain("eager_eot_threshold=0.4");
-    expect(url).toContain("eot_timeout_ms=7000");
-    expect(url).toContain("keyterm=Syrinx");
-  });
-
-  it("omits eager_eot_threshold by default (eager mode is opt-in)", async () => {
-    const local = await createLocalServer();
-    const { bus, plugin, started } = await startPlugin(local);
-
-    await plugin.close();
-    bus.stop();
-    await started;
-
-    const url = local.connectionUrls[0]!;
-    expect(url).toContain("eot_threshold=0.7");
-    expect(url).not.toContain("eager_eot_threshold=");
-  });
-
-  it("maps TurnInfo events onto the bus: Update→stt.interim, EndOfTurn→stt.result+eos.turn_complete", async () => {
-    const local = await createLocalServer();
-    const { bus, plugin, started } = await startPlugin(local);
-
-    const interims: SttInterimPacket[] = [];
-    const results: SttResultPacket[] = [];
-    const turnCompletes: EndOfSpeechPacket[] = [];
-    bus.on("stt.interim", (pkt) => {
-      interims.push(pkt as SttInterimPacket);
-    });
-    bus.on("stt.result", (pkt) => {
-      results.push(pkt as SttResultPacket);
-    });
-    bus.on("eos.turn_complete", (pkt) => {
-      turnCompletes.push(pkt as EndOfSpeechPacket);
-    });
-
-    const socket = local.sockets[0]!;
-    socket.send(turnInfo("Update", { transcript: "what are" }));
-    socket.send(
-      turnInfo("EndOfTurn", {
-        transcript: "what are the lab fees",
-        words: [
-          { word: "what", confidence: 0.99 },
-          { word: "are", confidence: 0.97 },
-        ],
-        end_of_turn_confidence: 0.91,
-      }),
-    );
-    await waitFor(turnCompletes);
-
-    await plugin.close();
-    bus.stop();
-    await started;
-
-    expect(interims.map((p) => p.text)).toContain("what are");
-    expect(results).toHaveLength(1);
-    expect(results[0]!.text).toBe("what are the lab fees");
-    expect(results[0]!.confidence).toBeCloseTo(0.98, 2);
-    expect(turnCompletes).toEqual([
-      expect.objectContaining({ contextId: "turn-1", text: "what are the lab fees" }),
-    ]);
-  });
-
-  it("emits eos.interim on EagerEndOfTurn and eos.retracted on TurnResumed", async () => {
-    const local = await createLocalServer();
-    const { bus, plugin, started } = await startPlugin(local, { eager_eot_threshold: 0.4 });
-
-    const eagers: InterimEndOfSpeechPacket[] = [];
-    const retractions: EndOfSpeechRetractedPacket[] = [];
-    bus.on("eos.interim", (pkt) => {
-      eagers.push(pkt as InterimEndOfSpeechPacket);
-    });
-    bus.on("eos.retracted", (pkt) => {
-      retractions.push(pkt as EndOfSpeechRetractedPacket);
-    });
-
-    const socket = local.sockets[0]!;
-    socket.send(turnInfo("EagerEndOfTurn", { transcript: "book a room", end_of_turn_confidence: 0.55 }));
-    await waitFor(eagers);
-    socket.send(turnInfo("TurnResumed"));
-    await waitFor(retractions);
-
-    await plugin.close();
-    bus.stop();
-    await started;
-
-    expect(eagers).toEqual([expect.objectContaining({ contextId: "turn-1", text: "book a room" })]);
-    expect(retractions).toEqual([expect.objectContaining({ contextId: "turn-1" })]);
-  });
-
-  it("emits vad.speech_started on StartOfTurn (Flux owns barge-in signalling)", async () => {
-    const local = await createLocalServer();
-    const { bus, plugin, started } = await startPlugin(local);
-
-    const speechStarts: unknown[] = [];
-    bus.on("vad.speech_started", (pkt) => {
-      speechStarts.push(pkt);
-    });
-
-    const socket = local.sockets[0]!;
-    socket.send(turnInfo("StartOfTurn", { transcript: "hello" }));
-    await waitFor(speechStarts);
-
-    await plugin.close();
-    bus.stop();
-    await started;
-
-    expect(speechStarts.length).toBe(1);
-  });
-});