@cheeko-ai/esp32-voice 2026.2.21

package/src/stt/deepgram.ts

@@ -0,0 +1,215 @@
+/**
+ * Deepgram streaming Speech-to-Text provider.
+ *
+ * Ported from cheekoclaw_bridge/deepgram_stt.py
+ *
+ * Uses Deepgram's WebSocket API for real-time speech recognition.
+ * Receives Opus frames from the ESP32 and streams them directly
+ * to Deepgram (Deepgram supports Opus natively).
+ *
+ * WebSocket URL: wss://api.deepgram.com/v1/listen
+ */
+
+import WebSocket from "ws";
+import type { SttProvider, SttProviderConfig, SttProviderMeta, SttTranscriptCallback } from "./stt-provider.js";
+import { sttRegistry } from "./stt-registry.js";
+
+const DEEPGRAM_WS_URL = "wss://api.deepgram.com/v1/listen";
+
+export class DeepgramSttProvider implements SttProvider {
+  readonly id = "deepgram";
+  readonly name = "Deepgram";
+  readonly streaming = true;
+
+  onTranscript: SttTranscriptCallback | null = null;
+  onSpeechEnd: (() => void | Promise<void>) | null = null;
+
+  private apiKey: string;
+  private model: string;
+  private language: string;
+  private ws: WebSocket | null = null;
+  private finalTranscript = "";
+  private finalizeResolve: ((value: string) => void) | null = null;
+  private finalizePromise: Promise<string> | null = null;
+
+  constructor(config: SttProviderConfig) {
+    this.apiKey = config.apiKey;
+    this.model = config.model ?? "nova-2";
+    this.language = config.language ?? "en";
+  }
+
+  async connect(): Promise<void> {
+    const params = new URLSearchParams({
+      encoding: "opus",
+      sample_rate: "16000",
+      channels: "1",
+      model: this.model,
+      language: this.language,
+      interim_results: "true",
+      punctuate: "true",
+      // Enable server-side VAD: fires speech_final when speech ends
+      endpointing: "300",
+      utterance_end_ms: "1000",
+    });
+
+    const url = `${DEEPGRAM_WS_URL}?${params.toString()}`;
+
+    return new Promise<void>((resolve, reject) => {
+      this.ws = new WebSocket(url, {
+        headers: { Authorization: `Token ${this.apiKey}` },
+      });
+
+      this.finalTranscript = "";
+      this.finalizePromise = new Promise<string>((res) => {
+        this.finalizeResolve = res;
+      });
+
+      this.ws.on("open", () => {
+        console.log("[deepgram-stt] Connected");
+        resolve();
+      });
+
+      this.ws.on("message", (data: Buffer) => {
+        this.handleMessage(data);
+      });
+
+      this.ws.on("error", (err) => {
+        console.error("[deepgram-stt] WebSocket error:", err.message);
+        reject(err);
+      });
+
+      this.ws.on("close", () => {
+        console.log("[deepgram-stt] Connection closed");
+        // Resolve finalize if still pending
+        if (this.finalizeResolve) {
+          this.finalizeResolve(this.finalTranscript);
+          this.finalizeResolve = null;
+        }
+      });
+    });
+  }
+
+  async sendAudio(audioData: Buffer): Promise<void> {
+    if (this.ws?.readyState === WebSocket.OPEN) {
+      this.ws.send(audioData);
+    }
+  }
+
+  async finalize(): Promise<string> {
+    // Send Deepgram's CloseStream message to trigger final results
+    if (this.ws?.readyState === WebSocket.OPEN) {
+      this.ws.send(JSON.stringify({ type: "CloseStream" }));
+    }
+
+    // Wait for the final transcript (with timeout)
+    if (this.finalizePromise) {
+      const timeoutPromise = new Promise<string>((resolve) => {
+        setTimeout(() => {
+          console.warn("[deepgram-stt] Timeout waiting for final transcript");
+          resolve(this.finalTranscript);
+        }, 5000);
+      });
+      return Promise.race([this.finalizePromise, timeoutPromise]);
+    }
+
+    return this.finalTranscript;
+  }
+
+  async close(): Promise<void> {
+    if (this.ws) {
+      try {
+        this.ws.close();
+      } catch {
+        // Ignore close errors
+      }
+      this.ws = null;
+    }
+  }
+
+  private handleMessage(data: Buffer): void {
+    try {
+      const msg = JSON.parse(data.toString());
+      const msgType = msg.type;
+
+      if (msgType === "Results") {
+        const alternatives = msg.channel?.alternatives ?? [];
+        if (alternatives.length === 0) return;
+
+        const text: string = alternatives[0].transcript ?? "";
+        const isFinal: boolean = msg.is_final ?? false;
+        const speechFinal: boolean = msg.speech_final ?? false;
+
+        if (text.trim()) {
+          // Fire transcript callback
+          if (this.onTranscript) {
+            const result = this.onTranscript(text, isFinal);
+            if (result instanceof Promise) {
+              result.catch((err) => console.error("[deepgram-stt] Transcript callback error:", err));
+            }
+          }
+
+          // Accumulate final segments
+          if (isFinal) {
+            if (this.finalTranscript) {
+              this.finalTranscript += " " + text;
+            } else {
+              this.finalTranscript = text;
+            }
+          }
+        }
+
+        // speech_final = server-side VAD detected end of utterance
+        // Resolve finalize promise immediately so processUtterance can proceed
+        if (speechFinal && this.finalizeResolve) {
+          console.log(`[deepgram-stt] speech_final — triggering utterance end (transcript: "${this.finalTranscript}")`);
+          this.finalizeResolve(this.finalTranscript);
+          this.finalizeResolve = null;
+          // Fire onSpeechEnd so the session calls processUtterance
+          if (this.onSpeechEnd) {
+            const result = this.onSpeechEnd();
+            if (result instanceof Promise) {
+              result.catch((err) => console.error("[deepgram-stt] onSpeechEnd error:", err));
+            }
+          }
+        }
+      } else if (msgType === "UtteranceEnd") {
+        // Fallback: Deepgram UtteranceEnd event (requires utterance_end_ms param)
+        console.log("[deepgram-stt] UtteranceEnd received");
+        if (this.finalizeResolve) {
+          this.finalizeResolve(this.finalTranscript);
+          this.finalizeResolve = null;
+        }
+        if (this.onSpeechEnd) {
+          const result = this.onSpeechEnd();
+          if (result instanceof Promise) {
+            result.catch((err) => console.error("[deepgram-stt] onSpeechEnd (UtteranceEnd) error:", err));
+          }
+        }
+      } else if (msgType === "Error") {
+        console.error("[deepgram-stt] Error:", msg);
+      }
+      // Ignore "Metadata" and other message types
+    } catch {
+      // Ignore parse errors
+    }
+  }
+}
+
+/** Deepgram STT provider metadata. */
+export const deepgramMeta: SttProviderMeta = {
+  id: "deepgram",
+  name: "Deepgram",
+  description: "Fast, accurate streaming STT with Nova-2 model. Supports Opus input natively.",
+  streaming: true,
+  envVar: "DEEPGRAM_API_KEY",
+  defaultModel: "nova-2",
+  docsUrl: "https://developers.deepgram.com/docs/streaming",
+};
+
+/** Factory function for creating Deepgram STT instances. */
+function createDeepgramStt(config: SttProviderConfig): SttProvider {
+  return new DeepgramSttProvider(config);
+}
+
+// Auto-register with the STT registry
+sttRegistry.register(deepgramMeta, createDeepgramStt);

package/src/stt/stt-provider.ts

@@ -0,0 +1,107 @@
+/**
+ * Speech-to-Text (STT) provider interface.
+ *
+ * All STT providers must implement this interface. Providers can be
+ * streaming (WebSocket-based, sending audio chunks in real time) or
+ * batch (send complete audio, receive transcript).
+ *
+ * Modeled after OpenClaw's multi-provider architecture — new providers
+ * can be added by implementing this interface and registering with
+ * the STT registry.
+ */
+
+export type SttTranscriptCallback = (text: string, isFinal: boolean) => void | Promise<void>;
+
+/** Called when the STT provider detects end of speech (server-side VAD). */
+export type SttSpeechEndCallback = () => void | Promise<void>;
+
+export interface SttProviderConfig {
+  /** Provider-specific API key. */
+  apiKey: string;
+  /** Model identifier (e.g., "nova-2" for Deepgram). */
+  model?: string;
+  /** Language code (e.g., "en", "es"). */
+  language?: string;
+  /** Additional provider-specific options. */
+  options?: Record<string, unknown>;
+}
+
+export interface SttProvider {
+  /** Unique provider identifier (e.g., "deepgram", "google", "whisper"). */
+  readonly id: string;
+
+  /** Human-readable provider name. */
+  readonly name: string;
+
+  /** Whether this provider supports real-time streaming via WebSocket. */
+  readonly streaming: boolean;
+
+  /**
+   * Called when a transcript (partial or final) is received.
+   * Set this before calling `connect()`.
+   */
+  onTranscript: SttTranscriptCallback | null;
+
+  /**
+   * Called when the STT provider detects end of speech via server-side VAD
+   * (e.g., Deepgram speech_final). Used to trigger utterance processing
+   * automatically when the firmware doesn't send a speech_end message.
+   */
+  onSpeechEnd?: SttSpeechEndCallback | null;
+
+  /**
+   * Open a connection to the STT service.
+   * For streaming providers, this opens a WebSocket.
+   * For batch providers, this may be a no-op.
+   */
+  connect(): Promise<void>;
+
+  /**
+   * Send an audio chunk to the STT service.
+   * For streaming providers: sends Opus/PCM frames in real time.
+   * For batch providers: buffers audio until `finalize()` is called.
+   *
+   * @param audioData - Raw audio bytes (Opus frames from ESP32)
+   */
+  sendAudio(audioData: Buffer): Promise<void>;
+
+  /**
+   * Signal end of audio and retrieve the final transcript.
+   * For streaming providers: sends a "close stream" signal and waits.
+   * For batch providers: sends buffered audio and waits for result.
+   *
+   * @returns The complete, final transcript text.
+   */
+  finalize(): Promise<string>;
+
+  /**
+   * Close the connection and release resources.
+   */
+  close(): Promise<void>;
+}
+
+/**
+ * Factory function type for creating STT provider instances.
+ * Each call creates a fresh provider for one utterance/session.
+ */
+export type SttProviderFactory = (config: SttProviderConfig) => SttProvider;
+
+/**
+ * Metadata about a registered STT provider.
+ */
+export interface SttProviderMeta {
+  /** Provider ID. */
+  id: string;
+  /** Human-readable name. */
+  name: string;
+  /** Short description. */
+  description: string;
+  /** Whether it supports streaming. */
+  streaming: boolean;
+  /** Required environment variable for the API key. */
+  envVar: string;
+  /** Default model identifier. */
+  defaultModel?: string;
+  /** Documentation URL. */
+  docsUrl?: string;
+}

package/src/stt/stt-registry.ts

@@ -0,0 +1,71 @@
+/**
+ * STT Provider Registry.
+ *
+ * Central registry for speech-to-text providers. Providers register
+ * themselves with a factory function, and the voice session creates
+ * instances as needed.
+ *
+ * Usage:
+ *   sttRegistry.register(deepgramMeta, createDeepgramStt);
+ *   const provider = sttRegistry.create("deepgram", { apiKey: "..." });
+ */
+
+import type { SttProvider, SttProviderConfig, SttProviderFactory, SttProviderMeta } from "./stt-provider.js";
+
+interface RegisteredSttProvider {
+  meta: SttProviderMeta;
+  factory: SttProviderFactory;
+}
+
+class SttRegistry {
+  private providers = new Map<string, RegisteredSttProvider>();
+
+  /**
+   * Register a new STT provider.
+   */
+  register(meta: SttProviderMeta, factory: SttProviderFactory): void {
+    if (this.providers.has(meta.id)) {
+      console.warn(`[stt-registry] Provider "${meta.id}" is already registered, overwriting.`);
+    }
+    this.providers.set(meta.id, { meta, factory });
+    console.log(`[stt-registry] Registered STT provider: ${meta.name} (${meta.id})`);
+  }
+
+  /**
+   * Create an instance of a registered STT provider.
+   */
+  create(providerId: string, config: SttProviderConfig): SttProvider {
+    const registered = this.providers.get(providerId);
+    if (!registered) {
+      const available = [...this.providers.keys()].join(", ");
+      throw new Error(
+        `STT provider "${providerId}" not found. Available: ${available || "none"}`,
+      );
+    }
+    return registered.factory(config);
+  }
+
+  /**
+   * Get metadata for a registered provider.
+   */
+  getMeta(providerId: string): SttProviderMeta | undefined {
+    return this.providers.get(providerId)?.meta;
+  }
+
+  /**
+   * List all registered providers.
+   */
+  list(): SttProviderMeta[] {
+    return [...this.providers.values()].map((p) => p.meta);
+  }
+
+  /**
+   * Check if a provider is registered.
+   */
+  has(providerId: string): boolean {
+    return this.providers.has(providerId);
+  }
+}
+
+/** Global STT provider registry. */
+export const sttRegistry = new SttRegistry();

package/src/tts/elevenlabs.ts

@@ -0,0 +1,215 @@
+/**
+ * ElevenLabs streaming Text-to-Speech provider.
+ *
+ * Ported from cheekoclaw_bridge/elevenlabs_tts.py
+ *
+ * Uses ElevenLabs' WebSocket API for real-time text-to-speech.
+ * Sends text and receives base64-encoded PCM audio (24kHz, 16-bit mono).
+ *
+ * WebSocket URL: wss://api.elevenlabs.io/v1/text-to-speech/{voice_id}/stream-input
+ */
+
+import WebSocket from "ws";
+import type { TtsProvider, TtsProviderConfig, TtsProviderMeta, TtsAudioCallback, TtsDoneCallback } from "./tts-provider.js";
+import { ttsRegistry } from "./tts-registry.js";
+
+const ELEVENLABS_WS_URL = "wss://api.elevenlabs.io/v1/text-to-speech/{voice_id}/stream-input";
+
+const DEFAULT_VOICE_ID = "21m00Tcm4TlvDq8ikWAM"; // Rachel
+const DEFAULT_MODEL_ID = "eleven_turbo_v2_5";
+
+export class ElevenLabsTtsProvider implements TtsProvider {
+  readonly id = "elevenlabs";
+  readonly name = "ElevenLabs";
+  readonly streaming = true;
+  readonly outputSampleRate = 24000;
+
+  onAudio: TtsAudioCallback | null = null;
+  onDone: TtsDoneCallback | null = null;
+
+  private apiKey: string;
+  private voiceId: string;
+  private modelId: string;
+  private ws: WebSocket | null = null;
+  private doneResolve: (() => void) | null = null;
+  private donePromise: Promise<void> | null = null;
+  // Serialises onAudio calls: each chunk waits for the previous one to finish
+  // (including any pacing sleeps) before being dispatched. Without this, all
+  // chunks are fired in parallel and the pacing in the callback is bypassed.
+  private audioChain: Promise<void> = Promise.resolve();
+  private isFinalReceived = false;
+
+  constructor(config: TtsProviderConfig) {
+    this.apiKey = config.apiKey;
+    this.voiceId = config.voiceId ?? DEFAULT_VOICE_ID;
+    this.modelId = config.model ?? DEFAULT_MODEL_ID;
+  }
+
+  async connect(): Promise<void> {
+    const url =
+      ELEVENLABS_WS_URL.replace("{voice_id}", this.voiceId) +
+      `?model_id=${this.modelId}&output_format=pcm_24000`;
+
+    return new Promise<void>((resolve, reject) => {
+      this.ws = new WebSocket(url, {
+        headers: { "xi-api-key": this.apiKey },
+      });
+
+      this.donePromise = new Promise<void>((res) => {
+        this.doneResolve = res;
+      });
+      // Reset chain and final flag for this connection
+      this.audioChain = Promise.resolve();
+      this.isFinalReceived = false;
+
+      this.ws.on("open", async () => {
+        console.log("[elevenlabs-tts] Connected");
+
+        // Send BOS (beginning of stream) message
+        const bos = {
+          text: " ",
+          voice_settings: {
+            stability: 0.5,
+            similarity_boost: 0.75,
+          },
+          generation_config: {
+            flush: true,
+          },
+        };
+
+        this.ws!.send(JSON.stringify(bos));
+        resolve();
+      });
+
+      this.ws.on("message", (data: Buffer) => {
+        this.handleMessage(data);
+      });
+
+      this.ws.on("error", (err) => {
+        console.error("[elevenlabs-tts] WebSocket error:", err.message);
+        reject(err);
+      });
+
+      this.ws.on("close", () => {
+        console.log("[elevenlabs-tts] Connection closed");
+        // If isFinal already triggered the chain drain, fireDone will be a no-op.
+        // Otherwise (unexpected close) drain the chain first then fire done.
+        if (!this.isFinalReceived) {
+          this.audioChain
+            .then(() => this.fireDone())
+            .catch(() => this.fireDone());
+        }
+      });
+    });
+  }
+
+  async synthesize(text: string): Promise<void> {
+    if (!this.ws || this.ws.readyState !== WebSocket.OPEN) {
+      throw new Error("[elevenlabs-tts] Not connected");
+    }
+
+    const msg = {
+      text,
+      generation_config: { flush: true },
+    };
+
+    this.ws.send(JSON.stringify(msg));
+  }
+
+  async flush(): Promise<void> {
+    // Send EOS (end of stream) — empty text signals end of input
+    if (this.ws?.readyState === WebSocket.OPEN) {
+      this.ws.send(JSON.stringify({ text: "" }));
+    }
+
+    // Wait for all audio to be delivered
+    if (this.donePromise) {
+      const timeoutPromise = new Promise<void>((resolve) => {
+        setTimeout(() => {
+          console.warn("[elevenlabs-tts] Timeout waiting for audio completion");
+          resolve();
+        }, 30000);
+      });
+      await Promise.race([this.donePromise, timeoutPromise]);
+    }
+  }
+
+  async close(): Promise<void> {
+    if (this.ws) {
+      try {
+        this.ws.close();
+      } catch {
+        // Ignore close errors
+      }
+      this.ws = null;
+    }
+  }
+
+  private handleMessage(data: Buffer): void {
+    try {
+      const msg = JSON.parse(data.toString());
+
+      // Audio chunk: base64-encoded PCM.
+      // Chain onto audioChain so each chunk is processed *after* the previous
+      // one finishes — including any pacing sleeps in the onAudio callback.
+      const audioB64: string | undefined = msg.audio;
+      if (audioB64) {
+        const pcmBytes = Buffer.from(audioB64, "base64");
+        if (pcmBytes.length > 0 && this.onAudio) {
+          const cb = this.onAudio;
+          this.audioChain = this.audioChain
+            .then(() => cb(pcmBytes))
+            .catch((err) => console.error("[elevenlabs-tts] Audio callback error:", err));
+        }
+      }
+
+      // isFinal: ElevenLabs signals all audio has been sent.
+      // We must wait for the entire audioChain to drain before firing done,
+      // so the caller (flush) only unblocks after all pacing sleeps complete.
+      if (msg.isFinal) {
+        console.log("[elevenlabs-tts] Stream complete (isFinal)");
+        this.isFinalReceived = true;
+        this.audioChain
+          .then(() => this.fireDone())
+          .catch(() => this.fireDone());
+      }
+    } catch {
+      // Ignore parse errors
+    }
+  }
+
+  private fireDone(): void {
+    if (this.onDone) {
+      const result = this.onDone();
+      if (result instanceof Promise) {
+        result.catch((err) => console.error("[elevenlabs-tts] Done callback error:", err));
+      }
+    }
+    if (this.doneResolve) {
+      this.doneResolve();
+      this.doneResolve = null;
+    }
+  }
+}
+
+/** ElevenLabs TTS provider metadata. */
+export const elevenlabsMeta: TtsProviderMeta = {
+  id: "elevenlabs",
+  name: "ElevenLabs",
+  description:
+    "High-quality streaming TTS with natural-sounding voices. Supports WebSocket streaming for low latency.",
+  streaming: true,
+  envVar: "ELEVENLABS_API_KEY",
+  defaultVoiceId: DEFAULT_VOICE_ID,
+  defaultModel: DEFAULT_MODEL_ID,
+  outputSampleRate: 24000,
+  docsUrl: "https://elevenlabs.io/docs/api-reference/text-to-speech-websockets",
+};
+
+/** Factory function for creating ElevenLabs TTS instances. */
+function createElevenLabsTts(config: TtsProviderConfig): TtsProvider {
+  return new ElevenLabsTtsProvider(config);
+}
+
+// Auto-register with the TTS registry
+ttsRegistry.register(elevenlabsMeta, createElevenLabsTts);