@cohortapp/agent-sdk 2.3.0

package/scripts/huddle/audio-bridge.mjs

@@ -0,0 +1,664 @@
+#!/usr/bin/env node
+/**
+ * Maestro Huddle — Audio Bridge
+ *
+ * Bidirectional audio pipeline between Slack huddles and the agent's voice AI:
+ *
+ *   INBOUND (what others say):
+ *     BlackHole 2ch → sox capture → PCM stream → Deepgram STT → transcript
+ *
+ *   OUTBOUND (what the agent says):
+ *     Response text → ElevenLabs TTS → MP3/PCM → sox playback → BlackHole 16ch
+ *
+ * The two paths use separate virtual audio devices to prevent feedback loops.
+ *
+ * Dependencies:
+ *   - sox (brew install sox) — audio capture and playback
+ *   - BlackHole 2ch — Slack speaker output (capture device)
+ *   - BlackHole 16ch — Slack microphone input (playback device)
+ *   - Deepgram API key (real-time STT)
+ *   - ElevenLabs API key (TTS)
+ */
+
+import { EventEmitter } from "node:events";
+import { spawn, execFileSync } from "node:child_process";
+import { WebSocket } from "ws";
+import { WebSocketServer } from "ws";
+import { writeFile, unlink } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { createServer } from "node:http";
+
+// ---------------------------------------------------------------------------
+// Configuration
+// ---------------------------------------------------------------------------
+
+const CAPTURE_DEVICE = process.env.HUDDLE_CAPTURE_DEVICE || "BlackHole 2ch";
+const PLAYBACK_DEVICE = process.env.HUDDLE_PLAYBACK_DEVICE || "BlackHole 16ch";
+
+// Capture mode: "webaudio" bypasses BlackHole for inbound audio (fixes echo),
+// "sox" uses the original BlackHole 2ch pipeline.
+const CAPTURE_MODE = process.env.HUDDLE_CAPTURE_MODE || "webaudio";
+const CAPTURE_WS_PORT = parseInt(process.env.HUDDLE_CAPTURE_WS_PORT || "3201", 10);
+
+const DEEPGRAM_API_KEY = process.env.DEEPGRAM_API_KEY || "";
+const DEEPGRAM_WS_URL = "wss://api.deepgram.com/v1/listen";
+const DEEPGRAM_PARAMS = new URLSearchParams({
+  model: "nova-2",
+  language: "en",
+  smart_format: "true",
+  punctuate: "true",
+  interim_results: "true",
+  utterance_end_ms: "1500",    // 1.5s silence = utterance boundary
+  vad_events: "true",          // Voice activity detection
+  endpointing: "800",          // 800ms endpoint detection
+  encoding: "linear16",
+  sample_rate: "16000",
+  channels: "1",
+});
+
+const ELEVENLABS_API_KEY = process.env.ELEVENLABS_API_KEY || "";
+const ELEVENLABS_VOICE_ID = process.env.ELEVENLABS_VOICE_ID || "EXAVITQu4vr4xnSDxMaL"; // Sarah
+const ELEVENLABS_MODEL = process.env.ELEVENLABS_MODEL || "eleven_flash_v2_5";
+const ELEVENLABS_API_URL = "https://api.elevenlabs.io/v1/text-to-speech";
+
+const SAMPLE_RATE = 16000;
+const CHANNELS = 1;
+const BIT_DEPTH = 16;
+
+// Silence detection: how many ms of silence before we consider speech done
+const SPEECH_TIMEOUT_MS = 1500;
+
+/**
+ * AudioBridge manages bidirectional audio for huddle voice participation.
+ *
+ * Events:
+ *   - 'transcript'        — Final transcript ready { text, isFinal }
+ *   - 'transcript:interim' — Interim transcript { text }
+ *   - 'speech:start'      — Voice activity detected (someone started talking)
+ *   - 'speech:end'        — Voice activity ended (silence detected)
+ *   - 'tts:start'         — TTS playback started { text }
+ *   - 'tts:end'           — TTS playback finished
+ *   - 'error'             — Error occurred { component, message }
+ *   - 'capture:started'   — Audio capture started
+ *   - 'capture:stopped'   — Audio capture stopped
+ */
+export class AudioBridge extends EventEmitter {
+  constructor(options = {}) {
+    super();
+    this.captureDevice = options.captureDevice || CAPTURE_DEVICE;
+    this.playbackDevice = options.playbackDevice || PLAYBACK_DEVICE;
+    this.deepgramKey = options.deepgramKey || DEEPGRAM_API_KEY;
+    this.elevenLabsKey = options.elevenLabsKey || ELEVENLABS_API_KEY;
+    this.voiceId = options.voiceId || ELEVENLABS_VOICE_ID;
+
+    // State
+    this.captureProcess = null;
+    this.deepgramWs = null;
+    this.isCapturing = false;
+    this.isSpeaking = false;        // the agent is currently speaking (TTS playing)
+    this.speechTimer = null;
+    this.currentUtterance = "";     // Accumulated speech from current speaker
+    this.ttsQueue = [];             // Queue of texts waiting to be spoken
+    this.ttsProcessing = false;
+    this.captureMode = options.captureMode || CAPTURE_MODE;
+    this.captureWsServer = null;    // WebSocket server for Web Audio capture
+    this.captureWsClient = null;    // Connected Web Audio client
+  }
+
+  // -------------------------------------------------------------------------
+  // Lifecycle
+  // -------------------------------------------------------------------------
+
+  /**
+   * Start the audio bridge — begin capturing from BlackHole and streaming
+   * to Deepgram for real-time transcription.
+   */
+  async start() {
+    if (this.isCapturing) return;
+
+    if (!this.deepgramKey) {
+      throw new Error("DEEPGRAM_API_KEY required for real-time STT");
+    }
+    if (!this.elevenLabsKey) {
+      throw new Error("ELEVENLABS_API_KEY required for TTS");
+    }
+
+    // Start Deepgram WebSocket first
+    await this._connectDeepgram();
+
+    // Start audio capture
+    if (this.captureMode === "webaudio") {
+      await this._startWebAudioCapture();
+    } else {
+      this._startCapture();
+    }
+
+    this.isCapturing = true;
+    this.emit("capture:started");
+  }
+
+  /**
+   * Stop the audio bridge — kill capture process and close Deepgram connection.
+   */
+  async stop() {
+    this.isCapturing = false;
+
+    // Stop capture
+    if (this.captureProcess) {
+      this.captureProcess.kill("SIGTERM");
+      this.captureProcess = null;
+    }
+
+    // Stop Web Audio capture server
+    if (this.captureWsServer) {
+      if (this.captureWsClient) {
+        this.captureWsClient.close();
+        this.captureWsClient = null;
+      }
+      this.captureWsServer.close();
+      this.captureWsServer = null;
+    }
+
+    // Close Deepgram
+    if (this.deepgramWs) {
+      // Send close frame to flush final transcript
+      try {
+        this.deepgramWs.send(JSON.stringify({ type: "CloseStream" }));
+      } catch { /* ignore */ }
+      this.deepgramWs.close();
+      this.deepgramWs = null;
+    }
+
+    if (this.speechTimer) {
+      clearTimeout(this.speechTimer);
+      this.speechTimer = null;
+    }
+
+    this.emit("capture:stopped");
+  }
+
+  // -------------------------------------------------------------------------
+  // Inbound: Audio Capture → Deepgram STT
+  // -------------------------------------------------------------------------
+
+  _startCapture() {
+    // Use sox to capture from BlackHole 2ch and output raw PCM to stdout
+    this.captureProcess = spawn("sox", [
+      "-t", "coreaudio", this.captureDevice,   // Input: BlackHole 2ch
+      "-t", "raw",                              // Output: raw PCM
+      "-r", String(SAMPLE_RATE),                // 16kHz
+      "-e", "signed-integer",                   // Signed int
+      "-b", String(BIT_DEPTH),                  // 16-bit
+      "-c", String(CHANNELS),                   // Mono
+      "-",                                      // stdout
+    ], {
+      stdio: ["ignore", "pipe", "pipe"],
+    });
+
+    this.captureProcess.stdout.on("data", (chunk) => {
+      // Don't send audio to Deepgram while the agent is speaking
+      // This prevents the STT from transcribing the agent's own voice
+      if (this.isSpeaking) return;
+
+      if (this.deepgramWs && this.deepgramWs.readyState === WebSocket.OPEN) {
+        this.deepgramWs.send(chunk);
+      }
+    });
+
+    this.captureProcess.stderr.on("data", (data) => {
+      const msg = data.toString().trim();
+      // sox prints informational messages to stderr — only log errors
+      if (msg && !msg.startsWith("sox WARN")) {
+        console.error(`[CAPTURE] ${msg}`);
+      }
+    });
+
+    this.captureProcess.on("exit", (code) => {
+      if (this.isCapturing) {
+        // Unexpected exit — restart
+        console.error(`[CAPTURE] sox exited with code ${code}, restarting...`);
+        setTimeout(() => {
+          if (this.isCapturing) this._startCapture();
+        }, 1000);
+      }
+    });
+  }
+
+  // -------------------------------------------------------------------------
+  // Inbound: Web Audio API capture (echo-free alternative to sox/BlackHole)
+  //
+  // Architecture:
+  //   1. A local WebSocket server receives PCM audio from the browser
+  //   2. A ScriptProcessorNode injected into Slack's renderer via CDP
+  //      captures the incoming WebRTC audio stream (before AEC)
+  //   3. PCM chunks are forwarded to Deepgram STT
+  //
+  // This bypasses Slack's speaker output entirely, eliminating the AEC
+  // echo artifacts that occur with virtual audio devices.
+  // -------------------------------------------------------------------------
+
+  async _startWebAudioCapture() {
+    // Step 1: Start a local WebSocket server to receive PCM from the browser
+    await this._startCaptureWsServer();
+
+    // Step 2: Inject the Web Audio capture script into Slack's renderer via CDP
+    await this._injectWebAudioCapture();
+
+    console.log(`[CAPTURE] Web Audio mode — listening on ws://127.0.0.1:${CAPTURE_WS_PORT}`);
+  }
+
+  _startCaptureWsServer() {
+    return new Promise((resolve) => {
+      const httpServer = createServer();
+      this.captureWsServer = new WebSocketServer({ server: httpServer });
+
+      this.captureWsServer.on("connection", (ws) => {
+        console.log("[CAPTURE] Browser audio stream connected");
+        this.captureWsClient = ws;
+
+        ws.on("message", (data) => {
+          // Don't forward to Deepgram while the agent is speaking
+          if (this.isSpeaking) return;
+
+          if (this.deepgramWs && this.deepgramWs.readyState === WebSocket.OPEN) {
+            this.deepgramWs.send(data);
+          }
+        });
+
+        ws.on("close", () => {
+          console.log("[CAPTURE] Browser audio stream disconnected");
+          this.captureWsClient = null;
+        });
+      });
+
+      httpServer.listen(CAPTURE_WS_PORT, "127.0.0.1", () => {
+        resolve();
+      });
+    });
+  }
+
+  /**
+   * Inject a Web Audio capture script into Slack's main renderer via CDP.
+   * The script finds the huddle audio element, taps its MediaStream with
+   * a ScriptProcessorNode, and streams PCM to our local WebSocket server.
+   */
+  async _injectWebAudioCapture() {
+    // Find the Slack main page CDP target
+    const resp = await fetch("http://localhost:9222/json");
+    const targets = await resp.json();
+    const mainTarget = targets.find(
+      (t) => t.type === "page" && t.url?.includes("app.slack.com/client")
+    );
+    if (!mainTarget) {
+      console.error("[CAPTURE] No Slack CDP target found for Web Audio injection");
+      return;
+    }
+
+    const captureScript = `
+      (async () => {
+        if (window._agentAudioCapture) {
+          console.log("[AGENT-CAPTURE] Already injected");
+          return "already-injected";
+        }
+
+        // Find the audio element carrying huddle audio
+        const findAudioStream = () => {
+          const audioEls = document.querySelectorAll("audio");
+          for (const el of audioEls) {
+            if (el.srcObject && el.srcObject.getAudioTracks().length > 0) {
+              return el.srcObject;
+            }
+          }
+          // Fallback: capture from the element itself
+          for (const el of audioEls) {
+            if (!el.paused && el.captureStream) {
+              return el.captureStream();
+            }
+          }
+          return null;
+        };
+
+        // Retry until we find an active audio stream
+        let stream = null;
+        for (let i = 0; i < 30; i++) {
+          stream = findAudioStream();
+          if (stream) break;
+          await new Promise(r => setTimeout(r, 1000));
+        }
+
+        if (!stream) {
+          console.error("[AGENT-CAPTURE] No audio stream found after 30s");
+          return "no-stream";
+        }
+
+        console.log("[AGENT-CAPTURE] Found audio stream with",
+          stream.getAudioTracks().length, "tracks");
+
+        // Connect to the agent's capture WebSocket server
+        const ws = new WebSocket("ws://127.0.0.1:${CAPTURE_WS_PORT}");
+
+        ws.onopen = () => {
+          console.log("[AGENT-CAPTURE] Connected to capture server");
+
+          // Create AudioContext and ScriptProcessor to capture PCM
+          const ctx = new AudioContext({ sampleRate: 16000 });
+          const source = ctx.createMediaStreamSource(stream);
+
+          // ScriptProcessorNode: 4096 buffer, 1 input channel, 1 output channel
+          const processor = ctx.createScriptProcessor(4096, 1, 1);
+
+          processor.onaudioprocess = (e) => {
+            if (ws.readyState !== WebSocket.OPEN) return;
+
+            const float32 = e.inputBuffer.getChannelData(0);
+
+            // Convert Float32 to Int16 PCM (what Deepgram expects)
+            const int16 = new Int16Array(float32.length);
+            for (let i = 0; i < float32.length; i++) {
+              const s = Math.max(-1, Math.min(1, float32[i]));
+              int16[i] = s < 0 ? s * 0x8000 : s * 0x7FFF;
+            }
+
+            ws.send(int16.buffer);
+          };
+
+          source.connect(processor);
+          processor.connect(ctx.destination); // Required for processor to work
+
+          window._agentAudioCapture = { ctx, source, processor, ws };
+          console.log("[AGENT-CAPTURE] Audio capture pipeline active (16kHz PCM)");
+        };
+
+        ws.onerror = (e) => {
+          console.error("[AGENT-CAPTURE] WebSocket error:", e.message || e);
+        };
+
+        ws.onclose = () => {
+          console.log("[AGENT-CAPTURE] WebSocket closed — cleaning up");
+          if (window._agentAudioCapture) {
+            window._agentAudioCapture.processor.disconnect();
+            window._agentAudioCapture.source.disconnect();
+            window._agentAudioCapture.ctx.close();
+            window._agentAudioCapture = null;
+          }
+        };
+
+        return "injected";
+      })()
+    `;
+
+    // Inject via CDP
+    const { WebSocket: WS } = await import("ws");
+    return new Promise((resolve) => {
+      const ws = new WS(mainTarget.webSocketDebuggerUrl);
+      ws.on("error", () => resolve());
+      ws.on("open", () => {
+        ws.send(JSON.stringify({
+          id: 1,
+          method: "Runtime.evaluate",
+          params: { expression: captureScript, returnByValue: true, awaitPromise: true },
+        }));
+      });
+      ws.on("message", (data) => {
+        const msg = JSON.parse(data.toString());
+        if (msg.id === 1) {
+          const result = msg.result?.result?.value || "unknown";
+          console.log(`[CAPTURE] CDP injection result: ${result}`);
+          ws.close();
+          resolve();
+        }
+      });
+      setTimeout(() => { ws.close(); resolve(); }, 35000);
+    });
+  }
+
+  async _connectDeepgram() {
+    const url = `${DEEPGRAM_WS_URL}?${DEEPGRAM_PARAMS.toString()}`;
+
+    return new Promise((resolve, reject) => {
+      this.deepgramWs = new WebSocket(url, {
+        headers: {
+          Authorization: `Token ${this.deepgramKey}`,
+        },
+      });
+
+      const timeout = setTimeout(() => {
+        reject(new Error("Deepgram connection timed out"));
+      }, 10000);
+
+      this.deepgramWs.on("open", () => {
+        clearTimeout(timeout);
+        console.log("[DEEPGRAM] Connected — real-time STT active");
+        resolve();
+      });
+
+      this.deepgramWs.on("message", (raw) => {
+        try {
+          const msg = JSON.parse(raw.toString());
+          this._handleDeepgramMessage(msg);
+        } catch (err) {
+          console.error("[DEEPGRAM] Parse error:", err.message);
+        }
+      });
+
+      this.deepgramWs.on("error", (err) => {
+        console.error("[DEEPGRAM] WebSocket error:", err.message);
+        this.emit("error", { component: "deepgram", message: err.message });
+      });
+
+      this.deepgramWs.on("close", (code, reason) => {
+        console.log(`[DEEPGRAM] Closed: ${code} ${reason}`);
+        // Reconnect if bridge is still active
+        if (this.isCapturing) {
+          setTimeout(() => this._connectDeepgram().catch(() => {}), 2000);
+        }
+      });
+    });
+  }
+
+  _handleDeepgramMessage(msg) {
+    // Voice Activity Detection events
+    if (msg.type === "SpeechStarted") {
+      this.emit("speech:start");
+      if (this.speechTimer) {
+        clearTimeout(this.speechTimer);
+        this.speechTimer = null;
+      }
+      return;
+    }
+
+    if (msg.type === "UtteranceEnd") {
+      // Utterance boundary — finalize accumulated text
+      if (this.currentUtterance.trim()) {
+        this.emit("transcript", {
+          text: this.currentUtterance.trim(),
+          isFinal: true,
+        });
+        this.currentUtterance = "";
+      }
+      this.emit("speech:end");
+      return;
+    }
+
+    // Transcript results
+    if (msg.type === "Results" && msg.channel) {
+      const alt = msg.channel.alternatives?.[0];
+      if (!alt) return;
+
+      const transcript = alt.transcript;
+      if (!transcript) return;
+
+      if (msg.is_final) {
+        // Final result for this segment
+        this.currentUtterance += (this.currentUtterance ? " " : "") + transcript;
+
+        // Reset speech timeout
+        if (this.speechTimer) clearTimeout(this.speechTimer);
+        this.speechTimer = setTimeout(() => {
+          if (this.currentUtterance.trim()) {
+            this.emit("transcript", {
+              text: this.currentUtterance.trim(),
+              isFinal: true,
+            });
+            this.currentUtterance = "";
+            this.emit("speech:end");
+          }
+        }, SPEECH_TIMEOUT_MS);
+      } else {
+        // Interim result — show in real-time but don't accumulate
+        const interim = this.currentUtterance
+          ? this.currentUtterance + " " + transcript
+          : transcript;
+        this.emit("transcript:interim", { text: interim });
+      }
+    }
+  }
+
+  // -------------------------------------------------------------------------
+  // Outbound: Text → ElevenLabs TTS → BlackHole 16ch
+  // -------------------------------------------------------------------------
+
+  /**
+   * Speak text through the huddle. Queues if already speaking.
+   * @param {string} text - The text for the agent to say
+   * @returns {Promise<void>} Resolves when speech playback completes
+   */
+  async speak(text) {
+    if (!text || !text.trim()) return;
+
+    return new Promise((resolve) => {
+      this.ttsQueue.push({ text: text.trim(), resolve });
+      this._processTtsQueue();
+    });
+  }
+
+  async _processTtsQueue() {
+    if (this.ttsProcessing || this.ttsQueue.length === 0) return;
+    this.ttsProcessing = true;
+
+    while (this.ttsQueue.length > 0) {
+      const { text, resolve } = this.ttsQueue.shift();
+
+      try {
+        this.isSpeaking = true;
+        this.emit("tts:start", { text });
+
+        // Generate TTS audio
+        const audioBuffer = await this._generateTts(text);
+
+        // Play audio to BlackHole 16ch (Slack's mic input)
+        await this._playAudio(audioBuffer);
+
+        this.emit("tts:end");
+      } catch (err) {
+        console.error(`[TTS] Error: ${err.message}`);
+        this.emit("error", { component: "tts", message: err.message });
+      } finally {
+        this.isSpeaking = false;
+        resolve();
+      }
+    }
+
+    this.ttsProcessing = false;
+  }
+
+  /**
+   * Generate speech audio from ElevenLabs API.
+   * @param {string} text
+   * @returns {Promise<Buffer>} Raw audio data (MP3)
+   */
+  async _generateTts(text) {
+    const url = `${ELEVENLABS_API_URL}/${this.voiceId}`;
+
+    const response = await fetch(url, {
+      method: "POST",
+      headers: {
+        "xi-api-key": this.elevenLabsKey,
+        "Content-Type": "application/json",
+        Accept: "audio/mpeg",
+      },
+      body: JSON.stringify({
+        text,
+        model_id: ELEVENLABS_MODEL,
+        voice_settings: {
+          stability: 0.5,
+          similarity_boost: 0.8,
+          speed: 1.0,
+        },
+      }),
+    });
+
+    if (!response.ok) {
+      const errText = await response.text().catch(() => "");
+      throw new Error(`ElevenLabs API error ${response.status}: ${errText}`);
+    }
+
+    const arrayBuffer = await response.arrayBuffer();
+    return Buffer.from(arrayBuffer);
+  }
+
+  /**
+   * Play audio buffer to the playback device (BlackHole 16ch).
+   * Uses sox to decode MP3 and output to the CoreAudio device.
+   * @param {Buffer} audioBuffer - MP3 audio data
+   * @returns {Promise<void>}
+   */
+  async _playAudio(audioBuffer) {
+    // Write MP3 to a temp file (sox needs seekable input for MP3)
+    const tmpFile = join(tmpdir(), `agent-tts-${Date.now()}.mp3`);
+    await writeFile(tmpFile, audioBuffer);
+
+    return new Promise((resolve, reject) => {
+      const player = spawn("sox", [
+        tmpFile,                                  // Input: MP3 file
+        "-t", "coreaudio", this.playbackDevice,   // Output: BlackHole 16ch
+      ], {
+        stdio: ["ignore", "ignore", "pipe"],
+      });
+
+      let stderrBuf = "";
+      player.stderr.on("data", (d) => { stderrBuf += d.toString(); });
+
+      player.on("exit", async (code) => {
+        // Clean up temp file
+        await unlink(tmpFile).catch(() => {});
+
+        if (code === 0) {
+          resolve();
+        } else {
+          reject(new Error(`sox playback exited ${code}: ${stderrBuf.trim()}`));
+        }
+      });
+
+      player.on("error", async (err) => {
+        await unlink(tmpFile).catch(() => {});
+        reject(err);
+      });
+    });
+  }
+
+  // -------------------------------------------------------------------------
+  // Utilities
+  // -------------------------------------------------------------------------
+
+  /**
+   * Check if the audio devices are available and working.
+   * @returns {Promise<{capture: boolean, playback: boolean}>}
+   */
+  async checkDevices() {
+    const result = { capture: false, playback: false };
+
+    try {
+      const devices = execFileSync("SwitchAudioSource", ["-a"], { encoding: "utf8" });
+      result.capture = devices.includes(this.captureDevice);
+      result.playback = devices.includes(this.playbackDevice);
+    } catch {
+      // SwitchAudioSource not available
+    }
+
+    return result;
+  }
+}
+
+export default AudioBridge;