@openclaw/voice-call 2026.1.29

package/src/media-stream.test.ts

@@ -0,0 +1,97 @@
+import { describe, expect, it } from "vitest";
+
+import type {
+  OpenAIRealtimeSTTProvider,
+  RealtimeSTTSession,
+} from "./providers/stt-openai-realtime.js";
+import { MediaStreamHandler } from "./media-stream.js";
+
+const createStubSession = (): RealtimeSTTSession => ({
+  connect: async () => {},
+  sendAudio: () => {},
+  waitForTranscript: async () => "",
+  onPartial: () => {},
+  onTranscript: () => {},
+  onSpeechStart: () => {},
+  close: () => {},
+  isConnected: () => true,
+});
+
+const createStubSttProvider = (): OpenAIRealtimeSTTProvider =>
+  ({
+    createSession: () => createStubSession(),
+  }) as unknown as OpenAIRealtimeSTTProvider;
+
+const flush = async (): Promise<void> => {
+  await new Promise((resolve) => setTimeout(resolve, 0));
+};
+
+const waitForAbort = (signal: AbortSignal): Promise<void> =>
+  new Promise((resolve) => {
+    if (signal.aborted) {
+      resolve();
+      return;
+    }
+    signal.addEventListener("abort", () => resolve(), { once: true });
+  });
+
+describe("MediaStreamHandler TTS queue", () => {
+  it("serializes TTS playback and resolves in order", async () => {
+    const handler = new MediaStreamHandler({
+      sttProvider: createStubSttProvider(),
+    });
+    const started: number[] = [];
+    const finished: number[] = [];
+
+    let resolveFirst!: () => void;
+    const firstGate = new Promise<void>((resolve) => {
+      resolveFirst = resolve;
+    });
+
+    const first = handler.queueTts("stream-1", async () => {
+      started.push(1);
+      await firstGate;
+      finished.push(1);
+    });
+    const second = handler.queueTts("stream-1", async () => {
+      started.push(2);
+      finished.push(2);
+    });
+
+    await flush();
+    expect(started).toEqual([1]);
+
+    resolveFirst();
+    await first;
+    await second;
+
+    expect(started).toEqual([1, 2]);
+    expect(finished).toEqual([1, 2]);
+  });
+
+  it("cancels active playback and clears queued items", async () => {
+    const handler = new MediaStreamHandler({
+      sttProvider: createStubSttProvider(),
+    });
+
+    let queuedRan = false;
+    const started: string[] = [];
+
+    const active = handler.queueTts("stream-1", async (signal) => {
+      started.push("active");
+      await waitForAbort(signal);
+    });
+    void handler.queueTts("stream-1", async () => {
+      queuedRan = true;
+    });
+
+    await flush();
+    expect(started).toEqual(["active"]);
+
+    handler.clearTtsQueue("stream-1");
+    await active;
+    await flush();
+
+    expect(queuedRan).toBe(false);
+  });
+});

package/src/media-stream.ts

@@ -0,0 +1,393 @@
+/**
+ * Media Stream Handler
+ *
+ * Handles bidirectional audio streaming between Twilio and the AI services.
+ * - Receives mu-law audio from Twilio via WebSocket
+ * - Forwards to OpenAI Realtime STT for transcription
+ * - Sends TTS audio back to Twilio
+ */
+
+import type { IncomingMessage } from "node:http";
+import type { Duplex } from "node:stream";
+
+import { WebSocket, WebSocketServer } from "ws";
+
+import type {
+  OpenAIRealtimeSTTProvider,
+  RealtimeSTTSession,
+} from "./providers/stt-openai-realtime.js";
+
+/**
+ * Configuration for the media stream handler.
+ */
+export interface MediaStreamConfig {
+  /** STT provider for transcription */
+  sttProvider: OpenAIRealtimeSTTProvider;
+  /** Callback when transcript is received */
+  onTranscript?: (callId: string, transcript: string) => void;
+  /** Callback for partial transcripts (streaming UI) */
+  onPartialTranscript?: (callId: string, partial: string) => void;
+  /** Callback when stream connects */
+  onConnect?: (callId: string, streamSid: string) => void;
+  /** Callback when speech starts (barge-in) */
+  onSpeechStart?: (callId: string) => void;
+  /** Callback when stream disconnects */
+  onDisconnect?: (callId: string) => void;
+}
+
+/**
+ * Active media stream session.
+ */
+interface StreamSession {
+  callId: string;
+  streamSid: string;
+  ws: WebSocket;
+  sttSession: RealtimeSTTSession;
+}
+
+type TtsQueueEntry = {
+  playFn: (signal: AbortSignal) => Promise<void>;
+  controller: AbortController;
+  resolve: () => void;
+  reject: (error: unknown) => void;
+};
+
+/**
+ * Manages WebSocket connections for Twilio media streams.
+ */
+export class MediaStreamHandler {
+  private wss: WebSocketServer | null = null;
+  private sessions = new Map<string, StreamSession>();
+  private config: MediaStreamConfig;
+  /** TTS playback queues per stream (serialize audio to prevent overlap) */
+  private ttsQueues = new Map<string, TtsQueueEntry[]>();
+  /** Whether TTS is currently playing per stream */
+  private ttsPlaying = new Map<string, boolean>();
+  /** Active TTS playback controllers per stream */
+  private ttsActiveControllers = new Map<string, AbortController>();
+
+  constructor(config: MediaStreamConfig) {
+    this.config = config;
+  }
+
+  /**
+   * Handle WebSocket upgrade for media stream connections.
+   */
+  handleUpgrade(request: IncomingMessage, socket: Duplex, head: Buffer): void {
+    if (!this.wss) {
+      this.wss = new WebSocketServer({ noServer: true });
+      this.wss.on("connection", (ws, req) => this.handleConnection(ws, req));
+    }
+
+    this.wss.handleUpgrade(request, socket, head, (ws) => {
+      this.wss?.emit("connection", ws, request);
+    });
+  }
+
+  /**
+   * Handle new WebSocket connection from Twilio.
+   */
+  private async handleConnection(
+    ws: WebSocket,
+    _request: IncomingMessage,
+  ): Promise<void> {
+    let session: StreamSession | null = null;
+
+    ws.on("message", async (data: Buffer) => {
+      try {
+        const message = JSON.parse(data.toString()) as TwilioMediaMessage;
+
+        switch (message.event) {
+          case "connected":
+            console.log("[MediaStream] Twilio connected");
+            break;
+
+          case "start":
+            session = await this.handleStart(ws, message);
+            break;
+
+          case "media":
+            if (session && message.media?.payload) {
+              // Forward audio to STT
+              const audioBuffer = Buffer.from(message.media.payload, "base64");
+              session.sttSession.sendAudio(audioBuffer);
+            }
+            break;
+
+          case "stop":
+            if (session) {
+              this.handleStop(session);
+              session = null;
+            }
+            break;
+        }
+      } catch (error) {
+        console.error("[MediaStream] Error processing message:", error);
+      }
+    });
+
+    ws.on("close", () => {
+      if (session) {
+        this.handleStop(session);
+      }
+    });
+
+    ws.on("error", (error) => {
+      console.error("[MediaStream] WebSocket error:", error);
+    });
+  }
+
+  /**
+   * Handle stream start event.
+   */
+  private async handleStart(
+    ws: WebSocket,
+    message: TwilioMediaMessage,
+  ): Promise<StreamSession> {
+    const streamSid = message.streamSid || "";
+    const callSid = message.start?.callSid || "";
+
+    console.log(
+      `[MediaStream] Stream started: ${streamSid} (call: ${callSid})`,
+    );
+
+    // Create STT session
+    const sttSession = this.config.sttProvider.createSession();
+
+    // Set up transcript callbacks
+    sttSession.onPartial((partial) => {
+      this.config.onPartialTranscript?.(callSid, partial);
+    });
+
+    sttSession.onTranscript((transcript) => {
+      this.config.onTranscript?.(callSid, transcript);
+    });
+
+    sttSession.onSpeechStart(() => {
+      this.config.onSpeechStart?.(callSid);
+    });
+
+    const session: StreamSession = {
+      callId: callSid,
+      streamSid,
+      ws,
+      sttSession,
+    };
+
+    this.sessions.set(streamSid, session);
+
+    // Notify connection BEFORE STT connect so TTS can work even if STT fails
+    this.config.onConnect?.(callSid, streamSid);
+
+    // Connect to OpenAI STT (non-blocking, log errors but don't fail the call)
+    sttSession.connect().catch((err) => {
+      console.warn(
+        `[MediaStream] STT connection failed (TTS still works):`,
+        err.message,
+      );
+    });
+
+    return session;
+  }
+
+  /**
+   * Handle stream stop event.
+   */
+  private handleStop(session: StreamSession): void {
+    console.log(`[MediaStream] Stream stopped: ${session.streamSid}`);
+
+    this.clearTtsState(session.streamSid);
+    session.sttSession.close();
+    this.sessions.delete(session.streamSid);
+    this.config.onDisconnect?.(session.callId);
+  }
+
+  /**
+   * Get an active session with an open WebSocket, or undefined if unavailable.
+   */
+  private getOpenSession(streamSid: string): StreamSession | undefined {
+    const session = this.sessions.get(streamSid);
+    return session?.ws.readyState === WebSocket.OPEN ? session : undefined;
+  }
+
+  /**
+   * Send a message to a stream's WebSocket if available.
+   */
+  private sendToStream(streamSid: string, message: unknown): void {
+    const session = this.getOpenSession(streamSid);
+    session?.ws.send(JSON.stringify(message));
+  }
+
+  /**
+   * Send audio to a specific stream (for TTS playback).
+   * Audio should be mu-law encoded at 8kHz mono.
+   */
+  sendAudio(streamSid: string, muLawAudio: Buffer): void {
+    this.sendToStream(streamSid, {
+      event: "media",
+      streamSid,
+      media: { payload: muLawAudio.toString("base64") },
+    });
+  }
+
+  /**
+   * Send a mark event to track audio playback position.
+   */
+  sendMark(streamSid: string, name: string): void {
+    this.sendToStream(streamSid, {
+      event: "mark",
+      streamSid,
+      mark: { name },
+    });
+  }
+
+  /**
+   * Clear audio buffer (interrupt playback).
+   */
+  clearAudio(streamSid: string): void {
+    this.sendToStream(streamSid, { event: "clear", streamSid });
+  }
+
+  /**
+   * Queue a TTS operation for sequential playback.
+   * Only one TTS operation plays at a time per stream to prevent overlap.
+   */
+  async queueTts(
+    streamSid: string,
+    playFn: (signal: AbortSignal) => Promise<void>,
+  ): Promise<void> {
+    const queue = this.getTtsQueue(streamSid);
+    let resolveEntry: () => void;
+    let rejectEntry: (error: unknown) => void;
+    const promise = new Promise<void>((resolve, reject) => {
+      resolveEntry = resolve;
+      rejectEntry = reject;
+    });
+
+    queue.push({
+      playFn,
+      controller: new AbortController(),
+      resolve: resolveEntry!,
+      reject: rejectEntry!,
+    });
+
+    if (!this.ttsPlaying.get(streamSid)) {
+      void this.processQueue(streamSid);
+    }
+
+    return promise;
+  }
+
+  /**
+   * Clear TTS queue and interrupt current playback (barge-in).
+   */
+  clearTtsQueue(streamSid: string): void {
+    const queue = this.getTtsQueue(streamSid);
+    queue.length = 0;
+    this.ttsActiveControllers.get(streamSid)?.abort();
+    this.clearAudio(streamSid);
+  }
+
+  /**
+   * Get active session by call ID.
+   */
+  getSessionByCallId(callId: string): StreamSession | undefined {
+    return [...this.sessions.values()].find(
+      (session) => session.callId === callId,
+    );
+  }
+
+  /**
+   * Close all sessions.
+   */
+  closeAll(): void {
+    for (const session of this.sessions.values()) {
+      this.clearTtsState(session.streamSid);
+      session.sttSession.close();
+      session.ws.close();
+    }
+    this.sessions.clear();
+  }
+
+  private getTtsQueue(streamSid: string): TtsQueueEntry[] {
+    const existing = this.ttsQueues.get(streamSid);
+    if (existing) return existing;
+    const queue: TtsQueueEntry[] = [];
+    this.ttsQueues.set(streamSid, queue);
+    return queue;
+  }
+
+  /**
+   * Process the TTS queue for a stream.
+   * Uses iterative approach to avoid stack accumulation from recursion.
+   */
+  private async processQueue(streamSid: string): Promise<void> {
+    this.ttsPlaying.set(streamSid, true);
+
+    while (true) {
+      const queue = this.ttsQueues.get(streamSid);
+      if (!queue || queue.length === 0) {
+        this.ttsPlaying.set(streamSid, false);
+        this.ttsActiveControllers.delete(streamSid);
+        return;
+      }
+
+      const entry = queue.shift()!;
+      this.ttsActiveControllers.set(streamSid, entry.controller);
+
+      try {
+        await entry.playFn(entry.controller.signal);
+        entry.resolve();
+      } catch (error) {
+        if (entry.controller.signal.aborted) {
+          entry.resolve();
+        } else {
+          console.error("[MediaStream] TTS playback error:", error);
+          entry.reject(error);
+        }
+      } finally {
+        if (this.ttsActiveControllers.get(streamSid) === entry.controller) {
+          this.ttsActiveControllers.delete(streamSid);
+        }
+      }
+    }
+  }
+
+  private clearTtsState(streamSid: string): void {
+    const queue = this.ttsQueues.get(streamSid);
+    if (queue) queue.length = 0;
+    this.ttsActiveControllers.get(streamSid)?.abort();
+    this.ttsActiveControllers.delete(streamSid);
+    this.ttsPlaying.delete(streamSid);
+    this.ttsQueues.delete(streamSid);
+  }
+}
+
+/**
+ * Twilio Media Stream message format.
+ */
+interface TwilioMediaMessage {
+  event: "connected" | "start" | "media" | "stop" | "mark" | "clear";
+  sequenceNumber?: string;
+  streamSid?: string;
+  start?: {
+    streamSid: string;
+    accountSid: string;
+    callSid: string;
+    tracks: string[];
+    mediaFormat: {
+      encoding: string;
+      sampleRate: number;
+      channels: number;
+    };
+  };
+  media?: {
+    track?: string;
+    chunk?: string;
+    timestamp?: string;
+    payload?: string;
+  };
+  mark?: {
+    name: string;
+  };
+}

package/src/providers/base.ts

@@ -0,0 +1,67 @@
+import type {
+  HangupCallInput,
+  InitiateCallInput,
+  InitiateCallResult,
+  PlayTtsInput,
+  ProviderName,
+  ProviderWebhookParseResult,
+  StartListeningInput,
+  StopListeningInput,
+  WebhookContext,
+  WebhookVerificationResult,
+} from "../types.js";
+
+/**
+ * Abstract base interface for voice call providers.
+ *
+ * Each provider (Telnyx, Twilio, etc.) implements this interface to provide
+ * a consistent API for the call manager.
+ *
+ * Responsibilities:
+ * - Webhook verification and event parsing
+ * - Outbound call initiation and hangup
+ * - Media control (TTS playback, STT listening)
+ */
+export interface VoiceCallProvider {
+  /** Provider identifier */
+  readonly name: ProviderName;
+
+  /**
+   * Verify webhook signature/HMAC before processing.
+   * Must be called before parseWebhookEvent.
+   */
+  verifyWebhook(ctx: WebhookContext): WebhookVerificationResult;
+
+  /**
+   * Parse provider-specific webhook payload into normalized events.
+   * Returns events and optional response to send back to provider.
+   */
+  parseWebhookEvent(ctx: WebhookContext): ProviderWebhookParseResult;
+
+  /**
+   * Initiate an outbound call.
+   * @returns Provider call ID and status
+   */
+  initiateCall(input: InitiateCallInput): Promise<InitiateCallResult>;
+
+  /**
+   * Hang up an active call.
+   */
+  hangupCall(input: HangupCallInput): Promise<void>;
+
+  /**
+   * Play TTS audio to the caller.
+   * The provider should handle streaming if supported.
+   */
+  playTts(input: PlayTtsInput): Promise<void>;
+
+  /**
+   * Start listening for user speech (activate STT).
+   */
+  startListening(input: StartListeningInput): Promise<void>;
+
+  /**
+   * Stop listening for user speech (deactivate STT).
+   */
+  stopListening(input: StopListeningInput): Promise<void>;
+}

package/src/providers/index.ts

@@ -0,0 +1,10 @@
+export type { VoiceCallProvider } from "./base.js";
+export { MockProvider } from "./mock.js";
+export {
+  OpenAIRealtimeSTTProvider,
+  type RealtimeSTTConfig,
+  type RealtimeSTTSession,
+} from "./stt-openai-realtime.js";
+export { TelnyxProvider } from "./telnyx.js";
+export { TwilioProvider } from "./twilio.js";
+export { PlivoProvider } from "./plivo.js";