@clawdbot/voice-call 0.1.0

package/src/providers/stt-openai-realtime.ts

@@ -0,0 +1,303 @@
+/**
+ * OpenAI Realtime STT Provider
+ *
+ * Uses the OpenAI Realtime API for streaming transcription with:
+ * - Direct mu-law audio support (no conversion needed)
+ * - Built-in server-side VAD for turn detection
+ * - Low-latency streaming transcription
+ * - Partial transcript callbacks for real-time UI updates
+ */
+
+import WebSocket from "ws";
+
+/**
+ * Configuration for OpenAI Realtime STT.
+ */
+export interface RealtimeSTTConfig {
+  /** OpenAI API key */
+  apiKey: string;
+  /** Model to use (default: gpt-4o-transcribe) */
+  model?: string;
+  /** Silence duration in ms before considering speech ended (default: 800) */
+  silenceDurationMs?: number;
+  /** VAD threshold 0-1 (default: 0.5) */
+  vadThreshold?: number;
+}
+
+/**
+ * Session for streaming audio and receiving transcripts.
+ */
+export interface RealtimeSTTSession {
+  /** Connect to the transcription service */
+  connect(): Promise<void>;
+  /** Send mu-law audio data (8kHz mono) */
+  sendAudio(audio: Buffer): void;
+  /** Wait for next complete transcript (after VAD detects end of speech) */
+  waitForTranscript(timeoutMs?: number): Promise<string>;
+  /** Set callback for partial transcripts (streaming) */
+  onPartial(callback: (partial: string) => void): void;
+  /** Set callback for final transcripts */
+  onTranscript(callback: (transcript: string) => void): void;
+  /** Close the session */
+  close(): void;
+  /** Check if session is connected */
+  isConnected(): boolean;
+}
+
+/**
+ * Provider factory for OpenAI Realtime STT sessions.
+ */
+export class OpenAIRealtimeSTTProvider {
+  readonly name = "openai-realtime";
+  private apiKey: string;
+  private model: string;
+  private silenceDurationMs: number;
+  private vadThreshold: number;
+
+  constructor(config: RealtimeSTTConfig) {
+    if (!config.apiKey) {
+      throw new Error("OpenAI API key required for Realtime STT");
+    }
+    this.apiKey = config.apiKey;
+    this.model = config.model || "gpt-4o-transcribe";
+    this.silenceDurationMs = config.silenceDurationMs || 800;
+    this.vadThreshold = config.vadThreshold || 0.5;
+  }
+
+  /**
+   * Create a new realtime transcription session.
+   */
+  createSession(): RealtimeSTTSession {
+    return new OpenAIRealtimeSTTSession(
+      this.apiKey,
+      this.model,
+      this.silenceDurationMs,
+      this.vadThreshold,
+    );
+  }
+}
+
+/**
+ * WebSocket-based session for real-time speech-to-text.
+ */
+class OpenAIRealtimeSTTSession implements RealtimeSTTSession {
+  private static readonly MAX_RECONNECT_ATTEMPTS = 5;
+  private static readonly RECONNECT_DELAY_MS = 1000;
+
+  private ws: WebSocket | null = null;
+  private connected = false;
+  private closed = false;
+  private reconnectAttempts = 0;
+  private pendingTranscript = "";
+  private onTranscriptCallback: ((transcript: string) => void) | null = null;
+  private onPartialCallback: ((partial: string) => void) | null = null;
+
+  constructor(
+    private readonly apiKey: string,
+    private readonly model: string,
+    private readonly silenceDurationMs: number,
+    private readonly vadThreshold: number,
+  ) {}
+
+  async connect(): Promise<void> {
+    this.closed = false;
+    this.reconnectAttempts = 0;
+    return this.doConnect();
+  }
+
+  private async doConnect(): Promise<void> {
+    return new Promise((resolve, reject) => {
+      const url = "wss://api.openai.com/v1/realtime?intent=transcription";
+
+      this.ws = new WebSocket(url, {
+        headers: {
+          Authorization: `Bearer ${this.apiKey}`,
+          "OpenAI-Beta": "realtime=v1",
+        },
+      });
+
+      this.ws.on("open", () => {
+        console.log("[RealtimeSTT] WebSocket connected");
+        this.connected = true;
+        this.reconnectAttempts = 0;
+
+        // Configure the transcription session
+        this.sendEvent({
+          type: "transcription_session.update",
+          session: {
+            input_audio_format: "g711_ulaw",
+            input_audio_transcription: {
+              model: this.model,
+            },
+            turn_detection: {
+              type: "server_vad",
+              threshold: this.vadThreshold,
+              prefix_padding_ms: 300,
+              silence_duration_ms: this.silenceDurationMs,
+            },
+          },
+        });
+
+        resolve();
+      });
+
+      this.ws.on("message", (data: Buffer) => {
+        try {
+          const event = JSON.parse(data.toString());
+          this.handleEvent(event);
+        } catch (e) {
+          console.error("[RealtimeSTT] Failed to parse event:", e);
+        }
+      });
+
+      this.ws.on("error", (error) => {
+        console.error("[RealtimeSTT] WebSocket error:", error);
+        if (!this.connected) reject(error);
+      });
+
+      this.ws.on("close", (code, reason) => {
+        console.log(
+          `[RealtimeSTT] WebSocket closed (code: ${code}, reason: ${reason?.toString() || "none"})`,
+        );
+        this.connected = false;
+
+        // Attempt reconnection if not intentionally closed
+        if (!this.closed) {
+          void this.attemptReconnect();
+        }
+      });
+
+      setTimeout(() => {
+        if (!this.connected) {
+          reject(new Error("Realtime STT connection timeout"));
+        }
+      }, 10000);
+    });
+  }
+
+  private async attemptReconnect(): Promise<void> {
+    if (this.closed) {
+      return;
+    }
+
+    if (
+      this.reconnectAttempts >= OpenAIRealtimeSTTSession.MAX_RECONNECT_ATTEMPTS
+    ) {
+      console.error(
+        `[RealtimeSTT] Max reconnect attempts (${OpenAIRealtimeSTTSession.MAX_RECONNECT_ATTEMPTS}) reached`,
+      );
+      return;
+    }
+
+    this.reconnectAttempts++;
+    const delay =
+      OpenAIRealtimeSTTSession.RECONNECT_DELAY_MS *
+      2 ** (this.reconnectAttempts - 1);
+    console.log(
+      `[RealtimeSTT] Reconnecting ${this.reconnectAttempts}/${OpenAIRealtimeSTTSession.MAX_RECONNECT_ATTEMPTS} in ${delay}ms...`,
+    );
+
+    await new Promise((resolve) => setTimeout(resolve, delay));
+
+    if (this.closed) {
+      return;
+    }
+
+    try {
+      await this.doConnect();
+      console.log("[RealtimeSTT] Reconnected successfully");
+    } catch (error) {
+      console.error("[RealtimeSTT] Reconnect failed:", error);
+    }
+  }
+
+  private handleEvent(event: {
+    type: string;
+    delta?: string;
+    transcript?: string;
+    error?: unknown;
+  }): void {
+    switch (event.type) {
+      case "transcription_session.created":
+      case "transcription_session.updated":
+      case "input_audio_buffer.speech_stopped":
+      case "input_audio_buffer.committed":
+        console.log(`[RealtimeSTT] ${event.type}`);
+        break;
+
+      case "conversation.item.input_audio_transcription.delta":
+        if (event.delta) {
+          this.pendingTranscript += event.delta;
+          this.onPartialCallback?.(this.pendingTranscript);
+        }
+        break;
+
+      case "conversation.item.input_audio_transcription.completed":
+        if (event.transcript) {
+          console.log(`[RealtimeSTT] Transcript: ${event.transcript}`);
+          this.onTranscriptCallback?.(event.transcript);
+        }
+        this.pendingTranscript = "";
+        break;
+
+      case "input_audio_buffer.speech_started":
+        console.log("[RealtimeSTT] Speech started");
+        this.pendingTranscript = "";
+        break;
+
+      case "error":
+        console.error("[RealtimeSTT] Error:", event.error);
+        break;
+    }
+  }
+
+  private sendEvent(event: unknown): void {
+    if (this.ws?.readyState === WebSocket.OPEN) {
+      this.ws.send(JSON.stringify(event));
+    }
+  }
+
+  sendAudio(muLawData: Buffer): void {
+    if (!this.connected) return;
+    this.sendEvent({
+      type: "input_audio_buffer.append",
+      audio: muLawData.toString("base64"),
+    });
+  }
+
+  onPartial(callback: (partial: string) => void): void {
+    this.onPartialCallback = callback;
+  }
+
+  onTranscript(callback: (transcript: string) => void): void {
+    this.onTranscriptCallback = callback;
+  }
+
+  async waitForTranscript(timeoutMs = 30000): Promise<string> {
+    return new Promise((resolve, reject) => {
+      const timeout = setTimeout(() => {
+        this.onTranscriptCallback = null;
+        reject(new Error("Transcript timeout"));
+      }, timeoutMs);
+
+      this.onTranscriptCallback = (transcript) => {
+        clearTimeout(timeout);
+        this.onTranscriptCallback = null;
+        resolve(transcript);
+      };
+    });
+  }
+
+  close(): void {
+    this.closed = true;
+    if (this.ws) {
+      this.ws.close();
+      this.ws = null;
+    }
+    this.connected = false;
+  }
+
+  isConnected(): boolean {
+    return this.connected;
+  }
+}

package/src/providers/telnyx.ts

@@ -0,0 +1,364 @@
+import crypto from "node:crypto";
+
+import type { TelnyxConfig } from "../config.js";
+import type {
+  EndReason,
+  HangupCallInput,
+  InitiateCallInput,
+  InitiateCallResult,
+  NormalizedEvent,
+  PlayTtsInput,
+  ProviderWebhookParseResult,
+  StartListeningInput,
+  StopListeningInput,
+  WebhookContext,
+  WebhookVerificationResult,
+} from "../types.js";
+import type { VoiceCallProvider } from "./base.js";
+
+/**
+ * Telnyx Voice API provider implementation.
+ *
+ * Uses Telnyx Call Control API v2 for managing calls.
+ * @see https://developers.telnyx.com/docs/api/v2/call-control
+ */
+export class TelnyxProvider implements VoiceCallProvider {
+  readonly name = "telnyx" as const;
+
+  private readonly apiKey: string;
+  private readonly connectionId: string;
+  private readonly publicKey: string | undefined;
+  private readonly baseUrl = "https://api.telnyx.com/v2";
+
+  constructor(config: TelnyxConfig) {
+    if (!config.apiKey) {
+      throw new Error("Telnyx API key is required");
+    }
+    if (!config.connectionId) {
+      throw new Error("Telnyx connection ID is required");
+    }
+
+    this.apiKey = config.apiKey;
+    this.connectionId = config.connectionId;
+    this.publicKey = config.publicKey;
+  }
+
+  /**
+   * Make an authenticated request to the Telnyx API.
+   */
+  private async apiRequest<T = unknown>(
+    endpoint: string,
+    body: Record<string, unknown>,
+    options?: { allowNotFound?: boolean },
+  ): Promise<T> {
+    const response = await fetch(`${this.baseUrl}${endpoint}`, {
+      method: "POST",
+      headers: {
+        Authorization: `Bearer ${this.apiKey}`,
+        "Content-Type": "application/json",
+      },
+      body: JSON.stringify(body),
+    });
+
+    if (!response.ok) {
+      if (options?.allowNotFound && response.status === 404) {
+        return undefined as T;
+      }
+      const errorText = await response.text();
+      throw new Error(`Telnyx API error: ${response.status} ${errorText}`);
+    }
+
+    const text = await response.text();
+    return text ? (JSON.parse(text) as T) : (undefined as T);
+  }
+
+  /**
+   * Verify Telnyx webhook signature using Ed25519.
+   */
+  verifyWebhook(ctx: WebhookContext): WebhookVerificationResult {
+    if (!this.publicKey) {
+      // No public key configured, skip verification (not recommended for production)
+      return { ok: true };
+    }
+
+    const signature = ctx.headers["telnyx-signature-ed25519"];
+    const timestamp = ctx.headers["telnyx-timestamp"];
+
+    if (!signature || !timestamp) {
+      return { ok: false, reason: "Missing signature or timestamp header" };
+    }
+
+    const signatureStr = Array.isArray(signature) ? signature[0] : signature;
+    const timestampStr = Array.isArray(timestamp) ? timestamp[0] : timestamp;
+
+    if (!signatureStr || !timestampStr) {
+      return { ok: false, reason: "Empty signature or timestamp" };
+    }
+
+    try {
+      const signedPayload = `${timestampStr}|${ctx.rawBody}`;
+      const signatureBuffer = Buffer.from(signatureStr, "base64");
+      const publicKeyBuffer = Buffer.from(this.publicKey, "base64");
+
+      const isValid = crypto.verify(
+        null, // Ed25519 doesn't use a digest
+        Buffer.from(signedPayload),
+        {
+          key: publicKeyBuffer,
+          format: "der",
+          type: "spki",
+        },
+        signatureBuffer,
+      );
+
+      if (!isValid) {
+        return { ok: false, reason: "Invalid signature" };
+      }
+
+      // Check timestamp is within 5 minutes
+      const eventTime = parseInt(timestampStr, 10) * 1000;
+      const now = Date.now();
+      if (Math.abs(now - eventTime) > 5 * 60 * 1000) {
+        return { ok: false, reason: "Timestamp too old" };
+      }
+
+      return { ok: true };
+    } catch (err) {
+      return {
+        ok: false,
+        reason: `Verification error: ${err instanceof Error ? err.message : String(err)}`,
+      };
+    }
+  }
+
+  /**
+   * Parse Telnyx webhook event into normalized format.
+   */
+  parseWebhookEvent(ctx: WebhookContext): ProviderWebhookParseResult {
+    try {
+      const payload = JSON.parse(ctx.rawBody);
+      const data = payload.data;
+
+      if (!data || !data.event_type) {
+        return { events: [], statusCode: 200 };
+      }
+
+      const event = this.normalizeEvent(data);
+      return {
+        events: event ? [event] : [],
+        statusCode: 200,
+      };
+    } catch {
+      return { events: [], statusCode: 400 };
+    }
+  }
+
+  /**
+   * Convert Telnyx event to normalized event format.
+   */
+  private normalizeEvent(data: TelnyxEvent): NormalizedEvent | null {
+    // Decode client_state from Base64 (we encode it in initiateCall)
+    let callId = "";
+    if (data.payload?.client_state) {
+      try {
+        callId = Buffer.from(data.payload.client_state, "base64").toString(
+          "utf8",
+        );
+      } catch {
+        // Fallback if not valid Base64
+        callId = data.payload.client_state;
+      }
+    }
+    if (!callId) {
+      callId = data.payload?.call_control_id || "";
+    }
+
+    const baseEvent = {
+      id: data.id || crypto.randomUUID(),
+      callId,
+      providerCallId: data.payload?.call_control_id,
+      timestamp: Date.now(),
+    };
+
+    switch (data.event_type) {
+      case "call.initiated":
+        return { ...baseEvent, type: "call.initiated" };
+
+      case "call.ringing":
+        return { ...baseEvent, type: "call.ringing" };
+
+      case "call.answered":
+        return { ...baseEvent, type: "call.answered" };
+
+      case "call.bridged":
+        return { ...baseEvent, type: "call.active" };
+
+      case "call.speak.started":
+        return {
+          ...baseEvent,
+          type: "call.speaking",
+          text: data.payload?.text || "",
+        };
+
+      case "call.transcription":
+        return {
+          ...baseEvent,
+          type: "call.speech",
+          transcript: data.payload?.transcription || "",
+          isFinal: data.payload?.is_final ?? true,
+          confidence: data.payload?.confidence,
+        };
+
+      case "call.hangup":
+        return {
+          ...baseEvent,
+          type: "call.ended",
+          reason: this.mapHangupCause(data.payload?.hangup_cause),
+        };
+
+      case "call.dtmf.received":
+        return {
+          ...baseEvent,
+          type: "call.dtmf",
+          digits: data.payload?.digit || "",
+        };
+
+      default:
+        return null;
+    }
+  }
+
+  /**
+   * Map Telnyx hangup cause to normalized end reason.
+   * @see https://developers.telnyx.com/docs/api/v2/call-control/Call-Commands#hangup-causes
+   */
+  private mapHangupCause(cause?: string): EndReason {
+    switch (cause) {
+      case "normal_clearing":
+      case "normal_unspecified":
+        return "completed";
+      case "originator_cancel":
+        return "hangup-bot";
+      case "call_rejected":
+      case "user_busy":
+        return "busy";
+      case "no_answer":
+      case "no_user_response":
+        return "no-answer";
+      case "destination_out_of_order":
+      case "network_out_of_order":
+      case "service_unavailable":
+      case "recovery_on_timer_expire":
+        return "failed";
+      case "machine_detected":
+      case "fax_detected":
+        return "voicemail";
+      case "user_hangup":
+      case "subscriber_absent":
+        return "hangup-user";
+      default:
+        // Unknown cause - log it for debugging and return completed
+        if (cause) {
+          console.warn(`[telnyx] Unknown hangup cause: ${cause}`);
+        }
+        return "completed";
+    }
+  }
+
+  /**
+   * Initiate an outbound call via Telnyx API.
+   */
+  async initiateCall(input: InitiateCallInput): Promise<InitiateCallResult> {
+    const result = await this.apiRequest<TelnyxCallResponse>("/calls", {
+      connection_id: this.connectionId,
+      to: input.to,
+      from: input.from,
+      webhook_url: input.webhookUrl,
+      webhook_url_method: "POST",
+      client_state: Buffer.from(input.callId).toString("base64"),
+      timeout_secs: 30,
+    });
+
+    return {
+      providerCallId: result.data.call_control_id,
+      status: "initiated",
+    };
+  }
+
+  /**
+   * Hang up a call via Telnyx API.
+   */
+  async hangupCall(input: HangupCallInput): Promise<void> {
+    await this.apiRequest(
+      `/calls/${input.providerCallId}/actions/hangup`,
+      { command_id: crypto.randomUUID() },
+      { allowNotFound: true },
+    );
+  }
+
+  /**
+   * Play TTS audio via Telnyx speak action.
+   */
+  async playTts(input: PlayTtsInput): Promise<void> {
+    await this.apiRequest(`/calls/${input.providerCallId}/actions/speak`, {
+      command_id: crypto.randomUUID(),
+      payload: input.text,
+      voice: input.voice || "female",
+      language: input.locale || "en-US",
+    });
+  }
+
+  /**
+   * Start transcription (STT) via Telnyx.
+   */
+  async startListening(input: StartListeningInput): Promise<void> {
+    await this.apiRequest(
+      `/calls/${input.providerCallId}/actions/transcription_start`,
+      {
+        command_id: crypto.randomUUID(),
+        language: input.language || "en",
+      },
+    );
+  }
+
+  /**
+   * Stop transcription via Telnyx.
+   */
+  async stopListening(input: StopListeningInput): Promise<void> {
+    await this.apiRequest(
+      `/calls/${input.providerCallId}/actions/transcription_stop`,
+      { command_id: crypto.randomUUID() },
+      { allowNotFound: true },
+    );
+  }
+}
+
+// -----------------------------------------------------------------------------
+// Telnyx-specific types
+// -----------------------------------------------------------------------------
+
+interface TelnyxEvent {
+  id?: string;
+  event_type: string;
+  payload?: {
+    call_control_id?: string;
+    client_state?: string;
+    text?: string;
+    transcription?: string;
+    is_final?: boolean;
+    confidence?: number;
+    hangup_cause?: string;
+    digit?: string;
+    [key: string]: unknown;
+  };
+}
+
+interface TelnyxCallResponse {
+  data: {
+    call_control_id: string;
+    call_leg_id: string;
+    call_session_id: string;
+    is_alive: boolean;
+    record_type: string;
+  };
+}