@wovin/tranz 0.1.35 → 0.2.0

package/src/utils/transcription/realtime.ts

@@ -0,0 +1,821 @@
+/**
+ * Realtime transcription API for Mistral's WebSocket-based transcription service
+ *
+ * Provides a simple, event-driven interface for streaming audio transcription.
+ * Users provide audio as AsyncIterable<Uint8Array> and receive typed events.
+ *
+ * Browser-compatible: Uses native WebSocket in browsers/Deno, 'ws' package in Node.js
+ *
+ * @example
+ * ```typescript
+ * import { createRealtimeTranscriber } from '@wovin/tranz/realtime'
+ *
+ * const transcriber = createRealtimeTranscriber({
+ *   apiKey: process.env.MISTRAL_API_KEY,
+ * })
+ *
+ * for await (const event of transcriber.transcribe(audioStream)) {
+ *   if (event.type === 'transcription.text.delta') {
+ *     process.stdout.write(event.text)
+ *   }
+ * }
+ * ```
+ */
+
+import { getWebSocketImpl } from "./runtime.js";
+
+// ============================================================================
+// Type Definitions
+// ============================================================================
+
+/**
+ * Audio encoding formats supported by the transcription service
+ */
+export enum AudioEncoding {
+  PcmS16le = "pcm_s16le",
+  PcmS16be = "pcm_s16be",
+  PcmU16le = "pcm_u16le",
+  PcmU16be = "pcm_u16be",
+  PcmS24le = "pcm_s24le",
+  PcmS24be = "pcm_s24be",
+  PcmU24le = "pcm_u24le",
+  PcmU24be = "pcm_u24be",
+  PcmS32le = "pcm_s32le",
+  PcmS32be = "pcm_s32be",
+  PcmU32le = "pcm_u32le",
+  PcmU32be = "pcm_u32be",
+  PcmF32le = "pcm_f32le",
+  PcmF32be = "pcm_f32be",
+  PcmF64le = "pcm_f64le",
+  PcmF64be = "pcm_f64be",
+}
+
+/**
+ * Audio format configuration for realtime transcription
+ */
+export interface AudioFormat {
+  /** Audio encoding format (default: pcm_s16le) */
+  encoding: AudioEncoding;
+  /** Sample rate in Hz (default: 16000) */
+  sampleRate: number;
+}
+
+/**
+ * Configuration for creating a realtime transcriber
+ */
+export interface RealtimeConfig {
+  /** Mistral API key */
+  apiKey: string;
+  /** Model ID (default: voxtral-mini-transcribe-realtime-2602) */
+  model?: string;
+  /** WebSocket base URL (default: wss://api.mistral.ai) */
+  baseUrl?: string;
+}
+
+/**
+ * Options for transcription
+ */
+export interface TranscribeOptions {
+  /** Audio format configuration (optional, uses defaults if not provided) */
+  audioFormat?: Partial<AudioFormat>;
+}
+
+/**
+ * Union type for all realtime transcription events
+ * These events are yielded as the transcription progresses
+ */
+export type RealtimeEvent =
+  | SessionCreatedEvent
+  | SessionUpdatedEvent
+  | TranscriptionTextDeltaEvent
+  | TranscriptionLanguageEvent
+  | TranscriptionSegmentEvent
+  | TranscriptionDoneEvent
+  | ErrorEvent;
+
+/**
+ * Session created event - emitted when WebSocket connection is established
+ */
+export interface SessionCreatedEvent {
+  type: "session.created";
+  session: {
+    id: string;
+    audioFormat: AudioFormat;
+  };
+}
+
+/**
+ * Session updated event - emitted when audio format is confirmed
+ */
+export interface SessionUpdatedEvent {
+  type: "session.updated";
+  session: {
+    audioFormat: AudioFormat;
+  };
+}
+
+/**
+ * Text delta event - emitted as transcription text arrives in chunks
+ * This is the primary event for displaying real-time transcription
+ */
+export interface TranscriptionTextDeltaEvent {
+  type: "transcription.text.delta";
+  text: string;
+}
+
+/**
+ * Language detection event - emitted when audio language is detected
+ */
+export interface TranscriptionLanguageEvent {
+  type: "transcription.language";
+  audioLanguage: string;
+}
+
+/**
+ * Segment event - emitted for timestamped segments
+ * NOTE: WebSocket realtime API does NOT support this - included for completeness
+ */
+export interface TranscriptionSegmentEvent {
+  type: "transcription.segment";
+  start?: number;
+  end?: number;
+  text: string;
+  speakerId?: string;
+}
+
+/**
+ * Transcription done event - emitted when transcription completes
+ * Contains the complete transcript
+ */
+export interface TranscriptionDoneEvent {
+  type: "transcription.done";
+  text: string;
+  language?: string;
+}
+
+/**
+ * Error event - emitted when an error occurs
+ */
+export interface ErrorEvent {
+  type: "error";
+  error: {
+    message: string | unknown;
+    code?: string;
+  };
+}
+
+// ============================================================================
+// Main API
+// ============================================================================
+
+/**
+ * Realtime transcriber interface
+ */
+export interface RealtimeTranscriber {
+  /**
+   * Transcribe audio stream and yield events as they arrive
+   *
+   * @param audioStream - AsyncIterable of audio chunks (Uint8Array)
+   * @param options - Optional transcription options
+   * @returns AsyncIterable of transcription events
+   *
+   * @example
+   * ```typescript
+   * const transcriber = createRealtimeTranscriber({ apiKey: 'xxx' })
+   *
+   * for await (const event of transcriber.transcribe(audioStream)) {
+   *   if (event.type === 'transcription.text.delta') {
+   *     console.log(event.text)
+   *   } else if (event.type === 'transcription.done') {
+   *     console.log('Complete:', event.text)
+   *     break
+   *   }
+   * }
+   * ```
+   */
+  transcribe(
+    audioStream: AsyncIterable<Uint8Array>,
+    options?: TranscribeOptions
+  ): AsyncIterable<RealtimeEvent>;
+}
+
+/**
+ * Create a realtime transcriber instance
+ *
+ * @param config - Configuration including API key and optional model/baseUrl
+ * @returns RealtimeTranscriber instance
+ *
+ * @example
+ * ```typescript
+ * const transcriber = createRealtimeTranscriber({
+ *   apiKey: process.env.MISTRAL_API_KEY,
+ *   model: 'voxtral-mini-transcribe-realtime-2602', // optional
+ *   baseUrl: 'wss://api.mistral.ai', // optional
+ * })
+ * ```
+ */
+export function createRealtimeTranscriber(
+  config: RealtimeConfig
+): RealtimeTranscriber {
+  // Check if running in browser - not supported yet
+  const isBrowser =
+    typeof window !== "undefined" &&
+    typeof document !== "undefined" &&
+    typeof navigator !== "undefined";
+
+  if (isBrowser) {
+    throw new Error(
+      "Realtime transcription is not yet supported in browsers. " +
+        "Browser WebSocket API does not support authentication headers required by Mistral API. " +
+        "Use this API in Node.js or server-side environments only. " +
+        "See: https://github.com/wovin/tranz/issues"
+    );
+  }
+
+  const model = config.model ?? "voxtral-mini-transcribe-realtime-2602";
+  const baseUrl = config.baseUrl ?? "wss://api.mistral.ai";
+
+  return {
+    async *transcribe(
+      audioStream: AsyncIterable<Uint8Array>,
+      options?: TranscribeOptions
+    ): AsyncIterable<RealtimeEvent> {
+      // Merge default audio format with user options
+      const audioFormat: AudioFormat = {
+        encoding: options?.audioFormat?.encoding ?? AudioEncoding.PcmS16le,
+        sampleRate: options?.audioFormat?.sampleRate ?? 16000,
+      };
+
+      // Create WebSocket connection
+      const connection = await createConnection(
+        config.apiKey,
+        baseUrl,
+        model,
+        audioFormat
+      );
+
+      try {
+        // Start audio sending task
+        let stopRequested = false;
+        const sendAudioTask = (async () => {
+          try {
+            for await (const chunk of audioStream) {
+              if (stopRequested || connection.isClosed) {
+                break;
+              }
+              await connection.sendAudio(chunk);
+            }
+          } finally {
+            await connection.endAudio();
+          }
+        })();
+
+        // Yield events as they arrive
+        for await (const event of connection.events()) {
+          yield event;
+
+          if (event.type === "transcription.done" || event.type === "error") {
+            break;
+          }
+        }
+
+        // Wait for audio sending to complete
+        await sendAudioTask;
+      } finally {
+        await connection.close();
+
+        // Clean up audio stream if possible
+        const maybeReturn = (
+          audioStream as {
+            return?: () => Promise<IteratorResult<Uint8Array>>;
+          }
+        ).return;
+        if (typeof maybeReturn === "function") {
+          await maybeReturn.call(audioStream);
+        }
+      }
+    },
+  };
+}
+
+// ============================================================================
+// WebSocket Connection Implementation
+// ============================================================================
+
+interface Connection {
+  isClosed: boolean;
+  events(): AsyncGenerator<RealtimeEvent>;
+  sendAudio(chunk: Uint8Array): Promise<void>;
+  endAudio(): Promise<void>;
+  close(): Promise<void>;
+}
+
+async function createConnection(
+  apiKey: string,
+  baseUrl: string,
+  model: string,
+  audioFormat: AudioFormat
+): Promise<Connection> {
+  const WebSocketImpl = await getWebSocketImpl();
+
+  // Build WebSocket URL
+  const wsUrl = buildWebSocketUrl(baseUrl, model, apiKey);
+
+  // Detect if this is Node.js 'ws' package or browser WebSocket
+  const isNodeWs = typeof process !== "undefined" && process.versions?.node;
+
+  // Create WebSocket
+  // Browser WebSocket doesn't support headers in constructor, Node.js ws does
+  const ws = isNodeWs
+    ? new (WebSocketImpl as any)(wsUrl, {
+        headers: {
+          Authorization: `Bearer ${apiKey}`,
+        },
+      })
+    : new WebSocketImpl(wsUrl);
+
+  // Wait for connection and session creation
+  const session = await waitForSession(ws as WebSocket);
+
+  let closed = false;
+
+  const websocket = ws as WebSocket;
+
+  const connection: Connection = {
+    get isClosed() {
+      return closed || websocket.readyState === 2 || websocket.readyState === 3;
+    },
+
+    async *events(): AsyncGenerator<RealtimeEvent> {
+      type QueueItem = {
+        kind: "message" | "close" | "error";
+        data?: unknown;
+        error?: Error;
+      };
+      const queue: QueueItem[] = [];
+      let resolver: ((item: QueueItem) => void) | null = null;
+      let done = false;
+
+      const push = (item: QueueItem) => {
+        if (done) return;
+        if (resolver) {
+          const resolve = resolver;
+          resolver = null;
+          resolve(item);
+          return;
+        }
+        queue.push(item);
+      };
+
+      const handleMessage = (event: MessageEvent) => {
+        push({ kind: "message", data: event.data });
+      };
+
+      const handleClose = () => {
+        closed = true;
+        push({ kind: "close" });
+      };
+
+      const handleError = (event: Event) => {
+        push({
+          kind: "error",
+          error: new Error("WebSocket connection error"),
+        });
+      };
+
+      websocket.addEventListener("message", handleMessage);
+      websocket.addEventListener("close", handleClose);
+      websocket.addEventListener("error", handleError);
+
+      try {
+        while (true) {
+          const item =
+            queue.length > 0
+              ? queue.shift()!
+              : await new Promise<QueueItem>((resolve) => {
+                  resolver = resolve;
+                });
+
+          if (item.kind === "close") break;
+
+          if (item.kind === "error") {
+            const error =
+              item.error ?? new Error("WebSocket connection error");
+            yield {
+              type: "error",
+              error: { message: error.message },
+            } as ErrorEvent;
+            continue;
+          }
+
+          const event = parseRealtimeEvent(item.data);
+          yield event;
+        }
+      } finally {
+        done = true;
+        websocket.removeEventListener("message", handleMessage);
+        websocket.removeEventListener("close", handleClose);
+        websocket.removeEventListener("error", handleError);
+        if (resolver !== null) {
+          const resolve = resolver;
+          resolver = null;
+          resolve({ kind: "close" });
+        }
+      }
+    },
+
+    async sendAudio(chunk: Uint8Array): Promise<void> {
+      if (connection.isClosed) {
+        throw new Error("Connection is closed");
+      }
+
+      const base64Audio = arrayBufferToBase64(chunk);
+      const message = {
+        type: "input_audio.append",
+        audio: base64Audio,
+      };
+
+      await sendJson(websocket, message);
+    },
+
+    async endAudio(): Promise<void> {
+      if (connection.isClosed) return;
+      await sendJson(websocket, { type: "input_audio.end" });
+    },
+
+    async close(): Promise<void> {
+      if (closed) return;
+      closed = true;
+
+      if (websocket.readyState === 3) return;
+
+      await new Promise<void>((resolve) => {
+        const finalize = () => {
+          websocket.removeEventListener("close", finalize);
+          resolve();
+        };
+        websocket.addEventListener("close", finalize);
+        websocket.close(1000, "");
+      });
+    },
+  };
+
+  return connection;
+}
+
+function buildWebSocketUrl(
+  baseUrl: string,
+  model: string,
+  apiKey: string
+): string {
+  const url = new URL("v1/audio/transcriptions/realtime", baseUrl);
+  url.searchParams.set("model", model);
+  return url.toString();
+}
+
+async function waitForSession(ws: WebSocket): Promise<SessionCreatedEvent> {
+  return new Promise((resolve, reject) => {
+    const timeout = setTimeout(() => {
+      cleanup();
+      ws.close();
+      reject(new Error("Timeout waiting for session creation"));
+    }, 10000);
+
+    const cleanup = () => {
+      clearTimeout(timeout);
+      ws.removeEventListener("message", handleMessage);
+      ws.removeEventListener("close", handleClose);
+      ws.removeEventListener("error", handleError);
+    };
+
+    const handleMessage = (event: MessageEvent) => {
+      try {
+        const parsed = parseRealtimeEvent(event.data);
+        if (parsed.type === "session.created") {
+          cleanup();
+          resolve(parsed as SessionCreatedEvent);
+        } else if (parsed.type === "error") {
+          cleanup();
+          ws.close();
+          reject(
+            new Error(
+              `Realtime transcription error: ${JSON.stringify(parsed.error)}`
+            )
+          );
+        }
+      } catch (err) {
+        cleanup();
+        ws.close();
+        reject(err);
+      }
+    };
+
+    const handleClose = () => {
+      cleanup();
+      reject(new Error("WebSocket closed during handshake"));
+    };
+
+    const handleError = () => {
+      cleanup();
+      reject(new Error("WebSocket error during handshake"));
+    };
+
+    ws.addEventListener("message", handleMessage);
+    ws.addEventListener("close", handleClose);
+    ws.addEventListener("error", handleError);
+  });
+}
+
+function parseRealtimeEvent(data: unknown): RealtimeEvent {
+  try {
+    const text =
+      typeof data === "string" ? data : new TextDecoder().decode(data as ArrayBuffer);
+    const payload = JSON.parse(text);
+
+    if (typeof payload.type !== "string") {
+      return {
+        type: "error",
+        error: { message: "Invalid event: missing type" },
+      };
+    }
+
+    return payload as RealtimeEvent;
+  } catch (err) {
+    return {
+      type: "error",
+      error: { message: `Failed to parse event: ${err}` },
+    };
+  }
+}
+
+async function sendJson(ws: WebSocket, payload: unknown): Promise<void> {
+  return new Promise((resolve, reject) => {
+    const message = JSON.stringify(payload);
+
+    // Node.js 'ws' package uses callback
+    if (typeof (ws as any).send === "function") {
+      const send = (ws as any).send.bind(ws);
+      // Try callback signature (Node.js ws)
+      try {
+        send(message, (err: Error | undefined) => {
+          if (err) reject(err);
+          else resolve();
+        });
+      } catch {
+        // Fallback for browser WebSocket (no callback)
+        ws.send(message);
+        resolve();
+      }
+    } else {
+      ws.send(message);
+      resolve();
+    }
+  });
+}
+
+function arrayBufferToBase64(buffer: Uint8Array): string {
+  // Browser
+  if (typeof btoa !== "undefined") {
+    const binary = Array.from(buffer)
+      .map((byte) => String.fromCharCode(byte))
+      .join("");
+    return btoa(binary);
+  }
+
+  // Node.js
+  if (typeof Buffer !== "undefined") {
+    return Buffer.from(buffer).toString("base64");
+  }
+
+  throw new Error("No base64 encoding available");
+}
+
+// ============================================================================
+// Helper Functions
+// ============================================================================
+
+/**
+ * Result from audio capture - includes stream and stop function
+ */
+export interface AudioCaptureResult {
+  /** AsyncGenerator yielding audio chunks */
+  stream: AsyncGenerator<Uint8Array, void, unknown>;
+  /** Function to stop audio capture */
+  stop: () => void;
+}
+
+/**
+ * Capture audio from microphone using SoX `rec` command (Node.js only)
+ *
+ * Yields PCM 16-bit signed little-endian mono audio chunks suitable for
+ * realtime transcription.
+ *
+ * **Requirements:**
+ * - SoX audio tools must be installed
+ *   - macOS: `brew install sox`
+ *   - Linux: `sudo apt install sox`
+ *
+ * **Note:** This is Node.js only. For browser audio capture, use `captureAudioFromBrowser()`
+ *
+ * @param sampleRate - Sample rate in Hz (default: 16000)
+ * @returns Object with audio stream and stop function
+ *
+ * @example
+ * ```typescript
+ * const { stream, stop } = captureAudioFromMicrophone(16000)
+ *
+ * try {
+ *   for await (const event of transcriber.transcribe(stream)) {
+ *     // ... handle events
+ *   }
+ * } finally {
+ *   stop() // Clean up audio capture
+ * }
+ * ```
+ */
+export async function captureAudioFromMicrophone(
+  sampleRate: number = 16000
+): Promise<AudioCaptureResult> {
+  // Check if we're in Node.js
+  if (typeof process === "undefined" || !process.versions?.node) {
+    throw new Error(
+      "captureAudioFromMicrophone() is Node.js only. Use captureAudioFromBrowser() in browsers."
+    );
+  }
+
+  // Dynamic import for Node.js child_process (ES module compatible)
+  const { spawn } = await import("node:child_process");
+
+  const recorder = spawn(
+    "rec",
+    [
+      "-q", // Quiet mode
+      "-t",
+      "raw", // Raw PCM output
+      "-b",
+      "16", // 16-bit samples
+      "-e",
+      "signed-integer", // Signed PCM
+      "-r",
+      String(sampleRate), // Sample rate
+      "-c",
+      "1", // Mono (1 channel)
+      "-", // Output to stdout
+    ],
+    { stdio: ["ignore", "pipe", "ignore"] }
+  );
+
+  recorder.on("error", (err: any) => {
+    if (err.code === "ENOENT") {
+      console.error(
+        "\nError: 'rec' command not found. Please install SoX:",
+        "\n  macOS: brew install sox",
+        "\n  Linux: sudo apt install sox"
+      );
+      process.exit(1);
+    }
+    throw err;
+  });
+
+  const stream = (async function* () {
+    try {
+      if (!recorder.stdout) {
+        throw new Error("Failed to create audio capture stream");
+      }
+      for await (const chunk of recorder.stdout) {
+        yield new Uint8Array(chunk as Buffer);
+      }
+    } finally {
+      if (!recorder.killed) {
+        recorder.kill("SIGTERM");
+      }
+    }
+  })();
+
+  const stop = () => {
+    if (!recorder.killed) {
+      recorder.kill("SIGTERM");
+    }
+  };
+
+  return { stream, stop };
+}
+
+/**
+ * Capture audio from browser microphone using Web Audio API
+ *
+ * **CURRENTLY DISABLED** - Browser support is not available yet due to
+ * WebSocket authentication limitations with Mistral API.
+ *
+ * @deprecated Browser realtime transcription is not yet supported.
+ *             Use captureAudioFromMicrophone() in Node.js instead.
+ *
+ * @param sampleRate - Target sample rate in Hz (default: 16000)
+ * @returns Object with audio stream and stop function
+ *
+ * @throws Error - Always throws as browser mode is disabled
+ *
+ * @todo Enable when Mistral API supports browser WebSocket authentication
+ * @todo Migrate to AudioWorklet for better performance
+ *       See: https://web.dev/patterns/media/microphone-process/
+ */
+export async function captureAudioFromBrowser(
+  sampleRate: number = 16000
+): Promise<AudioCaptureResult> {
+  throw new Error(
+    "Browser realtime transcription is not yet supported. " +
+      "Browser WebSocket API does not support authentication headers required by Mistral API. " +
+      "Use captureAudioFromMicrophone() in Node.js environments instead."
+  );
+
+  // Implementation disabled - kept for future when auth is resolved
+  /* istanbul ignore next */
+  if (typeof navigator === "undefined" || !navigator.mediaDevices) {
+    throw new Error(
+      "captureAudioFromBrowser() requires a browser environment with getUserMedia support"
+    );
+  }
+
+  const mediaStream = await navigator.mediaDevices.getUserMedia({
+    audio: {
+      channelCount: 1,
+      sampleRate,
+      echoCancellation: true,
+      noiseSuppression: true,
+    },
+  });
+
+  const audioContext = new AudioContext({ sampleRate });
+  const source = audioContext.createMediaStreamSource(mediaStream);
+  const processor = audioContext.createScriptProcessor(4096, 1, 1);
+
+  let stopped = false;
+  const chunks: Int16Array[] = [];
+  let resolver: ((value: IteratorResult<Uint8Array>) => void) | null = null;
+
+  processor.onaudioprocess = (event) => {
+    if (stopped) return;
+
+    const inputData = event.inputBuffer.getChannelData(0);
+    const pcm16 = new Int16Array(inputData.length);
+
+    // Convert Float32 to PCM S16LE
+    for (let i = 0; i < inputData.length; i++) {
+      const sample = Math.max(-1, Math.min(1, inputData[i]));
+      pcm16[i] = sample < 0 ? sample * 0x8000 : sample * 0x7fff;
+    }
+
+    // Convert to Uint8Array (little-endian)
+    const uint8 = new Uint8Array(pcm16.length * 2);
+    for (let i = 0; i < pcm16.length; i++) {
+      uint8[i * 2] = pcm16[i] & 0xff;
+      uint8[i * 2 + 1] = (pcm16[i] >> 8) & 0xff;
+    }
+
+    if (resolver) {
+      const resolve = resolver;
+      resolver = null;
+      resolve({ value: uint8, done: false });
+    } else {
+      chunks.push(pcm16);
+    }
+  };
+
+  source.connect(processor);
+  processor.connect(audioContext.destination);
+
+  const stream = (async function* () {
+    try {
+      while (!stopped) {
+        if (chunks.length > 0) {
+          const pcm16 = chunks.shift()!;
+          const uint8 = new Uint8Array(pcm16.length * 2);
+          for (let i = 0; i < pcm16.length; i++) {
+            uint8[i * 2] = pcm16[i] & 0xff;
+            uint8[i * 2 + 1] = (pcm16[i] >> 8) & 0xff;
+          }
+          yield uint8;
+        } else {
+          await new Promise<IteratorResult<Uint8Array>>((resolve) => {
+            resolver = resolve;
+          });
+        }
+      }
+    } finally {
+      processor.disconnect();
+      source.disconnect();
+      mediaStream.getTracks().forEach((track) => track.stop());
+      await audioContext.close();
+    }
+  })();
+
+  const stop = () => {
+    stopped = true;
+    if (resolver) {
+      resolver({ value: undefined, done: true });
+      resolver = null;
+    }
+  };
+
+  return { stream, stop };
+}