@mooncompany/uplink-chat 0.5.0

package/server/realtime/tts-stream.js

@@ -0,0 +1,446 @@
+/**
+ * TTS Streaming Module — Streams text-to-speech as PCM 24kHz 16-bit mono chunks
+ *
+ * Supports two engines:
+ *   - 'openai'  — OpenAI TTS API (POST /v1/audio/speech, response_format=pcm)
+ *   - 'edge'    — Microsoft Edge TTS via raw WebSocket (no npm dependency)
+ *
+ * Usage:
+ *   import { streamTTS, splitSentences } from './tts-stream.js';
+ *
+ *   for await (const pcmChunk of streamTTS('Hello world', config)) {
+ *     // pcmChunk is a Buffer of PCM 24kHz 16-bit mono audio
+ *   }
+ *
+ * Config shape:
+ *   {
+ *     engine: 'openai' | 'edge',
+ *     openaiApiKey: string,
+ *     openaiTtsVoice: string,   // alloy, echo, fable, onyx, nova, shimmer
+ *     openaiTtsModel: string,   // tts-1, tts-1-hd
+ *     edgeTtsVoice: string,     // e.g. 'en-US-AriaNeural'
+ *   }
+ */
+
+import { WebSocket } from 'ws';
+import { randomUUID } from 'crypto';
+import { createLogger } from '../logger.js';
+
+const log = createLogger('tts-stream');
+
+// ─── Constants ──────────────────────────────────────────────────────────────
+
+const OPENAI_TTS_URL = 'https://api.openai.com/v1/audio/speech';
+const OPENAI_TTS_TIMEOUT_MS = 30_000;
+
+// Edge TTS WebSocket endpoint (Microsoft Cognitive Services)
+const EDGE_TTS_WS_URL = 'wss://speech.platform.bing.com/consumer/speech/synthesize/readaloud/edge/v1';
+const EDGE_TTS_TRUSTED_TOKEN = '6A5AA1D4EAFF4E9FB37E23D68491D6F4';
+const EDGE_TTS_TIMEOUT_MS = 30_000;
+
+// ─── Sentence Splitting ────────────────────────────────────────────────────
+
+/**
+ * Split text into sentence-level chunks for streaming TTS.
+ * Splits at sentence boundaries (. ! ? followed by space or end)
+ * and at newlines. Returns an array of non-empty strings.
+ *
+ * @param {string} text - Full response text
+ * @returns {string[]} - Array of sentence chunks
+ */
+export function splitSentences(text) {
+  if (!text || typeof text !== 'string') return [];
+
+  // First split on newlines (paragraph boundaries)
+  const paragraphs = text.split(/\n+/);
+  const sentences = [];
+
+  for (const para of paragraphs) {
+    const trimmed = para.trim();
+    if (!trimmed) continue;
+
+    // Split on sentence-ending punctuation followed by whitespace or end-of-string.
+    // Handles: . ! ? and also ellipsis (...)
+    // Keeps the punctuation with the sentence.
+    const parts = trimmed.match(/[^.!?]*[.!?]+[\s]?|[^.!?]+$/g);
+
+    if (parts) {
+      for (const part of parts) {
+        const s = part.trim();
+        if (s) sentences.push(s);
+      }
+    } else {
+      // No sentence-ending punctuation found — return whole paragraph
+      sentences.push(trimmed);
+    }
+  }
+
+  return sentences;
+}
+
+/**
+ * Clean text for TTS — strip markdown, emojis, code blocks, etc.
+ * @param {string} text
+ * @returns {string}
+ */
+function cleanForTTS(text) {
+  return text
+    .replace(/```[\s\S]*?```/g, '')        // Code blocks
+    .replace(/`([^`]+)`/g, '$1')           // Inline code → just the text
+    .replace(/\*\*([^*]+)\*\*/g, '$1')     // Bold
+    .replace(/\*([^*]+)\*/g, '$1')         // Italic
+    .replace(/__([^_]+)__/g, '$1')         // Bold alt
+    .replace(/_([^_]+)_/g, '$1')           // Italic alt
+    .replace(/~~([^~]+)~~/g, '$1')         // Strikethrough
+    .replace(/\[([^\]]*)\]\([^)]*\)/g, '$1') // Links → link text
+    .replace(/#{1,6}\s/g, '')              // Headers
+    .replace(/^[-*+]\s/gm, '')             // List bullets
+    .replace(/^\d+\.\s/gm, '')             // Numbered lists
+    .replace(/^>\s?/gm, '')                // Block quotes
+    .replace(/\|/g, ' ')                   // Table pipes
+    .replace(/---+/g, '')                  // Horizontal rules
+    .replace(/[\u{1F600}-\u{1F64F}]/gu, '') // Emojis
+    .replace(/[\u{1F300}-\u{1F5FF}]/gu, '')
+    .replace(/[\u{1F680}-\u{1F6FF}]/gu, '')
+    .replace(/[\u{2600}-\u{26FF}]/gu, '')
+    .replace(/[\u{2700}-\u{27BF}]/gu, '')
+    .replace(/\s+/g, ' ')                 // Collapse whitespace
+    .trim();
+}
+
+// ─── OpenAI TTS Streaming ──────────────────────────────────────────────────
+
+/**
+ * Stream TTS from OpenAI API as PCM 24kHz 16-bit mono chunks.
+ *
+ * OpenAI's /v1/audio/speech endpoint with response_format=pcm returns
+ * raw PCM 24000 Hz, 16-bit signed little-endian mono audio.
+ *
+ * @param {string} text - Text to synthesize
+ * @param {Object} config - TTS configuration
+ * @yields {Buffer} PCM audio chunks
+ */
+async function* streamOpenAITTS(text, config) {
+  const apiKey = config.openaiApiKey;
+  if (!apiKey) {
+    throw new Error('OpenAI API key required for OpenAI TTS');
+  }
+
+  const voice = config.openaiTtsVoice || 'nova';
+  const model = config.openaiTtsModel || 'tts-1';
+  const cleanText = cleanForTTS(text);
+
+  if (!cleanText) {
+    log.warn('Empty text after cleaning — skipping TTS');
+    return;
+  }
+
+  log.debug(`OpenAI TTS: voice=${voice}, model=${model}, text="${cleanText.substring(0, 80)}..."`);
+
+  const controller = new AbortController();
+  const timeout = setTimeout(() => controller.abort(), OPENAI_TTS_TIMEOUT_MS);
+
+  try {
+    const response = await fetch(OPENAI_TTS_URL, {
+      method: 'POST',
+      headers: {
+        'Authorization': `Bearer ${apiKey}`,
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify({
+        model,
+        input: cleanText,
+        voice,
+        response_format: 'pcm', // Raw PCM 24kHz 16-bit mono
+      }),
+      signal: controller.signal,
+    });
+
+    clearTimeout(timeout);
+
+    if (!response.ok) {
+      const errorText = await response.text().catch(() => 'Unknown error');
+      throw new Error(`OpenAI TTS API error ${response.status}: ${errorText}`);
+    }
+
+    if (!response.body) {
+      throw new Error('OpenAI TTS response has no body');
+    }
+
+    // Stream the response body as PCM chunks
+    // PCM 16-bit requires 2-byte alignment — buffer partial bytes
+    const reader = response.body.getReader();
+    let remainder = null; // leftover byte from previous chunk
+    try {
+      while (true) {
+        const { done, value } = await reader.read();
+        if (done) break;
+        if (!value || value.length === 0) continue;
+
+        let chunk = Buffer.from(value);
+
+        // Prepend any leftover byte from previous chunk
+        if (remainder) {
+          chunk = Buffer.concat([remainder, chunk]);
+          remainder = null;
+        }
+
+        // If odd number of bytes, save the last byte for next chunk
+        if (chunk.length % 2 !== 0) {
+          remainder = chunk.slice(-1);
+          chunk = chunk.slice(0, -1);
+        }
+
+        if (chunk.length > 0) {
+          yield chunk;
+        }
+      }
+      // Flush any remaining byte (pad with zero)
+      if (remainder && remainder.length > 0) {
+        yield Buffer.concat([remainder, Buffer.alloc(1)]);
+      }
+    } finally {
+      reader.releaseLock();
+    }
+  } catch (err) {
+    clearTimeout(timeout);
+    if (err.name === 'AbortError') {
+      throw new Error('OpenAI TTS request timed out');
+    }
+    throw err;
+  }
+}
+
+// ─── Edge TTS Streaming ────────────────────────────────────────────────────
+
+/**
+ * Generate SSML for Edge TTS.
+ * @param {string} text - Text to speak
+ * @param {string} voice - Voice name (e.g. 'en-US-AriaNeural')
+ * @returns {string} SSML string
+ */
+function buildSSML(text, voice) {
+  // Escape XML special characters
+  const escaped = text
+    .replace(/&/g, '&')
+    .replace(/</g, '&lt;')
+    .replace(/>/g, '&gt;')
+    .replace(/"/g, '&quot;')
+    .replace(/'/g, '&apos;');
+
+  return `<speak version='1.0' xmlns='http://www.w3.org/2001/10/synthesis' xml:lang='en-US'>` +
+    `<voice name='${voice}'>${escaped}</voice></speak>`;
+}
+
+/**
+ * Generate an Edge TTS request ID (32-char hex, no dashes).
+ * @returns {string}
+ */
+function edgeRequestId() {
+  return randomUUID().replace(/-/g, '');
+}
+
+/**
+ * Build the Edge TTS WebSocket URL with required query parameters.
+ * @returns {string}
+ */
+function buildEdgeTTSUrl() {
+  return `${EDGE_TTS_WS_URL}?TrustedClientToken=${EDGE_TTS_TRUSTED_TOKEN}&ConnectionId=${edgeRequestId()}`;
+}
+
+/**
+ * Stream TTS from Edge TTS via raw WebSocket as PCM 24kHz 16-bit mono.
+ *
+ * Edge TTS natively outputs MP3. To get PCM we request a raw audio format
+ * via the output format header. Edge supports `raw-24khz-16bit-mono-pcm`.
+ *
+ * @param {string} text - Text to synthesize
+ * @param {Object} config - TTS configuration
+ * @yields {Buffer} PCM audio chunks
+ */
+async function* streamEdgeTTS(text, config) {
+  const voice = config.edgeTtsVoice || 'en-US-AriaNeural';
+  const cleanText = cleanForTTS(text);
+
+  if (!cleanText) {
+    log.warn('Empty text after cleaning — skipping Edge TTS');
+    return;
+  }
+
+  log.debug(`Edge TTS: voice=${voice}, text="${cleanText.substring(0, 80)}..."`);
+
+  const requestId = edgeRequestId();
+
+  // Yield PCM chunks from the WebSocket via a promise-based async iterator
+  const chunks = [];
+  let resolve;
+  let reject;
+  let done = false;
+  let error = null;
+
+  // Queue mechanism for async iteration
+  const queue = [];
+  let waiter = null;
+
+  function enqueue(chunk) {
+    if (waiter) {
+      const w = waiter;
+      waiter = null;
+      w({ value: chunk, done: false });
+    } else {
+      queue.push(chunk);
+    }
+  }
+
+  function finish(err) {
+    done = true;
+    error = err || null;
+    if (waiter) {
+      const w = waiter;
+      waiter = null;
+      if (err) {
+        w(Promise.reject(err));
+      } else {
+        w({ value: undefined, done: true });
+      }
+    }
+  }
+
+  // Connect
+  const ws = new WebSocket(buildEdgeTTSUrl(), {
+    headers: {
+      'Origin': 'chrome-extension://jdiccldimpdaibmpdkjnbmckianbfold',
+      'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36 Edg/130.0.0.0',
+    },
+  });
+
+  const connectionTimeout = setTimeout(() => {
+    ws.close();
+    finish(new Error('Edge TTS connection timed out'));
+  }, EDGE_TTS_TIMEOUT_MS);
+
+  ws.on('open', () => {
+    log.debug('Edge TTS WebSocket connected');
+
+    // 1. Send speech config
+    const configMessage =
+      `Content-Type:application/json; charset=utf-8\r\n` +
+      `Path:speech.config\r\n\r\n` +
+      JSON.stringify({
+        context: {
+          synthesis: {
+            audio: {
+              metadataOptions: { sentenceBoundaryEnabled: false, wordBoundaryEnabled: false },
+              outputFormat: 'raw-24khz-16bit-mono-pcm',
+            },
+          },
+        },
+      });
+    ws.send(configMessage);
+
+    // 2. Send SSML synthesis request
+    const ssml = buildSSML(cleanText, voice);
+    const synthMessage =
+      `X-RequestId:${requestId}\r\n` +
+      `Content-Type:application/ssml+xml\r\n` +
+      `Path:ssml\r\n\r\n` +
+      ssml;
+    ws.send(synthMessage);
+  });
+
+  ws.on('message', (data, isBinary) => {
+    if (isBinary) {
+      // Binary message: contains header + audio data
+      // The header is separated from audio by a known pattern
+      // Format: text-header\r\n\r\n<binary audio data>
+      // OR it could be prefixed with a 2-byte header length
+      const buf = Buffer.isBuffer(data) ? data : Buffer.from(data);
+
+      // Edge TTS binary frames: first 2 bytes = header length (big-endian)
+      // followed by header text, then raw audio
+      if (buf.length > 2) {
+        const headerLen = buf.readUInt16BE(0);
+        if (headerLen > 0 && headerLen < buf.length) {
+          const audioData = buf.subarray(2 + headerLen);
+          if (audioData.length > 0) {
+            enqueue(audioData);
+          }
+        }
+      }
+    } else {
+      // Text message — check for turn.end
+      const msg = data.toString();
+      if (msg.includes('Path:turn.end')) {
+        clearTimeout(connectionTimeout);
+        ws.close();
+        finish(null);
+      }
+    }
+  });
+
+  ws.on('error', (err) => {
+    clearTimeout(connectionTimeout);
+    log.error('Edge TTS WebSocket error:', err.message);
+    finish(new Error(`Edge TTS error: ${err.message}`));
+  });
+
+  ws.on('close', () => {
+    clearTimeout(connectionTimeout);
+    if (!done) {
+      finish(null);
+    }
+  });
+
+  // Async iteration
+  while (true) {
+    if (queue.length > 0) {
+      yield queue.shift();
+    } else if (done) {
+      if (error) throw error;
+      return;
+    } else {
+      // Wait for next chunk or finish
+      const result = await new Promise((res, rej) => {
+        waiter = res;
+      });
+      if (result.done) return;
+      yield result.value;
+    }
+  }
+}
+
+// ─── Public API ─────────────────────────────────────────────────────────────
+
+/**
+ * Stream TTS audio as PCM 24kHz 16-bit mono chunks.
+ *
+ * @param {string} text - Text to synthesize (single sentence or short chunk)
+ * @param {Object} config - TTS configuration
+ * @param {string} config.engine - 'openai' or 'edge'
+ * @param {string} [config.openaiApiKey] - Required for OpenAI engine
+ * @param {string} [config.openaiTtsVoice] - OpenAI voice (default: 'nova')
+ * @param {string} [config.openaiTtsModel] - OpenAI model (default: 'tts-1')
+ * @param {string} [config.edgeTtsVoice] - Edge TTS voice (default: 'en-US-AriaNeural')
+ * @yields {Buffer} PCM audio chunks (24kHz, 16-bit signed LE, mono)
+ */
+export async function* streamTTS(text, config) {
+  if (!text || typeof text !== 'string') {
+    log.warn('streamTTS called with empty text');
+    return;
+  }
+
+  const engine = config.engine || 'openai';
+
+  switch (engine) {
+    case 'openai':
+      yield* streamOpenAITTS(text, config);
+      break;
+
+    case 'edge':
+      yield* streamEdgeTTS(text, config);
+      break;
+
+    default:
+      throw new Error(`Unknown TTS engine: ${engine}`);
+  }
+}