@decentchat/decentchat-plugin 0.1.9

package/src/huddle/SpeechToText.ts

@@ -0,0 +1,223 @@
+import { execFile } from 'child_process';
+import { promisify } from 'util';
+import { writeFile, readFile, unlink } from 'fs/promises';
+import { randomBytes } from 'crypto';
+import { join } from 'path';
+import { tmpdir } from 'os';
+
+const execFileAsync = promisify(execFile);
+
+export interface STTOptions {
+  engine?: 'whisper-cpp' | 'whisper-python' | 'openai' | 'groq';
+  model?: string;
+  language?: string;
+  apiKey?: string;        // For openai/groq engines
+  log?: { info: (s: string) => void; warn?: (s: string) => void };
+}
+
+const DEFAULT_MODEL = 'base.en';
+const MODEL_DIR = '/opt/homebrew/share/whisper-cpp/models';
+const WHISPER_BIN = 'whisper-cli';
+const EXEC_TIMEOUT = 30_000;
+
+/**
+ * SpeechToText — converts PCM audio buffers to text.
+ * Supports local whisper-cpp or cloud APIs (OpenAI, Groq).
+ */
+export class SpeechToText {
+  private engine: string;
+  private modelPath: string;
+  private model: string;
+  private language?: string;
+  private apiKey?: string;
+  private log?: { info: (s: string) => void; warn?: (s: string) => void };
+
+  constructor(opts?: STTOptions) {
+    this.engine = opts?.engine ?? 'whisper-cpp';
+    this.model = opts?.model ?? DEFAULT_MODEL;
+    this.modelPath = join(MODEL_DIR, `ggml-${this.model}.bin`);
+    this.language = opts?.language;
+    this.apiKey = opts?.apiKey;
+    this.log = opts?.log;
+  }
+
+  /**
+   * Convert PCM buffer (16-bit signed LE, mono) to text.
+   */
+  async transcribe(pcmBuffer: Buffer, sampleRate = 48000): Promise<string> {
+    if (this.engine === 'openai' || this.engine === 'groq') {
+      return this.transcribeCloud(pcmBuffer, sampleRate);
+    }
+    return this.transcribeLocal(pcmBuffer, sampleRate);
+  }
+
+  /**
+   * Cloud transcription via OpenAI or Groq Whisper API.
+   */
+  private async transcribeCloud(pcmBuffer: Buffer, sampleRate: number): Promise<string> {
+    const wavBuffer = this.createWavBuffer(pcmBuffer, sampleRate);
+    const duration = (pcmBuffer.length / 2 / sampleRate).toFixed(1);
+
+    const isGroq = this.engine === 'groq';
+    const baseUrl = isGroq
+      ? 'https://api.groq.com/openai/v1'
+      : 'https://api.openai.com/v1';
+    // Only use this.model for cloud if it looks like a cloud model name
+    // (contains 'whisper'). Otherwise use the provider default.
+    const isCloudModel = this.model.includes('whisper');
+    const model = isGroq
+      ? (isCloudModel ? this.model : 'whisper-large-v3-turbo')
+      : (isCloudModel ? this.model : 'whisper-1');
+    const key = this.apiKey
+      ?? (isGroq ? process.env.GROQ_API_KEY : process.env.OPENAI_API_KEY)
+      ?? '';
+
+    if (!key) {
+      this.log?.warn?.(`[STT] No API key for ${this.engine} — set ${isGroq ? 'GROQ_API_KEY' : 'OPENAI_API_KEY'}`);
+      return '';
+    }
+
+    this.log?.info(`[STT] ${this.engine} transcribe: ${duration}s audio, model=${model}${this.language ? ', lang=' + this.language : ''}`);
+    const start = Date.now();
+
+    // Build multipart form data
+    const boundary = '----STTBoundary' + randomBytes(8).toString('hex');
+    const parts: Buffer[] = [];
+
+    // File part
+    parts.push(Buffer.from(
+      `--${boundary}\r\nContent-Disposition: form-data; name="file"; filename="audio.wav"\r\nContent-Type: audio/wav\r\n\r\n`
+    ));
+    parts.push(wavBuffer);
+    parts.push(Buffer.from('\r\n'));
+
+    // Model part
+    parts.push(Buffer.from(
+      `--${boundary}\r\nContent-Disposition: form-data; name="model"\r\n\r\n${model}\r\n`
+    ));
+
+    // Language part (optional)
+    if (this.language) {
+      parts.push(Buffer.from(
+        `--${boundary}\r\nContent-Disposition: form-data; name="language"\r\n\r\n${this.language}\r\n`
+      ));
+    }
+
+    // Response format
+    parts.push(Buffer.from(
+      `--${boundary}\r\nContent-Disposition: form-data; name="response_format"\r\n\r\ntext\r\n`
+    ));
+
+    parts.push(Buffer.from(`--${boundary}--\r\n`));
+
+    const body = Buffer.concat(parts);
+
+    const response = await fetch(`${baseUrl}/audio/transcriptions`, {
+      method: 'POST',
+      headers: {
+        'Authorization': `Bearer ${key}`,
+        'Content-Type': `multipart/form-data; boundary=${boundary}`,
+      },
+      body,
+    });
+
+    const elapsed = Date.now() - start;
+
+    if (!response.ok) {
+      const err = await response.text().catch(() => 'unknown');
+      this.log?.warn?.(`[STT] ${this.engine} error ${response.status}: ${err}`);
+      return '';
+    }
+
+    const text = (await response.text()).trim();
+    this.log?.info(`[STT] ${this.engine} transcribed in ${elapsed}ms: "${text.slice(0, 60)}${text.length > 60 ? '...' : ''}"`);
+    return text;
+  }
+
+  /**
+   * Local transcription via whisper-cli.
+   */
+  private async transcribeLocal(pcmBuffer: Buffer, sampleRate: number): Promise<string> {
+    const id = randomBytes(6).toString('hex');
+    const tmp = tmpdir();
+    const inputWav = join(tmp, `stt-${id}.wav`);
+    const resampledWav = join(tmp, `stt-${id}-16k.wav`);
+    const outputBase = join(tmp, `stt-${id}-out`);
+    const outputTxt = `${outputBase}.txt`;
+
+    const tempFiles = [inputWav, resampledWav, outputTxt];
+
+    try {
+      // 1. Write PCM to WAV
+      const wavBuffer = this.createWavBuffer(pcmBuffer, sampleRate);
+      await writeFile(inputWav, wavBuffer);
+      this.log?.info(`[STT] Wrote ${wavBuffer.length} bytes WAV → ${inputWav}`);
+
+      // 2. Resample to 16 kHz mono via ffmpeg
+      await execFileAsync('ffmpeg', [
+        '-i', inputWav,
+        '-ar', '16000',
+        '-ac', '1',
+        '-y', resampledWav,
+      ], { timeout: EXEC_TIMEOUT });
+      this.log?.info(`[STT] Resampled to 16 kHz → ${resampledWav}`);
+
+      // 3. Run whisper-cli
+      const args = [
+        '--model', this.modelPath,
+        '--output-txt',
+        '--output-file', outputBase,
+        '--no-timestamps',
+      ];
+      if (this.language) {
+        args.push('--language', this.language);
+      }
+      args.push(resampledWav);
+      this.log?.info(`[STT] whisper-cli args: ${args.join(' ')}`);
+      await execFileAsync(WHISPER_BIN, args, { timeout: EXEC_TIMEOUT });
+      this.log?.info(`[STT] whisper-cli finished`);
+
+      // 4. Read the generated .txt
+      const text = await readFile(outputTxt, 'utf-8');
+      return text.trim();
+    } finally {
+      await Promise.all(
+        tempFiles.map(f => unlink(f).catch(() => {})),
+      );
+      this.log?.info(`[STT] Cleaned up temp files`);
+    }
+  }
+
+  /**
+   * Create a valid WAV (RIFF) buffer from raw 16-bit signed LE PCM data.
+   */
+  private createWavBuffer(pcm: Buffer, sampleRate: number): Buffer {
+    const numChannels = 1;
+    const bitsPerSample = 16;
+    const byteRate = sampleRate * numChannels * (bitsPerSample / 8);
+    const blockAlign = numChannels * (bitsPerSample / 8);
+    const dataSize = pcm.length;
+    const headerSize = 44;
+
+    const header = Buffer.alloc(headerSize);
+    let offset = 0;
+
+    header.write('RIFF', offset); offset += 4;
+    header.writeUInt32LE(dataSize + headerSize - 8, offset); offset += 4;
+    header.write('WAVE', offset); offset += 4;
+
+    header.write('fmt ', offset); offset += 4;
+    header.writeUInt32LE(16, offset); offset += 4;
+    header.writeUInt16LE(1, offset); offset += 2;
+    header.writeUInt16LE(numChannels, offset); offset += 2;
+    header.writeUInt32LE(sampleRate, offset); offset += 4;
+    header.writeUInt32LE(byteRate, offset); offset += 4;
+    header.writeUInt16LE(blockAlign, offset); offset += 2;
+    header.writeUInt16LE(bitsPerSample, offset); offset += 2;
+
+    header.write('data', offset); offset += 4;
+    header.writeUInt32LE(dataSize, offset); offset += 4;
+
+    return Buffer.concat([header, pcm]);
+  }
+}

package/src/huddle/TextToSpeech.ts

@@ -0,0 +1,260 @@
+import OpusScript from 'opusscript';
+
+export interface TTSOptions {
+  apiKey: string;
+  voiceId?: string;       // Default: 'EXAVITQu4vr4xnSDxMaL' (Rachel)
+  model?: string;         // Default: 'eleven_turbo_v2_5' (multilingual)
+  language?: string;      // Language code for multilingual models (e.g. 'sk', 'en')
+  sampleRate?: number;    // Output sample rate for Opus: 48000
+  log?: { info: (s: string) => void };
+}
+
+const DEFAULT_VOICE_ID = 'EXAVITQu4vr4xnSDxMaL';
+const DEFAULT_MODEL = 'eleven_turbo_v2_5';
+const DEFAULT_SAMPLE_RATE = 48000;
+const ELEVENLABS_PCM_RATE = 24000;
+const FRAME_DURATION_MS = 20;
+const OPUS_PT = 111;
+const DEFAULT_SSRC = 1234;
+
+export class TextToSpeech {
+  private apiKey: string;
+  private voiceId: string;
+  private model: string;
+  private language?: string;
+  private sampleRate: number;
+  private log?: { info: (s: string) => void };
+  private encoder: OpusScript;
+
+  constructor(opts: TTSOptions) {
+    this.apiKey = opts.apiKey;
+    this.voiceId = opts.voiceId ?? DEFAULT_VOICE_ID;
+    this.model = opts.model ?? DEFAULT_MODEL;
+    this.language = opts.language;
+    this.sampleRate = opts.sampleRate ?? DEFAULT_SAMPLE_RATE;
+    this.log = opts.log;
+    this.encoder = new OpusScript(this.sampleRate, 2, OpusScript.Application.AUDIO);
+  }
+
+  /**
+   * Convert text to a sequence of RTP packets containing Opus-encoded audio.
+   * Each packet represents a 20ms frame.
+   */
+  async speak(text: string): Promise<Buffer[]> {
+    this.log?.info(`TTS: synthesizing "${text.slice(0, 60)}${text.length > 60 ? '...' : ''}"`);
+
+    // 1. Fetch PCM audio from ElevenLabs
+    const pcm24k = await this.fetchPcmFromElevenLabs(text);
+    this.log?.info(`TTS: received ${pcm24k.length} bytes of PCM @ ${ELEVENLABS_PCM_RATE}Hz`);
+
+    // 2. Resample 24kHz → 48kHz
+    //    ElevenLabs occasionally returns odd byte counts; truncate to even
+    //    since PCM 16-bit requires 2 bytes per sample.
+    const pcmEven = pcm24k.length % 2 !== 0 ? pcm24k.subarray(0, pcm24k.length - 1) : pcm24k;
+    const pcm48k = this.resample(pcmEven, ELEVENLABS_PCM_RATE, this.sampleRate);
+    this.log?.info(`TTS: resampled to ${pcm48k.length} bytes @ ${this.sampleRate}Hz`);
+
+    // 3. Chunk into 20ms frames and Opus encode
+    const samplesPerFrame = (this.sampleRate * FRAME_DURATION_MS) / 1000; // 960 at 48kHz
+    const bytesPerFrame = samplesPerFrame * 2; // 16-bit samples = 2 bytes each
+    const packets: Buffer[] = [];
+    let seq = 0;
+    let timestamp = 0;
+
+    for (let offset = 0; offset + bytesPerFrame <= pcm48k.length; offset += bytesPerFrame) {
+      const pcmFrame = pcm48k.subarray(offset, offset + bytesPerFrame);
+      // Convert mono PCM to stereo (interleaved L,R) for the 2-channel Opus encoder.
+      // The SDP advertises sprop-stereo=1, so the browser expects stereo frames.
+      const stereoPcm = Buffer.alloc(pcmFrame.length * 2);
+      for (let i = 0; i < samplesPerFrame; i++) {
+        const sample = pcmFrame.readInt16LE(i * 2);
+        stereoPcm.writeInt16LE(sample, i * 4);     // L
+        stereoPcm.writeInt16LE(sample, i * 4 + 2); // R
+      }
+      const opusFrame = this.encoder.encode(stereoPcm, samplesPerFrame);
+
+      const rtpPacket = this.createRtpPacket(
+        Buffer.from(opusFrame),
+        seq,
+        timestamp,
+        DEFAULT_SSRC,
+        OPUS_PT,
+        seq === 0  // Marker bit on first packet (start of talkspurt)
+      );
+      packets.push(rtpPacket);
+
+      seq++;
+      timestamp += samplesPerFrame;
+    }
+
+    this.log?.info(`TTS: encoded ${packets.length} RTP packets (${(packets.length * FRAME_DURATION_MS / 1000).toFixed(1)}s)`);
+    return packets;
+  }
+
+  /**
+   * Like speak(), but returns raw Opus frames (no RTP headers).
+   * For use with node-datachannel's media handler which adds RTP headers itself.
+   */
+  async speakRaw(text: string): Promise<Buffer[]> {
+    // 1. Get PCM from ElevenLabs
+    this.log?.info(`TTS: synthesizing (raw) "${text.slice(0, 60)}${text.length > 60 ? "..." : ""}"`);
+    const pcm = await this.fetchPcmFromElevenLabs(text);
+    this.log?.info(`TTS: received ${pcm.length} bytes of PCM @ 24000Hz`);
+    if (!pcm || pcm.length === 0) return [];
+
+    // 2. Resample to 48kHz
+    const pcmEven = pcm.length % 2 !== 0 ? pcm.subarray(0, pcm.length - 1) : pcm;
+    const pcm48k = this.resample(pcmEven, 24000 as number, this.sampleRate);
+    this.log?.info(`TTS: resampled to ${pcm48k.length} bytes @ 48000Hz (raw mode)`);
+
+    // 3. Chunk into 20ms frames and Opus encode (no RTP wrapping)
+    const samplesPerFrame = (this.sampleRate * FRAME_DURATION_MS) / 1000;
+    const bytesPerFrame = samplesPerFrame * 2;
+    const frames: Buffer[] = [];
+
+    // Use a FRESH encoder per call — persistent encoder state causes Chrome decode issues.
+    const OpusScript = (await import('opusscript')).default;
+    const freshEncoder = new OpusScript(this.sampleRate, 2, OpusScript.Application.AUDIO);
+
+    for (let offset = 0; offset + bytesPerFrame <= pcm48k.length; offset += bytesPerFrame) {
+      const pcmFrame = pcm48k.subarray(offset, offset + bytesPerFrame);
+      // Convert mono PCM to stereo (interleaved L,R) for the 2-channel Opus encoder.
+      // The SDP advertises sprop-stereo=1, so the browser expects stereo frames.
+      const stereoPcm = Buffer.alloc(pcmFrame.length * 2);
+      for (let i = 0; i < samplesPerFrame; i++) {
+        const sample = pcmFrame.readInt16LE(i * 2);
+        stereoPcm.writeInt16LE(sample, i * 4);     // L
+        stereoPcm.writeInt16LE(sample, i * 4 + 2); // R
+      }
+      const opusFrame = freshEncoder.encode(stereoPcm, samplesPerFrame);
+      frames.push(Buffer.from(opusFrame));
+    }
+    freshEncoder.delete();
+
+    this.log?.info(`TTS: encoded ${frames.length} raw Opus frames (${(frames.length * FRAME_DURATION_MS / 1000).toFixed(1)}s)`);
+
+    // DIAGNOSTIC: dump resampled PCM and Opus frames for offline analysis
+    try {
+      const fs = await import('fs');
+      fs.writeFileSync('/tmp/tts_debug_pcm48k.raw', pcm48k);
+      fs.writeFileSync('/tmp/tts_debug_frames.json', JSON.stringify(frames.map(f => Buffer.from(f).toString('base64'))));
+      this.log?.info(`TTS: DIAG dumped ${pcm48k.length}b PCM + ${frames.length} frames to /tmp/tts_debug_*`);
+    } catch {}
+
+    return frames;
+  }
+
+  /**
+   * Fetch raw PCM 16-bit 24kHz mono audio from ElevenLabs streaming TTS API.
+   */
+  private async fetchPcmFromElevenLabs(text: string): Promise<Buffer> {
+    // output_format is a QUERY parameter, not a body parameter.
+    // Without it, ElevenLabs returns MP3 (audio/mpeg) instead of raw PCM.
+    const url = `https://api.elevenlabs.io/v1/text-to-speech/${this.voiceId}/stream?output_format=pcm_24000`;
+
+    const response = await fetch(url, {
+      method: 'POST',
+      headers: {
+        'xi-api-key': this.apiKey,
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify({
+        text,
+        model_id: this.model,
+        ...(this.language ? { language_code: this.language } : {}),
+      }),
+    });
+
+    if (!response.ok) {
+      const errorText = await response.text().catch(() => 'unknown error');
+      throw new Error(`ElevenLabs API error ${response.status}: ${errorText}`);
+    }
+
+    // Accumulate response body into a single buffer
+    const chunks: Uint8Array[] = [];
+    const reader = response.body?.getReader();
+    if (!reader) throw new Error('No response body from ElevenLabs');
+
+    while (true) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      if (value) chunks.push(value);
+    }
+
+    // Concatenate all chunks
+    const totalLength = chunks.reduce((sum, c) => sum + c.length, 0);
+    const result = Buffer.alloc(totalLength);
+    let pos = 0;
+    for (const chunk of chunks) {
+      result.set(chunk, pos);
+      pos += chunk.length;
+    }
+
+    return result;
+  }
+
+  /**
+   * Resample PCM 16-bit mono audio using linear interpolation.
+   */
+  resample(input: Buffer, fromRate: number, toRate: number): Buffer {
+    if (fromRate === toRate) return input;
+
+    // Defensive: ensure even byte count for 16-bit PCM
+    const safeInput = input.length % 2 !== 0 ? input.subarray(0, input.length - 1) : input;
+    const inputSamples = safeInput.length / 2; // 16-bit = 2 bytes per sample
+    const ratio = fromRate / toRate;
+    const outputSamples = Math.floor(inputSamples / ratio);
+    const output = Buffer.alloc(outputSamples * 2);
+
+    for (let i = 0; i < outputSamples; i++) {
+      const srcPos = i * ratio;
+      const srcIndex = Math.floor(srcPos);
+      const frac = srcPos - srcIndex;
+
+      const s0 = safeInput.readInt16LE(srcIndex * 2);
+      const s1 = srcIndex + 1 < inputSamples
+        ? safeInput.readInt16LE((srcIndex + 1) * 2)
+        : s0;
+
+      const interpolated = Math.round(s0 + frac * (s1 - s0));
+      // Clamp to Int16 range
+      const clamped = Math.max(-32768, Math.min(32767, interpolated));
+      output.writeInt16LE(clamped, i * 2);
+    }
+
+    return output;
+  }
+
+  /**
+   * Create an RTP packet with the given Opus payload.
+   *
+   * RTP Header (12 bytes):
+   *   Byte 0:    0x80 (V=2, no padding, no extension, CC=0)
+   *   Byte 1:    payload type
+   *   Bytes 2-3: sequence number (big-endian)
+   *   Bytes 4-7: timestamp (big-endian, increments by 960 per 20ms frame)
+   *   Bytes 8-11: SSRC (big-endian)
+   */
+  createRtpPacket(payload: Buffer, seq: number, timestamp: number, ssrc: number, pt: number, marker = false): Buffer {
+    const header = Buffer.alloc(12);
+
+    header[0] = 0x80;          // V=2
+    header[1] = (marker ? 0x80 : 0) | (pt & 0x7f);  // Marker bit + payload type
+    header.writeUInt16BE(seq & 0xffff, 2);
+    header.writeUInt32BE(timestamp >>> 0, 4);
+    header.writeUInt32BE(ssrc >>> 0, 8);
+
+    return Buffer.concat([header, payload]);
+  }
+
+  /**
+   * Cleanup OpusScript encoder resources.
+   */
+  destroy(): void {
+    try {
+      this.encoder.delete();
+    } catch {
+      // Already destroyed or not supported
+    }
+  }
+}

package/src/huddle/index.ts

@@ -0,0 +1,8 @@
+export { BotHuddleManager } from './BotHuddleManager.js';
+export type { BotHuddleCallbacks, BotHuddleConfig, BotHuddleState, BotHuddleParticipant } from './BotHuddleManager.js';
+export { AudioPipeline } from './AudioPipeline.js';
+export type { AudioPipelineOptions } from './AudioPipeline.js';
+export { SpeechToText } from './SpeechToText.js';
+export type { STTOptions } from './SpeechToText.js';
+export { TextToSpeech } from './TextToSpeech.js';
+export type { TTSOptions } from './TextToSpeech.js';