@pompeii-labs/audio 0.0.1

package/dist/voice.js

@@ -0,0 +1,756 @@
+'use strict';
+
+var sdk = require('@deepgram/sdk');
+var elevenlabsJs = require('@elevenlabs/elevenlabs-js');
+var hume = require('hume');
+var OpenAI = require('openai');
+
+function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; }
+
+var OpenAI__default = /*#__PURE__*/_interopDefault(OpenAI);
+
+// src/helpers/bufferToInt16Array.ts
+function bufferToInt16Array(buffer) {
+  return new Int16Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / 2);
+}
+
+// src/decoders/mulaw.ts
+function mulawToPcm16(mulawData) {
+  const pcmData = new Int16Array(mulawData.length);
+  for (let i = 0; i < mulawData.length; i++) {
+    pcmData[i] = mulawToLinear(mulawData[i]);
+  }
+  return pcmData;
+}
+function mulawToLinear(mulawByte) {
+  const inverted = mulawByte ^ 255;
+  const sign = inverted & 128;
+  const segment = (inverted & 112) >> 4;
+  const step = inverted & 15;
+  let linear;
+  if (segment === 0) {
+    linear = (step << 1) + 1;
+  } else {
+    linear = (step << 1) + 1 + 32 << segment + 2;
+  }
+  linear -= 33;
+  return sign ? -linear : linear;
+}
+
+// src/encoders/mulaw.ts
+var BIAS = 132;
+var CLIP = 32635;
+var encodeTable = [
+  0,
+  0,
+  1,
+  1,
+  2,
+  2,
+  2,
+  2,
+  3,
+  3,
+  3,
+  3,
+  3,
+  3,
+  3,
+  3,
+  4,
+  4,
+  4,
+  4,
+  4,
+  4,
+  4,
+  4,
+  4,
+  4,
+  4,
+  4,
+  4,
+  4,
+  4,
+  4,
+  5,
+  5,
+  5,
+  5,
+  5,
+  5,
+  5,
+  5,
+  5,
+  5,
+  5,
+  5,
+  5,
+  5,
+  5,
+  5,
+  5,
+  5,
+  5,
+  5,
+  5,
+  5,
+  5,
+  5,
+  5,
+  5,
+  5,
+  5,
+  5,
+  5,
+  5,
+  5,
+  6,
+  6,
+  6,
+  6,
+  6,
+  6,
+  6,
+  6,
+  6,
+  6,
+  6,
+  6,
+  6,
+  6,
+  6,
+  6,
+  6,
+  6,
+  6,
+  6,
+  6,
+  6,
+  6,
+  6,
+  6,
+  6,
+  6,
+  6,
+  6,
+  6,
+  6,
+  6,
+  6,
+  6,
+  6,
+  6,
+  6,
+  6,
+  6,
+  6,
+  6,
+  6,
+  6,
+  6,
+  6,
+  6,
+  6,
+  6,
+  6,
+  6,
+  6,
+  6,
+  6,
+  6,
+  6,
+  6,
+  6,
+  6,
+  6,
+  6,
+  6,
+  6,
+  6,
+  6,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7,
+  7
+];
+function encodeSample(sample) {
+  const sign = sample >> 8 & 128;
+  if (sign !== 0) sample = -sample;
+  sample = sample + BIAS;
+  if (sample > CLIP) sample = CLIP;
+  const exponent = encodeTable[sample >> 7 & 255];
+  const mantissa = sample >> exponent + 3 & 15;
+  return ~(sign | exponent << 4 | mantissa);
+}
+function pcm16ToMulaw(pcmData) {
+  const mulawData = new Uint8Array(pcmData.length);
+  for (let i = 0; i < pcmData.length; i++) {
+    mulawData[i] = encodeSample(pcmData[i]);
+  }
+  return mulawData;
+}
+
+// src/helpers/int16ArrayToBuffer.ts
+function int16ArrayToBuffer(int16Array) {
+  return Buffer.from(int16Array.buffer, int16Array.byteOffset, int16Array.byteLength);
+}
+
+// src/helpers/convertAudioFormat.ts
+function encodePcm(audio, encoding) {
+  switch (encoding) {
+    case "mulaw":
+      return Buffer.from(pcm16ToMulaw(audio));
+    case "pcm":
+      return int16ArrayToBuffer(audio);
+    default:
+      throw new Error(`Could not encode audio: Unsupported encoding: ${encoding}`);
+  }
+}
+function decodeToPcm(audio, encoding) {
+  switch (encoding) {
+    case "mulaw":
+      return mulawToPcm16(audio);
+    case "pcm":
+      return bufferToInt16Array(audio);
+    default:
+      throw new Error(`Could not decode audio: Unsupported encoding: ${encoding}`);
+  }
+}
+
+// src/helpers/generateFadeOutSamples.ts
+function generateFadeOutSamples(lastSampleValue, fadeDurationMs, sampleRate) {
+  const fadeNumSamples = Math.ceil(fadeDurationMs / 1e3 * sampleRate);
+  const fadeSamples = new Int16Array(fadeNumSamples);
+  for (let i = 0; i < fadeNumSamples; i++) {
+    const progress = 1 - i / (fadeNumSamples - 1);
+    fadeSamples[i] = Math.round(lastSampleValue * progress);
+  }
+  return new Uint8Array(fadeSamples.buffer);
+}
+
+// src/helpers/resamplePcm.ts
+function resamplePcm(pcm, originalSampleRate, targetSampleRate) {
+  const ratio = originalSampleRate / targetSampleRate;
+  const newLength = Math.floor(pcm.length / ratio);
+  const newSamples = new Int16Array(newLength);
+  for (let i = 0; i < newSamples.length; i++) {
+    const exactPos = i * ratio;
+    const lowerIndex = Math.floor(exactPos);
+    const upperIndex = Math.min(lowerIndex + 1, pcm.length - 1);
+    const fraction = exactPos - lowerIndex;
+    const lowerSample = pcm[lowerIndex];
+    const upperSample = pcm[upperIndex];
+    newSamples[i] = Math.round(lowerSample + (upperSample - lowerSample) * fraction);
+  }
+  return newSamples;
+}
+
+// src/voice/helpers.ts
+function splitTextIntoChunks(text, targetLength = 100) {
+  const endOfSentencePunctuation = [".", "!", "?"];
+  const sentences = [];
+  for (let i = targetLength; i < text.length; i++) {
+    if (endOfSentencePunctuation.includes(text[i])) {
+      sentences.push(text.slice(0, i + 1));
+      text = text.slice(i + 1);
+      i = targetLength;
+    }
+  }
+  return sentences;
+}
+
+// src/voice/client.ts
+var uniformSampleRate = 48e3;
+var MagmaFlow = class {
+  constructor(args) {
+    this.textBuffer = "";
+    this.textQueue = [];
+    this.generatingAudio = false;
+    this.audioBuffer = [];
+    this.stt = args.stt;
+    this.tts = args.tts;
+    this.inputFormat = args.inputFormat;
+    this.outputFormat = args.outputFormat;
+    this.onAudioOutput = args.onAudioOutput;
+    this.tts.onOutput = (audio) => {
+      if (!audio) {
+        const lastChunk = this.audioBuffer[this.audioBuffer.length - 1];
+        if (lastChunk) {
+          const lastChunkSamples = bufferToInt16Array(lastChunk);
+          const lastSampleValue = lastChunkSamples[lastChunkSamples.length - 1];
+          this.audioBuffer.push(
+            Buffer.from(generateFadeOutSamples(lastSampleValue, 500, 48e3))
+          );
+        }
+        this.sendAudio();
+        this.generatingAudio = false;
+        this.generateAudio();
+        return;
+      }
+      this.audioBuffer.push(audio);
+    };
+    this.stt.onOutput = args.onTextOutput;
+    this.stt.onSpeechDetected = args.onSpeechDetected;
+  }
+  inputAudio(audio) {
+    const decodedAudio = decodeToPcm(audio, this.inputFormat.encoding);
+    const resampledPCM = resamplePcm(decodedAudio, this.inputFormat.sampleRate, 48e3);
+    this.stt.input(int16ArrayToBuffer(resampledPCM));
+  }
+  inputText(text) {
+    if (!text) {
+      this.textQueue.push(this.textBuffer);
+      this.textBuffer = "";
+      this.generateAudio();
+      return;
+    }
+    this.textBuffer += text;
+    const chunks = splitTextIntoChunks(this.textBuffer, 50);
+    for (const chunk of chunks) {
+      this.textQueue.push(chunk);
+      this.textBuffer = this.textBuffer.slice(chunk.length);
+      this.generateAudio();
+    }
+  }
+  generateAudio() {
+    if (this.generatingAudio) return;
+    const chunk = this.textQueue.shift();
+    if (!chunk) return;
+    this.generatingAudio = true;
+    this.tts.input(chunk);
+  }
+  sendAudio() {
+    if (this.audioBuffer.length === 0) return;
+    const concatenatedBuffer = Buffer.concat(this.audioBuffer);
+    const resampledPCM = resamplePcm(
+      bufferToInt16Array(concatenatedBuffer),
+      uniformSampleRate,
+      this.outputFormat.sampleRate
+    );
+    const encodedAudio = encodePcm(resampledPCM, this.outputFormat.encoding);
+    this.audioBuffer = [];
+    try {
+      this.onAudioOutput(encodedAudio);
+    } catch (error) {
+      console.error("Audio output callback error:", error);
+    }
+  }
+  kill() {
+    this.stt.kill();
+    this.tts.kill();
+    this.audioBuffer = [];
+    this.textQueue = [];
+    this.textBuffer = "";
+    this.generatingAudio = false;
+  }
+};
+
+// src/voice/speechToText/base.ts
+var MagmaFlowSpeechToText = class {
+  onSpeechDetected() {
+    console.log(`[Default STT] Speech detected`);
+  }
+  onOutput(text) {
+    console.log(`[Default STT] Output: ${text}`);
+  }
+  constructor() {
+  }
+};
+var kKeepAliveInterval = 5e3;
+var DeepgramModel = /* @__PURE__ */ ((DeepgramModel2) => {
+  DeepgramModel2["NOVA_3"] = "nova-3";
+  return DeepgramModel2;
+})(DeepgramModel || {});
+var DeepgramLanguage = /* @__PURE__ */ ((DeepgramLanguage2) => {
+  DeepgramLanguage2["EN_US"] = "en-US";
+  return DeepgramLanguage2;
+})(DeepgramLanguage || {});
+var DeepgramSTT = class extends MagmaFlowSpeechToText {
+  constructor(args) {
+    super();
+    this.connection = null;
+    this.textBuffer = "";
+    this.config = {
+      model: args.model,
+      vad_events: true,
+      interim_results: true,
+      encoding: "linear16",
+      sample_rate: 48e3,
+      channels: 1,
+      ...args.config
+    };
+    this.client = args.client ?? new sdk.DeepgramClient({
+      key: process.env.DEEPGRAM_API_KEY
+    });
+  }
+  setup() {
+    this.connection = this.client.listen.live(this.config);
+    this.connection.on(sdk.LiveTranscriptionEvents.Error, (event) => {
+      console.log(`[Deepgram] Error: ${JSON.stringify(event)}`);
+    });
+    this.connection.on(sdk.LiveTranscriptionEvents.Close, (event) => {
+      console.log(`[Deepgram] Close: ${JSON.stringify(event)}`);
+    });
+    this.connection.on(sdk.LiveTranscriptionEvents.Open, this.onOpen.bind(this));
+    this.connection.on(sdk.LiveTranscriptionEvents.Unhandled, (event) => {
+      console.log(`[Deepgram] Unhandled event: ${JSON.stringify(event)}`);
+    });
+    this.connection.on(
+      sdk.LiveTranscriptionEvents.Transcript,
+      this.handleTranscriptionEvent.bind(this)
+    );
+    this.connection.on(sdk.LiveTranscriptionEvents.UtteranceEnd, (event) => {
+      console.log(`[Deepgram] Utterance end: ${JSON.stringify(event)}`);
+    });
+  }
+  input(audio) {
+    if (!this.connection) {
+      this.setup();
+      return this.input(audio);
+    }
+    this.connection?.send(audio.buffer);
+  }
+  flush() {
+    this.connection?.finalize();
+  }
+  kill() {
+    this.connection?.requestClose();
+    this.connection = null;
+  }
+  handleTranscriptionEvent(transcriptionEvent) {
+    const transcriptOption = transcriptionEvent.channel.alternatives[0];
+    if (transcriptOption.transcript.trim() === "") {
+      return;
+    }
+    this.onSpeechDetected();
+    if (transcriptionEvent.is_final || transcriptionEvent.speech_final || transcriptionEvent.from_finalize) {
+      const confidencePct = Math.round(transcriptOption.confidence * 100);
+      const text = `[transcription confidence=${confidencePct}%]: ${transcriptOption.transcript}`;
+      this.textBuffer += text + " ";
+      if (transcriptionEvent.speech_final) {
+        this.onOutput(this.textBuffer);
+        this.textBuffer = "";
+      }
+    }
+  }
+  onOpen() {
+    console.log(`[Deepgram] Connected`);
+    this.keepAlive();
+  }
+  keepAlive() {
+    setTimeout(() => {
+      if (this.connection?.isConnected()) {
+        this.connection.keepAlive();
+        this.keepAlive();
+      } else {
+        return;
+      }
+    }, kKeepAliveInterval);
+  }
+};
+
+// src/voice/textToSpeech/base.ts
+var MagmaFlowTextToSpeech = class {
+  onOutput(audio) {
+    console.log("[Default TTS] Output:", audio);
+  }
+  constructor() {
+  }
+};
+var DeepgramTTS = class extends MagmaFlowTextToSpeech {
+  constructor(args) {
+    super();
+    this.client = args.client ?? new sdk.DeepgramClient({ key: process.env.DEEPGRAM_API_KEY });
+  }
+  async setup() {
+  }
+  input(text) {
+    if (!text) {
+      return;
+    }
+    this.client.speak.request(
+      {
+        text
+      },
+      {
+        sample_rate: 48e3,
+        encoding: "linear16",
+        model: "aura-2-thalia-en",
+        container: "none"
+      }
+    ).then(async (response) => {
+      const stream = await response.getStream();
+      if (!stream) {
+        return;
+      }
+      for await (const chunk of stream) {
+        this.output(Buffer.from(chunk));
+      }
+      this.output(null);
+      console.log("[Deepgram] Finished:", text);
+    });
+  }
+  output(audio) {
+    this.onOutput(audio);
+  }
+  kill() {
+  }
+  reset() {
+  }
+};
+var ElevenVoice = /* @__PURE__ */ ((ElevenVoice2) => {
+  ElevenVoice2["chris"] = "iP95p4xoKVk53GoZ742B";
+  ElevenVoice2["josh"] = "TxGEqnHWrfWFTfGW9XjX";
+  ElevenVoice2["rachel"] = "21m00Tcm4TlvDq8ikWAM";
+  ElevenVoice2["laura"] = "FGY2WhTYpPnrIDTdsKH5";
+  ElevenVoice2["felicity"] = "aTbnroHRGIomiKpqAQR8";
+  return ElevenVoice2;
+})(ElevenVoice || {});
+var ElevenLabsTTS = class extends MagmaFlowTextToSpeech {
+  constructor(args) {
+    super();
+    this.client = args.client ?? new elevenlabsJs.ElevenLabsClient({
+      apiKey: process.env.ELEVENLABS_API_KEY
+    });
+    this.model = args.model;
+    this.voice = args.voice;
+    this.config = args.config ?? {};
+  }
+  async setup() {
+  }
+  input(text) {
+    if (!text) {
+      return;
+    }
+    this.client.textToSpeech.stream(this.voice, {
+      text,
+      outputFormat: "pcm_48000",
+      modelId: this.model,
+      ...this.config
+    }).then(async (stream) => {
+      for await (const chunk of stream) {
+        this.output(chunk);
+      }
+      this.output(null);
+      console.log("[ElevenLabs] Finished:", text);
+    });
+  }
+  output(audio) {
+    this.onOutput(audio);
+  }
+  kill() {
+  }
+  reset() {
+  }
+};
+var HumeTTS = class extends MagmaFlowTextToSpeech {
+  constructor(args) {
+    super();
+    this.client = args.client ?? new hume.HumeClient({ apiKey: process.env.HUME_API_KEY });
+  }
+  async setup() {
+  }
+  input(text) {
+    if (!text) {
+      return;
+    }
+    this.client.tts.synthesizeJsonStreaming({
+      utterances: [
+        {
+          text
+        }
+      ],
+      format: {
+        type: "pcm"
+      },
+      instantMode: true
+    }).then(async (stream) => {
+      for await (const chunk of stream) {
+        this.output(Buffer.from(chunk.audio, "base64"));
+      }
+      this.output(null);
+      console.log("[Hume] Finished:", text);
+    });
+  }
+  output(audio) {
+    this.onOutput(audio);
+  }
+  kill() {
+  }
+  reset() {
+  }
+};
+var WhisperTTS = class extends MagmaFlowTextToSpeech {
+  constructor(args) {
+    super();
+    this.client = args.client ?? new OpenAI__default.default({ apiKey: process.env.OPENAI_API_KEY });
+  }
+  async setup() {
+  }
+  input(text) {
+    if (!text) {
+      return;
+    }
+    this.client.audio.speech.create({
+      model: "gpt-4o-mini-tts",
+      voice: "alloy",
+      input: text,
+      response_format: "pcm"
+    }).then(async (res) => {
+      const result = await res.arrayBuffer();
+      const resampledPCM = resamplePcm(
+        bufferToInt16Array(Buffer.from(result)),
+        24e3,
+        48e3
+      );
+      this.output(int16ArrayToBuffer(resampledPCM));
+      this.output(null);
+      console.log("[Whisper] Finished:", text);
+    });
+  }
+  output(audio) {
+    this.onOutput(audio);
+  }
+  kill() {
+  }
+  reset() {
+  }
+};
+
+exports.DeepgramLanguage = DeepgramLanguage;
+exports.DeepgramModel = DeepgramModel;
+exports.DeepgramSTT = DeepgramSTT;
+exports.DeepgramTTS = DeepgramTTS;
+exports.ElevenLabsTTS = ElevenLabsTTS;
+exports.ElevenVoice = ElevenVoice;
+exports.HumeTTS = HumeTTS;
+exports.MagmaFlow = MagmaFlow;
+exports.MagmaFlowSpeechToText = MagmaFlowSpeechToText;
+exports.MagmaFlowTextToSpeech = MagmaFlowTextToSpeech;
+exports.WhisperTTS = WhisperTTS;
+exports.splitTextIntoChunks = splitTextIntoChunks;