@tensamin/audio 0.1.1 → 0.1.3

package/dist/livekit/integration.js

@@ -0,0 +1,585 @@
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/livekit/integration.ts
+var integration_exports = {};
+__export(integration_exports, {
+  attachProcessingToTrack: () => attachProcessingToTrack
+});
+module.exports = __toCommonJS(integration_exports);
+
+// src/pipeline/audio-pipeline.ts
+var import_mitt = __toESM(require("mitt"));
+
+// src/context/audio-context.ts
+var sharedContext = null;
+var activePipelines = 0;
+function getAudioContext(options) {
+  if (typeof window === "undefined" || typeof AudioContext === "undefined") {
+    throw new Error(
+      "AudioContext is not supported in this environment (browser only)."
+    );
+  }
+  if (!sharedContext || sharedContext.state === "closed") {
+    sharedContext = new AudioContext(options);
+  }
+  return sharedContext;
+}
+function registerPipeline() {
+  activePipelines++;
+}
+function unregisterPipeline() {
+  activePipelines = Math.max(0, activePipelines - 1);
+}
+
+// src/noise-suppression/rnnoise-node.ts
+var RNNoisePlugin = class {
+  name = "rnnoise-ns";
+  wasmBuffer = null;
+  async createNode(context, config) {
+    const { loadRnnoise, RnnoiseWorkletNode } = await import("@sapphi-red/web-noise-suppressor");
+    if (!config?.enabled) {
+      console.log("Noise suppression disabled, using passthrough node");
+      const pass = context.createGain();
+      return pass;
+    }
+    if (!config?.wasmUrl || !config?.simdUrl || !config?.workletUrl) {
+      const error = new Error(
+        `RNNoisePlugin requires 'wasmUrl', 'simdUrl', and 'workletUrl' to be configured. Please download the assets from @sapphi-red/web-noise-suppressor and provide the URLs in the config. Current config: wasmUrl=${config?.wasmUrl}, simdUrl=${config?.simdUrl}, workletUrl=${config?.workletUrl}
+To disable noise suppression, set noiseSuppression.enabled to false.`
+      );
+      console.error(error.message);
+      throw error;
+    }
+    try {
+      if (!this.wasmBuffer) {
+        console.log("Loading RNNoise WASM binary...");
+        this.wasmBuffer = await loadRnnoise({
+          url: config.wasmUrl,
+          simdUrl: config.simdUrl
+        });
+        console.log("RNNoise WASM loaded successfully");
+      }
+    } catch (error) {
+      const err = new Error(
+        `Failed to load RNNoise WASM binary: ${error instanceof Error ? error.message : String(error)}`
+      );
+      console.error(err);
+      throw err;
+    }
+    const workletUrl = config.workletUrl;
+    try {
+      await context.audioWorklet.addModule(workletUrl);
+      console.log("RNNoise worklet loaded successfully");
+    } catch (e) {
+      const error = new Error(
+        `Failed to load RNNoise worklet from ${workletUrl}: ${e instanceof Error ? e.message : String(e)}. Ensure the workletUrl points to a valid RNNoise worklet script.`
+      );
+      console.error(error.message);
+      throw error;
+    }
+    try {
+      const node = new RnnoiseWorkletNode(context, {
+        wasmBinary: this.wasmBuffer,
+        maxChannels: 1
+        // Mono for now
+      });
+      console.log("RNNoise worklet node created successfully");
+      return node;
+    } catch (error) {
+      const err = new Error(
+        `Failed to create RNNoise worklet node: ${error instanceof Error ? error.message : String(error)}`
+      );
+      console.error(err);
+      throw err;
+    }
+  }
+};
+
+// src/vad/vad-node.ts
+var energyVadWorkletCode = `
+class EnergyVadProcessor extends AudioWorkletProcessor {
+  constructor() {
+    super();
+    this.smoothing = 0.95;
+    this.energy = 0;
+    this.noiseFloor = 0.001;
+  }
+
+  process(inputs, outputs, parameters) {
+    const input = inputs[0];
+    if (!input || !input.length) return true;
+    const channel = input[0];
+    
+    // Calculate RMS
+    let sum = 0;
+    for (let i = 0; i < channel.length; i++) {
+      sum += channel[i] * channel[i];
+    }
+    const rms = Math.sqrt(sum / channel.length);
+
+    // Simple adaptive noise floor (very basic)
+    if (rms < this.noiseFloor) {
+      this.noiseFloor = this.noiseFloor * 0.99 + rms * 0.01;
+    } else {
+      this.noiseFloor = this.noiseFloor * 0.999 + rms * 0.001;
+    }
+
+    // Calculate "probability" based on SNR
+    // This is a heuristic mapping from energy to 0-1
+    const snr = rms / (this.noiseFloor + 1e-6);
+    const probability = Math.min(1, Math.max(0, (snr - 1.5) / 10)); // Arbitrary scaling
+
+    this.port.postMessage({ probability });
+
+    return true;
+  }
+}
+registerProcessor('energy-vad-processor', EnergyVadProcessor);
+`;
+var EnergyVADPlugin = class {
+  name = "energy-vad";
+  async createNode(context, config, onDecision) {
+    if (!config?.enabled) {
+      console.log("VAD disabled, using passthrough node");
+      const pass = context.createGain();
+      return pass;
+    }
+    const blob = new Blob([energyVadWorkletCode], {
+      type: "application/javascript"
+    });
+    const url = URL.createObjectURL(blob);
+    try {
+      await context.audioWorklet.addModule(url);
+      console.log("Energy VAD worklet loaded successfully");
+    } catch (e) {
+      const error = new Error(
+        `Failed to load Energy VAD worklet: ${e instanceof Error ? e.message : String(e)}`
+      );
+      console.error(error.message);
+      URL.revokeObjectURL(url);
+      throw error;
+    }
+    URL.revokeObjectURL(url);
+    let node;
+    try {
+      node = new AudioWorkletNode(context, "energy-vad-processor");
+      console.log("Energy VAD node created successfully");
+    } catch (e) {
+      const error = new Error(
+        `Failed to create Energy VAD node: ${e instanceof Error ? e.message : String(e)}`
+      );
+      console.error(error.message);
+      throw error;
+    }
+    node.port.onmessage = (event) => {
+      try {
+        const { probability } = event.data;
+        if (typeof probability === "number" && !isNaN(probability)) {
+          onDecision(probability);
+        } else {
+          console.warn("Invalid VAD probability received:", event.data);
+        }
+      } catch (error) {
+        console.error("Error in VAD message handler:", error);
+      }
+    };
+    node.port.onmessageerror = (event) => {
+      console.error("VAD port message error:", event);
+    };
+    return node;
+  }
+};
+
+// src/extensibility/plugins.ts
+var nsPlugins = /* @__PURE__ */ new Map();
+var vadPlugins = /* @__PURE__ */ new Map();
+var defaultNs = new RNNoisePlugin();
+nsPlugins.set(defaultNs.name, defaultNs);
+var defaultVad = new EnergyVADPlugin();
+vadPlugins.set(defaultVad.name, defaultVad);
+function getNoiseSuppressionPlugin(name) {
+  if (!name) return defaultNs;
+  const plugin = nsPlugins.get(name);
+  if (!plugin) {
+    console.warn(
+      `Noise suppression plugin '${name}' not found, falling back to default.`
+    );
+    return defaultNs;
+  }
+  return plugin;
+}
+function getVADPlugin(name) {
+  if (!name) return defaultVad;
+  const plugin = vadPlugins.get(name);
+  if (!plugin) {
+    console.warn(`VAD plugin '${name}' not found, falling back to default.`);
+    return defaultVad;
+  }
+  return plugin;
+}
+
+// src/vad/vad-state.ts
+var VADStateMachine = class {
+  config;
+  currentState = "silent";
+  lastSpeechTime = 0;
+  speechStartTime = 0;
+  frameDurationMs = 20;
+  // Assumed frame duration, updated by calls
+  constructor(config) {
+    this.config = {
+      enabled: config?.enabled ?? true,
+      pluginName: config?.pluginName ?? "energy-vad",
+      startThreshold: config?.startThreshold ?? 0.5,
+      stopThreshold: config?.stopThreshold ?? 0.4,
+      hangoverMs: config?.hangoverMs ?? 300,
+      preRollMs: config?.preRollMs ?? 200
+    };
+  }
+  updateConfig(config) {
+    this.config = { ...this.config, ...config };
+  }
+  processFrame(probability, timestamp) {
+    const { startThreshold, stopThreshold, hangoverMs } = this.config;
+    let newState = this.currentState;
+    if (this.currentState === "silent" || this.currentState === "speech_ending") {
+      if (probability >= startThreshold) {
+        newState = "speech_starting";
+        this.speechStartTime = timestamp;
+        this.lastSpeechTime = timestamp;
+      } else {
+        newState = "silent";
+      }
+    } else if (this.currentState === "speech_starting" || this.currentState === "speaking") {
+      if (probability >= stopThreshold) {
+        newState = "speaking";
+        this.lastSpeechTime = timestamp;
+      } else {
+        const timeSinceSpeech = timestamp - this.lastSpeechTime;
+        if (timeSinceSpeech < hangoverMs) {
+          newState = "speaking";
+        } else {
+          newState = "speech_ending";
+        }
+      }
+    }
+    if (newState === "speech_starting") newState = "speaking";
+    if (newState === "speech_ending") newState = "silent";
+    this.currentState = newState;
+    return {
+      isSpeaking: newState === "speaking",
+      probability,
+      state: newState
+    };
+  }
+};
+
+// src/pipeline/audio-pipeline.ts
+async function createAudioPipeline(sourceTrack, config = {}) {
+  const context = getAudioContext();
+  registerPipeline();
+  const nsEnabled = config.noiseSuppression?.enabled !== false && Boolean(config.noiseSuppression?.wasmUrl && config.noiseSuppression?.simdUrl && config.noiseSuppression?.workletUrl);
+  const vadEnabled = config.vad?.enabled !== false;
+  const fullConfig = {
+    noiseSuppression: {
+      enabled: nsEnabled,
+      ...config.noiseSuppression
+    },
+    vad: {
+      enabled: vadEnabled,
+      ...config.vad
+    },
+    output: {
+      speechGain: 1,
+      silenceGain: vadEnabled ? 0 : 1,
+      // If no VAD, always output audio
+      gainRampTime: 0.02,
+      ...config.output
+    },
+    livekit: { manageTrackMute: false, ...config.livekit }
+  };
+  console.log("Audio pipeline config:", {
+    noiseSuppression: fullConfig.noiseSuppression?.enabled,
+    vad: fullConfig.vad?.enabled,
+    output: fullConfig.output
+  });
+  if (!sourceTrack || sourceTrack.kind !== "audio") {
+    throw new Error("createAudioPipeline requires a valid audio MediaStreamTrack");
+  }
+  if (sourceTrack.readyState === "ended") {
+    throw new Error("Cannot create pipeline from an ended MediaStreamTrack");
+  }
+  const sourceStream = new MediaStream([sourceTrack]);
+  const sourceNode = context.createMediaStreamSource(sourceStream);
+  let nsNode;
+  let vadNode;
+  const emitter = (0, import_mitt.default)();
+  try {
+    const nsPlugin = getNoiseSuppressionPlugin(
+      fullConfig.noiseSuppression?.pluginName
+    );
+    nsNode = await nsPlugin.createNode(
+      context,
+      fullConfig.noiseSuppression
+    );
+  } catch (error) {
+    const err = error instanceof Error ? error : new Error(String(error));
+    console.error("Failed to create noise suppression node:", err);
+    emitter.emit("error", err);
+    throw err;
+  }
+  const vadStateMachine = new VADStateMachine(fullConfig.vad);
+  try {
+    const vadPlugin = getVADPlugin(fullConfig.vad?.pluginName);
+    vadNode = await vadPlugin.createNode(
+      context,
+      fullConfig.vad,
+      (prob) => {
+        try {
+          const timestamp = context.currentTime * 1e3;
+          const newState = vadStateMachine.processFrame(prob, timestamp);
+          if (newState.state !== lastVadState.state || Math.abs(newState.probability - lastVadState.probability) > 0.1) {
+            emitter.emit("vadChange", newState);
+            lastVadState = newState;
+            updateGain(newState);
+          }
+        } catch (vadError) {
+          const err = vadError instanceof Error ? vadError : new Error(String(vadError));
+          console.error("Error in VAD callback:", err);
+          emitter.emit("error", err);
+        }
+      }
+    );
+  } catch (error) {
+    const err = error instanceof Error ? error : new Error(String(error));
+    console.error("Failed to create VAD node:", err);
+    emitter.emit("error", err);
+    throw err;
+  }
+  let lastVadState = {
+    isSpeaking: false,
+    probability: 0,
+    state: "silent"
+  };
+  const splitter = context.createGain();
+  sourceNode.connect(nsNode);
+  nsNode.connect(splitter);
+  splitter.connect(vadNode);
+  const delayNode = context.createDelay(1);
+  const preRollSeconds = (fullConfig.vad?.preRollMs ?? 200) / 1e3;
+  delayNode.delayTime.value = preRollSeconds;
+  const gainNode = context.createGain();
+  gainNode.gain.value = fullConfig.output?.silenceGain ?? 0;
+  const destination = context.createMediaStreamDestination();
+  try {
+    splitter.connect(delayNode);
+    delayNode.connect(gainNode);
+    gainNode.connect(destination);
+  } catch (error) {
+    const err = error instanceof Error ? error : new Error(String(error));
+    console.error("Failed to wire audio pipeline:", err);
+    emitter.emit("error", err);
+    throw err;
+  }
+  function updateGain(state) {
+    try {
+      const { speechGain, silenceGain, gainRampTime } = fullConfig.output;
+      const targetGain = state.isSpeaking ? speechGain ?? 1 : silenceGain ?? 0;
+      const now = context.currentTime;
+      gainNode.gain.setTargetAtTime(targetGain, now, gainRampTime ?? 0.02);
+    } catch (error) {
+      const err = error instanceof Error ? error : new Error(String(error));
+      console.error("Failed to update gain:", err);
+      emitter.emit("error", err);
+    }
+  }
+  const audioTracks = destination.stream.getAudioTracks();
+  console.log("Destination stream tracks:", {
+    count: audioTracks.length,
+    tracks: audioTracks.map((t) => ({
+      id: t.id,
+      label: t.label,
+      enabled: t.enabled,
+      readyState: t.readyState
+    }))
+  });
+  if (audioTracks.length === 0) {
+    const err = new Error(
+      "Failed to create processed audio track: destination stream has no audio tracks. This may indicate an issue with the audio graph connection."
+    );
+    console.error(err);
+    emitter.emit("error", err);
+    throw err;
+  }
+  const processedTrack = audioTracks[0];
+  if (!processedTrack || processedTrack.readyState === "ended") {
+    const err = new Error("Processed audio track is invalid or ended");
+    console.error(err);
+    emitter.emit("error", err);
+    throw err;
+  }
+  console.log("Audio pipeline created successfully:", {
+    sourceTrack: {
+      id: sourceTrack.id,
+      label: sourceTrack.label,
+      readyState: sourceTrack.readyState
+    },
+    processedTrack: {
+      id: processedTrack.id,
+      label: processedTrack.label,
+      readyState: processedTrack.readyState
+    },
+    config: {
+      noiseSuppression: fullConfig.noiseSuppression?.enabled,
+      vad: fullConfig.vad?.enabled
+    }
+  });
+  function dispose() {
+    try {
+      sourceNode.disconnect();
+      nsNode.disconnect();
+      splitter.disconnect();
+      vadNode.disconnect();
+      delayNode.disconnect();
+      gainNode.disconnect();
+      destination.stream.getTracks().forEach((t) => t.stop());
+      unregisterPipeline();
+    } catch (error) {
+      console.error("Error during pipeline disposal:", error);
+    }
+  }
+  return {
+    processedTrack,
+    events: emitter,
+    get state() {
+      return lastVadState;
+    },
+    setConfig: (newConfig) => {
+      try {
+        if (newConfig.vad) {
+          vadStateMachine.updateConfig(newConfig.vad);
+        }
+      } catch (error) {
+        const err = error instanceof Error ? error : new Error(String(error));
+        console.error("Failed to update config:", err);
+        emitter.emit("error", err);
+      }
+    },
+    dispose
+  };
+}
+
+// src/livekit/integration.ts
+async function attachProcessingToTrack(track, config = {}) {
+  if (!track) {
+    throw new Error("attachProcessingToTrack requires a valid LocalAudioTrack");
+  }
+  const originalTrack = track.mediaStreamTrack;
+  if (!originalTrack) {
+    throw new Error("LocalAudioTrack has no underlying MediaStreamTrack");
+  }
+  if (originalTrack.readyState === "ended") {
+    throw new Error("Cannot attach processing to an ended MediaStreamTrack");
+  }
+  let pipeline;
+  try {
+    console.log("Creating audio processing pipeline...");
+    pipeline = await createAudioPipeline(originalTrack, config);
+    console.log("Audio processing pipeline created successfully");
+  } catch (error) {
+    const err = new Error(
+      `Failed to create audio pipeline: ${error instanceof Error ? error.message : String(error)}`
+    );
+    console.error(err);
+    throw err;
+  }
+  if (!pipeline.processedTrack) {
+    throw new Error("Pipeline did not return a processed track");
+  }
+  try {
+    console.log("Replacing LiveKit track with processed track...");
+    await track.replaceTrack(pipeline.processedTrack);
+    console.log("LiveKit track replaced successfully");
+  } catch (error) {
+    pipeline.dispose();
+    const err = new Error(
+      `Failed to replace LiveKit track: ${error instanceof Error ? error.message : String(error)}`
+    );
+    console.error(err);
+    throw err;
+  }
+  if (config.livekit?.manageTrackMute) {
+    let isVadMuted = false;
+    pipeline.events.on("vadChange", async (state) => {
+      try {
+        if (state.isSpeaking) {
+          if (isVadMuted) {
+            await track.unmute();
+            isVadMuted = false;
+          }
+        } else {
+          if (!track.isMuted) {
+            await track.mute();
+            isVadMuted = true;
+          }
+        }
+      } catch (error) {
+        console.error("Error handling VAD-based track muting:", error);
+      }
+    });
+  }
+  pipeline.events.on("error", (error) => {
+    console.error("Audio pipeline error:", error);
+  });
+  const originalDispose = pipeline.dispose;
+  pipeline.dispose = () => {
+    try {
+      if (originalTrack.readyState === "live") {
+        console.log("Restoring original track...");
+        track.replaceTrack(originalTrack).catch((error) => {
+          console.error("Failed to restore original track:", error);
+        });
+      }
+      originalDispose();
+    } catch (error) {
+      console.error("Error during pipeline disposal:", error);
+      try {
+        originalDispose();
+      } catch (disposeError) {
+        console.error("Error calling original dispose:", disposeError);
+      }
+    }
+  };
+  return pipeline;
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  attachProcessingToTrack
+});

package/dist/livekit/integration.mjs

@@ -0,0 +1,12 @@
+import {
+  attachProcessingToTrack
+} from "../chunk-XMTQPMQ6.mjs";
+import "../chunk-EXH2PNUE.mjs";
+import "../chunk-JJASCVEW.mjs";
+import "../chunk-OZ7KMC4S.mjs";
+import "../chunk-6P2RDBW5.mjs";
+import "../chunk-XO6B3D4A.mjs";
+import "../chunk-R5JVHKWA.mjs";
+export {
+  attachProcessingToTrack
+};

package/dist/noise-suppression/rnnoise-node.d.mts

@@ -0,0 +1,10 @@
+import { NoiseSuppressionPlugin, AudioProcessingConfig } from '../types.mjs';
+import 'mitt';
+
+declare class RNNoisePlugin implements NoiseSuppressionPlugin {
+    name: string;
+    private wasmBuffer;
+    createNode(context: AudioContext, config: AudioProcessingConfig["noiseSuppression"]): Promise<AudioNode>;
+}
+
+export { RNNoisePlugin };

package/dist/noise-suppression/rnnoise-node.d.ts

@@ -0,0 +1,10 @@
+import { NoiseSuppressionPlugin, AudioProcessingConfig } from '../types.js';
+import 'mitt';
+
+declare class RNNoisePlugin implements NoiseSuppressionPlugin {
+    name: string;
+    private wasmBuffer;
+    createNode(context: AudioContext, config: AudioProcessingConfig["noiseSuppression"]): Promise<AudioNode>;
+}
+
+export { RNNoisePlugin };