@tensamin/audio 0.1.0 → 0.1.2

package/README.md

@@ -18,6 +18,18 @@ bun add @tensamin/audio livekit-client
 pnpm install @tensamin/audio livekit-client
 ```
 
+## Setup Assets
+
+This library uses WASM and AudioWorklets for processing. You must manually download these assets and serve them from your application (e.g., in your `public/` folder).
+
+1.  Download the following files from `https://unpkg.com/@sapphi-red/web-noise-suppressor@0.3.5/dist/`:
+
+- `rnnoise.wasm`
+- `rnnoise_simd.wasm`
+- `noise-suppressor-worklet.min.js`
+
+2.  Place them in your project's public directory (e.g., `public/audio-processor/`).
+
 ## Usage
 
 ### Basic Usage (Raw MediaStream)
@@ -31,7 +43,12 @@ const track = stream.getAudioTracks()[0];
 
 // Create pipeline
 const pipeline = await createAudioPipeline(track, {
-  noiseSuppression: { enabled: true },
+  noiseSuppression: {
+    enabled: true,
+    wasmUrl: "/audio-processor/rnnoise.wasm",
+    simdUrl: "/audio-processor/rnnoise_simd.wasm",
+    workletUrl: "/audio-processor/noise-suppressor-worklet.min.js",
+  },
   vad: { enabled: true },
 });
 
@@ -56,7 +73,12 @@ const localTrack = await LocalAudioTrack.create();
 
 // Attach processing (replaces the underlying track)
 const pipeline = await attachProcessingToTrack(localTrack, {
-  noiseSuppression: { enabled: true },
+  noiseSuppression: {
+    enabled: true,
+    wasmUrl: "/audio-processor/rnnoise.wasm",
+    simdUrl: "/audio-processor/rnnoise_simd.wasm",
+    workletUrl: "/audio-processor/noise-suppressor-worklet.min.js",
+  },
   vad: { enabled: true },
   livekit: { manageTrackMute: true }, // Optional: mute the track object itself
 });

package/dist/chunk-FS635GMR.mjs

@@ -0,0 +1,47 @@
+import {
+  RNNoisePlugin
+} from "./chunk-SDTOKWM2.mjs";
+import {
+  EnergyVADPlugin
+} from "./chunk-UMU2KIB6.mjs";
+
+// src/extensibility/plugins.ts
+var nsPlugins = /* @__PURE__ */ new Map();
+var vadPlugins = /* @__PURE__ */ new Map();
+var defaultNs = new RNNoisePlugin();
+nsPlugins.set(defaultNs.name, defaultNs);
+var defaultVad = new EnergyVADPlugin();
+vadPlugins.set(defaultVad.name, defaultVad);
+function registerNoiseSuppressionPlugin(plugin) {
+  nsPlugins.set(plugin.name, plugin);
+}
+function registerVADPlugin(plugin) {
+  vadPlugins.set(plugin.name, plugin);
+}
+function getNoiseSuppressionPlugin(name) {
+  if (!name) return defaultNs;
+  const plugin = nsPlugins.get(name);
+  if (!plugin) {
+    console.warn(
+      `Noise suppression plugin '${name}' not found, falling back to default.`
+    );
+    return defaultNs;
+  }
+  return plugin;
+}
+function getVADPlugin(name) {
+  if (!name) return defaultVad;
+  const plugin = vadPlugins.get(name);
+  if (!plugin) {
+    console.warn(`VAD plugin '${name}' not found, falling back to default.`);
+    return defaultVad;
+  }
+  return plugin;
+}
+
+export {
+  registerNoiseSuppressionPlugin,
+  registerVADPlugin,
+  getNoiseSuppressionPlugin,
+  getVADPlugin
+};

package/dist/chunk-HFSKQ33X.mjs

@@ -0,0 +1,38 @@
+import {
+  createAudioPipeline
+} from "./chunk-QU7E5HBA.mjs";
+
+// src/livekit/integration.ts
+async function attachProcessingToTrack(track, config = {}) {
+  const originalTrack = track.mediaStreamTrack;
+  const pipeline = await createAudioPipeline(originalTrack, config);
+  await track.replaceTrack(pipeline.processedTrack);
+  if (config.livekit?.manageTrackMute) {
+    let isVadMuted = false;
+    pipeline.events.on("vadChange", async (state) => {
+      if (state.isSpeaking) {
+        if (isVadMuted) {
+          await track.unmute();
+          isVadMuted = false;
+        }
+      } else {
+        if (!track.isMuted) {
+          await track.mute();
+          isVadMuted = true;
+        }
+      }
+    });
+  }
+  const originalDispose = pipeline.dispose;
+  pipeline.dispose = () => {
+    if (originalTrack.readyState === "live") {
+      track.replaceTrack(originalTrack).catch(console.error);
+    }
+    originalDispose();
+  };
+  return pipeline;
+}
+
+export {
+  attachProcessingToTrack
+};

package/{src/vad/vad-state.ts → dist/chunk-JJASCVEW.mjs}

@@ -1,36 +1,28 @@
-import type { AudioProcessingConfig, VADState } from "../types.js";
-
-export class VADStateMachine {
-  private config: Required<NonNullable<AudioProcessingConfig["vad"]>>;
-  private currentState: VADState["state"] = "silent";
-  private lastSpeechTime = 0;
-  private speechStartTime = 0;
-  private frameDurationMs = 20; // Assumed frame duration, updated by calls
-
-  constructor(config: AudioProcessingConfig["vad"]) {
+// src/vad/vad-state.ts
+var VADStateMachine = class {
+  config;
+  currentState = "silent";
+  lastSpeechTime = 0;
+  speechStartTime = 0;
+  frameDurationMs = 20;
+  // Assumed frame duration, updated by calls
+  constructor(config) {
     this.config = {
       enabled: config?.enabled ?? true,
       pluginName: config?.pluginName ?? "energy-vad",
       startThreshold: config?.startThreshold ?? 0.5,
       stopThreshold: config?.stopThreshold ?? 0.4,
       hangoverMs: config?.hangoverMs ?? 300,
-      preRollMs: config?.preRollMs ?? 200,
+      preRollMs: config?.preRollMs ?? 200
     };
   }
-
-  updateConfig(config: Partial<AudioProcessingConfig["vad"]>) {
+  updateConfig(config) {
     this.config = { ...this.config, ...config };
   }
-
-  processFrame(probability: number, timestamp: number): VADState {
+  processFrame(probability, timestamp) {
     const { startThreshold, stopThreshold, hangoverMs } = this.config;
-
     let newState = this.currentState;
-
-    if (
-      this.currentState === "silent" ||
-      this.currentState === "speech_ending"
-    ) {
+    if (this.currentState === "silent" || this.currentState === "speech_ending") {
       if (probability >= startThreshold) {
         newState = "speech_starting";
         this.speechStartTime = timestamp;
@@ -38,34 +30,30 @@ export class VADStateMachine {
       } else {
         newState = "silent";
       }
-    } else if (
-      this.currentState === "speech_starting" ||
-      this.currentState === "speaking"
-    ) {
+    } else if (this.currentState === "speech_starting" || this.currentState === "speaking") {
       if (probability >= stopThreshold) {
         newState = "speaking";
         this.lastSpeechTime = timestamp;
       } else {
-        // Check hangover
         const timeSinceSpeech = timestamp - this.lastSpeechTime;
         if (timeSinceSpeech < hangoverMs) {
-          newState = "speaking"; // Still in hangover
+          newState = "speaking";
         } else {
           newState = "speech_ending";
         }
       }
     }
-
-    // Transition from starting/ending to stable states
     if (newState === "speech_starting") newState = "speaking";
     if (newState === "speech_ending") newState = "silent";
-
     this.currentState = newState;
-
     return {
       isSpeaking: newState === "speaking",
       probability,
-      state: newState,
+      state: newState
     };
   }
-}
+};
+
+export {
+  VADStateMachine
+};

package/dist/chunk-OZ7KMC4S.mjs

@@ -0,0 +1,46 @@
+// src/context/audio-context.ts
+var sharedContext = null;
+var activePipelines = 0;
+function getAudioContext(options) {
+  if (typeof window === "undefined" || typeof AudioContext === "undefined") {
+    throw new Error(
+      "AudioContext is not supported in this environment (browser only)."
+    );
+  }
+  if (!sharedContext || sharedContext.state === "closed") {
+    sharedContext = new AudioContext(options);
+  }
+  return sharedContext;
+}
+function registerPipeline() {
+  activePipelines++;
+}
+function unregisterPipeline() {
+  activePipelines = Math.max(0, activePipelines - 1);
+}
+async function resumeAudioContext() {
+  if (sharedContext && sharedContext.state === "suspended") {
+    await sharedContext.resume();
+  }
+}
+async function suspendAudioContext() {
+  if (sharedContext && sharedContext.state === "running") {
+    await sharedContext.suspend();
+  }
+}
+async function closeAudioContext() {
+  if (sharedContext && sharedContext.state !== "closed") {
+    await sharedContext.close();
+  }
+  sharedContext = null;
+  activePipelines = 0;
+}
+
+export {
+  getAudioContext,
+  registerPipeline,
+  unregisterPipeline,
+  resumeAudioContext,
+  suspendAudioContext,
+  closeAudioContext
+};

package/dist/chunk-QU7E5HBA.mjs

@@ -0,0 +1,106 @@
+import {
+  VADStateMachine
+} from "./chunk-JJASCVEW.mjs";
+import {
+  getAudioContext,
+  registerPipeline,
+  unregisterPipeline
+} from "./chunk-OZ7KMC4S.mjs";
+import {
+  getNoiseSuppressionPlugin,
+  getVADPlugin
+} from "./chunk-FS635GMR.mjs";
+
+// src/pipeline/audio-pipeline.ts
+import mitt from "mitt";
+async function createAudioPipeline(sourceTrack, config = {}) {
+  const context = getAudioContext();
+  registerPipeline();
+  const fullConfig = {
+    noiseSuppression: { enabled: true, ...config.noiseSuppression },
+    vad: { enabled: true, ...config.vad },
+    output: {
+      speechGain: 1,
+      silenceGain: 0,
+      gainRampTime: 0.02,
+      ...config.output
+    },
+    livekit: { manageTrackMute: false, ...config.livekit }
+  };
+  const sourceStream = new MediaStream([sourceTrack]);
+  const sourceNode = context.createMediaStreamSource(sourceStream);
+  const nsPlugin = getNoiseSuppressionPlugin(
+    fullConfig.noiseSuppression?.pluginName
+  );
+  const nsNode = await nsPlugin.createNode(
+    context,
+    fullConfig.noiseSuppression
+  );
+  const vadPlugin = getVADPlugin(fullConfig.vad?.pluginName);
+  const vadStateMachine = new VADStateMachine(fullConfig.vad);
+  const emitter = mitt();
+  const vadNode = await vadPlugin.createNode(
+    context,
+    fullConfig.vad,
+    (prob) => {
+      const timestamp = context.currentTime * 1e3;
+      const newState = vadStateMachine.processFrame(prob, timestamp);
+      if (newState.state !== lastVadState.state || Math.abs(newState.probability - lastVadState.probability) > 0.1) {
+        emitter.emit("vadChange", newState);
+        lastVadState = newState;
+        updateGain(newState);
+      }
+    }
+  );
+  let lastVadState = {
+    isSpeaking: false,
+    probability: 0,
+    state: "silent"
+  };
+  const splitter = context.createGain();
+  sourceNode.connect(nsNode);
+  nsNode.connect(splitter);
+  splitter.connect(vadNode);
+  const delayNode = context.createDelay(1);
+  const preRollSeconds = (fullConfig.vad?.preRollMs ?? 200) / 1e3;
+  delayNode.delayTime.value = preRollSeconds;
+  const gainNode = context.createGain();
+  gainNode.gain.value = fullConfig.output?.silenceGain ?? 0;
+  const destination = context.createMediaStreamDestination();
+  splitter.connect(delayNode);
+  delayNode.connect(gainNode);
+  gainNode.connect(destination);
+  function updateGain(state) {
+    const { speechGain, silenceGain, gainRampTime } = fullConfig.output;
+    const targetGain = state.isSpeaking ? speechGain ?? 1 : silenceGain ?? 0;
+    const now = context.currentTime;
+    gainNode.gain.setTargetAtTime(targetGain, now, gainRampTime ?? 0.02);
+  }
+  function dispose() {
+    sourceNode.disconnect();
+    nsNode.disconnect();
+    splitter.disconnect();
+    vadNode.disconnect();
+    delayNode.disconnect();
+    gainNode.disconnect();
+    destination.stream.getTracks().forEach((t) => t.stop());
+    unregisterPipeline();
+  }
+  return {
+    processedTrack: destination.stream.getAudioTracks()[0],
+    events: emitter,
+    get state() {
+      return lastVadState;
+    },
+    setConfig: (newConfig) => {
+      if (newConfig.vad) {
+        vadStateMachine.updateConfig(newConfig.vad);
+      }
+    },
+    dispose
+  };
+}
+
+export {
+  createAudioPipeline
+};

package/dist/chunk-SDTOKWM2.mjs

@@ -0,0 +1,39 @@
+// src/noise-suppression/rnnoise-node.ts
+var RNNoisePlugin = class {
+  name = "rnnoise-ns";
+  wasmBuffer = null;
+  async createNode(context, config) {
+    const { loadRnnoise, RnnoiseWorkletNode } = await import("@sapphi-red/web-noise-suppressor");
+    if (!config?.enabled) {
+      const pass = context.createGain();
+      return pass;
+    }
+    if (!config?.wasmUrl || !config?.simdUrl || !config?.workletUrl) {
+      throw new Error(
+        "RNNoisePlugin requires 'wasmUrl', 'simdUrl', and 'workletUrl' to be configured. Please download the assets and provide the URLs."
+      );
+    }
+    if (!this.wasmBuffer) {
+      this.wasmBuffer = await loadRnnoise({
+        url: config.wasmUrl,
+        simdUrl: config.simdUrl
+      });
+    }
+    const workletUrl = config.workletUrl;
+    try {
+      await context.audioWorklet.addModule(workletUrl);
+    } catch (e) {
+      console.warn("Failed to add RNNoise worklet module:", e);
+    }
+    const node = new RnnoiseWorkletNode(context, {
+      wasmBinary: this.wasmBuffer,
+      maxChannels: 1
+      // Mono for now
+    });
+    return node;
+  }
+};
+
+export {
+  RNNoisePlugin
+};

package/{src/vad/vad-node.ts → dist/chunk-UMU2KIB6.mjs}

@@ -1,7 +1,5 @@
-import type { AudioProcessingConfig, VADPlugin } from "../types.js";
-
-// Inline AudioWorklet processor for Energy VAD
-const energyVadWorkletCode = `
+// src/vad/vad-node.ts
+var energyVadWorkletCode = `
 class EnergyVadProcessor extends AudioWorkletProcessor {
   constructor() {
     super();
@@ -41,21 +39,13 @@ class EnergyVadProcessor extends AudioWorkletProcessor {
 }
 registerProcessor('energy-vad-processor', EnergyVadProcessor);
 `;
-
-export class EnergyVADPlugin implements VADPlugin {
+var EnergyVADPlugin = class {
   name = "energy-vad";
-
-  async createNode(
-    context: AudioContext,
-    config: AudioProcessingConfig["vad"],
-    onDecision: (probability: number) => void,
-  ): Promise<AudioNode> {
-    // 1. Create Worklet
+  async createNode(context, config, onDecision) {
     const blob = new Blob([energyVadWorkletCode], {
-      type: "application/javascript",
+      type: "application/javascript"
     });
     const url = URL.createObjectURL(blob);
-
     try {
       await context.audioWorklet.addModule(url);
     } catch (e) {
@@ -64,15 +54,15 @@ export class EnergyVADPlugin implements VADPlugin {
     } finally {
       URL.revokeObjectURL(url);
     }
-
-    // 2. Create Node
     const node = new AudioWorkletNode(context, "energy-vad-processor");
-
     node.port.onmessage = (event) => {
       const { probability } = event.data;
       onDecision(probability);
     };
-
     return node;
   }
-}
+};
+
+export {
+  EnergyVADPlugin
+};

package/dist/context/audio-context.d.mts

@@ -0,0 +1,32 @@
+/**
+ * Manages a shared AudioContext for the application.
+ */
+/**
+ * Gets the shared AudioContext, creating it if necessary.
+ * @param options Optional AudioContextOptions
+ */
+declare function getAudioContext(options?: AudioContextOptions): AudioContext;
+/**
+ * Registers a pipeline usage. Keeps track of active users.
+ */
+declare function registerPipeline(): void;
+/**
+ * Unregisters a pipeline usage.
+ * Optionally closes the context if no pipelines are active (not implemented by default to avoid churn).
+ */
+declare function unregisterPipeline(): void;
+/**
+ * Resumes the shared AudioContext.
+ * Should be called in response to a user gesture.
+ */
+declare function resumeAudioContext(): Promise<void>;
+/**
+ * Suspends the shared AudioContext.
+ */
+declare function suspendAudioContext(): Promise<void>;
+/**
+ * Closes the shared AudioContext and releases resources.
+ */
+declare function closeAudioContext(): Promise<void>;
+
+export { closeAudioContext, getAudioContext, registerPipeline, resumeAudioContext, suspendAudioContext, unregisterPipeline };

package/dist/context/audio-context.d.ts

@@ -0,0 +1,32 @@
+/**
+ * Manages a shared AudioContext for the application.
+ */
+/**
+ * Gets the shared AudioContext, creating it if necessary.
+ * @param options Optional AudioContextOptions
+ */
+declare function getAudioContext(options?: AudioContextOptions): AudioContext;
+/**
+ * Registers a pipeline usage. Keeps track of active users.
+ */
+declare function registerPipeline(): void;
+/**
+ * Unregisters a pipeline usage.
+ * Optionally closes the context if no pipelines are active (not implemented by default to avoid churn).
+ */
+declare function unregisterPipeline(): void;
+/**
+ * Resumes the shared AudioContext.
+ * Should be called in response to a user gesture.
+ */
+declare function resumeAudioContext(): Promise<void>;
+/**
+ * Suspends the shared AudioContext.
+ */
+declare function suspendAudioContext(): Promise<void>;
+/**
+ * Closes the shared AudioContext and releases resources.
+ */
+declare function closeAudioContext(): Promise<void>;
+
+export { closeAudioContext, getAudioContext, registerPipeline, resumeAudioContext, suspendAudioContext, unregisterPipeline };

package/dist/context/audio-context.js

@@ -0,0 +1,75 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/context/audio-context.ts
+var audio_context_exports = {};
+__export(audio_context_exports, {
+  closeAudioContext: () => closeAudioContext,
+  getAudioContext: () => getAudioContext,
+  registerPipeline: () => registerPipeline,
+  resumeAudioContext: () => resumeAudioContext,
+  suspendAudioContext: () => suspendAudioContext,
+  unregisterPipeline: () => unregisterPipeline
+});
+module.exports = __toCommonJS(audio_context_exports);
+var sharedContext = null;
+var activePipelines = 0;
+function getAudioContext(options) {
+  if (typeof window === "undefined" || typeof AudioContext === "undefined") {
+    throw new Error(
+      "AudioContext is not supported in this environment (browser only)."
+    );
+  }
+  if (!sharedContext || sharedContext.state === "closed") {
+    sharedContext = new AudioContext(options);
+  }
+  return sharedContext;
+}
+function registerPipeline() {
+  activePipelines++;
+}
+function unregisterPipeline() {
+  activePipelines = Math.max(0, activePipelines - 1);
+}
+async function resumeAudioContext() {
+  if (sharedContext && sharedContext.state === "suspended") {
+    await sharedContext.resume();
+  }
+}
+async function suspendAudioContext() {
+  if (sharedContext && sharedContext.state === "running") {
+    await sharedContext.suspend();
+  }
+}
+async function closeAudioContext() {
+  if (sharedContext && sharedContext.state !== "closed") {
+    await sharedContext.close();
+  }
+  sharedContext = null;
+  activePipelines = 0;
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  closeAudioContext,
+  getAudioContext,
+  registerPipeline,
+  resumeAudioContext,
+  suspendAudioContext,
+  unregisterPipeline
+});

package/dist/context/audio-context.mjs

@@ -0,0 +1,16 @@
+import {
+  closeAudioContext,
+  getAudioContext,
+  registerPipeline,
+  resumeAudioContext,
+  suspendAudioContext,
+  unregisterPipeline
+} from "../chunk-OZ7KMC4S.mjs";
+export {
+  closeAudioContext,
+  getAudioContext,
+  registerPipeline,
+  resumeAudioContext,
+  suspendAudioContext,
+  unregisterPipeline
+};

package/dist/extensibility/plugins.d.mts

@@ -0,0 +1,9 @@
+import { NoiseSuppressionPlugin, VADPlugin } from '../types.mjs';
+import 'mitt';
+
+declare function registerNoiseSuppressionPlugin(plugin: NoiseSuppressionPlugin): void;
+declare function registerVADPlugin(plugin: VADPlugin): void;
+declare function getNoiseSuppressionPlugin(name?: string): NoiseSuppressionPlugin;
+declare function getVADPlugin(name?: string): VADPlugin;
+
+export { getNoiseSuppressionPlugin, getVADPlugin, registerNoiseSuppressionPlugin, registerVADPlugin };

package/dist/extensibility/plugins.d.ts

@@ -0,0 +1,9 @@
+import { NoiseSuppressionPlugin, VADPlugin } from '../types.js';
+import 'mitt';
+
+declare function registerNoiseSuppressionPlugin(plugin: NoiseSuppressionPlugin): void;
+declare function registerVADPlugin(plugin: VADPlugin): void;
+declare function getNoiseSuppressionPlugin(name?: string): NoiseSuppressionPlugin;
+declare function getVADPlugin(name?: string): VADPlugin;
+
+export { getNoiseSuppressionPlugin, getVADPlugin, registerNoiseSuppressionPlugin, registerVADPlugin };