@tensamin/audio 0.1.1 → 0.1.2

package/src/extensibility/plugins.ts

@@ -1,45 +0,0 @@
-import type { NoiseSuppressionPlugin, VADPlugin } from "../types.js";
-import { RNNoisePlugin } from "../noise-suppression/rnnoise-node.js";
-import { EnergyVADPlugin } from "../vad/vad-node.js";
-
-const nsPlugins = new Map<string, NoiseSuppressionPlugin>();
-const vadPlugins = new Map<string, VADPlugin>();
-
-// Register defaults
-const defaultNs = new RNNoisePlugin();
-nsPlugins.set(defaultNs.name, defaultNs);
-
-const defaultVad = new EnergyVADPlugin();
-vadPlugins.set(defaultVad.name, defaultVad);
-
-export function registerNoiseSuppressionPlugin(plugin: NoiseSuppressionPlugin) {
-  nsPlugins.set(plugin.name, plugin);
-}
-
-export function registerVADPlugin(plugin: VADPlugin) {
-  vadPlugins.set(plugin.name, plugin);
-}
-
-export function getNoiseSuppressionPlugin(
-  name?: string,
-): NoiseSuppressionPlugin {
-  if (!name) return defaultNs;
-  const plugin = nsPlugins.get(name);
-  if (!plugin) {
-    console.warn(
-      `Noise suppression plugin '${name}' not found, falling back to default.`,
-    );
-    return defaultNs;
-  }
-  return plugin;
-}
-
-export function getVADPlugin(name?: string): VADPlugin {
-  if (!name) return defaultVad;
-  const plugin = vadPlugins.get(name);
-  if (!plugin) {
-    console.warn(`VAD plugin '${name}' not found, falling back to default.`);
-    return defaultVad;
-  }
-  return plugin;
-}

package/src/index.ts

@@ -1,8 +0,0 @@
-export * from "./types.js";
-export * from "./context/audio-context.js";
-export * from "./pipeline/audio-pipeline.js";
-export * from "./livekit/integration.js";
-export * from "./extensibility/plugins.js";
-export * from "./noise-suppression/rnnoise-node.js";
-export * from "./vad/vad-node.js";
-export * from "./vad/vad-state.js";

package/src/livekit/integration.ts

@@ -1,61 +0,0 @@
-import type { LocalAudioTrack } from "livekit-client";
-import { createAudioPipeline } from "../pipeline/audio-pipeline.js";
-import type { AudioPipelineHandle, AudioProcessingConfig } from "../types.js";
-
-/**
- * Attaches the audio processing pipeline to a LiveKit LocalAudioTrack.
- * This replaces the underlying MediaStreamTrack with the processed one.
- */
-export async function attachProcessingToTrack(
-  track: LocalAudioTrack,
-  config: AudioProcessingConfig = {},
-): Promise<AudioPipelineHandle> {
-  // 1. Get the original track
-  const originalTrack = track.mediaStreamTrack;
-
-  // 2. Create pipeline
-  const pipeline = await createAudioPipeline(originalTrack, config);
-
-  // 3. Replace the track in LiveKit
-  // Use replaceTrack which is the public API to swap the underlying MediaStreamTrack.
-  await track.replaceTrack(pipeline.processedTrack);
-
-  // 4. Handle intelligent muting if enabled
-  if (config.livekit?.manageTrackMute) {
-    let isVadMuted = false;
-
-    pipeline.events.on("vadChange", async (state) => {
-      if (state.isSpeaking) {
-        if (isVadMuted) {
-          // Only unmute if we were the ones who muted it
-          // And check if the track is not globally muted by user?
-          // This is tricky. If user muted manually, track.isMuted is true.
-          // We should probably check a separate flag or assume VAD overrides only when "active".
-          // For safety, we only unmute if we muted.
-          await track.unmute();
-          isVadMuted = false;
-        }
-      } else {
-        // Silence
-        if (!track.isMuted) {
-          await track.mute();
-          isVadMuted = true;
-        }
-      }
-    });
-  }
-
-  // 5. Handle cleanup
-  const originalDispose = pipeline.dispose;
-  pipeline.dispose = () => {
-    // Restore original track?
-    // Or just stop.
-    // If we dispose, we should probably try to restore the original track if it's still alive.
-    if (originalTrack.readyState === "live") {
-      track.replaceTrack(originalTrack).catch(console.error);
-    }
-    originalDispose();
-  };
-
-  return pipeline;
-}

package/src/noise-suppression/rnnoise-node.ts

@@ -1,62 +0,0 @@
-import {
-  RnnoiseWorkletNode,
-  loadRnnoise,
-} from "@sapphi-red/web-noise-suppressor";
-import type {
-  AudioProcessingConfig,
-  NoiseSuppressionPlugin,
-} from "../types.js";
-
-// Default URLs (can be overridden by config)
-// These defaults assume the assets are served from the same origin or a known CDN.
-// In a real package, we might want to bundle them or require the user to provide them.
-const DEFAULT_WASM_URL =
-  "https://unpkg.com/@sapphi-red/web-noise-suppressor@0.3.5/dist/rnnoise.wasm";
-const DEFAULT_SIMD_WASM_URL =
-  "https://unpkg.com/@sapphi-red/web-noise-suppressor@0.3.5/dist/rnnoise_simd.wasm";
-const DEFAULT_WORKLET_URL =
-  "https://unpkg.com/@sapphi-red/web-noise-suppressor@0.3.5/dist/noise-suppressor-worklet.min.js";
-
-export class RNNoisePlugin implements NoiseSuppressionPlugin {
-  name = "rnnoise-ns";
-  private wasmBuffer: ArrayBuffer | null = null;
-
-  async createNode(
-    context: AudioContext,
-    config: AudioProcessingConfig["noiseSuppression"],
-  ): Promise<AudioNode> {
-    if (!config?.enabled) {
-      // Return a passthrough gain node if disabled but requested (though pipeline usually handles this)
-      const pass = context.createGain();
-      return pass;
-    }
-
-    // 1. Load WASM if not loaded
-    // We use the library's loader which handles SIMD detection if we provide both URLs.
-    // But wait, loadRnnoise returns ArrayBuffer.
-    if (!this.wasmBuffer) {
-      this.wasmBuffer = await loadRnnoise({
-        url: config.wasmUrl || DEFAULT_WASM_URL,
-        simdUrl: DEFAULT_SIMD_WASM_URL, // We should probably allow config for this too, but for now default is fine.
-      });
-    }
-
-    // 2. Load Worklet
-    const workletUrl = config.workletUrl || DEFAULT_WORKLET_URL;
-
-    try {
-      await context.audioWorklet.addModule(workletUrl);
-    } catch (e) {
-      console.warn("Failed to add RNNoise worklet module:", e);
-      // Proceeding, assuming it might be already loaded.
-    }
-
-    // 3. Create Node
-    const node = new RnnoiseWorkletNode(context, {
-      wasmBinary: this.wasmBuffer,
-      maxChannels: 1, // Mono for now
-    });
-
-    return node;
-  }
-}

package/src/pipeline/audio-pipeline.ts

@@ -1,154 +0,0 @@
-import mitt from "mitt";
-import {
-  getAudioContext,
-  registerPipeline,
-  unregisterPipeline,
-} from "../context/audio-context.js";
-import {
-  getNoiseSuppressionPlugin,
-  getVADPlugin,
-} from "../extensibility/plugins.js";
-import { VADStateMachine } from "../vad/vad-state.js";
-import type {
-  AudioPipelineEvents,
-  AudioPipelineHandle,
-  AudioProcessingConfig,
-  VADState,
-} from "../types.js";
-
-export async function createAudioPipeline(
-  sourceTrack: MediaStreamTrack,
-  config: AudioProcessingConfig = {},
-): Promise<AudioPipelineHandle> {
-  const context = getAudioContext();
-  registerPipeline();
-
-  // Defaults
-  const fullConfig: AudioProcessingConfig = {
-    noiseSuppression: { enabled: true, ...config.noiseSuppression },
-    vad: { enabled: true, ...config.vad },
-    output: {
-      speechGain: 1.0,
-      silenceGain: 0.0,
-      gainRampTime: 0.02,
-      ...config.output,
-    },
-    livekit: { manageTrackMute: false, ...config.livekit },
-  };
-
-  // 1. Source
-  const sourceStream = new MediaStream([sourceTrack]);
-  const sourceNode = context.createMediaStreamSource(sourceStream);
-
-  // 2. Noise Suppression
-  const nsPlugin = getNoiseSuppressionPlugin(
-    fullConfig.noiseSuppression?.pluginName,
-  );
-  const nsNode = await nsPlugin.createNode(
-    context,
-    fullConfig.noiseSuppression,
-  );
-
-  // 3. VAD
-  const vadPlugin = getVADPlugin(fullConfig.vad?.pluginName);
-  const vadStateMachine = new VADStateMachine(fullConfig.vad);
-  const emitter = mitt<AudioPipelineEvents>();
-
-  const vadNode = await vadPlugin.createNode(
-    context,
-    fullConfig.vad,
-    (prob) => {
-      const timestamp = context.currentTime * 1000;
-      const newState = vadStateMachine.processFrame(prob, timestamp);
-
-      // Emit if state changed or periodically?
-      // For now, emit on change.
-      if (
-        newState.state !== lastVadState.state ||
-        Math.abs(newState.probability - lastVadState.probability) > 0.1
-      ) {
-        emitter.emit("vadChange", newState);
-        lastVadState = newState;
-        updateGain(newState);
-      }
-    },
-  );
-
-  let lastVadState: VADState = {
-    isSpeaking: false,
-    probability: 0,
-    state: "silent",
-  };
-
-  // 4. Pipeline Wiring
-  // Source -> NS -> Splitter
-  // Splitter -> VAD
-  // Splitter -> Delay -> Gain -> Destination
-
-  const splitter = context.createGain(); // Using Gain as splitter (fan-out)
-
-  sourceNode.connect(nsNode);
-  nsNode.connect(splitter);
-
-  // Path 1: VAD
-  splitter.connect(vadNode);
-  // vadNode usually doesn't output audio, or we don't connect it to destination.
-
-  // Path 2: Audio Output
-  const delayNode = context.createDelay(1.0); // Max 1 sec
-  const preRollSeconds = (fullConfig.vad?.preRollMs ?? 200) / 1000;
-  delayNode.delayTime.value = preRollSeconds;
-
-  const gainNode = context.createGain();
-  gainNode.gain.value = fullConfig.output?.silenceGain ?? 0;
-
-  const destination = context.createMediaStreamDestination();
-
-  splitter.connect(delayNode);
-  delayNode.connect(gainNode);
-  gainNode.connect(destination);
-
-  // Helper to update gain
-  function updateGain(state: VADState) {
-    const { speechGain, silenceGain, gainRampTime } = fullConfig.output!;
-    const targetGain = state.isSpeaking
-      ? (speechGain ?? 1.0)
-      : (silenceGain ?? 0.0);
-
-    // Ramp to target
-    const now = context.currentTime;
-    gainNode.gain.setTargetAtTime(targetGain, now, gainRampTime ?? 0.02);
-  }
-
-  // Handle disposal
-  function dispose() {
-    sourceNode.disconnect();
-    nsNode.disconnect();
-    splitter.disconnect();
-    vadNode.disconnect();
-    delayNode.disconnect();
-    gainNode.disconnect();
-
-    // Stop tracks? No, we don't own the source track.
-    // But we own the destination track.
-    destination.stream.getTracks().forEach((t) => t.stop());
-
-    unregisterPipeline();
-  }
-
-  return {
-    processedTrack: destination.stream.getAudioTracks()[0]!,
-    events: emitter,
-    get state() {
-      return lastVadState;
-    },
-    setConfig: (newConfig) => {
-      // TODO: Implement runtime config updates
-      // For now, just update the VAD state machine config
-      if (newConfig.vad) {
-        vadStateMachine.updateConfig(newConfig.vad);
-      }
-    },
-    dispose,
-  };
-}

package/src/types.ts

@@ -1,167 +0,0 @@
-import type { LocalAudioTrack, TrackPublication } from "livekit-client";
-import type { Emitter } from "mitt";
-
-/**
- * Configuration for the audio processing pipeline.
- */
-export interface AudioProcessingConfig {
-  /**
-   * Noise suppression configuration.
-   */
-  noiseSuppression?: {
-    enabled: boolean;
-    /**
-     * Path or URL to the RNNoise WASM binary.
-     * If not provided, the default from @sapphi-red/web-noise-suppressor will be used (if bundler supports it).
-     */
-    wasmUrl?: string;
-    /**
-     * Path or URL to the RNNoise worklet script.
-     */
-    workletUrl?: string;
-    /**
-     * Plugin name to use. Defaults to 'rnnoise-ns'.
-     */
-    pluginName?: string;
-  };
-
-  /**
-   * Voice Activity Detection (VAD) configuration.
-   */
-  vad?: {
-    enabled: boolean;
-    /**
-     * Plugin name to use. Defaults to 'rnnoise-vad' or 'energy-vad'.
-     */
-    pluginName?: string;
-    /**
-     * Probability threshold for speech onset (0-1).
-     * Default: 0.5
-     */
-    startThreshold?: number;
-    /**
-     * Probability threshold for speech offset (0-1).
-     * Default: 0.4
-     */
-    stopThreshold?: number;
-    /**
-     * Time in ms to wait after speech stops before considering it silent.
-     * Default: 300ms
-     */
-    hangoverMs?: number;
-    /**
-     * Time in ms of audio to buffer before speech onset to avoid cutting the start.
-     * Default: 200ms
-     */
-    preRollMs?: number;
-  };
-
-  /**
-   * Output gain and muting configuration.
-   */
-  output?: {
-    /**
-     * Gain to apply when speaking (0-1+). Default: 1.0
-     */
-    speechGain?: number;
-    /**
-     * Gain to apply when silent (0-1). Default: 0.0 (mute)
-     */
-    silenceGain?: number;
-    /**
-     * Time in seconds to ramp gain changes. Default: 0.02
-     */
-    gainRampTime?: number;
-  };
-
-  /**
-   * LiveKit integration configuration.
-   */
-  livekit?: {
-    /**
-     * Whether to call track.mute()/unmute() on the LocalAudioTrack based on VAD.
-     * This saves bandwidth but has more signaling overhead.
-     * Default: false (uses gain gating only)
-     */
-    manageTrackMute?: boolean;
-  };
-}
-
-/**
- * Represents the state of Voice Activity Detection.
- */
-export interface VADState {
-  /**
-   * Whether speech is currently detected (after hysteresis).
-   */
-  isSpeaking: boolean;
-  /**
-   * Raw probability of speech from the VAD model (0-1).
-   */
-  probability: number;
-  /**
-   * Current state enum.
-   */
-  state: "silent" | "speech_starting" | "speaking" | "speech_ending";
-}
-
-/**
- * Events emitted by the audio pipeline.
- */
-export type AudioPipelineEvents = {
-  vadChange: VADState;
-  error: Error;
-};
-
-/**
- * Handle to a running audio processing pipeline.
- */
-export interface AudioPipelineHandle {
-  /**
-   * The processed MediaStreamTrack.
-   */
-  readonly processedTrack: MediaStreamTrack;
-
-  /**
-   * Event emitter for VAD state and errors.
-   */
-  readonly events: Emitter<AudioPipelineEvents>;
-
-  /**
-   * Current VAD state.
-   */
-  readonly state: VADState;
-
-  /**
-   * Update configuration at runtime.
-   */
-  setConfig(config: Partial<AudioProcessingConfig>): void;
-
-  /**
-   * Stop processing and release resources.
-   */
-  dispose(): void;
-}
-
-/**
- * Interface for a Noise Suppression Plugin.
- */
-export interface NoiseSuppressionPlugin {
-  name: string;
-  createNode(
-    context: AudioContext,
-    config: AudioProcessingConfig["noiseSuppression"],
-  ): Promise<AudioNode>;
-}
-
-/**
- * Interface for a VAD Plugin.
- */
-export interface VADPlugin {
-  name: string;
-  createNode(
-    context: AudioContext,
-    config: AudioProcessingConfig["vad"],
-    onDecision: (probability: number) => void,
-  ): Promise<AudioNode>;
-}

package/tsconfig.json

@@ -1,46 +0,0 @@
-{
-  // Visit https://aka.ms/tsconfig to read more about this file
-  "compilerOptions": {
-    // File Layout
-    "rootDir": "./src",
-    "outDir": "./dist",
-
-    // Environment Settings
-    // See also https://aka.ms/tsconfig/module
-    "moduleResolution": "bundler",
-    "module": "esnext",
-    "target": "esnext",
-    "types": [],
-    // For nodejs:
-    // "lib": ["esnext"],
-    // "types": ["node"],
-    // and npm install -D @types/node
-
-    // Other Outputs
-    "sourceMap": true,
-    "declaration": true,
-    "declarationMap": true,
-
-    // Stricter Typechecking Options
-    "noUncheckedIndexedAccess": true,
-    "exactOptionalPropertyTypes": true,
-
-    // Style Options
-    // "noImplicitReturns": true,
-    // "noImplicitOverride": true,
-    // "noUnusedLocals": true,
-    // "noUnusedParameters": true,
-    // "noFallthroughCasesInSwitch": true,
-    // "noPropertyAccessFromIndexSignature": true,
-
-    // Recommended Options
-    "strict": true,
-    "jsx": "react-jsx",
-    "verbatimModuleSyntax": true,
-    "isolatedModules": true,
-    "noUncheckedSideEffectImports": true,
-    "moduleDetection": "force",
-    "skipLibCheck": true,
-    "noEmit": true
-  }
-}