@tensamin/audio 0.1.1 → 0.1.3

package/src/pipeline/audio-pipeline.ts

@@ -1,154 +0,0 @@
-import mitt from "mitt";
-import {
-  getAudioContext,
-  registerPipeline,
-  unregisterPipeline,
-} from "../context/audio-context.js";
-import {
-  getNoiseSuppressionPlugin,
-  getVADPlugin,
-} from "../extensibility/plugins.js";
-import { VADStateMachine } from "../vad/vad-state.js";
-import type {
-  AudioPipelineEvents,
-  AudioPipelineHandle,
-  AudioProcessingConfig,
-  VADState,
-} from "../types.js";
-
-export async function createAudioPipeline(
-  sourceTrack: MediaStreamTrack,
-  config: AudioProcessingConfig = {},
-): Promise<AudioPipelineHandle> {
-  const context = getAudioContext();
-  registerPipeline();
-
-  // Defaults
-  const fullConfig: AudioProcessingConfig = {
-    noiseSuppression: { enabled: true, ...config.noiseSuppression },
-    vad: { enabled: true, ...config.vad },
-    output: {
-      speechGain: 1.0,
-      silenceGain: 0.0,
-      gainRampTime: 0.02,
-      ...config.output,
-    },
-    livekit: { manageTrackMute: false, ...config.livekit },
-  };
-
-  // 1. Source
-  const sourceStream = new MediaStream([sourceTrack]);
-  const sourceNode = context.createMediaStreamSource(sourceStream);
-
-  // 2. Noise Suppression
-  const nsPlugin = getNoiseSuppressionPlugin(
-    fullConfig.noiseSuppression?.pluginName,
-  );
-  const nsNode = await nsPlugin.createNode(
-    context,
-    fullConfig.noiseSuppression,
-  );
-
-  // 3. VAD
-  const vadPlugin = getVADPlugin(fullConfig.vad?.pluginName);
-  const vadStateMachine = new VADStateMachine(fullConfig.vad);
-  const emitter = mitt<AudioPipelineEvents>();
-
-  const vadNode = await vadPlugin.createNode(
-    context,
-    fullConfig.vad,
-    (prob) => {
-      const timestamp = context.currentTime * 1000;
-      const newState = vadStateMachine.processFrame(prob, timestamp);
-
-      // Emit if state changed or periodically?
-      // For now, emit on change.
-      if (
-        newState.state !== lastVadState.state ||
-        Math.abs(newState.probability - lastVadState.probability) > 0.1
-      ) {
-        emitter.emit("vadChange", newState);
-        lastVadState = newState;
-        updateGain(newState);
-      }
-    },
-  );
-
-  let lastVadState: VADState = {
-    isSpeaking: false,
-    probability: 0,
-    state: "silent",
-  };
-
-  // 4. Pipeline Wiring
-  // Source -> NS -> Splitter
-  // Splitter -> VAD
-  // Splitter -> Delay -> Gain -> Destination
-
-  const splitter = context.createGain(); // Using Gain as splitter (fan-out)
-
-  sourceNode.connect(nsNode);
-  nsNode.connect(splitter);
-
-  // Path 1: VAD
-  splitter.connect(vadNode);
-  // vadNode usually doesn't output audio, or we don't connect it to destination.
-
-  // Path 2: Audio Output
-  const delayNode = context.createDelay(1.0); // Max 1 sec
-  const preRollSeconds = (fullConfig.vad?.preRollMs ?? 200) / 1000;
-  delayNode.delayTime.value = preRollSeconds;
-
-  const gainNode = context.createGain();
-  gainNode.gain.value = fullConfig.output?.silenceGain ?? 0;
-
-  const destination = context.createMediaStreamDestination();
-
-  splitter.connect(delayNode);
-  delayNode.connect(gainNode);
-  gainNode.connect(destination);
-
-  // Helper to update gain
-  function updateGain(state: VADState) {
-    const { speechGain, silenceGain, gainRampTime } = fullConfig.output!;
-    const targetGain = state.isSpeaking
-      ? (speechGain ?? 1.0)
-      : (silenceGain ?? 0.0);
-
-    // Ramp to target
-    const now = context.currentTime;
-    gainNode.gain.setTargetAtTime(targetGain, now, gainRampTime ?? 0.02);
-  }
-
-  // Handle disposal
-  function dispose() {
-    sourceNode.disconnect();
-    nsNode.disconnect();
-    splitter.disconnect();
-    vadNode.disconnect();
-    delayNode.disconnect();
-    gainNode.disconnect();
-
-    // Stop tracks? No, we don't own the source track.
-    // But we own the destination track.
-    destination.stream.getTracks().forEach((t) => t.stop());
-
-    unregisterPipeline();
-  }
-
-  return {
-    processedTrack: destination.stream.getAudioTracks()[0]!,
-    events: emitter,
-    get state() {
-      return lastVadState;
-    },
-    setConfig: (newConfig) => {
-      // TODO: Implement runtime config updates
-      // For now, just update the VAD state machine config
-      if (newConfig.vad) {
-        vadStateMachine.updateConfig(newConfig.vad);
-      }
-    },
-    dispose,
-  };
-}

package/src/types.ts

@@ -1,167 +0,0 @@
-import type { LocalAudioTrack, TrackPublication } from "livekit-client";
-import type { Emitter } from "mitt";
-
-/**
- * Configuration for the audio processing pipeline.
- */
-export interface AudioProcessingConfig {
-  /**
-   * Noise suppression configuration.
-   */
-  noiseSuppression?: {
-    enabled: boolean;
-    /**
-     * Path or URL to the RNNoise WASM binary.
-     * If not provided, the default from @sapphi-red/web-noise-suppressor will be used (if bundler supports it).
-     */
-    wasmUrl?: string;
-    /**
-     * Path or URL to the RNNoise worklet script.
-     */
-    workletUrl?: string;
-    /**
-     * Plugin name to use. Defaults to 'rnnoise-ns'.
-     */
-    pluginName?: string;
-  };
-
-  /**
-   * Voice Activity Detection (VAD) configuration.
-   */
-  vad?: {
-    enabled: boolean;
-    /**
-     * Plugin name to use. Defaults to 'rnnoise-vad' or 'energy-vad'.
-     */
-    pluginName?: string;
-    /**
-     * Probability threshold for speech onset (0-1).
-     * Default: 0.5
-     */
-    startThreshold?: number;
-    /**
-     * Probability threshold for speech offset (0-1).
-     * Default: 0.4
-     */
-    stopThreshold?: number;
-    /**
-     * Time in ms to wait after speech stops before considering it silent.
-     * Default: 300ms
-     */
-    hangoverMs?: number;
-    /**
-     * Time in ms of audio to buffer before speech onset to avoid cutting the start.
-     * Default: 200ms
-     */
-    preRollMs?: number;
-  };
-
-  /**
-   * Output gain and muting configuration.
-   */
-  output?: {
-    /**
-     * Gain to apply when speaking (0-1+). Default: 1.0
-     */
-    speechGain?: number;
-    /**
-     * Gain to apply when silent (0-1). Default: 0.0 (mute)
-     */
-    silenceGain?: number;
-    /**
-     * Time in seconds to ramp gain changes. Default: 0.02
-     */
-    gainRampTime?: number;
-  };
-
-  /**
-   * LiveKit integration configuration.
-   */
-  livekit?: {
-    /**
-     * Whether to call track.mute()/unmute() on the LocalAudioTrack based on VAD.
-     * This saves bandwidth but has more signaling overhead.
-     * Default: false (uses gain gating only)
-     */
-    manageTrackMute?: boolean;
-  };
-}
-
-/**
- * Represents the state of Voice Activity Detection.
- */
-export interface VADState {
-  /**
-   * Whether speech is currently detected (after hysteresis).
-   */
-  isSpeaking: boolean;
-  /**
-   * Raw probability of speech from the VAD model (0-1).
-   */
-  probability: number;
-  /**
-   * Current state enum.
-   */
-  state: "silent" | "speech_starting" | "speaking" | "speech_ending";
-}
-
-/**
- * Events emitted by the audio pipeline.
- */
-export type AudioPipelineEvents = {
-  vadChange: VADState;
-  error: Error;
-};
-
-/**
- * Handle to a running audio processing pipeline.
- */
-export interface AudioPipelineHandle {
-  /**
-   * The processed MediaStreamTrack.
-   */
-  readonly processedTrack: MediaStreamTrack;
-
-  /**
-   * Event emitter for VAD state and errors.
-   */
-  readonly events: Emitter<AudioPipelineEvents>;
-
-  /**
-   * Current VAD state.
-   */
-  readonly state: VADState;
-
-  /**
-   * Update configuration at runtime.
-   */
-  setConfig(config: Partial<AudioProcessingConfig>): void;
-
-  /**
-   * Stop processing and release resources.
-   */
-  dispose(): void;
-}
-
-/**
- * Interface for a Noise Suppression Plugin.
- */
-export interface NoiseSuppressionPlugin {
-  name: string;
-  createNode(
-    context: AudioContext,
-    config: AudioProcessingConfig["noiseSuppression"],
-  ): Promise<AudioNode>;
-}
-
-/**
- * Interface for a VAD Plugin.
- */
-export interface VADPlugin {
-  name: string;
-  createNode(
-    context: AudioContext,
-    config: AudioProcessingConfig["vad"],
-    onDecision: (probability: number) => void,
-  ): Promise<AudioNode>;
-}

package/src/vad/vad-node.ts

@@ -1,78 +0,0 @@
-import type { AudioProcessingConfig, VADPlugin } from "../types.js";
-
-// Inline AudioWorklet processor for Energy VAD
-const energyVadWorkletCode = `
-class EnergyVadProcessor extends AudioWorkletProcessor {
-  constructor() {
-    super();
-    this.smoothing = 0.95;
-    this.energy = 0;
-    this.noiseFloor = 0.001;
-  }
-
-  process(inputs, outputs, parameters) {
-    const input = inputs[0];
-    if (!input || !input.length) return true;
-    const channel = input[0];
-    
-    // Calculate RMS
-    let sum = 0;
-    for (let i = 0; i < channel.length; i++) {
-      sum += channel[i] * channel[i];
-    }
-    const rms = Math.sqrt(sum / channel.length);
-
-    // Simple adaptive noise floor (very basic)
-    if (rms < this.noiseFloor) {
-      this.noiseFloor = this.noiseFloor * 0.99 + rms * 0.01;
-    } else {
-      this.noiseFloor = this.noiseFloor * 0.999 + rms * 0.001;
-    }
-
-    // Calculate "probability" based on SNR
-    // This is a heuristic mapping from energy to 0-1
-    const snr = rms / (this.noiseFloor + 1e-6);
-    const probability = Math.min(1, Math.max(0, (snr - 1.5) / 10)); // Arbitrary scaling
-
-    this.port.postMessage({ probability });
-
-    return true;
-  }
-}
-registerProcessor('energy-vad-processor', EnergyVadProcessor);
-`;
-
-export class EnergyVADPlugin implements VADPlugin {
-  name = "energy-vad";
-
-  async createNode(
-    context: AudioContext,
-    config: AudioProcessingConfig["vad"],
-    onDecision: (probability: number) => void,
-  ): Promise<AudioNode> {
-    // 1. Create Worklet
-    const blob = new Blob([energyVadWorkletCode], {
-      type: "application/javascript",
-    });
-    const url = URL.createObjectURL(blob);
-
-    try {
-      await context.audioWorklet.addModule(url);
-    } catch (e) {
-      console.warn("Failed to add Energy VAD worklet:", e);
-      throw e;
-    } finally {
-      URL.revokeObjectURL(url);
-    }
-
-    // 2. Create Node
-    const node = new AudioWorkletNode(context, "energy-vad-processor");
-
-    node.port.onmessage = (event) => {
-      const { probability } = event.data;
-      onDecision(probability);
-    };
-
-    return node;
-  }
-}

package/tsconfig.json

@@ -1,46 +0,0 @@
-{
-  // Visit https://aka.ms/tsconfig to read more about this file
-  "compilerOptions": {
-    // File Layout
-    "rootDir": "./src",
-    "outDir": "./dist",
-
-    // Environment Settings
-    // See also https://aka.ms/tsconfig/module
-    "moduleResolution": "bundler",
-    "module": "esnext",
-    "target": "esnext",
-    "types": [],
-    // For nodejs:
-    // "lib": ["esnext"],
-    // "types": ["node"],
-    // and npm install -D @types/node
-
-    // Other Outputs
-    "sourceMap": true,
-    "declaration": true,
-    "declarationMap": true,
-
-    // Stricter Typechecking Options
-    "noUncheckedIndexedAccess": true,
-    "exactOptionalPropertyTypes": true,
-
-    // Style Options
-    // "noImplicitReturns": true,
-    // "noImplicitOverride": true,
-    // "noUnusedLocals": true,
-    // "noUnusedParameters": true,
-    // "noFallthroughCasesInSwitch": true,
-    // "noPropertyAccessFromIndexSignature": true,
-
-    // Recommended Options
-    "strict": true,
-    "jsx": "react-jsx",
-    "verbatimModuleSyntax": true,
-    "isolatedModules": true,
-    "noUncheckedSideEffectImports": true,
-    "moduleDetection": "force",
-    "skipLibCheck": true,
-    "noEmit": true
-  }
-}