@newgameplusinc/odyssey-audio-video-sdk-dev 1.0.57 → 1.0.58

package/dist/MLNoiseSuppressor.js

@@ -1,439 +0,0 @@
-"use strict";
-/**
- * ML-Based Noise Suppressor for Odyssey MediaSoup SDK
- * Uses trained TensorFlow.js BiLSTM model for real-time noise suppression
- *
- * Architecture: BiLSTM (256 units x 2) + Dense layers
- * Input: Mel-spectrogram features (16 frames x 128 mels)
- * Output: Noise suppression mask (0-1 per frequency bin)
- *
- * Trained on: LibriSpeech + UrbanSound8K + MS-SNSD datasets
- * Performance: val_loss=0.038, SNR improvement ~12dB
- */
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || (function () {
-    var ownKeys = function(o) {
-        ownKeys = Object.getOwnPropertyNames || function (o) {
-            var ar = [];
-            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
-            return ar;
-        };
-        return ownKeys(o);
-    };
-    return function (mod) {
-        if (mod && mod.__esModule) return mod;
-        var result = {};
-        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
-        __setModuleDefault(result, mod);
-        return result;
-    };
-})();
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.MLNoiseSuppressor = void 0;
-const tf = __importStar(require("@tensorflow/tfjs"));
-class MLNoiseSuppressor {
-    constructor() {
-        this.model = null;
-        this.config = null;
-        this.normStats = null;
-        this.audioContext = null;
-        this.isInitialized = false;
-        // Real-time processing state
-        this.processingNode = null;
-        this.highPassFilter = null;
-        // Frame buffer for sequence-based processing
-        this.frameBuffer = [];
-        this.prevMask = null;
-        // Temporal smoothing (CRITICAL for quality - like Apple!)
-        this.SMOOTHING_ALPHA = 0.85; // Higher = smoother transitions
-        // Mel filterbank cache
-        this.melFilterbank = null;
-        this.fftSize = 512;
-        // FFT workspace
-        this.hannWindow = null;
-    }
-    /**
-     * Initialize the ML noise suppressor
-     * @param modelUrl URL to the model.json file
-     * @param audioContext Web Audio API AudioContext
-     */
-    async initialize(modelUrl, audioContext) {
-        console.log("🚀 Initializing ML Noise Suppressor (BiLSTM v2)...");
-        this.audioContext = audioContext;
-        try {
-            // Load model
-            console.log(`📂 Loading model from ${modelUrl}`);
-            this.model = await tf.loadLayersModel(modelUrl);
-            console.log("✅ Model loaded successfully");
-            console.log(`   Parameters: ${this.model.countParams().toLocaleString()}`);
-            // Load config
-            const baseUrl = modelUrl.substring(0, modelUrl.lastIndexOf("/"));
-            const configUrl = `${baseUrl}/model_config.json`;
-            const configResponse = await fetch(configUrl);
-            this.config = await configResponse.json();
-            console.log("⚙️  Config loaded:", this.config);
-            // Load normalization stats
-            const normUrl = `${baseUrl}/normalization_stats.json`;
-            const normResponse = await fetch(normUrl);
-            this.normStats = await normResponse.json();
-            console.log(`📏 Normalization stats: mean=${this.normStats.mean.toFixed(4)}, std=${this.normStats.std.toFixed(4)}`);
-            // Initialize FFT workspace
-            this.fftSize = this.config.frame_size || 512;
-            this.hannWindow = this.createHannWindow(this.fftSize);
-            // Create mel filterbank
-            this.melFilterbank = this.createMelFilterbank(this.fftSize, this.config.sample_rate, this.config.n_mels, 20, // fmin
-            8000 // fmax for voice
-            );
-            this.isInitialized = true;
-            console.log("✅ ML Noise Suppressor initialized!");
-        }
-        catch (error) {
-            console.error("❌ Failed to initialize ML Noise Suppressor:", error);
-            throw error;
-        }
-    }
-    /**
-     * Create Hann window for FFT
-     */
-    createHannWindow(size) {
-        const window = new Float32Array(size);
-        for (let i = 0; i < size; i++) {
-            window[i] = 0.5 * (1 - Math.cos((2 * Math.PI * i) / (size - 1)));
-        }
-        return window;
-    }
-    /**
-     * Create mel filterbank matrix
-     */
-    createMelFilterbank(fftSize, sampleRate, nMels, fmin, fmax) {
-        const nFft = Math.floor(fftSize / 2) + 1;
-        // Convert Hz to Mel scale
-        const hzToMel = (hz) => 2595 * Math.log10(1 + hz / 700);
-        const melToHz = (mel) => 700 * (Math.pow(10, mel / 2595) - 1);
-        const melMin = hzToMel(fmin);
-        const melMax = hzToMel(fmax);
-        // Create mel center frequencies
-        const melPoints = [];
-        for (let i = 0; i < nMels + 2; i++) {
-            melPoints.push(melMin + ((melMax - melMin) * i) / (nMels + 1));
-        }
-        // Convert to Hz
-        const hzPoints = melPoints.map(melToHz);
-        // Convert to FFT bins
-        const binPoints = hzPoints.map((hz) => Math.floor(((fftSize + 1) * hz) / sampleRate));
-        // Create triangular filterbank
-        const filterbank = [];
-        for (let m = 0; m < nMels; m++) {
-            const filter = new Float32Array(nFft);
-            const left = binPoints[m];
-            const center = binPoints[m + 1];
-            const right = binPoints[m + 2];
-            // Rising slope
-            for (let k = left; k < center && k < nFft; k++) {
-                filter[k] = (k - left) / (center - left);
-            }
-            // Falling slope
-            for (let k = center; k < right && k < nFft; k++) {
-                filter[k] = (right - k) / (right - center);
-            }
-            filterbank.push(filter);
-        }
-        return filterbank;
-    }
-    /**
-     * Compute FFT magnitude spectrum (optimized DFT for real-time)
-     */
-    computeFFT(frame) {
-        const N = frame.length;
-        const magnitude = new Float32Array(Math.floor(N / 2) + 1);
-        // Apply Hann window
-        const windowed = new Float32Array(N);
-        for (let i = 0; i < N; i++) {
-            windowed[i] = frame[i] * (this.hannWindow?.[i] || 1);
-        }
-        // Compute DFT for positive frequencies only
-        for (let k = 0; k <= N / 2; k++) {
-            let real = 0;
-            let imag = 0;
-            const twoPiKOverN = (2 * Math.PI * k) / N;
-            for (let n = 0; n < N; n++) {
-                const angle = twoPiKOverN * n;
-                real += windowed[n] * Math.cos(angle);
-                imag -= windowed[n] * Math.sin(angle);
-            }
-            magnitude[k] = Math.sqrt(real * real + imag * imag);
-        }
-        return magnitude;
-    }
-    /**
-     * Compute mel-spectrogram features from audio frame
-     */
-    computeMelFeatures(audio) {
-        if (!this.config || !this.melFilterbank) {
-            throw new Error("Config or filterbank not loaded");
-        }
-        // Compute FFT magnitude
-        const spectrum = this.computeFFT(audio);
-        // Apply mel filterbank and log compression
-        const melFeatures = new Array(this.config.n_mels);
-        for (let m = 0; m < this.config.n_mels; m++) {
-            let sum = 0;
-            const filter = this.melFilterbank[m];
-            for (let k = 0; k < spectrum.length && k < filter.length; k++) {
-                sum += spectrum[k] * spectrum[k] * filter[k]; // Power spectrum
-            }
-            // Log compression (matching training)
-            melFeatures[m] = Math.log(Math.max(sum, 1e-10) + 1);
-        }
-        return melFeatures;
-    }
-    /**
-     * Process audio buffer with ML noise suppression
-     * @param inputBuffer Audio buffer to process (Float32Array)
-     * @returns Processed audio buffer
-     */
-    async processAudio(inputBuffer) {
-        if (!this.isInitialized || !this.model || !this.config || !this.normStats) {
-            return inputBuffer;
-        }
-        try {
-            const hopLength = this.config.hop_length;
-            const frameSize = this.config.frame_size || 512;
-            const numFrames = Math.floor((inputBuffer.length - frameSize) / hopLength) + 1;
-            if (numFrames < 1) {
-                return inputBuffer;
-            }
-            // Extract mel features for each frame
-            const features = [];
-            for (let i = 0; i < numFrames; i++) {
-                const start = i * hopLength;
-                const frame = inputBuffer.slice(start, start + frameSize);
-                const melFeatures = this.computeMelFeatures(frame);
-                features.push(melFeatures);
-            }
-            // Add to frame buffer for sequence processing
-            this.frameBuffer.push(...features);
-            // Keep only recent frames (2x sequence length for overlap)
-            const seqLength = this.config.sequence_length;
-            while (this.frameBuffer.length > seqLength * 2) {
-                this.frameBuffer.shift();
-            }
-            // Need enough frames for one sequence
-            if (this.frameBuffer.length < seqLength) {
-                return inputBuffer; // Not enough frames yet, pass through
-            }
-            // Create sequence from recent frames
-            const sequence = this.frameBuffer.slice(-seqLength);
-            // Normalize features (using training stats)
-            const normalizedSeq = sequence.map((frame) => frame.map((val) => (val - this.normStats.mean) / this.normStats.std));
-            // Run model inference
-            const mask = await tf.tidy(() => {
-                const inputTensor = tf.tensor3d([normalizedSeq]);
-                const output = this.model.predict(inputTensor);
-                return output.arraySync();
-            });
-            // Get mask for the last frame (most recent prediction)
-            const lastMaskFrame = mask[0][seqLength - 1];
-            const currentMask = new Float32Array(lastMaskFrame);
-            // Apply temporal smoothing (CRITICAL for Apple-quality audio!)
-            const smoothedMask = this.applyTemporalSmoothing(currentMask);
-            // Apply mask to audio with voice preservation
-            const output = this.applyMaskWithVoicePreservation(inputBuffer, smoothedMask, numFrames);
-            return output;
-        }
-        catch (error) {
-            console.error("❌ Error processing audio:", error);
-            return inputBuffer;
-        }
-    }
-    /**
-     * Apply temporal smoothing to reduce artifacts (Apple-style)
-     */
-    applyTemporalSmoothing(currentMask) {
-        if (!this.prevMask || this.prevMask.length !== currentMask.length) {
-            this.prevMask = new Float32Array(currentMask);
-            return currentMask;
-        }
-        const smoothed = new Float32Array(currentMask.length);
-        for (let i = 0; i < currentMask.length; i++) {
-            // Exponential moving average for smooth transitions
-            smoothed[i] =
-                this.SMOOTHING_ALPHA * currentMask[i] +
-                    (1 - this.SMOOTHING_ALPHA) * this.prevMask[i];
-            // Never completely mute (preserve minimum 3% - prevents artifacts)
-            smoothed[i] = Math.max(0.03, Math.min(1.0, smoothed[i]));
-        }
-        this.prevMask = smoothed;
-        return smoothed;
-    }
-    /**
-     * Apply mask with voice frequency preservation
-     */
-    applyMaskWithVoicePreservation(audio, mask, numFrames) {
-        const output = new Float32Array(audio.length);
-        const hopLength = this.config.hop_length;
-        const nMels = this.config.n_mels;
-        // Calculate frequency-weighted gain
-        // Voice fundamentals are in lower mel bins, preserve them more
-        let voiceGain = 0;
-        let noiseGain = 0;
-        // Lower 1/4 of mels = voice fundamentals (80-500Hz)
-        const voiceBins = Math.floor(nMels / 4);
-        for (let i = 0; i < voiceBins; i++) {
-            voiceGain += mask[i];
-        }
-        voiceGain /= voiceBins;
-        // Upper 3/4 = potentially noise
-        for (let i = voiceBins; i < nMels; i++) {
-            noiseGain += mask[i];
-        }
-        noiseGain /= nMels - voiceBins;
-        // Blend gains (favor voice preservation)
-        const avgGain = voiceGain * 0.7 + noiseGain * 0.3;
-        // Apply gain per sample
-        for (let i = 0; i < audio.length; i++) {
-            // Use smooth gain
-            let gain = avgGain;
-            // Boost if mask indicates strong voice (> 0.5)
-            if (avgGain > 0.5) {
-                gain = Math.min(1.0, avgGain * 1.05);
-            }
-            output[i] = audio[i] * gain;
-        }
-        // Apply soft fade at edges to prevent clicks
-        const fadeLen = Math.min(64, output.length / 10);
-        for (let i = 0; i < fadeLen; i++) {
-            const fade = i / fadeLen;
-            output[i] *= fade;
-            output[output.length - 1 - i] *= fade;
-        }
-        return output;
-    }
-    /**
-     * Process MediaStream with ML noise suppression
-     * @param inputStream MediaStream to process
-     * @returns Cleaned MediaStream
-     */
-    async processMediaStream(inputStream) {
-        if (!this.audioContext || !this.isInitialized) {
-            console.warn("⚠️  ML Noise Suppressor not initialized, returning original stream");
-            return inputStream;
-        }
-        try {
-            console.log("🎤 [ML] Setting up BiLSTM noise suppression pipeline...");
-            // Create MediaStreamSource from input
-            const source = this.audioContext.createMediaStreamSource(inputStream);
-            // Create high-pass filter (remove <80Hz rumble - like Apple)
-            this.highPassFilter = this.audioContext.createBiquadFilter();
-            this.highPassFilter.type = "highpass";
-            this.highPassFilter.frequency.value = 80;
-            this.highPassFilter.Q.value = 0.7;
-            // Create destination for output
-            const destination = this.audioContext.createMediaStreamDestination();
-            // Create ScriptProcessor for real-time ML processing
-            // Buffer size of 2048 = ~42ms latency at 48kHz (acceptable for real-time)
-            const bufferSize = 2048;
-            this.processingNode = this.audioContext.createScriptProcessor(bufferSize, 1, 1);
-            let frameCount = 0;
-            const startTime = performance.now();
-            // Double-buffering for async ML processing
-            // We store the PREVIOUS processed result and output it in the NEXT callback
-            // This adds one buffer of latency but ensures we never output zeros
-            let previousProcessedBuffer = null;
-            let processingInFlight = false;
-            // Process audio frames with ML model
-            // IMPORTANT: onaudioprocess is synchronous! We use double-buffering to handle async ML
-            this.processingNode.onaudioprocess = (event) => {
-                const inputData = event.inputBuffer.getChannelData(0);
-                const outputData = event.outputBuffer.getChannelData(0);
-                frameCount++;
-                // OUTPUT: Use previously processed audio (or passthrough if not ready yet)
-                if (previousProcessedBuffer) {
-                    outputData.set(previousProcessedBuffer);
-                }
-                else {
-                    // First frame or ML not ready - pass through original audio
-                    outputData.set(inputData);
-                }
-                // PROCESS: Start async ML processing for the NEXT frame
-                // Only start new processing if previous one is complete
-                if (!processingInFlight) {
-                    processingInFlight = true;
-                    const inputCopy = new Float32Array(inputData);
-                    // Fire-and-forget async processing
-                    this.processAudio(inputCopy)
-                        .then((processed) => {
-                        previousProcessedBuffer = processed;
-                        processingInFlight = false;
-                    })
-                        .catch((error) => {
-                        // On error, store the original audio for passthrough
-                        previousProcessedBuffer = inputCopy;
-                        processingInFlight = false;
-                    });
-                }
-                // Log performance every ~4 seconds
-                if (frameCount % 100 === 0) {
-                    const elapsed = (performance.now() - startTime) / 1000;
-                    const fps = frameCount / elapsed;
-                    console.log(`🎤 [ML] BiLSTM: ${frameCount} frames @ ${fps.toFixed(1)} fps`);
-                }
-            };
-            // Connect: source -> highpass -> BiLSTM processor -> destination
-            source.connect(this.highPassFilter);
-            this.highPassFilter.connect(this.processingNode);
-            this.processingNode.connect(destination);
-            console.log("✅ [ML] Pipeline: mic → highpass(80Hz) → BiLSTM(256x2) → output");
-            console.log("✅ [ML] Latency: ~42ms, Sample rate: 48kHz");
-            return destination.stream;
-        }
-        catch (error) {
-            console.error("❌ [ML] Failed to process MediaStream:", error);
-            return inputStream;
-        }
-    }
-    /**
-     * Cleanup resources
-     */
-    dispose() {
-        if (this.processingNode) {
-            this.processingNode.disconnect();
-            this.processingNode = null;
-        }
-        if (this.highPassFilter) {
-            this.highPassFilter.disconnect();
-            this.highPassFilter = null;
-        }
-        if (this.model) {
-            this.model.dispose();
-            this.model = null;
-        }
-        this.frameBuffer = [];
-        this.prevMask = null;
-        this.melFilterbank = null;
-        this.isInitialized = false;
-        console.log("🗑️  ML Noise Suppressor disposed");
-    }
-    /**
-     * Check if initialized
-     */
-    isReady() {
-        return this.isInitialized;
-    }
-}
-exports.MLNoiseSuppressor = MLNoiseSuppressor;

package/dist/UltimateMLNoiseSuppressor.d.ts

@@ -1,74 +0,0 @@
-/**
- * ULTIMATE ML Noise Suppressor - Enhanced for Apple/Google Meet Quality
- * Features:
- * 1. Temporal smoothing (exponential moving average)
- * 2. Voice frequency preservation (80-500 Hz)
- * 3. Sub-bass filtering (remove < 80 Hz)
- * 4. Adaptive processing
- * 5. WebAssembly acceleration
- */
-export declare class UltimateMLNoiseSuppressor {
-    private model;
-    private config;
-    private normStats;
-    private audioContext;
-    private isInitialized;
-    private prevMask;
-    private readonly SMOOTHING_ALPHA;
-    private highPassFilter;
-    private voiceBandFilter;
-    private processingQueue;
-    private isProcessing;
-    /**
-     * Initialize with enhanced setup
-     */
-    initialize(modelUrl: string, audioContext: AudioContext): Promise<void>;
-    /**
-     * Setup filters for voice frequency preservation
-     */
-    private setupVoiceFilters;
-    /**
-     * Process audio with ULTIMATE quality
-     * NOTE: This runs in the AudioWorklet thread. It must be synchronous and fast.
-     * The heavy ML inference should ideally happen in a Worker, communicating via SharedArrayBuffer.
-     * For this implementation, we use a simplified frame-based approach.
-     */
-    processAudio(inputBuffer: Float32Array): Float32Array;
-    /**
-     * Placeholder for async processing (to be moved to a Web Worker)
-     */
-    processFrameAsync(inputBuffer: Float32Array): Promise<void>;
-    /**
-     * CRITICAL: Temporal smoothing (biggest quality improvement!)
-     */
-    private applyTemporalSmoothing;
-    /**
-     * Apply high-pass filter to remove rumble
-     */
-    private applyHighPassFilter;
-    /**
-     * Apply mask with voice frequency preservation
-     */
-    private applyMaskWithVoicePreservation;
-    /**
-     * Extract mel-spectrogram features
-     */
-    private extractMelFeatures;
-    /**
-     * Compute mel bin (simplified)
-     */
-    private computeMelBin;
-    /**
-     * Create sequences for LSTM input
-     */
-    private createSequences;
-    /**
-     * Reset processing state (call when switching audio streams)
-     */
-    reset(): void;
-    /**
-     * Get processing latency
-     */
-    getLatency(): number;
-}
-export default UltimateMLNoiseSuppressor;