@newgameplusinc/odyssey-audio-video-sdk-dev 1.0.55 → 1.0.57

package/dist/MLNoiseSuppressor.js

@@ -351,26 +351,48 @@ class MLNoiseSuppressor {
             this.processingNode = this.audioContext.createScriptProcessor(bufferSize, 1, 1);
             let frameCount = 0;
             const startTime = performance.now();
+            // Double-buffering for async ML processing
+            // We store the PREVIOUS processed result and output it in the NEXT callback
+            // This adds one buffer of latency but ensures we never output zeros
+            let previousProcessedBuffer = null;
+            let processingInFlight = false;
             // Process audio frames with ML model
-            this.processingNode.onaudioprocess = async (event) => {
+            // IMPORTANT: onaudioprocess is synchronous! We use double-buffering to handle async ML
+            this.processingNode.onaudioprocess = (event) => {
                 const inputData = event.inputBuffer.getChannelData(0);
                 const outputData = event.outputBuffer.getChannelData(0);
                 frameCount++;
-                try {
-                    // Process with BiLSTM model
-                    const processed = await this.processAudio(new Float32Array(inputData));
-                    outputData.set(processed);
-                    // Log performance every ~4 seconds
-                    if (frameCount % 100 === 0) {
-                        const elapsed = (performance.now() - startTime) / 1000;
-                        const fps = frameCount / elapsed;
-                        console.log(`🎤 [ML] BiLSTM: ${frameCount} frames @ ${fps.toFixed(1)} fps`);
-                    }
+                // OUTPUT: Use previously processed audio (or passthrough if not ready yet)
+                if (previousProcessedBuffer) {
+                    outputData.set(previousProcessedBuffer);
                 }
-                catch (error) {
-                    // On error, pass through original audio
+                else {
+                    // First frame or ML not ready - pass through original audio
                     outputData.set(inputData);
                 }
+                // PROCESS: Start async ML processing for the NEXT frame
+                // Only start new processing if previous one is complete
+                if (!processingInFlight) {
+                    processingInFlight = true;
+                    const inputCopy = new Float32Array(inputData);
+                    // Fire-and-forget async processing
+                    this.processAudio(inputCopy)
+                        .then((processed) => {
+                        previousProcessedBuffer = processed;
+                        processingInFlight = false;
+                    })
+                        .catch((error) => {
+                        // On error, store the original audio for passthrough
+                        previousProcessedBuffer = inputCopy;
+                        processingInFlight = false;
+                    });
+                }
+                // Log performance every ~4 seconds
+                if (frameCount % 100 === 0) {
+                    const elapsed = (performance.now() - startTime) / 1000;
+                    const fps = frameCount / elapsed;
+                    console.log(`🎤 [ML] BiLSTM: ${frameCount} frames @ ${fps.toFixed(1)} fps`);
+                }
             };
             // Connect: source -> highpass -> BiLSTM processor -> destination
             source.connect(this.highPassFilter);

package/dist/MediasoupManager.js

@@ -41,7 +41,7 @@ class MediasoupManager {
         this.recvTransport = null;
         this.producers = new Map();
         this.consumers = new Map();
-        this.participantId = '';
+        this.participantId = "";
         this.socket = socket;
         this.device = new mediasoupClient.Device();
     }
@@ -100,7 +100,7 @@ class MediasoupManager {
                 // Emit event so parent SDK can recreate producers
                 this.socket.emit("transport-failed", {
                     participantId: this.participantId,
-                    direction: "send"
+                    direction: "send",
                 });
             }
         });
@@ -126,30 +126,46 @@ class MediasoupManager {
     async produce(track, appData) {
         if (!this.sendTransport)
             throw new Error("Send transport not initialized");
+        console.log(`📤 [MediaSoup] Producing ${track.kind} track (transport state: ${this.sendTransport.connectionState})`);
         // Configure simulcast for video tracks for adaptive bitrate
         const produceOptions = { track, appData };
-        if (track.kind === 'video') {
+        if (track.kind === "video") {
             produceOptions.encodings = [
                 // Low quality layer - 100 kbps, good for poor connections
-                { rid: 'r0', active: true, maxBitrate: 100000, scaleResolutionDownBy: 4 },
+                {
+                    rid: "r0",
+                    active: true,
+                    maxBitrate: 100000,
+                    scaleResolutionDownBy: 4,
+                },
                 // Medium quality layer - 300 kbps, balanced
-                { rid: 'r1', active: true, maxBitrate: 300000, scaleResolutionDownBy: 2 },
+                {
+                    rid: "r1",
+                    active: true,
+                    maxBitrate: 300000,
+                    scaleResolutionDownBy: 2,
+                },
                 // High quality layer - 900 kbps, full resolution
-                { rid: 'r2', active: true, maxBitrate: 900000, scaleResolutionDownBy: 1 }
+                {
+                    rid: "r2",
+                    active: true,
+                    maxBitrate: 900000,
+                    scaleResolutionDownBy: 1,
+                },
             ];
             // VP8 codec for simulcast support
             produceOptions.codecOptions = {
-                videoGoogleStartBitrate: 1000
+                videoGoogleStartBitrate: 1000,
             };
         }
         const producer = await this.sendTransport.produce(produceOptions);
         // Handle producer events
-        producer.on('transportclose', () => {
+        producer.on("transportclose", () => {
             console.warn(`⚠️ [MediaSoup] Producer ${producer.id.substring(0, 8)} transport closed`);
             this.producers.delete(producer.id);
             // The main SDK (index.ts) should handle recreation
         });
-        producer.on('trackended', () => {
+        producer.on("trackended", () => {
             console.warn(`⚠️ [MediaSoup] Producer ${producer.id.substring(0, 8)} track ended`);
             producer.close();
             this.producers.delete(producer.id);
@@ -167,12 +183,12 @@ class MediasoupManager {
             rtpParameters: data.rtpParameters,
         });
         // Handle consumer events
-        consumer.on('transportclose', () => {
+        consumer.on("transportclose", () => {
             console.warn(`⚠️ [MediaSoup] Consumer ${consumer.id.substring(0, 8)} transport closed`);
             this.consumers.delete(consumer.id);
             // The main SDK (index.ts) should handle recreation
         });
-        consumer.on('trackended', () => {
+        consumer.on("trackended", () => {
             console.warn(`⚠️ [MediaSoup] Consumer ${consumer.id.substring(0, 8)} track ended`);
             consumer.close();
             this.consumers.delete(consumer.id);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@newgameplusinc/odyssey-audio-video-sdk-dev",
-  "version": "1.0.55",
+  "version": "1.0.57",
   "description": "Odyssey Spatial Audio & Video SDK using MediaSoup for real-time communication with AI-powered noise suppression",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",