@newgameplusinc/odyssey-audio-video-sdk-dev 1.0.9 → 1.0.11

package/dist/SpatialAudioManager.d.ts

@@ -1,13 +1,35 @@
 import { EventManager } from "./EventManager";
 import { Position } from "./types";
+type SpatialAudioDistanceConfig = {
+    refDistance?: number;
+    maxDistance?: number;
+    rolloffFactor?: number;
+    unit?: "auto" | "meters" | "centimeters";
+};
+type DenoiserOptions = {
+    enabled?: boolean;
+    threshold?: number;
+    noiseFloor?: number;
+    release?: number;
+};
+type SpatialAudioOptions = {
+    distance?: SpatialAudioDistanceConfig;
+    denoiser?: DenoiserOptions;
+};
 export declare class SpatialAudioManager extends EventManager {
     private audioContext;
     private participantNodes;
     private masterGainNode;
     private monitoringIntervals;
     private compressor;
+    private options;
+    private denoiseWorkletReady;
+    private denoiseWorkletUrl?;
+    private denoiserWasmBytes?;
+    private listenerPosition;
+    private listenerInitialized;
     private listenerDirection;
-    constructor();
+    constructor(options?: SpatialAudioOptions);
     getAudioContext(): AudioContext;
     /**
      * Setup spatial audio for a participant
@@ -24,7 +46,7 @@ export declare class SpatialAudioManager extends EventManager {
      * @param track Audio track from MediaSoup consumer
      * @param bypassSpatialization For testing - bypasses 3D positioning
      */
-    setupSpatialAudioForParticipant(participantId: string, track: MediaStreamTrack, bypassSpatialization?: boolean): void;
+    setupSpatialAudioForParticipant(participantId: string, track: MediaStreamTrack, bypassSpatialization?: boolean): Promise<void>;
     private startMonitoring;
     /**
      * Update spatial audio position and orientation for a participant
@@ -65,7 +87,17 @@ export declare class SpatialAudioManager extends EventManager {
      * @param lookAtPos Look-at position (where camera is pointing)
      */
     setListenerFromLSD(listenerPos: Position, cameraPos: Position, lookAtPos: Position): void;
+    private applyListenerTransform;
     removeParticipant(participantId: string): void;
     resumeAudioContext(): Promise<void>;
     getAudioContextState(): AudioContextState;
+    private getDistanceConfig;
+    private applySpatialBoostIfNeeded;
+    private getDistanceBetween;
+    private calculateDistanceGain;
+    private normalizePositionUnits;
+    private isDenoiserEnabled;
+    private ensureDenoiseWorklet;
+    private resolveOptions;
 }
+export {};

package/dist/SpatialAudioManager.js

@@ -3,14 +3,18 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.SpatialAudioManager = void 0;
 const EventManager_1 = require("./EventManager");
 class SpatialAudioManager extends EventManager_1.EventManager {
-    constructor() {
+    constructor(options) {
         super();
         this.participantNodes = new Map();
         this.monitoringIntervals = new Map();
+        this.denoiseWorkletReady = null;
+        this.listenerPosition = { x: 0, y: 0, z: 0 };
+        this.listenerInitialized = false;
         this.listenerDirection = {
             forward: { x: 0, y: 1, z: 0 },
             up: { x: 0, y: 0, z: 1 },
         };
+        this.options = this.resolveOptions(options);
         // Use high sample rate for best audio quality
         this.audioContext = new AudioContext({ sampleRate: 48000 });
         // Master gain
@@ -54,10 +58,10 @@ class SpatialAudioManager extends EventManager_1.EventManager {
      * @param track Audio track from MediaSoup consumer
      * @param bypassSpatialization For testing - bypasses 3D positioning
      */
-    setupSpatialAudioForParticipant(participantId, track, bypassSpatialization = false // Default to false
+    async setupSpatialAudioForParticipant(participantId, track, bypassSpatialization = false // Default to false
     ) {
         if (this.audioContext.state === "suspended") {
-            this.audioContext.resume();
+            await this.audioContext.resume();
         }
         // Create stream with noise suppression constraints
         const stream = new MediaStream([track]);
@@ -65,6 +69,29 @@ class SpatialAudioManager extends EventManager_1.EventManager {
         const panner = this.audioContext.createPanner();
         const analyser = this.audioContext.createAnalyser();
         const gain = this.audioContext.createGain();
+        let denoiseNode;
+        if (this.isDenoiserEnabled() && typeof this.audioContext.audioWorklet !== "undefined") {
+            try {
+                await this.ensureDenoiseWorklet();
+                denoiseNode = new AudioWorkletNode(this.audioContext, "odyssey-denoise", {
+                    numberOfInputs: 1,
+                    numberOfOutputs: 1,
+                    processorOptions: {
+                        enabled: this.options.denoiser?.enabled !== false,
+                        threshold: this.options.denoiser?.threshold,
+                        noiseFloor: this.options.denoiser?.noiseFloor,
+                        release: this.options.denoiser?.release,
+                        wasmBytes: this.denoiserWasmBytes
+                            ? this.denoiserWasmBytes.slice(0)
+                            : null,
+                    },
+                });
+            }
+            catch (error) {
+                console.warn("⚠️ Failed to initialize denoiser worklet. Falling back to raw audio.", error);
+                denoiseNode = undefined;
+            }
+        }
         // Create BiquadFilter nodes for static/noise reduction
         // Based on: https://tagdiwalaviral.medium.com/struggles-of-noise-reduction-in-rtc-part-2-2526f8179442
         const highpassFilter = this.audioContext.createBiquadFilter();
@@ -76,28 +103,32 @@ class SpatialAudioManager extends EventManager_1.EventManager {
         lowpassFilter.frequency.value = 7500; // Below 8kHz to avoid flat/muffled sound
         lowpassFilter.Q.value = 1.0; // Quality factor
         // Configure Panner for realistic 3D spatial audio
+        const distanceConfig = this.getDistanceConfig();
         panner.panningModel = "HRTF"; // Head-Related Transfer Function for realistic 3D
         panner.distanceModel = "inverse"; // Natural distance falloff
-        panner.refDistance = 100; // Distance at which volume = 1.0 (1 meter in Unreal = 100cm)
-        panner.maxDistance = 1500; // Maximum audible distance (15 meters)
-        panner.rolloffFactor = 1.5; // How quickly sound fades with distance
+        panner.refDistance = distanceConfig.refDistance ?? 1.2;
+        panner.maxDistance = distanceConfig.maxDistance ?? 30;
+        panner.rolloffFactor = distanceConfig.rolloffFactor ?? 1.35; // How quickly sound fades with distance
         panner.coneInnerAngle = 360; // Omnidirectional sound source
         panner.coneOuterAngle = 360;
         panner.coneOuterGain = 0.3; // Some sound even outside cone
         // Configure gain for individual participant volume control
         gain.gain.value = 1.0;
+        let currentNode = source;
+        if (denoiseNode) {
+            currentNode.connect(denoiseNode);
+            currentNode = denoiseNode;
+        }
+        currentNode.connect(highpassFilter);
+        highpassFilter.connect(lowpassFilter);
         if (bypassSpatialization) {
             console.log(`🔊 TESTING: Connecting audio directly to destination (bypassing spatial audio) for ${participantId}`);
-            source.connect(highpassFilter);
-            highpassFilter.connect(lowpassFilter);
             lowpassFilter.connect(analyser);
             analyser.connect(this.masterGainNode);
         }
         else {
             // Standard spatialized path with full audio chain
             // Audio Chain: source -> filters -> panner -> analyser -> gain -> masterGain -> compressor -> destination
-            source.connect(highpassFilter);
-            highpassFilter.connect(lowpassFilter);
             lowpassFilter.connect(panner);
             panner.connect(analyser);
             analyser.connect(gain);
@@ -110,6 +141,7 @@ class SpatialAudioManager extends EventManager_1.EventManager {
             gain,
             highpassFilter,
             lowpassFilter,
+            denoiseNode,
             stream,
         });
         console.log(`🎧 Spatial audio setup complete for ${participantId}:`, {
@@ -197,10 +229,13 @@ class SpatialAudioManager extends EventManager_1.EventManager {
     updateSpatialAudio(participantId, position, direction) {
         const nodes = this.participantNodes.get(participantId);
         if (nodes?.panner) {
+            const distanceConfig = this.getDistanceConfig();
+            const normalizedPosition = this.normalizePositionUnits(position);
+            const targetPosition = this.applySpatialBoostIfNeeded(normalizedPosition);
             // Update position (where the sound is coming from)
-            nodes.panner.positionX.setValueAtTime(position.x, this.audioContext.currentTime);
-            nodes.panner.positionY.setValueAtTime(position.y, this.audioContext.currentTime);
-            nodes.panner.positionZ.setValueAtTime(position.z, this.audioContext.currentTime);
+            nodes.panner.positionX.setValueAtTime(targetPosition.x, this.audioContext.currentTime);
+            nodes.panner.positionY.setValueAtTime(targetPosition.y, this.audioContext.currentTime);
+            nodes.panner.positionZ.setValueAtTime(targetPosition.z, this.audioContext.currentTime);
             // Update orientation (where the participant is facing)
             // This makes the audio source directional based on participant's direction
             if (direction) {
@@ -217,66 +252,36 @@ class SpatialAudioManager extends EventManager_1.EventManager {
                     nodes.panner.orientationZ.setValueAtTime(normZ, this.audioContext.currentTime);
                 }
             }
-        }
-    }
-    setListenerPosition(position, orientation) {
-        const { listener } = this.audioContext;
-        if (listener) {
-            // Store listener direction for reference
-            this.listenerDirection = {
-                forward: {
-                    x: orientation.forwardX,
-                    y: orientation.forwardY,
-                    z: orientation.forwardZ,
-                },
-                up: {
-                    x: orientation.upX,
-                    y: orientation.upY,
-                    z: orientation.upZ,
-                },
-            };
-            // Use setPosition and setOrientation for atomic updates if available
-            if (listener.positionX) {
-                listener.positionX.setValueAtTime(position.x, this.audioContext.currentTime);
-                listener.positionY.setValueAtTime(position.y, this.audioContext.currentTime);
-                listener.positionZ.setValueAtTime(position.z, this.audioContext.currentTime);
-            }
-            if (listener.forwardX) {
-                listener.forwardX.setValueAtTime(orientation.forwardX, this.audioContext.currentTime);
-                listener.forwardY.setValueAtTime(orientation.forwardY, this.audioContext.currentTime);
-                listener.forwardZ.setValueAtTime(orientation.forwardZ, this.audioContext.currentTime);
-                listener.upX.setValueAtTime(orientation.upX, this.audioContext.currentTime);
-                listener.upY.setValueAtTime(orientation.upY, this.audioContext.currentTime);
-                listener.upZ.setValueAtTime(orientation.upZ, this.audioContext.currentTime);
-            }
-            // Log spatial audio updates occasionally
-            if (Math.random() < 0.01) {
-                console.log(`🎧 [Spatial Audio] Listener updated:`, {
-                    position: { x: position.x.toFixed(1), y: position.y.toFixed(1), z: position.z.toFixed(1) },
-                    forward: {
-                        x: orientation.forwardX.toFixed(2),
-                        y: orientation.forwardY.toFixed(2),
-                        z: orientation.forwardZ.toFixed(2),
-                    },
-                    up: {
-                        x: orientation.upX.toFixed(2),
-                        y: orientation.upY.toFixed(2),
-                        z: orientation.upZ.toFixed(2),
-                    },
+            const listenerPos = this.listenerPosition;
+            const distance = this.getDistanceBetween(listenerPos, targetPosition);
+            const distanceGain = this.calculateDistanceGain(distanceConfig, distance);
+            nodes.gain.gain.setTargetAtTime(distanceGain, this.audioContext.currentTime, 0.05);
+            if (Math.random() < 0.02) {
+                console.log("🎚️ [Spatial Audio] Distance gain", {
+                    participantId,
+                    distance: distance.toFixed(2),
+                    gain: distanceGain.toFixed(2),
                 });
             }
         }
     }
+    setListenerPosition(position, orientation) {
+        const normalizedPosition = this.normalizePositionUnits(position);
+        this.applyListenerTransform(normalizedPosition, orientation);
+    }
     /**
      * Update listener orientation from LSD camera direction
      * @param cameraPos Camera position in world space
      * @param lookAtPos Look-at position (where camera is pointing)
      */
     setListenerFromLSD(listenerPos, cameraPos, lookAtPos) {
+        const normalizedListener = this.normalizePositionUnits(listenerPos);
+        const normalizedCamera = this.normalizePositionUnits(cameraPos);
+        const normalizedLookAt = this.normalizePositionUnits(lookAtPos);
         // Calculate forward vector (from camera to look-at point)
-        const forwardX = lookAtPos.x - cameraPos.x;
-        const forwardY = lookAtPos.y - cameraPos.y;
-        const forwardZ = lookAtPos.z - cameraPos.z;
+        const forwardX = normalizedLookAt.x - normalizedCamera.x;
+        const forwardY = normalizedLookAt.y - normalizedCamera.y;
+        const forwardZ = normalizedLookAt.z - normalizedCamera.z;
         // Normalize forward vector
         const forwardLen = Math.sqrt(forwardX * forwardX + forwardY * forwardY + forwardZ * forwardZ);
         if (forwardLen < 0.001) {
@@ -294,7 +299,7 @@ class SpatialAudioManager extends EventManager_1.EventManager {
         const rightLen = Math.sqrt(rightX * rightX + rightY * rightY + rightZ * rightZ);
         if (rightLen < 0.001) {
             // Forward is parallel to world up, use fallback
-            this.setListenerPosition(listenerPos, {
+            this.applyListenerTransform(normalizedListener, {
                 forwardX: fwdX,
                 forwardY: fwdY,
                 forwardZ: fwdZ,
@@ -311,7 +316,7 @@ class SpatialAudioManager extends EventManager_1.EventManager {
         const upX = fwdY * rZ - fwdZ * rY;
         const upY = fwdZ * rX - fwdX * rZ;
         const upZ = fwdX * rY - fwdY * rX;
-        this.setListenerPosition(listenerPos, {
+        this.applyListenerTransform(normalizedListener, {
             forwardX: fwdX,
             forwardY: fwdY,
             forwardZ: fwdZ,
@@ -320,6 +325,58 @@ class SpatialAudioManager extends EventManager_1.EventManager {
             upZ,
         });
     }
+    applyListenerTransform(normalizedPosition, orientation) {
+        const { listener } = this.audioContext;
+        if (!listener) {
+            return;
+        }
+        this.listenerPosition = { ...normalizedPosition };
+        this.listenerInitialized = true;
+        this.listenerDirection = {
+            forward: {
+                x: orientation.forwardX,
+                y: orientation.forwardY,
+                z: orientation.forwardZ,
+            },
+            up: {
+                x: orientation.upX,
+                y: orientation.upY,
+                z: orientation.upZ,
+            },
+        };
+        if (listener.positionX) {
+            listener.positionX.setValueAtTime(normalizedPosition.x, this.audioContext.currentTime);
+            listener.positionY.setValueAtTime(normalizedPosition.y, this.audioContext.currentTime);
+            listener.positionZ.setValueAtTime(normalizedPosition.z, this.audioContext.currentTime);
+        }
+        if (listener.forwardX) {
+            listener.forwardX.setValueAtTime(orientation.forwardX, this.audioContext.currentTime);
+            listener.forwardY.setValueAtTime(orientation.forwardY, this.audioContext.currentTime);
+            listener.forwardZ.setValueAtTime(orientation.forwardZ, this.audioContext.currentTime);
+            listener.upX.setValueAtTime(orientation.upX, this.audioContext.currentTime);
+            listener.upY.setValueAtTime(orientation.upY, this.audioContext.currentTime);
+            listener.upZ.setValueAtTime(orientation.upZ, this.audioContext.currentTime);
+        }
+        if (Math.random() < 0.01) {
+            console.log(`🎧 [Spatial Audio] Listener updated:`, {
+                position: {
+                    x: normalizedPosition.x.toFixed(2),
+                    y: normalizedPosition.y.toFixed(2),
+                    z: normalizedPosition.z.toFixed(2),
+                },
+                forward: {
+                    x: orientation.forwardX.toFixed(2),
+                    y: orientation.forwardY.toFixed(2),
+                    z: orientation.forwardZ.toFixed(2),
+                },
+                up: {
+                    x: orientation.upX.toFixed(2),
+                    y: orientation.upY.toFixed(2),
+                    z: orientation.upZ.toFixed(2),
+                },
+            });
+        }
+    }
     removeParticipant(participantId) {
         // Stop monitoring
         if (this.monitoringIntervals.has(participantId)) {
@@ -332,6 +389,9 @@ class SpatialAudioManager extends EventManager_1.EventManager {
             nodes.panner.disconnect();
             nodes.analyser.disconnect();
             nodes.gain.disconnect();
+            if (nodes.denoiseNode) {
+                nodes.denoiseNode.disconnect();
+            }
             nodes.stream.getTracks().forEach((track) => track.stop());
             this.participantNodes.delete(participantId);
             console.log(`🗑️ Removed participant ${participantId} from spatial audio.`);
@@ -346,5 +406,189 @@ class SpatialAudioManager extends EventManager_1.EventManager {
     getAudioContextState() {
         return this.audioContext.state;
     }
+    getDistanceConfig() {
+        return {
+            refDistance: this.options.distance?.refDistance ?? 1.2,
+            maxDistance: this.options.distance?.maxDistance ?? 30,
+            rolloffFactor: this.options.distance?.rolloffFactor ?? 1.35,
+            unit: this.options.distance?.unit ?? "auto",
+        };
+    }
+    applySpatialBoostIfNeeded(position) {
+        if (!this.listenerInitialized) {
+            return position;
+        }
+        const boost = (this.options.distance?.rolloffFactor || 1) * 0.85;
+        if (!isFinite(boost) || boost <= 1.01) {
+            return position;
+        }
+        const listener = this.listenerPosition;
+        return {
+            x: listener.x + (position.x - listener.x) * boost,
+            y: listener.y + (position.y - listener.y) * Math.min(boost, 1.2),
+            z: listener.z + (position.z - listener.z) * boost,
+        };
+    }
+    getDistanceBetween(a, b) {
+        const dx = b.x - a.x;
+        const dy = b.y - a.y;
+        const dz = b.z - a.z;
+        return Math.sqrt(dx * dx + dy * dy + dz * dz);
+    }
+    calculateDistanceGain(config, distance) {
+        if (!this.listenerInitialized) {
+            return 1;
+        }
+        if (distance <= config.refDistance) {
+            return 1;
+        }
+        if (distance >= config.maxDistance) {
+            return 0;
+        }
+        const normalized = (distance - config.refDistance) /
+            Math.max(config.maxDistance - config.refDistance, 0.001);
+        const shaped = Math.pow(Math.max(0, 1 - normalized), Math.max(1.2, config.rolloffFactor * 1.05));
+        return Math.min(1, Math.max(0.01, shaped));
+    }
+    normalizePositionUnits(position) {
+        const distanceConfig = this.getDistanceConfig();
+        if (distanceConfig.unit === "meters") {
+            return { ...position };
+        }
+        if (distanceConfig.unit === "centimeters") {
+            return {
+                x: position.x / 100,
+                y: position.y / 100,
+                z: position.z / 100,
+            };
+        }
+        const maxAxis = Math.max(Math.abs(position.x), Math.abs(position.y), Math.abs(position.z));
+        if (maxAxis > 50) {
+            // Likely centimeters coming from server
+            return {
+                x: position.x / 100,
+                y: position.y / 100,
+                z: position.z / 100,
+            };
+        }
+        return { ...position };
+    }
+    isDenoiserEnabled() {
+        return this.options.denoiser?.enabled !== false;
+    }
+    async ensureDenoiseWorklet() {
+        if (!this.isDenoiserEnabled()) {
+            return;
+        }
+        if (!("audioWorklet" in this.audioContext)) {
+            console.warn("⚠️ AudioWorklet not supported in this browser. Disabling denoiser.");
+            this.options.denoiser = {
+                ...(this.options.denoiser || {}),
+                enabled: false,
+            };
+            return;
+        }
+        if (this.denoiseWorkletReady) {
+            return this.denoiseWorkletReady;
+        }
+        const processorSource = `class OdysseyDenoiseProcessor extends AudioWorkletProcessor {
+  constructor(options) {
+    super();
+    const cfg = (options && options.processorOptions) || {};
+    this.enabled = cfg.enabled !== false;
+    this.threshold = typeof cfg.threshold === 'number' ? cfg.threshold : 0.012;
+    this.noiseFloor = typeof cfg.noiseFloor === 'number' ? cfg.noiseFloor : 0.004;
+    this.release = typeof cfg.release === 'number' ? cfg.release : 0.18;
+    this.smoothedLevel = this.noiseFloor;
+  }
+
+  process(inputs, outputs) {
+    const input = inputs[0];
+    const output = outputs[0];
+    if (!input || input.length === 0 || !output || output.length === 0) {
+      return true;
+    }
+
+    for (let channel = 0; channel < input.length; channel++) {
+      const inChannel = input[channel];
+      const outChannel = output[channel];
+      if (!inChannel || !outChannel) {
+        continue;
+      }
+
+      let sum = 0;
+      for (let i = 0; i < inChannel.length; i++) {
+        const sample = inChannel[i];
+        sum += sample * sample;
+      }
+
+      const rms = Math.sqrt(sum / inChannel.length);
+      this.smoothedLevel += (rms - this.smoothedLevel) * this.release;
+      const dynamicThreshold = Math.max(
+        this.noiseFloor,
+        this.threshold * 0.6 + this.smoothedLevel * 0.4
+      );
+
+      if (!this.enabled || rms >= dynamicThreshold) {
+        for (let i = 0; i < inChannel.length; i++) {
+          outChannel[i] = inChannel[i];
+        }
+      } else {
+        for (let i = 0; i < inChannel.length; i++) {
+          outChannel[i] = 0;
+        }
+      }
+    }
+
+    return true;
+  }
+}
+
+registerProcessor('odyssey-denoise', OdysseyDenoiseProcessor);
+`;
+        const blob = new Blob([processorSource], {
+            type: "application/javascript",
+        });
+        this.denoiseWorkletUrl = URL.createObjectURL(blob);
+        this.denoiseWorkletReady = this.audioContext.audioWorklet
+            .addModule(this.denoiseWorkletUrl)
+            .catch((error) => {
+            console.error("❌ Failed to register denoise worklet", error);
+            this.options.denoiser = {
+                ...(this.options.denoiser || {}),
+                enabled: false,
+            };
+            throw error;
+        });
+        return this.denoiseWorkletReady;
+    }
+    resolveOptions(options) {
+        const distanceDefaults = {
+            refDistance: 1.2,
+            maxDistance: 30,
+            rolloffFactor: 1.35,
+            unit: "auto",
+        };
+        const denoiserDefaults = {
+            enabled: true,
+            threshold: 0.012,
+            noiseFloor: 0.004,
+            release: 0.18,
+        };
+        return {
+            distance: {
+                refDistance: options?.distance?.refDistance ?? distanceDefaults.refDistance,
+                maxDistance: options?.distance?.maxDistance ?? distanceDefaults.maxDistance,
+                rolloffFactor: options?.distance?.rolloffFactor ?? distanceDefaults.rolloffFactor,
+                unit: options?.distance?.unit ?? distanceDefaults.unit,
+            },
+            denoiser: {
+                enabled: options?.denoiser?.enabled ?? denoiserDefaults.enabled,
+                threshold: options?.denoiser?.threshold ?? denoiserDefaults.threshold,
+                noiseFloor: options?.denoiser?.noiseFloor ?? denoiserDefaults.noiseFloor,
+                release: options?.denoiser?.release ?? denoiserDefaults.release,
+            },
+        };
+    }
 }
 exports.SpatialAudioManager = SpatialAudioManager;

package/dist/index.js

@@ -425,7 +425,7 @@ class OdysseySpatialComms extends EventManager_1.EventManager {
                     else {
                         console.log("🎧 SDK: Setting up spatial audio for REMOTE participant", participant.participantId);
                         // Setup spatial audio with full 3D positioning
-                        this.spatialAudioManager.setupSpatialAudioForParticipant(participant.participantId, track, false // Enable spatial audio
+                        await this.spatialAudioManager.setupSpatialAudioForParticipant(participant.participantId, track, false // Enable spatial audio
                         );
                         // Update spatial audio position for this participant
                         this.spatialAudioManager.updateSpatialAudio(participant.participantId, data.position);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@newgameplusinc/odyssey-audio-video-sdk-dev",
-  "version": "1.0.9",
+  "version": "1.0.11",
   "description": "Odyssey Spatial Audio & Video SDK using MediaSoup for real-time communication",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",