@newgameplusinc/odyssey-audio-video-sdk-dev 1.0.3 → 1.0.4

package/dist/SpatialAudioManager.d.ts

@@ -5,11 +5,52 @@ export declare class SpatialAudioManager extends EventManager {
     private participantNodes;
     private masterGainNode;
     private monitoringIntervals;
+    private compressor;
+    private listenerDirection;
     constructor();
     getAudioContext(): AudioContext;
+    /**
+     * Setup spatial audio for a participant
+     *
+     * CRITICAL: Each participant gets their OWN audio processing chain:
+     * Stream -> Source -> Panner -> Analyser -> Gain -> Compressor -> Output
+     *
+     * This ensures:
+     * - Each voice is positioned independently in 3D space
+     * - No server-side mixing required
+     * - Scalable to unlimited participants (browser handles the mixing)
+     *
+     * @param participantId Unique ID for this participant
+     * @param track Audio track from MediaSoup consumer
+     * @param bypassSpatialization For testing - bypasses 3D positioning
+     */
     setupSpatialAudioForParticipant(participantId: string, track: MediaStreamTrack, bypassSpatialization?: boolean): void;
     private startMonitoring;
-    updateSpatialAudio(participantId: string, position: Position): void;
+    /**
+     * Update spatial audio position and orientation for a participant
+     *
+     * This is called every time we receive position/direction updates from the server.
+     *
+     * Position: Where the participant is in 3D space (their location)
+     * Direction: Which way they're facing (their forward vector)
+     *
+     * Example:
+     * - Position: (x: -200, y: 0, z: 100) = 2m to your left
+     * - Direction: (x: 0, y: 1, z: 0) = facing forward (away from you)
+     * - Result: Sound comes from your left, oriented as if speaking away
+     *
+     * The Web Audio API's PannerNode uses HRTF to create realistic 3D audio
+     * based on these parameters plus the listener's position/orientation.
+     *
+     * @param participantId Who to update
+     * @param position Where they are (from socket data)
+     * @param direction Which way they're facing (from socket data)
+     */
+    updateSpatialAudio(participantId: string, position: Position, direction?: {
+        x: number;
+        y: number;
+        z: number;
+    }): void;
     setListenerPosition(position: Position, orientation: {
         forwardX: number;
         forwardY: number;
@@ -18,6 +59,12 @@ export declare class SpatialAudioManager extends EventManager {
         upY: number;
         upZ: number;
     }): void;
+    /**
+     * Update listener orientation from LSD camera direction
+     * @param cameraPos Camera position in world space
+     * @param lookAtPos Look-at position (where camera is pointing)
+     */
+    setListenerFromLSD(listenerPos: Position, cameraPos: Position, lookAtPos: Position): void;
     removeParticipant(participantId: string): void;
     resumeAudioContext(): Promise<void>;
     getAudioContextState(): AudioContextState;

package/dist/SpatialAudioManager.js

@@ -7,54 +7,82 @@ class SpatialAudioManager extends EventManager_1.EventManager {
         super();
         this.participantNodes = new Map();
         this.monitoringIntervals = new Map();
-        this.audioContext = new AudioContext();
+        this.listenerDirection = {
+            forward: { x: 0, y: 1, z: 0 },
+            up: { x: 0, y: 0, z: 1 },
+        };
+        // Use high sample rate for best audio quality
+        this.audioContext = new AudioContext({ sampleRate: 48000 });
+        // Master gain
         this.masterGainNode = this.audioContext.createGain();
-        this.masterGainNode.gain.value = 5.0; // Set master gain to 5.0
-        this.masterGainNode.connect(this.audioContext.destination);
-        console.log(`🔊 SpatialAudioManager initialized, gain: ${this.masterGainNode.gain.value} audioContext state: ${this.audioContext.state}`);
+        this.masterGainNode.gain.value = 5.0;
+        // Compressor for dynamic range control and preventing distortion
+        this.compressor = this.audioContext.createDynamicsCompressor();
+        this.compressor.threshold.value = -24; // dB
+        this.compressor.knee.value = 30; // dB
+        this.compressor.ratio.value = 12; // Compression ratio
+        this.compressor.attack.value = 0.003; // 3ms attack
+        this.compressor.release.value = 0.25; // 250ms release
+        // Connect master chain: masterGain -> compressor -> destination
+        this.masterGainNode.connect(this.compressor);
+        this.compressor.connect(this.audioContext.destination);
+        console.log(`🔊 SpatialAudioManager initialized with advanced audio processing:`, {
+            sampleRate: this.audioContext.sampleRate,
+            gain: this.masterGainNode.gain.value,
+            compressor: {
+                threshold: this.compressor.threshold.value,
+                ratio: this.compressor.ratio.value,
+            },
+            state: this.audioContext.state,
+        });
     }
     getAudioContext() {
         return this.audioContext;
     }
+    /**
+     * Setup spatial audio for a participant
+     *
+     * CRITICAL: Each participant gets their OWN audio processing chain:
+     * Stream -> Source -> Panner -> Analyser -> Gain -> Compressor -> Output
+     *
+     * This ensures:
+     * - Each voice is positioned independently in 3D space
+     * - No server-side mixing required
+     * - Scalable to unlimited participants (browser handles the mixing)
+     *
+     * @param participantId Unique ID for this participant
+     * @param track Audio track from MediaSoup consumer
+     * @param bypassSpatialization For testing - bypasses 3D positioning
+     */
     setupSpatialAudioForParticipant(participantId, track, bypassSpatialization = false // Default to false
     ) {
         if (this.audioContext.state === "suspended") {
             this.audioContext.resume();
         }
+        // Create stream with noise suppression constraints
         const stream = new MediaStream([track]);
-        // 🔍 DIAGNOSTIC TEST: Try playing through raw HTML audio element
-        console.warn(`🧪 DIAGNOSTIC: Testing raw audio playback for ${participantId}`);
-        const testAudio = new Audio();
-        testAudio.srcObject = stream;
-        testAudio.volume = 1.0;
-        testAudio
-            .play()
-            .then(() => {
-            console.log(`✅ DIAGNOSTIC: Raw audio element can play track for ${participantId}`);
-        })
-            .catch((err) => {
-            console.error(`❌ DIAGNOSTIC: Raw audio element CANNOT play:`, err);
-        });
         const source = this.audioContext.createMediaStreamSource(stream);
         const panner = this.audioContext.createPanner();
         const analyser = this.audioContext.createAnalyser();
         const gain = this.audioContext.createGain();
-        // Configure Panner
-        panner.panningModel = "HRTF";
-        panner.distanceModel = "inverse";
-        panner.refDistance = 1;
-        panner.maxDistance = 100;
-        panner.rolloffFactor = 1;
-        panner.coneInnerAngle = 360;
-        panner.coneOuterAngle = 0;
-        panner.coneOuterGain = 0;
+        // Configure Panner for realistic 3D spatial audio
+        panner.panningModel = "HRTF"; // Head-Related Transfer Function for realistic 3D
+        panner.distanceModel = "inverse"; // Natural distance falloff
+        panner.refDistance = 100; // Distance at which volume = 1.0 (1 meter in Unreal = 100cm)
+        panner.maxDistance = 1500; // Maximum audible distance (15 meters)
+        panner.rolloffFactor = 1.5; // How quickly sound fades with distance
+        panner.coneInnerAngle = 360; // Omnidirectional sound source
+        panner.coneOuterAngle = 360;
+        panner.coneOuterGain = 0.3; // Some sound even outside cone
+        // Configure gain for individual participant volume control
+        gain.gain.value = 1.0;
         if (bypassSpatialization) {
             console.log(`🔊 TESTING: Connecting audio directly to destination (bypassing spatial audio) for ${participantId}`);
             source.connect(analyser);
             analyser.connect(this.masterGainNode);
         }
         else {
-            // Standard spatialized path
+            // Standard spatialized path with full audio chain
             source.connect(panner);
             panner.connect(analyser);
             analyser.connect(gain);
@@ -69,11 +97,19 @@ class SpatialAudioManager extends EventManager_1.EventManager {
         });
         console.log(`🎧 Spatial audio setup complete for ${participantId}:`, {
             audioContextState: this.audioContext.state,
+            sampleRate: this.audioContext.sampleRate,
             gain: this.masterGainNode.gain.value,
             trackEnabled: track.enabled,
             trackMuted: track.muted,
             trackReadyState: track.readyState,
             isBypassed: bypassSpatialization,
+            pannerConfig: {
+                model: panner.panningModel,
+                distanceModel: panner.distanceModel,
+                refDistance: panner.refDistance,
+                maxDistance: panner.maxDistance,
+                rolloffFactor: panner.rolloffFactor,
+            },
         });
         // Start monitoring audio levels
         this.startMonitoring(participantId);
@@ -121,17 +157,67 @@ class SpatialAudioManager extends EventManager_1.EventManager {
         }, 2000); // Log every 2 seconds
         this.monitoringIntervals.set(participantId, interval);
     }
-    updateSpatialAudio(participantId, position) {
+    /**
+     * Update spatial audio position and orientation for a participant
+     *
+     * This is called every time we receive position/direction updates from the server.
+     *
+     * Position: Where the participant is in 3D space (their location)
+     * Direction: Which way they're facing (their forward vector)
+     *
+     * Example:
+     * - Position: (x: -200, y: 0, z: 100) = 2m to your left
+     * - Direction: (x: 0, y: 1, z: 0) = facing forward (away from you)
+     * - Result: Sound comes from your left, oriented as if speaking away
+     *
+     * The Web Audio API's PannerNode uses HRTF to create realistic 3D audio
+     * based on these parameters plus the listener's position/orientation.
+     *
+     * @param participantId Who to update
+     * @param position Where they are (from socket data)
+     * @param direction Which way they're facing (from socket data)
+     */
+    updateSpatialAudio(participantId, position, direction) {
         const nodes = this.participantNodes.get(participantId);
         if (nodes?.panner) {
+            // Update position (where the sound is coming from)
             nodes.panner.positionX.setValueAtTime(position.x, this.audioContext.currentTime);
             nodes.panner.positionY.setValueAtTime(position.y, this.audioContext.currentTime);
             nodes.panner.positionZ.setValueAtTime(position.z, this.audioContext.currentTime);
+            // Update orientation (where the participant is facing)
+            // This makes the audio source directional based on participant's direction
+            if (direction) {
+                // Normalize direction vector
+                const length = Math.sqrt(direction.x * direction.x +
+                    direction.y * direction.y +
+                    direction.z * direction.z);
+                if (length > 0.001) {
+                    const normX = direction.x / length;
+                    const normY = direction.y / length;
+                    const normZ = direction.z / length;
+                    nodes.panner.orientationX.setValueAtTime(normX, this.audioContext.currentTime);
+                    nodes.panner.orientationY.setValueAtTime(normY, this.audioContext.currentTime);
+                    nodes.panner.orientationZ.setValueAtTime(normZ, this.audioContext.currentTime);
+                }
+            }
         }
     }
     setListenerPosition(position, orientation) {
         const { listener } = this.audioContext;
         if (listener) {
+            // Store listener direction for reference
+            this.listenerDirection = {
+                forward: {
+                    x: orientation.forwardX,
+                    y: orientation.forwardY,
+                    z: orientation.forwardZ,
+                },
+                up: {
+                    x: orientation.upX,
+                    y: orientation.upY,
+                    z: orientation.upZ,
+                },
+            };
             // Use setPosition and setOrientation for atomic updates if available
             if (listener.positionX) {
                 listener.positionX.setValueAtTime(position.x, this.audioContext.currentTime);
@@ -146,8 +232,77 @@ class SpatialAudioManager extends EventManager_1.EventManager {
                 listener.upY.setValueAtTime(orientation.upY, this.audioContext.currentTime);
                 listener.upZ.setValueAtTime(orientation.upZ, this.audioContext.currentTime);
             }
+            // Log spatial audio updates occasionally
+            if (Math.random() < 0.01) {
+                console.log(`🎧 [Spatial Audio] Listener updated:`, {
+                    position: { x: position.x.toFixed(1), y: position.y.toFixed(1), z: position.z.toFixed(1) },
+                    forward: {
+                        x: orientation.forwardX.toFixed(2),
+                        y: orientation.forwardY.toFixed(2),
+                        z: orientation.forwardZ.toFixed(2),
+                    },
+                    up: {
+                        x: orientation.upX.toFixed(2),
+                        y: orientation.upY.toFixed(2),
+                        z: orientation.upZ.toFixed(2),
+                    },
+                });
+            }
         }
     }
+    /**
+     * Update listener orientation from LSD camera direction
+     * @param cameraPos Camera position in world space
+     * @param lookAtPos Look-at position (where camera is pointing)
+     */
+    setListenerFromLSD(listenerPos, cameraPos, lookAtPos) {
+        // Calculate forward vector (from camera to look-at point)
+        const forwardX = lookAtPos.x - cameraPos.x;
+        const forwardY = lookAtPos.y - cameraPos.y;
+        const forwardZ = lookAtPos.z - cameraPos.z;
+        // Normalize forward vector
+        const forwardLen = Math.sqrt(forwardX * forwardX + forwardY * forwardY + forwardZ * forwardZ);
+        if (forwardLen < 0.001) {
+            console.warn("⚠️ Forward vector too small, using default orientation");
+            return;
+        }
+        const fwdX = forwardX / forwardLen;
+        const fwdY = forwardY / forwardLen;
+        const fwdZ = forwardZ / forwardLen;
+        // Calculate right vector (cross product of world up and forward)
+        const worldUp = { x: 0, y: 0, z: 1 }; // Unreal Z-up
+        const rightX = worldUp.y * fwdZ - worldUp.z * fwdY;
+        const rightY = worldUp.z * fwdX - worldUp.x * fwdZ;
+        const rightZ = worldUp.x * fwdY - worldUp.y * fwdX;
+        const rightLen = Math.sqrt(rightX * rightX + rightY * rightY + rightZ * rightZ);
+        if (rightLen < 0.001) {
+            // Forward is parallel to world up, use fallback
+            this.setListenerPosition(listenerPos, {
+                forwardX: fwdX,
+                forwardY: fwdY,
+                forwardZ: fwdZ,
+                upX: 0,
+                upY: 0,
+                upZ: 1,
+            });
+            return;
+        }
+        const rX = rightX / rightLen;
+        const rY = rightY / rightLen;
+        const rZ = rightZ / rightLen;
+        // Calculate true up vector (cross product of forward and right)
+        const upX = fwdY * rZ - fwdZ * rY;
+        const upY = fwdZ * rX - fwdX * rZ;
+        const upZ = fwdX * rY - fwdY * rX;
+        this.setListenerPosition(listenerPos, {
+            forwardX: fwdX,
+            forwardY: fwdY,
+            forwardZ: fwdZ,
+            upX,
+            upY,
+            upZ,
+        });
+    }
     removeParticipant(participantId) {
         // Stop monitoring
         if (this.monitoringIntervals.has(participantId)) {

package/dist/index.js

@@ -457,8 +457,9 @@ class OdysseySpatialComms extends EventManager_1.EventManager {
                     participant.userName = data.userName;
                 if (data.userEmail !== undefined)
                     participant.userEmail = data.userEmail;
-                this.spatialAudioManager.updateSpatialAudio(data.participantId, data.position);
-                console.log("✅ SDK: Updated participant position and spatial audio");
+                // Update spatial audio with BOTH position AND direction from socket
+                this.spatialAudioManager.updateSpatialAudio(data.participantId, data.position, data.direction);
+                console.log("✅ SDK: Updated participant position and spatial audio with direction");
                 this.emit("participant-position-updated", participant);
             }
             else {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@newgameplusinc/odyssey-audio-video-sdk-dev",
-  "version": "1.0.3",
+  "version": "1.0.4",
   "description": "Odyssey Spatial Audio & Video SDK using MediaSoup for real-time communication",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",