@newgameplusinc/odyssey-audio-video-sdk-dev 1.0.0

package/README.md

@@ -0,0 +1,106 @@
+# Odyssey Spatial Audio SDK
+
+A comprehensive SDK for real-time spatial audio and video communication using MediaSoup, designed for immersive multi-user experiences in the Odyssey platform.
+
+## Purpose
+
+This package provides a complete WebRTC-based spatial audio and video solution that:
+- Manages MediaSoup connections for audio/video streaming
+- Implements spatial audio using Web Audio API with HRTF
+- Handles participant management with user profile data (bodyHeight, bodyShape, email, etc.)
+- Provides real-time position tracking for immersive spatial experiences
+
+## Installation
+
+You can install this package from npm:
+
+```bash
+npm install @newgameplusinc/odyssey-spatial-sdk-wrapper
+```
+
+Or install locally:
+
+```bash
+npm install ../mediasoup-sdk-test
+```
+
+## Usage
+
+### 1. Initialize the SDK
+
+```typescript
+import { OdysseySpatialComms } from "@newgameplusinc/odyssey-spatial-sdk-wrapper";
+
+// Initialize with your MediaSoup server URL
+const sdk = new OdysseySpatialComms("https://your-mediasoup-server.com");
+```
+
+### 2. Join a Room with User Profile Data
+
+```typescript
+const participant = await sdk.joinRoom({
+  roomId: "my-room",
+  userId: "user-123",
+  deviceId: "device-456",
+  position: { x: 0, y: 0, z: 0 },
+  direction: { x: 0, y: 0, z: 1 },
+  bodyHeight: "0.5",    // User's avatar height from Firebase
+  bodyShape: "4",       // User's avatar body shape from Firebase
+  userName: "John Doe", // User's display name
+  userEmail: "john@example.com" // User's email
+});
+```
+
+### 3. Produce Audio/Video Tracks
+
+```typescript
+// Get user media
+const stream = await navigator.mediaDevices.getUserMedia({ 
+  audio: true, 
+  video: true 
+});
+
+// Produce audio track
+const audioTrack = stream.getAudioTracks()[0];
+await sdk.produceTrack(audioTrack);
+
+// Produce video track
+const videoTrack = stream.getVideoTracks()[0];
+await sdk.produceTrack(videoTrack);
+```
+
+### 4. Update Position for Spatial Audio
+
+```typescript
+sdk.updatePosition(
+  { x: 10, y: 0, z: 5 },  // New position
+  { x: 0, y: 0, z: 1 }    // New direction
+);
+```
+
+### 5. Listen to Events
+
+```typescript
+// New participant joined
+sdk.on("new-participant", (participant) => {
+  console.log("New participant:", participant.userName, participant.bodyHeight);
+});
+
+// Participant left
+sdk.on("participant-left", (participantId) => {
+  console.log("Participant left:", participantId);
+});
+
+// Consumer created (receiving audio/video from remote participant)
+sdk.on("consumer-created", ({ participant, track, consumer }) => {
+  console.log("Receiving", track.kind, "from", participant.userName);
+});
+```
+
+## Build
+
+To build the package, run:
+
+```bash
+npm run build
+```

package/dist/EventManager.d.ts

@@ -0,0 +1,14 @@
+import { EventEmitter } from "events";
+import { OdysseyEvent } from "./types";
+/**
+ * A clean, standalone event bus for the SDK, completely decoupled from the socket.
+ * This ensures that only processed, structured SDK events are emitted to the application.
+ * It extends the Node.js EventEmitter to provide a standard and reliable event API.
+ */
+export declare class EventManager extends EventEmitter {
+    constructor();
+    on(event: OdysseyEvent, listener: (...args: any[]) => void): this;
+    emit(event: OdysseyEvent, ...args: any[]): boolean;
+    off(event: OdysseyEvent, listener: (...args: any[]) => void): this;
+    removeAllListeners(event?: OdysseyEvent): this;
+}

package/dist/EventManager.js

@@ -0,0 +1,28 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.EventManager = void 0;
+const events_1 = require("events");
+/**
+ * A clean, standalone event bus for the SDK, completely decoupled from the socket.
+ * This ensures that only processed, structured SDK events are emitted to the application.
+ * It extends the Node.js EventEmitter to provide a standard and reliable event API.
+ */
+class EventManager extends events_1.EventEmitter {
+    constructor() {
+        super();
+    }
+    // Type-safe overrides for the methods we use
+    on(event, listener) {
+        return super.on(event, listener);
+    }
+    emit(event, ...args) {
+        return super.emit(event, ...args);
+    }
+    off(event, listener) {
+        return super.off(event, listener);
+    }
+    removeAllListeners(event) {
+        return super.removeAllListeners(event);
+    }
+}
+exports.EventManager = EventManager;

package/dist/MediasoupManager.d.ts

@@ -0,0 +1,33 @@
+import * as mediasoupClient from "mediasoup-client";
+import { types } from "mediasoup-client";
+import { Socket } from "socket.io-client";
+export declare class MediasoupManager {
+    private device;
+    private socket;
+    private sendTransport;
+    private recvTransport;
+    private producers;
+    private consumers;
+    constructor(socket: Socket);
+    loadDevice(routerRtpCapabilities: types.RtpCapabilities): Promise<void>;
+    sendDeviceRtpCapabilities(participantId: string): void;
+    createSendTransport(participantId: string): Promise<void>;
+    createRecvTransport(participantId: string): Promise<void>;
+    private connectSendTransport;
+    private connectRecvTransport;
+    produce(track: MediaStreamTrack): Promise<types.Producer>;
+    consume(data: {
+        consumerId: string;
+        producerId: string;
+        kind: "audio" | "video";
+        rtpParameters: any;
+        participantId: string;
+    }): Promise<{
+        consumer: types.Consumer;
+        track: MediaStreamTrack;
+    }>;
+    resumeConsumer(consumerId: string): Promise<void>;
+    private createWebRtcTransport;
+    close(): void;
+    getConsumers(): Map<string, mediasoupClient.types.Consumer<mediasoupClient.types.AppData>>;
+}

package/dist/MediasoupManager.js

@@ -0,0 +1,189 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MediasoupManager = void 0;
+const mediasoupClient = __importStar(require("mediasoup-client"));
+class MediasoupManager {
+    constructor(socket) {
+        this.sendTransport = null;
+        this.recvTransport = null;
+        this.producers = new Map();
+        this.consumers = new Map();
+        this.socket = socket;
+        this.device = new mediasoupClient.Device();
+    }
+    async loadDevice(routerRtpCapabilities) {
+        if (this.device.loaded) {
+            console.warn("Device already loaded.");
+            return;
+        }
+        await this.device.load({ routerRtpCapabilities });
+    }
+    sendDeviceRtpCapabilities(participantId) {
+        this.socket.emit("device-rtp-capabilities", {
+            participantId,
+            rtpCapabilities: this.device.rtpCapabilities,
+        });
+    }
+    async createSendTransport(participantId) {
+        const params = await this.createWebRtcTransport("send", participantId);
+        this.sendTransport = this.device.createSendTransport(params);
+        this.connectSendTransport();
+    }
+    async createRecvTransport(participantId) {
+        const params = await this.createWebRtcTransport("recv", participantId);
+        this.recvTransport = this.device.createRecvTransport(params);
+        this.connectRecvTransport();
+    }
+    connectSendTransport() {
+        this.sendTransport?.on("connect", async ({ dtlsParameters }, callback, errback) => {
+            this.socket.emit("connect-transport", { transportId: this.sendTransport.id, dtlsParameters }, (response) => {
+                if (response.error)
+                    errback(new Error(response.error));
+                else
+                    callback();
+            });
+        });
+        this.sendTransport?.on("produce", async ({ kind, rtpParameters, appData, }, callback, errback) => {
+            this.socket.emit("produce", {
+                transportId: this.sendTransport.id,
+                kind,
+                rtpParameters,
+                appData,
+            }, (response) => {
+                if (response.error)
+                    errback(new Error(response.error));
+                else if (response.producerId)
+                    callback({ id: response.producerId });
+            });
+        });
+    }
+    connectRecvTransport() {
+        this.recvTransport?.on("connect", async ({ dtlsParameters }, callback, errback) => {
+            this.socket.emit("connect-transport", { transportId: this.recvTransport.id, dtlsParameters }, (response) => {
+                if (response.error)
+                    errback(new Error(response.error));
+                else
+                    callback();
+            });
+        });
+    }
+    async produce(track) {
+        if (!this.sendTransport)
+            throw new Error("Send transport not initialized");
+        const producer = await this.sendTransport.produce({ track });
+        this.producers.set(producer.id, producer);
+        return producer;
+    }
+    async consume(data) {
+        if (!this.recvTransport)
+            throw new Error("Receive transport not set up");
+        console.log(`📥 Creating consumer for ${data.participantId}:`, {
+            consumerId: data.consumerId,
+            producerId: data.producerId,
+            kind: data.kind,
+        });
+        const consumer = await this.recvTransport.consume({
+            id: data.consumerId,
+            producerId: data.producerId,
+            kind: data.kind,
+            rtpParameters: data.rtpParameters,
+        });
+        this.consumers.set(consumer.id, consumer);
+        console.log(`✅ Consumer created, track details:`, {
+            consumerId: consumer.id,
+            trackId: consumer.track.id,
+            trackKind: consumer.track.kind,
+            trackEnabled: consumer.track.enabled,
+            trackMuted: consumer.track.muted,
+            trackReadyState: consumer.track.readyState,
+            consumerPaused: consumer.paused,
+            consumerClosed: consumer.closed,
+        });
+        return { consumer, track: consumer.track };
+    }
+    async resumeConsumer(consumerId) {
+        console.log(`▶️ Resuming consumer ${consumerId}...`);
+        return new Promise((resolve, reject) => {
+            this.socket.emit("resume-consumer", { consumerId }, (response) => {
+                if (response.error) {
+                    console.error(`❌ Failed to resume consumer ${consumerId}:`, response.error);
+                    reject(new Error(response.error));
+                }
+                else {
+                    console.log(`✅ Consumer ${consumerId} resumed successfully on server`);
+                    resolve();
+                }
+            });
+        });
+    }
+    async createWebRtcTransport(direction, participantId) {
+        return new Promise((resolve, reject) => {
+            // Set up listener for transport-created event
+            const handleTransportCreated = (data) => {
+                if (data.type === direction) {
+                    // Remove listener to prevent memory leaks
+                    this.socket.off("transport-created", handleTransportCreated);
+                    if (data.error) {
+                        return reject(new Error(data.error));
+                    }
+                    resolve(data.params);
+                }
+            };
+            this.socket.on("transport-created", handleTransportCreated);
+            // Emit the request
+            this.socket.emit("create-transport", { direction, participantId });
+            // Add timeout to prevent hanging forever
+            setTimeout(() => {
+                this.socket.off("transport-created", handleTransportCreated);
+                reject(new Error(`Timeout waiting for ${direction} transport`));
+            }, 10000);
+        });
+    }
+    close() {
+        this.producers.forEach((p) => p.close());
+        this.consumers.forEach((c) => c.close());
+        if (this.sendTransport)
+            this.sendTransport.close();
+        if (this.recvTransport)
+            this.recvTransport.close();
+        this.producers.clear();
+        this.consumers.clear();
+    }
+    getConsumers() {
+        return this.consumers;
+    }
+}
+exports.MediasoupManager = MediasoupManager;

package/dist/SpatialAudioManager.d.ts

@@ -0,0 +1,24 @@
+import { EventManager } from "./EventManager";
+import { Position } from "./types";
+export declare class SpatialAudioManager extends EventManager {
+    private audioContext;
+    private participantNodes;
+    private masterGainNode;
+    private monitoringIntervals;
+    constructor();
+    getAudioContext(): AudioContext;
+    setupSpatialAudioForParticipant(participantId: string, track: MediaStreamTrack, bypassSpatialization?: boolean): void;
+    private startMonitoring;
+    updateSpatialAudio(participantId: string, position: Position): void;
+    setListenerPosition(position: Position, orientation: {
+        forwardX: number;
+        forwardY: number;
+        forwardZ: number;
+        upX: number;
+        upY: number;
+        upZ: number;
+    }): void;
+    removeParticipant(participantId: string): void;
+    resumeAudioContext(): Promise<void>;
+    getAudioContextState(): AudioContextState;
+}

package/dist/SpatialAudioManager.js

@@ -0,0 +1,178 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SpatialAudioManager = void 0;
+const EventManager_1 = require("./EventManager");
+class SpatialAudioManager extends EventManager_1.EventManager {
+    constructor() {
+        super();
+        this.participantNodes = new Map();
+        this.monitoringIntervals = new Map();
+        this.audioContext = new AudioContext();
+        this.masterGainNode = this.audioContext.createGain();
+        this.masterGainNode.gain.value = 5.0; // Set master gain to 5.0
+        this.masterGainNode.connect(this.audioContext.destination);
+        console.log(`🔊 SpatialAudioManager initialized, gain: ${this.masterGainNode.gain.value} audioContext state: ${this.audioContext.state}`);
+    }
+    getAudioContext() {
+        return this.audioContext;
+    }
+    setupSpatialAudioForParticipant(participantId, track, bypassSpatialization = false // Default to false
+    ) {
+        if (this.audioContext.state === "suspended") {
+            this.audioContext.resume();
+        }
+        const stream = new MediaStream([track]);
+        // 🔍 DIAGNOSTIC TEST: Try playing through raw HTML audio element
+        console.warn(`🧪 DIAGNOSTIC: Testing raw audio playback for ${participantId}`);
+        const testAudio = new Audio();
+        testAudio.srcObject = stream;
+        testAudio.volume = 1.0;
+        testAudio
+            .play()
+            .then(() => {
+            console.log(`✅ DIAGNOSTIC: Raw audio element can play track for ${participantId}`);
+        })
+            .catch((err) => {
+            console.error(`❌ DIAGNOSTIC: Raw audio element CANNOT play:`, err);
+        });
+        const source = this.audioContext.createMediaStreamSource(stream);
+        const panner = this.audioContext.createPanner();
+        const analyser = this.audioContext.createAnalyser();
+        const gain = this.audioContext.createGain();
+        // Configure Panner
+        panner.panningModel = "HRTF";
+        panner.distanceModel = "inverse";
+        panner.refDistance = 1;
+        panner.maxDistance = 100;
+        panner.rolloffFactor = 1;
+        panner.coneInnerAngle = 360;
+        panner.coneOuterAngle = 0;
+        panner.coneOuterGain = 0;
+        if (bypassSpatialization) {
+            console.log(`🔊 TESTING: Connecting audio directly to destination (bypassing spatial audio) for ${participantId}`);
+            source.connect(analyser);
+            analyser.connect(this.masterGainNode);
+        }
+        else {
+            // Standard spatialized path
+            source.connect(panner);
+            panner.connect(analyser);
+            analyser.connect(gain);
+            gain.connect(this.masterGainNode);
+        }
+        this.participantNodes.set(participantId, {
+            source,
+            panner,
+            analyser,
+            gain,
+            stream,
+        });
+        console.log(`🎧 Spatial audio setup complete for ${participantId}:`, {
+            audioContextState: this.audioContext.state,
+            gain: this.masterGainNode.gain.value,
+            trackEnabled: track.enabled,
+            trackMuted: track.muted,
+            trackReadyState: track.readyState,
+            isBypassed: bypassSpatialization,
+        });
+        // Start monitoring audio levels
+        this.startMonitoring(participantId);
+    }
+    startMonitoring(participantId) {
+        const nodes = this.participantNodes.get(participantId);
+        if (!nodes)
+            return;
+        const { analyser, stream } = nodes;
+        const dataArray = new Uint8Array(analyser.frequencyBinCount);
+        // Clear any existing interval for this participant
+        if (this.monitoringIntervals.has(participantId)) {
+            clearInterval(this.monitoringIntervals.get(participantId));
+        }
+        const interval = setInterval(() => {
+            analyser.getByteTimeDomainData(dataArray);
+            let sum = 0;
+            for (const amplitude of dataArray) {
+                sum += Math.abs(amplitude - 128);
+            }
+            const average = sum / dataArray.length;
+            const audioLevel = (average / 128) * 255; // Scale to 0-255
+            console.log(`📊 Audio level for ${participantId}: ${audioLevel.toFixed(2)} (0-255 scale)`);
+            if (audioLevel < 1.0) {
+                console.warn(`⚠️ NO AUDIO DATA detected for ${participantId}! Track may be silent or not transmitting.`);
+                console.info(`💡 Check: 1) Is microphone unmuted? 2) Is correct mic selected? 3) Is mic working in system settings?`);
+            }
+            // Check track status after 2 seconds
+            setTimeout(() => {
+                const track = stream.getAudioTracks()[0];
+                if (track) {
+                    console.log(`🔊 Audio track status after 2s for ${participantId}:`, {
+                        trackEnabled: track.enabled,
+                        trackMuted: track.muted,
+                        trackReadyState: track.readyState,
+                        audioContextState: this.audioContext.state,
+                        pannerPosition: {
+                            x: nodes.panner.positionX.value,
+                            y: nodes.panner.positionY.value,
+                            z: nodes.panner.positionZ.value,
+                        },
+                    });
+                }
+            }, 2000);
+        }, 2000); // Log every 2 seconds
+        this.monitoringIntervals.set(participantId, interval);
+    }
+    updateSpatialAudio(participantId, position) {
+        const nodes = this.participantNodes.get(participantId);
+        if (nodes?.panner) {
+            nodes.panner.positionX.setValueAtTime(position.x, this.audioContext.currentTime);
+            nodes.panner.positionY.setValueAtTime(position.y, this.audioContext.currentTime);
+            nodes.panner.positionZ.setValueAtTime(position.z, this.audioContext.currentTime);
+        }
+    }
+    setListenerPosition(position, orientation) {
+        const { listener } = this.audioContext;
+        if (listener) {
+            // Use setPosition and setOrientation for atomic updates if available
+            if (listener.positionX) {
+                listener.positionX.setValueAtTime(position.x, this.audioContext.currentTime);
+                listener.positionY.setValueAtTime(position.y, this.audioContext.currentTime);
+                listener.positionZ.setValueAtTime(position.z, this.audioContext.currentTime);
+            }
+            if (listener.forwardX) {
+                listener.forwardX.setValueAtTime(orientation.forwardX, this.audioContext.currentTime);
+                listener.forwardY.setValueAtTime(orientation.forwardY, this.audioContext.currentTime);
+                listener.forwardZ.setValueAtTime(orientation.forwardZ, this.audioContext.currentTime);
+                listener.upX.setValueAtTime(orientation.upX, this.audioContext.currentTime);
+                listener.upY.setValueAtTime(orientation.upY, this.audioContext.currentTime);
+                listener.upZ.setValueAtTime(orientation.upZ, this.audioContext.currentTime);
+            }
+        }
+    }
+    removeParticipant(participantId) {
+        // Stop monitoring
+        if (this.monitoringIntervals.has(participantId)) {
+            clearInterval(this.monitoringIntervals.get(participantId));
+            this.monitoringIntervals.delete(participantId);
+        }
+        const nodes = this.participantNodes.get(participantId);
+        if (nodes) {
+            nodes.source.disconnect();
+            nodes.panner.disconnect();
+            nodes.analyser.disconnect();
+            nodes.gain.disconnect();
+            nodes.stream.getTracks().forEach((track) => track.stop());
+            this.participantNodes.delete(participantId);
+            console.log(`🗑️ Removed participant ${participantId} from spatial audio.`);
+        }
+    }
+    async resumeAudioContext() {
+        if (this.audioContext.state === "suspended") {
+            await this.audioContext.resume();
+            console.log("✅ Audio context has been resumed successfully.");
+        }
+    }
+    getAudioContextState() {
+        return this.audioContext.state;
+    }
+}
+exports.SpatialAudioManager = SpatialAudioManager;

package/dist/index.d.ts

@@ -0,0 +1,43 @@
+import { EventManager } from "./EventManager";
+import { Direction, MediaState, OdysseyEvent, Participant, Position, RoomJoinedData } from "./types";
+export declare class OdysseySpatialComms extends EventManager {
+    private socket;
+    room: {
+        id: string;
+        participants: Map<string, Participant>;
+    } | null;
+    private localParticipant;
+    private mediasoupManager;
+    private spatialAudioManager;
+    constructor(serverUrl: string);
+    on(event: OdysseyEvent, listener: (...args: any[]) => void): this;
+    emit(event: OdysseyEvent, ...args: any[]): boolean;
+    joinRoom(data: {
+        roomId: string;
+        userId: string;
+        deviceId: string;
+        participantId?: string;
+        position: Position;
+        direction: Direction;
+        bodyHeight?: string;
+        bodyShape?: string;
+        userName?: string;
+        userEmail?: string;
+    }): Promise<Participant>;
+    leaveRoom(): void;
+    resumeAudio(): Promise<void>;
+    getAudioContextState(): AudioContextState;
+    produceTrack(track: MediaStreamTrack): Promise<any>;
+    updatePosition(position: Position, direction: Direction): void;
+    updateMediaState(mediaState: MediaState): void;
+    setListenerPosition(position: Position, orientation: {
+        forwardX: number;
+        forwardY: number;
+        forwardZ: number;
+        upX: number;
+        upY: number;
+        upZ: number;
+    }): void;
+    private listenForEvents;
+}
+export type { Direction, MediaState, OdysseyEvent, Participant, Position, RoomJoinedData, };

package/dist/index.js

@@ -0,0 +1,490 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.OdysseySpatialComms = void 0;
+const socket_io_client_1 = require("socket.io-client");
+const EventManager_1 = require("./EventManager");
+const MediasoupManager_1 = require("./MediasoupManager");
+const SpatialAudioManager_1 = require("./SpatialAudioManager");
+class OdysseySpatialComms extends EventManager_1.EventManager {
+    constructor(serverUrl) {
+        super(); // Initialize the EventEmitter base class
+        this.room = null;
+        this.localParticipant = null;
+        this.socket = (0, socket_io_client_1.io)(serverUrl, {
+            transports: ["websocket"],
+        });
+        this.mediasoupManager = new MediasoupManager_1.MediasoupManager(this.socket);
+        this.spatialAudioManager = new SpatialAudioManager_1.SpatialAudioManager();
+        // Set max listeners to prevent warning
+        this.setMaxListeners(50);
+        this.listenForEvents();
+        this.emit("connected");
+    }
+    // Override to ensure we're using the EventEmitter's on, not the socket's
+    on(event, listener) {
+        // Explicitly call EventEmitter's on method
+        return super.on(event, listener);
+    }
+    // Override to ensure we're using the EventEmitter's emit, not the socket's
+    emit(event, ...args) {
+        // Explicitly call EventEmitter's emit method
+        return super.emit(event, ...args);
+    }
+    async joinRoom(data) {
+        return new Promise((resolve, reject) => {
+            // Create a one-time listener for room-joined event
+            const handleRoomJoined = async (roomData) => {
+                try {
+                    console.log("✅ Successfully joined room:", roomData);
+                    console.log("👥 [SDK] Participants received from server:", roomData.participants);
+                    console.log("🔍 [SDK] Each participant data:", roomData.participants.map((p) => ({
+                        participantId: p.participantId,
+                        bodyHeight: p.bodyHeight,
+                        bodyShape: p.bodyShape,
+                        userName: p.userName,
+                        userEmail: p.userEmail,
+                    })));
+                    // Remove the listener to prevent memory leaks
+                    this.socket.off("room-joined", handleRoomJoined);
+                    // 1. Load Mediasoup device
+                    await this.mediasoupManager.loadDevice(roomData.routerRtpCapabilities);
+                    // 2. Create transports
+                    await this.mediasoupManager.createSendTransport(roomData.participantId);
+                    await this.mediasoupManager.createRecvTransport(roomData.participantId);
+                    // 3. Store local participant info
+                    const localParticipantData = roomData.participants.find((p) => p.participantId === roomData.participantId);
+                    if (!localParticipantData) {
+                        throw new Error("Could not find local participant in room data");
+                    }
+                    this.localParticipant = {
+                        ...localParticipantData,
+                        isLocal: true,
+                        producers: new Map(),
+                        consumers: new Map(),
+                    };
+                    // 4. Initialize room state
+                    this.room = {
+                        id: roomData.roomId,
+                        participants: new Map(),
+                    };
+                    // 5. Process all participants from the server
+                    for (const pData of roomData.participants) {
+                        const participant = {
+                            ...pData,
+                            isLocal: pData.participantId === this.localParticipant.participantId,
+                            producers: new Map(),
+                            consumers: new Map(),
+                        };
+                        this.room.participants.set(pData.participantId, participant);
+                    }
+                    // 6. Note: Device RTP capabilities will be sent after producing tracks
+                    // This ensures transports are fully connected via DTLS handshake
+                    console.log("⏳ Transports created, will send RTP capabilities after producing tracks");
+                    this.emit("room-joined", roomData);
+                    resolve(this.localParticipant);
+                }
+                catch (error) {
+                    this.socket.off("room-joined", handleRoomJoined);
+                    reject(error);
+                }
+            };
+            // Register the listener
+            this.socket.on("room-joined", handleRoomJoined);
+            // Log the exact data being sent to the server
+            console.log("🚀 [SDK] Emitting 'join-room' event to server with data:", {
+                roomId: data.roomId,
+                userId: data.userId,
+                deviceId: data.deviceId,
+                participantId: data.participantId,
+                position: data.position,
+                direction: data.direction,
+                bodyHeight: data.bodyHeight,
+                bodyShape: data.bodyShape,
+                userName: data.userName,
+                userEmail: data.userEmail,
+            });
+            // Emit join-room request
+            this.socket.emit("join-room", data);
+        });
+    }
+    leaveRoom() {
+        if (!this.socket.connected)
+            return;
+        this.socket.emit("leave-room");
+        this.mediasoupManager.close();
+        this.socket.disconnect();
+        this.room = null;
+        this.localParticipant = null;
+        this.removeAllListeners(); // Clean up all listeners
+        console.log("🚪 SDK: Left room and cleaned up resources.");
+    }
+    async resumeAudio() {
+        await this.spatialAudioManager.resumeAudioContext();
+    }
+    getAudioContextState() {
+        return this.spatialAudioManager.getAudioContextState();
+    }
+    async produceTrack(track) {
+        const producer = await this.mediasoupManager.produce(track);
+        if (this.localParticipant) {
+            const isFirstProducer = this.localParticipant.producers.size === 0;
+            this.localParticipant.producers.set(producer.id, producer);
+            if (track.kind === "audio") {
+                this.localParticipant.audioTrack = track;
+                this.localParticipant.mediaState.audio = true;
+            }
+            else if (track.kind === "video") {
+                this.localParticipant.videoTrack = track;
+                this.localParticipant.mediaState.video = true;
+            }
+            // Send device RTP capabilities after first track is produced
+            // This ensures transports are fully connected via DTLS
+            if (isFirstProducer) {
+                console.log("📡 Sending device RTP capabilities after first track produced");
+                this.mediasoupManager.sendDeviceRtpCapabilities(this.localParticipant.participantId);
+            }
+            // Notify others of media state change
+            this.updateMediaState(this.localParticipant.mediaState);
+        }
+        return producer;
+    }
+    updatePosition(position, direction) {
+        if (this.localParticipant && this.room) {
+            this.localParticipant.position = position;
+            this.localParticipant.direction = direction;
+            this.socket.emit("update-position", {
+                participantId: this.localParticipant.participantId,
+                conferenceId: this.room.id,
+                roomId: this.room.id,
+                position,
+                direction,
+            });
+        }
+    }
+    updateMediaState(mediaState) {
+        if (this.localParticipant && this.room) {
+            this.localParticipant.mediaState = mediaState;
+            this.socket.emit("update-media-state", {
+                participantId: this.localParticipant.participantId,
+                conferenceId: this.room.id,
+                roomId: this.room.id,
+                mediaState,
+            });
+        }
+    }
+    setListenerPosition(position, orientation) {
+        this.spatialAudioManager.setListenerPosition(position, orientation);
+    }
+    listenForEvents() {
+        this.socket.on("new-participant", (participantData) => {
+            console.log("👋 SDK: new-participant event received:", {
+                participantId: participantData.participantId,
+                conferenceId: participantData.conferenceId,
+                roomId: participantData.roomId,
+                position: participantData.position,
+                direction: participantData.direction,
+                mediaState: participantData.mediaState,
+                timestamp: participantData.timestamp,
+                bodyHeight: participantData.bodyHeight,
+                bodyShape: participantData.bodyShape,
+                userName: participantData.userName,
+                userEmail: participantData.userEmail,
+            });
+            console.log("🔍 [SDK] New participant USER DATA:", {
+                bodyHeight: participantData.bodyHeight,
+                bodyShape: participantData.bodyShape,
+                userName: participantData.userName,
+                userEmail: participantData.userEmail,
+            });
+            if (this.room) {
+                const newParticipant = {
+                    participantId: participantData.participantId,
+                    userId: participantData.userId,
+                    deviceId: participantData.deviceId,
+                    position: participantData.position,
+                    direction: participantData.direction,
+                    mediaState: participantData.mediaState,
+                    isLocal: false,
+                    producers: new Map(),
+                    consumers: new Map(),
+                    bodyHeight: participantData.bodyHeight,
+                    bodyShape: participantData.bodyShape,
+                    userName: participantData.userName,
+                    userEmail: participantData.userEmail,
+                };
+                this.room.participants.set(participantData.participantId, newParticipant);
+                console.log("✅ SDK: New participant added to room, total participants:", this.room.participants.size);
+                console.log("✅ SDK: New participant full data:", newParticipant);
+                this.emit("new-participant", newParticipant);
+            }
+        });
+        this.socket.on("participant-left", (data) => {
+            if (this.room) {
+                this.room.participants.delete(data.participantId);
+                this.spatialAudioManager.removeParticipant(data.participantId);
+                this.emit("participant-left", data.participantId);
+            }
+        });
+        this.socket.on("consumer-created", async (data) => {
+            console.log("📥 SDK: consumer-created event received:", {
+                consumerId: data.consumerId,
+                producerId: data.producerId,
+                kind: data.kind,
+                participantId: data.participantId,
+                conferenceId: data.conferenceId,
+                roomId: data.roomId,
+                position: data.position,
+                direction: data.direction,
+                mediaState: data.mediaState,
+                timestamp: data.timestamp,
+                bodyHeight: data.bodyHeight,
+                bodyShape: data.bodyShape,
+                userName: data.userName,
+                userEmail: data.userEmail,
+            });
+            const { consumer, track } = await this.mediasoupManager.consume(data);
+            this.socket.on("all-participants-update", (payload) => {
+                console.log("📦 SDK: all-participants-update event received:", payload);
+                if (!this.room) {
+                    this.room = {
+                        id: payload.roomId,
+                        participants: new Map(),
+                    };
+                }
+                const activeParticipantIds = new Set();
+                for (const snapshot of payload.participants) {
+                    activeParticipantIds.add(snapshot.participantId);
+                    let participant = this.room?.participants.get(snapshot.participantId);
+                    const normalizedPosition = {
+                        x: snapshot.position?.x ?? 0,
+                        y: snapshot.position?.y ?? 0,
+                        z: snapshot.position?.z ?? 0,
+                    };
+                    const normalizedDirection = {
+                        x: snapshot.direction?.x ?? 0,
+                        y: snapshot.direction?.y ?? 0,
+                        z: snapshot.direction?.z ?? 1,
+                    };
+                    const normalizedMediaState = {
+                        audio: snapshot.mediaState?.audio ?? false,
+                        video: snapshot.mediaState?.video ?? false,
+                        sharescreen: snapshot.mediaState?.sharescreen ?? false,
+                    };
+                    if (!participant) {
+                        participant = {
+                            participantId: snapshot.participantId,
+                            userId: snapshot.userId,
+                            deviceId: snapshot.deviceId,
+                            isLocal: this.localParticipant?.participantId ===
+                                snapshot.participantId,
+                            audioTrack: undefined,
+                            videoTrack: undefined,
+                            producers: new Map(),
+                            consumers: new Map(),
+                            position: normalizedPosition,
+                            direction: normalizedDirection,
+                            mediaState: normalizedMediaState,
+                        };
+                    }
+                    else {
+                        participant.userId = snapshot.userId;
+                        participant.deviceId = snapshot.deviceId;
+                        participant.position = normalizedPosition;
+                        participant.direction = normalizedDirection;
+                        participant.mediaState = normalizedMediaState;
+                    }
+                    participant.bodyHeight = snapshot.bodyHeight;
+                    participant.bodyShape = snapshot.bodyShape;
+                    participant.userName = snapshot.userName;
+                    participant.userEmail = snapshot.userEmail;
+                    participant.isLocal =
+                        this.localParticipant?.participantId === snapshot.participantId;
+                    this.room?.participants.set(snapshot.participantId, participant);
+                    if (participant.isLocal) {
+                        this.localParticipant = participant;
+                    }
+                }
+                if (this.room) {
+                    for (const existingId of Array.from(this.room.participants.keys())) {
+                        if (!activeParticipantIds.has(existingId)) {
+                            this.room.participants.delete(existingId);
+                        }
+                    }
+                }
+                const normalizedParticipants = this.room
+                    ? Array.from(this.room.participants.values())
+                    : [];
+                this.emit("all-participants-update", normalizedParticipants);
+            });
+            // Resume the consumer to start receiving media (non-blocking)
+            this.mediasoupManager
+                .resumeConsumer(consumer.id)
+                .then(() => {
+                console.log(`▶️ SDK: Consumer resumed for ${data.participantId}, kind: ${data.kind}`);
+            })
+                .catch((err) => {
+                console.error(`❌ SDK: Failed to resume consumer for ${data.participantId}:`, err);
+            });
+            let participant = this.room?.participants.get(data.participantId);
+            // If participant doesn't exist yet, create it with the data from the event
+            if (!participant && this.room) {
+                console.log("🆕 SDK: Creating participant entry from consumer-created event with user profile data");
+                participant = {
+                    participantId: data.participantId,
+                    userId: data.participantId.split(":")[0] || data.participantId,
+                    deviceId: data.participantId.split(":")[1] || data.participantId,
+                    isLocal: false,
+                    position: data.position,
+                    direction: data.direction,
+                    mediaState: data.mediaState,
+                    producers: new Map(),
+                    consumers: new Map(),
+                    // Include user profile data from consumer-created event
+                    bodyHeight: data.bodyHeight,
+                    bodyShape: data.bodyShape,
+                    userName: data.userName,
+                    userEmail: data.userEmail,
+                };
+                this.room.participants.set(data.participantId, participant);
+                console.log("✅ SDK: Created participant WITH user profile data:", {
+                    participantId: data.participantId,
+                    bodyHeight: data.bodyHeight,
+                    bodyShape: data.bodyShape,
+                    userName: data.userName,
+                    userEmail: data.userEmail,
+                });
+            }
+            console.log("🔍 SDK: Participant lookup result:", {
+                found: !!participant,
+                participantId: data.participantId,
+                totalParticipants: this.room?.participants.size,
+                participantIds: Array.from(this.room?.participants.keys() || []),
+            });
+            if (participant) {
+                // Update participant data with latest from server
+                participant.position = data.position;
+                participant.direction = data.direction;
+                participant.mediaState = data.mediaState;
+                // Update user profile data if provided (or preserve existing if not provided)
+                if (data.bodyHeight !== undefined) {
+                    participant.bodyHeight = data.bodyHeight;
+                }
+                if (data.bodyShape !== undefined) {
+                    participant.bodyShape = data.bodyShape;
+                }
+                if (data.userName !== undefined) {
+                    participant.userName = data.userName;
+                }
+                if (data.userEmail !== undefined) {
+                    participant.userEmail = data.userEmail;
+                }
+                participant.consumers.set(consumer.id, consumer);
+                if (track.kind === "audio") {
+                    participant.audioTrack = track;
+                    console.log("🎧 SDK: Setting up spatial audio for", participant.participantId);
+                    // Setup spatial audio with full 3D positioning
+                    this.spatialAudioManager.setupSpatialAudioForParticipant(participant.participantId, track, false // Enable spatial audio
+                    );
+                    // Update spatial audio position for this participant
+                    this.spatialAudioManager.updateSpatialAudio(participant.participantId, data.position);
+                    console.log("📍 SDK: Spatial audio position set:", data.position);
+                }
+                else if (track.kind === "video") {
+                    participant.videoTrack = track;
+                    console.log("📹 SDK: Video track set for", participant.participantId);
+                }
+                console.log("✅ SDK: Emitting consumer-created event with participant:", {
+                    participantId: participant.participantId,
+                    hasPosition: !!participant.position,
+                    position: participant.position,
+                    trackKind: track.kind,
+                    conferenceId: data.conferenceId,
+                    roomId: data.roomId,
+                });
+                this.emit("consumer-created", {
+                    participant,
+                    track,
+                    consumer,
+                });
+            }
+            else {
+                console.error("❌ SDK: Participant not found for consumer-created event:", data.participantId);
+            }
+        });
+        this.socket.on("participant-position-updated", (data) => {
+            console.log("📍 SDK: participant-position-updated event received:", {
+                participantId: data.participantId,
+                conferenceId: data.conferenceId,
+                roomId: data.roomId,
+                position: data.position,
+                direction: data.direction,
+                mediaState: data.mediaState,
+                consumerIds: data.consumerIds,
+                timestamp: data.timestamp,
+                bodyHeight: data.bodyHeight,
+                bodyShape: data.bodyShape,
+                userName: data.userName,
+                userEmail: data.userEmail,
+            });
+            console.log("🔍 [SDK] Position update USER DATA:", {
+                participantId: data.participantId,
+                bodyHeight: data.bodyHeight,
+                bodyShape: data.bodyShape,
+                userName: data.userName,
+                userEmail: data.userEmail,
+            });
+            const participant = this.room?.participants.get(data.participantId);
+            if (participant) {
+                participant.position = data.position;
+                participant.direction = data.direction;
+                participant.mediaState = data.mediaState;
+                // Update user data if provided
+                if (data.bodyHeight !== undefined)
+                    participant.bodyHeight = data.bodyHeight;
+                if (data.bodyShape !== undefined)
+                    participant.bodyShape = data.bodyShape;
+                if (data.userName !== undefined)
+                    participant.userName = data.userName;
+                if (data.userEmail !== undefined)
+                    participant.userEmail = data.userEmail;
+                this.spatialAudioManager.updateSpatialAudio(data.participantId, data.position);
+                console.log("✅ SDK: Updated participant position and spatial audio");
+                this.emit("participant-position-updated", participant);
+            }
+            else {
+                console.warn("⚠️ SDK: Participant not found for position update:", data.participantId);
+                console.warn("⚠️ SDK: Available participants in room:", Array.from(this.room?.participants.keys() || []));
+            }
+        });
+        this.socket.on("participant-media-state-updated", (data) => {
+            console.log("🔄 SDK: participant-media-state-updated event received:", {
+                participantId: data.participantId,
+                conferenceId: data.conferenceId,
+                roomId: data.roomId,
+                mediaState: data.mediaState,
+                position: data.position,
+                direction: data.direction,
+                timestamp: data.timestamp,
+            });
+            const participant = this.room?.participants.get(data.participantId);
+            if (participant) {
+                participant.mediaState = data.mediaState;
+                participant.position = data.position;
+                participant.direction = data.direction;
+                console.log("✅ SDK: Updated participant media state");
+                this.emit("participant-media-state-updated", participant);
+            }
+            else {
+                console.warn("⚠️ SDK: Participant not found for media state update:", data.participantId);
+            }
+        });
+        this.socket.on("error", (error) => {
+            console.error("Socket error:", error);
+            this.emit("error", error);
+        });
+        this.socket.on("disconnect", () => {
+            this.emit("disconnected");
+        });
+    }
+}
+exports.OdysseySpatialComms = OdysseySpatialComms;

package/dist/types.d.ts

@@ -0,0 +1,58 @@
+export interface Position {
+    x: number;
+    y: number;
+    z: number;
+}
+export interface Direction {
+    x: number;
+    y: number;
+    z: number;
+}
+export interface MediaState {
+    audio: boolean;
+    video: boolean;
+    sharescreen: boolean;
+}
+export interface Participant {
+    participantId: string;
+    userId: string;
+    deviceId: string;
+    name?: string;
+    avatarUrl?: string;
+    isLocal: boolean;
+    audioTrack?: MediaStreamTrack;
+    videoTrack?: MediaStreamTrack;
+    producers: Map<string, any>;
+    consumers: Map<string, any>;
+    position: Position;
+    direction: Direction;
+    mediaState: MediaState;
+    bodyHeight?: string;
+    bodyShape?: string;
+    userName?: string;
+    userEmail?: string;
+}
+export interface RoomJoinedData {
+    participants: Participant[];
+    routerRtpCapabilities: any;
+    participantId: string;
+    roomId: string;
+    iceServers: any[];
+}
+export interface ParticipantsSnapshotEvent {
+    roomId: string;
+    participants: Array<{
+        participantId: string;
+        userId: string;
+        deviceId: string;
+        position: Position;
+        direction: Direction;
+        bodyHeight: string;
+        bodyShape: string;
+        userName: string;
+        userEmail: string;
+        mediaState: MediaState;
+    }>;
+    timestamp: number;
+}
+export type OdysseyEvent = "connected" | "disconnected" | "room-joined" | "all-participants-update" | "new-participant" | "participant-left" | "producer-created" | "consumer-created" | "participant-media-state-updated" | "participant-position-updated" | "error";

package/dist/types.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/package.json

@@ -0,0 +1,50 @@
+{
+  "name": "@newgameplusinc/odyssey-audio-video-sdk-dev",
+  "version": "1.0.0",
+  "description": "Odyssey Spatial Audio & Video SDK using MediaSoup for real-time communication",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "scripts": {
+    "build": "tsc",
+    "dev": "tsc --watch",
+    "test": "jest",
+    "lint": "eslint src/**/*.ts",
+    "prepublishOnly": "npm run build",
+    "publish:npm": "npm publish --access public",
+    "version:patch": "npm version patch",
+    "version:minor": "npm version minor",
+    "version:major": "npm version major"
+  },
+  "keywords": [
+    "spatial-audio",
+    "webrtc",
+    "mediasoup",
+    "conference",
+    "real-time",
+    "audio",
+    "video",
+    "odyssey"
+  ],
+  "author": "New Game Plus Inc",
+  "license": "MIT",
+  "dependencies": {
+    "socket.io-client": "^4.7.2",
+    "webrtc-adapter": "^8.2.3",
+    "mediasoup-client": "^3.6.90",
+    "events": "^3.3.0"
+  },
+  "devDependencies": {
+    "@types/node": "^20.0.0",
+    "typescript": "^5.0.0",
+    "jest": "^29.0.0",
+    "@types/jest": "^29.0.0"
+  },
+  "files": [
+    "dist"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/New-Game-Plus-Inc/mediasoup-sdk-test.git"
+  },
+  "homepage": "https://github.com/New-Game-Plus-Inc/mediasoup-sdk-test"
+}