@newgameplusinc/odyssey-audio-video-sdk-dev 1.0.261 → 1.0.263
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -65,9 +65,7 @@ class MLNoiseSuppressor {
|
|
|
65
65
|
await tf.ready();
|
|
66
66
|
// Load model - use standard loadLayersModel instead of fromMemory
|
|
67
67
|
// This handles weight loading automatically
|
|
68
|
-
console.log(`[MLNoiseSuppressor] Loading model from: ${modelUrl}`);
|
|
69
68
|
this.model = await tf.loadLayersModel(modelUrl);
|
|
70
|
-
console.log(`[MLNoiseSuppressor] Model loaded successfully`);
|
|
71
69
|
// Load config
|
|
72
70
|
const baseUrl = modelUrl.substring(0, modelUrl.lastIndexOf('/'));
|
|
73
71
|
const configResponse = await fetch(`${baseUrl}/model_config.json`);
|
|
@@ -81,7 +79,6 @@ class MLNoiseSuppressor {
|
|
|
81
79
|
this.normStats = { mean: 0, std: 1 };
|
|
82
80
|
}
|
|
83
81
|
this.isInitialized = true;
|
|
84
|
-
console.log(`[MLNoiseSuppressor] Initialization complete`);
|
|
85
82
|
}
|
|
86
83
|
catch (error) {
|
|
87
84
|
console.error(`[MLNoiseSuppressor] Initialization failed:`, error);
|
|
@@ -62,25 +62,11 @@ class SpatialAudioChannel {
|
|
|
62
62
|
* Setup spatial audio for a participant
|
|
63
63
|
*/
|
|
64
64
|
async setupParticipant(participantId, track, bypassSpatialization = false) {
|
|
65
|
-
console.log(`🎧 [SpatialAudioChannel] Setting up participant ${participantId.substring(0, 8)}`, {
|
|
66
|
-
trackKind: track.kind,
|
|
67
|
-
trackEnabled: track.enabled,
|
|
68
|
-
trackMuted: track.muted,
|
|
69
|
-
trackReadyState: track.readyState,
|
|
70
|
-
bypassSpatialization,
|
|
71
|
-
audioContextState: this.audioContext.state,
|
|
72
|
-
});
|
|
73
65
|
if (this.audioContext.state === 'suspended') {
|
|
74
|
-
console.log(`🎧 [SpatialAudioChannel] Resuming suspended AudioContext...`);
|
|
75
66
|
await this.audioContext.resume();
|
|
76
|
-
console.log(`🎧 [SpatialAudioChannel] AudioContext resumed, state: ${this.audioContext.state}`);
|
|
77
67
|
}
|
|
78
68
|
const stream = new MediaStream([track]);
|
|
79
69
|
const source = this.audioContext.createMediaStreamSource(stream);
|
|
80
|
-
console.log(`🎧 [SpatialAudioChannel] Created MediaStreamSource for ${participantId.substring(0, 8)}`, {
|
|
81
|
-
streamActive: stream.active,
|
|
82
|
-
streamTracks: stream.getTracks().length,
|
|
83
|
-
});
|
|
84
70
|
// Create all audio nodes
|
|
85
71
|
const panner = this.createPanner();
|
|
86
72
|
const stereoPanner = this.audioContext.createStereoPanner();
|
|
@@ -138,13 +124,6 @@ class SpatialAudioChannel {
|
|
|
138
124
|
dynamicLowpass,
|
|
139
125
|
stream,
|
|
140
126
|
});
|
|
141
|
-
console.log(`🎧 [SpatialAudioChannel] ✅ Audio pipeline setup complete for ${participantId.substring(0, 8)}`, {
|
|
142
|
-
bypassSpatialization,
|
|
143
|
-
masterGainValue: this.masterGainNode.gain.value,
|
|
144
|
-
participantGainValue: gain.gain.value,
|
|
145
|
-
audioContextState: this.audioContext.state,
|
|
146
|
-
connectedToDestination: true,
|
|
147
|
-
});
|
|
148
127
|
}
|
|
149
128
|
/**
|
|
150
129
|
* Update spatial audio for a participant
|
|
@@ -179,18 +158,6 @@ class SpatialAudioChannel {
|
|
|
179
158
|
(0, gain_smoothing_1.applyStereoPanSmooth)(nodes.stereoPanner, panValue, this.audioContext, 0.05);
|
|
180
159
|
// Apply gain
|
|
181
160
|
(0, gain_smoothing_1.applyGainSmooth)(nodes.gain, gainValue, this.audioContext, 0.1);
|
|
182
|
-
// ========== DEBUG: SPATIAL AUDIO CALCULATION ==========
|
|
183
|
-
console.log(`🔊 [SpatialAudio] Participant ${participantId}:`, {
|
|
184
|
-
speakerPos: speakerHead,
|
|
185
|
-
listenerPos: listenerPos,
|
|
186
|
-
distance: distance.toFixed(2) + 'm',
|
|
187
|
-
rawPan: rawPan.toFixed(3),
|
|
188
|
-
smoothedPan: smoothedPan.toFixed(3),
|
|
189
|
-
panValue: panValue.toFixed(3),
|
|
190
|
-
gainPercent: gainPercent.toFixed(1) + '%',
|
|
191
|
-
gainValue: gainValue.toFixed(3),
|
|
192
|
-
panning: { left: panning.left.toFixed(1), right: panning.right.toFixed(1) },
|
|
193
|
-
});
|
|
194
161
|
}
|
|
195
162
|
/**
|
|
196
163
|
* Set listener position from LSD data
|
|
@@ -203,13 +170,6 @@ class SpatialAudioChannel {
|
|
|
203
170
|
if (rot && typeof rot.y === 'number') {
|
|
204
171
|
this.listenerState.right = (0, pan_calc_1.calculateListenerRight)(rot.y);
|
|
205
172
|
}
|
|
206
|
-
// ========== DEBUG: LISTENER STATE UPDATE ==========
|
|
207
|
-
console.log('👂 [SpatialAudio] Listener state updated:', {
|
|
208
|
-
position: this.listenerState.position,
|
|
209
|
-
right: this.listenerState.right,
|
|
210
|
-
yawDegrees: rot?.y,
|
|
211
|
-
initialized: this.listenerState.initialized,
|
|
212
|
-
});
|
|
213
173
|
}
|
|
214
174
|
/**
|
|
215
175
|
* Toggle spatialization for a participant
|
|
@@ -239,7 +199,6 @@ class SpatialAudioChannel {
|
|
|
239
199
|
// Ignore - may already be routed correctly
|
|
240
200
|
}
|
|
241
201
|
}, fadeTime * 1000);
|
|
242
|
-
console.log(`[SpatialAudioChannel] ✅ Enabled spatialization for: ${participantId}`);
|
|
243
202
|
}
|
|
244
203
|
else {
|
|
245
204
|
// CROSSFADE: Connect direct path BEFORE disconnecting stereo panner
|
|
@@ -258,7 +217,6 @@ class SpatialAudioChannel {
|
|
|
258
217
|
// Already disconnected
|
|
259
218
|
}
|
|
260
219
|
}, fadeTime * 1000);
|
|
261
|
-
console.log(`[SpatialAudioChannel] ✅ Disabled spatialization (huddle mode) for: ${participantId}`);
|
|
262
220
|
}
|
|
263
221
|
}
|
|
264
222
|
catch (error) {
|
|
@@ -317,12 +275,10 @@ class SpatialAudioChannel {
|
|
|
317
275
|
*/
|
|
318
276
|
async initializeMLNoiseSuppression(modelPath) {
|
|
319
277
|
try {
|
|
320
|
-
console.log(`[SpatialAudioChannel] Initializing ML noise suppression: ${modelPath}`);
|
|
321
278
|
this.mlNoiseSuppressor = new MLNoiseSuppressor_1.MLNoiseSuppressor();
|
|
322
279
|
await this.mlNoiseSuppressor.initialize(modelPath);
|
|
323
280
|
if (this.mlNoiseSuppressor.isReady()) {
|
|
324
281
|
this.noiseSuppressionMode = 'ml';
|
|
325
|
-
console.log('[SpatialAudioChannel] ML noise suppression initialized successfully');
|
|
326
282
|
}
|
|
327
283
|
else {
|
|
328
284
|
throw new Error('ML model failed to load');
|
|
@@ -68,10 +68,6 @@ class MediasoupManager {
|
|
|
68
68
|
const params = await this.createWebRtcTransport('send', participantId);
|
|
69
69
|
// Extract iceServers from params and pass them correctly
|
|
70
70
|
const { iceServers, ...transportParams } = params;
|
|
71
|
-
console.log(`🎧 [MediasoupManager] Creating send transport with ICE servers:`, {
|
|
72
|
-
hasIceServers: !!iceServers,
|
|
73
|
-
iceServerCount: iceServers?.length || 0,
|
|
74
|
-
});
|
|
75
71
|
this.sendTransport = this.device.createSendTransport({
|
|
76
72
|
...transportParams,
|
|
77
73
|
iceServers: iceServers || [],
|
|
@@ -82,11 +78,6 @@ class MediasoupManager {
|
|
|
82
78
|
const params = await this.createWebRtcTransport('recv', participantId);
|
|
83
79
|
// Extract iceServers from params and pass them correctly
|
|
84
80
|
const { iceServers, ...transportParams } = params;
|
|
85
|
-
console.log(`🎧 [MediasoupManager] Creating recv transport with ICE servers:`, {
|
|
86
|
-
hasIceServers: !!iceServers,
|
|
87
|
-
iceServerCount: iceServers?.length || 0,
|
|
88
|
-
iceServers: iceServers,
|
|
89
|
-
});
|
|
90
81
|
this.recvTransport = this.device.createRecvTransport({
|
|
91
82
|
...transportParams,
|
|
92
83
|
iceServers: iceServers || [],
|
|
@@ -96,10 +87,12 @@ class MediasoupManager {
|
|
|
96
87
|
connectSendTransport() {
|
|
97
88
|
this.sendTransport?.on('connect', async ({ dtlsParameters }, callback, errback) => {
|
|
98
89
|
this.socket.emit('connect-transport', { transportId: this.sendTransport.id, dtlsParameters }, (response) => {
|
|
99
|
-
if (response.error)
|
|
90
|
+
if (response.error) {
|
|
100
91
|
errback(new Error(response.error));
|
|
101
|
-
|
|
92
|
+
}
|
|
93
|
+
else {
|
|
102
94
|
callback();
|
|
95
|
+
}
|
|
103
96
|
});
|
|
104
97
|
});
|
|
105
98
|
this.sendTransport?.on('produce', async ({ kind, rtpParameters, appData, }, callback, errback) => {
|
|
@@ -109,10 +102,12 @@ class MediasoupManager {
|
|
|
109
102
|
rtpParameters,
|
|
110
103
|
appData,
|
|
111
104
|
}, (response) => {
|
|
112
|
-
if (response.error)
|
|
105
|
+
if (response.error) {
|
|
113
106
|
errback(new Error(response.error));
|
|
114
|
-
|
|
107
|
+
}
|
|
108
|
+
else if (response.producerId) {
|
|
115
109
|
callback({ id: response.producerId });
|
|
110
|
+
}
|
|
116
111
|
});
|
|
117
112
|
});
|
|
118
113
|
this.sendTransport?.on('connectionstatechange', (state) => {
|
|
@@ -127,32 +122,19 @@ class MediasoupManager {
|
|
|
127
122
|
}
|
|
128
123
|
connectRecvTransport() {
|
|
129
124
|
this.recvTransport?.on('connect', async ({ dtlsParameters }, callback, errback) => {
|
|
130
|
-
console.log(`🎧 [MediasoupManager] Recv transport connecting, DTLS params:`, {
|
|
131
|
-
transportId: this.recvTransport.id.substring(0, 8),
|
|
132
|
-
});
|
|
133
125
|
this.socket.emit('connect-transport', { transportId: this.recvTransport.id, dtlsParameters }, (response) => {
|
|
134
126
|
if (response.error) {
|
|
135
|
-
console.error(`🎧 [MediasoupManager] ❌ Recv transport connect failed:`, response.error);
|
|
136
127
|
errback(new Error(response.error));
|
|
137
128
|
}
|
|
138
129
|
else {
|
|
139
|
-
console.log(`🎧 [MediasoupManager] ✅ Recv transport connected`);
|
|
140
130
|
callback();
|
|
141
131
|
}
|
|
142
132
|
});
|
|
143
133
|
});
|
|
144
134
|
this.recvTransport?.on('connectionstatechange', (state) => {
|
|
145
|
-
console.log(`🎧 [MediasoupManager] Recv transport connection state changed: ${state}`);
|
|
146
135
|
if (state === 'failed' || state === 'closed') {
|
|
147
|
-
|
|
136
|
+
// Handle transport failure
|
|
148
137
|
}
|
|
149
|
-
if (state === 'connected') {
|
|
150
|
-
console.log(`🎧 [MediasoupManager] ✅ Recv transport fully connected!`);
|
|
151
|
-
}
|
|
152
|
-
});
|
|
153
|
-
// Also monitor ICE state separately
|
|
154
|
-
this.recvTransport?.on('icegatheringstatechange', (state) => {
|
|
155
|
-
console.log(`🎧 [MediasoupManager] Recv transport ICE gathering state: ${state}`);
|
|
156
138
|
});
|
|
157
139
|
}
|
|
158
140
|
async produce(track, appData) {
|
|
@@ -183,27 +165,12 @@ class MediasoupManager {
|
|
|
183
165
|
async consume(data) {
|
|
184
166
|
if (!this.recvTransport)
|
|
185
167
|
throw new Error('Receive transport not set up');
|
|
186
|
-
console.log(`🎧 [MediasoupManager] Creating consumer for ${data.participantId.substring(0, 8)}`, {
|
|
187
|
-
consumerId: data.consumerId.substring(0, 8),
|
|
188
|
-
producerId: data.producerId.substring(0, 8),
|
|
189
|
-
kind: data.kind,
|
|
190
|
-
recvTransportId: this.recvTransport.id.substring(0, 8),
|
|
191
|
-
recvTransportConnectionState: this.recvTransport.connectionState,
|
|
192
|
-
});
|
|
193
168
|
const consumer = await this.recvTransport.consume({
|
|
194
169
|
id: data.consumerId,
|
|
195
170
|
producerId: data.producerId,
|
|
196
171
|
kind: data.kind,
|
|
197
172
|
rtpParameters: data.rtpParameters,
|
|
198
173
|
});
|
|
199
|
-
console.log(`🎧 [MediasoupManager] Consumer created for ${data.participantId.substring(0, 8)}`, {
|
|
200
|
-
consumerId: consumer.id.substring(0, 8),
|
|
201
|
-
trackKind: consumer.track.kind,
|
|
202
|
-
trackEnabled: consumer.track.enabled,
|
|
203
|
-
trackMuted: consumer.track.muted,
|
|
204
|
-
trackReadyState: consumer.track.readyState,
|
|
205
|
-
consumerPaused: consumer.paused,
|
|
206
|
-
});
|
|
207
174
|
consumer.on('transportclose', () => {
|
|
208
175
|
this.consumers.delete(consumer.id);
|
|
209
176
|
});
|
package/dist/index.js
CHANGED
|
@@ -211,15 +211,6 @@ class OdysseySpatialComms extends EventManager_1.EventManager {
|
|
|
211
211
|
pan: spatialData?.pan,
|
|
212
212
|
dxLocal: spatialData?.dxLocal,
|
|
213
213
|
};
|
|
214
|
-
// ========== DEBUG: SDK SENDING TO SERVER ==========
|
|
215
|
-
console.log("📤 [SDK->Server] updatePosition:", {
|
|
216
|
-
participantId: updateData.participantId,
|
|
217
|
-
position: updateData.position,
|
|
218
|
-
direction: updateData.direction,
|
|
219
|
-
rot: updateData.rot,
|
|
220
|
-
pan: updateData.pan,
|
|
221
|
-
dxLocal: updateData.dxLocal,
|
|
222
|
-
});
|
|
223
214
|
this.socket.emit("update-position", updateData);
|
|
224
215
|
}
|
|
225
216
|
}
|
|
@@ -238,13 +229,6 @@ class OdysseySpatialComms extends EventManager_1.EventManager {
|
|
|
238
229
|
this.spatialAudioManager.setListenerPosition(position, orientation);
|
|
239
230
|
}
|
|
240
231
|
setListenerFromLSD(listenerPos, cameraPos, lookAtPos, rot) {
|
|
241
|
-
// ========== DEBUG: LISTENER POSITION UPDATE ==========
|
|
242
|
-
console.log("🎧 [SDK] setListenerFromLSD:", {
|
|
243
|
-
listenerPos,
|
|
244
|
-
cameraPos,
|
|
245
|
-
lookAtPos,
|
|
246
|
-
rot,
|
|
247
|
-
});
|
|
248
232
|
this.spatialAudioManager.setListenerFromLSD(listenerPos, cameraPos, lookAtPos, rot);
|
|
249
233
|
}
|
|
250
234
|
listenForEvents() {
|
|
@@ -399,19 +383,11 @@ class OdysseySpatialComms extends EventManager_1.EventManager {
|
|
|
399
383
|
participant.consumers.set(consumer.id, consumer);
|
|
400
384
|
if (track.kind === "audio") {
|
|
401
385
|
participant.audioTrack = track;
|
|
402
|
-
console.log(`🎧 [SDK] Received audio consumer for ${participant.participantId.substring(0, 8)}`, {
|
|
403
|
-
consumerId: consumer.id.substring(0, 8),
|
|
404
|
-
trackEnabled: track.enabled,
|
|
405
|
-
trackMuted: track.muted,
|
|
406
|
-
trackReadyState: track.readyState,
|
|
407
|
-
consumerPaused: consumer.paused,
|
|
408
|
-
});
|
|
409
386
|
// CRITICAL: Do NOT setup spatial audio for local participant (yourself)
|
|
410
387
|
// This prevents hearing your own microphone (loopback)
|
|
411
388
|
const isLocalParticipant = participant.participantId ===
|
|
412
389
|
this.localParticipant?.participantId;
|
|
413
390
|
if (isLocalParticipant) {
|
|
414
|
-
console.log(`🎧 [SDK] Skipping audio setup for local participant (prevent loopback)`);
|
|
415
391
|
// Do NOT connect this audio to Web Audio API
|
|
416
392
|
return; // Exit early to prevent any audio processing
|
|
417
393
|
}
|
|
@@ -419,20 +395,16 @@ class OdysseySpatialComms extends EventManager_1.EventManager {
|
|
|
419
395
|
// Check if participant is in a huddle (non-spatial channel)
|
|
420
396
|
const participantChannel = participant.currentChannel || "spatial";
|
|
421
397
|
const isInHuddle = participantChannel !== "spatial";
|
|
422
|
-
console.log(`🎧 [SDK] Setting up spatial audio for ${participant.participantId.substring(0, 8)}, huddle: ${isInHuddle}`);
|
|
423
398
|
// Setup spatial audio - bypass 3D positioning for huddle members
|
|
424
399
|
await this.spatialAudioManager.setupSpatialAudioForParticipant(participant.participantId, track, isInHuddle // Bypass spatialization if in huddle
|
|
425
400
|
);
|
|
426
401
|
}
|
|
427
402
|
// NOW resume the consumer after audio pipeline is ready
|
|
428
|
-
console.log(`🎧 [SDK] Resuming consumer ${consumer.id.substring(0, 8)}...`);
|
|
429
403
|
this.mediasoupManager
|
|
430
404
|
.resumeConsumer(consumer.id)
|
|
431
|
-
.then(() => {
|
|
432
|
-
console.log(`🎧 [SDK] ✅ Consumer ${consumer.id.substring(0, 8)} resumed successfully`);
|
|
433
|
-
})
|
|
405
|
+
.then(() => { })
|
|
434
406
|
.catch((err) => {
|
|
435
|
-
console.error(
|
|
407
|
+
console.error(`[SDK] Failed to resume consumer:`, err);
|
|
436
408
|
});
|
|
437
409
|
}
|
|
438
410
|
else if (track.kind === "video") {
|
|
@@ -462,16 +434,6 @@ class OdysseySpatialComms extends EventManager_1.EventManager {
|
|
|
462
434
|
}
|
|
463
435
|
});
|
|
464
436
|
this.socket.on("participant-position-updated", (data) => {
|
|
465
|
-
// ========== DEBUG: SDK RECEIVED FROM SERVER ==========
|
|
466
|
-
console.log("📥 [SDK<-Server] participant-position-updated:", {
|
|
467
|
-
participantId: data.participantId,
|
|
468
|
-
position: data.position,
|
|
469
|
-
direction: data.direction,
|
|
470
|
-
rot: data.rot,
|
|
471
|
-
pan: data.pan,
|
|
472
|
-
dxLocal: data.dxLocal,
|
|
473
|
-
currentChannel: data.currentChannel,
|
|
474
|
-
});
|
|
475
437
|
const participant = this.room?.participants.get(data.participantId);
|
|
476
438
|
if (participant) {
|
|
477
439
|
participant.position = data.position;
|
package/package.json
CHANGED