@newgameplusinc/odyssey-audio-video-sdk-dev 1.0.262 → 1.0.263
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -65,9 +65,7 @@ class MLNoiseSuppressor {
|
|
|
65
65
|
await tf.ready();
|
|
66
66
|
// Load model - use standard loadLayersModel instead of fromMemory
|
|
67
67
|
// This handles weight loading automatically
|
|
68
|
-
console.log(`[MLNoiseSuppressor] Loading model from: ${modelUrl}`);
|
|
69
68
|
this.model = await tf.loadLayersModel(modelUrl);
|
|
70
|
-
console.log(`[MLNoiseSuppressor] Model loaded successfully`);
|
|
71
69
|
// Load config
|
|
72
70
|
const baseUrl = modelUrl.substring(0, modelUrl.lastIndexOf('/'));
|
|
73
71
|
const configResponse = await fetch(`${baseUrl}/model_config.json`);
|
|
@@ -81,7 +79,6 @@ class MLNoiseSuppressor {
|
|
|
81
79
|
this.normStats = { mean: 0, std: 1 };
|
|
82
80
|
}
|
|
83
81
|
this.isInitialized = true;
|
|
84
|
-
console.log(`[MLNoiseSuppressor] Initialization complete`);
|
|
85
82
|
}
|
|
86
83
|
catch (error) {
|
|
87
84
|
console.error(`[MLNoiseSuppressor] Initialization failed:`, error);
|
|
@@ -62,25 +62,11 @@ class SpatialAudioChannel {
|
|
|
62
62
|
* Setup spatial audio for a participant
|
|
63
63
|
*/
|
|
64
64
|
async setupParticipant(participantId, track, bypassSpatialization = false) {
|
|
65
|
-
console.log(`🎧 [SpatialAudioChannel] Setting up participant ${participantId.substring(0, 8)}`, {
|
|
66
|
-
trackKind: track.kind,
|
|
67
|
-
trackEnabled: track.enabled,
|
|
68
|
-
trackMuted: track.muted,
|
|
69
|
-
trackReadyState: track.readyState,
|
|
70
|
-
bypassSpatialization,
|
|
71
|
-
audioContextState: this.audioContext.state,
|
|
72
|
-
});
|
|
73
65
|
if (this.audioContext.state === 'suspended') {
|
|
74
|
-
console.log(`🎧 [SpatialAudioChannel] Resuming suspended AudioContext...`);
|
|
75
66
|
await this.audioContext.resume();
|
|
76
|
-
console.log(`🎧 [SpatialAudioChannel] AudioContext resumed, state: ${this.audioContext.state}`);
|
|
77
67
|
}
|
|
78
68
|
const stream = new MediaStream([track]);
|
|
79
69
|
const source = this.audioContext.createMediaStreamSource(stream);
|
|
80
|
-
console.log(`🎧 [SpatialAudioChannel] Created MediaStreamSource for ${participantId.substring(0, 8)}`, {
|
|
81
|
-
streamActive: stream.active,
|
|
82
|
-
streamTracks: stream.getTracks().length,
|
|
83
|
-
});
|
|
84
70
|
// Create all audio nodes
|
|
85
71
|
const panner = this.createPanner();
|
|
86
72
|
const stereoPanner = this.audioContext.createStereoPanner();
|
|
@@ -138,13 +124,6 @@ class SpatialAudioChannel {
|
|
|
138
124
|
dynamicLowpass,
|
|
139
125
|
stream,
|
|
140
126
|
});
|
|
141
|
-
console.log(`🎧 [SpatialAudioChannel] ✅ Audio pipeline setup complete for ${participantId.substring(0, 8)}`, {
|
|
142
|
-
bypassSpatialization,
|
|
143
|
-
masterGainValue: this.masterGainNode.gain.value,
|
|
144
|
-
participantGainValue: gain.gain.value,
|
|
145
|
-
audioContextState: this.audioContext.state,
|
|
146
|
-
connectedToDestination: true,
|
|
147
|
-
});
|
|
148
127
|
}
|
|
149
128
|
/**
|
|
150
129
|
* Update spatial audio for a participant
|
|
@@ -179,18 +158,6 @@ class SpatialAudioChannel {
|
|
|
179
158
|
(0, gain_smoothing_1.applyStereoPanSmooth)(nodes.stereoPanner, panValue, this.audioContext, 0.05);
|
|
180
159
|
// Apply gain
|
|
181
160
|
(0, gain_smoothing_1.applyGainSmooth)(nodes.gain, gainValue, this.audioContext, 0.1);
|
|
182
|
-
// ========== DEBUG: SPATIAL AUDIO CALCULATION ==========
|
|
183
|
-
console.log(`🔊 [SpatialAudio] Participant ${participantId}:`, {
|
|
184
|
-
speakerPos: speakerHead,
|
|
185
|
-
listenerPos: listenerPos,
|
|
186
|
-
distance: distance.toFixed(2) + 'm',
|
|
187
|
-
rawPan: rawPan.toFixed(3),
|
|
188
|
-
smoothedPan: smoothedPan.toFixed(3),
|
|
189
|
-
panValue: panValue.toFixed(3),
|
|
190
|
-
gainPercent: gainPercent.toFixed(1) + '%',
|
|
191
|
-
gainValue: gainValue.toFixed(3),
|
|
192
|
-
panning: { left: panning.left.toFixed(1), right: panning.right.toFixed(1) },
|
|
193
|
-
});
|
|
194
161
|
}
|
|
195
162
|
/**
|
|
196
163
|
* Set listener position from LSD data
|
|
@@ -203,13 +170,6 @@ class SpatialAudioChannel {
|
|
|
203
170
|
if (rot && typeof rot.y === 'number') {
|
|
204
171
|
this.listenerState.right = (0, pan_calc_1.calculateListenerRight)(rot.y);
|
|
205
172
|
}
|
|
206
|
-
// ========== DEBUG: LISTENER STATE UPDATE ==========
|
|
207
|
-
console.log('👂 [SpatialAudio] Listener state updated:', {
|
|
208
|
-
position: this.listenerState.position,
|
|
209
|
-
right: this.listenerState.right,
|
|
210
|
-
yawDegrees: rot?.y,
|
|
211
|
-
initialized: this.listenerState.initialized,
|
|
212
|
-
});
|
|
213
173
|
}
|
|
214
174
|
/**
|
|
215
175
|
* Toggle spatialization for a participant
|
|
@@ -239,7 +199,6 @@ class SpatialAudioChannel {
|
|
|
239
199
|
// Ignore - may already be routed correctly
|
|
240
200
|
}
|
|
241
201
|
}, fadeTime * 1000);
|
|
242
|
-
console.log(`[SpatialAudioChannel] ✅ Enabled spatialization for: ${participantId}`);
|
|
243
202
|
}
|
|
244
203
|
else {
|
|
245
204
|
// CROSSFADE: Connect direct path BEFORE disconnecting stereo panner
|
|
@@ -258,7 +217,6 @@ class SpatialAudioChannel {
|
|
|
258
217
|
// Already disconnected
|
|
259
218
|
}
|
|
260
219
|
}, fadeTime * 1000);
|
|
261
|
-
console.log(`[SpatialAudioChannel] ✅ Disabled spatialization (huddle mode) for: ${participantId}`);
|
|
262
220
|
}
|
|
263
221
|
}
|
|
264
222
|
catch (error) {
|
|
@@ -317,12 +275,10 @@ class SpatialAudioChannel {
|
|
|
317
275
|
*/
|
|
318
276
|
async initializeMLNoiseSuppression(modelPath) {
|
|
319
277
|
try {
|
|
320
|
-
console.log(`[SpatialAudioChannel] Initializing ML noise suppression: ${modelPath}`);
|
|
321
278
|
this.mlNoiseSuppressor = new MLNoiseSuppressor_1.MLNoiseSuppressor();
|
|
322
279
|
await this.mlNoiseSuppressor.initialize(modelPath);
|
|
323
280
|
if (this.mlNoiseSuppressor.isReady()) {
|
|
324
281
|
this.noiseSuppressionMode = 'ml';
|
|
325
|
-
console.log('[SpatialAudioChannel] ML noise suppression initialized successfully');
|
|
326
282
|
}
|
|
327
283
|
else {
|
|
328
284
|
throw new Error('ML model failed to load');
|
|
@@ -68,10 +68,6 @@ class MediasoupManager {
|
|
|
68
68
|
const params = await this.createWebRtcTransport('send', participantId);
|
|
69
69
|
// Extract iceServers from params and pass them correctly
|
|
70
70
|
const { iceServers, ...transportParams } = params;
|
|
71
|
-
console.log(`🎧 [MediasoupManager] Creating send transport with ICE servers:`, {
|
|
72
|
-
hasIceServers: !!iceServers,
|
|
73
|
-
iceServerCount: iceServers?.length || 0,
|
|
74
|
-
});
|
|
75
71
|
this.sendTransport = this.device.createSendTransport({
|
|
76
72
|
...transportParams,
|
|
77
73
|
iceServers: iceServers || [],
|
|
@@ -82,11 +78,6 @@ class MediasoupManager {
|
|
|
82
78
|
const params = await this.createWebRtcTransport('recv', participantId);
|
|
83
79
|
// Extract iceServers from params and pass them correctly
|
|
84
80
|
const { iceServers, ...transportParams } = params;
|
|
85
|
-
console.log(`🎧 [MediasoupManager] Creating recv transport with ICE servers:`, {
|
|
86
|
-
hasIceServers: !!iceServers,
|
|
87
|
-
iceServerCount: iceServers?.length || 0,
|
|
88
|
-
iceServers: iceServers,
|
|
89
|
-
});
|
|
90
81
|
this.recvTransport = this.device.createRecvTransport({
|
|
91
82
|
...transportParams,
|
|
92
83
|
iceServers: iceServers || [],
|
|
@@ -95,22 +86,16 @@ class MediasoupManager {
|
|
|
95
86
|
}
|
|
96
87
|
connectSendTransport() {
|
|
97
88
|
this.sendTransport?.on('connect', async ({ dtlsParameters }, callback, errback) => {
|
|
98
|
-
console.log(`🎤 [MediasoupManager] Send transport connecting, DTLS params:`, {
|
|
99
|
-
transportId: this.sendTransport.id.substring(0, 8),
|
|
100
|
-
});
|
|
101
89
|
this.socket.emit('connect-transport', { transportId: this.sendTransport.id, dtlsParameters }, (response) => {
|
|
102
90
|
if (response.error) {
|
|
103
|
-
console.error(`🎤 [MediasoupManager] ❌ Send transport connect failed:`, response.error);
|
|
104
91
|
errback(new Error(response.error));
|
|
105
92
|
}
|
|
106
93
|
else {
|
|
107
|
-
console.log(`🎤 [MediasoupManager] ✅ Send transport connected`);
|
|
108
94
|
callback();
|
|
109
95
|
}
|
|
110
96
|
});
|
|
111
97
|
});
|
|
112
98
|
this.sendTransport?.on('produce', async ({ kind, rtpParameters, appData, }, callback, errback) => {
|
|
113
|
-
console.log(`🎤 [MediasoupManager] Producing ${kind} track via socket...`);
|
|
114
99
|
this.socket.emit('produce', {
|
|
115
100
|
transportId: this.sendTransport.id,
|
|
116
101
|
kind,
|
|
@@ -118,22 +103,15 @@ class MediasoupManager {
|
|
|
118
103
|
appData,
|
|
119
104
|
}, (response) => {
|
|
120
105
|
if (response.error) {
|
|
121
|
-
console.error(`🎤 [MediasoupManager] ❌ Produce failed:`, response.error);
|
|
122
106
|
errback(new Error(response.error));
|
|
123
107
|
}
|
|
124
108
|
else if (response.producerId) {
|
|
125
|
-
console.log(`🎤 [MediasoupManager] ✅ Produce success, producerId:`, response.producerId.substring(0, 8));
|
|
126
109
|
callback({ id: response.producerId });
|
|
127
110
|
}
|
|
128
111
|
});
|
|
129
112
|
});
|
|
130
113
|
this.sendTransport?.on('connectionstatechange', (state) => {
|
|
131
|
-
console.log(`🎤 [MediasoupManager] Send transport connection state changed: ${state}`);
|
|
132
|
-
if (state === 'connected') {
|
|
133
|
-
console.log(`🎤 [MediasoupManager] ✅ Send transport fully connected!`);
|
|
134
|
-
}
|
|
135
114
|
if (state === 'failed' || state === 'closed') {
|
|
136
|
-
console.error(`🎤 [MediasoupManager] ❌ Send transport ${state}`);
|
|
137
115
|
this.producers.clear();
|
|
138
116
|
this.socket.emit('transport-failed', {
|
|
139
117
|
participantId: this.participantId,
|
|
@@ -141,53 +119,27 @@ class MediasoupManager {
|
|
|
141
119
|
});
|
|
142
120
|
}
|
|
143
121
|
});
|
|
144
|
-
// Also monitor ICE state separately
|
|
145
|
-
this.sendTransport?.on('icegatheringstatechange', (state) => {
|
|
146
|
-
console.log(`🎤 [MediasoupManager] Send transport ICE gathering state: ${state}`);
|
|
147
|
-
});
|
|
148
122
|
}
|
|
149
123
|
connectRecvTransport() {
|
|
150
124
|
this.recvTransport?.on('connect', async ({ dtlsParameters }, callback, errback) => {
|
|
151
|
-
console.log(`🎧 [MediasoupManager] Recv transport connecting, DTLS params:`, {
|
|
152
|
-
transportId: this.recvTransport.id.substring(0, 8),
|
|
153
|
-
});
|
|
154
125
|
this.socket.emit('connect-transport', { transportId: this.recvTransport.id, dtlsParameters }, (response) => {
|
|
155
126
|
if (response.error) {
|
|
156
|
-
console.error(`🎧 [MediasoupManager] ❌ Recv transport connect failed:`, response.error);
|
|
157
127
|
errback(new Error(response.error));
|
|
158
128
|
}
|
|
159
129
|
else {
|
|
160
|
-
console.log(`🎧 [MediasoupManager] ✅ Recv transport connected`);
|
|
161
130
|
callback();
|
|
162
131
|
}
|
|
163
132
|
});
|
|
164
133
|
});
|
|
165
134
|
this.recvTransport?.on('connectionstatechange', (state) => {
|
|
166
|
-
console.log(`🎧 [MediasoupManager] Recv transport connection state changed: ${state}`);
|
|
167
135
|
if (state === 'failed' || state === 'closed') {
|
|
168
|
-
|
|
169
|
-
}
|
|
170
|
-
if (state === 'connected') {
|
|
171
|
-
console.log(`🎧 [MediasoupManager] ✅ Recv transport fully connected!`);
|
|
136
|
+
// Handle transport failure
|
|
172
137
|
}
|
|
173
138
|
});
|
|
174
|
-
// Also monitor ICE state separately
|
|
175
|
-
this.recvTransport?.on('icegatheringstatechange', (state) => {
|
|
176
|
-
console.log(`🎧 [MediasoupManager] Recv transport ICE gathering state: ${state}`);
|
|
177
|
-
});
|
|
178
139
|
}
|
|
179
140
|
async produce(track, appData) {
|
|
180
141
|
if (!this.sendTransport)
|
|
181
142
|
throw new Error('Send transport not initialized');
|
|
182
|
-
// Debug: Log track state BEFORE producing
|
|
183
|
-
console.log(`🎤 [MediasoupManager] Producing ${track.kind} track:`, {
|
|
184
|
-
trackId: track.id.substring(0, 8),
|
|
185
|
-
enabled: track.enabled,
|
|
186
|
-
muted: track.muted,
|
|
187
|
-
readyState: track.readyState,
|
|
188
|
-
label: track.label,
|
|
189
|
-
sendTransportState: this.sendTransport.connectionState,
|
|
190
|
-
});
|
|
191
143
|
const produceOptions = { track, appData };
|
|
192
144
|
if (track.kind === 'video') {
|
|
193
145
|
produceOptions.encodings = [
|
|
@@ -200,12 +152,6 @@ class MediasoupManager {
|
|
|
200
152
|
};
|
|
201
153
|
}
|
|
202
154
|
const producer = await this.sendTransport.produce(produceOptions);
|
|
203
|
-
console.log(`🎤 [MediasoupManager] Producer created:`, {
|
|
204
|
-
producerId: producer.id.substring(0, 8),
|
|
205
|
-
kind: producer.kind,
|
|
206
|
-
paused: producer.paused,
|
|
207
|
-
closed: producer.closed,
|
|
208
|
-
});
|
|
209
155
|
producer.on('transportclose', () => {
|
|
210
156
|
this.producers.delete(producer.id);
|
|
211
157
|
});
|
|
@@ -219,27 +165,12 @@ class MediasoupManager {
|
|
|
219
165
|
async consume(data) {
|
|
220
166
|
if (!this.recvTransport)
|
|
221
167
|
throw new Error('Receive transport not set up');
|
|
222
|
-
console.log(`🎧 [MediasoupManager] Creating consumer for ${data.participantId.substring(0, 8)}`, {
|
|
223
|
-
consumerId: data.consumerId.substring(0, 8),
|
|
224
|
-
producerId: data.producerId.substring(0, 8),
|
|
225
|
-
kind: data.kind,
|
|
226
|
-
recvTransportId: this.recvTransport.id.substring(0, 8),
|
|
227
|
-
recvTransportConnectionState: this.recvTransport.connectionState,
|
|
228
|
-
});
|
|
229
168
|
const consumer = await this.recvTransport.consume({
|
|
230
169
|
id: data.consumerId,
|
|
231
170
|
producerId: data.producerId,
|
|
232
171
|
kind: data.kind,
|
|
233
172
|
rtpParameters: data.rtpParameters,
|
|
234
173
|
});
|
|
235
|
-
console.log(`🎧 [MediasoupManager] Consumer created for ${data.participantId.substring(0, 8)}`, {
|
|
236
|
-
consumerId: consumer.id.substring(0, 8),
|
|
237
|
-
trackKind: consumer.track.kind,
|
|
238
|
-
trackEnabled: consumer.track.enabled,
|
|
239
|
-
trackMuted: consumer.track.muted,
|
|
240
|
-
trackReadyState: consumer.track.readyState,
|
|
241
|
-
consumerPaused: consumer.paused,
|
|
242
|
-
});
|
|
243
174
|
consumer.on('transportclose', () => {
|
|
244
175
|
this.consumers.delete(consumer.id);
|
|
245
176
|
});
|
package/dist/index.js
CHANGED
|
@@ -211,15 +211,6 @@ class OdysseySpatialComms extends EventManager_1.EventManager {
|
|
|
211
211
|
pan: spatialData?.pan,
|
|
212
212
|
dxLocal: spatialData?.dxLocal,
|
|
213
213
|
};
|
|
214
|
-
// ========== DEBUG: SDK SENDING TO SERVER ==========
|
|
215
|
-
console.log("📤 [SDK->Server] updatePosition:", {
|
|
216
|
-
participantId: updateData.participantId,
|
|
217
|
-
position: updateData.position,
|
|
218
|
-
direction: updateData.direction,
|
|
219
|
-
rot: updateData.rot,
|
|
220
|
-
pan: updateData.pan,
|
|
221
|
-
dxLocal: updateData.dxLocal,
|
|
222
|
-
});
|
|
223
214
|
this.socket.emit("update-position", updateData);
|
|
224
215
|
}
|
|
225
216
|
}
|
|
@@ -238,13 +229,6 @@ class OdysseySpatialComms extends EventManager_1.EventManager {
|
|
|
238
229
|
this.spatialAudioManager.setListenerPosition(position, orientation);
|
|
239
230
|
}
|
|
240
231
|
setListenerFromLSD(listenerPos, cameraPos, lookAtPos, rot) {
|
|
241
|
-
// ========== DEBUG: LISTENER POSITION UPDATE ==========
|
|
242
|
-
console.log("🎧 [SDK] setListenerFromLSD:", {
|
|
243
|
-
listenerPos,
|
|
244
|
-
cameraPos,
|
|
245
|
-
lookAtPos,
|
|
246
|
-
rot,
|
|
247
|
-
});
|
|
248
232
|
this.spatialAudioManager.setListenerFromLSD(listenerPos, cameraPos, lookAtPos, rot);
|
|
249
233
|
}
|
|
250
234
|
listenForEvents() {
|
|
@@ -399,19 +383,11 @@ class OdysseySpatialComms extends EventManager_1.EventManager {
|
|
|
399
383
|
participant.consumers.set(consumer.id, consumer);
|
|
400
384
|
if (track.kind === "audio") {
|
|
401
385
|
participant.audioTrack = track;
|
|
402
|
-
console.log(`🎧 [SDK] Received audio consumer for ${participant.participantId.substring(0, 8)}`, {
|
|
403
|
-
consumerId: consumer.id.substring(0, 8),
|
|
404
|
-
trackEnabled: track.enabled,
|
|
405
|
-
trackMuted: track.muted,
|
|
406
|
-
trackReadyState: track.readyState,
|
|
407
|
-
consumerPaused: consumer.paused,
|
|
408
|
-
});
|
|
409
386
|
// CRITICAL: Do NOT setup spatial audio for local participant (yourself)
|
|
410
387
|
// This prevents hearing your own microphone (loopback)
|
|
411
388
|
const isLocalParticipant = participant.participantId ===
|
|
412
389
|
this.localParticipant?.participantId;
|
|
413
390
|
if (isLocalParticipant) {
|
|
414
|
-
console.log(`🎧 [SDK] Skipping audio setup for local participant (prevent loopback)`);
|
|
415
391
|
// Do NOT connect this audio to Web Audio API
|
|
416
392
|
return; // Exit early to prevent any audio processing
|
|
417
393
|
}
|
|
@@ -419,20 +395,16 @@ class OdysseySpatialComms extends EventManager_1.EventManager {
|
|
|
419
395
|
// Check if participant is in a huddle (non-spatial channel)
|
|
420
396
|
const participantChannel = participant.currentChannel || "spatial";
|
|
421
397
|
const isInHuddle = participantChannel !== "spatial";
|
|
422
|
-
console.log(`🎧 [SDK] Setting up spatial audio for ${participant.participantId.substring(0, 8)}, huddle: ${isInHuddle}`);
|
|
423
398
|
// Setup spatial audio - bypass 3D positioning for huddle members
|
|
424
399
|
await this.spatialAudioManager.setupSpatialAudioForParticipant(participant.participantId, track, isInHuddle // Bypass spatialization if in huddle
|
|
425
400
|
);
|
|
426
401
|
}
|
|
427
402
|
// NOW resume the consumer after audio pipeline is ready
|
|
428
|
-
console.log(`🎧 [SDK] Resuming consumer ${consumer.id.substring(0, 8)}...`);
|
|
429
403
|
this.mediasoupManager
|
|
430
404
|
.resumeConsumer(consumer.id)
|
|
431
|
-
.then(() => {
|
|
432
|
-
console.log(`🎧 [SDK] ✅ Consumer ${consumer.id.substring(0, 8)} resumed successfully`);
|
|
433
|
-
})
|
|
405
|
+
.then(() => { })
|
|
434
406
|
.catch((err) => {
|
|
435
|
-
console.error(
|
|
407
|
+
console.error(`[SDK] Failed to resume consumer:`, err);
|
|
436
408
|
});
|
|
437
409
|
}
|
|
438
410
|
else if (track.kind === "video") {
|
|
@@ -462,16 +434,6 @@ class OdysseySpatialComms extends EventManager_1.EventManager {
|
|
|
462
434
|
}
|
|
463
435
|
});
|
|
464
436
|
this.socket.on("participant-position-updated", (data) => {
|
|
465
|
-
// ========== DEBUG: SDK RECEIVED FROM SERVER ==========
|
|
466
|
-
console.log("📥 [SDK<-Server] participant-position-updated:", {
|
|
467
|
-
participantId: data.participantId,
|
|
468
|
-
position: data.position,
|
|
469
|
-
direction: data.direction,
|
|
470
|
-
rot: data.rot,
|
|
471
|
-
pan: data.pan,
|
|
472
|
-
dxLocal: data.dxLocal,
|
|
473
|
-
currentChannel: data.currentChannel,
|
|
474
|
-
});
|
|
475
437
|
const participant = this.room?.participants.get(data.participantId);
|
|
476
438
|
if (participant) {
|
|
477
439
|
participant.position = data.position;
|
package/package.json
CHANGED