@newgameplusinc/odyssey-audio-video-sdk-dev 1.0.57 → 1.0.59
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -255
- package/dist/MediasoupManager.js +11 -27
- package/dist/index.d.ts +8 -15
- package/dist/index.js +31 -76
- package/package.json +3 -4
- package/dist/MLNoiseSuppressor.d.ts +0 -76
- package/dist/MLNoiseSuppressor.js +0 -439
- package/dist/UltimateMLNoiseSuppressor.d.ts +0 -74
- package/dist/UltimateMLNoiseSuppressor.js +0 -309
package/README.md
CHANGED
|
@@ -11,69 +11,12 @@ It mirrors the production SDK used by Odyssey V2 and ships ready-to-drop into an
|
|
|
11
11
|
## Feature Highlights
|
|
12
12
|
- 🔌 **One class to rule it all** – `OdysseySpatialComms` wires transports, producers, consumers, and room state.
|
|
13
13
|
- 🧭 **Accurate pose propagation** – `updatePosition()` streams listener pose to the SFU while `participant-position-updated` keeps the local store in sync.
|
|
14
|
-
- 🤖 **AI-Powered Noise Suppression** – Deep learning model (TensorFlow.js) runs client-side to remove background noise BEFORE audio reaches MediaSoup. Uses trained LSTM-based mask prediction for superior noise cancellation without affecting voice quality.
|
|
15
14
|
- 🎧 **Studio-grade spatial audio** – each remote participant gets a dedicated Web Audio graph: denoiser → high-pass → low-pass → HRTF `PannerNode` → adaptive gain → master compressor. Uses Web Audio API's HRTF panning model for accurate left/right/front/back positioning based on distance and direction, with custom AudioWorklet processors for noise cancellation and voice tuning.
|
|
16
15
|
- 🎥 **Camera-ready streams** – video tracks are exposed separately so UI layers can render muted `<video>` tags while audio stays inside Web Audio.
|
|
17
16
|
- 🔁 **EventEmitter contract** – subscribe to `room-joined`, `consumer-created`, `participant-position-updated`, etc., without touching Socket.IO directly.
|
|
18
17
|
|
|
19
18
|
## Quick Start
|
|
20
19
|
|
|
21
|
-
### With ML Noise Suppression (Recommended)
|
|
22
|
-
|
|
23
|
-
```ts
|
|
24
|
-
import {
|
|
25
|
-
OdysseySpatialComms,
|
|
26
|
-
Direction,
|
|
27
|
-
Position,
|
|
28
|
-
} from "@newgameplusinc/odyssey-audio-video-sdk-dev";
|
|
29
|
-
|
|
30
|
-
const sdk = new OdysseySpatialComms("https://mediasoup-server.example.com");
|
|
31
|
-
|
|
32
|
-
// 1) Initialize ML noise suppression (place model files in public/models/)
|
|
33
|
-
await sdk.initializeMLNoiseSuppression(
|
|
34
|
-
'/models/odyssey_noise_suppressor_v1/model.json'
|
|
35
|
-
);
|
|
36
|
-
|
|
37
|
-
// 2) Join a room
|
|
38
|
-
await sdk.joinRoom({
|
|
39
|
-
roomId: "demo-room",
|
|
40
|
-
userId: "user-123",
|
|
41
|
-
deviceId: "device-123",
|
|
42
|
-
position: { x: 0, y: 0, z: 0 },
|
|
43
|
-
direction: { x: 0, y: 1, z: 0 },
|
|
44
|
-
});
|
|
45
|
-
|
|
46
|
-
// 3) Produce local media (ML cleaning applied automatically to audio)
|
|
47
|
-
const stream = await navigator.mediaDevices.getUserMedia({
|
|
48
|
-
audio: {
|
|
49
|
-
echoCancellation: true,
|
|
50
|
-
noiseSuppression: false, // Disable browser NS, use ML instead!
|
|
51
|
-
autoGainControl: true,
|
|
52
|
-
sampleRate: 48000,
|
|
53
|
-
},
|
|
54
|
-
video: true
|
|
55
|
-
});
|
|
56
|
-
for (const track of stream.getTracks()) {
|
|
57
|
-
await sdk.produceTrack(track); // ML processes audio tracks automatically
|
|
58
|
-
}
|
|
59
|
-
|
|
60
|
-
// 4) Toggle ML noise suppression on/off
|
|
61
|
-
sdk.toggleMLNoiseSuppression(true); // or false
|
|
62
|
-
|
|
63
|
-
// 5) Handle remote tracks
|
|
64
|
-
sdk.on("consumer-created", async ({ participant, track }) => {
|
|
65
|
-
if (track.kind === "video") {
|
|
66
|
-
attachVideo(track, participant.participantId);
|
|
67
|
-
}
|
|
68
|
-
});
|
|
69
|
-
|
|
70
|
-
// 6) Keep spatial audio honest
|
|
71
|
-
sdk.updatePosition(currentPos, currentDir);
|
|
72
|
-
sdk.setListenerFromLSD(listenerPos, cameraPos, lookAtPos);
|
|
73
|
-
```
|
|
74
|
-
|
|
75
|
-
### Without ML Noise Suppression (Legacy)
|
|
76
|
-
|
|
77
20
|
```ts
|
|
78
21
|
import {
|
|
79
22
|
OdysseySpatialComms,
|
|
@@ -113,83 +56,23 @@ sdk.setListenerFromLSD(listenerPos, cameraPos, lookAtPos);
|
|
|
113
56
|
## Audio Flow (Server ↔ Browser)
|
|
114
57
|
|
|
115
58
|
```
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
│
|
|
122
|
-
│
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
│ SDK: mediasoupManager.produce() │
|
|
134
|
-
│ (Sends clean track to server) │
|
|
135
|
-
└────────┬─────────────────────────────────┘
|
|
136
|
-
│
|
|
137
|
-
│ WebRTC/RTP
|
|
138
|
-
▼
|
|
139
|
-
┌─────────────────────────────────────────────┐
|
|
140
|
-
│ SERVER-SIDE ROUTING │
|
|
141
|
-
└─────────────────────────────────────────────┘
|
|
142
|
-
|
|
143
|
-
┌──────────────┐ update-position ┌──────────────┐ route clean audio ┌──────────────────┐
|
|
144
|
-
│ Browser LSD │ ──────────────────▶ │ MediaSoup SFU│ ───────────────────▶ │ Other Clients │
|
|
145
|
-
│ (Unreal data)│ │ + Socket.IO │ │ (Receive RTP) │
|
|
146
|
-
└──────────────┘ └──────┬───────┘ └──────┬───────────┘
|
|
147
|
-
│ │
|
|
148
|
-
│ consumer-created event │
|
|
149
|
-
▼ ▼
|
|
150
|
-
┌─────────────────────────────────────────────┐
|
|
151
|
-
│ REMOTE AUDIO PLAYBACK │
|
|
152
|
-
└─────────────────────────────────────────────┘
|
|
153
|
-
|
|
154
|
-
┌──────────────────┐
|
|
155
|
-
│ SDK Event Bus │
|
|
156
|
-
│ (EventManager) │
|
|
157
|
-
└────────┬─────────┘
|
|
158
|
-
│ track + pose
|
|
159
|
-
▼
|
|
160
|
-
┌──────────────────┐
|
|
161
|
-
│ SpatialAudioMgr │
|
|
162
|
-
│ (Web Audio API) │
|
|
163
|
-
│ • Denoiser │◀─── Traditional noise reduction
|
|
164
|
-
│ • HP/LP Filters │ (runs on received audio)
|
|
165
|
-
│ • HRTF Panner │
|
|
166
|
-
│ • Distance Gain │
|
|
167
|
-
│ • Compressor │
|
|
168
|
-
└────────┬─────────┘
|
|
169
|
-
│
|
|
170
|
-
▼
|
|
171
|
-
┌──────────────────┐
|
|
172
|
-
│ Web Audio Graph │
|
|
173
|
-
└────────┬─────────┘
|
|
174
|
-
│
|
|
175
|
-
▼
|
|
176
|
-
Listener ears (Left/Right)
|
|
177
|
-
│
|
|
178
|
-
▼
|
|
179
|
-
System Output
|
|
180
|
-
```
|
|
181
|
-
|
|
182
|
-
### ML Noise Suppression Pipeline (Client-Side)
|
|
183
|
-
```
|
|
184
|
-
Mic → getUserMedia()
|
|
185
|
-
↓
|
|
186
|
-
Vue: sdk.produceTrack(audioTrack)
|
|
187
|
-
↓
|
|
188
|
-
SDK: mlNoiseSuppressor.processMediaStream() [TensorFlow.js runs here]
|
|
189
|
-
↓
|
|
190
|
-
SDK: mediasoupManager.produce(cleanTrack)
|
|
191
|
-
↓
|
|
192
|
-
MediaSoup Server → Other participants hear clean audio ✅
|
|
59
|
+
┌──────────────┐ update-position ┌──────────────┐ pose + tracks ┌──────────────────┐
|
|
60
|
+
│ Browser LSD │ ──────────────────▶ │ MediaSoup SFU│ ────────────────▶ │ SDK Event Bus │
|
|
61
|
+
│ (Unreal data)│ │ + Socket.IO │ │ (EventManager) │
|
|
62
|
+
└──────┬───────┘ └──────┬───────┘ └──────────┬────────┘
|
|
63
|
+
│ │ track + pose
|
|
64
|
+
│ │ ▼
|
|
65
|
+
│ ┌────────▼────────┐ ┌──────────────────┐
|
|
66
|
+
│ audio RTP │ consumer-created│ │ SpatialAudioMgr │
|
|
67
|
+
└──────────────────────────▶│ setup per-user │◀──────────────────────│ (Web Audio API) │
|
|
68
|
+
└────────┬────────┘ │ - Denoiser │
|
|
69
|
+
│ │ - HP / LP │
|
|
70
|
+
│ │ - HRTF Panner │
|
|
71
|
+
▼ │ - Gain + Comp │
|
|
72
|
+
Web Audio Graph └──────────┬───────┘
|
|
73
|
+
│ │
|
|
74
|
+
▼ ▼
|
|
75
|
+
Listener ears (Left/Right) System Output
|
|
193
76
|
```
|
|
194
77
|
|
|
195
78
|
### Web Audio Algorithms
|
|
@@ -236,121 +119,7 @@ These layers run entirely in Web Audio, so you can ship “AirPods-style” back
|
|
|
236
119
|
3. **Position + direction updates** – every `participant-position-updated` event calls `updateSpatialAudio(participantId, position, direction)`. The position feeds the panner’s XYZ, while the direction vector sets the source orientation so voices project forward relative to avatar facing.
|
|
237
120
|
4. **Distance-aware gain** – the manager stores the latest listener pose and computes the Euclidean distance to each remote participant on every update. A custom rolloff curve adjusts gain before the compressor, giving the “someone on my left / far away” perception without blowing out master levels.
|
|
238
121
|
5. **Left/right rendering** – because the panner uses `panningModel = "HRTF"`, browsers feed the processed signal into the user’s audio hardware with head-related transfer functions, producing natural interaural time/intensity differences.
|
|
239
|
-
## ML Noise Suppression (Deep Learning Pre-Processing)
|
|
240
|
-
|
|
241
|
-
**NEW:** The SDK now includes an optional **AI-powered noise suppression** layer that runs **BEFORE** audio reaches MediaSoup, using a trained TensorFlow.js model.
|
|
242
|
-
|
|
243
|
-
### Why ML Noise Suppression?
|
|
244
|
-
- **Superior noise removal** – Deep learning models learn complex noise patterns that traditional DSP can't handle (keyboard typing, paper rustling, traffic, etc.)
|
|
245
|
-
- **Voice preservation** – LSTM-based mask prediction preserves natural voice quality while removing background noise
|
|
246
|
-
- **Client-side processing** – Runs entirely in the browser using TensorFlow.js (WebGL/WebAssembly acceleration)
|
|
247
|
-
- **Privacy-first** – Audio never leaves the user's device; processing happens locally
|
|
248
|
-
- **Zero latency** – <10ms processing time per frame, suitable for real-time communication
|
|
249
|
-
|
|
250
|
-
### Architecture
|
|
251
|
-
```
|
|
252
|
-
Raw Mic Audio → ML Model (TF.js) → Clean Audio → MediaSoup → Traditional Denoiser → Spatial Audio
|
|
253
|
-
```
|
|
254
|
-
|
|
255
|
-
The ML model applies **mask-based spectral subtraction** trained on diverse noise datasets:
|
|
256
|
-
1. Extracts mel-spectrogram from raw audio
|
|
257
|
-
2. Predicts a noise mask (0-1 per frequency bin) using Bidirectional LSTM
|
|
258
|
-
3. Applies mask to remove noise while preserving speech
|
|
259
|
-
4. Reconstructs clean audio waveform
|
|
260
|
-
|
|
261
|
-
### Setup ML Noise Suppression
|
|
262
|
-
|
|
263
|
-
**1. Place Model Files:**
|
|
264
|
-
```
|
|
265
|
-
YourApp/public/models/odyssey_noise_suppressor_v1/
|
|
266
|
-
├── model.json # TF.js model architecture
|
|
267
|
-
├── group1-shard*.bin # Model weights (multiple files)
|
|
268
|
-
├── normalization_stats.json # Preprocessing parameters
|
|
269
|
-
└── model_config.json # Audio config (48kHz, n_mels, etc.)
|
|
270
|
-
```
|
|
271
|
-
|
|
272
|
-
**2. Initialize in Code:**
|
|
273
|
-
```ts
|
|
274
|
-
const sdk = new OdysseySpatialComms('wss://your-server.com');
|
|
275
|
-
|
|
276
|
-
// Initialize ML noise suppression
|
|
277
|
-
try {
|
|
278
|
-
await sdk.initializeMLNoiseSuppression(
|
|
279
|
-
'/models/odyssey_noise_suppressor_v1/model.json'
|
|
280
|
-
);
|
|
281
|
-
console.log('✅ ML Noise Suppression enabled');
|
|
282
|
-
} catch (error) {
|
|
283
|
-
console.error('ML initialization failed:', error);
|
|
284
|
-
// Graceful degradation - SDK continues without ML
|
|
285
|
-
}
|
|
286
|
-
|
|
287
|
-
// Produce audio tracks (ML cleaning applied automatically)
|
|
288
|
-
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
|
|
289
|
-
await sdk.produceTrack(stream.getAudioTracks()[0]);
|
|
290
|
-
|
|
291
|
-
// Toggle ML on/off at runtime
|
|
292
|
-
sdk.toggleMLNoiseSuppression(false); // Disable
|
|
293
|
-
sdk.toggleMLNoiseSuppression(true); // Re-enable
|
|
294
|
-
|
|
295
|
-
// Check ML status
|
|
296
|
-
if (sdk.isMLNoiseSuppressionEnabled()) {
|
|
297
|
-
console.log('ML is active');
|
|
298
|
-
}
|
|
299
|
-
```
|
|
300
|
-
|
|
301
|
-
**3. Recommended Audio Constraints:**
|
|
302
|
-
```ts
|
|
303
|
-
const stream = await navigator.mediaDevices.getUserMedia({
|
|
304
|
-
audio: {
|
|
305
|
-
echoCancellation: true, // Keep echo cancellation
|
|
306
|
-
noiseSuppression: false, // Disable browser NS (ML replaces it)
|
|
307
|
-
autoGainControl: true, // Keep AGC
|
|
308
|
-
sampleRate: 48000, // Match model training (48kHz)
|
|
309
|
-
},
|
|
310
|
-
});
|
|
311
|
-
```
|
|
312
|
-
|
|
313
|
-
### ML Model Details
|
|
314
|
-
- **Architecture:** Bidirectional LSTM (2 layers, 256 units) + Dense layers
|
|
315
|
-
- **Input:** 48kHz audio → Mel-spectrogram (128 bins, 8-frame sequences)
|
|
316
|
-
- **Output:** Time-frequency mask (0-1 values per bin)
|
|
317
|
-
- **Latency:** ~5-8ms per chunk (AudioWorklet processing)
|
|
318
|
-
- **Model Size:** ~2-3 MB (quantized to uint8)
|
|
319
|
-
- **Training:** LibriSpeech (clean speech) + AudioSet (noise) datasets
|
|
320
|
-
|
|
321
|
-
### When to Use ML vs Traditional Denoiser
|
|
322
|
-
|
|
323
|
-
| Feature | ML Noise Suppression | Traditional Denoiser (AudioWorklet) |
|
|
324
|
-
|---------|---------------------|-------------------------------------|
|
|
325
|
-
| **Noise Types** | Complex (keyboard, traffic, music) | Stationary (fan, HVAC, hiss) |
|
|
326
|
-
| **Voice Quality** | Excellent (learned patterns) | Good (spectral shaping) |
|
|
327
|
-
| **CPU Usage** | Medium (TF.js optimized) | Low (simple DSP) |
|
|
328
|
-
| **Latency** | ~5-8ms | ~1-2ms |
|
|
329
|
-
| **Use Case** | Noisy environments | Quiet rooms with constant noise |
|
|
330
|
-
|
|
331
|
-
**Best Practice:** Enable **both** for maximum quality:
|
|
332
|
-
- ML suppresses complex noise (pre-MediaSoup)
|
|
333
|
-
- Traditional denoiser handles residual stationary noise (post-receive)
|
|
334
|
-
|
|
335
|
-
### Troubleshooting
|
|
336
|
-
|
|
337
|
-
**Model fails to load:**
|
|
338
|
-
- Ensure model files are served as static assets (check browser Network tab)
|
|
339
|
-
- Verify CORS headers if serving from CDN
|
|
340
|
-
- Check browser console for TensorFlow.js errors
|
|
341
|
-
|
|
342
|
-
**High CPU usage:**
|
|
343
|
-
- TF.js automatically uses WebGL when available (much faster)
|
|
344
|
-
- Disable ML on low-end devices: `sdk.toggleMLNoiseSuppression(false)`
|
|
345
|
-
|
|
346
|
-
**Voice sounds muffled:**
|
|
347
|
-
- Model trained on 48kHz audio; ensure mic uses same sample rate
|
|
348
|
-
- Check if browser is downsampling to 16kHz (some mobile browsers do this)
|
|
349
122
|
|
|
350
|
-
**Doesn't remove all noise:**
|
|
351
|
-
- ML works best on noise types seen during training
|
|
352
|
-
- Combine with traditional denoiser for residual cleanup
|
|
353
|
-
- Extremely loud noise (>30 dB SNR) may leak through
|
|
354
123
|
## Video Flow (Capture ↔ Rendering)
|
|
355
124
|
|
|
356
125
|
```
|
|
@@ -368,19 +137,17 @@ const stream = await navigator.mediaDevices.getUserMedia({
|
|
|
368
137
|
```
|
|
369
138
|
|
|
370
139
|
## Core Classes
|
|
371
|
-
- `src/index.ts` – `OdysseySpatialComms` (socket lifecycle, producers/consumers, event surface
|
|
140
|
+
- `src/index.ts` – `OdysseySpatialComms` (socket lifecycle, producers/consumers, event surface).
|
|
372
141
|
- `src/MediasoupManager.ts` – transport helpers for produce/consume/resume.
|
|
373
142
|
- `src/SpatialAudioManager.ts` – Web Audio orchestration (listener transforms, per-participant chains, denoiser, distance math).
|
|
374
|
-
- `src/MLNoiseSuppressor.ts` – TensorFlow.js-based deep learning noise suppression (mel-spectrogram extraction, LSTM inference, mask application).
|
|
375
143
|
- `src/EventManager.ts` – lightweight EventEmitter used by the entire SDK.
|
|
376
144
|
|
|
377
145
|
## Integration Checklist
|
|
378
146
|
1. **Instantiate once** per page/tab and keep it in a store (Vuex, Redux, Zustand, etc.).
|
|
379
|
-
2. **
|
|
380
|
-
3. **
|
|
381
|
-
4. **
|
|
382
|
-
5. **
|
|
383
|
-
6. **Monitor logs** – browser console shows `🎧 SDK`, `📍 SDK`, `🎚️ [Spatial Audio]`, and `🎤 ML` statements for every critical hop.
|
|
147
|
+
2. **Pipe LSD/Lap data** from your rendering engine into `updatePosition()` + `setListenerFromLSD()` at ~10 Hz.
|
|
148
|
+
3. **Render videos muted** – never attach remote audio tracks straight to DOM; let `SpatialAudioManager` own playback.
|
|
149
|
+
4. **Push avatar telemetry back to Unreal** so `remoteSpatialData` can render minimaps/circles (see Odyssey V2 `sendMediaSoupParticipantsToUnreal`).
|
|
150
|
+
5. **Monitor logs** – browser console shows `🎧 SDK`, `📍 SDK`, and `🎚️ [Spatial Audio]` statements for every critical hop.
|
|
384
151
|
|
|
385
152
|
## Server Contract (Socket.IO events)
|
|
386
153
|
| Event | Direction | Payload |
|
package/dist/MediasoupManager.js
CHANGED
|
@@ -41,7 +41,7 @@ class MediasoupManager {
|
|
|
41
41
|
this.recvTransport = null;
|
|
42
42
|
this.producers = new Map();
|
|
43
43
|
this.consumers = new Map();
|
|
44
|
-
this.participantId =
|
|
44
|
+
this.participantId = '';
|
|
45
45
|
this.socket = socket;
|
|
46
46
|
this.device = new mediasoupClient.Device();
|
|
47
47
|
}
|
|
@@ -100,7 +100,7 @@ class MediasoupManager {
|
|
|
100
100
|
// Emit event so parent SDK can recreate producers
|
|
101
101
|
this.socket.emit("transport-failed", {
|
|
102
102
|
participantId: this.participantId,
|
|
103
|
-
direction: "send"
|
|
103
|
+
direction: "send"
|
|
104
104
|
});
|
|
105
105
|
}
|
|
106
106
|
});
|
|
@@ -126,46 +126,30 @@ class MediasoupManager {
|
|
|
126
126
|
async produce(track, appData) {
|
|
127
127
|
if (!this.sendTransport)
|
|
128
128
|
throw new Error("Send transport not initialized");
|
|
129
|
-
console.log(`📤 [MediaSoup] Producing ${track.kind} track (transport state: ${this.sendTransport.connectionState})`);
|
|
130
129
|
// Configure simulcast for video tracks for adaptive bitrate
|
|
131
130
|
const produceOptions = { track, appData };
|
|
132
|
-
if (track.kind ===
|
|
131
|
+
if (track.kind === 'video') {
|
|
133
132
|
produceOptions.encodings = [
|
|
134
133
|
// Low quality layer - 100 kbps, good for poor connections
|
|
135
|
-
{
|
|
136
|
-
rid: "r0",
|
|
137
|
-
active: true,
|
|
138
|
-
maxBitrate: 100000,
|
|
139
|
-
scaleResolutionDownBy: 4,
|
|
140
|
-
},
|
|
134
|
+
{ rid: 'r0', active: true, maxBitrate: 100000, scaleResolutionDownBy: 4 },
|
|
141
135
|
// Medium quality layer - 300 kbps, balanced
|
|
142
|
-
{
|
|
143
|
-
rid: "r1",
|
|
144
|
-
active: true,
|
|
145
|
-
maxBitrate: 300000,
|
|
146
|
-
scaleResolutionDownBy: 2,
|
|
147
|
-
},
|
|
136
|
+
{ rid: 'r1', active: true, maxBitrate: 300000, scaleResolutionDownBy: 2 },
|
|
148
137
|
// High quality layer - 900 kbps, full resolution
|
|
149
|
-
{
|
|
150
|
-
rid: "r2",
|
|
151
|
-
active: true,
|
|
152
|
-
maxBitrate: 900000,
|
|
153
|
-
scaleResolutionDownBy: 1,
|
|
154
|
-
},
|
|
138
|
+
{ rid: 'r2', active: true, maxBitrate: 900000, scaleResolutionDownBy: 1 }
|
|
155
139
|
];
|
|
156
140
|
// VP8 codec for simulcast support
|
|
157
141
|
produceOptions.codecOptions = {
|
|
158
|
-
videoGoogleStartBitrate: 1000
|
|
142
|
+
videoGoogleStartBitrate: 1000
|
|
159
143
|
};
|
|
160
144
|
}
|
|
161
145
|
const producer = await this.sendTransport.produce(produceOptions);
|
|
162
146
|
// Handle producer events
|
|
163
|
-
producer.on(
|
|
147
|
+
producer.on('transportclose', () => {
|
|
164
148
|
console.warn(`⚠️ [MediaSoup] Producer ${producer.id.substring(0, 8)} transport closed`);
|
|
165
149
|
this.producers.delete(producer.id);
|
|
166
150
|
// The main SDK (index.ts) should handle recreation
|
|
167
151
|
});
|
|
168
|
-
producer.on(
|
|
152
|
+
producer.on('trackended', () => {
|
|
169
153
|
console.warn(`⚠️ [MediaSoup] Producer ${producer.id.substring(0, 8)} track ended`);
|
|
170
154
|
producer.close();
|
|
171
155
|
this.producers.delete(producer.id);
|
|
@@ -183,12 +167,12 @@ class MediasoupManager {
|
|
|
183
167
|
rtpParameters: data.rtpParameters,
|
|
184
168
|
});
|
|
185
169
|
// Handle consumer events
|
|
186
|
-
consumer.on(
|
|
170
|
+
consumer.on('transportclose', () => {
|
|
187
171
|
console.warn(`⚠️ [MediaSoup] Consumer ${consumer.id.substring(0, 8)} transport closed`);
|
|
188
172
|
this.consumers.delete(consumer.id);
|
|
189
173
|
// The main SDK (index.ts) should handle recreation
|
|
190
174
|
});
|
|
191
|
-
consumer.on(
|
|
175
|
+
consumer.on('trackended', () => {
|
|
192
176
|
console.warn(`⚠️ [MediaSoup] Consumer ${consumer.id.substring(0, 8)} track ended`);
|
|
193
177
|
consumer.close();
|
|
194
178
|
this.consumers.delete(consumer.id);
|
package/dist/index.d.ts
CHANGED
|
@@ -10,11 +10,17 @@ export declare class OdysseySpatialComms extends EventManager {
|
|
|
10
10
|
private localParticipant;
|
|
11
11
|
private mediasoupManager;
|
|
12
12
|
private spatialAudioManager;
|
|
13
|
-
private mlNoiseSuppressor;
|
|
14
|
-
private mlNoiseSuppressionEnabled;
|
|
15
13
|
constructor(serverUrl: string, spatialOptions?: SpatialAudioOptions);
|
|
16
14
|
on(event: OdysseyEvent, listener: (...args: any[]) => void): this;
|
|
17
15
|
emit(event: OdysseyEvent, ...args: any[]): boolean;
|
|
16
|
+
/**
|
|
17
|
+
* Initialize ML-based noise suppression
|
|
18
|
+
* Note: This SDK uses AudioWorklet-based denoising configured via SpatialAudioOptions.
|
|
19
|
+
* This method is provided for API compatibility but the actual noise reduction
|
|
20
|
+
* is handled by the SpatialAudioManager's denoiser configuration.
|
|
21
|
+
* @param modelPath Path to the ML model (currently not used)
|
|
22
|
+
*/
|
|
23
|
+
initializeMLNoiseSuppression(modelPath: string): Promise<void>;
|
|
18
24
|
joinRoom(data: {
|
|
19
25
|
roomId: string;
|
|
20
26
|
userId: string;
|
|
@@ -30,19 +36,6 @@ export declare class OdysseySpatialComms extends EventManager {
|
|
|
30
36
|
leaveRoom(): void;
|
|
31
37
|
resumeAudio(): Promise<void>;
|
|
32
38
|
getAudioContextState(): AudioContextState;
|
|
33
|
-
/**
|
|
34
|
-
* Initialize ML noise suppression
|
|
35
|
-
* @param modelUrl - URL to model.json (e.g., '/models/odyssey_noise_suppressor_v1/model.json')
|
|
36
|
-
*/
|
|
37
|
-
initializeMLNoiseSuppression(modelUrl: string): Promise<void>;
|
|
38
|
-
/**
|
|
39
|
-
* Toggle ML noise suppression on/off
|
|
40
|
-
*/
|
|
41
|
-
toggleMLNoiseSuppression(enabled: boolean): void;
|
|
42
|
-
/**
|
|
43
|
-
* Check if ML noise suppression is enabled
|
|
44
|
-
*/
|
|
45
|
-
isMLNoiseSuppressionEnabled(): boolean;
|
|
46
39
|
produceTrack(track: MediaStreamTrack, appData?: {
|
|
47
40
|
isScreenshare?: boolean;
|
|
48
41
|
}): Promise<any>;
|
package/dist/index.js
CHANGED
|
@@ -5,14 +5,11 @@ const socket_io_client_1 = require("socket.io-client");
|
|
|
5
5
|
const EventManager_1 = require("./EventManager");
|
|
6
6
|
const MediasoupManager_1 = require("./MediasoupManager");
|
|
7
7
|
const SpatialAudioManager_1 = require("./SpatialAudioManager");
|
|
8
|
-
const MLNoiseSuppressor_1 = require("./MLNoiseSuppressor");
|
|
9
8
|
class OdysseySpatialComms extends EventManager_1.EventManager {
|
|
10
9
|
constructor(serverUrl, spatialOptions) {
|
|
11
10
|
super(); // Initialize the EventEmitter base class
|
|
12
11
|
this.room = null;
|
|
13
12
|
this.localParticipant = null;
|
|
14
|
-
this.mlNoiseSuppressor = null;
|
|
15
|
-
this.mlNoiseSuppressionEnabled = false;
|
|
16
13
|
this.socket = (0, socket_io_client_1.io)(serverUrl, {
|
|
17
14
|
transports: ["websocket"],
|
|
18
15
|
});
|
|
@@ -33,6 +30,20 @@ class OdysseySpatialComms extends EventManager_1.EventManager {
|
|
|
33
30
|
// Explicitly call EventEmitter's emit method
|
|
34
31
|
return super.emit(event, ...args);
|
|
35
32
|
}
|
|
33
|
+
/**
|
|
34
|
+
* Initialize ML-based noise suppression
|
|
35
|
+
* Note: This SDK uses AudioWorklet-based denoising configured via SpatialAudioOptions.
|
|
36
|
+
* This method is provided for API compatibility but the actual noise reduction
|
|
37
|
+
* is handled by the SpatialAudioManager's denoiser configuration.
|
|
38
|
+
* @param modelPath Path to the ML model (currently not used)
|
|
39
|
+
*/
|
|
40
|
+
async initializeMLNoiseSuppression(modelPath) {
|
|
41
|
+
console.log(`[OdysseySpatialComms] ML Noise Suppression initialization called with model: ${modelPath}`);
|
|
42
|
+
console.log("[OdysseySpatialComms] Note: Noise reduction is handled by AudioWorklet denoiser in SpatialAudioManager");
|
|
43
|
+
// This is a stub method for API compatibility
|
|
44
|
+
// The actual noise suppression is handled by the SpatialAudioManager's denoiser
|
|
45
|
+
return Promise.resolve();
|
|
46
|
+
}
|
|
36
47
|
async joinRoom(data) {
|
|
37
48
|
return new Promise((resolve, reject) => {
|
|
38
49
|
// Create a one-time listener for room-joined event
|
|
@@ -104,71 +115,8 @@ class OdysseySpatialComms extends EventManager_1.EventManager {
|
|
|
104
115
|
getAudioContextState() {
|
|
105
116
|
return this.spatialAudioManager.getAudioContextState();
|
|
106
117
|
}
|
|
107
|
-
/**
|
|
108
|
-
* Initialize ML noise suppression
|
|
109
|
-
* @param modelUrl - URL to model.json (e.g., '/models/odyssey_noise_suppressor_v1/model.json')
|
|
110
|
-
*/
|
|
111
|
-
async initializeMLNoiseSuppression(modelUrl) {
|
|
112
|
-
if (this.mlNoiseSuppressor) {
|
|
113
|
-
console.log('ML Noise Suppression already initialized');
|
|
114
|
-
return;
|
|
115
|
-
}
|
|
116
|
-
try {
|
|
117
|
-
console.log('🎤 Initializing ML Noise Suppression...');
|
|
118
|
-
this.mlNoiseSuppressor = new MLNoiseSuppressor_1.MLNoiseSuppressor();
|
|
119
|
-
await this.mlNoiseSuppressor.initialize(modelUrl, this.spatialAudioManager.getAudioContext());
|
|
120
|
-
this.mlNoiseSuppressionEnabled = true;
|
|
121
|
-
console.log('✅ ML Noise Suppression enabled');
|
|
122
|
-
}
|
|
123
|
-
catch (error) {
|
|
124
|
-
console.error('❌ Failed to initialize ML Noise Suppression:', error);
|
|
125
|
-
this.mlNoiseSuppressor = null;
|
|
126
|
-
this.mlNoiseSuppressionEnabled = false;
|
|
127
|
-
throw error;
|
|
128
|
-
}
|
|
129
|
-
}
|
|
130
|
-
/**
|
|
131
|
-
* Toggle ML noise suppression on/off
|
|
132
|
-
*/
|
|
133
|
-
toggleMLNoiseSuppression(enabled) {
|
|
134
|
-
if (!this.mlNoiseSuppressor) {
|
|
135
|
-
console.warn('ML Noise Suppression not initialized. Call initializeMLNoiseSuppression() first.');
|
|
136
|
-
return;
|
|
137
|
-
}
|
|
138
|
-
this.mlNoiseSuppressionEnabled = enabled;
|
|
139
|
-
console.log(`🎤 ML Noise Suppression: ${enabled ? 'ON' : 'OFF'}`);
|
|
140
|
-
}
|
|
141
|
-
/**
|
|
142
|
-
* Check if ML noise suppression is enabled
|
|
143
|
-
*/
|
|
144
|
-
isMLNoiseSuppressionEnabled() {
|
|
145
|
-
return this.mlNoiseSuppressionEnabled && this.mlNoiseSuppressor !== null;
|
|
146
|
-
}
|
|
147
118
|
async produceTrack(track, appData) {
|
|
148
|
-
|
|
149
|
-
let processedTrack = track;
|
|
150
|
-
// Apply ML noise suppression to audio BEFORE sending to MediaSoup
|
|
151
|
-
if (track.kind === 'audio' && this.mlNoiseSuppressionEnabled && this.mlNoiseSuppressor) {
|
|
152
|
-
try {
|
|
153
|
-
console.log('🎤 [SDK] Applying ML noise suppression to audio...');
|
|
154
|
-
const inputStream = new MediaStream([track]);
|
|
155
|
-
console.log('🎤 [SDK] Created input stream with track');
|
|
156
|
-
const cleanedStream = await this.mlNoiseSuppressor.processMediaStream(inputStream);
|
|
157
|
-
console.log('🎤 [SDK] Got cleaned stream from ML');
|
|
158
|
-
processedTrack = cleanedStream.getAudioTracks()[0];
|
|
159
|
-
console.log(`✅ [SDK] ML noise suppression applied - processed track state: ${processedTrack.readyState}`);
|
|
160
|
-
}
|
|
161
|
-
catch (error) {
|
|
162
|
-
console.error('❌ [SDK] ML noise suppression failed, using original track:', error);
|
|
163
|
-
processedTrack = track; // Fallback to original track
|
|
164
|
-
}
|
|
165
|
-
}
|
|
166
|
-
else {
|
|
167
|
-
console.log(`ℹ️ [SDK] Skipping ML - kind: ${track.kind}, ML enabled: ${this.mlNoiseSuppressionEnabled}`);
|
|
168
|
-
}
|
|
169
|
-
console.log(`📤 [SDK] Producing track to MediaSoup - kind: ${processedTrack.kind}, state: ${processedTrack.readyState}`);
|
|
170
|
-
const producer = await this.mediasoupManager.produce(processedTrack, appData);
|
|
171
|
-
console.log(`✅ [SDK] Producer created - id: ${producer.id}, kind: ${producer.kind}`);
|
|
119
|
+
const producer = await this.mediasoupManager.produce(track, appData);
|
|
172
120
|
if (this.localParticipant) {
|
|
173
121
|
const isFirstProducer = this.localParticipant.producers.size === 0;
|
|
174
122
|
this.localParticipant.producers.set(producer.id, producer);
|
|
@@ -208,17 +156,22 @@ class OdysseySpatialComms extends EventManager_1.EventManager {
|
|
|
208
156
|
this.localParticipant.producers.clear();
|
|
209
157
|
const tracksToReproduce = [];
|
|
210
158
|
// Collect audio track
|
|
211
|
-
if (this.localParticipant.audioTrack &&
|
|
159
|
+
if (this.localParticipant.audioTrack &&
|
|
160
|
+
this.localParticipant.audioTrack.readyState === "live") {
|
|
212
161
|
tracksToReproduce.push({ track: this.localParticipant.audioTrack });
|
|
213
162
|
}
|
|
214
163
|
// Collect video track
|
|
215
|
-
if (this.localParticipant.videoTrack &&
|
|
164
|
+
if (this.localParticipant.videoTrack &&
|
|
165
|
+
this.localParticipant.videoTrack.readyState === "live") {
|
|
216
166
|
tracksToReproduce.push({ track: this.localParticipant.videoTrack });
|
|
217
167
|
}
|
|
218
168
|
// Collect screenshare track
|
|
219
169
|
const screenshareTrack = this.localParticipant.screenshareTrack;
|
|
220
|
-
if (screenshareTrack && screenshareTrack.readyState ===
|
|
221
|
-
tracksToReproduce.push({
|
|
170
|
+
if (screenshareTrack && screenshareTrack.readyState === "live") {
|
|
171
|
+
tracksToReproduce.push({
|
|
172
|
+
track: screenshareTrack,
|
|
173
|
+
appData: { isScreenshare: true },
|
|
174
|
+
});
|
|
222
175
|
}
|
|
223
176
|
// Recreate producers
|
|
224
177
|
for (const { track, appData } of tracksToReproduce) {
|
|
@@ -298,8 +251,7 @@ class OdysseySpatialComms extends EventManager_1.EventManager {
|
|
|
298
251
|
participantId: snapshot.participantId,
|
|
299
252
|
userId: snapshot.userId,
|
|
300
253
|
deviceId: snapshot.deviceId,
|
|
301
|
-
isLocal: this.localParticipant?.participantId ===
|
|
302
|
-
snapshot.participantId,
|
|
254
|
+
isLocal: this.localParticipant?.participantId === snapshot.participantId,
|
|
303
255
|
audioTrack: undefined,
|
|
304
256
|
videoTrack: undefined,
|
|
305
257
|
producers: new Map(),
|
|
@@ -323,7 +275,8 @@ class OdysseySpatialComms extends EventManager_1.EventManager {
|
|
|
323
275
|
participant.isLocal =
|
|
324
276
|
this.localParticipant?.participantId === snapshot.participantId;
|
|
325
277
|
// CRITICAL: Store channel data for huddle detection
|
|
326
|
-
participant.currentChannel =
|
|
278
|
+
participant.currentChannel =
|
|
279
|
+
snapshot.currentChannel || "spatial";
|
|
327
280
|
this.room?.participants.set(snapshot.participantId, participant);
|
|
328
281
|
if (participant.isLocal) {
|
|
329
282
|
this.localParticipant = participant;
|
|
@@ -418,7 +371,8 @@ class OdysseySpatialComms extends EventManager_1.EventManager {
|
|
|
418
371
|
participant.audioTrack = track;
|
|
419
372
|
// CRITICAL: Do NOT setup spatial audio for local participant (yourself)
|
|
420
373
|
// This prevents hearing your own microphone (loopback)
|
|
421
|
-
const isLocalParticipant = participant.participantId ===
|
|
374
|
+
const isLocalParticipant = participant.participantId ===
|
|
375
|
+
this.localParticipant?.participantId;
|
|
422
376
|
if (isLocalParticipant) {
|
|
423
377
|
// Do NOT connect this audio to Web Audio API
|
|
424
378
|
return; // Exit early to prevent any audio processing
|
|
@@ -546,7 +500,8 @@ class OdysseySpatialComms extends EventManager_1.EventManager {
|
|
|
546
500
|
}
|
|
547
501
|
}
|
|
548
502
|
// Update local participant if it's them
|
|
549
|
-
if (this.localParticipant?.participantId === data.participantId &&
|
|
503
|
+
if (this.localParticipant?.participantId === data.participantId &&
|
|
504
|
+
this.localParticipant !== null) {
|
|
550
505
|
this.localParticipant.currentChannel = data.channelId;
|
|
551
506
|
}
|
|
552
507
|
this.emit("participant-channel-changed", data);
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@newgameplusinc/odyssey-audio-video-sdk-dev",
|
|
3
|
-
"version": "1.0.
|
|
4
|
-
"description": "Odyssey Spatial Audio & Video SDK using MediaSoup for real-time communication
|
|
3
|
+
"version": "1.0.59",
|
|
4
|
+
"description": "Odyssey Spatial Audio & Video SDK using MediaSoup for real-time communication",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
7
7
|
"scripts": {
|
|
@@ -31,8 +31,7 @@
|
|
|
31
31
|
"socket.io-client": "^4.7.2",
|
|
32
32
|
"webrtc-adapter": "^8.2.3",
|
|
33
33
|
"mediasoup-client": "^3.6.90",
|
|
34
|
-
"events": "^3.3.0"
|
|
35
|
-
"@tensorflow/tfjs": "^4.22.0"
|
|
34
|
+
"events": "^3.3.0"
|
|
36
35
|
},
|
|
37
36
|
"devDependencies": {
|
|
38
37
|
"@types/node": "^20.0.0",
|