@newgameplusinc/odyssey-audio-video-sdk-dev 1.0.57 → 1.0.58
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -255
- package/dist/MediasoupManager.js +11 -27
- package/dist/index.d.ts +0 -15
- package/dist/index.js +1 -67
- package/package.json +3 -4
- package/dist/MLNoiseSuppressor.d.ts +0 -76
- package/dist/MLNoiseSuppressor.js +0 -439
- package/dist/UltimateMLNoiseSuppressor.d.ts +0 -74
- package/dist/UltimateMLNoiseSuppressor.js +0 -309
package/README.md
CHANGED
|
@@ -11,69 +11,12 @@ It mirrors the production SDK used by Odyssey V2 and ships ready-to-drop into an
|
|
|
11
11
|
## Feature Highlights
|
|
12
12
|
- 🔌 **One class to rule it all** – `OdysseySpatialComms` wires transports, producers, consumers, and room state.
|
|
13
13
|
- 🧭 **Accurate pose propagation** – `updatePosition()` streams listener pose to the SFU while `participant-position-updated` keeps the local store in sync.
|
|
14
|
-
- 🤖 **AI-Powered Noise Suppression** – Deep learning model (TensorFlow.js) runs client-side to remove background noise BEFORE audio reaches MediaSoup. Uses trained LSTM-based mask prediction for superior noise cancellation without affecting voice quality.
|
|
15
14
|
- 🎧 **Studio-grade spatial audio** – each remote participant gets a dedicated Web Audio graph: denoiser → high-pass → low-pass → HRTF `PannerNode` → adaptive gain → master compressor. Uses Web Audio API's HRTF panning model for accurate left/right/front/back positioning based on distance and direction, with custom AudioWorklet processors for noise cancellation and voice tuning.
|
|
16
15
|
- 🎥 **Camera-ready streams** – video tracks are exposed separately so UI layers can render muted `<video>` tags while audio stays inside Web Audio.
|
|
17
16
|
- 🔁 **EventEmitter contract** – subscribe to `room-joined`, `consumer-created`, `participant-position-updated`, etc., without touching Socket.IO directly.
|
|
18
17
|
|
|
19
18
|
## Quick Start
|
|
20
19
|
|
|
21
|
-
### With ML Noise Suppression (Recommended)
|
|
22
|
-
|
|
23
|
-
```ts
|
|
24
|
-
import {
|
|
25
|
-
OdysseySpatialComms,
|
|
26
|
-
Direction,
|
|
27
|
-
Position,
|
|
28
|
-
} from "@newgameplusinc/odyssey-audio-video-sdk-dev";
|
|
29
|
-
|
|
30
|
-
const sdk = new OdysseySpatialComms("https://mediasoup-server.example.com");
|
|
31
|
-
|
|
32
|
-
// 1) Initialize ML noise suppression (place model files in public/models/)
|
|
33
|
-
await sdk.initializeMLNoiseSuppression(
|
|
34
|
-
'/models/odyssey_noise_suppressor_v1/model.json'
|
|
35
|
-
);
|
|
36
|
-
|
|
37
|
-
// 2) Join a room
|
|
38
|
-
await sdk.joinRoom({
|
|
39
|
-
roomId: "demo-room",
|
|
40
|
-
userId: "user-123",
|
|
41
|
-
deviceId: "device-123",
|
|
42
|
-
position: { x: 0, y: 0, z: 0 },
|
|
43
|
-
direction: { x: 0, y: 1, z: 0 },
|
|
44
|
-
});
|
|
45
|
-
|
|
46
|
-
// 3) Produce local media (ML cleaning applied automatically to audio)
|
|
47
|
-
const stream = await navigator.mediaDevices.getUserMedia({
|
|
48
|
-
audio: {
|
|
49
|
-
echoCancellation: true,
|
|
50
|
-
noiseSuppression: false, // Disable browser NS, use ML instead!
|
|
51
|
-
autoGainControl: true,
|
|
52
|
-
sampleRate: 48000,
|
|
53
|
-
},
|
|
54
|
-
video: true
|
|
55
|
-
});
|
|
56
|
-
for (const track of stream.getTracks()) {
|
|
57
|
-
await sdk.produceTrack(track); // ML processes audio tracks automatically
|
|
58
|
-
}
|
|
59
|
-
|
|
60
|
-
// 4) Toggle ML noise suppression on/off
|
|
61
|
-
sdk.toggleMLNoiseSuppression(true); // or false
|
|
62
|
-
|
|
63
|
-
// 5) Handle remote tracks
|
|
64
|
-
sdk.on("consumer-created", async ({ participant, track }) => {
|
|
65
|
-
if (track.kind === "video") {
|
|
66
|
-
attachVideo(track, participant.participantId);
|
|
67
|
-
}
|
|
68
|
-
});
|
|
69
|
-
|
|
70
|
-
// 6) Keep spatial audio honest
|
|
71
|
-
sdk.updatePosition(currentPos, currentDir);
|
|
72
|
-
sdk.setListenerFromLSD(listenerPos, cameraPos, lookAtPos);
|
|
73
|
-
```
|
|
74
|
-
|
|
75
|
-
### Without ML Noise Suppression (Legacy)
|
|
76
|
-
|
|
77
20
|
```ts
|
|
78
21
|
import {
|
|
79
22
|
OdysseySpatialComms,
|
|
@@ -113,83 +56,23 @@ sdk.setListenerFromLSD(listenerPos, cameraPos, lookAtPos);
|
|
|
113
56
|
## Audio Flow (Server ↔ Browser)
|
|
114
57
|
|
|
115
58
|
```
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
│
|
|
122
|
-
│
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
│ SDK: mediasoupManager.produce() │
|
|
134
|
-
│ (Sends clean track to server) │
|
|
135
|
-
└────────┬─────────────────────────────────┘
|
|
136
|
-
│
|
|
137
|
-
│ WebRTC/RTP
|
|
138
|
-
▼
|
|
139
|
-
┌─────────────────────────────────────────────┐
|
|
140
|
-
│ SERVER-SIDE ROUTING │
|
|
141
|
-
└─────────────────────────────────────────────┘
|
|
142
|
-
|
|
143
|
-
┌──────────────┐ update-position ┌──────────────┐ route clean audio ┌──────────────────┐
|
|
144
|
-
│ Browser LSD │ ──────────────────▶ │ MediaSoup SFU│ ───────────────────▶ │ Other Clients │
|
|
145
|
-
│ (Unreal data)│ │ + Socket.IO │ │ (Receive RTP) │
|
|
146
|
-
└──────────────┘ └──────┬───────┘ └──────┬───────────┘
|
|
147
|
-
│ │
|
|
148
|
-
│ consumer-created event │
|
|
149
|
-
▼ ▼
|
|
150
|
-
┌─────────────────────────────────────────────┐
|
|
151
|
-
│ REMOTE AUDIO PLAYBACK │
|
|
152
|
-
└─────────────────────────────────────────────┘
|
|
153
|
-
|
|
154
|
-
┌──────────────────┐
|
|
155
|
-
│ SDK Event Bus │
|
|
156
|
-
│ (EventManager) │
|
|
157
|
-
└────────┬─────────┘
|
|
158
|
-
│ track + pose
|
|
159
|
-
▼
|
|
160
|
-
┌──────────────────┐
|
|
161
|
-
│ SpatialAudioMgr │
|
|
162
|
-
│ (Web Audio API) │
|
|
163
|
-
│ • Denoiser │◀─── Traditional noise reduction
|
|
164
|
-
│ • HP/LP Filters │ (runs on received audio)
|
|
165
|
-
│ • HRTF Panner │
|
|
166
|
-
│ • Distance Gain │
|
|
167
|
-
│ • Compressor │
|
|
168
|
-
└────────┬─────────┘
|
|
169
|
-
│
|
|
170
|
-
▼
|
|
171
|
-
┌──────────────────┐
|
|
172
|
-
│ Web Audio Graph │
|
|
173
|
-
└────────┬─────────┘
|
|
174
|
-
│
|
|
175
|
-
▼
|
|
176
|
-
Listener ears (Left/Right)
|
|
177
|
-
│
|
|
178
|
-
▼
|
|
179
|
-
System Output
|
|
180
|
-
```
|
|
181
|
-
|
|
182
|
-
### ML Noise Suppression Pipeline (Client-Side)
|
|
183
|
-
```
|
|
184
|
-
Mic → getUserMedia()
|
|
185
|
-
↓
|
|
186
|
-
Vue: sdk.produceTrack(audioTrack)
|
|
187
|
-
↓
|
|
188
|
-
SDK: mlNoiseSuppressor.processMediaStream() [TensorFlow.js runs here]
|
|
189
|
-
↓
|
|
190
|
-
SDK: mediasoupManager.produce(cleanTrack)
|
|
191
|
-
↓
|
|
192
|
-
MediaSoup Server → Other participants hear clean audio ✅
|
|
59
|
+
┌──────────────┐ update-position ┌──────────────┐ pose + tracks ┌──────────────────┐
|
|
60
|
+
│ Browser LSD │ ──────────────────▶ │ MediaSoup SFU│ ────────────────▶ │ SDK Event Bus │
|
|
61
|
+
│ (Unreal data)│ │ + Socket.IO │ │ (EventManager) │
|
|
62
|
+
└──────┬───────┘ └──────┬───────┘ └──────────┬────────┘
|
|
63
|
+
│ │ track + pose
|
|
64
|
+
│ │ ▼
|
|
65
|
+
│ ┌────────▼────────┐ ┌──────────────────┐
|
|
66
|
+
│ audio RTP │ consumer-created│ │ SpatialAudioMgr │
|
|
67
|
+
└──────────────────────────▶│ setup per-user │◀──────────────────────│ (Web Audio API) │
|
|
68
|
+
└────────┬────────┘ │ - Denoiser │
|
|
69
|
+
│ │ - HP / LP │
|
|
70
|
+
│ │ - HRTF Panner │
|
|
71
|
+
▼ │ - Gain + Comp │
|
|
72
|
+
Web Audio Graph └──────────┬───────┘
|
|
73
|
+
│ │
|
|
74
|
+
▼ ▼
|
|
75
|
+
Listener ears (Left/Right) System Output
|
|
193
76
|
```
|
|
194
77
|
|
|
195
78
|
### Web Audio Algorithms
|
|
@@ -236,121 +119,7 @@ These layers run entirely in Web Audio, so you can ship “AirPods-style” back
|
|
|
236
119
|
3. **Position + direction updates** – every `participant-position-updated` event calls `updateSpatialAudio(participantId, position, direction)`. The position feeds the panner’s XYZ, while the direction vector sets the source orientation so voices project forward relative to avatar facing.
|
|
237
120
|
4. **Distance-aware gain** – the manager stores the latest listener pose and computes the Euclidean distance to each remote participant on every update. A custom rolloff curve adjusts gain before the compressor, giving the “someone on my left / far away” perception without blowing out master levels.
|
|
238
121
|
5. **Left/right rendering** – because the panner uses `panningModel = "HRTF"`, browsers feed the processed signal into the user’s audio hardware with head-related transfer functions, producing natural interaural time/intensity differences.
|
|
239
|
-
## ML Noise Suppression (Deep Learning Pre-Processing)
|
|
240
|
-
|
|
241
|
-
**NEW:** The SDK now includes an optional **AI-powered noise suppression** layer that runs **BEFORE** audio reaches MediaSoup, using a trained TensorFlow.js model.
|
|
242
|
-
|
|
243
|
-
### Why ML Noise Suppression?
|
|
244
|
-
- **Superior noise removal** – Deep learning models learn complex noise patterns that traditional DSP can't handle (keyboard typing, paper rustling, traffic, etc.)
|
|
245
|
-
- **Voice preservation** – LSTM-based mask prediction preserves natural voice quality while removing background noise
|
|
246
|
-
- **Client-side processing** – Runs entirely in the browser using TensorFlow.js (WebGL/WebAssembly acceleration)
|
|
247
|
-
- **Privacy-first** – Audio never leaves the user's device; processing happens locally
|
|
248
|
-
- **Zero latency** – <10ms processing time per frame, suitable for real-time communication
|
|
249
|
-
|
|
250
|
-
### Architecture
|
|
251
|
-
```
|
|
252
|
-
Raw Mic Audio → ML Model (TF.js) → Clean Audio → MediaSoup → Traditional Denoiser → Spatial Audio
|
|
253
|
-
```
|
|
254
|
-
|
|
255
|
-
The ML model applies **mask-based spectral subtraction** trained on diverse noise datasets:
|
|
256
|
-
1. Extracts mel-spectrogram from raw audio
|
|
257
|
-
2. Predicts a noise mask (0-1 per frequency bin) using Bidirectional LSTM
|
|
258
|
-
3. Applies mask to remove noise while preserving speech
|
|
259
|
-
4. Reconstructs clean audio waveform
|
|
260
|
-
|
|
261
|
-
### Setup ML Noise Suppression
|
|
262
|
-
|
|
263
|
-
**1. Place Model Files:**
|
|
264
|
-
```
|
|
265
|
-
YourApp/public/models/odyssey_noise_suppressor_v1/
|
|
266
|
-
├── model.json # TF.js model architecture
|
|
267
|
-
├── group1-shard*.bin # Model weights (multiple files)
|
|
268
|
-
├── normalization_stats.json # Preprocessing parameters
|
|
269
|
-
└── model_config.json # Audio config (48kHz, n_mels, etc.)
|
|
270
|
-
```
|
|
271
|
-
|
|
272
|
-
**2. Initialize in Code:**
|
|
273
|
-
```ts
|
|
274
|
-
const sdk = new OdysseySpatialComms('wss://your-server.com');
|
|
275
|
-
|
|
276
|
-
// Initialize ML noise suppression
|
|
277
|
-
try {
|
|
278
|
-
await sdk.initializeMLNoiseSuppression(
|
|
279
|
-
'/models/odyssey_noise_suppressor_v1/model.json'
|
|
280
|
-
);
|
|
281
|
-
console.log('✅ ML Noise Suppression enabled');
|
|
282
|
-
} catch (error) {
|
|
283
|
-
console.error('ML initialization failed:', error);
|
|
284
|
-
// Graceful degradation - SDK continues without ML
|
|
285
|
-
}
|
|
286
|
-
|
|
287
|
-
// Produce audio tracks (ML cleaning applied automatically)
|
|
288
|
-
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
|
|
289
|
-
await sdk.produceTrack(stream.getAudioTracks()[0]);
|
|
290
|
-
|
|
291
|
-
// Toggle ML on/off at runtime
|
|
292
|
-
sdk.toggleMLNoiseSuppression(false); // Disable
|
|
293
|
-
sdk.toggleMLNoiseSuppression(true); // Re-enable
|
|
294
|
-
|
|
295
|
-
// Check ML status
|
|
296
|
-
if (sdk.isMLNoiseSuppressionEnabled()) {
|
|
297
|
-
console.log('ML is active');
|
|
298
|
-
}
|
|
299
|
-
```
|
|
300
|
-
|
|
301
|
-
**3. Recommended Audio Constraints:**
|
|
302
|
-
```ts
|
|
303
|
-
const stream = await navigator.mediaDevices.getUserMedia({
|
|
304
|
-
audio: {
|
|
305
|
-
echoCancellation: true, // Keep echo cancellation
|
|
306
|
-
noiseSuppression: false, // Disable browser NS (ML replaces it)
|
|
307
|
-
autoGainControl: true, // Keep AGC
|
|
308
|
-
sampleRate: 48000, // Match model training (48kHz)
|
|
309
|
-
},
|
|
310
|
-
});
|
|
311
|
-
```
|
|
312
|
-
|
|
313
|
-
### ML Model Details
|
|
314
|
-
- **Architecture:** Bidirectional LSTM (2 layers, 256 units) + Dense layers
|
|
315
|
-
- **Input:** 48kHz audio → Mel-spectrogram (128 bins, 8-frame sequences)
|
|
316
|
-
- **Output:** Time-frequency mask (0-1 values per bin)
|
|
317
|
-
- **Latency:** ~5-8ms per chunk (AudioWorklet processing)
|
|
318
|
-
- **Model Size:** ~2-3 MB (quantized to uint8)
|
|
319
|
-
- **Training:** LibriSpeech (clean speech) + AudioSet (noise) datasets
|
|
320
|
-
|
|
321
|
-
### When to Use ML vs Traditional Denoiser
|
|
322
|
-
|
|
323
|
-
| Feature | ML Noise Suppression | Traditional Denoiser (AudioWorklet) |
|
|
324
|
-
|---------|---------------------|-------------------------------------|
|
|
325
|
-
| **Noise Types** | Complex (keyboard, traffic, music) | Stationary (fan, HVAC, hiss) |
|
|
326
|
-
| **Voice Quality** | Excellent (learned patterns) | Good (spectral shaping) |
|
|
327
|
-
| **CPU Usage** | Medium (TF.js optimized) | Low (simple DSP) |
|
|
328
|
-
| **Latency** | ~5-8ms | ~1-2ms |
|
|
329
|
-
| **Use Case** | Noisy environments | Quiet rooms with constant noise |
|
|
330
|
-
|
|
331
|
-
**Best Practice:** Enable **both** for maximum quality:
|
|
332
|
-
- ML suppresses complex noise (pre-MediaSoup)
|
|
333
|
-
- Traditional denoiser handles residual stationary noise (post-receive)
|
|
334
|
-
|
|
335
|
-
### Troubleshooting
|
|
336
|
-
|
|
337
|
-
**Model fails to load:**
|
|
338
|
-
- Ensure model files are served as static assets (check browser Network tab)
|
|
339
|
-
- Verify CORS headers if serving from CDN
|
|
340
|
-
- Check browser console for TensorFlow.js errors
|
|
341
|
-
|
|
342
|
-
**High CPU usage:**
|
|
343
|
-
- TF.js automatically uses WebGL when available (much faster)
|
|
344
|
-
- Disable ML on low-end devices: `sdk.toggleMLNoiseSuppression(false)`
|
|
345
|
-
|
|
346
|
-
**Voice sounds muffled:**
|
|
347
|
-
- Model trained on 48kHz audio; ensure mic uses same sample rate
|
|
348
|
-
- Check if browser is downsampling to 16kHz (some mobile browsers do this)
|
|
349
122
|
|
|
350
|
-
**Doesn't remove all noise:**
|
|
351
|
-
- ML works best on noise types seen during training
|
|
352
|
-
- Combine with traditional denoiser for residual cleanup
|
|
353
|
-
- Extremely loud noise (>30 dB SNR) may leak through
|
|
354
123
|
## Video Flow (Capture ↔ Rendering)
|
|
355
124
|
|
|
356
125
|
```
|
|
@@ -368,19 +137,17 @@ const stream = await navigator.mediaDevices.getUserMedia({
|
|
|
368
137
|
```
|
|
369
138
|
|
|
370
139
|
## Core Classes
|
|
371
|
-
- `src/index.ts` – `OdysseySpatialComms` (socket lifecycle, producers/consumers, event surface
|
|
140
|
+
- `src/index.ts` – `OdysseySpatialComms` (socket lifecycle, producers/consumers, event surface).
|
|
372
141
|
- `src/MediasoupManager.ts` – transport helpers for produce/consume/resume.
|
|
373
142
|
- `src/SpatialAudioManager.ts` – Web Audio orchestration (listener transforms, per-participant chains, denoiser, distance math).
|
|
374
|
-
- `src/MLNoiseSuppressor.ts` – TensorFlow.js-based deep learning noise suppression (mel-spectrogram extraction, LSTM inference, mask application).
|
|
375
143
|
- `src/EventManager.ts` – lightweight EventEmitter used by the entire SDK.
|
|
376
144
|
|
|
377
145
|
## Integration Checklist
|
|
378
146
|
1. **Instantiate once** per page/tab and keep it in a store (Vuex, Redux, Zustand, etc.).
|
|
379
|
-
2. **
|
|
380
|
-
3. **
|
|
381
|
-
4. **
|
|
382
|
-
5. **
|
|
383
|
-
6. **Monitor logs** – browser console shows `🎧 SDK`, `📍 SDK`, `🎚️ [Spatial Audio]`, and `🎤 ML` statements for every critical hop.
|
|
147
|
+
2. **Pipe LSD/Lap data** from your rendering engine into `updatePosition()` + `setListenerFromLSD()` at ~10 Hz.
|
|
148
|
+
3. **Render videos muted** – never attach remote audio tracks straight to DOM; let `SpatialAudioManager` own playback.
|
|
149
|
+
4. **Push avatar telemetry back to Unreal** so `remoteSpatialData` can render minimaps/circles (see Odyssey V2 `sendMediaSoupParticipantsToUnreal`).
|
|
150
|
+
5. **Monitor logs** – browser console shows `🎧 SDK`, `📍 SDK`, and `🎚️ [Spatial Audio]` statements for every critical hop.
|
|
384
151
|
|
|
385
152
|
## Server Contract (Socket.IO events)
|
|
386
153
|
| Event | Direction | Payload |
|
package/dist/MediasoupManager.js
CHANGED
|
@@ -41,7 +41,7 @@ class MediasoupManager {
|
|
|
41
41
|
this.recvTransport = null;
|
|
42
42
|
this.producers = new Map();
|
|
43
43
|
this.consumers = new Map();
|
|
44
|
-
this.participantId =
|
|
44
|
+
this.participantId = '';
|
|
45
45
|
this.socket = socket;
|
|
46
46
|
this.device = new mediasoupClient.Device();
|
|
47
47
|
}
|
|
@@ -100,7 +100,7 @@ class MediasoupManager {
|
|
|
100
100
|
// Emit event so parent SDK can recreate producers
|
|
101
101
|
this.socket.emit("transport-failed", {
|
|
102
102
|
participantId: this.participantId,
|
|
103
|
-
direction: "send"
|
|
103
|
+
direction: "send"
|
|
104
104
|
});
|
|
105
105
|
}
|
|
106
106
|
});
|
|
@@ -126,46 +126,30 @@ class MediasoupManager {
|
|
|
126
126
|
async produce(track, appData) {
|
|
127
127
|
if (!this.sendTransport)
|
|
128
128
|
throw new Error("Send transport not initialized");
|
|
129
|
-
console.log(`📤 [MediaSoup] Producing ${track.kind} track (transport state: ${this.sendTransport.connectionState})`);
|
|
130
129
|
// Configure simulcast for video tracks for adaptive bitrate
|
|
131
130
|
const produceOptions = { track, appData };
|
|
132
|
-
if (track.kind ===
|
|
131
|
+
if (track.kind === 'video') {
|
|
133
132
|
produceOptions.encodings = [
|
|
134
133
|
// Low quality layer - 100 kbps, good for poor connections
|
|
135
|
-
{
|
|
136
|
-
rid: "r0",
|
|
137
|
-
active: true,
|
|
138
|
-
maxBitrate: 100000,
|
|
139
|
-
scaleResolutionDownBy: 4,
|
|
140
|
-
},
|
|
134
|
+
{ rid: 'r0', active: true, maxBitrate: 100000, scaleResolutionDownBy: 4 },
|
|
141
135
|
// Medium quality layer - 300 kbps, balanced
|
|
142
|
-
{
|
|
143
|
-
rid: "r1",
|
|
144
|
-
active: true,
|
|
145
|
-
maxBitrate: 300000,
|
|
146
|
-
scaleResolutionDownBy: 2,
|
|
147
|
-
},
|
|
136
|
+
{ rid: 'r1', active: true, maxBitrate: 300000, scaleResolutionDownBy: 2 },
|
|
148
137
|
// High quality layer - 900 kbps, full resolution
|
|
149
|
-
{
|
|
150
|
-
rid: "r2",
|
|
151
|
-
active: true,
|
|
152
|
-
maxBitrate: 900000,
|
|
153
|
-
scaleResolutionDownBy: 1,
|
|
154
|
-
},
|
|
138
|
+
{ rid: 'r2', active: true, maxBitrate: 900000, scaleResolutionDownBy: 1 }
|
|
155
139
|
];
|
|
156
140
|
// VP8 codec for simulcast support
|
|
157
141
|
produceOptions.codecOptions = {
|
|
158
|
-
videoGoogleStartBitrate: 1000
|
|
142
|
+
videoGoogleStartBitrate: 1000
|
|
159
143
|
};
|
|
160
144
|
}
|
|
161
145
|
const producer = await this.sendTransport.produce(produceOptions);
|
|
162
146
|
// Handle producer events
|
|
163
|
-
producer.on(
|
|
147
|
+
producer.on('transportclose', () => {
|
|
164
148
|
console.warn(`⚠️ [MediaSoup] Producer ${producer.id.substring(0, 8)} transport closed`);
|
|
165
149
|
this.producers.delete(producer.id);
|
|
166
150
|
// The main SDK (index.ts) should handle recreation
|
|
167
151
|
});
|
|
168
|
-
producer.on(
|
|
152
|
+
producer.on('trackended', () => {
|
|
169
153
|
console.warn(`⚠️ [MediaSoup] Producer ${producer.id.substring(0, 8)} track ended`);
|
|
170
154
|
producer.close();
|
|
171
155
|
this.producers.delete(producer.id);
|
|
@@ -183,12 +167,12 @@ class MediasoupManager {
|
|
|
183
167
|
rtpParameters: data.rtpParameters,
|
|
184
168
|
});
|
|
185
169
|
// Handle consumer events
|
|
186
|
-
consumer.on(
|
|
170
|
+
consumer.on('transportclose', () => {
|
|
187
171
|
console.warn(`⚠️ [MediaSoup] Consumer ${consumer.id.substring(0, 8)} transport closed`);
|
|
188
172
|
this.consumers.delete(consumer.id);
|
|
189
173
|
// The main SDK (index.ts) should handle recreation
|
|
190
174
|
});
|
|
191
|
-
consumer.on(
|
|
175
|
+
consumer.on('trackended', () => {
|
|
192
176
|
console.warn(`⚠️ [MediaSoup] Consumer ${consumer.id.substring(0, 8)} track ended`);
|
|
193
177
|
consumer.close();
|
|
194
178
|
this.consumers.delete(consumer.id);
|
package/dist/index.d.ts
CHANGED
|
@@ -10,8 +10,6 @@ export declare class OdysseySpatialComms extends EventManager {
|
|
|
10
10
|
private localParticipant;
|
|
11
11
|
private mediasoupManager;
|
|
12
12
|
private spatialAudioManager;
|
|
13
|
-
private mlNoiseSuppressor;
|
|
14
|
-
private mlNoiseSuppressionEnabled;
|
|
15
13
|
constructor(serverUrl: string, spatialOptions?: SpatialAudioOptions);
|
|
16
14
|
on(event: OdysseyEvent, listener: (...args: any[]) => void): this;
|
|
17
15
|
emit(event: OdysseyEvent, ...args: any[]): boolean;
|
|
@@ -30,19 +28,6 @@ export declare class OdysseySpatialComms extends EventManager {
|
|
|
30
28
|
leaveRoom(): void;
|
|
31
29
|
resumeAudio(): Promise<void>;
|
|
32
30
|
getAudioContextState(): AudioContextState;
|
|
33
|
-
/**
|
|
34
|
-
* Initialize ML noise suppression
|
|
35
|
-
* @param modelUrl - URL to model.json (e.g., '/models/odyssey_noise_suppressor_v1/model.json')
|
|
36
|
-
*/
|
|
37
|
-
initializeMLNoiseSuppression(modelUrl: string): Promise<void>;
|
|
38
|
-
/**
|
|
39
|
-
* Toggle ML noise suppression on/off
|
|
40
|
-
*/
|
|
41
|
-
toggleMLNoiseSuppression(enabled: boolean): void;
|
|
42
|
-
/**
|
|
43
|
-
* Check if ML noise suppression is enabled
|
|
44
|
-
*/
|
|
45
|
-
isMLNoiseSuppressionEnabled(): boolean;
|
|
46
31
|
produceTrack(track: MediaStreamTrack, appData?: {
|
|
47
32
|
isScreenshare?: boolean;
|
|
48
33
|
}): Promise<any>;
|
package/dist/index.js
CHANGED
|
@@ -5,14 +5,11 @@ const socket_io_client_1 = require("socket.io-client");
|
|
|
5
5
|
const EventManager_1 = require("./EventManager");
|
|
6
6
|
const MediasoupManager_1 = require("./MediasoupManager");
|
|
7
7
|
const SpatialAudioManager_1 = require("./SpatialAudioManager");
|
|
8
|
-
const MLNoiseSuppressor_1 = require("./MLNoiseSuppressor");
|
|
9
8
|
class OdysseySpatialComms extends EventManager_1.EventManager {
|
|
10
9
|
constructor(serverUrl, spatialOptions) {
|
|
11
10
|
super(); // Initialize the EventEmitter base class
|
|
12
11
|
this.room = null;
|
|
13
12
|
this.localParticipant = null;
|
|
14
|
-
this.mlNoiseSuppressor = null;
|
|
15
|
-
this.mlNoiseSuppressionEnabled = false;
|
|
16
13
|
this.socket = (0, socket_io_client_1.io)(serverUrl, {
|
|
17
14
|
transports: ["websocket"],
|
|
18
15
|
});
|
|
@@ -104,71 +101,8 @@ class OdysseySpatialComms extends EventManager_1.EventManager {
|
|
|
104
101
|
getAudioContextState() {
|
|
105
102
|
return this.spatialAudioManager.getAudioContextState();
|
|
106
103
|
}
|
|
107
|
-
/**
|
|
108
|
-
* Initialize ML noise suppression
|
|
109
|
-
* @param modelUrl - URL to model.json (e.g., '/models/odyssey_noise_suppressor_v1/model.json')
|
|
110
|
-
*/
|
|
111
|
-
async initializeMLNoiseSuppression(modelUrl) {
|
|
112
|
-
if (this.mlNoiseSuppressor) {
|
|
113
|
-
console.log('ML Noise Suppression already initialized');
|
|
114
|
-
return;
|
|
115
|
-
}
|
|
116
|
-
try {
|
|
117
|
-
console.log('🎤 Initializing ML Noise Suppression...');
|
|
118
|
-
this.mlNoiseSuppressor = new MLNoiseSuppressor_1.MLNoiseSuppressor();
|
|
119
|
-
await this.mlNoiseSuppressor.initialize(modelUrl, this.spatialAudioManager.getAudioContext());
|
|
120
|
-
this.mlNoiseSuppressionEnabled = true;
|
|
121
|
-
console.log('✅ ML Noise Suppression enabled');
|
|
122
|
-
}
|
|
123
|
-
catch (error) {
|
|
124
|
-
console.error('❌ Failed to initialize ML Noise Suppression:', error);
|
|
125
|
-
this.mlNoiseSuppressor = null;
|
|
126
|
-
this.mlNoiseSuppressionEnabled = false;
|
|
127
|
-
throw error;
|
|
128
|
-
}
|
|
129
|
-
}
|
|
130
|
-
/**
|
|
131
|
-
* Toggle ML noise suppression on/off
|
|
132
|
-
*/
|
|
133
|
-
toggleMLNoiseSuppression(enabled) {
|
|
134
|
-
if (!this.mlNoiseSuppressor) {
|
|
135
|
-
console.warn('ML Noise Suppression not initialized. Call initializeMLNoiseSuppression() first.');
|
|
136
|
-
return;
|
|
137
|
-
}
|
|
138
|
-
this.mlNoiseSuppressionEnabled = enabled;
|
|
139
|
-
console.log(`🎤 ML Noise Suppression: ${enabled ? 'ON' : 'OFF'}`);
|
|
140
|
-
}
|
|
141
|
-
/**
|
|
142
|
-
* Check if ML noise suppression is enabled
|
|
143
|
-
*/
|
|
144
|
-
isMLNoiseSuppressionEnabled() {
|
|
145
|
-
return this.mlNoiseSuppressionEnabled && this.mlNoiseSuppressor !== null;
|
|
146
|
-
}
|
|
147
104
|
async produceTrack(track, appData) {
|
|
148
|
-
|
|
149
|
-
let processedTrack = track;
|
|
150
|
-
// Apply ML noise suppression to audio BEFORE sending to MediaSoup
|
|
151
|
-
if (track.kind === 'audio' && this.mlNoiseSuppressionEnabled && this.mlNoiseSuppressor) {
|
|
152
|
-
try {
|
|
153
|
-
console.log('🎤 [SDK] Applying ML noise suppression to audio...');
|
|
154
|
-
const inputStream = new MediaStream([track]);
|
|
155
|
-
console.log('🎤 [SDK] Created input stream with track');
|
|
156
|
-
const cleanedStream = await this.mlNoiseSuppressor.processMediaStream(inputStream);
|
|
157
|
-
console.log('🎤 [SDK] Got cleaned stream from ML');
|
|
158
|
-
processedTrack = cleanedStream.getAudioTracks()[0];
|
|
159
|
-
console.log(`✅ [SDK] ML noise suppression applied - processed track state: ${processedTrack.readyState}`);
|
|
160
|
-
}
|
|
161
|
-
catch (error) {
|
|
162
|
-
console.error('❌ [SDK] ML noise suppression failed, using original track:', error);
|
|
163
|
-
processedTrack = track; // Fallback to original track
|
|
164
|
-
}
|
|
165
|
-
}
|
|
166
|
-
else {
|
|
167
|
-
console.log(`ℹ️ [SDK] Skipping ML - kind: ${track.kind}, ML enabled: ${this.mlNoiseSuppressionEnabled}`);
|
|
168
|
-
}
|
|
169
|
-
console.log(`📤 [SDK] Producing track to MediaSoup - kind: ${processedTrack.kind}, state: ${processedTrack.readyState}`);
|
|
170
|
-
const producer = await this.mediasoupManager.produce(processedTrack, appData);
|
|
171
|
-
console.log(`✅ [SDK] Producer created - id: ${producer.id}, kind: ${producer.kind}`);
|
|
105
|
+
const producer = await this.mediasoupManager.produce(track, appData);
|
|
172
106
|
if (this.localParticipant) {
|
|
173
107
|
const isFirstProducer = this.localParticipant.producers.size === 0;
|
|
174
108
|
this.localParticipant.producers.set(producer.id, producer);
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@newgameplusinc/odyssey-audio-video-sdk-dev",
|
|
3
|
-
"version": "1.0.
|
|
4
|
-
"description": "Odyssey Spatial Audio & Video SDK using MediaSoup for real-time communication
|
|
3
|
+
"version": "1.0.58",
|
|
4
|
+
"description": "Odyssey Spatial Audio & Video SDK using MediaSoup for real-time communication",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
7
7
|
"scripts": {
|
|
@@ -31,8 +31,7 @@
|
|
|
31
31
|
"socket.io-client": "^4.7.2",
|
|
32
32
|
"webrtc-adapter": "^8.2.3",
|
|
33
33
|
"mediasoup-client": "^3.6.90",
|
|
34
|
-
"events": "^3.3.0"
|
|
35
|
-
"@tensorflow/tfjs": "^4.22.0"
|
|
34
|
+
"events": "^3.3.0"
|
|
36
35
|
},
|
|
37
36
|
"devDependencies": {
|
|
38
37
|
"@types/node": "^20.0.0",
|
|
@@ -1,76 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* ML-Based Noise Suppressor for Odyssey MediaSoup SDK
|
|
3
|
-
* Uses trained TensorFlow.js BiLSTM model for real-time noise suppression
|
|
4
|
-
*
|
|
5
|
-
* Architecture: BiLSTM (256 units x 2) + Dense layers
|
|
6
|
-
* Input: Mel-spectrogram features (16 frames x 128 mels)
|
|
7
|
-
* Output: Noise suppression mask (0-1 per frequency bin)
|
|
8
|
-
*
|
|
9
|
-
* Trained on: LibriSpeech + UrbanSound8K + MS-SNSD datasets
|
|
10
|
-
* Performance: val_loss=0.038, SNR improvement ~12dB
|
|
11
|
-
*/
|
|
12
|
-
export declare class MLNoiseSuppressor {
|
|
13
|
-
private model;
|
|
14
|
-
private config;
|
|
15
|
-
private normStats;
|
|
16
|
-
private audioContext;
|
|
17
|
-
private isInitialized;
|
|
18
|
-
private processingNode;
|
|
19
|
-
private highPassFilter;
|
|
20
|
-
private frameBuffer;
|
|
21
|
-
private prevMask;
|
|
22
|
-
private readonly SMOOTHING_ALPHA;
|
|
23
|
-
private melFilterbank;
|
|
24
|
-
private fftSize;
|
|
25
|
-
private hannWindow;
|
|
26
|
-
/**
|
|
27
|
-
* Initialize the ML noise suppressor
|
|
28
|
-
* @param modelUrl URL to the model.json file
|
|
29
|
-
* @param audioContext Web Audio API AudioContext
|
|
30
|
-
*/
|
|
31
|
-
initialize(modelUrl: string, audioContext: AudioContext): Promise<void>;
|
|
32
|
-
/**
|
|
33
|
-
* Create Hann window for FFT
|
|
34
|
-
*/
|
|
35
|
-
private createHannWindow;
|
|
36
|
-
/**
|
|
37
|
-
* Create mel filterbank matrix
|
|
38
|
-
*/
|
|
39
|
-
private createMelFilterbank;
|
|
40
|
-
/**
|
|
41
|
-
* Compute FFT magnitude spectrum (optimized DFT for real-time)
|
|
42
|
-
*/
|
|
43
|
-
private computeFFT;
|
|
44
|
-
/**
|
|
45
|
-
* Compute mel-spectrogram features from audio frame
|
|
46
|
-
*/
|
|
47
|
-
private computeMelFeatures;
|
|
48
|
-
/**
|
|
49
|
-
* Process audio buffer with ML noise suppression
|
|
50
|
-
* @param inputBuffer Audio buffer to process (Float32Array)
|
|
51
|
-
* @returns Processed audio buffer
|
|
52
|
-
*/
|
|
53
|
-
processAudio(inputBuffer: Float32Array): Promise<Float32Array>;
|
|
54
|
-
/**
|
|
55
|
-
* Apply temporal smoothing to reduce artifacts (Apple-style)
|
|
56
|
-
*/
|
|
57
|
-
private applyTemporalSmoothing;
|
|
58
|
-
/**
|
|
59
|
-
* Apply mask with voice frequency preservation
|
|
60
|
-
*/
|
|
61
|
-
private applyMaskWithVoicePreservation;
|
|
62
|
-
/**
|
|
63
|
-
* Process MediaStream with ML noise suppression
|
|
64
|
-
* @param inputStream MediaStream to process
|
|
65
|
-
* @returns Cleaned MediaStream
|
|
66
|
-
*/
|
|
67
|
-
processMediaStream(inputStream: MediaStream): Promise<MediaStream>;
|
|
68
|
-
/**
|
|
69
|
-
* Cleanup resources
|
|
70
|
-
*/
|
|
71
|
-
dispose(): void;
|
|
72
|
-
/**
|
|
73
|
-
* Check if initialized
|
|
74
|
-
*/
|
|
75
|
-
isReady(): boolean;
|
|
76
|
-
}
|