@newgameplusinc/odyssey-audio-video-sdk-dev 1.0.57 → 1.0.58

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -11,69 +11,12 @@ It mirrors the production SDK used by Odyssey V2 and ships ready-to-drop into an
11
11
  ## Feature Highlights
12
12
  - 🔌 **One class to rule it all** – `OdysseySpatialComms` wires transports, producers, consumers, and room state.
13
13
  - 🧭 **Accurate pose propagation** – `updatePosition()` streams listener pose to the SFU while `participant-position-updated` keeps the local store in sync.
14
- - 🤖 **AI-Powered Noise Suppression** – Deep learning model (TensorFlow.js) runs client-side to remove background noise BEFORE audio reaches MediaSoup. Uses trained LSTM-based mask prediction for superior noise cancellation without affecting voice quality.
15
14
  - 🎧 **Studio-grade spatial audio** – each remote participant gets a dedicated Web Audio graph: denoiser → high-pass → low-pass → HRTF `PannerNode` → adaptive gain → master compressor. Uses Web Audio API's HRTF panning model for accurate left/right/front/back positioning based on distance and direction, with custom AudioWorklet processors for noise cancellation and voice tuning.
16
15
  - 🎥 **Camera-ready streams** – video tracks are exposed separately so UI layers can render muted `<video>` tags while audio stays inside Web Audio.
17
16
  - 🔁 **EventEmitter contract** – subscribe to `room-joined`, `consumer-created`, `participant-position-updated`, etc., without touching Socket.IO directly.
18
17
 
19
18
  ## Quick Start
20
19
 
21
- ### With ML Noise Suppression (Recommended)
22
-
23
- ```ts
24
- import {
25
- OdysseySpatialComms,
26
- Direction,
27
- Position,
28
- } from "@newgameplusinc/odyssey-audio-video-sdk-dev";
29
-
30
- const sdk = new OdysseySpatialComms("https://mediasoup-server.example.com");
31
-
32
- // 1) Initialize ML noise suppression (place model files in public/models/)
33
- await sdk.initializeMLNoiseSuppression(
34
- '/models/odyssey_noise_suppressor_v1/model.json'
35
- );
36
-
37
- // 2) Join a room
38
- await sdk.joinRoom({
39
- roomId: "demo-room",
40
- userId: "user-123",
41
- deviceId: "device-123",
42
- position: { x: 0, y: 0, z: 0 },
43
- direction: { x: 0, y: 1, z: 0 },
44
- });
45
-
46
- // 3) Produce local media (ML cleaning applied automatically to audio)
47
- const stream = await navigator.mediaDevices.getUserMedia({
48
- audio: {
49
- echoCancellation: true,
50
- noiseSuppression: false, // Disable browser NS, use ML instead!
51
- autoGainControl: true,
52
- sampleRate: 48000,
53
- },
54
- video: true
55
- });
56
- for (const track of stream.getTracks()) {
57
- await sdk.produceTrack(track); // ML processes audio tracks automatically
58
- }
59
-
60
- // 4) Toggle ML noise suppression on/off
61
- sdk.toggleMLNoiseSuppression(true); // or false
62
-
63
- // 5) Handle remote tracks
64
- sdk.on("consumer-created", async ({ participant, track }) => {
65
- if (track.kind === "video") {
66
- attachVideo(track, participant.participantId);
67
- }
68
- });
69
-
70
- // 6) Keep spatial audio honest
71
- sdk.updatePosition(currentPos, currentDir);
72
- sdk.setListenerFromLSD(listenerPos, cameraPos, lookAtPos);
73
- ```
74
-
75
- ### Without ML Noise Suppression (Legacy)
76
-
77
20
  ```ts
78
21
  import {
79
22
  OdysseySpatialComms,
@@ -113,83 +56,23 @@ sdk.setListenerFromLSD(listenerPos, cameraPos, lookAtPos);
113
56
  ## Audio Flow (Server ↔ Browser)
114
57
 
115
58
  ```
116
- ┌─────────────────────────────────────────────┐
117
- CLIENT-SIDE PROCESSING
118
- └─────────────────────────────────────────────┘
119
-
120
- ┌──────────────┐ getUserMedia ┌──────────────────────┐ ML Processing ┌──────────────────┐
121
- Microphone ────────────────▶ │ Vue: produceTrack() │ ───────────────▶ │ SDK: ML Noise │
122
- (Raw Audio) │ │ (SDK method call) │ │ Suppressor │
123
- └──────────────┘ └──────────────────────┘ (TF.js Model)
124
- Load model.json
125
- • Mel-spectrogram
126
- LSTM inference
127
- Mask apply
128
- └────────┬─────────┘
129
-
130
- Clean Audio
131
-
132
- ┌──────────────────────────────────────────┐
133
- │ SDK: mediasoupManager.produce() │
134
- │ (Sends clean track to server) │
135
- └────────┬─────────────────────────────────┘
136
-
137
- │ WebRTC/RTP
138
-
139
- ┌─────────────────────────────────────────────┐
140
- │ SERVER-SIDE ROUTING │
141
- └─────────────────────────────────────────────┘
142
-
143
- ┌──────────────┐ update-position ┌──────────────┐ route clean audio ┌──────────────────┐
144
- │ Browser LSD │ ──────────────────▶ │ MediaSoup SFU│ ───────────────────▶ │ Other Clients │
145
- │ (Unreal data)│ │ + Socket.IO │ │ (Receive RTP) │
146
- └──────────────┘ └──────┬───────┘ └──────┬───────────┘
147
- │ │
148
- │ consumer-created event │
149
- ▼ ▼
150
- ┌─────────────────────────────────────────────┐
151
- │ REMOTE AUDIO PLAYBACK │
152
- └─────────────────────────────────────────────┘
153
-
154
- ┌──────────────────┐
155
- │ SDK Event Bus │
156
- │ (EventManager) │
157
- └────────┬─────────┘
158
- │ track + pose
159
-
160
- ┌──────────────────┐
161
- │ SpatialAudioMgr │
162
- │ (Web Audio API) │
163
- │ • Denoiser │◀─── Traditional noise reduction
164
- │ • HP/LP Filters │ (runs on received audio)
165
- │ • HRTF Panner │
166
- │ • Distance Gain │
167
- │ • Compressor │
168
- └────────┬─────────┘
169
-
170
-
171
- ┌──────────────────┐
172
- │ Web Audio Graph │
173
- └────────┬─────────┘
174
-
175
-
176
- Listener ears (Left/Right)
177
-
178
-
179
- System Output
180
- ```
181
-
182
- ### ML Noise Suppression Pipeline (Client-Side)
183
- ```
184
- Mic → getUserMedia()
185
-
186
- Vue: sdk.produceTrack(audioTrack)
187
-
188
- SDK: mlNoiseSuppressor.processMediaStream() [TensorFlow.js runs here]
189
-
190
- SDK: mediasoupManager.produce(cleanTrack)
191
-
192
- MediaSoup Server → Other participants hear clean audio ✅
59
+ ┌──────────────┐ update-position ┌──────────────┐ pose + tracks ┌──────────────────┐
60
+ Browser LSD ──────────────────▶ │ MediaSoup SFU│ ────────────────▶ │ SDK Event Bus │
61
+ │ (Unreal data)│ │ + Socket.IO │ │ (EventManager) │
62
+ └──────┬───────┘ └──────┬───────┘ └──────────┬────────┘
63
+ │ │ track + pose
64
+
65
+ ┌────────▼────────┐ ┌──────────────────┐
66
+ audio RTP consumer-created│ │ SpatialAudioMgr │
67
+ └──────────────────────────▶│ setup per-user │◀──────────────────────│ (Web Audio API)
68
+ └────────┬────────┘ - Denoiser
69
+ │ - HP / LP
70
+ │ - HRTF Panner
71
+ ▼ │ - Gain + Comp │
72
+ Web Audio Graph └──────────┬───────┘
73
+ │ │
74
+
75
+ Listener ears (Left/Right) System Output
193
76
  ```
194
77
 
195
78
  ### Web Audio Algorithms
@@ -236,121 +119,7 @@ These layers run entirely in Web Audio, so you can ship “AirPods-style” back
236
119
  3. **Position + direction updates** – every `participant-position-updated` event calls `updateSpatialAudio(participantId, position, direction)`. The position feeds the panner’s XYZ, while the direction vector sets the source orientation so voices project forward relative to avatar facing.
237
120
  4. **Distance-aware gain** – the manager stores the latest listener pose and computes the Euclidean distance to each remote participant on every update. A custom rolloff curve adjusts gain before the compressor, giving the “someone on my left / far away” perception without blowing out master levels.
238
121
  5. **Left/right rendering** – because the panner uses `panningModel = "HRTF"`, browsers feed the processed signal into the user’s audio hardware with head-related transfer functions, producing natural interaural time/intensity differences.
239
- ## ML Noise Suppression (Deep Learning Pre-Processing)
240
-
241
- **NEW:** The SDK now includes an optional **AI-powered noise suppression** layer that runs **BEFORE** audio reaches MediaSoup, using a trained TensorFlow.js model.
242
-
243
- ### Why ML Noise Suppression?
244
- - **Superior noise removal** – Deep learning models learn complex noise patterns that traditional DSP can't handle (keyboard typing, paper rustling, traffic, etc.)
245
- - **Voice preservation** – LSTM-based mask prediction preserves natural voice quality while removing background noise
246
- - **Client-side processing** – Runs entirely in the browser using TensorFlow.js (WebGL/WebAssembly acceleration)
247
- - **Privacy-first** – Audio never leaves the user's device; processing happens locally
248
- - **Zero latency** – <10ms processing time per frame, suitable for real-time communication
249
-
250
- ### Architecture
251
- ```
252
- Raw Mic Audio → ML Model (TF.js) → Clean Audio → MediaSoup → Traditional Denoiser → Spatial Audio
253
- ```
254
-
255
- The ML model applies **mask-based spectral subtraction** trained on diverse noise datasets:
256
- 1. Extracts mel-spectrogram from raw audio
257
- 2. Predicts a noise mask (0-1 per frequency bin) using Bidirectional LSTM
258
- 3. Applies mask to remove noise while preserving speech
259
- 4. Reconstructs clean audio waveform
260
-
261
- ### Setup ML Noise Suppression
262
-
263
- **1. Place Model Files:**
264
- ```
265
- YourApp/public/models/odyssey_noise_suppressor_v1/
266
- ├── model.json # TF.js model architecture
267
- ├── group1-shard*.bin # Model weights (multiple files)
268
- ├── normalization_stats.json # Preprocessing parameters
269
- └── model_config.json # Audio config (48kHz, n_mels, etc.)
270
- ```
271
-
272
- **2. Initialize in Code:**
273
- ```ts
274
- const sdk = new OdysseySpatialComms('wss://your-server.com');
275
-
276
- // Initialize ML noise suppression
277
- try {
278
- await sdk.initializeMLNoiseSuppression(
279
- '/models/odyssey_noise_suppressor_v1/model.json'
280
- );
281
- console.log('✅ ML Noise Suppression enabled');
282
- } catch (error) {
283
- console.error('ML initialization failed:', error);
284
- // Graceful degradation - SDK continues without ML
285
- }
286
-
287
- // Produce audio tracks (ML cleaning applied automatically)
288
- const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
289
- await sdk.produceTrack(stream.getAudioTracks()[0]);
290
-
291
- // Toggle ML on/off at runtime
292
- sdk.toggleMLNoiseSuppression(false); // Disable
293
- sdk.toggleMLNoiseSuppression(true); // Re-enable
294
-
295
- // Check ML status
296
- if (sdk.isMLNoiseSuppressionEnabled()) {
297
- console.log('ML is active');
298
- }
299
- ```
300
-
301
- **3. Recommended Audio Constraints:**
302
- ```ts
303
- const stream = await navigator.mediaDevices.getUserMedia({
304
- audio: {
305
- echoCancellation: true, // Keep echo cancellation
306
- noiseSuppression: false, // Disable browser NS (ML replaces it)
307
- autoGainControl: true, // Keep AGC
308
- sampleRate: 48000, // Match model training (48kHz)
309
- },
310
- });
311
- ```
312
-
313
- ### ML Model Details
314
- - **Architecture:** Bidirectional LSTM (2 layers, 256 units) + Dense layers
315
- - **Input:** 48kHz audio → Mel-spectrogram (128 bins, 8-frame sequences)
316
- - **Output:** Time-frequency mask (0-1 values per bin)
317
- - **Latency:** ~5-8ms per chunk (AudioWorklet processing)
318
- - **Model Size:** ~2-3 MB (quantized to uint8)
319
- - **Training:** LibriSpeech (clean speech) + AudioSet (noise) datasets
320
-
321
- ### When to Use ML vs Traditional Denoiser
322
-
323
- | Feature | ML Noise Suppression | Traditional Denoiser (AudioWorklet) |
324
- |---------|---------------------|-------------------------------------|
325
- | **Noise Types** | Complex (keyboard, traffic, music) | Stationary (fan, HVAC, hiss) |
326
- | **Voice Quality** | Excellent (learned patterns) | Good (spectral shaping) |
327
- | **CPU Usage** | Medium (TF.js optimized) | Low (simple DSP) |
328
- | **Latency** | ~5-8ms | ~1-2ms |
329
- | **Use Case** | Noisy environments | Quiet rooms with constant noise |
330
-
331
- **Best Practice:** Enable **both** for maximum quality:
332
- - ML suppresses complex noise (pre-MediaSoup)
333
- - Traditional denoiser handles residual stationary noise (post-receive)
334
-
335
- ### Troubleshooting
336
-
337
- **Model fails to load:**
338
- - Ensure model files are served as static assets (check browser Network tab)
339
- - Verify CORS headers if serving from CDN
340
- - Check browser console for TensorFlow.js errors
341
-
342
- **High CPU usage:**
343
- - TF.js automatically uses WebGL when available (much faster)
344
- - Disable ML on low-end devices: `sdk.toggleMLNoiseSuppression(false)`
345
-
346
- **Voice sounds muffled:**
347
- - Model trained on 48kHz audio; ensure mic uses same sample rate
348
- - Check if browser is downsampling to 16kHz (some mobile browsers do this)
349
122
 
350
- **Doesn't remove all noise:**
351
- - ML works best on noise types seen during training
352
- - Combine with traditional denoiser for residual cleanup
353
- - Extremely loud noise (>30 dB SNR) may leak through
354
123
  ## Video Flow (Capture ↔ Rendering)
355
124
 
356
125
  ```
@@ -368,19 +137,17 @@ const stream = await navigator.mediaDevices.getUserMedia({
368
137
  ```
369
138
 
370
139
  ## Core Classes
371
- - `src/index.ts` – `OdysseySpatialComms` (socket lifecycle, producers/consumers, event surface, ML noise suppression integration).
140
+ - `src/index.ts` – `OdysseySpatialComms` (socket lifecycle, producers/consumers, event surface).
372
141
  - `src/MediasoupManager.ts` – transport helpers for produce/consume/resume.
373
142
  - `src/SpatialAudioManager.ts` – Web Audio orchestration (listener transforms, per-participant chains, denoiser, distance math).
374
- - `src/MLNoiseSuppressor.ts` – TensorFlow.js-based deep learning noise suppression (mel-spectrogram extraction, LSTM inference, mask application).
375
143
  - `src/EventManager.ts` – lightweight EventEmitter used by the entire SDK.
376
144
 
377
145
  ## Integration Checklist
378
146
  1. **Instantiate once** per page/tab and keep it in a store (Vuex, Redux, Zustand, etc.).
379
- 2. **(Optional) Initialize ML noise suppression** Call `await sdk.initializeMLNoiseSuppression('/models/odyssey_noise_suppressor_v1/model.json')` after instantiation for AI-powered noise cancellation.
380
- 3. **Pipe LSD/Lap data** from your rendering engine into `updatePosition()` + `setListenerFromLSD()` at ~10 Hz.
381
- 4. **Render videos muted** never attach remote audio tracks straight to DOM; let `SpatialAudioManager` own playback.
382
- 5. **Push avatar telemetry back to Unreal** so `remoteSpatialData` can render minimaps/circles (see Odyssey V2 `sendMediaSoupParticipantsToUnreal`).
383
- 6. **Monitor logs** – browser console shows `🎧 SDK`, `📍 SDK`, `🎚️ [Spatial Audio]`, and `🎤 ML` statements for every critical hop.
147
+ 2. **Pipe LSD/Lap data** from your rendering engine into `updatePosition()` + `setListenerFromLSD()` at ~10 Hz.
148
+ 3. **Render videos muted** never attach remote audio tracks straight to DOM; let `SpatialAudioManager` own playback.
149
+ 4. **Push avatar telemetry back to Unreal** so `remoteSpatialData` can render minimaps/circles (see Odyssey V2 `sendMediaSoupParticipantsToUnreal`).
150
+ 5. **Monitor logs** browser console shows `🎧 SDK`, `📍 SDK`, and `🎚️ [Spatial Audio]` statements for every critical hop.
384
151
 
385
152
  ## Server Contract (Socket.IO events)
386
153
  | Event | Direction | Payload |
@@ -41,7 +41,7 @@ class MediasoupManager {
41
41
  this.recvTransport = null;
42
42
  this.producers = new Map();
43
43
  this.consumers = new Map();
44
- this.participantId = "";
44
+ this.participantId = '';
45
45
  this.socket = socket;
46
46
  this.device = new mediasoupClient.Device();
47
47
  }
@@ -100,7 +100,7 @@ class MediasoupManager {
100
100
  // Emit event so parent SDK can recreate producers
101
101
  this.socket.emit("transport-failed", {
102
102
  participantId: this.participantId,
103
- direction: "send",
103
+ direction: "send"
104
104
  });
105
105
  }
106
106
  });
@@ -126,46 +126,30 @@ class MediasoupManager {
126
126
  async produce(track, appData) {
127
127
  if (!this.sendTransport)
128
128
  throw new Error("Send transport not initialized");
129
- console.log(`📤 [MediaSoup] Producing ${track.kind} track (transport state: ${this.sendTransport.connectionState})`);
130
129
  // Configure simulcast for video tracks for adaptive bitrate
131
130
  const produceOptions = { track, appData };
132
- if (track.kind === "video") {
131
+ if (track.kind === 'video') {
133
132
  produceOptions.encodings = [
134
133
  // Low quality layer - 100 kbps, good for poor connections
135
- {
136
- rid: "r0",
137
- active: true,
138
- maxBitrate: 100000,
139
- scaleResolutionDownBy: 4,
140
- },
134
+ { rid: 'r0', active: true, maxBitrate: 100000, scaleResolutionDownBy: 4 },
141
135
  // Medium quality layer - 300 kbps, balanced
142
- {
143
- rid: "r1",
144
- active: true,
145
- maxBitrate: 300000,
146
- scaleResolutionDownBy: 2,
147
- },
136
+ { rid: 'r1', active: true, maxBitrate: 300000, scaleResolutionDownBy: 2 },
148
137
  // High quality layer - 900 kbps, full resolution
149
- {
150
- rid: "r2",
151
- active: true,
152
- maxBitrate: 900000,
153
- scaleResolutionDownBy: 1,
154
- },
138
+ { rid: 'r2', active: true, maxBitrate: 900000, scaleResolutionDownBy: 1 }
155
139
  ];
156
140
  // VP8 codec for simulcast support
157
141
  produceOptions.codecOptions = {
158
- videoGoogleStartBitrate: 1000,
142
+ videoGoogleStartBitrate: 1000
159
143
  };
160
144
  }
161
145
  const producer = await this.sendTransport.produce(produceOptions);
162
146
  // Handle producer events
163
- producer.on("transportclose", () => {
147
+ producer.on('transportclose', () => {
164
148
  console.warn(`⚠️ [MediaSoup] Producer ${producer.id.substring(0, 8)} transport closed`);
165
149
  this.producers.delete(producer.id);
166
150
  // The main SDK (index.ts) should handle recreation
167
151
  });
168
- producer.on("trackended", () => {
152
+ producer.on('trackended', () => {
169
153
  console.warn(`⚠️ [MediaSoup] Producer ${producer.id.substring(0, 8)} track ended`);
170
154
  producer.close();
171
155
  this.producers.delete(producer.id);
@@ -183,12 +167,12 @@ class MediasoupManager {
183
167
  rtpParameters: data.rtpParameters,
184
168
  });
185
169
  // Handle consumer events
186
- consumer.on("transportclose", () => {
170
+ consumer.on('transportclose', () => {
187
171
  console.warn(`⚠️ [MediaSoup] Consumer ${consumer.id.substring(0, 8)} transport closed`);
188
172
  this.consumers.delete(consumer.id);
189
173
  // The main SDK (index.ts) should handle recreation
190
174
  });
191
- consumer.on("trackended", () => {
175
+ consumer.on('trackended', () => {
192
176
  console.warn(`⚠️ [MediaSoup] Consumer ${consumer.id.substring(0, 8)} track ended`);
193
177
  consumer.close();
194
178
  this.consumers.delete(consumer.id);
package/dist/index.d.ts CHANGED
@@ -10,8 +10,6 @@ export declare class OdysseySpatialComms extends EventManager {
10
10
  private localParticipant;
11
11
  private mediasoupManager;
12
12
  private spatialAudioManager;
13
- private mlNoiseSuppressor;
14
- private mlNoiseSuppressionEnabled;
15
13
  constructor(serverUrl: string, spatialOptions?: SpatialAudioOptions);
16
14
  on(event: OdysseyEvent, listener: (...args: any[]) => void): this;
17
15
  emit(event: OdysseyEvent, ...args: any[]): boolean;
@@ -30,19 +28,6 @@ export declare class OdysseySpatialComms extends EventManager {
30
28
  leaveRoom(): void;
31
29
  resumeAudio(): Promise<void>;
32
30
  getAudioContextState(): AudioContextState;
33
- /**
34
- * Initialize ML noise suppression
35
- * @param modelUrl - URL to model.json (e.g., '/models/odyssey_noise_suppressor_v1/model.json')
36
- */
37
- initializeMLNoiseSuppression(modelUrl: string): Promise<void>;
38
- /**
39
- * Toggle ML noise suppression on/off
40
- */
41
- toggleMLNoiseSuppression(enabled: boolean): void;
42
- /**
43
- * Check if ML noise suppression is enabled
44
- */
45
- isMLNoiseSuppressionEnabled(): boolean;
46
31
  produceTrack(track: MediaStreamTrack, appData?: {
47
32
  isScreenshare?: boolean;
48
33
  }): Promise<any>;
package/dist/index.js CHANGED
@@ -5,14 +5,11 @@ const socket_io_client_1 = require("socket.io-client");
5
5
  const EventManager_1 = require("./EventManager");
6
6
  const MediasoupManager_1 = require("./MediasoupManager");
7
7
  const SpatialAudioManager_1 = require("./SpatialAudioManager");
8
- const MLNoiseSuppressor_1 = require("./MLNoiseSuppressor");
9
8
  class OdysseySpatialComms extends EventManager_1.EventManager {
10
9
  constructor(serverUrl, spatialOptions) {
11
10
  super(); // Initialize the EventEmitter base class
12
11
  this.room = null;
13
12
  this.localParticipant = null;
14
- this.mlNoiseSuppressor = null;
15
- this.mlNoiseSuppressionEnabled = false;
16
13
  this.socket = (0, socket_io_client_1.io)(serverUrl, {
17
14
  transports: ["websocket"],
18
15
  });
@@ -104,71 +101,8 @@ class OdysseySpatialComms extends EventManager_1.EventManager {
104
101
  getAudioContextState() {
105
102
  return this.spatialAudioManager.getAudioContextState();
106
103
  }
107
- /**
108
- * Initialize ML noise suppression
109
- * @param modelUrl - URL to model.json (e.g., '/models/odyssey_noise_suppressor_v1/model.json')
110
- */
111
- async initializeMLNoiseSuppression(modelUrl) {
112
- if (this.mlNoiseSuppressor) {
113
- console.log('ML Noise Suppression already initialized');
114
- return;
115
- }
116
- try {
117
- console.log('🎤 Initializing ML Noise Suppression...');
118
- this.mlNoiseSuppressor = new MLNoiseSuppressor_1.MLNoiseSuppressor();
119
- await this.mlNoiseSuppressor.initialize(modelUrl, this.spatialAudioManager.getAudioContext());
120
- this.mlNoiseSuppressionEnabled = true;
121
- console.log('✅ ML Noise Suppression enabled');
122
- }
123
- catch (error) {
124
- console.error('❌ Failed to initialize ML Noise Suppression:', error);
125
- this.mlNoiseSuppressor = null;
126
- this.mlNoiseSuppressionEnabled = false;
127
- throw error;
128
- }
129
- }
130
- /**
131
- * Toggle ML noise suppression on/off
132
- */
133
- toggleMLNoiseSuppression(enabled) {
134
- if (!this.mlNoiseSuppressor) {
135
- console.warn('ML Noise Suppression not initialized. Call initializeMLNoiseSuppression() first.');
136
- return;
137
- }
138
- this.mlNoiseSuppressionEnabled = enabled;
139
- console.log(`🎤 ML Noise Suppression: ${enabled ? 'ON' : 'OFF'}`);
140
- }
141
- /**
142
- * Check if ML noise suppression is enabled
143
- */
144
- isMLNoiseSuppressionEnabled() {
145
- return this.mlNoiseSuppressionEnabled && this.mlNoiseSuppressor !== null;
146
- }
147
104
  async produceTrack(track, appData) {
148
- console.log(`🎬 [SDK] produceTrack called - kind: ${track.kind}, enabled: ${track.enabled}, readyState: ${track.readyState}`);
149
- let processedTrack = track;
150
- // Apply ML noise suppression to audio BEFORE sending to MediaSoup
151
- if (track.kind === 'audio' && this.mlNoiseSuppressionEnabled && this.mlNoiseSuppressor) {
152
- try {
153
- console.log('🎤 [SDK] Applying ML noise suppression to audio...');
154
- const inputStream = new MediaStream([track]);
155
- console.log('🎤 [SDK] Created input stream with track');
156
- const cleanedStream = await this.mlNoiseSuppressor.processMediaStream(inputStream);
157
- console.log('🎤 [SDK] Got cleaned stream from ML');
158
- processedTrack = cleanedStream.getAudioTracks()[0];
159
- console.log(`✅ [SDK] ML noise suppression applied - processed track state: ${processedTrack.readyState}`);
160
- }
161
- catch (error) {
162
- console.error('❌ [SDK] ML noise suppression failed, using original track:', error);
163
- processedTrack = track; // Fallback to original track
164
- }
165
- }
166
- else {
167
- console.log(`ℹ️ [SDK] Skipping ML - kind: ${track.kind}, ML enabled: ${this.mlNoiseSuppressionEnabled}`);
168
- }
169
- console.log(`📤 [SDK] Producing track to MediaSoup - kind: ${processedTrack.kind}, state: ${processedTrack.readyState}`);
170
- const producer = await this.mediasoupManager.produce(processedTrack, appData);
171
- console.log(`✅ [SDK] Producer created - id: ${producer.id}, kind: ${producer.kind}`);
105
+ const producer = await this.mediasoupManager.produce(track, appData);
172
106
  if (this.localParticipant) {
173
107
  const isFirstProducer = this.localParticipant.producers.size === 0;
174
108
  this.localParticipant.producers.set(producer.id, producer);
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@newgameplusinc/odyssey-audio-video-sdk-dev",
3
- "version": "1.0.57",
4
- "description": "Odyssey Spatial Audio & Video SDK using MediaSoup for real-time communication with AI-powered noise suppression",
3
+ "version": "1.0.58",
4
+ "description": "Odyssey Spatial Audio & Video SDK using MediaSoup for real-time communication",
5
5
  "main": "dist/index.js",
6
6
  "types": "dist/index.d.ts",
7
7
  "scripts": {
@@ -31,8 +31,7 @@
31
31
  "socket.io-client": "^4.7.2",
32
32
  "webrtc-adapter": "^8.2.3",
33
33
  "mediasoup-client": "^3.6.90",
34
- "events": "^3.3.0",
35
- "@tensorflow/tfjs": "^4.22.0"
34
+ "events": "^3.3.0"
36
35
  },
37
36
  "devDependencies": {
38
37
  "@types/node": "^20.0.0",
@@ -1,76 +0,0 @@
1
- /**
2
- * ML-Based Noise Suppressor for Odyssey MediaSoup SDK
3
- * Uses trained TensorFlow.js BiLSTM model for real-time noise suppression
4
- *
5
- * Architecture: BiLSTM (256 units x 2) + Dense layers
6
- * Input: Mel-spectrogram features (16 frames x 128 mels)
7
- * Output: Noise suppression mask (0-1 per frequency bin)
8
- *
9
- * Trained on: LibriSpeech + UrbanSound8K + MS-SNSD datasets
10
- * Performance: val_loss=0.038, SNR improvement ~12dB
11
- */
12
- export declare class MLNoiseSuppressor {
13
- private model;
14
- private config;
15
- private normStats;
16
- private audioContext;
17
- private isInitialized;
18
- private processingNode;
19
- private highPassFilter;
20
- private frameBuffer;
21
- private prevMask;
22
- private readonly SMOOTHING_ALPHA;
23
- private melFilterbank;
24
- private fftSize;
25
- private hannWindow;
26
- /**
27
- * Initialize the ML noise suppressor
28
- * @param modelUrl URL to the model.json file
29
- * @param audioContext Web Audio API AudioContext
30
- */
31
- initialize(modelUrl: string, audioContext: AudioContext): Promise<void>;
32
- /**
33
- * Create Hann window for FFT
34
- */
35
- private createHannWindow;
36
- /**
37
- * Create mel filterbank matrix
38
- */
39
- private createMelFilterbank;
40
- /**
41
- * Compute FFT magnitude spectrum (optimized DFT for real-time)
42
- */
43
- private computeFFT;
44
- /**
45
- * Compute mel-spectrogram features from audio frame
46
- */
47
- private computeMelFeatures;
48
- /**
49
- * Process audio buffer with ML noise suppression
50
- * @param inputBuffer Audio buffer to process (Float32Array)
51
- * @returns Processed audio buffer
52
- */
53
- processAudio(inputBuffer: Float32Array): Promise<Float32Array>;
54
- /**
55
- * Apply temporal smoothing to reduce artifacts (Apple-style)
56
- */
57
- private applyTemporalSmoothing;
58
- /**
59
- * Apply mask with voice frequency preservation
60
- */
61
- private applyMaskWithVoicePreservation;
62
- /**
63
- * Process MediaStream with ML noise suppression
64
- * @param inputStream MediaStream to process
65
- * @returns Cleaned MediaStream
66
- */
67
- processMediaStream(inputStream: MediaStream): Promise<MediaStream>;
68
- /**
69
- * Cleanup resources
70
- */
71
- dispose(): void;
72
- /**
73
- * Check if initialized
74
- */
75
- isReady(): boolean;
76
- }