@newgameplusinc/odyssey-audio-video-sdk-dev 1.0.16 β 1.0.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +0 -9
- package/dist/SpatialAudioManager.d.ts +8 -18
- package/dist/SpatialAudioManager.js +104 -312
- package/dist/index.d.ts +0 -1
- package/dist/index.js +0 -3
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -13,7 +13,6 @@ It mirrors the production SDK used by Odyssey V2 and ships ready-to-drop into an
|
|
|
13
13
|
- π§ **Accurate pose propagation** β `updatePosition()` streams listener pose to the SFU while `participant-position-updated` keeps the local store in sync.
|
|
14
14
|
- π§ **Studio-grade spatial audio** β each remote participant gets a dedicated Web Audio graph: denoiser β high-pass β low-pass β HRTF `PannerNode` β adaptive gain β master compressor.
|
|
15
15
|
- π₯ **Camera-ready streams** β video tracks are exposed separately so UI layers can render muted `<video>` tags while audio stays inside Web Audio.
|
|
16
|
-
- ποΈ **Clean microphone uplink (optβin)** β when `outboundTuning.enabled=true`, `enhanceOutgoingAudioTrack` runs mic input through denoiser + EQ + compressor before hitting the SFU.
|
|
17
16
|
- π **EventEmitter contract** β subscribe to `room-joined`, `consumer-created`, `participant-position-updated`, etc., without touching Socket.IO directly.
|
|
18
17
|
|
|
19
18
|
## Quick Start
|
|
@@ -81,7 +80,6 @@ sdk.setListenerFromLSD(listenerPos, cameraPos, lookAtPos);
|
|
|
81
80
|
- **Orientation math** β `setListenerFromLSD()` builds forward/right/up vectors from camera/LookAt to keep the listener aligned with head movement.
|
|
82
81
|
- **Dynamic distance gain** β `updateSpatialAudio()` measures distance from listener β source and applies a smooth rolloff curve, so distant avatars fade to silence.
|
|
83
82
|
- **Noise handling** β optional AudioWorklet denoiser plus high/low-pass filters trim rumble & hiss before HRTF processing.
|
|
84
|
-
- **Dynamic gate (opt-in)** β enable via `noiseGate.enabled=true` to let the SDK automatically clamp remote tracks when theyβre idle.
|
|
85
83
|
|
|
86
84
|
#### How Spatial Audio Is Built
|
|
87
85
|
1. **Telemetry ingestion** β each LSD packet is passed through `setListenerFromLSD(listenerPos, cameraPos, lookAtPos)` so the Web Audio listener matches the playerβs real head/camera pose.
|
|
@@ -91,13 +89,6 @@ sdk.setListenerFromLSD(listenerPos, cameraPos, lookAtPos);
|
|
|
91
89
|
4. **Distance-aware gain** β the manager stores the latest listener pose and computes the Euclidean distance to each remote participant on every update. A custom rolloff curve adjusts gain before the compressor, giving the βsomeone on my left / far awayβ perception without blowing out master levels.
|
|
92
90
|
5. **Left/right rendering** β because the panner uses `panningModel = "HRTF"`, browsers feed the processed signal into the userβs audio hardware with head-related transfer functions, producing natural interaural time/intensity differences.
|
|
93
91
|
|
|
94
|
-
#### How Microphone Audio Is Tuned Before Sending (Opt-In)
|
|
95
|
-
> Disabled by default. Enable via `new SpatialAudioManager({ outboundTuning: { enabled: true } })`.
|
|
96
|
-
1. **Hardware constraints first** β the SDK requests `noiseSuppression`, `echoCancellation`, and `autoGainControl` on the raw `MediaStreamTrack` (plus Chromium-specific `goog*` flags).
|
|
97
|
-
2. **Web Audio pre-flight** β `enhanceOutgoingAudioTrack(track)` clones the mic into a dedicated `AudioContext` and chain: `Denoiser β 50/60β―Hz notches β Low-shelf rumble cut β High-pass (95β―Hz) β Low-pass (7.2β―kHz) β High-shelf tame β Presence boost β Dynamics compressor β Adaptive gate`.
|
|
98
|
-
3. **Adaptive gate** β a lightweight RMS monitor clamps the gate gain when only background hiss remains, but opens instantly when speech energy rises.
|
|
99
|
-
4. **Clean stream to SFU** β the processed track is what you pass to `produceTrack`, so every participant receives the filtered audio (and your local store uses the same track for mute toggles). Toggle the feature off to fall back to raw WebRTC audio instantly.
|
|
100
|
-
|
|
101
92
|
## Video Flow (Capture β Rendering)
|
|
102
93
|
|
|
103
94
|
```
|
|
@@ -12,17 +12,9 @@ type DenoiserOptions = {
|
|
|
12
12
|
noiseFloor?: number;
|
|
13
13
|
release?: number;
|
|
14
14
|
};
|
|
15
|
-
type NoiseGateOptions = {
|
|
16
|
-
enabled?: boolean;
|
|
17
|
-
};
|
|
18
|
-
type OutboundTuningOptions = {
|
|
19
|
-
enabled?: boolean;
|
|
20
|
-
};
|
|
21
15
|
type SpatialAudioOptions = {
|
|
22
16
|
distance?: SpatialAudioDistanceConfig;
|
|
23
17
|
denoiser?: DenoiserOptions;
|
|
24
|
-
noiseGate?: NoiseGateOptions;
|
|
25
|
-
outboundTuning?: OutboundTuningOptions;
|
|
26
18
|
};
|
|
27
19
|
export declare class SpatialAudioManager extends EventManager {
|
|
28
20
|
private audioContext;
|
|
@@ -31,16 +23,12 @@ export declare class SpatialAudioManager extends EventManager {
|
|
|
31
23
|
private monitoringIntervals;
|
|
32
24
|
private compressor;
|
|
33
25
|
private options;
|
|
26
|
+
private denoiseWorkletReady;
|
|
34
27
|
private denoiseWorkletUrl?;
|
|
35
28
|
private denoiserWasmBytes?;
|
|
36
|
-
private denoiseContextPromises;
|
|
37
29
|
private listenerPosition;
|
|
38
30
|
private listenerInitialized;
|
|
39
|
-
private stabilityState;
|
|
40
|
-
private outgoingProcessors;
|
|
41
31
|
private listenerDirection;
|
|
42
|
-
private noiseGateEnabled;
|
|
43
|
-
private outboundTuningEnabled;
|
|
44
32
|
constructor(options?: SpatialAudioOptions);
|
|
45
33
|
getAudioContext(): AudioContext;
|
|
46
34
|
/**
|
|
@@ -59,9 +47,7 @@ export declare class SpatialAudioManager extends EventManager {
|
|
|
59
47
|
* @param bypassSpatialization For testing - bypasses 3D positioning
|
|
60
48
|
*/
|
|
61
49
|
setupSpatialAudioForParticipant(participantId: string, track: MediaStreamTrack, bypassSpatialization?: boolean): Promise<void>;
|
|
62
|
-
enhanceOutgoingAudioTrack(track: MediaStreamTrack): Promise<MediaStreamTrack>;
|
|
63
50
|
private startMonitoring;
|
|
64
|
-
private handleTrackStability;
|
|
65
51
|
/**
|
|
66
52
|
* Update spatial audio position and orientation for a participant
|
|
67
53
|
*
|
|
@@ -110,10 +96,14 @@ export declare class SpatialAudioManager extends EventManager {
|
|
|
110
96
|
private getDistanceBetween;
|
|
111
97
|
private calculateDistanceGain;
|
|
112
98
|
private normalizePositionUnits;
|
|
99
|
+
private getVectorFromListener;
|
|
100
|
+
private applyDirectionalSuppression;
|
|
101
|
+
private calculateClarityScore;
|
|
102
|
+
private calculateProximityWeight;
|
|
103
|
+
private calculateDirectionFocus;
|
|
104
|
+
private normalizeVector;
|
|
105
|
+
private clamp;
|
|
113
106
|
private isDenoiserEnabled;
|
|
114
|
-
private applyHardwareNoiseConstraints;
|
|
115
|
-
private startOutboundMonitor;
|
|
116
|
-
private cleanupOutboundProcessor;
|
|
117
107
|
private ensureDenoiseWorklet;
|
|
118
108
|
private resolveOptions;
|
|
119
109
|
}
|
|
@@ -7,23 +7,19 @@ class SpatialAudioManager extends EventManager_1.EventManager {
|
|
|
7
7
|
super();
|
|
8
8
|
this.participantNodes = new Map();
|
|
9
9
|
this.monitoringIntervals = new Map();
|
|
10
|
-
this.
|
|
10
|
+
this.denoiseWorkletReady = null;
|
|
11
11
|
this.listenerPosition = { x: 0, y: 0, z: 0 };
|
|
12
12
|
this.listenerInitialized = false;
|
|
13
|
-
this.stabilityState = new Map();
|
|
14
|
-
this.outgoingProcessors = new Map();
|
|
15
13
|
this.listenerDirection = {
|
|
16
14
|
forward: { x: 0, y: 1, z: 0 },
|
|
17
15
|
up: { x: 0, y: 0, z: 1 },
|
|
18
16
|
};
|
|
19
17
|
this.options = this.resolveOptions(options);
|
|
20
|
-
this.noiseGateEnabled = this.options.noiseGate?.enabled ?? false;
|
|
21
|
-
this.outboundTuningEnabled = this.options.outboundTuning?.enabled ?? false;
|
|
22
18
|
// Use high sample rate for best audio quality
|
|
23
19
|
this.audioContext = new AudioContext({ sampleRate: 48000 });
|
|
24
20
|
// Master gain
|
|
25
21
|
this.masterGainNode = this.audioContext.createGain();
|
|
26
|
-
this.masterGainNode.gain.value =
|
|
22
|
+
this.masterGainNode.gain.value = 5.0;
|
|
27
23
|
// Compressor for dynamic range control and preventing distortion
|
|
28
24
|
this.compressor = this.audioContext.createDynamicsCompressor();
|
|
29
25
|
this.compressor.threshold.value = -24; // dB
|
|
@@ -73,7 +69,7 @@ class SpatialAudioManager extends EventManager_1.EventManager {
|
|
|
73
69
|
const panner = this.audioContext.createPanner();
|
|
74
70
|
const analyser = this.audioContext.createAnalyser();
|
|
75
71
|
const gain = this.audioContext.createGain();
|
|
76
|
-
const
|
|
72
|
+
const proximityGain = this.audioContext.createGain();
|
|
77
73
|
let denoiseNode;
|
|
78
74
|
if (this.isDenoiserEnabled() && typeof this.audioContext.audioWorklet !== "undefined") {
|
|
79
75
|
try {
|
|
@@ -107,8 +103,11 @@ class SpatialAudioManager extends EventManager_1.EventManager {
|
|
|
107
103
|
lowpassFilter.type = "lowpass";
|
|
108
104
|
lowpassFilter.frequency.value = 7500; // Below 8kHz to avoid flat/muffled sound
|
|
109
105
|
lowpassFilter.Q.value = 1.0; // Quality factor
|
|
110
|
-
|
|
111
|
-
|
|
106
|
+
const dynamicLowpass = this.audioContext.createBiquadFilter();
|
|
107
|
+
dynamicLowpass.type = "lowpass";
|
|
108
|
+
dynamicLowpass.frequency.value = 7600;
|
|
109
|
+
dynamicLowpass.Q.value = 0.8;
|
|
110
|
+
proximityGain.gain.value = 1.0;
|
|
112
111
|
// Configure Panner for realistic 3D spatial audio
|
|
113
112
|
const distanceConfig = this.getDistanceConfig();
|
|
114
113
|
panner.panningModel = "HRTF"; // Head-Related Transfer Function for realistic 3D
|
|
@@ -128,20 +127,17 @@ class SpatialAudioManager extends EventManager_1.EventManager {
|
|
|
128
127
|
}
|
|
129
128
|
currentNode.connect(highpassFilter);
|
|
130
129
|
highpassFilter.connect(lowpassFilter);
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
lowpassFilter.connect(noiseGate);
|
|
134
|
-
postFilterNode = noiseGate;
|
|
135
|
-
}
|
|
130
|
+
lowpassFilter.connect(dynamicLowpass);
|
|
131
|
+
dynamicLowpass.connect(proximityGain);
|
|
136
132
|
if (bypassSpatialization) {
|
|
137
133
|
console.log(`π TESTING: Connecting audio directly to destination (bypassing spatial audio) for ${participantId}`);
|
|
138
|
-
|
|
134
|
+
proximityGain.connect(analyser);
|
|
139
135
|
analyser.connect(this.masterGainNode);
|
|
140
136
|
}
|
|
141
137
|
else {
|
|
142
138
|
// Standard spatialized path with full audio chain
|
|
143
|
-
// Audio Chain: source -> filters ->
|
|
144
|
-
|
|
139
|
+
// Audio Chain: source -> filters -> panner -> analyser -> gain -> masterGain -> compressor -> destination
|
|
140
|
+
proximityGain.connect(panner);
|
|
145
141
|
panner.connect(analyser);
|
|
146
142
|
analyser.connect(gain);
|
|
147
143
|
gain.connect(this.masterGainNode);
|
|
@@ -151,21 +147,13 @@ class SpatialAudioManager extends EventManager_1.EventManager {
|
|
|
151
147
|
panner,
|
|
152
148
|
analyser,
|
|
153
149
|
gain,
|
|
154
|
-
|
|
150
|
+
proximityGain,
|
|
155
151
|
highpassFilter,
|
|
156
152
|
lowpassFilter,
|
|
153
|
+
dynamicLowpass,
|
|
157
154
|
denoiseNode,
|
|
158
155
|
stream,
|
|
159
156
|
});
|
|
160
|
-
this.stabilityState.set(participantId, {
|
|
161
|
-
smoothedLevel: 0,
|
|
162
|
-
targetGain: 1,
|
|
163
|
-
networkMuted: false,
|
|
164
|
-
});
|
|
165
|
-
if (this.noiseGateEnabled && typeof track.onmute !== "undefined") {
|
|
166
|
-
track.onmute = () => this.handleTrackStability(participantId, true);
|
|
167
|
-
track.onunmute = () => this.handleTrackStability(participantId, false);
|
|
168
|
-
}
|
|
169
157
|
console.log(`π§ Spatial audio setup complete for ${participantId}:`, {
|
|
170
158
|
audioContextState: this.audioContext.state,
|
|
171
159
|
sampleRate: this.audioContext.sampleRate,
|
|
@@ -182,146 +170,15 @@ class SpatialAudioManager extends EventManager_1.EventManager {
|
|
|
182
170
|
rolloffFactor: panner.rolloffFactor,
|
|
183
171
|
},
|
|
184
172
|
});
|
|
185
|
-
// Start monitoring audio levels
|
|
186
|
-
|
|
187
|
-
this.startMonitoring(participantId);
|
|
188
|
-
}
|
|
189
|
-
}
|
|
190
|
-
async enhanceOutgoingAudioTrack(track) {
|
|
191
|
-
if (track.kind !== "audio" || !this.outboundTuningEnabled) {
|
|
192
|
-
return track;
|
|
193
|
-
}
|
|
194
|
-
const existingProcessor = Array.from(this.outgoingProcessors.values()).find((processor) => processor.originalTrack === track);
|
|
195
|
-
if (existingProcessor) {
|
|
196
|
-
return existingProcessor.processedTrack;
|
|
197
|
-
}
|
|
198
|
-
await this.applyHardwareNoiseConstraints(track);
|
|
199
|
-
const context = new AudioContext({ sampleRate: 48000 });
|
|
200
|
-
await context.resume();
|
|
201
|
-
const sourceStream = new MediaStream([track]);
|
|
202
|
-
const source = context.createMediaStreamSource(sourceStream);
|
|
203
|
-
let current = source;
|
|
204
|
-
let denoiseNode;
|
|
205
|
-
if (this.isDenoiserEnabled() && typeof context.audioWorklet !== "undefined") {
|
|
206
|
-
try {
|
|
207
|
-
await this.ensureDenoiseWorklet(context);
|
|
208
|
-
denoiseNode = new AudioWorkletNode(context, "odyssey-denoise", {
|
|
209
|
-
numberOfInputs: 1,
|
|
210
|
-
numberOfOutputs: 1,
|
|
211
|
-
processorOptions: {
|
|
212
|
-
enabled: true,
|
|
213
|
-
threshold: this.options.denoiser?.threshold,
|
|
214
|
-
noiseFloor: this.options.denoiser?.noiseFloor,
|
|
215
|
-
release: this.options.denoiser?.release,
|
|
216
|
-
wasmBytes: this.denoiserWasmBytes
|
|
217
|
-
? this.denoiserWasmBytes.slice(0)
|
|
218
|
-
: null,
|
|
219
|
-
},
|
|
220
|
-
});
|
|
221
|
-
current.connect(denoiseNode);
|
|
222
|
-
current = denoiseNode;
|
|
223
|
-
}
|
|
224
|
-
catch (error) {
|
|
225
|
-
console.warn("β οΈ Outgoing denoiser unavailable, continuing without it.", error);
|
|
226
|
-
}
|
|
227
|
-
}
|
|
228
|
-
const notch60 = context.createBiquadFilter();
|
|
229
|
-
notch60.type = "notch";
|
|
230
|
-
notch60.frequency.value = 60;
|
|
231
|
-
notch60.Q.value = 24;
|
|
232
|
-
current.connect(notch60);
|
|
233
|
-
current = notch60;
|
|
234
|
-
const notch50 = context.createBiquadFilter();
|
|
235
|
-
notch50.type = "notch";
|
|
236
|
-
notch50.frequency.value = 50;
|
|
237
|
-
notch50.Q.value = 24;
|
|
238
|
-
current.connect(notch50);
|
|
239
|
-
current = notch50;
|
|
240
|
-
const lowShelf = context.createBiquadFilter();
|
|
241
|
-
lowShelf.type = "lowshelf";
|
|
242
|
-
lowShelf.frequency.value = 120;
|
|
243
|
-
lowShelf.gain.value = -3;
|
|
244
|
-
current.connect(lowShelf);
|
|
245
|
-
current = lowShelf;
|
|
246
|
-
const highpassFilter = context.createBiquadFilter();
|
|
247
|
-
highpassFilter.type = "highpass";
|
|
248
|
-
highpassFilter.frequency.value = 95;
|
|
249
|
-
highpassFilter.Q.value = 0.8;
|
|
250
|
-
current.connect(highpassFilter);
|
|
251
|
-
current = highpassFilter;
|
|
252
|
-
const lowpassFilter = context.createBiquadFilter();
|
|
253
|
-
lowpassFilter.type = "lowpass";
|
|
254
|
-
lowpassFilter.frequency.value = 7200;
|
|
255
|
-
lowpassFilter.Q.value = 0.8;
|
|
256
|
-
current.connect(lowpassFilter);
|
|
257
|
-
current = lowpassFilter;
|
|
258
|
-
const hissShelf = context.createBiquadFilter();
|
|
259
|
-
hissShelf.type = "highshelf";
|
|
260
|
-
hissShelf.frequency.value = 6400;
|
|
261
|
-
hissShelf.gain.value = -4;
|
|
262
|
-
current.connect(hissShelf);
|
|
263
|
-
current = hissShelf;
|
|
264
|
-
const presenceBoost = context.createBiquadFilter();
|
|
265
|
-
presenceBoost.type = "peaking";
|
|
266
|
-
presenceBoost.frequency.value = 2400;
|
|
267
|
-
presenceBoost.Q.value = 1.1;
|
|
268
|
-
presenceBoost.gain.value = 2.4;
|
|
269
|
-
current.connect(presenceBoost);
|
|
270
|
-
current = presenceBoost;
|
|
271
|
-
const compressor = context.createDynamicsCompressor();
|
|
272
|
-
compressor.threshold.value = -18;
|
|
273
|
-
compressor.knee.value = 16;
|
|
274
|
-
compressor.ratio.value = 3.2;
|
|
275
|
-
compressor.attack.value = 0.002;
|
|
276
|
-
compressor.release.value = 0.22;
|
|
277
|
-
current.connect(compressor);
|
|
278
|
-
current = compressor;
|
|
279
|
-
const postCompressorTap = context.createGain();
|
|
280
|
-
postCompressorTap.gain.value = 1.05;
|
|
281
|
-
current.connect(postCompressorTap);
|
|
282
|
-
current = postCompressorTap;
|
|
283
|
-
const analyser = context.createAnalyser();
|
|
284
|
-
analyser.fftSize = 512;
|
|
285
|
-
current.connect(analyser);
|
|
286
|
-
const gate = context.createGain();
|
|
287
|
-
gate.gain.value = 1;
|
|
288
|
-
current.connect(gate);
|
|
289
|
-
const destination = context.createMediaStreamDestination();
|
|
290
|
-
gate.connect(destination);
|
|
291
|
-
const processedTrack = destination.stream.getAudioTracks()[0];
|
|
292
|
-
processedTrack.contentHint = "speech";
|
|
293
|
-
const processorId = processedTrack.id;
|
|
294
|
-
const monitor = this.startOutboundMonitor(processorId, analyser, gate);
|
|
295
|
-
const cleanup = () => this.cleanupOutboundProcessor(processorId);
|
|
296
|
-
processedTrack.addEventListener("ended", cleanup);
|
|
297
|
-
track.addEventListener("ended", cleanup);
|
|
298
|
-
this.outgoingProcessors.set(processorId, {
|
|
299
|
-
context,
|
|
300
|
-
sourceStream,
|
|
301
|
-
destinationStream: destination.stream,
|
|
302
|
-
analyser,
|
|
303
|
-
gate,
|
|
304
|
-
monitor,
|
|
305
|
-
originalTrack: track,
|
|
306
|
-
processedTrack,
|
|
307
|
-
cleanupListener: cleanup,
|
|
308
|
-
});
|
|
309
|
-
console.log("ποΈ [SDK] Outgoing audio tuned", {
|
|
310
|
-
originalTrackId: track.id,
|
|
311
|
-
processedTrackId: processedTrack.id,
|
|
312
|
-
});
|
|
313
|
-
return processedTrack;
|
|
173
|
+
// Start monitoring audio levels
|
|
174
|
+
this.startMonitoring(participantId);
|
|
314
175
|
}
|
|
315
176
|
startMonitoring(participantId) {
|
|
316
|
-
if (!this.noiseGateEnabled) {
|
|
317
|
-
return;
|
|
318
|
-
}
|
|
319
177
|
const nodes = this.participantNodes.get(participantId);
|
|
320
178
|
if (!nodes)
|
|
321
179
|
return;
|
|
322
|
-
const { analyser, stream
|
|
180
|
+
const { analyser, stream } = nodes;
|
|
323
181
|
const dataArray = new Uint8Array(analyser.frequencyBinCount);
|
|
324
|
-
let lastTrackLog = 0;
|
|
325
182
|
// Clear any existing interval for this participant
|
|
326
183
|
if (this.monitoringIntervals.has(participantId)) {
|
|
327
184
|
clearInterval(this.monitoringIntervals.get(participantId));
|
|
@@ -334,48 +191,16 @@ class SpatialAudioManager extends EventManager_1.EventManager {
|
|
|
334
191
|
}
|
|
335
192
|
const average = sum / dataArray.length;
|
|
336
193
|
const audioLevel = (average / 128) * 255; // Scale to 0-255
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
if (stability) {
|
|
340
|
-
const smoothing = 0.2;
|
|
341
|
-
stability.smoothedLevel =
|
|
342
|
-
stability.smoothedLevel * (1 - smoothing) + normalizedLevel * smoothing;
|
|
343
|
-
const gateOpenThreshold = 0.028; // tuned for speech presence
|
|
344
|
-
const gateCloseThreshold = 0.012;
|
|
345
|
-
const noiseFloorGain = 0.12;
|
|
346
|
-
let targetGain = stability.targetGain;
|
|
347
|
-
if (stability.networkMuted) {
|
|
348
|
-
targetGain = 0;
|
|
349
|
-
}
|
|
350
|
-
else if (stability.smoothedLevel < gateCloseThreshold) {
|
|
351
|
-
targetGain = 0;
|
|
352
|
-
}
|
|
353
|
-
else if (stability.smoothedLevel < gateOpenThreshold) {
|
|
354
|
-
targetGain = noiseFloorGain;
|
|
355
|
-
}
|
|
356
|
-
else {
|
|
357
|
-
targetGain = 1;
|
|
358
|
-
}
|
|
359
|
-
if (Math.abs(targetGain - stability.targetGain) > 0.05) {
|
|
360
|
-
const ramp = targetGain > stability.targetGain ? 0.04 : 0.18;
|
|
361
|
-
noiseGate.gain.setTargetAtTime(targetGain, this.audioContext.currentTime, ramp);
|
|
362
|
-
stability.targetGain = targetGain;
|
|
363
|
-
}
|
|
364
|
-
if (Math.random() < 0.05) {
|
|
365
|
-
console.log(`ποΈ [NoiseGate] ${participantId}`, {
|
|
366
|
-
level: stability.smoothedLevel.toFixed(3),
|
|
367
|
-
gain: stability.targetGain.toFixed(2),
|
|
368
|
-
});
|
|
369
|
-
}
|
|
370
|
-
}
|
|
371
|
-
if (audioLevel < 1.0 && Math.random() < 0.2) {
|
|
194
|
+
console.log(`π Audio level for ${participantId}: ${audioLevel.toFixed(2)} (0-255 scale)`);
|
|
195
|
+
if (audioLevel < 1.0) {
|
|
372
196
|
console.warn(`β οΈ NO AUDIO DATA detected for ${participantId}! Track may be silent or not transmitting.`);
|
|
197
|
+
console.info(`π‘ Check: 1) Is microphone unmuted? 2) Is correct mic selected? 3) Is mic working in system settings?`);
|
|
373
198
|
}
|
|
374
|
-
|
|
375
|
-
|
|
199
|
+
// Check track status after 2 seconds
|
|
200
|
+
setTimeout(() => {
|
|
376
201
|
const track = stream.getAudioTracks()[0];
|
|
377
202
|
if (track) {
|
|
378
|
-
console.log(`π Audio track status for ${participantId}:`, {
|
|
203
|
+
console.log(`π Audio track status after 2s for ${participantId}:`, {
|
|
379
204
|
trackEnabled: track.enabled,
|
|
380
205
|
trackMuted: track.muted,
|
|
381
206
|
trackReadyState: track.readyState,
|
|
@@ -387,23 +212,10 @@ class SpatialAudioManager extends EventManager_1.EventManager {
|
|
|
387
212
|
},
|
|
388
213
|
});
|
|
389
214
|
}
|
|
390
|
-
}
|
|
391
|
-
},
|
|
215
|
+
}, 2000);
|
|
216
|
+
}, 2000); // Log every 2 seconds
|
|
392
217
|
this.monitoringIntervals.set(participantId, interval);
|
|
393
218
|
}
|
|
394
|
-
handleTrackStability(participantId, muted) {
|
|
395
|
-
if (!this.noiseGateEnabled) {
|
|
396
|
-
return;
|
|
397
|
-
}
|
|
398
|
-
const nodes = this.participantNodes.get(participantId);
|
|
399
|
-
if (!nodes)
|
|
400
|
-
return;
|
|
401
|
-
const stability = this.stabilityState.get(participantId);
|
|
402
|
-
if (stability) {
|
|
403
|
-
stability.networkMuted = muted;
|
|
404
|
-
}
|
|
405
|
-
nodes.noiseGate.gain.setTargetAtTime(muted ? 0 : 1, this.audioContext.currentTime, muted ? 0.05 : 0.2);
|
|
406
|
-
}
|
|
407
219
|
/**
|
|
408
220
|
* Update spatial audio position and orientation for a participant
|
|
409
221
|
*
|
|
@@ -451,7 +263,9 @@ class SpatialAudioManager extends EventManager_1.EventManager {
|
|
|
451
263
|
}
|
|
452
264
|
}
|
|
453
265
|
const listenerPos = this.listenerPosition;
|
|
266
|
+
const vectorToSource = this.getVectorFromListener(targetPosition);
|
|
454
267
|
const distance = this.getDistanceBetween(listenerPos, targetPosition);
|
|
268
|
+
this.applyDirectionalSuppression(participantId, distance, vectorToSource);
|
|
455
269
|
const distanceGain = this.calculateDistanceGain(distanceConfig, distance);
|
|
456
270
|
nodes.gain.gain.setTargetAtTime(distanceGain, this.audioContext.currentTime, 0.05);
|
|
457
271
|
if (Math.random() < 0.02) {
|
|
@@ -587,18 +401,11 @@ class SpatialAudioManager extends EventManager_1.EventManager {
|
|
|
587
401
|
nodes.panner.disconnect();
|
|
588
402
|
nodes.analyser.disconnect();
|
|
589
403
|
nodes.gain.disconnect();
|
|
590
|
-
nodes.noiseGate.disconnect();
|
|
591
404
|
if (nodes.denoiseNode) {
|
|
592
405
|
nodes.denoiseNode.disconnect();
|
|
593
406
|
}
|
|
594
|
-
const track = nodes.stream.getAudioTracks()[0];
|
|
595
|
-
if (track) {
|
|
596
|
-
track.onmute = null;
|
|
597
|
-
track.onunmute = null;
|
|
598
|
-
}
|
|
599
407
|
nodes.stream.getTracks().forEach((track) => track.stop());
|
|
600
408
|
this.participantNodes.delete(participantId);
|
|
601
|
-
this.stabilityState.delete(participantId);
|
|
602
409
|
console.log(`ποΈ Removed participant ${participantId} from spatial audio.`);
|
|
603
410
|
}
|
|
604
411
|
}
|
|
@@ -653,7 +460,7 @@ class SpatialAudioManager extends EventManager_1.EventManager {
|
|
|
653
460
|
const normalized = (distance - config.refDistance) /
|
|
654
461
|
Math.max(config.maxDistance - config.refDistance, 0.001);
|
|
655
462
|
const shaped = Math.pow(Math.max(0, 1 - normalized), Math.max(1.2, config.rolloffFactor * 1.05));
|
|
656
|
-
return Math.min(1, Math.max(0.
|
|
463
|
+
return Math.min(1, Math.max(0.01, shaped));
|
|
657
464
|
}
|
|
658
465
|
normalizePositionUnits(position) {
|
|
659
466
|
const distanceConfig = this.getDistanceConfig();
|
|
@@ -678,82 +485,83 @@ class SpatialAudioManager extends EventManager_1.EventManager {
|
|
|
678
485
|
}
|
|
679
486
|
return { ...position };
|
|
680
487
|
}
|
|
681
|
-
|
|
682
|
-
|
|
488
|
+
getVectorFromListener(targetPosition) {
|
|
489
|
+
if (!this.listenerInitialized) {
|
|
490
|
+
return { ...targetPosition };
|
|
491
|
+
}
|
|
492
|
+
return {
|
|
493
|
+
x: targetPosition.x - this.listenerPosition.x,
|
|
494
|
+
y: targetPosition.y - this.listenerPosition.y,
|
|
495
|
+
z: targetPosition.z - this.listenerPosition.z,
|
|
496
|
+
};
|
|
683
497
|
}
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
],
|
|
498
|
+
applyDirectionalSuppression(participantId, distance, vectorToSource) {
|
|
499
|
+
const nodes = this.participantNodes.get(participantId);
|
|
500
|
+
if (!nodes) {
|
|
501
|
+
return;
|
|
502
|
+
}
|
|
503
|
+
const clarityScore = this.calculateClarityScore(distance, vectorToSource);
|
|
504
|
+
const targetGain = 0.55 + clarityScore * 0.6; // 0.55 β 1.15
|
|
505
|
+
const targetLowpass = 3200 + clarityScore * 4200; // 3.2kHz β ~7.4kHz
|
|
506
|
+
nodes.proximityGain.gain.setTargetAtTime(targetGain, this.audioContext.currentTime, 0.08);
|
|
507
|
+
nodes.dynamicLowpass.frequency.setTargetAtTime(targetLowpass, this.audioContext.currentTime, 0.12);
|
|
508
|
+
if (Math.random() < 0.005) {
|
|
509
|
+
console.log("ποΈ [Spatial Audio] Directional tuning", {
|
|
510
|
+
participantId,
|
|
511
|
+
distance: distance.toFixed(2),
|
|
512
|
+
clarityScore: clarityScore.toFixed(2),
|
|
513
|
+
targetGain: targetGain.toFixed(2),
|
|
514
|
+
lowpassHz: targetLowpass.toFixed(0),
|
|
702
515
|
});
|
|
703
516
|
}
|
|
704
|
-
|
|
705
|
-
|
|
517
|
+
}
|
|
518
|
+
calculateClarityScore(distance, vectorToSource) {
|
|
519
|
+
const proximityWeight = this.calculateProximityWeight(distance);
|
|
520
|
+
const focusWeight = this.calculateDirectionFocus(vectorToSource);
|
|
521
|
+
return this.clamp(0.2 + proximityWeight * 0.6 + focusWeight * 0.2, 0, 1);
|
|
522
|
+
}
|
|
523
|
+
calculateProximityWeight(distance) {
|
|
524
|
+
const closeRange = 1.2;
|
|
525
|
+
const fadeRange = 12;
|
|
526
|
+
if (distance <= closeRange) {
|
|
527
|
+
return 1;
|
|
528
|
+
}
|
|
529
|
+
if (distance >= fadeRange) {
|
|
530
|
+
return 0;
|
|
706
531
|
}
|
|
707
|
-
|
|
532
|
+
return 1 - (distance - closeRange) / (fadeRange - closeRange);
|
|
708
533
|
}
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
}
|
|
718
|
-
const level = (sum / dataArray.length) / 128;
|
|
719
|
-
smoothedLevel = smoothedLevel * 0.7 + level * 0.3;
|
|
720
|
-
let targetGain = 1;
|
|
721
|
-
if (smoothedLevel < 0.02) {
|
|
722
|
-
targetGain = 0;
|
|
723
|
-
}
|
|
724
|
-
else if (smoothedLevel < 0.05) {
|
|
725
|
-
targetGain = 0.45;
|
|
726
|
-
}
|
|
727
|
-
else {
|
|
728
|
-
targetGain = 1;
|
|
729
|
-
}
|
|
730
|
-
gate.gain.setTargetAtTime(targetGain, gate.context.currentTime, targetGain > gate.gain.value ? 0.02 : 0.08);
|
|
731
|
-
if (Math.random() < 0.03) {
|
|
732
|
-
console.log("ποΈ [SDK] Outgoing gate", {
|
|
733
|
-
processorId,
|
|
734
|
-
level: smoothedLevel.toFixed(3),
|
|
735
|
-
gain: targetGain.toFixed(2),
|
|
736
|
-
});
|
|
737
|
-
}
|
|
738
|
-
}, 200);
|
|
534
|
+
calculateDirectionFocus(vectorToSource) {
|
|
535
|
+
if (!this.listenerInitialized) {
|
|
536
|
+
return 0.5;
|
|
537
|
+
}
|
|
538
|
+
const forward = this.normalizeVector(this.listenerDirection.forward);
|
|
539
|
+
const source = this.normalizeVector(vectorToSource, { x: 0, y: 0, z: 1 });
|
|
540
|
+
const dot = forward.x * source.x + forward.y * source.y + forward.z * source.z;
|
|
541
|
+
return this.clamp((dot + 1) / 2, 0, 1);
|
|
739
542
|
}
|
|
740
|
-
|
|
741
|
-
const
|
|
742
|
-
if (
|
|
743
|
-
return;
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
this.outgoingProcessors.delete(processorId);
|
|
543
|
+
normalizeVector(vector, fallback = { x: 0, y: 0, z: 1 }) {
|
|
544
|
+
const length = Math.hypot(vector.x, vector.y, vector.z);
|
|
545
|
+
if (length < 1e-4) {
|
|
546
|
+
return { ...fallback };
|
|
547
|
+
}
|
|
548
|
+
return {
|
|
549
|
+
x: vector.x / length,
|
|
550
|
+
y: vector.y / length,
|
|
551
|
+
z: vector.z / length,
|
|
552
|
+
};
|
|
751
553
|
}
|
|
752
|
-
|
|
554
|
+
clamp(value, min, max) {
|
|
555
|
+
return Math.min(max, Math.max(min, value));
|
|
556
|
+
}
|
|
557
|
+
isDenoiserEnabled() {
|
|
558
|
+
return this.options.denoiser?.enabled !== false;
|
|
559
|
+
}
|
|
560
|
+
async ensureDenoiseWorklet() {
|
|
753
561
|
if (!this.isDenoiserEnabled()) {
|
|
754
562
|
return;
|
|
755
563
|
}
|
|
756
|
-
if (!("audioWorklet" in
|
|
564
|
+
if (!("audioWorklet" in this.audioContext)) {
|
|
757
565
|
console.warn("β οΈ AudioWorklet not supported in this browser. Disabling denoiser.");
|
|
758
566
|
this.options.denoiser = {
|
|
759
567
|
...(this.options.denoiser || {}),
|
|
@@ -761,9 +569,8 @@ class SpatialAudioManager extends EventManager_1.EventManager {
|
|
|
761
569
|
};
|
|
762
570
|
return;
|
|
763
571
|
}
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
return existingPromise;
|
|
572
|
+
if (this.denoiseWorkletReady) {
|
|
573
|
+
return this.denoiseWorkletReady;
|
|
767
574
|
}
|
|
768
575
|
const processorSource = `class OdysseyDenoiseProcessor extends AudioWorkletProcessor {
|
|
769
576
|
constructor(options) {
|
|
@@ -820,13 +627,11 @@ class SpatialAudioManager extends EventManager_1.EventManager {
|
|
|
820
627
|
|
|
821
628
|
registerProcessor('odyssey-denoise', OdysseyDenoiseProcessor);
|
|
822
629
|
`;
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
}
|
|
829
|
-
const promise = targetContext.audioWorklet
|
|
630
|
+
const blob = new Blob([processorSource], {
|
|
631
|
+
type: "application/javascript",
|
|
632
|
+
});
|
|
633
|
+
this.denoiseWorkletUrl = URL.createObjectURL(blob);
|
|
634
|
+
this.denoiseWorkletReady = this.audioContext.audioWorklet
|
|
830
635
|
.addModule(this.denoiseWorkletUrl)
|
|
831
636
|
.catch((error) => {
|
|
832
637
|
console.error("β Failed to register denoise worklet", error);
|
|
@@ -836,8 +641,7 @@ registerProcessor('odyssey-denoise', OdysseyDenoiseProcessor);
|
|
|
836
641
|
};
|
|
837
642
|
throw error;
|
|
838
643
|
});
|
|
839
|
-
this.
|
|
840
|
-
return promise;
|
|
644
|
+
return this.denoiseWorkletReady;
|
|
841
645
|
}
|
|
842
646
|
resolveOptions(options) {
|
|
843
647
|
const distanceDefaults = {
|
|
@@ -852,12 +656,6 @@ registerProcessor('odyssey-denoise', OdysseyDenoiseProcessor);
|
|
|
852
656
|
noiseFloor: 0.004,
|
|
853
657
|
release: 0.18,
|
|
854
658
|
};
|
|
855
|
-
const noiseGateDefaults = {
|
|
856
|
-
enabled: true,
|
|
857
|
-
};
|
|
858
|
-
const outboundDefaults = {
|
|
859
|
-
enabled: false,
|
|
860
|
-
};
|
|
861
659
|
return {
|
|
862
660
|
distance: {
|
|
863
661
|
refDistance: options?.distance?.refDistance ?? distanceDefaults.refDistance,
|
|
@@ -871,12 +669,6 @@ registerProcessor('odyssey-denoise', OdysseyDenoiseProcessor);
|
|
|
871
669
|
noiseFloor: options?.denoiser?.noiseFloor ?? denoiserDefaults.noiseFloor,
|
|
872
670
|
release: options?.denoiser?.release ?? denoiserDefaults.release,
|
|
873
671
|
},
|
|
874
|
-
noiseGate: {
|
|
875
|
-
enabled: options?.noiseGate?.enabled ?? noiseGateDefaults.enabled,
|
|
876
|
-
},
|
|
877
|
-
outboundTuning: {
|
|
878
|
-
enabled: options?.outboundTuning?.enabled ?? outboundDefaults.enabled,
|
|
879
|
-
},
|
|
880
672
|
};
|
|
881
673
|
}
|
|
882
674
|
}
|
package/dist/index.d.ts
CHANGED
|
@@ -26,7 +26,6 @@ export declare class OdysseySpatialComms extends EventManager {
|
|
|
26
26
|
}): Promise<Participant>;
|
|
27
27
|
leaveRoom(): void;
|
|
28
28
|
resumeAudio(): Promise<void>;
|
|
29
|
-
enhanceOutgoingAudioTrack(track: MediaStreamTrack): Promise<MediaStreamTrack>;
|
|
30
29
|
getAudioContextState(): AudioContextState;
|
|
31
30
|
produceTrack(track: MediaStreamTrack): Promise<any>;
|
|
32
31
|
updatePosition(position: Position, direction: Direction, spatialData?: {
|
package/dist/index.js
CHANGED
|
@@ -121,9 +121,6 @@ class OdysseySpatialComms extends EventManager_1.EventManager {
|
|
|
121
121
|
async resumeAudio() {
|
|
122
122
|
await this.spatialAudioManager.resumeAudioContext();
|
|
123
123
|
}
|
|
124
|
-
async enhanceOutgoingAudioTrack(track) {
|
|
125
|
-
return this.spatialAudioManager.enhanceOutgoingAudioTrack(track);
|
|
126
|
-
}
|
|
127
124
|
getAudioContextState() {
|
|
128
125
|
return this.spatialAudioManager.getAudioContextState();
|
|
129
126
|
}
|
package/package.json
CHANGED