@newgameplusinc/odyssey-audio-video-sdk-dev 1.0.10 β 1.0.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +128 -91
- package/dist/SpatialAudioManager.d.ts +14 -1
- package/dist/SpatialAudioManager.js +401 -75
- package/dist/index.d.ts +1 -0
- package/dist/index.js +3 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -1,106 +1,143 @@
|
|
|
1
|
-
# Odyssey
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
1
|
+
# Odyssey Audio/Video SDK (MediaSoup + Web Audio)
|
|
2
|
+
|
|
3
|
+
This package exposes `OdysseySpatialComms`, a thin TypeScript client that glues together:
|
|
4
|
+
|
|
5
|
+
- **MediaSoup SFU** for ultra-low-latency audio/video routing
|
|
6
|
+
- **Web Audio API** for Apple-like spatial mixing via `SpatialAudioManager`
|
|
7
|
+
- **Socket telemetry** (position + direction) so every browser hears/see everyone exactly where they are in the 3D world
|
|
8
|
+
|
|
9
|
+
It mirrors the production SDK used by Odyssey V2 and ships ready-to-drop into any Web UI (Vue, React, plain JS).
|
|
10
|
+
|
|
11
|
+
## Feature Highlights
|
|
12
|
+
- π **One class to rule it all** β `OdysseySpatialComms` wires transports, producers, consumers, and room state.
|
|
13
|
+
- π§ **Accurate pose propagation** β `updatePosition()` streams listener pose to the SFU while `participant-position-updated` keeps the local store in sync.
|
|
14
|
+
- π§ **Studio-grade spatial audio** β each remote participant gets a dedicated Web Audio graph: denoiser β high-pass β low-pass β HRTF `PannerNode` β adaptive gain β master compressor.
|
|
15
|
+
- π₯ **Camera-ready streams** β video tracks are exposed separately so UI layers can render muted `<video>` tags while audio stays inside Web Audio.
|
|
16
|
+
- ποΈ **Clean microphone uplink** β optional `enhanceOutgoingAudioTrack` helper runs mic input through denoiser + EQ + compressor before hitting the SFU.
|
|
17
|
+
- π **EventEmitter contract** β subscribe to `room-joined`, `consumer-created`, `participant-position-updated`, etc., without touching Socket.IO directly.
|
|
18
|
+
|
|
19
|
+
## Quick Start
|
|
20
|
+
|
|
21
|
+
```ts
|
|
22
|
+
import {
|
|
23
|
+
OdysseySpatialComms,
|
|
24
|
+
Direction,
|
|
25
|
+
Position,
|
|
26
|
+
} from "@newgameplusinc/odyssey-audio-video-sdk-dev";
|
|
27
|
+
|
|
28
|
+
const sdk = new OdysseySpatialComms("https://mediasoup-server.example.com");
|
|
29
|
+
|
|
30
|
+
// 1) Join a room
|
|
31
|
+
await sdk.joinRoom({
|
|
32
|
+
roomId: "demo-room",
|
|
33
|
+
userId: "user-123",
|
|
34
|
+
deviceId: "device-123",
|
|
35
|
+
position: { x: 0, y: 0, z: 0 },
|
|
36
|
+
direction: { x: 0, y: 1, z: 0 },
|
|
37
|
+
});
|
|
20
38
|
|
|
21
|
-
|
|
39
|
+
// 2) Produce local media
|
|
40
|
+
const stream = await navigator.mediaDevices.getUserMedia({ audio: true, video: true });
|
|
41
|
+
for (const track of stream.getTracks()) {
|
|
42
|
+
await sdk.produceTrack(track);
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
// 3) Handle remote tracks
|
|
46
|
+
sdk.on("consumer-created", async ({ participant, track }) => {
|
|
47
|
+
if (track.kind === "video") {
|
|
48
|
+
attachVideo(track, participant.participantId);
|
|
49
|
+
}
|
|
50
|
+
});
|
|
22
51
|
|
|
23
|
-
|
|
24
|
-
|
|
52
|
+
// 4) Keep spatial audio honest
|
|
53
|
+
sdk.updatePosition(currentPos, currentDir);
|
|
54
|
+
sdk.setListenerFromLSD(listenerPos, cameraPos, lookAtPos);
|
|
25
55
|
```
|
|
26
56
|
|
|
27
|
-
##
|
|
28
|
-
|
|
29
|
-
### 1. Initialize the SDK
|
|
57
|
+
## Audio Flow (Server β Browser)
|
|
30
58
|
|
|
31
|
-
```typescript
|
|
32
|
-
import { OdysseySpatialComms } from "@newgameplusinc/odyssey-spatial-sdk-wrapper";
|
|
33
|
-
|
|
34
|
-
// Initialize with your MediaSoup server URL
|
|
35
|
-
const sdk = new OdysseySpatialComms("https://your-mediasoup-server.com");
|
|
36
59
|
```
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
60
|
+
ββββββββββββββββ update-position ββββββββββββββββ pose + tracks ββββββββββββββββββββ
|
|
61
|
+
β Browser LSD β βββββββββββββββββββΆ β MediaSoup SFUβ βββββββββββββββββΆ β SDK Event Bus β
|
|
62
|
+
β (Unreal data)β β + Socket.IO β β (EventManager) β
|
|
63
|
+
ββββββββ¬ββββββββ ββββββββ¬ββββββββ ββββββββββββ¬βββββββββ
|
|
64
|
+
β β track + pose
|
|
65
|
+
β β βΌ
|
|
66
|
+
β ββββββββββΌβββββββββ ββββββββββββββββββββ
|
|
67
|
+
β audio RTP β consumer-createdβ β SpatialAudioMgr β
|
|
68
|
+
ββββββββββββββββββββββββββββΆβ setup per-user βββββββββββββββββββββββββ (Web Audio API) β
|
|
69
|
+
ββββββββββ¬βββββββββ β - Denoiser β
|
|
70
|
+
β β - HP / LP β
|
|
71
|
+
β β - HRTF Panner β
|
|
72
|
+
βΌ β - Gain + Comp β
|
|
73
|
+
Web Audio Graph ββββββββββββ¬ββββββββ
|
|
74
|
+
β β
|
|
75
|
+
βΌ βΌ
|
|
76
|
+
Listener ears (Left/Right) System Output
|
|
52
77
|
```
|
|
53
78
|
|
|
54
|
-
###
|
|
79
|
+
### Web Audio Algorithms
|
|
80
|
+
- **Coordinate normalization** β Unreal sends centimeters; `SpatialAudioManager` auto-detects large values and converts to meters once.
|
|
81
|
+
- **Orientation math** β `setListenerFromLSD()` builds forward/right/up vectors from camera/LookAt to keep the listener aligned with head movement.
|
|
82
|
+
- **Dynamic distance gain** β `updateSpatialAudio()` measures distance from listener β source and applies a smooth rolloff curve, so distant avatars fade to silence.
|
|
83
|
+
- **Noise handling** β optional AudioWorklet denoiser plus high/low-pass filters trim rumble & hiss before HRTF processing.
|
|
55
84
|
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
// Produce audio track
|
|
64
|
-
const audioTrack = stream.getAudioTracks()[0];
|
|
65
|
-
await sdk.produceTrack(audioTrack);
|
|
85
|
+
#### How Spatial Audio Is Built
|
|
86
|
+
1. **Telemetry ingestion** β each LSD packet is passed through `setListenerFromLSD(listenerPos, cameraPos, lookAtPos)` so the Web Audio listener matches the playerβs real head/camera pose.
|
|
87
|
+
2. **Per-participant node graph** β when `consumer-created` yields a remote audio track, `setupSpatialAudioForParticipant()` spins up an isolated graph:
|
|
88
|
+
`MediaStreamSource β (optional) Denoiser Worklet β High-Pass β Low-Pass β Panner(HRTF) β Gain β Master Compressor`.
|
|
89
|
+
3. **Position + direction updates** β every `participant-position-updated` event calls `updateSpatialAudio(participantId, position, direction)`. The position feeds the pannerβs XYZ, while the direction vector sets the source orientation so voices project forward relative to avatar facing.
|
|
90
|
+
4. **Distance-aware gain** β the manager stores the latest listener pose and computes the Euclidean distance to each remote participant on every update. A custom rolloff curve adjusts gain before the compressor, giving the βsomeone on my left / far awayβ perception without blowing out master levels.
|
|
91
|
+
5. **Left/right rendering** β because the panner uses `panningModel = "HRTF"`, browsers feed the processed signal into the userβs audio hardware with head-related transfer functions, producing natural interaural time/intensity differences.
|
|
66
92
|
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
93
|
+
#### How Microphone Audio Is Tuned Before Sending
|
|
94
|
+
1. **Hardware constraints first** β the SDK requests `noiseSuppression`, `echoCancellation`, and `autoGainControl` on the raw `MediaStreamTrack` (plus Chromium-specific `goog*` flags).
|
|
95
|
+
2. **Web Audio pre-flight** β `enhanceOutgoingAudioTrack(track)` clones the mic into a dedicated `AudioContext` and chain: `Denoiser β 50/60β―Hz notches β Low-shelf rumble cut β High-pass (95β―Hz) β Low-pass (7.2β―kHz) β High-shelf tame β Presence boost β Dynamics compressor β Adaptive gate`.
|
|
96
|
+
3. **Adaptive gate** β a lightweight RMS monitor clamps the gate gain when only background hiss remains, but opens instantly when speech energy rises.
|
|
97
|
+
4. **Clean stream to SFU** β the processed track is what you pass to `produceTrack`, so every participant receives the filtered audio (and your local store uses the same track for mute toggles).
|
|
71
98
|
|
|
72
|
-
|
|
99
|
+
## Video Flow (Capture β Rendering)
|
|
73
100
|
|
|
74
|
-
```typescript
|
|
75
|
-
sdk.updatePosition(
|
|
76
|
-
{ x: 10, y: 0, z: 5 }, // New position
|
|
77
|
-
{ x: 0, y: 0, z: 1 } // New direction
|
|
78
|
-
);
|
|
79
101
|
```
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
console.log("Participant left:", participantId);
|
|
92
|
-
});
|
|
93
|
-
|
|
94
|
-
// Consumer created (receiving audio/video from remote participant)
|
|
95
|
-
sdk.on("consumer-created", ({ participant, track, consumer }) => {
|
|
96
|
-
console.log("Receiving", track.kind, "from", participant.userName);
|
|
97
|
-
});
|
|
102
|
+
ββββββββββββββββ produceTrack ββββββββββββββββ RTP ββββββββββββββββ
|
|
103
|
+
β getUserMedia β ββββββββββββββββΆ β MediaSoup SDKβ βββββββΆ β MediaSoup SFUβ
|
|
104
|
+
ββββββββ¬ββββββββ β (Odyssey) β ββββββββ¬ββββββββ
|
|
105
|
+
β ββββββββ¬ββββββββ β
|
|
106
|
+
β consumer-created β track β
|
|
107
|
+
βΌ βΌ β
|
|
108
|
+
ββββββββββββββββ ββββββββββββββββ β
|
|
109
|
+
β Vue/React UI β ββββββββββββββββ β SDK Event Bus β ββββββββββββββββ
|
|
110
|
+
β (muted video β β exposes media β
|
|
111
|
+
β elements) β β tracks β
|
|
112
|
+
ββββββββββββββββ ββββββββββββββββ
|
|
98
113
|
```
|
|
99
114
|
|
|
100
|
-
##
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
115
|
+
## Core Classes
|
|
116
|
+
- `src/index.ts` β `OdysseySpatialComms` (socket lifecycle, producers/consumers, event surface).
|
|
117
|
+
- `src/MediasoupManager.ts` β transport helpers for produce/consume/resume.
|
|
118
|
+
- `src/SpatialAudioManager.ts` β Web Audio orchestration (listener transforms, per-participant chains, denoiser, distance math).
|
|
119
|
+
- `src/EventManager.ts` β lightweight EventEmitter used by the entire SDK.
|
|
120
|
+
|
|
121
|
+
## Integration Checklist
|
|
122
|
+
1. **Instantiate once** per page/tab and keep it in a store (Vuex, Redux, Zustand, etc.).
|
|
123
|
+
2. **Pipe LSD/Lap data** from your rendering engine into `updatePosition()` + `setListenerFromLSD()` at ~10 Hz.
|
|
124
|
+
3. **Render videos muted** β never attach remote audio tracks straight to DOM; let `SpatialAudioManager` own playback.
|
|
125
|
+
4. **Push avatar telemetry back to Unreal** so `remoteSpatialData` can render minimaps/circles (see Odyssey V2 `sendMediaSoupParticipantsToUnreal`).
|
|
126
|
+
5. **Monitor logs** β browser console shows `π§ SDK`, `π SDK`, and `ποΈ [Spatial Audio]` statements for every critical hop.
|
|
127
|
+
|
|
128
|
+
## Server Contract (Socket.IO events)
|
|
129
|
+
| Event | Direction | Payload |
|
|
130
|
+
|-------|-----------|---------|
|
|
131
|
+
| `join-room` | client β server | `{roomId, userId, deviceId, position, direction}` |
|
|
132
|
+
| `room-joined` | server β client | `RoomJoinedData` (router caps, participants snapshot) |
|
|
133
|
+
| `update-position` | client β server | `{participantId, conferenceId, position, direction}` |
|
|
134
|
+
| `participant-position-updated` | server β client | `{participantId, position, direction, mediaState}` |
|
|
135
|
+
| `consumer-created` | server β client | `{participantId, track(kind), position, direction}` |
|
|
136
|
+
| `participant-media-state-updated` | server β client | `{participantId, mediaState}` |
|
|
137
|
+
|
|
138
|
+
## Development Tips
|
|
139
|
+
- Run `pnpm install && pnpm build` inside `mediasoup-sdk-test` to publish a fresh build.
|
|
140
|
+
- Use `pnpm watch` while iterating so TypeScript outputs live under `dist/`.
|
|
141
|
+
- The SDK targets evergreen browsers; for Safari <16.4 you may need to polyfill AudioWorklets or disable the denoiser via `new SpatialAudioManager({ denoiser: { enabled: false } })`.
|
|
142
|
+
|
|
143
|
+
Have questions or want to extend the SDK? Start with `SpatialAudioManager` β thatβs where most of the βreal-worldβ behavior (distance feel, stereo cues, denoiser) lives.
|
|
@@ -23,9 +23,13 @@ export declare class SpatialAudioManager extends EventManager {
|
|
|
23
23
|
private monitoringIntervals;
|
|
24
24
|
private compressor;
|
|
25
25
|
private options;
|
|
26
|
-
private denoiseWorkletReady;
|
|
27
26
|
private denoiseWorkletUrl?;
|
|
28
27
|
private denoiserWasmBytes?;
|
|
28
|
+
private denoiseContextPromises;
|
|
29
|
+
private listenerPosition;
|
|
30
|
+
private listenerInitialized;
|
|
31
|
+
private stabilityState;
|
|
32
|
+
private outgoingProcessors;
|
|
29
33
|
private listenerDirection;
|
|
30
34
|
constructor(options?: SpatialAudioOptions);
|
|
31
35
|
getAudioContext(): AudioContext;
|
|
@@ -45,7 +49,9 @@ export declare class SpatialAudioManager extends EventManager {
|
|
|
45
49
|
* @param bypassSpatialization For testing - bypasses 3D positioning
|
|
46
50
|
*/
|
|
47
51
|
setupSpatialAudioForParticipant(participantId: string, track: MediaStreamTrack, bypassSpatialization?: boolean): Promise<void>;
|
|
52
|
+
enhanceOutgoingAudioTrack(track: MediaStreamTrack): Promise<MediaStreamTrack>;
|
|
48
53
|
private startMonitoring;
|
|
54
|
+
private handleTrackStability;
|
|
49
55
|
/**
|
|
50
56
|
* Update spatial audio position and orientation for a participant
|
|
51
57
|
*
|
|
@@ -85,12 +91,19 @@ export declare class SpatialAudioManager extends EventManager {
|
|
|
85
91
|
* @param lookAtPos Look-at position (where camera is pointing)
|
|
86
92
|
*/
|
|
87
93
|
setListenerFromLSD(listenerPos: Position, cameraPos: Position, lookAtPos: Position): void;
|
|
94
|
+
private applyListenerTransform;
|
|
88
95
|
removeParticipant(participantId: string): void;
|
|
89
96
|
resumeAudioContext(): Promise<void>;
|
|
90
97
|
getAudioContextState(): AudioContextState;
|
|
91
98
|
private getDistanceConfig;
|
|
99
|
+
private applySpatialBoostIfNeeded;
|
|
100
|
+
private getDistanceBetween;
|
|
101
|
+
private calculateDistanceGain;
|
|
92
102
|
private normalizePositionUnits;
|
|
93
103
|
private isDenoiserEnabled;
|
|
104
|
+
private applyHardwareNoiseConstraints;
|
|
105
|
+
private startOutboundMonitor;
|
|
106
|
+
private cleanupOutboundProcessor;
|
|
94
107
|
private ensureDenoiseWorklet;
|
|
95
108
|
private resolveOptions;
|
|
96
109
|
}
|
|
@@ -7,7 +7,11 @@ class SpatialAudioManager extends EventManager_1.EventManager {
|
|
|
7
7
|
super();
|
|
8
8
|
this.participantNodes = new Map();
|
|
9
9
|
this.monitoringIntervals = new Map();
|
|
10
|
-
this.
|
|
10
|
+
this.denoiseContextPromises = new WeakMap();
|
|
11
|
+
this.listenerPosition = { x: 0, y: 0, z: 0 };
|
|
12
|
+
this.listenerInitialized = false;
|
|
13
|
+
this.stabilityState = new Map();
|
|
14
|
+
this.outgoingProcessors = new Map();
|
|
11
15
|
this.listenerDirection = {
|
|
12
16
|
forward: { x: 0, y: 1, z: 0 },
|
|
13
17
|
up: { x: 0, y: 0, z: 1 },
|
|
@@ -67,6 +71,7 @@ class SpatialAudioManager extends EventManager_1.EventManager {
|
|
|
67
71
|
const panner = this.audioContext.createPanner();
|
|
68
72
|
const analyser = this.audioContext.createAnalyser();
|
|
69
73
|
const gain = this.audioContext.createGain();
|
|
74
|
+
const noiseGate = this.audioContext.createGain();
|
|
70
75
|
let denoiseNode;
|
|
71
76
|
if (this.isDenoiserEnabled() && typeof this.audioContext.audioWorklet !== "undefined") {
|
|
72
77
|
try {
|
|
@@ -100,6 +105,8 @@ class SpatialAudioManager extends EventManager_1.EventManager {
|
|
|
100
105
|
lowpassFilter.type = "lowpass";
|
|
101
106
|
lowpassFilter.frequency.value = 7500; // Below 8kHz to avoid flat/muffled sound
|
|
102
107
|
lowpassFilter.Q.value = 1.0; // Quality factor
|
|
108
|
+
// Adaptive noise gate defaults
|
|
109
|
+
noiseGate.gain.value = 1.0;
|
|
103
110
|
// Configure Panner for realistic 3D spatial audio
|
|
104
111
|
const distanceConfig = this.getDistanceConfig();
|
|
105
112
|
panner.panningModel = "HRTF"; // Head-Related Transfer Function for realistic 3D
|
|
@@ -119,15 +126,16 @@ class SpatialAudioManager extends EventManager_1.EventManager {
|
|
|
119
126
|
}
|
|
120
127
|
currentNode.connect(highpassFilter);
|
|
121
128
|
highpassFilter.connect(lowpassFilter);
|
|
129
|
+
lowpassFilter.connect(noiseGate);
|
|
122
130
|
if (bypassSpatialization) {
|
|
123
131
|
console.log(`π TESTING: Connecting audio directly to destination (bypassing spatial audio) for ${participantId}`);
|
|
124
|
-
|
|
132
|
+
noiseGate.connect(analyser);
|
|
125
133
|
analyser.connect(this.masterGainNode);
|
|
126
134
|
}
|
|
127
135
|
else {
|
|
128
136
|
// Standard spatialized path with full audio chain
|
|
129
|
-
// Audio Chain: source -> filters -> panner -> analyser -> gain -> masterGain -> compressor -> destination
|
|
130
|
-
|
|
137
|
+
// Audio Chain: source -> filters -> noiseGate -> panner -> analyser -> gain -> masterGain -> compressor -> destination
|
|
138
|
+
noiseGate.connect(panner);
|
|
131
139
|
panner.connect(analyser);
|
|
132
140
|
analyser.connect(gain);
|
|
133
141
|
gain.connect(this.masterGainNode);
|
|
@@ -137,11 +145,21 @@ class SpatialAudioManager extends EventManager_1.EventManager {
|
|
|
137
145
|
panner,
|
|
138
146
|
analyser,
|
|
139
147
|
gain,
|
|
148
|
+
noiseGate,
|
|
140
149
|
highpassFilter,
|
|
141
150
|
lowpassFilter,
|
|
142
151
|
denoiseNode,
|
|
143
152
|
stream,
|
|
144
153
|
});
|
|
154
|
+
this.stabilityState.set(participantId, {
|
|
155
|
+
smoothedLevel: 0,
|
|
156
|
+
targetGain: 1,
|
|
157
|
+
networkMuted: false,
|
|
158
|
+
});
|
|
159
|
+
if (typeof track.onmute !== "undefined") {
|
|
160
|
+
track.onmute = () => this.handleTrackStability(participantId, true);
|
|
161
|
+
track.onunmute = () => this.handleTrackStability(participantId, false);
|
|
162
|
+
}
|
|
145
163
|
console.log(`π§ Spatial audio setup complete for ${participantId}:`, {
|
|
146
164
|
audioContextState: this.audioContext.state,
|
|
147
165
|
sampleRate: this.audioContext.sampleRate,
|
|
@@ -161,12 +179,138 @@ class SpatialAudioManager extends EventManager_1.EventManager {
|
|
|
161
179
|
// Start monitoring audio levels
|
|
162
180
|
this.startMonitoring(participantId);
|
|
163
181
|
}
|
|
182
|
+
async enhanceOutgoingAudioTrack(track) {
|
|
183
|
+
if (track.kind !== "audio") {
|
|
184
|
+
return track;
|
|
185
|
+
}
|
|
186
|
+
const existingProcessor = Array.from(this.outgoingProcessors.values()).find((processor) => processor.originalTrack === track);
|
|
187
|
+
if (existingProcessor) {
|
|
188
|
+
return existingProcessor.processedTrack;
|
|
189
|
+
}
|
|
190
|
+
await this.applyHardwareNoiseConstraints(track);
|
|
191
|
+
const context = new AudioContext({ sampleRate: 48000 });
|
|
192
|
+
await context.resume();
|
|
193
|
+
const sourceStream = new MediaStream([track]);
|
|
194
|
+
const source = context.createMediaStreamSource(sourceStream);
|
|
195
|
+
let current = source;
|
|
196
|
+
let denoiseNode;
|
|
197
|
+
if (this.isDenoiserEnabled() && typeof context.audioWorklet !== "undefined") {
|
|
198
|
+
try {
|
|
199
|
+
await this.ensureDenoiseWorklet(context);
|
|
200
|
+
denoiseNode = new AudioWorkletNode(context, "odyssey-denoise", {
|
|
201
|
+
numberOfInputs: 1,
|
|
202
|
+
numberOfOutputs: 1,
|
|
203
|
+
processorOptions: {
|
|
204
|
+
enabled: true,
|
|
205
|
+
threshold: this.options.denoiser?.threshold,
|
|
206
|
+
noiseFloor: this.options.denoiser?.noiseFloor,
|
|
207
|
+
release: this.options.denoiser?.release,
|
|
208
|
+
wasmBytes: this.denoiserWasmBytes
|
|
209
|
+
? this.denoiserWasmBytes.slice(0)
|
|
210
|
+
: null,
|
|
211
|
+
},
|
|
212
|
+
});
|
|
213
|
+
current.connect(denoiseNode);
|
|
214
|
+
current = denoiseNode;
|
|
215
|
+
}
|
|
216
|
+
catch (error) {
|
|
217
|
+
console.warn("β οΈ Outgoing denoiser unavailable, continuing without it.", error);
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
const notch60 = context.createBiquadFilter();
|
|
221
|
+
notch60.type = "notch";
|
|
222
|
+
notch60.frequency.value = 60;
|
|
223
|
+
notch60.Q.value = 24;
|
|
224
|
+
current.connect(notch60);
|
|
225
|
+
current = notch60;
|
|
226
|
+
const notch50 = context.createBiquadFilter();
|
|
227
|
+
notch50.type = "notch";
|
|
228
|
+
notch50.frequency.value = 50;
|
|
229
|
+
notch50.Q.value = 24;
|
|
230
|
+
current.connect(notch50);
|
|
231
|
+
current = notch50;
|
|
232
|
+
const lowShelf = context.createBiquadFilter();
|
|
233
|
+
lowShelf.type = "lowshelf";
|
|
234
|
+
lowShelf.frequency.value = 120;
|
|
235
|
+
lowShelf.gain.value = -3;
|
|
236
|
+
current.connect(lowShelf);
|
|
237
|
+
current = lowShelf;
|
|
238
|
+
const highpassFilter = context.createBiquadFilter();
|
|
239
|
+
highpassFilter.type = "highpass";
|
|
240
|
+
highpassFilter.frequency.value = 95;
|
|
241
|
+
highpassFilter.Q.value = 0.8;
|
|
242
|
+
current.connect(highpassFilter);
|
|
243
|
+
current = highpassFilter;
|
|
244
|
+
const lowpassFilter = context.createBiquadFilter();
|
|
245
|
+
lowpassFilter.type = "lowpass";
|
|
246
|
+
lowpassFilter.frequency.value = 7200;
|
|
247
|
+
lowpassFilter.Q.value = 0.8;
|
|
248
|
+
current.connect(lowpassFilter);
|
|
249
|
+
current = lowpassFilter;
|
|
250
|
+
const hissShelf = context.createBiquadFilter();
|
|
251
|
+
hissShelf.type = "highshelf";
|
|
252
|
+
hissShelf.frequency.value = 6400;
|
|
253
|
+
hissShelf.gain.value = -4;
|
|
254
|
+
current.connect(hissShelf);
|
|
255
|
+
current = hissShelf;
|
|
256
|
+
const presenceBoost = context.createBiquadFilter();
|
|
257
|
+
presenceBoost.type = "peaking";
|
|
258
|
+
presenceBoost.frequency.value = 2400;
|
|
259
|
+
presenceBoost.Q.value = 1.1;
|
|
260
|
+
presenceBoost.gain.value = 2.4;
|
|
261
|
+
current.connect(presenceBoost);
|
|
262
|
+
current = presenceBoost;
|
|
263
|
+
const compressor = context.createDynamicsCompressor();
|
|
264
|
+
compressor.threshold.value = -18;
|
|
265
|
+
compressor.knee.value = 16;
|
|
266
|
+
compressor.ratio.value = 3.2;
|
|
267
|
+
compressor.attack.value = 0.002;
|
|
268
|
+
compressor.release.value = 0.22;
|
|
269
|
+
current.connect(compressor);
|
|
270
|
+
current = compressor;
|
|
271
|
+
const postCompressorTap = context.createGain();
|
|
272
|
+
postCompressorTap.gain.value = 1.05;
|
|
273
|
+
current.connect(postCompressorTap);
|
|
274
|
+
current = postCompressorTap;
|
|
275
|
+
const analyser = context.createAnalyser();
|
|
276
|
+
analyser.fftSize = 512;
|
|
277
|
+
current.connect(analyser);
|
|
278
|
+
const gate = context.createGain();
|
|
279
|
+
gate.gain.value = 1;
|
|
280
|
+
current.connect(gate);
|
|
281
|
+
const destination = context.createMediaStreamDestination();
|
|
282
|
+
gate.connect(destination);
|
|
283
|
+
const processedTrack = destination.stream.getAudioTracks()[0];
|
|
284
|
+
processedTrack.contentHint = "speech";
|
|
285
|
+
const processorId = processedTrack.id;
|
|
286
|
+
const monitor = this.startOutboundMonitor(processorId, analyser, gate);
|
|
287
|
+
const cleanup = () => this.cleanupOutboundProcessor(processorId);
|
|
288
|
+
processedTrack.addEventListener("ended", cleanup);
|
|
289
|
+
track.addEventListener("ended", cleanup);
|
|
290
|
+
this.outgoingProcessors.set(processorId, {
|
|
291
|
+
context,
|
|
292
|
+
sourceStream,
|
|
293
|
+
destinationStream: destination.stream,
|
|
294
|
+
analyser,
|
|
295
|
+
gate,
|
|
296
|
+
monitor,
|
|
297
|
+
originalTrack: track,
|
|
298
|
+
processedTrack,
|
|
299
|
+
cleanupListener: cleanup,
|
|
300
|
+
});
|
|
301
|
+
console.log("ποΈ [SDK] Outgoing audio tuned", {
|
|
302
|
+
originalTrackId: track.id,
|
|
303
|
+
processedTrackId: processedTrack.id,
|
|
304
|
+
});
|
|
305
|
+
return processedTrack;
|
|
306
|
+
}
|
|
164
307
|
startMonitoring(participantId) {
|
|
165
308
|
const nodes = this.participantNodes.get(participantId);
|
|
166
309
|
if (!nodes)
|
|
167
310
|
return;
|
|
168
|
-
const { analyser, stream } = nodes;
|
|
311
|
+
const { analyser, stream, noiseGate } = nodes;
|
|
169
312
|
const dataArray = new Uint8Array(analyser.frequencyBinCount);
|
|
313
|
+
let lastTrackLog = 0;
|
|
170
314
|
// Clear any existing interval for this participant
|
|
171
315
|
if (this.monitoringIntervals.has(participantId)) {
|
|
172
316
|
clearInterval(this.monitoringIntervals.get(participantId));
|
|
@@ -179,16 +323,47 @@ class SpatialAudioManager extends EventManager_1.EventManager {
|
|
|
179
323
|
}
|
|
180
324
|
const average = sum / dataArray.length;
|
|
181
325
|
const audioLevel = (average / 128) * 255; // Scale to 0-255
|
|
182
|
-
|
|
183
|
-
|
|
326
|
+
const normalizedLevel = audioLevel / 255;
|
|
327
|
+
const stability = this.stabilityState.get(participantId);
|
|
328
|
+
if (stability) {
|
|
329
|
+
const smoothing = 0.2;
|
|
330
|
+
stability.smoothedLevel =
|
|
331
|
+
stability.smoothedLevel * (1 - smoothing) + normalizedLevel * smoothing;
|
|
332
|
+
const gateOpenThreshold = 0.035; // empirical speech/noise split
|
|
333
|
+
const gateCloseThreshold = 0.015;
|
|
334
|
+
let targetGain = stability.targetGain;
|
|
335
|
+
if (stability.networkMuted) {
|
|
336
|
+
targetGain = 0;
|
|
337
|
+
}
|
|
338
|
+
else if (stability.smoothedLevel < gateCloseThreshold) {
|
|
339
|
+
targetGain = 0;
|
|
340
|
+
}
|
|
341
|
+
else if (stability.smoothedLevel < gateOpenThreshold) {
|
|
342
|
+
targetGain = 0.35;
|
|
343
|
+
}
|
|
344
|
+
else {
|
|
345
|
+
targetGain = 1;
|
|
346
|
+
}
|
|
347
|
+
if (Math.abs(targetGain - stability.targetGain) > 0.05) {
|
|
348
|
+
const ramp = targetGain > stability.targetGain ? 0.03 : 0.12;
|
|
349
|
+
noiseGate.gain.setTargetAtTime(targetGain, this.audioContext.currentTime, ramp);
|
|
350
|
+
stability.targetGain = targetGain;
|
|
351
|
+
}
|
|
352
|
+
if (Math.random() < 0.05) {
|
|
353
|
+
console.log(`ποΈ [NoiseGate] ${participantId}`, {
|
|
354
|
+
level: stability.smoothedLevel.toFixed(3),
|
|
355
|
+
gain: stability.targetGain.toFixed(2),
|
|
356
|
+
});
|
|
357
|
+
}
|
|
358
|
+
}
|
|
359
|
+
if (audioLevel < 1.0 && Math.random() < 0.2) {
|
|
184
360
|
console.warn(`β οΈ NO AUDIO DATA detected for ${participantId}! Track may be silent or not transmitting.`);
|
|
185
|
-
console.info(`π‘ Check: 1) Is microphone unmuted? 2) Is correct mic selected? 3) Is mic working in system settings?`);
|
|
186
361
|
}
|
|
187
|
-
|
|
188
|
-
|
|
362
|
+
if (Date.now() - lastTrackLog > 2000) {
|
|
363
|
+
lastTrackLog = Date.now();
|
|
189
364
|
const track = stream.getAudioTracks()[0];
|
|
190
365
|
if (track) {
|
|
191
|
-
console.log(`π Audio track status
|
|
366
|
+
console.log(`π Audio track status for ${participantId}:`, {
|
|
192
367
|
trackEnabled: track.enabled,
|
|
193
368
|
trackMuted: track.muted,
|
|
194
369
|
trackReadyState: track.readyState,
|
|
@@ -200,10 +375,20 @@ class SpatialAudioManager extends EventManager_1.EventManager {
|
|
|
200
375
|
},
|
|
201
376
|
});
|
|
202
377
|
}
|
|
203
|
-
}
|
|
204
|
-
},
|
|
378
|
+
}
|
|
379
|
+
}, 250); // Adaptive monitoring ~4x per second
|
|
205
380
|
this.monitoringIntervals.set(participantId, interval);
|
|
206
381
|
}
|
|
382
|
+
handleTrackStability(participantId, muted) {
|
|
383
|
+
const nodes = this.participantNodes.get(participantId);
|
|
384
|
+
if (!nodes)
|
|
385
|
+
return;
|
|
386
|
+
const stability = this.stabilityState.get(participantId);
|
|
387
|
+
if (stability) {
|
|
388
|
+
stability.networkMuted = muted;
|
|
389
|
+
}
|
|
390
|
+
nodes.noiseGate.gain.setTargetAtTime(muted ? 0 : 1, this.audioContext.currentTime, muted ? 0.05 : 0.2);
|
|
391
|
+
}
|
|
207
392
|
/**
|
|
208
393
|
* Update spatial audio position and orientation for a participant
|
|
209
394
|
*
|
|
@@ -227,11 +412,13 @@ class SpatialAudioManager extends EventManager_1.EventManager {
|
|
|
227
412
|
updateSpatialAudio(participantId, position, direction) {
|
|
228
413
|
const nodes = this.participantNodes.get(participantId);
|
|
229
414
|
if (nodes?.panner) {
|
|
415
|
+
const distanceConfig = this.getDistanceConfig();
|
|
230
416
|
const normalizedPosition = this.normalizePositionUnits(position);
|
|
417
|
+
const targetPosition = this.applySpatialBoostIfNeeded(normalizedPosition);
|
|
231
418
|
// Update position (where the sound is coming from)
|
|
232
|
-
nodes.panner.positionX.setValueAtTime(
|
|
233
|
-
nodes.panner.positionY.setValueAtTime(
|
|
234
|
-
nodes.panner.positionZ.setValueAtTime(
|
|
419
|
+
nodes.panner.positionX.setValueAtTime(targetPosition.x, this.audioContext.currentTime);
|
|
420
|
+
nodes.panner.positionY.setValueAtTime(targetPosition.y, this.audioContext.currentTime);
|
|
421
|
+
nodes.panner.positionZ.setValueAtTime(targetPosition.z, this.audioContext.currentTime);
|
|
235
422
|
// Update orientation (where the participant is facing)
|
|
236
423
|
// This makes the audio source directional based on participant's direction
|
|
237
424
|
if (direction) {
|
|
@@ -248,57 +435,23 @@ class SpatialAudioManager extends EventManager_1.EventManager {
|
|
|
248
435
|
nodes.panner.orientationZ.setValueAtTime(normZ, this.audioContext.currentTime);
|
|
249
436
|
}
|
|
250
437
|
}
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
x: orientation.forwardX,
|
|
261
|
-
y: orientation.forwardY,
|
|
262
|
-
z: orientation.forwardZ,
|
|
263
|
-
},
|
|
264
|
-
up: {
|
|
265
|
-
x: orientation.upX,
|
|
266
|
-
y: orientation.upY,
|
|
267
|
-
z: orientation.upZ,
|
|
268
|
-
},
|
|
269
|
-
};
|
|
270
|
-
// Use setPosition and setOrientation for atomic updates if available
|
|
271
|
-
if (listener.positionX) {
|
|
272
|
-
listener.positionX.setValueAtTime(normalizedPosition.x, this.audioContext.currentTime);
|
|
273
|
-
listener.positionY.setValueAtTime(normalizedPosition.y, this.audioContext.currentTime);
|
|
274
|
-
listener.positionZ.setValueAtTime(normalizedPosition.z, this.audioContext.currentTime);
|
|
275
|
-
}
|
|
276
|
-
if (listener.forwardX) {
|
|
277
|
-
listener.forwardX.setValueAtTime(orientation.forwardX, this.audioContext.currentTime);
|
|
278
|
-
listener.forwardY.setValueAtTime(orientation.forwardY, this.audioContext.currentTime);
|
|
279
|
-
listener.forwardZ.setValueAtTime(orientation.forwardZ, this.audioContext.currentTime);
|
|
280
|
-
listener.upX.setValueAtTime(orientation.upX, this.audioContext.currentTime);
|
|
281
|
-
listener.upY.setValueAtTime(orientation.upY, this.audioContext.currentTime);
|
|
282
|
-
listener.upZ.setValueAtTime(orientation.upZ, this.audioContext.currentTime);
|
|
283
|
-
}
|
|
284
|
-
// Log spatial audio updates occasionally
|
|
285
|
-
if (Math.random() < 0.01) {
|
|
286
|
-
console.log(`π§ [Spatial Audio] Listener updated:`, {
|
|
287
|
-
position: { x: position.x.toFixed(1), y: position.y.toFixed(1), z: position.z.toFixed(1) },
|
|
288
|
-
forward: {
|
|
289
|
-
x: orientation.forwardX.toFixed(2),
|
|
290
|
-
y: orientation.forwardY.toFixed(2),
|
|
291
|
-
z: orientation.forwardZ.toFixed(2),
|
|
292
|
-
},
|
|
293
|
-
up: {
|
|
294
|
-
x: orientation.upX.toFixed(2),
|
|
295
|
-
y: orientation.upY.toFixed(2),
|
|
296
|
-
z: orientation.upZ.toFixed(2),
|
|
297
|
-
},
|
|
438
|
+
const listenerPos = this.listenerPosition;
|
|
439
|
+
const distance = this.getDistanceBetween(listenerPos, targetPosition);
|
|
440
|
+
const distanceGain = this.calculateDistanceGain(distanceConfig, distance);
|
|
441
|
+
nodes.gain.gain.setTargetAtTime(distanceGain, this.audioContext.currentTime, 0.05);
|
|
442
|
+
if (Math.random() < 0.02) {
|
|
443
|
+
console.log("ποΈ [Spatial Audio] Distance gain", {
|
|
444
|
+
participantId,
|
|
445
|
+
distance: distance.toFixed(2),
|
|
446
|
+
gain: distanceGain.toFixed(2),
|
|
298
447
|
});
|
|
299
448
|
}
|
|
300
449
|
}
|
|
301
450
|
}
|
|
451
|
+
setListenerPosition(position, orientation) {
|
|
452
|
+
const normalizedPosition = this.normalizePositionUnits(position);
|
|
453
|
+
this.applyListenerTransform(normalizedPosition, orientation);
|
|
454
|
+
}
|
|
302
455
|
/**
|
|
303
456
|
* Update listener orientation from LSD camera direction
|
|
304
457
|
* @param cameraPos Camera position in world space
|
|
@@ -329,7 +482,7 @@ class SpatialAudioManager extends EventManager_1.EventManager {
|
|
|
329
482
|
const rightLen = Math.sqrt(rightX * rightX + rightY * rightY + rightZ * rightZ);
|
|
330
483
|
if (rightLen < 0.001) {
|
|
331
484
|
// Forward is parallel to world up, use fallback
|
|
332
|
-
this.
|
|
485
|
+
this.applyListenerTransform(normalizedListener, {
|
|
333
486
|
forwardX: fwdX,
|
|
334
487
|
forwardY: fwdY,
|
|
335
488
|
forwardZ: fwdZ,
|
|
@@ -346,7 +499,7 @@ class SpatialAudioManager extends EventManager_1.EventManager {
|
|
|
346
499
|
const upX = fwdY * rZ - fwdZ * rY;
|
|
347
500
|
const upY = fwdZ * rX - fwdX * rZ;
|
|
348
501
|
const upZ = fwdX * rY - fwdY * rX;
|
|
349
|
-
this.
|
|
502
|
+
this.applyListenerTransform(normalizedListener, {
|
|
350
503
|
forwardX: fwdX,
|
|
351
504
|
forwardY: fwdY,
|
|
352
505
|
forwardZ: fwdZ,
|
|
@@ -355,6 +508,58 @@ class SpatialAudioManager extends EventManager_1.EventManager {
|
|
|
355
508
|
upZ,
|
|
356
509
|
});
|
|
357
510
|
}
|
|
511
|
+
applyListenerTransform(normalizedPosition, orientation) {
|
|
512
|
+
const { listener } = this.audioContext;
|
|
513
|
+
if (!listener) {
|
|
514
|
+
return;
|
|
515
|
+
}
|
|
516
|
+
this.listenerPosition = { ...normalizedPosition };
|
|
517
|
+
this.listenerInitialized = true;
|
|
518
|
+
this.listenerDirection = {
|
|
519
|
+
forward: {
|
|
520
|
+
x: orientation.forwardX,
|
|
521
|
+
y: orientation.forwardY,
|
|
522
|
+
z: orientation.forwardZ,
|
|
523
|
+
},
|
|
524
|
+
up: {
|
|
525
|
+
x: orientation.upX,
|
|
526
|
+
y: orientation.upY,
|
|
527
|
+
z: orientation.upZ,
|
|
528
|
+
},
|
|
529
|
+
};
|
|
530
|
+
if (listener.positionX) {
|
|
531
|
+
listener.positionX.setValueAtTime(normalizedPosition.x, this.audioContext.currentTime);
|
|
532
|
+
listener.positionY.setValueAtTime(normalizedPosition.y, this.audioContext.currentTime);
|
|
533
|
+
listener.positionZ.setValueAtTime(normalizedPosition.z, this.audioContext.currentTime);
|
|
534
|
+
}
|
|
535
|
+
if (listener.forwardX) {
|
|
536
|
+
listener.forwardX.setValueAtTime(orientation.forwardX, this.audioContext.currentTime);
|
|
537
|
+
listener.forwardY.setValueAtTime(orientation.forwardY, this.audioContext.currentTime);
|
|
538
|
+
listener.forwardZ.setValueAtTime(orientation.forwardZ, this.audioContext.currentTime);
|
|
539
|
+
listener.upX.setValueAtTime(orientation.upX, this.audioContext.currentTime);
|
|
540
|
+
listener.upY.setValueAtTime(orientation.upY, this.audioContext.currentTime);
|
|
541
|
+
listener.upZ.setValueAtTime(orientation.upZ, this.audioContext.currentTime);
|
|
542
|
+
}
|
|
543
|
+
if (Math.random() < 0.01) {
|
|
544
|
+
console.log(`π§ [Spatial Audio] Listener updated:`, {
|
|
545
|
+
position: {
|
|
546
|
+
x: normalizedPosition.x.toFixed(2),
|
|
547
|
+
y: normalizedPosition.y.toFixed(2),
|
|
548
|
+
z: normalizedPosition.z.toFixed(2),
|
|
549
|
+
},
|
|
550
|
+
forward: {
|
|
551
|
+
x: orientation.forwardX.toFixed(2),
|
|
552
|
+
y: orientation.forwardY.toFixed(2),
|
|
553
|
+
z: orientation.forwardZ.toFixed(2),
|
|
554
|
+
},
|
|
555
|
+
up: {
|
|
556
|
+
x: orientation.upX.toFixed(2),
|
|
557
|
+
y: orientation.upY.toFixed(2),
|
|
558
|
+
z: orientation.upZ.toFixed(2),
|
|
559
|
+
},
|
|
560
|
+
});
|
|
561
|
+
}
|
|
562
|
+
}
|
|
358
563
|
removeParticipant(participantId) {
|
|
359
564
|
// Stop monitoring
|
|
360
565
|
if (this.monitoringIntervals.has(participantId)) {
|
|
@@ -367,11 +572,18 @@ class SpatialAudioManager extends EventManager_1.EventManager {
|
|
|
367
572
|
nodes.panner.disconnect();
|
|
368
573
|
nodes.analyser.disconnect();
|
|
369
574
|
nodes.gain.disconnect();
|
|
575
|
+
nodes.noiseGate.disconnect();
|
|
370
576
|
if (nodes.denoiseNode) {
|
|
371
577
|
nodes.denoiseNode.disconnect();
|
|
372
578
|
}
|
|
579
|
+
const track = nodes.stream.getAudioTracks()[0];
|
|
580
|
+
if (track) {
|
|
581
|
+
track.onmute = null;
|
|
582
|
+
track.onunmute = null;
|
|
583
|
+
}
|
|
373
584
|
nodes.stream.getTracks().forEach((track) => track.stop());
|
|
374
585
|
this.participantNodes.delete(participantId);
|
|
586
|
+
this.stabilityState.delete(participantId);
|
|
375
587
|
console.log(`ποΈ Removed participant ${participantId} from spatial audio.`);
|
|
376
588
|
}
|
|
377
589
|
}
|
|
@@ -392,6 +604,42 @@ class SpatialAudioManager extends EventManager_1.EventManager {
|
|
|
392
604
|
unit: this.options.distance?.unit ?? "auto",
|
|
393
605
|
};
|
|
394
606
|
}
|
|
607
|
+
applySpatialBoostIfNeeded(position) {
|
|
608
|
+
if (!this.listenerInitialized) {
|
|
609
|
+
return position;
|
|
610
|
+
}
|
|
611
|
+
const boost = (this.options.distance?.rolloffFactor || 1) * 0.85;
|
|
612
|
+
if (!isFinite(boost) || boost <= 1.01) {
|
|
613
|
+
return position;
|
|
614
|
+
}
|
|
615
|
+
const listener = this.listenerPosition;
|
|
616
|
+
return {
|
|
617
|
+
x: listener.x + (position.x - listener.x) * boost,
|
|
618
|
+
y: listener.y + (position.y - listener.y) * Math.min(boost, 1.2),
|
|
619
|
+
z: listener.z + (position.z - listener.z) * boost,
|
|
620
|
+
};
|
|
621
|
+
}
|
|
622
|
+
getDistanceBetween(a, b) {
|
|
623
|
+
const dx = b.x - a.x;
|
|
624
|
+
const dy = b.y - a.y;
|
|
625
|
+
const dz = b.z - a.z;
|
|
626
|
+
return Math.sqrt(dx * dx + dy * dy + dz * dz);
|
|
627
|
+
}
|
|
628
|
+
calculateDistanceGain(config, distance) {
|
|
629
|
+
if (!this.listenerInitialized) {
|
|
630
|
+
return 1;
|
|
631
|
+
}
|
|
632
|
+
if (distance <= config.refDistance) {
|
|
633
|
+
return 1;
|
|
634
|
+
}
|
|
635
|
+
if (distance >= config.maxDistance) {
|
|
636
|
+
return 0;
|
|
637
|
+
}
|
|
638
|
+
const normalized = (distance - config.refDistance) /
|
|
639
|
+
Math.max(config.maxDistance - config.refDistance, 0.001);
|
|
640
|
+
const shaped = Math.pow(Math.max(0, 1 - normalized), Math.max(1.2, config.rolloffFactor * 1.05));
|
|
641
|
+
return Math.min(1, Math.max(0.01, shaped));
|
|
642
|
+
}
|
|
395
643
|
normalizePositionUnits(position) {
|
|
396
644
|
const distanceConfig = this.getDistanceConfig();
|
|
397
645
|
if (distanceConfig.unit === "meters") {
|
|
@@ -418,11 +666,85 @@ class SpatialAudioManager extends EventManager_1.EventManager {
|
|
|
418
666
|
isDenoiserEnabled() {
|
|
419
667
|
return this.options.denoiser?.enabled !== false;
|
|
420
668
|
}
|
|
421
|
-
async
|
|
669
|
+
async applyHardwareNoiseConstraints(track) {
|
|
670
|
+
try {
|
|
671
|
+
await track.applyConstraints({
|
|
672
|
+
echoCancellation: true,
|
|
673
|
+
noiseSuppression: true,
|
|
674
|
+
autoGainControl: true,
|
|
675
|
+
advanced: [
|
|
676
|
+
{
|
|
677
|
+
echoCancellation: true,
|
|
678
|
+
noiseSuppression: true,
|
|
679
|
+
autoGainControl: true,
|
|
680
|
+
googEchoCancellation: true,
|
|
681
|
+
googNoiseSuppression: true,
|
|
682
|
+
googAutoGainControl: true,
|
|
683
|
+
googHighpassFilter: true,
|
|
684
|
+
googTypingNoiseDetection: true,
|
|
685
|
+
},
|
|
686
|
+
],
|
|
687
|
+
});
|
|
688
|
+
}
|
|
689
|
+
catch (error) {
|
|
690
|
+
console.warn("β οΈ Unable to apply hardware audio constraints", error);
|
|
691
|
+
}
|
|
692
|
+
track.contentHint = "speech";
|
|
693
|
+
}
|
|
694
|
+
startOutboundMonitor(processorId, analyser, gate) {
|
|
695
|
+
const dataArray = new Uint8Array(analyser.fftSize);
|
|
696
|
+
let smoothedLevel = 0;
|
|
697
|
+
return setInterval(() => {
|
|
698
|
+
analyser.getByteTimeDomainData(dataArray);
|
|
699
|
+
let sum = 0;
|
|
700
|
+
for (const value of dataArray) {
|
|
701
|
+
sum += Math.abs(value - 128);
|
|
702
|
+
}
|
|
703
|
+
const level = (sum / dataArray.length) / 128;
|
|
704
|
+
smoothedLevel = smoothedLevel * 0.7 + level * 0.3;
|
|
705
|
+
let targetGain = 1;
|
|
706
|
+
if (smoothedLevel < 0.02) {
|
|
707
|
+
targetGain = 0;
|
|
708
|
+
}
|
|
709
|
+
else if (smoothedLevel < 0.05) {
|
|
710
|
+
targetGain = 0.45;
|
|
711
|
+
}
|
|
712
|
+
else {
|
|
713
|
+
targetGain = 1;
|
|
714
|
+
}
|
|
715
|
+
gate.gain.setTargetAtTime(targetGain, gate.context.currentTime, targetGain > gate.gain.value ? 0.02 : 0.08);
|
|
716
|
+
if (Math.random() < 0.03) {
|
|
717
|
+
console.log("ποΈ [SDK] Outgoing gate", {
|
|
718
|
+
processorId,
|
|
719
|
+
level: smoothedLevel.toFixed(3),
|
|
720
|
+
gain: targetGain.toFixed(2),
|
|
721
|
+
});
|
|
722
|
+
}
|
|
723
|
+
}, 200);
|
|
724
|
+
}
|
|
725
|
+
cleanupOutboundProcessor(processorId) {
|
|
726
|
+
const processor = this.outgoingProcessors.get(processorId);
|
|
727
|
+
if (!processor)
|
|
728
|
+
return;
|
|
729
|
+
clearInterval(processor.monitor);
|
|
730
|
+
processor.processedTrack.removeEventListener("ended", processor.cleanupListener);
|
|
731
|
+
processor.originalTrack.removeEventListener("ended", processor.cleanupListener);
|
|
732
|
+
try {
|
|
733
|
+
processor.originalTrack.stop();
|
|
734
|
+
}
|
|
735
|
+
catch (error) {
|
|
736
|
+
console.warn("β οΈ Unable to stop original track during cleanup", error);
|
|
737
|
+
}
|
|
738
|
+
processor.destinationStream.getTracks().forEach((t) => t.stop());
|
|
739
|
+
processor.sourceStream.getTracks().forEach((t) => t.stop());
|
|
740
|
+
processor.context.close();
|
|
741
|
+
this.outgoingProcessors.delete(processorId);
|
|
742
|
+
}
|
|
743
|
+
async ensureDenoiseWorklet(targetContext = this.audioContext) {
|
|
422
744
|
if (!this.isDenoiserEnabled()) {
|
|
423
745
|
return;
|
|
424
746
|
}
|
|
425
|
-
if (!("audioWorklet" in
|
|
747
|
+
if (!("audioWorklet" in targetContext)) {
|
|
426
748
|
console.warn("β οΈ AudioWorklet not supported in this browser. Disabling denoiser.");
|
|
427
749
|
this.options.denoiser = {
|
|
428
750
|
...(this.options.denoiser || {}),
|
|
@@ -430,8 +752,9 @@ class SpatialAudioManager extends EventManager_1.EventManager {
|
|
|
430
752
|
};
|
|
431
753
|
return;
|
|
432
754
|
}
|
|
433
|
-
|
|
434
|
-
|
|
755
|
+
const existingPromise = this.denoiseContextPromises.get(targetContext);
|
|
756
|
+
if (existingPromise) {
|
|
757
|
+
return existingPromise;
|
|
435
758
|
}
|
|
436
759
|
const processorSource = `class OdysseyDenoiseProcessor extends AudioWorkletProcessor {
|
|
437
760
|
constructor(options) {
|
|
@@ -488,11 +811,13 @@ class SpatialAudioManager extends EventManager_1.EventManager {
|
|
|
488
811
|
|
|
489
812
|
registerProcessor('odyssey-denoise', OdysseyDenoiseProcessor);
|
|
490
813
|
`;
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
814
|
+
if (!this.denoiseWorkletUrl) {
|
|
815
|
+
const blob = new Blob([processorSource], {
|
|
816
|
+
type: "application/javascript",
|
|
817
|
+
});
|
|
818
|
+
this.denoiseWorkletUrl = URL.createObjectURL(blob);
|
|
819
|
+
}
|
|
820
|
+
const promise = targetContext.audioWorklet
|
|
496
821
|
.addModule(this.denoiseWorkletUrl)
|
|
497
822
|
.catch((error) => {
|
|
498
823
|
console.error("β Failed to register denoise worklet", error);
|
|
@@ -502,7 +827,8 @@ registerProcessor('odyssey-denoise', OdysseyDenoiseProcessor);
|
|
|
502
827
|
};
|
|
503
828
|
throw error;
|
|
504
829
|
});
|
|
505
|
-
|
|
830
|
+
this.denoiseContextPromises.set(targetContext, promise);
|
|
831
|
+
return promise;
|
|
506
832
|
}
|
|
507
833
|
resolveOptions(options) {
|
|
508
834
|
const distanceDefaults = {
|
package/dist/index.d.ts
CHANGED
|
@@ -26,6 +26,7 @@ export declare class OdysseySpatialComms extends EventManager {
|
|
|
26
26
|
}): Promise<Participant>;
|
|
27
27
|
leaveRoom(): void;
|
|
28
28
|
resumeAudio(): Promise<void>;
|
|
29
|
+
enhanceOutgoingAudioTrack(track: MediaStreamTrack): Promise<MediaStreamTrack>;
|
|
29
30
|
getAudioContextState(): AudioContextState;
|
|
30
31
|
produceTrack(track: MediaStreamTrack): Promise<any>;
|
|
31
32
|
updatePosition(position: Position, direction: Direction, spatialData?: {
|
package/dist/index.js
CHANGED
|
@@ -121,6 +121,9 @@ class OdysseySpatialComms extends EventManager_1.EventManager {
|
|
|
121
121
|
async resumeAudio() {
|
|
122
122
|
await this.spatialAudioManager.resumeAudioContext();
|
|
123
123
|
}
|
|
124
|
+
async enhanceOutgoingAudioTrack(track) {
|
|
125
|
+
return this.spatialAudioManager.enhanceOutgoingAudioTrack(track);
|
|
126
|
+
}
|
|
124
127
|
getAudioContextState() {
|
|
125
128
|
return this.spatialAudioManager.getAudioContextState();
|
|
126
129
|
}
|
package/package.json
CHANGED