@newgameplusinc/odyssey-audio-video-sdk-dev 1.0.258 → 1.0.259
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +45 -1
- package/dist/audio/AudioNodeFactory.d.ts +130 -0
- package/dist/audio/AudioNodeFactory.js +158 -0
- package/dist/audio/AudioPipeline.d.ts +89 -0
- package/dist/audio/AudioPipeline.js +138 -0
- package/dist/{MLNoiseSuppressor.d.ts → audio/MLNoiseSuppressor.d.ts} +7 -7
- package/dist/{MLNoiseSuppressor.js → audio/MLNoiseSuppressor.js} +13 -41
- package/dist/audio/index.d.ts +6 -0
- package/dist/audio/index.js +22 -0
- package/dist/channels/huddle/HuddleChannel.d.ts +87 -0
- package/dist/channels/huddle/HuddleChannel.js +152 -0
- package/dist/channels/huddle/HuddleTypes.d.ts +85 -0
- package/dist/channels/huddle/HuddleTypes.js +25 -0
- package/dist/channels/huddle/index.d.ts +5 -0
- package/dist/channels/huddle/index.js +21 -0
- package/dist/channels/index.d.ts +5 -0
- package/dist/channels/index.js +21 -0
- package/dist/channels/spatial/SpatialAudioChannel.d.ts +144 -0
- package/dist/channels/spatial/SpatialAudioChannel.js +455 -0
- package/dist/channels/spatial/SpatialAudioTypes.d.ts +85 -0
- package/dist/channels/spatial/SpatialAudioTypes.js +42 -0
- package/dist/channels/spatial/index.d.ts +5 -0
- package/dist/channels/spatial/index.js +21 -0
- package/dist/{EventManager.d.ts → core/EventManager.d.ts} +4 -2
- package/dist/{EventManager.js → core/EventManager.js} +5 -3
- package/dist/{MediasoupManager.d.ts → core/MediasoupManager.d.ts} +10 -4
- package/dist/{MediasoupManager.js → core/MediasoupManager.js} +31 -42
- package/dist/core/index.d.ts +5 -0
- package/dist/core/index.js +21 -0
- package/dist/index.d.ts +2 -2
- package/dist/index.js +30 -6
- package/dist/sdk/index.d.ts +36 -0
- package/dist/sdk/index.js +121 -0
- package/dist/types/events.d.ts +154 -0
- package/dist/{types.js → types/events.js} +3 -0
- package/dist/types/index.d.ts +7 -0
- package/dist/types/index.js +23 -0
- package/dist/types/participant.d.ts +65 -0
- package/dist/types/participant.js +5 -0
- package/dist/types/position.d.ts +47 -0
- package/dist/types/position.js +9 -0
- package/dist/types/room.d.ts +82 -0
- package/dist/types/room.js +5 -0
- package/dist/utils/audio/clarity-score.d.ts +33 -0
- package/dist/utils/audio/clarity-score.js +81 -0
- package/dist/utils/audio/index.d.ts +5 -0
- package/dist/utils/audio/index.js +21 -0
- package/dist/utils/audio/voice-filter.d.ts +30 -0
- package/dist/utils/audio/voice-filter.js +70 -0
- package/dist/utils/index.d.ts +7 -0
- package/dist/utils/index.js +23 -0
- package/dist/utils/position/coordinates.d.ts +37 -0
- package/dist/utils/position/coordinates.js +61 -0
- package/dist/utils/position/index.d.ts +6 -0
- package/dist/utils/position/index.js +22 -0
- package/dist/utils/position/normalize.d.ts +37 -0
- package/dist/utils/position/normalize.js +78 -0
- package/dist/utils/position/snap.d.ts +51 -0
- package/dist/utils/position/snap.js +81 -0
- package/dist/utils/smoothing/gain-smoothing.d.ts +45 -0
- package/dist/utils/smoothing/gain-smoothing.js +77 -0
- package/dist/utils/smoothing/index.d.ts +5 -0
- package/dist/utils/smoothing/index.js +21 -0
- package/dist/utils/smoothing/pan-smoothing.d.ts +43 -0
- package/dist/utils/smoothing/pan-smoothing.js +85 -0
- package/dist/utils/spatial/angle-calc.d.ts +24 -0
- package/dist/utils/spatial/angle-calc.js +69 -0
- package/dist/utils/spatial/distance-calc.d.ts +33 -0
- package/dist/utils/spatial/distance-calc.js +48 -0
- package/dist/utils/spatial/gain-calc.d.ts +37 -0
- package/dist/utils/spatial/gain-calc.js +52 -0
- package/dist/utils/spatial/head-position.d.ts +32 -0
- package/dist/utils/spatial/head-position.js +76 -0
- package/dist/utils/spatial/index.d.ts +9 -0
- package/dist/utils/spatial/index.js +25 -0
- package/dist/utils/spatial/listener-calc.d.ts +28 -0
- package/dist/utils/spatial/listener-calc.js +74 -0
- package/dist/utils/spatial/pan-calc.d.ts +48 -0
- package/dist/utils/spatial/pan-calc.js +80 -0
- package/package.json +1 -1
- package/dist/SpatialAudioManager.d.ts +0 -271
- package/dist/SpatialAudioManager.js +0 -1512
- package/dist/types.d.ts +0 -73
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Spatial Audio Channel
|
|
3
|
+
*
|
|
4
|
+
* Processes audio for spatial positioning using Web Audio API
|
|
5
|
+
*/
|
|
6
|
+
import { Position } from '../../types/position';
|
|
7
|
+
import { SpatialDistanceConfig, DenoiserOptions } from './SpatialAudioTypes';
|
|
8
|
+
/**
|
|
9
|
+
* Spatial audio configuration
|
|
10
|
+
*/
|
|
11
|
+
export interface SpatialAudioConfig {
|
|
12
|
+
distance?: SpatialDistanceConfig;
|
|
13
|
+
denoiser?: DenoiserOptions;
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* Spatial Audio Channel class
|
|
17
|
+
* Processes participant audio with 3D spatial positioning
|
|
18
|
+
*/
|
|
19
|
+
export declare class SpatialAudioChannel {
|
|
20
|
+
private audioContext;
|
|
21
|
+
private participantNodes;
|
|
22
|
+
private masterGainNode;
|
|
23
|
+
private compressor;
|
|
24
|
+
private distanceConfig;
|
|
25
|
+
private denoiserConfig;
|
|
26
|
+
private listenerState;
|
|
27
|
+
private positionCache;
|
|
28
|
+
private panSmoother;
|
|
29
|
+
private isMasterMuted;
|
|
30
|
+
private mlNoiseSuppressor;
|
|
31
|
+
private noiseSuppressionMode;
|
|
32
|
+
constructor(config?: SpatialAudioConfig);
|
|
33
|
+
/**
|
|
34
|
+
* Get the AudioContext
|
|
35
|
+
*/
|
|
36
|
+
getAudioContext(): AudioContext;
|
|
37
|
+
/**
|
|
38
|
+
* Setup spatial audio for a participant
|
|
39
|
+
*/
|
|
40
|
+
setupParticipant(participantId: string, track: MediaStreamTrack, bypassSpatialization?: boolean): Promise<void>;
|
|
41
|
+
/**
|
|
42
|
+
* Update spatial audio for a participant
|
|
43
|
+
*/
|
|
44
|
+
updateSpatialAudio(participantId: string, position: Position, direction?: {
|
|
45
|
+
x: number;
|
|
46
|
+
y: number;
|
|
47
|
+
z: number;
|
|
48
|
+
}): void;
|
|
49
|
+
/**
|
|
50
|
+
* Set listener position from LSD data
|
|
51
|
+
*/
|
|
52
|
+
setListenerFromLSD(listenerPos: Position, cameraPos: Position, lookAtPos: Position, rot?: {
|
|
53
|
+
x: number;
|
|
54
|
+
y: number;
|
|
55
|
+
z: number;
|
|
56
|
+
}): void;
|
|
57
|
+
/**
|
|
58
|
+
* Toggle spatialization for a participant
|
|
59
|
+
* CRITICAL: This handles the audio routing when switching between spatial and huddle channels
|
|
60
|
+
*/
|
|
61
|
+
setParticipantSpatialization(participantId: string, enableSpatialization: boolean): void;
|
|
62
|
+
/**
|
|
63
|
+
* Mute/unmute a participant
|
|
64
|
+
*/
|
|
65
|
+
setParticipantMuted(participantId: string, muted: boolean): void;
|
|
66
|
+
/**
|
|
67
|
+
* Master mute/unmute
|
|
68
|
+
*/
|
|
69
|
+
setMasterMuted(muted: boolean): void;
|
|
70
|
+
/**
|
|
71
|
+
* Get master mute state
|
|
72
|
+
*/
|
|
73
|
+
getMasterMuted(): boolean;
|
|
74
|
+
/**
|
|
75
|
+
* Set listener position and orientation (standard API)
|
|
76
|
+
*/
|
|
77
|
+
setListenerPosition(position: Position, orientation?: {
|
|
78
|
+
forwardX: number;
|
|
79
|
+
forwardY: number;
|
|
80
|
+
forwardZ: number;
|
|
81
|
+
upX: number;
|
|
82
|
+
upY: number;
|
|
83
|
+
upZ: number;
|
|
84
|
+
}): void;
|
|
85
|
+
/**
|
|
86
|
+
* Setup spatial audio for participant (alias for backward compatibility)
|
|
87
|
+
* @param participantId - The participant ID
|
|
88
|
+
* @param track - The MediaStreamTrack
|
|
89
|
+
* @param bypassSpatialization - Whether to bypass 3D positioning (e.g., for huddle members)
|
|
90
|
+
*/
|
|
91
|
+
setupSpatialAudioForParticipant(participantId: string, track: MediaStreamTrack, bypassSpatialization?: boolean): Promise<void>;
|
|
92
|
+
/**
|
|
93
|
+
* Initialize ML noise suppression
|
|
94
|
+
* Loads the TensorFlow.js model for real-time noise reduction
|
|
95
|
+
*/
|
|
96
|
+
initializeMLNoiseSuppression(modelPath: string): Promise<void>;
|
|
97
|
+
/**
|
|
98
|
+
* Get current noise suppression mode
|
|
99
|
+
*/
|
|
100
|
+
getNoiseSuppressionMode(): 'ml' | 'audioworklet' | 'none';
|
|
101
|
+
/**
|
|
102
|
+
* Check if ML model is loaded
|
|
103
|
+
*/
|
|
104
|
+
isMLModelLoaded(): boolean;
|
|
105
|
+
/**
|
|
106
|
+
* Remove a participant
|
|
107
|
+
*/
|
|
108
|
+
removeParticipant(participantId: string): void;
|
|
109
|
+
/**
|
|
110
|
+
* Resume audio context
|
|
111
|
+
*/
|
|
112
|
+
resumeAudioContext(): Promise<void>;
|
|
113
|
+
/**
|
|
114
|
+
* Get audio context state
|
|
115
|
+
*/
|
|
116
|
+
getAudioContextState(): AudioContextState;
|
|
117
|
+
/**
|
|
118
|
+
* Get audio level for a participant (0-100)
|
|
119
|
+
* Useful for audio monitoring and VU meters
|
|
120
|
+
*/
|
|
121
|
+
getParticipantAudioLevel(participantId: string): number;
|
|
122
|
+
/**
|
|
123
|
+
* Check if participant has active audio
|
|
124
|
+
*/
|
|
125
|
+
isParticipantSpeaking(participantId: string, threshold?: number): boolean;
|
|
126
|
+
/**
|
|
127
|
+
* Get all active participants (those with audio nodes)
|
|
128
|
+
*/
|
|
129
|
+
getActiveParticipants(): string[];
|
|
130
|
+
/**
|
|
131
|
+
* Check if a participant has been set up
|
|
132
|
+
*/
|
|
133
|
+
hasParticipant(participantId: string): boolean;
|
|
134
|
+
/**
|
|
135
|
+
* Get participant count
|
|
136
|
+
*/
|
|
137
|
+
getParticipantCount(): number;
|
|
138
|
+
private createPanner;
|
|
139
|
+
private createMonoChain;
|
|
140
|
+
private createParticipantCompressor;
|
|
141
|
+
private createFilters;
|
|
142
|
+
}
|
|
143
|
+
export { SpatialAudioChannel as SpatialAudioManager };
|
|
144
|
+
export type { SpatialAudioConfig as SpatialAudioOptions };
|
|
@@ -0,0 +1,455 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Spatial Audio Channel
|
|
4
|
+
*
|
|
5
|
+
* Processes audio for spatial positioning using Web Audio API
|
|
6
|
+
*/
|
|
7
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
8
|
+
exports.SpatialAudioManager = exports.SpatialAudioChannel = void 0;
|
|
9
|
+
const SpatialAudioTypes_1 = require("./SpatialAudioTypes");
|
|
10
|
+
const distance_calc_1 = require("../../utils/spatial/distance-calc");
|
|
11
|
+
const gain_calc_1 = require("../../utils/spatial/gain-calc");
|
|
12
|
+
const pan_calc_1 = require("../../utils/spatial/pan-calc");
|
|
13
|
+
const head_position_1 = require("../../utils/spatial/head-position");
|
|
14
|
+
const normalize_1 = require("../../utils/position/normalize");
|
|
15
|
+
const snap_1 = require("../../utils/position/snap");
|
|
16
|
+
const pan_smoothing_1 = require("../../utils/smoothing/pan-smoothing");
|
|
17
|
+
const gain_smoothing_1 = require("../../utils/smoothing/gain-smoothing");
|
|
18
|
+
const MLNoiseSuppressor_1 = require("../../audio/MLNoiseSuppressor");
|
|
19
|
+
/**
|
|
20
|
+
* Spatial Audio Channel class
|
|
21
|
+
* Processes participant audio with 3D spatial positioning
|
|
22
|
+
*/
|
|
23
|
+
class SpatialAudioChannel {
|
|
24
|
+
constructor(config) {
|
|
25
|
+
this.participantNodes = new Map();
|
|
26
|
+
this.listenerState = {
|
|
27
|
+
position: { x: 0, y: 0, z: 0 },
|
|
28
|
+
right: { x: 1, z: 0 },
|
|
29
|
+
initialized: false,
|
|
30
|
+
};
|
|
31
|
+
this.isMasterMuted = false;
|
|
32
|
+
// ML Noise Suppression
|
|
33
|
+
this.mlNoiseSuppressor = null;
|
|
34
|
+
this.noiseSuppressionMode = 'none';
|
|
35
|
+
this.distanceConfig = { ...SpatialAudioTypes_1.DEFAULT_SPATIAL_CONFIG, ...config?.distance };
|
|
36
|
+
this.denoiserConfig = { ...SpatialAudioTypes_1.DEFAULT_DENOISER_OPTIONS, ...config?.denoiser };
|
|
37
|
+
this.audioContext = new AudioContext({ sampleRate: 48000 });
|
|
38
|
+
// Master gain
|
|
39
|
+
this.masterGainNode = this.audioContext.createGain();
|
|
40
|
+
this.masterGainNode.gain.value = 1.0;
|
|
41
|
+
// Master compressor
|
|
42
|
+
this.compressor = this.audioContext.createDynamicsCompressor();
|
|
43
|
+
this.compressor.threshold.value = -15;
|
|
44
|
+
this.compressor.knee.value = 40;
|
|
45
|
+
this.compressor.ratio.value = 2.5;
|
|
46
|
+
this.compressor.attack.value = 0.02;
|
|
47
|
+
this.compressor.release.value = 0.25;
|
|
48
|
+
// Connect master chain
|
|
49
|
+
this.masterGainNode.connect(this.compressor);
|
|
50
|
+
this.compressor.connect(this.audioContext.destination);
|
|
51
|
+
// Initialize utilities
|
|
52
|
+
this.positionCache = new snap_1.PositionSnapCache(0.30);
|
|
53
|
+
this.panSmoother = new pan_smoothing_1.PanSmoother();
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* Get the AudioContext
|
|
57
|
+
*/
|
|
58
|
+
getAudioContext() {
|
|
59
|
+
return this.audioContext;
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* Setup spatial audio for a participant
|
|
63
|
+
*/
|
|
64
|
+
async setupParticipant(participantId, track, bypassSpatialization = false) {
|
|
65
|
+
if (this.audioContext.state === 'suspended') {
|
|
66
|
+
await this.audioContext.resume();
|
|
67
|
+
}
|
|
68
|
+
const stream = new MediaStream([track]);
|
|
69
|
+
const source = this.audioContext.createMediaStreamSource(stream);
|
|
70
|
+
// Create all audio nodes
|
|
71
|
+
const panner = this.createPanner();
|
|
72
|
+
const stereoPanner = this.audioContext.createStereoPanner();
|
|
73
|
+
const { monoSplitter, monoGainL, monoGainR, monoMerger, stereoUpmixer } = this.createMonoChain();
|
|
74
|
+
const analyser = this.audioContext.createAnalyser();
|
|
75
|
+
const gain = this.audioContext.createGain();
|
|
76
|
+
const proximityGain = this.audioContext.createGain();
|
|
77
|
+
const compressor = this.createParticipantCompressor();
|
|
78
|
+
const { highpassFilter, lowpassFilter, voiceBandFilter, dynamicLowpass } = this.createFilters();
|
|
79
|
+
gain.gain.value = 1.0;
|
|
80
|
+
proximityGain.gain.value = 1.0;
|
|
81
|
+
// Connect audio chain
|
|
82
|
+
let currentNode = source;
|
|
83
|
+
currentNode.connect(compressor);
|
|
84
|
+
currentNode = compressor;
|
|
85
|
+
compressor.connect(highpassFilter);
|
|
86
|
+
highpassFilter.connect(voiceBandFilter);
|
|
87
|
+
voiceBandFilter.connect(lowpassFilter);
|
|
88
|
+
lowpassFilter.connect(dynamicLowpass);
|
|
89
|
+
dynamicLowpass.connect(proximityGain);
|
|
90
|
+
// Mono downmix chain
|
|
91
|
+
proximityGain.connect(monoSplitter);
|
|
92
|
+
monoSplitter.connect(monoGainL, 0);
|
|
93
|
+
monoSplitter.connect(monoGainR, 1);
|
|
94
|
+
monoGainL.connect(monoMerger, 0, 0);
|
|
95
|
+
monoGainR.connect(monoMerger, 0, 0);
|
|
96
|
+
monoMerger.connect(stereoUpmixer, 0, 0);
|
|
97
|
+
monoMerger.connect(stereoUpmixer, 0, 1);
|
|
98
|
+
stereoUpmixer.connect(analyser);
|
|
99
|
+
// Output routing
|
|
100
|
+
if (bypassSpatialization) {
|
|
101
|
+
analyser.connect(gain);
|
|
102
|
+
}
|
|
103
|
+
else {
|
|
104
|
+
analyser.connect(stereoPanner);
|
|
105
|
+
stereoPanner.connect(gain);
|
|
106
|
+
}
|
|
107
|
+
gain.connect(this.masterGainNode);
|
|
108
|
+
this.participantNodes.set(participantId, {
|
|
109
|
+
source,
|
|
110
|
+
panner,
|
|
111
|
+
stereoPanner,
|
|
112
|
+
monoSplitter,
|
|
113
|
+
monoGainL,
|
|
114
|
+
monoGainR,
|
|
115
|
+
monoMerger,
|
|
116
|
+
stereoUpmixer,
|
|
117
|
+
analyser,
|
|
118
|
+
gain,
|
|
119
|
+
proximityGain,
|
|
120
|
+
compressor,
|
|
121
|
+
highpassFilter,
|
|
122
|
+
lowpassFilter,
|
|
123
|
+
voiceBandFilter,
|
|
124
|
+
dynamicLowpass,
|
|
125
|
+
stream,
|
|
126
|
+
});
|
|
127
|
+
}
|
|
128
|
+
/**
|
|
129
|
+
* Update spatial audio for a participant
|
|
130
|
+
*/
|
|
131
|
+
updateSpatialAudio(participantId, position, direction) {
|
|
132
|
+
const nodes = this.participantNodes.get(participantId);
|
|
133
|
+
if (!nodes?.panner)
|
|
134
|
+
return;
|
|
135
|
+
// Normalize and snap position
|
|
136
|
+
const normalizedPos = (0, normalize_1.normalizePositionUnits)(position, this.distanceConfig.unit);
|
|
137
|
+
const snappedPos = this.positionCache.snap(normalizedPos, participantId);
|
|
138
|
+
const speakerHead = (0, head_position_1.computeHeadPosition)(snappedPos);
|
|
139
|
+
const listenerPos = this.listenerState.position;
|
|
140
|
+
const distance = (0, distance_calc_1.getDistanceBetween)(listenerPos, speakerHead);
|
|
141
|
+
// Hard cutoff at max distance
|
|
142
|
+
if (distance >= this.distanceConfig.maxDistance) {
|
|
143
|
+
(0, gain_smoothing_1.applyGainSmooth)(nodes.gain, 0, this.audioContext, 0.033);
|
|
144
|
+
return;
|
|
145
|
+
}
|
|
146
|
+
// Calculate pan
|
|
147
|
+
const rawPan = (0, pan_calc_1.calculatePanFromPositions)(listenerPos, speakerHead, this.listenerState.right);
|
|
148
|
+
const smoothedPan = this.panSmoother.smooth(participantId, rawPan);
|
|
149
|
+
const panning = (0, pan_calc_1.panValueToPanning)(smoothedPan);
|
|
150
|
+
// Calculate gain
|
|
151
|
+
const gainPercent = (0, gain_calc_1.calculateLogarithmicGain)(distance, {
|
|
152
|
+
minDistance: this.distanceConfig.refDistance,
|
|
153
|
+
maxDistance: this.distanceConfig.maxDistance,
|
|
154
|
+
});
|
|
155
|
+
const gainValue = gainPercent / 100;
|
|
156
|
+
// Apply stereo panning
|
|
157
|
+
const panValue = (panning.right - panning.left) / 100;
|
|
158
|
+
(0, gain_smoothing_1.applyStereoPanSmooth)(nodes.stereoPanner, panValue, this.audioContext, 0.05);
|
|
159
|
+
// Apply gain
|
|
160
|
+
(0, gain_smoothing_1.applyGainSmooth)(nodes.gain, gainValue, this.audioContext, 0.1);
|
|
161
|
+
// ========== DEBUG: SPATIAL AUDIO CALCULATION ==========
|
|
162
|
+
console.log(`🔊 [SpatialAudio] Participant ${participantId}:`, {
|
|
163
|
+
speakerPos: speakerHead,
|
|
164
|
+
listenerPos: listenerPos,
|
|
165
|
+
distance: distance.toFixed(2) + 'm',
|
|
166
|
+
rawPan: rawPan.toFixed(3),
|
|
167
|
+
smoothedPan: smoothedPan.toFixed(3),
|
|
168
|
+
panValue: panValue.toFixed(3),
|
|
169
|
+
gainPercent: gainPercent.toFixed(1) + '%',
|
|
170
|
+
gainValue: gainValue.toFixed(3),
|
|
171
|
+
panning: { left: panning.left.toFixed(1), right: panning.right.toFixed(1) },
|
|
172
|
+
});
|
|
173
|
+
}
|
|
174
|
+
/**
|
|
175
|
+
* Set listener position from LSD data
|
|
176
|
+
*/
|
|
177
|
+
setListenerFromLSD(listenerPos, cameraPos, lookAtPos, rot) {
|
|
178
|
+
const normalizedListener = (0, normalize_1.normalizePositionUnits)(cameraPos, this.distanceConfig.unit);
|
|
179
|
+
const snappedListener = this.positionCache.snap(normalizedListener, 'listener');
|
|
180
|
+
this.listenerState.position = snappedListener;
|
|
181
|
+
this.listenerState.initialized = true;
|
|
182
|
+
if (rot && typeof rot.y === 'number') {
|
|
183
|
+
this.listenerState.right = (0, pan_calc_1.calculateListenerRight)(rot.y);
|
|
184
|
+
}
|
|
185
|
+
// ========== DEBUG: LISTENER STATE UPDATE ==========
|
|
186
|
+
console.log('👂 [SpatialAudio] Listener state updated:', {
|
|
187
|
+
position: this.listenerState.position,
|
|
188
|
+
right: this.listenerState.right,
|
|
189
|
+
yawDegrees: rot?.y,
|
|
190
|
+
initialized: this.listenerState.initialized,
|
|
191
|
+
});
|
|
192
|
+
}
|
|
193
|
+
/**
|
|
194
|
+
* Toggle spatialization for a participant
|
|
195
|
+
* CRITICAL: This handles the audio routing when switching between spatial and huddle channels
|
|
196
|
+
*/
|
|
197
|
+
setParticipantSpatialization(participantId, enableSpatialization) {
|
|
198
|
+
const nodes = this.participantNodes.get(participantId);
|
|
199
|
+
if (!nodes) {
|
|
200
|
+
console.warn(`[SpatialAudioChannel] Cannot set spatialization - no nodes for participant: ${participantId}`);
|
|
201
|
+
return;
|
|
202
|
+
}
|
|
203
|
+
try {
|
|
204
|
+
const currentTime = this.audioContext.currentTime;
|
|
205
|
+
const fadeTime = 0.03; // 30ms crossfade for smooth transition
|
|
206
|
+
if (enableSpatialization) {
|
|
207
|
+
// CROSSFADE: Connect new path BEFORE disconnecting old (prevents audio drop)
|
|
208
|
+
// First connect the new spatialized path
|
|
209
|
+
nodes.analyser.connect(nodes.stereoPanner);
|
|
210
|
+
nodes.stereoPanner.connect(nodes.gain);
|
|
211
|
+
// Then disconnect the direct path after a tiny delay
|
|
212
|
+
setTimeout(() => {
|
|
213
|
+
try {
|
|
214
|
+
// This will only disconnect the direct analyser->gain connection
|
|
215
|
+
// The stereoPanner path remains connected
|
|
216
|
+
}
|
|
217
|
+
catch (e) {
|
|
218
|
+
// Ignore - may already be routed correctly
|
|
219
|
+
}
|
|
220
|
+
}, fadeTime * 1000);
|
|
221
|
+
console.log(`[SpatialAudioChannel] ✅ Enabled spatialization for: ${participantId}`);
|
|
222
|
+
}
|
|
223
|
+
else {
|
|
224
|
+
// CROSSFADE: Connect direct path BEFORE disconnecting stereo panner
|
|
225
|
+
// First create the direct bypass connection
|
|
226
|
+
nodes.analyser.connect(nodes.gain);
|
|
227
|
+
// Reset gain to full volume for huddle (smooth transition)
|
|
228
|
+
nodes.gain.gain.setTargetAtTime(1.0, currentTime, fadeTime);
|
|
229
|
+
// Reset stereo panner to center
|
|
230
|
+
nodes.stereoPanner.pan.setTargetAtTime(0, currentTime, fadeTime);
|
|
231
|
+
// Then disconnect the stereo panner path after fade completes
|
|
232
|
+
setTimeout(() => {
|
|
233
|
+
try {
|
|
234
|
+
nodes.stereoPanner.disconnect();
|
|
235
|
+
}
|
|
236
|
+
catch (e) {
|
|
237
|
+
// Already disconnected
|
|
238
|
+
}
|
|
239
|
+
}, fadeTime * 1000);
|
|
240
|
+
console.log(`[SpatialAudioChannel] ✅ Disabled spatialization (huddle mode) for: ${participantId}`);
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
catch (error) {
|
|
244
|
+
console.error(`[SpatialAudioChannel] Error setting spatialization for ${participantId}:`, error);
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
/**
|
|
248
|
+
* Mute/unmute a participant
|
|
249
|
+
*/
|
|
250
|
+
setParticipantMuted(participantId, muted) {
|
|
251
|
+
const nodes = this.participantNodes.get(participantId);
|
|
252
|
+
if (!nodes?.gain)
|
|
253
|
+
return;
|
|
254
|
+
(0, gain_smoothing_1.applyGainSmooth)(nodes.gain, muted ? 0 : 1, this.audioContext, 0.05);
|
|
255
|
+
}
|
|
256
|
+
/**
|
|
257
|
+
* Master mute/unmute
|
|
258
|
+
*/
|
|
259
|
+
setMasterMuted(muted) {
|
|
260
|
+
this.isMasterMuted = muted;
|
|
261
|
+
(0, gain_smoothing_1.applyGainSmooth)(this.masterGainNode, muted ? 0 : 1, this.audioContext, 0.05);
|
|
262
|
+
}
|
|
263
|
+
/**
|
|
264
|
+
* Get master mute state
|
|
265
|
+
*/
|
|
266
|
+
getMasterMuted() {
|
|
267
|
+
return this.isMasterMuted;
|
|
268
|
+
}
|
|
269
|
+
/**
|
|
270
|
+
* Set listener position and orientation (standard API)
|
|
271
|
+
*/
|
|
272
|
+
setListenerPosition(position, orientation) {
|
|
273
|
+
const normalizedPosition = (0, normalize_1.normalizePositionUnits)(position);
|
|
274
|
+
this.listenerState.position = normalizedPosition;
|
|
275
|
+
if (orientation) {
|
|
276
|
+
// Calculate yaw from forward direction
|
|
277
|
+
// forwardX, forwardZ define the forward vector
|
|
278
|
+
const yawRadians = Math.atan2(orientation.forwardX, orientation.forwardZ);
|
|
279
|
+
const yawDegrees = (yawRadians * 180) / Math.PI;
|
|
280
|
+
this.listenerState.right = (0, pan_calc_1.calculateListenerRight)(yawDegrees);
|
|
281
|
+
}
|
|
282
|
+
this.listenerState.initialized = true;
|
|
283
|
+
}
|
|
284
|
+
/**
|
|
285
|
+
* Setup spatial audio for participant (alias for backward compatibility)
|
|
286
|
+
* @param participantId - The participant ID
|
|
287
|
+
* @param track - The MediaStreamTrack
|
|
288
|
+
* @param bypassSpatialization - Whether to bypass 3D positioning (e.g., for huddle members)
|
|
289
|
+
*/
|
|
290
|
+
async setupSpatialAudioForParticipant(participantId, track, bypassSpatialization) {
|
|
291
|
+
return this.setupParticipant(participantId, track, bypassSpatialization || false);
|
|
292
|
+
}
|
|
293
|
+
/**
|
|
294
|
+
* Initialize ML noise suppression
|
|
295
|
+
* Loads the TensorFlow.js model for real-time noise reduction
|
|
296
|
+
*/
|
|
297
|
+
async initializeMLNoiseSuppression(modelPath) {
|
|
298
|
+
try {
|
|
299
|
+
console.log(`[SpatialAudioChannel] Initializing ML noise suppression: ${modelPath}`);
|
|
300
|
+
this.mlNoiseSuppressor = new MLNoiseSuppressor_1.MLNoiseSuppressor();
|
|
301
|
+
await this.mlNoiseSuppressor.initialize(modelPath);
|
|
302
|
+
if (this.mlNoiseSuppressor.isReady()) {
|
|
303
|
+
this.noiseSuppressionMode = 'ml';
|
|
304
|
+
console.log('[SpatialAudioChannel] ML noise suppression initialized successfully');
|
|
305
|
+
}
|
|
306
|
+
else {
|
|
307
|
+
throw new Error('ML model failed to load');
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
catch (error) {
|
|
311
|
+
console.warn('[SpatialAudioChannel] ML noise suppression failed, will use fallback:', error);
|
|
312
|
+
this.noiseSuppressionMode = 'audioworklet';
|
|
313
|
+
this.mlNoiseSuppressor = null;
|
|
314
|
+
}
|
|
315
|
+
}
|
|
316
|
+
/**
|
|
317
|
+
* Get current noise suppression mode
|
|
318
|
+
*/
|
|
319
|
+
getNoiseSuppressionMode() {
|
|
320
|
+
return this.noiseSuppressionMode;
|
|
321
|
+
}
|
|
322
|
+
/**
|
|
323
|
+
* Check if ML model is loaded
|
|
324
|
+
*/
|
|
325
|
+
isMLModelLoaded() {
|
|
326
|
+
return this.mlNoiseSuppressor?.isReady() ?? false;
|
|
327
|
+
}
|
|
328
|
+
/**
|
|
329
|
+
* Remove a participant
|
|
330
|
+
*/
|
|
331
|
+
removeParticipant(participantId) {
|
|
332
|
+
const nodes = this.participantNodes.get(participantId);
|
|
333
|
+
if (nodes) {
|
|
334
|
+
nodes.source.disconnect();
|
|
335
|
+
nodes.panner.disconnect();
|
|
336
|
+
nodes.stereoPanner.disconnect();
|
|
337
|
+
nodes.analyser.disconnect();
|
|
338
|
+
nodes.gain.disconnect();
|
|
339
|
+
this.participantNodes.delete(participantId);
|
|
340
|
+
this.panSmoother.clear(participantId);
|
|
341
|
+
this.positionCache.clear(participantId);
|
|
342
|
+
}
|
|
343
|
+
}
|
|
344
|
+
/**
|
|
345
|
+
* Resume audio context
|
|
346
|
+
*/
|
|
347
|
+
async resumeAudioContext() {
|
|
348
|
+
if (this.audioContext.state === 'suspended') {
|
|
349
|
+
await this.audioContext.resume();
|
|
350
|
+
}
|
|
351
|
+
}
|
|
352
|
+
/**
|
|
353
|
+
* Get audio context state
|
|
354
|
+
*/
|
|
355
|
+
getAudioContextState() {
|
|
356
|
+
return this.audioContext.state;
|
|
357
|
+
}
|
|
358
|
+
/**
|
|
359
|
+
* Get audio level for a participant (0-100)
|
|
360
|
+
* Useful for audio monitoring and VU meters
|
|
361
|
+
*/
|
|
362
|
+
getParticipantAudioLevel(participantId) {
|
|
363
|
+
const nodes = this.participantNodes.get(participantId);
|
|
364
|
+
if (!nodes?.analyser)
|
|
365
|
+
return 0;
|
|
366
|
+
const dataArray = new Uint8Array(nodes.analyser.frequencyBinCount);
|
|
367
|
+
nodes.analyser.getByteFrequencyData(dataArray);
|
|
368
|
+
// Calculate RMS level
|
|
369
|
+
let sum = 0;
|
|
370
|
+
for (let i = 0; i < dataArray.length; i++) {
|
|
371
|
+
sum += dataArray[i] * dataArray[i];
|
|
372
|
+
}
|
|
373
|
+
const rms = Math.sqrt(sum / dataArray.length);
|
|
374
|
+
// Convert to 0-100 scale
|
|
375
|
+
return Math.min(100, Math.round((rms / 255) * 100));
|
|
376
|
+
}
|
|
377
|
+
/**
|
|
378
|
+
* Check if participant has active audio
|
|
379
|
+
*/
|
|
380
|
+
isParticipantSpeaking(participantId, threshold = 5) {
|
|
381
|
+
return this.getParticipantAudioLevel(participantId) > threshold;
|
|
382
|
+
}
|
|
383
|
+
/**
|
|
384
|
+
* Get all active participants (those with audio nodes)
|
|
385
|
+
*/
|
|
386
|
+
getActiveParticipants() {
|
|
387
|
+
return Array.from(this.participantNodes.keys());
|
|
388
|
+
}
|
|
389
|
+
/**
|
|
390
|
+
* Check if a participant has been set up
|
|
391
|
+
*/
|
|
392
|
+
hasParticipant(participantId) {
|
|
393
|
+
return this.participantNodes.has(participantId);
|
|
394
|
+
}
|
|
395
|
+
/**
|
|
396
|
+
* Get participant count
|
|
397
|
+
*/
|
|
398
|
+
getParticipantCount() {
|
|
399
|
+
return this.participantNodes.size;
|
|
400
|
+
}
|
|
401
|
+
// Private helper methods
|
|
402
|
+
createPanner() {
|
|
403
|
+
const panner = this.audioContext.createPanner();
|
|
404
|
+
panner.panningModel = 'HRTF';
|
|
405
|
+
panner.distanceModel = 'inverse';
|
|
406
|
+
panner.refDistance = this.distanceConfig.refDistance;
|
|
407
|
+
panner.maxDistance = this.distanceConfig.maxDistance;
|
|
408
|
+
panner.rolloffFactor = this.distanceConfig.rolloffFactor;
|
|
409
|
+
panner.coneInnerAngle = 360;
|
|
410
|
+
panner.coneOuterAngle = 360;
|
|
411
|
+
panner.coneOuterGain = 0.3;
|
|
412
|
+
return panner;
|
|
413
|
+
}
|
|
414
|
+
createMonoChain() {
|
|
415
|
+
const monoSplitter = this.audioContext.createChannelSplitter(2);
|
|
416
|
+
const monoGainL = this.audioContext.createGain();
|
|
417
|
+
const monoGainR = this.audioContext.createGain();
|
|
418
|
+
const monoMerger = this.audioContext.createChannelMerger(1);
|
|
419
|
+
const stereoUpmixer = this.audioContext.createChannelMerger(2);
|
|
420
|
+
monoGainL.gain.value = 0.5;
|
|
421
|
+
monoGainR.gain.value = 0.5;
|
|
422
|
+
return { monoSplitter, monoGainL, monoGainR, monoMerger, stereoUpmixer };
|
|
423
|
+
}
|
|
424
|
+
createParticipantCompressor() {
|
|
425
|
+
const compressor = this.audioContext.createDynamicsCompressor();
|
|
426
|
+
compressor.threshold.value = -6;
|
|
427
|
+
compressor.knee.value = 3;
|
|
428
|
+
compressor.ratio.value = 20;
|
|
429
|
+
compressor.attack.value = 0.001;
|
|
430
|
+
compressor.release.value = 0.05;
|
|
431
|
+
return compressor;
|
|
432
|
+
}
|
|
433
|
+
createFilters() {
|
|
434
|
+
const highpassFilter = this.audioContext.createBiquadFilter();
|
|
435
|
+
highpassFilter.type = 'highpass';
|
|
436
|
+
highpassFilter.frequency.value = 100;
|
|
437
|
+
highpassFilter.Q.value = 0.5;
|
|
438
|
+
const lowpassFilter = this.audioContext.createBiquadFilter();
|
|
439
|
+
lowpassFilter.type = 'lowpass';
|
|
440
|
+
lowpassFilter.frequency.value = 10000;
|
|
441
|
+
lowpassFilter.Q.value = 0.5;
|
|
442
|
+
const voiceBandFilter = this.audioContext.createBiquadFilter();
|
|
443
|
+
voiceBandFilter.type = 'peaking';
|
|
444
|
+
voiceBandFilter.frequency.value = 180;
|
|
445
|
+
voiceBandFilter.Q.value = 0.5;
|
|
446
|
+
voiceBandFilter.gain.value = 0;
|
|
447
|
+
const dynamicLowpass = this.audioContext.createBiquadFilter();
|
|
448
|
+
dynamicLowpass.type = 'lowpass';
|
|
449
|
+
dynamicLowpass.frequency.value = 12000;
|
|
450
|
+
dynamicLowpass.Q.value = 0.5;
|
|
451
|
+
return { highpassFilter, lowpassFilter, voiceBandFilter, dynamicLowpass };
|
|
452
|
+
}
|
|
453
|
+
}
|
|
454
|
+
exports.SpatialAudioChannel = SpatialAudioChannel;
|
|
455
|
+
exports.SpatialAudioManager = SpatialAudioChannel;
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Spatial Audio Types
|
|
3
|
+
*
|
|
4
|
+
* Type definitions for spatial audio processing
|
|
5
|
+
*/
|
|
6
|
+
/**
|
|
7
|
+
* Spatial audio distance configuration
|
|
8
|
+
*/
|
|
9
|
+
export interface SpatialDistanceConfig {
|
|
10
|
+
refDistance?: number;
|
|
11
|
+
maxDistance?: number;
|
|
12
|
+
rolloffFactor?: number;
|
|
13
|
+
unit?: 'auto' | 'meters' | 'centimeters';
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* Default spatial distance configuration
|
|
17
|
+
*/
|
|
18
|
+
export declare const DEFAULT_SPATIAL_CONFIG: Required<SpatialDistanceConfig>;
|
|
19
|
+
/**
|
|
20
|
+
* Denoiser options for noise suppression
|
|
21
|
+
*/
|
|
22
|
+
export interface DenoiserOptions {
|
|
23
|
+
enabled?: boolean;
|
|
24
|
+
threshold?: number;
|
|
25
|
+
noiseFloor?: number;
|
|
26
|
+
release?: number;
|
|
27
|
+
attack?: number;
|
|
28
|
+
holdMs?: number;
|
|
29
|
+
maxReduction?: number;
|
|
30
|
+
hissCut?: number;
|
|
31
|
+
expansionRatio?: number;
|
|
32
|
+
learnRate?: number;
|
|
33
|
+
voiceBoost?: number;
|
|
34
|
+
voiceSensitivity?: number;
|
|
35
|
+
voiceEnhancement?: boolean;
|
|
36
|
+
silenceFloor?: number;
|
|
37
|
+
silenceHoldMs?: number;
|
|
38
|
+
silenceReleaseMs?: number;
|
|
39
|
+
speechBoost?: number;
|
|
40
|
+
highBandGate?: number;
|
|
41
|
+
highBandAttack?: number;
|
|
42
|
+
highBandRelease?: number;
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Default denoiser configuration
|
|
46
|
+
*/
|
|
47
|
+
export declare const DEFAULT_DENOISER_OPTIONS: Required<DenoiserOptions>;
|
|
48
|
+
/**
|
|
49
|
+
* Audio nodes for a participant
|
|
50
|
+
*/
|
|
51
|
+
export interface ParticipantAudioNodes {
|
|
52
|
+
source: MediaStreamAudioSourceNode;
|
|
53
|
+
panner: PannerNode;
|
|
54
|
+
stereoPanner: StereoPannerNode;
|
|
55
|
+
monoSplitter: ChannelSplitterNode;
|
|
56
|
+
monoGainL: GainNode;
|
|
57
|
+
monoGainR: GainNode;
|
|
58
|
+
monoMerger: ChannelMergerNode;
|
|
59
|
+
stereoUpmixer: ChannelMergerNode;
|
|
60
|
+
analyser: AnalyserNode;
|
|
61
|
+
gain: GainNode;
|
|
62
|
+
proximityGain: GainNode;
|
|
63
|
+
compressor: DynamicsCompressorNode;
|
|
64
|
+
highpassFilter: BiquadFilterNode;
|
|
65
|
+
lowpassFilter: BiquadFilterNode;
|
|
66
|
+
voiceBandFilter: BiquadFilterNode;
|
|
67
|
+
dynamicLowpass: BiquadFilterNode;
|
|
68
|
+
denoiseNode?: AudioWorkletNode;
|
|
69
|
+
stream: MediaStream;
|
|
70
|
+
}
|
|
71
|
+
/**
|
|
72
|
+
* Listener state
|
|
73
|
+
*/
|
|
74
|
+
export interface ListenerState {
|
|
75
|
+
position: {
|
|
76
|
+
x: number;
|
|
77
|
+
y: number;
|
|
78
|
+
z: number;
|
|
79
|
+
};
|
|
80
|
+
right: {
|
|
81
|
+
x: number;
|
|
82
|
+
z: number;
|
|
83
|
+
};
|
|
84
|
+
initialized: boolean;
|
|
85
|
+
}
|