@newgameplusinc/alpha-spatial-comms 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1092 -0
- package/dist/index.d.mts +342 -0
- package/dist/index.d.ts +342 -0
- package/dist/index.js +3275 -0
- package/dist/index.js.map +1 -0
- package/dist/index.mjs +3252 -0
- package/dist/index.mjs.map +1 -0
- package/package.json +85 -0
package/dist/index.mjs
ADDED
|
@@ -0,0 +1,3252 @@
|
|
|
1
|
+
import { io } from 'socket.io-client';
|
|
2
|
+
import { Device } from 'mediasoup-client';
|
|
3
|
+
import * as tf from '@tensorflow/tfjs';
|
|
4
|
+
import { StreamChat } from 'stream-chat';
|
|
5
|
+
|
|
6
|
+
// src/index.ts
|
|
7
|
+
|
|
8
|
+
// src/core/EventManager.ts
|
|
9
|
+
function createEventBus() {
|
|
10
|
+
const listeners = /* @__PURE__ */ new Map();
|
|
11
|
+
function on(event, listener) {
|
|
12
|
+
let set = listeners.get(event);
|
|
13
|
+
if (!set) {
|
|
14
|
+
set = /* @__PURE__ */ new Set();
|
|
15
|
+
listeners.set(event, set);
|
|
16
|
+
}
|
|
17
|
+
set.add(listener);
|
|
18
|
+
}
|
|
19
|
+
function off(event, listener) {
|
|
20
|
+
listeners.get(event)?.delete(listener);
|
|
21
|
+
}
|
|
22
|
+
function emit(event, ...args) {
|
|
23
|
+
listeners.get(event)?.forEach((fn) => fn(...args));
|
|
24
|
+
}
|
|
25
|
+
function removeAllListeners(event) {
|
|
26
|
+
if (event) {
|
|
27
|
+
listeners.delete(event);
|
|
28
|
+
} else {
|
|
29
|
+
listeners.clear();
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
return { on, off, emit, removeAllListeners };
|
|
33
|
+
}
|
|
34
|
+
function createMediasoupManager(socket) {
|
|
35
|
+
const device = new Device();
|
|
36
|
+
let sendTransport = null;
|
|
37
|
+
let recvTransport = null;
|
|
38
|
+
const producers = /* @__PURE__ */ new Map();
|
|
39
|
+
const consumers = /* @__PURE__ */ new Map();
|
|
40
|
+
let participantId = "";
|
|
41
|
+
let sendTransportState = "new";
|
|
42
|
+
let recvTransportState = "new";
|
|
43
|
+
let iceRestartAttempts = 0;
|
|
44
|
+
const MAX_ICE_RESTART_ATTEMPTS = 3;
|
|
45
|
+
const internalBus = createEventBus();
|
|
46
|
+
function emitTransportEvent(event, data) {
|
|
47
|
+
internalBus.emit(event, data);
|
|
48
|
+
}
|
|
49
|
+
async function loadDevice(routerRtpCapabilities) {
|
|
50
|
+
if (device.loaded) return;
|
|
51
|
+
await device.load({ routerRtpCapabilities });
|
|
52
|
+
}
|
|
53
|
+
function sendDeviceRtpCapabilities(pid) {
|
|
54
|
+
participantId = pid;
|
|
55
|
+
socket.emit("device-rtp-capabilities", {
|
|
56
|
+
participantId: pid,
|
|
57
|
+
rtpCapabilities: device.rtpCapabilities
|
|
58
|
+
});
|
|
59
|
+
}
|
|
60
|
+
function createWebRtcTransport(direction, pid) {
|
|
61
|
+
return new Promise((resolve, reject) => {
|
|
62
|
+
const handleTransportCreated = (data) => {
|
|
63
|
+
if (data.type === direction) {
|
|
64
|
+
socket.off("transport-created", handleTransportCreated);
|
|
65
|
+
if (data.error) return reject(new Error(data.error));
|
|
66
|
+
resolve(data.params);
|
|
67
|
+
}
|
|
68
|
+
};
|
|
69
|
+
socket.on("transport-created", handleTransportCreated);
|
|
70
|
+
socket.emit("create-transport", { direction, participantId: pid });
|
|
71
|
+
setTimeout(() => {
|
|
72
|
+
socket.off("transport-created", handleTransportCreated);
|
|
73
|
+
reject(new Error(`Timeout waiting for ${direction} transport`));
|
|
74
|
+
}, 1e4);
|
|
75
|
+
});
|
|
76
|
+
}
|
|
77
|
+
function attemptIceRestart(direction) {
|
|
78
|
+
if (iceRestartAttempts >= MAX_ICE_RESTART_ATTEMPTS) {
|
|
79
|
+
emitTransportEvent("transport-failed", {
|
|
80
|
+
direction,
|
|
81
|
+
reason: "max-ice-restarts"
|
|
82
|
+
});
|
|
83
|
+
return;
|
|
84
|
+
}
|
|
85
|
+
iceRestartAttempts++;
|
|
86
|
+
const transport = direction === "send" ? sendTransport : recvTransport;
|
|
87
|
+
if (!transport) return;
|
|
88
|
+
socket.emit(
|
|
89
|
+
"request-ice-restart",
|
|
90
|
+
{ participantId, transportId: transport.id, direction },
|
|
91
|
+
async (response) => {
|
|
92
|
+
if (response.error) {
|
|
93
|
+
setTimeout(() => attemptIceRestart(direction), 2e3);
|
|
94
|
+
return;
|
|
95
|
+
}
|
|
96
|
+
if (response.iceParameters) {
|
|
97
|
+
try {
|
|
98
|
+
await transport.restartIce({ iceParameters: response.iceParameters });
|
|
99
|
+
} catch {
|
|
100
|
+
setTimeout(() => attemptIceRestart(direction), 2e3);
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
);
|
|
105
|
+
}
|
|
106
|
+
function connectSendTransport() {
|
|
107
|
+
sendTransport?.on(
|
|
108
|
+
"connect",
|
|
109
|
+
async ({ dtlsParameters }, callback, errback) => {
|
|
110
|
+
socket.emit(
|
|
111
|
+
"connect-transport",
|
|
112
|
+
{ transportId: sendTransport.id, dtlsParameters },
|
|
113
|
+
(response) => {
|
|
114
|
+
if (response.error) {
|
|
115
|
+
errback(new Error(response.error));
|
|
116
|
+
} else {
|
|
117
|
+
callback();
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
);
|
|
121
|
+
}
|
|
122
|
+
);
|
|
123
|
+
sendTransport?.on(
|
|
124
|
+
"produce",
|
|
125
|
+
async ({
|
|
126
|
+
kind,
|
|
127
|
+
rtpParameters,
|
|
128
|
+
appData
|
|
129
|
+
}, callback, errback) => {
|
|
130
|
+
socket.emit(
|
|
131
|
+
"produce",
|
|
132
|
+
{
|
|
133
|
+
transportId: sendTransport.id,
|
|
134
|
+
kind,
|
|
135
|
+
rtpParameters,
|
|
136
|
+
appData
|
|
137
|
+
},
|
|
138
|
+
(response) => {
|
|
139
|
+
if (response.error) {
|
|
140
|
+
errback(new Error(response.error));
|
|
141
|
+
} else if (response.producerId) {
|
|
142
|
+
callback({ id: response.producerId });
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
);
|
|
146
|
+
}
|
|
147
|
+
);
|
|
148
|
+
sendTransport?.on("connectionstatechange", (state) => {
|
|
149
|
+
sendTransportState = state;
|
|
150
|
+
switch (state) {
|
|
151
|
+
case "connected":
|
|
152
|
+
iceRestartAttempts = 0;
|
|
153
|
+
emitTransportEvent("transport-connected", { direction: "send" });
|
|
154
|
+
break;
|
|
155
|
+
case "disconnected":
|
|
156
|
+
emitTransportEvent("transport-disconnected", { direction: "send" });
|
|
157
|
+
attemptIceRestart("send");
|
|
158
|
+
break;
|
|
159
|
+
case "failed":
|
|
160
|
+
producers.clear();
|
|
161
|
+
emitTransportEvent("transport-failed", { direction: "send" });
|
|
162
|
+
socket.emit("transport-failed", { participantId, direction: "send" });
|
|
163
|
+
break;
|
|
164
|
+
case "closed":
|
|
165
|
+
producers.clear();
|
|
166
|
+
emitTransportEvent("transport-closed", { direction: "send" });
|
|
167
|
+
break;
|
|
168
|
+
case "connecting":
|
|
169
|
+
emitTransportEvent("transport-connecting", { direction: "send" });
|
|
170
|
+
break;
|
|
171
|
+
}
|
|
172
|
+
});
|
|
173
|
+
}
|
|
174
|
+
function connectRecvTransport() {
|
|
175
|
+
recvTransport?.on(
|
|
176
|
+
"connect",
|
|
177
|
+
async ({ dtlsParameters }, callback, errback) => {
|
|
178
|
+
socket.emit(
|
|
179
|
+
"connect-transport",
|
|
180
|
+
{ transportId: recvTransport.id, dtlsParameters },
|
|
181
|
+
(response) => {
|
|
182
|
+
if (response.error) {
|
|
183
|
+
errback(new Error(response.error));
|
|
184
|
+
} else {
|
|
185
|
+
callback();
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
);
|
|
189
|
+
}
|
|
190
|
+
);
|
|
191
|
+
recvTransport?.on("connectionstatechange", (state) => {
|
|
192
|
+
recvTransportState = state;
|
|
193
|
+
switch (state) {
|
|
194
|
+
case "connected":
|
|
195
|
+
iceRestartAttempts = 0;
|
|
196
|
+
emitTransportEvent("transport-connected", { direction: "recv" });
|
|
197
|
+
break;
|
|
198
|
+
case "disconnected":
|
|
199
|
+
emitTransportEvent("transport-disconnected", { direction: "recv" });
|
|
200
|
+
attemptIceRestart("recv");
|
|
201
|
+
break;
|
|
202
|
+
case "failed":
|
|
203
|
+
emitTransportEvent("transport-failed", { direction: "recv" });
|
|
204
|
+
socket.emit("transport-failed", { participantId, direction: "recv" });
|
|
205
|
+
break;
|
|
206
|
+
case "closed":
|
|
207
|
+
emitTransportEvent("transport-closed", { direction: "recv" });
|
|
208
|
+
break;
|
|
209
|
+
case "connecting":
|
|
210
|
+
emitTransportEvent("transport-connecting", { direction: "recv" });
|
|
211
|
+
break;
|
|
212
|
+
}
|
|
213
|
+
});
|
|
214
|
+
}
|
|
215
|
+
async function createSendTransport(pid) {
|
|
216
|
+
const params = await createWebRtcTransport("send", pid);
|
|
217
|
+
const { iceServers, ...transportParams } = params;
|
|
218
|
+
sendTransport = device.createSendTransport({
|
|
219
|
+
...transportParams,
|
|
220
|
+
iceServers: iceServers || []
|
|
221
|
+
});
|
|
222
|
+
connectSendTransport();
|
|
223
|
+
}
|
|
224
|
+
async function createRecvTransport(pid) {
|
|
225
|
+
const params = await createWebRtcTransport("recv", pid);
|
|
226
|
+
const { iceServers, ...transportParams } = params;
|
|
227
|
+
recvTransport = device.createRecvTransport({
|
|
228
|
+
...transportParams,
|
|
229
|
+
iceServers: iceServers || []
|
|
230
|
+
});
|
|
231
|
+
connectRecvTransport();
|
|
232
|
+
}
|
|
233
|
+
async function produce(track, appData) {
|
|
234
|
+
if (!sendTransport) throw new Error("Send transport not initialized");
|
|
235
|
+
const produceOptions = { track, appData };
|
|
236
|
+
const isScreenshare = appData?.isScreenshare === true;
|
|
237
|
+
if (track.kind === "video") {
|
|
238
|
+
if (isScreenshare) {
|
|
239
|
+
produceOptions.encodings = [
|
|
240
|
+
{
|
|
241
|
+
rid: "r0",
|
|
242
|
+
active: true,
|
|
243
|
+
maxBitrate: 5e5,
|
|
244
|
+
maxFramerate: 10,
|
|
245
|
+
scaleResolutionDownBy: 4
|
|
246
|
+
},
|
|
247
|
+
{
|
|
248
|
+
rid: "r1",
|
|
249
|
+
active: true,
|
|
250
|
+
maxBitrate: 15e5,
|
|
251
|
+
maxFramerate: 15,
|
|
252
|
+
scaleResolutionDownBy: 2
|
|
253
|
+
},
|
|
254
|
+
{
|
|
255
|
+
rid: "r2",
|
|
256
|
+
active: true,
|
|
257
|
+
maxBitrate: 3e6,
|
|
258
|
+
maxFramerate: 15,
|
|
259
|
+
scaleResolutionDownBy: 1
|
|
260
|
+
}
|
|
261
|
+
];
|
|
262
|
+
produceOptions.codecOptions = { videoGoogleStartBitrate: 1500 };
|
|
263
|
+
} else {
|
|
264
|
+
produceOptions.encodings = [
|
|
265
|
+
{
|
|
266
|
+
rid: "r0",
|
|
267
|
+
active: true,
|
|
268
|
+
maxBitrate: 1e5,
|
|
269
|
+
scaleResolutionDownBy: 4
|
|
270
|
+
},
|
|
271
|
+
{
|
|
272
|
+
rid: "r1",
|
|
273
|
+
active: true,
|
|
274
|
+
maxBitrate: 3e5,
|
|
275
|
+
scaleResolutionDownBy: 2
|
|
276
|
+
},
|
|
277
|
+
{
|
|
278
|
+
rid: "r2",
|
|
279
|
+
active: true,
|
|
280
|
+
maxBitrate: 9e5,
|
|
281
|
+
scaleResolutionDownBy: 1
|
|
282
|
+
}
|
|
283
|
+
];
|
|
284
|
+
produceOptions.codecOptions = { videoGoogleStartBitrate: 1e3 };
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
const producer = await sendTransport.produce(produceOptions);
|
|
288
|
+
producer.on("transportclose", () => {
|
|
289
|
+
producers.delete(producer.id);
|
|
290
|
+
});
|
|
291
|
+
producer.on("trackended", () => {
|
|
292
|
+
producer.close();
|
|
293
|
+
producers.delete(producer.id);
|
|
294
|
+
});
|
|
295
|
+
producers.set(producer.id, producer);
|
|
296
|
+
return producer;
|
|
297
|
+
}
|
|
298
|
+
async function consume(data) {
|
|
299
|
+
if (!recvTransport) throw new Error("Receive transport not set up");
|
|
300
|
+
const consumer = await recvTransport.consume({
|
|
301
|
+
id: data.consumerId,
|
|
302
|
+
producerId: data.producerId,
|
|
303
|
+
kind: data.kind,
|
|
304
|
+
rtpParameters: data.rtpParameters
|
|
305
|
+
});
|
|
306
|
+
consumer.on("transportclose", () => {
|
|
307
|
+
consumers.delete(consumer.id);
|
|
308
|
+
});
|
|
309
|
+
consumer.on("trackended", () => {
|
|
310
|
+
consumer.close();
|
|
311
|
+
consumers.delete(consumer.id);
|
|
312
|
+
});
|
|
313
|
+
consumers.set(consumer.id, consumer);
|
|
314
|
+
return { consumer, track: consumer.track };
|
|
315
|
+
}
|
|
316
|
+
async function resumeConsumer(consumerId) {
|
|
317
|
+
return new Promise((resolve, reject) => {
|
|
318
|
+
socket.emit(
|
|
319
|
+
"resume-consumer",
|
|
320
|
+
{ consumerId, participantId },
|
|
321
|
+
(response) => {
|
|
322
|
+
if (response.error) {
|
|
323
|
+
reject(new Error(response.error));
|
|
324
|
+
} else {
|
|
325
|
+
resolve();
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
);
|
|
329
|
+
});
|
|
330
|
+
}
|
|
331
|
+
function getTransportStates() {
|
|
332
|
+
return { send: sendTransportState, recv: recvTransportState };
|
|
333
|
+
}
|
|
334
|
+
function getConsumers() {
|
|
335
|
+
return consumers;
|
|
336
|
+
}
|
|
337
|
+
function close() {
|
|
338
|
+
producers.forEach((p) => p.close());
|
|
339
|
+
consumers.forEach((c) => c.close());
|
|
340
|
+
if (sendTransport) sendTransport.close();
|
|
341
|
+
if (recvTransport) recvTransport.close();
|
|
342
|
+
producers.clear();
|
|
343
|
+
consumers.clear();
|
|
344
|
+
}
|
|
345
|
+
return {
|
|
346
|
+
loadDevice,
|
|
347
|
+
sendDeviceRtpCapabilities,
|
|
348
|
+
createSendTransport,
|
|
349
|
+
createRecvTransport,
|
|
350
|
+
produce,
|
|
351
|
+
consume,
|
|
352
|
+
resumeConsumer,
|
|
353
|
+
getTransportStates,
|
|
354
|
+
getConsumers,
|
|
355
|
+
close,
|
|
356
|
+
onTransportEvent: (event, listener) => internalBus.on(event, listener)
|
|
357
|
+
};
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
// src/channels/spatial/SpatialAudioTypes.ts
|
|
361
|
+
var DEFAULT_SPATIAL_CONFIG = {
|
|
362
|
+
refDistance: 0.5,
|
|
363
|
+
// Full volume at 0.5m or closer
|
|
364
|
+
maxDistance: 15,
|
|
365
|
+
// Silent at 15m (audio cutoff at 15.1m+)
|
|
366
|
+
rolloffFactor: 1,
|
|
367
|
+
// Unused - we use quadratic falloff
|
|
368
|
+
unit: "auto"
|
|
369
|
+
};
|
|
370
|
+
|
|
371
|
+
// src/utils/spatial/distance-calc.ts
|
|
372
|
+
function getDistanceBetween(a, b) {
|
|
373
|
+
const dx = b.x - a.x;
|
|
374
|
+
const dy = b.y - a.y;
|
|
375
|
+
const dz = b.z - a.z;
|
|
376
|
+
return Math.sqrt(dx * dx + dy * dy + dz * dz);
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
// src/utils/spatial/gain-calc.ts
|
|
380
|
+
var DEFAULT_GAIN_CONFIG = {
|
|
381
|
+
minDistance: 0.5,
|
|
382
|
+
// Full volume at 0.5m or closer
|
|
383
|
+
maxDistance: 15
|
|
384
|
+
// Silent at 15m
|
|
385
|
+
};
|
|
386
|
+
function calculateLogarithmicGain(distance, config = {}) {
|
|
387
|
+
const { minDistance, maxDistance } = {
|
|
388
|
+
...DEFAULT_GAIN_CONFIG,
|
|
389
|
+
...config
|
|
390
|
+
};
|
|
391
|
+
if (distance >= maxDistance) return 0;
|
|
392
|
+
if (distance <= minDistance) return 100;
|
|
393
|
+
const range = maxDistance - minDistance;
|
|
394
|
+
const normalizedDistance = (distance - minDistance) / range;
|
|
395
|
+
const remainingRatio = 1 - normalizedDistance;
|
|
396
|
+
const gain = 100 * remainingRatio * remainingRatio * remainingRatio;
|
|
397
|
+
return Math.max(0, gain);
|
|
398
|
+
}
|
|
399
|
+
|
|
400
|
+
// src/utils/spatial/pan-calc.ts
|
|
401
|
+
function calculateListenerRight(yawDegrees) {
|
|
402
|
+
const yawRad = yawDegrees * Math.PI / 180;
|
|
403
|
+
return {
|
|
404
|
+
x: Math.cos(yawRad),
|
|
405
|
+
z: -Math.sin(yawRad)
|
|
406
|
+
};
|
|
407
|
+
}
|
|
408
|
+
function calculatePanFromPositions(listenerPos, sourcePos, listenerRight) {
|
|
409
|
+
const vecToSource = {
|
|
410
|
+
x: sourcePos.x - listenerPos.x,
|
|
411
|
+
z: sourcePos.z - listenerPos.z
|
|
412
|
+
};
|
|
413
|
+
const dxLocal = vecToSource.x * listenerRight.x + vecToSource.z * listenerRight.z;
|
|
414
|
+
const listenerForward = { x: -listenerRight.z, z: listenerRight.x };
|
|
415
|
+
const dzLocal = vecToSource.x * listenerForward.x + vecToSource.z * listenerForward.z;
|
|
416
|
+
const angleToSource = Math.atan2(dxLocal, dzLocal);
|
|
417
|
+
const rawPanValue = Math.sin(angleToSource);
|
|
418
|
+
return rawPanValue;
|
|
419
|
+
}
|
|
420
|
+
|
|
421
|
+
// src/utils/spatial/head-position.ts
|
|
422
|
+
var DEFAULT_HEAD_HEIGHT = 1.6;
|
|
423
|
+
function computeHeadPosition(bodyPosition, headHeight = DEFAULT_HEAD_HEIGHT) {
|
|
424
|
+
return {
|
|
425
|
+
x: bodyPosition.x,
|
|
426
|
+
y: bodyPosition.y + headHeight,
|
|
427
|
+
z: bodyPosition.z
|
|
428
|
+
};
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
// src/utils/position/normalize.ts
|
|
432
|
+
function normalizePositionUnits(position, unit = "auto") {
|
|
433
|
+
if (unit === "meters") {
|
|
434
|
+
return { ...position };
|
|
435
|
+
}
|
|
436
|
+
if (unit === "centimeters") {
|
|
437
|
+
return {
|
|
438
|
+
x: position.x / 100,
|
|
439
|
+
y: position.y / 100,
|
|
440
|
+
z: position.z / 100
|
|
441
|
+
};
|
|
442
|
+
}
|
|
443
|
+
const maxAxis = Math.max(
|
|
444
|
+
Math.abs(position.x),
|
|
445
|
+
Math.abs(position.y),
|
|
446
|
+
Math.abs(position.z)
|
|
447
|
+
);
|
|
448
|
+
if (maxAxis > 50) {
|
|
449
|
+
return {
|
|
450
|
+
x: position.x / 100,
|
|
451
|
+
y: position.y / 100,
|
|
452
|
+
z: position.z / 100
|
|
453
|
+
};
|
|
454
|
+
}
|
|
455
|
+
return { ...position };
|
|
456
|
+
}
|
|
457
|
+
|
|
458
|
+
// src/utils/position/snap.ts
|
|
459
|
+
var SNAP_CONFIG = {
|
|
460
|
+
threshold: 0.3
|
|
461
|
+
};
|
|
462
|
+
function createPositionSnapCache(threshold = SNAP_CONFIG.threshold) {
|
|
463
|
+
const cache = /* @__PURE__ */ new Map();
|
|
464
|
+
function snap(position, id) {
|
|
465
|
+
const cached = cache.get(id);
|
|
466
|
+
if (!cached || cached.x === 0 && cached.y === 0 && cached.z === 0) {
|
|
467
|
+
cache.set(id, { ...position });
|
|
468
|
+
return position;
|
|
469
|
+
}
|
|
470
|
+
const dx = position.x - cached.x;
|
|
471
|
+
const dy = position.y - cached.y;
|
|
472
|
+
const dz = position.z - cached.z;
|
|
473
|
+
const movedDistance = Math.sqrt(dx * dx + dy * dy + dz * dz);
|
|
474
|
+
if (movedDistance > threshold) {
|
|
475
|
+
cache.set(id, { ...position });
|
|
476
|
+
return position;
|
|
477
|
+
}
|
|
478
|
+
return cached;
|
|
479
|
+
}
|
|
480
|
+
return {
|
|
481
|
+
snap,
|
|
482
|
+
clear: (id) => {
|
|
483
|
+
cache.delete(id);
|
|
484
|
+
},
|
|
485
|
+
clearAll: () => {
|
|
486
|
+
cache.clear();
|
|
487
|
+
}
|
|
488
|
+
};
|
|
489
|
+
}
|
|
490
|
+
|
|
491
|
+
// src/utils/smoothing/pan-smoothing.ts
|
|
492
|
+
var DEFAULT_PAN_SMOOTHER_OPTIONS = {
|
|
493
|
+
smoothingFactor: 0.3,
|
|
494
|
+
changeThreshold: 0.02,
|
|
495
|
+
centerDeadZone: 0.03
|
|
496
|
+
};
|
|
497
|
+
function createPanSmoother(options = {}) {
|
|
498
|
+
const opts = { ...DEFAULT_PAN_SMOOTHER_OPTIONS, ...options };
|
|
499
|
+
const smoothedValues = /* @__PURE__ */ new Map();
|
|
500
|
+
const lastUpdateTime = /* @__PURE__ */ new Map();
|
|
501
|
+
function smooth(participantId, newPanValue) {
|
|
502
|
+
const previousPan = smoothedValues.get(participantId);
|
|
503
|
+
const now = Date.now();
|
|
504
|
+
const lastTime = lastUpdateTime.get(participantId) || 0;
|
|
505
|
+
const timeSinceLastUpdate = now - lastTime;
|
|
506
|
+
let targetPan = newPanValue;
|
|
507
|
+
if (Math.abs(newPanValue) < opts.centerDeadZone) {
|
|
508
|
+
targetPan = 0;
|
|
509
|
+
}
|
|
510
|
+
if (previousPan === void 0) {
|
|
511
|
+
smoothedValues.set(participantId, targetPan);
|
|
512
|
+
lastUpdateTime.set(participantId, now);
|
|
513
|
+
return targetPan;
|
|
514
|
+
}
|
|
515
|
+
const panChange = Math.abs(targetPan - previousPan);
|
|
516
|
+
if (panChange < opts.changeThreshold) {
|
|
517
|
+
return previousPan;
|
|
518
|
+
}
|
|
519
|
+
let effectiveSmoothingFactor = opts.smoothingFactor;
|
|
520
|
+
const signFlipped = previousPan > 0 && targetPan < 0 || previousPan < 0 && targetPan > 0;
|
|
521
|
+
const bothNearCenter = Math.abs(previousPan) < 0.15 && Math.abs(targetPan) < 0.15;
|
|
522
|
+
const isLikelyJitter = signFlipped && panChange > 1 && timeSinceLastUpdate < 60;
|
|
523
|
+
if (isLikelyJitter) {
|
|
524
|
+
effectiveSmoothingFactor = 0.7;
|
|
525
|
+
} else if (bothNearCenter) {
|
|
526
|
+
effectiveSmoothingFactor = Math.min(opts.smoothingFactor + 0.1, 0.5);
|
|
527
|
+
}
|
|
528
|
+
const smoothedPan = previousPan * effectiveSmoothingFactor + targetPan * (1 - effectiveSmoothingFactor);
|
|
529
|
+
const finalPan = Math.abs(smoothedPan) < opts.centerDeadZone ? 0 : smoothedPan;
|
|
530
|
+
smoothedValues.set(participantId, finalPan);
|
|
531
|
+
lastUpdateTime.set(participantId, now);
|
|
532
|
+
return finalPan;
|
|
533
|
+
}
|
|
534
|
+
function clear(participantId) {
|
|
535
|
+
smoothedValues.delete(participantId);
|
|
536
|
+
lastUpdateTime.delete(participantId);
|
|
537
|
+
}
|
|
538
|
+
function clearAll() {
|
|
539
|
+
smoothedValues.clear();
|
|
540
|
+
lastUpdateTime.clear();
|
|
541
|
+
}
|
|
542
|
+
return { smooth, clear, clearAll };
|
|
543
|
+
}
|
|
544
|
+
|
|
545
|
+
// src/utils/smoothing/gain-smoothing.ts
|
|
546
|
+
function applyGainSmooth(gainNode, targetGain, audioContext, timeConstant = 0.1) {
|
|
547
|
+
try {
|
|
548
|
+
gainNode.gain.setTargetAtTime(
|
|
549
|
+
targetGain,
|
|
550
|
+
audioContext.currentTime,
|
|
551
|
+
timeConstant
|
|
552
|
+
);
|
|
553
|
+
} catch (err) {
|
|
554
|
+
gainNode.gain.value = targetGain;
|
|
555
|
+
}
|
|
556
|
+
}
|
|
557
|
+
function applyStereoPanSmooth(stereoPanner, panValue, audioContext, timeConstant = 0.05) {
|
|
558
|
+
try {
|
|
559
|
+
stereoPanner.pan.setTargetAtTime(
|
|
560
|
+
panValue,
|
|
561
|
+
audioContext.currentTime,
|
|
562
|
+
timeConstant
|
|
563
|
+
);
|
|
564
|
+
} catch (err) {
|
|
565
|
+
stereoPanner.pan.value = panValue;
|
|
566
|
+
}
|
|
567
|
+
}
|
|
568
|
+
var L2RegularizerCompat = class {
|
|
569
|
+
constructor(config) {
|
|
570
|
+
this.l2 = config?.l2 ?? 0;
|
|
571
|
+
}
|
|
572
|
+
apply(x) {
|
|
573
|
+
return tf.tidy(
|
|
574
|
+
() => tf.sum(tf.square(x)).mul(tf.scalar(this.l2))
|
|
575
|
+
);
|
|
576
|
+
}
|
|
577
|
+
getConfig() {
|
|
578
|
+
return { l2: this.l2 };
|
|
579
|
+
}
|
|
580
|
+
};
|
|
581
|
+
L2RegularizerCompat.className = "L2";
|
|
582
|
+
tf.serialization.registerClass(L2RegularizerCompat);
|
|
583
|
+
var _GRUCellResetAfterSupport = class _GRUCellResetAfterSupport extends tf.layers.Layer {
|
|
584
|
+
constructor(config) {
|
|
585
|
+
super(config);
|
|
586
|
+
this.dropoutMask = null;
|
|
587
|
+
this.recurrentDropoutMask = null;
|
|
588
|
+
this.units = config.units;
|
|
589
|
+
this.resetAfter = (config.resetAfter ?? config.reset_after) !== false;
|
|
590
|
+
this.useBiasFlag = (config.useBias ?? config.use_bias) !== false;
|
|
591
|
+
this.stateSize = this.units;
|
|
592
|
+
this.activationFn = _GRUCellResetAfterSupport.makeActivation(
|
|
593
|
+
config.activation || "tanh"
|
|
594
|
+
);
|
|
595
|
+
this.recurrentActivationFn = _GRUCellResetAfterSupport.makeActivation(
|
|
596
|
+
config.recurrentActivation ?? config.recurrent_activation ?? "sigmoid"
|
|
597
|
+
// Keras default (model.json uses 'sigmoid', NOT 'hardSigmoid')
|
|
598
|
+
);
|
|
599
|
+
}
|
|
600
|
+
build(inputShape) {
|
|
601
|
+
let shape;
|
|
602
|
+
if (Array.isArray(inputShape[0])) {
|
|
603
|
+
shape = inputShape[0];
|
|
604
|
+
} else {
|
|
605
|
+
shape = inputShape;
|
|
606
|
+
}
|
|
607
|
+
const inputDim = shape[shape.length - 1];
|
|
608
|
+
this.kernelW = this.addWeight(
|
|
609
|
+
"kernel",
|
|
610
|
+
[inputDim, this.units * 3],
|
|
611
|
+
"float32",
|
|
612
|
+
tf.initializers.glorotUniform({})
|
|
613
|
+
);
|
|
614
|
+
this.recurrentKernelW = this.addWeight(
|
|
615
|
+
"recurrent_kernel",
|
|
616
|
+
[this.units, this.units * 3],
|
|
617
|
+
"float32",
|
|
618
|
+
tf.initializers.orthogonal({})
|
|
619
|
+
);
|
|
620
|
+
if (this.useBiasFlag) {
|
|
621
|
+
const biasShape = this.resetAfter ? [2, this.units * 3] : [this.units * 3];
|
|
622
|
+
this.biasW = this.addWeight(
|
|
623
|
+
"bias",
|
|
624
|
+
biasShape,
|
|
625
|
+
"float32",
|
|
626
|
+
tf.initializers.zeros()
|
|
627
|
+
);
|
|
628
|
+
}
|
|
629
|
+
this.built = true;
|
|
630
|
+
}
|
|
631
|
+
call(inputs, _kwargs) {
|
|
632
|
+
return tf.tidy(() => {
|
|
633
|
+
const input = inputs[0];
|
|
634
|
+
const hPrev = inputs[1];
|
|
635
|
+
let matrixX = input.matMul(this.kernelW.read());
|
|
636
|
+
if (this.resetAfter) {
|
|
637
|
+
let bKernel = null;
|
|
638
|
+
let bRecurrent = null;
|
|
639
|
+
if (this.useBiasFlag && this.biasW) {
|
|
640
|
+
const b = this.biasW.read();
|
|
641
|
+
bKernel = b.slice([0, 0], [1, -1]).squeeze([0]);
|
|
642
|
+
bRecurrent = b.slice([1, 0], [1, -1]).squeeze([0]);
|
|
643
|
+
}
|
|
644
|
+
if (bKernel) matrixX = matrixX.add(bKernel);
|
|
645
|
+
const matrixInner = hPrev.matMul(this.recurrentKernelW.read());
|
|
646
|
+
const [xZ, xR, xH] = tf.split(matrixX, 3, matrixX.rank - 1);
|
|
647
|
+
const [iZ, iR, iH] = tf.split(matrixInner, 3, matrixInner.rank - 1);
|
|
648
|
+
let rZ = iZ, rR = iR, rH = iH;
|
|
649
|
+
if (bRecurrent) {
|
|
650
|
+
const [rbZ, rbR, rbH] = tf.split(bRecurrent, 3, 0);
|
|
651
|
+
rZ = iZ.add(rbZ);
|
|
652
|
+
rR = iR.add(rbR);
|
|
653
|
+
rH = iH.add(rbH);
|
|
654
|
+
}
|
|
655
|
+
const z = this.recurrentActivationFn(xZ.add(rZ));
|
|
656
|
+
const r = this.recurrentActivationFn(xR.add(rR));
|
|
657
|
+
const hh = this.activationFn(xH.add(r.mul(rH)));
|
|
658
|
+
const h = tf.add(tf.mul(z, hPrev), tf.mul(tf.sub(tf.scalar(1), z), hh));
|
|
659
|
+
return [h, h];
|
|
660
|
+
} else {
|
|
661
|
+
if (this.useBiasFlag && this.biasW) {
|
|
662
|
+
matrixX = matrixX.add(this.biasW.read());
|
|
663
|
+
}
|
|
664
|
+
const rk = this.recurrentKernelW.read();
|
|
665
|
+
const [rk1, rk2] = tf.split(
|
|
666
|
+
rk,
|
|
667
|
+
[2 * this.units, this.units],
|
|
668
|
+
rk.rank - 1
|
|
669
|
+
);
|
|
670
|
+
const matrixInner = hPrev.matMul(rk1);
|
|
671
|
+
const [xZ, xR, xH] = tf.split(matrixX, 3, matrixX.rank - 1);
|
|
672
|
+
const [recZ, recR] = tf.split(matrixInner, 2, matrixInner.rank - 1);
|
|
673
|
+
const z = this.recurrentActivationFn(xZ.add(recZ));
|
|
674
|
+
const r = this.recurrentActivationFn(xR.add(recR));
|
|
675
|
+
const recH = tf.mul(r, hPrev).matMul(rk2);
|
|
676
|
+
const hh = this.activationFn(xH.add(recH));
|
|
677
|
+
const h = tf.add(tf.mul(z, hPrev), tf.mul(tf.sub(tf.scalar(1), z), hh));
|
|
678
|
+
return [h, h];
|
|
679
|
+
}
|
|
680
|
+
});
|
|
681
|
+
}
|
|
682
|
+
static makeActivation(name) {
|
|
683
|
+
switch (name.toLowerCase()) {
|
|
684
|
+
case "tanh":
|
|
685
|
+
return (x) => x.tanh();
|
|
686
|
+
case "sigmoid":
|
|
687
|
+
return (x) => x.sigmoid();
|
|
688
|
+
case "hardsigmoid":
|
|
689
|
+
case "hard_sigmoid":
|
|
690
|
+
return (x) => tf.clipByValue(x.mul(tf.scalar(0.2)).add(tf.scalar(0.5)), 0, 1);
|
|
691
|
+
case "relu":
|
|
692
|
+
return (x) => x.relu();
|
|
693
|
+
case "linear":
|
|
694
|
+
return (x) => x;
|
|
695
|
+
default:
|
|
696
|
+
return (x) => x.sigmoid();
|
|
697
|
+
}
|
|
698
|
+
}
|
|
699
|
+
getConfig() {
|
|
700
|
+
return {
|
|
701
|
+
...super.getConfig(),
|
|
702
|
+
units: this.units,
|
|
703
|
+
useBias: this.useBiasFlag,
|
|
704
|
+
resetAfter: this.resetAfter
|
|
705
|
+
};
|
|
706
|
+
}
|
|
707
|
+
};
|
|
708
|
+
// Must match TF.js serialization class name
|
|
709
|
+
_GRUCellResetAfterSupport.className = "GRUCell";
|
|
710
|
+
var GRUCellResetAfterSupport = _GRUCellResetAfterSupport;
|
|
711
|
+
tf.serialization.registerClass(GRUCellResetAfterSupport);
|
|
712
|
+
var GRULayerWithResetAfter = class {
|
|
713
|
+
static fromConfig(_cls, config) {
|
|
714
|
+
const layerName = config.name;
|
|
715
|
+
const cell = new GRUCellResetAfterSupport({
|
|
716
|
+
...config,
|
|
717
|
+
name: `gru_cell`,
|
|
718
|
+
// → weights: "gru_96/gru_cell/kernel" ✅
|
|
719
|
+
// Normalise keys: Keras JSON is snake_case; TF.js internals are camelCase
|
|
720
|
+
useBias: config.useBias ?? config.use_bias ?? true,
|
|
721
|
+
recurrentActivation: config.recurrentActivation ?? config.recurrent_activation ?? "sigmoid",
|
|
722
|
+
resetAfter: (config.resetAfter ?? config.reset_after) !== false,
|
|
723
|
+
returnSequences: config.returnSequences ?? config.return_sequences ?? false
|
|
724
|
+
});
|
|
725
|
+
return tf.layers.rnn({
|
|
726
|
+
cell,
|
|
727
|
+
returnSequences: config.returnSequences ?? config.return_sequences ?? false,
|
|
728
|
+
returnState: config.returnState ?? config.return_state ?? false,
|
|
729
|
+
goBackwards: config.goBackwards ?? config.go_backwards ?? false,
|
|
730
|
+
stateful: config.stateful ?? false,
|
|
731
|
+
unroll: config.unroll ?? false,
|
|
732
|
+
// Always use float32 — model.json exports dtype as a Keras Policy object
|
|
733
|
+
// (e.g. {class_name:"Policy", config:{name:"mixed_float16"}}) which TF.js
|
|
734
|
+
// cannot use as a DataType string. Weights are float32 anyway (UINT8 quantized
|
|
735
|
+
// with original_dtype=float32), so float32 is always correct here.
|
|
736
|
+
dtype: "float32",
|
|
737
|
+
name: layerName
|
|
738
|
+
// e.g. "gru_96" — RNN layer keeps this name
|
|
739
|
+
});
|
|
740
|
+
}
|
|
741
|
+
};
|
|
742
|
+
GRULayerWithResetAfter.className = "GRU";
|
|
743
|
+
tf.serialization.registerClass(GRULayerWithResetAfter);
|
|
744
|
+
var WEBRTC_SAMPLE_RATE = 48e3;
|
|
745
|
+
function createMLNoiseSuppressor() {
|
|
746
|
+
const instance = new MLNoiseSuppressor();
|
|
747
|
+
return {
|
|
748
|
+
initialize: (url) => instance.initialize(url),
|
|
749
|
+
isReady: () => instance.isReady(),
|
|
750
|
+
getModelConfig: () => instance.getModelConfig(),
|
|
751
|
+
getInfo: () => instance.getInfo(),
|
|
752
|
+
computeGainsFromFeatures: (f, s, n, p) => instance.computeGainsFromFeatures(f, s, n, p),
|
|
753
|
+
clearParticipantSmoothingState: (id) => instance.clearParticipantSmoothingState(id),
|
|
754
|
+
processAudioSync: (buf) => instance.processAudioSync(buf),
|
|
755
|
+
reset: () => instance.reset(),
|
|
756
|
+
dispose: () => instance.dispose()
|
|
757
|
+
};
|
|
758
|
+
}
|
|
759
|
+
var MLNoiseSuppressor = class _MLNoiseSuppressor {
|
|
760
|
+
constructor() {
|
|
761
|
+
this.model = null;
|
|
762
|
+
this.config = null;
|
|
763
|
+
this.normStats = null;
|
|
764
|
+
this.isInitialized = false;
|
|
765
|
+
// Temporal smoothing state — per-participant to avoid cross-participant corruption
|
|
766
|
+
// when multiple async computeGainsFromFeatures() calls run concurrently.
|
|
767
|
+
this.prevMaskMap = /* @__PURE__ */ new Map();
|
|
768
|
+
// Faster smoothing: 0.70 means 30% of new value per frame (~3 frames to stabilise).
|
|
769
|
+
// Old 0.92 was too slow — first speech frame had smoothed max≈0.08 which fell below
|
|
770
|
+
// the isSpeechFrame threshold and silenced the first ~200ms of every utterance.
|
|
771
|
+
this.SMOOTHING_ALPHA = 0.7;
|
|
772
|
+
// Mel filterbank cache (built once, reused every frame)
|
|
773
|
+
this.melFilterbank = null;
|
|
774
|
+
// ── Google Meet-style ring buffer architecture ─────────────────────────────
|
|
775
|
+
// Process audio in exact hop_length (80 sample) steps at 16kHz.
|
|
776
|
+
// Each hop: STFT → apply cached FFT gains → ISTFT → OLA.
|
|
777
|
+
// Model runs async, updates cachedFftGains after each inference.
|
|
778
|
+
// Audio thread cost: ~2ms. Model inference: async, never blocks.
|
|
779
|
+
// Persistent input ring buffer (16kHz) — accumulates samples across callbacks.
|
|
780
|
+
this.inRing = new Float32Array(8192);
|
|
781
|
+
this.inRingLen = 0;
|
|
782
|
+
// OLA accumulator (16kHz) — each hop's ISTFT output is overlap-added here.
|
|
783
|
+
this.outAccum = new Float32Array(16384);
|
|
784
|
+
this.outNorm = new Float32Array(16384);
|
|
785
|
+
// sum of hann² for normalisation
|
|
786
|
+
// Per-FFT-bin gains from last completed async inference.
|
|
787
|
+
// All-ones until first inference completes → transparent passthrough.
|
|
788
|
+
this.cachedFftGains = null;
|
|
789
|
+
this.inferenceInFlight = false;
|
|
790
|
+
// Hann window cache (allocated once per N_FFT)
|
|
791
|
+
this.hannWinCache = null;
|
|
792
|
+
// Rolling mel frame history — seq_len frames of mel features for model input.
|
|
793
|
+
this.melFrameHistory = [];
|
|
794
|
+
// Resampling state (for 16kHz models)
|
|
795
|
+
this.needsResampling = false;
|
|
796
|
+
this.resampleRatio = 1;
|
|
797
|
+
// WEBRTC_SAMPLE_RATE / model.sample_rate
|
|
798
|
+
// Legacy fields (kept for isReady / reset API compatibility)
|
|
799
|
+
this.lastComplexFrames = [];
|
|
800
|
+
this.VOICE_FUNDAMENTAL_MIN = 80;
|
|
801
|
+
this.VOICE_FUNDAMENTAL_MAX = 500;
|
|
802
|
+
this.frameCounter = 0;
|
|
803
|
+
this.skipFrames = 0;
|
|
804
|
+
this.lastProcessedOutput = null;
|
|
805
|
+
this.processingTimes = [];
|
|
806
|
+
this.MAX_PROCESSING_TIME_MS = 42;
|
|
807
|
+
}
|
|
808
|
+
/**
|
|
809
|
+
* Initialize ML model for noise suppression
|
|
810
|
+
* @param modelUrl Path to model.json file
|
|
811
|
+
*/
|
|
812
|
+
async initialize(modelUrl) {
|
|
813
|
+
try {
|
|
814
|
+
try {
|
|
815
|
+
await tf.setBackend("cpu");
|
|
816
|
+
} catch {
|
|
817
|
+
await tf.setBackend("webgl");
|
|
818
|
+
}
|
|
819
|
+
await tf.ready();
|
|
820
|
+
console.log(
|
|
821
|
+
`[MLNoiseSuppressor] TF.js backend ready: ${tf.getBackend()}`
|
|
822
|
+
);
|
|
823
|
+
const baseUrlForWeights = modelUrl.split("?")[0].replace(/model\.json$/, "");
|
|
824
|
+
const [modelJsonResp, binResp] = await Promise.all([
|
|
825
|
+
fetch(modelUrl),
|
|
826
|
+
fetch(`${baseUrlForWeights}group1-shard1of1.bin`)
|
|
827
|
+
]);
|
|
828
|
+
const modelJsonData = await modelJsonResp.json();
|
|
829
|
+
const weightData = await binResp.arrayBuffer();
|
|
830
|
+
const remappedWeightSpecs = modelJsonData.weightsManifest[0].weights.map(
|
|
831
|
+
(w) => ({ ...w, name: w.name.replace("/gru_cell/", "/") })
|
|
832
|
+
);
|
|
833
|
+
this.model = await tf.loadLayersModel(
|
|
834
|
+
tf.io.fromMemory({
|
|
835
|
+
modelTopology: modelJsonData.modelTopology,
|
|
836
|
+
weightSpecs: remappedWeightSpecs,
|
|
837
|
+
weightData,
|
|
838
|
+
format: modelJsonData.format,
|
|
839
|
+
generatedBy: modelJsonData.generatedBy,
|
|
840
|
+
convertedBy: modelJsonData.convertedBy
|
|
841
|
+
})
|
|
842
|
+
);
|
|
843
|
+
console.log(
|
|
844
|
+
`[MLNoiseSuppressor] Model loaded \u2014 ${this.model.countParams().toLocaleString()} params | backend: ${tf.getBackend()} | url: ${modelUrl}`
|
|
845
|
+
);
|
|
846
|
+
const baseUrl = modelUrl.substring(0, modelUrl.lastIndexOf("/"));
|
|
847
|
+
const configResponse = await fetch(`${baseUrl}/model_config.json`);
|
|
848
|
+
this.config = await configResponse.json();
|
|
849
|
+
try {
|
|
850
|
+
const normResponse = await fetch(`${baseUrl}/normalization_stats.json`);
|
|
851
|
+
this.normStats = await normResponse.json();
|
|
852
|
+
console.log(
|
|
853
|
+
`[MLNoiseSuppressor] Normalization stats loaded \u2014 mean: ${this.normStats.mean.toFixed(4)}, std: ${this.normStats.std.toFixed(4)}`
|
|
854
|
+
);
|
|
855
|
+
} catch (e) {
|
|
856
|
+
console.warn(
|
|
857
|
+
`[MLNoiseSuppressor] normalization_stats.json not found, using defaults (mean=0, std=1)`
|
|
858
|
+
);
|
|
859
|
+
this.normStats = { mean: 0, std: 1 };
|
|
860
|
+
}
|
|
861
|
+
const modelSampleRate = this.config.sample_rate || 48e3;
|
|
862
|
+
if (modelSampleRate !== WEBRTC_SAMPLE_RATE) {
|
|
863
|
+
this.needsResampling = true;
|
|
864
|
+
this.resampleRatio = WEBRTC_SAMPLE_RATE / modelSampleRate;
|
|
865
|
+
console.log(
|
|
866
|
+
`[MLNoiseSuppressor] Resampling enabled: ${WEBRTC_SAMPLE_RATE}Hz \u2192 ${modelSampleRate}Hz (ratio: ${this.resampleRatio})`
|
|
867
|
+
);
|
|
868
|
+
} else {
|
|
869
|
+
this.needsResampling = false;
|
|
870
|
+
this.resampleRatio = 1;
|
|
871
|
+
}
|
|
872
|
+
const seqLen = this.config.sequence_length || 8;
|
|
873
|
+
const nMels = this.config.n_mels || 40;
|
|
874
|
+
console.log(
|
|
875
|
+
`[MLNoiseSuppressor] Warming up model (${seqLen} \xD7 ${nMels})...`
|
|
876
|
+
);
|
|
877
|
+
const warmupInput = tf.zeros([1, seqLen, nMels]);
|
|
878
|
+
for (let w = 0; w < 3; w++) {
|
|
879
|
+
const warmupOut = this.model.predict(warmupInput);
|
|
880
|
+
warmupOut.dataSync();
|
|
881
|
+
warmupOut.dispose();
|
|
882
|
+
}
|
|
883
|
+
warmupInput.dispose();
|
|
884
|
+
const diagInput = tf.fill([1, seqLen, nMels], 0.5);
|
|
885
|
+
const diagOut = this.model.predict(diagInput);
|
|
886
|
+
const diagData = new Float32Array(await diagOut.dataSync());
|
|
887
|
+
const diagMax = Math.max(...Array.from(diagData));
|
|
888
|
+
const diagMin = Math.min(...Array.from(diagData));
|
|
889
|
+
diagOut.dispose();
|
|
890
|
+
diagInput.dispose();
|
|
891
|
+
console.log(
|
|
892
|
+
`[MLNoiseSuppressor] \u26A0\uFE0F Weight check (input=0.5): output max=${diagMax.toFixed(4)} min=${diagMin.toFixed(4)} \u2014 ${diagMax > 0.01 ? "\u2705 weights LOADED" : "\u274C weights NOT loaded (all-zero output)"}`
|
|
893
|
+
);
|
|
894
|
+
console.log(
|
|
895
|
+
`[MLNoiseSuppressor] \u{1F4CB} TF.js weight variable names (${this.model.weights.length} total):`
|
|
896
|
+
);
|
|
897
|
+
this.model.weights.forEach((w, i) => {
|
|
898
|
+
const vals = w.read().dataSync();
|
|
899
|
+
const wMax = Math.max(...Array.from(vals).slice(0, 20));
|
|
900
|
+
const wMin = Math.min(...Array.from(vals).slice(0, 20));
|
|
901
|
+
console.log(
|
|
902
|
+
` [${i}] ${w.name} shape=${JSON.stringify(w.shape)} val_range=[${wMin.toFixed(4)}, ${wMax.toFixed(4)}]${Math.abs(wMax) < 1e-6 && Math.abs(wMin) < 1e-6 ? " \u2190 \u274C ALL ZEROS (not loaded!)" : ""}`
|
|
903
|
+
);
|
|
904
|
+
});
|
|
905
|
+
console.log(`[MLNoiseSuppressor] Warmup done`);
|
|
906
|
+
this.isInitialized = true;
|
|
907
|
+
console.log(`[MLNoiseSuppressor] \u2705 Ready \u2014 noise suppression is ACTIVE`);
|
|
908
|
+
console.log(
|
|
909
|
+
`[MLNoiseSuppressor] Config: ${modelSampleRate}Hz, ${this.config.n_mels} mels, n_fft=${this.config.n_fft || 2048}`
|
|
910
|
+
);
|
|
911
|
+
} catch (error) {
|
|
912
|
+
console.error(`[MLNoiseSuppressor] \u274C Initialization failed:`, error);
|
|
913
|
+
throw error;
|
|
914
|
+
}
|
|
915
|
+
}
|
|
916
|
+
/**
|
|
917
|
+
* Google Meet-style ring buffer noise suppression.
|
|
918
|
+
*
|
|
919
|
+
* Trains / Google Meet process audio in exact hop_length steps (80 samples = 5ms @ 16kHz).
|
|
920
|
+
* Each step:
|
|
921
|
+
* 1. Extract one STFT frame (n_fft=512) with Hann window — zero-pad if near end
|
|
922
|
+
* 2. Apply cached per-FFT-bin gains (from last async inference) → ISTFT
|
|
923
|
+
* 3. Overlap-add into persistent output ring buffer
|
|
924
|
+
* 4. Fire async inference (non-blocking) to update gains for next steps
|
|
925
|
+
*
|
|
926
|
+
* Audio thread cost: ~2ms (pure JS STFT + OLA).
|
|
927
|
+
* Model cost: async, never blocks audio thread.
|
|
928
|
+
* Mask staleness: ≤1 hop (5ms) — imperceptible for noise suppression.
|
|
929
|
+
*/
|
|
930
|
+
processAudioSync(inputBuffer) {
|
|
931
|
+
if (!this.isInitialized || !this.model || !this.config) {
|
|
932
|
+
return inputBuffer;
|
|
933
|
+
}
|
|
934
|
+
const N_FFT = this.config.n_fft || 512;
|
|
935
|
+
const HOP = this.config.hop_length || 80;
|
|
936
|
+
const N_MELS = this.config.n_mels || 40;
|
|
937
|
+
const SEQ_LEN = this.config.sequence_length || 8;
|
|
938
|
+
const bins = N_FFT / 2 + 1;
|
|
939
|
+
if (!this.hannWinCache || this.hannWinCache.length !== N_FFT) {
|
|
940
|
+
this.hannWinCache = new Float32Array(N_FFT);
|
|
941
|
+
for (let i = 0; i < N_FFT; i++)
|
|
942
|
+
this.hannWinCache[i] = 0.5 * (1 - Math.cos(2 * Math.PI * i / (N_FFT - 1)));
|
|
943
|
+
}
|
|
944
|
+
const hann = this.hannWinCache;
|
|
945
|
+
try {
|
|
946
|
+
const in16 = this.needsResampling ? this.downsample(inputBuffer, this.resampleRatio) : inputBuffer;
|
|
947
|
+
if (this.inRingLen + in16.length > this.inRing.length) {
|
|
948
|
+
const grown = new Float32Array(
|
|
949
|
+
Math.max((this.inRingLen + in16.length) * 2, 8192)
|
|
950
|
+
);
|
|
951
|
+
grown.set(this.inRing.subarray(0, this.inRingLen));
|
|
952
|
+
this.inRing = grown;
|
|
953
|
+
}
|
|
954
|
+
this.inRing.set(in16, this.inRingLen);
|
|
955
|
+
this.inRingLen += in16.length;
|
|
956
|
+
const maxOut = this.inRingLen + N_FFT;
|
|
957
|
+
if (maxOut > this.outAccum.length) {
|
|
958
|
+
const g1 = new Float32Array(maxOut * 2);
|
|
959
|
+
g1.set(this.outAccum);
|
|
960
|
+
this.outAccum = g1;
|
|
961
|
+
const g2 = new Float32Array(maxOut * 2);
|
|
962
|
+
g2.set(this.outNorm);
|
|
963
|
+
this.outNorm = g2;
|
|
964
|
+
}
|
|
965
|
+
let offset = 0;
|
|
966
|
+
while (offset + HOP <= this.inRingLen) {
|
|
967
|
+
const frame = new Float32Array(N_FFT);
|
|
968
|
+
for (let i = 0; i < N_FFT; i++) {
|
|
969
|
+
const idx = offset + i;
|
|
970
|
+
frame[i] = (idx < this.inRingLen ? this.inRing[idx] : 0) * hann[i];
|
|
971
|
+
}
|
|
972
|
+
const { real: fR, imag: fI } = _MLNoiseSuppressor.fftForward(frame);
|
|
973
|
+
const power = new Float32Array(bins);
|
|
974
|
+
for (let k = 0; k < bins; k++) power[k] = fR[k] * fR[k] + fI[k] * fI[k];
|
|
975
|
+
const melFrame = this.computeMelFrame(power, N_MELS);
|
|
976
|
+
this.melFrameHistory.push(melFrame);
|
|
977
|
+
if (this.melFrameHistory.length > SEQ_LEN)
|
|
978
|
+
this.melFrameHistory = this.melFrameHistory.slice(-SEQ_LEN);
|
|
979
|
+
const gains = this.cachedFftGains ?? new Float32Array(bins).fill(1);
|
|
980
|
+
const mR = new Float32Array(N_FFT);
|
|
981
|
+
const mI = new Float32Array(N_FFT);
|
|
982
|
+
for (let k = 0; k < bins; k++) {
|
|
983
|
+
mR[k] = fR[k] * gains[k];
|
|
984
|
+
mI[k] = fI[k] * gains[k];
|
|
985
|
+
if (k > 0 && k < N_FFT / 2) {
|
|
986
|
+
mR[N_FFT - k] = mR[k];
|
|
987
|
+
mI[N_FFT - k] = -mI[k];
|
|
988
|
+
}
|
|
989
|
+
}
|
|
990
|
+
const recon = _MLNoiseSuppressor.ifftFromComplex(mR, mI);
|
|
991
|
+
for (let i = 0; i < N_FFT; i++) {
|
|
992
|
+
const pos = offset + i;
|
|
993
|
+
this.outAccum[pos] += recon[i] * hann[i];
|
|
994
|
+
this.outNorm[pos] += hann[i] * hann[i];
|
|
995
|
+
}
|
|
996
|
+
if (!this.inferenceInFlight) {
|
|
997
|
+
this.runInferenceAsync(this.melFrameHistory.slice(), SEQ_LEN, N_MELS);
|
|
998
|
+
}
|
|
999
|
+
offset += HOP;
|
|
1000
|
+
}
|
|
1001
|
+
const out16 = new Float32Array(in16.length);
|
|
1002
|
+
const ready2 = Math.min(in16.length, offset);
|
|
1003
|
+
for (let i = 0; i < ready2; i++) {
|
|
1004
|
+
out16[i] = this.outNorm[i] > 1e-10 ? this.outAccum[i] / this.outNorm[i] : in16[i];
|
|
1005
|
+
}
|
|
1006
|
+
for (let i = ready2; i < in16.length; i++) out16[i] = in16[i];
|
|
1007
|
+
this.inRing.copyWithin(0, offset);
|
|
1008
|
+
this.inRingLen -= offset;
|
|
1009
|
+
this.outAccum.copyWithin(0, offset);
|
|
1010
|
+
this.outNorm.copyWithin(0, offset);
|
|
1011
|
+
this.outAccum.fill(0, this.inRingLen + N_FFT);
|
|
1012
|
+
this.outNorm.fill(0, this.inRingLen + N_FFT);
|
|
1013
|
+
return this.needsResampling ? this.upsample(out16, this.resampleRatio, inputBuffer.length) : out16;
|
|
1014
|
+
} catch {
|
|
1015
|
+
return inputBuffer;
|
|
1016
|
+
}
|
|
1017
|
+
}
|
|
1018
|
+
/**
|
|
1019
|
+
* Compute mel-spectrogram features for one STFT frame.
|
|
1020
|
+
* Matches training pipeline: power_to_db(ref=max) → (db+80)/80 clipped [0,1].
|
|
1021
|
+
*/
|
|
1022
|
+
computeMelFrame(powerSpec, N_MELS) {
|
|
1023
|
+
if (!this.config) return Array(N_MELS).fill(0);
|
|
1024
|
+
const N_FFT = this.config.n_fft || 512;
|
|
1025
|
+
const SR = this.config.sample_rate || 16e3;
|
|
1026
|
+
if (!this.melFilterbank)
|
|
1027
|
+
this.melFilterbank = _MLNoiseSuppressor.buildMelFilterbank(
|
|
1028
|
+
N_MELS,
|
|
1029
|
+
N_FFT,
|
|
1030
|
+
SR
|
|
1031
|
+
);
|
|
1032
|
+
const melPow = new Float32Array(N_MELS);
|
|
1033
|
+
for (let m = 0; m < N_MELS; m++) {
|
|
1034
|
+
let s = 0;
|
|
1035
|
+
const filt = this.melFilterbank[m];
|
|
1036
|
+
for (let k = 0; k < filt.length && k < powerSpec.length; k++)
|
|
1037
|
+
s += filt[k] * powerSpec[k];
|
|
1038
|
+
melPow[m] = s;
|
|
1039
|
+
}
|
|
1040
|
+
let maxP = 1e-10;
|
|
1041
|
+
for (let m = 0; m < N_MELS; m++) if (melPow[m] > maxP) maxP = melPow[m];
|
|
1042
|
+
const refDb = 10 * Math.log10(maxP);
|
|
1043
|
+
const out = new Array(N_MELS);
|
|
1044
|
+
for (let m = 0; m < N_MELS; m++) {
|
|
1045
|
+
const db = 10 * Math.log10(melPow[m] + 1e-10) - refDb;
|
|
1046
|
+
out[m] = Math.max(0, Math.min(1, (db + 80) / 80));
|
|
1047
|
+
}
|
|
1048
|
+
return out;
|
|
1049
|
+
}
|
|
1050
|
+
/**
|
|
1051
|
+
* Run model inference asynchronously.
|
|
1052
|
+
* Updates cachedFftGains (per-FFT-bin gains) — audio thread applies them each hop.
|
|
1053
|
+
* Never awaited from processAudioSync; inferenceInFlight prevents overlapping calls.
|
|
1054
|
+
*/
|
|
1055
|
+
async runInferenceAsync(historySnapshot, SEQ_LEN, N_MELS) {
|
|
1056
|
+
this.inferenceInFlight = true;
|
|
1057
|
+
try {
|
|
1058
|
+
const histLen = historySnapshot.length;
|
|
1059
|
+
const padCount = SEQ_LEN - histLen;
|
|
1060
|
+
const flat = new Array(SEQ_LEN * N_MELS).fill(0);
|
|
1061
|
+
for (let f = 0; f < histLen; f++)
|
|
1062
|
+
for (let m = 0; m < N_MELS; m++)
|
|
1063
|
+
flat[(padCount + f) * N_MELS + m] = historySnapshot[f][m];
|
|
1064
|
+
const N_FFT = this.config.n_fft || 512;
|
|
1065
|
+
const SR = this.config.sample_rate || 16e3;
|
|
1066
|
+
const bins = N_FFT / 2 + 1;
|
|
1067
|
+
const inp = tf.tensor(flat, [1, SEQ_LEN, N_MELS]);
|
|
1068
|
+
const out = this.model.predict(inp);
|
|
1069
|
+
const maskFlat = new Float32Array(await out.data());
|
|
1070
|
+
inp.dispose();
|
|
1071
|
+
out.dispose();
|
|
1072
|
+
const mask40 = new Float32Array(N_MELS);
|
|
1073
|
+
for (let m = 0; m < N_MELS; m++)
|
|
1074
|
+
mask40[m] = maskFlat[(SEQ_LEN - 1) * N_MELS + m];
|
|
1075
|
+
const smoothed = this.applyTemporalSmoothing(mask40);
|
|
1076
|
+
if (!this.melFilterbank)
|
|
1077
|
+
this.melFilterbank = _MLNoiseSuppressor.buildMelFilterbank(
|
|
1078
|
+
N_MELS,
|
|
1079
|
+
N_FFT,
|
|
1080
|
+
SR
|
|
1081
|
+
);
|
|
1082
|
+
const gains = new Float32Array(bins);
|
|
1083
|
+
const filtTotal = new Float32Array(bins);
|
|
1084
|
+
for (let m = 0; m < N_MELS; m++) {
|
|
1085
|
+
const filt = this.melFilterbank[m];
|
|
1086
|
+
for (let k = 0; k < filt.length && k < bins; k++) {
|
|
1087
|
+
gains[k] += filt[k] * smoothed[m];
|
|
1088
|
+
filtTotal[k] += filt[k];
|
|
1089
|
+
}
|
|
1090
|
+
}
|
|
1091
|
+
for (let k = 0; k < bins; k++) {
|
|
1092
|
+
gains[k] = filtTotal[k] > 1e-8 ? gains[k] / filtTotal[k] : 1;
|
|
1093
|
+
gains[k] = Math.max(0, Math.min(1, gains[k]));
|
|
1094
|
+
}
|
|
1095
|
+
this.cachedFftGains = gains;
|
|
1096
|
+
} catch {
|
|
1097
|
+
} finally {
|
|
1098
|
+
this.inferenceInFlight = false;
|
|
1099
|
+
}
|
|
1100
|
+
}
|
|
1101
|
+
/**
|
|
1102
|
+
* Downsample audio buffer (e.g., 48kHz → 16kHz)
|
|
1103
|
+
* Simple linear interpolation for speed
|
|
1104
|
+
*/
|
|
1105
|
+
downsample(input, ratio) {
|
|
1106
|
+
const outputLength = Math.floor(input.length / ratio);
|
|
1107
|
+
const output = new Float32Array(outputLength);
|
|
1108
|
+
for (let i = 0; i < outputLength; i++) {
|
|
1109
|
+
const srcIndex = i * ratio;
|
|
1110
|
+
const srcIndexFloor = Math.floor(srcIndex);
|
|
1111
|
+
const frac = srcIndex - srcIndexFloor;
|
|
1112
|
+
if (srcIndexFloor + 1 < input.length) {
|
|
1113
|
+
output[i] = input[srcIndexFloor] * (1 - frac) + input[srcIndexFloor + 1] * frac;
|
|
1114
|
+
} else {
|
|
1115
|
+
output[i] = input[srcIndexFloor];
|
|
1116
|
+
}
|
|
1117
|
+
}
|
|
1118
|
+
return output;
|
|
1119
|
+
}
|
|
1120
|
+
/**
|
|
1121
|
+
* Upsample audio buffer (e.g., 16kHz → 48kHz)
|
|
1122
|
+
* Linear interpolation to match original length
|
|
1123
|
+
*/
|
|
1124
|
+
upsample(input, ratio, targetLength) {
|
|
1125
|
+
const output = new Float32Array(targetLength);
|
|
1126
|
+
for (let i = 0; i < targetLength; i++) {
|
|
1127
|
+
const srcIndex = i / ratio;
|
|
1128
|
+
const srcIndexFloor = Math.floor(srcIndex);
|
|
1129
|
+
const frac = srcIndex - srcIndexFloor;
|
|
1130
|
+
if (srcIndexFloor + 1 < input.length) {
|
|
1131
|
+
output[i] = input[srcIndexFloor] * (1 - frac) + input[srcIndexFloor + 1] * frac;
|
|
1132
|
+
} else if (srcIndexFloor < input.length) {
|
|
1133
|
+
output[i] = input[srcIndexFloor];
|
|
1134
|
+
} else {
|
|
1135
|
+
output[i] = input[input.length - 1];
|
|
1136
|
+
}
|
|
1137
|
+
}
|
|
1138
|
+
return output;
|
|
1139
|
+
}
|
|
1140
|
+
/**
|
|
1141
|
+
* Extract mel-spectrogram features matching the training pipeline exactly:
|
|
1142
|
+
* librosa.feature.melspectrogram(n_fft, hop_length, n_mels) — values from config
|
|
1143
|
+
* → power_to_db(ref=np.max)
|
|
1144
|
+
* → (mel_db + 80) / 80 clipped to [0, 1]
|
|
1145
|
+
* No z-score normalisation applied (model trained on raw [0,1] values).
|
|
1146
|
+
*
|
|
1147
|
+
* Config driven: works with 16kHz/40mels or 48kHz/128mels depending on model_config.json
|
|
1148
|
+
*/
|
|
1149
|
+
extractFeatures(audio) {
|
|
1150
|
+
if (!this.config) return [[Array(128).fill(0)][0]];
|
|
1151
|
+
const N_FFT = this.config?.n_fft || 2048;
|
|
1152
|
+
const HOP = this.config.hop_length || 256;
|
|
1153
|
+
const N_MELS = this.config.n_mels || 128;
|
|
1154
|
+
const SR = this.config.sample_rate || 48e3;
|
|
1155
|
+
if (!this.melFilterbank) {
|
|
1156
|
+
this.melFilterbank = _MLNoiseSuppressor.buildMelFilterbank(
|
|
1157
|
+
N_MELS,
|
|
1158
|
+
N_FFT,
|
|
1159
|
+
SR
|
|
1160
|
+
);
|
|
1161
|
+
}
|
|
1162
|
+
const hannWindow = new Float32Array(N_FFT);
|
|
1163
|
+
for (let i = 0; i < N_FFT; i++) {
|
|
1164
|
+
hannWindow[i] = 0.5 * (1 - Math.cos(2 * Math.PI * i / (N_FFT - 1)));
|
|
1165
|
+
}
|
|
1166
|
+
const numFrames = Math.max(1, Math.floor((audio.length - N_FFT) / HOP) + 1);
|
|
1167
|
+
const features = [];
|
|
1168
|
+
this.lastComplexFrames = [];
|
|
1169
|
+
for (let fi = 0; fi < numFrames; fi++) {
|
|
1170
|
+
const start = fi * HOP;
|
|
1171
|
+
const frame = new Float32Array(N_FFT);
|
|
1172
|
+
for (let i = 0; i < N_FFT; i++) {
|
|
1173
|
+
frame[i] = (start + i < audio.length ? audio[start + i] : 0) * hannWindow[i];
|
|
1174
|
+
}
|
|
1175
|
+
const { real: fftReal, imag: fftImag } = _MLNoiseSuppressor.fftForward(frame);
|
|
1176
|
+
const bins = N_FFT / 2 + 1;
|
|
1177
|
+
this.lastComplexFrames.push({
|
|
1178
|
+
real: fftReal.slice(0, bins),
|
|
1179
|
+
imag: fftImag.slice(0, bins)
|
|
1180
|
+
});
|
|
1181
|
+
const powerSpec = new Float32Array(bins);
|
|
1182
|
+
for (let k = 0; k < bins; k++) {
|
|
1183
|
+
powerSpec[k] = fftReal[k] * fftReal[k] + fftImag[k] * fftImag[k];
|
|
1184
|
+
}
|
|
1185
|
+
const melPower = new Float32Array(N_MELS);
|
|
1186
|
+
for (let m = 0; m < N_MELS; m++) {
|
|
1187
|
+
let sum2 = 0;
|
|
1188
|
+
const filt = this.melFilterbank[m];
|
|
1189
|
+
for (let k = 0; k < filt.length; k++) sum2 += filt[k] * powerSpec[k];
|
|
1190
|
+
melPower[m] = sum2;
|
|
1191
|
+
}
|
|
1192
|
+
let maxPow = 1e-10;
|
|
1193
|
+
for (let m = 0; m < N_MELS; m++)
|
|
1194
|
+
if (melPower[m] > maxPow) maxPow = melPower[m];
|
|
1195
|
+
const refDb = 10 * Math.log10(maxPow);
|
|
1196
|
+
const melNorm = new Array(N_MELS);
|
|
1197
|
+
for (let m = 0; m < N_MELS; m++) {
|
|
1198
|
+
const db = 10 * Math.log10(melPower[m] + 1e-10) - refDb;
|
|
1199
|
+
melNorm[m] = Math.max(0, Math.min(1, (db + 80) / 80));
|
|
1200
|
+
}
|
|
1201
|
+
features.push(melNorm);
|
|
1202
|
+
}
|
|
1203
|
+
return features;
|
|
1204
|
+
}
|
|
1205
|
+
/**
|
|
1206
|
+
* Cooley-Tukey radix-2 FFT — returns full complex spectrum.
|
|
1207
|
+
* Accepts optional imaginary input for use as IFFT building block.
|
|
1208
|
+
* frameReal.length must be a power of 2.
|
|
1209
|
+
*/
|
|
1210
|
+
static fftForward(frameReal, frameImag) {
|
|
1211
|
+
const N = frameReal.length;
|
|
1212
|
+
const real = new Float32Array(frameReal);
|
|
1213
|
+
const imag = frameImag ? new Float32Array(frameImag) : new Float32Array(N);
|
|
1214
|
+
let j = 0;
|
|
1215
|
+
for (let i = 1; i < N; i++) {
|
|
1216
|
+
let bit = N >> 1;
|
|
1217
|
+
while (j & bit) {
|
|
1218
|
+
j ^= bit;
|
|
1219
|
+
bit >>= 1;
|
|
1220
|
+
}
|
|
1221
|
+
j ^= bit;
|
|
1222
|
+
if (i < j) {
|
|
1223
|
+
let tmp = real[i];
|
|
1224
|
+
real[i] = real[j];
|
|
1225
|
+
real[j] = tmp;
|
|
1226
|
+
tmp = imag[i];
|
|
1227
|
+
imag[i] = imag[j];
|
|
1228
|
+
imag[j] = tmp;
|
|
1229
|
+
}
|
|
1230
|
+
}
|
|
1231
|
+
for (let len = 2; len <= N; len <<= 1) {
|
|
1232
|
+
const halfLen = len >> 1;
|
|
1233
|
+
const ang = -2 * Math.PI / len;
|
|
1234
|
+
const wRe = Math.cos(ang);
|
|
1235
|
+
const wIm = Math.sin(ang);
|
|
1236
|
+
for (let i = 0; i < N; i += len) {
|
|
1237
|
+
let curRe = 1, curIm = 0;
|
|
1238
|
+
for (let k = 0; k < halfLen; k++) {
|
|
1239
|
+
const uRe = real[i + k], uIm = imag[i + k];
|
|
1240
|
+
const vRe = real[i + k + halfLen] * curRe - imag[i + k + halfLen] * curIm;
|
|
1241
|
+
const vIm = real[i + k + halfLen] * curIm + imag[i + k + halfLen] * curRe;
|
|
1242
|
+
real[i + k] = uRe + vRe;
|
|
1243
|
+
imag[i + k] = uIm + vIm;
|
|
1244
|
+
real[i + k + halfLen] = uRe - vRe;
|
|
1245
|
+
imag[i + k + halfLen] = uIm - vIm;
|
|
1246
|
+
const newRe = curRe * wRe - curIm * wIm;
|
|
1247
|
+
curIm = curRe * wIm + curIm * wRe;
|
|
1248
|
+
curRe = newRe;
|
|
1249
|
+
}
|
|
1250
|
+
}
|
|
1251
|
+
}
|
|
1252
|
+
return { real, imag };
|
|
1253
|
+
}
|
|
1254
|
+
/**
|
|
1255
|
+
* IFFT via conjugate trick: ifft(X) = conj(fft(conj(X))) / N
|
|
1256
|
+
* Returns the real part of the time-domain signal.
|
|
1257
|
+
*/
|
|
1258
|
+
static ifftFromComplex(real, imag) {
|
|
1259
|
+
const N = real.length;
|
|
1260
|
+
const conjImag = new Float32Array(N);
|
|
1261
|
+
for (let i = 0; i < N; i++) conjImag[i] = -imag[i];
|
|
1262
|
+
const { real: outReal} = _MLNoiseSuppressor.fftForward(
|
|
1263
|
+
real,
|
|
1264
|
+
conjImag
|
|
1265
|
+
);
|
|
1266
|
+
const output = new Float32Array(N);
|
|
1267
|
+
for (let i = 0; i < N; i++) {
|
|
1268
|
+
output[i] = outReal[i] / N;
|
|
1269
|
+
}
|
|
1270
|
+
return output;
|
|
1271
|
+
}
|
|
1272
|
+
/**
|
|
1273
|
+
* Build an N-band triangular mel filterbank matching librosa's defaults.
|
|
1274
|
+
* Uses Slaney mel scale (htk=False, librosa default): linear below 1 kHz,
|
|
1275
|
+
* logarithmic above. This matches librosa.feature.melspectrogram used in
|
|
1276
|
+
* Colab training — HTK formula would produce different band-frequency
|
|
1277
|
+
* assignments and degrade suppression quality.
|
|
1278
|
+
*/
|
|
1279
|
+
static buildMelFilterbank(nMels, nFft, sr) {
|
|
1280
|
+
const bins = nFft / 2 + 1;
|
|
1281
|
+
const F_SP = 200 / 3;
|
|
1282
|
+
const MIN_LOG_HZ = 1e3;
|
|
1283
|
+
const MIN_LOG_MEL = MIN_LOG_HZ / F_SP;
|
|
1284
|
+
const LOGSTEP = Math.log(6.4) / 27;
|
|
1285
|
+
const hzToMel = (hz) => hz < MIN_LOG_HZ ? hz / F_SP : MIN_LOG_MEL + Math.log(hz / MIN_LOG_HZ) / LOGSTEP;
|
|
1286
|
+
const melToHz = (mel) => mel < MIN_LOG_MEL ? mel * F_SP : MIN_LOG_HZ * Math.exp(LOGSTEP * (mel - MIN_LOG_MEL));
|
|
1287
|
+
const melMin = hzToMel(0);
|
|
1288
|
+
const melMax = hzToMel(sr / 2);
|
|
1289
|
+
const melPts = new Float32Array(nMels + 2);
|
|
1290
|
+
for (let i = 0; i < nMels + 2; i++) {
|
|
1291
|
+
melPts[i] = melToHz(melMin + i / (nMels + 1) * (melMax - melMin));
|
|
1292
|
+
}
|
|
1293
|
+
const fftFreqs = new Float32Array(bins);
|
|
1294
|
+
for (let k = 0; k < bins; k++) fftFreqs[k] = k * sr / nFft;
|
|
1295
|
+
const filters = [];
|
|
1296
|
+
for (let m = 0; m < nMels; m++) {
|
|
1297
|
+
const filt = new Float32Array(bins);
|
|
1298
|
+
const left = melPts[m];
|
|
1299
|
+
const center = melPts[m + 1];
|
|
1300
|
+
const right = melPts[m + 2];
|
|
1301
|
+
for (let k = 0; k < bins; k++) {
|
|
1302
|
+
const f = fftFreqs[k];
|
|
1303
|
+
if (f >= left && f <= center) {
|
|
1304
|
+
filt[k] = (f - left) / (center - left + 1e-10);
|
|
1305
|
+
} else if (f > center && f <= right) {
|
|
1306
|
+
filt[k] = (right - f) / (right - center + 1e-10);
|
|
1307
|
+
}
|
|
1308
|
+
}
|
|
1309
|
+
filters.push(filt);
|
|
1310
|
+
}
|
|
1311
|
+
return filters;
|
|
1312
|
+
}
|
|
1313
|
+
/**
|
|
1314
|
+
* Apply temporal smoothing to the 40-band mel mask.
|
|
1315
|
+
*
|
|
1316
|
+
* Standard IRM smoothing: alpha weighted toward the PREVIOUS mask (heavy weight)
|
|
1317
|
+
* to prevent "musical noise" — rapid per-band gain changes that create tonal
|
|
1318
|
+
* artifacts. SMOOTHING_ALPHA=0.85 means 85% previous + 15% new each frame.
|
|
1319
|
+
*
|
|
1320
|
+
* BUG FIX: the old code had `alpha * current + (1-alpha) * prev` which was
|
|
1321
|
+
* effectively 85% tracking the current mask — almost no smoothing, causing chirps.
|
|
1322
|
+
*/
|
|
1323
|
+
applyTemporalSmoothing(currentMask, participantId = "__default__") {
|
|
1324
|
+
const smoothed = new Float32Array(currentMask.length);
|
|
1325
|
+
const prev = this.prevMaskMap.get(participantId);
|
|
1326
|
+
if (!prev || prev.length !== currentMask.length) {
|
|
1327
|
+
const init = new Float32Array(currentMask);
|
|
1328
|
+
this.prevMaskMap.set(participantId, init);
|
|
1329
|
+
return init;
|
|
1330
|
+
}
|
|
1331
|
+
for (let i = 0; i < currentMask.length; i++) {
|
|
1332
|
+
smoothed[i] = this.SMOOTHING_ALPHA * prev[i] + (1 - this.SMOOTHING_ALPHA) * currentMask[i];
|
|
1333
|
+
smoothed[i] = Math.max(0, Math.min(1, smoothed[i]));
|
|
1334
|
+
}
|
|
1335
|
+
this.prevMaskMap.set(participantId, smoothed);
|
|
1336
|
+
return smoothed;
|
|
1337
|
+
}
|
|
1338
|
+
/**
|
|
1339
|
+
* Apply the 40-band spectral mask via STFT → gain → ISTFT (overlap-add).
|
|
1340
|
+
*
|
|
1341
|
+
* Replaces the old applyMaskToAudio which incorrectly used frequency-domain mask
|
|
1342
|
+
* values as time-domain sample gains — producing distortion rather than clean
|
|
1343
|
+
* suppression. Correct approach:
|
|
1344
|
+
* 1. Map 40-band mel mask → per-FFT-bin gain via transpose mel filterbank
|
|
1345
|
+
* 2. Apply gains to the saved complex STFT frames
|
|
1346
|
+
* 3. Reconstruct via synthesis Hann window + overlap-add
|
|
1347
|
+
*/
|
|
1348
|
+
applyMaskViaOLA(audio, mask40, N_FFT, HOP, N_MELS) {
|
|
1349
|
+
if (!this.melFilterbank || this.lastComplexFrames.length === 0) {
|
|
1350
|
+
return audio;
|
|
1351
|
+
}
|
|
1352
|
+
const bins = N_FFT / 2 + 1;
|
|
1353
|
+
const fftGains = new Float32Array(bins);
|
|
1354
|
+
const filtTotals = new Float32Array(bins);
|
|
1355
|
+
for (let m = 0; m < N_MELS; m++) {
|
|
1356
|
+
const filt = this.melFilterbank[m];
|
|
1357
|
+
for (let k = 0; k < filt.length && k < bins; k++) {
|
|
1358
|
+
fftGains[k] += filt[k] * mask40[m];
|
|
1359
|
+
filtTotals[k] += filt[k];
|
|
1360
|
+
}
|
|
1361
|
+
}
|
|
1362
|
+
for (let k = 0; k < bins; k++) {
|
|
1363
|
+
fftGains[k] = filtTotals[k] > 1e-8 ? fftGains[k] / filtTotals[k] : 1;
|
|
1364
|
+
fftGains[k] = Math.max(0.3, Math.min(1, fftGains[k]));
|
|
1365
|
+
}
|
|
1366
|
+
const hannWindow = new Float32Array(N_FFT);
|
|
1367
|
+
for (let i = 0; i < N_FFT; i++) {
|
|
1368
|
+
hannWindow[i] = 0.5 * (1 - Math.cos(2 * Math.PI * i / (N_FFT - 1)));
|
|
1369
|
+
}
|
|
1370
|
+
const output = new Float32Array(audio.length);
|
|
1371
|
+
const normFactor = new Float32Array(audio.length);
|
|
1372
|
+
for (let fi = 0; fi < this.lastComplexFrames.length; fi++) {
|
|
1373
|
+
const start = fi * HOP;
|
|
1374
|
+
const { real: origReal, imag: origImag } = this.lastComplexFrames[fi];
|
|
1375
|
+
const maskedReal = new Float32Array(N_FFT);
|
|
1376
|
+
const maskedImag = new Float32Array(N_FFT);
|
|
1377
|
+
for (let k = 0; k < bins; k++) {
|
|
1378
|
+
maskedReal[k] = origReal[k] * fftGains[k];
|
|
1379
|
+
maskedImag[k] = origImag[k] * fftGains[k];
|
|
1380
|
+
if (k > 0 && k < N_FFT / 2) {
|
|
1381
|
+
maskedReal[N_FFT - k] = maskedReal[k];
|
|
1382
|
+
maskedImag[N_FFT - k] = -maskedImag[k];
|
|
1383
|
+
}
|
|
1384
|
+
}
|
|
1385
|
+
const reconstructed = _MLNoiseSuppressor.ifftFromComplex(
|
|
1386
|
+
maskedReal,
|
|
1387
|
+
maskedImag
|
|
1388
|
+
);
|
|
1389
|
+
for (let i = 0; i < N_FFT; i++) {
|
|
1390
|
+
const pos = start + i;
|
|
1391
|
+
if (pos < audio.length) {
|
|
1392
|
+
output[pos] += reconstructed[i] * hannWindow[i];
|
|
1393
|
+
normFactor[pos] += hannWindow[i] * hannWindow[i];
|
|
1394
|
+
}
|
|
1395
|
+
}
|
|
1396
|
+
}
|
|
1397
|
+
for (let i = 0; i < output.length; i++) {
|
|
1398
|
+
output[i] = normFactor[i] > 1e-10 ? output[i] / normFactor[i] : audio[i];
|
|
1399
|
+
}
|
|
1400
|
+
return output;
|
|
1401
|
+
}
|
|
1402
|
+
/**
|
|
1403
|
+
* Reset processing state (call when participant reconnects or audio track resets)
|
|
1404
|
+
*/
|
|
1405
|
+
reset() {
|
|
1406
|
+
this.prevMaskMap.clear();
|
|
1407
|
+
this.melFrameHistory = [];
|
|
1408
|
+
this.lastComplexFrames = [];
|
|
1409
|
+
this.inRing = new Float32Array(8192);
|
|
1410
|
+
this.inRingLen = 0;
|
|
1411
|
+
this.outAccum = new Float32Array(16384);
|
|
1412
|
+
this.outNorm = new Float32Array(16384);
|
|
1413
|
+
this.cachedFftGains = null;
|
|
1414
|
+
this.inferenceInFlight = false;
|
|
1415
|
+
}
|
|
1416
|
+
/**
|
|
1417
|
+
* Check if ML processor is ready
|
|
1418
|
+
*/
|
|
1419
|
+
isReady() {
|
|
1420
|
+
return this.isInitialized && this.model !== null;
|
|
1421
|
+
}
|
|
1422
|
+
/**
|
|
1423
|
+
* Return the loaded model config so callers (e.g. AudioWorkletNode setup)
|
|
1424
|
+
* can read n_fft, hop_length, n_mels, sample_rate, sequence_length.
|
|
1425
|
+
*/
|
|
1426
|
+
getModelConfig() {
|
|
1427
|
+
return this.config;
|
|
1428
|
+
}
|
|
1429
|
+
/**
|
|
1430
|
+
* Run one inference pass from pre-computed mel features and return per-FFT-bin
|
|
1431
|
+
* gains. Called by SpatialAudioChannel for each AudioWorkletNode message.
|
|
1432
|
+
*
|
|
1433
|
+
* @param features Flat Float32Array of shape [SEQ_LEN × N_MELS] (row-major).
|
|
1434
|
+
* Produced by the AudioWorklet processor and transferred via postMessage.
|
|
1435
|
+
* @param seqLen Sequence length (default from config).
|
|
1436
|
+
* @param nMels Number of mel bands (default from config).
|
|
1437
|
+
* @returns Float32Array of length N_FFT/2+1 with per-bin gains in [0,1].
|
|
1438
|
+
*/
|
|
1439
|
+
async computeGainsFromFeatures(features, seqLen, nMels, participantId = "__default__") {
|
|
1440
|
+
if (!this.model || !this.config) {
|
|
1441
|
+
const bins2 = (this.config?.n_fft || 512) / 2 + 1;
|
|
1442
|
+
return new Float32Array(bins2).fill(1);
|
|
1443
|
+
}
|
|
1444
|
+
const N_FFT = this.config.n_fft || 512;
|
|
1445
|
+
const SR = this.config.sample_rate || 16e3;
|
|
1446
|
+
const bins = N_FFT / 2 + 1;
|
|
1447
|
+
try {
|
|
1448
|
+
const inp = tf.tensor(Array.from(features), [
|
|
1449
|
+
1,
|
|
1450
|
+
seqLen,
|
|
1451
|
+
nMels
|
|
1452
|
+
]);
|
|
1453
|
+
const out = this.model.predict(inp);
|
|
1454
|
+
const maskFlat = new Float32Array(await out.data());
|
|
1455
|
+
inp.dispose();
|
|
1456
|
+
out.dispose();
|
|
1457
|
+
const rawMask = new Float32Array(nMels);
|
|
1458
|
+
for (let m = 0; m < nMels; m++) {
|
|
1459
|
+
rawMask[m] = Math.max(
|
|
1460
|
+
0,
|
|
1461
|
+
Math.min(1, maskFlat[(seqLen - 1) * nMels + m])
|
|
1462
|
+
);
|
|
1463
|
+
}
|
|
1464
|
+
let rawMax = 0;
|
|
1465
|
+
for (let m = 0; m < nMels; m++) {
|
|
1466
|
+
if (rawMask[m] > rawMax) rawMax = rawMask[m];
|
|
1467
|
+
}
|
|
1468
|
+
if (rawMax < 5e-3) {
|
|
1469
|
+
const passthroughCount = _MLNoiseSuppressor._passthroughCount || 0;
|
|
1470
|
+
_MLNoiseSuppressor._passthroughCount = passthroughCount + 1;
|
|
1471
|
+
const lastPassthroughLog = _MLNoiseSuppressor._lastPassthroughLog || 0;
|
|
1472
|
+
if (Date.now() - lastPassthroughLog > 2e3) {
|
|
1473
|
+
_MLNoiseSuppressor._lastPassthroughLog = Date.now();
|
|
1474
|
+
console.warn(
|
|
1475
|
+
`[ML] \u274C PASSTHROUGH (rawMax=${rawMax.toFixed(5)}) \u2014 model output near-zero. Count since last log: ${passthroughCount}. Weights likely NOT loaded. Check [MLNoiseSuppressor] weight name dump above.`
|
|
1476
|
+
);
|
|
1477
|
+
_MLNoiseSuppressor._passthroughCount = 0;
|
|
1478
|
+
}
|
|
1479
|
+
const passthrough = new Float32Array(bins).fill(1);
|
|
1480
|
+
return passthrough;
|
|
1481
|
+
}
|
|
1482
|
+
let isSpeechFrame = rawMax >= 0.108;
|
|
1483
|
+
const SF_HOLDOVER = 8;
|
|
1484
|
+
const sfMap = _MLNoiseSuppressor._sfHoldoverMap || (_MLNoiseSuppressor._sfHoldoverMap = /* @__PURE__ */ new Map());
|
|
1485
|
+
const sfHold = sfMap.get(participantId) || 0;
|
|
1486
|
+
if (isSpeechFrame) {
|
|
1487
|
+
sfMap.set(participantId, SF_HOLDOVER);
|
|
1488
|
+
} else if (sfHold > 0) {
|
|
1489
|
+
isSpeechFrame = true;
|
|
1490
|
+
sfMap.set(participantId, sfHold - 1);
|
|
1491
|
+
} else {
|
|
1492
|
+
sfMap.set(participantId, 0);
|
|
1493
|
+
}
|
|
1494
|
+
const smoothed = this.applyTemporalSmoothing(rawMask, participantId);
|
|
1495
|
+
if (!this.melFilterbank) {
|
|
1496
|
+
this.melFilterbank = _MLNoiseSuppressor.buildMelFilterbank(
|
|
1497
|
+
nMels,
|
|
1498
|
+
N_FFT,
|
|
1499
|
+
SR
|
|
1500
|
+
);
|
|
1501
|
+
}
|
|
1502
|
+
const gains = new Float32Array(bins);
|
|
1503
|
+
const filtTotal = new Float32Array(bins);
|
|
1504
|
+
for (let m = 0; m < nMels; m++) {
|
|
1505
|
+
const filt = this.melFilterbank[m];
|
|
1506
|
+
for (let k = 0; k < filt.length && k < bins; k++) {
|
|
1507
|
+
gains[k] += filt[k] * smoothed[m];
|
|
1508
|
+
filtTotal[k] += filt[k];
|
|
1509
|
+
}
|
|
1510
|
+
}
|
|
1511
|
+
for (let k = 0; k < bins; k++) {
|
|
1512
|
+
gains[k] = filtTotal[k] > 1e-8 ? gains[k] / filtTotal[k] : 1;
|
|
1513
|
+
gains[k] = Math.max(0, Math.min(1, gains[k]));
|
|
1514
|
+
}
|
|
1515
|
+
const IRM_SPEECH_FLOOR = 0.05;
|
|
1516
|
+
if (isSpeechFrame) {
|
|
1517
|
+
for (let k = 1; k < bins; k++) {
|
|
1518
|
+
gains[k] = Math.max(IRM_SPEECH_FLOOR, gains[k]);
|
|
1519
|
+
}
|
|
1520
|
+
}
|
|
1521
|
+
gains[0] = isSpeechFrame ? 1 : 0;
|
|
1522
|
+
const now = Date.now();
|
|
1523
|
+
const lastLog = _MLNoiseSuppressor._lastGainLog || /* @__PURE__ */ new Map();
|
|
1524
|
+
const lastT = lastLog.get(participantId) || 0;
|
|
1525
|
+
if (!isSpeechFrame || now - lastT > 1e3) {
|
|
1526
|
+
lastLog.set(participantId, now);
|
|
1527
|
+
_MLNoiseSuppressor._lastGainLog = lastLog;
|
|
1528
|
+
let gainSum = 0;
|
|
1529
|
+
for (let k = 1; k < bins; k++) gainSum += gains[k];
|
|
1530
|
+
const meanIRM = gainSum / (bins - 1);
|
|
1531
|
+
const suppPct = ((1 - meanIRM) * 100).toFixed(0);
|
|
1532
|
+
console.log(
|
|
1533
|
+
`[ML] ${isSpeechFrame ? "\u{1F399}\uFE0F SPEECH" : "\u{1F507} NOISE "} p=${participantId.substring(0, 8)} rawMax=${rawMax.toFixed(3)} suppression=${suppPct}% gate=${gains[0].toFixed(0)}`
|
|
1534
|
+
);
|
|
1535
|
+
}
|
|
1536
|
+
return gains;
|
|
1537
|
+
} catch {
|
|
1538
|
+
return new Float32Array(bins).fill(1);
|
|
1539
|
+
}
|
|
1540
|
+
}
|
|
1541
|
+
/**
|
|
1542
|
+
* Get model info
|
|
1543
|
+
*/
|
|
1544
|
+
getInfo() {
|
|
1545
|
+
return {
|
|
1546
|
+
initialized: this.isInitialized,
|
|
1547
|
+
backend: tf.getBackend(),
|
|
1548
|
+
modelLoaded: this.model !== null
|
|
1549
|
+
};
|
|
1550
|
+
}
|
|
1551
|
+
/**
|
|
1552
|
+
* Remove per-participant smoothing state when a participant disconnects.
|
|
1553
|
+
* Keeps prevMaskMap from growing unboundedly in long sessions.
|
|
1554
|
+
*/
|
|
1555
|
+
clearParticipantSmoothingState(participantId) {
|
|
1556
|
+
this.prevMaskMap.delete(participantId);
|
|
1557
|
+
}
|
|
1558
|
+
/**
|
|
1559
|
+
* Cleanup resources
|
|
1560
|
+
*/
|
|
1561
|
+
dispose() {
|
|
1562
|
+
if (this.model) {
|
|
1563
|
+
this.model.dispose();
|
|
1564
|
+
this.model = null;
|
|
1565
|
+
}
|
|
1566
|
+
this.prevMaskMap.clear();
|
|
1567
|
+
this.isInitialized = false;
|
|
1568
|
+
}
|
|
1569
|
+
};
|
|
1570
|
+
|
|
1571
|
+
// src/utils/debug.ts
|
|
1572
|
+
var isDebugEnabled = () => typeof window !== "undefined" && window.ODYSSEY_DEBUG === true;
|
|
1573
|
+
var debugLog = (category, message, data) => {
|
|
1574
|
+
if (!isDebugEnabled()) return;
|
|
1575
|
+
const timestamp = (/* @__PURE__ */ new Date()).toISOString().slice(11, 23);
|
|
1576
|
+
if (data !== void 0) {
|
|
1577
|
+
console.log(`[${timestamp}][SDK:${category}] ${message}`, data);
|
|
1578
|
+
} else {
|
|
1579
|
+
console.log(`[${timestamp}][SDK:${category}] ${message}`);
|
|
1580
|
+
}
|
|
1581
|
+
};
|
|
1582
|
+
|
|
1583
|
+
// src/channels/spatial/SpatialAudioChannel.ts
|
|
1584
|
+
function createSpatialAudioManager(config) {
|
|
1585
|
+
const distanceConfig = {
|
|
1586
|
+
...DEFAULT_SPATIAL_CONFIG,
|
|
1587
|
+
...config?.distance
|
|
1588
|
+
};
|
|
1589
|
+
({
|
|
1590
|
+
...config?.denoiser
|
|
1591
|
+
});
|
|
1592
|
+
const audioContext = new AudioContext({ sampleRate: 48e3 });
|
|
1593
|
+
const resumeOnGesture = () => {
|
|
1594
|
+
if (audioContext.state === "suspended") {
|
|
1595
|
+
audioContext.resume().catch(() => {
|
|
1596
|
+
});
|
|
1597
|
+
}
|
|
1598
|
+
document.removeEventListener("click", resumeOnGesture);
|
|
1599
|
+
document.removeEventListener("keydown", resumeOnGesture);
|
|
1600
|
+
document.removeEventListener("touchstart", resumeOnGesture);
|
|
1601
|
+
};
|
|
1602
|
+
document.addEventListener("click", resumeOnGesture);
|
|
1603
|
+
document.addEventListener("keydown", resumeOnGesture);
|
|
1604
|
+
document.addEventListener("touchstart", resumeOnGesture);
|
|
1605
|
+
const masterGainNode = audioContext.createGain();
|
|
1606
|
+
masterGainNode.gain.value = 2.5;
|
|
1607
|
+
const masterCompressor = audioContext.createDynamicsCompressor();
|
|
1608
|
+
masterCompressor.threshold.value = 0;
|
|
1609
|
+
masterCompressor.knee.value = 0;
|
|
1610
|
+
masterCompressor.ratio.value = 1;
|
|
1611
|
+
masterCompressor.attack.value = 3e-3;
|
|
1612
|
+
masterCompressor.release.value = 0.25;
|
|
1613
|
+
masterGainNode.connect(masterCompressor);
|
|
1614
|
+
masterCompressor.connect(audioContext.destination);
|
|
1615
|
+
const participantNodes = /* @__PURE__ */ new Map();
|
|
1616
|
+
const listenerState = {
|
|
1617
|
+
position: { x: 0, y: 0, z: 0 },
|
|
1618
|
+
right: { x: 1, z: 0 },
|
|
1619
|
+
initialized: false
|
|
1620
|
+
};
|
|
1621
|
+
const positionCache = createPositionSnapCache(0.3);
|
|
1622
|
+
const panSmoother = createPanSmoother();
|
|
1623
|
+
let isMasterMuted = false;
|
|
1624
|
+
const lastGainValues = /* @__PURE__ */ new Map();
|
|
1625
|
+
const lastPanValues = /* @__PURE__ */ new Map();
|
|
1626
|
+
const GAIN_CHANGE_THRESHOLD = 0.02;
|
|
1627
|
+
const PAN_CHANGE_THRESHOLD = 0.03;
|
|
1628
|
+
const lastUpdateTime = /* @__PURE__ */ new Map();
|
|
1629
|
+
const MIN_UPDATE_INTERVAL_MS = 50;
|
|
1630
|
+
let mlNoiseSuppressor = null;
|
|
1631
|
+
let noiseSuppressionMode = "none";
|
|
1632
|
+
const inferenceInFlight = /* @__PURE__ */ new Set();
|
|
1633
|
+
let workletRegistered = false;
|
|
1634
|
+
let workletUrl = null;
|
|
1635
|
+
let outgoingDenoiseChain = null;
|
|
1636
|
+
function createPanner() {
|
|
1637
|
+
const panner = audioContext.createPanner();
|
|
1638
|
+
panner.panningModel = "HRTF";
|
|
1639
|
+
panner.distanceModel = "inverse";
|
|
1640
|
+
panner.refDistance = distanceConfig.refDistance;
|
|
1641
|
+
panner.maxDistance = distanceConfig.maxDistance;
|
|
1642
|
+
panner.rolloffFactor = distanceConfig.rolloffFactor;
|
|
1643
|
+
panner.coneInnerAngle = 360;
|
|
1644
|
+
panner.coneOuterAngle = 360;
|
|
1645
|
+
panner.coneOuterGain = 0.3;
|
|
1646
|
+
return panner;
|
|
1647
|
+
}
|
|
1648
|
+
function createMonoChain() {
|
|
1649
|
+
const monoSplitter = audioContext.createChannelSplitter(2);
|
|
1650
|
+
const monoGainL = audioContext.createGain();
|
|
1651
|
+
const monoGainR = audioContext.createGain();
|
|
1652
|
+
const monoMerger = audioContext.createChannelMerger(1);
|
|
1653
|
+
const stereoUpmixer = audioContext.createChannelMerger(2);
|
|
1654
|
+
monoGainL.gain.value = 0.5;
|
|
1655
|
+
monoGainR.gain.value = 0.5;
|
|
1656
|
+
return { monoSplitter, monoGainL, monoGainR, monoMerger, stereoUpmixer };
|
|
1657
|
+
}
|
|
1658
|
+
function createParticipantCompressor() {
|
|
1659
|
+
const comp = audioContext.createDynamicsCompressor();
|
|
1660
|
+
comp.threshold.value = -30;
|
|
1661
|
+
comp.knee.value = 12;
|
|
1662
|
+
comp.ratio.value = 3;
|
|
1663
|
+
comp.attack.value = 3e-3;
|
|
1664
|
+
comp.release.value = 0.15;
|
|
1665
|
+
return comp;
|
|
1666
|
+
}
|
|
1667
|
+
function createFilters() {
|
|
1668
|
+
const highpassFilter = audioContext.createBiquadFilter();
|
|
1669
|
+
highpassFilter.type = "highpass";
|
|
1670
|
+
highpassFilter.frequency.value = 100;
|
|
1671
|
+
highpassFilter.Q.value = 0.5;
|
|
1672
|
+
const lowpassFilter = audioContext.createBiquadFilter();
|
|
1673
|
+
lowpassFilter.type = "lowpass";
|
|
1674
|
+
lowpassFilter.frequency.value = 1e4;
|
|
1675
|
+
lowpassFilter.Q.value = 0.5;
|
|
1676
|
+
const voiceBandFilter = audioContext.createBiquadFilter();
|
|
1677
|
+
voiceBandFilter.type = "peaking";
|
|
1678
|
+
voiceBandFilter.frequency.value = 180;
|
|
1679
|
+
voiceBandFilter.Q.value = 0.5;
|
|
1680
|
+
voiceBandFilter.gain.value = 0;
|
|
1681
|
+
const dynamicLowpass = audioContext.createBiquadFilter();
|
|
1682
|
+
dynamicLowpass.type = "lowpass";
|
|
1683
|
+
dynamicLowpass.frequency.value = 12e3;
|
|
1684
|
+
dynamicLowpass.Q.value = 0.5;
|
|
1685
|
+
return { highpassFilter, lowpassFilter, voiceBandFilter, dynamicLowpass };
|
|
1686
|
+
}
|
|
1687
|
+
function teardownOutgoingDenoiseChain(chain = outgoingDenoiseChain) {
|
|
1688
|
+
const localPid = "__local_outgoing__";
|
|
1689
|
+
inferenceInFlight.delete(localPid);
|
|
1690
|
+
if (mlNoiseSuppressor) {
|
|
1691
|
+
mlNoiseSuppressor.clearParticipantSmoothingState(localPid);
|
|
1692
|
+
}
|
|
1693
|
+
if (!chain) return;
|
|
1694
|
+
try {
|
|
1695
|
+
chain.denoiseNode.port.close();
|
|
1696
|
+
} catch {
|
|
1697
|
+
}
|
|
1698
|
+
try {
|
|
1699
|
+
chain.source.disconnect();
|
|
1700
|
+
} catch {
|
|
1701
|
+
}
|
|
1702
|
+
try {
|
|
1703
|
+
chain.denoiseNode.disconnect();
|
|
1704
|
+
} catch {
|
|
1705
|
+
}
|
|
1706
|
+
if (outgoingDenoiseChain === chain) {
|
|
1707
|
+
outgoingDenoiseChain = null;
|
|
1708
|
+
}
|
|
1709
|
+
}
|
|
1710
|
+
function getAudioContext() {
|
|
1711
|
+
return audioContext;
|
|
1712
|
+
}
|
|
1713
|
+
async function setupParticipant(participantId, track, bypassSpatialization = false) {
|
|
1714
|
+
if (audioContext.state === "suspended") {
|
|
1715
|
+
await audioContext.resume();
|
|
1716
|
+
}
|
|
1717
|
+
const stream = new MediaStream([track]);
|
|
1718
|
+
const source = audioContext.createMediaStreamSource(stream);
|
|
1719
|
+
const panner = createPanner();
|
|
1720
|
+
const stereoPanner = audioContext.createStereoPanner();
|
|
1721
|
+
const { monoSplitter, monoGainL, monoGainR, monoMerger, stereoUpmixer } = createMonoChain();
|
|
1722
|
+
const analyser = audioContext.createAnalyser();
|
|
1723
|
+
const gain = audioContext.createGain();
|
|
1724
|
+
const proximityGain = audioContext.createGain();
|
|
1725
|
+
const comp = createParticipantCompressor();
|
|
1726
|
+
const { highpassFilter, lowpassFilter, voiceBandFilter, dynamicLowpass } = createFilters();
|
|
1727
|
+
gain.gain.value = 1;
|
|
1728
|
+
proximityGain.gain.value = 1;
|
|
1729
|
+
let noiseNode = null;
|
|
1730
|
+
if (workletRegistered && mlNoiseSuppressor) {
|
|
1731
|
+
const cfg = mlNoiseSuppressor.getModelConfig();
|
|
1732
|
+
noiseNode = new AudioWorkletNode(audioContext, "noise-suppressor", {
|
|
1733
|
+
numberOfInputs: 1,
|
|
1734
|
+
numberOfOutputs: 1,
|
|
1735
|
+
outputChannelCount: [1]
|
|
1736
|
+
});
|
|
1737
|
+
noiseNode.port.postMessage({
|
|
1738
|
+
type: "config",
|
|
1739
|
+
n_fft: cfg?.n_fft ?? 512,
|
|
1740
|
+
hop_length: cfg?.hop_length ?? 80,
|
|
1741
|
+
n_mels: cfg?.n_mels ?? 40,
|
|
1742
|
+
sample_rate: cfg?.sample_rate ?? 16e3,
|
|
1743
|
+
sequence_length: cfg?.sequence_length ?? 8
|
|
1744
|
+
});
|
|
1745
|
+
const workletRef = noiseNode;
|
|
1746
|
+
const seqLen = cfg?.sequence_length ?? 8;
|
|
1747
|
+
const nMels = cfg?.n_mels ?? 40;
|
|
1748
|
+
const _pid = participantId;
|
|
1749
|
+
noiseNode.port.onmessage = async (e) => {
|
|
1750
|
+
if (e.data.type === "mel_features" && mlNoiseSuppressor?.isReady()) {
|
|
1751
|
+
if (inferenceInFlight.has(_pid)) return;
|
|
1752
|
+
inferenceInFlight.add(_pid);
|
|
1753
|
+
try {
|
|
1754
|
+
const gains = await mlNoiseSuppressor.computeGainsFromFeatures(
|
|
1755
|
+
new Float32Array(e.data.features),
|
|
1756
|
+
seqLen,
|
|
1757
|
+
nMels,
|
|
1758
|
+
_pid
|
|
1759
|
+
);
|
|
1760
|
+
try {
|
|
1761
|
+
workletRef.port.postMessage({ type: "gains", gains }, [
|
|
1762
|
+
gains.buffer
|
|
1763
|
+
]);
|
|
1764
|
+
} catch {
|
|
1765
|
+
}
|
|
1766
|
+
} finally {
|
|
1767
|
+
inferenceInFlight.delete(_pid);
|
|
1768
|
+
}
|
|
1769
|
+
}
|
|
1770
|
+
};
|
|
1771
|
+
}
|
|
1772
|
+
if (noiseNode) {
|
|
1773
|
+
source.connect(noiseNode);
|
|
1774
|
+
noiseNode.connect(comp);
|
|
1775
|
+
} else {
|
|
1776
|
+
source.connect(comp);
|
|
1777
|
+
}
|
|
1778
|
+
comp.connect(highpassFilter);
|
|
1779
|
+
highpassFilter.connect(voiceBandFilter);
|
|
1780
|
+
voiceBandFilter.connect(lowpassFilter);
|
|
1781
|
+
lowpassFilter.connect(dynamicLowpass);
|
|
1782
|
+
dynamicLowpass.connect(proximityGain);
|
|
1783
|
+
proximityGain.connect(monoSplitter);
|
|
1784
|
+
monoSplitter.connect(monoGainL, 0);
|
|
1785
|
+
monoSplitter.connect(monoGainR, 1);
|
|
1786
|
+
monoGainL.connect(monoMerger, 0, 0);
|
|
1787
|
+
monoGainR.connect(monoMerger, 0, 0);
|
|
1788
|
+
monoMerger.connect(stereoUpmixer, 0, 0);
|
|
1789
|
+
monoMerger.connect(stereoUpmixer, 0, 1);
|
|
1790
|
+
stereoUpmixer.connect(analyser);
|
|
1791
|
+
if (bypassSpatialization) {
|
|
1792
|
+
analyser.connect(gain);
|
|
1793
|
+
} else {
|
|
1794
|
+
analyser.connect(stereoPanner);
|
|
1795
|
+
stereoPanner.connect(gain);
|
|
1796
|
+
}
|
|
1797
|
+
gain.connect(masterGainNode);
|
|
1798
|
+
participantNodes.set(participantId, {
|
|
1799
|
+
source,
|
|
1800
|
+
panner,
|
|
1801
|
+
stereoPanner,
|
|
1802
|
+
monoSplitter,
|
|
1803
|
+
monoGainL,
|
|
1804
|
+
monoGainR,
|
|
1805
|
+
monoMerger,
|
|
1806
|
+
stereoUpmixer,
|
|
1807
|
+
analyser,
|
|
1808
|
+
gain,
|
|
1809
|
+
proximityGain,
|
|
1810
|
+
compressor: comp,
|
|
1811
|
+
highpassFilter,
|
|
1812
|
+
lowpassFilter,
|
|
1813
|
+
voiceBandFilter,
|
|
1814
|
+
dynamicLowpass,
|
|
1815
|
+
denoiseNode: noiseNode ?? void 0,
|
|
1816
|
+
stream
|
|
1817
|
+
});
|
|
1818
|
+
lastGainValues.set(participantId, 1);
|
|
1819
|
+
lastPanValues.set(participantId, 0);
|
|
1820
|
+
}
|
|
1821
|
+
function updateSpatialAudio(participantId, position, direction) {
|
|
1822
|
+
const nodes = participantNodes.get(participantId);
|
|
1823
|
+
if (!nodes?.panner) return;
|
|
1824
|
+
if (!listenerState.initialized) return;
|
|
1825
|
+
const now = performance.now();
|
|
1826
|
+
const lastTime = lastUpdateTime.get(participantId) || 0;
|
|
1827
|
+
if (now - lastTime < MIN_UPDATE_INTERVAL_MS) return;
|
|
1828
|
+
lastUpdateTime.set(participantId, now);
|
|
1829
|
+
const normalizedPos = normalizePositionUnits(position, distanceConfig.unit);
|
|
1830
|
+
const snappedPos = positionCache.snap(normalizedPos, participantId);
|
|
1831
|
+
const speakerHead = computeHeadPosition(snappedPos);
|
|
1832
|
+
const listenerPos = listenerState.position;
|
|
1833
|
+
const distance = getDistanceBetween(listenerPos, speakerHead);
|
|
1834
|
+
if (distance >= distanceConfig.maxDistance) {
|
|
1835
|
+
const lastGain2 = lastGainValues.get(participantId) ?? 1;
|
|
1836
|
+
if (lastGain2 > 0.01) {
|
|
1837
|
+
applyGainSmooth(nodes.gain, 0, audioContext, 0.1);
|
|
1838
|
+
lastGainValues.set(participantId, 0);
|
|
1839
|
+
}
|
|
1840
|
+
return;
|
|
1841
|
+
}
|
|
1842
|
+
const rawPan = calculatePanFromPositions(
|
|
1843
|
+
listenerPos,
|
|
1844
|
+
speakerHead,
|
|
1845
|
+
listenerState.right
|
|
1846
|
+
);
|
|
1847
|
+
const smoothedPan = panSmoother.smooth(participantId, rawPan);
|
|
1848
|
+
const gainPercent = calculateLogarithmicGain(distance, {
|
|
1849
|
+
minDistance: distanceConfig.refDistance,
|
|
1850
|
+
maxDistance: distanceConfig.maxDistance
|
|
1851
|
+
});
|
|
1852
|
+
const gainValue = gainPercent / 100;
|
|
1853
|
+
const panValue = smoothedPan;
|
|
1854
|
+
const lastPan = lastPanValues.get(participantId) ?? 0;
|
|
1855
|
+
const lastGain = lastGainValues.get(participantId) ?? 1;
|
|
1856
|
+
const panChanged = Math.abs(panValue - lastPan) > PAN_CHANGE_THRESHOLD;
|
|
1857
|
+
const gainChanged = Math.abs(gainValue - lastGain) > GAIN_CHANGE_THRESHOLD;
|
|
1858
|
+
if (panChanged || gainChanged) {
|
|
1859
|
+
const panDirection = panValue > 0.3 ? "RIGHT" : panValue < -0.3 ? "LEFT" : "CENTER";
|
|
1860
|
+
const panPercent = Math.abs(panValue * 100).toFixed(0);
|
|
1861
|
+
debugLog("SPATIAL", `${participantId.slice(0, 8)}`, {
|
|
1862
|
+
dist: distance.toFixed(1) + "m",
|
|
1863
|
+
gain: (gainValue * 100).toFixed(0) + "%",
|
|
1864
|
+
pan: `${panPercent}% ${panDirection}`,
|
|
1865
|
+
rawPan: rawPan.toFixed(2),
|
|
1866
|
+
listenerRight: `(${listenerState.right.x.toFixed(2)}, ${listenerState.right.z.toFixed(2)})`
|
|
1867
|
+
});
|
|
1868
|
+
}
|
|
1869
|
+
if (panChanged) {
|
|
1870
|
+
applyStereoPanSmooth(nodes.stereoPanner, panValue, audioContext, 0.08);
|
|
1871
|
+
lastPanValues.set(participantId, panValue);
|
|
1872
|
+
}
|
|
1873
|
+
const GAIN_THRESHOLD = 0.025;
|
|
1874
|
+
const isFirstCalculation = !lastGainValues.has(participantId);
|
|
1875
|
+
const significantChange = Math.abs(gainValue - lastGain) > GAIN_THRESHOLD;
|
|
1876
|
+
if (isFirstCalculation || significantChange) {
|
|
1877
|
+
applyGainSmooth(nodes.gain, gainValue, audioContext, 0.15);
|
|
1878
|
+
lastGainValues.set(participantId, gainValue);
|
|
1879
|
+
}
|
|
1880
|
+
}
|
|
1881
|
+
function setListenerFromLSD(listenerPos, cameraPos, lookAtPos, rot) {
|
|
1882
|
+
const normalizedListener = normalizePositionUnits(
|
|
1883
|
+
cameraPos,
|
|
1884
|
+
distanceConfig.unit
|
|
1885
|
+
);
|
|
1886
|
+
const snappedListener = positionCache.snap(normalizedListener, "listener");
|
|
1887
|
+
listenerState.position = snappedListener;
|
|
1888
|
+
listenerState.initialized = true;
|
|
1889
|
+
if (rot && typeof rot.y === "number") {
|
|
1890
|
+
listenerState.right = calculateListenerRight(rot.y);
|
|
1891
|
+
} else if (lookAtPos && cameraPos) {
|
|
1892
|
+
const dx = lookAtPos.x - cameraPos.x;
|
|
1893
|
+
const dz = lookAtPos.z - cameraPos.z;
|
|
1894
|
+
const yawRadians = Math.atan2(dx, dz);
|
|
1895
|
+
const yawDegrees = yawRadians * 180 / Math.PI;
|
|
1896
|
+
listenerState.right = calculateListenerRight(yawDegrees);
|
|
1897
|
+
}
|
|
1898
|
+
}
|
|
1899
|
+
function setParticipantSpatialization(participantId, enableSpatialization) {
|
|
1900
|
+
const nodes = participantNodes.get(participantId);
|
|
1901
|
+
if (!nodes) {
|
|
1902
|
+
console.warn(
|
|
1903
|
+
`[SpatialAudioChannel] Cannot set spatialization - no nodes for participant: ${participantId}`
|
|
1904
|
+
);
|
|
1905
|
+
return;
|
|
1906
|
+
}
|
|
1907
|
+
try {
|
|
1908
|
+
const currentTime = audioContext.currentTime;
|
|
1909
|
+
const fadeTime = 0.03;
|
|
1910
|
+
if (enableSpatialization) {
|
|
1911
|
+
nodes.analyser.connect(nodes.stereoPanner);
|
|
1912
|
+
nodes.stereoPanner.connect(nodes.gain);
|
|
1913
|
+
setTimeout(() => {
|
|
1914
|
+
try {
|
|
1915
|
+
} catch (e) {
|
|
1916
|
+
}
|
|
1917
|
+
}, fadeTime * 1e3);
|
|
1918
|
+
} else {
|
|
1919
|
+
nodes.analyser.connect(nodes.gain);
|
|
1920
|
+
nodes.gain.gain.setTargetAtTime(1, currentTime, fadeTime);
|
|
1921
|
+
nodes.stereoPanner.pan.setTargetAtTime(0, currentTime, fadeTime);
|
|
1922
|
+
setTimeout(() => {
|
|
1923
|
+
try {
|
|
1924
|
+
nodes.stereoPanner.disconnect();
|
|
1925
|
+
} catch (e) {
|
|
1926
|
+
}
|
|
1927
|
+
}, fadeTime * 1e3);
|
|
1928
|
+
}
|
|
1929
|
+
} catch (error) {
|
|
1930
|
+
console.error(
|
|
1931
|
+
`[SpatialAudioChannel] Error setting spatialization for ${participantId}:`,
|
|
1932
|
+
error
|
|
1933
|
+
);
|
|
1934
|
+
}
|
|
1935
|
+
}
|
|
1936
|
+
function setParticipantMuted(participantId, muted) {
|
|
1937
|
+
const nodes = participantNodes.get(participantId);
|
|
1938
|
+
if (!nodes?.gain) return;
|
|
1939
|
+
applyGainSmooth(nodes.gain, muted ? 0 : 1, audioContext, 0.05);
|
|
1940
|
+
}
|
|
1941
|
+
function setMasterMuted(muted) {
|
|
1942
|
+
isMasterMuted = muted;
|
|
1943
|
+
applyGainSmooth(masterGainNode, muted ? 0 : 1, audioContext, 0.05);
|
|
1944
|
+
}
|
|
1945
|
+
function getMasterMuted() {
|
|
1946
|
+
return isMasterMuted;
|
|
1947
|
+
}
|
|
1948
|
+
function setListenerPosition(position, orientation) {
|
|
1949
|
+
const normalizedPosition = normalizePositionUnits(position);
|
|
1950
|
+
positionCache.snap(normalizedPosition, "listener");
|
|
1951
|
+
listenerState.position = normalizedPosition;
|
|
1952
|
+
if (orientation) {
|
|
1953
|
+
const yawRadians = Math.atan2(orientation.forwardX, orientation.forwardZ);
|
|
1954
|
+
const yawDegrees = yawRadians * 180 / Math.PI;
|
|
1955
|
+
listenerState.right = calculateListenerRight(yawDegrees);
|
|
1956
|
+
}
|
|
1957
|
+
listenerState.initialized = true;
|
|
1958
|
+
}
|
|
1959
|
+
async function setupSpatialAudioForParticipant(participantId, track, bypassSpatialization) {
|
|
1960
|
+
return setupParticipant(
|
|
1961
|
+
participantId,
|
|
1962
|
+
track,
|
|
1963
|
+
bypassSpatialization || false
|
|
1964
|
+
);
|
|
1965
|
+
}
|
|
1966
|
+
async function initializeMLNoiseSuppression(modelPath, workletUrlParam) {
|
|
1967
|
+
if (workletUrlParam && !workletRegistered) {
|
|
1968
|
+
try {
|
|
1969
|
+
await audioContext.audioWorklet.addModule(workletUrlParam);
|
|
1970
|
+
workletRegistered = true;
|
|
1971
|
+
workletUrl = workletUrlParam;
|
|
1972
|
+
console.log(
|
|
1973
|
+
`[SpatialAudioChannel] AudioWorklet registered: ${workletUrlParam}`
|
|
1974
|
+
);
|
|
1975
|
+
} catch (e) {
|
|
1976
|
+
console.warn(
|
|
1977
|
+
"[SpatialAudioChannel] AudioWorklet registration failed \u2014 falling back to passthrough:",
|
|
1978
|
+
e
|
|
1979
|
+
);
|
|
1980
|
+
}
|
|
1981
|
+
}
|
|
1982
|
+
mlNoiseSuppressor = createMLNoiseSuppressor();
|
|
1983
|
+
await mlNoiseSuppressor.initialize(modelPath);
|
|
1984
|
+
if (!mlNoiseSuppressor.isReady()) {
|
|
1985
|
+
mlNoiseSuppressor = null;
|
|
1986
|
+
throw new Error(
|
|
1987
|
+
`[SpatialAudioChannel] ML model loaded from ${modelPath} but isReady() returned false`
|
|
1988
|
+
);
|
|
1989
|
+
}
|
|
1990
|
+
noiseSuppressionMode = "ml";
|
|
1991
|
+
console.log(
|
|
1992
|
+
`[SpatialAudioChannel] ML noise suppression ACTIVE \u2014 model loaded from ${modelPath}`
|
|
1993
|
+
);
|
|
1994
|
+
}
|
|
1995
|
+
async function enhanceOutgoingAudioTrack(track) {
|
|
1996
|
+
if (track.kind !== "audio") return track;
|
|
1997
|
+
if (!workletRegistered || !mlNoiseSuppressor?.isReady()) return track;
|
|
1998
|
+
if (outgoingDenoiseChain && outgoingDenoiseChain.outputTrack.id === track.id && outgoingDenoiseChain.outputTrack.readyState === "live") {
|
|
1999
|
+
return outgoingDenoiseChain.outputTrack;
|
|
2000
|
+
}
|
|
2001
|
+
if (outgoingDenoiseChain && outgoingDenoiseChain.inputTrack.id === track.id && outgoingDenoiseChain.outputTrack.readyState === "live") {
|
|
2002
|
+
return outgoingDenoiseChain.outputTrack;
|
|
2003
|
+
}
|
|
2004
|
+
const cfg = mlNoiseSuppressor.getModelConfig();
|
|
2005
|
+
if (!cfg) return track;
|
|
2006
|
+
try {
|
|
2007
|
+
if (audioContext.state === "suspended") {
|
|
2008
|
+
await audioContext.resume();
|
|
2009
|
+
}
|
|
2010
|
+
const previousChain = outgoingDenoiseChain;
|
|
2011
|
+
const stream = new MediaStream([track]);
|
|
2012
|
+
const source = audioContext.createMediaStreamSource(stream);
|
|
2013
|
+
const denoiseNode = new AudioWorkletNode(
|
|
2014
|
+
audioContext,
|
|
2015
|
+
"noise-suppressor",
|
|
2016
|
+
{
|
|
2017
|
+
numberOfInputs: 1,
|
|
2018
|
+
numberOfOutputs: 1,
|
|
2019
|
+
outputChannelCount: [1]
|
|
2020
|
+
}
|
|
2021
|
+
);
|
|
2022
|
+
const destination = audioContext.createMediaStreamDestination();
|
|
2023
|
+
denoiseNode.port.postMessage({
|
|
2024
|
+
type: "config",
|
|
2025
|
+
n_fft: cfg.n_fft ?? 512,
|
|
2026
|
+
hop_length: cfg.hop_length ?? 80,
|
|
2027
|
+
n_mels: cfg.n_mels ?? 40,
|
|
2028
|
+
sample_rate: cfg.sample_rate ?? 16e3,
|
|
2029
|
+
sequence_length: cfg.sequence_length ?? 8
|
|
2030
|
+
});
|
|
2031
|
+
const seqLen = cfg.sequence_length ?? 8;
|
|
2032
|
+
const nMels = cfg.n_mels ?? 40;
|
|
2033
|
+
const localPid = "__local_outgoing__";
|
|
2034
|
+
const workletRef = denoiseNode;
|
|
2035
|
+
denoiseNode.port.onmessage = async (e) => {
|
|
2036
|
+
if (e.data.type === "mel_features" && mlNoiseSuppressor?.isReady()) {
|
|
2037
|
+
if (inferenceInFlight.has(localPid)) return;
|
|
2038
|
+
inferenceInFlight.add(localPid);
|
|
2039
|
+
try {
|
|
2040
|
+
const gains = await mlNoiseSuppressor.computeGainsFromFeatures(
|
|
2041
|
+
new Float32Array(e.data.features),
|
|
2042
|
+
seqLen,
|
|
2043
|
+
nMels,
|
|
2044
|
+
localPid
|
|
2045
|
+
);
|
|
2046
|
+
workletRef.port.postMessage({ type: "gains", gains }, [gains.buffer]);
|
|
2047
|
+
} catch {
|
|
2048
|
+
} finally {
|
|
2049
|
+
inferenceInFlight.delete(localPid);
|
|
2050
|
+
}
|
|
2051
|
+
}
|
|
2052
|
+
};
|
|
2053
|
+
source.connect(denoiseNode);
|
|
2054
|
+
denoiseNode.connect(destination);
|
|
2055
|
+
const enhancedTrack = destination.stream.getAudioTracks()[0];
|
|
2056
|
+
if (!enhancedTrack) {
|
|
2057
|
+
try {
|
|
2058
|
+
source.disconnect();
|
|
2059
|
+
} catch {
|
|
2060
|
+
}
|
|
2061
|
+
try {
|
|
2062
|
+
denoiseNode.disconnect();
|
|
2063
|
+
} catch {
|
|
2064
|
+
}
|
|
2065
|
+
return track;
|
|
2066
|
+
}
|
|
2067
|
+
enhancedTrack.enabled = track.enabled;
|
|
2068
|
+
track.addEventListener("ended", () => teardownOutgoingDenoiseChain(), {
|
|
2069
|
+
once: true
|
|
2070
|
+
});
|
|
2071
|
+
enhancedTrack.addEventListener(
|
|
2072
|
+
"ended",
|
|
2073
|
+
() => teardownOutgoingDenoiseChain(),
|
|
2074
|
+
{ once: true }
|
|
2075
|
+
);
|
|
2076
|
+
outgoingDenoiseChain = {
|
|
2077
|
+
inputTrack: track,
|
|
2078
|
+
outputTrack: enhancedTrack,
|
|
2079
|
+
source,
|
|
2080
|
+
denoiseNode,
|
|
2081
|
+
destination
|
|
2082
|
+
};
|
|
2083
|
+
if (previousChain) {
|
|
2084
|
+
teardownOutgoingDenoiseChain(previousChain);
|
|
2085
|
+
}
|
|
2086
|
+
console.log(
|
|
2087
|
+
"[SpatialAudioChannel] Local outgoing ML denoise ACTIVE (AudioWorklet)"
|
|
2088
|
+
);
|
|
2089
|
+
return enhancedTrack;
|
|
2090
|
+
} catch (e) {
|
|
2091
|
+
console.warn(
|
|
2092
|
+
"[SpatialAudioChannel] Failed to enhance outgoing track, using original track:",
|
|
2093
|
+
e
|
|
2094
|
+
);
|
|
2095
|
+
return track;
|
|
2096
|
+
}
|
|
2097
|
+
}
|
|
2098
|
+
function getNoiseSuppressionMode() {
|
|
2099
|
+
return noiseSuppressionMode;
|
|
2100
|
+
}
|
|
2101
|
+
function isMLModelLoaded() {
|
|
2102
|
+
return mlNoiseSuppressor?.isReady() ?? false;
|
|
2103
|
+
}
|
|
2104
|
+
function removeParticipant(participantId) {
|
|
2105
|
+
const nodes = participantNodes.get(participantId);
|
|
2106
|
+
if (nodes) {
|
|
2107
|
+
debugLog("AUDIO", `Removing participant: ${participantId.slice(0, 8)}`, {
|
|
2108
|
+
totalBefore: participantNodes.size,
|
|
2109
|
+
totalAfter: participantNodes.size - 1
|
|
2110
|
+
});
|
|
2111
|
+
nodes.denoiseNode?.port.close();
|
|
2112
|
+
nodes.denoiseNode?.disconnect();
|
|
2113
|
+
nodes.source.disconnect();
|
|
2114
|
+
nodes.panner.disconnect();
|
|
2115
|
+
nodes.stereoPanner.disconnect();
|
|
2116
|
+
nodes.analyser.disconnect();
|
|
2117
|
+
nodes.gain.disconnect();
|
|
2118
|
+
participantNodes.delete(participantId);
|
|
2119
|
+
panSmoother.clear(participantId);
|
|
2120
|
+
positionCache.clear(participantId);
|
|
2121
|
+
lastGainValues.delete(participantId);
|
|
2122
|
+
lastPanValues.delete(participantId);
|
|
2123
|
+
lastUpdateTime.delete(participantId);
|
|
2124
|
+
inferenceInFlight.delete(participantId);
|
|
2125
|
+
if (mlNoiseSuppressor) {
|
|
2126
|
+
mlNoiseSuppressor.clearParticipantSmoothingState(participantId);
|
|
2127
|
+
}
|
|
2128
|
+
} else {
|
|
2129
|
+
debugLog(
|
|
2130
|
+
"AUDIO",
|
|
2131
|
+
`Remove called but participant not found: ${participantId.slice(0, 8)}`
|
|
2132
|
+
);
|
|
2133
|
+
}
|
|
2134
|
+
}
|
|
2135
|
+
async function resumeAudioContext() {
|
|
2136
|
+
if (audioContext.state === "suspended") {
|
|
2137
|
+
await audioContext.resume();
|
|
2138
|
+
}
|
|
2139
|
+
}
|
|
2140
|
+
function getAudioContextState() {
|
|
2141
|
+
return audioContext.state;
|
|
2142
|
+
}
|
|
2143
|
+
function getParticipantAudioLevel(participantId) {
|
|
2144
|
+
const nodes = participantNodes.get(participantId);
|
|
2145
|
+
if (!nodes?.analyser) return 0;
|
|
2146
|
+
const dataArray = new Uint8Array(nodes.analyser.frequencyBinCount);
|
|
2147
|
+
nodes.analyser.getByteFrequencyData(dataArray);
|
|
2148
|
+
let sum2 = 0;
|
|
2149
|
+
for (let i = 0; i < dataArray.length; i++) {
|
|
2150
|
+
sum2 += dataArray[i] * dataArray[i];
|
|
2151
|
+
}
|
|
2152
|
+
const rms = Math.sqrt(sum2 / dataArray.length);
|
|
2153
|
+
return Math.min(100, Math.round(rms / 255 * 100));
|
|
2154
|
+
}
|
|
2155
|
+
function isParticipantSpeaking(participantId, threshold = 5) {
|
|
2156
|
+
return getParticipantAudioLevel(participantId) > threshold;
|
|
2157
|
+
}
|
|
2158
|
+
function getActiveParticipants() {
|
|
2159
|
+
return Array.from(participantNodes.keys());
|
|
2160
|
+
}
|
|
2161
|
+
function hasParticipant(participantId) {
|
|
2162
|
+
return participantNodes.has(participantId);
|
|
2163
|
+
}
|
|
2164
|
+
function getParticipantCount() {
|
|
2165
|
+
return participantNodes.size;
|
|
2166
|
+
}
|
|
2167
|
+
return {
|
|
2168
|
+
getAudioContext,
|
|
2169
|
+
setupParticipant,
|
|
2170
|
+
setupSpatialAudioForParticipant,
|
|
2171
|
+
updateSpatialAudio,
|
|
2172
|
+
setListenerFromLSD,
|
|
2173
|
+
setListenerPosition,
|
|
2174
|
+
setParticipantSpatialization,
|
|
2175
|
+
setParticipantMuted,
|
|
2176
|
+
setMasterMuted,
|
|
2177
|
+
getMasterMuted,
|
|
2178
|
+
initializeMLNoiseSuppression,
|
|
2179
|
+
enhanceOutgoingAudioTrack,
|
|
2180
|
+
getNoiseSuppressionMode,
|
|
2181
|
+
isMLModelLoaded,
|
|
2182
|
+
removeParticipant,
|
|
2183
|
+
resumeAudioContext,
|
|
2184
|
+
getAudioContextState,
|
|
2185
|
+
getParticipantAudioLevel,
|
|
2186
|
+
isParticipantSpeaking,
|
|
2187
|
+
getActiveParticipants,
|
|
2188
|
+
hasParticipant,
|
|
2189
|
+
getParticipantCount
|
|
2190
|
+
};
|
|
2191
|
+
}
|
|
2192
|
+
|
|
2193
|
+
// src/screen-share-live-in-space/screen-share-live-manager.ts
|
|
2194
|
+
function createScreenShareLiveManager(socket) {
|
|
2195
|
+
let isBroadcasting = false;
|
|
2196
|
+
let currentProducerId = null;
|
|
2197
|
+
function findScreenshareProducerId(participant) {
|
|
2198
|
+
for (const [producerId, producer] of participant.producers) {
|
|
2199
|
+
if (producer.appData?.isScreenshare) {
|
|
2200
|
+
return producerId;
|
|
2201
|
+
}
|
|
2202
|
+
}
|
|
2203
|
+
return null;
|
|
2204
|
+
}
|
|
2205
|
+
async function startBroadcast(room, participant, serverUrl) {
|
|
2206
|
+
if (!room || !participant) {
|
|
2207
|
+
throw new Error("Must be in a room to start space live broadcast");
|
|
2208
|
+
}
|
|
2209
|
+
const screenshareProducerId = findScreenshareProducerId(participant);
|
|
2210
|
+
if (!screenshareProducerId) {
|
|
2211
|
+
throw new Error("No active screen share to broadcast to space");
|
|
2212
|
+
}
|
|
2213
|
+
return new Promise((resolve, reject) => {
|
|
2214
|
+
const request = {
|
|
2215
|
+
roomId: room.id,
|
|
2216
|
+
producerId: screenshareProducerId,
|
|
2217
|
+
participantId: participant.participantId,
|
|
2218
|
+
userName: participant.userName || "Unknown"
|
|
2219
|
+
};
|
|
2220
|
+
const timestamp = Date.now();
|
|
2221
|
+
socket.emit(
|
|
2222
|
+
"start-space-live-broadcast",
|
|
2223
|
+
request,
|
|
2224
|
+
(response) => {
|
|
2225
|
+
if (response.error) {
|
|
2226
|
+
reject(new Error(response.error));
|
|
2227
|
+
} else {
|
|
2228
|
+
isBroadcasting = true;
|
|
2229
|
+
currentProducerId = screenshareProducerId;
|
|
2230
|
+
debugLog("SPACE-LIVE", "Started space live broadcast", {
|
|
2231
|
+
producerId: screenshareProducerId
|
|
2232
|
+
});
|
|
2233
|
+
resolve({
|
|
2234
|
+
producerId: screenshareProducerId,
|
|
2235
|
+
roomId: room.id,
|
|
2236
|
+
serverUrl,
|
|
2237
|
+
timestamp
|
|
2238
|
+
});
|
|
2239
|
+
}
|
|
2240
|
+
}
|
|
2241
|
+
);
|
|
2242
|
+
});
|
|
2243
|
+
}
|
|
2244
|
+
async function stopBroadcast(room, participantId) {
|
|
2245
|
+
if (!room) {
|
|
2246
|
+
isBroadcasting = false;
|
|
2247
|
+
currentProducerId = null;
|
|
2248
|
+
return;
|
|
2249
|
+
}
|
|
2250
|
+
return new Promise((resolve) => {
|
|
2251
|
+
const request = {
|
|
2252
|
+
roomId: room.id,
|
|
2253
|
+
participantId
|
|
2254
|
+
};
|
|
2255
|
+
socket.emit("stop-space-live-broadcast", request, () => {
|
|
2256
|
+
isBroadcasting = false;
|
|
2257
|
+
currentProducerId = null;
|
|
2258
|
+
debugLog("SPACE-LIVE", "Stopped space live broadcast");
|
|
2259
|
+
resolve();
|
|
2260
|
+
});
|
|
2261
|
+
});
|
|
2262
|
+
}
|
|
2263
|
+
function getStatus(roomId, callback) {
|
|
2264
|
+
if (!roomId) {
|
|
2265
|
+
callback({ isActive: false });
|
|
2266
|
+
return;
|
|
2267
|
+
}
|
|
2268
|
+
socket.emit("get-space-live-broadcast-status", { roomId }, callback);
|
|
2269
|
+
}
|
|
2270
|
+
function onBroadcastAvailable(callback) {
|
|
2271
|
+
socket.on("space-live-broadcast-available", callback);
|
|
2272
|
+
}
|
|
2273
|
+
function onBroadcastStopped(callback) {
|
|
2274
|
+
socket.on("space-live-broadcast-stopped", callback);
|
|
2275
|
+
}
|
|
2276
|
+
function removeListeners() {
|
|
2277
|
+
socket.off("space-live-broadcast-available");
|
|
2278
|
+
socket.off("space-live-broadcast-stopped");
|
|
2279
|
+
}
|
|
2280
|
+
function cleanup() {
|
|
2281
|
+
isBroadcasting = false;
|
|
2282
|
+
currentProducerId = null;
|
|
2283
|
+
removeListeners();
|
|
2284
|
+
}
|
|
2285
|
+
return {
|
|
2286
|
+
get isActive() {
|
|
2287
|
+
return isBroadcasting;
|
|
2288
|
+
},
|
|
2289
|
+
get broadcastProducerId() {
|
|
2290
|
+
return currentProducerId;
|
|
2291
|
+
},
|
|
2292
|
+
startBroadcast,
|
|
2293
|
+
stopBroadcast,
|
|
2294
|
+
getStatus,
|
|
2295
|
+
onBroadcastAvailable,
|
|
2296
|
+
onBroadcastStopped,
|
|
2297
|
+
removeListeners,
|
|
2298
|
+
cleanup
|
|
2299
|
+
};
|
|
2300
|
+
}
|
|
2301
|
+
|
|
2302
|
+
// src/core/room.ts
|
|
2303
|
+
function createRoomActions(ctx) {
|
|
2304
|
+
const { socket, mediasoupManager, state, emitEvent } = ctx;
|
|
2305
|
+
async function joinRoom(data) {
|
|
2306
|
+
if (!socket.connected) {
|
|
2307
|
+
await new Promise((resolve, reject) => {
|
|
2308
|
+
const CONNECT_TIMEOUT_MS = 1e4;
|
|
2309
|
+
const timer = setTimeout(() => {
|
|
2310
|
+
socket.off("connect", onConnect);
|
|
2311
|
+
socket.off("connect_error", onError);
|
|
2312
|
+
reject(
|
|
2313
|
+
new Error(
|
|
2314
|
+
"[OdysseySDK] Connection timed out. Verify the server is reachable and your apiKey/userToken are valid."
|
|
2315
|
+
)
|
|
2316
|
+
);
|
|
2317
|
+
}, CONNECT_TIMEOUT_MS);
|
|
2318
|
+
const onConnect = () => {
|
|
2319
|
+
clearTimeout(timer);
|
|
2320
|
+
socket.off("connect_error", onError);
|
|
2321
|
+
resolve();
|
|
2322
|
+
};
|
|
2323
|
+
const onError = (err) => {
|
|
2324
|
+
clearTimeout(timer);
|
|
2325
|
+
socket.off("connect", onConnect);
|
|
2326
|
+
reject(new Error(`[OdysseySDK] Connection failed: ${err.message}`));
|
|
2327
|
+
};
|
|
2328
|
+
socket.once("connect", onConnect);
|
|
2329
|
+
socket.once("connect_error", onError);
|
|
2330
|
+
});
|
|
2331
|
+
}
|
|
2332
|
+
return new Promise((resolve, reject) => {
|
|
2333
|
+
const handleRoomJoined = async (roomData) => {
|
|
2334
|
+
try {
|
|
2335
|
+
socket.off("room-joined", handleRoomJoined);
|
|
2336
|
+
await mediasoupManager.loadDevice(roomData.routerRtpCapabilities);
|
|
2337
|
+
await mediasoupManager.createSendTransport(roomData.participantId);
|
|
2338
|
+
await mediasoupManager.createRecvTransport(roomData.participantId);
|
|
2339
|
+
mediasoupManager.sendDeviceRtpCapabilities(roomData.participantId);
|
|
2340
|
+
const localParticipantData = roomData.participants.find(
|
|
2341
|
+
(p) => p.participantId === roomData.participantId
|
|
2342
|
+
);
|
|
2343
|
+
if (!localParticipantData) {
|
|
2344
|
+
throw new Error("Could not find local participant in room data");
|
|
2345
|
+
}
|
|
2346
|
+
state.localParticipant = {
|
|
2347
|
+
...localParticipantData,
|
|
2348
|
+
isLocal: true,
|
|
2349
|
+
producers: /* @__PURE__ */ new Map(),
|
|
2350
|
+
consumers: /* @__PURE__ */ new Map(),
|
|
2351
|
+
currentChannel: "spatial"
|
|
2352
|
+
};
|
|
2353
|
+
state.room = { id: roomData.roomId, participants: /* @__PURE__ */ new Map() };
|
|
2354
|
+
for (const pData of roomData.participants) {
|
|
2355
|
+
const participant = {
|
|
2356
|
+
...pData,
|
|
2357
|
+
isLocal: pData.participantId === state.localParticipant.participantId,
|
|
2358
|
+
producers: /* @__PURE__ */ new Map(),
|
|
2359
|
+
consumers: /* @__PURE__ */ new Map()
|
|
2360
|
+
};
|
|
2361
|
+
state.room.participants.set(pData.participantId, participant);
|
|
2362
|
+
}
|
|
2363
|
+
emitEvent("room-joined", roomData);
|
|
2364
|
+
resolve(state.localParticipant);
|
|
2365
|
+
} catch (error) {
|
|
2366
|
+
socket.off("room-joined", handleRoomJoined);
|
|
2367
|
+
reject(error);
|
|
2368
|
+
}
|
|
2369
|
+
};
|
|
2370
|
+
socket.on("room-joined", handleRoomJoined);
|
|
2371
|
+
socket.emit("join-room", data);
|
|
2372
|
+
});
|
|
2373
|
+
}
|
|
2374
|
+
function leaveRoom() {
|
|
2375
|
+
if (!socket.connected) return;
|
|
2376
|
+
socket.emit("leave-room");
|
|
2377
|
+
mediasoupManager.close();
|
|
2378
|
+
socket.disconnect();
|
|
2379
|
+
state.room = null;
|
|
2380
|
+
state.localParticipant = null;
|
|
2381
|
+
ctx.bus.removeAllListeners();
|
|
2382
|
+
}
|
|
2383
|
+
function updatePosition(position, direction, spatialData) {
|
|
2384
|
+
if (state.localParticipant && state.room) {
|
|
2385
|
+
state.localParticipant.position = position;
|
|
2386
|
+
state.localParticipant.direction = direction;
|
|
2387
|
+
socket.emit("update-position", {
|
|
2388
|
+
participantId: state.localParticipant.participantId,
|
|
2389
|
+
conferenceId: state.room.id,
|
|
2390
|
+
roomId: state.room.id,
|
|
2391
|
+
position,
|
|
2392
|
+
direction,
|
|
2393
|
+
rot: spatialData?.rot,
|
|
2394
|
+
cameraDistance: spatialData?.cameraDistance,
|
|
2395
|
+
screenPos: spatialData?.screenPos,
|
|
2396
|
+
pan: spatialData?.pan,
|
|
2397
|
+
dxLocal: spatialData?.dxLocal
|
|
2398
|
+
});
|
|
2399
|
+
}
|
|
2400
|
+
}
|
|
2401
|
+
function updateMediaState(mediaState) {
|
|
2402
|
+
if (state.localParticipant && state.room) {
|
|
2403
|
+
state.localParticipant.mediaState = mediaState;
|
|
2404
|
+
socket.emit("update-media-state", {
|
|
2405
|
+
participantId: state.localParticipant.participantId,
|
|
2406
|
+
conferenceId: state.room.id,
|
|
2407
|
+
roomId: state.room.id,
|
|
2408
|
+
mediaState
|
|
2409
|
+
});
|
|
2410
|
+
}
|
|
2411
|
+
}
|
|
2412
|
+
return { joinRoom, leaveRoom, updatePosition, updateMediaState };
|
|
2413
|
+
}
|
|
2414
|
+
|
|
2415
|
+
// src/producers/tracks.ts
|
|
2416
|
+
function createTrackActions(ctx) {
|
|
2417
|
+
const { socket, state, mediasoupManager, updateMediaState } = ctx;
|
|
2418
|
+
async function produceTrack(track, appData) {
|
|
2419
|
+
const producer = await mediasoupManager.produce(track, appData);
|
|
2420
|
+
if (state.localParticipant) {
|
|
2421
|
+
state.localParticipant.producers.set(producer.id, producer);
|
|
2422
|
+
if (track.kind === "audio") {
|
|
2423
|
+
state.localParticipant.audioTrack = track;
|
|
2424
|
+
state.localParticipant.mediaState.audio = true;
|
|
2425
|
+
} else if (track.kind === "video") {
|
|
2426
|
+
if (appData?.isScreenshare) {
|
|
2427
|
+
state.localParticipant.screenshareTrack = track;
|
|
2428
|
+
state.localParticipant.mediaState.sharescreen = true;
|
|
2429
|
+
} else {
|
|
2430
|
+
state.localParticipant.videoTrack = track;
|
|
2431
|
+
state.localParticipant.mediaState.video = true;
|
|
2432
|
+
}
|
|
2433
|
+
}
|
|
2434
|
+
updateMediaState(state.localParticipant.mediaState);
|
|
2435
|
+
}
|
|
2436
|
+
return producer;
|
|
2437
|
+
}
|
|
2438
|
+
async function recreateProducers() {
|
|
2439
|
+
if (!state.localParticipant) return;
|
|
2440
|
+
state.localParticipant.producers.clear();
|
|
2441
|
+
const tracksToReproduce = [];
|
|
2442
|
+
if (state.localParticipant.audioTrack && state.localParticipant.audioTrack.readyState === "live") {
|
|
2443
|
+
tracksToReproduce.push({ track: state.localParticipant.audioTrack });
|
|
2444
|
+
}
|
|
2445
|
+
if (state.localParticipant.videoTrack && state.localParticipant.videoTrack.readyState === "live") {
|
|
2446
|
+
tracksToReproduce.push({ track: state.localParticipant.videoTrack });
|
|
2447
|
+
}
|
|
2448
|
+
const screenshareTrack = state.localParticipant.screenshareTrack;
|
|
2449
|
+
if (screenshareTrack && screenshareTrack.readyState === "live") {
|
|
2450
|
+
tracksToReproduce.push({
|
|
2451
|
+
track: screenshareTrack,
|
|
2452
|
+
appData: { isScreenshare: true }
|
|
2453
|
+
});
|
|
2454
|
+
}
|
|
2455
|
+
for (const { track, appData } of tracksToReproduce) {
|
|
2456
|
+
try {
|
|
2457
|
+
await produceTrack(track, appData);
|
|
2458
|
+
} catch (error) {
|
|
2459
|
+
}
|
|
2460
|
+
}
|
|
2461
|
+
}
|
|
2462
|
+
async function stopScreenShare() {
|
|
2463
|
+
if (!state.localParticipant || !state.room) return;
|
|
2464
|
+
const screenshareTrack = state.localParticipant.screenshareTrack;
|
|
2465
|
+
if (screenshareTrack) {
|
|
2466
|
+
screenshareTrack.stop();
|
|
2467
|
+
state.localParticipant.screenshareTrack = null;
|
|
2468
|
+
}
|
|
2469
|
+
for (const [producerId, producer] of state.localParticipant.producers) {
|
|
2470
|
+
if (producer.appData?.isScreenshare) {
|
|
2471
|
+
producer.close();
|
|
2472
|
+
state.localParticipant.producers.delete(producerId);
|
|
2473
|
+
break;
|
|
2474
|
+
}
|
|
2475
|
+
}
|
|
2476
|
+
return new Promise((resolve) => {
|
|
2477
|
+
socket.emit(
|
|
2478
|
+
"stop-screenshare",
|
|
2479
|
+
{ participantId: state.localParticipant.participantId },
|
|
2480
|
+
() => {
|
|
2481
|
+
state.localParticipant.mediaState.sharescreen = false;
|
|
2482
|
+
resolve();
|
|
2483
|
+
}
|
|
2484
|
+
);
|
|
2485
|
+
});
|
|
2486
|
+
}
|
|
2487
|
+
async function stopVideoProducer() {
|
|
2488
|
+
if (!state.localParticipant || !state.room) return;
|
|
2489
|
+
const videoTrack = state.localParticipant.videoTrack;
|
|
2490
|
+
if (videoTrack) {
|
|
2491
|
+
videoTrack.stop();
|
|
2492
|
+
state.localParticipant.videoTrack = void 0;
|
|
2493
|
+
}
|
|
2494
|
+
for (const [producerId, producer] of state.localParticipant.producers) {
|
|
2495
|
+
if (producer.kind === "video" && !producer.appData?.isScreenshare) {
|
|
2496
|
+
producer.close();
|
|
2497
|
+
state.localParticipant.producers.delete(producerId);
|
|
2498
|
+
break;
|
|
2499
|
+
}
|
|
2500
|
+
}
|
|
2501
|
+
return new Promise((resolve) => {
|
|
2502
|
+
socket.emit(
|
|
2503
|
+
"stop-video",
|
|
2504
|
+
{ participantId: state.localParticipant.participantId },
|
|
2505
|
+
() => {
|
|
2506
|
+
state.localParticipant.mediaState.video = false;
|
|
2507
|
+
resolve();
|
|
2508
|
+
}
|
|
2509
|
+
);
|
|
2510
|
+
});
|
|
2511
|
+
}
|
|
2512
|
+
return { produceTrack, recreateProducers, stopScreenShare, stopVideoProducer };
|
|
2513
|
+
}
|
|
2514
|
+
|
|
2515
|
+
// src/events/listeners.ts
|
|
2516
|
+
function registerSocketListeners(ctx) {
|
|
2517
|
+
const { socket, state, mediasoupManager, spatialAudioManager, emitEvent } = ctx;
|
|
2518
|
+
socket.on("all-participants-update", (payload) => {
|
|
2519
|
+
if (!state.room) {
|
|
2520
|
+
state.room = { id: payload.roomId, participants: /* @__PURE__ */ new Map() };
|
|
2521
|
+
}
|
|
2522
|
+
const activeParticipantIds = /* @__PURE__ */ new Set();
|
|
2523
|
+
for (const snapshot of payload.participants) {
|
|
2524
|
+
activeParticipantIds.add(snapshot.participantId);
|
|
2525
|
+
let participant = state.room?.participants.get(snapshot.participantId);
|
|
2526
|
+
const normalizedPosition = {
|
|
2527
|
+
x: snapshot.position?.x ?? 0,
|
|
2528
|
+
y: snapshot.position?.y ?? 0,
|
|
2529
|
+
z: snapshot.position?.z ?? 0
|
|
2530
|
+
};
|
|
2531
|
+
const normalizedDirection = {
|
|
2532
|
+
x: snapshot.direction?.x ?? 0,
|
|
2533
|
+
y: snapshot.direction?.y ?? 0,
|
|
2534
|
+
z: snapshot.direction?.z ?? 1
|
|
2535
|
+
};
|
|
2536
|
+
const normalizedMediaState = {
|
|
2537
|
+
audio: snapshot.mediaState?.audio ?? false,
|
|
2538
|
+
video: snapshot.mediaState?.video ?? false,
|
|
2539
|
+
sharescreen: snapshot.mediaState?.sharescreen ?? false
|
|
2540
|
+
};
|
|
2541
|
+
if (!participant) {
|
|
2542
|
+
participant = {
|
|
2543
|
+
participantId: snapshot.participantId,
|
|
2544
|
+
userId: snapshot.userId,
|
|
2545
|
+
deviceId: snapshot.deviceId,
|
|
2546
|
+
isLocal: state.localParticipant?.participantId === snapshot.participantId,
|
|
2547
|
+
audioTrack: void 0,
|
|
2548
|
+
videoTrack: void 0,
|
|
2549
|
+
producers: /* @__PURE__ */ new Map(),
|
|
2550
|
+
consumers: /* @__PURE__ */ new Map(),
|
|
2551
|
+
position: normalizedPosition,
|
|
2552
|
+
direction: normalizedDirection,
|
|
2553
|
+
mediaState: normalizedMediaState
|
|
2554
|
+
};
|
|
2555
|
+
} else {
|
|
2556
|
+
participant.userId = snapshot.userId;
|
|
2557
|
+
participant.deviceId = snapshot.deviceId;
|
|
2558
|
+
participant.position = normalizedPosition;
|
|
2559
|
+
participant.direction = normalizedDirection;
|
|
2560
|
+
participant.mediaState = normalizedMediaState;
|
|
2561
|
+
}
|
|
2562
|
+
participant.bodyHeight = snapshot.bodyHeight;
|
|
2563
|
+
participant.bodyShape = snapshot.bodyShape;
|
|
2564
|
+
participant.userName = snapshot.userName;
|
|
2565
|
+
participant.userEmail = snapshot.userEmail;
|
|
2566
|
+
participant.isLocal = state.localParticipant?.participantId === snapshot.participantId;
|
|
2567
|
+
participant.currentChannel = snapshot.currentChannel || "spatial";
|
|
2568
|
+
state.room?.participants.set(snapshot.participantId, participant);
|
|
2569
|
+
if (participant.isLocal) {
|
|
2570
|
+
state.localParticipant = participant;
|
|
2571
|
+
}
|
|
2572
|
+
}
|
|
2573
|
+
if (state.room) {
|
|
2574
|
+
for (const existingId of Array.from(state.room.participants.keys())) {
|
|
2575
|
+
if (!activeParticipantIds.has(existingId)) {
|
|
2576
|
+
state.room.participants.delete(existingId);
|
|
2577
|
+
}
|
|
2578
|
+
}
|
|
2579
|
+
}
|
|
2580
|
+
const normalizedParticipants = state.room ? Array.from(state.room.participants.values()) : [];
|
|
2581
|
+
emitEvent("all-participants-update", normalizedParticipants);
|
|
2582
|
+
});
|
|
2583
|
+
socket.on("new-participant", (participantData) => {
|
|
2584
|
+
if (state.room) {
|
|
2585
|
+
const newParticipant = {
|
|
2586
|
+
participantId: participantData.participantId,
|
|
2587
|
+
userId: participantData.userId,
|
|
2588
|
+
deviceId: participantData.deviceId,
|
|
2589
|
+
position: participantData.position,
|
|
2590
|
+
direction: participantData.direction,
|
|
2591
|
+
mediaState: participantData.mediaState,
|
|
2592
|
+
isLocal: false,
|
|
2593
|
+
producers: /* @__PURE__ */ new Map(),
|
|
2594
|
+
consumers: /* @__PURE__ */ new Map(),
|
|
2595
|
+
bodyHeight: participantData.bodyHeight,
|
|
2596
|
+
bodyShape: participantData.bodyShape,
|
|
2597
|
+
userName: participantData.userName,
|
|
2598
|
+
userEmail: participantData.userEmail
|
|
2599
|
+
};
|
|
2600
|
+
newParticipant.currentChannel = participantData.currentChannel || "spatial";
|
|
2601
|
+
state.room.participants.set(
|
|
2602
|
+
participantData.participantId,
|
|
2603
|
+
newParticipant
|
|
2604
|
+
);
|
|
2605
|
+
emitEvent("new-participant", newParticipant);
|
|
2606
|
+
}
|
|
2607
|
+
});
|
|
2608
|
+
socket.on("participant-left", (data) => {
|
|
2609
|
+
if (state.room) {
|
|
2610
|
+
state.room.participants.delete(data.participantId);
|
|
2611
|
+
spatialAudioManager.removeParticipant(data.participantId);
|
|
2612
|
+
emitEvent("participant-left", data.participantId);
|
|
2613
|
+
}
|
|
2614
|
+
});
|
|
2615
|
+
socket.on(
|
|
2616
|
+
"consumer-created",
|
|
2617
|
+
async (data) => {
|
|
2618
|
+
debugLog("CONSUMER", `Full data for ${data.userName}`, {
|
|
2619
|
+
participantId: data.participantId,
|
|
2620
|
+
position: data.position,
|
|
2621
|
+
channel: data.currentChannel,
|
|
2622
|
+
mediaState: data.mediaState
|
|
2623
|
+
});
|
|
2624
|
+
let consumer;
|
|
2625
|
+
let track;
|
|
2626
|
+
try {
|
|
2627
|
+
const result = await mediasoupManager.consume(data);
|
|
2628
|
+
consumer = result.consumer;
|
|
2629
|
+
track = result.track;
|
|
2630
|
+
} catch (err) {
|
|
2631
|
+
console.warn(
|
|
2632
|
+
`[SDK] consume() skipped for ${data.participantId?.substring(0, 8)} \u2014 transport not ready (${err?.message}).`
|
|
2633
|
+
);
|
|
2634
|
+
return;
|
|
2635
|
+
}
|
|
2636
|
+
let participant = state.room?.participants.get(data.participantId);
|
|
2637
|
+
if (!participant && state.room) {
|
|
2638
|
+
participant = {
|
|
2639
|
+
participantId: data.participantId,
|
|
2640
|
+
userId: data.participantId.split(":")[0] || data.participantId,
|
|
2641
|
+
deviceId: data.participantId.split(":")[1] || data.participantId,
|
|
2642
|
+
isLocal: false,
|
|
2643
|
+
position: data.position,
|
|
2644
|
+
direction: data.direction,
|
|
2645
|
+
mediaState: data.mediaState,
|
|
2646
|
+
producers: /* @__PURE__ */ new Map(),
|
|
2647
|
+
consumers: /* @__PURE__ */ new Map(),
|
|
2648
|
+
bodyHeight: data.bodyHeight,
|
|
2649
|
+
bodyShape: data.bodyShape,
|
|
2650
|
+
userName: data.userName,
|
|
2651
|
+
userEmail: data.userEmail
|
|
2652
|
+
};
|
|
2653
|
+
participant.currentChannel = data.currentChannel || "spatial";
|
|
2654
|
+
state.room.participants.set(data.participantId, participant);
|
|
2655
|
+
}
|
|
2656
|
+
if (participant) {
|
|
2657
|
+
participant.position = data.position;
|
|
2658
|
+
participant.direction = data.direction;
|
|
2659
|
+
participant.mediaState = data.mediaState;
|
|
2660
|
+
participant.currentChannel = data.currentChannel || participant.currentChannel || "spatial";
|
|
2661
|
+
if (data.bodyHeight !== void 0)
|
|
2662
|
+
participant.bodyHeight = data.bodyHeight;
|
|
2663
|
+
if (data.bodyShape !== void 0)
|
|
2664
|
+
participant.bodyShape = data.bodyShape;
|
|
2665
|
+
if (data.userName !== void 0) participant.userName = data.userName;
|
|
2666
|
+
if (data.userEmail !== void 0)
|
|
2667
|
+
participant.userEmail = data.userEmail;
|
|
2668
|
+
participant.consumers.set(consumer.id, consumer);
|
|
2669
|
+
if (track.kind === "audio") {
|
|
2670
|
+
participant.audioTrack = track;
|
|
2671
|
+
const isLocalParticipant = participant.participantId === state.localParticipant?.participantId;
|
|
2672
|
+
if (isLocalParticipant) {
|
|
2673
|
+
return;
|
|
2674
|
+
}
|
|
2675
|
+
const participantChannel = participant.currentChannel || "spatial";
|
|
2676
|
+
const isInHuddle = participantChannel !== "spatial";
|
|
2677
|
+
await spatialAudioManager.setupSpatialAudioForParticipant(
|
|
2678
|
+
participant.participantId,
|
|
2679
|
+
track,
|
|
2680
|
+
isInHuddle
|
|
2681
|
+
);
|
|
2682
|
+
mediasoupManager.resumeConsumer(consumer.id).then(() => {
|
|
2683
|
+
}).catch((err) => {
|
|
2684
|
+
console.error(`[SDK] Failed to resume consumer:`, err);
|
|
2685
|
+
});
|
|
2686
|
+
} else if (track.kind === "video") {
|
|
2687
|
+
const isScreenshare = data.appData?.isScreenshare;
|
|
2688
|
+
if (isScreenshare) {
|
|
2689
|
+
participant.screenshareTrack = track;
|
|
2690
|
+
} else {
|
|
2691
|
+
participant.videoTrack = track;
|
|
2692
|
+
}
|
|
2693
|
+
const resumeWithRetry = async (retries = 3) => {
|
|
2694
|
+
for (let i = 0; i < retries; i++) {
|
|
2695
|
+
try {
|
|
2696
|
+
await mediasoupManager.resumeConsumer(consumer.id);
|
|
2697
|
+
return;
|
|
2698
|
+
} catch (err) {
|
|
2699
|
+
if (i < retries - 1) {
|
|
2700
|
+
await new Promise((resolve) => setTimeout(resolve, 200));
|
|
2701
|
+
} else {
|
|
2702
|
+
console.error(
|
|
2703
|
+
`[SDK] Failed to resume video consumer after ${retries} attempts:`,
|
|
2704
|
+
err
|
|
2705
|
+
);
|
|
2706
|
+
}
|
|
2707
|
+
}
|
|
2708
|
+
}
|
|
2709
|
+
};
|
|
2710
|
+
resumeWithRetry();
|
|
2711
|
+
}
|
|
2712
|
+
emitEvent("consumer-created", {
|
|
2713
|
+
participant,
|
|
2714
|
+
track,
|
|
2715
|
+
consumer,
|
|
2716
|
+
isScreenshare: data.appData?.isScreenshare,
|
|
2717
|
+
appData: data.appData
|
|
2718
|
+
});
|
|
2719
|
+
}
|
|
2720
|
+
}
|
|
2721
|
+
);
|
|
2722
|
+
socket.on(
|
|
2723
|
+
"participant-position-updated",
|
|
2724
|
+
(data) => {
|
|
2725
|
+
const participant = state.room?.participants.get(data.participantId);
|
|
2726
|
+
if (participant) {
|
|
2727
|
+
debugLog(
|
|
2728
|
+
"POSITION",
|
|
2729
|
+
`${data.userName || data.participantId.slice(0, 8)}`,
|
|
2730
|
+
{
|
|
2731
|
+
pos: {
|
|
2732
|
+
x: data.position?.x?.toFixed(1),
|
|
2733
|
+
y: data.position?.y?.toFixed(1)
|
|
2734
|
+
},
|
|
2735
|
+
channel: data.currentChannel
|
|
2736
|
+
}
|
|
2737
|
+
);
|
|
2738
|
+
participant.position = data.position;
|
|
2739
|
+
participant.direction = data.direction;
|
|
2740
|
+
participant.mediaState = data.mediaState;
|
|
2741
|
+
if (data.bodyHeight !== void 0)
|
|
2742
|
+
participant.bodyHeight = data.bodyHeight;
|
|
2743
|
+
if (data.bodyShape !== void 0)
|
|
2744
|
+
participant.bodyShape = data.bodyShape;
|
|
2745
|
+
if (data.userName !== void 0) participant.userName = data.userName;
|
|
2746
|
+
if (data.userEmail !== void 0)
|
|
2747
|
+
participant.userEmail = data.userEmail;
|
|
2748
|
+
if (data.currentChannel !== void 0)
|
|
2749
|
+
participant.currentChannel = data.currentChannel;
|
|
2750
|
+
if (data.pan !== void 0) participant.pan = data.pan;
|
|
2751
|
+
if (data.dxLocal !== void 0)
|
|
2752
|
+
participant.dxLocal = data.dxLocal;
|
|
2753
|
+
if (data.rot !== void 0) participant.rot = data.rot;
|
|
2754
|
+
const participantChannel = participant.currentChannel;
|
|
2755
|
+
const isInHuddle = participantChannel !== "spatial";
|
|
2756
|
+
if (!isInHuddle) {
|
|
2757
|
+
spatialAudioManager.updateSpatialAudio(
|
|
2758
|
+
data.participantId,
|
|
2759
|
+
data.position,
|
|
2760
|
+
data.rot || data.direction
|
|
2761
|
+
);
|
|
2762
|
+
}
|
|
2763
|
+
emitEvent("participant-position-updated", participant);
|
|
2764
|
+
}
|
|
2765
|
+
}
|
|
2766
|
+
);
|
|
2767
|
+
socket.on(
|
|
2768
|
+
"participant-media-state-updated",
|
|
2769
|
+
(data) => {
|
|
2770
|
+
const participant = state.room?.participants.get(data.participantId);
|
|
2771
|
+
if (participant) {
|
|
2772
|
+
participant.mediaState = data.mediaState;
|
|
2773
|
+
participant.position = data.position;
|
|
2774
|
+
participant.direction = data.direction;
|
|
2775
|
+
emitEvent("participant-media-state-updated", participant);
|
|
2776
|
+
}
|
|
2777
|
+
}
|
|
2778
|
+
);
|
|
2779
|
+
socket.on("error", (error) => {
|
|
2780
|
+
emitEvent("error", error);
|
|
2781
|
+
});
|
|
2782
|
+
socket.on("disconnect", () => {
|
|
2783
|
+
emitEvent("disconnected");
|
|
2784
|
+
});
|
|
2785
|
+
socket.on("huddle-invite-received", (data) => {
|
|
2786
|
+
emitEvent("huddle-invite-received", data);
|
|
2787
|
+
});
|
|
2788
|
+
socket.on("private-huddle-started", (data) => {
|
|
2789
|
+
if (state.localParticipant) {
|
|
2790
|
+
state.localParticipant.currentChannel = data.channelId;
|
|
2791
|
+
}
|
|
2792
|
+
emitEvent("private-huddle-started", data);
|
|
2793
|
+
});
|
|
2794
|
+
socket.on("huddle-updated", (data) => {
|
|
2795
|
+
emitEvent("huddle-updated", data);
|
|
2796
|
+
});
|
|
2797
|
+
socket.on("huddle-invite-rejected", (data) => {
|
|
2798
|
+
emitEvent("huddle-invite-rejected", data);
|
|
2799
|
+
});
|
|
2800
|
+
socket.on("huddle-ended", (data) => {
|
|
2801
|
+
if (state.localParticipant) {
|
|
2802
|
+
state.localParticipant.currentChannel = "spatial";
|
|
2803
|
+
}
|
|
2804
|
+
emitEvent("huddle-ended", data);
|
|
2805
|
+
});
|
|
2806
|
+
socket.on("participant-channel-changed", (data) => {
|
|
2807
|
+
const participant = state.room?.participants.get(data.participantId);
|
|
2808
|
+
if (participant) {
|
|
2809
|
+
participant.currentChannel = data.channelId;
|
|
2810
|
+
const isInSpatialChannel = data.channelId === "spatial";
|
|
2811
|
+
if (participant.audioTrack) {
|
|
2812
|
+
spatialAudioManager.setParticipantSpatialization(
|
|
2813
|
+
data.participantId,
|
|
2814
|
+
isInSpatialChannel
|
|
2815
|
+
);
|
|
2816
|
+
}
|
|
2817
|
+
const myChannel = state.localParticipant?.currentChannel || "spatial";
|
|
2818
|
+
const theirChannel = data.channelId || "spatial";
|
|
2819
|
+
if (myChannel !== theirChannel) {
|
|
2820
|
+
participant.screenshareTrack = null;
|
|
2821
|
+
}
|
|
2822
|
+
}
|
|
2823
|
+
if (state.localParticipant?.participantId === data.participantId) {
|
|
2824
|
+
state.localParticipant.currentChannel = data.channelId;
|
|
2825
|
+
}
|
|
2826
|
+
emitEvent("participant-channel-changed", data);
|
|
2827
|
+
});
|
|
2828
|
+
socket.on(
|
|
2829
|
+
"consumer-closed",
|
|
2830
|
+
(data) => {
|
|
2831
|
+
emitEvent("consumer-closed", data);
|
|
2832
|
+
}
|
|
2833
|
+
);
|
|
2834
|
+
socket.on("mute-requested", (data) => {
|
|
2835
|
+
emitEvent("mute-requested", data);
|
|
2836
|
+
});
|
|
2837
|
+
socket.on(
|
|
2838
|
+
"space-live-broadcast-available",
|
|
2839
|
+
(data) => {
|
|
2840
|
+
debugLog("SPACE-LIVE-BROADCAST", "Broadcast available", data);
|
|
2841
|
+
emitEvent("space-live-broadcast-available", data);
|
|
2842
|
+
}
|
|
2843
|
+
);
|
|
2844
|
+
socket.on("space-live-broadcast-stopped", (data) => {
|
|
2845
|
+
debugLog("SPACE-LIVE-BROADCAST", "Broadcast stopped", data);
|
|
2846
|
+
emitEvent("space-live-broadcast-stopped", data);
|
|
2847
|
+
});
|
|
2848
|
+
}
|
|
2849
|
+
function createChatManager(options) {
|
|
2850
|
+
const { serverUrl } = options;
|
|
2851
|
+
let client = null;
|
|
2852
|
+
let activeChannel = null;
|
|
2853
|
+
let onMessageCallback = null;
|
|
2854
|
+
function handleNewMessage(event) {
|
|
2855
|
+
const message = event.message;
|
|
2856
|
+
if (message && onMessageCallback) {
|
|
2857
|
+
onMessageCallback(message);
|
|
2858
|
+
}
|
|
2859
|
+
}
|
|
2860
|
+
async function fetchStreamKey(userId) {
|
|
2861
|
+
const response = await fetch(
|
|
2862
|
+
`${serverUrl}/api/chat/stream-key?userId=${userId}`,
|
|
2863
|
+
{
|
|
2864
|
+
method: "GET",
|
|
2865
|
+
headers: { "Content-Type": "application/json" }
|
|
2866
|
+
}
|
|
2867
|
+
);
|
|
2868
|
+
if (!response.ok) {
|
|
2869
|
+
throw new Error(`Failed to get stream key: HTTP ${response.status}`);
|
|
2870
|
+
}
|
|
2871
|
+
const data = await response.json();
|
|
2872
|
+
if (!data.success || !data.consumerKey) {
|
|
2873
|
+
throw new Error("Stream key not available");
|
|
2874
|
+
}
|
|
2875
|
+
return data.consumerKey;
|
|
2876
|
+
}
|
|
2877
|
+
async function fetchStreamToken(userId) {
|
|
2878
|
+
const response = await fetch(`${serverUrl}/api/chat/stream-token`, {
|
|
2879
|
+
method: "POST",
|
|
2880
|
+
headers: { "Content-Type": "application/json" },
|
|
2881
|
+
body: JSON.stringify({ userId })
|
|
2882
|
+
});
|
|
2883
|
+
if (!response.ok) {
|
|
2884
|
+
throw new Error(`Failed to get stream token: HTTP ${response.status}`);
|
|
2885
|
+
}
|
|
2886
|
+
const data = await response.json();
|
|
2887
|
+
if (!data.success || !data.token) {
|
|
2888
|
+
throw new Error("Stream token not available");
|
|
2889
|
+
}
|
|
2890
|
+
return data.token;
|
|
2891
|
+
}
|
|
2892
|
+
async function init(data) {
|
|
2893
|
+
const { userId, userName, channelId, channelName } = data;
|
|
2894
|
+
const key = await fetchStreamKey(userId);
|
|
2895
|
+
client = StreamChat.getInstance(key);
|
|
2896
|
+
const token = await fetchStreamToken(userId);
|
|
2897
|
+
debugLog("Chat", `Connecting user ${userId}...`);
|
|
2898
|
+
const user = await client.connectUser(
|
|
2899
|
+
{ id: userId, name: userName || userId },
|
|
2900
|
+
token
|
|
2901
|
+
);
|
|
2902
|
+
if (!user) throw new Error("Could not connect chat user");
|
|
2903
|
+
debugLog("Chat", "Connected successfully");
|
|
2904
|
+
const channel = client.channel("livestream", channelId, {
|
|
2905
|
+
name: channelName || channelId
|
|
2906
|
+
});
|
|
2907
|
+
await channel.watch();
|
|
2908
|
+
channel.on("message.new", handleNewMessage);
|
|
2909
|
+
activeChannel = channel;
|
|
2910
|
+
}
|
|
2911
|
+
async function sendMessage(message) {
|
|
2912
|
+
if (!activeChannel) throw new Error("Chat not initialized");
|
|
2913
|
+
return activeChannel.sendMessage(message);
|
|
2914
|
+
}
|
|
2915
|
+
async function sendNotification(data) {
|
|
2916
|
+
return sendMessage({
|
|
2917
|
+
notification: {
|
|
2918
|
+
isNotification: true,
|
|
2919
|
+
text: data.text,
|
|
2920
|
+
participants: data.participants,
|
|
2921
|
+
sender: data.sender
|
|
2922
|
+
}
|
|
2923
|
+
});
|
|
2924
|
+
}
|
|
2925
|
+
async function leave() {
|
|
2926
|
+
if (activeChannel) {
|
|
2927
|
+
activeChannel.off("message.new", handleNewMessage);
|
|
2928
|
+
await activeChannel.stopWatching();
|
|
2929
|
+
}
|
|
2930
|
+
if (client) {
|
|
2931
|
+
await client.disconnectUser();
|
|
2932
|
+
}
|
|
2933
|
+
activeChannel = null;
|
|
2934
|
+
client = null;
|
|
2935
|
+
}
|
|
2936
|
+
function getChannel() {
|
|
2937
|
+
return activeChannel;
|
|
2938
|
+
}
|
|
2939
|
+
return {
|
|
2940
|
+
init,
|
|
2941
|
+
sendMessage,
|
|
2942
|
+
sendNotification,
|
|
2943
|
+
leave,
|
|
2944
|
+
getChannel,
|
|
2945
|
+
// Expose for SDK event wiring
|
|
2946
|
+
set onMessage(cb) {
|
|
2947
|
+
onMessageCallback = cb;
|
|
2948
|
+
}
|
|
2949
|
+
};
|
|
2950
|
+
}
|
|
2951
|
+
|
|
2952
|
+
// src/index.ts
|
|
2953
|
+
var ODYSSEY_SERVER_URL = "https://dev-spatial-comms.2x22.com";
|
|
2954
|
+
function createOdysseySpatialComms(apiKey, userToken, spatialOptions) {
|
|
2955
|
+
console.log(
|
|
2956
|
+
`[OdysseySDK] \u{1F517} Server URL: ${ODYSSEY_SERVER_URL}`
|
|
2957
|
+
);
|
|
2958
|
+
if (!apiKey || typeof apiKey !== "string" || !apiKey.trim()) {
|
|
2959
|
+
throw new Error("[OdysseySDK] A valid apiKey is required.");
|
|
2960
|
+
}
|
|
2961
|
+
const bus = createEventBus();
|
|
2962
|
+
const socket = io(ODYSSEY_SERVER_URL, {
|
|
2963
|
+
transports: ["websocket"],
|
|
2964
|
+
auth: { apiKey, token: userToken }
|
|
2965
|
+
});
|
|
2966
|
+
let sessionStart = null;
|
|
2967
|
+
socket.on("connect", () => {
|
|
2968
|
+
sessionStart = Date.now();
|
|
2969
|
+
debugLog("Connection", `\u2705 Connected to server socketId=${socket.id}`);
|
|
2970
|
+
});
|
|
2971
|
+
socket.on(
|
|
2972
|
+
"connect_error",
|
|
2973
|
+
(err) => debugLog("Connection", `\u274C Connection error: ${err.message}`)
|
|
2974
|
+
);
|
|
2975
|
+
socket.on("disconnect", (reason) => {
|
|
2976
|
+
if (sessionStart !== null) {
|
|
2977
|
+
const secs = Math.floor((Date.now() - sessionStart) / 1e3);
|
|
2978
|
+
const mins = Math.floor(secs / 60);
|
|
2979
|
+
const remainSecs = secs % 60;
|
|
2980
|
+
const duration = mins > 0 ? `${mins}m ${remainSecs}s` : `${remainSecs}s`;
|
|
2981
|
+
console.log(
|
|
2982
|
+
`[OdysseySDK] \u{1F4CA} Session ended \u2014 duration: ${duration} (${secs}s raw)`
|
|
2983
|
+
);
|
|
2984
|
+
sessionStart = null;
|
|
2985
|
+
}
|
|
2986
|
+
debugLog("Connection", `\u{1F50C} Disconnected reason=${reason}`);
|
|
2987
|
+
});
|
|
2988
|
+
const mediasoupManager = createMediasoupManager(socket);
|
|
2989
|
+
const spatialAudioManager = createSpatialAudioManager(spatialOptions);
|
|
2990
|
+
const screenShareLiveManager = createScreenShareLiveManager(socket);
|
|
2991
|
+
const state = { room: null, localParticipant: null };
|
|
2992
|
+
function emitEvent(event, ...args) {
|
|
2993
|
+
bus.emit(event, ...args);
|
|
2994
|
+
}
|
|
2995
|
+
bus.emit("connected");
|
|
2996
|
+
const chatManager = createChatManager({
|
|
2997
|
+
serverUrl: ODYSSEY_SERVER_URL
|
|
2998
|
+
});
|
|
2999
|
+
const {
|
|
3000
|
+
joinRoom: _joinRoom,
|
|
3001
|
+
leaveRoom: _leaveRoom,
|
|
3002
|
+
updatePosition,
|
|
3003
|
+
updateMediaState
|
|
3004
|
+
} = createRoomActions({ socket, bus, mediasoupManager, state, emitEvent });
|
|
3005
|
+
const { produceTrack, recreateProducers, stopScreenShare, stopVideoProducer } = createTrackActions({ socket, state, mediasoupManager, updateMediaState });
|
|
3006
|
+
registerSocketListeners({
|
|
3007
|
+
socket,
|
|
3008
|
+
state,
|
|
3009
|
+
mediasoupManager,
|
|
3010
|
+
spatialAudioManager,
|
|
3011
|
+
emitEvent
|
|
3012
|
+
});
|
|
3013
|
+
async function joinRoom(data) {
|
|
3014
|
+
return _joinRoom(data);
|
|
3015
|
+
}
|
|
3016
|
+
function leaveRoom() {
|
|
3017
|
+
chatManager.leave().catch(() => {
|
|
3018
|
+
});
|
|
3019
|
+
_leaveRoom();
|
|
3020
|
+
}
|
|
3021
|
+
async function initializeMLNoiseSuppression(modelPath, workletUrl) {
|
|
3022
|
+
try {
|
|
3023
|
+
await spatialAudioManager.initializeMLNoiseSuppression(
|
|
3024
|
+
modelPath,
|
|
3025
|
+
workletUrl
|
|
3026
|
+
);
|
|
3027
|
+
} catch (error) {
|
|
3028
|
+
}
|
|
3029
|
+
}
|
|
3030
|
+
async function resumeAudio() {
|
|
3031
|
+
await spatialAudioManager.resumeAudioContext();
|
|
3032
|
+
}
|
|
3033
|
+
function getAudioContextState() {
|
|
3034
|
+
return spatialAudioManager.getAudioContextState();
|
|
3035
|
+
}
|
|
3036
|
+
async function enhanceOutgoingAudioTrack(track) {
|
|
3037
|
+
return spatialAudioManager.enhanceOutgoingAudioTrack(track);
|
|
3038
|
+
}
|
|
3039
|
+
function isMLNoiseSuppressionActive() {
|
|
3040
|
+
return spatialAudioManager.getNoiseSuppressionMode() === "ml";
|
|
3041
|
+
}
|
|
3042
|
+
function getNoiseSuppressionMode() {
|
|
3043
|
+
return spatialAudioManager.getNoiseSuppressionMode();
|
|
3044
|
+
}
|
|
3045
|
+
function setMasterMuted(muted) {
|
|
3046
|
+
spatialAudioManager.setMasterMuted(muted);
|
|
3047
|
+
}
|
|
3048
|
+
function getMasterMuted() {
|
|
3049
|
+
return spatialAudioManager.getMasterMuted();
|
|
3050
|
+
}
|
|
3051
|
+
async function startSpaceLiveBroadcast() {
|
|
3052
|
+
if (!state.localParticipant || !state.room) {
|
|
3053
|
+
throw new Error("Must be in a room to start space live broadcast");
|
|
3054
|
+
}
|
|
3055
|
+
return screenShareLiveManager.startBroadcast(
|
|
3056
|
+
state.room,
|
|
3057
|
+
state.localParticipant,
|
|
3058
|
+
ODYSSEY_SERVER_URL
|
|
3059
|
+
);
|
|
3060
|
+
}
|
|
3061
|
+
async function stopSpaceLiveBroadcast() {
|
|
3062
|
+
return screenShareLiveManager.stopBroadcast(
|
|
3063
|
+
state.room,
|
|
3064
|
+
state.localParticipant?.participantId
|
|
3065
|
+
);
|
|
3066
|
+
}
|
|
3067
|
+
function getSpaceLiveBroadcastStatus(callback) {
|
|
3068
|
+
screenShareLiveManager.getStatus(state.room?.id || null, callback);
|
|
3069
|
+
}
|
|
3070
|
+
function setListenerPosition(position, orientation) {
|
|
3071
|
+
spatialAudioManager.setListenerPosition(position, orientation);
|
|
3072
|
+
}
|
|
3073
|
+
function setListenerFromLSD(listenerPos, cameraPos, lookAtPos, rot) {
|
|
3074
|
+
spatialAudioManager.setListenerFromLSD(
|
|
3075
|
+
listenerPos,
|
|
3076
|
+
cameraPos,
|
|
3077
|
+
lookAtPos,
|
|
3078
|
+
rot
|
|
3079
|
+
);
|
|
3080
|
+
}
|
|
3081
|
+
async function sendHuddleInvite(toParticipantId) {
|
|
3082
|
+
if (!state.localParticipant || !state.room)
|
|
3083
|
+
return { success: false, error: "Not in a room" };
|
|
3084
|
+
return new Promise((resolve) => {
|
|
3085
|
+
socket.emit(
|
|
3086
|
+
"send-huddle-invite",
|
|
3087
|
+
{
|
|
3088
|
+
fromParticipantId: state.localParticipant.participantId,
|
|
3089
|
+
toParticipantId,
|
|
3090
|
+
roomId: state.room.id
|
|
3091
|
+
},
|
|
3092
|
+
(response) => resolve(response)
|
|
3093
|
+
);
|
|
3094
|
+
});
|
|
3095
|
+
}
|
|
3096
|
+
async function acceptHuddleInvite(inviteId) {
|
|
3097
|
+
if (!state.localParticipant)
|
|
3098
|
+
return { success: false, error: "Not in a room" };
|
|
3099
|
+
return new Promise((resolve) => {
|
|
3100
|
+
socket.emit(
|
|
3101
|
+
"accept-huddle-invite",
|
|
3102
|
+
{ inviteId, participantId: state.localParticipant.participantId },
|
|
3103
|
+
(response) => resolve(response)
|
|
3104
|
+
);
|
|
3105
|
+
});
|
|
3106
|
+
}
|
|
3107
|
+
async function rejectHuddleInvite(inviteId) {
|
|
3108
|
+
if (!state.localParticipant)
|
|
3109
|
+
return { success: false, error: "Not in a room" };
|
|
3110
|
+
return new Promise((resolve) => {
|
|
3111
|
+
socket.emit(
|
|
3112
|
+
"reject-huddle-invite",
|
|
3113
|
+
{ inviteId, participantId: state.localParticipant.participantId },
|
|
3114
|
+
(response) => resolve(response)
|
|
3115
|
+
);
|
|
3116
|
+
});
|
|
3117
|
+
}
|
|
3118
|
+
async function joinHuddle() {
|
|
3119
|
+
if (!state.localParticipant || !state.room)
|
|
3120
|
+
return { success: false, error: "Not in a room" };
|
|
3121
|
+
return new Promise((resolve) => {
|
|
3122
|
+
socket.emit(
|
|
3123
|
+
"join-huddle",
|
|
3124
|
+
{
|
|
3125
|
+
participantId: state.localParticipant.participantId,
|
|
3126
|
+
roomId: state.room.id
|
|
3127
|
+
},
|
|
3128
|
+
(response) => {
|
|
3129
|
+
if (response.success && state.localParticipant) {
|
|
3130
|
+
state.localParticipant.currentChannel = response.channelId;
|
|
3131
|
+
}
|
|
3132
|
+
resolve(response);
|
|
3133
|
+
}
|
|
3134
|
+
);
|
|
3135
|
+
});
|
|
3136
|
+
}
|
|
3137
|
+
async function leaveHuddle() {
|
|
3138
|
+
if (!state.localParticipant || !state.room)
|
|
3139
|
+
return { success: false, error: "Not in a room" };
|
|
3140
|
+
return new Promise((resolve) => {
|
|
3141
|
+
socket.emit(
|
|
3142
|
+
"leave-huddle",
|
|
3143
|
+
{
|
|
3144
|
+
participantId: state.localParticipant.participantId,
|
|
3145
|
+
roomId: state.room.id
|
|
3146
|
+
},
|
|
3147
|
+
(response) => {
|
|
3148
|
+
if (response.success && state.localParticipant) {
|
|
3149
|
+
state.localParticipant.currentChannel = response.channelId;
|
|
3150
|
+
}
|
|
3151
|
+
resolve(response);
|
|
3152
|
+
}
|
|
3153
|
+
);
|
|
3154
|
+
});
|
|
3155
|
+
}
|
|
3156
|
+
async function getParticipantChannel(participantId) {
|
|
3157
|
+
return new Promise((resolve) => {
|
|
3158
|
+
socket.emit(
|
|
3159
|
+
"get-participant-channel",
|
|
3160
|
+
{ participantId },
|
|
3161
|
+
(response) => {
|
|
3162
|
+
resolve(response.channelId || "spatial");
|
|
3163
|
+
}
|
|
3164
|
+
);
|
|
3165
|
+
});
|
|
3166
|
+
}
|
|
3167
|
+
async function muteParticipant(participantId) {
|
|
3168
|
+
if (!state.localParticipant || !state.room)
|
|
3169
|
+
return { success: false, error: "Not in a room" };
|
|
3170
|
+
return new Promise((resolve) => {
|
|
3171
|
+
socket.emit(
|
|
3172
|
+
"mute-participant",
|
|
3173
|
+
{
|
|
3174
|
+
targetParticipantId: participantId,
|
|
3175
|
+
roomId: state.room.id,
|
|
3176
|
+
requesterId: state.localParticipant.participantId
|
|
3177
|
+
},
|
|
3178
|
+
(response) => resolve(response)
|
|
3179
|
+
);
|
|
3180
|
+
});
|
|
3181
|
+
}
|
|
3182
|
+
function getCurrentChannel() {
|
|
3183
|
+
return state.localParticipant?.currentChannel || "spatial";
|
|
3184
|
+
}
|
|
3185
|
+
return {
|
|
3186
|
+
get room() {
|
|
3187
|
+
return state.room;
|
|
3188
|
+
},
|
|
3189
|
+
on: (event, listener) => {
|
|
3190
|
+
bus.on(event, listener);
|
|
3191
|
+
},
|
|
3192
|
+
off: (event, listener) => {
|
|
3193
|
+
bus.off(event, listener);
|
|
3194
|
+
},
|
|
3195
|
+
initializeMLNoiseSuppression,
|
|
3196
|
+
joinRoom,
|
|
3197
|
+
leaveRoom,
|
|
3198
|
+
resumeAudio,
|
|
3199
|
+
getAudioContextState,
|
|
3200
|
+
enhanceOutgoingAudioTrack,
|
|
3201
|
+
isMLNoiseSuppressionActive,
|
|
3202
|
+
getNoiseSuppressionMode,
|
|
3203
|
+
setMasterMuted,
|
|
3204
|
+
getMasterMuted,
|
|
3205
|
+
produceTrack,
|
|
3206
|
+
recreateProducers,
|
|
3207
|
+
updatePosition,
|
|
3208
|
+
updateMediaState,
|
|
3209
|
+
stopScreenShare,
|
|
3210
|
+
stopVideoProducer,
|
|
3211
|
+
startSpaceLiveBroadcast,
|
|
3212
|
+
stopSpaceLiveBroadcast,
|
|
3213
|
+
getSpaceLiveBroadcastStatus,
|
|
3214
|
+
get isSpaceLiveBroadcasting() {
|
|
3215
|
+
return screenShareLiveManager.isActive;
|
|
3216
|
+
},
|
|
3217
|
+
setListenerPosition,
|
|
3218
|
+
setListenerFromLSD,
|
|
3219
|
+
sendHuddleInvite,
|
|
3220
|
+
acceptHuddleInvite,
|
|
3221
|
+
rejectHuddleInvite,
|
|
3222
|
+
joinHuddle,
|
|
3223
|
+
leaveHuddle,
|
|
3224
|
+
getParticipantChannel,
|
|
3225
|
+
muteParticipant,
|
|
3226
|
+
getCurrentChannel,
|
|
3227
|
+
chat: {
|
|
3228
|
+
init: (data) => chatManager.init(data),
|
|
3229
|
+
sendMessage: (message) => chatManager.sendMessage(message),
|
|
3230
|
+
sendNotification: (data) => chatManager.sendNotification(data),
|
|
3231
|
+
leave: () => chatManager.leave(),
|
|
3232
|
+
onMessage: (callback) => {
|
|
3233
|
+
chatManager.onMessage = callback;
|
|
3234
|
+
}
|
|
3235
|
+
}
|
|
3236
|
+
};
|
|
3237
|
+
}
|
|
3238
|
+
var SpatialCommsSDK = {
|
|
3239
|
+
/**
|
|
3240
|
+
* Create a new SDK client instance.
|
|
3241
|
+
* @param apiKey SDK API key issued by Odyssey (from payment service)
|
|
3242
|
+
* @param userToken JWT access token for the authenticated user (from SSO)
|
|
3243
|
+
* @param spatialOptions Optional spatial audio configuration
|
|
3244
|
+
*/
|
|
3245
|
+
create(apiKey, userToken, spatialOptions) {
|
|
3246
|
+
return createOdysseySpatialComms(apiKey, userToken, spatialOptions);
|
|
3247
|
+
}
|
|
3248
|
+
};
|
|
3249
|
+
|
|
3250
|
+
export { SpatialCommsSDK, createOdysseySpatialComms };
|
|
3251
|
+
//# sourceMappingURL=index.mjs.map
|
|
3252
|
+
//# sourceMappingURL=index.mjs.map
|