@livedigital/client 3.28.1 → 3.29.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/dist/config/ConfigService.d.ts +6 -1
  2. package/dist/constants/events.d.ts +1 -0
  3. package/dist/constants/events.ts +1 -0
  4. package/dist/engine/IntegrationsService.d.ts +28 -0
  5. package/dist/engine/index.d.ts +7 -3
  6. package/dist/engine/media/index.d.ts +10 -5
  7. package/dist/engine/media/streamEffects/ProcessorsCache.d.ts +11 -0
  8. package/dist/engine/media/streamEffects/audio/AudioTrackProcessor.d.ts +6 -0
  9. package/dist/engine/media/streamEffects/audio/asdk/ASDKTrackProcessor.d.ts +23 -0
  10. package/dist/engine/media/streamEffects/audio/noiseSuppression/{TrackProcessor.d.ts → RNNoiseTrackProcessor.d.ts} +3 -1
  11. package/dist/engine/media/tracks/DefaultAudioTrack.d.ts +11 -2
  12. package/dist/helpers/loader.d.ts +3 -0
  13. package/dist/index.es.js +4 -4
  14. package/dist/index.js +19 -19
  15. package/dist/inversify.factories.d.ts +2 -1
  16. package/dist/inversify.tokens.d.ts +3 -0
  17. package/dist/types/client.d.ts +6 -0
  18. package/dist/types/container.d.ts +7 -0
  19. package/dist/types/engine.d.ts +1 -0
  20. package/dist/types/media.d.ts +5 -4
  21. package/package.json +2 -1
  22. package/src/config/ConfigService.ts +13 -2
  23. package/src/constants/events.ts +1 -0
  24. package/src/engine/IntegrationsService.ts +155 -0
  25. package/src/engine/index.ts +22 -56
  26. package/src/engine/media/index.ts +79 -27
  27. package/src/engine/media/streamEffects/ProcessorsCache.ts +41 -0
  28. package/src/engine/media/streamEffects/audio/AudioTrackProcessor.ts +6 -0
  29. package/src/engine/media/streamEffects/audio/asdk/ASDKTrackProcessor.ts +155 -0
  30. package/src/engine/media/streamEffects/audio/asdk/atsrb.d.ts +37 -0
  31. package/src/engine/media/streamEffects/audio/asdk/errorBus.d.ts +42 -0
  32. package/src/engine/media/streamEffects/audio/noiseSuppression/{TrackProcessor.ts → RNNoiseTrackProcessor.ts} +32 -6
  33. package/src/engine/media/streamEffects/video/esdk/TrackProcessor.ts +5 -2
  34. package/src/engine/media/tracks/DefaultAudioTrack.ts +39 -14
  35. package/src/engine/media/tracks/DefaultBaseTrack.ts +9 -0
  36. package/src/helpers/loader.ts +49 -0
  37. package/src/index.ts +4 -1
  38. package/src/inversify.config.ts +29 -2
  39. package/src/inversify.factories.ts +47 -2
  40. package/src/inversify.tokens.ts +3 -0
  41. package/src/types/client.ts +6 -0
  42. package/src/types/container.ts +8 -0
  43. package/src/types/engine.ts +1 -0
  44. package/src/types/media.ts +5 -4
  45. package/dist/helpers/config.d.ts +0 -10
  46. package/src/helpers/config.ts +0 -31
@@ -1,7 +1,8 @@
1
1
  import { interfaces } from 'inversify';
2
- import { AudioTrackFactory, PeerConsumerFactory, PeerFactory, PeerTrackFactory, VideoTrackFactory } from './types/container';
2
+ import { AudioTrackFactory, AudioTrackProcessorFactory, PeerConsumerFactory, PeerFactory, PeerTrackFactory, VideoTrackFactory } from './types/container';
3
3
  export declare function peerTrackFactory(context: interfaces.Context): PeerTrackFactory;
4
4
  export declare function peerConsumerFactory(context: interfaces.Context): PeerConsumerFactory;
5
5
  export declare function peerFactory(context: interfaces.Context): PeerFactory;
6
6
  export declare function audioTrackFactory(context: interfaces.Context): AudioTrackFactory;
7
7
  export declare function videoTrackFactory(context: interfaces.Context): VideoTrackFactory;
8
+ export declare function audioTrackProcessorFactory(context: interfaces.Context): AudioTrackProcessorFactory;
@@ -26,6 +26,9 @@ export declare enum TOKEN {
26
26
  PeerConsumerFactory = "PeerConsumerFactory",
27
27
  AudioTrackFactory = "AudioTrackFactory",
28
28
  VideoTrackFactory = "VideoTrackFactory",
29
+ AudioTrackProcessorFactory = "AudioTrackProcessorFactory",
30
+ IntegrationsService = "IntegrationsService",
31
+ ProcessorsCache = "ProcessorsCache",
29
32
  StatsHandler = "StatsHandler",
30
33
  DataChannelsManager = "DataChannelsManager"
31
34
  }
@@ -4,6 +4,12 @@ import { LogLevel, LogMessageHandler } from './common';
4
4
  import { IssuesHandler, NetworkScoresUpdatedHandler } from './engine';
5
5
  import { InitEffectsSDKParams } from './media';
6
6
  export declare type ClientParams = {
7
+ denoiser?: 'rnnoise' | 'asdk';
8
+ asdk?: {
9
+ customerId: string;
10
+ version: '2.3.5';
11
+ localDir?: string;
12
+ };
7
13
  clientEventEmitter?: EnhancedEventEmitter;
8
14
  internalEventEmitter?: EnhancedEventEmitter;
9
15
  network?: {
@@ -1,4 +1,5 @@
1
1
  import Logger from '../engine/Logger';
2
+ import { AudioTrackProcessor } from '../engine/media/streamEffects/audio/AudioTrackProcessor';
2
3
  import DefaultAudioTrack from '../engine/media/tracks/DefaultAudioTrack';
3
4
  import DefaultVideoTrack from '../engine/media/tracks/DefaultVideoTrack';
4
5
  import PeerTrack from '../engine/media/tracks/PeerTrack';
@@ -6,6 +7,7 @@ import Peer from '../engine/Peer';
6
7
  import PeerConsumer, { PeerConsumerConstructorParams } from '../engine/PeerConsumer';
7
8
  import { CreatePeerParams } from '../engine/Peers';
8
9
  import { CreatePeerTrackParams } from '../engine/RemotePeerTracks';
10
+ import { TrackLabel } from './common';
9
11
  import { AudioTrackParams, BaseTrackParams } from './media';
10
12
  export declare type PeerTrackFactoryParams = CreatePeerTrackParams & {
11
13
  onClose: () => void;
@@ -17,3 +19,8 @@ export declare type PeerConsumerFactory = (params: CreatePeerConsumerParams) =>
17
19
  export declare type LoggerFactory = (namespace: string) => Logger;
18
20
  export declare type AudioTrackFactory = (params: AudioTrackParams) => DefaultAudioTrack;
19
21
  export declare type VideoTrackFactory = (params: BaseTrackParams) => DefaultVideoTrack;
22
+ export declare type CreateAudioTrackProcessorParams = {
23
+ noiseSuppressor?: WebAssembly.WebAssemblyInstantiatedSource;
24
+ trackLabel: TrackLabel;
25
+ };
26
+ export declare type AudioTrackProcessorFactory = (params: CreateAudioTrackProcessorParams) => AudioTrackProcessor;
@@ -70,6 +70,7 @@ export declare type ClientObserverEvents = {
70
70
  [CLIENT_EVENTS.forcedDisconnect]: [];
71
71
  [CLIENT_EVENTS.rejectUnauthorized]: [];
72
72
  [CLIENT_EVENTS.channelStateInconsistent]: [ChannelStateInconsistentPayload];
73
+ [CLIENT_EVENTS.denoiserInitializing]: [boolean];
73
74
  };
74
75
  export declare type InternalObserverEvents = {
75
76
  [INTERNAL_CLIENT_EVENTS.trackProduced]: [Track | BaseTrack];
@@ -4,14 +4,15 @@ import { AppData, RtpEncodingParameters } from 'mediasoup-client/lib/types';
4
4
  import { ParsedOutboundVideoStreamStats, ParsedOutboundAudioStreamStats } from 'webrtc-issue-detector';
5
5
  import Logger from '../engine/Logger';
6
6
  import Media from '../engine/media';
7
- import { MyPeer } from '../engine/MyPeer';
8
- import Network from '../engine/network';
7
+ import { AudioTrackProcessor } from '../engine/media/streamEffects/audio/AudioTrackProcessor';
9
8
  import type { Effects } from '../engine/media/streamEffects/video/esdk/TrackProcessor';
10
9
  import MediaStreamTrackManager from '../engine/media/tracks/MediaStreamTrackManager';
10
+ import { MyPeer } from '../engine/MyPeer';
11
+ import Network from '../engine/network';
12
+ import SocketIO from '../engine/network/Socket';
11
13
  import EnhancedEventEmitter from '../EnhancedEventEmitter';
12
14
  import { BaseTrackInfo, EncoderConfig, TrackLabel, TrackOutboundStats, TrackProduceParams, TrackTransformParams } from './common';
13
15
  import { InternalObserverEvents } from './engine';
14
- import SocketIO from '../engine/network/Socket';
15
16
  import StatsHandler from '../engine/WebRTCStats/StatsHandler';
16
17
  export interface BaseTrackDependenciesParams {
17
18
  internalEventEmitter: EnhancedEventEmitter<InternalObserverEvents>;
@@ -66,6 +67,7 @@ export interface TrackWithEffects extends BaseTrack {
66
67
  isEffectsProcessing: boolean;
67
68
  }
68
69
  export interface AudioTrack extends BaseTrack, TrackWithEncodings, TrackWithEffects {
70
+ prepareTrackProcessor(trackProcessor?: AudioTrackProcessor): void;
69
71
  enableNoiseSuppression(): Promise<void>;
70
72
  disableNoiseSuppression(): Promise<void>;
71
73
  pause(): Promise<void>;
@@ -89,7 +91,6 @@ export interface VideoTrack extends BaseTrack, TrackWithEncodings, TrackWithEffe
89
91
  }
90
92
  export declare type Track = AudioTrack | VideoTrack;
91
93
  export interface AudioTrackParams extends BaseTrackParams {
92
- noiseSuppressor?: WebAssembly.WebAssemblyInstantiatedSource;
93
94
  }
94
95
  export interface MediaTracksFactory {
95
96
  createAudioTrack: (params: Omit<AudioTrackParams, 'logger'>) => BaseTrack;
package/package.json CHANGED
@@ -2,7 +2,7 @@
2
2
  "name": "@livedigital/client",
3
3
  "author": "vlprojects",
4
4
  "license": "MIT",
5
- "version": "3.28.1",
5
+ "version": "3.29.0",
6
6
  "private": false,
7
7
  "bugs": {
8
8
  "url": "https://github.com/vlprojects/livedigital-sdk/issues"
@@ -50,6 +50,7 @@
50
50
  "fflate": "^0.8.2",
51
51
  "hash-sum": "^2.0.0",
52
52
  "inversify": "^6.0.2",
53
+ "load-script": "^2.0.0",
53
54
  "mediasoup-client": "3.7.18",
54
55
  "message-batcher": "^0.0.2",
55
56
  "qs": "^6.11.0",
@@ -4,11 +4,15 @@ import { LoadBalancerApiClientParams } from '../engine/network/LoadBalancerClien
4
4
  import * as client from '../types/client';
5
5
  import { InitEffectsSDKParams } from '../types/media';
6
6
 
7
- type ParamsToOmit = 'clientEventEmitter' | 'internalEventEmitter' | 'network' | 'effectsSDKParams';
8
-
7
+ type ParamsToOmit = 'clientEventEmitter' | 'internalEventEmitter' | 'network' | 'effectsSDKParams' | 'asdk';
9
8
  export type ProcessedParams = Required<Omit<client.ClientParams, ParamsToOmit>> & {
10
9
  loadbalancer: LoadBalancerApiClientParams,
11
10
  effectsSDKParams?: InitEffectsSDKParams,
11
+ asdk: {
12
+ customerId: string
13
+ version: '2.3.5',
14
+ localDir?: string,
15
+ },
12
16
  };
13
17
 
14
18
  const loadbalancerDefaults: LoadBalancerApiClientParams = {
@@ -16,6 +20,11 @@ const loadbalancerDefaults: LoadBalancerApiClientParams = {
16
20
  timeout: 5000,
17
21
  };
18
22
 
23
+ const asdkDefaults: client.ClientParams['asdk'] = {
24
+ customerId: '',
25
+ version: '2.3.5',
26
+ };
27
+
19
28
  const noop = () => { };
20
29
 
21
30
  @injectable()
@@ -24,6 +33,8 @@ export default class ConfigService {
24
33
 
25
34
  constructor(params: client.ClientParams) {
26
35
  this.#params = {
36
+ denoiser: params.denoiser || 'rnnoise',
37
+ asdk: Object.assign(asdkDefaults, params.asdk || {}),
27
38
  loadbalancer: Object.assign(loadbalancerDefaults, params.network?.loadbalancer || {}),
28
39
  logLevel: params.logLevel ?? LogLevels.Error,
29
40
  sendAnalytics: params.sendAnalytics ?? false,
@@ -38,6 +38,7 @@ export const CLIENT_EVENTS = {
38
38
  connectionRestored: 'connection-restored',
39
39
  forcedDisconnect: 'forced-disconnect',
40
40
  rejectUnauthorized: 'reject-unauthorized',
41
+ denoiserInitializing: 'denoiser-initializing',
41
42
  } as const;
42
43
 
43
44
  export const TRACK_EVENTS = {
@@ -0,0 +1,155 @@
1
+ import { inject, injectable } from 'inversify';
2
+ import ConfigService from '../config/ConfigService';
3
+ import { loadScriptWithFallback } from '../helpers/loader';
4
+ import { TOKEN } from '../inversify.tokens';
5
+ import { TrackLabel } from '../types/common';
6
+ import { AudioTrackProcessorFactory, LoggerFactory } from '../types/container';
7
+ import Logger from './Logger';
8
+ import { ASDKTrackProcessor } from './media/streamEffects/audio/asdk/ASDKTrackProcessor';
9
+ import WasmModuleCompiler from './media/streamEffects/audio/noiseSuppression/WasmModuleCompiler';
10
+ import { ProcessorsCache } from './media/streamEffects/ProcessorsCache';
11
+
12
+ export type Integrations = 'asdk' | 'rnnoise'; // later 'esdk' should be added
13
+ export type IntegrationContext = {
14
+ integrated: boolean,
15
+ initPromise: Promise<void>,
16
+ };
17
+
18
+ export interface Integration {
19
+ type: Integrations,
20
+ enabled: boolean; // flag to show that integration is requested/enabled through ClientParams
21
+ isInitialized: boolean;
22
+ waitForInitialization: Promise<void>;
23
+ }
24
+
25
+ @injectable()
26
+ export class IntegrationsService {
27
+ readonly #integrationsStorage: Map<Integrations, IntegrationContext> = new Map();
28
+
29
+ readonly #logger: Logger;
30
+
31
+ constructor(
32
+ @inject(TOKEN.Config) private readonly config: ConfigService,
33
+ @inject(TOKEN.ProcessorsCache) private readonly processorsCache: ProcessorsCache,
34
+ @inject(TOKEN.AudioTrackProcessorFactory) private readonly audioTrackProcessorFactory: AudioTrackProcessorFactory,
35
+ @inject(TOKEN.LoggerFactory) loggerFactory: LoggerFactory,
36
+ ) {
37
+ this.#logger = loggerFactory('IntegrationsService');
38
+ }
39
+
40
+ init() { // must not be async
41
+ const denoiser = this.config.get('denoiser');
42
+ if (denoiser === 'asdk') {
43
+ this.#logger.info('Start integration of ASDK', {
44
+ denoiser,
45
+ });
46
+
47
+ const context: IntegrationContext = {
48
+ integrated: false,
49
+ initPromise: this.integrateASDK(),
50
+ };
51
+
52
+ this.#integrationsStorage.set('asdk', context);
53
+ } else if (denoiser === 'rnnoise') {
54
+ this.#logger.info('Start integration of RNNoise', {
55
+ denoiser,
56
+ });
57
+
58
+ const context: IntegrationContext = {
59
+ integrated: false,
60
+ initPromise: this.integrateRNNoise(),
61
+ };
62
+
63
+ this.#integrationsStorage.set('rnnoise', context);
64
+ }
65
+ }
66
+
67
+ private async integrateASDK(): Promise<void> {
68
+ const asdkParams = this.config.get('asdk');
69
+
70
+ if (!window.atsvb) {
71
+ // https://effectssdk.ai/sdk/audio/dev/2.3.5/atsvb-web.js
72
+ const remoteUrl = `https://effectssdk.ai/sdk/audio/dev/${asdkParams.version}/atsvb-web.js`;
73
+ const localUrl = asdkParams.localDir ?? `${asdkParams.localDir}/${asdkParams.version}/atsvb-web.js`;
74
+
75
+ await loadScriptWithFallback(remoteUrl, localUrl);
76
+ }
77
+
78
+ if (!window.atsvb) {
79
+ this.#logger.info('Audio Effects SDK is not loaded, skipping initialization', { case: 'initAudioSDK' });
80
+
81
+ throw new Error('windows.atsvb is still missing');
82
+ }
83
+
84
+ const processor = this.audioTrackProcessorFactory({
85
+ trackLabel: TrackLabel.Microphone,
86
+ }) as ASDKTrackProcessor;
87
+
88
+ await processor.init();
89
+
90
+ this.processorsCache.add('asdk', TrackLabel.Microphone, processor);
91
+
92
+ const context = this.#integrationsStorage.get('asdk');
93
+
94
+ if (!context) {
95
+ throw Error('IntegrationContext is missing for ASDK');
96
+ }
97
+
98
+ context.integrated = true;
99
+ }
100
+
101
+ get audio(): Integration {
102
+ const denoiser = this.config.get('denoiser');
103
+
104
+ if (denoiser === 'asdk') {
105
+ return this.getASDKState();
106
+ }
107
+
108
+ return this.getRNNoiseState();
109
+ }
110
+
111
+ private getASDKState(): Integration {
112
+ const asdk = this.#integrationsStorage.get('asdk');
113
+
114
+ return {
115
+ type: 'asdk',
116
+ enabled: Boolean(asdk),
117
+ isInitialized: asdk?.integrated ?? false,
118
+ waitForInitialization: asdk?.initPromise ?? Promise.resolve(),
119
+ };
120
+ }
121
+
122
+ private getRNNoiseState(): Integration {
123
+ const rnnoise = this.#integrationsStorage.get('rnnoise');
124
+
125
+ return {
126
+ type: 'rnnoise',
127
+ enabled: Boolean(rnnoise),
128
+ isInitialized: rnnoise?.integrated ?? false,
129
+ waitForInitialization: rnnoise?.initPromise ?? Promise.resolve(),
130
+ };
131
+ }
132
+
133
+ private async integrateRNNoise(): Promise<void> {
134
+ const noiseSuppressor = await WasmModuleCompiler.compileModule(this.config.get('staticFilesPath'));
135
+
136
+ const processor = this.audioTrackProcessorFactory({
137
+ trackLabel: TrackLabel.Microphone,
138
+ noiseSuppressor,
139
+ });
140
+
141
+ this.processorsCache.add('rnnoise', TrackLabel.Microphone, processor);
142
+
143
+ const context = this.#integrationsStorage.get('rnnoise');
144
+
145
+ if (!context) {
146
+ throw Error('IntegrationContext is missing for ASDK');
147
+ }
148
+
149
+ context.integrated = true;
150
+ }
151
+
152
+ reset() {
153
+ this.#integrationsStorage.clear();
154
+ }
155
+ }
@@ -4,6 +4,7 @@ import { MediaKind, RtpCapabilities } from 'mediasoup-client/lib/RtpParameters';
4
4
  import { serializeError } from 'serialize-error';
5
5
  import WebRTCIssueDetector from 'webrtc-issue-detector';
6
6
  import clientMetaProvider from '../ClientMetaProvider';
7
+ import ConfigService from '../config/ConfigService';
7
8
  import { MAX_JOIN_ATTEMPTS, SOCKET_ERROR_CODE_UNAUTHORIZED } from '../constants/common';
8
9
  import {
9
10
  CHANNEL_EVENTS, CLIENT_EVENTS, INTERNAL_CLIENT_EVENTS, NETWORK_OBSERVER_EVENTS,
@@ -46,8 +47,10 @@ import ChannelStateSynchronizer from './ChannelStateSynchronizer/ChannelStateSyn
46
47
  import EventsQueue from './EventsQueue';
47
48
  import ChannelEventHandler from './handlers/ChannelEventHandler';
48
49
  import MediaSoupEventHandler from './handlers/MediaSoupEventHandler';
50
+ import { IntegrationsService } from './IntegrationsService';
49
51
  import Logger from './Logger';
50
52
  import Media from './media';
53
+ import { ProcessorsCache } from './media/streamEffects/ProcessorsCache';
51
54
  import VideoTrack from './media/tracks/DefaultVideoTrack';
52
55
  import { MyPeer } from './MyPeer';
53
56
  import Network from './network';
@@ -74,12 +77,17 @@ class Engine {
74
77
 
75
78
  #isConnectionLost = false;
76
79
 
80
+ #denoiserInitializing: boolean = false;
81
+
77
82
  #transportStatsProvider?: TransportsStatsProvider;
78
83
 
79
84
  constructor(
85
+ @inject(TOKEN.Config) private readonly config: ConfigService,
80
86
  @inject(TOKEN.CallState) private readonly state: CallState,
81
87
  @inject(TOKEN.MyPeer) private readonly myPeer: MyPeer,
82
88
  @inject(TOKEN.LoggerFactory) loggerFactory: (namespace: string) => Logger,
89
+ @inject(TOKEN.IntegrationsService) private readonly integrationsService: IntegrationsService,
90
+ @inject(TOKEN.ProcessorsCache) private readonly processorsCache: ProcessorsCache,
83
91
  @inject(TOKEN.MediaSoupEventHandler) private readonly mediaSoupEventsHandler: MediaSoupEventHandler,
84
92
  @inject(TOKEN.ChannelEventHandler) private readonly channelEventsHandler: ChannelEventHandler,
85
93
  @inject(TOKEN.ChannelStateSynchronizer) readonly channelStateSynchronizer: ChannelStateSynchronizer,
@@ -112,7 +120,7 @@ class Engine {
112
120
  try {
113
121
  const { rtpCapabilities } = await this.socket
114
122
  .request('router.getRtpCapabilities') as { rtpCapabilities: RtpCapabilities };
115
- await this.media.createNoiseSuppressor();
123
+ this.integrationsService.init();
116
124
  await this.media.loadDevice(rtpCapabilities);
117
125
 
118
126
  this.#logger.info('Supported capabilities', {
@@ -184,6 +192,9 @@ class Engine {
184
192
  await this.media.clearTracks();
185
193
  }
186
194
 
195
+ await this.processorsCache.reset();
196
+ this.integrationsService.reset();
197
+
187
198
  this.eventsQueue.removeListener('message');
188
199
  this.dataChannelsManager.closeAllDataChannels();
189
200
 
@@ -208,46 +219,6 @@ class Engine {
208
219
  }
209
220
  }
210
221
 
211
- private async releaseForRejoin(): Promise<void> {
212
- if (this.isReleasing) {
213
- return;
214
- }
215
-
216
- this.isReleasing = true;
217
-
218
- try {
219
- this.state.isJoined = false;
220
- this.state.reset();
221
- this.myPeer.delete();
222
- this.webRtcIssueDetector?.stopWatchingNewPeerConnections();
223
-
224
- this.peers.reset();
225
-
226
- await this.network.closeSendTransport();
227
- await this.network.closeReceiveTransport();
228
-
229
- this.eventsQueue.removeListener('message');
230
- this.dataChannelsManager.closeAllDataChannels();
231
-
232
- this.channelEventsHandler.reset();
233
- this.mediaSoupEventsHandler.reset();
234
-
235
- this.removeClientEventEmitterListeners();
236
-
237
- await this.network.reset();
238
- this.socket.disconnect();
239
- clientMetaProvider.clear();
240
- this.#logger.debug('release()', {
241
- socketId: this.myPeerId,
242
- });
243
- } catch (error: unknown) {
244
- this.#logger.error('release()', { error });
245
- throw new Error('Error release engine');
246
- } finally {
247
- this.isReleasing = false;
248
- }
249
- }
250
-
251
222
  private removeClientEventEmitterListeners(): void {
252
223
  Array.from(Object.values(INTERNAL_CLIENT_EVENTS)).forEach((x) => this.internalEventEmitter.removeAllListeners(x));
253
224
  }
@@ -283,7 +254,7 @@ class Engine {
283
254
  actionName: 'joinWithRetry',
284
255
  logger: this.#loggerFactory('RetryJoinNode'),
285
256
  onError: async () => {
286
- await this.releaseForRejoin();
257
+ await this.release(true);
287
258
  },
288
259
  });
289
260
  } catch (error) {
@@ -353,24 +324,23 @@ class Engine {
353
324
  audio: options?.audio,
354
325
  });
355
326
 
356
- track.setLabel(options?.isPreview ? TrackLabel.MicrophonePreview : TrackLabel.Microphone);
327
+ const trackLabel = options?.isPreview ? TrackLabel.MicrophonePreview : TrackLabel.Microphone;
328
+
329
+ track.setLabel(trackLabel);
357
330
  track.setStopTrackOnPause(Boolean(options?.stopTrackOnPause));
358
331
  if (options?.encoderConfig) {
359
332
  track.setEncoderConfig(options.encoderConfig);
360
333
  }
361
334
 
362
- if (track.kind === 'audio' && options?.noiseSuppression) {
363
- try {
364
- await (track as AudioTrack).enableNoiseSuppression();
365
- } catch (error) {
366
- this.#logger.warn('Failed to start noise suppression, featured original track', {
367
- label: track.getLabel(),
368
- });
369
- }
335
+ if (track.kind === 'audio' && track.getLabel() === TrackLabel.Microphone && !this.#denoiserInitializing) {
336
+ this.media.initDenoiser(track as AudioTrack, options); // must be non-blocking, don't await
370
337
  }
371
338
 
372
339
  this.media.setTrack(track);
373
- this.#logger.debug('createMicrophoneAudioTrack()', { options });
340
+ this.#logger.debug('createMicrophoneAudioTrack()', {
341
+ options,
342
+ trackLabel: track.getLabel(),
343
+ });
374
344
  return track as AudioTrack;
375
345
  } catch (error) {
376
346
  this.#logger.error('createMicrophoneAudioTrack()', { error, options });
@@ -435,10 +405,6 @@ class Engine {
435
405
  return this.media.deleteTrack(tracks);
436
406
  }
437
407
 
438
- public get canUseNoiseSuppression(): boolean {
439
- return this.media.isNoiseSuppressorLoaded;
440
- }
441
-
442
408
  public getNodeActiveStreamsStat(): NodeActiveStreamsStat {
443
409
  const node = this.socket.getServerUrl();
444
410
  const inboundActiveStreams = this.peers.hosts()
@@ -1,13 +1,17 @@
1
1
  import { inject, injectable } from 'inversify';
2
2
  import { Device } from 'mediasoup-client';
3
3
  import { RtpCapabilities, RtpCodecCapability } from 'mediasoup-client/lib/RtpParameters';
4
- import ConfigService from '../../config/ConfigService';
4
+ import { serializeError } from 'serialize-error';
5
+ import ConfigService, { ProcessedParams } from '../../config/ConfigService';
5
6
  import { SCREEN_SHARING_SIMULCAST_ENCODINGS, WEBCAM_SIMULCAST_ENCODINGS } from '../../constants/encodings';
7
+ import { CLIENT_EVENTS } from '../../constants/events';
6
8
  import { VIDEO_CONSTRAINS } from '../../constants/videoConstrains';
9
+ import EnhancedEventEmitter from '../../EnhancedEventEmitter';
7
10
  import { detectHandlerName } from '../../helpers/media';
8
11
  import { TOKEN } from '../../inversify.tokens';
9
12
  import {
10
13
  CreateCameraVideoTrackOptions,
14
+ CreateMicrophoneAudioTrackOptions,
11
15
  CreateScreenMediaOptions,
12
16
  CreateTracksPayload,
13
17
  CreateVideoTrackParams,
@@ -17,12 +21,18 @@ import {
17
21
  import type { AudioTrackFactory, VideoTrackFactory } from '../../types/container';
18
22
  import {
19
23
  AudioTrack,
20
- BaseTrack, BaseTrackParams, InitEffectsSDKParams, TrackWithEncodings,
24
+ BaseTrack,
25
+ BaseTrackParams,
26
+ InitEffectsSDKParams,
27
+ TrackWithEncodings,
21
28
  VideoTrack,
22
29
  } from '../../types/media';
30
+ import { IntegrationsService } from '../IntegrationsService';
23
31
  import Logger from '../Logger';
24
- import WasmModuleCompiler from './streamEffects/audio/noiseSuppression/WasmModuleCompiler';
32
+ import { ProcessorsCache } from './streamEffects/ProcessorsCache';
25
33
  import EffectsSDKTrackProcessor from './streamEffects/video/esdk/TrackProcessor';
34
+ import DefaultAudioTrack from './tracks/DefaultAudioTrack';
35
+ import DefaultVideoTrack from './tracks/DefaultVideoTrack';
26
36
  import MediaStreamTrackManager from './tracks/MediaStreamTrackManager';
27
37
 
28
38
  @injectable()
@@ -37,21 +47,28 @@ class Media {
37
47
 
38
48
  #effectsSDKParams: InitEffectsSDKParams | null = null;
39
49
 
40
- #noiseSuppressor?: WebAssembly.WebAssemblyInstantiatedSource;
41
-
42
50
  #effectsSDKInitialized = false;
43
51
 
44
52
  #effectsProcessors = new Map<string, EffectsSDKTrackProcessor>();
45
53
 
54
+ #denoiserInitializing = false;
55
+
56
+ #denoiser: ProcessedParams['denoiser'];
57
+
46
58
  constructor(
47
59
  @inject(TOKEN.Config) private readonly config: ConfigService,
48
60
  @inject(TOKEN.LoggerFactory) private readonly loggerFactory: (namespace: string) => Logger,
49
61
  @inject(TOKEN.AudioTrackFactory) private readonly audioTrackFactory: AudioTrackFactory,
62
+ @inject(TOKEN.ClientEventEmitter) private readonly clientEventEmitter: EnhancedEventEmitter,
63
+ @inject(TOKEN.IntegrationsService) private readonly integrationsService: IntegrationsService,
64
+ @inject(TOKEN.ProcessorsCache) private readonly processorsCache: ProcessorsCache,
50
65
  @inject(TOKEN.VideoTrackFactory) private readonly videoTrackFactory: VideoTrackFactory,
51
66
  @inject(TOKEN.MediaStreamTrackManager) private readonly mediaStreamTrackManager: MediaStreamTrackManager,
52
67
  ) {
53
68
  const effectsSDKParams = this.config.get('effectsSDKParams');
54
69
 
70
+ this.#denoiser = this.config.get('denoiser');
71
+
55
72
  this.#logger = this.loggerFactory('Media');
56
73
  this.#effectsSDKParams = effectsSDKParams || null;
57
74
  }
@@ -64,10 +81,6 @@ class Media {
64
81
  return this.device;
65
82
  }
66
83
 
67
- get isNoiseSuppressorLoaded(): boolean {
68
- return !!this.#noiseSuppressor;
69
- }
70
-
71
84
  get effectsSDKInitialized(): boolean {
72
85
  return this.#effectsSDKInitialized;
73
86
  }
@@ -87,19 +100,6 @@ class Media {
87
100
  }
88
101
  }
89
102
 
90
- async createNoiseSuppressor(): Promise<void> {
91
- try {
92
- if (this.#noiseSuppressor) {
93
- this.#logger.warn('createNoiseSuppressor()', { message: 'Noise suppression module already created' });
94
- return;
95
- }
96
-
97
- this.#noiseSuppressor = await WasmModuleCompiler.compileModule(this.config.get('staticFilesPath'));
98
- } catch (error) {
99
- this.#logger.warn('createNoiseSuppressor()', { message: 'Can not load wasm module', error });
100
- }
101
- }
102
-
103
103
  getTrackCodec(track: TrackWithEncodings): RtpCodecCapability | undefined {
104
104
  const { rtpCapabilities, rtpCapabilities: { codecs } } = this.mediasoupDevice;
105
105
  if (!codecs) {
@@ -117,11 +117,10 @@ class Media {
117
117
  constraints,
118
118
  };
119
119
 
120
- let track;
120
+ let track: DefaultAudioTrack | DefaultVideoTrack;
121
121
  if (mediaStreamTrack.kind === 'audio') {
122
122
  track = this.audioTrackFactory({
123
123
  ...params,
124
- noiseSuppressor: this.#noiseSuppressor,
125
124
  loggerNamespace: 'AudioTrack',
126
125
  });
127
126
  } else {
@@ -210,9 +209,9 @@ class Media {
210
209
  this.tracks.set(track.getLabel(), track);
211
210
  }
212
211
 
213
- initEffectsSDK(): Promise<boolean> {
212
+ async initEffectsSDK(): Promise<boolean> {
214
213
  if (this.#effectsSDKInitialized) {
215
- return Promise.resolve(true);
214
+ return true;
216
215
  }
217
216
 
218
217
  if (!this.#effectsSDKParams) {
@@ -240,7 +239,7 @@ class Media {
240
239
  this.#logger.info('Effects SDK initialized', { case: 'initEffectsSDK' });
241
240
  resolve(true);
242
241
  } catch (error) {
243
- this.#logger.error('Failed to initilize Effects SDK', { error, case: 'initEffectsSDK' });
242
+ this.#logger.error('Failed to initialize Effects SDK', { error, case: 'initEffectsSDK' });
244
243
  reject(error);
245
244
  }
246
245
  };
@@ -256,6 +255,59 @@ class Media {
256
255
  });
257
256
  }
258
257
 
258
+ async initDenoiser(track: AudioTrack, options?: CreateMicrophoneAudioTrackOptions) {
259
+ if (this.#denoiserInitializing) {
260
+ return;
261
+ }
262
+
263
+ try {
264
+ this.#denoiserInitializing = true;
265
+ this.clientEventEmitter.emit(CLIENT_EVENTS.denoiserInitializing, true);
266
+ await this.integrationsService.audio.waitForInitialization;
267
+
268
+ this.#logger.debug('Denoiser initialized, activating', {
269
+ case: 'Denoiser',
270
+ type: this.integrationsService.audio.type,
271
+ trackLabel: track.getLabel(),
272
+ });
273
+
274
+ const audioTrack = track as AudioTrack;
275
+ const trackProcessor = this.processorsCache.get(this.#denoiser, TrackLabel.Microphone);
276
+ if (!trackProcessor) {
277
+ this.#logger.error('No cached trackProcessor found', {
278
+ case: 'Denoiser',
279
+ trackLabel: track.getLabel(),
280
+ type: this.integrationsService.audio.type,
281
+ });
282
+
283
+ return;
284
+ }
285
+
286
+ audioTrack.prepareTrackProcessor(trackProcessor);
287
+
288
+ this.#logger.debug('Denoiser trackProcessor set, check if activation is needed', {
289
+ case: 'Denoiser',
290
+ type: this.#denoiser,
291
+ shouldEnableDenoiser: options?.noiseSuppression,
292
+ trackLabel: track.getLabel(),
293
+ });
294
+
295
+ if (options?.noiseSuppression) {
296
+ await (track as AudioTrack).enableNoiseSuppression();
297
+ }
298
+ } catch (error) {
299
+ this.#logger.warn('Failed to start noise suppression, featured original track', {
300
+ case: 'Denoiser',
301
+ type: this.#denoiser,
302
+ error: serializeError(error),
303
+ label: track.getLabel(),
304
+ });
305
+ } finally {
306
+ this.clientEventEmitter.emit(CLIENT_EVENTS.denoiserInitializing, false);
307
+ this.#denoiserInitializing = false;
308
+ }
309
+ }
310
+
259
311
  async createEffectsSDKTrackProcessor(trackLabel?: TrackLabel) {
260
312
  if (trackLabel) {
261
313
  const processor = this.#effectsProcessors.get(trackLabel);