@livedigital/client 3.28.1 → 3.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config/ConfigService.d.ts +6 -1
- package/dist/constants/events.d.ts +1 -0
- package/dist/constants/events.ts +1 -0
- package/dist/engine/IntegrationsService.d.ts +28 -0
- package/dist/engine/index.d.ts +7 -3
- package/dist/engine/media/index.d.ts +10 -5
- package/dist/engine/media/streamEffects/ProcessorsCache.d.ts +11 -0
- package/dist/engine/media/streamEffects/audio/AudioTrackProcessor.d.ts +6 -0
- package/dist/engine/media/streamEffects/audio/asdk/ASDKTrackProcessor.d.ts +23 -0
- package/dist/engine/media/streamEffects/audio/noiseSuppression/{TrackProcessor.d.ts → RNNoiseTrackProcessor.d.ts} +3 -1
- package/dist/engine/media/tracks/DefaultAudioTrack.d.ts +11 -2
- package/dist/helpers/loader.d.ts +3 -0
- package/dist/index.es.js +4 -4
- package/dist/index.js +19 -19
- package/dist/inversify.factories.d.ts +2 -1
- package/dist/inversify.tokens.d.ts +3 -0
- package/dist/types/client.d.ts +6 -0
- package/dist/types/container.d.ts +7 -0
- package/dist/types/engine.d.ts +1 -0
- package/dist/types/media.d.ts +5 -4
- package/package.json +2 -1
- package/src/config/ConfigService.ts +13 -2
- package/src/constants/events.ts +1 -0
- package/src/engine/IntegrationsService.ts +155 -0
- package/src/engine/index.ts +22 -56
- package/src/engine/media/index.ts +79 -27
- package/src/engine/media/streamEffects/ProcessorsCache.ts +41 -0
- package/src/engine/media/streamEffects/audio/AudioTrackProcessor.ts +6 -0
- package/src/engine/media/streamEffects/audio/asdk/ASDKTrackProcessor.ts +155 -0
- package/src/engine/media/streamEffects/audio/asdk/atsrb.d.ts +37 -0
- package/src/engine/media/streamEffects/audio/asdk/errorBus.d.ts +42 -0
- package/src/engine/media/streamEffects/audio/noiseSuppression/{TrackProcessor.ts → RNNoiseTrackProcessor.ts} +32 -6
- package/src/engine/media/streamEffects/video/esdk/TrackProcessor.ts +5 -2
- package/src/engine/media/tracks/DefaultAudioTrack.ts +39 -14
- package/src/engine/media/tracks/DefaultBaseTrack.ts +9 -0
- package/src/helpers/loader.ts +49 -0
- package/src/index.ts +4 -1
- package/src/inversify.config.ts +29 -2
- package/src/inversify.factories.ts +47 -2
- package/src/inversify.tokens.ts +3 -0
- package/src/types/client.ts +6 -0
- package/src/types/container.ts +8 -0
- package/src/types/engine.ts +1 -0
- package/src/types/media.ts +5 -4
- package/dist/helpers/config.d.ts +0 -10
- package/src/helpers/config.ts +0 -31
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import { interfaces } from 'inversify';
|
|
2
|
-
import { AudioTrackFactory, PeerConsumerFactory, PeerFactory, PeerTrackFactory, VideoTrackFactory } from './types/container';
|
|
2
|
+
import { AudioTrackFactory, AudioTrackProcessorFactory, PeerConsumerFactory, PeerFactory, PeerTrackFactory, VideoTrackFactory } from './types/container';
|
|
3
3
|
export declare function peerTrackFactory(context: interfaces.Context): PeerTrackFactory;
|
|
4
4
|
export declare function peerConsumerFactory(context: interfaces.Context): PeerConsumerFactory;
|
|
5
5
|
export declare function peerFactory(context: interfaces.Context): PeerFactory;
|
|
6
6
|
export declare function audioTrackFactory(context: interfaces.Context): AudioTrackFactory;
|
|
7
7
|
export declare function videoTrackFactory(context: interfaces.Context): VideoTrackFactory;
|
|
8
|
+
export declare function audioTrackProcessorFactory(context: interfaces.Context): AudioTrackProcessorFactory;
|
|
@@ -26,6 +26,9 @@ export declare enum TOKEN {
|
|
|
26
26
|
PeerConsumerFactory = "PeerConsumerFactory",
|
|
27
27
|
AudioTrackFactory = "AudioTrackFactory",
|
|
28
28
|
VideoTrackFactory = "VideoTrackFactory",
|
|
29
|
+
AudioTrackProcessorFactory = "AudioTrackProcessorFactory",
|
|
30
|
+
IntegrationsService = "IntegrationsService",
|
|
31
|
+
ProcessorsCache = "ProcessorsCache",
|
|
29
32
|
StatsHandler = "StatsHandler",
|
|
30
33
|
DataChannelsManager = "DataChannelsManager"
|
|
31
34
|
}
|
package/dist/types/client.d.ts
CHANGED
|
@@ -4,6 +4,12 @@ import { LogLevel, LogMessageHandler } from './common';
|
|
|
4
4
|
import { IssuesHandler, NetworkScoresUpdatedHandler } from './engine';
|
|
5
5
|
import { InitEffectsSDKParams } from './media';
|
|
6
6
|
export declare type ClientParams = {
|
|
7
|
+
denoiser?: 'rnnoise' | 'asdk';
|
|
8
|
+
asdk?: {
|
|
9
|
+
customerId: string;
|
|
10
|
+
version: '2.3.5';
|
|
11
|
+
localDir?: string;
|
|
12
|
+
};
|
|
7
13
|
clientEventEmitter?: EnhancedEventEmitter;
|
|
8
14
|
internalEventEmitter?: EnhancedEventEmitter;
|
|
9
15
|
network?: {
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import Logger from '../engine/Logger';
|
|
2
|
+
import { AudioTrackProcessor } from '../engine/media/streamEffects/audio/AudioTrackProcessor';
|
|
2
3
|
import DefaultAudioTrack from '../engine/media/tracks/DefaultAudioTrack';
|
|
3
4
|
import DefaultVideoTrack from '../engine/media/tracks/DefaultVideoTrack';
|
|
4
5
|
import PeerTrack from '../engine/media/tracks/PeerTrack';
|
|
@@ -6,6 +7,7 @@ import Peer from '../engine/Peer';
|
|
|
6
7
|
import PeerConsumer, { PeerConsumerConstructorParams } from '../engine/PeerConsumer';
|
|
7
8
|
import { CreatePeerParams } from '../engine/Peers';
|
|
8
9
|
import { CreatePeerTrackParams } from '../engine/RemotePeerTracks';
|
|
10
|
+
import { TrackLabel } from './common';
|
|
9
11
|
import { AudioTrackParams, BaseTrackParams } from './media';
|
|
10
12
|
export declare type PeerTrackFactoryParams = CreatePeerTrackParams & {
|
|
11
13
|
onClose: () => void;
|
|
@@ -17,3 +19,8 @@ export declare type PeerConsumerFactory = (params: CreatePeerConsumerParams) =>
|
|
|
17
19
|
export declare type LoggerFactory = (namespace: string) => Logger;
|
|
18
20
|
export declare type AudioTrackFactory = (params: AudioTrackParams) => DefaultAudioTrack;
|
|
19
21
|
export declare type VideoTrackFactory = (params: BaseTrackParams) => DefaultVideoTrack;
|
|
22
|
+
export declare type CreateAudioTrackProcessorParams = {
|
|
23
|
+
noiseSuppressor?: WebAssembly.WebAssemblyInstantiatedSource;
|
|
24
|
+
trackLabel: TrackLabel;
|
|
25
|
+
};
|
|
26
|
+
export declare type AudioTrackProcessorFactory = (params: CreateAudioTrackProcessorParams) => AudioTrackProcessor;
|
package/dist/types/engine.d.ts
CHANGED
|
@@ -70,6 +70,7 @@ export declare type ClientObserverEvents = {
|
|
|
70
70
|
[CLIENT_EVENTS.forcedDisconnect]: [];
|
|
71
71
|
[CLIENT_EVENTS.rejectUnauthorized]: [];
|
|
72
72
|
[CLIENT_EVENTS.channelStateInconsistent]: [ChannelStateInconsistentPayload];
|
|
73
|
+
[CLIENT_EVENTS.denoiserInitializing]: [boolean];
|
|
73
74
|
};
|
|
74
75
|
export declare type InternalObserverEvents = {
|
|
75
76
|
[INTERNAL_CLIENT_EVENTS.trackProduced]: [Track | BaseTrack];
|
package/dist/types/media.d.ts
CHANGED
|
@@ -4,14 +4,15 @@ import { AppData, RtpEncodingParameters } from 'mediasoup-client/lib/types';
|
|
|
4
4
|
import { ParsedOutboundVideoStreamStats, ParsedOutboundAudioStreamStats } from 'webrtc-issue-detector';
|
|
5
5
|
import Logger from '../engine/Logger';
|
|
6
6
|
import Media from '../engine/media';
|
|
7
|
-
import {
|
|
8
|
-
import Network from '../engine/network';
|
|
7
|
+
import { AudioTrackProcessor } from '../engine/media/streamEffects/audio/AudioTrackProcessor';
|
|
9
8
|
import type { Effects } from '../engine/media/streamEffects/video/esdk/TrackProcessor';
|
|
10
9
|
import MediaStreamTrackManager from '../engine/media/tracks/MediaStreamTrackManager';
|
|
10
|
+
import { MyPeer } from '../engine/MyPeer';
|
|
11
|
+
import Network from '../engine/network';
|
|
12
|
+
import SocketIO from '../engine/network/Socket';
|
|
11
13
|
import EnhancedEventEmitter from '../EnhancedEventEmitter';
|
|
12
14
|
import { BaseTrackInfo, EncoderConfig, TrackLabel, TrackOutboundStats, TrackProduceParams, TrackTransformParams } from './common';
|
|
13
15
|
import { InternalObserverEvents } from './engine';
|
|
14
|
-
import SocketIO from '../engine/network/Socket';
|
|
15
16
|
import StatsHandler from '../engine/WebRTCStats/StatsHandler';
|
|
16
17
|
export interface BaseTrackDependenciesParams {
|
|
17
18
|
internalEventEmitter: EnhancedEventEmitter<InternalObserverEvents>;
|
|
@@ -66,6 +67,7 @@ export interface TrackWithEffects extends BaseTrack {
|
|
|
66
67
|
isEffectsProcessing: boolean;
|
|
67
68
|
}
|
|
68
69
|
export interface AudioTrack extends BaseTrack, TrackWithEncodings, TrackWithEffects {
|
|
70
|
+
prepareTrackProcessor(trackProcessor?: AudioTrackProcessor): void;
|
|
69
71
|
enableNoiseSuppression(): Promise<void>;
|
|
70
72
|
disableNoiseSuppression(): Promise<void>;
|
|
71
73
|
pause(): Promise<void>;
|
|
@@ -89,7 +91,6 @@ export interface VideoTrack extends BaseTrack, TrackWithEncodings, TrackWithEffe
|
|
|
89
91
|
}
|
|
90
92
|
export declare type Track = AudioTrack | VideoTrack;
|
|
91
93
|
export interface AudioTrackParams extends BaseTrackParams {
|
|
92
|
-
noiseSuppressor?: WebAssembly.WebAssemblyInstantiatedSource;
|
|
93
94
|
}
|
|
94
95
|
export interface MediaTracksFactory {
|
|
95
96
|
createAudioTrack: (params: Omit<AudioTrackParams, 'logger'>) => BaseTrack;
|
package/package.json
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
"name": "@livedigital/client",
|
|
3
3
|
"author": "vlprojects",
|
|
4
4
|
"license": "MIT",
|
|
5
|
-
"version": "3.
|
|
5
|
+
"version": "3.29.0",
|
|
6
6
|
"private": false,
|
|
7
7
|
"bugs": {
|
|
8
8
|
"url": "https://github.com/vlprojects/livedigital-sdk/issues"
|
|
@@ -50,6 +50,7 @@
|
|
|
50
50
|
"fflate": "^0.8.2",
|
|
51
51
|
"hash-sum": "^2.0.0",
|
|
52
52
|
"inversify": "^6.0.2",
|
|
53
|
+
"load-script": "^2.0.0",
|
|
53
54
|
"mediasoup-client": "3.7.18",
|
|
54
55
|
"message-batcher": "^0.0.2",
|
|
55
56
|
"qs": "^6.11.0",
|
|
@@ -4,11 +4,15 @@ import { LoadBalancerApiClientParams } from '../engine/network/LoadBalancerClien
|
|
|
4
4
|
import * as client from '../types/client';
|
|
5
5
|
import { InitEffectsSDKParams } from '../types/media';
|
|
6
6
|
|
|
7
|
-
type ParamsToOmit = 'clientEventEmitter' | 'internalEventEmitter' | 'network' | 'effectsSDKParams';
|
|
8
|
-
|
|
7
|
+
type ParamsToOmit = 'clientEventEmitter' | 'internalEventEmitter' | 'network' | 'effectsSDKParams' | 'asdk';
|
|
9
8
|
export type ProcessedParams = Required<Omit<client.ClientParams, ParamsToOmit>> & {
|
|
10
9
|
loadbalancer: LoadBalancerApiClientParams,
|
|
11
10
|
effectsSDKParams?: InitEffectsSDKParams,
|
|
11
|
+
asdk: {
|
|
12
|
+
customerId: string
|
|
13
|
+
version: '2.3.5',
|
|
14
|
+
localDir?: string,
|
|
15
|
+
},
|
|
12
16
|
};
|
|
13
17
|
|
|
14
18
|
const loadbalancerDefaults: LoadBalancerApiClientParams = {
|
|
@@ -16,6 +20,11 @@ const loadbalancerDefaults: LoadBalancerApiClientParams = {
|
|
|
16
20
|
timeout: 5000,
|
|
17
21
|
};
|
|
18
22
|
|
|
23
|
+
const asdkDefaults: client.ClientParams['asdk'] = {
|
|
24
|
+
customerId: '',
|
|
25
|
+
version: '2.3.5',
|
|
26
|
+
};
|
|
27
|
+
|
|
19
28
|
const noop = () => { };
|
|
20
29
|
|
|
21
30
|
@injectable()
|
|
@@ -24,6 +33,8 @@ export default class ConfigService {
|
|
|
24
33
|
|
|
25
34
|
constructor(params: client.ClientParams) {
|
|
26
35
|
this.#params = {
|
|
36
|
+
denoiser: params.denoiser || 'rnnoise',
|
|
37
|
+
asdk: Object.assign(asdkDefaults, params.asdk || {}),
|
|
27
38
|
loadbalancer: Object.assign(loadbalancerDefaults, params.network?.loadbalancer || {}),
|
|
28
39
|
logLevel: params.logLevel ?? LogLevels.Error,
|
|
29
40
|
sendAnalytics: params.sendAnalytics ?? false,
|
package/src/constants/events.ts
CHANGED
|
@@ -38,6 +38,7 @@ export const CLIENT_EVENTS = {
|
|
|
38
38
|
connectionRestored: 'connection-restored',
|
|
39
39
|
forcedDisconnect: 'forced-disconnect',
|
|
40
40
|
rejectUnauthorized: 'reject-unauthorized',
|
|
41
|
+
denoiserInitializing: 'denoiser-initializing',
|
|
41
42
|
} as const;
|
|
42
43
|
|
|
43
44
|
export const TRACK_EVENTS = {
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
import { inject, injectable } from 'inversify';
|
|
2
|
+
import ConfigService from '../config/ConfigService';
|
|
3
|
+
import { loadScriptWithFallback } from '../helpers/loader';
|
|
4
|
+
import { TOKEN } from '../inversify.tokens';
|
|
5
|
+
import { TrackLabel } from '../types/common';
|
|
6
|
+
import { AudioTrackProcessorFactory, LoggerFactory } from '../types/container';
|
|
7
|
+
import Logger from './Logger';
|
|
8
|
+
import { ASDKTrackProcessor } from './media/streamEffects/audio/asdk/ASDKTrackProcessor';
|
|
9
|
+
import WasmModuleCompiler from './media/streamEffects/audio/noiseSuppression/WasmModuleCompiler';
|
|
10
|
+
import { ProcessorsCache } from './media/streamEffects/ProcessorsCache';
|
|
11
|
+
|
|
12
|
+
export type Integrations = 'asdk' | 'rnnoise'; // later 'esdk' should be added
|
|
13
|
+
export type IntegrationContext = {
|
|
14
|
+
integrated: boolean,
|
|
15
|
+
initPromise: Promise<void>,
|
|
16
|
+
};
|
|
17
|
+
|
|
18
|
+
export interface Integration {
|
|
19
|
+
type: Integrations,
|
|
20
|
+
enabled: boolean; // flag to show that integration is requested/enabled through ClientParams
|
|
21
|
+
isInitialized: boolean;
|
|
22
|
+
waitForInitialization: Promise<void>;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
@injectable()
|
|
26
|
+
export class IntegrationsService {
|
|
27
|
+
readonly #integrationsStorage: Map<Integrations, IntegrationContext> = new Map();
|
|
28
|
+
|
|
29
|
+
readonly #logger: Logger;
|
|
30
|
+
|
|
31
|
+
constructor(
|
|
32
|
+
@inject(TOKEN.Config) private readonly config: ConfigService,
|
|
33
|
+
@inject(TOKEN.ProcessorsCache) private readonly processorsCache: ProcessorsCache,
|
|
34
|
+
@inject(TOKEN.AudioTrackProcessorFactory) private readonly audioTrackProcessorFactory: AudioTrackProcessorFactory,
|
|
35
|
+
@inject(TOKEN.LoggerFactory) loggerFactory: LoggerFactory,
|
|
36
|
+
) {
|
|
37
|
+
this.#logger = loggerFactory('IntegrationsService');
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
init() { // must not be async
|
|
41
|
+
const denoiser = this.config.get('denoiser');
|
|
42
|
+
if (denoiser === 'asdk') {
|
|
43
|
+
this.#logger.info('Start integration of ASDK', {
|
|
44
|
+
denoiser,
|
|
45
|
+
});
|
|
46
|
+
|
|
47
|
+
const context: IntegrationContext = {
|
|
48
|
+
integrated: false,
|
|
49
|
+
initPromise: this.integrateASDK(),
|
|
50
|
+
};
|
|
51
|
+
|
|
52
|
+
this.#integrationsStorage.set('asdk', context);
|
|
53
|
+
} else if (denoiser === 'rnnoise') {
|
|
54
|
+
this.#logger.info('Start integration of RNNoise', {
|
|
55
|
+
denoiser,
|
|
56
|
+
});
|
|
57
|
+
|
|
58
|
+
const context: IntegrationContext = {
|
|
59
|
+
integrated: false,
|
|
60
|
+
initPromise: this.integrateRNNoise(),
|
|
61
|
+
};
|
|
62
|
+
|
|
63
|
+
this.#integrationsStorage.set('rnnoise', context);
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
private async integrateASDK(): Promise<void> {
|
|
68
|
+
const asdkParams = this.config.get('asdk');
|
|
69
|
+
|
|
70
|
+
if (!window.atsvb) {
|
|
71
|
+
// https://effectssdk.ai/sdk/audio/dev/2.3.5/atsvb-web.js
|
|
72
|
+
const remoteUrl = `https://effectssdk.ai/sdk/audio/dev/${asdkParams.version}/atsvb-web.js`;
|
|
73
|
+
const localUrl = asdkParams.localDir ?? `${asdkParams.localDir}/${asdkParams.version}/atsvb-web.js`;
|
|
74
|
+
|
|
75
|
+
await loadScriptWithFallback(remoteUrl, localUrl);
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
if (!window.atsvb) {
|
|
79
|
+
this.#logger.info('Audio Effects SDK is not loaded, skipping initialization', { case: 'initAudioSDK' });
|
|
80
|
+
|
|
81
|
+
throw new Error('windows.atsvb is still missing');
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
const processor = this.audioTrackProcessorFactory({
|
|
85
|
+
trackLabel: TrackLabel.Microphone,
|
|
86
|
+
}) as ASDKTrackProcessor;
|
|
87
|
+
|
|
88
|
+
await processor.init();
|
|
89
|
+
|
|
90
|
+
this.processorsCache.add('asdk', TrackLabel.Microphone, processor);
|
|
91
|
+
|
|
92
|
+
const context = this.#integrationsStorage.get('asdk');
|
|
93
|
+
|
|
94
|
+
if (!context) {
|
|
95
|
+
throw Error('IntegrationContext is missing for ASDK');
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
context.integrated = true;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
get audio(): Integration {
|
|
102
|
+
const denoiser = this.config.get('denoiser');
|
|
103
|
+
|
|
104
|
+
if (denoiser === 'asdk') {
|
|
105
|
+
return this.getASDKState();
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
return this.getRNNoiseState();
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
private getASDKState(): Integration {
|
|
112
|
+
const asdk = this.#integrationsStorage.get('asdk');
|
|
113
|
+
|
|
114
|
+
return {
|
|
115
|
+
type: 'asdk',
|
|
116
|
+
enabled: Boolean(asdk),
|
|
117
|
+
isInitialized: asdk?.integrated ?? false,
|
|
118
|
+
waitForInitialization: asdk?.initPromise ?? Promise.resolve(),
|
|
119
|
+
};
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
private getRNNoiseState(): Integration {
|
|
123
|
+
const rnnoise = this.#integrationsStorage.get('rnnoise');
|
|
124
|
+
|
|
125
|
+
return {
|
|
126
|
+
type: 'rnnoise',
|
|
127
|
+
enabled: Boolean(rnnoise),
|
|
128
|
+
isInitialized: rnnoise?.integrated ?? false,
|
|
129
|
+
waitForInitialization: rnnoise?.initPromise ?? Promise.resolve(),
|
|
130
|
+
};
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
private async integrateRNNoise(): Promise<void> {
|
|
134
|
+
const noiseSuppressor = await WasmModuleCompiler.compileModule(this.config.get('staticFilesPath'));
|
|
135
|
+
|
|
136
|
+
const processor = this.audioTrackProcessorFactory({
|
|
137
|
+
trackLabel: TrackLabel.Microphone,
|
|
138
|
+
noiseSuppressor,
|
|
139
|
+
});
|
|
140
|
+
|
|
141
|
+
this.processorsCache.add('rnnoise', TrackLabel.Microphone, processor);
|
|
142
|
+
|
|
143
|
+
const context = this.#integrationsStorage.get('rnnoise');
|
|
144
|
+
|
|
145
|
+
if (!context) {
|
|
146
|
+
throw Error('IntegrationContext is missing for ASDK');
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
context.integrated = true;
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
reset() {
|
|
153
|
+
this.#integrationsStorage.clear();
|
|
154
|
+
}
|
|
155
|
+
}
|
package/src/engine/index.ts
CHANGED
|
@@ -4,6 +4,7 @@ import { MediaKind, RtpCapabilities } from 'mediasoup-client/lib/RtpParameters';
|
|
|
4
4
|
import { serializeError } from 'serialize-error';
|
|
5
5
|
import WebRTCIssueDetector from 'webrtc-issue-detector';
|
|
6
6
|
import clientMetaProvider from '../ClientMetaProvider';
|
|
7
|
+
import ConfigService from '../config/ConfigService';
|
|
7
8
|
import { MAX_JOIN_ATTEMPTS, SOCKET_ERROR_CODE_UNAUTHORIZED } from '../constants/common';
|
|
8
9
|
import {
|
|
9
10
|
CHANNEL_EVENTS, CLIENT_EVENTS, INTERNAL_CLIENT_EVENTS, NETWORK_OBSERVER_EVENTS,
|
|
@@ -46,8 +47,10 @@ import ChannelStateSynchronizer from './ChannelStateSynchronizer/ChannelStateSyn
|
|
|
46
47
|
import EventsQueue from './EventsQueue';
|
|
47
48
|
import ChannelEventHandler from './handlers/ChannelEventHandler';
|
|
48
49
|
import MediaSoupEventHandler from './handlers/MediaSoupEventHandler';
|
|
50
|
+
import { IntegrationsService } from './IntegrationsService';
|
|
49
51
|
import Logger from './Logger';
|
|
50
52
|
import Media from './media';
|
|
53
|
+
import { ProcessorsCache } from './media/streamEffects/ProcessorsCache';
|
|
51
54
|
import VideoTrack from './media/tracks/DefaultVideoTrack';
|
|
52
55
|
import { MyPeer } from './MyPeer';
|
|
53
56
|
import Network from './network';
|
|
@@ -74,12 +77,17 @@ class Engine {
|
|
|
74
77
|
|
|
75
78
|
#isConnectionLost = false;
|
|
76
79
|
|
|
80
|
+
#denoiserInitializing: boolean = false;
|
|
81
|
+
|
|
77
82
|
#transportStatsProvider?: TransportsStatsProvider;
|
|
78
83
|
|
|
79
84
|
constructor(
|
|
85
|
+
@inject(TOKEN.Config) private readonly config: ConfigService,
|
|
80
86
|
@inject(TOKEN.CallState) private readonly state: CallState,
|
|
81
87
|
@inject(TOKEN.MyPeer) private readonly myPeer: MyPeer,
|
|
82
88
|
@inject(TOKEN.LoggerFactory) loggerFactory: (namespace: string) => Logger,
|
|
89
|
+
@inject(TOKEN.IntegrationsService) private readonly integrationsService: IntegrationsService,
|
|
90
|
+
@inject(TOKEN.ProcessorsCache) private readonly processorsCache: ProcessorsCache,
|
|
83
91
|
@inject(TOKEN.MediaSoupEventHandler) private readonly mediaSoupEventsHandler: MediaSoupEventHandler,
|
|
84
92
|
@inject(TOKEN.ChannelEventHandler) private readonly channelEventsHandler: ChannelEventHandler,
|
|
85
93
|
@inject(TOKEN.ChannelStateSynchronizer) readonly channelStateSynchronizer: ChannelStateSynchronizer,
|
|
@@ -112,7 +120,7 @@ class Engine {
|
|
|
112
120
|
try {
|
|
113
121
|
const { rtpCapabilities } = await this.socket
|
|
114
122
|
.request('router.getRtpCapabilities') as { rtpCapabilities: RtpCapabilities };
|
|
115
|
-
|
|
123
|
+
this.integrationsService.init();
|
|
116
124
|
await this.media.loadDevice(rtpCapabilities);
|
|
117
125
|
|
|
118
126
|
this.#logger.info('Supported capabilities', {
|
|
@@ -184,6 +192,9 @@ class Engine {
|
|
|
184
192
|
await this.media.clearTracks();
|
|
185
193
|
}
|
|
186
194
|
|
|
195
|
+
await this.processorsCache.reset();
|
|
196
|
+
this.integrationsService.reset();
|
|
197
|
+
|
|
187
198
|
this.eventsQueue.removeListener('message');
|
|
188
199
|
this.dataChannelsManager.closeAllDataChannels();
|
|
189
200
|
|
|
@@ -208,46 +219,6 @@ class Engine {
|
|
|
208
219
|
}
|
|
209
220
|
}
|
|
210
221
|
|
|
211
|
-
private async releaseForRejoin(): Promise<void> {
|
|
212
|
-
if (this.isReleasing) {
|
|
213
|
-
return;
|
|
214
|
-
}
|
|
215
|
-
|
|
216
|
-
this.isReleasing = true;
|
|
217
|
-
|
|
218
|
-
try {
|
|
219
|
-
this.state.isJoined = false;
|
|
220
|
-
this.state.reset();
|
|
221
|
-
this.myPeer.delete();
|
|
222
|
-
this.webRtcIssueDetector?.stopWatchingNewPeerConnections();
|
|
223
|
-
|
|
224
|
-
this.peers.reset();
|
|
225
|
-
|
|
226
|
-
await this.network.closeSendTransport();
|
|
227
|
-
await this.network.closeReceiveTransport();
|
|
228
|
-
|
|
229
|
-
this.eventsQueue.removeListener('message');
|
|
230
|
-
this.dataChannelsManager.closeAllDataChannels();
|
|
231
|
-
|
|
232
|
-
this.channelEventsHandler.reset();
|
|
233
|
-
this.mediaSoupEventsHandler.reset();
|
|
234
|
-
|
|
235
|
-
this.removeClientEventEmitterListeners();
|
|
236
|
-
|
|
237
|
-
await this.network.reset();
|
|
238
|
-
this.socket.disconnect();
|
|
239
|
-
clientMetaProvider.clear();
|
|
240
|
-
this.#logger.debug('release()', {
|
|
241
|
-
socketId: this.myPeerId,
|
|
242
|
-
});
|
|
243
|
-
} catch (error: unknown) {
|
|
244
|
-
this.#logger.error('release()', { error });
|
|
245
|
-
throw new Error('Error release engine');
|
|
246
|
-
} finally {
|
|
247
|
-
this.isReleasing = false;
|
|
248
|
-
}
|
|
249
|
-
}
|
|
250
|
-
|
|
251
222
|
private removeClientEventEmitterListeners(): void {
|
|
252
223
|
Array.from(Object.values(INTERNAL_CLIENT_EVENTS)).forEach((x) => this.internalEventEmitter.removeAllListeners(x));
|
|
253
224
|
}
|
|
@@ -283,7 +254,7 @@ class Engine {
|
|
|
283
254
|
actionName: 'joinWithRetry',
|
|
284
255
|
logger: this.#loggerFactory('RetryJoinNode'),
|
|
285
256
|
onError: async () => {
|
|
286
|
-
await this.
|
|
257
|
+
await this.release(true);
|
|
287
258
|
},
|
|
288
259
|
});
|
|
289
260
|
} catch (error) {
|
|
@@ -353,24 +324,23 @@ class Engine {
|
|
|
353
324
|
audio: options?.audio,
|
|
354
325
|
});
|
|
355
326
|
|
|
356
|
-
|
|
327
|
+
const trackLabel = options?.isPreview ? TrackLabel.MicrophonePreview : TrackLabel.Microphone;
|
|
328
|
+
|
|
329
|
+
track.setLabel(trackLabel);
|
|
357
330
|
track.setStopTrackOnPause(Boolean(options?.stopTrackOnPause));
|
|
358
331
|
if (options?.encoderConfig) {
|
|
359
332
|
track.setEncoderConfig(options.encoderConfig);
|
|
360
333
|
}
|
|
361
334
|
|
|
362
|
-
if (track.kind === 'audio' &&
|
|
363
|
-
|
|
364
|
-
await (track as AudioTrack).enableNoiseSuppression();
|
|
365
|
-
} catch (error) {
|
|
366
|
-
this.#logger.warn('Failed to start noise suppression, featured original track', {
|
|
367
|
-
label: track.getLabel(),
|
|
368
|
-
});
|
|
369
|
-
}
|
|
335
|
+
if (track.kind === 'audio' && track.getLabel() === TrackLabel.Microphone && !this.#denoiserInitializing) {
|
|
336
|
+
this.media.initDenoiser(track as AudioTrack, options); // must be non-blocking, don't await
|
|
370
337
|
}
|
|
371
338
|
|
|
372
339
|
this.media.setTrack(track);
|
|
373
|
-
this.#logger.debug('createMicrophoneAudioTrack()', {
|
|
340
|
+
this.#logger.debug('createMicrophoneAudioTrack()', {
|
|
341
|
+
options,
|
|
342
|
+
trackLabel: track.getLabel(),
|
|
343
|
+
});
|
|
374
344
|
return track as AudioTrack;
|
|
375
345
|
} catch (error) {
|
|
376
346
|
this.#logger.error('createMicrophoneAudioTrack()', { error, options });
|
|
@@ -435,10 +405,6 @@ class Engine {
|
|
|
435
405
|
return this.media.deleteTrack(tracks);
|
|
436
406
|
}
|
|
437
407
|
|
|
438
|
-
public get canUseNoiseSuppression(): boolean {
|
|
439
|
-
return this.media.isNoiseSuppressorLoaded;
|
|
440
|
-
}
|
|
441
|
-
|
|
442
408
|
public getNodeActiveStreamsStat(): NodeActiveStreamsStat {
|
|
443
409
|
const node = this.socket.getServerUrl();
|
|
444
410
|
const inboundActiveStreams = this.peers.hosts()
|
|
@@ -1,13 +1,17 @@
|
|
|
1
1
|
import { inject, injectable } from 'inversify';
|
|
2
2
|
import { Device } from 'mediasoup-client';
|
|
3
3
|
import { RtpCapabilities, RtpCodecCapability } from 'mediasoup-client/lib/RtpParameters';
|
|
4
|
-
import
|
|
4
|
+
import { serializeError } from 'serialize-error';
|
|
5
|
+
import ConfigService, { ProcessedParams } from '../../config/ConfigService';
|
|
5
6
|
import { SCREEN_SHARING_SIMULCAST_ENCODINGS, WEBCAM_SIMULCAST_ENCODINGS } from '../../constants/encodings';
|
|
7
|
+
import { CLIENT_EVENTS } from '../../constants/events';
|
|
6
8
|
import { VIDEO_CONSTRAINS } from '../../constants/videoConstrains';
|
|
9
|
+
import EnhancedEventEmitter from '../../EnhancedEventEmitter';
|
|
7
10
|
import { detectHandlerName } from '../../helpers/media';
|
|
8
11
|
import { TOKEN } from '../../inversify.tokens';
|
|
9
12
|
import {
|
|
10
13
|
CreateCameraVideoTrackOptions,
|
|
14
|
+
CreateMicrophoneAudioTrackOptions,
|
|
11
15
|
CreateScreenMediaOptions,
|
|
12
16
|
CreateTracksPayload,
|
|
13
17
|
CreateVideoTrackParams,
|
|
@@ -17,12 +21,18 @@ import {
|
|
|
17
21
|
import type { AudioTrackFactory, VideoTrackFactory } from '../../types/container';
|
|
18
22
|
import {
|
|
19
23
|
AudioTrack,
|
|
20
|
-
BaseTrack,
|
|
24
|
+
BaseTrack,
|
|
25
|
+
BaseTrackParams,
|
|
26
|
+
InitEffectsSDKParams,
|
|
27
|
+
TrackWithEncodings,
|
|
21
28
|
VideoTrack,
|
|
22
29
|
} from '../../types/media';
|
|
30
|
+
import { IntegrationsService } from '../IntegrationsService';
|
|
23
31
|
import Logger from '../Logger';
|
|
24
|
-
import
|
|
32
|
+
import { ProcessorsCache } from './streamEffects/ProcessorsCache';
|
|
25
33
|
import EffectsSDKTrackProcessor from './streamEffects/video/esdk/TrackProcessor';
|
|
34
|
+
import DefaultAudioTrack from './tracks/DefaultAudioTrack';
|
|
35
|
+
import DefaultVideoTrack from './tracks/DefaultVideoTrack';
|
|
26
36
|
import MediaStreamTrackManager from './tracks/MediaStreamTrackManager';
|
|
27
37
|
|
|
28
38
|
@injectable()
|
|
@@ -37,21 +47,28 @@ class Media {
|
|
|
37
47
|
|
|
38
48
|
#effectsSDKParams: InitEffectsSDKParams | null = null;
|
|
39
49
|
|
|
40
|
-
#noiseSuppressor?: WebAssembly.WebAssemblyInstantiatedSource;
|
|
41
|
-
|
|
42
50
|
#effectsSDKInitialized = false;
|
|
43
51
|
|
|
44
52
|
#effectsProcessors = new Map<string, EffectsSDKTrackProcessor>();
|
|
45
53
|
|
|
54
|
+
#denoiserInitializing = false;
|
|
55
|
+
|
|
56
|
+
#denoiser: ProcessedParams['denoiser'];
|
|
57
|
+
|
|
46
58
|
constructor(
|
|
47
59
|
@inject(TOKEN.Config) private readonly config: ConfigService,
|
|
48
60
|
@inject(TOKEN.LoggerFactory) private readonly loggerFactory: (namespace: string) => Logger,
|
|
49
61
|
@inject(TOKEN.AudioTrackFactory) private readonly audioTrackFactory: AudioTrackFactory,
|
|
62
|
+
@inject(TOKEN.ClientEventEmitter) private readonly clientEventEmitter: EnhancedEventEmitter,
|
|
63
|
+
@inject(TOKEN.IntegrationsService) private readonly integrationsService: IntegrationsService,
|
|
64
|
+
@inject(TOKEN.ProcessorsCache) private readonly processorsCache: ProcessorsCache,
|
|
50
65
|
@inject(TOKEN.VideoTrackFactory) private readonly videoTrackFactory: VideoTrackFactory,
|
|
51
66
|
@inject(TOKEN.MediaStreamTrackManager) private readonly mediaStreamTrackManager: MediaStreamTrackManager,
|
|
52
67
|
) {
|
|
53
68
|
const effectsSDKParams = this.config.get('effectsSDKParams');
|
|
54
69
|
|
|
70
|
+
this.#denoiser = this.config.get('denoiser');
|
|
71
|
+
|
|
55
72
|
this.#logger = this.loggerFactory('Media');
|
|
56
73
|
this.#effectsSDKParams = effectsSDKParams || null;
|
|
57
74
|
}
|
|
@@ -64,10 +81,6 @@ class Media {
|
|
|
64
81
|
return this.device;
|
|
65
82
|
}
|
|
66
83
|
|
|
67
|
-
get isNoiseSuppressorLoaded(): boolean {
|
|
68
|
-
return !!this.#noiseSuppressor;
|
|
69
|
-
}
|
|
70
|
-
|
|
71
84
|
get effectsSDKInitialized(): boolean {
|
|
72
85
|
return this.#effectsSDKInitialized;
|
|
73
86
|
}
|
|
@@ -87,19 +100,6 @@ class Media {
|
|
|
87
100
|
}
|
|
88
101
|
}
|
|
89
102
|
|
|
90
|
-
async createNoiseSuppressor(): Promise<void> {
|
|
91
|
-
try {
|
|
92
|
-
if (this.#noiseSuppressor) {
|
|
93
|
-
this.#logger.warn('createNoiseSuppressor()', { message: 'Noise suppression module already created' });
|
|
94
|
-
return;
|
|
95
|
-
}
|
|
96
|
-
|
|
97
|
-
this.#noiseSuppressor = await WasmModuleCompiler.compileModule(this.config.get('staticFilesPath'));
|
|
98
|
-
} catch (error) {
|
|
99
|
-
this.#logger.warn('createNoiseSuppressor()', { message: 'Can not load wasm module', error });
|
|
100
|
-
}
|
|
101
|
-
}
|
|
102
|
-
|
|
103
103
|
getTrackCodec(track: TrackWithEncodings): RtpCodecCapability | undefined {
|
|
104
104
|
const { rtpCapabilities, rtpCapabilities: { codecs } } = this.mediasoupDevice;
|
|
105
105
|
if (!codecs) {
|
|
@@ -117,11 +117,10 @@ class Media {
|
|
|
117
117
|
constraints,
|
|
118
118
|
};
|
|
119
119
|
|
|
120
|
-
let track;
|
|
120
|
+
let track: DefaultAudioTrack | DefaultVideoTrack;
|
|
121
121
|
if (mediaStreamTrack.kind === 'audio') {
|
|
122
122
|
track = this.audioTrackFactory({
|
|
123
123
|
...params,
|
|
124
|
-
noiseSuppressor: this.#noiseSuppressor,
|
|
125
124
|
loggerNamespace: 'AudioTrack',
|
|
126
125
|
});
|
|
127
126
|
} else {
|
|
@@ -210,9 +209,9 @@ class Media {
|
|
|
210
209
|
this.tracks.set(track.getLabel(), track);
|
|
211
210
|
}
|
|
212
211
|
|
|
213
|
-
initEffectsSDK(): Promise<boolean> {
|
|
212
|
+
async initEffectsSDK(): Promise<boolean> {
|
|
214
213
|
if (this.#effectsSDKInitialized) {
|
|
215
|
-
return
|
|
214
|
+
return true;
|
|
216
215
|
}
|
|
217
216
|
|
|
218
217
|
if (!this.#effectsSDKParams) {
|
|
@@ -240,7 +239,7 @@ class Media {
|
|
|
240
239
|
this.#logger.info('Effects SDK initialized', { case: 'initEffectsSDK' });
|
|
241
240
|
resolve(true);
|
|
242
241
|
} catch (error) {
|
|
243
|
-
this.#logger.error('Failed to
|
|
242
|
+
this.#logger.error('Failed to initialize Effects SDK', { error, case: 'initEffectsSDK' });
|
|
244
243
|
reject(error);
|
|
245
244
|
}
|
|
246
245
|
};
|
|
@@ -256,6 +255,59 @@ class Media {
|
|
|
256
255
|
});
|
|
257
256
|
}
|
|
258
257
|
|
|
258
|
+
async initDenoiser(track: AudioTrack, options?: CreateMicrophoneAudioTrackOptions) {
|
|
259
|
+
if (this.#denoiserInitializing) {
|
|
260
|
+
return;
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
try {
|
|
264
|
+
this.#denoiserInitializing = true;
|
|
265
|
+
this.clientEventEmitter.emit(CLIENT_EVENTS.denoiserInitializing, true);
|
|
266
|
+
await this.integrationsService.audio.waitForInitialization;
|
|
267
|
+
|
|
268
|
+
this.#logger.debug('Denoiser initialized, activating', {
|
|
269
|
+
case: 'Denoiser',
|
|
270
|
+
type: this.integrationsService.audio.type,
|
|
271
|
+
trackLabel: track.getLabel(),
|
|
272
|
+
});
|
|
273
|
+
|
|
274
|
+
const audioTrack = track as AudioTrack;
|
|
275
|
+
const trackProcessor = this.processorsCache.get(this.#denoiser, TrackLabel.Microphone);
|
|
276
|
+
if (!trackProcessor) {
|
|
277
|
+
this.#logger.error('No cached trackProcessor found', {
|
|
278
|
+
case: 'Denoiser',
|
|
279
|
+
trackLabel: track.getLabel(),
|
|
280
|
+
type: this.integrationsService.audio.type,
|
|
281
|
+
});
|
|
282
|
+
|
|
283
|
+
return;
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
audioTrack.prepareTrackProcessor(trackProcessor);
|
|
287
|
+
|
|
288
|
+
this.#logger.debug('Denoiser trackProcessor set, check if activation is needed', {
|
|
289
|
+
case: 'Denoiser',
|
|
290
|
+
type: this.#denoiser,
|
|
291
|
+
shouldEnableDenoiser: options?.noiseSuppression,
|
|
292
|
+
trackLabel: track.getLabel(),
|
|
293
|
+
});
|
|
294
|
+
|
|
295
|
+
if (options?.noiseSuppression) {
|
|
296
|
+
await (track as AudioTrack).enableNoiseSuppression();
|
|
297
|
+
}
|
|
298
|
+
} catch (error) {
|
|
299
|
+
this.#logger.warn('Failed to start noise suppression, featured original track', {
|
|
300
|
+
case: 'Denoiser',
|
|
301
|
+
type: this.#denoiser,
|
|
302
|
+
error: serializeError(error),
|
|
303
|
+
label: track.getLabel(),
|
|
304
|
+
});
|
|
305
|
+
} finally {
|
|
306
|
+
this.clientEventEmitter.emit(CLIENT_EVENTS.denoiserInitializing, false);
|
|
307
|
+
this.#denoiserInitializing = false;
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
|
|
259
311
|
async createEffectsSDKTrackProcessor(trackLabel?: TrackLabel) {
|
|
260
312
|
if (trackLabel) {
|
|
261
313
|
const processor = this.#effectsProcessors.get(trackLabel);
|