@livedigital/client 3.48.0 → 3.49.0

package/src/engine/media/index.ts

@@ -1,574 +0,0 @@
-import { inject, injectable } from 'inversify';
-import { Device, type types } from 'mediasoup-client';
-import { serializeError } from 'serialize-error';
-import ConfigService, { ProcessedParams } from '../../config/ConfigService';
-import { SCREEN_SHARING_SIMULCAST_ENCODINGS, WEBCAM_SIMULCAST_ENCODINGS } from '../../constants/encodings';
-import { CLIENT_EVENTS } from '../../constants/events';
-import { SpatialLayer, VIDEO_CONSTRAINS } from '../../constants/videoConstrains';
-import EnhancedEventEmitter from '../../EnhancedEventEmitter';
-import OperationError from '../../errors/OperationError';
-import { detectHandlerName } from '../../helpers/media';
-import { TOKEN } from '../../inversify.tokens';
-import {
-  CreateCameraVideoTrackOptions,
-  CreateMicrophoneAudioTrackOptions,
-  CreateScreenMediaOptions,
-  CreateTracksPayload,
-  CreateVideoTrackParams,
-  ProducerRequestMaxSpatialLayer,
-  TrackLabel,
-  TransformParams,
-} from '../../types/common';
-import type { AudioTrackFactory, VideoTrackFactory } from '../../types/container';
-import {
-  AudioTrack,
-  BaseTrack,
-  BaseTrackParams,
-  InitEffectsSDKParams,
-  TrackWithEncodings,
-  VideoTrack,
-} from '../../types/media';
-import { IntegrationsService } from '../IntegrationsService';
-import Logger from '../Logger';
-import SignalingApiClient from '../signalingApiClient/SignalingApiClient';
-import { ASDKTrackProcessor } from './streamEffects/audio/asdk/ASDKTrackProcessor';
-import { ProcessorsCache } from './streamEffects/ProcessorsCache';
-import EffectsSDKTrackProcessor from './streamEffects/video/esdk/TrackProcessor';
-import DefaultAudioTrack from './tracks/DefaultAudioTrack';
-import DefaultVideoTrack from './tracks/DefaultVideoTrack';
-import MediaStreamTrackManager from './tracks/MediaStreamTrackManager';
-
-export interface DenoiserFallbackContext {
-  reason: string;
-  options?: CreateMicrophoneAudioTrackOptions;
-  error?: Error;
-}
-
-@injectable()
-class Media {
-  public isDeviceLoaded = false;
-
-  private device?: Device;
-
-  private readonly tracks = new Map<TrackLabel, BaseTrack>();
-
-  readonly #logger: Logger;
-
-  readonly #effectsSDKParams: InitEffectsSDKParams | null = null;
-
-  #effectsSDKInitialized = false;
-
-  readonly #effectsProcessors = new Map<string, EffectsSDKTrackProcessor>();
-
-  #denoiserInitializing = false;
-
-  #denoiserFallbacked = false;
-
-  #denoiser: ProcessedParams['denoiser'];
-
-  constructor(
-    @inject(TOKEN.Config) private readonly config: ConfigService,
-    @inject(TOKEN.LoggerFactory) private readonly loggerFactory: (namespace: string) => Logger,
-    @inject(TOKEN.AudioTrackFactory) private readonly audioTrackFactory: AudioTrackFactory,
-    @inject(TOKEN.ClientEventEmitter) private readonly clientEventEmitter: EnhancedEventEmitter,
-    @inject(TOKEN.IntegrationsService) private readonly integrationsService: IntegrationsService,
-    @inject(TOKEN.ProcessorsCache) private readonly processorsCache: ProcessorsCache,
-    @inject(TOKEN.VideoTrackFactory) private readonly videoTrackFactory: VideoTrackFactory,
-    @inject(TOKEN.MediaStreamTrackManager) private readonly mediaStreamTrackManager: MediaStreamTrackManager,
-    @inject(TOKEN.SignalingApiClient) private readonly signalingApiClient: SignalingApiClient,
-  ) {
-    const effectsSDKParams = this.config.get('effectsSDKParams');
-
-    this.#denoiser = this.config.get('denoiser');
-
-    this.#logger = this.loggerFactory('Media');
-    this.#effectsSDKParams = effectsSDKParams ?? null;
-  }
-
-  get mediasoupDevice(): Device {
-    this.device ??= this.createDevice();
-
-    return this.device;
-  }
-
-  get effectsSDKInitialized(): boolean {
-    return this.#effectsSDKInitialized;
-  }
-
-  get effectsSDKParams(): InitEffectsSDKParams | null {
-    return this.#effectsSDKParams;
-  }
-
-  getTrack(label: TrackLabel): BaseTrack | undefined {
-    return this.tracks.get(label);
-  }
-
-  async loadDevice(routerRtpCapabilities: types.RtpCapabilities): Promise<void> {
-    if (!this.isDeviceLoaded) {
-      await this.mediasoupDevice.load({ routerRtpCapabilities });
-      this.isDeviceLoaded = true;
-    }
-  }
-
-  getTrackCodec(track: TrackWithEncodings): types.RtpCodecCapability | undefined {
-    const { rtpCapabilities, rtpCapabilities: { codecs } } = this.mediasoupDevice;
-
-    if (!codecs) {
-      this.#logger.error('getTrackCodec()', { label: track.getLabel, rtpCapabilities });
-
-      return undefined;
-    }
-
-    return codecs.find((c) => c.mimeType.toLowerCase() === `${ track.kind }/${ track.getPreferredCodec() }`);
-  }
-
-  private createTracks({ constraints, mediaStreamTracks }: CreateTracksPayload): BaseTrack[] {
-    return mediaStreamTracks.map((mediaStreamTrack) => {
-      const params: Omit<BaseTrackParams, 'loggerNamespace'> = {
-        mediaStreamTrack,
-        constraints,
-      };
-
-      let track: DefaultAudioTrack | DefaultVideoTrack | undefined = undefined;
-
-      if (mediaStreamTrack.kind === 'audio') {
-        track = this.audioTrackFactory({
-          ...params,
-          loggerNamespace: 'AudioTrack',
-        });
-      } else {
-        track = this.videoTrackFactory({
-          ...params,
-          loggerNamespace: 'VideoTrack',
-        });
-      }
-
-      this.#logger.debug('createTrack() track created', {
-        trackId: track.id,
-        kind: track.kind,
-      });
-
-      return track as unknown as BaseTrack;
-    });
-  }
-
-  async createUserMediaTracks(constraints: MediaStreamConstraints): Promise<BaseTrack[]> {
-    const mediaStreamTracks = await this.mediaStreamTrackManager.createUserMediaTracks(constraints);
-
-    return this.createTracks({ constraints, mediaStreamTracks });
-  }
-
-  async createDisplayMediaTracks(constraints: MediaStreamConstraints): Promise<BaseTrack[]> {
-    const mediaStreamTracks = await this.mediaStreamTrackManager.createDisplayMediaTracks(constraints);
-
-    return this.createTracks({ constraints, mediaStreamTracks });
-  }
-
-  createCustomMediaTrack(
-    mediaStreamTrack: MediaStreamTrack,
-    constraints: MediaStreamConstraints,
-  ): BaseTrack {
-    return this.createTracks({ constraints, mediaStreamTracks: [mediaStreamTrack] })[0];
-  }
-
-  async deleteTrack(track: BaseTrack): Promise<void> {
-    const logContext = {
-      trackId: track.id,
-      label: track.getLabel(),
-      producerId: track.producerId,
-    };
-
-    try {
-      if ('disableNoiseSuppression' in track) {
-        await (track as AudioTrack).disableNoiseSuppression();
-      }
-
-      if ('disableEffects' in track) {
-        await (track as VideoTrack).disableEffects();
-      }
-    } catch (error) {
-      this.#logger.warn('deleteTrack()', { error, ...logContext });
-    }
-
-    try {
-      this.tracks.delete(track.getLabel());
-      track.stopMediaStreamTrack();
-      await track.closeProducer();
-    } catch (error) {
-      this.#logger.error('deleteTrack()', { error });
-    } finally {
-      this.#logger.debug('deleteTrack()', logContext);
-    }
-  }
-
-  getAllTracks(): BaseTrack[] {
-    return Array.from(this.tracks.values());
-  }
-
-  publishedTracks(): BaseTrack[] {
-    return this.getAllTracks().filter((track) => track.isPublished);
-  }
-
-  clearTracks(): void {
-    try {
-      this.getAllTracks().forEach((track) => track.stopMediaStreamTrack());
-    } catch (error) {
-      this.#logger.warn('clearTracks()', { error });
-    } finally {
-      this.tracks.clear();
-    }
-  }
-
-  async unpublishTracks(): Promise<void> {
-    try {
-      await Promise.all(this.publishedTracks().map(async (track) => await track.unpublish(false)));
-    } catch (error) {
-      this.#logger.warn('unpublishTracks()', { error });
-    }
-  }
-
-  setTrack(track: BaseTrack): void {
-    this.tracks.set(track.getLabel(), track);
-  }
-
-  async initEffectsSDK(): Promise<boolean> {
-    if (this.#effectsSDKInitialized) {
-      return true;
-    }
-
-    if (!this.#effectsSDKParams) {
-      this.#logger.error('Effects SDK config params not provided', { case: 'initEffectsSDK' });
-      throw new OperationError(
-        'effects_sdk_config_missing',
-        new Error('Effects SDK config not provided'),
-      );
-    }
-
-    const init = async (resolve: (value: boolean) => void, reject: (error: Error) => void): Promise<void> => {
-      if (!window.tsvb) {
-        this.#logger.info('Effects SDK not loaded, skipping initialization', { case: 'initEffectsSDK' });
-        resolve(false);
-
-        return;
-      }
-
-      this.#logger.info('Effects SDK initializing', { case: 'initEffectsSDK' });
-
-      try {
-        /* cant use Promise.all(), something breaks in parallel preload() */
-        this.#effectsProcessors.set(
-          TrackLabel.Camera, await this.createEffectsSDKTrackProcessor(TrackLabel.Camera),
-        );
-        this.#effectsProcessors.set(
-          TrackLabel.CameraPreview, await this.createEffectsSDKTrackProcessor(TrackLabel.CameraPreview),
-        );
-        this.#effectsSDKInitialized = true;
-        this.#logger.info('Effects SDK initialized', { case: 'initEffectsSDK' });
-        resolve(true);
-      } catch (error: unknown) {
-        this.#logger.error('Failed to initialize Effects SDK', { error, case: 'initEffectsSDK' });
-        reject(error as Error);
-      }
-    };
-
-    return await new Promise((resolve, reject) => {
-      if (!window.tsvb) {
-        this.#logger.info('Effects SDK tsvb-web.js is not ready, waiting', { case: 'initEffectsSDK' });
-        window.addEventListener('load', async () => await init(resolve, reject));
-
-        return;
-      }
-
-      init(resolve, reject).catch((error: unknown) => {
-        this.#logger.error('Failed to execute init() in initEffectsSDK', {
-          error: serializeError(error),
-        });
-      });
-    });
-  }
-
-  async initDenoiser(track: AudioTrack, options?: CreateMicrophoneAudioTrackOptions): Promise<void> {
-    if (this.#denoiserInitializing) {
-      return;
-    }
-
-    try {
-      this.#denoiserInitializing = true;
-      this.clientEventEmitter.emit(CLIENT_EVENTS.denoiserInitializing, true);
-
-      try {
-        await this.integrationsService.audio.waitForInitialization;
-      } catch (error: unknown) {
-        if (this.#denoiser === 'asdk') {
-          this.#denoiserFallbacked = true;
-          this.#denoiser = 'rnnoise';
-          this.config.set('denoiser', this.#denoiser);
-
-          this.#logger.debug('Set config to rnnoise denoiser', {
-            case: 'Denoiser',
-            error: serializeError(error),
-            type: this.config.get('denoiser'),
-          });
-
-          this.clientEventEmitter.emit(CLIENT_EVENTS.denoiserFallback);
-
-          this.integrationsService.init();
-          await this.integrationsService.audio.waitForInitialization;
-        }
-      }
-
-      this.#logger.debug('Denoiser initialized, activating', {
-        case: 'Denoiser',
-        type: this.integrationsService.audio.type,
-        trackLabel: track.getLabel(),
-      });
-
-      const audioTrack = track;
-      const trackProcessor = this.processorsCache.get(this.#denoiser, TrackLabel.Microphone);
-
-      if (!trackProcessor) {
-        this.#logger.error('No cached trackProcessor found', {
-          case: 'Denoiser',
-          trackLabel: track.getLabel(),
-          type: this.integrationsService.audio.type,
-        });
-
-        return;
-      }
-
-      if (trackProcessor.type === 'asdk') {
-        (trackProcessor as ASDKTrackProcessor).setOnError((reason, asdkError) => {
-          this.clientEventEmitter.emit(CLIENT_EVENTS.denoiserInitializing, true);
-          this.fallbackToRNoise(track, { reason, options, error: asdkError })
-            .catch((error: unknown) => {
-              this.#logger.error('Failed to fallback to RNNoise through onError', {
-                case: 'Denoiser',
-                type: this.#denoiser,
-                error: serializeError(error),
-                label: track.getLabel(),
-              });
-            })
-            .then(() => {
-              this.clientEventEmitter.emit(CLIENT_EVENTS.denoiserInitializing, false);
-            })
-            .catch(() => { /* no operation */ });
-        });
-      }
-
-      audioTrack.setTrackProcessor(trackProcessor);
-
-      this.#logger.debug('Denoiser trackProcessor set, check if activation is needed', {
-        case: 'Denoiser',
-        type: this.#denoiser,
-        shouldEnableDenoiser: options?.noiseSuppression,
-        trackLabel: track.getLabel(),
-      });
-
-      if (options?.noiseSuppression) {
-        await (track).enableNoiseSuppression();
-      }
-    } catch (error) {
-      this.#logger.warn('Failed to start noise suppression, featured original track', {
-        case: 'Denoiser',
-        type: this.#denoiser,
-        error: serializeError(error),
-        label: track.getLabel(),
-      });
-
-      if (this.#denoiser === 'asdk') {
-        try {
-          this.#logger.warn('Fallback to RNNoise', {
-            error: serializeError(error),
-          });
-          await this.fallbackToRNoise(track, { options, reason: 'ASDK init error', error });
-        } catch (error2) {
-          this.#logger.error('Failed to fallback to RNNoise through catch', {
-            case: 'Denoiser',
-            type: this.#denoiser,
-            error: serializeError(error2),
-            label: track.getLabel(),
-          });
-        }
-      }
-    } finally {
-      this.clientEventEmitter.emit(CLIENT_EVENTS.denoiserInitializing, false);
-      this.#denoiserInitializing = false;
-    }
-  }
-
-  async fallbackToRNoise(track: AudioTrack, context: DenoiserFallbackContext): Promise<void> {
-    if (this.#denoiserFallbacked) {
-      return;
-    }
-
-    this.#denoiserFallbacked = true;
-
-    if (this.#denoiser === 'rnnoise') {
-      return;
-    }
-
-    this.clientEventEmitter.emit(CLIENT_EVENTS.denoiserFallback);
-
-    this.#logger.info('Fallback to RNNoise', {
-      reason: context?.reason,
-      error: serializeError(context?.error),
-      case: 'Denoiser',
-    });
-
-    const audioTrack = track;
-
-    await audioTrack.disableNoiseSuppression();
-    audioTrack.unsetTrackProcessor();
-
-    this.#denoiser = 'rnnoise';
-    this.config.set('denoiser', this.#denoiser);
-    this.#logger.debug('Set config to rnnoise denoiser', {
-      case: 'Denoiser',
-      type: this.config.get('denoiser'),
-    });
-
-    this.integrationsService.init();
-    await this.integrationsService.audio.waitForInitialization;
-
-    const trackProcessor = this.processorsCache.get(this.#denoiser, TrackLabel.Microphone);
-
-    if (!trackProcessor) {
-      this.#logger.error('No cached trackProcessor found', {
-        case: 'Denoiser',
-        trackLabel: track.getLabel(),
-        type: this.integrationsService.audio.type,
-      });
-
-      return;
-    }
-
-    audioTrack.setTrackProcessor(trackProcessor);
-
-    if (context?.options?.noiseSuppression) {
-      await audioTrack.enableNoiseSuppression();
-    }
-  }
-
-  async createEffectsSDKTrackProcessor(trackLabel?: TrackLabel): Promise<EffectsSDKTrackProcessor> {
-    if (trackLabel) {
-      const processor = this.#effectsProcessors.get(trackLabel);
-
-      if (processor) {
-        return processor;
-      }
-    }
-
-    if (!this.#effectsSDKParams) {
-      this.#logger.error('Effects SDK config params not provided', { case: 'initEffectsSDK' });
-      throw new OperationError(
-        'effects_sdk_config_missing',
-        new Error('Effects SDK config not provided'),
-      );
-    }
-
-    const processor = new EffectsSDKTrackProcessor({
-      loggerFactory: this.loggerFactory,
-      customerId: this.config.get('effectsSDKParams')?.customerId,
-      sdk_url: this.config.get('effectsSDKParams')?.sdk_url,
-      api_url: this.config.get('effectsSDKParams')?.api_url,
-      config: this.config,
-    });
-
-    await processor.init();
-    this.#logger.info('Track effects processor initialized', { trackLabel });
-
-    return processor;
-  }
-
-  async handleProducerSetMaxSpatialLayer({
-    producerId,
-    spatialLayer: rawSpatialLayer,
-  }: ProducerRequestMaxSpatialLayer): Promise<void> {
-    const track = this.getAllTracks().find((t) => t.getProducer()?.id === producerId);
-
-    if (!track) {
-      this.#logger.warn('producerRequestMaxSpatialLayer()', { message: 'Producer not found' });
-
-      return;
-    }
-
-    if (!(track instanceof DefaultVideoTrack)) {
-      this.#logger.warn('producerRequestMaxSpatialLayer()', { message: 'Wrong producer kind' });
-
-      return;
-    }
-
-    const spatialLayer = rawSpatialLayer ?? SpatialLayer.LOW;
-    const currentMaxSpatialLayer = track.getMaxSpatialLayer();
-
-    if (currentMaxSpatialLayer === spatialLayer) {
-      this.#logger.debug('producerRequestMaxSpatialLayer()', {
-        message: 'Skip set max spatial layer',
-        requestedSpatialLayer: spatialLayer,
-        currentMaxSpatialLayer,
-      });
-
-      return;
-    }
-
-    if (track.getEncodings().length > 1) { // no need to do this operation for the only encoding item
-      await track.setMaxSpatialLayer(spatialLayer);
-    }
-
-    await this.signalingApiClient.producer.setProducerMaxSpatialLayer(producerId, {
-      maxSpatialLayer: spatialLayer,
-    });
-    await track.adjustEncodingPresetToCurrentMaxSpatialLayer(spatialLayer);
-  }
-
-  static getVideoTrackTransformParams(track: MediaStreamTrack): TransformParams {
-    const caps = track.getSettings();
-
-    return {
-      width: caps.width ?? VIDEO_CONSTRAINS.hd.width,
-      height: caps.height ?? VIDEO_CONSTRAINS.hd.height,
-    };
-  }
-
-  static getScreenVideoTrackParams(options?: CreateScreenMediaOptions): CreateVideoTrackParams {
-    const videoTrackOptions = options?.video ?? {
-      frameRate: { max: 20 },
-      width: { max: VIDEO_CONSTRAINS.hd.width },
-      height: { max: VIDEO_CONSTRAINS.hd.height },
-    };
-
-    return {
-      videoTrackOptions,
-      encoderConfig: options?.videoEncoderConfig ?? {
-        encodings: SCREEN_SHARING_SIMULCAST_ENCODINGS,
-      },
-    };
-  }
-
-  static getCameraVideoTrackParams(options?: CreateCameraVideoTrackOptions): CreateVideoTrackParams {
-    const videoTrackOptions = options?.video ?? {
-      frameRate: { max: 24 },
-      width: { max: VIDEO_CONSTRAINS.hd.width },
-      height: { max: VIDEO_CONSTRAINS.hd.height },
-    };
-
-    return {
-      videoTrackOptions,
-      encoderConfig: options?.encoderConfig ?? {
-        encodings: WEBCAM_SIMULCAST_ENCODINGS,
-      },
-    };
-  }
-
-  private createDevice(): Device {
-    const handlerName = detectHandlerName();
-
-    this.#logger.debug('Creating a media device', {
-      handlerName: handlerName ?? 'not specified',
-    });
-
-    return new Device({ handlerName });
-  }
-}
-
-export default Media;

package/src/engine/media/producer/PendingClosureProducerStorage.ts

@@ -1,37 +0,0 @@
-import { injectable } from 'inversify';
-import type { TrackLabel } from '../../../types/common';
-
-export interface PendingClosureProducer {
-  label: TrackLabel;
-  producerId: string;
-}
-
-@injectable()
-class PendingClosureProducerStorage {
-  readonly #pending = new Map<TrackLabel, PendingClosureProducer>();
-
-  set(label: TrackLabel, producerId: string): void {
-    this.#pending.set(label, {
-      label,
-      producerId,
-    });
-  }
-
-  get(label: TrackLabel): PendingClosureProducer | undefined {
-    return this.#pending.get(label);
-  }
-
-  has(label: TrackLabel): boolean {
-    return this.#pending.has(label);
-  }
-
-  delete(label: TrackLabel): void {
-    this.#pending.delete(label);
-  }
-
-  entries(): PendingClosureProducer[] {
-    return Array.from(this.#pending.values());
-  }
-}
-
-export default PendingClosureProducerStorage;