@livedigital/client 3.32.0-createCustomMediaTracks.1 → 3.32.0

package/dist/types/common.d.ts

@@ -119,11 +119,6 @@ export declare type CreateScreenMediaOptions = BaseVideoTrackOptions & BaseAudio
     videoEncoderConfig?: VideoEncoderConfig;
     audioEncoderConfig?: AudioEncoderConfig;
 };
-export declare type CreateCustomMediaOptions = BaseVideoTrackOptions & BaseAudioTrackOptions & {
-    videoEncoderConfig?: VideoEncoderConfig;
-    audioEncoderConfig?: AudioEncoderConfig;
-    mediaStream: MediaStream;
-};
 export declare enum TrackLabel {
     Camera = "camera",
     CameraPreview = "camera-preview",

package/dist/types/media.d.ts

@@ -68,6 +68,7 @@ export interface TrackWithEffects extends BaseTrack {
 }
 export interface AudioTrack extends BaseTrack, TrackWithEncodings, TrackWithEffects {
     prepareTrackProcessor(trackProcessor?: AudioTrackProcessor): void;
+    unsetTrackProcessor(): void;
     enableNoiseSuppression(): Promise<void>;
     disableNoiseSuppression(): Promise<void>;
     pause(): Promise<void>;

package/package.json

@@ -2,7 +2,7 @@
   "name": "@livedigital/client",
   "author": "vlprojects",
   "license": "MIT",
-  "version": "3.32.0-createCustomMediaTracks.1",
+  "version": "3.32.0",
   "private": false,
   "bugs": {
     "url": "https://github.com/vlprojects/livedigital-sdk/issues"
@@ -85,6 +85,7 @@
     "@types/sinon-chai": "^3.2.9",
     "@typescript-eslint/eslint-plugin": "^4.32.0",
     "@typescript-eslint/parser": "^4.32.0",
+    "audio-effects-sdk": "^2.4.2",
     "axios-mock-adapter": "^1.21.2",
     "chai": "^4.3.7",
     "chai-as-promised": "^7.1.1",

package/src/config/ConfigService.ts

@@ -54,4 +54,8 @@ export default class ConfigService {
   get<K extends keyof ProcessedParams>(key: K): ProcessedParams[K] {
     return this.#params[key] as ProcessedParams[K];
   }
+
+  set<K extends keyof ProcessedParams, V extends ProcessedParams[K]>(key: K, value: V) {
+    this.#params[key] = value;
+  }
 }

package/src/engine/IntegrationsService.ts

@@ -144,7 +144,7 @@ export class IntegrationsService {
     const context = this.#integrationsStorage.get('rnnoise');
 
     if (!context) {
-      throw Error('IntegrationContext is missing for ASDK');
+      throw Error('IntegrationContext is missing for RNNoise');
     }
 
     context.integrated = true;

package/src/engine/index.ts

@@ -17,7 +17,6 @@ import { retryAsync } from '../helpers/retry';
 import { TOKEN } from '../inversify.tokens';
 import {
   CreateCameraVideoTrackOptions,
-  CreateCustomMediaOptions,
   CreateMicrophoneAudioTrackOptions,
   CreateScreenMediaOptions,
   JoinChannelParams,
@@ -402,65 +401,6 @@ class Engine {
     }
   }
 
-  async createCustomMediaTracks(options?: CreateCustomMediaOptions): Promise<Track[]> {
-    const videoTrackParams = Media.getScreenVideoTrackParams(options);
-
-    if (!options?.mediaStream) {
-      throw new Error('MediaStream is required');
-    }
-
-    try {
-      const tracks = await this.media.createCustomMediaTracks(options.mediaStream, {
-        video: options?.video,
-        audio: options?.audio,
-      });
-
-      tracks.forEach((track) => {
-        track.mediaStreamTrack.addEventListener('ended', () => {
-          track.unpublish().catch((error: Error) => {
-            this.#logger.error('Failed to unpublish media track', {
-              track,
-              error: serializeError(error),
-            });
-          });
-        }, { once: true });
-
-        track.setStopTrackOnPause(Boolean(options?.stopTrackOnPause));
-        if (track instanceof VideoTrack) {
-          track.mediaStreamTrack.contentHint = 'text';
-          track.setLabel(TrackLabel.ScreenVideo);
-          track.setEncoderConfig(videoTrackParams.encoderConfig);
-          const transformParams = Media.getVideoTrackTransformParams(track.mediaStreamTrack);
-          track.setTransformParams(transformParams);
-          this.media.setTrack(track);
-          this.#logger.debug('createScreenMediaTrack()', {
-            trackParams: videoTrackParams,
-            transformParams,
-            label: track.logCtx,
-          });
-
-          return;
-        }
-
-        track.setLabel(TrackLabel.ScreenAudio);
-        if (options?.audioEncoderConfig) {
-          track.setEncoderConfig(options.audioEncoderConfig);
-        }
-
-        this.media.setTrack(track);
-        this.#logger.debug('createCustomMediaTrack()', {
-          trackParams: options?.audio,
-          encoderConfig: options?.audioEncoderConfig,
-        });
-      });
-
-      return tracks as Track[];
-    } catch (error) {
-      this.#logger.error('createCustomMediaTrack()', { error, trackParams: videoTrackParams });
-      throw new Error(`Can not create custom media tracks: ${error.message}`);
-    }
-  }
-
   deleteTrack(tracks: BaseTrack): Promise<void> {
     return this.media.deleteTrack(tracks);
   }

package/src/engine/media/index.ts

@@ -29,6 +29,7 @@ import {
 } from '../../types/media';
 import { IntegrationsService } from '../IntegrationsService';
 import Logger from '../Logger';
+import { ASDKTrackProcessor } from './streamEffects/audio/asdk/ASDKTrackProcessor';
 import { ProcessorsCache } from './streamEffects/ProcessorsCache';
 import EffectsSDKTrackProcessor from './streamEffects/video/esdk/TrackProcessor';
 import DefaultAudioTrack from './tracks/DefaultAudioTrack';
@@ -53,6 +54,8 @@ class Media {
 
   #denoiserInitializing = false;
 
+  #denoiserFallbacked = false;
+
   #denoiser: ProcessedParams['denoiser'];
 
   constructor(
@@ -149,11 +152,6 @@ class Media {
     return this.createTracks({ constraints, mediaStreamTracks });
   }
 
-  async createCustomMediaTracks(mediaStream: MediaStream, constraints: MediaStreamConstraints): Promise<BaseTrack[]> {
-    const mediaStreamTracks = mediaStream.getTracks();
-    return this.createTracks({ constraints, mediaStreamTracks });
-  }
-
   async deleteTrack(track: BaseTrack) {
     const logContext = {
       trackId: track.id,
@@ -288,6 +286,24 @@ class Media {
         return;
       }
 
+      if (this.#denoiser === 'asdk') {
+        (trackProcessor as ASDKTrackProcessor).setOnError(() => {
+          this.clientEventEmitter.emit(CLIENT_EVENTS.denoiserInitializing, true);
+          this.fallbackToRNoise(track, options)
+            .catch((error) => {
+              this.#logger.error('Failed to fallback to RNNoise through onError', {
+                case: 'Denoiser',
+                type: this.#denoiser,
+                error: serializeError(error),
+                label: track.getLabel(),
+              });
+            })
+            .then(() => {
+              this.clientEventEmitter.emit(CLIENT_EVENTS.denoiserInitializing, false);
+            });
+        });
+      }
+
       audioTrack.prepareTrackProcessor(trackProcessor);
 
       this.#logger.debug('Denoiser trackProcessor set, check if activation is needed', {
@@ -307,12 +323,67 @@ class Media {
         error: serializeError(error),
         label: track.getLabel(),
       });
+
+      if (this.#denoiser === 'asdk') {
+        try {
+          this.#logger.warn('Fallback to RNNoise', {
+            error: serializeError(error),
+          });
+          await this.fallbackToRNoise(track, options);
+        } catch (error2) {
+          this.#logger.error('Failed to fallback to RNNoise through catch', {
+            case: 'Denoiser',
+            type: this.#denoiser,
+            error: serializeError(error2),
+            label: track.getLabel(),
+          });
+        }
+      }
     } finally {
       this.clientEventEmitter.emit(CLIENT_EVENTS.denoiserInitializing, false);
       this.#denoiserInitializing = false;
     }
   }
 
+  async fallbackToRNoise(track: AudioTrack, options?: CreateMicrophoneAudioTrackOptions) {
+    this.#logger.debug('Fallback to RNNoise denoiser', {
+      case: 'Denoiser',
+    });
+
+    const audioTrack = track as AudioTrack;
+
+    await audioTrack.disableNoiseSuppression();
+    audioTrack.unsetTrackProcessor();
+
+    this.#denoiser = 'rnnoise';
+    this.config.set('denoiser', this.#denoiser);
+    this.#logger.debug('Set config to rnnoise denoiser', {
+      case: 'Denoiser',
+      type: this.config.get('denoiser'),
+    });
+
+    this.integrationsService.init();
+    await this.integrationsService.audio.waitForInitialization;
+
+    const trackProcessor = this.processorsCache.get(this.#denoiser, TrackLabel.Microphone);
+
+    if (!trackProcessor) {
+      this.#logger.error('No cached trackProcessor found', {
+        case: 'Denoiser',
+        trackLabel: track.getLabel(),
+        type: this.integrationsService.audio.type,
+      });
+
+      return;
+    }
+
+    audioTrack.prepareTrackProcessor(trackProcessor);
+
+    if (options?.noiseSuppression) {
+      await audioTrack.enableNoiseSuppression();
+    }
+  }
+
   async createEffectsSDKTrackProcessor(trackLabel?: TrackLabel) {
     if (trackLabel) {
       const processor = this.#effectsProcessors.get(trackLabel);

package/src/engine/media/streamEffects/audio/asdk/ASDKTrackProcessor.ts

@@ -1,9 +1,8 @@
 import { serializeError } from 'serialize-error';
+import type { atsvb, Config } from 'audio-effects-sdk';
 import { LoggerFactory } from '../../../../../types/container';
 import Logger from '../../../../Logger';
 import { AudioTrackProcessor } from '../AudioTrackProcessor';
-import type { atsvb, Config } from './atsrb';
-import { ErrorObject } from './errorBus';
 
 declare global {
   interface Window {
@@ -11,10 +10,13 @@ declare global {
   }
 }
 
+const noop = () => { };
+
 export type AudioTrackProcessorParams = {
   asdkCustomerId: string,
   loggerFactory: LoggerFactory,
   onModelsLoaded?: () => void,
+  onError?: () => void,
 };
 
 export class ASDKTrackProcessor implements AudioTrackProcessor {
@@ -32,21 +34,64 @@ export class ASDKTrackProcessor implements AudioTrackProcessor {
 
   readonly type = 'asdk';
 
+  #onError: () => void;
+
   constructor(params: AudioTrackProcessorParams) {
     const { loggerFactory } = params;
     this.#logger = loggerFactory('ASDKTrackProcessor');
+    this.#onError = typeof params.onError === 'function' ? params.onError : noop;
 
     // eslint-disable-next-line new-cap
     this.#atsvb = new window.atsvb(params.asdkCustomerId);
 
-    const config: Config = {
+    this.#atsvb.config({
       preset: this.#preset,
-      sampleRate: this.#sampleRate,
-    };
-
-    this.#atsvb.config(config);
+      sample_rate: this.#sampleRate,
+      sdk_url: 'https://effectssdk.ai/sdk/audio/',
+      // path to wasm files
+      wasmPaths: {
+        'ort-wasm.wasm': 'https://effectssdk.ai/sdk/audio/dev/2.4.2/ort-wasm.wasm',
+        'ort-wasm-simd.wasm': 'https://effectssdk.ai/sdk/audio/dev/2.4.2/ort-wasm-simd.wasm',
+      },
+    });
+
+    this.#atsvb.onError((error) => {
+      const {
+        message, type, cause, ...rest
+      } = error;
+
+      if (type === 'error') {
+        this.#logger.error(message, {
+          error: serializeError(cause),
+          cause,
+          ...rest,
+          type: this.type,
+          case: 'Denoiser',
+        });
+        this.#onError();
+        this.#onError = noop;
+      }
+
+      if (type === 'warning') {
+        this.#logger.warn(message, {
+          ...rest,
+          type: this.type,
+          case: 'Denoiser',
+        });
+      }
+
+      if (type === 'info') {
+        this.#logger.info(message, {
+          ...rest,
+          type: this.type,
+          case: 'Denoiser',
+        });
+      }
+    });
+  }
 
-    this.#atsvb.onError(this.atsvbErrorHandler.bind(this));
+  setOnError(onError: () => void): void {
+    this.#onError = onError;
   }
 
   async init() {
@@ -102,13 +147,15 @@ export class ASDKTrackProcessor implements AudioTrackProcessor {
         track,
       });
 
+      this.#onError();
+      this.#onError = noop;
+
       throw new Error('Can not process this track');
     }
   }
 
   async stopProcessing() {
     try {
-      this.#atsvb.clear();
       this.#atsvb.stop();
     } catch (error) {
       this.#logger.error('stopProcessing()', {
@@ -120,36 +167,4 @@ export class ASDKTrackProcessor implements AudioTrackProcessor {
       this.#originalTrack = null;
     }
   }
-
-  private atsvbErrorHandler(error: ErrorObject) {
-    const {
-      message, type, cause, ...rest
-    } = error;
-
-    if (type === 'error') {
-      this.#logger.error(message, {
-        error: serializeError(cause),
-        cause,
-        ...rest,
-        type: this.type,
-        case: 'Denoiser',
-      });
-    }
-
-    if (type === 'warning') {
-      this.#logger.warn(message, {
-        ...rest,
-        type: this.type,
-        case: 'Denoiser',
-      });
-    }
-
-    if (type === 'info') {
-      this.#logger.info(message, {
-        ...rest,
-        type: this.type,
-        case: 'Denoiser',
-      });
-    }
-  }
 }

package/src/engine/media/tracks/DefaultAudioTrack.ts

@@ -37,6 +37,10 @@ class DefaultAudioTrack extends DefaultBaseTrack implements AudioTrack {
     this.#trackProcessor = trackProcessor;
   }
 
+  unsetTrackProcessor() {
+    this.#trackProcessor = undefined;
+  }
+
   get isEffectsProcessing(): boolean {
     return Boolean(this.#trackProcessor?.isProcessing);
   }

package/src/index.ts

@@ -10,7 +10,6 @@ import { ClientParams } from './types/client';
 import {
   AvailableMediaDevices,
   CreateCameraVideoTrackOptions,
-  CreateCustomMediaOptions,
   CreateMicrophoneAudioTrackOptions,
   CreateScreenMediaOptions,
   JoinChannelParams, Role,
@@ -122,10 +121,6 @@ class Client {
     return this.#engine.createScreenMediaTracks(options);
   }
 
-  createCustomMediaTracks(options?: CreateCustomMediaOptions): Promise<Track[]> {
-    return this.#engine.createCustomMediaTracks(options);
-  }
-
   deleteTrack(track: Track): Promise<void> {
     return this.#engine.deleteTrack(track);
   }

package/src/inversify.factories.ts

@@ -155,10 +155,11 @@ export function audioTrackProcessorFactory(context: interfaces.Context): AudioTr
   const loggerFactory = context.container.get<LoggerFactory>(TOKEN.LoggerFactory);
   const config = context.container.get<ConfigService>(TOKEN.Config);
   const processorsCache = context.container.get<ProcessorsCache>(TOKEN.ProcessorsCache);
-  const denoiser = config.get('denoiser');
   const asdk = config.get('asdk');
 
   return (params: CreateAudioTrackProcessorParams): AudioTrackProcessor => {
+    const denoiser = config.get('denoiser');
+
     if (processorsCache.has(denoiser, params.trackLabel)) {
       throw new Error(`Track processor for type:${denoiser}, label:${params.trackLabel} already created`);
     }

package/src/types/common.ts

@@ -154,12 +154,6 @@ export type CreateScreenMediaOptions = BaseVideoTrackOptions & BaseAudioTrackOpt
   audioEncoderConfig?: AudioEncoderConfig,
 };
 
-export type CreateCustomMediaOptions = BaseVideoTrackOptions & BaseAudioTrackOptions & {
-  videoEncoderConfig?: VideoEncoderConfig,
-  audioEncoderConfig?: AudioEncoderConfig,
-  mediaStream: MediaStream,
-};
-
 export enum TrackLabel {
   Camera = 'camera',
   CameraPreview = 'camera-preview',

package/src/types/media.ts

@@ -87,6 +87,7 @@ export interface TrackWithEffects extends BaseTrack {
 
 export interface AudioTrack extends BaseTrack, TrackWithEncodings, TrackWithEffects {
   prepareTrackProcessor(trackProcessor?: AudioTrackProcessor): void;
+  unsetTrackProcessor(): void;
   enableNoiseSuppression(): Promise<void>;
   disableNoiseSuppression(): Promise<void>;
   pause(): Promise<void>;