@livedigital/client 3.48.0 → 3.49.0

package/src/engine/media/producer/ProducerCloseManager.ts

@@ -1,115 +0,0 @@
-import { inject, injectable } from 'inversify';
-import { serializeError } from 'serialize-error';
-import NotFoundError from '../../../errors/NotFoundError';
-import { retryAsync } from '../../../helpers/retry';
-import { TOKEN } from '../../../inversify.tokens';
-import type { TrackLabel } from '../../../types/common';
-import type Logger from '../../Logger';
-import type Network from '../../network';
-import PendingClosureProducerStorage, {
-  PendingClosureProducer,
-} from './PendingClosureProducerStorage';
-
-@injectable()
-class ProducerCloseManager {
-  readonly #logger: Logger;
-
-  constructor(
-    @inject(TOKEN.Network) private readonly network: Network,
-    @inject(TOKEN.PendingClosureProducerStorage) private readonly storage: PendingClosureProducerStorage,
-    @inject(TOKEN.LoggerFactory) loggerFactory: (namespace: string) => Logger,
-  ) {
-    this.#logger = loggerFactory('ProducerCloseManager');
-  }
-
-  async close(label: TrackLabel, producerId: string): Promise<void> {
-    try {
-      const closed = await this.sendCloseRequest(producerId);
-      this.storage.delete(label);
-
-      if (!closed) {
-        return;
-      }
-
-      this.#logger.info('Remote producer closed', { label, producerId });
-    } catch (error) {
-      this.storage.set(label, producerId);
-      this.#logger.warn('Failed to close remote producer, queued for retry', {
-        label,
-        producerId,
-        error: serializeError(error),
-      });
-    }
-  }
-
-  async ensureNoPending(label: TrackLabel): Promise<void> {
-    const pending = this.storage.get(label);
-
-    if (!pending) {
-      return;
-    }
-
-    await this.flushEntry(pending);
-  }
-
-  async flushPending(): Promise<void> {
-    const entries = this.storage.entries();
-
-    for (const entry of entries) {
-      try {
-        await this.flushEntry(entry);
-      } catch {}
-    }
-  }
-
-  private async flushEntry(entry: PendingClosureProducer): Promise<void> {
-    try {
-      await retryAsync(
-        async () => {
-          await this.sendCloseRequest(entry.producerId);
-        },
-        {
-          maxRetries: 5,
-          minBackoffDelayMs: 200,
-          maxBackoffDelayMs: 1000,
-          actionName: 'closeRemoteProducer',
-          logger: this.#logger,
-        },
-      );
-
-      this.storage.delete(entry.label);
-      this.#logger.info('Pending remote producer closed', {
-        label: entry.label,
-        producerId: entry.producerId,
-      });
-    } catch (error) {
-      this.#logger.warn('Failed to close pending remote producer', {
-        label: entry.label,
-        producerId: entry.producerId,
-        error: serializeError(error),
-      });
-
-      throw error;
-    }
-  }
-
-  private async sendCloseRequest(producerId: string): Promise<boolean> {
-    try {
-      await this.network.closeRemoteProducer(producerId);
-
-      return true;
-    } catch (error) {
-      if (error instanceof NotFoundError) {
-        this.#logger.debug('Remote producer maybe already closed', {
-          producerId,
-        });
-
-        return false;
-      }
-
-      throw error;
-    }
-  }
-}
-
-export default ProducerCloseManager;

package/src/engine/media/streamEffects/ProcessorsCache.ts

@@ -1,41 +0,0 @@
-import { injectable } from 'inversify';
-import { TrackLabel } from '../../../types/common';
-import { AudioTrackProcessor } from './audio/AudioTrackProcessor';
-
-export type ProcessorType = 'asdk' | 'rnnoise';
-
-@injectable()
-export class ProcessorsCache {
-  readonly #audioProcessorsCache = new Map<string, AudioTrackProcessor>();
-
-  add(type: ProcessorType, label: TrackLabel, processor: AudioTrackProcessor): void {
-    const key = this.getKey(type, label);
-
-    if (this.#audioProcessorsCache.has(key)) {
-      throw new Error('Attempt to rewrite cached track processor');
-    }
-
-    this.#audioProcessorsCache.set(key, processor);
-  }
-
-  get(type: ProcessorType, label: TrackLabel): AudioTrackProcessor | undefined {
-    return this.#audioProcessorsCache.get(this.getKey(type, label));
-  }
-
-  has(type: ProcessorType, label: TrackLabel): boolean {
-    return this.#audioProcessorsCache.has(this.getKey(type, label));
-  }
-
-  private getKey(type: ProcessorType, label: TrackLabel): string {
-    return `${ type }:${ label }`;
-  }
-
-  reset(): void {
-    for (const processor of this.#audioProcessorsCache.values()) {
-      processor.stopProcessing();
-      processor.release();
-    }
-
-    this.#audioProcessorsCache.clear();
-  }
-}

package/src/engine/media/streamEffects/audio/AudioTrackProcessor.ts

@@ -1,7 +0,0 @@
-export interface AudioTrackProcessor {
-  readonly type: 'asdk' | 'rnnoise';
-  processTrack: (track: MediaStreamTrack) => Promise<MediaStreamTrack>;
-  stopProcessing: () => void;
-  get isProcessing(): boolean;
-  release: () => void;
-}

package/src/engine/media/streamEffects/audio/asdk/ASDKTrackProcessor.ts

@@ -1,275 +0,0 @@
-import { serializeError } from 'serialize-error';
-import type { atsvb } from 'audio-effects-sdk';
-import type { LoggerFactory } from '../../../../../types/container';
-import type Logger from '../../../../Logger';
-import type { AudioTrackProcessor } from '../AudioTrackProcessor';
-import { TEN_SECONDS_IN_MS, DEBOUNCE_LIMIT, ASDK_VERSION } from '../../../../../constants/common';
-import type ConfigService from '../../../../../config/ConfigService';
-import { URL } from 'url';
-
-enum ErrorType {
-  INFO = 'info',
-  WARNING = 'warning',
-  ERROR = 'error'
-}
-enum ErrorCode {
-  PERFORMANCE_STOP = 1001,
-  REDUCE_LATENCY = 1002,
-}
-enum ErrorEmitter {
-  ATSVB = 'atsvb',
-  STREAM_PROCESSOR = 'stream_processor',
-  ML_INFERENCE = 'ml_inference',
-  MODEL = 'model',
-  WORKLET = 'worklet',
-  WORKER = 'worker',
-  MODEL_LOADER = 'model_loader',
-}
-
-interface ErrorObject {
-  message: string;
-  type: ErrorType;
-  code?: ErrorCode;
-  emitter?: ErrorEmitter;
-  cause?: Error;
-  data?: unknown;
-}
-
-declare global {
-  interface Window {
-    atsvb: typeof atsvb;
-  }
-}
-
-const noop = (): void => { /* no operation */ };
-
-export interface AudioTrackProcessorParams {
-  asdkCustomerId: string;
-  config: ConfigService;
-  loggerFactory: LoggerFactory;
-  onModelsLoaded?: () => void;
-  onError?: () => void;
-}
-
-export class ASDKTrackProcessor implements AudioTrackProcessor {
-  readonly #atsvb: atsvb;
-
-  readonly #logger: Logger;
-
-  #originalTrack: MediaStreamTrack | null = null;
-
-  readonly #debounceCounters = new Map<number, number>();
-
-  #track?: MediaStreamTrack;
-
-  readonly type = 'asdk';
-
-  #onError: (reason: string, error: Error | undefined) => void;
-
-  constructor(params: AudioTrackProcessorParams) {
-    const { loggerFactory } = params;
-    this.#logger = loggerFactory('ASDKTrackProcessor');
-    this.#onError = typeof params.onError === 'function' ? params.onError : noop;
-
-    this.#atsvb = new window.atsvb(params.asdkCustomerId);
-
-    const staticFilesPath = params.config.get('staticFilesPath');
-    const {
-      // eslint-disable-next-line @typescript-eslint/naming-convention
-      localDir, version, sdk_url, api_url,
-    } = params.config.get('asdk');
-
-    let pathToWasm = `https://effectssdk.ai/sdk/audio/dev/${ version ?? ASDK_VERSION }`;
-
-    if (localDir) {
-      const url = new URL(`${ localDir }/${ version ?? ASDK_VERSION }`, staticFilesPath);
-      pathToWasm = url.href;
-    }
-
-    const asdkParams: Record<string, unknown> = {
-      preset: 'balanced',
-      sample_rate: 48_000,
-      api_url,
-      wasmPaths: {
-        'ort-wasm.wasm': `${ pathToWasm }/ort-wasm.wasm`,
-        'ort-wasm-simd.wasm': `${ pathToWasm }/ort-wasm-simd.wasm`,
-      },
-    };
-
-    const shouldLoadFromLocal = !sdk_url && localDir;
-    const sdkUrl = shouldLoadFromLocal ? `${ pathToWasm }/` : sdk_url;
-
-    if (shouldLoadFromLocal) {
-      this.#logger.warn('Using static ASDK path due to missing config', {
-        case: 'Denoiser',
-        type: 'asdk',
-        ...params.config.get('asdk'),
-        sdkUrl: asdkParams.sdk_url,
-      });
-    }
-
-    if (sdkUrl) {
-      asdkParams.sdk_url = sdkUrl;
-    }
-
-    this.#atsvb.config(asdkParams);
-
-    this.#atsvb.onError(this.onError.bind(this));
-  }
-
-  setOnError(onError: (reason: string, error: Error | undefined) => void): void {
-    this.#onError = onError;
-  }
-
-  async init(): Promise<void> {
-    await this.#atsvb.preload();
-  }
-
-  release(): void {
-    this.#atsvb.stop();
-    this.#atsvb.clear();
-  }
-
-  get isProcessing(): boolean {
-    return !!this.#track;
-  }
-
-  async processTrack(track: MediaStreamTrack): Promise<MediaStreamTrack> {
-    if (this.#originalTrack === track || track.readyState === 'ended') {
-      return this.#atsvb.getStream()?.getAudioTracks()[0] || track;
-    }
-
-    this.stopProcessing();
-
-    try {
-      const outTrackPromise = new Promise<MediaStreamTrack>((resolve, reject) => {
-        this.#atsvb.onReady = () => {
-          try {
-            const newStream = this.#atsvb.getStream();
-            const outTrack = newStream?.getAudioTracks()[0];
-
-            if (!outTrack) {
-              resolve(track);
-
-              return;
-            }
-
-            this.#originalTrack = track;
-            this.#track = outTrack;
-
-            resolve(outTrack);
-            this.#atsvb.run();
-          } catch (error) {
-            reject(error as Error);
-          } finally {
-            this.#atsvb.onReady = undefined;
-          }
-        };
-      });
-
-      this.#atsvb.useStream(new MediaStream([track]));
-
-      const outTrack = await outTrackPromise;
-
-      this.#logger.info('processTrack()', {
-        case: 'Denoiser',
-        type: this.type,
-        track: track.id,
-      });
-
-      return outTrack;
-    } catch (error) {
-      this.#logger.error('processTrack()', {
-        error: serializeError(error),
-        type: this.type,
-        case: 'Denoiser',
-        track,
-      });
-
-      this.#onError('ASDK processTrack() error', error as Error);
-      this.#onError = noop;
-
-      throw new Error('Can not process this track');
-    }
-  }
-
-  stopProcessing(): void {
-    try {
-      this.#atsvb.stop();
-    } catch (error) {
-      this.#logger.error('stopProcessing()', {
-        type: this.type,
-        case: 'Denoiser',
-        error: serializeError(error),
-      });
-    } finally {
-      this.#originalTrack = null;
-    }
-  }
-
-  getUpdatedDebounceCounter(errorCode: number): number {
-    let value = this.#debounceCounters.get(errorCode) ?? 0;
-
-    value += 1;
-    this.#debounceCounters.set(errorCode, value);
-
-    setTimeout(() => {
-      const currentValue = this.#debounceCounters.get(errorCode) ?? 0;
-
-      if (currentValue <= 0) {
-        this.#debounceCounters.set(errorCode, 0);
-
-        return;
-      }
-
-      this.#debounceCounters.set(errorCode, currentValue - 1);
-    }, TEN_SECONDS_IN_MS);
-
-    return value;
-  }
-
-  onError(error: ErrorObject): void {
-    const {
-      message, type, cause, ...rest
-    } = error;
-
-    if (type === ErrorType.ERROR) {
-      this.#logger.error(message, {
-        error: serializeError(cause),
-        cause,
-        ...rest,
-        type: this.type,
-        case: 'Denoiser',
-      });
-
-      if (rest.code === ErrorCode.REDUCE_LATENCY || rest.code === ErrorCode.PERFORMANCE_STOP) {
-        const debounceCounter = this.getUpdatedDebounceCounter(rest.code);
-
-        if (debounceCounter > DEBOUNCE_LIMIT) {
-          this.#debounceCounters.set(rest.code, 0);
-          this.#onError(`ASDK performance issue(${ rest.code }): ${ message }`, cause);
-          this.#onError = noop;
-        }
-      } else {
-        this.#onError(`ASDK runtime error: ${ message }`, cause);
-        this.#onError = noop;
-      }
-    }
-
-    if (type === ErrorType.WARNING) {
-      this.#logger.warn(message, {
-        ...rest,
-        type: this.type,
-        case: 'Denoiser',
-      });
-    }
-
-    if (type === ErrorType.INFO) {
-      this.#logger.info(message, {
-        ...rest,
-        type: this.type,
-        case: 'Denoiser',
-      });
-    }
-  }
-}

package/src/engine/media/streamEffects/audio/asdk/atsrb.d.ts

@@ -1,36 +0,0 @@
-import type { ErrorObject } from './errorBus';
-
-export interface Config {
-  preset: 'speed' | 'balanced' | 'quality',
-  sampleRate: 16000 | 32000 | 44100 | 48000,
-}
-
-export declare class atsvb {
-  private readonly customer_id: string;
-
-  private readonly streamProcessor: any;
-
-  private readonly _isRunning: boolean;
-
-  private readonly _currentInitialization: Promise<void> | null;
-
-  clear(): void;
-  config(config: Config): void;
-  constructor(customer_id: string);
-  getAudioTrack(): MediaStreamTrack[] | undefined;
-  getCustomerId(): string;
-  getStream(): MediaStream | undefined;
-  init(stream: MediaStream): Promise<void>;
-  initImpl(stream: MediaStream, prevPromise: Promise<void>): Promise<void>;
-  onError(f: (e: ErrorObject) => void): void;
-  onReady?: () => void;
-  preload(): Promise<void>;
-  run(): void;
-  setDenoisePower(power: number): void;
-  setDroppedFramesThreshold(threshold: number): void;
-  setPreset(preset: Config['preset'], sampleRate: Config['sampleRate']): Promise<void>;
-  setPresetImpl(preset: Config['preset'], sampleRate: Config['sampleRate'], prevPromise: Promise<void>): Promise<void>;
-  stop(): void;
-  useAudioTrack(track: MediaStreamTrack): void;
-  useStream(stream: MediaStream): void;
-}

package/src/engine/media/streamEffects/audio/asdk/errorBus.d.ts

@@ -1,47 +0,0 @@
-export type CallbackFunction = (n: ErrorObject) => void;
-
-export declare enum ErrorType {
-    INFO = "info",
-    WARNING = "warning",
-    ERROR = "error"
-}
-declare class _ErrorBus {
-  private static readonly _instance;
-
-  static getInstance(): _ErrorBus;
-  private readonly onErrorFunction?;
-  private constructor();
-  subscribe(f: CallbackFunction): void;
-  notify(n: ErrorObjectLight): void;
-}
-export declare const ErrorBus: _ErrorBus;
-export interface ErrorObject {
-  message: string;
-  type: ErrorType;
-  emitter?: ErrorEmitter;
-  cause?: Error;
-  data?: any;
-}
-type ErrorObjectLight = Omit<ErrorObject, 'type'> & {
-  type?: ErrorType;
-};
-export declare enum ErrorType {
-  INFO = 'info',
-  WARNING = 'warning',
-  ERROR = 'error',
-}
-export declare enum ErrorEmitter {
-  TSVB = 'tsvb',
-  COMPONENTS_SYSTEM = 'components_system',
-  SRTEAM_PROCESSOR = 'stream_processor',
-  ML_INFERENCE = 'ml_inference',
-  PRESET_INIT = 'preset_init',
-  RENDERER = 'renderer',
-  RECORDER = 'recorder',
-  EFFECT_VIRTUAL_BACKGROUND = 'effect_virtual_background',
-  EFFECT_COLOR_CORRECTION = 'effect_color_correction',
-  EFFECT_COLOR_FILTER = 'effect_color_filter',
-  EFFECT_SMART_ZOOM = 'effect_smart_zoom',
-  EFFECT_LOW_LIGHT = 'effect_low_light',
-}
-export {};

package/src/engine/media/streamEffects/audio/noiseSuppression/NoiseSuppressionAudioWorkletNode.ts

@@ -1,35 +0,0 @@
-import type ConfigService from '../../../../../config/ConfigService';
-import { container } from '../../../../../inversify.config';
-import { TOKEN } from '../../../../../inversify.tokens';
-import type { NoiseSuppressorParams } from '../../../../../types/streamEffects';
-
-export default class NoiseSuppressionAudioWorkletNode extends AudioWorkletNode {
-  constructor(params: NoiseSuppressorParams) {
-    super(params.context, 'rnnoise', {
-      channelCountMode: 'explicit',
-      channelCount: 1,
-      channelInterpretation: 'speakers',
-      numberOfInputs: 1,
-      numberOfOutputs: 1,
-      outputChannelCount: [1],
-      processorOptions: {
-        module: params.noiseSuppressor.module,
-      },
-    });
-
-    this.port.onmessage = ({ data }) => {
-      const e: Event = Object.assign(new Event('status'), data);
-      this.dispatchEvent(e);
-    };
-  }
-
-  update(keepalive: boolean): void {
-    this.port.postMessage(keepalive);
-  }
-
-  static async register(context: AudioContext): Promise<void> {
-    // TODO: refactor after TT-187 merge (audio effects)
-    const config = container.get<ConfigService>(TOKEN.Config);
-    await context.audioWorklet.addModule(`${ config.get('staticFilesPath') }/noiseSuppressionWorklet.js`);
-  }
-}

package/src/engine/media/streamEffects/audio/noiseSuppression/NoiseSuppressionNodeProvider.ts

@@ -1,44 +0,0 @@
-import type { NoiseSuppressionNode, NoiseSuppressorParams } from '../../../../../types/streamEffects';
-import NoiseSuppressionAudioWorkletNode from './NoiseSuppressionAudioWorkletNode';
-import NoiseSuppressionScriptProcessor from './NoiseSuppressionScriptProcessorNode';
-import { isSafari } from '../../../../../helpers/browser';
-
-export default class NoiseSuppressionNodeProvider {
-  readonly #context: AudioContext;
-
-  readonly #noiseSuppressor: WebAssembly.WebAssemblyInstantiatedSource;
-
-  #noiseSuppressionNode?: NoiseSuppressionAudioWorkletNode | NoiseSuppressionScriptProcessor;
-
-  constructor({ context, noiseSuppressor }: NoiseSuppressorParams) {
-    this.#context = context;
-    this.#noiseSuppressor = noiseSuppressor;
-  }
-
-  async createNoiseSuppressionNode(): Promise<NoiseSuppressionNode> {
-    return isSafari() ? this.createScriptProcessorNode() : await this.createAudioWorkletNode();
-  }
-
-  stop(): void {
-    this.#noiseSuppressionNode?.update(false);
-  }
-
-  private async createAudioWorkletNode(): Promise<NoiseSuppressionAudioWorkletNode> {
-    await NoiseSuppressionAudioWorkletNode.register(this.#context);
-    this.#noiseSuppressionNode = new NoiseSuppressionAudioWorkletNode({
-      context: this.#context,
-      noiseSuppressor: this.#noiseSuppressor,
-    });
-
-    return this.#noiseSuppressionNode;
-  }
-
-  private createScriptProcessorNode(): ScriptProcessorNode {
-    this.#noiseSuppressionNode = new NoiseSuppressionScriptProcessor({
-      context: this.#context,
-      noiseSuppressor: this.#noiseSuppressor,
-    });
-
-    return this.#noiseSuppressionNode.noiseSuppressionScriptProcessorNode;
-  }
-}

package/src/engine/media/streamEffects/audio/noiseSuppression/NoiseSuppressionScriptProcessorNode.ts

@@ -1,69 +0,0 @@
-import type { NoiseSuppressorParams, RNNoiseInstance } from '../../../../../types/streamEffects';
-
-const BYTES_PER_FLOAT32 = 4;
-
-export default class NoiseSuppressionScriptProcessor {
-  readonly #instance: RNNoiseInstance;
-
-  readonly #context: AudioContext;
-
-  readonly #circularBuffer: Float32Array;
-
-  readonly #state: number;
-
-  readonly #processor: ScriptProcessorNode;
-
-  #alive = true;
-
-  constructor(params: NoiseSuppressorParams) {
-    this.#instance = params.noiseSuppressor.instance.exports as unknown as RNNoiseInstance;
-    this.#context = params.context;
-    this.#circularBuffer = new Float32Array(this.#instance.memory.buffer);
-    this.#state = this.#instance.newState();
-    this.#processor = this.#context.createScriptProcessor();
-    this.setOnAudioProcess();
-  }
-
-  get noiseSuppressionScriptProcessorNode(): ScriptProcessorNode {
-    return this.#processor;
-  }
-
-  update(keepalive: boolean): void {
-    if (!keepalive) {
-      this.#alive = false;
-      this.#instance.deleteState(this.#state);
-    }
-
-    const e = Object.assign(new Event('status'), { vadProb: this.#instance.getVadProb(this.#state) });
-    this.#processor.dispatchEvent(e);
-  }
-
-  private setOnAudioProcess(): void {
-    this.#processor.addEventListener('audioprocess', ({ inputBuffer, outputBuffer }) => {
-      if (!this.#alive) {
-        return;
-      }
-
-      /*
-        - В WebAssembly все адреса считаются в байтах.
-        - В JavaScript для работы с тем же куском памяти мы используем Float32Array, где элемент занимает 4 байта.
-        - Чтобы из «адреса_в_байтах» → «индекса_в_Float32Array», нужно byteOffset / 4.
-        - Аналогично при работе с Int16Array делили бы на 2, с Float64Array — на 8.
-        - RNNoise обрабатывает по 10 мс (480 сэмплов при 48 кГц). Если ваш ScriptProcessorNode настроен на 128 сэмплов,
-        то только каждое четвёртое срабатывание реально отдаст результат — именно поэтому проверка
-        if (pointer) встречается почти на каждом вызове.
-        - getVadProb() даёт оценку «есть ли речь», её можно использовать, например, чтобы зажигать индикатор говорящего.
-        Так что вся «магия» в двух местах деления на 4 — это простое соответствие между байтовыми адресами WASM и
-        индексами Float32Array, а pointer — это адрес начала готового,
-        уже очищенного от шума блока в том же кольцевом буфере.
-      */
-      this.#circularBuffer.set(inputBuffer.getChannelData(0), this.#instance.getInput(this.#state) / BYTES_PER_FLOAT32);
-      const output = outputBuffer.getChannelData(0);
-      const pointer = this.#instance.pipe(this.#state, output.length) / BYTES_PER_FLOAT32;
-
-      if (pointer) {
-        output.set(this.#circularBuffer.subarray(pointer, pointer + output.length));
-      }
-    });
-  }
-}