@livedigital/client 2.39.0 → 2.40.0-video-effects.2

package/README.md

@@ -17,6 +17,14 @@ import Client from '@livedigital/client';
 const client = new Client();
 ```
 
+## Plugins
+Now you can use the noise reduction and virtual backgrounds functionality. To do this, copy all the files from the
+/files directory to the root directory of your project, and add tflite loader scripts to your index.html
+```
+<script src="%PUBLIC_URL%/tflite/tflite.js"></script>
+<script src="%PUBLIC_URL%/tflite/tflite-simd.js"></script>
+```
+
 ## Debugging
 ### Log Levels
 LiveDigital client uses the [debug](https://www.npmjs.com/package/debug) module to log to the browser console.

package/dist/engine/index.d.ts

@@ -50,9 +50,10 @@ declare class Engine {
     createCameraVideoTrack(options?: CreateCameraVideoTrackOptions): Promise<Track>;
     createMicrophoneAudioTrack(options?: CreateMicrophoneAudioTrackOptions): Promise<Track>;
     createScreenMediaTracks(options?: CreateScreenMediaOptions): Promise<Track[]>;
-    deleteTrack(tracks: Track): void;
+    deleteTrack(tracks: Track): Promise<void>;
     get channelId(): string | undefined;
     get appId(): string | undefined;
+    get canUseNoiseSuppression(): boolean;
     private connectToSocketServerWithRetry;
     private connectToSocketServer;
     private getAvailableNode;

package/dist/engine/media/index.d.ts

@@ -9,12 +9,14 @@ declare class Media {
     private tracks;
     constructor(params: CreateMediaParams);
     get mediasoupDevice(): Device;
+    get isNoiseSuppressorLoaded(): boolean;
     loadDevice(routerRtpCapabilities: RtpCapabilities): Promise<void>;
+    createNoiseSuppressor(): Promise<void>;
     getTrackCodec(track: Track): RtpCodecCapability | undefined;
     private createTracks;
     createUserMediaTracks(constraints: MediaStreamConstraints): Promise<Track[]>;
     createDisplayMediaTracks(constraints: MediaStreamConstraints): Promise<Track[]>;
-    deleteTrack(track: Track): void;
+    deleteTrack(track: Track): Promise<void>;
     getAllTracks(): Track[];
     clearTracks(): Promise<void>;
     static getVideoTrackTransformParams(track: MediaStreamTrack): TransformParams;

package/dist/engine/media/streamEffects/audio/noiseSuppression/NoiseSuppressionAudioWorkletNode.d.ts

@@ -0,0 +1,6 @@
+import { NoiseSuppressorParams } from '../../../../../types/streamEffects';
+export default class NoiseSuppressionAudioWorkletNode extends AudioWorkletNode {
+    constructor(params: NoiseSuppressorParams);
+    update(keepalive: boolean): void;
+    static register(context: AudioContext): Promise<void>;
+}

package/dist/engine/media/streamEffects/audio/noiseSuppression/NoiseSuppressionNodeProvider.d.ts

@@ -0,0 +1,9 @@
+import { NoiseSuppressionNode, NoiseSuppressorParams } from '../../../../../types/streamEffects';
+export default class NoiseSuppressionNodeProvider {
+    #private;
+    constructor({ context, noiseSuppressor }: NoiseSuppressorParams);
+    createNoiseSuppressionNode(): Promise<NoiseSuppressionNode>;
+    stop(): void;
+    private createAudioWorkletNode;
+    private createScriptProcessorNode;
+}

package/dist/engine/media/streamEffects/audio/noiseSuppression/NoiseSuppressionScriptProcessorNode.d.ts

@@ -0,0 +1,8 @@
+import { NoiseSuppressorParams } from '../../../../../types/streamEffects';
+export default class NoiseSuppressionScriptProcessor {
+    #private;
+    constructor(params: NoiseSuppressorParams);
+    get noiseSuppressionScriptProcessorNode(): ScriptProcessorNode;
+    update(keepalive: boolean): void;
+    private setOnAudioProcess;
+}

package/dist/engine/media/streamEffects/audio/noiseSuppression/TrackProcessor.d.ts

@@ -0,0 +1,13 @@
+import { LogLevel } from '../../../../../types/common';
+interface TrackProcessorParams {
+    noiseSuppressor: WebAssembly.WebAssemblyInstantiatedSource;
+    logLevel: LogLevel;
+}
+export default class TrackProcessor {
+    #private;
+    constructor(params: TrackProcessorParams);
+    get isProcessing(): boolean;
+    processTrack(track: MediaStreamTrack): Promise<MediaStreamTrack>;
+    stopProcessing(): Promise<void>;
+}
+export {};

package/dist/engine/media/streamEffects/audio/noiseSuppression/WasmModuleCompiler.d.ts

@@ -0,0 +1,3 @@
+export default class WasmModuleCompiler {
+    static compileModule(binaryFilePath: string): Promise<WebAssembly.WebAssemblyInstantiatedSource>;
+}

package/dist/engine/media/streamEffects/video/virtual-background/TensorFlowLoader.d.ts

@@ -0,0 +1,13 @@
+import { LogLevel } from '../../../../../types/common';
+import { SegmentationConfig, TFLite } from './types';
+export default class TensorFlowLoader {
+    #private;
+    constructor(logLevel: LogLevel);
+    get isLoaded(): boolean;
+    get tfliteModule(): TFLite | undefined;
+    loadTFLite(): Promise<void>;
+    loadTFLiteModel(segmentationConfig: SegmentationConfig): Promise<void>;
+    private loadTFLiteRegular;
+    private loadTFLiteSIMD;
+    private static getTFLiteModelFileName;
+}

package/dist/engine/media/streamEffects/video/virtual-background/TrackProcessor.d.ts

@@ -0,0 +1,16 @@
+import { GetProcessedTrackParams } from './types';
+import { LogLevel } from '../../../../../types/common';
+interface VideoTrackProcessorParams {
+    logLevel: LogLevel;
+}
+export default class TrackProcessor {
+    #private;
+    constructor(params: VideoTrackProcessorParams);
+    get isProcessing(): boolean;
+    processTrack(params: GetProcessedTrackParams): Promise<MediaStreamTrack>;
+    stopProcessing(): void;
+    private static buildWebGL2Pipeline;
+    private static buildCanvas2dPipeline;
+    private static buildSourcePlayback;
+}
+export {};

package/dist/engine/media/streamEffects/video/virtual-background/constants.d.ts

@@ -0,0 +1,7 @@
+import { InputResolution, PipeLineConfig } from './types';
+export declare const INPUT_RESOLUTIONS: {
+    [resolution in InputResolution]: [number, number];
+};
+export declare const BLUR_PIPELINE_CONFIG: PipeLineConfig;
+export declare const BLUR_IN_BACKGROUND_TASK_PIPELINE_CONFIG: PipeLineConfig;
+export declare const SAFARI_BLUR_PIPELINE_CONFIG: PipeLineConfig;

package/dist/engine/media/streamEffects/video/virtual-background/helpers/buildTimerWorker.d.ts

@@ -0,0 +1,5 @@
+export default function buildTimerWorker(): {
+    setTimeout: (callback: () => void, timeoutMs: number) => number;
+    clearTimeout: (callbackId: number) => void;
+    terminate: () => void;
+};

package/dist/engine/media/streamEffects/video/virtual-background/pipelines/canvas2d/canvas2dPipeline.d.ts

@@ -0,0 +1,13 @@
+import { BackgroundConfig, SourcePlayback, PostProcessingConfig, SegmentationConfig, TFLite } from '../../types';
+export interface BuildCanvas2dPipeLineParams {
+    sourcePlayback: SourcePlayback;
+    backgroundConfig: BackgroundConfig;
+    segmentationConfig: SegmentationConfig;
+    canvas: HTMLCanvasElement;
+    tflite: TFLite;
+}
+export default function buildCanvas2dPipeline({ sourcePlayback, backgroundConfig, segmentationConfig, canvas, tflite, }: BuildCanvas2dPipeLineParams): {
+    render: () => Promise<void>;
+    updatePostProcessingConfig: (newPostProcessingConfig: PostProcessingConfig) => void;
+    cleanUp: () => void;
+};

package/dist/engine/media/streamEffects/video/virtual-background/pipelines/helpers/webglHelper.d.ts

@@ -0,0 +1,8 @@
+export declare const glsl: (template: {
+    raw: readonly string[] | ArrayLike<string>;
+}, ...substitutions: any[]) => string;
+export declare function createProgram(gl: WebGL2RenderingContext, vertexShader: WebGLShader, fragmentShader: WebGLShader): WebGLProgram;
+export declare function compileShader(gl: WebGL2RenderingContext, shaderType: number, shaderSource: string): WebGLShader;
+export declare function createTexture(gl: WebGL2RenderingContext, internalformat: number, width: number, height: number, minFilter?: number, magFilter?: number): WebGLTexture | null;
+export declare function readPixelsAsync(gl: WebGL2RenderingContext, x: number, y: number, width: number, height: number, format: number, type: number, dest: ArrayBufferView): Promise<ArrayBufferView>;
+export declare function createPiplelineStageProgram(gl: WebGL2RenderingContext, vertexShader: WebGLShader, fragmentShader: WebGLShader, positionBuffer: WebGLBuffer, texCoordBuffer: WebGLBuffer): WebGLProgram;

package/dist/engine/media/streamEffects/video/virtual-background/pipelines/webgl2/backgroundBlurStage.d.ts

@@ -0,0 +1,6 @@
+export declare type BackgroundBlurStage = {
+    render(): void;
+    updateCoverage(coverage: [number, number]): void;
+    cleanUp(): void;
+};
+export declare function buildBackgroundBlurStage(gl: WebGL2RenderingContext, vertexShader: WebGLShader, positionBuffer: WebGLBuffer, texCoordBuffer: WebGLBuffer, personMaskTexture: WebGLTexture, canvas: HTMLCanvasElement): BackgroundBlurStage;

package/dist/engine/media/streamEffects/video/virtual-background/pipelines/webgl2/backgroundImageStage.d.ts

@@ -0,0 +1,9 @@
+import { BlendMode } from '../../types';
+export declare type BackgroundImageStage = {
+    render(): void;
+    updateCoverage(coverage: [number, number]): void;
+    updateLightWrapping(lightWrapping: number): void;
+    updateBlendMode(blendMode: BlendMode): void;
+    cleanUp(): void;
+};
+export declare function buildBackgroundImageStage(gl: WebGL2RenderingContext, positionBuffer: WebGLBuffer, texCoordBuffer: WebGLBuffer, personMaskTexture: WebGLTexture, backgroundImage: HTMLImageElement | null, canvas: HTMLCanvasElement): BackgroundImageStage;

package/dist/engine/media/streamEffects/video/virtual-background/pipelines/webgl2/buildJointBilateralFilterStage.d.ts

@@ -0,0 +1,7 @@
+import { SegmentationConfig } from '../../types';
+export default function buildJointBilateralFilterStage(gl: WebGL2RenderingContext, vertexShader: WebGLShader, positionBuffer: WebGLBuffer, texCoordBuffer: WebGLBuffer, inputTexture: WebGLTexture, segmentationConfig: SegmentationConfig, outputTexture: WebGLTexture, canvas: HTMLCanvasElement): {
+    render: () => void;
+    updateSigmaSpace: (sigmaSpace: number) => void;
+    updateSigmaColor: (sigmaColor: number) => void;
+    cleanUp: () => void;
+};

package/dist/engine/media/streamEffects/video/virtual-background/pipelines/webgl2/buildLoadSegmentationStage.d.ts

@@ -0,0 +1,5 @@
+import { SegmentationConfig, TFLite } from '../../types';
+export default function buildLoadSegmentationStage(gl: WebGL2RenderingContext, vertexShader: WebGLShader, positionBuffer: WebGLBuffer, texCoordBuffer: WebGLBuffer, segmentationConfig: SegmentationConfig, tflite: TFLite, outputTexture: WebGLTexture): {
+    render: () => void;
+    cleanUp: () => void;
+};

package/dist/engine/media/streamEffects/video/virtual-background/pipelines/webgl2/buildResizingStage.d.ts

@@ -0,0 +1,5 @@
+import { SegmentationConfig, TFLite } from '../../types';
+export default function buildResizingStage(gl: WebGL2RenderingContext, vertexShader: WebGLShader, positionBuffer: WebGLBuffer, texCoordBuffer: WebGLBuffer, segmentationConfig: SegmentationConfig, tflite: TFLite): {
+    render: () => Promise<void>;
+    cleanUp: () => void;
+};

package/dist/engine/media/streamEffects/video/virtual-background/pipelines/webgl2/buildSoftmaxStage.d.ts

@@ -0,0 +1,5 @@
+import { SegmentationConfig, TFLite } from '../../types';
+export default function buildSoftmaxStage(gl: WebGL2RenderingContext, vertexShader: WebGLShader, positionBuffer: WebGLBuffer, texCoordBuffer: WebGLBuffer, segmentationConfig: SegmentationConfig, tflite: TFLite, outputTexture: WebGLTexture): {
+    render: () => void;
+    cleanUp: () => void;
+};

package/dist/engine/media/streamEffects/video/virtual-background/pipelines/webgl2/webgl2Pipeline.d.ts

@@ -0,0 +1,14 @@
+import { SourcePlayback, BackgroundConfig, PostProcessingConfig, SegmentationConfig, TFLite } from '../../types';
+export interface BuildWebGL2PipeLineParams {
+    sourcePlayback: SourcePlayback;
+    backgroundImage: HTMLImageElement | null;
+    backgroundConfig: BackgroundConfig;
+    segmentationConfig: SegmentationConfig;
+    canvas: HTMLCanvasElement;
+    tflite: TFLite;
+}
+export default function buildWebGL2Pipeline({ sourcePlayback, backgroundImage, backgroundConfig, segmentationConfig, canvas, tflite, }: BuildWebGL2PipeLineParams): {
+    render: () => Promise<void>;
+    updatePostProcessingConfig: (postProcessingConfig: PostProcessingConfig) => void;
+    cleanUp: () => void;
+};

package/dist/engine/media/streamEffects/video/virtual-background/types.d.ts

@@ -0,0 +1,81 @@
+/// <reference types="emscripten" />
+export interface TFLite extends EmscriptenModule {
+    _getModelBufferMemoryOffset(): number;
+    _getInputMemoryOffset(): number;
+    _getInputHeight(): number;
+    _getInputWidth(): number;
+    _getInputChannelCount(): number;
+    _getOutputMemoryOffset(): number;
+    _getOutputHeight(): number;
+    _getOutputWidth(): number;
+    _getOutputChannelCount(): number;
+    _loadModel(bufferSize: number): number;
+    _runInference(): number;
+}
+export declare type BlendMode = 'screen' | 'linearDodge';
+export interface TimerWorker {
+    setTimeout: (callback: () => void, timeoutMs: number) => number;
+    clearTimeout: (callbackId: number) => void;
+    terminate: () => void;
+}
+export declare type PostProcessingConfig = {
+    smoothSegmentationMask: boolean;
+    jointBilateralFilter: JointBilateralFilterConfig;
+    coverage: [number, number];
+    lightWrapping: number;
+    blendMode: BlendMode;
+};
+export declare type JointBilateralFilterConfig = {
+    sigmaSpace: number;
+    sigmaColor: number;
+};
+export declare type SourcePlayback = {
+    htmlElement: HTMLImageElement | HTMLVideoElement;
+    width: number;
+    height: number;
+};
+export declare type BackgroundConfig = {
+    type: 'none' | 'blur' | 'image';
+    url?: string;
+    imageSource: HTMLImageElement | null;
+};
+export declare type PipelineType = 'canvas2dCpu' | 'webgl2';
+export declare type SegmentationModel = 'meet' | 'mlkit';
+export declare type SegmentationBackend = 'webgl' | 'wasm' | 'wasmSimd';
+export declare type InputResolution = '640x360' | '256x256' | '256x144' | '160x96';
+export declare type SegmentationConfig = {
+    model: SegmentationModel;
+    backend: SegmentationBackend;
+    inputResolution: InputResolution;
+    pipeline: PipelineType;
+    targetFps: number;
+    deferInputResizing: boolean;
+};
+export interface PipeLine {
+    render: () => Promise<void>;
+    updatePostProcessingConfig: (postProcessingConfig: PostProcessingConfig) => void;
+    cleanUp: () => void;
+}
+export interface PipeLineConfig {
+    backgroundConfig: BackgroundConfig;
+    segmentationConfig: SegmentationConfig;
+    postprocessingConfig: PostProcessingConfig;
+}
+export interface BuildPipeLineConfig {
+    config: PipeLineConfig;
+    sourcePlayback: SourcePlayback;
+    destinationPlayback: HTMLCanvasElement;
+    tflite: TFLite;
+}
+export interface TrackProcessingContext {
+    pipeline: PipeLine;
+    source: HTMLVideoElement | HTMLImageElement;
+    destination: HTMLCanvasElement;
+    track: MediaStreamTrack;
+    timerWorker: TimerWorker;
+    timerId: number;
+}
+export declare type GetProcessedTrackParams = {
+    track: MediaStreamTrack;
+    pipelineConfig: PipeLineConfig;
+};

package/dist/engine/media/tracks/AudioTrack.d.ts

@@ -1,13 +1,20 @@
 import { ProducerCodecOptions } from 'mediasoup-client/lib/Producer';
 import { RtpEncodingParameters } from 'mediasoup-client/lib/RtpParameters';
-import { AudioCodec, AudioEncoderConfig } from '../../../types/common';
-import BaseTrack from './BaseTrack';
-import TrackWithCodecOptions from './TrackWithCodecOptions';
-declare class AudioTrack extends BaseTrack implements TrackWithCodecOptions {
+import { AudioCodec, AudioEncoderConfig, TrackWithEncodings } from '../../../types/common';
+import BaseTrack, { BaseTrackConstructorParams } from './BaseTrack';
+interface AudioTrackParams extends BaseTrackConstructorParams {
+    noiseSuppressor?: WebAssembly.WebAssemblyInstantiatedSource;
+}
+declare class AudioTrack extends BaseTrack implements TrackWithEncodings {
+    #private;
+    constructor(params: AudioTrackParams);
+    get isEffectsProcessing(): boolean;
     getEncoderConfig(): AudioEncoderConfig;
     getEncodings(): RtpEncodingParameters[];
     getCodecOptions(): ProducerCodecOptions;
     getPreferredCodec(): AudioCodec;
+    enableNoiseSuppression(): Promise<void>;
+    disableNoiseSuppression(): Promise<void>;
     publish(): Promise<void>;
 }
 export default AudioTrack;

package/dist/engine/media/tracks/BaseTrack.d.ts

@@ -51,6 +51,7 @@ declare class BaseTrack {
     resume(): Promise<void>;
     getInfo(): Promise<BaseTrackInfo>;
     getStats(): Promise<TrackOutboundStats | undefined>;
+    replaceTrack(track: MediaStreamTrack): Promise<void>;
     private resumeProducer;
     private reopenMediaStreamTrack;
     private getOutboundRTPStreamStats;

package/dist/engine/media/tracks/VideoTrack.d.ts

@@ -1,11 +1,12 @@
 import { ProducerCodecOptions } from 'mediasoup-client/lib/Producer';
 import { RtpEncodingParameters } from 'mediasoup-client/lib/RtpParameters';
-import { TrackPublishParams, VideoCodec, VideoEncoderConfig } from '../../../types/common';
-import BaseTrack from './BaseTrack';
-import TrackWithCodecOptions from './TrackWithCodecOptions';
-import TrackWithEncodings from './TrackWithEncodings';
-declare class VideoTrack extends BaseTrack implements TrackWithCodecOptions, TrackWithEncodings {
+import { TrackPublishParams, TrackWithEncodings, VideoCodec, VideoEncoderConfig } from '../../../types/common';
+import BaseTrack, { BaseTrackConstructorParams } from './BaseTrack';
+declare class VideoTrack extends BaseTrack implements TrackWithEncodings {
+    #private;
     transformParams: {};
+    constructor(params: BaseTrackConstructorParams);
+    get isEffectsProcessing(): boolean;
     getEncoderConfig(): VideoEncoderConfig;
     getCodecOptions(): ProducerCodecOptions;
     getPreferredCodec(): VideoCodec;
@@ -13,6 +14,10 @@ declare class VideoTrack extends BaseTrack implements TrackWithCodecOptions, Tra
     setTransformParams(transformParams: {}): void;
     setMaxSpatialLayer(spatialLayer: number): Promise<void>;
     getMaxSpatialLayer(): number | undefined;
+    enableBlur(): Promise<void>;
+    disableBlur(): Promise<void>;
     publish(params?: TrackPublishParams): Promise<void>;
+    private watchOrientation;
+    private watchDocumentVisibility;
 }
 export default VideoTrack;

package/dist/helpers/browser.d.ts

@@ -0,0 +1 @@
+export declare const isSafari: () => boolean;

package/dist/helpers/media.d.ts

@@ -1,2 +1,3 @@
 import { types } from 'mediasoup-client';
 export declare const detectHandlerName: (userAgent?: string | undefined) => types.BuiltinHandlerName | undefined;
+export declare const getNewAudioContext: (contextOptions?: AudioContextOptions | undefined) => AudioContext | null;

package/dist/index.d.ts

@@ -23,13 +23,14 @@ declare class Client {
     get peers(): Peer[];
     get availableVideoDevices(): MediaDeviceInfo[];
     get availableAudioDevices(): MediaDeviceInfo[];
+    get canUseNoiseSuppression(): boolean;
     detectDevices(): Promise<AvailableMediaDevices>;
     join(params: JoinChannelParams): Promise<void>;
     leave(): Promise<void>;
     createCameraVideoTrack(options?: CreateCameraVideoTrackOptions): Promise<Track>;
     createMicrophoneAudioTrack(options?: CreateMicrophoneAudioTrackOptions): Promise<Track>;
     createScreenMediaTracks(options?: CreateScreenMediaOptions): Promise<Track[]>;
-    deleteTrack(track: Track): void;
+    deleteTrack(track: Track): Promise<void>;
     loadPeers(role?: Role): Promise<void>;
     updateAppData(appData: Record<string, unknown>): Promise<void>;
     getTransportsInfo(): Promise<TransportsStateInfo>;