@livedigital/client 3.11.0 → 3.11.1

package/README.md

@@ -19,9 +19,10 @@ const client = new Client();
 
 ## Plugins
 Now you can use the noise reduction and virtual backgrounds functionality. To do this, copy all the files from the
-/files directory to the root directory of your project, and add tsvb-web.js scripts to your index.html
+/files directory to the root directory of your project, and add tflite loader scripts to your index.html
 ```
-<script src="%PUBLIC_URL%/esdk/2.7.4/tsvb-web.js"></script>
+<script src="%PUBLIC_URL%/tflite/tflite.js"></script>
+<script src="%PUBLIC_URL%/tflite/tflite-simd.js"></script>
 ```
 
 ## Debugging
@@ -30,17 +31,17 @@ LiveDigital client uses the [debug](https://www.npmjs.com/package/debug) module
 
 There are three log severities:
 
-- **debug**: verbose information logged to console.info
-- **warn**: warning messages logged to console.warn
-- **error**: error messages logged to console.error
+- **debug**: verbose information logged to console.info  
+- **warn**: warning messages logged to console.warn  
+- **error**: error messages logged to console.error  
 
 All the logs generated by LiveDigital client have a namespace starting with “LiveDigital”
 plus colon, followed by the log severity in upper case plus colon (just if “warn” or “error”),
-and followed by the internal component name (if any) and the log message.
+and followed by the internal component name (if any) and the log message.  
 
 ### Enable Logging
 By default, logging is turned off. In order to enable it, the debug key in the browser's
-localStorage must be set.
+localStorage must be set.  
 >Check the [debug](https://www.npmjs.com/package/debug) module documentation for further information regarding how to filter specific log messages based on namespace matching rules.
 
 ### Example

package/dist/engine/media/streamEffects/video/virtual-background/TensorFlowLoader.d.ts

@@ -0,0 +1,13 @@
+import { LogLevel } from '../../../../../types/common';
+import { SegmentationConfig, TFLite } from './types';
+export default class TensorFlowLoader {
+    #private;
+    constructor(logLevel: LogLevel);
+    get isLoaded(): boolean;
+    get tfliteModule(): TFLite | undefined;
+    loadTFLite(): Promise<void>;
+    loadTFLiteModel(segmentationConfig: SegmentationConfig): Promise<void>;
+    private loadTFLiteRegular;
+    private loadTFLiteSIMD;
+    private static getTFLiteModelFileName;
+}

package/dist/engine/media/streamEffects/video/virtual-background/TrackProcessor.d.ts

@@ -0,0 +1,16 @@
+import { GetProcessedTrackParams } from './types';
+import { LogLevel } from '../../../../../types/common';
+interface VideoTrackProcessorParams {
+    logLevel: LogLevel;
+}
+export default class TrackProcessor {
+    #private;
+    constructor(params: VideoTrackProcessorParams);
+    get isProcessing(): boolean;
+    processTrack(params: GetProcessedTrackParams): Promise<MediaStreamTrack>;
+    stopProcessing(): void;
+    private static buildWebGL2Pipeline;
+    private static buildCanvas2dPipeline;
+    private static buildSourcePlayback;
+}
+export {};

package/dist/engine/media/streamEffects/video/virtual-background/constants.d.ts

@@ -0,0 +1,7 @@
+import { InputResolution, PipeLineConfig } from './types';
+export declare const INPUT_RESOLUTIONS: {
+    [resolution in InputResolution]: [number, number];
+};
+export declare const BLUR_PIPELINE_CONFIG: PipeLineConfig;
+export declare const BLUR_IN_BACKGROUND_TASK_PIPELINE_CONFIG: PipeLineConfig;
+export declare const SAFARI_BLUR_PIPELINE_CONFIG: PipeLineConfig;

package/dist/engine/media/streamEffects/video/virtual-background/helpers/buildTimerWorker.d.ts

@@ -0,0 +1,5 @@
+export default function buildTimerWorker(): {
+    setTimeout: (callback: () => void, timeoutMs: number) => number;
+    clearTimeout: (callbackId: number) => void;
+    terminate: () => void;
+};

package/dist/engine/media/streamEffects/video/virtual-background/pipelines/canvas2d/canvas2dPipeline.d.ts

@@ -0,0 +1,13 @@
+import { BackgroundConfig, SourcePlayback, PostProcessingConfig, SegmentationConfig, TFLite } from '../../types';
+export interface BuildCanvas2dPipeLineParams {
+    sourcePlayback: SourcePlayback;
+    backgroundConfig: BackgroundConfig;
+    segmentationConfig: SegmentationConfig;
+    canvas: HTMLCanvasElement;
+    tflite: TFLite;
+}
+export default function buildCanvas2dPipeline({ sourcePlayback, backgroundConfig, segmentationConfig, canvas, tflite, }: BuildCanvas2dPipeLineParams): {
+    render: () => Promise<void>;
+    updatePostProcessingConfig: (newPostProcessingConfig: PostProcessingConfig) => void;
+    cleanUp: () => void;
+};

package/dist/engine/media/streamEffects/video/virtual-background/pipelines/helpers/webglHelper.d.ts

@@ -0,0 +1,8 @@
+export declare const glsl: (template: {
+    raw: readonly string[] | ArrayLike<string>;
+}, ...substitutions: any[]) => string;
+export declare function createProgram(gl: WebGL2RenderingContext, vertexShader: WebGLShader, fragmentShader: WebGLShader): WebGLProgram;
+export declare function compileShader(gl: WebGL2RenderingContext, shaderType: number, shaderSource: string): WebGLShader;
+export declare function createTexture(gl: WebGL2RenderingContext, internalformat: number, width: number, height: number, minFilter?: number, magFilter?: number): WebGLTexture | null;
+export declare function readPixelsAsync(gl: WebGL2RenderingContext, x: number, y: number, width: number, height: number, format: number, type: number, dest: ArrayBufferView): Promise<ArrayBufferView>;
+export declare function createPiplelineStageProgram(gl: WebGL2RenderingContext, vertexShader: WebGLShader, fragmentShader: WebGLShader, positionBuffer: WebGLBuffer, texCoordBuffer: WebGLBuffer): WebGLProgram;

package/dist/engine/media/streamEffects/video/virtual-background/pipelines/webgl2/backgroundBlurStage.d.ts

@@ -0,0 +1,6 @@
+export declare type BackgroundBlurStage = {
+    render(): void;
+    updateCoverage(coverage: [number, number]): void;
+    cleanUp(): void;
+};
+export declare function buildBackgroundBlurStage(gl: WebGL2RenderingContext, vertexShader: WebGLShader, positionBuffer: WebGLBuffer, texCoordBuffer: WebGLBuffer, personMaskTexture: WebGLTexture, canvas: HTMLCanvasElement): BackgroundBlurStage;

package/dist/engine/media/streamEffects/video/virtual-background/pipelines/webgl2/backgroundImageStage.d.ts

@@ -0,0 +1,9 @@
+import { BlendMode } from '../../types';
+export declare type BackgroundImageStage = {
+    render(): void;
+    updateCoverage(coverage: [number, number]): void;
+    updateLightWrapping(lightWrapping: number): void;
+    updateBlendMode(blendMode: BlendMode): void;
+    cleanUp(): void;
+};
+export declare function buildBackgroundImageStage(gl: WebGL2RenderingContext, positionBuffer: WebGLBuffer, texCoordBuffer: WebGLBuffer, personMaskTexture: WebGLTexture, backgroundImage: HTMLImageElement | null, canvas: HTMLCanvasElement): BackgroundImageStage;

package/dist/engine/media/streamEffects/video/virtual-background/pipelines/webgl2/buildJointBilateralFilterStage.d.ts

@@ -0,0 +1,7 @@
+import { SegmentationConfig } from '../../types';
+export default function buildJointBilateralFilterStage(gl: WebGL2RenderingContext, vertexShader: WebGLShader, positionBuffer: WebGLBuffer, texCoordBuffer: WebGLBuffer, inputTexture: WebGLTexture, segmentationConfig: SegmentationConfig, outputTexture: WebGLTexture, canvas: HTMLCanvasElement): {
+    render: () => void;
+    updateSigmaSpace: (sigmaSpace: number) => void;
+    updateSigmaColor: (sigmaColor: number) => void;
+    cleanUp: () => void;
+};

package/dist/engine/media/streamEffects/video/virtual-background/pipelines/webgl2/buildLoadSegmentationStage.d.ts

@@ -0,0 +1,5 @@
+import { SegmentationConfig, TFLite } from '../../types';
+export default function buildLoadSegmentationStage(gl: WebGL2RenderingContext, vertexShader: WebGLShader, positionBuffer: WebGLBuffer, texCoordBuffer: WebGLBuffer, segmentationConfig: SegmentationConfig, tflite: TFLite, outputTexture: WebGLTexture): {
+    render: () => void;
+    cleanUp: () => void;
+};

package/dist/engine/media/streamEffects/video/virtual-background/pipelines/webgl2/buildResizingStage.d.ts

@@ -0,0 +1,5 @@
+import { SegmentationConfig, TFLite } from '../../types';
+export default function buildResizingStage(gl: WebGL2RenderingContext, vertexShader: WebGLShader, positionBuffer: WebGLBuffer, texCoordBuffer: WebGLBuffer, segmentationConfig: SegmentationConfig, tflite: TFLite): {
+    render: () => Promise<void>;
+    cleanUp: () => void;
+};

package/dist/engine/media/streamEffects/video/virtual-background/pipelines/webgl2/buildSoftmaxStage.d.ts

@@ -0,0 +1,5 @@
+import { SegmentationConfig, TFLite } from '../../types';
+export default function buildSoftmaxStage(gl: WebGL2RenderingContext, vertexShader: WebGLShader, positionBuffer: WebGLBuffer, texCoordBuffer: WebGLBuffer, segmentationConfig: SegmentationConfig, tflite: TFLite, outputTexture: WebGLTexture): {
+    render: () => void;
+    cleanUp: () => void;
+};

package/dist/engine/media/streamEffects/video/virtual-background/pipelines/webgl2/webgl2Pipeline.d.ts

@@ -0,0 +1,14 @@
+import { SourcePlayback, BackgroundConfig, PostProcessingConfig, SegmentationConfig, TFLite } from '../../types';
+export interface BuildWebGL2PipeLineParams {
+    sourcePlayback: SourcePlayback;
+    backgroundImage: HTMLImageElement | null;
+    backgroundConfig: BackgroundConfig;
+    segmentationConfig: SegmentationConfig;
+    canvas: HTMLCanvasElement;
+    tflite: TFLite;
+}
+export default function buildWebGL2Pipeline({ sourcePlayback, backgroundImage, backgroundConfig, segmentationConfig, canvas, tflite, }: BuildWebGL2PipeLineParams): {
+    render: () => Promise<void>;
+    updatePostProcessingConfig: (postProcessingConfig: PostProcessingConfig) => void;
+    cleanUp: () => void;
+};

package/dist/engine/media/streamEffects/video/virtual-background/types.d.ts

@@ -0,0 +1,81 @@
+/// <reference types="emscripten" />
+export interface TFLite extends EmscriptenModule {
+    _getModelBufferMemoryOffset(): number;
+    _getInputMemoryOffset(): number;
+    _getInputHeight(): number;
+    _getInputWidth(): number;
+    _getInputChannelCount(): number;
+    _getOutputMemoryOffset(): number;
+    _getOutputHeight(): number;
+    _getOutputWidth(): number;
+    _getOutputChannelCount(): number;
+    _loadModel(bufferSize: number): number;
+    _runInference(): number;
+}
+export declare type BlendMode = 'screen' | 'linearDodge';
+export interface TimerWorker {
+    setTimeout: (callback: () => void, timeoutMs: number) => number;
+    clearTimeout: (callbackId: number) => void;
+    terminate: () => void;
+}
+export declare type PostProcessingConfig = {
+    smoothSegmentationMask: boolean;
+    jointBilateralFilter: JointBilateralFilterConfig;
+    coverage: [number, number];
+    lightWrapping: number;
+    blendMode: BlendMode;
+};
+export declare type JointBilateralFilterConfig = {
+    sigmaSpace: number;
+    sigmaColor: number;
+};
+export declare type SourcePlayback = {
+    htmlElement: HTMLImageElement | HTMLVideoElement;
+    width: number;
+    height: number;
+};
+export declare type BackgroundConfig = {
+    type: 'none' | 'blur' | 'image';
+    url?: string;
+    imageSource: HTMLImageElement | null;
+};
+export declare type PipelineType = 'canvas2dCpu' | 'webgl2';
+export declare type SegmentationModel = 'meet' | 'mlkit';
+export declare type SegmentationBackend = 'webgl' | 'wasm' | 'wasmSimd';
+export declare type InputResolution = '640x360' | '256x256' | '256x144' | '160x96';
+export declare type SegmentationConfig = {
+    model: SegmentationModel;
+    backend: SegmentationBackend;
+    inputResolution: InputResolution;
+    pipeline: PipelineType;
+    targetFps: number;
+    deferInputResizing: boolean;
+};
+export interface PipeLine {
+    render: () => Promise<void>;
+    updatePostProcessingConfig: (postProcessingConfig: PostProcessingConfig) => void;
+    cleanUp: () => void;
+}
+export interface PipeLineConfig {
+    backgroundConfig: BackgroundConfig;
+    segmentationConfig: SegmentationConfig;
+    postprocessingConfig: PostProcessingConfig;
+}
+export interface BuildPipeLineConfig {
+    config: PipeLineConfig;
+    sourcePlayback: SourcePlayback;
+    destinationPlayback: HTMLCanvasElement;
+    tflite: TFLite;
+}
+export interface TrackProcessingContext {
+    pipeline: PipeLine;
+    source: HTMLVideoElement | HTMLImageElement;
+    destination: HTMLCanvasElement;
+    track: MediaStreamTrack;
+    timerWorker: TimerWorker;
+    timerId: number;
+}
+export declare type GetProcessedTrackParams = {
+    track: MediaStreamTrack;
+    pipelineConfig: PipeLineConfig;
+};

package/dist/engine/media/tracks/DefaultVideoTrack.d.ts

@@ -2,7 +2,6 @@ import { ProducerCodecOptions } from 'mediasoup-client/lib/Producer';
 import { RtpEncodingParameters } from 'mediasoup-client/lib/RtpParameters';
 import { VideoCodec, VideoEncoderConfig } from '../../../types/common';
 import DefaultBaseTrack from './DefaultBaseTrack';
-import type { Effects } from '../streamEffects/video/esdk/TrackProcessor';
 import { BaseTrackParams, VideoTrack, VideoTrackPublishParams } from '../../../types/media';
 declare class DefaultVideoTrack extends DefaultBaseTrack implements VideoTrack {
     #private;
@@ -16,13 +15,10 @@ declare class DefaultVideoTrack extends DefaultBaseTrack implements VideoTrack {
     setTransformParams(transformParams: {}): void;
     setMaxSpatialLayer(spatialLayer: number): Promise<void>;
     getMaxSpatialLayer(): number | undefined;
-    enableEffects(): Promise<void>;
-    disableEffects(): Promise<void>;
+    enableBlur(): Promise<void>;
+    disableBlur(): Promise<void>;
     publish(params?: VideoTrackPublishParams): Promise<void>;
     private watchOrientation;
-    applyEffects(effects: Effects): Promise<void>;
-    resetEffects(): Promise<void>;
-    onEffectReady(callback: () => void): void;
-    get effectReadyProgress(): number;
+    private watchDocumentVisibility;
 }
 export default DefaultVideoTrack;