@livekit/track-processors 0.6.0 → 0.7.0

package/dist/src/transformers/types.d.ts

@@ -7,27 +7,30 @@ export interface VideoTransformerInitOptions extends TrackTransformerInitOptions
 }
 export interface AudioTransformerInitOptions extends TrackTransformerInitOptions {
 }
-export interface VideoTrackTransformer<Options extends Record<string, unknown>> extends BaseTrackTransformer<VideoTransformerInitOptions, VideoFrame> {
+export interface VideoTrackTransformer<Options extends Record<string, unknown>> extends BaseTrackTransformer<VideoTransformerInitOptions, VideoFrame, TrackTransformerDestroyOptions> {
     init: (options: VideoTransformerInitOptions) => void;
-    destroy: () => void;
+    destroy: (options?: TrackTransformerDestroyOptions) => void;
     restart: (options: VideoTransformerInitOptions) => void;
     transform: (frame: VideoFrame, controller: TransformStreamDefaultController) => void;
     transformer?: TransformStream;
     update: (options: Options) => void;
 }
-export interface AudioTrackTransformer<Options extends Record<string, unknown>> extends BaseTrackTransformer<AudioTransformerInitOptions, AudioData> {
+export interface AudioTrackTransformer<Options extends Record<string, unknown>> extends BaseTrackTransformer<AudioTransformerInitOptions, AudioData, TrackTransformerDestroyOptions> {
     init: (options: AudioTransformerInitOptions) => void;
-    destroy: () => void;
+    destroy: (options: TrackTransformerDestroyOptions) => void;
     restart: (options: AudioTransformerInitOptions) => void;
     transform: (frame: AudioData, controller: TransformStreamDefaultController) => void;
     transformer?: TransformStream;
     update: (options: Options) => void;
 }
+export type TrackTransformerDestroyOptions = {
+    willProcessorRestart: boolean;
+};
 export type TrackTransformer<Options extends Record<string, unknown>> = VideoTrackTransformer<Options> | AudioTrackTransformer<Options>;
-export interface BaseTrackTransformer<T extends TrackTransformerInitOptions, DataType extends VideoFrame | AudioData> {
-    init: (options: T) => void;
-    destroy: () => void;
-    restart: (options: T) => void;
+export interface BaseTrackTransformer<InitOpts extends TrackTransformerInitOptions, DataType extends VideoFrame | AudioData, DestroyOpts extends TrackTransformerDestroyOptions = TrackTransformerDestroyOptions> {
+    init: (options: InitOpts) => void;
+    destroy: (options: DestroyOpts) => void;
+    restart: (options: InitOpts) => void;
     transform: (frame: DataType, controller: TransformStreamDefaultController) => void;
     transformer?: TransformStream;
 }

package/dist/src/webgl/index.d.ts

@@ -3,5 +3,6 @@ export declare const setupWebGL: (canvas: OffscreenCanvas | HTMLCanvasElement) =
     updateMask: (mask: WebGLTexture) => void;
     setBackgroundImage: (image: ImageBitmap | null) => Promise<void>;
     setBlurRadius: (radius: number | null) => void;
+    setBackgroundDisabled: (disabled: boolean) => void;
     cleanup: () => void;
 } | undefined;

package/dist/src/webgl/shader-programs/compositeShader.d.ts

@@ -13,6 +13,7 @@ export declare function createCompositeProgram(gl: WebGL2RenderingContext): {
         mask: WebGLUniformLocation;
         frame: WebGLUniformLocation;
         background: WebGLUniformLocation;
+        disableBackground: WebGLUniformLocation;
         stepWidth: WebGLUniformLocation;
     };
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@livekit/track-processors",
-  "version": "0.6.0",
+  "version": "0.7.0",
   "description": "LiveKit track processors",
   "main": "dist/index.js",
   "module": "dist/index.mjs",
@@ -20,8 +20,8 @@
     "@mediapipe/tasks-vision": "0.10.14"
   },
   "peerDependencies": {
-    "livekit-client": "^1.12.0 || ^2.1.0",
-    "@types/dom-mediacapture-transform": "^0.1.9"
+    "@types/dom-mediacapture-transform": "^0.1.9",
+    "livekit-client": "^1.12.0 || ^2.1.0"
   },
   "devDependencies": {
     "@changesets/cli": "^2.26.2",

package/src/ProcessorWrapper.ts

@@ -1,6 +1,9 @@
-import type { ProcessorOptions, Track, TrackProcessor } from 'livekit-client';
-import { TrackTransformer } from './transformers';
+import { type ProcessorOptions, type Track, type TrackProcessor } from 'livekit-client';
+import { TrackTransformer, TrackTransformerDestroyOptions } from './transformers';
 import { createCanvas, waitForTrackResolution } from './utils';
+import { LoggerNames, getLogger } from './logger';
+
+type ProcessorWrapperLifecycleState = 'idle' | 'initializing' | 'running' | 'media-exhausted' | 'destroying' | 'destroyed';
 
 export interface ProcessorWrapperOptions {
   /**
@@ -10,7 +13,10 @@ export interface ProcessorWrapperOptions {
   maxFps?: number;
 }
 
-export default class ProcessorWrapper<TransformerOptions extends Record<string, unknown>>
+export default class ProcessorWrapper<
+  TransformerOptions extends Record<string, unknown>,
+  Transformer extends TrackTransformer<TransformerOptions> = TrackTransformer<TransformerOptions>,
+>
   implements TrackProcessor<Track.Kind>
 {
   /**
@@ -58,7 +64,7 @@ export default class ProcessorWrapper<TransformerOptions extends Record<string,
 
   processedTrack?: MediaStreamTrack;
 
-  transformer: TrackTransformer<TransformerOptions>;
+  transformer: Transformer;
 
   // For tracking whether we're using the stream API fallback
   private useStreamFallback = false;
@@ -77,10 +83,12 @@ export default class ProcessorWrapper<TransformerOptions extends Record<string,
   // FPS control for fallback implementation
   private maxFps: number;
 
-  private symbol?: Symbol;
+  private log = getLogger(LoggerNames.ProcessorWrapper);
+
+  private lifecycleState: ProcessorWrapperLifecycleState = 'idle';
 
   constructor(
-    transformer: TrackTransformer<TransformerOptions>,
+    transformer: Transformer,
     name: string,
     options: ProcessorWrapperOptions = {},
   ) {
@@ -140,6 +148,8 @@ export default class ProcessorWrapper<TransformerOptions extends Record<string,
   }
 
   async init(opts: ProcessorOptions<Track.Kind>): Promise<void> {
+    this.log.debug('Init called');
+    this.lifecycleState = 'initializing';
     await this.setup(opts);
 
     if (!this.canvas) {
@@ -156,6 +166,7 @@ export default class ProcessorWrapper<TransformerOptions extends Record<string,
     } else {
       this.initStreamProcessorPath();
     }
+    this.lifecycleState = 'running';
   }
 
   private initStreamProcessorPath() {
@@ -168,20 +179,20 @@ export default class ProcessorWrapper<TransformerOptions extends Record<string,
     const readableStream = this.processor.readable;
     const pipedStream = readableStream.pipeThrough(this.transformer!.transformer!);
 
-    const symbol = Symbol('stream');
-    this.symbol = symbol;
-
     pipedStream
       .pipeTo(this.trackGenerator.writable)
-      // destroy processor if stream finishes
-      .then(() => this.destroy(symbol))
+      // if stream finishes, the media to process is exhausted
+      .then(() => this.handleMediaExhausted())
       // destroy processor if stream errors - unless it's an abort error
       .catch((e) => {
         if (e instanceof DOMException && e.name === 'AbortError') {
-          console.log('stream processor path aborted');
+          this.log.log('stream processor path aborted');
+        } else if (e instanceof DOMException && e.name === 'InvalidStateError' && e.message === 'Stream closed') {
+          this.log.log('stream processor underlying stream closed');
+          this.handleMediaExhausted();
         } else {
-          console.error('error when trying to pipe', e);
-          this.destroy(symbol);
+          this.log.error('error when trying to pipe', e);
+          this.destroy();
         }
       });
 
@@ -224,7 +235,7 @@ export default class ProcessorWrapper<TransformerOptions extends Record<string,
         // @ts-ignore - The controller expects both VideoFrame & AudioData but we're only using VideoFrame
         this.transformer.transform(frame, controller);
       } catch (e) {
-        console.error('Error in transform:', e);
+        this.log.error('Error in transform:', e);
         frame.close();
       }
     };
@@ -235,6 +246,7 @@ export default class ProcessorWrapper<TransformerOptions extends Record<string,
 
   private startRenderLoop() {
     if (!this.sourceDummy || !(this.sourceDummy instanceof HTMLVideoElement)) {
+      this.handleMediaExhausted();
       return;
     }
 
@@ -257,11 +269,12 @@ export default class ProcessorWrapper<TransformerOptions extends Record<string,
         !this.sourceDummy ||
         !(this.sourceDummy instanceof HTMLVideoElement)
       ) {
+        this.handleMediaExhausted();
         return;
       }
 
       if (this.sourceDummy.paused) {
-        console.warn('Video is paused, trying to play');
+        this.log.warn('Video is paused, trying to play');
         this.sourceDummy.play();
         return;
       }
@@ -298,7 +311,7 @@ export default class ProcessorWrapper<TransformerOptions extends Record<string,
               window.location.hostname === '127.0.0.1';
 
             if (isDevelopment && now - lastFpsLog > 5000) {
-              console.debug(
+              this.log.debug(
                 `[${this.name}] Estimated video FPS: ${estimatedVideoFps.toFixed(
                   1,
                 )}, Processing at: ${(frameCount / 5).toFixed(1)} FPS`,
@@ -333,7 +346,7 @@ export default class ProcessorWrapper<TransformerOptions extends Record<string,
             }
           }
         } catch (e) {
-          console.error('Error in render loop:', e);
+          this.log.error('Error in render loop:', e);
         }
       }
       this.animationFrameId = requestAnimationFrame(renderLoop);
@@ -343,7 +356,8 @@ export default class ProcessorWrapper<TransformerOptions extends Record<string,
   }
 
   async restart(opts: ProcessorOptions<Track.Kind>): Promise<void> {
-    await this.destroy();
+    this.log.debug('Restart called');
+    await this.destroy({ willProcessorRestart: true });
     await this.init(opts);
   }
 
@@ -356,11 +370,18 @@ export default class ProcessorWrapper<TransformerOptions extends Record<string,
     await this.transformer.update(options[0]);
   }
 
-  async destroy(symbol?: Symbol) {
-    if (symbol && this.symbol !== symbol) {
-      // If the symbol is provided, we only destroy if it matches the current symbol
+  /** Called if the media pipeline no longer can read frames to process from the source media */
+  private async handleMediaExhausted() {
+    this.log.debug('Media was exhausted from source');
+    if (this.lifecycleState !== 'running') {
       return;
     }
+    this.lifecycleState = 'media-exhausted'
+    await this.cleanup();
+  }
+
+  /** Tears down the media stack logic initialized in initStreamProcessorPath / initFallbackPath */
+  private async cleanup() {
     if (this.useStreamFallback) {
       this.processingEnabled = false;
       if (this.animationFrameId) {
@@ -372,9 +393,28 @@ export default class ProcessorWrapper<TransformerOptions extends Record<string,
       }
       this.capturedStream?.getTracks().forEach((track) => track.stop());
     } else {
+      // NOTE: closing writableControl below terminates the stream in initStreamProcessorPath /
+      // calls the .then(...) which calls this.handleMediaExhausted
       await this.processor?.writableControl?.close();
       this.trackGenerator?.stop();
     }
-    await this.transformer.destroy();
+  }
+
+  async destroy(transformerDestroyOptions: TrackTransformerDestroyOptions = { willProcessorRestart: false }) {
+    this.log.debug(`Destroy called - lifecycleState=${this.lifecycleState}, transformerDestroyOptions=${JSON.stringify(transformerDestroyOptions)}`);
+    switch (this.lifecycleState) {
+      case 'running':
+      case 'media-exhausted':
+        this.lifecycleState = 'destroying';
+
+        await this.cleanup();
+
+        await this.transformer.destroy(transformerDestroyOptions);
+        this.lifecycleState = 'destroyed';
+        break;
+
+      default:
+        break;
+    }
   }
 }

package/src/index.ts

@@ -14,6 +14,9 @@ export {
   BackgroundTransformer,
   type ProcessorWrapperOptions,
 };
+export * from './logger';
+
+const DEFAULT_BLUR_RADIUS = 10;
 
 /**
  * Determines if the current browser supports background processors
@@ -27,16 +30,212 @@ export const supportsBackgroundProcessors = () =>
 export const supportsModernBackgroundProcessors = () =>
   BackgroundTransformer.isSupported && ProcessorWrapper.hasModernApiSupport;
 
-export interface BackgroundProcessorOptions extends ProcessorWrapperOptions {
+type SwitchBackgroundProcessorBackgroundBlurOptions = {
+  mode: 'background-blur';
+  /** If unspecified, defaults to {@link DEFAULT_BLUR_RADIUS} */
   blurRadius?: number;
-  imagePath?: string;
+};
+
+type SwitchBackgroundProcessorVirtualBackgroundOptions = {
+  mode: 'virtual-background';
+  imagePath: string;
+};
+
+type SwitchBackgroundProcessorDisabledOptions = {
+  mode: 'disabled';
+};
+
+export type SwitchBackgroundProcessorOptions =
+| SwitchBackgroundProcessorDisabledOptions
+| SwitchBackgroundProcessorBackgroundBlurOptions
+| SwitchBackgroundProcessorVirtualBackgroundOptions
+
+type BackgroundProcessorCommonOptions = ProcessorWrapperOptions & {
   segmenterOptions?: SegmenterOptions;
   assetPaths?: { tasksVisionFileSet?: string; modelAssetPath?: string };
   onFrameProcessed?: (stats: FrameProcessingStats) => void;
+};
+
+type BackgroundProcessorModeOptions = BackgroundProcessorCommonOptions & SwitchBackgroundProcessorOptions;
+type BackgroundProcessorLegacyOptions = BackgroundProcessorCommonOptions & {
+  mode?: never;
+  blurRadius?: number;
+  imagePath?: string;
+};
+
+export type BackgroundProcessorOptions =
+  | BackgroundProcessorModeOptions
+  | BackgroundProcessorLegacyOptions;
+
+class BackgroundProcessorWrapper extends ProcessorWrapper<BackgroundOptions, BackgroundTransformer> {
+  get mode(): BackgroundProcessorModeOptions['mode'] | 'legacy' {
+    const options = this.transformer.options;
+
+    if (options.backgroundDisabled) {
+      return 'disabled';
+    }
+
+    if (typeof options.imagePath === 'string' && typeof options.blurRadius === 'undefined') {
+      return 'virtual-background';
+    }
+
+    if (typeof options.imagePath === 'undefined') {
+      return 'background-blur';
+    }
+
+    return 'legacy';
+  }
+
+  async switchTo(options: SwitchBackgroundProcessorOptions) {
+    switch (options.mode) {
+      case 'background-blur':
+        await this.updateTransformerOptions({
+          imagePath: undefined,
+          blurRadius: options.blurRadius ?? DEFAULT_BLUR_RADIUS,
+          backgroundDisabled: false,
+        });
+        break;
+      case 'virtual-background':
+        await this.updateTransformerOptions({
+          imagePath: options.imagePath,
+          blurRadius: undefined,
+          backgroundDisabled: false,
+        });
+        break;
+      case 'disabled':
+        await this.updateTransformerOptions({ imagePath: undefined, backgroundDisabled: true });
+        break;
+    }
+  }
 }
 
+/**
+ * Instantiates a background processor that supports blurring the background of a user's local
+ * video or replacing the user's background with a virtual background image, and supports switching
+ * the active mode later on the fly to avoid visual artifacts.
+ *
+ * @example
+ * const camTrack = currentRoom.localParticipant.getTrackPublication(Track.Source.Camera)!.track as LocalVideoTrack;
+ * const processor = BackgroundProcessor({ mode: 'background-blur', blurRadius: 10 });
+ * camTrack.setProcessor(processor);
+ *
+ * // Change to background image:
+ * processor.switchToVirtualBackground('path/to/image.png');
+ * // Change back to background blur:
+ * processor.switchToBackgroundBlur(10);
+ */
+export const BackgroundProcessor = (
+  options: BackgroundProcessorOptions,
+  name = 'background-processor',
+) => {
+  const isTransformerSupported = BackgroundTransformer.isSupported;
+  const isProcessorSupported = ProcessorWrapper.isSupported;
+
+  if (!isTransformerSupported) {
+    throw new Error('Background transformer is not supported in this browser');
+  }
+
+  if (!isProcessorSupported) {
+    throw new Error(
+      'Neither MediaStreamTrackProcessor nor canvas.captureStream() fallback is supported in this browser',
+    );
+  }
+
+  // Extract transformer-specific options and processor options
+  let transformer, processorOpts;
+  switch (options.mode) {
+    case 'background-blur': {
+      const {
+        // eslint-disable-next-line no-unused-vars
+        mode,
+        blurRadius = DEFAULT_BLUR_RADIUS,
+        segmenterOptions,
+        assetPaths,
+        onFrameProcessed,
+        ...rest
+      } = options;
+
+      processorOpts = rest;
+      transformer = new BackgroundTransformer({
+        blurRadius,
+        segmenterOptions,
+        assetPaths,
+        onFrameProcessed,
+      });
+      break;
+    }
+
+    case 'virtual-background': {
+      const {
+        // eslint-disable-next-line no-unused-vars
+        mode,
+        imagePath,
+        segmenterOptions,
+        assetPaths,
+        onFrameProcessed,
+        ...rest
+      } = options;
+
+      processorOpts = rest;
+      transformer = new BackgroundTransformer({
+        imagePath,
+        segmenterOptions,
+        assetPaths,
+        onFrameProcessed,
+      });
+      break;
+    }
+
+    case 'disabled': {
+      const {
+        segmenterOptions,
+        assetPaths,
+        onFrameProcessed,
+        ...rest
+      } = options;
+
+      processorOpts = rest;
+      transformer = new BackgroundTransformer({
+        segmenterOptions,
+        assetPaths,
+        onFrameProcessed,
+      });
+      break;
+    }
+
+    default: {
+      const {
+        blurRadius,
+        imagePath,
+        segmenterOptions,
+        assetPaths,
+        onFrameProcessed,
+        ...rest
+      } = options;
+
+      processorOpts = rest;
+      transformer = new BackgroundTransformer({
+        blurRadius,
+        imagePath,
+        segmenterOptions,
+        assetPaths,
+        onFrameProcessed,
+      });
+      break;
+    }
+  }
+
+  const processor = new BackgroundProcessorWrapper(transformer, name, processorOpts);
+
+  return processor;
+};
+
+/**
+ * Instantiates a background processor that is configured in blur mode.
+ * @deprecated Use `BackgroundProcessor({ mode: 'background-blur', blurRadius: 10,  ... })` instead.
+ */
 export const BackgroundBlur = (
-  blurRadius: number = 10,
+  blurRadius: number = DEFAULT_BLUR_RADIUS,
   segmenterOptions?: SegmenterOptions,
   onFrameProcessed?: (stats: FrameProcessingStats) => void,
   processorOptions?: ProcessorWrapperOptions,
@@ -52,6 +251,10 @@ export const BackgroundBlur = (
   );
 };
 
+/**
+ * Instantiates a background processor that is configured in virtual background mode.
+ * @deprecated Use `BackgroundProcessor({ mode: 'virtual-background', imagePath: '...', ... })` instead.
+ */
 export const VirtualBackground = (
   imagePath: string,
   segmenterOptions?: SegmenterOptions,
@@ -69,42 +272,3 @@ export const VirtualBackground = (
   );
 };
 
-export const BackgroundProcessor = (
-  options: BackgroundProcessorOptions,
-  name = 'background-processor',
-) => {
-  const isTransformerSupported = BackgroundTransformer.isSupported;
-  const isProcessorSupported = ProcessorWrapper.isSupported;
-
-  if (!isTransformerSupported) {
-    throw new Error('Background transformer is not supported in this browser');
-  }
-
-  if (!isProcessorSupported) {
-    throw new Error(
-      'Neither MediaStreamTrackProcessor nor canvas.captureStream() fallback is supported in this browser',
-    );
-  }
-
-  // Extract transformer-specific options and processor options
-  const {
-    blurRadius,
-    imagePath,
-    segmenterOptions,
-    assetPaths,
-    onFrameProcessed,
-    ...processorOpts
-  } = options;
-
-  const transformer = new BackgroundTransformer({
-    blurRadius,
-    imagePath,
-    segmenterOptions,
-    assetPaths,
-    onFrameProcessed,
-  });
-
-  const processor = new ProcessorWrapper(transformer, name, processorOpts);
-
-  return processor;
-};

package/src/logger.ts

@@ -0,0 +1,74 @@
+import { getLogger as clientGetLogger } from 'livekit-client';
+
+export enum LogLevel {
+  trace = 0,
+  debug = 1,
+  info = 2,
+  warn = 3,
+  error = 4,
+  silent = 5,
+}
+
+export enum LoggerNames {
+  ProcessorWrapper = 'livekit-processor-wrapper',
+  BackgroundProcessor = 'livekit-background-processor',
+  WebGl = 'livekit-track-processor-web-gl',
+}
+
+type LogLevelString = keyof typeof LogLevel;
+
+export type StructuredLogger = ReturnType<typeof clientGetLogger>;
+
+let livekitLogger = getLogger('livekit');
+const livekitLoggers = Object.values(LoggerNames).map((name) => getLogger(name));
+
+livekitLogger.setDefaultLevel(LogLevel.info);
+
+export default livekitLogger as StructuredLogger;
+
+/**
+ * @internal
+ */
+export function getLogger(name: string) {
+  return clientGetLogger(name);
+}
+
+export function setLogLevel(level: LogLevel | LogLevelString, loggerName?: LoggerNames) {
+  if (loggerName) {
+    getLogger(loggerName).setLevel(level);
+  } else {
+    for (const logger of livekitLoggers) {
+      logger.setLevel(level);
+    }
+  }
+}
+
+export type LogExtension = (level: LogLevel, msg: string, context?: object) => void;
+
+/**
+ * use this to hook into the logging function to allow sending internal livekit logs to third party services
+ * if set, the browser logs will lose their stacktrace information (see https://github.com/pimterry/loglevel#writing-plugins)
+ */
+export function setLogExtension(extension: LogExtension, logger?: StructuredLogger) {
+  const loggers = logger ? [logger] : livekitLoggers;
+
+  loggers.forEach((logR) => {
+    const originalFactory = logR.methodFactory;
+
+    logR.methodFactory = (methodName, configLevel, loggerName) => {
+      const rawMethod = originalFactory(methodName, configLevel, loggerName);
+
+      const logLevel = LogLevel[methodName as LogLevelString];
+      const needLog = logLevel >= configLevel && logLevel < LogLevel.silent;
+
+      return (msg, context?: [msg: string, context: object]) => {
+        if (context) rawMethod(msg, context);
+        else rawMethod(msg);
+        if (needLog) {
+          extension(logLevel, msg, context);
+        }
+      };
+    };
+    logR.setLevel(logR.getLevel());
+  });
+}