@livekit/track-processors 0.6.0 → 0.7.0

package/src/transformers/BackgroundTransformer.ts

@@ -1,7 +1,8 @@
 import * as vision from '@mediapipe/tasks-vision';
+import { getLogger, LoggerNames } from '../logger';
 import { dependencies } from '../../package.json';
 import VideoTransformer from './VideoTransformer';
-import { VideoTransformerInitOptions } from './types';
+import { TrackTransformerDestroyOptions, VideoTransformerInitOptions } from './types';
 
 export type SegmenterOptions = Partial<vision.ImageSegmenterOptions['baseOptions']>;
 
@@ -14,6 +15,7 @@ export interface FrameProcessingStats {
 export type BackgroundOptions = {
   blurRadius?: number;
   imagePath?: string;
+  backgroundDisabled?: boolean;
   /** cannot be updated through the `update` method, needs a restart */
   segmenterOptions?: SegmenterOptions;
   /** cannot be updated through the `update` method, needs a restart */
@@ -36,7 +38,7 @@ export default class BackgroundProcessor extends VideoTransformer<BackgroundOpti
 
   segmentationResults: vision.ImageSegmenterResult | undefined;
 
-  backgroundImage: ImageBitmap | null = null;
+  backgroundImageAndPath: { imageData: ImageBitmap, path: string } | null = null;
 
   options: BackgroundOptions;
 
@@ -44,6 +46,8 @@ export default class BackgroundProcessor extends VideoTransformer<BackgroundOpti
 
   isFirstFrame = true;
 
+  private log = getLogger(LoggerNames.ProcessorWrapper);
+
   constructor(opts: BackgroundOptions) {
     super();
     this.options = opts;
@@ -75,45 +79,57 @@ export default class BackgroundProcessor extends VideoTransformer<BackgroundOpti
     });
 
     // Skip loading the image here if update already loaded the image below
-    if (this.options?.imagePath && !this.backgroundImage) {
-      await this.loadBackground(this.options.imagePath).catch((err) =>
-        console.error('Error while loading processor background image: ', err),
+    if (this.options?.imagePath) {
+      await this.loadAndSetBackground(this.options.imagePath).catch((err) =>
+        this.log.error('Error while loading processor background image: ', err),
       );
     }
-    if (this.options.blurRadius) {
+    if (typeof this.options.blurRadius === 'number') {
       this.gl?.setBlurRadius(this.options.blurRadius);
     }
+    this.gl?.setBackgroundDisabled(this.options.backgroundDisabled ?? false);
   }
 
-  async destroy() {
+  async destroy(options?: TrackTransformerDestroyOptions) {
     await super.destroy();
     await this.imageSegmenter?.close();
-    this.backgroundImage = null;
-    this.isFirstFrame = true;
+    this.backgroundImageAndPath = null;
+
+    if (!options?.willProcessorRestart) {
+      this.isFirstFrame = true;
+    }
   }
 
-  async loadBackground(path: string) {
-    const img = new Image();
+  async loadAndSetBackground(path: string) {
+    if (!this.backgroundImageAndPath || this.backgroundImageAndPath?.path !== path) {
+      const img = new Image();
 
-    await new Promise((resolve, reject) => {
-      img.crossOrigin = 'Anonymous';
-      img.onload = () => resolve(img);
-      img.onerror = (err) => reject(err);
-      img.src = path;
-    });
-    const imageData = await createImageBitmap(img);
-    this.gl?.setBackgroundImage(imageData);
+      await new Promise((resolve, reject) => {
+        img.crossOrigin = 'Anonymous';
+        img.onload = () => resolve(img);
+        img.onerror = (err) => reject(err);
+        img.src = path;
+      });
+      const imageData = await createImageBitmap(img);
+      this.backgroundImageAndPath = { imageData, path };
+    }
+    this.gl?.setBackgroundImage(this.backgroundImageAndPath.imageData);
   }
 
   async transform(frame: VideoFrame, controller: TransformStreamDefaultController<VideoFrame>) {
     let enqueuedFrame = false;
     try {
       if (!(frame instanceof VideoFrame) || frame.codedWidth === 0 || frame.codedHeight === 0) {
-        console.debug('empty frame detected, ignoring');
+        this.log.debug('empty frame detected, ignoring');
         return;
       }
 
-      if (this.isDisabled) {
+      let skipProcessingFrame = this.isDisabled ?? this.options.backgroundDisabled ?? false;
+      if (typeof this.options.blurRadius !== 'number' && typeof this.options.imagePath !== 'string') {
+        skipProcessingFrame = true;
+      }
+
+      if (skipProcessingFrame) {
         controller.enqueue(frame);
         enqueuedFrame = true;
         return;
@@ -187,7 +203,7 @@ export default class BackgroundProcessor extends VideoTransformer<BackgroundOpti
       }
       await segmentationPromise;
     } catch (e) {
-      console.error('Error while processing frame: ', e);
+      this.log.error('Error while processing frame: ', e);
     } finally {
       if (!enqueuedFrame) {
         frame.close();
@@ -200,10 +216,11 @@ export default class BackgroundProcessor extends VideoTransformer<BackgroundOpti
 
     this.gl?.setBlurRadius(opts.blurRadius ?? null);
     if (opts.imagePath) {
-      await this.loadBackground(opts.imagePath);
+      await this.loadAndSetBackground(opts.imagePath);
     } else {
       this.gl?.setBackgroundImage(null);
     }
+    this.gl?.setBackgroundDisabled(opts.backgroundDisabled ?? false);
   }
 
   private async drawFrame(frame: VideoFrame) {

package/src/transformers/VideoTransformer.ts

@@ -15,7 +15,7 @@ export default abstract class VideoTransformer<Options extends Record<string, un
 
   gl?: ReturnType<typeof setupWebGL>;
 
-  protected isDisabled?: Boolean = false;
+  protected isDisabled?: boolean = false;
 
   async init({
     outputCanvas,

package/src/transformers/types.ts

@@ -10,9 +10,9 @@ export interface VideoTransformerInitOptions extends TrackTransformerInitOptions
 export interface AudioTransformerInitOptions extends TrackTransformerInitOptions {}
 
 export interface VideoTrackTransformer<Options extends Record<string, unknown>>
-  extends BaseTrackTransformer<VideoTransformerInitOptions, VideoFrame> {
+  extends BaseTrackTransformer<VideoTransformerInitOptions, VideoFrame, TrackTransformerDestroyOptions> {
   init: (options: VideoTransformerInitOptions) => void;
-  destroy: () => void;
+  destroy: (options?: TrackTransformerDestroyOptions) => void;
   restart: (options: VideoTransformerInitOptions) => void;
   transform: (frame: VideoFrame, controller: TransformStreamDefaultController) => void;
   transformer?: TransformStream;
@@ -20,26 +20,29 @@ export interface VideoTrackTransformer<Options extends Record<string, unknown>>
 }
 
 export interface AudioTrackTransformer<Options extends Record<string, unknown>>
-  extends BaseTrackTransformer<AudioTransformerInitOptions, AudioData> {
+  extends BaseTrackTransformer<AudioTransformerInitOptions, AudioData, TrackTransformerDestroyOptions> {
   init: (options: AudioTransformerInitOptions) => void;
-  destroy: () => void;
+  destroy: (options: TrackTransformerDestroyOptions) => void;
   restart: (options: AudioTransformerInitOptions) => void;
   transform: (frame: AudioData, controller: TransformStreamDefaultController) => void;
   transformer?: TransformStream;
   update: (options: Options) => void;
 }
 
+export type TrackTransformerDestroyOptions = { willProcessorRestart: boolean };
+
 export type TrackTransformer<Options extends Record<string, unknown>> =
   | VideoTrackTransformer<Options>
   | AudioTrackTransformer<Options>;
 
 export interface BaseTrackTransformer<
-  T extends TrackTransformerInitOptions,
+  InitOpts extends TrackTransformerInitOptions,
   DataType extends VideoFrame | AudioData,
+  DestroyOpts extends TrackTransformerDestroyOptions = TrackTransformerDestroyOptions,
 > {
-  init: (options: T) => void;
-  destroy: () => void;
-  restart: (options: T) => void;
+  init: (options: InitOpts) => void;
+  destroy: (options: DestroyOpts) => void;
+  restart: (options: InitOpts) => void;
   transform: (frame: DataType, controller: TransformStreamDefaultController) => void;
   transformer?: TransformStream;
 }

package/src/webgl/index.ts

@@ -4,6 +4,7 @@
  * - downsample the video texture in background blur scenario before applying the (gaussian) blur for better performance
  *
  */
+import { getLogger, LoggerNames} from '../logger';
 import { applyBlur, createBlurProgram } from './shader-programs/blurShader';
 import { createBoxBlurProgram } from './shader-programs/boxBlurShader';
 import { createCompositeProgram } from './shader-programs/compositeShader';
@@ -16,6 +17,8 @@ import {
   resizeImageToCover,
 } from './utils';
 
+const log = getLogger(LoggerNames.WebGl);
+
 export const setupWebGL = (canvas: OffscreenCanvas | HTMLCanvasElement) => {
   const gl = canvas.getContext('webgl2', {
     antialias: true,
@@ -27,7 +30,7 @@ export const setupWebGL = (canvas: OffscreenCanvas | HTMLCanvasElement) => {
   const downsampleFactor = 4;
 
   if (!gl) {
-    console.error('Failed to create WebGL context');
+    log.error('Failed to create WebGL context');
     return undefined;
   }
 
@@ -42,6 +45,7 @@ export const setupWebGL = (canvas: OffscreenCanvas | HTMLCanvasElement) => {
     mask: maskTextureLocation,
     frame: frameTextureLocation,
     background: bgTextureLocation,
+    disableBackground: disableBackgroundLocation,
   } = composite.uniformLocations;
 
   // Create the blur program using the same vertex shader source
@@ -103,14 +107,17 @@ export const setupWebGL = (canvas: OffscreenCanvas | HTMLCanvasElement) => {
     createFramebuffer(gl, finalMaskTextures[1], canvas.width, canvas.height),
   ];
 
+  let backgroundImageDisabled = false;
+
   // Set up uniforms for the composite shader
   gl.useProgram(compositeProgram);
+  gl.uniform1i(disableBackgroundLocation, backgroundImageDisabled ? 1 : 0);
   gl.uniform1i(bgTextureLocation, 0);
   gl.uniform1i(frameTextureLocation, 1);
   gl.uniform1i(maskTextureLocation, 2);
 
   // Store custom background image
-  let customBackgroundImage: ImageBitmap | ImageData = getEmptyImageData();
+  let customBackgroundImage: ImageBitmap | ImageData | null = null;
 
   function renderFrame(frame: VideoFrame) {
     if (frame.codedWidth === 0 || finalMaskTextures.length === 0) {
@@ -149,7 +156,7 @@ export const setupWebGL = (canvas: OffscreenCanvas | HTMLCanvasElement) => {
         bgBlurFrameBuffers,
         bgBlurTextures,
       );
-    } else {
+    } else if (customBackgroundImage) {
       gl.activeTexture(gl.TEXTURE0);
       gl.bindTexture(gl.TEXTURE_2D, bgTexture);
       gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, customBackgroundImage);
@@ -170,6 +177,7 @@ export const setupWebGL = (canvas: OffscreenCanvas | HTMLCanvasElement) => {
     gl.activeTexture(gl.TEXTURE0);
     gl.bindTexture(gl.TEXTURE_2D, backgroundTexture);
     gl.uniform1i(bgTextureLocation, 0);
+    gl.uniform1i(disableBackgroundLocation, backgroundImageDisabled ? 1 : 0);
 
     // Set frame texture
     gl.activeTexture(gl.TEXTURE1);
@@ -189,9 +197,10 @@ export const setupWebGL = (canvas: OffscreenCanvas | HTMLCanvasElement) => {
    */
   async function setBackgroundImage(image: ImageBitmap | null) {
     // Clear existing background
-    customBackgroundImage = getEmptyImageData();
+    customBackgroundImage = null;
 
     if (image) {
+      customBackgroundImage = getEmptyImageData();
       try {
         // Resize and crop the image to cover the canvas
         const croppedImage = await resizeImageToCover(image, canvas.width, canvas.height);
@@ -199,16 +208,16 @@ export const setupWebGL = (canvas: OffscreenCanvas | HTMLCanvasElement) => {
         // Store the cropped and resized image
         customBackgroundImage = croppedImage;
       } catch (error) {
-        console.error(
+        log.error(
           'Error processing background image, falling back to black background:',
           error,
         );
       }
-    }
 
-    gl.activeTexture(gl.TEXTURE0);
-    gl.bindTexture(gl.TEXTURE_2D, bgTexture);
-    gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, customBackgroundImage);
+      gl.activeTexture(gl.TEXTURE0);
+      gl.bindTexture(gl.TEXTURE_2D, bgTexture);
+      gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, customBackgroundImage);
+    }
   }
 
   function setBlurRadius(radius: number | null) {
@@ -216,6 +225,10 @@ export const setupWebGL = (canvas: OffscreenCanvas | HTMLCanvasElement) => {
     setBackgroundImage(null);
   }
 
+  function setBackgroundDisabled(disabled: boolean) {
+    backgroundImageDisabled = disabled;
+  }
+
   function updateMask(mask: WebGLTexture) {
     // Use the existing applyBlur function to apply the first blur pass
     // The second blur pass will be written to finalMaskTextures[writeMaskIndex]
@@ -282,12 +295,12 @@ export const setupWebGL = (canvas: OffscreenCanvas | HTMLCanvasElement) => {
       if (customBackgroundImage instanceof ImageBitmap) {
         customBackgroundImage.close();
       }
-      customBackgroundImage = getEmptyImageData();
+      customBackgroundImage = null;
     }
     bgBlurTextures = [];
     bgBlurFrameBuffers = [];
     finalMaskTextures = [];
   }
 
-  return { renderFrame, updateMask, setBackgroundImage, setBlurRadius, cleanup };
+  return { renderFrame, updateMask, setBackgroundImage, setBlurRadius, setBackgroundDisabled, cleanup };
 };

package/src/webgl/shader-programs/compositeShader.ts

@@ -6,29 +6,33 @@ export const compositeFragmentShader = glsl`#version 300 es
   precision mediump float;
   in vec2 texCoords;
   uniform sampler2D background;
+  uniform bool disableBackground;
   uniform sampler2D frame;
   uniform sampler2D mask;
   out vec4 fragColor;
   
   void main() {
-      
     vec4 frameTex = texture(frame, texCoords);
-    vec4 bgTex = texture(background, texCoords);
 
-    float maskVal = texture(mask, texCoords).r;
+    if (disableBackground) {
+      fragColor = frameTex;
+    } else {
+      vec4 bgTex = texture(background, texCoords);
 
-    // Compute screen-space gradient to detect edge sharpness
-    float grad = length(vec2(dFdx(maskVal), dFdy(maskVal)));
+      float maskVal = texture(mask, texCoords).r;
 
-    float edgeSoftness = 2.0; // higher = softer
-    
-    // Create a smooth edge around binary transition
-    float smoothAlpha = smoothstep(0.5 - grad * edgeSoftness, 0.5 + grad * edgeSoftness, maskVal);
+      // Compute screen-space gradient to detect edge sharpness
+      float grad = length(vec2(dFdx(maskVal), dFdy(maskVal)));
 
-    // Optional: preserve frame alpha, or override as fully opaque
-    vec4 blended = mix(bgTex, vec4(frameTex.rgb, 1.0), 1.0 - smoothAlpha);
-    
-    fragColor = blended;
+      float edgeSoftness = 2.0; // higher = softer
+
+      // Create a smooth edge around binary transition
+      float smoothAlpha = smoothstep(0.5 - grad * edgeSoftness, 0.5 + grad * edgeSoftness, maskVal);
+
+      // Optional: preserve frame alpha, or override as fully opaque
+      vec4 blended = mix(bgTex, vec4(frameTex.rgb, 1.0), 1.0 - smoothAlpha);
+      fragColor = blended;
+    }
   
   }
 `;
@@ -51,6 +55,7 @@ export function createCompositeProgram(gl: WebGL2RenderingContext) {
     mask: gl.getUniformLocation(compositeProgram, 'mask')!,
     frame: gl.getUniformLocation(compositeProgram, 'frame')!,
     background: gl.getUniformLocation(compositeProgram, 'background')!,
+    disableBackground: gl.getUniformLocation(compositeProgram, 'disableBackground')!,
     stepWidth: gl.getUniformLocation(compositeProgram, 'u_stepWidth')!,
   };
 

package/src/webgl/utils.ts

@@ -1,3 +1,7 @@
+import { getLogger, LoggerNames} from '../logger';
+
+const log = getLogger(LoggerNames.WebGl);
+
 /**
  * Initialize a WebGL texture
  */
@@ -24,7 +28,7 @@ export function createShader(
   gl.shaderSource(shader, source);
   gl.compileShader(shader);
   if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) {
-    console.error('Shader compile failed:', gl.getShaderInfoLog(shader));
+    log.error('Shader compile failed:', gl.getShaderInfoLog(shader));
     gl.deleteShader(shader);
     throw new Error('Shader compile failed');
   }
@@ -41,7 +45,7 @@ export function createProgram(
   gl.attachShader(program, fs);
   gl.linkProgram(program);
   if (!gl.getProgramParameter(program, gl.LINK_STATUS)) {
-    console.error('Program link failed:', gl.getProgramInfoLog(program));
+    log.error('Program link failed:', gl.getProgramInfoLog(program));
     throw new Error('Program link failed');
   }
   return program;