@livekit/track-processors 0.4.0 → 0.5.0

package/dist/index.mjs

@@ -17,13 +17,29 @@ async function waitForTrackResolution(track) {
 }
 
 // src/ProcessorWrapper.ts
-var ProcessorWrapper = class {
-  static get isSupported() {
-    return typeof MediaStreamTrackGenerator !== "undefined" && typeof MediaStreamTrackProcessor !== "undefined";
-  }
-  constructor(transformer, name) {
+var ProcessorWrapper = class _ProcessorWrapper {
+  constructor(transformer, name, options = {}) {
+    // For tracking whether we're using the stream API fallback
+    this.useStreamFallback = false;
+    this.processingEnabled = false;
+    var _a;
     this.name = name;
     this.transformer = transformer;
+    this.maxFps = (_a = options.maxFps) != null ? _a : 30;
+  }
+  /**
+   * Determines if the Processor is supported on the current browser
+   */
+  static get isSupported() {
+    const hasStreamProcessor = typeof MediaStreamTrackGenerator !== "undefined" && typeof MediaStreamTrackProcessor !== "undefined";
+    const hasFallbackSupport = typeof HTMLCanvasElement !== "undefined" && typeof VideoFrame !== "undefined" && "captureStream" in HTMLCanvasElement.prototype;
+    return hasStreamProcessor || hasFallbackSupport;
+  }
+  /**
+   * Determines if modern browser APIs are supported, which yield better performance
+   */
+  static get hasModernApiSupport() {
+    return typeof MediaStreamTrackGenerator !== "undefined" && typeof MediaStreamTrackProcessor !== "undefined";
   }
   async setup(opts) {
     this.source = opts.track;
@@ -36,41 +52,187 @@ var ProcessorWrapper = class {
       this.sourceDummy.height = height != null ? height : 300;
       this.sourceDummy.width = width != null ? width : 300;
     }
-    this.processor = new MediaStreamTrackProcessor({ track: this.source });
-    this.trackGenerator = new MediaStreamTrackGenerator({
-      kind: "video",
-      signalTarget: this.source
-    });
-    this.canvas = new OffscreenCanvas(width != null ? width : 300, height != null ? height : 300);
+    this.useStreamFallback = !_ProcessorWrapper.hasModernApiSupport;
+    if (this.useStreamFallback) {
+      const existingCanvas = document.querySelector(
+        'canvas[data-livekit-processor="' + this.name + '"]'
+      );
+      if (existingCanvas) {
+        this.displayCanvas = existingCanvas;
+        this.displayCanvas.width = width != null ? width : 300;
+        this.displayCanvas.height = height != null ? height : 300;
+      } else {
+        this.displayCanvas = document.createElement("canvas");
+        this.displayCanvas.width = width != null ? width : 300;
+        this.displayCanvas.height = height != null ? height : 300;
+        this.displayCanvas.style.display = "none";
+        this.displayCanvas.dataset.livekitProcessor = this.name;
+        document.body.appendChild(this.displayCanvas);
+      }
+      this.renderContext = this.displayCanvas.getContext("2d");
+      this.capturedStream = this.displayCanvas.captureStream();
+      this.canvas = new OffscreenCanvas(width != null ? width : 300, height != null ? height : 300);
+    } else {
+      this.processor = new MediaStreamTrackProcessor({ track: this.source });
+      this.trackGenerator = new MediaStreamTrackGenerator({
+        kind: "video",
+        signalTarget: this.source
+      });
+      this.canvas = new OffscreenCanvas(width != null ? width : 300, height != null ? height : 300);
+    }
   }
   async init(opts) {
     await this.setup(opts);
-    if (!this.canvas || !this.processor || !this.trackGenerator) {
-      throw new TypeError("Expected both canvas and processor to be defined after setup");
+    if (!this.canvas) {
+      throw new TypeError("Expected canvas to be defined after setup");
     }
-    const readableStream = this.processor.readable;
     await this.transformer.init({
       outputCanvas: this.canvas,
       inputElement: this.sourceDummy
     });
+    if (this.useStreamFallback) {
+      this.initFallbackPath();
+    } else {
+      this.initStreamProcessorPath();
+    }
+  }
+  initStreamProcessorPath() {
+    if (!this.processor || !this.trackGenerator) {
+      throw new TypeError(
+        "Expected processor and trackGenerator to be defined for stream processor path"
+      );
+    }
+    const readableStream = this.processor.readable;
     const pipedStream = readableStream.pipeThrough(this.transformer.transformer);
     pipedStream.pipeTo(this.trackGenerator.writable).catch((e) => console.error("error when trying to pipe", e)).finally(() => this.destroy());
     this.processedTrack = this.trackGenerator;
   }
+  initFallbackPath() {
+    if (!this.capturedStream || !this.source || !this.canvas || !this.renderContext) {
+      throw new TypeError("Missing required components for fallback implementation");
+    }
+    this.processedTrack = this.capturedStream.getVideoTracks()[0];
+    this.processingEnabled = true;
+    this.frameCallback = (frame) => {
+      if (!this.processingEnabled || !frame) {
+        frame.close();
+        return;
+      }
+      const controller = {
+        enqueue: (processedFrame) => {
+          if (this.renderContext && this.displayCanvas) {
+            this.renderContext.drawImage(
+              processedFrame,
+              0,
+              0,
+              this.displayCanvas.width,
+              this.displayCanvas.height
+            );
+            processedFrame.close();
+          }
+        }
+      };
+      try {
+        this.transformer.transform(frame, controller);
+      } catch (e) {
+        console.error("Error in transform:", e);
+        frame.close();
+      }
+    };
+    this.startRenderLoop();
+  }
+  startRenderLoop() {
+    if (!this.sourceDummy || !(this.sourceDummy instanceof HTMLVideoElement)) {
+      return;
+    }
+    let lastVideoTimestamp = -1;
+    let lastFrameTime = 0;
+    const videoElement = this.sourceDummy;
+    const minFrameInterval = 1e3 / this.maxFps;
+    let estimatedVideoFps = this.maxFps;
+    let frameTimeHistory = [];
+    let lastVideoTimeChange = 0;
+    let frameCount = 0;
+    let lastFpsLog = 0;
+    const renderLoop = () => {
+      if (!this.processingEnabled || !this.sourceDummy || !(this.sourceDummy instanceof HTMLVideoElement)) {
+        return;
+      }
+      const videoTime = videoElement.currentTime;
+      const now = performance.now();
+      const timeSinceLastFrame = now - lastFrameTime;
+      const hasNewFrame = videoTime !== lastVideoTimestamp;
+      if (hasNewFrame) {
+        if (lastVideoTimeChange > 0) {
+          const timeBetweenFrames = now - lastVideoTimeChange;
+          frameTimeHistory.push(timeBetweenFrames);
+          if (frameTimeHistory.length > 10) {
+            frameTimeHistory.shift();
+          }
+          if (frameTimeHistory.length > 2) {
+            const avgFrameTime = frameTimeHistory.reduce((sum, time) => sum + time, 0) / frameTimeHistory.length;
+            estimatedVideoFps = 1e3 / avgFrameTime;
+            const isDevelopment = typeof window !== "undefined" && window.location.hostname === "localhost" || window.location.hostname === "127.0.0.1";
+            if (isDevelopment && now - lastFpsLog > 5e3) {
+              console.debug(
+                `[${this.name}] Estimated video FPS: ${estimatedVideoFps.toFixed(
+                  1
+                )}, Processing at: ${(frameCount / 5).toFixed(1)} FPS`
+              );
+              frameCount = 0;
+              lastFpsLog = now;
+            }
+          }
+        }
+        lastVideoTimeChange = now;
+      }
+      const timeThresholdMet = timeSinceLastFrame >= minFrameInterval;
+      if (hasNewFrame && timeThresholdMet) {
+        lastVideoTimestamp = videoTime;
+        lastFrameTime = now;
+        frameCount++;
+        try {
+          const frame = new VideoFrame(videoElement);
+          if (this.frameCallback) {
+            this.frameCallback(frame);
+          } else {
+            frame.close();
+          }
+        } catch (e) {
+          console.error("Error in render loop:", e);
+        }
+      }
+      this.animationFrameId = requestAnimationFrame(renderLoop);
+    };
+    this.animationFrameId = requestAnimationFrame(renderLoop);
+  }
   async restart(opts) {
     await this.destroy();
-    return this.init(opts);
+    await this.init(opts);
   }
   async restartTransformer(...options) {
-    this.transformer.restart(options[0]);
+    await this.transformer.restart(options[0]);
   }
   async updateTransformerOptions(...options) {
-    this.transformer.update(options[0]);
+    await this.transformer.update(options[0]);
   }
   async destroy() {
-    var _a;
+    var _a, _b, _c, _d;
+    if (this.useStreamFallback) {
+      this.processingEnabled = false;
+      if (this.animationFrameId) {
+        cancelAnimationFrame(this.animationFrameId);
+        this.animationFrameId = void 0;
+      }
+      if (this.displayCanvas && this.displayCanvas.parentNode) {
+        this.displayCanvas.parentNode.removeChild(this.displayCanvas);
+      }
+      (_a = this.capturedStream) == null ? void 0 : _a.getTracks().forEach((track) => track.stop());
+    } else {
+      await ((_c = (_b = this.processor) == null ? void 0 : _b.writableControl) == null ? void 0 : _c.close());
+      (_d = this.trackGenerator) == null ? void 0 : _d.stop();
+    }
     await this.transformer.destroy();
-    (_a = this.trackGenerator) == null ? void 0 : _a.stop();
   }
 };
 
@@ -79,7 +241,348 @@ import * as vision from "@mediapipe/tasks-vision";
 
 // package.json
 var dependencies = {
-  "@mediapipe/tasks-vision": "0.10.21"
+  "@mediapipe/tasks-vision": "^0.10.22-rc.20250304"
+};
+
+// src/webgl/index.ts
+var blurFragmentShader = `
+  precision highp float;
+  varying vec2 texCoords;
+  uniform sampler2D u_texture;
+  uniform vec2 u_texelSize;
+  uniform vec2 u_direction;
+  uniform float u_radius;
+
+  void main() {
+    float sigma = u_radius;
+    float twoSigmaSq = 2.0 * sigma * sigma;
+    float totalWeight = 0.0;
+    vec3 result = vec3(0.0);
+    const int MAX_SAMPLES = 16;
+    int radius = int(min(float(MAX_SAMPLES), ceil(u_radius)));
+
+    for (int i = -MAX_SAMPLES; i <= MAX_SAMPLES; ++i) {
+      float offset = float(i);
+      if (abs(offset) > float(radius)) continue;
+      float weight = exp(-(offset * offset) / twoSigmaSq);
+      vec2 sampleCoord = texCoords + u_direction * u_texelSize * offset;
+      result += texture2D(u_texture, sampleCoord).rgb * weight;
+      totalWeight += weight;
+    }
+
+    gl_FragColor = vec4(result / totalWeight, 1.0);
+  }
+`;
+var createShaderProgram = (gl) => {
+  const vs = `
+      attribute vec2 position;
+      varying vec2 texCoords;
+    
+      void main() {
+        texCoords = (position + 1.0) / 2.0;
+        texCoords.y = 1.0 - texCoords.y;
+        gl_Position = vec4(position, 0, 1.0);
+      }
+    `;
+  const cS = `
+      precision highp float;
+      varying vec2 texCoords;
+      uniform sampler2D background;
+      uniform sampler2D frame;
+      uniform sampler2D mask;
+      void main() {
+          vec4 maskTex = texture2D(mask, texCoords); 
+          vec4 frameTex = texture2D(frame, texCoords);
+          vec4 bgTex = texture2D(background, texCoords);
+       
+      
+        float a = maskTex.r;
+  
+        gl_FragColor = mix(bgTex, vec4(frameTex.rgb, 1.0), 1.0 - a);
+      
+      }
+    `;
+  const vertexShader = gl.createShader(gl.VERTEX_SHADER);
+  if (!vertexShader) {
+    throw Error("can not create vertex shader");
+  }
+  gl.shaderSource(vertexShader, vs);
+  gl.compileShader(vertexShader);
+  const compositeShader = gl.createShader(gl.FRAGMENT_SHADER);
+  if (!compositeShader) {
+    throw Error("can not create fragment shader");
+  }
+  gl.shaderSource(compositeShader, cS);
+  gl.compileShader(compositeShader);
+  const compositeProgram = gl.createProgram();
+  if (!compositeProgram) {
+    throw Error("can not create composite program");
+  }
+  gl.attachShader(compositeProgram, vertexShader);
+  gl.attachShader(compositeProgram, compositeShader);
+  gl.linkProgram(compositeProgram);
+  let blurProgram = null;
+  let blurVertexShader = null;
+  let blurFrag = null;
+  let blurUniforms = null;
+  blurFrag = gl.createShader(gl.FRAGMENT_SHADER);
+  if (!blurFrag) {
+    throw Error("can not create blur shader");
+  }
+  gl.shaderSource(blurFrag, blurFragmentShader);
+  gl.compileShader(blurFrag);
+  if (!gl.getShaderParameter(blurFrag, gl.COMPILE_STATUS)) {
+    const info = gl.getShaderInfoLog(blurFrag);
+    throw Error(`Failed to compile blur shader: ${info}`);
+  }
+  blurVertexShader = gl.createShader(gl.VERTEX_SHADER);
+  if (!blurVertexShader) {
+    throw Error("can not create blur vertex shader");
+  }
+  gl.shaderSource(blurVertexShader, vs);
+  gl.compileShader(blurVertexShader);
+  blurProgram = gl.createProgram();
+  if (!blurProgram) {
+    throw Error("can not create blur program");
+  }
+  gl.attachShader(blurProgram, blurVertexShader);
+  gl.attachShader(blurProgram, blurFrag);
+  gl.linkProgram(blurProgram);
+  if (!gl.getProgramParameter(blurProgram, gl.LINK_STATUS)) {
+    const info = gl.getProgramInfoLog(blurProgram);
+    throw Error(`Failed to link blur program: ${info}`);
+  }
+  blurUniforms = {
+    position: gl.getAttribLocation(blurProgram, "position"),
+    texture: gl.getUniformLocation(blurProgram, "u_texture"),
+    texelSize: gl.getUniformLocation(blurProgram, "u_texelSize"),
+    direction: gl.getUniformLocation(blurProgram, "u_direction"),
+    radius: gl.getUniformLocation(blurProgram, "u_radius")
+  };
+  return {
+    vertexShader,
+    compositeShader,
+    blurShader: blurFrag,
+    compositeProgram,
+    blurProgram,
+    attribLocations: {
+      position: gl.getAttribLocation(compositeProgram, "position")
+    },
+    uniformLocations: {
+      mask: gl.getUniformLocation(compositeProgram, "mask"),
+      frame: gl.getUniformLocation(compositeProgram, "frame"),
+      background: gl.getUniformLocation(compositeProgram, "background")
+    },
+    blurUniforms
+  };
+};
+function initTexture(gl, texIndex) {
+  const texRef = gl.TEXTURE0 + texIndex;
+  gl.activeTexture(texRef);
+  const texture = gl.createTexture();
+  gl.bindTexture(gl.TEXTURE_2D, texture);
+  gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
+  gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
+  gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
+  gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR);
+  gl.bindTexture(gl.TEXTURE_2D, texture);
+  return texture;
+}
+function createFramebuffer(gl, texture, width, height) {
+  const framebuffer = gl.createFramebuffer();
+  gl.bindFramebuffer(gl.FRAMEBUFFER, framebuffer);
+  gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, texture, 0);
+  gl.bindTexture(gl.TEXTURE_2D, texture);
+  gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, width, height, 0, gl.RGBA, gl.UNSIGNED_BYTE, null);
+  const status = gl.checkFramebufferStatus(gl.FRAMEBUFFER);
+  if (status !== gl.FRAMEBUFFER_COMPLETE) {
+    throw new Error("Framebuffer not complete");
+  }
+  gl.bindFramebuffer(gl.FRAMEBUFFER, null);
+  return framebuffer;
+}
+var createVertexBuffer = (gl) => {
+  if (!gl) {
+    return null;
+  }
+  const vertexBuffer = gl.createBuffer();
+  gl.bindBuffer(gl.ARRAY_BUFFER, vertexBuffer);
+  gl.bufferData(
+    gl.ARRAY_BUFFER,
+    new Float32Array([-1, -1, -1, 1, 1, 1, -1, -1, 1, 1, 1, -1]),
+    gl.STATIC_DRAW
+  );
+  return vertexBuffer;
+};
+var setupWebGL = (canvas) => {
+  const gl = canvas.getContext("webgl2", { premultipliedAlpha: false });
+  let blurRadius = null;
+  if (!gl) {
+    return void 0;
+  }
+  gl.enable(gl.BLEND);
+  gl.blendFunc(gl.SRC_ALPHA, gl.ONE_MINUS_SRC_ALPHA);
+  const {
+    compositeProgram,
+    blurProgram,
+    attribLocations: { position: positionLocation },
+    uniformLocations: {
+      mask: maskTextureLocation,
+      frame: frameTextureLocation,
+      background: bgTextureLocation
+    },
+    blurUniforms
+  } = createShaderProgram(gl);
+  const bgTexture = initTexture(gl, 0);
+  const frameTexture = initTexture(gl, 1);
+  const vertexBuffer = createVertexBuffer(gl);
+  let processTextures = [];
+  let processFramebuffers = [];
+  processTextures.push(initTexture(gl, 3));
+  processTextures.push(initTexture(gl, 4));
+  processFramebuffers.push(createFramebuffer(gl, processTextures[0], canvas.width, canvas.height));
+  processFramebuffers.push(createFramebuffer(gl, processTextures[1], canvas.width, canvas.height));
+  gl.useProgram(compositeProgram);
+  gl.uniform1i(bgTextureLocation, 0);
+  gl.uniform1i(frameTextureLocation, 1);
+  gl.uniform1i(maskTextureLocation, 2);
+  let customBackgroundImage = null;
+  function applyBlur(sourceTexture, width, height) {
+    if (!blurRadius || !blurProgram || !blurUniforms)
+      return bgTexture;
+    gl.useProgram(blurProgram);
+    gl.bindBuffer(gl.ARRAY_BUFFER, vertexBuffer);
+    gl.vertexAttribPointer(blurUniforms.position, 2, gl.FLOAT, false, 0, 0);
+    gl.enableVertexAttribArray(blurUniforms.position);
+    const texelWidth = 1 / width;
+    const texelHeight = 1 / height;
+    gl.bindFramebuffer(gl.FRAMEBUFFER, processFramebuffers[0]);
+    gl.viewport(0, 0, width, height);
+    gl.activeTexture(gl.TEXTURE0);
+    gl.bindTexture(gl.TEXTURE_2D, sourceTexture);
+    gl.uniform1i(blurUniforms.texture, 0);
+    gl.uniform2f(blurUniforms.texelSize, texelWidth, texelHeight);
+    gl.uniform2f(blurUniforms.direction, 1, 0);
+    gl.uniform1f(blurUniforms.radius, blurRadius);
+    gl.drawArrays(gl.TRIANGLES, 0, 6);
+    gl.bindFramebuffer(gl.FRAMEBUFFER, processFramebuffers[1]);
+    gl.viewport(0, 0, width, height);
+    gl.activeTexture(gl.TEXTURE0);
+    gl.bindTexture(gl.TEXTURE_2D, processTextures[0]);
+    gl.uniform1i(blurUniforms.texture, 0);
+    gl.uniform2f(blurUniforms.direction, 0, 1);
+    gl.drawArrays(gl.TRIANGLES, 0, 6);
+    gl.bindFramebuffer(gl.FRAMEBUFFER, null);
+    return processTextures[1];
+  }
+  function render(frame, mask) {
+    if (frame.codedWidth === 0 || mask.width === 0) {
+      return;
+    }
+    const width = frame.displayWidth;
+    const height = frame.displayHeight;
+    gl.activeTexture(gl.TEXTURE1);
+    gl.bindTexture(gl.TEXTURE_2D, frameTexture);
+    gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, frame);
+    let backgroundTexture = bgTexture;
+    if (customBackgroundImage) {
+      gl.activeTexture(gl.TEXTURE0);
+      gl.bindTexture(gl.TEXTURE_2D, bgTexture);
+      gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, customBackgroundImage);
+      backgroundTexture = bgTexture;
+    } else if (blurRadius) {
+      backgroundTexture = applyBlur(frameTexture, width, height);
+    }
+    const maskTexture = mask.getAsWebGLTexture();
+    gl.viewport(0, 0, width, height);
+    gl.clearColor(1, 1, 1, 1);
+    gl.clear(gl.COLOR_BUFFER_BIT);
+    gl.useProgram(compositeProgram);
+    gl.bindBuffer(gl.ARRAY_BUFFER, vertexBuffer);
+    gl.vertexAttribPointer(positionLocation, 2, gl.FLOAT, false, 0, 0);
+    gl.enableVertexAttribArray(positionLocation);
+    gl.activeTexture(gl.TEXTURE0);
+    gl.bindTexture(gl.TEXTURE_2D, backgroundTexture);
+    gl.uniform1i(bgTextureLocation, 0);
+    gl.activeTexture(gl.TEXTURE1);
+    gl.bindTexture(gl.TEXTURE_2D, frameTexture);
+    gl.uniform1i(frameTextureLocation, 1);
+    gl.activeTexture(gl.TEXTURE2);
+    gl.bindTexture(gl.TEXTURE_2D, maskTexture);
+    gl.uniform1i(maskTextureLocation, 2);
+    gl.drawArrays(gl.TRIANGLES, 0, 6);
+    mask.close();
+  }
+  async function setBackgroundImage(image) {
+    customBackgroundImage = null;
+    if (image) {
+      try {
+        const canvasWidth = canvas.width;
+        const canvasHeight = canvas.height;
+        const imgAspect = image.width / image.height;
+        const canvasAspect = canvasWidth / canvasHeight;
+        let sx = 0;
+        let sy = 0;
+        let sWidth = image.width;
+        let sHeight = image.height;
+        if (imgAspect > canvasAspect) {
+          sWidth = Math.round(image.height * canvasAspect);
+          sx = Math.round((image.width - sWidth) / 2);
+        } else if (imgAspect < canvasAspect) {
+          sHeight = Math.round(image.width / canvasAspect);
+          sy = Math.round((image.height - sHeight) / 2);
+        }
+        const croppedImage = await createImageBitmap(image, sx, sy, sWidth, sHeight, {
+          resizeWidth: canvasWidth,
+          resizeHeight: canvasHeight,
+          resizeQuality: "medium"
+        });
+        customBackgroundImage = croppedImage;
+        gl.activeTexture(gl.TEXTURE0);
+        gl.bindTexture(gl.TEXTURE_2D, bgTexture);
+        gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, croppedImage);
+      } catch (error) {
+        console.error("Error processing background image:", error);
+        customBackgroundImage = image;
+        gl.activeTexture(gl.TEXTURE0);
+        gl.bindTexture(gl.TEXTURE_2D, bgTexture);
+        gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, image);
+      }
+    } else {
+      const emptyImage = new ImageData(2, 2);
+      emptyImage.data[0] = 0;
+      emptyImage.data[1] = 0;
+      emptyImage.data[2] = 0;
+      emptyImage.data[3] = 0;
+      gl.activeTexture(gl.TEXTURE0);
+      gl.bindTexture(gl.TEXTURE_2D, bgTexture);
+      gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, emptyImage);
+    }
+  }
+  function setBlurRadius(radius) {
+    blurRadius = radius;
+    setBackgroundImage(null);
+  }
+  function cleanup() {
+    gl.deleteProgram(compositeProgram);
+    gl.deleteProgram(blurProgram);
+    gl.deleteTexture(bgTexture);
+    gl.deleteTexture(frameTexture);
+    for (const texture of processTextures) {
+      gl.deleteTexture(texture);
+    }
+    for (const framebuffer of processFramebuffers) {
+      gl.deleteFramebuffer(framebuffer);
+    }
+    gl.deleteBuffer(vertexBuffer);
+    if (customBackgroundImage) {
+      customBackgroundImage.close();
+      customBackgroundImage = null;
+    }
+    processTextures = [];
+    processFramebuffers = [];
+  }
+  return { render, setBackgroundImage, setBlurRadius, cleanup };
 };
 
 // src/transformers/VideoTransformer.ts
@@ -91,7 +594,6 @@ var VideoTransformer = class {
     outputCanvas,
     inputElement: inputVideo
   }) {
-    var _a;
     if (!(inputVideo instanceof HTMLVideoElement)) {
       throw TypeError("Video transformer needs a HTMLVideoElement as input");
     }
@@ -100,21 +602,29 @@ var VideoTransformer = class {
     });
     this.canvas = outputCanvas || null;
     if (outputCanvas) {
-      this.ctx = ((_a = this.canvas) == null ? void 0 : _a.getContext("2d")) || void 0;
+      this.gl = setupWebGL(
+        this.canvas || new OffscreenCanvas(inputVideo.videoWidth, inputVideo.videoHeight)
+      );
     }
     this.inputVideo = inputVideo;
     this.isDisabled = false;
   }
   async restart({ outputCanvas, inputElement: inputVideo }) {
+    var _a;
     this.canvas = outputCanvas || null;
-    this.ctx = this.canvas.getContext("2d") || void 0;
+    (_a = this.gl) == null ? void 0 : _a.cleanup();
+    this.gl = setupWebGL(
+      this.canvas || new OffscreenCanvas(inputVideo.videoWidth, inputVideo.videoHeight)
+    );
     this.inputVideo = inputVideo;
     this.isDisabled = false;
   }
   async destroy() {
+    var _a;
     this.isDisabled = true;
     this.canvas = void 0;
-    this.ctx = void 0;
+    (_a = this.gl) == null ? void 0 : _a.cleanup();
+    this.gl = void 0;
   }
 };
 
@@ -127,10 +637,10 @@ var BackgroundProcessor = class extends VideoTransformer {
     this.update(opts);
   }
   static get isSupported() {
-    return typeof OffscreenCanvas !== "undefined";
+    return typeof OffscreenCanvas !== "undefined" && typeof VideoFrame !== "undefined" && typeof createImageBitmap !== "undefined" && !!document.createElement("canvas").getContext("webgl2");
   }
   async init({ outputCanvas, inputElement: inputVideo }) {
-    var _a, _b, _c, _d, _e;
+    var _a, _b, _c, _d, _e, _f;
     await super.init({ outputCanvas, inputElement: inputVideo });
     const fileSet = await vision.FilesetResolver.forVisionTasks(
       (_b = (_a = this.options.assetPaths) == null ? void 0 : _a.tasksVisionFileSet) != null ? _b : `https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@${dependencies["@mediapipe/tasks-vision"]}/wasm`
@@ -141,6 +651,7 @@ var BackgroundProcessor = class extends VideoTransformer {
         delegate: "GPU",
         ...this.options.segmenterOptions
       },
+      canvas: this.canvas,
       runningMode: "VIDEO",
       outputCategoryMask: true,
       outputConfidenceMasks: false
@@ -150,6 +661,9 @@ var BackgroundProcessor = class extends VideoTransformer {
         (err) => console.error("Error while loading processor background image: ", err)
       );
     }
+    if (this.options.blurRadius) {
+      (_f = this.gl) == null ? void 0 : _f.setBlurRadius(this.options.blurRadius);
+    }
   }
   async destroy() {
     var _a;
@@ -158,6 +672,7 @@ var BackgroundProcessor = class extends VideoTransformer {
     this.backgroundImage = null;
   }
   async loadBackground(path) {
+    var _a;
     const img = new Image();
     await new Promise((resolve, reject) => {
       img.crossOrigin = "Anonymous";
@@ -166,12 +681,12 @@ var BackgroundProcessor = class extends VideoTransformer {
       img.src = path;
     });
     const imageData = await createImageBitmap(img);
-    this.backgroundImage = imageData;
+    (_a = this.gl) == null ? void 0 : _a.setBackgroundImage(imageData);
   }
   async transform(frame, controller) {
     var _a;
     try {
-      if (!(frame instanceof VideoFrame)) {
+      if (!(frame instanceof VideoFrame) || frame.codedWidth === 0 || frame.codedHeight === 0) {
         console.debug("empty frame detected, ignoring");
         return;
       }
@@ -182,118 +697,97 @@ var BackgroundProcessor = class extends VideoTransformer {
       if (!this.canvas) {
         throw TypeError("Canvas needs to be initialized first");
       }
+      this.canvas.width = frame.displayWidth;
+      this.canvas.height = frame.displayHeight;
       let startTimeMs = performance.now();
-      (_a = this.imageSegmenter) == null ? void 0 : _a.segmentForVideo(
-        this.inputVideo,
-        startTimeMs,
-        (result) => this.segmentationResults = result
-      );
-      if (this.blurRadius) {
-        await this.blurBackground(frame);
-      } else {
-        await this.drawVirtualBackground(frame);
-      }
-      const newFrame = new VideoFrame(this.canvas, {
-        timestamp: frame.timestamp || Date.now()
+      (_a = this.imageSegmenter) == null ? void 0 : _a.segmentForVideo(frame, startTimeMs, (result) => {
+        var _a2, _b;
+        const segmentationTimeMs = performance.now() - startTimeMs;
+        this.segmentationResults = result;
+        this.drawFrame(frame);
+        if (this.canvas && this.canvas.width > 0 && this.canvas.height > 0) {
+          const newFrame = new VideoFrame(this.canvas, {
+            timestamp: frame.timestamp || Date.now()
+          });
+          const filterTimeMs = performance.now() - startTimeMs - segmentationTimeMs;
+          const stats = {
+            processingTimeMs: performance.now() - startTimeMs,
+            segmentationTimeMs,
+            filterTimeMs
+          };
+          (_b = (_a2 = this.options).onFrameProcessed) == null ? void 0 : _b.call(_a2, stats);
+          controller.enqueue(newFrame);
+        } else {
+          controller.enqueue(frame);
+        }
+        frame.close();
       });
-      controller.enqueue(newFrame);
-    } finally {
+    } catch (e) {
+      console.error("Error while processing frame: ", e);
       frame == null ? void 0 : frame.close();
     }
   }
   async update(opts) {
-    this.options = opts;
+    var _a;
+    this.options = { ...this.options, ...opts };
     if (opts.blurRadius) {
-      this.blurRadius = opts.blurRadius;
+      (_a = this.gl) == null ? void 0 : _a.setBlurRadius(opts.blurRadius);
     } else if (opts.imagePath) {
       await this.loadBackground(opts.imagePath);
     }
   }
-  async drawVirtualBackground(frame) {
-    var _a;
-    if (!this.canvas || !this.ctx || !this.segmentationResults || !this.inputVideo)
-      return;
-    if (((_a = this.segmentationResults) == null ? void 0 : _a.categoryMask) && this.segmentationResults.categoryMask.width > 0) {
-      this.ctx.globalCompositeOperation = "copy";
-      const bitmap = await maskToBitmap(
-        this.segmentationResults.categoryMask,
-        this.segmentationResults.categoryMask.width,
-        this.segmentationResults.categoryMask.height
-      );
-      this.ctx.drawImage(bitmap, 0, 0, this.canvas.width, this.canvas.height);
-      this.ctx.filter = "none";
-      this.ctx.globalCompositeOperation = "source-in";
-      if (this.backgroundImage) {
-        this.ctx.drawImage(
-          this.backgroundImage,
-          0,
-          0,
-          this.backgroundImage.width,
-          this.backgroundImage.height,
-          0,
-          0,
-          this.canvas.width,
-          this.canvas.height
-        );
-      } else {
-        this.ctx.fillStyle = "#00FF00";
-        this.ctx.fillRect(0, 0, this.canvas.width, this.canvas.height);
-      }
-      this.ctx.globalCompositeOperation = "destination-over";
-    }
-    this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);
-  }
-  async blurBackground(frame) {
-    var _a, _b, _c;
-    if (!this.ctx || !this.canvas || !((_b = (_a = this.segmentationResults) == null ? void 0 : _a.categoryMask) == null ? void 0 : _b.canvas) || !this.inputVideo) {
+  async drawFrame(frame) {
+    if (!this.canvas || !this.gl || !this.segmentationResults || !this.inputVideo)
       return;
-    }
-    this.ctx.save();
-    this.ctx.globalCompositeOperation = "copy";
-    if (((_c = this.segmentationResults) == null ? void 0 : _c.categoryMask) && this.segmentationResults.categoryMask.width > 0) {
-      const bitmap = await maskToBitmap(
-        this.segmentationResults.categoryMask,
-        this.segmentationResults.categoryMask.width,
-        this.segmentationResults.categoryMask.height
-      );
-      this.ctx.globalCompositeOperation = "copy";
-      this.ctx.drawImage(bitmap, 0, 0, this.canvas.width, this.canvas.height);
-      this.ctx.filter = "none";
-      this.ctx.globalCompositeOperation = "source-out";
-      this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);
-      this.ctx.globalCompositeOperation = "destination-over";
-      this.ctx.filter = `blur(${this.blurRadius}px)`;
-      this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);
-      this.ctx.restore();
+    const mask = this.segmentationResults.categoryMask;
+    if (mask) {
+      this.gl.render(frame, mask);
     }
   }
 };
-function maskToBitmap(mask, videoWidth, videoHeight) {
-  const dataArray = new Uint8ClampedArray(videoWidth * videoHeight * 4);
-  const result = mask.getAsUint8Array();
-  for (let i = 0; i < result.length; i += 1) {
-    dataArray[i * 4] = result[i];
-    dataArray[i * 4 + 1] = result[i];
-    dataArray[i * 4 + 2] = result[i];
-    dataArray[i * 4 + 3] = result[i];
-  }
-  const dataNew = new ImageData(dataArray, videoWidth, videoHeight);
-  return createImageBitmap(dataNew);
-}
 
 // src/index.ts
-var BackgroundBlur = (blurRadius = 10, segmenterOptions) => {
-  return BackgroundProcessor2({ blurRadius, segmenterOptions }, "background-blur");
+var supportsBackgroundProcessors = () => BackgroundProcessor.isSupported && ProcessorWrapper.isSupported;
+var supportsModernBackgroundProcessors = () => BackgroundProcessor.isSupported && ProcessorWrapper.hasModernApiSupport;
+var BackgroundBlur = (blurRadius = 10, segmenterOptions, onFrameProcessed, processorOptions) => {
+  return BackgroundProcessor2(
+    {
+      blurRadius,
+      segmenterOptions,
+      onFrameProcessed,
+      ...processorOptions
+    },
+    "background-blur"
+  );
 };
-var VirtualBackground = (imagePath, segmenterOptions) => {
-  return BackgroundProcessor2({ imagePath, segmenterOptions }, "virtual-background");
+var VirtualBackground = (imagePath, segmenterOptions, onFrameProcessed, processorOptions) => {
+  return BackgroundProcessor2(
+    {
+      imagePath,
+      segmenterOptions,
+      onFrameProcessed,
+      ...processorOptions
+    },
+    "virtual-background"
+  );
 };
 var BackgroundProcessor2 = (options, name = "background-processor") => {
-  const isProcessorSupported = ProcessorWrapper.isSupported && BackgroundProcessor.isSupported;
+  const isTransformerSupported = BackgroundProcessor.isSupported;
+  const isProcessorSupported = ProcessorWrapper.isSupported;
+  if (!isTransformerSupported) {
+    throw new Error("Background transformer is not supported in this browser");
+  }
   if (!isProcessorSupported) {
-    throw new Error("processor is not supported in this browser");
+    throw new Error(
+      "Neither MediaStreamTrackProcessor nor canvas.captureStream() fallback is supported in this browser"
+    );
   }
-  const processor = new ProcessorWrapper(new BackgroundProcessor(options), name);
+  const { blurRadius, imagePath, segmenterOptions, onFrameProcessed, ...processorOpts } = options;
+  const processor = new ProcessorWrapper(
+    new BackgroundProcessor({ blurRadius, imagePath, segmenterOptions, onFrameProcessed }),
+    name,
+    processorOpts
+  );
   return processor;
 };
 export {
@@ -302,6 +796,8 @@ export {
   BackgroundProcessor as BackgroundTransformer,
   ProcessorWrapper,
   VideoTransformer,
-  VirtualBackground
+  VirtualBackground,
+  supportsBackgroundProcessors,
+  supportsModernBackgroundProcessors
 };
 //# sourceMappingURL=index.mjs.map