@livekit/track-processors 0.5.4 → 0.5.5

package/dist/index.js

@@ -264,6 +264,28 @@ function initTexture(gl, texIndex) {
   gl.bindTexture(gl.TEXTURE_2D, texture);
   return texture;
 }
+function createShader(gl, type, source) {
+  const shader = gl.createShader(type);
+  gl.shaderSource(shader, source);
+  gl.compileShader(shader);
+  if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) {
+    console.error("Shader compile failed:", gl.getShaderInfoLog(shader));
+    gl.deleteShader(shader);
+    throw new Error("Shader compile failed");
+  }
+  return shader;
+}
+function createProgram(gl, vs, fs) {
+  const program = gl.createProgram();
+  gl.attachShader(program, vs);
+  gl.attachShader(program, fs);
+  gl.linkProgram(program);
+  if (!gl.getProgramParameter(program, gl.LINK_STATUS)) {
+    console.error("Program link failed:", gl.getProgramInfoLog(program));
+    throw new Error("Program link failed");
+  }
+  return program;
+}
 function createFramebuffer(gl, texture, width, height) {
   const framebuffer = gl.createFramebuffer();
   gl.bindFramebuffer(gl.FRAMEBUFFER, framebuffer);
@@ -328,7 +350,7 @@ var vertexShaderSource = (flipY = true) => `#version 300 es
 
 // src/webgl/shader-programs/blurShader.ts
 var blurFragmentShader = glsl`#version 300 es
-  precision highp float;
+  precision mediump float;
   in vec2 texCoords;
   uniform sampler2D u_texture;
   uniform vec2 u_texelSize;
@@ -357,33 +379,9 @@ var blurFragmentShader = glsl`#version 300 es
   }
 `;
 function createBlurProgram(gl) {
-  const blurFrag = gl.createShader(gl.FRAGMENT_SHADER);
-  if (!blurFrag) {
-    throw Error("cannot create blur shader");
-  }
-  gl.shaderSource(blurFrag, blurFragmentShader);
-  gl.compileShader(blurFrag);
-  if (!gl.getShaderParameter(blurFrag, gl.COMPILE_STATUS)) {
-    const info = gl.getShaderInfoLog(blurFrag);
-    throw Error(`Failed to compile blur shader: ${info}`);
-  }
-  const blurVertexShader = gl.createShader(gl.VERTEX_SHADER);
-  if (!blurVertexShader) {
-    throw Error("cannot create blur vertex shader");
-  }
-  gl.shaderSource(blurVertexShader, vertexShaderSource());
-  gl.compileShader(blurVertexShader);
-  const blurProgram = gl.createProgram();
-  if (!blurProgram) {
-    throw Error("cannot create blur program");
-  }
-  gl.attachShader(blurProgram, blurVertexShader);
-  gl.attachShader(blurProgram, blurFrag);
-  gl.linkProgram(blurProgram);
-  if (!gl.getProgramParameter(blurProgram, gl.LINK_STATUS)) {
-    const info = gl.getProgramInfoLog(blurProgram);
-    throw Error(`Failed to link blur program: ${info}`);
-  }
+  const blurVertexShader = createShader(gl, gl.VERTEX_SHADER, vertexShaderSource());
+  const blurFrag = createShader(gl, gl.FRAGMENT_SHADER, blurFragmentShader);
+  const blurProgram = createProgram(gl, blurVertexShader, blurFrag);
   const blurUniforms = {
     position: gl.getAttribLocation(blurProgram, "position"),
     texture: gl.getUniformLocation(blurProgram, "u_texture"),
@@ -458,37 +456,9 @@ void main() {
 }
 `;
 function createBoxBlurProgram(gl) {
-  const vertexShader = gl.createShader(gl.VERTEX_SHADER);
-  if (!vertexShader) {
-    throw Error("cannot create vertex shader");
-  }
-  gl.shaderSource(vertexShader, vertexShaderSource());
-  gl.compileShader(vertexShader);
-  if (!gl.getShaderParameter(vertexShader, gl.COMPILE_STATUS)) {
-    const info = gl.getShaderInfoLog(vertexShader);
-    throw Error(`Failed to compile vertex shader: ${info}`);
-  }
-  const fragmentShader = gl.createShader(gl.FRAGMENT_SHADER);
-  if (!fragmentShader) {
-    throw Error("cannot create fragment shader");
-  }
-  gl.shaderSource(fragmentShader, boxBlurFragmentShader);
-  gl.compileShader(fragmentShader);
-  if (!gl.getShaderParameter(fragmentShader, gl.COMPILE_STATUS)) {
-    const info = gl.getShaderInfoLog(fragmentShader);
-    throw Error(`Failed to compile box blur shader: ${info}`);
-  }
-  const program = gl.createProgram();
-  if (!program) {
-    throw Error("cannot create box blur program");
-  }
-  gl.attachShader(program, vertexShader);
-  gl.attachShader(program, fragmentShader);
-  gl.linkProgram(program);
-  if (!gl.getProgramParameter(program, gl.LINK_STATUS)) {
-    const info = gl.getProgramInfoLog(program);
-    throw Error(`Failed to link box blur program: ${info}`);
-  }
+  const vertexShader = createShader(gl, gl.VERTEX_SHADER, vertexShaderSource());
+  const fragmentShader = createShader(gl, gl.FRAGMENT_SHADER, boxBlurFragmentShader);
+  const program = createProgram(gl, vertexShader, fragmentShader);
   const uniforms = {
     position: gl.getAttribLocation(program, "position"),
     texture: gl.getUniformLocation(program, "u_texture"),
@@ -506,7 +476,7 @@ function createBoxBlurProgram(gl) {
 
 // src/webgl/shader-programs/compositeShader.ts
 var compositeFragmentShader = glsl`#version 300 es
-  precision highp float;
+  precision mediump float;
   in vec2 texCoords;
   uniform sampler2D background;
   uniform sampler2D frame;
@@ -536,37 +506,9 @@ var compositeFragmentShader = glsl`#version 300 es
   }
 `;
 function createCompositeProgram(gl) {
-  const vertexShader = gl.createShader(gl.VERTEX_SHADER);
-  if (!vertexShader) {
-    throw Error("cannot create vertex shader");
-  }
-  gl.shaderSource(vertexShader, vertexShaderSource());
-  gl.compileShader(vertexShader);
-  if (!gl.getShaderParameter(vertexShader, gl.COMPILE_STATUS)) {
-    const info = gl.getShaderInfoLog(vertexShader);
-    throw Error(`Failed to compile vertex shader: ${info}`);
-  }
-  const compositeShader = gl.createShader(gl.FRAGMENT_SHADER);
-  if (!compositeShader) {
-    throw Error("cannot create fragment shader");
-  }
-  gl.shaderSource(compositeShader, compositeFragmentShader);
-  gl.compileShader(compositeShader);
-  if (!gl.getShaderParameter(compositeShader, gl.COMPILE_STATUS)) {
-    const info = gl.getShaderInfoLog(compositeShader);
-    throw Error(`Failed to compile composite shader: ${info}`);
-  }
-  const compositeProgram = gl.createProgram();
-  if (!compositeProgram) {
-    throw Error("cannot create composite program");
-  }
-  gl.attachShader(compositeProgram, vertexShader);
-  gl.attachShader(compositeProgram, compositeShader);
-  gl.linkProgram(compositeProgram);
-  if (!gl.getProgramParameter(compositeProgram, gl.LINK_STATUS)) {
-    const info = gl.getProgramInfoLog(compositeProgram);
-    throw Error(`Failed to link composite program: ${info}`);
-  }
+  const vertexShader = createShader(gl, gl.VERTEX_SHADER, vertexShaderSource());
+  const compositeShader = createShader(gl, gl.FRAGMENT_SHADER, compositeFragmentShader);
+  const compositeProgram = createProgram(gl, vertexShader, compositeShader);
   const attribLocations = {
     position: gl.getAttribLocation(compositeProgram, "position")
   };
@@ -585,6 +527,63 @@ function createCompositeProgram(gl) {
   };
 }
 
+// src/webgl/shader-programs/downSampler.ts
+function createDownSampler(gl, width, height) {
+  const texture = gl.createTexture();
+  gl.bindTexture(gl.TEXTURE_2D, texture);
+  gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, width, height, 0, gl.RGBA, gl.UNSIGNED_BYTE, null);
+  gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
+  gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR);
+  gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
+  gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
+  const framebuffer = gl.createFramebuffer();
+  gl.bindFramebuffer(gl.FRAMEBUFFER, framebuffer);
+  gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, texture, 0);
+  const vertexSource = `
+      attribute vec2 position;
+      varying vec2 v_uv;
+      void main() {
+        v_uv = (position + 1.0) * 0.5;
+        gl_Position = vec4(position, 0.0, 1.0);
+      }
+    `;
+  const fragmentSource = `
+      precision mediump float;
+      varying vec2 v_uv;
+      uniform sampler2D u_texture;
+      void main() {
+        gl_FragColor = texture2D(u_texture, v_uv);
+      }
+    `;
+  const vertShader = createShader(gl, gl.VERTEX_SHADER, vertexSource);
+  const fragShader = createShader(gl, gl.FRAGMENT_SHADER, fragmentSource);
+  const program = createProgram(gl, vertShader, fragShader);
+  const uniforms = {
+    texture: gl.getUniformLocation(program, "u_texture"),
+    position: gl.getAttribLocation(program, "position")
+  };
+  return {
+    framebuffer,
+    texture,
+    program,
+    uniforms
+  };
+}
+function applyDownsampling(gl, inputTexture, downSampler, vertexBuffer, width, height) {
+  gl.useProgram(downSampler.program);
+  gl.bindFramebuffer(gl.FRAMEBUFFER, downSampler.framebuffer);
+  gl.viewport(0, 0, width, height);
+  gl.bindBuffer(gl.ARRAY_BUFFER, vertexBuffer);
+  gl.enableVertexAttribArray(downSampler.uniforms.position);
+  gl.vertexAttribPointer(downSampler.uniforms.position, 2, gl.FLOAT, false, 0, 0);
+  gl.activeTexture(gl.TEXTURE0);
+  gl.bindTexture(gl.TEXTURE_2D, inputTexture);
+  gl.uniform1i(downSampler.uniforms.texture, 0);
+  gl.drawArrays(gl.TRIANGLES, 0, 6);
+  gl.bindFramebuffer(gl.FRAMEBUFFER, null);
+  return downSampler.texture;
+}
+
 // src/webgl/index.ts
 var setupWebGL = (canvas) => {
   const gl = canvas.getContext("webgl2", {
@@ -592,6 +591,8 @@ var setupWebGL = (canvas) => {
     premultipliedAlpha: true
   });
   let blurRadius = null;
+  let maskBlurRadius = 8;
+  const downsampleFactor = 4;
   if (!gl) {
     console.error("Failed to create WebGL context");
     return void 0;
@@ -620,27 +621,36 @@ var setupWebGL = (canvas) => {
   }
   let bgBlurTextures = [];
   let bgBlurFrameBuffers = [];
-  let maskBlurTextures = [];
-  let maskBlurFrameBuffers = [];
+  let blurredMaskTexture = null;
+  let finalMaskTextures = [];
+  let readMaskIndex = 0;
+  let writeMaskIndex = 1;
   bgBlurTextures.push(initTexture(gl, 3));
   bgBlurTextures.push(initTexture(gl, 4));
-  bgBlurFrameBuffers.push(createFramebuffer(gl, bgBlurTextures[0], canvas.width, canvas.height));
-  bgBlurFrameBuffers.push(createFramebuffer(gl, bgBlurTextures[1], canvas.width, canvas.height));
-  maskBlurTextures.push(initTexture(gl, 5));
-  maskBlurTextures.push(initTexture(gl, 6));
-  maskBlurFrameBuffers.push(
-    createFramebuffer(gl, maskBlurTextures[0], canvas.width, canvas.height)
+  const bgBlurTextureWidth = Math.floor(canvas.width / downsampleFactor);
+  const bgBlurTextureHeight = Math.floor(canvas.height / downsampleFactor);
+  const downSampler = createDownSampler(gl, bgBlurTextureWidth, bgBlurTextureHeight);
+  bgBlurFrameBuffers.push(
+    createFramebuffer(gl, bgBlurTextures[0], bgBlurTextureWidth, bgBlurTextureHeight)
   );
-  maskBlurFrameBuffers.push(
-    createFramebuffer(gl, maskBlurTextures[1], canvas.width, canvas.height)
+  bgBlurFrameBuffers.push(
+    createFramebuffer(gl, bgBlurTextures[1], bgBlurTextureWidth, bgBlurTextureHeight)
   );
+  const tempMaskTexture = initTexture(gl, 5);
+  const tempMaskFrameBuffer = createFramebuffer(gl, tempMaskTexture, canvas.width, canvas.height);
+  finalMaskTextures.push(initTexture(gl, 6));
+  finalMaskTextures.push(initTexture(gl, 7));
+  const finalMaskFrameBuffers = [
+    createFramebuffer(gl, finalMaskTextures[0], canvas.width, canvas.height),
+    createFramebuffer(gl, finalMaskTextures[1], canvas.width, canvas.height)
+  ];
   gl.useProgram(compositeProgram);
   gl.uniform1i(bgTextureLocation, 0);
   gl.uniform1i(frameTextureLocation, 1);
   gl.uniform1i(maskTextureLocation, 2);
   let customBackgroundImage = emptyImageData;
-  function render(frame, mask) {
-    if (frame.codedWidth === 0 || mask.width === 0) {
+  function renderFrame(frame) {
+    if (frame.codedWidth === 0 || finalMaskTextures.length === 0) {
       return;
     }
     const width = frame.displayWidth;
@@ -650,11 +660,19 @@ var setupWebGL = (canvas) => {
     gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, frame);
     let backgroundTexture = bgTexture;
     if (blurRadius) {
-      backgroundTexture = applyBlur(
+      const downSampledFrameTexture = applyDownsampling(
         gl,
         frameTexture,
-        width,
-        height,
+        downSampler,
+        vertexBuffer,
+        bgBlurTextureWidth,
+        bgBlurTextureHeight
+      );
+      backgroundTexture = applyBlur(
+        gl,
+        downSampledFrameTexture,
+        bgBlurTextureWidth,
+        bgBlurTextureHeight,
         blurRadius,
         blurProgram,
         blurUniforms,
@@ -668,19 +686,6 @@ var setupWebGL = (canvas) => {
       gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, customBackgroundImage);
       backgroundTexture = bgTexture;
     }
-    const blurredMaskTexture = applyBlur(
-      gl,
-      mask.getAsWebGLTexture(),
-      width,
-      height,
-      blurRadius || 1,
-      // Use a default blur radius if not set
-      boxBlurProgram,
-      boxBlurUniforms,
-      vertexBuffer,
-      maskBlurFrameBuffers,
-      maskBlurTextures
-    );
     gl.viewport(0, 0, width, height);
     gl.clearColor(1, 1, 1, 1);
     gl.clear(gl.COLOR_BUFFER_BIT);
@@ -695,10 +700,9 @@ var setupWebGL = (canvas) => {
     gl.bindTexture(gl.TEXTURE_2D, frameTexture);
     gl.uniform1i(frameTextureLocation, 1);
     gl.activeTexture(gl.TEXTURE2);
-    gl.bindTexture(gl.TEXTURE_2D, blurredMaskTexture);
+    gl.bindTexture(gl.TEXTURE_2D, finalMaskTextures[readMaskIndex]);
     gl.uniform1i(maskTextureLocation, 2);
     gl.drawArrays(gl.TRIANGLES, 0, 6);
-    mask.close();
   }
   async function setBackgroundImage(image) {
     customBackgroundImage = emptyImageData;
@@ -718,28 +722,56 @@ var setupWebGL = (canvas) => {
     gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, customBackgroundImage);
   }
   function setBlurRadius(radius) {
-    blurRadius = radius;
+    blurRadius = radius ? Math.max(1, Math.floor(radius / downsampleFactor)) : null;
     setBackgroundImage(null);
   }
+  function updateMask(mask) {
+    const tempFramebuffers = [tempMaskFrameBuffer, finalMaskFrameBuffers[writeMaskIndex]];
+    const tempTextures = [tempMaskTexture, finalMaskTextures[writeMaskIndex]];
+    applyBlur(
+      gl,
+      mask,
+      canvas.width,
+      canvas.height,
+      maskBlurRadius || 1,
+      boxBlurProgram,
+      boxBlurUniforms,
+      vertexBuffer,
+      tempFramebuffers,
+      tempTextures
+    );
+    readMaskIndex = writeMaskIndex;
+    writeMaskIndex = 1 - writeMaskIndex;
+  }
   function cleanup() {
     gl.deleteProgram(compositeProgram);
     gl.deleteProgram(blurProgram);
     gl.deleteProgram(boxBlurProgram);
     gl.deleteTexture(bgTexture);
     gl.deleteTexture(frameTexture);
+    gl.deleteTexture(tempMaskTexture);
+    gl.deleteFramebuffer(tempMaskFrameBuffer);
     for (const texture of bgBlurTextures) {
       gl.deleteTexture(texture);
     }
     for (const framebuffer of bgBlurFrameBuffers) {
       gl.deleteFramebuffer(framebuffer);
     }
-    for (const texture of maskBlurTextures) {
+    for (const texture of finalMaskTextures) {
       gl.deleteTexture(texture);
     }
-    for (const framebuffer of maskBlurFrameBuffers) {
+    for (const framebuffer of finalMaskFrameBuffers) {
       gl.deleteFramebuffer(framebuffer);
     }
     gl.deleteBuffer(vertexBuffer);
+    if (blurredMaskTexture) {
+      gl.deleteTexture(blurredMaskTexture);
+    }
+    if (downSampler) {
+      gl.deleteTexture(downSampler.texture);
+      gl.deleteFramebuffer(downSampler.framebuffer);
+      gl.deleteProgram(downSampler.program);
+    }
     if (customBackgroundImage) {
       if (customBackgroundImage instanceof ImageBitmap) {
         customBackgroundImage.close();
@@ -748,10 +780,9 @@ var setupWebGL = (canvas) => {
     }
     bgBlurTextures = [];
     bgBlurFrameBuffers = [];
-    maskBlurTextures = [];
-    maskBlurFrameBuffers = [];
+    finalMaskTextures = [];
   }
-  return { render, setBackgroundImage, setBlurRadius, cleanup };
+  return { renderFrame, updateMask, setBackgroundImage, setBlurRadius, cleanup };
 };
 
 // src/transformers/VideoTransformer.ts
@@ -802,6 +833,7 @@ var BackgroundProcessor = class extends VideoTransformer {
   constructor(opts) {
     super();
     this.backgroundImage = null;
+    this.segmentationTimeMs = 0;
     this.options = opts;
     this.update(opts);
   }
@@ -853,7 +885,7 @@ var BackgroundProcessor = class extends VideoTransformer {
     (_a = this.gl) == null ? void 0 : _a.setBackgroundImage(imageData);
   }
   async transform(frame, controller) {
-    var _a;
+    var _a, _b;
     try {
       if (!(frame instanceof VideoFrame) || frame.codedWidth === 0 || frame.codedHeight === 0) {
         console.debug("empty frame detected, ignoring");
@@ -863,37 +895,49 @@ var BackgroundProcessor = class extends VideoTransformer {
         controller.enqueue(frame);
         return;
       }
+      const frameTimeMs = Date.now();
       if (!this.canvas) {
         throw TypeError("Canvas needs to be initialized first");
       }
       this.canvas.width = frame.displayWidth;
       this.canvas.height = frame.displayHeight;
-      let startTimeMs = performance.now();
-      (_a = this.imageSegmenter) == null ? void 0 : _a.segmentForVideo(frame, startTimeMs, (result) => {
-        var _a2, _b;
-        const segmentationTimeMs = performance.now() - startTimeMs;
-        this.segmentationResults = result;
-        this.drawFrame(frame);
-        if (this.canvas && this.canvas.width > 0 && this.canvas.height > 0) {
-          const newFrame = new VideoFrame(this.canvas, {
-            timestamp: frame.timestamp || Date.now()
+      const segmentationPromise = new Promise((resolve, reject) => {
+        var _a2;
+        try {
+          let segmentationStartTimeMs = performance.now();
+          (_a2 = this.imageSegmenter) == null ? void 0 : _a2.segmentForVideo(frame, segmentationStartTimeMs, (result) => {
+            this.segmentationTimeMs = performance.now() - segmentationStartTimeMs;
+            this.segmentationResults = result;
+            this.updateMask(result.categoryMask);
+            result.close();
+            resolve();
           });
-          const filterTimeMs = performance.now() - startTimeMs - segmentationTimeMs;
-          const stats = {
-            processingTimeMs: performance.now() - startTimeMs,
-            segmentationTimeMs,
-            filterTimeMs
-          };
-          (_b = (_a2 = this.options).onFrameProcessed) == null ? void 0 : _b.call(_a2, stats);
-          controller.enqueue(newFrame);
-        } else {
-          controller.enqueue(frame);
+        } catch (e) {
+          reject(e);
         }
-        frame.close();
       });
+      const filterStartTimeMs = performance.now();
+      this.drawFrame(frame);
+      if (this.canvas && this.canvas.width > 0 && this.canvas.height > 0) {
+        const newFrame = new VideoFrame(this.canvas, {
+          timestamp: frame.timestamp || frameTimeMs
+        });
+        controller.enqueue(newFrame);
+        const filterTimeMs = performance.now() - filterStartTimeMs;
+        const stats = {
+          processingTimeMs: this.segmentationTimeMs + filterTimeMs,
+          segmentationTimeMs: this.segmentationTimeMs,
+          filterTimeMs
+        };
+        (_b = (_a = this.options).onFrameProcessed) == null ? void 0 : _b.call(_a, stats);
+      } else {
+        controller.enqueue(frame);
+      }
+      await segmentationPromise;
     } catch (e) {
       console.error("Error while processing frame: ", e);
-      frame == null ? void 0 : frame.close();
+    } finally {
+      frame.close();
     }
   }
   async update(opts) {
@@ -906,12 +950,16 @@ var BackgroundProcessor = class extends VideoTransformer {
     }
   }
   async drawFrame(frame) {
-    if (!this.canvas || !this.gl || !this.segmentationResults || !this.inputVideo)
+    var _a;
+    if (!this.gl)
       return;
-    const mask = this.segmentationResults.categoryMask;
-    if (mask) {
-      this.gl.render(frame, mask);
-    }
+    (_a = this.gl) == null ? void 0 : _a.renderFrame(frame);
+  }
+  async updateMask(mask) {
+    var _a;
+    if (!mask)
+      return;
+    (_a = this.gl) == null ? void 0 : _a.updateMask(mask.getAsWebGLTexture());
   }
 };