@livekit/track-processors 0.5.4 → 0.5.6

package/src/transformers/VideoTransformer.ts

@@ -1,3 +1,4 @@
+import { createCanvas } from '../utils';
 import { setupWebGL } from '../webgl/index';
 import { VideoTrackTransformer, VideoTransformerInitOptions } from './types';
 
@@ -6,7 +7,7 @@ export default abstract class VideoTransformer<Options extends Record<string, un
 {
   transformer?: TransformStream;
 
-  canvas?: OffscreenCanvas;
+  canvas?: OffscreenCanvas | HTMLCanvasElement;
 
   // ctx?: OffscreenCanvasRenderingContext2D;
 
@@ -31,7 +32,7 @@ export default abstract class VideoTransformer<Options extends Record<string, un
     if (outputCanvas) {
       // this.ctx = this.canvas?.getContext('2d') || undefined;
       this.gl = setupWebGL(
-        this.canvas || new OffscreenCanvas(inputVideo.videoWidth, inputVideo.videoHeight),
+        this.canvas || createCanvas(inputVideo.videoWidth, inputVideo.videoHeight),
       );
     }
     this.inputVideo = inputVideo;
@@ -42,7 +43,7 @@ export default abstract class VideoTransformer<Options extends Record<string, un
     this.canvas = outputCanvas || null;
     this.gl?.cleanup();
     this.gl = setupWebGL(
-      this.canvas || new OffscreenCanvas(inputVideo.videoWidth, inputVideo.videoHeight),
+      this.canvas || createCanvas(inputVideo.videoWidth, inputVideo.videoHeight),
     );
 
     this.inputVideo = inputVideo;

package/src/transformers/types.ts

@@ -3,7 +3,7 @@ export type TrackTransformerInitOptions = {
 };
 
 export interface VideoTransformerInitOptions extends TrackTransformerInitOptions {
-  outputCanvas: OffscreenCanvas;
+  outputCanvas: OffscreenCanvas | HTMLCanvasElement;
   inputElement: HTMLVideoElement;
 }
 

package/src/utils.ts

@@ -1,6 +1,5 @@
 /* eslint-disable @typescript-eslint/naming-convention */
-export const supportsProcessor = typeof MediaStreamTrackGenerator !== 'undefined';
-export const supportsOffscreenCanvas = typeof OffscreenCanvas !== 'undefined';
+export const supportsOffscreenCanvas = () => typeof OffscreenCanvas !== 'undefined';
 
 async function sleep(time: number) {
   return new Promise((resolve) => setTimeout(resolve, time));
@@ -23,3 +22,13 @@ export async function waitForTrackResolution(track: MediaStreamTrack) {
   }
   return { width: undefined, height: undefined };
 }
+
+export function createCanvas(width: number, height: number) {
+  if (supportsOffscreenCanvas()) {
+    return new OffscreenCanvas(width, height);
+  }
+  const canvas = document.createElement('canvas');
+  canvas.width = width;
+  canvas.height = height;
+  return canvas;
+}

package/src/webgl/index.ts

@@ -1,22 +1,30 @@
-import { MPMask } from '@mediapipe/tasks-vision';
+/**
+ * WebGL setup for the mask processor
+ * potential improvements:
+ * - downsample the video texture in background blur scenario before applying the (gaussian) blur for better performance
+ *
+ */
 import { applyBlur, createBlurProgram } from './shader-programs/blurShader';
 import { createBoxBlurProgram } from './shader-programs/boxBlurShader';
 import { createCompositeProgram } from './shader-programs/compositeShader';
+import { applyDownsampling, createDownSampler } from './shader-programs/downSampler';
 import {
   createFramebuffer,
   createVertexBuffer,
-  emptyImageData,
+  getEmptyImageData,
   initTexture,
   resizeImageToCover,
 } from './utils';
 
-export const setupWebGL = (canvas: OffscreenCanvas) => {
+export const setupWebGL = (canvas: OffscreenCanvas | HTMLCanvasElement) => {
   const gl = canvas.getContext('webgl2', {
     antialias: true,
     premultipliedAlpha: true,
   }) as WebGL2RenderingContext;
 
   let blurRadius: number | null = null;
+  let maskBlurRadius: number | null = 8;
+  const downsampleFactor = 4;
 
   if (!gl) {
     console.error('Failed to create WebGL context');
@@ -57,29 +65,44 @@ export const setupWebGL = (canvas: OffscreenCanvas) => {
   // Create additional textures and framebuffers for processing
   let bgBlurTextures: WebGLTexture[] = [];
   let bgBlurFrameBuffers: WebGLFramebuffer[] = [];
-  let maskBlurTextures: WebGLTexture[] = [];
-  let maskBlurFrameBuffers: WebGLFramebuffer[] = [];
+  let blurredMaskTexture: WebGLTexture | null = null;
+
+  // For double buffering the final mask
+  let finalMaskTextures: WebGLTexture[] = [];
+  let readMaskIndex = 0; // Index for renderFrame to read from
+  let writeMaskIndex = 1; // Index for updateMask to write to
 
   // Create textures for background processing (blur)
   bgBlurTextures.push(initTexture(gl, 3)); // For blur pass 1
   bgBlurTextures.push(initTexture(gl, 4)); // For blur pass 2
 
-  // Create framebuffers for background processing
-  bgBlurFrameBuffers.push(createFramebuffer(gl, bgBlurTextures[0], canvas.width, canvas.height));
-  bgBlurFrameBuffers.push(createFramebuffer(gl, bgBlurTextures[1], canvas.width, canvas.height));
+  const bgBlurTextureWidth = Math.floor(canvas.width / downsampleFactor);
+  const bgBlurTextureHeight = Math.floor(canvas.height / downsampleFactor);
 
-  // Create textures for mask processing (blur)
-  maskBlurTextures.push(initTexture(gl, 5)); // For mask blur pass 1
-  maskBlurTextures.push(initTexture(gl, 6)); // For mask blur pass 2
+  const downSampler = createDownSampler(gl, bgBlurTextureWidth, bgBlurTextureHeight);
 
-  // Create framebuffers for mask processing
-  maskBlurFrameBuffers.push(
-    createFramebuffer(gl, maskBlurTextures[0], canvas.width, canvas.height),
+  // Create framebuffers for background processing
+  bgBlurFrameBuffers.push(
+    createFramebuffer(gl, bgBlurTextures[0], bgBlurTextureWidth, bgBlurTextureHeight),
   );
-  maskBlurFrameBuffers.push(
-    createFramebuffer(gl, maskBlurTextures[1], canvas.width, canvas.height),
+  bgBlurFrameBuffers.push(
+    createFramebuffer(gl, bgBlurTextures[1], bgBlurTextureWidth, bgBlurTextureHeight),
   );
 
+  // Initialize texture for the first mask blur pass
+  const tempMaskTexture = initTexture(gl, 5);
+  const tempMaskFrameBuffer = createFramebuffer(gl, tempMaskTexture, canvas.width, canvas.height);
+
+  // Initialize two textures for double-buffering the final mask
+  finalMaskTextures.push(initTexture(gl, 6)); // For reading in renderFrame
+  finalMaskTextures.push(initTexture(gl, 7)); // For writing in updateMask
+
+  // Create framebuffers for the final mask textures
+  const finalMaskFrameBuffers = [
+    createFramebuffer(gl, finalMaskTextures[0], canvas.width, canvas.height),
+    createFramebuffer(gl, finalMaskTextures[1], canvas.width, canvas.height),
+  ];
+
   // Set up uniforms for the composite shader
   gl.useProgram(compositeProgram);
   gl.uniform1i(bgTextureLocation, 0);
@@ -87,10 +110,10 @@ export const setupWebGL = (canvas: OffscreenCanvas) => {
   gl.uniform1i(maskTextureLocation, 2);
 
   // Store custom background image
-  let customBackgroundImage: ImageBitmap | ImageData = emptyImageData;
+  let customBackgroundImage: ImageBitmap | ImageData = getEmptyImageData();
 
-  function render(frame: VideoFrame, mask: MPMask) {
-    if (frame.codedWidth === 0 || mask.width === 0) {
+  function renderFrame(frame: VideoFrame) {
+    if (frame.codedWidth === 0 || finalMaskTextures.length === 0) {
       return;
     }
 
@@ -106,11 +129,19 @@ export const setupWebGL = (canvas: OffscreenCanvas) => {
     let backgroundTexture = bgTexture;
 
     if (blurRadius) {
-      backgroundTexture = applyBlur(
+      const downSampledFrameTexture = applyDownsampling(
         gl,
         frameTexture,
-        width,
-        height,
+        downSampler,
+        vertexBuffer!,
+        bgBlurTextureWidth,
+        bgBlurTextureHeight,
+      );
+      backgroundTexture = applyBlur(
+        gl,
+        downSampledFrameTexture,
+        bgBlurTextureWidth,
+        bgBlurTextureHeight,
         blurRadius,
         blurProgram,
         blurUniforms,
@@ -125,20 +156,6 @@ export const setupWebGL = (canvas: OffscreenCanvas) => {
       backgroundTexture = bgTexture;
     }
 
-    // Apply box blur to mask texture
-    const blurredMaskTexture = applyBlur(
-      gl,
-      mask.getAsWebGLTexture(),
-      width,
-      height,
-      blurRadius || 1.0, // Use a default blur radius if not set
-      boxBlurProgram,
-      boxBlurUniforms,
-      vertexBuffer!,
-      maskBlurFrameBuffers,
-      maskBlurTextures,
-    );
-
     // Render the final composite
     gl.viewport(0, 0, width, height);
     gl.clearColor(1.0, 1.0, 1.0, 1.0);
@@ -159,13 +176,11 @@ export const setupWebGL = (canvas: OffscreenCanvas) => {
     gl.bindTexture(gl.TEXTURE_2D, frameTexture);
     gl.uniform1i(frameTextureLocation, 1);
 
-    // Set blurred mask texture
+    // Set mask texture - always read from the current read index
     gl.activeTexture(gl.TEXTURE2);
-    gl.bindTexture(gl.TEXTURE_2D, blurredMaskTexture);
+    gl.bindTexture(gl.TEXTURE_2D, finalMaskTextures[readMaskIndex]);
     gl.uniform1i(maskTextureLocation, 2);
     gl.drawArrays(gl.TRIANGLES, 0, 6);
-
-    mask.close();
   }
 
   /**
@@ -174,7 +189,7 @@ export const setupWebGL = (canvas: OffscreenCanvas) => {
    */
   async function setBackgroundImage(image: ImageBitmap | null) {
     // Clear existing background
-    customBackgroundImage = emptyImageData;
+    customBackgroundImage = getEmptyImageData();
 
     if (image) {
       try {
@@ -197,42 +212,82 @@ export const setupWebGL = (canvas: OffscreenCanvas) => {
   }
 
   function setBlurRadius(radius: number | null) {
-    blurRadius = radius;
+    blurRadius = radius ? Math.max(1, Math.floor(radius / downsampleFactor)) : null; // we are downsampling the blur texture, so decrease the radius here for better performance with a similar visual result
     setBackgroundImage(null);
   }
 
+  function updateMask(mask: WebGLTexture) {
+    // Use the existing applyBlur function to apply the first blur pass
+    // The second blur pass will be written to finalMaskTextures[writeMaskIndex]
+
+    // Create temporary arrays for the single blur operation
+    const tempFramebuffers = [tempMaskFrameBuffer, finalMaskFrameBuffers[writeMaskIndex]];
+
+    const tempTextures = [tempMaskTexture, finalMaskTextures[writeMaskIndex]];
+
+    // Apply the blur using the existing function
+    applyBlur(
+      gl,
+      mask,
+      canvas.width,
+      canvas.height,
+      maskBlurRadius || 1.0,
+      boxBlurProgram,
+      boxBlurUniforms,
+      vertexBuffer!,
+      tempFramebuffers,
+      tempTextures,
+    );
+
+    // Swap indices for the next frame
+    readMaskIndex = writeMaskIndex;
+    writeMaskIndex = 1 - writeMaskIndex;
+  }
+
   function cleanup() {
     gl.deleteProgram(compositeProgram);
     gl.deleteProgram(blurProgram);
     gl.deleteProgram(boxBlurProgram);
     gl.deleteTexture(bgTexture);
     gl.deleteTexture(frameTexture);
+    gl.deleteTexture(tempMaskTexture);
+    gl.deleteFramebuffer(tempMaskFrameBuffer);
+
     for (const texture of bgBlurTextures) {
       gl.deleteTexture(texture);
     }
     for (const framebuffer of bgBlurFrameBuffers) {
       gl.deleteFramebuffer(framebuffer);
     }
-    for (const texture of maskBlurTextures) {
+    for (const texture of finalMaskTextures) {
       gl.deleteTexture(texture);
     }
-    for (const framebuffer of maskBlurFrameBuffers) {
+    for (const framebuffer of finalMaskFrameBuffers) {
       gl.deleteFramebuffer(framebuffer);
     }
     gl.deleteBuffer(vertexBuffer);
 
+    if (blurredMaskTexture) {
+      gl.deleteTexture(blurredMaskTexture);
+    }
+
+    if (downSampler) {
+      gl.deleteTexture(downSampler.texture);
+      gl.deleteFramebuffer(downSampler.framebuffer);
+      gl.deleteProgram(downSampler.program);
+    }
+
     // Release any ImageBitmap resources
     if (customBackgroundImage) {
       if (customBackgroundImage instanceof ImageBitmap) {
         customBackgroundImage.close();
       }
-      customBackgroundImage = emptyImageData;
+      customBackgroundImage = getEmptyImageData();
     }
     bgBlurTextures = [];
     bgBlurFrameBuffers = [];
-    maskBlurTextures = [];
-    maskBlurFrameBuffers = [];
+    finalMaskTextures = [];
   }
 
-  return { render, setBackgroundImage, setBlurRadius, cleanup };
+  return { renderFrame, updateMask, setBackgroundImage, setBlurRadius, cleanup };
 };

package/src/webgl/shader-programs/blurShader.ts

@@ -1,9 +1,9 @@
-import { glsl } from '../utils';
+import { createProgram, createShader, glsl } from '../utils';
 import { vertexShaderSource } from './vertexShader';
 
 // Define the blur fragment shader
 export const blurFragmentShader = glsl`#version 300 es
-  precision highp float;
+  precision mediump float;
   in vec2 texCoords;
   uniform sampler2D u_texture;
   uniform vec2 u_texelSize;
@@ -33,42 +33,10 @@ export const blurFragmentShader = glsl`#version 300 es
 `;
 
 export function createBlurProgram(gl: WebGL2RenderingContext) {
-  // Create blur shader
-  const blurFrag = gl.createShader(gl.FRAGMENT_SHADER);
-  if (!blurFrag) {
-    throw Error('cannot create blur shader');
-  }
-  gl.shaderSource(blurFrag, blurFragmentShader);
-  gl.compileShader(blurFrag);
-
-  // Get compile status and log errors if any
-  if (!gl.getShaderParameter(blurFrag, gl.COMPILE_STATUS)) {
-    const info = gl.getShaderInfoLog(blurFrag);
-    throw Error(`Failed to compile blur shader: ${info}`);
-  }
+  const blurVertexShader = createShader(gl, gl.VERTEX_SHADER, vertexShaderSource());
+  const blurFrag = createShader(gl, gl.FRAGMENT_SHADER, blurFragmentShader);
 
-  // Create blur vertex shader
-  const blurVertexShader = gl.createShader(gl.VERTEX_SHADER);
-  if (!blurVertexShader) {
-    throw Error('cannot create blur vertex shader');
-  }
-  gl.shaderSource(blurVertexShader, vertexShaderSource());
-  gl.compileShader(blurVertexShader);
-
-  // Create blur program
-  const blurProgram = gl.createProgram();
-  if (!blurProgram) {
-    throw Error('cannot create blur program');
-  }
-  gl.attachShader(blurProgram, blurVertexShader);
-  gl.attachShader(blurProgram, blurFrag);
-  gl.linkProgram(blurProgram);
-
-  // Check blur program link status
-  if (!gl.getProgramParameter(blurProgram, gl.LINK_STATUS)) {
-    const info = gl.getProgramInfoLog(blurProgram);
-    throw Error(`Failed to link blur program: ${info}`);
-  }
+  const blurProgram = createProgram(gl, blurVertexShader, blurFrag);
 
   // Get uniform locations
   const blurUniforms = {

package/src/webgl/shader-programs/boxBlurShader.ts

@@ -1,4 +1,4 @@
-import { glsl } from '../utils';
+import { createProgram, createShader, glsl } from '../utils';
 import { vertexShaderSource } from './vertexShader';
 
 export const boxBlurFragmentShader = glsl`#version 300 es
@@ -37,48 +37,10 @@ void main() {
  * Create the box blur shader program
  */
 export function createBoxBlurProgram(gl: WebGL2RenderingContext) {
-  // Create vertex shader
-  const vertexShader = gl.createShader(gl.VERTEX_SHADER);
-  if (!vertexShader) {
-    throw Error('cannot create vertex shader');
-  }
-  gl.shaderSource(vertexShader, vertexShaderSource());
-  gl.compileShader(vertexShader);
-
-  // Check vertex shader compilation
-  if (!gl.getShaderParameter(vertexShader, gl.COMPILE_STATUS)) {
-    const info = gl.getShaderInfoLog(vertexShader);
-    throw Error(`Failed to compile vertex shader: ${info}`);
-  }
-
-  // Create fragment shader
-  const fragmentShader = gl.createShader(gl.FRAGMENT_SHADER);
-  if (!fragmentShader) {
-    throw Error('cannot create fragment shader');
-  }
-  gl.shaderSource(fragmentShader, boxBlurFragmentShader);
-  gl.compileShader(fragmentShader);
+  const vertexShader = createShader(gl, gl.VERTEX_SHADER, vertexShaderSource());
+  const fragmentShader = createShader(gl, gl.FRAGMENT_SHADER, boxBlurFragmentShader);
 
-  // Check fragment shader compilation
-  if (!gl.getShaderParameter(fragmentShader, gl.COMPILE_STATUS)) {
-    const info = gl.getShaderInfoLog(fragmentShader);
-    throw Error(`Failed to compile box blur shader: ${info}`);
-  }
-
-  // Create the program
-  const program = gl.createProgram();
-  if (!program) {
-    throw Error('cannot create box blur program');
-  }
-  gl.attachShader(program, vertexShader);
-  gl.attachShader(program, fragmentShader);
-  gl.linkProgram(program);
-
-  // Check program link status
-  if (!gl.getProgramParameter(program, gl.LINK_STATUS)) {
-    const info = gl.getProgramInfoLog(program);
-    throw Error(`Failed to link box blur program: ${info}`);
-  }
+  const program = createProgram(gl, vertexShader, fragmentShader);
 
   // Get attribute and uniform locations
   const uniforms = {

package/src/webgl/shader-programs/compositeShader.ts

@@ -1,9 +1,9 @@
-import { glsl } from '../utils';
+import { createProgram, createShader, glsl } from '../utils';
 import { vertexShaderSource } from './vertexShader';
 
 // Fragment shader source for compositing
 export const compositeFragmentShader = glsl`#version 300 es
-  precision highp float;
+  precision mediump float;
   in vec2 texCoords;
   uniform sampler2D background;
   uniform sampler2D frame;
@@ -37,48 +37,10 @@ export const compositeFragmentShader = glsl`#version 300 es
  * Create the composite shader program
  */
 export function createCompositeProgram(gl: WebGL2RenderingContext) {
-  // Create vertex shader
-  const vertexShader = gl.createShader(gl.VERTEX_SHADER);
-  if (!vertexShader) {
-    throw Error('cannot create vertex shader');
-  }
-  gl.shaderSource(vertexShader, vertexShaderSource());
-  gl.compileShader(vertexShader);
-
-  // Check vertex shader compilation
-  if (!gl.getShaderParameter(vertexShader, gl.COMPILE_STATUS)) {
-    const info = gl.getShaderInfoLog(vertexShader);
-    throw Error(`Failed to compile vertex shader: ${info}`);
-  }
-
-  // Create fragment shader
-  const compositeShader = gl.createShader(gl.FRAGMENT_SHADER);
-  if (!compositeShader) {
-    throw Error('cannot create fragment shader');
-  }
-  gl.shaderSource(compositeShader, compositeFragmentShader);
-  gl.compileShader(compositeShader);
+  const vertexShader = createShader(gl, gl.VERTEX_SHADER, vertexShaderSource());
+  const compositeShader = createShader(gl, gl.FRAGMENT_SHADER, compositeFragmentShader);
 
-  // Check fragment shader compilation
-  if (!gl.getShaderParameter(compositeShader, gl.COMPILE_STATUS)) {
-    const info = gl.getShaderInfoLog(compositeShader);
-    throw Error(`Failed to compile composite shader: ${info}`);
-  }
-
-  // Create the program
-  const compositeProgram = gl.createProgram();
-  if (!compositeProgram) {
-    throw Error('cannot create composite program');
-  }
-  gl.attachShader(compositeProgram, vertexShader);
-  gl.attachShader(compositeProgram, compositeShader);
-  gl.linkProgram(compositeProgram);
-
-  // Check program link status
-  if (!gl.getProgramParameter(compositeProgram, gl.LINK_STATUS)) {
-    const info = gl.getProgramInfoLog(compositeProgram);
-    throw Error(`Failed to link composite program: ${info}`);
-  }
+  const compositeProgram = createProgram(gl, vertexShader, compositeShader);
 
   // Get attribute and uniform locations
   const attribLocations = {

package/src/webgl/shader-programs/downSampler.ts

@@ -0,0 +1,94 @@
+import { createProgram, createShader } from '../utils';
+
+export function createDownSampler(
+  gl: WebGL2RenderingContext,
+  width: number,
+  height: number,
+): {
+  framebuffer: WebGLFramebuffer;
+  texture: WebGLTexture;
+  program: WebGLProgram;
+  uniforms: any;
+} {
+  // Create texture
+  const texture = gl.createTexture()!;
+  gl.bindTexture(gl.TEXTURE_2D, texture);
+  gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, width, height, 0, gl.RGBA, gl.UNSIGNED_BYTE, null);
+  gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
+  gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR);
+  gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
+  gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
+
+  // Create framebuffer
+  const framebuffer = gl.createFramebuffer()!;
+  gl.bindFramebuffer(gl.FRAMEBUFFER, framebuffer);
+  gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, texture, 0);
+
+  // Create shader program for copying
+  const vertexSource = `
+      attribute vec2 position;
+      varying vec2 v_uv;
+      void main() {
+        v_uv = (position + 1.0) * 0.5;
+        gl_Position = vec4(position, 0.0, 1.0);
+      }
+    `;
+
+  const fragmentSource = `
+      precision mediump float;
+      varying vec2 v_uv;
+      uniform sampler2D u_texture;
+      void main() {
+        gl_FragColor = texture2D(u_texture, v_uv);
+      }
+    `;
+
+  const vertShader = createShader(gl, gl.VERTEX_SHADER, vertexSource);
+  const fragShader = createShader(gl, gl.FRAGMENT_SHADER, fragmentSource);
+  const program = createProgram(gl, vertShader, fragShader);
+
+  const uniforms = {
+    texture: gl.getUniformLocation(program, 'u_texture'),
+    position: gl.getAttribLocation(program, 'position'),
+  };
+
+  return {
+    framebuffer,
+    texture,
+    program,
+    uniforms,
+  };
+}
+
+export function applyDownsampling(
+  gl: WebGL2RenderingContext,
+  inputTexture: WebGLTexture,
+  downSampler: {
+    framebuffer: WebGLFramebuffer;
+    texture: WebGLTexture;
+    program: WebGLProgram;
+    uniforms: any;
+  },
+  vertexBuffer: WebGLBuffer,
+  width: number,
+  height: number,
+): WebGLTexture {
+  gl.useProgram(downSampler.program);
+
+  gl.bindFramebuffer(gl.FRAMEBUFFER, downSampler.framebuffer);
+  gl.viewport(0, 0, width, height);
+
+  gl.bindBuffer(gl.ARRAY_BUFFER, vertexBuffer);
+  gl.enableVertexAttribArray(downSampler.uniforms.position);
+  gl.vertexAttribPointer(downSampler.uniforms.position, 2, gl.FLOAT, false, 0, 0);
+
+  gl.activeTexture(gl.TEXTURE0);
+  gl.bindTexture(gl.TEXTURE_2D, inputTexture);
+  gl.uniform1i(downSampler.uniforms.texture, 0);
+
+  gl.drawArrays(gl.TRIANGLES, 0, 6);
+
+  gl.bindFramebuffer(gl.FRAMEBUFFER, null);
+
+  return downSampler.texture;
+}

package/src/webgl/utils.ts

@@ -15,6 +15,38 @@ export function initTexture(gl: WebGL2RenderingContext, texIndex: number) {
   return texture;
 }
 
+export function createShader(
+  gl: WebGL2RenderingContext,
+  type: number,
+  source: string,
+): WebGLShader {
+  const shader = gl.createShader(type)!;
+  gl.shaderSource(shader, source);
+  gl.compileShader(shader);
+  if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) {
+    console.error('Shader compile failed:', gl.getShaderInfoLog(shader));
+    gl.deleteShader(shader);
+    throw new Error('Shader compile failed');
+  }
+  return shader;
+}
+
+export function createProgram(
+  gl: WebGL2RenderingContext,
+  vs: WebGLShader,
+  fs: WebGLShader,
+): WebGLProgram {
+  const program = gl.createProgram()!;
+  gl.attachShader(program, vs);
+  gl.attachShader(program, fs);
+  gl.linkProgram(program);
+  if (!gl.getProgramParameter(program, gl.LINK_STATUS)) {
+    console.error('Program link failed:', gl.getProgramInfoLog(program));
+    throw new Error('Program link failed');
+  }
+  return program;
+}
+
 /**
  * Create a WebGL framebuffer with the given texture as color attachment
  */
@@ -99,12 +131,20 @@ export async function resizeImageToCover(
   });
 }
 
-const emptyImageData = new ImageData(2, 2);
-emptyImageData.data[0] = 0;
-emptyImageData.data[1] = 0;
-emptyImageData.data[2] = 0;
-emptyImageData.data[3] = 0;
+let emptyImageData: ImageData | undefined;
+
+function getEmptyImageData() {
+  if (!emptyImageData) {
+    emptyImageData = new ImageData(2, 2);
+    emptyImageData.data[0] = 0;
+    emptyImageData.data[1] = 0;
+    emptyImageData.data[2] = 0;
+    emptyImageData.data[3] = 0;
+  }
+
+  return emptyImageData;
+}
 
 const glsl = (source: any) => source;
 
-export { emptyImageData, glsl };
+export { getEmptyImageData, glsl };