@livekit/track-processors 0.5.3 → 0.5.5

package/src/webgl/shader-programs/blurShader.ts

@@ -0,0 +1,108 @@
+import { createProgram, createShader, glsl } from '../utils';
+import { vertexShaderSource } from './vertexShader';
+
+// Define the blur fragment shader
+export const blurFragmentShader = glsl`#version 300 es
+  precision mediump float;
+  in vec2 texCoords;
+  uniform sampler2D u_texture;
+  uniform vec2 u_texelSize;
+  uniform vec2 u_direction;
+  uniform float u_radius;
+  out vec4 fragColor;
+
+  void main() {
+    float sigma = u_radius;
+    float twoSigmaSq = 2.0 * sigma * sigma;
+    float totalWeight = 0.0;
+    vec3 result = vec3(0.0);
+    const int MAX_SAMPLES = 16;
+    int radius = int(min(float(MAX_SAMPLES), ceil(u_radius)));
+
+    for (int i = -MAX_SAMPLES; i <= MAX_SAMPLES; ++i) {
+      float offset = float(i);
+      if (abs(offset) > float(radius)) continue;
+      float weight = exp(-(offset * offset) / twoSigmaSq);
+      vec2 sampleCoord = texCoords + u_direction * u_texelSize * offset;
+      result += texture(u_texture, sampleCoord).rgb * weight;
+      totalWeight += weight;
+    }
+
+    fragColor = vec4(result / totalWeight, 1.0);
+  }
+`;
+
+export function createBlurProgram(gl: WebGL2RenderingContext) {
+  const blurVertexShader = createShader(gl, gl.VERTEX_SHADER, vertexShaderSource());
+  const blurFrag = createShader(gl, gl.FRAGMENT_SHADER, blurFragmentShader);
+
+  const blurProgram = createProgram(gl, blurVertexShader, blurFrag);
+
+  // Get uniform locations
+  const blurUniforms = {
+    position: gl.getAttribLocation(blurProgram, 'position'),
+    texture: gl.getUniformLocation(blurProgram, 'u_texture'),
+    texelSize: gl.getUniformLocation(blurProgram, 'u_texelSize'),
+    direction: gl.getUniformLocation(blurProgram, 'u_direction'),
+    radius: gl.getUniformLocation(blurProgram, 'u_radius'),
+  };
+
+  return {
+    program: blurProgram,
+    shader: blurFrag,
+    vertexShader: blurVertexShader,
+    uniforms: blurUniforms,
+  };
+}
+
+export function applyBlur(
+  gl: WebGL2RenderingContext,
+  sourceTexture: WebGLTexture,
+  width: number,
+  height: number,
+  blurRadius: number,
+  blurProgram: WebGLProgram,
+  blurUniforms: any,
+  vertexBuffer: WebGLBuffer,
+  processFramebuffers: WebGLFramebuffer[],
+  processTextures: WebGLTexture[],
+) {
+  gl.useProgram(blurProgram);
+
+  // Set common attributes
+  gl.bindBuffer(gl.ARRAY_BUFFER, vertexBuffer);
+  gl.vertexAttribPointer(blurUniforms.position, 2, gl.FLOAT, false, 0, 0);
+  gl.enableVertexAttribArray(blurUniforms.position);
+
+  const texelWidth = 1.0 / width;
+  const texelHeight = 1.0 / height;
+
+  // First pass - horizontal blur
+  gl.bindFramebuffer(gl.FRAMEBUFFER, processFramebuffers[0]);
+  gl.viewport(0, 0, width, height);
+
+  gl.activeTexture(gl.TEXTURE0);
+  gl.bindTexture(gl.TEXTURE_2D, sourceTexture);
+  gl.uniform1i(blurUniforms.texture, 0);
+  gl.uniform2f(blurUniforms.texelSize, texelWidth, texelHeight);
+  gl.uniform2f(blurUniforms.direction, 1.0, 0.0); // Horizontal
+  gl.uniform1f(blurUniforms.radius, blurRadius);
+
+  gl.drawArrays(gl.TRIANGLES, 0, 6);
+
+  // Second pass - vertical blur
+  gl.bindFramebuffer(gl.FRAMEBUFFER, processFramebuffers[1]);
+  gl.viewport(0, 0, width, height);
+
+  gl.activeTexture(gl.TEXTURE0);
+  gl.bindTexture(gl.TEXTURE_2D, processTextures[0]);
+  gl.uniform1i(blurUniforms.texture, 0);
+  gl.uniform2f(blurUniforms.direction, 0.0, 1.0); // Vertical
+
+  gl.drawArrays(gl.TRIANGLES, 0, 6);
+
+  // Reset framebuffer
+  gl.bindFramebuffer(gl.FRAMEBUFFER, null);
+
+  return processTextures[1];
+}

package/src/webgl/shader-programs/boxBlurShader.ts

@@ -0,0 +1,60 @@
+import { createProgram, createShader, glsl } from '../utils';
+import { vertexShaderSource } from './vertexShader';
+
+export const boxBlurFragmentShader = glsl`#version 300 es
+precision mediump float;
+
+in vec2 texCoords;
+
+uniform sampler2D u_texture;
+uniform vec2 u_texelSize;    // 1.0 / texture size
+uniform vec2 u_direction;    // (1.0, 0.0) for horizontal, (0.0, 1.0) for vertical
+uniform float u_radius;      // blur radius in texels
+
+out vec4 fragColor;
+
+void main() {
+    vec3 sum = vec3(0.0);
+    float count = 0.0;
+
+    // Limit radius to avoid excessive loop cost
+    const int MAX_RADIUS = 16;
+    int radius = int(min(float(MAX_RADIUS), u_radius));
+
+    for (int i = -MAX_RADIUS; i <= MAX_RADIUS; ++i) {
+        if (abs(i) > radius) continue;
+
+        vec2 offset = u_direction * u_texelSize * float(i);
+        sum += texture(u_texture, texCoords + offset).rgb;
+        count += 1.0;
+  }
+
+  fragColor = vec4(sum / count, 1.0);
+}
+`;
+
+/**
+ * Create the box blur shader program
+ */
+export function createBoxBlurProgram(gl: WebGL2RenderingContext) {
+  const vertexShader = createShader(gl, gl.VERTEX_SHADER, vertexShaderSource());
+  const fragmentShader = createShader(gl, gl.FRAGMENT_SHADER, boxBlurFragmentShader);
+
+  const program = createProgram(gl, vertexShader, fragmentShader);
+
+  // Get attribute and uniform locations
+  const uniforms = {
+    position: gl.getAttribLocation(program, 'position'),
+    texture: gl.getUniformLocation(program, 'u_texture'),
+    texelSize: gl.getUniformLocation(program, 'u_texelSize'),
+    direction: gl.getUniformLocation(program, 'u_direction'),
+    radius: gl.getUniformLocation(program, 'u_radius'),
+  };
+
+  return {
+    program,
+    vertexShader,
+    fragmentShader,
+    uniforms,
+  };
+}

package/src/webgl/shader-programs/compositeShader.ts

@@ -0,0 +1,64 @@
+import { createProgram, createShader, glsl } from '../utils';
+import { vertexShaderSource } from './vertexShader';
+
+// Fragment shader source for compositing
+export const compositeFragmentShader = glsl`#version 300 es
+  precision mediump float;
+  in vec2 texCoords;
+  uniform sampler2D background;
+  uniform sampler2D frame;
+  uniform sampler2D mask;
+  out vec4 fragColor;
+  
+  void main() {
+      
+    vec4 frameTex = texture(frame, texCoords);
+    vec4 bgTex = texture(background, texCoords);
+
+    float maskVal = texture(mask, texCoords).r;
+
+    // Compute screen-space gradient to detect edge sharpness
+    float grad = length(vec2(dFdx(maskVal), dFdy(maskVal)));
+
+    float edgeSoftness = 2.0; // higher = softer
+    
+    // Create a smooth edge around binary transition
+    float smoothAlpha = smoothstep(0.5 - grad * edgeSoftness, 0.5 + grad * edgeSoftness, maskVal);
+
+    // Optional: preserve frame alpha, or override as fully opaque
+    vec4 blended = mix(bgTex, vec4(frameTex.rgb, 1.0), 1.0 - smoothAlpha);
+    
+    fragColor = blended;
+  
+  }
+`;
+
+/**
+ * Create the composite shader program
+ */
+export function createCompositeProgram(gl: WebGL2RenderingContext) {
+  const vertexShader = createShader(gl, gl.VERTEX_SHADER, vertexShaderSource());
+  const compositeShader = createShader(gl, gl.FRAGMENT_SHADER, compositeFragmentShader);
+
+  const compositeProgram = createProgram(gl, vertexShader, compositeShader);
+
+  // Get attribute and uniform locations
+  const attribLocations = {
+    position: gl.getAttribLocation(compositeProgram, 'position'),
+  };
+
+  const uniformLocations = {
+    mask: gl.getUniformLocation(compositeProgram, 'mask')!,
+    frame: gl.getUniformLocation(compositeProgram, 'frame')!,
+    background: gl.getUniformLocation(compositeProgram, 'background')!,
+    stepWidth: gl.getUniformLocation(compositeProgram, 'u_stepWidth')!,
+  };
+
+  return {
+    program: compositeProgram,
+    vertexShader,
+    fragmentShader: compositeShader,
+    attribLocations,
+    uniformLocations,
+  };
+}

package/src/webgl/shader-programs/downSampler.ts

@@ -0,0 +1,94 @@
+import { createProgram, createShader } from '../utils';
+
+export function createDownSampler(
+  gl: WebGL2RenderingContext,
+  width: number,
+  height: number,
+): {
+  framebuffer: WebGLFramebuffer;
+  texture: WebGLTexture;
+  program: WebGLProgram;
+  uniforms: any;
+} {
+  // Create texture
+  const texture = gl.createTexture()!;
+  gl.bindTexture(gl.TEXTURE_2D, texture);
+  gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, width, height, 0, gl.RGBA, gl.UNSIGNED_BYTE, null);
+  gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
+  gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR);
+  gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
+  gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
+
+  // Create framebuffer
+  const framebuffer = gl.createFramebuffer()!;
+  gl.bindFramebuffer(gl.FRAMEBUFFER, framebuffer);
+  gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, texture, 0);
+
+  // Create shader program for copying
+  const vertexSource = `
+      attribute vec2 position;
+      varying vec2 v_uv;
+      void main() {
+        v_uv = (position + 1.0) * 0.5;
+        gl_Position = vec4(position, 0.0, 1.0);
+      }
+    `;
+
+  const fragmentSource = `
+      precision mediump float;
+      varying vec2 v_uv;
+      uniform sampler2D u_texture;
+      void main() {
+        gl_FragColor = texture2D(u_texture, v_uv);
+      }
+    `;
+
+  const vertShader = createShader(gl, gl.VERTEX_SHADER, vertexSource);
+  const fragShader = createShader(gl, gl.FRAGMENT_SHADER, fragmentSource);
+  const program = createProgram(gl, vertShader, fragShader);
+
+  const uniforms = {
+    texture: gl.getUniformLocation(program, 'u_texture'),
+    position: gl.getAttribLocation(program, 'position'),
+  };
+
+  return {
+    framebuffer,
+    texture,
+    program,
+    uniforms,
+  };
+}
+
+export function applyDownsampling(
+  gl: WebGL2RenderingContext,
+  inputTexture: WebGLTexture,
+  downSampler: {
+    framebuffer: WebGLFramebuffer;
+    texture: WebGLTexture;
+    program: WebGLProgram;
+    uniforms: any;
+  },
+  vertexBuffer: WebGLBuffer,
+  width: number,
+  height: number,
+): WebGLTexture {
+  gl.useProgram(downSampler.program);
+
+  gl.bindFramebuffer(gl.FRAMEBUFFER, downSampler.framebuffer);
+  gl.viewport(0, 0, width, height);
+
+  gl.bindBuffer(gl.ARRAY_BUFFER, vertexBuffer);
+  gl.enableVertexAttribArray(downSampler.uniforms.position);
+  gl.vertexAttribPointer(downSampler.uniforms.position, 2, gl.FLOAT, false, 0, 0);
+
+  gl.activeTexture(gl.TEXTURE0);
+  gl.bindTexture(gl.TEXTURE_2D, inputTexture);
+  gl.uniform1i(downSampler.uniforms.texture, 0);
+
+  gl.drawArrays(gl.TRIANGLES, 0, 6);
+
+  gl.bindFramebuffer(gl.FRAMEBUFFER, null);
+
+  return downSampler.texture;
+}

package/src/webgl/shader-programs/vertexShader.ts

@@ -0,0 +1,11 @@
+// Vertex shader source
+export const vertexShaderSource = (flipY: boolean = true) => `#version 300 es
+  in vec2 position;
+  out vec2 texCoords;
+
+  void main() {
+    texCoords = (position + 1.0) / 2.0;
+    texCoords.y = ${flipY ? '1.0 - texCoords.y' : 'texCoords.y'};
+    gl_Position = vec4(position, 0, 1.0);
+  }
+`;

package/src/webgl/utils.ts

@@ -0,0 +1,142 @@
+/**
+ * Initialize a WebGL texture
+ */
+export function initTexture(gl: WebGL2RenderingContext, texIndex: number) {
+  const texRef = gl.TEXTURE0 + texIndex;
+  gl.activeTexture(texRef);
+  const texture = gl.createTexture();
+  gl.bindTexture(gl.TEXTURE_2D, texture);
+  gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
+  gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
+  gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
+  gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR);
+  gl.bindTexture(gl.TEXTURE_2D, texture);
+
+  return texture;
+}
+
+export function createShader(
+  gl: WebGL2RenderingContext,
+  type: number,
+  source: string,
+): WebGLShader {
+  const shader = gl.createShader(type)!;
+  gl.shaderSource(shader, source);
+  gl.compileShader(shader);
+  if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) {
+    console.error('Shader compile failed:', gl.getShaderInfoLog(shader));
+    gl.deleteShader(shader);
+    throw new Error('Shader compile failed');
+  }
+  return shader;
+}
+
+export function createProgram(
+  gl: WebGL2RenderingContext,
+  vs: WebGLShader,
+  fs: WebGLShader,
+): WebGLProgram {
+  const program = gl.createProgram()!;
+  gl.attachShader(program, vs);
+  gl.attachShader(program, fs);
+  gl.linkProgram(program);
+  if (!gl.getProgramParameter(program, gl.LINK_STATUS)) {
+    console.error('Program link failed:', gl.getProgramInfoLog(program));
+    throw new Error('Program link failed');
+  }
+  return program;
+}
+
+/**
+ * Create a WebGL framebuffer with the given texture as color attachment
+ */
+export function createFramebuffer(
+  gl: WebGL2RenderingContext,
+  texture: WebGLTexture,
+  width: number,
+  height: number,
+) {
+  const framebuffer = gl.createFramebuffer();
+  gl.bindFramebuffer(gl.FRAMEBUFFER, framebuffer);
+
+  // Set the texture as the color attachment
+  gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, texture, 0);
+
+  // Ensure texture dimensions match the provided width and height
+  gl.bindTexture(gl.TEXTURE_2D, texture);
+  gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, width, height, 0, gl.RGBA, gl.UNSIGNED_BYTE, null);
+
+  // Check if framebuffer is complete
+  const status = gl.checkFramebufferStatus(gl.FRAMEBUFFER);
+  if (status !== gl.FRAMEBUFFER_COMPLETE) {
+    throw new Error('Framebuffer not complete');
+  }
+
+  gl.bindFramebuffer(gl.FRAMEBUFFER, null);
+  return framebuffer;
+}
+
+/**
+ * Create a vertex buffer for a full-screen quad
+ */
+export function createVertexBuffer(gl: WebGL2RenderingContext): WebGLBuffer | null {
+  const vertexBuffer = gl.createBuffer();
+  gl.bindBuffer(gl.ARRAY_BUFFER, vertexBuffer);
+  gl.bufferData(
+    gl.ARRAY_BUFFER,
+    new Float32Array([-1, -1, -1, 1, 1, 1, -1, -1, 1, 1, 1, -1]),
+    gl.STATIC_DRAW,
+  );
+  return vertexBuffer;
+}
+
+/**
+ * Resizes and crops an image to cover a target canvas while maintaining aspect ratio
+ * @param image The source image
+ * @param targetWidth The target width
+ * @param targetHeight The target height
+ * @returns A cropped and resized ImageBitmap
+ */
+export async function resizeImageToCover(
+  image: ImageBitmap,
+  targetWidth: number,
+  targetHeight: number,
+): Promise<ImageBitmap> {
+  // Calculate dimensions and crop for "cover" mode
+  const imgAspect = image.width / image.height;
+  const targetAspect = targetWidth / targetHeight;
+
+  let sx = 0;
+  let sy = 0;
+  let sWidth = image.width;
+  let sHeight = image.height;
+
+  // For cover mode, we need to crop some parts of the image
+  // to ensure it covers the canvas while maintaining aspect ratio
+  if (imgAspect > targetAspect) {
+    // Image is wider than target - crop the sides
+    sWidth = Math.round(image.height * targetAspect);
+    sx = Math.round((image.width - sWidth) / 2); // Center the crop horizontally
+  } else if (imgAspect < targetAspect) {
+    // Image is taller than target - crop the top/bottom
+    sHeight = Math.round(image.width / targetAspect);
+    sy = Math.round((image.height - sHeight) / 2); // Center the crop vertically
+  }
+
+  // Create a new ImageBitmap with the cropped portion
+  return createImageBitmap(image, sx, sy, sWidth, sHeight, {
+    resizeWidth: targetWidth,
+    resizeHeight: targetHeight,
+    resizeQuality: 'medium',
+  });
+}
+
+const emptyImageData = new ImageData(2, 2);
+emptyImageData.data[0] = 0;
+emptyImageData.data[1] = 0;
+emptyImageData.data[2] = 0;
+emptyImageData.data[3] = 0;
+
+const glsl = (source: any) => source;
+
+export { emptyImageData, glsl };