@stream-io/video-filters-web 0.0.1

package/src/tflite.ts

@@ -0,0 +1,59 @@
+// @ts-expect-error - module is not declared
+import { createTFLiteSIMDModule } from './tflite-simd.js';
+import { packageName, version } from './version';
+
+// This is a WebAssembly module compiled from the TensorFlow Lite C++ library.
+const createTFLite = createTFLiteSIMDModule as (opts?: any) => Promise<TFLite>;
+
+export interface TFLite extends EmscriptenModule {
+  _getModelBufferMemoryOffset(): number;
+  _getInputMemoryOffset(): number;
+  _getInputHeight(): number;
+  _getInputWidth(): number;
+  _getInputChannelCount(): number;
+  _getOutputMemoryOffset(): number;
+  _getOutputHeight(): number;
+  _getOutputWidth(): number;
+  _getOutputChannelCount(): number;
+  _loadModel(bufferSize: number): number;
+  _runInference(): number;
+}
+
+export const loadTFLite = async (
+  options: {
+    basePath?: string;
+    tfFilePath?: string;
+    modelFilePath?: string;
+  } = {},
+) => {
+  const {
+    basePath = `https://unpkg.com/${packageName}@${version}/tf`,
+    tfFilePath = `${basePath}/tflite/tflite-simd.wasm`,
+    modelFilePath = `${basePath}/models/segm_full_v679.tflite`,
+  } = options;
+
+  const [tfLite, model] = await Promise.all([
+    createTFLite({ locateFile: () => tfFilePath }),
+    fetchModel(modelFilePath),
+  ]);
+
+  const modelBufferOffset = tfLite._getModelBufferMemoryOffset();
+  tfLite.HEAPU8.set(new Uint8Array(model), modelBufferOffset);
+  tfLite._loadModel(model.byteLength);
+
+  return tfLite;
+};
+
+let lastModelFilePath = '';
+let modelFileCache: ArrayBuffer | undefined;
+const fetchModel = async (modelFilePath: string) => {
+  const model =
+    modelFilePath === lastModelFilePath && modelFileCache
+      ? modelFileCache
+      : await fetch(modelFilePath).then((r) => r.arrayBuffer());
+
+  // Cache the model file for future use.
+  modelFileCache = model;
+  lastModelFilePath = modelFilePath;
+  return model;
+};

package/src/version.ts

@@ -0,0 +1,3 @@
+export const version = process.env.PKG_VERSION || '0.0.0-development';
+export const packageName =
+  process.env.PKG_NAME || '@stream-io/video-filters-web';

package/src/webgl2/backgroundBlurStage.ts

@@ -0,0 +1,307 @@
+import {
+  compileShader,
+  createPipelineStageProgram,
+  createTexture,
+  glsl,
+} from '../helpers/webglHelper';
+import type { BackgroundBlurLevel } from '../createRenderer';
+
+export type BackgroundBlurStage = {
+  render(): void;
+  updateCoverage(coverage: [number, number]): void;
+  cleanUp(): void;
+};
+
+export function buildBackgroundBlurStage(
+  gl: WebGL2RenderingContext,
+  vertexShader: WebGLShader,
+  positionBuffer: WebGLBuffer,
+  texCoordBuffer: WebGLBuffer,
+  personMaskTexture: WebGLTexture,
+  canvas: HTMLCanvasElement,
+  blurLevel: BackgroundBlurLevel,
+): BackgroundBlurStage {
+  const blurPass = buildBlurPass(
+    gl,
+    vertexShader,
+    positionBuffer,
+    texCoordBuffer,
+    personMaskTexture,
+    canvas,
+    blurLevel,
+  );
+  const blendPass = buildBlendPass(gl, positionBuffer, texCoordBuffer, canvas);
+
+  function render() {
+    blurPass.render();
+    blendPass.render();
+  }
+
+  function updateCoverage(coverage: [number, number]) {
+    blendPass.updateCoverage(coverage);
+  }
+
+  function cleanUp() {
+    blendPass.cleanUp();
+    blurPass.cleanUp();
+  }
+
+  return {
+    render,
+    updateCoverage,
+    cleanUp,
+  };
+}
+
+function buildBlurPass(
+  gl: WebGL2RenderingContext,
+  vertexShader: WebGLShader,
+  positionBuffer: WebGLBuffer,
+  texCoordBuffer: WebGLBuffer,
+  personMaskTexture: WebGLTexture,
+  canvas: HTMLCanvasElement,
+  blurLevel: BackgroundBlurLevel,
+) {
+  const weights =
+    blurLevel === 'low'
+      ? [0.227027027, 0.1545945946, 0.1016216216, 0.0340540541, 0.0142162162]
+      : blurLevel === 'medium'
+      ? [0.327027027, 0.1945945946, 0.1216216216, 0.0540540541, 0.0162162162]
+      : [0.627027027, 0.3445945946, 0.2216216216, 0.0540540541, 0.0162162162];
+
+  const fragmentShaderSource = glsl`#version 300 es
+
+    precision highp float;
+
+    uniform sampler2D u_inputFrame;
+    uniform sampler2D u_personMask;
+    uniform vec2 u_texelSize;
+
+    in vec2 v_texCoord;
+    out vec4 outColor;
+
+    const float offset[5] = float[](0.0, 1.0, 2.0, 3.0, 4.0);
+    const float weight[5] = float[](
+      ${weights.join(',')}
+    );
+
+    void main() {
+      vec4 centerColor = texture(u_inputFrame, v_texCoord);
+      float personMask = texture(u_personMask, v_texCoord).a;
+
+      vec4 frameColor = centerColor * weight[0] * (1.0 - personMask);
+
+      for (int i = 1; i < 5; i++) {
+        vec2 offset = vec2(offset[i]) * u_texelSize;
+
+        vec2 texCoord = v_texCoord + offset;
+        frameColor += texture(u_inputFrame, texCoord)
+           * weight[i]
+           * (1.0 - texture(u_personMask, texCoord).a);
+
+        texCoord = v_texCoord - offset;
+        frameColor += texture(u_inputFrame, texCoord)
+          * weight[i]
+          * (1.0 - texture(u_personMask, texCoord).a);
+      }
+      outColor = vec4(frameColor.rgb + (1.0 - frameColor.a) * centerColor.rgb, 1.0);
+    }
+  `;
+
+  const scale = 0.5;
+  const outputWidth = canvas.width * scale;
+  const outputHeight = canvas.height * scale;
+  const texelWidth = 1 / outputWidth;
+  const texelHeight = 1 / outputHeight;
+
+  const fragmentShader = compileShader(
+    gl,
+    gl.FRAGMENT_SHADER,
+    fragmentShaderSource,
+  );
+  const program = createPipelineStageProgram(
+    gl,
+    vertexShader,
+    fragmentShader,
+    positionBuffer,
+    texCoordBuffer,
+  );
+  const inputFrameLocation = gl.getUniformLocation(program, 'u_inputFrame');
+  const personMaskLocation = gl.getUniformLocation(program, 'u_personMask');
+  const texelSizeLocation = gl.getUniformLocation(program, 'u_texelSize');
+  const texture1 = createTexture(
+    gl,
+    gl.RGBA8,
+    outputWidth,
+    outputHeight,
+    gl.NEAREST,
+    // @ts-expect-error types are incomplete
+    gl.LINEAR,
+  );
+  const texture2 = createTexture(
+    gl,
+    gl.RGBA8,
+    outputWidth,
+    outputHeight,
+    gl.NEAREST,
+    // @ts-expect-error types are incomplete
+    gl.LINEAR,
+  );
+
+  const frameBuffer1 = gl.createFramebuffer();
+  gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer1);
+  gl.framebufferTexture2D(
+    gl.FRAMEBUFFER,
+    gl.COLOR_ATTACHMENT0,
+    gl.TEXTURE_2D,
+    texture1,
+    0,
+  );
+
+  const frameBuffer2 = gl.createFramebuffer();
+  gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer2);
+  gl.framebufferTexture2D(
+    gl.FRAMEBUFFER,
+    gl.COLOR_ATTACHMENT0,
+    gl.TEXTURE_2D,
+    texture2,
+    0,
+  );
+
+  gl.useProgram(program);
+  gl.uniform1i(personMaskLocation, 1);
+
+  function render() {
+    gl.viewport(0, 0, outputWidth, outputHeight);
+    gl.useProgram(program);
+    gl.uniform1i(inputFrameLocation, 0);
+    gl.activeTexture(gl.TEXTURE1);
+    gl.bindTexture(gl.TEXTURE_2D, personMaskTexture);
+
+    for (let i = 0; i < 3; i++) {
+      gl.uniform2f(texelSizeLocation, 0, texelHeight);
+      gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer1);
+      gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4);
+
+      gl.activeTexture(gl.TEXTURE2);
+      gl.bindTexture(gl.TEXTURE_2D, texture1);
+      gl.uniform1i(inputFrameLocation, 2);
+
+      gl.uniform2f(texelSizeLocation, texelWidth, 0);
+      gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer2);
+      gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4);
+
+      gl.bindTexture(gl.TEXTURE_2D, texture2);
+    }
+  }
+
+  function cleanUp() {
+    gl.deleteFramebuffer(frameBuffer2);
+    gl.deleteFramebuffer(frameBuffer1);
+    gl.deleteTexture(texture2);
+    gl.deleteTexture(texture1);
+    gl.deleteProgram(program);
+    gl.deleteShader(fragmentShader);
+  }
+
+  return {
+    render,
+    cleanUp,
+  };
+}
+
+function buildBlendPass(
+  gl: WebGL2RenderingContext,
+  positionBuffer: WebGLBuffer,
+  texCoordBuffer: WebGLBuffer,
+  canvas: HTMLCanvasElement,
+) {
+  const vertexShaderSource = glsl`#version 300 es
+
+    in vec2 a_position;
+    in vec2 a_texCoord;
+
+    out vec2 v_texCoord;
+
+    void main() {
+      // Flipping Y is required when rendering to canvas
+      gl_Position = vec4(a_position * vec2(1.0, -1.0), 0.0, 1.0);
+      v_texCoord = a_texCoord;
+    }
+  `;
+
+  const fragmentShaderSource = glsl`#version 300 es
+
+    precision highp float;
+
+    uniform sampler2D u_inputFrame;
+    uniform sampler2D u_personMask;
+    uniform sampler2D u_blurredInputFrame;
+    uniform vec2 u_coverage;
+
+    in vec2 v_texCoord;
+
+    out vec4 outColor;
+
+    void main() {
+      vec3 color = texture(u_inputFrame, v_texCoord).rgb;
+      vec3 blurredColor = texture(u_blurredInputFrame, v_texCoord).rgb;
+      float personMask = texture(u_personMask, v_texCoord).a;
+      personMask = smoothstep(u_coverage.x, u_coverage.y, personMask);
+      outColor = vec4(mix(blurredColor, color, personMask), 1.0);
+    }
+  `;
+
+  const { width: outputWidth, height: outputHeight } = canvas;
+
+  const vertexShader = compileShader(gl, gl.VERTEX_SHADER, vertexShaderSource);
+  const fragmentShader = compileShader(
+    gl,
+    gl.FRAGMENT_SHADER,
+    fragmentShaderSource,
+  );
+  const program = createPipelineStageProgram(
+    gl,
+    vertexShader,
+    fragmentShader,
+    positionBuffer,
+    texCoordBuffer,
+  );
+  const inputFrameLocation = gl.getUniformLocation(program, 'u_inputFrame');
+  const personMaskLocation = gl.getUniformLocation(program, 'u_personMask');
+  const blurredInputFrame = gl.getUniformLocation(
+    program,
+    'u_blurredInputFrame',
+  );
+  const coverageLocation = gl.getUniformLocation(program, 'u_coverage');
+
+  gl.useProgram(program);
+  gl.uniform1i(inputFrameLocation, 0);
+  gl.uniform1i(personMaskLocation, 1);
+  gl.uniform1i(blurredInputFrame, 2);
+  gl.uniform2f(coverageLocation, 0, 1);
+
+  function render() {
+    gl.viewport(0, 0, outputWidth, outputHeight);
+    gl.useProgram(program);
+    gl.bindFramebuffer(gl.FRAMEBUFFER, null);
+    gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4);
+  }
+
+  function updateCoverage(coverage: [number, number]) {
+    gl.useProgram(program);
+    gl.uniform2f(coverageLocation, coverage[0], coverage[1]);
+  }
+
+  function cleanUp() {
+    gl.deleteProgram(program);
+    gl.deleteShader(fragmentShader);
+    gl.deleteShader(vertexShader);
+  }
+
+  return {
+    render,
+    updateCoverage,
+    cleanUp,
+  };
+}

package/src/webgl2/backgroundImageStage.ts

@@ -0,0 +1,222 @@
+import {
+  compileShader,
+  createPipelineStageProgram,
+  createTexture,
+  glsl,
+} from '../helpers/webglHelper';
+
+export type BackgroundImageStage = {
+  render(): void;
+  updateCoverage(coverage: [number, number]): void;
+  updateLightWrapping(lightWrapping: number): void;
+  updateBlendMode(blendMode: 'screen' | 'linearDodge'): void;
+  cleanUp(): void;
+};
+
+export function buildBackgroundImageStage(
+  gl: WebGL2RenderingContext,
+  positionBuffer: WebGLBuffer,
+  texCoordBuffer: WebGLBuffer,
+  personMaskTexture: WebGLTexture,
+  backgroundImage: HTMLImageElement | undefined,
+  canvas: HTMLCanvasElement,
+): BackgroundImageStage {
+  const vertexShaderSource = glsl`#version 300 es
+
+    uniform vec2 u_backgroundScale;
+    uniform vec2 u_backgroundOffset;
+
+    in vec2 a_position;
+    in vec2 a_texCoord;
+
+    out vec2 v_texCoord;
+    out vec2 v_backgroundCoord;
+
+    void main() {
+      // Flipping Y is required when rendering to canvas
+      gl_Position = vec4(a_position * vec2(1.0, -1.0), 0.0, 1.0);
+      v_texCoord = a_texCoord;
+      v_backgroundCoord = a_texCoord * u_backgroundScale + u_backgroundOffset;
+    }
+  `;
+
+  const fragmentShaderSource = glsl`#version 300 es
+
+    precision highp float;
+
+    uniform sampler2D u_inputFrame;
+    uniform sampler2D u_personMask;
+    uniform sampler2D u_background;
+    uniform vec2 u_coverage;
+    uniform float u_lightWrapping;
+    uniform float u_blendMode;
+
+    in vec2 v_texCoord;
+    in vec2 v_backgroundCoord;
+
+    out vec4 outColor;
+
+    vec3 screen(vec3 a, vec3 b) {
+      return 1.0 - (1.0 - a) * (1.0 - b);
+    }
+
+    vec3 linearDodge(vec3 a, vec3 b) {
+      return a + b;
+    }
+
+    void main() {
+      vec3 frameColor = texture(u_inputFrame, v_texCoord).rgb;
+      vec3 backgroundColor = texture(u_background, v_backgroundCoord).rgb;
+      float personMask = texture(u_personMask, v_texCoord).a;
+      float lightWrapMask = 1.0 - max(0.0, personMask - u_coverage.y) / (1.0 - u_coverage.y);
+      vec3 lightWrap = u_lightWrapping * lightWrapMask * backgroundColor;
+
+      frameColor = u_blendMode * linearDodge(frameColor, lightWrap)
+        + (1.0 - u_blendMode) * screen(frameColor, lightWrap);
+      personMask = smoothstep(u_coverage.x, u_coverage.y, personMask);
+      outColor = vec4(frameColor * personMask + backgroundColor * (1.0 - personMask), 1.0);
+    }
+  `;
+
+  const { width: outputWidth, height: outputHeight } = canvas;
+  const outputRatio = outputWidth / outputHeight;
+
+  const vertexShader = compileShader(gl, gl.VERTEX_SHADER, vertexShaderSource);
+  const fragmentShader = compileShader(
+    gl,
+    gl.FRAGMENT_SHADER,
+    fragmentShaderSource,
+  );
+  const program = createPipelineStageProgram(
+    gl,
+    vertexShader,
+    fragmentShader,
+    positionBuffer,
+    texCoordBuffer,
+  );
+  const backgroundScaleLocation = gl.getUniformLocation(
+    program,
+    'u_backgroundScale',
+  );
+  const backgroundOffsetLocation = gl.getUniformLocation(
+    program,
+    'u_backgroundOffset',
+  );
+  const inputFrameLocation = gl.getUniformLocation(program, 'u_inputFrame');
+  const personMaskLocation = gl.getUniformLocation(program, 'u_personMask');
+  const backgroundLocation = gl.getUniformLocation(program, 'u_background');
+  const coverageLocation = gl.getUniformLocation(program, 'u_coverage');
+  const lightWrappingLocation = gl.getUniformLocation(
+    program,
+    'u_lightWrapping',
+  );
+  const blendModeLocation = gl.getUniformLocation(program, 'u_blendMode');
+
+  gl.useProgram(program);
+  gl.uniform2f(backgroundScaleLocation, 1, 1);
+  gl.uniform2f(backgroundOffsetLocation, 0, 0);
+  gl.uniform1i(inputFrameLocation, 0);
+  gl.uniform1i(personMaskLocation, 1);
+  gl.uniform2f(coverageLocation, 0, 1);
+  gl.uniform1f(lightWrappingLocation, 0);
+  gl.uniform1f(blendModeLocation, 0);
+
+  let backgroundTexture: WebGLTexture | null = null;
+  // TODO Find a better to handle background being loaded
+  if (backgroundImage?.complete) {
+    updateBackgroundImage(backgroundImage);
+  } else if (backgroundImage) {
+    backgroundImage.onload = () => {
+      updateBackgroundImage(backgroundImage);
+    };
+  }
+
+  function render() {
+    gl.viewport(0, 0, outputWidth, outputHeight);
+    gl.useProgram(program);
+    gl.activeTexture(gl.TEXTURE1);
+    gl.bindTexture(gl.TEXTURE_2D, personMaskTexture);
+    if (backgroundTexture !== null) {
+      gl.activeTexture(gl.TEXTURE2);
+      gl.bindTexture(gl.TEXTURE_2D, backgroundTexture);
+      // TODO Handle correctly the background not loaded yet
+      gl.uniform1i(backgroundLocation, 2);
+    }
+    gl.bindFramebuffer(gl.FRAMEBUFFER, null);
+    gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4);
+  }
+
+  function updateBackgroundImage(bgImage: HTMLImageElement) {
+    backgroundTexture = createTexture(
+      gl,
+      gl.RGBA8,
+      bgImage.naturalWidth,
+      bgImage.naturalHeight,
+      // @ts-expect-error types are incomplete
+      gl.LINEAR,
+      gl.LINEAR,
+    );
+    gl.texSubImage2D(
+      gl.TEXTURE_2D,
+      0,
+      0,
+      0,
+      bgImage.naturalWidth,
+      bgImage.naturalHeight,
+      gl.RGBA,
+      gl.UNSIGNED_BYTE,
+      bgImage,
+    );
+
+    let xOffset = 0;
+    let yOffset = 0;
+    let backgroundWidth = bgImage.naturalWidth;
+    let backgroundHeight = bgImage.naturalHeight;
+    const backgroundRatio = backgroundWidth / backgroundHeight;
+    if (backgroundRatio < outputRatio) {
+      backgroundHeight = backgroundWidth / outputRatio;
+      yOffset = (bgImage.naturalHeight - backgroundHeight) / 2;
+    } else {
+      backgroundWidth = backgroundHeight * outputRatio;
+      xOffset = (bgImage.naturalWidth - backgroundWidth) / 2;
+    }
+
+    const xScale = backgroundWidth / bgImage.naturalWidth;
+    const yScale = backgroundHeight / bgImage.naturalHeight;
+    xOffset /= bgImage.naturalWidth;
+    yOffset /= bgImage.naturalHeight;
+
+    gl.uniform2f(backgroundScaleLocation, xScale, yScale);
+    gl.uniform2f(backgroundOffsetLocation, xOffset, yOffset);
+  }
+
+  function updateCoverage(coverage: [number, number]) {
+    gl.useProgram(program);
+    gl.uniform2f(coverageLocation, coverage[0], coverage[1]);
+  }
+
+  function updateLightWrapping(lightWrapping: number) {
+    gl.useProgram(program);
+    gl.uniform1f(lightWrappingLocation, lightWrapping);
+  }
+
+  function updateBlendMode(blendMode: 'screen' | 'linearDodge') {
+    gl.useProgram(program);
+    gl.uniform1f(blendModeLocation, blendMode === 'screen' ? 0 : 1);
+  }
+
+  function cleanUp() {
+    gl.deleteTexture(backgroundTexture);
+    gl.deleteProgram(program);
+    gl.deleteShader(fragmentShader);
+    gl.deleteShader(vertexShader);
+  }
+
+  return {
+    render,
+    updateCoverage,
+    updateLightWrapping,
+    updateBlendMode,
+    cleanUp,
+  };
+}