@stream-io/video-filters-web 0.8.6 → 0.8.7

package/dist/esm/src/legacy/webgl2/jointBilateralFilterStage.js

@@ -0,0 +1,113 @@
+import { compileShader, createPipelineStageProgram, glsl } from '../helpers/webglHelper.js';
+
+function buildJointBilateralFilterStage(gl, vertexShader, positionBuffer, texCoordBuffer, inputTexture, outputTexture, canvas, segmentationConfig) {
+    const fragmentShaderSource = glsl `#version 300 es
+
+    precision highp float;
+
+    uniform sampler2D u_inputFrame;
+    uniform sampler2D u_segmentationMask;
+    uniform vec2 u_texelSize;
+    uniform float u_step;
+    uniform float u_radius;
+    uniform float u_offset;
+    uniform float u_sigmaTexel;
+    uniform float u_sigmaColor;
+
+    in vec2 v_texCoord;
+    out vec4 outColor;
+
+    float gaussian(float x, float sigma) {
+      float coeff = -0.5 / (sigma * sigma * 4.0 + 1.0e-6);
+      return exp((x * x) * coeff);
+    }
+
+    void main() {
+      vec2 centerCoord = v_texCoord;
+      vec3 centerColor = texture(u_inputFrame, centerCoord).rgb;
+      float newVal = 0.0;
+
+      float spaceWeight = 0.0;
+      float colorWeight = 0.0;
+      float totalWeight = 0.0;
+
+      // Subsample kernel space.
+      for (float i = -u_radius + u_offset; i <= u_radius; i += u_step) {
+        for (float j = -u_radius + u_offset; j <= u_radius; j += u_step) {
+          vec2 shift = vec2(j, i) * u_texelSize;
+          vec2 coord = vec2(centerCoord + shift);
+          vec3 frameColor = texture(u_inputFrame, coord).rgb;
+          float outVal = texture(u_segmentationMask, coord).a;
+
+          spaceWeight = gaussian(distance(centerCoord, coord), u_sigmaTexel);
+          colorWeight = gaussian(distance(centerColor, frameColor), u_sigmaColor);
+          totalWeight += spaceWeight * colorWeight;
+
+          newVal += spaceWeight * colorWeight * outVal;
+        }
+      }
+      newVal /= totalWeight;
+
+      outColor = vec4(vec3(0.0), newVal);
+    }
+  `;
+    const { width: segmentationWidth, height: segmentationHeight } = segmentationConfig;
+    const { width: outputWidth, height: outputHeight } = canvas;
+    const texelWidth = 1 / outputWidth;
+    const texelHeight = 1 / outputHeight;
+    const fragmentShader = compileShader(gl, gl.FRAGMENT_SHADER, fragmentShaderSource);
+    const program = createPipelineStageProgram(gl, vertexShader, fragmentShader, positionBuffer, texCoordBuffer);
+    const inputFrameLocation = gl.getUniformLocation(program, 'u_inputFrame');
+    const segmentationMaskLocation = gl.getUniformLocation(program, 'u_segmentationMask');
+    const texelSizeLocation = gl.getUniformLocation(program, 'u_texelSize');
+    const stepLocation = gl.getUniformLocation(program, 'u_step');
+    const radiusLocation = gl.getUniformLocation(program, 'u_radius');
+    const offsetLocation = gl.getUniformLocation(program, 'u_offset');
+    const sigmaTexelLocation = gl.getUniformLocation(program, 'u_sigmaTexel');
+    const sigmaColorLocation = gl.getUniformLocation(program, 'u_sigmaColor');
+    const frameBuffer = gl.createFramebuffer();
+    gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer);
+    gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, outputTexture, 0);
+    gl.useProgram(program);
+    gl.uniform1i(inputFrameLocation, 0);
+    gl.uniform1i(segmentationMaskLocation, 1);
+    gl.uniform2f(texelSizeLocation, texelWidth, texelHeight);
+    // Ensures default values are configured to prevent infinite
+    // loop in fragment shader
+    updateSigmaSpace(0);
+    updateSigmaColor(0);
+    function render() {
+        gl.viewport(0, 0, outputWidth, outputHeight);
+        gl.useProgram(program);
+        gl.activeTexture(gl.TEXTURE1);
+        gl.bindTexture(gl.TEXTURE_2D, inputTexture);
+        gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer);
+        gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4);
+    }
+    function updateSigmaSpace(sigmaSpace) {
+        sigmaSpace *= Math.max(outputWidth / segmentationWidth, outputHeight / segmentationHeight);
+        const kSparsityFactor = 0.66; // Higher is sparser.
+        const step = Math.max(1, Math.sqrt(sigmaSpace) * kSparsityFactor);
+        const radius = sigmaSpace;
+        const offset = step > 1 ? step * 0.5 : 0;
+        const sigmaTexel = Math.max(texelWidth, texelHeight) * sigmaSpace;
+        gl.useProgram(program);
+        gl.uniform1f(stepLocation, step);
+        gl.uniform1f(radiusLocation, radius);
+        gl.uniform1f(offsetLocation, offset);
+        gl.uniform1f(sigmaTexelLocation, sigmaTexel);
+    }
+    function updateSigmaColor(sigmaColor) {
+        gl.useProgram(program);
+        gl.uniform1f(sigmaColorLocation, sigmaColor);
+    }
+    function cleanUp() {
+        gl.deleteFramebuffer(frameBuffer);
+        gl.deleteProgram(program);
+        gl.deleteShader(fragmentShader);
+    }
+    return { render, updateSigmaSpace, updateSigmaColor, cleanUp };
+}
+
+export { buildJointBilateralFilterStage };
+//# sourceMappingURL=jointBilateralFilterStage.js.map

package/dist/esm/src/legacy/webgl2/jointBilateralFilterStage.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"jointBilateralFilterStage.js","sources":["../../../../../../../src/legacy/webgl2/jointBilateralFilterStage.ts"],"sourcesContent":["import {\n  compileShader,\n  createPipelineStageProgram,\n  glsl,\n} from '../helpers/webglHelper';\nimport { SegmentationParams } from '../segmentation';\n\nexport function buildJointBilateralFilterStage(\n  gl: WebGL2RenderingContext,\n  vertexShader: WebGLShader,\n  positionBuffer: WebGLBuffer,\n  texCoordBuffer: WebGLBuffer,\n  inputTexture: WebGLTexture,\n  outputTexture: WebGLTexture,\n  canvas: HTMLCanvasElement,\n  segmentationConfig: SegmentationParams,\n) {\n  const fragmentShaderSource = glsl`#version 300 es\n\n    precision highp float;\n\n    uniform sampler2D u_inputFrame;\n    uniform sampler2D u_segmentationMask;\n    uniform vec2 u_texelSize;\n    uniform float u_step;\n    uniform float u_radius;\n    uniform float u_offset;\n    uniform float u_sigmaTexel;\n    uniform float u_sigmaColor;\n\n    in vec2 v_texCoord;\n    out vec4 outColor;\n\n    float gaussian(float x, float sigma) {\n      float coeff = -0.5 / (sigma * sigma * 4.0 + 1.0e-6);\n      return exp((x * x) * coeff);\n    }\n\n    void main() {\n      vec2 centerCoord = v_texCoord;\n      vec3 centerColor = texture(u_inputFrame, centerCoord).rgb;\n      float newVal = 0.0;\n\n      float spaceWeight = 0.0;\n      float colorWeight = 0.0;\n      float totalWeight = 0.0;\n\n      // Subsample kernel space.\n      for (float i = -u_radius + u_offset; i <= u_radius; i += u_step) {\n        for (float j = -u_radius + u_offset; j <= u_radius; j += u_step) {\n          vec2 shift = vec2(j, i) * u_texelSize;\n          vec2 coord = vec2(centerCoord + shift);\n          vec3 frameColor = texture(u_inputFrame, coord).rgb;\n          float outVal = texture(u_segmentationMask, coord).a;\n\n          spaceWeight = gaussian(distance(centerCoord, coord), u_sigmaTexel);\n          colorWeight = gaussian(distance(centerColor, frameColor), u_sigmaColor);\n          totalWeight += spaceWeight * colorWeight;\n\n          newVal += spaceWeight * colorWeight * outVal;\n        }\n      }\n      newVal /= totalWeight;\n\n      outColor = vec4(vec3(0.0), newVal);\n    }\n  `;\n\n  const { width: segmentationWidth, height: segmentationHeight } =\n    segmentationConfig;\n  const { width: outputWidth, height: outputHeight } = canvas;\n  const texelWidth = 1 / outputWidth;\n  const texelHeight = 1 / outputHeight;\n\n  const fragmentShader = compileShader(\n    gl,\n    gl.FRAGMENT_SHADER,\n    fragmentShaderSource,\n  );\n  const program = createPipelineStageProgram(\n    gl,\n    vertexShader,\n    fragmentShader,\n    positionBuffer,\n    texCoordBuffer,\n  );\n  const inputFrameLocation = gl.getUniformLocation(program, 'u_inputFrame');\n  const segmentationMaskLocation = gl.getUniformLocation(\n    program,\n    'u_segmentationMask',\n  );\n  const texelSizeLocation = gl.getUniformLocation(program, 'u_texelSize');\n  const stepLocation = gl.getUniformLocation(program, 'u_step');\n  const radiusLocation = gl.getUniformLocation(program, 'u_radius');\n  const offsetLocation = gl.getUniformLocation(program, 'u_offset');\n  const sigmaTexelLocation = gl.getUniformLocation(program, 'u_sigmaTexel');\n  const sigmaColorLocation = gl.getUniformLocation(program, 'u_sigmaColor');\n\n  const frameBuffer = gl.createFramebuffer();\n  gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer);\n  gl.framebufferTexture2D(\n    gl.FRAMEBUFFER,\n    gl.COLOR_ATTACHMENT0,\n    gl.TEXTURE_2D,\n    outputTexture,\n    0,\n  );\n\n  gl.useProgram(program);\n  gl.uniform1i(inputFrameLocation, 0);\n  gl.uniform1i(segmentationMaskLocation, 1);\n  gl.uniform2f(texelSizeLocation, texelWidth, texelHeight);\n\n  // Ensures default values are configured to prevent infinite\n  // loop in fragment shader\n  updateSigmaSpace(0);\n  updateSigmaColor(0);\n\n  function render() {\n    gl.viewport(0, 0, outputWidth, outputHeight);\n    gl.useProgram(program);\n    gl.activeTexture(gl.TEXTURE1);\n    gl.bindTexture(gl.TEXTURE_2D, inputTexture);\n    gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer);\n    gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4);\n  }\n\n  function updateSigmaSpace(sigmaSpace: number) {\n    sigmaSpace *= Math.max(\n      outputWidth / segmentationWidth,\n      outputHeight / segmentationHeight,\n    );\n\n    const kSparsityFactor = 0.66; // Higher is sparser.\n    const step = Math.max(1, Math.sqrt(sigmaSpace) * kSparsityFactor);\n    const radius = sigmaSpace;\n    const offset = step > 1 ? step * 0.5 : 0;\n    const sigmaTexel = Math.max(texelWidth, texelHeight) * sigmaSpace;\n\n    gl.useProgram(program);\n    gl.uniform1f(stepLocation, step);\n    gl.uniform1f(radiusLocation, radius);\n    gl.uniform1f(offsetLocation, offset);\n    gl.uniform1f(sigmaTexelLocation, sigmaTexel);\n  }\n\n  function updateSigmaColor(sigmaColor: number) {\n    gl.useProgram(program);\n    gl.uniform1f(sigmaColorLocation, sigmaColor);\n  }\n\n  function cleanUp() {\n    gl.deleteFramebuffer(frameBuffer);\n    gl.deleteProgram(program);\n    gl.deleteShader(fragmentShader);\n  }\n\n  return { render, updateSigmaSpace, updateSigmaColor, cleanUp };\n}\n"],"names":[],"mappings":";;SAOgB,8BAA8B,CAC5C,EAA0B,EAC1B,YAAyB,EACzB,cAA2B,EAC3B,cAA2B,EAC3B,YAA0B,EAC1B,aAA2B,EAC3B,MAAyB,EACzB,kBAAsC,EAAA;IAEtC,MAAM,oBAAoB,GAAG,IAAI,CAAA,CAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiDhC;IAED,MAAM,EAAE,KAAK,EAAE,iBAAiB,EAAE,MAAM,EAAE,kBAAkB,EAAE,GAC5D,kBAAkB;IACpB,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,MAAM,EAAE,YAAY,EAAE,GAAG,MAAM;AAC3D,IAAA,MAAM,UAAU,GAAG,CAAC,GAAG,WAAW;AAClC,IAAA,MAAM,WAAW,GAAG,CAAC,GAAG,YAAY;AAEpC,IAAA,MAAM,cAAc,GAAG,aAAa,CAClC,EAAE,EACF,EAAE,CAAC,eAAe,EAClB,oBAAoB,CACrB;AACD,IAAA,MAAM,OAAO,GAAG,0BAA0B,CACxC,EAAE,EACF,YAAY,EACZ,cAAc,EACd,cAAc,EACd,cAAc,CACf;IACD,MAAM,kBAAkB,GAAG,EAAE,CAAC,kBAAkB,CAAC,OAAO,EAAE,cAAc,CAAC;IACzE,MAAM,wBAAwB,GAAG,EAAE,CAAC,kBAAkB,CACpD,OAAO,EACP,oBAAoB,CACrB;IACD,MAAM,iBAAiB,GAAG,EAAE,CAAC,kBAAkB,CAAC,OAAO,EAAE,aAAa,CAAC;IACvE,MAAM,YAAY,GAAG,EAAE,CAAC,kBAAkB,CAAC,OAAO,EAAE,QAAQ,CAAC;IAC7D,MAAM,cAAc,GAAG,EAAE,CAAC,kBAAkB,CAAC,OAAO,EAAE,UAAU,CAAC;IACjE,MAAM,cAAc,GAAG,EAAE,CAAC,kBAAkB,CAAC,OAAO,EAAE,UAAU,CAAC;IACjE,MAAM,kBAAkB,GAAG,EAAE,CAAC,kBAAkB,CAAC,OAAO,EAAE,cAAc,CAAC;IACzE,MAAM,kBAAkB,GAAG,EAAE,CAAC,kBAAkB,CAAC,OAAO,EAAE,cAAc,CAAC;AAEzE,IAAA,MAAM,WAAW,GAAG,EAAE,CAAC,iBAAiB,EAAE;IAC1C,EAAE,CAAC,eAAe,CAAC,EAAE,CAAC,WAAW,EAAE,WAAW,CAAC;AAC/C,IAAA,EAAE,CAAC,oBAAoB,CACrB,EAAE,CAAC,WAAW,EACd,EAAE,CAAC,iBAAiB,EACpB,EAAE,CAAC,UAAU,EACb,aAAa,EACb,CAAC,CACF;AAED,IAAA,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC;AACtB,IAAA,EAAE,CAAC,SAAS,CAAC,kBAAkB,EAAE,CAAC,CAAC;AACnC,IAAA,EAAE,CAAC,SAAS,CAAC,wBAAwB,EAAE,CAAC,CAAC;IACzC,EAAE,CAAC,SAAS,CAAC,iBAAiB,EAAE,UAAU,EAAE,WAAW,CAAC;;;IAIxD,gBAAgB,CAAC,CAAC,CAAC;IACnB,gBAAgB,CAAC,CAAC,CAAC;AAEnB,IAAA,SAAS,MAAM,GAAA;QACb,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,EAAE,WAAW,EAAE,YAAY,CAAC;AAC5C,QAAA,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC;AACtB,QAAA,EAAE,CAAC,aAAa,CAAC,EAAE,CAAC,QAAQ,CAAC;QAC7B,EAAE,CAAC,WAAW,CAAC,EAAE,CAAC,UAAU,EAAE,YAAY,CAAC;QAC3C,EAAE,CAAC,eAAe,CAAC,EAAE,CAAC,WAAW,EAAE,WAAW,CAAC;QAC/C,EAAE,CAAC,UAAU,CAAC,EAAE,CAAC,cAAc,EAAE,CAAC,EAAE,CAAC,CAAC;IACxC;IAEA,SAAS,gBAAgB,CAAC,UAAkB,EAAA;AAC1C,QAAA,UAAU,IAAI,IAAI,CAAC,GAAG,CACpB,WAAW,GAAG,iBAAiB,EAC/B,YAAY,GAAG,kBAAkB,CAClC;AAED,QAAA,MAAM,eAAe,GAAG,IAAI,CAAC;AAC7B,QAAA,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,eAAe,CAAC;QACjE,MAAM,MAAM,GAAG,UAAU;AACzB,QAAA,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,IAAI,GAAG,GAAG,GAAG,CAAC;AACxC,QAAA,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,WAAW,CAAC,GAAG,UAAU;AAEjE,QAAA,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC;AACtB,QAAA,EAAE,CAAC,SAAS,CAAC,YAAY,EAAE,IAAI,CAAC;AAChC,QAAA,EAAE,CAAC,SAAS,CAAC,cAAc,EAAE,MAAM,CAAC;AACpC,QAAA,EAAE,CAAC,SAAS,CAAC,cAAc,EAAE,MAAM,CAAC;AACpC,QAAA,EAAE,CAAC,SAAS,CAAC,kBAAkB,EAAE,UAAU,CAAC;IAC9C;IAEA,SAAS,gBAAgB,CAAC,UAAkB,EAAA;AAC1C,QAAA,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC;AACtB,QAAA,EAAE,CAAC,SAAS,CAAC,kBAAkB,EAAE,UAAU,CAAC;IAC9C;AAEA,IAAA,SAAS,OAAO,GAAA;AACd,QAAA,EAAE,CAAC,iBAAiB,CAAC,WAAW,CAAC;AACjC,QAAA,EAAE,CAAC,aAAa,CAAC,OAAO,CAAC;AACzB,QAAA,EAAE,CAAC,YAAY,CAAC,cAAc,CAAC;IACjC;IAEA,OAAO,EAAE,MAAM,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,OAAO,EAAE;AAChE;;;;"}

package/dist/esm/src/legacy/webgl2/resizingStage.js

@@ -0,0 +1,57 @@
+import { compileShader, createPipelineStageProgram, createTexture, glsl, readPixelsAsync } from '../helpers/webglHelper.js';
+
+function buildResizingStage(gl, vertexShader, positionBuffer, texCoordBuffer, tflite, segmentationConfig, onError) {
+    const fragmentShaderSource = glsl `#version 300 es
+
+    precision highp float;
+    uniform sampler2D u_inputFrame;
+    in vec2 v_texCoord;
+    out vec4 outColor;
+
+    void main() {
+      outColor = texture(u_inputFrame, v_texCoord);
+    }
+  `;
+    // TFLite memory will be accessed as float32
+    const tfliteInputMemoryOffset = tflite._getInputMemoryOffset() / 4;
+    const { width: outputWidth, height: outputHeight } = segmentationConfig;
+    const outputPixelCount = outputWidth * outputHeight;
+    const fragmentShader = compileShader(gl, gl.FRAGMENT_SHADER, fragmentShaderSource);
+    const program = createPipelineStageProgram(gl, vertexShader, fragmentShader, positionBuffer, texCoordBuffer);
+    const inputFrameLocation = gl.getUniformLocation(program, 'u_inputFrame');
+    const outputTexture = createTexture(gl, gl.RGBA8, outputWidth, outputHeight);
+    const frameBuffer = gl.createFramebuffer();
+    gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer);
+    gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, outputTexture, 0);
+    const outputPixels = new Uint8Array(outputPixelCount * 4);
+    gl.useProgram(program);
+    gl.uniform1i(inputFrameLocation, 0);
+    function render() {
+        gl.viewport(0, 0, outputWidth, outputHeight);
+        gl.useProgram(program);
+        gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer);
+        gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4);
+        // Downloads pixels asynchronously from GPU while rendering the current frame.
+        // The pixels will be available in the next frame render which results
+        // in offsets in the segmentation output but increases the frame rate.
+        readPixelsAsync(gl, 0, 0, outputWidth, outputHeight, gl.RGBA, gl.UNSIGNED_BYTE, outputPixels).catch((error) => {
+        });
+        for (let i = 0; i < outputPixelCount; i++) {
+            const tfliteIndex = tfliteInputMemoryOffset + i * 3;
+            const outputIndex = i * 4;
+            tflite.HEAPF32[tfliteIndex] = outputPixels[outputIndex] / 255;
+            tflite.HEAPF32[tfliteIndex + 1] = outputPixels[outputIndex + 1] / 255;
+            tflite.HEAPF32[tfliteIndex + 2] = outputPixels[outputIndex + 2] / 255;
+        }
+    }
+    function cleanUp() {
+        gl.deleteFramebuffer(frameBuffer);
+        gl.deleteTexture(outputTexture);
+        gl.deleteProgram(program);
+        gl.deleteShader(fragmentShader);
+    }
+    return { render, cleanUp };
+}
+
+export { buildResizingStage };
+//# sourceMappingURL=resizingStage.js.map

package/dist/esm/src/legacy/webgl2/resizingStage.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"resizingStage.js","sources":["../../../../../../../src/legacy/webgl2/resizingStage.ts"],"sourcesContent":["import { TFLite } from '../tflite';\nimport {\n  compileShader,\n  createPipelineStageProgram,\n  createTexture,\n  glsl,\n  readPixelsAsync,\n} from '../helpers/webglHelper';\nimport { SegmentationParams } from '../segmentation';\n\nexport function buildResizingStage(\n  gl: WebGL2RenderingContext,\n  vertexShader: WebGLShader,\n  positionBuffer: WebGLBuffer,\n  texCoordBuffer: WebGLBuffer,\n  tflite: TFLite,\n  segmentationConfig: SegmentationParams,\n  onError?: (error: any) => void,\n) {\n  const fragmentShaderSource = glsl`#version 300 es\n\n    precision highp float;\n    uniform sampler2D u_inputFrame;\n    in vec2 v_texCoord;\n    out vec4 outColor;\n\n    void main() {\n      outColor = texture(u_inputFrame, v_texCoord);\n    }\n  `;\n\n  // TFLite memory will be accessed as float32\n  const tfliteInputMemoryOffset = tflite._getInputMemoryOffset() / 4;\n\n  const { width: outputWidth, height: outputHeight } = segmentationConfig;\n  const outputPixelCount = outputWidth * outputHeight;\n\n  const fragmentShader = compileShader(\n    gl,\n    gl.FRAGMENT_SHADER,\n    fragmentShaderSource,\n  );\n  const program = createPipelineStageProgram(\n    gl,\n    vertexShader,\n    fragmentShader,\n    positionBuffer,\n    texCoordBuffer,\n  );\n  const inputFrameLocation = gl.getUniformLocation(program, 'u_inputFrame');\n  const outputTexture = createTexture(gl, gl.RGBA8, outputWidth, outputHeight);\n\n  const frameBuffer = gl.createFramebuffer();\n  gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer);\n  gl.framebufferTexture2D(\n    gl.FRAMEBUFFER,\n    gl.COLOR_ATTACHMENT0,\n    gl.TEXTURE_2D,\n    outputTexture,\n    0,\n  );\n  const outputPixels = new Uint8Array(outputPixelCount * 4);\n\n  gl.useProgram(program);\n  gl.uniform1i(inputFrameLocation, 0);\n\n  function render() {\n    gl.viewport(0, 0, outputWidth, outputHeight);\n    gl.useProgram(program);\n    gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer);\n    gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4);\n\n    // Downloads pixels asynchronously from GPU while rendering the current frame.\n    // The pixels will be available in the next frame render which results\n    // in offsets in the segmentation output but increases the frame rate.\n    readPixelsAsync(\n      gl,\n      0,\n      0,\n      outputWidth,\n      outputHeight,\n      gl.RGBA,\n      gl.UNSIGNED_BYTE,\n      outputPixels,\n    ).catch((error: any) => {\n      onError?.(error);\n    });\n\n    for (let i = 0; i < outputPixelCount; i++) {\n      const tfliteIndex = tfliteInputMemoryOffset + i * 3;\n      const outputIndex = i * 4;\n      tflite.HEAPF32[tfliteIndex] = outputPixels[outputIndex] / 255;\n      tflite.HEAPF32[tfliteIndex + 1] = outputPixels[outputIndex + 1] / 255;\n      tflite.HEAPF32[tfliteIndex + 2] = outputPixels[outputIndex + 2] / 255;\n    }\n  }\n\n  function cleanUp() {\n    gl.deleteFramebuffer(frameBuffer);\n    gl.deleteTexture(outputTexture);\n    gl.deleteProgram(program);\n    gl.deleteShader(fragmentShader);\n  }\n\n  return { render, cleanUp };\n}\n"],"names":[],"mappings":";;AAUM,SAAU,kBAAkB,CAChC,EAA0B,EAC1B,YAAyB,EACzB,cAA2B,EAC3B,cAA2B,EAC3B,MAAc,EACd,kBAAsC,EACtC,OAA8B,EAAA;IAE9B,MAAM,oBAAoB,GAAG,IAAI,CAAA,CAAA;;;;;;;;;;GAUhC;;IAGD,MAAM,uBAAuB,GAAG,MAAM,CAAC,qBAAqB,EAAE,GAAG,CAAC;IAElE,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,MAAM,EAAE,YAAY,EAAE,GAAG,kBAAkB;AACvE,IAAA,MAAM,gBAAgB,GAAG,WAAW,GAAG,YAAY;AAEnD,IAAA,MAAM,cAAc,GAAG,aAAa,CAClC,EAAE,EACF,EAAE,CAAC,eAAe,EAClB,oBAAoB,CACrB;AACD,IAAA,MAAM,OAAO,GAAG,0BAA0B,CACxC,EAAE,EACF,YAAY,EACZ,cAAc,EACd,cAAc,EACd,cAAc,CACf;IACD,MAAM,kBAAkB,GAAG,EAAE,CAAC,kBAAkB,CAAC,OAAO,EAAE,cAAc,CAAC;AACzE,IAAA,MAAM,aAAa,GAAG,aAAa,CAAC,EAAE,EAAE,EAAE,CAAC,KAAK,EAAE,WAAW,EAAE,YAAY,CAAC;AAE5E,IAAA,MAAM,WAAW,GAAG,EAAE,CAAC,iBAAiB,EAAE;IAC1C,EAAE,CAAC,eAAe,CAAC,EAAE,CAAC,WAAW,EAAE,WAAW,CAAC;AAC/C,IAAA,EAAE,CAAC,oBAAoB,CACrB,EAAE,CAAC,WAAW,EACd,EAAE,CAAC,iBAAiB,EACpB,EAAE,CAAC,UAAU,EACb,aAAa,EACb,CAAC,CACF;IACD,MAAM,YAAY,GAAG,IAAI,UAAU,CAAC,gBAAgB,GAAG,CAAC,CAAC;AAEzD,IAAA,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC;AACtB,IAAA,EAAE,CAAC,SAAS,CAAC,kBAAkB,EAAE,CAAC,CAAC;AAEnC,IAAA,SAAS,MAAM,GAAA;QACb,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,EAAE,WAAW,EAAE,YAAY,CAAC;AAC5C,QAAA,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC;QACtB,EAAE,CAAC,eAAe,CAAC,EAAE,CAAC,WAAW,EAAE,WAAW,CAAC;QAC/C,EAAE,CAAC,UAAU,CAAC,EAAE,CAAC,cAAc,EAAE,CAAC,EAAE,CAAC,CAAC;;;;QAKtC,eAAe,CACb,EAAE,EACF,CAAC,EACD,CAAC,EACD,WAAW,EACX,YAAY,EACZ,EAAE,CAAC,IAAI,EACP,EAAE,CAAC,aAAa,EAChB,YAAY,CACb,CAAC,KAAK,CAAC,CAAC,KAAU,KAAI;AAEvB,QAAA,CAAC,CAAC;AAEF,QAAA,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,gBAAgB,EAAE,CAAC,EAAE,EAAE;AACzC,YAAA,MAAM,WAAW,GAAG,uBAAuB,GAAG,CAAC,GAAG,CAAC;AACnD,YAAA,MAAM,WAAW,GAAG,CAAC,GAAG,CAAC;AACzB,YAAA,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,GAAG,YAAY,CAAC,WAAW,CAAC,GAAG,GAAG;AAC7D,YAAA,MAAM,CAAC,OAAO,CAAC,WAAW,GAAG,CAAC,CAAC,GAAG,YAAY,CAAC,WAAW,GAAG,CAAC,CAAC,GAAG,GAAG;AACrE,YAAA,MAAM,CAAC,OAAO,CAAC,WAAW,GAAG,CAAC,CAAC,GAAG,YAAY,CAAC,WAAW,GAAG,CAAC,CAAC,GAAG,GAAG;QACvE;IACF;AAEA,IAAA,SAAS,OAAO,GAAA;AACd,QAAA,EAAE,CAAC,iBAAiB,CAAC,WAAW,CAAC;AACjC,QAAA,EAAE,CAAC,aAAa,CAAC,aAAa,CAAC;AAC/B,QAAA,EAAE,CAAC,aAAa,CAAC,OAAO,CAAC;AACzB,QAAA,EAAE,CAAC,YAAY,CAAC,cAAc,CAAC;IACjC;AAEA,IAAA,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE;AAC5B;;;;"}

package/dist/esm/src/legacy/webgl2/softmaxStage.js

@@ -0,0 +1,51 @@
+import { compileShader, createPipelineStageProgram, createTexture, glsl } from '../helpers/webglHelper.js';
+
+function buildSoftmaxStage(gl, vertexShader, positionBuffer, texCoordBuffer, tflite, outputTexture, segmentationConfig) {
+    const fragmentShaderSource = glsl `#version 300 es
+
+    precision highp float;
+
+    uniform sampler2D u_inputSegmentation;
+    in vec2 v_texCoord;
+    out vec4 outColor;
+
+    void main() {
+      vec2 segmentation = texture(u_inputSegmentation, v_texCoord).rg;
+      float shift = max(segmentation.r, segmentation.g);
+      float backgroundExp = exp(segmentation.r - shift);
+      float personExp = exp(segmentation.g - shift);
+      outColor = vec4(vec3(0.0), personExp / (backgroundExp + personExp));
+    }
+  `;
+    // TFLite memory will be accessed as float32
+    const tfliteOutputMemoryOffset = tflite._getOutputMemoryOffset() / 4;
+    const { width: segmentationWidth, height: segmentationHeight } = segmentationConfig;
+    const fragmentShader = compileShader(gl, gl.FRAGMENT_SHADER, fragmentShaderSource);
+    const program = createPipelineStageProgram(gl, vertexShader, fragmentShader, positionBuffer, texCoordBuffer);
+    const inputLocation = gl.getUniformLocation(program, 'u_inputSegmentation');
+    const inputTexture = createTexture(gl, gl.RG32F, segmentationWidth, segmentationHeight);
+    const frameBuffer = gl.createFramebuffer();
+    gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer);
+    gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, outputTexture, 0);
+    gl.useProgram(program);
+    gl.uniform1i(inputLocation, 1);
+    function render() {
+        gl.viewport(0, 0, segmentationWidth, segmentationHeight);
+        gl.useProgram(program);
+        gl.activeTexture(gl.TEXTURE1);
+        gl.bindTexture(gl.TEXTURE_2D, inputTexture);
+        gl.texSubImage2D(gl.TEXTURE_2D, 0, 0, 0, segmentationWidth, segmentationHeight, gl.RG, gl.FLOAT, tflite.HEAPF32, tfliteOutputMemoryOffset);
+        gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer);
+        gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4);
+    }
+    function cleanUp() {
+        gl.deleteFramebuffer(frameBuffer);
+        gl.deleteTexture(inputTexture);
+        gl.deleteProgram(program);
+        gl.deleteShader(fragmentShader);
+    }
+    return { render, cleanUp };
+}
+
+export { buildSoftmaxStage };
+//# sourceMappingURL=softmaxStage.js.map

package/dist/esm/src/legacy/webgl2/softmaxStage.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"softmaxStage.js","sources":["../../../../../../../src/legacy/webgl2/softmaxStage.ts"],"sourcesContent":["import { TFLite } from '../tflite';\nimport {\n  compileShader,\n  createPipelineStageProgram,\n  createTexture,\n  glsl,\n} from '../helpers/webglHelper';\nimport { SegmentationParams } from '../segmentation';\n\nexport function buildSoftmaxStage(\n  gl: WebGL2RenderingContext,\n  vertexShader: WebGLShader,\n  positionBuffer: WebGLBuffer,\n  texCoordBuffer: WebGLBuffer,\n  tflite: TFLite,\n  outputTexture: WebGLTexture,\n  segmentationConfig: SegmentationParams,\n) {\n  const fragmentShaderSource = glsl`#version 300 es\n\n    precision highp float;\n\n    uniform sampler2D u_inputSegmentation;\n    in vec2 v_texCoord;\n    out vec4 outColor;\n\n    void main() {\n      vec2 segmentation = texture(u_inputSegmentation, v_texCoord).rg;\n      float shift = max(segmentation.r, segmentation.g);\n      float backgroundExp = exp(segmentation.r - shift);\n      float personExp = exp(segmentation.g - shift);\n      outColor = vec4(vec3(0.0), personExp / (backgroundExp + personExp));\n    }\n  `;\n\n  // TFLite memory will be accessed as float32\n  const tfliteOutputMemoryOffset = tflite._getOutputMemoryOffset() / 4;\n  const { width: segmentationWidth, height: segmentationHeight } =\n    segmentationConfig;\n\n  const fragmentShader = compileShader(\n    gl,\n    gl.FRAGMENT_SHADER,\n    fragmentShaderSource,\n  );\n  const program = createPipelineStageProgram(\n    gl,\n    vertexShader,\n    fragmentShader,\n    positionBuffer,\n    texCoordBuffer,\n  );\n  const inputLocation = gl.getUniformLocation(program, 'u_inputSegmentation');\n  const inputTexture = createTexture(\n    gl,\n    gl.RG32F,\n    segmentationWidth,\n    segmentationHeight,\n  );\n\n  const frameBuffer = gl.createFramebuffer();\n  gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer);\n  gl.framebufferTexture2D(\n    gl.FRAMEBUFFER,\n    gl.COLOR_ATTACHMENT0,\n    gl.TEXTURE_2D,\n    outputTexture,\n    0,\n  );\n\n  gl.useProgram(program);\n  gl.uniform1i(inputLocation, 1);\n\n  function render() {\n    gl.viewport(0, 0, segmentationWidth, segmentationHeight);\n    gl.useProgram(program);\n    gl.activeTexture(gl.TEXTURE1);\n    gl.bindTexture(gl.TEXTURE_2D, inputTexture);\n    gl.texSubImage2D(\n      gl.TEXTURE_2D,\n      0,\n      0,\n      0,\n      segmentationWidth,\n      segmentationHeight,\n      gl.RG,\n      gl.FLOAT,\n      tflite.HEAPF32,\n      tfliteOutputMemoryOffset,\n    );\n    gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer);\n    gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4);\n  }\n\n  function cleanUp() {\n    gl.deleteFramebuffer(frameBuffer);\n    gl.deleteTexture(inputTexture);\n    gl.deleteProgram(program);\n    gl.deleteShader(fragmentShader);\n  }\n\n  return { render, cleanUp };\n}\n"],"names":[],"mappings":";;AASM,SAAU,iBAAiB,CAC/B,EAA0B,EAC1B,YAAyB,EACzB,cAA2B,EAC3B,cAA2B,EAC3B,MAAc,EACd,aAA2B,EAC3B,kBAAsC,EAAA;IAEtC,MAAM,oBAAoB,GAAG,IAAI,CAAA,CAAA;;;;;;;;;;;;;;;GAehC;;IAGD,MAAM,wBAAwB,GAAG,MAAM,CAAC,sBAAsB,EAAE,GAAG,CAAC;IACpE,MAAM,EAAE,KAAK,EAAE,iBAAiB,EAAE,MAAM,EAAE,kBAAkB,EAAE,GAC5D,kBAAkB;AAEpB,IAAA,MAAM,cAAc,GAAG,aAAa,CAClC,EAAE,EACF,EAAE,CAAC,eAAe,EAClB,oBAAoB,CACrB;AACD,IAAA,MAAM,OAAO,GAAG,0BAA0B,CACxC,EAAE,EACF,YAAY,EACZ,cAAc,EACd,cAAc,EACd,cAAc,CACf;IACD,MAAM,aAAa,GAAG,EAAE,CAAC,kBAAkB,CAAC,OAAO,EAAE,qBAAqB,CAAC;AAC3E,IAAA,MAAM,YAAY,GAAG,aAAa,CAChC,EAAE,EACF,EAAE,CAAC,KAAK,EACR,iBAAiB,EACjB,kBAAkB,CACnB;AAED,IAAA,MAAM,WAAW,GAAG,EAAE,CAAC,iBAAiB,EAAE;IAC1C,EAAE,CAAC,eAAe,CAAC,EAAE,CAAC,WAAW,EAAE,WAAW,CAAC;AAC/C,IAAA,EAAE,CAAC,oBAAoB,CACrB,EAAE,CAAC,WAAW,EACd,EAAE,CAAC,iBAAiB,EACpB,EAAE,CAAC,UAAU,EACb,aAAa,EACb,CAAC,CACF;AAED,IAAA,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC;AACtB,IAAA,EAAE,CAAC,SAAS,CAAC,aAAa,EAAE,CAAC,CAAC;AAE9B,IAAA,SAAS,MAAM,GAAA;QACb,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,EAAE,iBAAiB,EAAE,kBAAkB,CAAC;AACxD,QAAA,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC;AACtB,QAAA,EAAE,CAAC,aAAa,CAAC,EAAE,CAAC,QAAQ,CAAC;QAC7B,EAAE,CAAC,WAAW,CAAC,EAAE,CAAC,UAAU,EAAE,YAAY,CAAC;AAC3C,QAAA,EAAE,CAAC,aAAa,CACd,EAAE,CAAC,UAAU,EACb,CAAC,EACD,CAAC,EACD,CAAC,EACD,iBAAiB,EACjB,kBAAkB,EAClB,EAAE,CAAC,EAAE,EACL,EAAE,CAAC,KAAK,EACR,MAAM,CAAC,OAAO,EACd,wBAAwB,CACzB;QACD,EAAE,CAAC,eAAe,CAAC,EAAE,CAAC,WAAW,EAAE,WAAW,CAAC;QAC/C,EAAE,CAAC,UAAU,CAAC,EAAE,CAAC,cAAc,EAAE,CAAC,EAAE,CAAC,CAAC;IACxC;AAEA,IAAA,SAAS,OAAO,GAAA;AACd,QAAA,EAAE,CAAC,iBAAiB,CAAC,WAAW,CAAC;AACjC,QAAA,EAAE,CAAC,aAAa,CAAC,YAAY,CAAC;AAC9B,QAAA,EAAE,CAAC,aAAa,CAAC,OAAO,CAAC;AACzB,QAAA,EAAE,CAAC,YAAY,CAAC,cAAc,CAAC;IACjC;AAEA,IAAA,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE;AAC5B;;;;"}

package/dist/esm/src/legacy/webgl2/webgl2Pipeline.js

@@ -0,0 +1,105 @@
+import { compileShader, createTexture, glsl } from '../helpers/webglHelper.js';
+import { buildBackgroundBlurStage } from './backgroundBlurStage.js';
+import { buildBackgroundImageStage } from './backgroundImageStage.js';
+import { buildJointBilateralFilterStage } from './jointBilateralFilterStage.js';
+import { buildResizingStage } from './resizingStage.js';
+import { buildSoftmaxStage } from './softmaxStage.js';
+
+function buildWebGL2Pipeline(videoSource, backgroundImage, blurLevel, backgroundFilter, canvas, tflite, segmentationConfig, onError) {
+    const gl = canvas.getContext('webgl2');
+    if (!gl)
+        throw new Error('WebGL2 is not supported');
+    if (gl.isContextLost())
+        throw new Error('WebGL2 context was lost');
+    const { width: frameWidth, height: frameHeight } = videoSource;
+    const { width: segmentationWidth, height: segmentationHeight } = segmentationConfig;
+    const vertexShaderSource = glsl `#version 300 es
+
+    in vec2 a_position;
+    in vec2 a_texCoord;
+    out vec2 v_texCoord;
+
+    void main() {
+      gl_Position = vec4(a_position, 0.0, 1.0);
+      v_texCoord = a_texCoord;
+    }
+  `;
+    const vertexShader = compileShader(gl, gl.VERTEX_SHADER, vertexShaderSource);
+    const vertexArray = gl.createVertexArray();
+    gl.bindVertexArray(vertexArray);
+    const positionBuffer = gl.createBuffer();
+    gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
+    gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([-1, -1, 1.0, -1, -1, 1.0, 1.0, 1.0]), gl.STATIC_DRAW);
+    const texCoordBuffer = gl.createBuffer();
+    gl.bindBuffer(gl.ARRAY_BUFFER, texCoordBuffer);
+    gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([0.0, 0.0, 1.0, 0.0, 0.0, 1.0, 1.0, 1.0]), gl.STATIC_DRAW);
+    // We don't use texStorage2D here because texImage2D seems faster
+    // to upload video texture than texSubImage2D even though the latter
+    // is supposed to be the recommended way:
+    // https://developer.mozilla.org/en-US/docs/Web/API/WebGL_API/WebGL_best_practices#use_texstorage_to_create_textures
+    const inputFrameTexture = gl.createTexture();
+    gl.bindTexture(gl.TEXTURE_2D, inputFrameTexture);
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
+    // TODO Rename segmentation and person mask to be more specific
+    const segmentationTexture = createTexture(gl, gl.RGBA8, segmentationWidth, segmentationHeight);
+    const personMaskTexture = createTexture(gl, gl.RGBA8, frameWidth, frameHeight);
+    const resizingStage = buildResizingStage(gl, vertexShader, positionBuffer, texCoordBuffer, tflite, segmentationConfig);
+    const loadSegmentationStage = buildSoftmaxStage(gl, vertexShader, positionBuffer, texCoordBuffer, tflite, segmentationTexture, segmentationConfig);
+    const jointBilateralFilterStage = buildJointBilateralFilterStage(gl, vertexShader, positionBuffer, texCoordBuffer, segmentationTexture, personMaskTexture, canvas, segmentationConfig);
+    const backgroundStage = backgroundFilter === 'blur'
+        ? buildBackgroundBlurStage(gl, vertexShader, positionBuffer, texCoordBuffer, personMaskTexture, canvas, blurLevel || 'high')
+        : buildBackgroundImageStage(gl, positionBuffer, texCoordBuffer, personMaskTexture, backgroundImage, canvas);
+    function render() {
+        gl.activeTexture(gl.TEXTURE0);
+        gl.bindTexture(gl.TEXTURE_2D, inputFrameTexture);
+        // texImage2D seems faster than texSubImage2D to upload
+        // video texture
+        gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, videoSource);
+        gl.bindVertexArray(vertexArray);
+        resizingStage.render();
+        tflite._runInference();
+        loadSegmentationStage.render();
+        jointBilateralFilterStage.render();
+        backgroundStage.render();
+    }
+    function updatePostProcessingConfig() {
+        jointBilateralFilterStage.updateSigmaSpace(1);
+        jointBilateralFilterStage.updateSigmaColor(0.1);
+        if (backgroundFilter === 'image') {
+            const backgroundImageStage = backgroundStage;
+            backgroundImageStage.updateCoverage([0.5, 0.75]);
+            backgroundImageStage.updateLightWrapping(0.3);
+            backgroundImageStage.updateBlendMode('screen');
+        }
+        else if (backgroundFilter === 'blur') {
+            const backgroundBlurStage = backgroundStage;
+            backgroundBlurStage.updateCoverage([0.5, 0.75]);
+        }
+        else {
+            // TODO Handle no background in a separate pipeline path
+            const backgroundImageStage = backgroundStage;
+            backgroundImageStage.updateCoverage([0, 0.9999]);
+            backgroundImageStage.updateLightWrapping(0);
+        }
+    }
+    function cleanUp() {
+        backgroundStage.cleanUp();
+        jointBilateralFilterStage.cleanUp();
+        loadSegmentationStage.cleanUp();
+        resizingStage.cleanUp();
+        gl.deleteTexture(personMaskTexture);
+        gl.deleteTexture(segmentationTexture);
+        gl.deleteTexture(inputFrameTexture);
+        gl.deleteBuffer(texCoordBuffer);
+        gl.deleteBuffer(positionBuffer);
+        gl.deleteVertexArray(vertexArray);
+        gl.deleteShader(vertexShader);
+    }
+    return { render, updatePostProcessingConfig, cleanUp };
+}
+
+export { buildWebGL2Pipeline };
+//# sourceMappingURL=webgl2Pipeline.js.map

package/dist/esm/src/legacy/webgl2/webgl2Pipeline.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"webgl2Pipeline.js","sources":["../../../../../../../src/legacy/webgl2/webgl2Pipeline.ts"],"sourcesContent":["import { TFLite } from '../tflite';\nimport { compileShader, createTexture, glsl } from '../helpers/webglHelper';\nimport {\n  BackgroundBlurStage,\n  buildBackgroundBlurStage,\n} from './backgroundBlurStage';\nimport {\n  BackgroundImageStage,\n  buildBackgroundImageStage,\n} from './backgroundImageStage';\nimport { buildJointBilateralFilterStage } from './jointBilateralFilterStage';\nimport { buildResizingStage } from './resizingStage';\nimport { buildSoftmaxStage } from './softmaxStage';\nimport { SegmentationParams } from '../segmentation';\nimport type { BackgroundBlurLevel, BackgroundFilter } from '../../types';\n\nexport function buildWebGL2Pipeline(\n  videoSource: HTMLVideoElement,\n  backgroundImage: HTMLImageElement | undefined,\n  blurLevel: BackgroundBlurLevel | undefined,\n  backgroundFilter: BackgroundFilter,\n  canvas: HTMLCanvasElement,\n  tflite: TFLite,\n  segmentationConfig: SegmentationParams,\n  onError?: (error: any) => void,\n) {\n  const gl = canvas.getContext('webgl2')!;\n  if (!gl) throw new Error('WebGL2 is not supported');\n  if (gl.isContextLost()) throw new Error('WebGL2 context was lost');\n\n  const { width: frameWidth, height: frameHeight } = videoSource;\n  const { width: segmentationWidth, height: segmentationHeight } =\n    segmentationConfig;\n\n  const vertexShaderSource = glsl`#version 300 es\n\n    in vec2 a_position;\n    in vec2 a_texCoord;\n    out vec2 v_texCoord;\n\n    void main() {\n      gl_Position = vec4(a_position, 0.0, 1.0);\n      v_texCoord = a_texCoord;\n    }\n  `;\n  const vertexShader = compileShader(gl, gl.VERTEX_SHADER, vertexShaderSource);\n\n  const vertexArray = gl.createVertexArray();\n  gl.bindVertexArray(vertexArray);\n\n  const positionBuffer = gl.createBuffer()!;\n  gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);\n  gl.bufferData(\n    gl.ARRAY_BUFFER,\n    new Float32Array([-1.0, -1.0, 1.0, -1.0, -1.0, 1.0, 1.0, 1.0]),\n    gl.STATIC_DRAW,\n  );\n\n  const texCoordBuffer = gl.createBuffer()!;\n  gl.bindBuffer(gl.ARRAY_BUFFER, texCoordBuffer);\n  gl.bufferData(\n    gl.ARRAY_BUFFER,\n    new Float32Array([0.0, 0.0, 1.0, 0.0, 0.0, 1.0, 1.0, 1.0]),\n    gl.STATIC_DRAW,\n  );\n\n  // We don't use texStorage2D here because texImage2D seems faster\n  // to upload video texture than texSubImage2D even though the latter\n  // is supposed to be the recommended way:\n  // https://developer.mozilla.org/en-US/docs/Web/API/WebGL_API/WebGL_best_practices#use_texstorage_to_create_textures\n  const inputFrameTexture = gl.createTexture();\n  gl.bindTexture(gl.TEXTURE_2D, inputFrameTexture);\n  gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);\n  gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);\n  gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);\n  gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);\n\n  // TODO Rename segmentation and person mask to be more specific\n  const segmentationTexture = createTexture(\n    gl,\n    gl.RGBA8,\n    segmentationWidth,\n    segmentationHeight,\n  )!;\n  const personMaskTexture = createTexture(\n    gl,\n    gl.RGBA8,\n    frameWidth,\n    frameHeight,\n  )!;\n\n  const resizingStage = buildResizingStage(\n    gl,\n    vertexShader,\n    positionBuffer,\n    texCoordBuffer,\n    tflite,\n    segmentationConfig,\n    onError,\n  );\n  const loadSegmentationStage = buildSoftmaxStage(\n    gl,\n    vertexShader,\n    positionBuffer,\n    texCoordBuffer,\n    tflite,\n    segmentationTexture,\n    segmentationConfig,\n  );\n\n  const jointBilateralFilterStage = buildJointBilateralFilterStage(\n    gl,\n    vertexShader,\n    positionBuffer,\n    texCoordBuffer,\n    segmentationTexture,\n    personMaskTexture,\n    canvas,\n    segmentationConfig,\n  );\n  const backgroundStage =\n    backgroundFilter === 'blur'\n      ? buildBackgroundBlurStage(\n          gl,\n          vertexShader,\n          positionBuffer,\n          texCoordBuffer,\n          personMaskTexture,\n          canvas,\n          blurLevel || 'high',\n        )\n      : buildBackgroundImageStage(\n          gl,\n          positionBuffer,\n          texCoordBuffer,\n          personMaskTexture,\n          backgroundImage,\n          canvas,\n        );\n\n  function render() {\n    gl.activeTexture(gl.TEXTURE0);\n    gl.bindTexture(gl.TEXTURE_2D, inputFrameTexture);\n\n    // texImage2D seems faster than texSubImage2D to upload\n    // video texture\n    gl.texImage2D(\n      gl.TEXTURE_2D,\n      0,\n      gl.RGBA,\n      gl.RGBA,\n      gl.UNSIGNED_BYTE,\n      videoSource,\n    );\n\n    gl.bindVertexArray(vertexArray);\n\n    resizingStage.render();\n\n    tflite._runInference();\n\n    loadSegmentationStage.render();\n    jointBilateralFilterStage.render();\n    backgroundStage.render();\n  }\n\n  function updatePostProcessingConfig() {\n    jointBilateralFilterStage.updateSigmaSpace(1);\n    jointBilateralFilterStage.updateSigmaColor(0.1);\n\n    if (backgroundFilter === 'image') {\n      const backgroundImageStage = backgroundStage as BackgroundImageStage;\n      backgroundImageStage.updateCoverage([0.5, 0.75]);\n      backgroundImageStage.updateLightWrapping(0.3);\n      backgroundImageStage.updateBlendMode('screen');\n    } else if (backgroundFilter === 'blur') {\n      const backgroundBlurStage = backgroundStage as BackgroundBlurStage;\n      backgroundBlurStage.updateCoverage([0.5, 0.75]);\n    } else {\n      // TODO Handle no background in a separate pipeline path\n      const backgroundImageStage = backgroundStage as BackgroundImageStage;\n      backgroundImageStage.updateCoverage([0, 0.9999]);\n      backgroundImageStage.updateLightWrapping(0);\n    }\n  }\n\n  function cleanUp() {\n    backgroundStage.cleanUp();\n    jointBilateralFilterStage.cleanUp();\n    loadSegmentationStage.cleanUp();\n    resizingStage.cleanUp();\n\n    gl.deleteTexture(personMaskTexture);\n    gl.deleteTexture(segmentationTexture);\n    gl.deleteTexture(inputFrameTexture);\n    gl.deleteBuffer(texCoordBuffer);\n    gl.deleteBuffer(positionBuffer);\n    gl.deleteVertexArray(vertexArray);\n    gl.deleteShader(vertexShader);\n  }\n\n  return { render, updatePostProcessingConfig, cleanUp };\n}\n"],"names":[],"mappings":";;;;;;;SAgBgB,mBAAmB,CACjC,WAA6B,EAC7B,eAA6C,EAC7C,SAA0C,EAC1C,gBAAkC,EAClC,MAAyB,EACzB,MAAc,EACd,kBAAsC,EACtC,OAA8B,EAAA;IAE9B,MAAM,EAAE,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAE;AACvC,IAAA,IAAI,CAAC,EAAE;AAAE,QAAA,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC;IACnD,IAAI,EAAE,CAAC,aAAa,EAAE;AAAE,QAAA,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC;IAElE,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,WAAW;IAC9D,MAAM,EAAE,KAAK,EAAE,iBAAiB,EAAE,MAAM,EAAE,kBAAkB,EAAE,GAC5D,kBAAkB;IAEpB,MAAM,kBAAkB,GAAG,IAAI,CAAA,CAAA;;;;;;;;;;GAU9B;AACD,IAAA,MAAM,YAAY,GAAG,aAAa,CAAC,EAAE,EAAE,EAAE,CAAC,aAAa,EAAE,kBAAkB,CAAC;AAE5E,IAAA,MAAM,WAAW,GAAG,EAAE,CAAC,iBAAiB,EAAE;AAC1C,IAAA,EAAE,CAAC,eAAe,CAAC,WAAW,CAAC;AAE/B,IAAA,MAAM,cAAc,GAAG,EAAE,CAAC,YAAY,EAAG;IACzC,EAAE,CAAC,UAAU,CAAC,EAAE,CAAC,YAAY,EAAE,cAAc,CAAC;AAC9C,IAAA,EAAE,CAAC,UAAU,CACX,EAAE,CAAC,YAAY,EACf,IAAI,YAAY,CAAC,CAAC,EAAI,EAAE,EAAI,EAAE,GAAG,EAAE,EAAI,EAAE,EAAI,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,EAC9D,EAAE,CAAC,WAAW,CACf;AAED,IAAA,MAAM,cAAc,GAAG,EAAE,CAAC,YAAY,EAAG;IACzC,EAAE,CAAC,UAAU,CAAC,EAAE,CAAC,YAAY,EAAE,cAAc,CAAC;AAC9C,IAAA,EAAE,CAAC,UAAU,CACX,EAAE,CAAC,YAAY,EACf,IAAI,YAAY,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,EAC1D,EAAE,CAAC,WAAW,CACf;;;;;AAMD,IAAA,MAAM,iBAAiB,GAAG,EAAE,CAAC,aAAa,EAAE;IAC5C,EAAE,CAAC,WAAW,CAAC,EAAE,CAAC,UAAU,EAAE,iBAAiB,CAAC;AAChD,IAAA,EAAE,CAAC,aAAa,CAAC,EAAE,CAAC,UAAU,EAAE,EAAE,CAAC,cAAc,EAAE,EAAE,CAAC,aAAa,CAAC;AACpE,IAAA,EAAE,CAAC,aAAa,CAAC,EAAE,CAAC,UAAU,EAAE,EAAE,CAAC,cAAc,EAAE,EAAE,CAAC,aAAa,CAAC;AACpE,IAAA,EAAE,CAAC,aAAa,CAAC,EAAE,CAAC,UAAU,EAAE,EAAE,CAAC,kBAAkB,EAAE,EAAE,CAAC,OAAO,CAAC;AAClE,IAAA,EAAE,CAAC,aAAa,CAAC,EAAE,CAAC,UAAU,EAAE,EAAE,CAAC,kBAAkB,EAAE,EAAE,CAAC,OAAO,CAAC;;AAGlE,IAAA,MAAM,mBAAmB,GAAG,aAAa,CACvC,EAAE,EACF,EAAE,CAAC,KAAK,EACR,iBAAiB,EACjB,kBAAkB,CAClB;AACF,IAAA,MAAM,iBAAiB,GAAG,aAAa,CACrC,EAAE,EACF,EAAE,CAAC,KAAK,EACR,UAAU,EACV,WAAW,CACX;AAEF,IAAA,MAAM,aAAa,GAAG,kBAAkB,CACtC,EAAE,EACF,YAAY,EACZ,cAAc,EACd,cAAc,EACd,MAAM,EACN,kBACO,CACR;AACD,IAAA,MAAM,qBAAqB,GAAG,iBAAiB,CAC7C,EAAE,EACF,YAAY,EACZ,cAAc,EACd,cAAc,EACd,MAAM,EACN,mBAAmB,EACnB,kBAAkB,CACnB;IAED,MAAM,yBAAyB,GAAG,8BAA8B,CAC9D,EAAE,EACF,YAAY,EACZ,cAAc,EACd,cAAc,EACd,mBAAmB,EACnB,iBAAiB,EACjB,MAAM,EACN,kBAAkB,CACnB;AACD,IAAA,MAAM,eAAe,GACnB,gBAAgB,KAAK;AACnB,UAAE,wBAAwB,CACtB,EAAE,EACF,YAAY,EACZ,cAAc,EACd,cAAc,EACd,iBAAiB,EACjB,MAAM,EACN,SAAS,IAAI,MAAM;AAEvB,UAAE,yBAAyB,CACvB,EAAE,EACF,cAAc,EACd,cAAc,EACd,iBAAiB,EACjB,eAAe,EACf,MAAM,CACP;AAEP,IAAA,SAAS,MAAM,GAAA;AACb,QAAA,EAAE,CAAC,aAAa,CAAC,EAAE,CAAC,QAAQ,CAAC;QAC7B,EAAE,CAAC,WAAW,CAAC,EAAE,CAAC,UAAU,EAAE,iBAAiB,CAAC;;;QAIhD,EAAE,CAAC,UAAU,CACX,EAAE,CAAC,UAAU,EACb,CAAC,EACD,EAAE,CAAC,IAAI,EACP,EAAE,CAAC,IAAI,EACP,EAAE,CAAC,aAAa,EAChB,WAAW,CACZ;AAED,QAAA,EAAE,CAAC,eAAe,CAAC,WAAW,CAAC;QAE/B,aAAa,CAAC,MAAM,EAAE;QAEtB,MAAM,CAAC,aAAa,EAAE;QAEtB,qBAAqB,CAAC,MAAM,EAAE;QAC9B,yBAAyB,CAAC,MAAM,EAAE;QAClC,eAAe,CAAC,MAAM,EAAE;IAC1B;AAEA,IAAA,SAAS,0BAA0B,GAAA;AACjC,QAAA,yBAAyB,CAAC,gBAAgB,CAAC,CAAC,CAAC;AAC7C,QAAA,yBAAyB,CAAC,gBAAgB,CAAC,GAAG,CAAC;AAE/C,QAAA,IAAI,gBAAgB,KAAK,OAAO,EAAE;YAChC,MAAM,oBAAoB,GAAG,eAAuC;YACpE,oBAAoB,CAAC,cAAc,CAAC,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;AAChD,YAAA,oBAAoB,CAAC,mBAAmB,CAAC,GAAG,CAAC;AAC7C,YAAA,oBAAoB,CAAC,eAAe,CAAC,QAAQ,CAAC;QAChD;AAAO,aAAA,IAAI,gBAAgB,KAAK,MAAM,EAAE;YACtC,MAAM,mBAAmB,GAAG,eAAsC;YAClE,mBAAmB,CAAC,cAAc,CAAC,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QACjD;aAAO;;YAEL,MAAM,oBAAoB,GAAG,eAAuC;YACpE,oBAAoB,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;AAChD,YAAA,oBAAoB,CAAC,mBAAmB,CAAC,CAAC,CAAC;QAC7C;IACF;AAEA,IAAA,SAAS,OAAO,GAAA;QACd,eAAe,CAAC,OAAO,EAAE;QACzB,yBAAyB,CAAC,OAAO,EAAE;QACnC,qBAAqB,CAAC,OAAO,EAAE;QAC/B,aAAa,CAAC,OAAO,EAAE;AAEvB,QAAA,EAAE,CAAC,aAAa,CAAC,iBAAiB,CAAC;AACnC,QAAA,EAAE,CAAC,aAAa,CAAC,mBAAmB,CAAC;AACrC,QAAA,EAAE,CAAC,aAAa,CAAC,iBAAiB,CAAC;AACnC,QAAA,EAAE,CAAC,YAAY,CAAC,cAAc,CAAC;AAC/B,QAAA,EAAE,CAAC,YAAY,CAAC,cAAc,CAAC;AAC/B,QAAA,EAAE,CAAC,iBAAiB,CAAC,WAAW,CAAC;AACjC,QAAA,EAAE,CAAC,YAAY,CAAC,YAAY,CAAC;IAC/B;AAEA,IAAA,OAAO,EAAE,MAAM,EAAE,0BAA0B,EAAE,OAAO,EAAE;AACxD;;;;"}

package/dist/esm/src/mediapipe.js

@@ -0,0 +1,16 @@
+import { packageName, version } from './version.js';
+
+let lastModelFilePath = '';
+let modelFileCache;
+const loadMediaPipe = async (options = {}) => {
+    const { basePath = `https://unpkg.com/${packageName}@${version}/mediapipe`, modelPath = `${basePath}/models/selfie_segmenter_landscape.tflite`, } = options;
+    const model = modelPath === lastModelFilePath && modelFileCache
+        ? modelFileCache
+        : await fetch(modelPath).then((r) => r.arrayBuffer());
+    modelFileCache = model;
+    lastModelFilePath = modelPath;
+    return model;
+};
+
+export { loadMediaPipe };
+//# sourceMappingURL=mediapipe.js.map

package/dist/esm/src/mediapipe.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"mediapipe.js","sources":["../../../../../src/mediapipe.ts"],"sourcesContent":["import { packageName, version } from './version';\n\nlet lastModelFilePath = '';\nlet modelFileCache: ArrayBuffer | undefined;\nexport const loadMediaPipe = async (\n  options: {\n    basePath?: string;\n    modelPath?: string;\n  } = {},\n): Promise<ArrayBuffer> => {\n  const {\n    basePath = `https://unpkg.com/${packageName}@${version}/mediapipe`,\n    modelPath = `${basePath}/models/selfie_segmenter_landscape.tflite`,\n  } = options;\n\n  const model =\n    modelPath === lastModelFilePath && modelFileCache\n      ? modelFileCache\n      : await fetch(modelPath).then((r) => r.arrayBuffer());\n\n  modelFileCache = model;\n  lastModelFilePath = modelPath;\n\n  return model;\n};\n"],"names":[],"mappings":";;AAEA,IAAI,iBAAiB,GAAG,EAAE;AAC1B,IAAI,cAAuC;MAC9B,aAAa,GAAG,OAC3B,OAAA,GAGI,EAAE,KACkB;AACxB,IAAA,MAAM,EACJ,QAAQ,GAAG,CAAA,kBAAA,EAAqB,WAAW,IAAI,OAAO,CAAA,UAAA,CAAY,EAClE,SAAS,GAAG,CAAA,EAAG,QAAQ,2CAA2C,GACnE,GAAG,OAAO;AAEX,IAAA,MAAM,KAAK,GACT,SAAS,KAAK,iBAAiB,IAAI;AACjC,UAAE;AACF,UAAE,MAAM,KAAK,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC;IAEzD,cAAc,GAAG,KAAK;IACtB,iBAAiB,GAAG,SAAS;AAE7B,IAAA,OAAO,KAAK;AACd;;;;"}

package/dist/esm/src/types.js

@@ -0,0 +1,11 @@
+/**
+ * Maps blur level to blur strength values.
+ */
+const BACKGROUND_BLUR_MAP = {
+    low: 3,
+    medium: 5,
+    high: 7,
+};
+
+export { BACKGROUND_BLUR_MAP };
+//# sourceMappingURL=types.js.map

package/dist/esm/src/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sources":["../../../../../src/types.ts"],"sourcesContent":["export type BackgroundSource = {\n  type: string;\n  media?: ImageBitmap | ReadableStream;\n  url: string;\n  video?: HTMLVideoElement;\n  track?: MediaStreamTrack;\n};\nexport type BackgroundFilter = 'blur' | 'image';\nexport type BackgroundBlurLevel = 'low' | 'medium' | 'high' | number;\n\n/**\n * Options for controlling the segmentation mask smoothing.\n * These values are passed to the WebGL shader that blends\n * consecutive segmentation masks over time.\n */\nexport interface SegmentationOptions {\n  /**\n   * Controls how fast the mask adapts to new segmentation results.\n   * Higher values make the mask react faster but may cause flickering.\n   * Lower values produce smoother transitions but increase latency.\n   * Value should be between 0 and 1.\n   * @default 0.8\n   */\n  smoothingFactor?: number;\n\n  /**\n   * Lower edge of the smoothstep function applied to the confidence mask.\n   * Confidence values below this are mapped to 0 (background).\n   * Value should be between 0 and 1, and less than smoothstepMax.\n   * @default 0.6\n   */\n  smoothstepMin?: number;\n\n  /**\n   * Upper edge of the smoothstep function applied to the confidence mask.\n   * Confidence values above this are mapped to 1 (foreground).\n   * Value should be between 0 and 1, and greater than smoothstepMin.\n   * @default 0.9\n   */\n  smoothstepMax?: number;\n}\n\nexport interface SegmenterOptions {\n  backgroundSource?: BackgroundSource | null;\n  bgBlur: number;\n  bgBlurRadius: number;\n  isSelfieMode: boolean;\n  segmentationOptions?: SegmentationOptions;\n}\n/**\n * Static configuration for the processor, defining which background\n * effect should be applied and how it should behave.\n */\nexport interface BackgroundOptions {\n  basePath?: string;\n  modelPath?: string;\n  backgroundFilter?: BackgroundFilter;\n  backgroundBlurLevel?: BackgroundBlurLevel;\n  backgroundImage?: string | undefined;\n  segmentationOptions?: SegmentationOptions;\n}\n\n/**\n * Performance statistics for video processing.\n */\nexport interface PerformanceStats {\n  delay: number;\n  fps: number;\n  timestamp: number;\n}\n\n/**\n * Runtime hooks for handling lifecycle or error events.\n */\nexport interface VideoTrackProcessorHooks {\n  onError?: (error: unknown) => void;\n  onStats?: (stats: PerformanceStats) => void;\n}\n\n/**\n * Maps blur level to blur strength values.\n */\nexport const BACKGROUND_BLUR_MAP: Record<'low' | 'medium' | 'high', number> = {\n  low: 3,\n  medium: 5,\n  high: 7,\n};\n"],"names":[],"mappings":"AA+EA;;AAEG;AACI,MAAM,mBAAmB,GAA8C;AAC5E,IAAA,GAAG,EAAE,CAAC;AACN,IAAA,MAAM,EAAE,CAAC;AACT,IAAA,IAAI,EAAE,CAAC;;;;;"}

package/dist/esm/src/version.js

@@ -0,0 +1,5 @@
+const version = "0.8.7";
+const packageName = "@stream-io/video-filters-web";
+
+export { packageName, version };
+//# sourceMappingURL=version.js.map

package/dist/esm/src/version.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"version.js","sources":["../../../../../src/version.ts"],"sourcesContent":["export const version = process.env.PKG_VERSION || '0.0.0-development';\nexport const packageName =\n  process.env.PKG_NAME || '@stream-io/video-filters-web';\n"],"names":[],"mappings":"AAAO,MAAM,OAAO,GAAG;AAChB,MAAM,WAAW,GACtB;;;;"}

package/dist/index.cjs.js

@@ -1561,7 +1561,7 @@ const createTFLiteSIMDModule = (__Module) => {
   return __Module.ready;
 };
 
-const version = "0.8.6";
+const version = "0.8.7";
 const packageName = "@stream-io/video-filters-web";
 
 // @ts-expect-error - module is not declared

package/package.json

@@ -1,14 +1,14 @@
 {
   "name": "@stream-io/video-filters-web",
-  "version": "0.8.6",
+  "version": "0.8.7",
   "main": "./dist/index.cjs.js",
-  "module": "./dist/index.es.js",
+  "module": "./dist/esm/index.js",
   "types": "./dist/index.d.ts",
   "license": "See license in LICENSE",
   "scripts": {
     "clean": "rimraf dist",
     "start": "rollup -c -w",
-    "build": "NODE_ENV=production rollup -c"
+    "build": "yarn clean && NODE_ENV=production rollup -c"
   },
   "repository": {
     "type": "git",