@pexip/media-processor 20.3.5 → 21.0.0

package/dist/common/backends/webgl/binaryMask.d.ts

@@ -0,0 +1,8 @@
+import type { TextureInfo } from './types';
+/**
+ * Create a binary mask renderer based on the provided threshold
+ */
+export declare const createBinaryMaskRenderer: (processor: OffscreenCanvas, width: number, height: number) => {
+    render: (mask: TextureInfo, threshold: number) => TextureInfo;
+    release: () => void;
+};

package/dist/common/backends/webgl/binaryMask.js

@@ -0,0 +1,95 @@
+import { assert } from '@pexip/utils';
+import { m3 } from '../../m3';
+import { RENDERING_EVENTS } from '../../constants';
+import { createGLProgram, createMaskTexture, createFrameBuffer, } from './webglUtils';
+import vertexSrc from './shaders/matrix.vert';
+import fragmentSrc from './shaders/binaryMask.frag';
+/**
+ * Create a binary mask renderer based on the provided threshold
+ */
+export const createBinaryMaskRenderer = (processor, width, height) => {
+    const { gl, program, attribLocations, uniformLocations, floatFullChannelSupported, floatSingleChannelSupported, release, } = createGLProgram(processor, {
+        attribs: {
+            position: 'a_position',
+            texCoord: 'a_texCoord',
+        },
+        uniforms: {
+            matrix: 'u_matrix',
+            mask: 'u_mask',
+            size: 'u_size',
+            threshold: 'u_threshold',
+        },
+        vertexSrc,
+        fragmentSrc,
+    });
+    const vao = gl.createVertexArray();
+    gl.bindVertexArray(vao);
+    // Position
+    const positionBuffer = gl.createBuffer();
+    gl.enableVertexAttribArray(attribLocations.position);
+    gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
+    gl.vertexAttribPointer(attribLocations.position, 2, gl.FLOAT, false, 0, 0);
+    gl.bufferData(gl.ARRAY_BUFFER, 
+    // prettier-ignore
+    new Float32Array([
+        0, 0,
+        0, 1,
+        1, 0,
+        1, 0,
+        0, 1,
+        1, 1,
+    ]), gl.STATIC_DRAW);
+    // Texture coordinates for the rectangle
+    const texCoordBuffer = gl.createBuffer();
+    gl.bindBuffer(gl.ARRAY_BUFFER, texCoordBuffer);
+    gl.bufferData(gl.ARRAY_BUFFER, 
+    // prettier-ignore
+    new Float32Array([
+        0, 0,
+        0, 1,
+        1, 0,
+        1, 0,
+        0, 1,
+        1, 1,
+    ]), gl.STATIC_DRAW);
+    gl.enableVertexAttribArray(attribLocations.texCoord);
+    gl.vertexAttribPointer(attribLocations.texCoord, 2, gl.FLOAT, false, 0, 0);
+    const outputTexture = createMaskTexture(gl, width, height, false, floatSingleChannelSupported, floatFullChannelSupported);
+    const frameBuffer = createFrameBuffer(gl, outputTexture);
+    return {
+        render: (mask, threshold) => {
+            assert(!gl.isContextLost(), RENDERING_EVENTS.ContextLost);
+            gl.clearColor(0, 0, 0, 0);
+            gl.clear(gl.COLOR_BUFFER_BIT);
+            gl.useProgram(program);
+            gl.bindVertexArray(vao);
+            gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer);
+            gl.viewport(0, 0, width, height);
+            // Textures
+            gl.activeTexture(gl.TEXTURE0);
+            gl.uniform1i(uniformLocations.mask, 0);
+            gl.bindTexture(gl.TEXTURE_2D, mask.texture);
+            gl.uniform2i(uniformLocations.size, mask.width, mask.height);
+            gl.uniform1f(uniformLocations.threshold, threshold);
+            // No Flip Y Coordinate for drawing on the frame buffer
+            // Project, position and scale to full-size of the canvas
+            const matrix = m3.projection(1, 1, 1);
+            gl.uniformMatrix3fv(uniformLocations.matrix, false, matrix);
+            gl.drawArrays(gl.TRIANGLES, 0, 6);
+            // Cleanup
+            gl.bindBuffer(gl.ARRAY_BUFFER, null);
+            gl.bindFramebuffer(gl.FRAMEBUFFER, null);
+            gl.bindTexture(gl.TEXTURE_2D, null);
+            gl.bindVertexArray(null);
+            return { texture: outputTexture, width, height };
+        },
+        release: () => {
+            gl.deleteBuffer(positionBuffer);
+            gl.deleteBuffer(texCoordBuffer);
+            gl.deleteTexture(outputTexture);
+            gl.deleteFramebuffer(frameBuffer);
+            gl.deleteVertexArray(vao);
+            release();
+        },
+    };
+};

package/dist/common/backends/webgl/blender.d.ts

@@ -1,3 +1,4 @@
+import type { RendererOptions } from '../../types/render';
 import type { TextureInfo } from './types';
 /**
  * Blend foreground and background according to the mask
@@ -7,6 +8,6 @@ export declare const createBlendRenderer: (processor: OffscreenCanvas) => {
         mask: TextureInfo;
         foreground: TextureInfo;
         background: TextureInfo;
-    }) => TextureInfo;
+    }, options: RendererOptions) => TextureInfo;
     release: () => void;
 };

package/dist/common/backends/webgl/blender.js

@@ -1,26 +1,27 @@
 import { assert } from '@pexip/utils';
 import { RENDERING_EVENTS } from '../../constants';
 import { m3 } from '../../m3';
-import { createGLProgram, createAndSetupTexture } from './webglUtils';
-import { vertexShaderSource } from './shaders/vertexShader';
-import { blendFragmentShaderSource } from './shaders/blendFragmentShader';
+import { createFrameBuffer, createGLProgram, createRGBAFrameTexture, } from './webglUtils';
+import vertexSrc from './shaders/matrix.vert';
+import fragmentSrc from './shaders/blend.frag';
 /**
  * Blend foreground and background according to the mask
  */
 export const createBlendRenderer = (processor) => {
-    const { gl, program, attribLocations, uniformLocations, release } = createGLProgram(processor, {
+    const { gl, program, attribLocations, floatFullChannelSupported, uniformLocations, release, } = createGLProgram(processor, {
         attribs: {
             position: 'a_position',
             texCoord: 'a_texCoord',
         },
         uniforms: {
             matrix: 'u_matrix',
+            mask: 'u_mask',
             frameBackground: 'u_frame_background',
             frameForeground: 'u_frame_foreground',
-            mask: 'u_mask',
+            confidenceThresholds: 'u_confidence_thresholds',
         },
-        vertexSrc: vertexShaderSource,
-        fragmentSrc: blendFragmentShaderSource,
+        vertexSrc,
+        fragmentSrc,
     });
     gl.enable(gl.BLEND);
     const vao = gl.createVertexArray();
@@ -55,18 +56,16 @@ export const createBlendRenderer = (processor) => {
     ]), gl.STATIC_DRAW);
     gl.enableVertexAttribArray(attribLocations.texCoord);
     gl.vertexAttribPointer(attribLocations.texCoord, 2, gl.FLOAT, false, 0, 0);
-    const outputTexture = createAndSetupTexture(gl);
-    gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, processor.width, processor.height, 0, gl.RGBA, gl.UNSIGNED_BYTE, null);
-    const frameBuffer = gl.createFramebuffer();
-    gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer);
-    gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, outputTexture, 0);
+    const outputTexture = createRGBAFrameTexture(gl, processor.width, processor.height, true, floatFullChannelSupported);
+    const frameBuffer = createFrameBuffer(gl, outputTexture);
     return {
-        render: ({ mask, foreground, background, }) => {
+        render: ({ mask, foreground, background, }, options) => {
             assert(!gl.isContextLost(), RENDERING_EVENTS.ContextLost);
+            gl.useProgram(program);
             gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer);
             gl.viewport(0, 0, processor.width, processor.height);
-            gl.useProgram(program);
             gl.bindVertexArray(vao);
+            gl.uniform2f(uniformLocations.confidenceThresholds, options.backgroundThreshold, options.foregroundThreshold);
             // Textures
             gl.activeTexture(gl.TEXTURE0);
             gl.uniform1i(uniformLocations.frameForeground, 0);
@@ -82,7 +81,11 @@ export const createBlendRenderer = (processor) => {
             const matrix = m3.projection(1, 1, 1);
             gl.uniformMatrix3fv(uniformLocations.matrix, false, matrix);
             gl.drawArrays(gl.TRIANGLES, 0, 6);
-            assert(outputTexture);
+            // Cleanup
+            gl.bindBuffer(gl.ARRAY_BUFFER, null);
+            gl.bindFramebuffer(gl.FRAMEBUFFER, null);
+            gl.bindTexture(gl.TEXTURE_2D, null);
+            gl.bindVertexArray(null);
             return {
                 texture: outputTexture,
                 width: processor.width,

package/dist/common/backends/webgl/blur.d.ts

@@ -2,7 +2,7 @@ import type { TextureInfo } from './types';
 /**
  * Blur provided frame
  */
-export declare const createBlurRenderer: (processor: OffscreenCanvas, pass: number) => {
+export declare const createBlurRenderer: (processor: OffscreenCanvas, pass: number, redOnly?: boolean) => {
     render: (frame: TextureInfo, pass: number) => TextureInfo;
     release: () => void;
 };

package/dist/common/backends/webgl/blur.js

@@ -2,15 +2,15 @@ import { assert } from '@pexip/utils';
 import { clamping } from '../../utils';
 import { RENDERING_EVENTS } from '../../constants';
 import { m3 } from '../../m3';
-import { createAndSetupTexture, createGLProgram } from './webglUtils';
-import { vertexShaderSource } from './shaders/vertexShader';
-import { dualFilterBlurFragmentShaderSource } from './shaders/dualFilterBlurFragmentShader';
+import { createFrameBuffer, createGLProgram, createMaskTexture, createRGBAFrameTexture, } from './webglUtils';
+import vertexSrc from './shaders/matrix.vert';
+import fragmentSrc from './shaders/dualFilterBlur.frag';
 const clampBlurAmount = clamping(1, 9);
 /**
  * Blur provided frame
  */
-export const createBlurRenderer = (processor, pass) => {
-    const { gl, program, attribLocations, uniformLocations, release } = createGLProgram(processor, {
+export const createBlurRenderer = (processor, pass, redOnly = false) => {
+    const { gl, program, attribLocations, uniformLocations, floatSingleChannelSupported, floatFullChannelSupported, release, } = createGLProgram(processor, {
         attribs: {
             position: 'a_position',
             texCoord: 'a_texCoord',
@@ -20,8 +20,8 @@ export const createBlurRenderer = (processor, pass) => {
             downsampling: 'u_downsampling',
             frame: 'u_frame',
         },
-        vertexSrc: vertexShaderSource,
-        fragmentSrc: dualFilterBlurFragmentShaderSource,
+        vertexSrc,
+        fragmentSrc,
     });
     const vao = gl.createVertexArray();
     gl.bindVertexArray(vao);
@@ -61,18 +61,13 @@ export const createBlurRenderer = (processor, pass) => {
         gl.viewport(0, 0, width, height);
     };
     const createFramebufferAndTexture = (idx) => {
-        const texture = createAndSetupTexture(gl);
-        assert(texture);
         // Avoid fractional value
         const width = Math.trunc(processor.width / 2 ** idx);
         const height = Math.trunc(processor.height / 2 ** idx);
-        gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, width, height, 0, gl.RGBA, gl.UNSIGNED_BYTE, null);
-        const frameBuffer = gl.createFramebuffer();
-        assert(frameBuffer);
-        gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer);
-        // Attach a texture to it.
-        const attachmentPoint = gl.COLOR_ATTACHMENT0;
-        gl.framebufferTexture2D(gl.FRAMEBUFFER, attachmentPoint, gl.TEXTURE_2D, texture, 0);
+        const texture = redOnly
+            ? createMaskTexture(gl, width, height, true, floatSingleChannelSupported, floatFullChannelSupported)
+            : createRGBAFrameTexture(gl, width, height, true, floatFullChannelSupported);
+        const frameBuffer = createFrameBuffer(gl, texture);
         return { texture, frameBuffer, width, height };
     };
     const generateBlurBuffers = (pass, offset = 0) => {
@@ -87,6 +82,13 @@ export const createBlurRenderer = (processor, pass) => {
     return {
         render: (frame, pass) => {
             assert(!gl.isContextLost(), RENDERING_EVENTS.ContextLost);
+            if (!pass) {
+                return {
+                    texture: frame.texture,
+                    width: frame.width,
+                    height: frame.height,
+                };
+            }
             gl.useProgram(program);
             gl.bindVertexArray(vao);
             // Frame texture
@@ -153,6 +155,11 @@ export const createBlurRenderer = (processor, pass) => {
                 // for the next draw, use the texture we just rendered to.
                 gl.bindTexture(gl.TEXTURE_2D, texture);
             }
+            // Cleanup
+            gl.bindBuffer(gl.ARRAY_BUFFER, null);
+            gl.bindFramebuffer(gl.FRAMEBUFFER, null);
+            gl.bindTexture(gl.TEXTURE_2D, null);
+            gl.bindVertexArray(null);
             return {
                 texture: blurBuffers[0]?.texture ?? frame.texture,
                 width: blurBuffers[0]?.width ?? frame.width,

package/dist/common/backends/webgl/canvas.js

@@ -2,8 +2,8 @@ import { assert } from '@pexip/utils';
 import { m3 } from '../../m3';
 import { RENDERING_EVENTS } from '../../constants';
 import { createGLProgram } from './webglUtils';
-import { vertexShaderSource } from './shaders/vertexShader';
-import { textureFragmentShaderSource } from './shaders/textureFragmentShader';
+import vertexSrc from './shaders/matrix.vert';
+import fragmentSrc from './shaders/texture.frag';
 /**
  * Render texture to the processing canvas framebuffer
  */
@@ -17,8 +17,8 @@ export const createCanvasRenderer = (processor) => {
             matrix: 'u_matrix',
             texture: 'u_texture',
         },
-        vertexSrc: vertexShaderSource,
-        fragmentSrc: textureFragmentShaderSource,
+        vertexSrc,
+        fragmentSrc,
     });
     const vao = gl.createVertexArray();
     gl.bindVertexArray(vao);
@@ -72,6 +72,11 @@ export const createCanvasRenderer = (processor) => {
             gl.activeTexture(gl.TEXTURE0);
             gl.bindTexture(gl.TEXTURE_2D, texture.texture);
             gl.drawArrays(gl.TRIANGLES, 0, 6);
+            // Cleanup
+            gl.bindBuffer(gl.ARRAY_BUFFER, null);
+            gl.bindFramebuffer(gl.FRAMEBUFFER, null);
+            gl.bindTexture(gl.TEXTURE_2D, null);
+            gl.bindVertexArray(null);
         },
         release: () => {
             gl.deleteBuffer(positionBuffer);

package/dist/common/backends/webgl/diffAbsMask.d.ts

@@ -0,0 +1,10 @@
+import type { TextureInfo } from './types';
+/**
+ * Create absulute difference renderer to caculate the difference between 2 textures
+ */
+export declare const createDiffAbsRenderer: (processor: OffscreenCanvas, width: number, height: number) => {
+    render: (textureA: TextureInfo, textureB: TextureInfo) => TextureInfo & {
+        frameBuffer: WebGLFramebuffer;
+    };
+    release: () => void;
+};

package/dist/common/backends/webgl/diffAbsMask.js

@@ -0,0 +1,103 @@
+import { assert } from '@pexip/utils';
+import { m3 } from '../../m3';
+import { RENDERING_EVENTS } from '../../constants';
+import { createGLProgram, createMaskTexture, createFrameBuffer, } from './webglUtils';
+import vertexSrc from './shaders/matrix.vert';
+import fragmentSrc from './shaders/diffAbs.frag';
+/**
+ * Create absulute difference renderer to caculate the difference between 2 textures
+ */
+export const createDiffAbsRenderer = (processor, width, height) => {
+    const { gl, program, attribLocations, uniformLocations, floatSingleChannelSupported, floatFullChannelSupported, release, } = createGLProgram(processor, {
+        attribs: {
+            position: 'a_position',
+            texCoord: 'a_texCoord',
+        },
+        uniforms: {
+            matrix: 'u_matrix',
+            texA: 'u_tex_a',
+            texB: 'u_tex_b',
+            size: 'u_size',
+        },
+        vertexSrc,
+        fragmentSrc,
+    });
+    const vao = gl.createVertexArray();
+    gl.bindVertexArray(vao);
+    // Position
+    const positionBuffer = gl.createBuffer();
+    gl.enableVertexAttribArray(attribLocations.position);
+    gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
+    gl.vertexAttribPointer(attribLocations.position, 2, gl.FLOAT, false, 0, 0);
+    gl.bufferData(gl.ARRAY_BUFFER, 
+    // prettier-ignore
+    new Float32Array([
+        0, 0,
+        0, 1,
+        1, 0,
+        1, 0,
+        0, 1,
+        1, 1,
+    ]), gl.STATIC_DRAW);
+    // Texture coordinates for the rectangle
+    const texCoordBuffer = gl.createBuffer();
+    gl.bindBuffer(gl.ARRAY_BUFFER, texCoordBuffer);
+    gl.bufferData(gl.ARRAY_BUFFER, 
+    // prettier-ignore
+    new Float32Array([
+        0, 0,
+        0, 1,
+        1, 0,
+        1, 0,
+        0, 1,
+        1, 1,
+    ]), gl.STATIC_DRAW);
+    gl.enableVertexAttribArray(attribLocations.texCoord);
+    gl.vertexAttribPointer(attribLocations.texCoord, 2, gl.FLOAT, false, 0, 0);
+    const outputTexturePing = createMaskTexture(gl, width, height, true, floatSingleChannelSupported, floatFullChannelSupported);
+    const frameBufferPing = createFrameBuffer(gl, outputTexturePing);
+    return {
+        render: (textureA, textureB) => {
+            assert(!gl.isContextLost(), RENDERING_EVENTS.ContextLost);
+            assert(textureA.width === textureB.width &&
+                textureA.height === textureB.height);
+            gl.useProgram(program);
+            gl.bindVertexArray(vao);
+            gl.uniform2i(uniformLocations.size, textureA.width, textureA.height);
+            gl.activeTexture(gl.TEXTURE0);
+            gl.uniform1i(uniformLocations.texA, 0);
+            gl.bindTexture(gl.TEXTURE_2D, textureA.texture);
+            gl.activeTexture(gl.TEXTURE1);
+            gl.uniform1i(uniformLocations.texB, 1);
+            gl.bindTexture(gl.TEXTURE_2D, textureB.texture);
+            // No Flip Y Coordinate for drawing on the frame buffer
+            // Project, position and scale to full-size of the canvas
+            const matrix = m3.projection(1, 1, 1);
+            gl.uniformMatrix3fv(uniformLocations.matrix, false, matrix);
+            gl.bindFramebuffer(gl.FRAMEBUFFER, frameBufferPing);
+            gl.viewport(0, 0, width, height);
+            gl.clearColor(0, 0, 0, 0);
+            gl.clear(gl.COLOR_BUFFER_BIT);
+            gl.drawArrays(gl.TRIANGLES, 0, 6);
+            // Cleanup
+            gl.bindBuffer(gl.ARRAY_BUFFER, null);
+            gl.bindFramebuffer(gl.FRAMEBUFFER, null);
+            gl.bindTexture(gl.TEXTURE_2D, null);
+            gl.bindVertexArray(null);
+            return {
+                texture: outputTexturePing,
+                frameBuffer: frameBufferPing,
+                width,
+                height,
+            };
+        },
+        release: () => {
+            gl.deleteBuffer(positionBuffer);
+            gl.deleteBuffer(texCoordBuffer);
+            gl.deleteTexture(outputTexturePing);
+            gl.deleteFramebuffer(frameBufferPing);
+            gl.deleteVertexArray(vao);
+            release();
+        },
+    };
+};

package/dist/common/backends/webgl/geodesicReconstruction.d.ts

@@ -0,0 +1,13 @@
+import type { RendererOptions } from '../../types/render';
+import type { TextureInfo } from './types';
+/**
+ * A Geodesic Reconstruction Renderer that performs geodesic dilation of
+ * a mask constrained by a reference mask.
+ */
+export declare const createGeodesicReconstructionRenderer: (processor: OffscreenCanvas, width: number, height: number) => {
+    render: ({ mask, maskRef }: {
+        mask: TextureInfo;
+        maskRef: TextureInfo;
+    }, options: Pick<RendererOptions, "morphDilateRadiusPx" | "morphPass">) => TextureInfo;
+    release: () => void;
+};

package/dist/common/backends/webgl/geodesicReconstruction.js

@@ -0,0 +1,176 @@
+import { assert } from '@pexip/utils';
+import { m3 } from '../../m3';
+import { RENDERING_EVENTS } from '../../constants';
+import { clamping } from '../../utils';
+import { createGLProgram, createMaskTexture, createFrameBuffer, } from './webglUtils';
+import vertexSrc from './shaders/matrix.vert';
+import fragmentSrc from './shaders/dilateClipToMask.frag';
+import { createDiffAbsRenderer } from './diffAbsMask';
+// Convergence threshold used when testing for geodesic reconstruction stability.
+const CONVERGENCE_MAX_ITERATIONS = 64;
+const MAX_RADIUS_PX = 64;
+const clampRadiusPx = clamping(0, MAX_RADIUS_PX);
+/**
+ * A Geodesic Reconstruction Renderer that performs geodesic dilation of
+ * a mask constrained by a reference mask.
+ */
+export const createGeodesicReconstructionRenderer = (processor, width, height) => {
+    const { gl, program, attribLocations, uniformLocations, floatSingleChannelSupported, floatFullChannelSupported, release, } = createGLProgram(processor, {
+        attribs: {
+            position: 'a_position',
+            texCoord: 'a_texCoord',
+        },
+        uniforms: {
+            matrix: 'u_matrix',
+            mask: 'u_mask',
+            maskRef: 'u_ref_mask',
+            texelSize: 'u_texel_size',
+            radiusPx: 'u_radius_px',
+            direction: 'u_direction',
+        },
+        vertexSrc,
+        fragmentSrc,
+    });
+    const vao = gl.createVertexArray();
+    gl.bindVertexArray(vao);
+    // Position
+    const positionBuffer = gl.createBuffer();
+    gl.enableVertexAttribArray(attribLocations.position);
+    gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
+    gl.vertexAttribPointer(attribLocations.position, 2, gl.FLOAT, false, 0, 0);
+    gl.bufferData(gl.ARRAY_BUFFER, 
+    // prettier-ignore
+    new Float32Array([
+        0, 0,
+        0, 1,
+        1, 0,
+        1, 0,
+        0, 1,
+        1, 1,
+    ]), gl.STATIC_DRAW);
+    // Texture coordinates for the rectangle
+    const texCoordBuffer = gl.createBuffer();
+    gl.bindBuffer(gl.ARRAY_BUFFER, texCoordBuffer);
+    gl.bufferData(gl.ARRAY_BUFFER, 
+    // prettier-ignore
+    new Float32Array([
+        0, 0,
+        0, 1,
+        1, 0,
+        1, 0,
+        0, 1,
+        1, 1,
+    ]), gl.STATIC_DRAW);
+    gl.enableVertexAttribArray(attribLocations.texCoord);
+    gl.vertexAttribPointer(attribLocations.texCoord, 2, gl.FLOAT, false, 0, 0);
+    const scaleFactor = processor.width / width;
+    const MIN_CCL_WIDTH = 32;
+    const MIN_CCL_HEIGHT = 18;
+    const convergenceWidth = Math.max(Math.floor(width / 4), MIN_CCL_WIDTH);
+    const convergenceHeight = Math.max(Math.floor(height / 4), MIN_CCL_HEIGHT);
+    const CONVERGENCE_EPSILON = convergenceWidth === MIN_CCL_WIDTH ? 1e-8 : 1e-4;
+    const convergenceSize = convergenceWidth * convergenceHeight;
+    const diffRenderer = createDiffAbsRenderer(processor, convergenceWidth, convergenceHeight);
+    const supportFloat = floatSingleChannelSupported || floatFullChannelSupported;
+    const readPixelsFormat = supportFloat ? gl.RGBA : gl.RED;
+    const readPixelsType = supportFloat ? gl.FLOAT : gl.UNSIGNED_BYTE;
+    const convergenceDiff = supportFloat
+        ? new Float32Array(convergenceSize * 4)
+        : new Uint8Array(convergenceSize);
+    const outputTexturePing = createMaskTexture(gl, width, height, false, floatSingleChannelSupported, floatFullChannelSupported);
+    const outputTexturePong = createMaskTexture(gl, width, height, false, floatSingleChannelSupported, floatFullChannelSupported);
+    const frameBufferPing = createFrameBuffer(gl, outputTexturePing);
+    const frameBufferPong = createFrameBuffer(gl, outputTexturePong);
+    return {
+        render: ({ mask, maskRef }, options) => {
+            assert(!gl.isContextLost(), RENDERING_EVENTS.ContextLost);
+            let write = frameBufferPing;
+            let read = mask.texture;
+            const dilate = () => {
+                gl.useProgram(program);
+                gl.bindVertexArray(vao);
+                gl.uniform1i(uniformLocations.radiusPx, clampRadiusPx(Math.floor(options.morphDilateRadiusPx / scaleFactor)));
+                gl.uniform2f(uniformLocations.texelSize, 1 / mask.width, 1 / mask.height);
+                // Reference Mask
+                gl.activeTexture(gl.TEXTURE1);
+                gl.uniform1i(uniformLocations.maskRef, 1);
+                gl.bindTexture(gl.TEXTURE_2D, maskRef.texture);
+                // No Flip Y Coordinate for drawing on the frame buffer
+                // Project, position and scale to full-size of the canvas
+                const matrix = m3.projection(1, 1, 1);
+                gl.uniformMatrix3fv(uniformLocations.matrix, false, matrix);
+                // Separable
+                for (let j = 0; j < 2; j++) {
+                    // Textures
+                    gl.activeTexture(gl.TEXTURE0);
+                    gl.uniform1i(uniformLocations.mask, 0);
+                    gl.bindTexture(gl.TEXTURE_2D, read);
+                    // Horizontal then Vertical
+                    gl.uniform2f(uniformLocations.direction, 1.0 - j, 0.0 + j);
+                    gl.bindFramebuffer(gl.FRAMEBUFFER, write);
+                    gl.viewport(0, 0, width, height);
+                    gl.drawArrays(gl.TRIANGLES, 0, 6);
+                    // Swap
+                    [read, write] =
+                        write === frameBufferPing
+                            ? [outputTexturePing, frameBufferPong]
+                            : [outputTexturePong, frameBufferPing];
+                }
+            };
+            for (let i = 0; i < options.morphPass; i++) {
+                dilate();
+            }
+            let convergent = false;
+            let count = 0;
+            while (!convergent) {
+                dilate();
+                const diff = diffRenderer.render({ texture: outputTexturePing, width, height }, { texture: outputTexturePong, width, height });
+                gl.bindFramebuffer(gl.FRAMEBUFFER, diff.frameBuffer);
+                gl.readPixels(0, 0, convergenceWidth, convergenceHeight, readPixelsFormat, readPixelsType, convergenceDiff);
+                let sum = 0;
+                let meanDiff = 0;
+                if (supportFloat) {
+                    for (let i = 0, j = 0; i < convergenceSize; ++i, j += 4) {
+                        //@ts-expect-error will not be undefined
+                        sum += convergenceDiff[j];
+                    }
+                    meanDiff = sum / convergenceSize;
+                }
+                else {
+                    for (let i = 0; i < convergenceSize; ++i) {
+                        //@ts-expect-error will not be undefined
+                        sum += convergenceDiff[i];
+                    }
+                    meanDiff = sum / convergenceSize / 255.0;
+                }
+                gl.bindFramebuffer(gl.FRAMEBUFFER, null);
+                if (meanDiff < CONVERGENCE_EPSILON ||
+                    count >= CONVERGENCE_MAX_ITERATIONS) {
+                    // console.log(
+                    //     `Geodesic reconstruction converged in ${count} iterations with ${meanDiff}`,
+                    //     {convergenceWidth, convergenceHeight, width, height},
+                    // );
+                    convergent = true;
+                }
+                count++;
+            }
+            // Cleanup
+            gl.bindBuffer(gl.ARRAY_BUFFER, null);
+            gl.bindFramebuffer(gl.FRAMEBUFFER, null);
+            gl.bindTexture(gl.TEXTURE_2D, null);
+            gl.bindVertexArray(null);
+            return { texture: read, width, height };
+        },
+        release: () => {
+            gl.deleteBuffer(positionBuffer);
+            gl.deleteBuffer(texCoordBuffer);
+            gl.deleteTexture(outputTexturePing);
+            gl.deleteTexture(outputTexturePong);
+            gl.deleteFramebuffer(frameBufferPing);
+            gl.deleteFramebuffer(frameBufferPong);
+            gl.deleteVertexArray(vao);
+            diffRenderer.release();
+            release();
+        },
+    };
+};

package/dist/common/backends/webgl/jointBilateralFilter.d.ts

@@ -0,0 +1,12 @@
+import type { RendererOptions } from '../../types/render';
+import type { TextureInfo } from './types';
+/**
+ * Joint Bilateral Filter
+ */
+export declare const createJointBilateralFilterRenderer: (processor: OffscreenCanvas, width: number, height: number) => {
+    render: ({ input, guide }: {
+        input: TextureInfo;
+        guide: TextureInfo;
+    }, options: RendererOptions) => TextureInfo;
+    release: () => void;
+};