@pexip/media-processor 20.3.5 → 22.0.0

package/dist/common/backends/webgl/renderer.js

@@ -1,23 +1,59 @@
 import { assert } from '@pexip/utils';
-import { PROCESSING_WIDTH, PROCESSING_HEIGHT, EDGE_BLUR_AMOUNT, BACKGROUND_BLUR_AMOUNT, RENDERING_EVENTS, } from '../../constants';
-import { createTextureRenderer } from './texture';
-import { createBlurRenderer } from './blur';
+import { BACKGROUND_BLUR_AMOUNT, PROCESSING_HEIGHT, PROCESSING_WIDTH, RENDERING_EVENTS, STRONG_EDGE_BLUR_AMOUNT, } from '../../constants';
+import { createLazyProps, handleWebGLContextLoss } from '../../utils';
+import { createBinaryMaskRenderer } from './binaryMask';
 import { createBlendRenderer } from './blender';
+import { createBlurRenderer } from './blur';
 import { createCanvasRenderer } from './canvas';
-import { createTextureToTextureRenderer } from './textureToTexture';
+import { createComponentSelector } from './selectComponent';
+import { createGeodesicReconstructionRenderer } from './geodesicReconstruction';
+import { createJointBilateralFilterRenderer } from './jointBilateralFilter';
+import { createMorphErodeRenderer } from './morphErode';
+import { createReplaceRenderer } from './replace';
 import { createSmoothingMaskRenderer } from './smoothingMask';
-import { createLazyProps, handleWebGLContextLoss } from '../../utils';
+import { createTentBlurRenderer } from './tentBlur';
+import { createTextureRenderer } from './texture';
+import { createTextureToTextureRenderer } from './textureToTexture';
 export const createRenderer = (renderEventHandlers, canvas = new OffscreenCanvas(PROCESSING_WIDTH, PROCESSING_HEIGHT)) => {
+    const halfWidth = Math.trunc(canvas.width / 2);
+    const halfHeight = Math.trunc(canvas.height / 2);
+    const quarterWidth = Math.trunc(canvas.width / 4);
+    const quarterHeight = Math.trunc(canvas.height / 4);
+    // Width and Height used for the Center Component selection
+    const cclWidth = 32;
+    const cclHeight = 18;
     const rendererCreator = {
-        videoFrameRenderer: () => createTextureRenderer(canvas),
-        maskRenderer: () => createTextureRenderer(canvas),
-        canvasRenderer: () => createCanvasRenderer(canvas),
-        backgroundImageRenderer: () => createTextureRenderer(canvas),
+        backgroundImageRenderer: () => createTextureRenderer(canvas, false),
         blendRenderer: () => createBlendRenderer(canvas),
         blurRenderer: () => createBlurRenderer(canvas, BACKGROUND_BLUR_AMOUNT),
-        maskBlurRenderer: () => createBlurRenderer(canvas, EDGE_BLUR_AMOUNT),
-        textureRenderer: () => createTextureToTextureRenderer(canvas),
-        smoothingMaskRenderer: () => createSmoothingMaskRenderer(canvas),
+        canvasRenderer: () => createCanvasRenderer(canvas),
+        frameDownsampler2: () => createTextureToTextureRenderer(canvas, false, true, halfWidth, halfHeight),
+        frameDownsampler4: () => createTextureToTextureRenderer(canvas, false, true, quarterWidth, quarterHeight),
+        geodesicReconstructionRenderer2: () => createGeodesicReconstructionRenderer(canvas, halfWidth, halfHeight),
+        geodesicReconstructionRenderer4: () => createGeodesicReconstructionRenderer(canvas, quarterWidth, quarterHeight),
+        jointBilateralFilterRenderer2: () => createJointBilateralFilterRenderer(canvas, halfWidth, halfHeight),
+        jointBilateralFilterRenderer4: () => createJointBilateralFilterRenderer(canvas, quarterWidth, quarterHeight),
+        jointBilateralFilterRenderer: () => createJointBilateralFilterRenderer(canvas, canvas.width, canvas.height),
+        lightWrapBlurRenderer: () => createBlurRenderer(canvas, BACKGROUND_BLUR_AMOUNT),
+        maskBlurRenderer: () => createTentBlurRenderer(canvas, canvas.width, canvas.height),
+        strongMaskBlurRenderer: () => createBlurRenderer(canvas, STRONG_EDGE_BLUR_AMOUNT),
+        maskDownsampler2: () => createTextureToTextureRenderer(canvas, true, false, halfWidth, halfHeight),
+        maskDownsampler4: () => createTextureToTextureRenderer(canvas, true, false, quarterWidth, quarterHeight),
+        maskRenderer: () => createTextureRenderer(canvas, true),
+        maskUpsampler2: () => createTextureToTextureRenderer(canvas, true, false, halfWidth, halfHeight),
+        maskUpsampler4: () => createTextureToTextureRenderer(canvas, true, false, quarterWidth, quarterHeight),
+        maskUpsampler: () => createTextureToTextureRenderer(canvas, true, false, canvas.width, canvas.height),
+        morphErodeRenderer: () => createMorphErodeRenderer(canvas, halfWidth, halfHeight),
+        morphErodeRenderer4: () => createMorphErodeRenderer(canvas, quarterWidth, quarterHeight),
+        prevMaskRenderer2: () => createTextureToTextureRenderer(canvas, true, false, halfWidth, halfHeight),
+        prevMaskRenderer4: () => createTextureToTextureRenderer(canvas, true, false, quarterWidth, quarterHeight),
+        replaceRenderer: () => createReplaceRenderer(canvas),
+        selectCenterComponent: () => createComponentSelector(canvas, cclWidth, cclHeight),
+        smoothingMaskRenderer4: () => createSmoothingMaskRenderer(canvas, quarterWidth, quarterHeight),
+        smoothingMaskRenderer2: () => createSmoothingMaskRenderer(canvas, halfWidth, halfHeight),
+        binaryMaskRenderer2: () => createBinaryMaskRenderer(canvas, halfWidth, halfHeight),
+        binaryMaskRenderer4: () => createBinaryMaskRenderer(canvas, quarterWidth, quarterHeight),
+        videoFrameRenderer: () => createTextureRenderer(canvas, false),
         release: (states) => () => {
             for (const key in states) {
                 if (key !== 'release') {
@@ -35,6 +71,15 @@ export const createRenderer = (renderEventHandlers, canvas = new OffscreenCanvas
         get videoFrameRenderer() {
             return lazyProps.videoFrameRenderer;
         },
+        get jointBilateralFilterRenderer() {
+            return lazyProps.jointBilateralFilterRenderer;
+        },
+        get jointBilateralFilterRenderer2() {
+            return lazyProps.jointBilateralFilterRenderer2;
+        },
+        get jointBilateralFilterRenderer4() {
+            return lazyProps.jointBilateralFilterRenderer4;
+        },
         get maskRenderer() {
             return lazyProps.maskRenderer;
         },
@@ -47,18 +92,74 @@ export const createRenderer = (renderEventHandlers, canvas = new OffscreenCanvas
         get blendRenderer() {
             return lazyProps.blendRenderer;
         },
+        get replaceRenderer() {
+            return lazyProps.replaceRenderer;
+        },
         get blurRenderer() {
             return lazyProps.blurRenderer;
         },
-        // TODO: Impl edge blur
+        get lightWrapBlurRenderer() {
+            return lazyProps.lightWrapBlurRenderer;
+        },
+        get prevMaskRenderer2() {
+            return lazyProps.prevMaskRenderer2;
+        },
+        get prevMaskRenderer4() {
+            return lazyProps.prevMaskRenderer4;
+        },
+        get selectCenterComponent() {
+            return lazyProps.selectCenterComponent;
+        },
+        get smoothingMaskRenderer2() {
+            return lazyProps.smoothingMaskRenderer2;
+        },
+        get smoothingMaskRenderer4() {
+            return lazyProps.smoothingMaskRenderer4;
+        },
+        get binaryMaskRenderer2() {
+            return lazyProps.binaryMaskRenderer2;
+        },
+        get binaryMaskRenderer4() {
+            return lazyProps.binaryMaskRenderer4;
+        },
+        get morphErodeRenderer() {
+            return lazyProps.morphErodeRenderer;
+        },
+        get morphErodeRenderer4() {
+            return lazyProps.morphErodeRenderer4;
+        },
         get maskBlurRenderer() {
             return lazyProps.maskBlurRenderer;
         },
-        get textureRenderer() {
-            return lazyProps.textureRenderer;
+        get strongMaskBlurRenderer() {
+            return lazyProps.strongMaskBlurRenderer;
+        },
+        get maskDownsampler2() {
+            return lazyProps.maskDownsampler2;
+        },
+        get maskDownsampler4() {
+            return lazyProps.maskDownsampler4;
+        },
+        get maskUpsampler() {
+            return lazyProps.maskUpsampler;
+        },
+        get maskUpsampler2() {
+            return lazyProps.maskUpsampler2;
+        },
+        get maskUpsampler4() {
+            return lazyProps.maskUpsampler4;
+        },
+        get frameDownsampler2() {
+            return lazyProps.frameDownsampler2;
         },
-        get smoothingMaskRenderer() {
-            return lazyProps.smoothingMaskRenderer;
+        get frameDownsampler4() {
+            return lazyProps.frameDownsampler4;
+        },
+        get geodesicReconstructionRenderer2() {
+            return lazyProps.geodesicReconstructionRenderer2;
+        },
+        get geodesicReconstructionRenderer4() {
+            return lazyProps.geodesicReconstructionRenderer4;
         },
     };
     const init = () => {
@@ -75,43 +176,92 @@ export const createRenderer = (renderEventHandlers, canvas = new OffscreenCanvas
     };
     const render = (mask, frame, options) => {
         assert(!props.gl?.isContextLost(), RENDERING_EVENTS.ContextLost);
-        const maskTexture = props.maskRenderer.render({
-            frame: mask,
-            width: mask.width,
-            height: mask.height,
-            timestamp: frame.timestamp,
-        });
-        const smoothedMask = props.smoothingMaskRenderer.render({
-            mask: maskTexture,
-            prevMask: props.prevMask,
-        }, options);
+        // 1. Upload current frame and raw mask
         const frameTexture = props.videoFrameRenderer.render(frame);
-        let background = frameTexture;
+        const maskTexture = props.maskRenderer.render(mask);
+        // 2. Joint Bilateral Filter
+        let smoothedMask;
+        let compositeMaskInput;
+        switch (options.downSampleFactor) {
+            case 4: {
+                const maskLowRes = props.maskDownsampler4.render(maskTexture);
+                const frameLowRes = props.frameDownsampler4.render(frameTexture);
+                const filteredMaskLowRes = props.jointBilateralFilterRenderer4.render({ input: maskLowRes, guide: frameLowRes }, options);
+                smoothedMask = props.smoothingMaskRenderer4.render({
+                    mask: filteredMaskLowRes,
+                    prevMask: props.prevMask4 ?? filteredMaskLowRes,
+                }, options);
+                if (options.excludeBystanders) {
+                    const binary = props.binaryMaskRenderer4.render(smoothedMask, options.foregroundThreshold);
+                    const eroded = props.morphErodeRenderer4.render({ mask: binary }, options);
+                    const selected = props.selectCenterComponent.render(eroded, options);
+                    const filted = props.maskUpsampler4.render(selected);
+                    smoothedMask = props.geodesicReconstructionRenderer4.render({ mask: filted, maskRef: smoothedMask }, options);
+                }
+                // Upsample to original size
+                const result = props.maskUpsampler.render(smoothedMask);
+                compositeMaskInput = props.strongMaskBlurRenderer.render(result, STRONG_EDGE_BLUR_AMOUNT);
+                break;
+            }
+            default: {
+                const maskLowRes = props.maskDownsampler2.render(maskTexture);
+                const frameLowRes = props.frameDownsampler2.render(frameTexture);
+                const filteredMaskLowRes = props.jointBilateralFilterRenderer2.render({ input: maskLowRes, guide: frameLowRes }, options);
+                smoothedMask = props.smoothingMaskRenderer2.render({
+                    mask: filteredMaskLowRes,
+                    prevMask: props.prevMask2 ?? filteredMaskLowRes,
+                }, options);
+                if (options.excludeBystanders) {
+                    const binary = props.binaryMaskRenderer2.render(smoothedMask, options.foregroundThreshold);
+                    const eroded = props.morphErodeRenderer.render({ mask: binary }, options);
+                    const selected = props.selectCenterComponent.render(eroded, options);
+                    const filted = props.maskUpsampler2.render(selected);
+                    smoothedMask = props.geodesicReconstructionRenderer2.render({ mask: filted, maskRef: smoothedMask }, options);
+                }
+                // Upsample to original size
+                const result = props.maskUpsampler.render(smoothedMask);
+                compositeMaskInput = props.maskBlurRenderer.render(result, options.edgeBlurAmount);
+                break;
+            }
+        }
+        // 5. Composite final output
+        let outputFrameTexture = frameTexture;
         if (options.effects === 'blur' && options.backgroundBlurAmount) {
-            background = props.blurRenderer.render(frameTexture, options.backgroundBlurAmount);
+            const background = props.blurRenderer.render(frameTexture, options.backgroundBlurAmount);
+            outputFrameTexture = props.blendRenderer.render({
+                mask: compositeMaskInput,
+                foreground: frameTexture,
+                background,
+            }, options);
         }
         else if (options?.effects === 'overlay' && options.backgroundImage) {
-            background = props.backgroundImageRenderer.render({
-                frame: options.backgroundImage,
-                timestamp: performance.now(),
-                width: options.backgroundImage.width,
-                height: options.backgroundImage.height,
-            });
+            const background = props.backgroundImageRenderer.render(options.backgroundImage);
+            outputFrameTexture = props.replaceRenderer.render({
+                mask: compositeMaskInput,
+                foreground: frameTexture,
+                background,
+                blurredBackground: props.lightWrapBlurRenderer.render(background, options.lightWrapBlurAmount),
+                timestamp: frame.timestamp,
+            }, options);
+        }
+        // 6. Output to present
+        props.canvasRenderer.render(outputFrameTexture);
+        // 7. Cloned the current smoothedMask & frame so it can be fed back to the next round
+        switch (options.downSampleFactor) {
+            case 4:
+                props.prevMask4 = props.prevMaskRenderer4.render(smoothedMask);
+                break;
+            default:
+                props.prevMask2 = props.prevMaskRenderer2.render(smoothedMask);
+                break;
         }
-        const blendedFrameTexture = props.blendRenderer.render({
-            mask: smoothedMask,
-            foreground: frameTexture,
-            background,
-        });
-        props.canvasRenderer.render(blendedFrameTexture);
-        // Cloned the current smoothedMask so it can be fed back to the next round
-        props.prevMask = props.textureRenderer.render(smoothedMask);
     };
     const cleanup = () => {
         props.canvasContextEventsCleanup();
         lazyProps.release();
         lazyProps = createLazyProps(rendererCreator);
-        props.prevMask = undefined;
+        props.prevMask2 = undefined;
+        props.prevMask4 = undefined;
     };
     const release = () => {
         cleanup();

package/dist/common/backends/webgl/replace.d.ts

@@ -0,0 +1,17 @@
+import type { TextureInfo } from './types';
+import type { RendererOptions } from '../../types/render';
+/**
+ * Create a background replacement renderer that composites
+ * the foreground and background as well as adding light wrap
+ * effect based on the mask and blurred background.
+ */
+export declare const createReplaceRenderer: (processor: OffscreenCanvas) => {
+    render: ({ mask, foreground, background, blurredBackground, timestamp, }: {
+        mask: TextureInfo;
+        foreground: TextureInfo;
+        background: TextureInfo;
+        blurredBackground: TextureInfo;
+        timestamp: number;
+    }, options: RendererOptions) => TextureInfo;
+    release: () => void;
+};

package/dist/common/backends/webgl/replace.js

@@ -0,0 +1,126 @@
+import { assert } from '@pexip/utils';
+import { RENDERING_EVENTS } from '../../constants';
+import { m3 } from '../../m3';
+import { createGLProgram, createRGBAFrameTexture, createFrameBuffer, } from './webglUtils';
+import vertexSrc from './shaders/matrix.vert';
+import fragmentSrc from './shaders/replace.frag';
+/**
+ * Create a background replacement renderer that composites
+ * the foreground and background as well as adding light wrap
+ * effect based on the mask and blurred background.
+ */
+export const createReplaceRenderer = (processor) => {
+    const { gl, program, attribLocations, floatFullChannelSupported, uniformLocations, release, } = createGLProgram(processor, {
+        attribs: {
+            position: 'a_position',
+            texCoord: 'a_texCoord',
+        },
+        uniforms: {
+            matrix: 'u_matrix',
+            frameBackground: 'u_frame_background',
+            frameForeground: 'u_frame_foreground',
+            blurBackground: 'u_blur_background',
+            mask: 'u_mask',
+            confidenceThresholds: 'u_confidence_thresholds',
+            lightWrap: 'u_light_wrap',
+            lightWrapTightness: 'u_light_wrap_tightness',
+            lightWrapEdgeBand: 'u_edge_band',
+            time: 'u_time',
+            parity: 'u_parity',
+            frameSize: 'u_frame_size',
+        },
+        vertexSrc,
+        fragmentSrc,
+    });
+    gl.enable(gl.BLEND);
+    const vao = gl.createVertexArray();
+    gl.bindVertexArray(vao);
+    // Position
+    const positionBuffer = gl.createBuffer();
+    gl.enableVertexAttribArray(attribLocations.position);
+    gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
+    gl.vertexAttribPointer(attribLocations.position, 2, gl.FLOAT, false, 0, 0);
+    gl.bufferData(gl.ARRAY_BUFFER, 
+    // prettier-ignore
+    new Float32Array([
+        0, 0,
+        0, 1,
+        1, 0,
+        1, 0,
+        0, 1,
+        1, 1,
+    ]), gl.STATIC_DRAW);
+    // Texture coordinates for the rectangle
+    const texCoordBuffer = gl.createBuffer();
+    gl.bindBuffer(gl.ARRAY_BUFFER, texCoordBuffer);
+    gl.bufferData(gl.ARRAY_BUFFER, 
+    // prettier-ignore
+    new Float32Array([
+        0, 0,
+        0, 1,
+        1, 0,
+        1, 0,
+        0, 1,
+        1, 1,
+    ]), gl.STATIC_DRAW);
+    gl.enableVertexAttribArray(attribLocations.texCoord);
+    gl.vertexAttribPointer(attribLocations.texCoord, 2, gl.FLOAT, false, 0, 0);
+    const outputTexture = createRGBAFrameTexture(gl, processor.width, processor.height, true, floatFullChannelSupported);
+    const frameBuffer = createFrameBuffer(gl, outputTexture);
+    let parity = -1;
+    return {
+        render: ({ mask, foreground, background, blurredBackground, timestamp, }, options) => {
+            assert(!gl.isContextLost(), RENDERING_EVENTS.ContextLost);
+            gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer);
+            gl.viewport(0, 0, processor.width, processor.height);
+            gl.useProgram(program);
+            gl.bindVertexArray(vao);
+            // Textures
+            gl.activeTexture(gl.TEXTURE0);
+            gl.uniform1i(uniformLocations.frameForeground, 0);
+            gl.bindTexture(gl.TEXTURE_2D, foreground.texture);
+            gl.activeTexture(gl.TEXTURE1);
+            gl.uniform1i(uniformLocations.frameBackground, 1);
+            gl.bindTexture(gl.TEXTURE_2D, background.texture);
+            gl.activeTexture(gl.TEXTURE2);
+            gl.uniform1i(uniformLocations.mask, 2);
+            gl.bindTexture(gl.TEXTURE_2D, mask.texture);
+            gl.activeTexture(gl.TEXTURE3);
+            gl.uniform1i(uniformLocations.blurBackground, 3);
+            gl.bindTexture(gl.TEXTURE_2D, blurredBackground.texture);
+            gl.uniform2f(uniformLocations.confidenceThresholds, options.backgroundThreshold, options.foregroundThreshold);
+            gl.uniform1f(uniformLocations.lightWrap, options.lightWrapIntensity);
+            gl.uniform1f(uniformLocations.lightWrapTightness, options.lightWrapTightness);
+            gl.uniform1f(uniformLocations.lightWrapEdgeBand, options.lightWrapEdgeBand);
+            parity = -parity;
+            gl.uniform1f(uniformLocations.parity, parity);
+            gl.uniform1f(uniformLocations.time, 
+            // Use seconds and wrap to a modest range to avoid mediump precision loss
+            (timestamp * 1e-6) % 1024.0);
+            gl.uniform2f(uniformLocations.frameSize, processor.width, processor.height);
+            // No Flip Y Coordinate for drawing on the frame buffer
+            // Project, position and scale to full-size of the canvas
+            const matrix = m3.projection(1, 1, 1);
+            gl.uniformMatrix3fv(uniformLocations.matrix, false, matrix);
+            gl.drawArrays(gl.TRIANGLES, 0, 6);
+            // Cleanup
+            gl.bindBuffer(gl.ARRAY_BUFFER, null);
+            gl.bindFramebuffer(gl.FRAMEBUFFER, null);
+            gl.bindTexture(gl.TEXTURE_2D, null);
+            gl.bindVertexArray(null);
+            return {
+                texture: outputTexture,
+                width: processor.width,
+                height: processor.height,
+            };
+        },
+        release: () => {
+            gl.deleteBuffer(positionBuffer);
+            gl.deleteBuffer(texCoordBuffer);
+            gl.deleteTexture(outputTexture);
+            gl.deleteFramebuffer(frameBuffer);
+            gl.deleteVertexArray(vao);
+            release();
+        },
+    };
+};

package/dist/common/backends/webgl/selectComponent.d.ts

@@ -0,0 +1,15 @@
+import type { RendererOptions } from '../../types/render';
+import type { TextureInfo } from './types';
+/**
+ * Based on Connected Component Labelling algorithm,
+ * find the center-most connected component in the mask and
+ * return a binary mask of just that component.
+ *
+ * @param processor - OffscreenCanvas for WebGL context
+ * @param width - target width of the downsampled mask to process (e.g. 32)
+ * @param height - target height of the downsampled mask to process (e.g. 18)
+ */
+export declare function createComponentSelector(processor: OffscreenCanvas, width: number, height: number): {
+    render: (softMask: TextureInfo, options: RendererOptions) => TextureInfo;
+    release: () => void;
+};

package/dist/common/backends/webgl/selectComponent.js

@@ -0,0 +1,102 @@
+import { assert } from '@pexip/utils';
+import { RENDERING_EVENTS } from '../../constants';
+import { createTextureToTextureRenderer } from './textureToTexture';
+import { scoreComponent, labelComponents, iou } from '../../utils';
+/**
+ * Based on Connected Component Labelling algorithm,
+ * find the center-most connected component in the mask and
+ * return a binary mask of just that component.
+ *
+ * @param processor - OffscreenCanvas for WebGL context
+ * @param width - target width of the downsampled mask to process (e.g. 32)
+ * @param height - target height of the downsampled mask to process (e.g. 18)
+ */
+export function createComponentSelector(processor, width, height) {
+    const gl = processor.getContext('webgl2', {
+        premultipliedAlpha: false,
+        powerPreference: 'high-performance',
+    });
+    const downSampler = createTextureToTextureRenderer(processor, true, false, width, height, false);
+    assert(gl, RENDERING_EVENTS.ContextCreationError);
+    const labels = new Int32Array(width * height);
+    const data = new Uint8Array(width * height);
+    const bin = new Uint8Array(width * height);
+    const bestMask = new Uint8Array(width * height);
+    const labelledMask = new Uint8Array(width * height);
+    let prevPrimaryScore = 0;
+    const outputTexture = gl.createTexture();
+    gl.bindTexture(gl.TEXTURE_2D, outputTexture);
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
+    const fbo = gl.createFramebuffer();
+    const config = {
+        // Frames persisted = -Math.log(1 + switchThreshold) / Math.log(decay)
+        // Width 0.3 switchThreshold and 0.95 decay, it persists about 6 frames
+        switchThreshold: 0.3,
+        decay: 0.95,
+        // Weights
+        area: 0.2,
+        height: 0.4,
+        center: 0.2,
+        iou: 0.2,
+    };
+    return {
+        render: (softMask, options) => {
+            const smallMask = downSampler.render(softMask);
+            // Read back single-channel 8-bit for simplicity here
+            gl.bindFramebuffer(gl.FRAMEBUFFER, fbo);
+            gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, smallMask.texture, 0);
+            gl.readPixels(0, 0, width, height, gl.RED, gl.UNSIGNED_BYTE, data);
+            gl.bindFramebuffer(gl.FRAMEBUFFER, null);
+            // Binarize using foregroundThreshold in 0..255 domain
+            const thresh = Math.max(0, Math.min(255, Math.floor(255 * options.foregroundThreshold)));
+            for (let i = 0; i < bin.length; i++) {
+                //@ts-expect-error will not be undefined
+                bin[i] = data[i] >= thresh ? 255 : 0;
+            }
+            let bestScore = -Infinity;
+            let bestLabel = 0;
+            const stats = labelComponents(bin, labels, width, height);
+            for (const [l, s] of stats.entries()) {
+                if (l === 0) {
+                    continue; // background
+                }
+                for (let i = 0; i < labels.length; i++) {
+                    labelledMask[i] = labels[i] === l ? 255 : 0;
+                }
+                const score = scoreComponent(s, iou(bestMask, labelledMask), {
+                    ...config,
+                    centerX: options.personCenter[0] * width,
+                    centerY: options.personCenter[1] * height,
+                }, width, height);
+                if (score > bestScore) {
+                    bestScore = score;
+                    bestLabel = l;
+                }
+            }
+            // Hysteresis: switch only if significantly better
+            const shouldSwitch = bestScore > prevPrimaryScore * (1.0 + config.switchThreshold);
+            if (shouldSwitch || prevPrimaryScore === 0) {
+                for (let i = 0; i < labels.length; i++) {
+                    bestMask[i] = labels[i] === bestLabel ? 255 : 0;
+                }
+                prevPrimaryScore = bestScore;
+            }
+            else {
+                prevPrimaryScore *= config.decay;
+            }
+            // Upload small filtered mask to GPU and return as TextureInfo (caller will upsample & combine)
+            gl.bindTexture(gl.TEXTURE_2D, outputTexture);
+            gl.texImage2D(gl.TEXTURE_2D, 0, gl.R8, width, height, 0, gl.RED, gl.UNSIGNED_BYTE, bestMask);
+            gl.bindTexture(gl.TEXTURE_2D, null);
+            return { texture: outputTexture, width, height };
+        },
+        release: () => {
+            gl.deleteTexture(outputTexture);
+            gl.deleteFramebuffer(fbo);
+            downSampler.release();
+        },
+    };
+}

package/dist/common/backends/webgl/smoothingMask.d.ts

@@ -3,9 +3,9 @@ import type { TextureInfo } from './types';
 /**
  * Smooth the mask edge
  */
-export declare const createSmoothingMaskRenderer: (processor: OffscreenCanvas) => {
-    render: ({ mask, prevMask, }: {
-        prevMask?: TextureInfo;
+export declare const createSmoothingMaskRenderer: (processor: OffscreenCanvas, width: number, height: number) => {
+    render: ({ mask, prevMask }: {
+        prevMask: TextureInfo;
         mask: TextureInfo;
     }, options: RendererOptions) => TextureInfo;
     release: () => void;

package/dist/common/backends/webgl/smoothingMask.js

@@ -1,14 +1,14 @@
 import { assert } from '@pexip/utils';
 import { m3 } from '../../m3';
 import { RENDERING_EVENTS } from '../../constants';
-import { createGLProgram, createAndSetupTexture } from './webglUtils';
-import { vertexShaderSource } from './shaders/vertexShader';
-import { smoothingMaskFragmentShaderSource } from './shaders/smoothingMaskFragmentShader';
+import { createGLProgram, createFrameBuffer, createMaskTexture, } from './webglUtils';
+import vertexSrc from './shaders/matrix.vert';
+import fragmentSrc from './shaders/smoothingMask.frag';
 /**
  * Smooth the mask edge
  */
-export const createSmoothingMaskRenderer = (processor) => {
-    const { gl, program, attribLocations, uniformLocations, release } = createGLProgram(processor, {
+export const createSmoothingMaskRenderer = (processor, width, height) => {
+    const { gl, program, attribLocations, uniformLocations, floatFullChannelSupported, floatSingleChannelSupported, release, } = createGLProgram(processor, {
         attribs: {
             position: 'a_position',
             texCoord: 'a_texCoord',
@@ -16,12 +16,13 @@ export const createSmoothingMaskRenderer = (processor) => {
         uniforms: {
             matrix: 'u_matrix',
             mask: 'u_mask',
-            maskCombineRatio: 'u_mask_combine_ratio',
-            foregroundThreshold: 'u_foreground_threshold',
             prevMask: 'u_mask_prev',
+            frame: 'u_frame',
+            prevFrame: 'u_frame_prev',
+            maskCombineRatio: 'u_mask_combine_ratio',
         },
-        vertexSrc: vertexShaderSource,
-        fragmentSrc: smoothingMaskFragmentShaderSource,
+        vertexSrc,
+        fragmentSrc,
     });
     const vao = gl.createVertexArray();
     gl.bindVertexArray(vao);
@@ -55,16 +56,13 @@ export const createSmoothingMaskRenderer = (processor) => {
     ]), gl.STATIC_DRAW);
     gl.enableVertexAttribArray(attribLocations.texCoord);
     gl.vertexAttribPointer(attribLocations.texCoord, 2, gl.FLOAT, false, 0, 0);
-    const outputTexture = createAndSetupTexture(gl);
-    gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, processor.width, processor.height, 0, gl.RGBA, gl.UNSIGNED_BYTE, null);
-    const frameBuffer = gl.createFramebuffer();
-    gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer);
-    gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, outputTexture, 0);
+    const outputTexture = createMaskTexture(gl, width, height, false, floatSingleChannelSupported, floatFullChannelSupported);
+    const frameBuffer = createFrameBuffer(gl, outputTexture);
     return {
-        render: ({ mask, prevMask, }, options) => {
+        render: ({ mask, prevMask }, options) => {
             assert(!gl.isContextLost(), RENDERING_EVENTS.ContextLost);
             gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer);
-            gl.viewport(0, 0, processor.width, processor.height);
+            gl.viewport(0, 0, mask.width, mask.height);
             gl.clearColor(0, 0, 0, 0);
             gl.clear(gl.COLOR_BUFFER_BIT);
             gl.useProgram(program);
@@ -75,19 +73,22 @@ export const createSmoothingMaskRenderer = (processor) => {
             gl.bindTexture(gl.TEXTURE_2D, mask.texture);
             gl.activeTexture(gl.TEXTURE1);
             gl.uniform1i(uniformLocations.prevMask, 1);
-            gl.bindTexture(gl.TEXTURE_2D, prevMask?.texture ?? mask.texture);
+            gl.bindTexture(gl.TEXTURE_2D, prevMask.texture);
             // No Flip Y Coordinate for drawing on the frame buffer
             // Project, position and scale to full-size of the canvas
             const matrix = m3.projection(1, 1, 1);
             gl.uniformMatrix3fv(uniformLocations.matrix, false, matrix);
             gl.uniform1f(uniformLocations.maskCombineRatio, options.maskCombineRatio);
-            gl.uniform1f(uniformLocations.foregroundThreshold, options.foregroundThreshold);
             gl.drawArrays(gl.TRIANGLES, 0, 6);
-            assert(outputTexture);
+            // Cleanup
+            gl.bindBuffer(gl.ARRAY_BUFFER, null);
+            gl.bindFramebuffer(gl.FRAMEBUFFER, null);
+            gl.bindTexture(gl.TEXTURE_2D, null);
+            gl.bindVertexArray(null);
             return {
                 texture: outputTexture,
-                width: processor.width,
-                height: processor.height,
+                width,
+                height,
             };
         },
         release: () => {

package/dist/common/backends/webgl/tentBlur.d.ts

@@ -0,0 +1,9 @@
+import type { TextureInfo } from './types';
+/**
+ * Create a tent blur renderer for small radius blurring of a mask.
+ * The blur is implemented as a separable filter.
+ */
+export declare const createTentBlurRenderer: (processor: OffscreenCanvas, width: number, height: number) => {
+    render: (mask: TextureInfo, radiusPx: number) => TextureInfo;
+    release: () => void;
+};