@pexip/media-processor 20.3.5 → 21.0.0

package/dist/common/backends/webgl/jointBilateralFilter.js

@@ -0,0 +1,108 @@
+import { assert } from '@pexip/utils';
+import { RENDERING_EVENTS } from '../../constants';
+import { m3 } from '../../m3';
+import { createGLProgram, createMaskTexture, createFrameBuffer, } from './webglUtils';
+import vertexSrc from './shaders/matrix.vert';
+import fragmentSrc from './shaders/jointBilateralFilter.frag';
+/**
+ * Joint Bilateral Filter
+ */
+export const createJointBilateralFilterRenderer = (processor, width, height) => {
+    const { gl, program, attribLocations, uniformLocations, release } = createGLProgram(processor, {
+        attribs: {
+            position: 'a_position',
+            texCoord: 'a_texCoord',
+        },
+        uniforms: {
+            frameGuide: 'u_frame_guide',
+            frameInput: 'u_frame_input',
+            matrix: 'u_matrix',
+            texelSize: 'u_texel_size',
+            sigmaSpace: 'u_sigma_space',
+            sigmaRangeLo: 'u_sigma_range_lo',
+            sigmaRangeHi: 'u_sigma_range_hi',
+            foregroundThreshold: 'u_foreground_threshold',
+            backgroundThreshold: 'u_background_threshold',
+        },
+        vertexSrc,
+        fragmentSrc,
+    });
+    gl.enable(gl.BLEND);
+    const vao = gl.createVertexArray();
+    gl.bindVertexArray(vao);
+    // Position
+    const positionBuffer = gl.createBuffer();
+    gl.enableVertexAttribArray(attribLocations.position);
+    gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
+    gl.vertexAttribPointer(attribLocations.position, 2, gl.FLOAT, false, 0, 0);
+    gl.bufferData(gl.ARRAY_BUFFER, 
+    // prettier-ignore
+    new Float32Array([
+        0, 0,
+        0, 1,
+        1, 0,
+        1, 0,
+        0, 1,
+        1, 1,
+    ]), gl.STATIC_DRAW);
+    // Texture coordinates for the rectangle
+    const texCoordBuffer = gl.createBuffer();
+    gl.bindBuffer(gl.ARRAY_BUFFER, texCoordBuffer);
+    gl.bufferData(gl.ARRAY_BUFFER, 
+    // prettier-ignore
+    new Float32Array([
+        0, 0,
+        0, 1,
+        1, 0,
+        1, 0,
+        0, 1,
+        1, 1,
+    ]), gl.STATIC_DRAW);
+    gl.enableVertexAttribArray(attribLocations.texCoord);
+    gl.vertexAttribPointer(attribLocations.texCoord, 2, gl.FLOAT, false, 0, 0);
+    const outputTexture = createMaskTexture(gl, width, height);
+    const frameBuffer = createFrameBuffer(gl, outputTexture);
+    const scaleFactor = processor.width / width;
+    return {
+        render: ({ input, guide }, options) => {
+            assert(!gl.isContextLost(), RENDERING_EVENTS.ContextLost);
+            assert(options.foregroundThreshold >= options.backgroundThreshold, 'foregroundThreshold must be >= backgroundThreshold');
+            gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer);
+            gl.viewport(0, 0, width, height);
+            gl.useProgram(program);
+            gl.bindVertexArray(vao);
+            // Textures
+            gl.activeTexture(gl.TEXTURE0);
+            gl.uniform1i(uniformLocations.frameInput, 0);
+            gl.bindTexture(gl.TEXTURE_2D, input.texture);
+            gl.activeTexture(gl.TEXTURE1);
+            gl.uniform1i(uniformLocations.frameGuide, 1);
+            gl.bindTexture(gl.TEXTURE_2D, guide.texture);
+            gl.uniform2f(uniformLocations.texelSize, 1.0 / input.width, 1.0 / input.height);
+            gl.uniform1f(uniformLocations.sigmaSpace, options.sigmaSpace / scaleFactor);
+            gl.uniform1f(uniformLocations.sigmaRangeLo, options.sigmaRangeLo);
+            gl.uniform1f(uniformLocations.sigmaRangeHi, options.sigmaRangeHi);
+            gl.uniform1f(uniformLocations.backgroundThreshold, options.backgroundThreshold);
+            gl.uniform1f(uniformLocations.foregroundThreshold, options.foregroundThreshold);
+            // No Flip Y Coordinate for drawing on the frame buffer
+            // Project, position and scale to full-size of the canvas
+            const matrix = m3.projection(1, 1, 1);
+            gl.uniformMatrix3fv(uniformLocations.matrix, false, matrix);
+            gl.drawArrays(gl.TRIANGLES, 0, 6);
+            // Cleanup
+            gl.bindBuffer(gl.ARRAY_BUFFER, null);
+            gl.bindFramebuffer(gl.FRAMEBUFFER, null);
+            gl.bindTexture(gl.TEXTURE_2D, null);
+            gl.bindVertexArray(null);
+            return { texture: outputTexture, width, height };
+        },
+        release: () => {
+            gl.deleteBuffer(positionBuffer);
+            gl.deleteBuffer(texCoordBuffer);
+            gl.deleteTexture(outputTexture);
+            gl.deleteFramebuffer(frameBuffer);
+            gl.deleteVertexArray(vao);
+            release();
+        },
+    };
+};

package/dist/common/backends/webgl/morphErode.d.ts

@@ -0,0 +1,13 @@
+import type { RendererOptions } from '../../types/render';
+import type { TextureInfo } from './types';
+/**
+ * Create a morphological erosion renderer that performs erosion of a mask with a
+ * given radius and number of passes.
+ * The erosion is implemented as a separable filter.
+ */
+export declare const createMorphErodeRenderer: (processor: OffscreenCanvas, width: number, height: number) => {
+    render: ({ mask }: {
+        mask: TextureInfo;
+    }, options: Pick<RendererOptions, "morphPass" | "morphErodeRadiusPx">) => TextureInfo;
+    release: () => void;
+};

package/dist/common/backends/webgl/morphErode.js

@@ -0,0 +1,121 @@
+import { assert } from '@pexip/utils';
+import { m3 } from '../../m3';
+import { RENDERING_EVENTS } from '../../constants';
+import { clamping } from '../../utils';
+import { createGLProgram, createMaskTexture, createFrameBuffer, } from './webglUtils';
+import vertexSrc from './shaders/matrix.vert';
+import fragmentSrc from './shaders/erode.frag';
+const MAX_RADIUS_PX = 64;
+const clampRadiusPx = clamping(0, MAX_RADIUS_PX);
+/**
+ * Create a morphological erosion renderer that performs erosion of a mask with a
+ * given radius and number of passes.
+ * The erosion is implemented as a separable filter.
+ */
+export const createMorphErodeRenderer = (processor, width, height) => {
+    const { gl, program, attribLocations, uniformLocations, floatSingleChannelSupported, floatFullChannelSupported, release, } = createGLProgram(processor, {
+        attribs: {
+            position: 'a_position',
+            texCoord: 'a_texCoord',
+        },
+        uniforms: {
+            matrix: 'u_matrix',
+            mask: 'u_mask',
+            texelSize: 'u_texel_size',
+            radiusPx: 'u_radius_px',
+            direction: 'u_direction',
+        },
+        vertexSrc,
+        fragmentSrc,
+    });
+    const vao = gl.createVertexArray();
+    gl.bindVertexArray(vao);
+    // Position
+    const positionBuffer = gl.createBuffer();
+    gl.enableVertexAttribArray(attribLocations.position);
+    gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
+    gl.vertexAttribPointer(attribLocations.position, 2, gl.FLOAT, false, 0, 0);
+    gl.bufferData(gl.ARRAY_BUFFER, 
+    // prettier-ignore
+    new Float32Array([
+        0, 0,
+        0, 1,
+        1, 0,
+        1, 0,
+        0, 1,
+        1, 1,
+    ]), gl.STATIC_DRAW);
+    // Texture coordinates for the rectangle
+    const texCoordBuffer = gl.createBuffer();
+    gl.bindBuffer(gl.ARRAY_BUFFER, texCoordBuffer);
+    gl.bufferData(gl.ARRAY_BUFFER, 
+    // prettier-ignore
+    new Float32Array([
+        0, 0,
+        0, 1,
+        1, 0,
+        1, 0,
+        0, 1,
+        1, 1,
+    ]), gl.STATIC_DRAW);
+    gl.enableVertexAttribArray(attribLocations.texCoord);
+    gl.vertexAttribPointer(attribLocations.texCoord, 2, gl.FLOAT, false, 0, 0);
+    const outputTexturePing = createMaskTexture(gl, width, height, false, floatSingleChannelSupported, floatFullChannelSupported);
+    const frameBufferPing = createFrameBuffer(gl, outputTexturePing);
+    const outputTexturePong = createMaskTexture(gl, width, height, false, floatSingleChannelSupported, floatFullChannelSupported);
+    const frameBufferPong = createFrameBuffer(gl, outputTexturePong);
+    const scaleFactor = processor.width / width;
+    return {
+        render: ({ mask }, options) => {
+            assert(!gl.isContextLost(), RENDERING_EVENTS.ContextLost);
+            gl.clearColor(0, 0, 0, 0);
+            gl.clear(gl.COLOR_BUFFER_BIT);
+            gl.useProgram(program);
+            gl.bindVertexArray(vao);
+            gl.uniform2f(uniformLocations.texelSize, 1 / mask.width, 1 / mask.height);
+            gl.uniform1i(uniformLocations.radiusPx, clampRadiusPx(Math.floor(options.morphErodeRadiusPx / scaleFactor)));
+            const matrix = m3.projection(1, 1, 1);
+            // No Flip Y Coordinate for drawing on the frame buffer
+            // Project, position and scale to full-size of the canvas
+            gl.uniformMatrix3fv(uniformLocations.matrix, false, matrix);
+            let write = frameBufferPing;
+            let read = mask.texture;
+            // Ping-pong framebuffers for multiple passes
+            for (let i = 0; i < options.morphPass; i++) {
+                // Separable
+                for (let j = 0; j < 2; j++) {
+                    // Textures
+                    gl.activeTexture(gl.TEXTURE0);
+                    gl.uniform1i(uniformLocations.mask, 0);
+                    gl.bindTexture(gl.TEXTURE_2D, read);
+                    // Horizontal then Vertical
+                    gl.uniform2f(uniformLocations.direction, 1.0 - j, 0.0 + j);
+                    gl.bindFramebuffer(gl.FRAMEBUFFER, write);
+                    gl.viewport(0, 0, mask.width, mask.height);
+                    gl.drawArrays(gl.TRIANGLES, 0, 6);
+                    // Swap
+                    [read, write] =
+                        write === frameBufferPing
+                            ? [outputTexturePing, frameBufferPong]
+                            : [outputTexturePong, frameBufferPing];
+                }
+            }
+            // Cleanup
+            gl.bindBuffer(gl.ARRAY_BUFFER, null);
+            gl.bindFramebuffer(gl.FRAMEBUFFER, null);
+            gl.bindTexture(gl.TEXTURE_2D, null);
+            gl.bindVertexArray(null);
+            return { texture: read, width, height };
+        },
+        release: () => {
+            gl.deleteBuffer(positionBuffer);
+            gl.deleteBuffer(texCoordBuffer);
+            gl.deleteTexture(outputTexturePing);
+            gl.deleteTexture(outputTexturePong);
+            gl.deleteFramebuffer(frameBufferPing);
+            gl.deleteFramebuffer(frameBufferPong);
+            gl.deleteVertexArray(vao);
+            release();
+        },
+    };
+};

package/dist/common/backends/webgl/renderer.js

@@ -1,23 +1,59 @@
 import { assert } from '@pexip/utils';
-import { PROCESSING_WIDTH, PROCESSING_HEIGHT, EDGE_BLUR_AMOUNT, BACKGROUND_BLUR_AMOUNT, RENDERING_EVENTS, } from '../../constants';
-import { createTextureRenderer } from './texture';
-import { createBlurRenderer } from './blur';
+import { BACKGROUND_BLUR_AMOUNT, PROCESSING_HEIGHT, PROCESSING_WIDTH, RENDERING_EVENTS, STRONG_EDGE_BLUR_AMOUNT, } from '../../constants';
+import { createLazyProps, handleWebGLContextLoss } from '../../utils';
+import { createBinaryMaskRenderer } from './binaryMask';
 import { createBlendRenderer } from './blender';
+import { createBlurRenderer } from './blur';
 import { createCanvasRenderer } from './canvas';
-import { createTextureToTextureRenderer } from './textureToTexture';
+import { createComponentSelector } from './selectComponent';
+import { createGeodesicReconstructionRenderer } from './geodesicReconstruction';
+import { createJointBilateralFilterRenderer } from './jointBilateralFilter';
+import { createMorphErodeRenderer } from './morphErode';
+import { createReplaceRenderer } from './replace';
 import { createSmoothingMaskRenderer } from './smoothingMask';
-import { createLazyProps, handleWebGLContextLoss } from '../../utils';
+import { createTentBlurRenderer } from './tentBlur';
+import { createTextureRenderer } from './texture';
+import { createTextureToTextureRenderer } from './textureToTexture';
 export const createRenderer = (renderEventHandlers, canvas = new OffscreenCanvas(PROCESSING_WIDTH, PROCESSING_HEIGHT)) => {
+    const halfWidth = Math.trunc(canvas.width / 2);
+    const halfHeight = Math.trunc(canvas.height / 2);
+    const quarterWidth = Math.trunc(canvas.width / 4);
+    const quarterHeight = Math.trunc(canvas.height / 4);
+    // Width and Height used for the Center Component selection
+    const cclWidth = 32;
+    const cclHeight = 18;
     const rendererCreator = {
-        videoFrameRenderer: () => createTextureRenderer(canvas),
-        maskRenderer: () => createTextureRenderer(canvas),
-        canvasRenderer: () => createCanvasRenderer(canvas),
-        backgroundImageRenderer: () => createTextureRenderer(canvas),
+        backgroundImageRenderer: () => createTextureRenderer(canvas, false),
         blendRenderer: () => createBlendRenderer(canvas),
         blurRenderer: () => createBlurRenderer(canvas, BACKGROUND_BLUR_AMOUNT),
-        maskBlurRenderer: () => createBlurRenderer(canvas, EDGE_BLUR_AMOUNT),
-        textureRenderer: () => createTextureToTextureRenderer(canvas),
-        smoothingMaskRenderer: () => createSmoothingMaskRenderer(canvas),
+        canvasRenderer: () => createCanvasRenderer(canvas),
+        frameDownsampler2: () => createTextureToTextureRenderer(canvas, false, true, halfWidth, halfHeight),
+        frameDownsampler4: () => createTextureToTextureRenderer(canvas, false, true, quarterWidth, quarterHeight),
+        geodesicReconstructionRenderer2: () => createGeodesicReconstructionRenderer(canvas, halfWidth, halfHeight),
+        geodesicReconstructionRenderer4: () => createGeodesicReconstructionRenderer(canvas, quarterWidth, quarterHeight),
+        jointBilateralFilterRenderer2: () => createJointBilateralFilterRenderer(canvas, halfWidth, halfHeight),
+        jointBilateralFilterRenderer4: () => createJointBilateralFilterRenderer(canvas, quarterWidth, quarterHeight),
+        jointBilateralFilterRenderer: () => createJointBilateralFilterRenderer(canvas, canvas.width, canvas.height),
+        lightWrapBlurRenderer: () => createBlurRenderer(canvas, BACKGROUND_BLUR_AMOUNT),
+        maskBlurRenderer: () => createTentBlurRenderer(canvas, canvas.width, canvas.height),
+        strongMaskBlurRenderer: () => createBlurRenderer(canvas, STRONG_EDGE_BLUR_AMOUNT),
+        maskDownsampler2: () => createTextureToTextureRenderer(canvas, true, false, halfWidth, halfHeight),
+        maskDownsampler4: () => createTextureToTextureRenderer(canvas, true, false, quarterWidth, quarterHeight),
+        maskRenderer: () => createTextureRenderer(canvas, true),
+        maskUpsampler2: () => createTextureToTextureRenderer(canvas, true, false, halfWidth, halfHeight),
+        maskUpsampler4: () => createTextureToTextureRenderer(canvas, true, false, quarterWidth, quarterHeight),
+        maskUpsampler: () => createTextureToTextureRenderer(canvas, true, false, canvas.width, canvas.height),
+        morphErodeRenderer: () => createMorphErodeRenderer(canvas, halfWidth, halfHeight),
+        morphErodeRenderer4: () => createMorphErodeRenderer(canvas, quarterWidth, quarterHeight),
+        prevMaskRenderer2: () => createTextureToTextureRenderer(canvas, true, false, halfWidth, halfHeight),
+        prevMaskRenderer4: () => createTextureToTextureRenderer(canvas, true, false, quarterWidth, quarterHeight),
+        replaceRenderer: () => createReplaceRenderer(canvas),
+        selectCenterComponent: () => createComponentSelector(canvas, cclWidth, cclHeight),
+        smoothingMaskRenderer4: () => createSmoothingMaskRenderer(canvas, quarterWidth, quarterHeight),
+        smoothingMaskRenderer2: () => createSmoothingMaskRenderer(canvas, halfWidth, halfHeight),
+        binaryMaskRenderer2: () => createBinaryMaskRenderer(canvas, halfWidth, halfHeight),
+        binaryMaskRenderer4: () => createBinaryMaskRenderer(canvas, quarterWidth, quarterHeight),
+        videoFrameRenderer: () => createTextureRenderer(canvas, false),
         release: (states) => () => {
             for (const key in states) {
                 if (key !== 'release') {
@@ -35,6 +71,15 @@ export const createRenderer = (renderEventHandlers, canvas = new OffscreenCanvas
         get videoFrameRenderer() {
             return lazyProps.videoFrameRenderer;
         },
+        get jointBilateralFilterRenderer() {
+            return lazyProps.jointBilateralFilterRenderer;
+        },
+        get jointBilateralFilterRenderer2() {
+            return lazyProps.jointBilateralFilterRenderer2;
+        },
+        get jointBilateralFilterRenderer4() {
+            return lazyProps.jointBilateralFilterRenderer4;
+        },
         get maskRenderer() {
             return lazyProps.maskRenderer;
         },
@@ -47,18 +92,74 @@ export const createRenderer = (renderEventHandlers, canvas = new OffscreenCanvas
         get blendRenderer() {
             return lazyProps.blendRenderer;
         },
+        get replaceRenderer() {
+            return lazyProps.replaceRenderer;
+        },
         get blurRenderer() {
             return lazyProps.blurRenderer;
         },
-        // TODO: Impl edge blur
+        get lightWrapBlurRenderer() {
+            return lazyProps.lightWrapBlurRenderer;
+        },
+        get prevMaskRenderer2() {
+            return lazyProps.prevMaskRenderer2;
+        },
+        get prevMaskRenderer4() {
+            return lazyProps.prevMaskRenderer4;
+        },
+        get selectCenterComponent() {
+            return lazyProps.selectCenterComponent;
+        },
+        get smoothingMaskRenderer2() {
+            return lazyProps.smoothingMaskRenderer2;
+        },
+        get smoothingMaskRenderer4() {
+            return lazyProps.smoothingMaskRenderer4;
+        },
+        get binaryMaskRenderer2() {
+            return lazyProps.binaryMaskRenderer2;
+        },
+        get binaryMaskRenderer4() {
+            return lazyProps.binaryMaskRenderer4;
+        },
+        get morphErodeRenderer() {
+            return lazyProps.morphErodeRenderer;
+        },
+        get morphErodeRenderer4() {
+            return lazyProps.morphErodeRenderer4;
+        },
         get maskBlurRenderer() {
             return lazyProps.maskBlurRenderer;
         },
-        get textureRenderer() {
-            return lazyProps.textureRenderer;
+        get strongMaskBlurRenderer() {
+            return lazyProps.strongMaskBlurRenderer;
+        },
+        get maskDownsampler2() {
+            return lazyProps.maskDownsampler2;
+        },
+        get maskDownsampler4() {
+            return lazyProps.maskDownsampler4;
+        },
+        get maskUpsampler() {
+            return lazyProps.maskUpsampler;
+        },
+        get maskUpsampler2() {
+            return lazyProps.maskUpsampler2;
+        },
+        get maskUpsampler4() {
+            return lazyProps.maskUpsampler4;
+        },
+        get frameDownsampler2() {
+            return lazyProps.frameDownsampler2;
+        },
+        get frameDownsampler4() {
+            return lazyProps.frameDownsampler4;
         },
-        get smoothingMaskRenderer() {
-            return lazyProps.smoothingMaskRenderer;
+        get geodesicReconstructionRenderer2() {
+            return lazyProps.geodesicReconstructionRenderer2;
+        },
+        get geodesicReconstructionRenderer4() {
+            return lazyProps.geodesicReconstructionRenderer4;
         },
     };
     const init = () => {
@@ -75,43 +176,102 @@ export const createRenderer = (renderEventHandlers, canvas = new OffscreenCanvas
     };
     const render = (mask, frame, options) => {
         assert(!props.gl?.isContextLost(), RENDERING_EVENTS.ContextLost);
+        // 1. Upload current frame and raw mask
+        const frameTexture = props.videoFrameRenderer.render(frame);
         const maskTexture = props.maskRenderer.render({
             frame: mask,
             width: mask.width,
             height: mask.height,
             timestamp: frame.timestamp,
         });
-        const smoothedMask = props.smoothingMaskRenderer.render({
-            mask: maskTexture,
-            prevMask: props.prevMask,
-        }, options);
-        const frameTexture = props.videoFrameRenderer.render(frame);
-        let background = frameTexture;
+        // 2. Joint Bilateral Filter
+        let smoothedMask;
+        let compositeMaskInput;
+        switch (options.downSampleFactor) {
+            case 4: {
+                const maskLowRes = props.maskDownsampler4.render(maskTexture);
+                const frameLowRes = props.frameDownsampler4.render(frameTexture);
+                const filteredMaskLowRes = props.jointBilateralFilterRenderer4.render({ input: maskLowRes, guide: frameLowRes }, options);
+                smoothedMask = props.smoothingMaskRenderer4.render({
+                    mask: filteredMaskLowRes,
+                    prevMask: props.prevMask4 ?? filteredMaskLowRes,
+                }, options);
+                if (options.excludeBystanders) {
+                    const binary = props.binaryMaskRenderer4.render(smoothedMask, options.foregroundThreshold);
+                    const eroded = props.morphErodeRenderer4.render({ mask: binary }, options);
+                    const selected = props.selectCenterComponent.render(eroded, options);
+                    const filted = props.maskUpsampler4.render(selected);
+                    smoothedMask = props.geodesicReconstructionRenderer4.render({ mask: filted, maskRef: smoothedMask }, options);
+                }
+                // Upsample to original size
+                const result = props.maskUpsampler.render(smoothedMask);
+                compositeMaskInput = props.strongMaskBlurRenderer.render(result, STRONG_EDGE_BLUR_AMOUNT);
+                break;
+            }
+            default: {
+                const maskLowRes = props.maskDownsampler2.render(maskTexture);
+                const frameLowRes = props.frameDownsampler2.render(frameTexture);
+                const filteredMaskLowRes = props.jointBilateralFilterRenderer2.render({ input: maskLowRes, guide: frameLowRes }, options);
+                smoothedMask = props.smoothingMaskRenderer2.render({
+                    mask: filteredMaskLowRes,
+                    prevMask: props.prevMask2 ?? filteredMaskLowRes,
+                }, options);
+                if (options.excludeBystanders) {
+                    const binary = props.binaryMaskRenderer2.render(smoothedMask, options.foregroundThreshold);
+                    const eroded = props.morphErodeRenderer.render({ mask: binary }, options);
+                    const selected = props.selectCenterComponent.render(eroded, options);
+                    const filted = props.maskUpsampler2.render(selected);
+                    smoothedMask = props.geodesicReconstructionRenderer2.render({ mask: filted, maskRef: smoothedMask }, options);
+                }
+                // Upsample to original size
+                const result = props.maskUpsampler.render(smoothedMask);
+                compositeMaskInput = props.maskBlurRenderer.render(result, options.edgeBlurAmount);
+                break;
+            }
+        }
+        // 5. Composite final output
+        let outputFrameTexture = frameTexture;
         if (options.effects === 'blur' && options.backgroundBlurAmount) {
-            background = props.blurRenderer.render(frameTexture, options.backgroundBlurAmount);
+            const background = props.blurRenderer.render(frameTexture, options.backgroundBlurAmount);
+            outputFrameTexture = props.blendRenderer.render({
+                mask: compositeMaskInput,
+                foreground: frameTexture,
+                background,
+            }, options);
         }
         else if (options?.effects === 'overlay' && options.backgroundImage) {
-            background = props.backgroundImageRenderer.render({
+            const background = props.backgroundImageRenderer.render({
                 frame: options.backgroundImage,
                 timestamp: performance.now(),
                 width: options.backgroundImage.width,
                 height: options.backgroundImage.height,
             });
+            outputFrameTexture = props.replaceRenderer.render({
+                mask: compositeMaskInput,
+                foreground: frameTexture,
+                background,
+                blurredBackground: props.lightWrapBlurRenderer.render(background, options.lightWrapBlurAmount),
+                timestamp: frame.timestamp,
+            }, options);
+        }
+        // 6. Output to present
+        props.canvasRenderer.render(outputFrameTexture);
+        // 7. Cloned the current smoothedMask & frame so it can be fed back to the next round
+        switch (options.downSampleFactor) {
+            case 4:
+                props.prevMask4 = props.prevMaskRenderer4.render(smoothedMask);
+                break;
+            default:
+                props.prevMask2 = props.prevMaskRenderer2.render(smoothedMask);
+                break;
         }
-        const blendedFrameTexture = props.blendRenderer.render({
-            mask: smoothedMask,
-            foreground: frameTexture,
-            background,
-        });
-        props.canvasRenderer.render(blendedFrameTexture);
-        // Cloned the current smoothedMask so it can be fed back to the next round
-        props.prevMask = props.textureRenderer.render(smoothedMask);
     };
     const cleanup = () => {
         props.canvasContextEventsCleanup();
         lazyProps.release();
         lazyProps = createLazyProps(rendererCreator);
-        props.prevMask = undefined;
+        props.prevMask2 = undefined;
+        props.prevMask4 = undefined;
     };
     const release = () => {
         cleanup();

package/dist/common/backends/webgl/replace.d.ts

@@ -0,0 +1,17 @@
+import type { TextureInfo } from './types';
+import type { RendererOptions } from '../../types/render';
+/**
+ * Create a background replacement renderer that composites
+ * the foreground and background as well as adding light wrap
+ * effect based on the mask and blurred background.
+ */
+export declare const createReplaceRenderer: (processor: OffscreenCanvas) => {
+    render: ({ mask, foreground, background, blurredBackground, timestamp, }: {
+        mask: TextureInfo;
+        foreground: TextureInfo;
+        background: TextureInfo;
+        blurredBackground: TextureInfo;
+        timestamp: number;
+    }, options: RendererOptions) => TextureInfo;
+    release: () => void;
+};

package/dist/common/backends/webgl/replace.js

@@ -0,0 +1,126 @@
+import { assert } from '@pexip/utils';
+import { RENDERING_EVENTS } from '../../constants';
+import { m3 } from '../../m3';
+import { createGLProgram, createRGBAFrameTexture, createFrameBuffer, } from './webglUtils';
+import vertexSrc from './shaders/matrix.vert';
+import fragmentSrc from './shaders/replace.frag';
+/**
+ * Create a background replacement renderer that composites
+ * the foreground and background as well as adding light wrap
+ * effect based on the mask and blurred background.
+ */
+export const createReplaceRenderer = (processor) => {
+    const { gl, program, attribLocations, floatFullChannelSupported, uniformLocations, release, } = createGLProgram(processor, {
+        attribs: {
+            position: 'a_position',
+            texCoord: 'a_texCoord',
+        },
+        uniforms: {
+            matrix: 'u_matrix',
+            frameBackground: 'u_frame_background',
+            frameForeground: 'u_frame_foreground',
+            blurBackground: 'u_blur_background',
+            mask: 'u_mask',
+            confidenceThresholds: 'u_confidence_thresholds',
+            lightWrap: 'u_light_wrap',
+            lightWrapTightness: 'u_light_wrap_tightness',
+            lightWrapEdgeBand: 'u_edge_band',
+            time: 'u_time',
+            parity: 'u_parity',
+            frameSize: 'u_frame_size',
+        },
+        vertexSrc,
+        fragmentSrc,
+    });
+    gl.enable(gl.BLEND);
+    const vao = gl.createVertexArray();
+    gl.bindVertexArray(vao);
+    // Position
+    const positionBuffer = gl.createBuffer();
+    gl.enableVertexAttribArray(attribLocations.position);
+    gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
+    gl.vertexAttribPointer(attribLocations.position, 2, gl.FLOAT, false, 0, 0);
+    gl.bufferData(gl.ARRAY_BUFFER, 
+    // prettier-ignore
+    new Float32Array([
+        0, 0,
+        0, 1,
+        1, 0,
+        1, 0,
+        0, 1,
+        1, 1,
+    ]), gl.STATIC_DRAW);
+    // Texture coordinates for the rectangle
+    const texCoordBuffer = gl.createBuffer();
+    gl.bindBuffer(gl.ARRAY_BUFFER, texCoordBuffer);
+    gl.bufferData(gl.ARRAY_BUFFER, 
+    // prettier-ignore
+    new Float32Array([
+        0, 0,
+        0, 1,
+        1, 0,
+        1, 0,
+        0, 1,
+        1, 1,
+    ]), gl.STATIC_DRAW);
+    gl.enableVertexAttribArray(attribLocations.texCoord);
+    gl.vertexAttribPointer(attribLocations.texCoord, 2, gl.FLOAT, false, 0, 0);
+    const outputTexture = createRGBAFrameTexture(gl, processor.width, processor.height, true, floatFullChannelSupported);
+    const frameBuffer = createFrameBuffer(gl, outputTexture);
+    let parity = -1;
+    return {
+        render: ({ mask, foreground, background, blurredBackground, timestamp, }, options) => {
+            assert(!gl.isContextLost(), RENDERING_EVENTS.ContextLost);
+            gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer);
+            gl.viewport(0, 0, processor.width, processor.height);
+            gl.useProgram(program);
+            gl.bindVertexArray(vao);
+            // Textures
+            gl.activeTexture(gl.TEXTURE0);
+            gl.uniform1i(uniformLocations.frameForeground, 0);
+            gl.bindTexture(gl.TEXTURE_2D, foreground.texture);
+            gl.activeTexture(gl.TEXTURE1);
+            gl.uniform1i(uniformLocations.frameBackground, 1);
+            gl.bindTexture(gl.TEXTURE_2D, background.texture);
+            gl.activeTexture(gl.TEXTURE2);
+            gl.uniform1i(uniformLocations.mask, 2);
+            gl.bindTexture(gl.TEXTURE_2D, mask.texture);
+            gl.activeTexture(gl.TEXTURE3);
+            gl.uniform1i(uniformLocations.blurBackground, 3);
+            gl.bindTexture(gl.TEXTURE_2D, blurredBackground.texture);
+            gl.uniform2f(uniformLocations.confidenceThresholds, options.backgroundThreshold, options.foregroundThreshold);
+            gl.uniform1f(uniformLocations.lightWrap, options.lightWrapIntensity);
+            gl.uniform1f(uniformLocations.lightWrapTightness, options.lightWrapTightness);
+            gl.uniform1f(uniformLocations.lightWrapEdgeBand, options.lightWrapEdgeBand);
+            parity = -parity;
+            gl.uniform1f(uniformLocations.parity, parity);
+            gl.uniform1f(uniformLocations.time, 
+            // Use seconds and wrap to a modest range to avoid mediump precision loss
+            (timestamp * 1e-6) % 1024.0);
+            gl.uniform2f(uniformLocations.frameSize, processor.width, processor.height);
+            // No Flip Y Coordinate for drawing on the frame buffer
+            // Project, position and scale to full-size of the canvas
+            const matrix = m3.projection(1, 1, 1);
+            gl.uniformMatrix3fv(uniformLocations.matrix, false, matrix);
+            gl.drawArrays(gl.TRIANGLES, 0, 6);
+            // Cleanup
+            gl.bindBuffer(gl.ARRAY_BUFFER, null);
+            gl.bindFramebuffer(gl.FRAMEBUFFER, null);
+            gl.bindTexture(gl.TEXTURE_2D, null);
+            gl.bindVertexArray(null);
+            return {
+                texture: outputTexture,
+                width: processor.width,
+                height: processor.height,
+            };
+        },
+        release: () => {
+            gl.deleteBuffer(positionBuffer);
+            gl.deleteBuffer(texCoordBuffer);
+            gl.deleteTexture(outputTexture);
+            gl.deleteFramebuffer(frameBuffer);
+            gl.deleteVertexArray(vao);
+            release();
+        },
+    };
+};