@pexip/media-processor 17.3.0 → 18.0.0

package/CHANGELOG.md

@@ -1,5 +1,50 @@
 # Changelog
 
+## 18.0.0
+
+### Major Changes
+
+- bfb8058: Replace @mediapipe/selfie_segmentation with @mediapipe/tasks-vision
+
+  - Replace @mediapipe/selfie_segmentation with @mediapipe/tasks-vision
+  - Move video processing into WebWorker
+  - Replace Canvas 2D with WebGL implementation
+  - Replace Gaussian Blur with Dual filtering blur
+  - Add Segmentation Mask Smoothing
+
+### Minor Changes
+
+- 0a5a136: Fix getting the correct size of the MediaStream from a video element
+- 6b75621: Return the processing result when no output provided
+- 218a2c0: Refactor lazy props
+- 9bcfc68: Avoid processing a frame which is disposed
+- 1543a66: Allow passing Background Image Binary
+
+  - Add `update()` API to allow passing Background Image for background
+    replacement
+
+- 68e8ec1: Standardize input size and ratio
+
+### Patch Changes
+
+- Updated dependencies [bfb8058]
+  - @pexip/utils@16.11.0
+
+## 17.4.0
+
+### Minor Changes
+
+- 24fe7e1: Fix lint related errors
+
+### Patch Changes
+
+- Updated dependencies [4a1908a]
+- Updated dependencies [fec1c90]
+- Updated dependencies [ec134d1]
+- Updated dependencies [24fe7e1]
+  - @pexip/utils@16.10.0
+  - @pexip/denoise@16.4.0
+
 ## 17.3.0
 
 ### Minor Changes

package/dist/common/backends/webgl/blender.d.ts

@@ -0,0 +1,11 @@
+import type { RendererOptions } from '../../types/render';
+import type { TextureInfo } from './types';
+export declare const createBlendRenderer: (processor: OffscreenCanvas) => {
+    render: ({ mask, prevMask, foreground, background, }: {
+        prevMask?: TextureInfo | undefined;
+        mask: TextureInfo;
+        foreground: TextureInfo;
+        background: TextureInfo;
+    }, options: RendererOptions) => TextureInfo;
+    release: () => void;
+};

package/dist/common/backends/webgl/blender.js

@@ -0,0 +1,104 @@
+import { assert } from '../../utils';
+import { m3 } from '../../m3';
+import { createGLProgram, createAndSetupTexture } from './webglUtils';
+import { vertexShaderSource } from './shaders/vertexShader';
+import { blendFragmentShaderSource } from './shaders/blendFragmentShader';
+export const createBlendRenderer = (processor) => {
+    const { gl, program, attribLocations, uniformLocations, release } = createGLProgram(processor, {
+        attribs: {
+            position: 'a_position',
+            texCoord: 'a_texCoord',
+        },
+        uniforms: {
+            matrix: 'u_matrix',
+            frameBackground: 'u_frame_background',
+            frameForeground: 'u_frame_foreground',
+            mask: 'u_mask',
+            maskCombineRatio: 'u_mask_combine_ratio',
+            foregroundThreshold: 'u_foreground_threshold',
+            prevMask: 'u_mask_prev',
+        },
+        vertexSrc: vertexShaderSource,
+        fragmentSrc: blendFragmentShaderSource,
+    });
+    gl.enable(gl.BLEND);
+    const vao = gl.createVertexArray();
+    gl.bindVertexArray(vao);
+    // Position
+    const positionBuffer = gl.createBuffer();
+    gl.enableVertexAttribArray(attribLocations.position);
+    gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
+    gl.vertexAttribPointer(attribLocations.position, 2, gl.FLOAT, false, 0, 0);
+    gl.bufferData(gl.ARRAY_BUFFER, 
+    // prettier-ignore
+    new Float32Array([
+        0, 0,
+        0, 1,
+        1, 0,
+        1, 0,
+        0, 1,
+        1, 1,
+    ]), gl.STATIC_DRAW);
+    // Texture coordinates for the rectangle
+    const texCoordBuffer = gl.createBuffer();
+    gl.bindBuffer(gl.ARRAY_BUFFER, texCoordBuffer);
+    gl.bufferData(gl.ARRAY_BUFFER, 
+    // prettier-ignore
+    new Float32Array([
+        0, 0,
+        0, 1,
+        1, 0,
+        1, 0,
+        0, 1,
+        1, 1,
+    ]), gl.STATIC_DRAW);
+    gl.enableVertexAttribArray(attribLocations.texCoord);
+    gl.vertexAttribPointer(attribLocations.texCoord, 2, gl.FLOAT, false, 0, 0);
+    const outputTexture = createAndSetupTexture(gl);
+    gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, processor.width, processor.height, 0, gl.RGBA, gl.UNSIGNED_BYTE, null);
+    const frameBuffer = gl.createFramebuffer();
+    gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer);
+    gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, outputTexture, 0);
+    return {
+        render: ({ mask, prevMask, foreground, background, }, options) => {
+            gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer);
+            gl.viewport(0, 0, processor.width, processor.height);
+            gl.useProgram(program);
+            gl.bindVertexArray(vao);
+            // Textures
+            gl.activeTexture(gl.TEXTURE0);
+            gl.uniform1i(uniformLocations.frameForeground, 0);
+            gl.bindTexture(gl.TEXTURE_2D, foreground.texture);
+            gl.activeTexture(gl.TEXTURE1);
+            gl.uniform1i(uniformLocations.frameBackground, 1);
+            gl.bindTexture(gl.TEXTURE_2D, background.texture);
+            gl.activeTexture(gl.TEXTURE2);
+            gl.uniform1i(uniformLocations.mask, 2);
+            gl.bindTexture(gl.TEXTURE_2D, mask.texture);
+            gl.activeTexture(gl.TEXTURE3);
+            gl.uniform1i(uniformLocations.prevMask, 3);
+            gl.bindTexture(gl.TEXTURE_2D, prevMask?.texture ?? mask.texture);
+            // No Flip Y Coordinate for drawing on the frame buffer
+            // Project, position and scale to full-size of the canvas
+            const matrix = m3.projection(1, 1, 1);
+            gl.uniformMatrix3fv(uniformLocations.matrix, false, matrix);
+            gl.uniform1f(uniformLocations.maskCombineRatio, options.maskCombineRatio);
+            gl.uniform1f(uniformLocations.foregroundThreshold, options.foregroundThreshold);
+            gl.drawArrays(gl.TRIANGLES, 0, 6);
+            assert(outputTexture);
+            return {
+                texture: outputTexture,
+                width: processor.width,
+                height: processor.height,
+            };
+        },
+        release: () => {
+            gl.deleteBuffer(positionBuffer);
+            gl.deleteBuffer(texCoordBuffer);
+            gl.deleteTexture(outputTexture);
+            gl.deleteFramebuffer(frameBuffer);
+            gl.deleteVertexArray(vao);
+            release();
+        },
+    };
+};

package/dist/common/backends/webgl/blur.d.ts

@@ -0,0 +1,5 @@
+import type { TextureInfo } from './types';
+export declare const createBlurRenderer: (processor: OffscreenCanvas, pass: number) => {
+    render: (frame: TextureInfo, pass: number) => TextureInfo;
+    release: () => void;
+};

package/dist/common/backends/webgl/blur.js

@@ -0,0 +1,153 @@
+import { assert, clamping } from '../../utils';
+import { m3 } from '../../m3';
+import { createAndSetupTexture, createGLProgram } from './webglUtils';
+import { vertexShaderSource } from './shaders/vertexShader';
+import { dualFilterBlurFragmentShaderSource } from './shaders/dualFilterBlurFragmentShader';
+const clampBlurAmount = clamping(1, 9);
+export const createBlurRenderer = (processor, pass) => {
+    const { gl, program, attribLocations, uniformLocations, release } = createGLProgram(processor, {
+        attribs: {
+            position: 'a_position',
+            texCoord: 'a_texCoord',
+        },
+        uniforms: {
+            matrix: 'u_matrix',
+            downsampling: 'u_downsampling',
+            frame: 'u_frame',
+        },
+        vertexSrc: vertexShaderSource,
+        fragmentSrc: dualFilterBlurFragmentShaderSource,
+    });
+    const vao = gl.createVertexArray();
+    gl.bindVertexArray(vao);
+    // Position
+    const positionBuffer = gl.createBuffer();
+    gl.enableVertexAttribArray(attribLocations.position);
+    gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
+    gl.vertexAttribPointer(attribLocations.position, 2, gl.FLOAT, false, 0, 0);
+    // Put a unit quad in the buffer
+    gl.bufferData(gl.ARRAY_BUFFER, 
+    // prettier-ignore
+    new Float32Array([
+        0, 0,
+        0, 1,
+        1, 0,
+        1, 0,
+        0, 1,
+        1, 1,
+    ]), gl.STATIC_DRAW);
+    // Texture coordinates for the rectangle
+    const texCoordBuffer = gl.createBuffer();
+    gl.bindBuffer(gl.ARRAY_BUFFER, texCoordBuffer);
+    gl.bufferData(gl.ARRAY_BUFFER, 
+    // prettier-ignore
+    new Float32Array([
+        0, 0,
+        0, 1,
+        1, 0,
+        1, 0,
+        0, 1,
+        1, 1,
+    ]), gl.STATIC_DRAW);
+    gl.enableVertexAttribArray(attribLocations.texCoord);
+    gl.vertexAttribPointer(attribLocations.texCoord, 2, gl.FLOAT, false, 0, 0);
+    const setFrameBuffer = (frameBuffer, width, height) => {
+        gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer);
+        gl.viewport(0, 0, width, height);
+    };
+    const createFramebufferAndTexture = (idx) => {
+        const texture = createAndSetupTexture(gl);
+        assert(texture);
+        // Avoid fractional value
+        const width = Math.trunc(processor.width / Math.pow(2, idx));
+        const height = Math.trunc(processor.height / Math.pow(2, idx));
+        gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, width, height, 0, gl.RGBA, gl.UNSIGNED_BYTE, null);
+        const frameBuffer = gl.createFramebuffer();
+        assert(frameBuffer);
+        gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer);
+        // Attach a texture to it.
+        const attachmentPoint = gl.COLOR_ATTACHMENT0;
+        gl.framebufferTexture2D(gl.FRAMEBUFFER, attachmentPoint, gl.TEXTURE_2D, texture, 0);
+        return { texture, frameBuffer, width, height };
+    };
+    const generateBlurBuffers = (pass, offset = 0) => {
+        const length = Math.max(clampBlurAmount(offset + pass) - offset, 0);
+        return Array.from({
+            length,
+        }).map((_, currentIdx) => createFramebufferAndTexture(offset + currentIdx));
+    };
+    // Generate multiple FrameBuffers, textures and the widths and heights are of size
+    // of half of the previous one. E.g. 640x480 -> 320x240 -> 160x120 -> 80x60
+    const blurBuffers = generateBlurBuffers(pass + 1);
+    return {
+        render: (frame, pass) => {
+            gl.useProgram(program);
+            gl.bindVertexArray(vao);
+            // Frame texture
+            gl.activeTexture(gl.TEXTURE0);
+            gl.uniform1i(uniformLocations.frame, 0);
+            gl.bindTexture(gl.TEXTURE_2D, frame.texture);
+            // Dual filter blur
+            // Downsampling
+            gl.uniform1i(uniformLocations.downsampling, 1);
+            // Adjust pass number accordingly
+            const currentPass = pass + 1;
+            if (currentPass > blurBuffers.length) {
+                generateBlurBuffers(currentPass - blurBuffers.length, blurBuffers.length).forEach(buffer => blurBuffers.push(buffer));
+            }
+            while (currentPass < blurBuffers.length) {
+                const buffer = blurBuffers.pop();
+                gl.deleteTexture(buffer?.texture ?? null);
+                gl.deleteFramebuffer(buffer?.frameBuffer ?? null);
+            }
+            // Skip the first pass
+            for (let i = 1; i < currentPass; i++) {
+                const { frameBuffer = null, texture = null, width = processor.width, height = processor.height, } = blurBuffers[i] ?? {};
+                // Setup to draw to the frame buffer
+                setFrameBuffer(frameBuffer, width, height);
+                const matrix = m3.projection(1, 1, 1);
+                gl.uniformMatrix3fv(uniformLocations.matrix, false, matrix);
+                gl.drawArrays(gl.TRIANGLES, 0, 6);
+                const status = gl.checkFramebufferStatus(gl.FRAMEBUFFER);
+                if (status !== gl.FRAMEBUFFER_COMPLETE) {
+                    throw new Error(`downsampling FrameBuffer Status: ${status}`);
+                }
+                // for the next draw, use the texture we just rendered to.
+                gl.bindTexture(gl.TEXTURE_2D, texture);
+            }
+            // Upsampling
+            gl.uniform1i(uniformLocations.downsampling, 0);
+            // Skip the first pass
+            for (let i = pass - 1; i >= 0; i--) {
+                const { frameBuffer = null, texture = null, width = processor.width, height = processor.height, } = blurBuffers[i] ?? {};
+                // console.log(`Drawing upsampling pass ${i}, ${width}x${height}`);
+                // Setup to draw to the frame buffer
+                setFrameBuffer(frameBuffer, width, height);
+                const matrix = m3.projection(1, 1, 1);
+                gl.uniformMatrix3fv(uniformLocations.matrix, false, matrix);
+                gl.drawArrays(gl.TRIANGLES, 0, 6);
+                const status = gl.checkFramebufferStatus(gl.FRAMEBUFFER);
+                if (status !== gl.FRAMEBUFFER_COMPLETE) {
+                    throw new Error(`Upsampling FrameBuffer Status: ${status}`);
+                }
+                // for the next draw, use the texture we just rendered to.
+                gl.bindTexture(gl.TEXTURE_2D, texture);
+            }
+            return {
+                texture: blurBuffers[0]?.texture ?? frame.texture,
+                width: blurBuffers[0]?.width ?? frame.width,
+                height: blurBuffers[0]?.height ?? frame.height,
+            };
+        },
+        release: () => {
+            blurBuffers.forEach(({ frameBuffer, texture }) => {
+                gl.deleteTexture(texture);
+                gl.deleteFramebuffer(frameBuffer);
+            });
+            gl.deleteBuffer(positionBuffer);
+            gl.deleteBuffer(texCoordBuffer);
+            gl.deleteVertexArray(vao);
+            release();
+        },
+    };
+};

package/dist/common/backends/webgl/canvas.d.ts

@@ -0,0 +1,5 @@
+import type { TextureInfo } from './types';
+export declare const createCanvasRenderer: (processor: OffscreenCanvas) => {
+    render: (texture: TextureInfo) => void;
+    release: () => void;
+};

package/dist/common/backends/webgl/canvas.js

@@ -0,0 +1,77 @@
+import { m3 } from '../../m3';
+import { createGLProgram } from './webglUtils';
+import { vertexShaderSource } from './shaders/vertexShader';
+import { textureFragmentShaderSource } from './shaders/textureFragmentShader';
+export const createCanvasRenderer = (processor) => {
+    const { gl, program, attribLocations, uniformLocations, release } = createGLProgram(processor, {
+        attribs: {
+            position: 'a_position',
+            texCoord: 'a_texCoord',
+        },
+        uniforms: {
+            matrix: 'u_matrix',
+            texture: 'u_texture',
+        },
+        vertexSrc: vertexShaderSource,
+        fragmentSrc: textureFragmentShaderSource,
+    });
+    const vao = gl.createVertexArray();
+    gl.bindVertexArray(vao);
+    // Position
+    const positionBuffer = gl.createBuffer();
+    gl.enableVertexAttribArray(attribLocations.position);
+    gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
+    gl.vertexAttribPointer(attribLocations.position, 2, gl.FLOAT, false, 0, 0);
+    // Put a unit quad in the buffer
+    gl.bufferData(gl.ARRAY_BUFFER, 
+    // prettier-ignore
+    new Float32Array([
+        0, 0,
+        0, 1,
+        1, 0,
+        1, 0,
+        0, 1,
+        1, 1,
+    ]), gl.STATIC_DRAW);
+    // Texture coordinates for the rectangle
+    const texCoordBuffer = gl.createBuffer();
+    gl.bindBuffer(gl.ARRAY_BUFFER, texCoordBuffer);
+    gl.bufferData(gl.ARRAY_BUFFER, 
+    // prettier-ignore
+    new Float32Array([
+        0, 0,
+        0, 1,
+        1, 0,
+        1, 0,
+        0, 1,
+        1, 1,
+    ]), gl.STATIC_DRAW);
+    gl.enableVertexAttribArray(attribLocations.texCoord);
+    gl.vertexAttribPointer(attribLocations.texCoord, 2, gl.FLOAT, false, 0, 0);
+    return {
+        render: (texture) => {
+            // Draw on the Canvas
+            gl.bindFramebuffer(gl.FRAMEBUFFER, null);
+            gl.viewport(0, 0, processor.width, processor.height);
+            gl.clearColor(0, 0, 0, 1);
+            gl.clear(gl.COLOR_BUFFER_BIT);
+            gl.useProgram(program);
+            gl.bindVertexArray(vao);
+            // Texture Unit setup
+            gl.uniform1i(uniformLocations.texture, 0);
+            // Project, position and scale to full-size of the canvas
+            const matrix = m3.projection(1, 1);
+            gl.uniformMatrix3fv(uniformLocations.matrix, false, matrix);
+            // Frame texture
+            gl.activeTexture(gl.TEXTURE0);
+            gl.bindTexture(gl.TEXTURE_2D, texture.texture);
+            gl.drawArrays(gl.TRIANGLES, 0, 6);
+        },
+        release: () => {
+            gl.deleteBuffer(positionBuffer);
+            gl.deleteBuffer(texCoordBuffer);
+            gl.deleteVertexArray(vao);
+            release();
+        },
+    };
+};

package/dist/common/backends/webgl/renderer.d.ts

@@ -0,0 +1,8 @@
+import type { Renderer, RendererOptions } from '../../types/render';
+import type { ImageMapIterable } from '../../types/media';
+interface Options extends Partial<RendererOptions> {
+    backgroundImageInits?: ImageMapIterable;
+    onError?: (error: Error) => void;
+}
+export declare const createRenderer: (canvas?: OffscreenCanvas, { backgroundBlurAmount, backgroundImageUrl, edgeBlurAmount, effects, foregroundThreshold, maskCombineRatio, onError, backgroundImageInits, }?: Options) => Renderer;
+export {};

package/dist/common/backends/webgl/renderer.js

@@ -0,0 +1,126 @@
+import { PROCESSING_WIDTH, PROCESSING_HEIGHT, EDGE_BLUR_AMOUNT, BACKGROUND_BLUR_AMOUNT, FOREGROUND_THRESHOLD, MASK_COMBINE_RATIO, RENDER_EFFECT, } from '../../constants';
+import { createObjectUpdater, createRemoteImageBitmap } from '../../utils';
+import { toTextureInfo } from './webglUtils';
+import { createTextureRenderer } from './texture';
+import { createBlurRenderer } from './blur';
+import { createBlendRenderer } from './blender';
+import { createCanvasRenderer } from './canvas';
+const createLazyProps = (stateBuilder) => {
+    const props = {};
+    return new Proxy(props, {
+        get(target, p, receiver) {
+            const prop = target[p];
+            if (!prop) {
+                target[p] = stateBuilder[p]?.(props);
+            }
+            return Reflect.get(target, p, receiver);
+        },
+    });
+};
+export const createRenderer = (canvas = new OffscreenCanvas(PROCESSING_WIDTH, PROCESSING_HEIGHT), { backgroundBlurAmount = BACKGROUND_BLUR_AMOUNT, backgroundImageUrl = '', edgeBlurAmount = EDGE_BLUR_AMOUNT, effects = RENDER_EFFECT.None, foregroundThreshold = FOREGROUND_THRESHOLD, maskCombineRatio = MASK_COMBINE_RATIO, onError, backgroundImageInits, } = {}) => {
+    const rendererCreator = {
+        videoFrameRenderer: () => createTextureRenderer(canvas),
+        canvasRenderer: () => createCanvasRenderer(canvas),
+        backgroundImageRenderer: () => createTextureRenderer(canvas),
+        blendRenderer: () => createBlendRenderer(canvas),
+        blurRenderer: () => createBlurRenderer(canvas, props.backgroundBlurAmount),
+        maskBlurRenderer: () => createBlurRenderer(canvas, props.edgeBlurAmount),
+        release: (states) => () => Object.keys(states).forEach(key => key !== 'release' &&
+            states[key]?.release()),
+    };
+    let lazyProps = createLazyProps(rendererCreator);
+    const props = {
+        backgroundBlurAmount,
+        backgroundImageUrl,
+        edgeBlurAmount,
+        effects,
+        foregroundThreshold,
+        maskCombineRatio,
+        canvas,
+        backgroundImages: createRemoteImageBitmap(backgroundImageInits),
+        // TODO: refactor to avoid repeated codes
+        get videoFrameRenderer() {
+            return lazyProps.videoFrameRenderer;
+        },
+        get canvasRenderer() {
+            return lazyProps.canvasRenderer;
+        },
+        get backgroundImageRenderer() {
+            return lazyProps.backgroundImageRenderer;
+        },
+        get blendRenderer() {
+            return lazyProps.blendRenderer;
+        },
+        get blurRenderer() {
+            return lazyProps.blurRenderer;
+        },
+        // TODO: Impl edge blur
+        get maskBlurRenderer() {
+            return lazyProps.maskBlurRenderer;
+        },
+    };
+    const updateProps = createObjectUpdater(props);
+    const update = options => {
+        if (options.backgroundImage) {
+            props.backgroundImages.setImage(options.backgroundImage.key, options.backgroundImage.image);
+        }
+        updateProps(options);
+    };
+    const handleError = (error) => {
+        if (error instanceof Error) {
+            return onError?.(error);
+        }
+        throw error;
+    };
+    const init = options => {
+        if (options) {
+            update(options);
+        }
+    };
+    const render = (mask, frame, options) => {
+        if (options) {
+            updateProps(options);
+        }
+        const frameTexture = props.videoFrameRenderer.render(frame);
+        let background = frameTexture;
+        if (props.effects === 'blur' && props.backgroundBlurAmount) {
+            background = props.blurRenderer.render(frameTexture, props.backgroundBlurAmount);
+        }
+        else if (props?.effects === 'overlay' && props.backgroundImageUrl) {
+            // Keep rendering with video frame until we get the background image
+            const backgroundImage = props.backgroundImages.getImage(props.backgroundImageUrl);
+            if (backgroundImage) {
+                background = props.backgroundImageRenderer.render({
+                    frame: backgroundImage,
+                    timestamp: performance.now(),
+                    width: backgroundImage.width,
+                    height: backgroundImage.height,
+                });
+            }
+            else {
+                props.backgroundImages
+                    .tryFetchingImage(props.backgroundImageUrl)
+                    .catch(error => {
+                    handleError(new Error(`FailedToLoadBackgroundImage: ${props.backgroundImageUrl}`, { cause: error }));
+                });
+            }
+        }
+        const blendedFrameTexture = props.blendRenderer.render({
+            prevMask: props.prevMask && toTextureInfo(props.prevMask),
+            mask: toTextureInfo(mask),
+            foreground: frameTexture,
+            background,
+        }, props);
+        props.canvasRenderer.render(blendedFrameTexture);
+        props.prevMask?.close();
+        props.prevMask = mask.clone();
+    };
+    const release = () => {
+        lazyProps.release();
+        lazyProps = createLazyProps(rendererCreator);
+        props.backgroundImages.clear();
+        props.prevMask?.close();
+        props.prevMask = undefined;
+    };
+    return { init, render, release, update };
+};

package/dist/common/backends/webgl/shaders/blendFragmentShader.d.ts

@@ -0,0 +1 @@
+export declare const blendFragmentShaderSource = "#version 300 es\nprecision mediump float;\n\n// our textures\nuniform sampler2D u_mask;\nuniform sampler2D u_mask_prev;\nuniform sampler2D u_frame_foreground;\nuniform sampler2D u_frame_background;\n\n// control flags\nuniform float u_mask_combine_ratio;\nuniform float u_foreground_threshold;\n\nin vec2 v_texCoord;\n\n// we need to declare an output for the fragment shader\nout vec4 outColor;\n\n// Source: https://github.com/google/mediapipe/blob/8750c3dca1bc00fbe47acf501b7c9c61f3f3bd3f/mediapipe/calculators/image/segmentation_smoothing_calculator.cc#L377\nfloat SmoothingMask(sampler2D prevMask, sampler2D currMask, vec2 uv, float ratio) {\n    float prevMaskValue = texture(prevMask, uv).r;\n    float newMaskValue = texture(currMask, uv).r;\n\n    /*\n    * Assume p := newMaskValue\n    * H(p) := 1 + (p * log(p) + (1-p) * log(1-p)) / log(2)\n    * uncertainty alpha(p) =\n    *   Clamp(1 - (1 - H(p)) * (1 - H(p)), 0, 1) [squaring the\n    * uncertainty]\n    *\n    * The following polynomial approximates uncertainty alpha as a\n    * function of (p + 0.5):\n    */\n    const float c1 = 5.68842;\n    const float c2 = -0.748699;\n    const float c3 = -57.8051;\n    const float c4 = 291.309;\n    const float c5 = -624.717;\n    float t = newMaskValue - 0.5;\n    float x = t * t;\n\n    float uncertainty =\n    1.0 - min(1.0, x * (c1 + x * (c2 + x * (c3 + x * (c4 + x * c5)))));\n\n    float outputValue = newMaskValue + (prevMaskValue - newMaskValue) *\n                            (uncertainty * ratio);\n    return outputValue;\n}\n\nvoid main() {\n    vec2 halfpixel = 0.5 / vec2(textureSize(u_frame_background, 0));\n\n    vec4 background = texture(u_frame_background, v_texCoord);\n    vec4 foreground = texture(u_frame_foreground, v_texCoord);\n    float maskAlpha = SmoothingMask(u_mask_prev, u_mask, v_texCoord, u_mask_combine_ratio);\n    float filteredMask = maskAlpha;\n    if (maskAlpha < u_foreground_threshold) {\n        filteredMask = 0.0;\n    }\n\n    outColor = mix(background, foreground, filteredMask);\n}\n";

package/dist/common/backends/webgl/shaders/blendFragmentShader.js

@@ -0,0 +1,63 @@
+export const blendFragmentShaderSource = `#version 300 es
+precision mediump float;
+
+// our textures
+uniform sampler2D u_mask;
+uniform sampler2D u_mask_prev;
+uniform sampler2D u_frame_foreground;
+uniform sampler2D u_frame_background;
+
+// control flags
+uniform float u_mask_combine_ratio;
+uniform float u_foreground_threshold;
+
+in vec2 v_texCoord;
+
+// we need to declare an output for the fragment shader
+out vec4 outColor;
+
+// Source: https://github.com/google/mediapipe/blob/8750c3dca1bc00fbe47acf501b7c9c61f3f3bd3f/mediapipe/calculators/image/segmentation_smoothing_calculator.cc#L377
+float SmoothingMask(sampler2D prevMask, sampler2D currMask, vec2 uv, float ratio) {
+    float prevMaskValue = texture(prevMask, uv).r;
+    float newMaskValue = texture(currMask, uv).r;
+
+    /*
+    * Assume p := newMaskValue
+    * H(p) := 1 + (p * log(p) + (1-p) * log(1-p)) / log(2)
+    * uncertainty alpha(p) =
+    *   Clamp(1 - (1 - H(p)) * (1 - H(p)), 0, 1) [squaring the
+    * uncertainty]
+    *
+    * The following polynomial approximates uncertainty alpha as a
+    * function of (p + 0.5):
+    */
+    const float c1 = 5.68842;
+    const float c2 = -0.748699;
+    const float c3 = -57.8051;
+    const float c4 = 291.309;
+    const float c5 = -624.717;
+    float t = newMaskValue - 0.5;
+    float x = t * t;
+
+    float uncertainty =
+    1.0 - min(1.0, x * (c1 + x * (c2 + x * (c3 + x * (c4 + x * c5)))));
+
+    float outputValue = newMaskValue + (prevMaskValue - newMaskValue) *
+                            (uncertainty * ratio);
+    return outputValue;
+}
+
+void main() {
+    vec2 halfpixel = 0.5 / vec2(textureSize(u_frame_background, 0));
+
+    vec4 background = texture(u_frame_background, v_texCoord);
+    vec4 foreground = texture(u_frame_foreground, v_texCoord);
+    float maskAlpha = SmoothingMask(u_mask_prev, u_mask, v_texCoord, u_mask_combine_ratio);
+    float filteredMask = maskAlpha;
+    if (maskAlpha < u_foreground_threshold) {
+        filteredMask = 0.0;
+    }
+
+    outColor = mix(background, foreground, filteredMask);
+}
+`;

package/dist/common/backends/webgl/shaders/dualFilterBlurFragmentShader.d.ts

@@ -0,0 +1 @@
+export declare const dualFilterBlurFragmentShaderSource = "#version 300 es\nprecision mediump float;\n\n// our textures\nuniform sampler2D u_frame;\n\n// control flags\nuniform int u_downsampling;\n\nin vec2 v_texCoord;\n\n// we need to declare an output for the fragment shader\nout vec4 outColor;\n\nvec4 Downsample(sampler2D image, vec2 uv, vec2 halfpixel) {\n    vec4 sum = texture(image, uv) * 4.0;\n    sum += texture(image, uv - halfpixel.xy);\n    sum += texture(image, uv + halfpixel.xy);\n    sum += texture(image, uv + vec2(halfpixel.x, -halfpixel.y));\n    sum += texture(image, uv - vec2(halfpixel.x, -halfpixel.y));\n    return sum / 8.0;\n}\n\nvec4 Upsample(sampler2D image, vec2 uv, vec2 halfpixel) {\n    vec4 sum = texture(image, uv + vec2(-halfpixel.x * 2.0, 0.0));\n    sum += texture(image, uv + vec2(-halfpixel.x, halfpixel.y)) * 2.0;\n    sum += texture(image, uv + vec2(0.0, halfpixel.y * 2.0));\n    sum += texture(image, uv + vec2(halfpixel.x, halfpixel.y)) * 2.0;\n    sum += texture(image, uv + vec2(halfpixel.x * 2.0, 0.0));\n    sum += texture(image, uv + vec2(halfpixel.x, -halfpixel.y)) * 2.0;\n    sum += texture(image, uv + vec2(0.0, -halfpixel.y * 2.0));\n    sum += texture(image, uv + vec2(-halfpixel.x, -halfpixel.y)) * 2.0;\n    return sum / 12.0;\n}\n\nvoid main() {\n    vec2 halfpixel = 0.5 / vec2(textureSize(u_frame, 0));\n\n    if (u_downsampling == 1) {\n        outColor = Downsample(u_frame, v_texCoord, halfpixel);\n        return;\n    }\n    outColor = Upsample(u_frame, v_texCoord, halfpixel);\n}\n";

package/dist/common/backends/webgl/shaders/dualFilterBlurFragmentShader.js

@@ -0,0 +1,45 @@
+export const dualFilterBlurFragmentShaderSource = `#version 300 es
+precision mediump float;
+
+// our textures
+uniform sampler2D u_frame;
+
+// control flags
+uniform int u_downsampling;
+
+in vec2 v_texCoord;
+
+// we need to declare an output for the fragment shader
+out vec4 outColor;
+
+vec4 Downsample(sampler2D image, vec2 uv, vec2 halfpixel) {
+    vec4 sum = texture(image, uv) * 4.0;
+    sum += texture(image, uv - halfpixel.xy);
+    sum += texture(image, uv + halfpixel.xy);
+    sum += texture(image, uv + vec2(halfpixel.x, -halfpixel.y));
+    sum += texture(image, uv - vec2(halfpixel.x, -halfpixel.y));
+    return sum / 8.0;
+}
+
+vec4 Upsample(sampler2D image, vec2 uv, vec2 halfpixel) {
+    vec4 sum = texture(image, uv + vec2(-halfpixel.x * 2.0, 0.0));
+    sum += texture(image, uv + vec2(-halfpixel.x, halfpixel.y)) * 2.0;
+    sum += texture(image, uv + vec2(0.0, halfpixel.y * 2.0));
+    sum += texture(image, uv + vec2(halfpixel.x, halfpixel.y)) * 2.0;
+    sum += texture(image, uv + vec2(halfpixel.x * 2.0, 0.0));
+    sum += texture(image, uv + vec2(halfpixel.x, -halfpixel.y)) * 2.0;
+    sum += texture(image, uv + vec2(0.0, -halfpixel.y * 2.0));
+    sum += texture(image, uv + vec2(-halfpixel.x, -halfpixel.y)) * 2.0;
+    return sum / 12.0;
+}
+
+void main() {
+    vec2 halfpixel = 0.5 / vec2(textureSize(u_frame, 0));
+
+    if (u_downsampling == 1) {
+        outColor = Downsample(u_frame, v_texCoord, halfpixel);
+        return;
+    }
+    outColor = Upsample(u_frame, v_texCoord, halfpixel);
+}
+`;

package/dist/common/backends/webgl/shaders/textureFragmentShader.d.ts

@@ -0,0 +1 @@
+export declare const textureFragmentShaderSource = "#version 300 es\nprecision mediump float;\n\n// Passed in from the vertex shader.\nin vec2 v_texCoord;\n\n// The texture.\nuniform sampler2D u_texture;\n\n// we need to declare an output for the fragment shader\nout vec4 outColor;\n\nvoid main() {\n  outColor = texture(u_texture, v_texCoord);\n}\n";