@pexip/media-processor 20.3.4 → 21.0.0

package/dist/common/backends/webgl/tentBlur.js

@@ -0,0 +1,118 @@
+import { assert } from '@pexip/utils';
+import { m3 } from '../../m3';
+import { RENDERING_EVENTS } from '../../constants';
+import { clamping } from '../../utils';
+import { createGLProgram, createMaskTexture, createFrameBuffer, } from './webglUtils';
+import vertexSrc from './shaders/matrix.vert';
+import fragmentSrc from './shaders/tentBlur.frag';
+const MAX_RADIUS_PX = 16;
+const clampRadiusPx = clamping(0, MAX_RADIUS_PX);
+/**
+ * Create a tent blur renderer for small radius blurring of a mask.
+ * The blur is implemented as a separable filter.
+ */
+export const createTentBlurRenderer = (processor, width, height) => {
+    const { gl, program, attribLocations, uniformLocations, floatSingleChannelSupported, floatFullChannelSupported, release, } = createGLProgram(processor, {
+        attribs: {
+            position: 'a_position',
+            texCoord: 'a_texCoord',
+        },
+        uniforms: {
+            matrix: 'u_matrix',
+            mask: 'u_mask',
+            texelSize: 'u_texel_size',
+            direction: 'u_direction',
+            radiusPx: 'u_radius_px',
+        },
+        vertexSrc,
+        fragmentSrc,
+    });
+    const vao = gl.createVertexArray();
+    gl.bindVertexArray(vao);
+    // Position
+    const positionBuffer = gl.createBuffer();
+    gl.enableVertexAttribArray(attribLocations.position);
+    gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
+    gl.vertexAttribPointer(attribLocations.position, 2, gl.FLOAT, false, 0, 0);
+    gl.bufferData(gl.ARRAY_BUFFER, 
+    // prettier-ignore
+    new Float32Array([
+        0, 0,
+        0, 1,
+        1, 0,
+        1, 0,
+        0, 1,
+        1, 1,
+    ]), gl.STATIC_DRAW);
+    // Texture coordinates for the rectangle
+    const texCoordBuffer = gl.createBuffer();
+    gl.bindBuffer(gl.ARRAY_BUFFER, texCoordBuffer);
+    gl.bufferData(gl.ARRAY_BUFFER, 
+    // prettier-ignore
+    new Float32Array([
+        0, 0,
+        0, 1,
+        1, 0,
+        1, 0,
+        0, 1,
+        1, 1,
+    ]), gl.STATIC_DRAW);
+    gl.enableVertexAttribArray(attribLocations.texCoord);
+    gl.vertexAttribPointer(attribLocations.texCoord, 2, gl.FLOAT, false, 0, 0);
+    const outputTexturePing = createMaskTexture(gl, width, height, false, floatSingleChannelSupported, floatFullChannelSupported);
+    const frameBufferPing = createFrameBuffer(gl, outputTexturePing);
+    const outputTexturePong = createMaskTexture(gl, width, height, false, floatSingleChannelSupported, floatFullChannelSupported);
+    const frameBufferPong = createFrameBuffer(gl, outputTexturePong);
+    const scaleFactor = processor.width / width;
+    return {
+        render: (mask, radiusPx) => {
+            assert(!gl.isContextLost(), RENDERING_EVENTS.ContextLost);
+            gl.clearColor(0, 0, 0, 0);
+            gl.clear(gl.COLOR_BUFFER_BIT);
+            gl.useProgram(program);
+            gl.bindVertexArray(vao);
+            gl.uniform2f(uniformLocations.texelSize, 1 / mask.width, 1 / mask.height);
+            gl.uniform1i(uniformLocations.radiusPx, clampRadiusPx(Math.floor(radiusPx / scaleFactor)));
+            const matrix = m3.projection(1, 1, 1);
+            // No Flip Y Coordinate for drawing on the frame buffer
+            // Project, position and scale to full-size of the canvas
+            gl.uniformMatrix3fv(uniformLocations.matrix, false, matrix);
+            let write = frameBufferPing;
+            let read = mask.texture;
+            // Ping-pong framebuffers for multiple passes
+            // Separable
+            for (let j = 0; j < 2; j++) {
+                // Textures
+                gl.activeTexture(gl.TEXTURE0);
+                gl.uniform1i(uniformLocations.mask, 0);
+                gl.bindTexture(gl.TEXTURE_2D, read);
+                // Horizontal then Vertical
+                gl.uniform2f(uniformLocations.direction, 1.0 - j, 0.0 + j);
+                gl.bindFramebuffer(gl.FRAMEBUFFER, write);
+                gl.viewport(0, 0, mask.width, mask.height);
+                gl.drawArrays(gl.TRIANGLES, 0, 6);
+                // Swap
+                [read, write] =
+                    write === frameBufferPing
+                        ? [outputTexturePing, frameBufferPong]
+                        : [outputTexturePong, frameBufferPing];
+            }
+            // Cleanup
+            gl.bindBuffer(gl.ARRAY_BUFFER, null);
+            gl.bindFramebuffer(gl.FRAMEBUFFER, null);
+            gl.bindTexture(gl.TEXTURE_2D, null);
+            gl.bindVertexArray(null);
+            return { texture: read, width, height };
+        },
+        release: () => {
+            gl.deleteBuffer(positionBuffer);
+            gl.deleteBuffer(texCoordBuffer);
+            gl.deleteTexture(outputTexturePing);
+            gl.deleteTexture(outputTexturePong);
+            gl.deleteFramebuffer(frameBufferPing);
+            gl.deleteFramebuffer(frameBufferPong);
+            gl.deleteVertexArray(vao);
+            release();
+        },
+    };
+};

package/dist/common/backends/webgl/texture.d.ts

@@ -3,7 +3,7 @@ import type { TextureInfo } from './types';
 /**
  * Render Video frame into texture
  */
-export declare const createTextureRenderer: (processor: OffscreenCanvas) => {
+export declare const createTextureRenderer: (processor: OffscreenCanvas, redOnly: boolean) => {
     render: (video: VideoFrameLike) => TextureInfo;
     release: () => void;
 };

package/dist/common/backends/webgl/texture.js

@@ -1,14 +1,14 @@
 import { assert } from '@pexip/utils';
 import { RENDERING_EVENTS } from '../../constants';
 import { m3 } from '../../m3';
-import { createAndSetupTexture, createGLProgram } from './webglUtils';
-import { vertexShaderSource } from './shaders/vertexShader';
-import { textureFragmentShaderSource } from './shaders/textureFragmentShader';
+import { createFrameBuffer, createGLProgram, createMaskTexture, createRGBAFrameTexture, } from './webglUtils';
+import vertexSrc from './shaders/matrix.vert';
+import fragmentSrc from './shaders/texture.frag';
 /**
  * Render Video frame into texture
  */
-export const createTextureRenderer = (processor) => {
-    const { gl, program, attribLocations, uniformLocations, release } = createGLProgram(processor, {
+export const createTextureRenderer = (processor, redOnly) => {
+    const { gl, program, attribLocations, uniformLocations, floatFullChannelSupported, floatSingleChannelSupported, release, } = createGLProgram(processor, {
         attribs: {
             position: 'a_position',
             texCoord: 'a_texCoord',
@@ -17,8 +17,8 @@ export const createTextureRenderer = (processor) => {
             matrix: 'u_matrix',
             texture: 'u_texture',
         },
-        vertexSrc: vertexShaderSource,
-        fragmentSrc: textureFragmentShaderSource,
+        vertexSrc,
+        fragmentSrc,
     });
     const vao = gl.createVertexArray();
     gl.bindVertexArray(vao);
@@ -53,12 +53,11 @@ export const createTextureRenderer = (processor) => {
     ]), gl.STATIC_DRAW);
     gl.enableVertexAttribArray(attribLocations.texCoord);
     gl.vertexAttribPointer(attribLocations.texCoord, 2, gl.FLOAT, false, 0, 0);
-    const frameTexture = createAndSetupTexture(gl);
-    const outputTexture = createAndSetupTexture(gl);
-    gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, processor.width, processor.height, 0, gl.RGBA, gl.UNSIGNED_BYTE, null);
-    const frameBuffer = gl.createFramebuffer();
-    gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer);
-    gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, outputTexture, 0);
+    const frameTexture = createRGBAFrameTexture(gl, processor.width, processor.height, true, floatFullChannelSupported);
+    const outputTexture = redOnly
+        ? createMaskTexture(gl, processor.width, processor.height, false, floatSingleChannelSupported, floatFullChannelSupported)
+        : createRGBAFrameTexture(gl, processor.width, processor.height, true, floatFullChannelSupported);
+    const frameBuffer = createFrameBuffer(gl, outputTexture);
     return {
         render: (video) => {
             assert(!gl.isContextLost(), RENDERING_EVENTS.ContextLost);
@@ -71,16 +70,19 @@ export const createTextureRenderer = (processor) => {
             gl.bindVertexArray(vao);
             // Texture Unit setup
             gl.uniform1i(uniformLocations.texture, 0);
-            // Project, position and scale to full-size of the canvas
+            // Project, position, and scale to full-size of the canvas
             const matrix = m3.projection(1, 1, 1);
             gl.uniformMatrix3fv(uniformLocations.matrix, false, matrix);
-            // Frame texture
+            // Upload the frame to texture
             gl.activeTexture(gl.TEXTURE0);
             gl.bindTexture(gl.TEXTURE_2D, frameTexture);
             gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, video.frame);
-            gl.generateMipmap(gl.TEXTURE_2D);
             gl.drawArrays(gl.TRIANGLES, 0, 6);
-            assert(outputTexture);
+            // Cleanup
+            gl.bindBuffer(gl.ARRAY_BUFFER, null);
+            gl.bindFramebuffer(gl.FRAMEBUFFER, null);
+            gl.bindTexture(gl.TEXTURE_2D, null);
+            gl.bindVertexArray(null);
             return {
                 texture: outputTexture,
                 width: processor.width,

package/dist/common/backends/webgl/textureToTexture.d.ts

@@ -2,7 +2,7 @@ import type { TextureInfo } from './types';
 /**
  * Render texture into another texture, i.e. clone
  */
-export declare const createTextureToTextureRenderer: (processor: OffscreenCanvas) => {
+export declare const createTextureToTextureRenderer: (processor: OffscreenCanvas, redOnly: boolean, linearSampling?: boolean, targetWidth?: number, targetHeight?: number, float?: boolean) => {
     render: (inputTexture: TextureInfo) => TextureInfo;
     release: () => void;
 };

package/dist/common/backends/webgl/textureToTexture.js

@@ -1,14 +1,14 @@
 import { assert } from '@pexip/utils';
 import { RENDERING_EVENTS } from '../../constants';
 import { m3 } from '../../m3';
-import { createAndSetupTexture, createGLProgram } from './webglUtils';
-import { vertexShaderSource } from './shaders/vertexShader';
-import { textureFragmentShaderSource } from './shaders/textureFragmentShader';
+import { createFrameBuffer, createGLProgram, createMaskTexture, createRGBAFrameTexture, } from './webglUtils';
+import vertexSrc from './shaders/matrix.vert';
+import fragmentSrc from './shaders/texture.frag';
 /**
  * Render texture into another texture, i.e. clone
  */
-export const createTextureToTextureRenderer = (processor) => {
-    const { gl, program, attribLocations, uniformLocations, release } = createGLProgram(processor, {
+export const createTextureToTextureRenderer = (processor, redOnly, linearSampling = false, targetWidth = processor.width, targetHeight = processor.height, float = true) => {
+    const { gl, program, attribLocations, uniformLocations, floatFullChannelSupported, floatSingleChannelSupported, release, } = createGLProgram(processor, {
         attribs: {
             position: 'a_position',
             texCoord: 'a_texCoord',
@@ -17,8 +17,8 @@ export const createTextureToTextureRenderer = (processor) => {
             matrix: 'u_matrix',
             texture: 'u_texture',
         },
-        vertexSrc: vertexShaderSource,
-        fragmentSrc: textureFragmentShaderSource,
+        vertexSrc,
+        fragmentSrc,
     });
     const vao = gl.createVertexArray();
     gl.bindVertexArray(vao);
@@ -53,17 +53,16 @@ export const createTextureToTextureRenderer = (processor) => {
     ]), gl.STATIC_DRAW);
     gl.enableVertexAttribArray(attribLocations.texCoord);
     gl.vertexAttribPointer(attribLocations.texCoord, 2, gl.FLOAT, false, 0, 0);
-    const outputTexture = createAndSetupTexture(gl);
-    gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, processor.width, processor.height, 0, gl.RGBA, gl.UNSIGNED_BYTE, null);
-    const frameBuffer = gl.createFramebuffer();
-    gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer);
-    gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, outputTexture, 0);
+    const outputTexture = redOnly
+        ? createMaskTexture(gl, targetWidth, targetHeight, linearSampling, float && floatSingleChannelSupported, float && floatFullChannelSupported)
+        : createRGBAFrameTexture(gl, targetWidth, targetHeight, linearSampling, float && floatFullChannelSupported);
+    const frameBuffer = createFrameBuffer(gl, outputTexture);
     return {
         render: (inputTexture) => {
             assert(!gl.isContextLost(), RENDERING_EVENTS.ContextLost);
             // Draw on the frame buffer
             gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer);
-            gl.viewport(0, 0, processor.width, processor.height);
+            gl.viewport(0, 0, targetWidth, targetHeight);
             gl.clearColor(0, 0, 0, 1);
             gl.clear(gl.COLOR_BUFFER_BIT);
             gl.useProgram(program);
@@ -76,11 +75,15 @@ export const createTextureToTextureRenderer = (processor) => {
             const matrix = m3.projection(1, 1, 1);
             gl.uniformMatrix3fv(uniformLocations.matrix, false, matrix);
             gl.drawArrays(gl.TRIANGLES, 0, 6);
-            assert(outputTexture);
+            // Cleanup
+            gl.bindBuffer(gl.ARRAY_BUFFER, null);
+            gl.bindFramebuffer(gl.FRAMEBUFFER, null);
+            gl.bindTexture(gl.TEXTURE_2D, null);
+            gl.bindVertexArray(null);
             return {
                 texture: outputTexture,
-                width: processor.width,
-                height: processor.height,
+                width: targetWidth,
+                height: targetHeight,
             };
         },
         release: () => {

package/dist/common/backends/webgl/webglUtils.d.ts

@@ -61,7 +61,11 @@ export declare const createProgram: (gl: WebGLRenderingContext, shaders: [WebGLS
  *        A parallel array to opt_attribs letting you assign locations.
  */
 export declare const createProgramFromSources: (gl: WebGLRenderingContext, [vs, fs]: [GLShaderSource, GLShaderSource], attribs?: string[], locations?: number[]) => [WebGLProgram, Release];
-export declare const createAndSetupTexture: (gl: WebGL2RenderingContext) => WebGLTexture;
+export declare const createAndSetupTexture: (gl: WebGL2RenderingContext, linear: boolean) => WebGLTexture;
+export declare const createRGBAFrameTexture: (gl: WebGL2RenderingContext, width: number, height: number, linear?: boolean, withFloat?: boolean) => WebGLTexture;
+export declare const createMaskTexture: (gl: WebGL2RenderingContext, width: number, height: number, linear?: boolean, withFloatSingle?: boolean, withFloatFull?: boolean) => WebGLTexture;
+export declare const createFrameBuffer: (gl: WebGL2RenderingContext, texture: WebGLTexture) => WebGLFramebuffer;
+export declare const support: (gl: WebGL2RenderingContext, internalFormat: GLint, format: GLenum, width?: number, height?: number) => boolean;
 type Location<S extends string> = S extends `u_${string}` ? ReturnType<WebGL2RenderingContext['getUniformLocation']> : S extends `a_${string}` ? ReturnType<WebGL2RenderingContext['getAttribLocation']> : never;
 export declare const getLocations: <T extends Record<string, string>>(gl: WebGL2RenderingContext, program: WebGLProgram, locations: T) => { [Key in keyof T]: Location<T[Key]>; };
 export declare const createGLProgram: <A extends Record<string, string>, U extends Record<string, string>>(processCanvas: OffscreenCanvas, { attribs, uniforms, vertexSrc, fragmentSrc, }: {
@@ -75,6 +79,8 @@ export declare const createGLProgram: <A extends Record<string, string>, U exten
     attribLocations: { [Key in keyof A]: Location<A[Key]>; };
     uniformLocations: { [Key_1 in keyof U]: Location<U[Key_1]>; };
     release: Release;
+    floatSingleChannelSupported: boolean;
+    floatFullChannelSupported: boolean;
 };
 export declare const toTextureInfo: (mask: MPMask) => TextureInfo;
 export {};

package/dist/common/backends/webgl/webglUtils.js

@@ -1,3 +1,5 @@
+import { assert } from '@pexip/utils';
+import { RENDERING_EVENTS } from '../../constants';
 /**
  * WebGL Shader type enum
  */
@@ -139,16 +141,69 @@ export const createProgramFromSources = (gl, [vs, fs], attribs, locations) => cr
     loadShader(gl, GL_SHADER_TYPE.VertexShader, vs),
     loadShader(gl, GL_SHADER_TYPE.FragmentShader, fs),
 ], attribs, locations);
-export const createAndSetupTexture = (gl) => {
+export const createAndSetupTexture = (gl, linear) => {
     const texture = gl.createTexture();
     gl.bindTexture(gl.TEXTURE_2D, texture);
-    // Set the parameters so we don't need mips and so we're not filtering
+    // Set the parameters so we don't need MIPS and so we're not filtering
     // and we don't repeat at the edges.
     gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
     gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, linear ? gl.LINEAR : gl.NEAREST);
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, linear ? gl.LINEAR : gl.NEAREST);
+    return texture;
+};
+export const createRGBAFrameTexture = (gl, width, height, linear = false, withFloat = false) => {
+    const texture = createAndSetupTexture(gl, linear);
+    if (withFloat) {
+        gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA16F, width, height, 0, gl.RGBA, gl.HALF_FLOAT, null);
+        gl.bindFramebuffer(gl.FRAMEBUFFER, null);
+        return texture;
+    }
+    gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, width, height, 0, gl.RGBA, gl.UNSIGNED_BYTE, null);
+    gl.bindTexture(gl.TEXTURE_2D, null);
+    return texture;
+};
+export const createMaskTexture = (gl, width, height, linear = false, withFloatSingle = false, withFloatFull = false) => {
+    const texture = createAndSetupTexture(gl, linear);
+    if (withFloatSingle) {
+        gl.texImage2D(gl.TEXTURE_2D, 0, gl.R16F, width, height, 0, gl.RED, gl.HALF_FLOAT, null);
+        gl.bindFramebuffer(gl.FRAMEBUFFER, null);
+        return texture;
+    }
+    if (withFloatFull) {
+        gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA16F, width, height, 0, gl.RGBA, gl.HALF_FLOAT, null);
+        gl.bindFramebuffer(gl.FRAMEBUFFER, null);
+        return texture;
+    }
+    gl.texImage2D(gl.TEXTURE_2D, 0, gl.R8, width, height, 0, gl.RED, gl.UNSIGNED_BYTE, null);
+    gl.bindTexture(gl.TEXTURE_2D, null);
+    return texture;
+};
+export const createFrameBuffer = (gl, texture) => {
+    const frameBuffer = gl.createFramebuffer();
+    gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer);
+    gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, texture, 0);
+    gl.bindFramebuffer(gl.FRAMEBUFFER, null);
+    return frameBuffer;
+};
+export const support = (gl, internalFormat, format, width = 2, height = 2) => {
+    // Create single-channel float16 texture
+    const tex = gl.createTexture();
+    gl.bindTexture(gl.TEXTURE_2D, tex);
+    gl.texImage2D(gl.TEXTURE_2D, 0, internalFormat, width, height, 0, format, gl.HALF_FLOAT, null);
     gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
     gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
-    return texture;
+    // Attach texture
+    const fb = createFrameBuffer(gl, tex);
+    gl.bindFramebuffer(gl.FRAMEBUFFER, fb);
+    // Check completeness
+    const complete = gl.checkFramebufferStatus(gl.FRAMEBUFFER) === gl.FRAMEBUFFER_COMPLETE;
+    // Clean up
+    gl.bindFramebuffer(gl.FRAMEBUFFER, null);
+    gl.deleteFramebuffer(fb);
+    gl.bindTexture(gl.TEXTURE_2D, null);
+    gl.deleteTexture(tex);
+    return complete;
 };
 export const getLocations = (gl, program, locations) => Object.keys(locations).reduce((acc, key) => {
     const k = key;
@@ -167,9 +222,7 @@ export const createGLProgram = (processCanvas, { attribs, uniforms, vertexSrc, f
         premultipliedAlpha: false,
         powerPreference: 'high-performance',
     });
-    if (!gl) {
-        throw new Error('WebGL2 is not supported');
-    }
+    assert(gl, RENDERING_EVENTS.ContextCreationError);
     gl.enable(gl.BLEND);
     gl.blendFunc(gl.SRC_ALPHA, gl.ONE_MINUS_SRC_ALPHA);
     assertShaderSource(vertexSrc);
@@ -178,12 +231,17 @@ export const createGLProgram = (processCanvas, { attribs, uniforms, vertexSrc, f
         vertexSrc,
         fragmentSrc,
     ]);
+    const hasFloat = !!gl.getExtension('EXT_color_buffer_float');
+    const floatSingleChannelSupported = hasFloat && support(gl, gl.R16F, gl.RED);
+    const floatFullChannelSupported = hasFloat && support(gl, gl.RGBA16F, gl.RGBA);
     return {
         gl,
         program,
         attribLocations: getLocations(gl, program, attribs),
         uniformLocations: getLocations(gl, program, uniforms),
         release,
+        floatSingleChannelSupported,
+        floatFullChannelSupported,
     };
 };
 export const toTextureInfo = (mask) => ({

package/dist/common/backends/webgpu/blender.d.ts

@@ -1,8 +1,13 @@
+import type { RendererOptions } from '../../types/render';
 /**
  *
  * Blend foreground and background according to the mask
  */
 export declare const createBlendRenderer: (processor: OffscreenCanvas, device: GPUDevice, presentationFormat: GPUTextureFormat, isContextLost: () => boolean) => {
-    render: (frameTexture: GPUTexture, bgTexture: GPUTexture, maskTexture: GPUTexture) => void;
+    render: (encoder: GPUCommandEncoder, { foreground, background, mask, }: {
+        foreground: GPUTexture;
+        background: GPUTexture;
+        mask: GPUTexture;
+    }, options: RendererOptions) => void;
     release: () => void;
 };

package/dist/common/backends/webgpu/blender.js

@@ -1,7 +1,7 @@
+import { makeShaderDataDefinitions, makeStructuredView } from 'webgpu-utils';
 import { assert } from '@pexip/utils';
 import { RENDERING_EVENTS } from '../../constants';
 import blenderShader from './shaders/blenderShader.wgsl';
-import { RENDER_PASS_COLOR_ATTACHMENT_CLEAR_VALUE } from './constants';
 /**
  *
  * Blend foreground and background according to the mask
@@ -16,63 +16,108 @@ export const createBlendRenderer = (processor, device, presentationFormat, isCon
         format: presentationFormat,
         alphaMode: 'premultiplied',
     });
+    const defs = makeShaderDataDefinitions(blenderShader);
+    // @ts-expect-error This is what we get from the API
+    const uniforms = makeStructuredView(defs.uniforms.thresholds);
+    // Cache shader module and pipeline at init time
+    const shaderModule = device.createShaderModule({
+        label: 'blender shader',
+        code: blenderShader,
+    });
+    const pipeline = device.createRenderPipeline({
+        label: 'blender pipeline',
+        layout: 'auto',
+        vertex: {
+            module: shaderModule,
+        },
+        fragment: {
+            module: shaderModule,
+            targets: [{ format: presentationFormat }],
+        },
+    });
+    // Cache sampler at init time
+    const linearSampler = device.createSampler({
+        addressModeU: 'clamp-to-edge',
+        addressModeV: 'clamp-to-edge',
+        magFilter: 'linear',
+        minFilter: 'linear',
+    });
+    const nearestSampler = device.createSampler({
+        addressModeU: 'clamp-to-edge',
+        addressModeV: 'clamp-to-edge',
+        magFilter: 'nearest',
+        minFilter: 'nearest',
+    });
+    const uniformsBuffer = device.createBuffer({
+        size: uniforms.arrayBuffer.byteLength,
+        usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST,
+    });
+    const updateUniforms = (options) => {
+        uniforms.set({
+            background: options.backgroundThreshold,
+            foreground: options.foregroundThreshold,
+        });
+        // Upload to GPU
+        device.queue.writeBuffer(uniformsBuffer, 0, uniforms.arrayBuffer);
+    };
+    const colorAttachment0 = {
+        loadOp: 'load',
+        storeOp: 'store',
+    };
+    const renderPassDescriptor = {
+        label: 'blender renderPass',
+        colorAttachments: [colorAttachment0],
+    };
+    const bindGroup0 = device.createBindGroup({
+        layout: pipeline.getBindGroupLayout(0),
+        entries: [
+            { binding: 0, resource: linearSampler },
+            { binding: 1, resource: nearestSampler },
+            { binding: 2, resource: { buffer: uniformsBuffer } },
+        ],
+    });
+    let cachedBindGroup1 = null;
+    let cachedMask = null;
+    let cachedBackground = null;
+    let cachedForeground = null;
+    let cachedBackgroundThreshold = -1;
+    let cachedForegroundThreshold = -1;
     return {
-        render: (frameTexture, bgTexture, maskTexture) => {
+        render: (encoder, { foreground, background, mask, }, options) => {
             assert(!isContextLost(), RENDERING_EVENTS.ContextLost);
-            const pipeline = device.createRenderPipeline({
-                label: 'blender pipeline',
-                layout: 'auto',
-                vertex: {
-                    module: device.createShaderModule({
-                        label: 'blender vertex',
-                        code: blenderShader,
-                    }),
-                },
-                fragment: {
-                    module: device.createShaderModule({
-                        label: 'blender fragment',
-                        code: blenderShader,
-                    }),
-                    targets: [{ format: presentationFormat }],
-                },
-            });
-            const encoder = device.createCommandEncoder({
-                label: 'blender encoder',
-            });
-            const pass = encoder.beginRenderPass({
-                label: 'blender renderPass',
-                colorAttachments: [
-                    {
-                        view: context.getCurrentTexture().createView(),
-                        clearValue: RENDER_PASS_COLOR_ATTACHMENT_CLEAR_VALUE,
-                        loadOp: 'clear',
-                        storeOp: 'store',
-                    },
-                ],
-            });
+            colorAttachment0.view = context.getCurrentTexture().createView();
+            if (!(cachedBackground === background &&
+                cachedForeground === foreground &&
+                cachedMask === mask)) {
+                cachedBindGroup1 = device.createBindGroup({
+                    layout: pipeline.getBindGroupLayout(1),
+                    entries: [
+                        { binding: 0, resource: background.createView() },
+                        { binding: 1, resource: foreground.createView() },
+                        { binding: 2, resource: mask.createView() },
+                    ],
+                });
+                cachedBackground = background;
+                cachedForeground = foreground;
+                cachedMask = mask;
+            }
+            if (!(cachedBackgroundThreshold === options.backgroundThreshold &&
+                cachedForegroundThreshold === options.foregroundThreshold)) {
+                updateUniforms(options);
+                cachedBackgroundThreshold = options.backgroundThreshold;
+                cachedForegroundThreshold = options.foregroundThreshold;
+            }
+            const pass = encoder.beginRenderPass(renderPassDescriptor);
             pass.setPipeline(pipeline);
-            const sampler = device.createSampler({
-                addressModeU: 'repeat',
-                addressModeV: 'repeat',
-                magFilter: 'nearest',
-            });
-            const bindGroup = device.createBindGroup({
-                layout: pipeline.getBindGroupLayout(0),
-                entries: [
-                    { binding: 0, resource: sampler },
-                    { binding: 1, resource: bgTexture.createView() },
-                    { binding: 2, resource: frameTexture.createView() },
-                    { binding: 3, resource: maskTexture.createView() },
-                ],
-            });
-            pass.setBindGroup(0, bindGroup);
+            pass.setBindGroup(0, bindGroup0);
+            pass.setBindGroup(1, cachedBindGroup1);
             pass.draw(6); // call our vertex shader 6 times
             pass.end();
-            const commandBuffer = encoder.finish();
-            device.queue.submit([commandBuffer]);
         },
         release: () => {
-            // Nothing
+            uniformsBuffer.destroy();
+            cachedBackgroundThreshold = -1;
+            cachedForegroundThreshold = -1;
         },
     };
 };

package/dist/common/backends/webgpu/constants.js

@@ -1 +1 @@
-export const RENDER_PASS_COLOR_ATTACHMENT_CLEAR_VALUE = [0.0, 0.0, 0.0, 0];
+export const RENDER_PASS_COLOR_ATTACHMENT_CLEAR_VALUE = [0.0, 0.0, 0.0, 1.0];

package/dist/common/backends/webgpu/dualFilterBlur.d.ts

@@ -1,5 +1,4 @@
-import type { VideoFrameLike } from '../../types/media';
-export declare const createBlurRenderer: (processor: OffscreenCanvas, device: GPUDevice, presentationFormat: GPUTextureFormat, isContextLost: () => boolean) => {
-    render: (frame: VideoFrameLike, pass: number) => GPUTexture;
+export declare const createBlurRenderer: (device: GPUDevice, width: number, height: number, textureFormat: GPUTextureFormat, isContextLost: () => boolean) => {
+    render: (encoder: GPUCommandEncoder, frame: GPUTexture, pass: number) => GPUTexture;
     release: () => void;
 };