@pexip/media-processor 20.3.5 → 21.0.0

package/dist/common/backends/webgpu/blender.d.ts

@@ -1,8 +1,13 @@
+import type { RendererOptions } from '../../types/render';
 /**
  *
  * Blend foreground and background according to the mask
  */
 export declare const createBlendRenderer: (processor: OffscreenCanvas, device: GPUDevice, presentationFormat: GPUTextureFormat, isContextLost: () => boolean) => {
-    render: (frameTexture: GPUTexture, bgTexture: GPUTexture, maskTexture: GPUTexture) => void;
+    render: (encoder: GPUCommandEncoder, { foreground, background, mask, }: {
+        foreground: GPUTexture;
+        background: GPUTexture;
+        mask: GPUTexture;
+    }, options: RendererOptions) => void;
     release: () => void;
 };

package/dist/common/backends/webgpu/blender.js

@@ -1,7 +1,7 @@
+import { makeShaderDataDefinitions, makeStructuredView } from 'webgpu-utils';
 import { assert } from '@pexip/utils';
 import { RENDERING_EVENTS } from '../../constants';
 import blenderShader from './shaders/blenderShader.wgsl';
-import { RENDER_PASS_COLOR_ATTACHMENT_CLEAR_VALUE } from './constants';
 /**
  *
  * Blend foreground and background according to the mask
@@ -16,63 +16,108 @@ export const createBlendRenderer = (processor, device, presentationFormat, isCon
         format: presentationFormat,
         alphaMode: 'premultiplied',
     });
+    const defs = makeShaderDataDefinitions(blenderShader);
+    // @ts-expect-error This is what we get from the API
+    const uniforms = makeStructuredView(defs.uniforms.thresholds);
+    // Cache shader module and pipeline at init time
+    const shaderModule = device.createShaderModule({
+        label: 'blender shader',
+        code: blenderShader,
+    });
+    const pipeline = device.createRenderPipeline({
+        label: 'blender pipeline',
+        layout: 'auto',
+        vertex: {
+            module: shaderModule,
+        },
+        fragment: {
+            module: shaderModule,
+            targets: [{ format: presentationFormat }],
+        },
+    });
+    // Cache sampler at init time
+    const linearSampler = device.createSampler({
+        addressModeU: 'clamp-to-edge',
+        addressModeV: 'clamp-to-edge',
+        magFilter: 'linear',
+        minFilter: 'linear',
+    });
+    const nearestSampler = device.createSampler({
+        addressModeU: 'clamp-to-edge',
+        addressModeV: 'clamp-to-edge',
+        magFilter: 'nearest',
+        minFilter: 'nearest',
+    });
+    const uniformsBuffer = device.createBuffer({
+        size: uniforms.arrayBuffer.byteLength,
+        usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST,
+    });
+    const updateUniforms = (options) => {
+        uniforms.set({
+            background: options.backgroundThreshold,
+            foreground: options.foregroundThreshold,
+        });
+        // Upload to GPU
+        device.queue.writeBuffer(uniformsBuffer, 0, uniforms.arrayBuffer);
+    };
+    const colorAttachment0 = {
+        loadOp: 'load',
+        storeOp: 'store',
+    };
+    const renderPassDescriptor = {
+        label: 'blender renderPass',
+        colorAttachments: [colorAttachment0],
+    };
+    const bindGroup0 = device.createBindGroup({
+        layout: pipeline.getBindGroupLayout(0),
+        entries: [
+            { binding: 0, resource: linearSampler },
+            { binding: 1, resource: nearestSampler },
+            { binding: 2, resource: { buffer: uniformsBuffer } },
+        ],
+    });
+    let cachedBindGroup1 = null;
+    let cachedMask = null;
+    let cachedBackground = null;
+    let cachedForeground = null;
+    let cachedBackgroundThreshold = -1;
+    let cachedForegroundThreshold = -1;
     return {
-        render: (frameTexture, bgTexture, maskTexture) => {
+        render: (encoder, { foreground, background, mask, }, options) => {
             assert(!isContextLost(), RENDERING_EVENTS.ContextLost);
-            const pipeline = device.createRenderPipeline({
-                label: 'blender pipeline',
-                layout: 'auto',
-                vertex: {
-                    module: device.createShaderModule({
-                        label: 'blender vertex',
-                        code: blenderShader,
-                    }),
-                },
-                fragment: {
-                    module: device.createShaderModule({
-                        label: 'blender fragment',
-                        code: blenderShader,
-                    }),
-                    targets: [{ format: presentationFormat }],
-                },
-            });
-            const encoder = device.createCommandEncoder({
-                label: 'blender encoder',
-            });
-            const pass = encoder.beginRenderPass({
-                label: 'blender renderPass',
-                colorAttachments: [
-                    {
-                        view: context.getCurrentTexture().createView(),
-                        clearValue: RENDER_PASS_COLOR_ATTACHMENT_CLEAR_VALUE,
-                        loadOp: 'clear',
-                        storeOp: 'store',
-                    },
-                ],
-            });
+            colorAttachment0.view = context.getCurrentTexture().createView();
+            if (!(cachedBackground === background &&
+                cachedForeground === foreground &&
+                cachedMask === mask)) {
+                cachedBindGroup1 = device.createBindGroup({
+                    layout: pipeline.getBindGroupLayout(1),
+                    entries: [
+                        { binding: 0, resource: background.createView() },
+                        { binding: 1, resource: foreground.createView() },
+                        { binding: 2, resource: mask.createView() },
+                    ],
+                });
+                cachedBackground = background;
+                cachedForeground = foreground;
+                cachedMask = mask;
+            }
+            if (!(cachedBackgroundThreshold === options.backgroundThreshold &&
+                cachedForegroundThreshold === options.foregroundThreshold)) {
+                updateUniforms(options);
+                cachedBackgroundThreshold = options.backgroundThreshold;
+                cachedForegroundThreshold = options.foregroundThreshold;
+            }
+            const pass = encoder.beginRenderPass(renderPassDescriptor);
             pass.setPipeline(pipeline);
-            const sampler = device.createSampler({
-                addressModeU: 'repeat',
-                addressModeV: 'repeat',
-                magFilter: 'nearest',
-            });
-            const bindGroup = device.createBindGroup({
-                layout: pipeline.getBindGroupLayout(0),
-                entries: [
-                    { binding: 0, resource: sampler },
-                    { binding: 1, resource: bgTexture.createView() },
-                    { binding: 2, resource: frameTexture.createView() },
-                    { binding: 3, resource: maskTexture.createView() },
-                ],
-            });
-            pass.setBindGroup(0, bindGroup);
+            pass.setBindGroup(0, bindGroup0);
+            pass.setBindGroup(1, cachedBindGroup1);
             pass.draw(6); // call our vertex shader 6 times
             pass.end();
-            const commandBuffer = encoder.finish();
-            device.queue.submit([commandBuffer]);
         },
         release: () => {
-            // Nothing
+            uniformsBuffer.destroy();
+            cachedBackgroundThreshold = -1;
+            cachedForegroundThreshold = -1;
         },
     };
 };

package/dist/common/backends/webgpu/constants.js

@@ -1 +1 @@
-export const RENDER_PASS_COLOR_ATTACHMENT_CLEAR_VALUE = [0.0, 0.0, 0.0, 0];
+export const RENDER_PASS_COLOR_ATTACHMENT_CLEAR_VALUE = [0.0, 0.0, 0.0, 1.0];

package/dist/common/backends/webgpu/dualFilterBlur.d.ts

@@ -1,5 +1,4 @@
-import type { VideoFrameLike } from '../../types/media';
-export declare const createBlurRenderer: (processor: OffscreenCanvas, device: GPUDevice, presentationFormat: GPUTextureFormat, isContextLost: () => boolean) => {
-    render: (frame: VideoFrameLike, pass: number) => GPUTexture;
+export declare const createBlurRenderer: (device: GPUDevice, width: number, height: number, textureFormat: GPUTextureFormat, isContextLost: () => boolean) => {
+    render: (encoder: GPUCommandEncoder, frame: GPUTexture, pass: number) => GPUTexture;
     release: () => void;
 };

package/dist/common/backends/webgpu/dualFilterBlur.js

@@ -1,192 +1,194 @@
 import { assert } from '@pexip/utils';
 import { RENDERING_EVENTS } from '../../constants';
-import blurExample from './shaders/dualBlurShader.wgsl';
+import blurComputeShader from './shaders/dualBlurComputeShader.wgsl';
 import { clamping } from '../../utils';
-import { RENDER_PASS_COLOR_ATTACHMENT_CLEAR_VALUE } from './constants';
 const clampBlurAmount = clamping(1, 9);
-export const createBlurRenderer = (processor, device, presentationFormat, isContextLost) => {
-    const context = processor.getContext('webgpu');
-    if (!context) {
-        throw Error('WebGPU not supported on the canvas.');
-    }
-    context.configure({
-        device,
-        format: presentationFormat,
-        alphaMode: 'premultiplied',
-    });
-    // Helper to create a texture of given size
-    const createTexture = (width, height) => device.createTexture({
-        format: presentationFormat,
-        size: [width, height],
-        usage: GPUTextureUsage.TEXTURE_BINDING |
-            GPUTextureUsage.COPY_DST |
-            GPUTextureUsage.RENDER_ATTACHMENT,
-    });
-    // Helper to create a uniform buffer to flip downsampling vs upsampling
-    const createFlipBuffer = (value) => {
-        const buffer = device.createBuffer({
-            size: 4,
-            mappedAtCreation: true,
-            usage: GPUBufferUsage.UNIFORM,
-        });
-        new Uint32Array(buffer.getMappedRange())[0] = value;
-        buffer.unmap();
-        return buffer;
-    };
-    const sampler = device.createSampler({
-        magFilter: 'linear',
-        minFilter: 'linear',
+export const createBlurRenderer = (device, width, height, textureFormat, isContextLost) => {
+    const shaderModule = device.createShaderModule({
+        label: 'dual blur compute shader',
+        code: blurComputeShader,
     });
-    const shaderModule = device.createShaderModule({ code: blurExample });
-    const pipeline = device.createRenderPipeline({
-        label: 'dual blur render pipeline',
+    const downsamplePipeline = device.createRenderPipeline({
+        label: 'dual blur downsample compute pipeline',
         layout: 'auto',
-        vertex: {
+        vertex: { module: shaderModule, entryPoint: 'vs_main' },
+        fragment: {
             module: shaderModule,
-            entryPoint: 'vert_main',
-            buffers: [],
+            targets: [{ format: textureFormat }],
+            entryPoint: 'downsample_fs',
         },
+    });
+    const upsamplePipeline = device.createRenderPipeline({
+        label: 'dual blur upsample compute pipeline',
+        layout: 'auto',
+        vertex: { module: shaderModule, entryPoint: 'vs_main' },
         fragment: {
             module: shaderModule,
-            entryPoint: 'frag_main',
-            targets: [{ format: presentationFormat }],
+            targets: [{ format: textureFormat }],
+            entryPoint: 'upsample_fs',
         },
-        primitive: { topology: 'triangle-list' },
     });
-    // --- State for persistent textures and buffers ---
+    const sampler = device.createSampler({
+        magFilter: 'linear',
+        minFilter: 'linear',
+        addressModeU: 'clamp-to-edge',
+        addressModeV: 'clamp-to-edge',
+    });
+    // Helper to create a texture of given size
+    const createTexture = (w, h) => device.createTexture({
+        label: 'dual filter blur texture',
+        format: textureFormat,
+        size: [w, h],
+        usage: GPUTextureUsage.TEXTURE_BINDING |
+            GPUTextureUsage.RENDER_ATTACHMENT,
+    });
+    // --- State for persistent textures and bind groups ---
     let blurTextures = [];
     let lastWidth = -1;
     let lastHeight = -1;
-    // Helper to (re)generate textures if needed
-    function ensureBlurTextures(pass, width, height) {
+    let downsampleRenderPassDescriptor = [];
+    let upsampleRenderPassDescriptor = [];
+    let downsampleBindGroups = [];
+    let upsampleBindGroups = [];
+    let cacheFrameTexture = null;
+    let cacheFirstDownsampleBindGroup = null;
+    let cacheFirstDownsampleRenderPassDescriptor = null;
+    // Helper to (re)generate textures and bind groups if needed
+    function ensureBlurTextures(pass, frameWidth, frameHeight) {
         const length = clampBlurAmount(pass + 1);
-        let needsUpdate = false;
         if (blurTextures.length !== length ||
-            lastWidth !== width ||
-            lastHeight !== height) {
+            lastWidth !== frameWidth ||
+            lastHeight !== frameHeight) {
             // Destroy old textures
             for (const t of blurTextures) {
                 t.texture.destroy();
             }
             blurTextures = [];
+            downsampleBindGroups = [];
+            downsampleRenderPassDescriptor = [];
+            upsampleBindGroups = [];
+            // Invalidate frame cache since firstDst view has changed
+            cacheFrameTexture = null;
+            cacheFirstDownsampleBindGroup = null;
             for (let i = 0; i < length; i++) {
-                const w = Math.max(Math.trunc(width / 2 ** i), 1);
-                const h = Math.max(Math.trunc(height / 2 ** i), 1);
+                const w = Math.max(Math.trunc(frameWidth / 2 ** i), 1);
+                const h = Math.max(Math.trunc(frameHeight / 2 ** i), 1);
+                const texture = createTexture(w, h);
                 blurTextures.push({
-                    texture: createTexture(w, h),
+                    texture,
+                    view: texture.createView(),
                     width: w,
                     height: h,
                 });
             }
-            lastWidth = width;
-            lastHeight = height;
-            needsUpdate = true;
-        }
-        return needsUpdate;
-    }
-    const flipDownBuffer = createFlipBuffer(0); // create once
-    const flipUpBuffer = createFlipBuffer(1); // create once
-    return {
-        render: (frame, pass) => {
-            assert(!isContextLost(), RENDERING_EVENTS.ContextLost);
-            // Ensure textures and buffers are up-to-date
-            ensureBlurTextures(pass, processor.width, processor.height);
-            // Upload frame to the largest texture (level 0)
-            const largestTexture = blurTextures[0];
-            assert(largestTexture);
-            device.queue.copyExternalImageToTexture({ source: frame.frame, flipY: false }, { texture: largestTexture.texture }, { width: largestTexture.width, height: largestTexture.height });
-            // Downsampling passes
-            for (let i = 1; i < blurTextures.length; i++) {
+            // Pre-create downsample bind groups: blurTextures[i-1] → blurTextures[i]
+            for (let i = 2; i < blurTextures.length; i++) {
                 const src = blurTextures[i - 1];
-                assert(src);
                 const dst = blurTextures[i];
+                assert(src);
                 assert(dst);
-                const bindGroup = device.createBindGroup({
-                    layout: pipeline.getBindGroupLayout(0),
+                downsampleBindGroups.push(device.createBindGroup({
+                    layout: downsamplePipeline.getBindGroupLayout(0),
                     entries: [
                         { binding: 0, resource: sampler },
-                        { binding: 1, resource: src.texture.createView() },
-                        { binding: 2, resource: { buffer: flipDownBuffer } },
+                        { binding: 1, resource: src.view },
                     ],
-                });
-                const encoder = device.createCommandEncoder();
-                const passEncoder = encoder.beginRenderPass({
+                }));
+                downsampleRenderPassDescriptor.push({
+                    label: `downsample render pass ${i}`,
                     colorAttachments: [
-                        {
-                            view: dst.texture.createView(),
-                            loadOp: 'clear',
-                            storeOp: 'store',
-                            clearValue: RENDER_PASS_COLOR_ATTACHMENT_CLEAR_VALUE,
-                        },
+                        { loadOp: 'load', storeOp: 'store', view: dst.view },
                     ],
                 });
-                passEncoder.setPipeline(pipeline);
-                passEncoder.setBindGroup(0, bindGroup);
-                passEncoder.draw(6);
-                passEncoder.end();
-                device.queue.submit([encoder.finish()]);
             }
-            // Upsampling passes
+            // Pre-create upsample bind groups: blurTextures[i+1] → blurTextures[i]
+            // Filled in descending order so upsampleBindGroups[0] is the deepest pair.
             for (let i = blurTextures.length - 2; i >= 0; i--) {
                 const src = blurTextures[i + 1];
-                assert(src);
                 const dst = blurTextures[i];
+                assert(src);
                 assert(dst);
-                const bindGroup = device.createBindGroup({
-                    layout: pipeline.getBindGroupLayout(0),
+                upsampleBindGroups.push(device.createBindGroup({
+                    layout: upsamplePipeline.getBindGroupLayout(0),
                     entries: [
                         { binding: 0, resource: sampler },
-                        { binding: 1, resource: src.texture.createView() },
-                        { binding: 2, resource: { buffer: flipUpBuffer } },
+                        { binding: 1, resource: src.view },
                     ],
-                });
-                const encoder = device.createCommandEncoder();
-                const passEncoder = encoder.beginRenderPass({
+                }));
+                upsampleRenderPassDescriptor.push({
+                    label: `upsample render pass ${i}`,
                     colorAttachments: [
-                        {
-                            view: dst.texture.createView(),
-                            loadOp: 'clear',
-                            storeOp: 'store',
-                            clearValue: RENDER_PASS_COLOR_ATTACHMENT_CLEAR_VALUE,
-                        },
+                        { loadOp: 'load', storeOp: 'store', view: dst.view },
                     ],
                 });
-                passEncoder.setPipeline(pipeline);
-                passEncoder.setBindGroup(0, bindGroup);
-                passEncoder.draw(6);
-                passEncoder.end();
-                device.queue.submit([encoder.finish()]);
             }
-            // Final pass: draw to canvas
+            lastWidth = frameWidth;
+            lastHeight = frameHeight;
+        }
+    }
+    return {
+        render: (encoder, frame, pass) => {
+            assert(!isContextLost(), RENDERING_EVENTS.ContextLost);
+            // Ensure textures and bind groups are up-to-date
+            ensureBlurTextures(pass, width, height);
+            const firstDst = blurTextures[1];
+            assert(firstDst);
+            // Cache the first downsample bind group (frame → blurTextures[1])
+            if (frame !== cacheFrameTexture) {
+                cacheFirstDownsampleBindGroup = device.createBindGroup({
+                    layout: downsamplePipeline.getBindGroupLayout(0),
+                    entries: [
+                        { binding: 0, resource: sampler },
+                        { binding: 1, resource: frame.createView() },
+                    ],
+                });
+                cacheFirstDownsampleRenderPassDescriptor = {
+                    label: `downsample render pass 0`,
+                    colorAttachments: [
+                        { loadOp: 'load', storeOp: 'store', view: firstDst.view },
+                    ],
+                };
+                cacheFrameTexture = frame;
+            }
+            assert(cacheFirstDownsampleRenderPassDescriptor);
+            // First downsampling: frame → blurTextures[1]
+            const firstComputePass = encoder.beginRenderPass(cacheFirstDownsampleRenderPassDescriptor);
+            firstComputePass.setPipeline(downsamplePipeline);
+            firstComputePass.setBindGroup(0, cacheFirstDownsampleBindGroup);
+            firstComputePass.draw(3);
+            firstComputePass.end();
+            // Subsequent downsampling passes: blurTextures[i-1] → blurTextures[i]
+            for (let i = 2; i < blurTextures.length; i++) {
+                const dst = blurTextures[i];
+                assert(dst);
+                // downsampleBindGroups[i-1] covers blurTextures[i-1] → blurTextures[i]
+                const bindGroup = downsampleBindGroups[i - 2];
+                const renderPassDescriptor = downsampleRenderPassDescriptor[i - 2];
+                assert(bindGroup);
+                assert(renderPassDescriptor);
+                const pass = encoder.beginRenderPass(renderPassDescriptor);
+                pass.setPipeline(downsamplePipeline);
+                pass.setBindGroup(0, bindGroup);
+                pass.draw(3);
+                pass.end();
+            }
+            // Upsampling passes (deepest level first, up to blurTextures[0])
+            for (let i = 0; i < upsampleBindGroups.length; i++) {
+                const dstIdx = blurTextures.length - 2 - i;
+                const dst = blurTextures[dstIdx];
+                assert(dst);
+                const bindGroup = upsampleBindGroups[i];
+                const renderPassDescriptor = upsampleRenderPassDescriptor[i];
+                assert(bindGroup);
+                assert(renderPassDescriptor);
+                const pass = encoder.beginRenderPass(renderPassDescriptor);
+                pass.setPipeline(upsamplePipeline);
+                pass.setBindGroup(0, bindGroup);
+                pass.draw(3);
+                pass.end();
+            }
             const finalTexture = blurTextures[0];
             assert(finalTexture);
-            const finalBindGroup = device.createBindGroup({
-                layout: pipeline.getBindGroupLayout(0),
-                entries: [
-                    { binding: 0, resource: sampler },
-                    {
-                        binding: 1,
-                        resource: finalTexture.texture.createView(),
-                    },
-                    { binding: 2, resource: { buffer: flipUpBuffer } },
-                ],
-            });
-            const encoder = device.createCommandEncoder();
-            const passEncoder = encoder.beginRenderPass({
-                colorAttachments: [
-                    {
-                        view: context.getCurrentTexture().createView(),
-                        loadOp: 'clear',
-                        storeOp: 'store',
-                        clearValue: RENDER_PASS_COLOR_ATTACHMENT_CLEAR_VALUE,
-                    },
-                ],
-            });
-            passEncoder.setPipeline(pipeline);
-            passEncoder.setBindGroup(0, finalBindGroup);
-            passEncoder.draw(6);
-            passEncoder.end();
-            device.queue.submit([encoder.finish()]);
             return finalTexture.texture;
         },
         release: () => {
@@ -194,8 +196,12 @@ export const createBlurRenderer = (processor, device, presentationFormat, isCont
                 t.texture.destroy();
             }
             blurTextures = [];
-            flipDownBuffer.destroy();
-            flipUpBuffer.destroy();
+            downsampleRenderPassDescriptor = [];
+            upsampleRenderPassDescriptor = [];
+            downsampleBindGroups = [];
+            upsampleBindGroups = [];
+            cacheFrameTexture = null;
+            cacheFirstDownsampleBindGroup = null;
         },
     };
 };

package/dist/common/backends/webgpu/jointBilateralFilter.d.ts

@@ -0,0 +1,7 @@
+import type { RendererOptions } from '../../types/render';
+type JointBilateralFilterParams = Pick<RendererOptions, 'sigmaSpace' | 'sigmaRangeLo' | 'sigmaRangeHi' | 'backgroundThreshold' | 'foregroundThreshold'>;
+export declare const createJointBilateralFilterRenderer: (processor: OffscreenCanvas, device: GPUDevice, isContextLost: () => boolean, width: number, height: number, maskFormat: GPUTextureFormat) => {
+    render: (encoder: GPUCommandEncoder, maskTexture: GPUTexture, guideTexture: GPUTexture, params: JointBilateralFilterParams) => GPUTexture;
+    release: () => void;
+};
+export {};