@pexip/media-processor 20.3.4 → 21.0.0

package/dist/common/backends/webgpu/dualFilterBlur.js

@@ -1,192 +1,194 @@
 import { assert } from '@pexip/utils';
 import { RENDERING_EVENTS } from '../../constants';
-import blurExample from './shaders/dualBlurShader.wgsl';
+import blurComputeShader from './shaders/dualBlurComputeShader.wgsl';
 import { clamping } from '../../utils';
-import { RENDER_PASS_COLOR_ATTACHMENT_CLEAR_VALUE } from './constants';
 const clampBlurAmount = clamping(1, 9);
-export const createBlurRenderer = (processor, device, presentationFormat, isContextLost) => {
-    const context = processor.getContext('webgpu');
-    if (!context) {
-        throw Error('WebGPU not supported on the canvas.');
-    }
-    context.configure({
-        device,
-        format: presentationFormat,
-        alphaMode: 'premultiplied',
-    });
-    // Helper to create a texture of given size
-    const createTexture = (width, height) => device.createTexture({
-        format: presentationFormat,
-        size: [width, height],
-        usage: GPUTextureUsage.TEXTURE_BINDING |
-            GPUTextureUsage.COPY_DST |
-            GPUTextureUsage.RENDER_ATTACHMENT,
-    });
-    // Helper to create a uniform buffer to flip downsampling vs upsampling
-    const createFlipBuffer = (value) => {
-        const buffer = device.createBuffer({
-            size: 4,
-            mappedAtCreation: true,
-            usage: GPUBufferUsage.UNIFORM,
-        });
-        new Uint32Array(buffer.getMappedRange())[0] = value;
-        buffer.unmap();
-        return buffer;
-    };
-    const sampler = device.createSampler({
-        magFilter: 'linear',
-        minFilter: 'linear',
+export const createBlurRenderer = (device, width, height, textureFormat, isContextLost) => {
+    const shaderModule = device.createShaderModule({
+        label: 'dual blur compute shader',
+        code: blurComputeShader,
     });
-    const shaderModule = device.createShaderModule({ code: blurExample });
-    const pipeline = device.createRenderPipeline({
-        label: 'dual blur render pipeline',
+    const downsamplePipeline = device.createRenderPipeline({
+        label: 'dual blur downsample compute pipeline',
         layout: 'auto',
-        vertex: {
+        vertex: { module: shaderModule, entryPoint: 'vs_main' },
+        fragment: {
             module: shaderModule,
-            entryPoint: 'vert_main',
-            buffers: [],
+            targets: [{ format: textureFormat }],
+            entryPoint: 'downsample_fs',
         },
+    });
+    const upsamplePipeline = device.createRenderPipeline({
+        label: 'dual blur upsample compute pipeline',
+        layout: 'auto',
+        vertex: { module: shaderModule, entryPoint: 'vs_main' },
         fragment: {
             module: shaderModule,
-            entryPoint: 'frag_main',
-            targets: [{ format: presentationFormat }],
+            targets: [{ format: textureFormat }],
+            entryPoint: 'upsample_fs',
         },
-        primitive: { topology: 'triangle-list' },
     });
-    // --- State for persistent textures and buffers ---
+    const sampler = device.createSampler({
+        magFilter: 'linear',
+        minFilter: 'linear',
+        addressModeU: 'clamp-to-edge',
+        addressModeV: 'clamp-to-edge',
+    });
+    // Helper to create a texture of given size
+    const createTexture = (w, h) => device.createTexture({
+        label: 'dual filter blur texture',
+        format: textureFormat,
+        size: [w, h],
+        usage: GPUTextureUsage.TEXTURE_BINDING |
+            GPUTextureUsage.RENDER_ATTACHMENT,
+    });
+    // --- State for persistent textures and bind groups ---
     let blurTextures = [];
     let lastWidth = -1;
     let lastHeight = -1;
-    // Helper to (re)generate textures if needed
-    function ensureBlurTextures(pass, width, height) {
+    let downsampleRenderPassDescriptor = [];
+    let upsampleRenderPassDescriptor = [];
+    let downsampleBindGroups = [];
+    let upsampleBindGroups = [];
+    let cacheFrameTexture = null;
+    let cacheFirstDownsampleBindGroup = null;
+    let cacheFirstDownsampleRenderPassDescriptor = null;
+    // Helper to (re)generate textures and bind groups if needed
+    function ensureBlurTextures(pass, frameWidth, frameHeight) {
         const length = clampBlurAmount(pass + 1);
-        let needsUpdate = false;
         if (blurTextures.length !== length ||
-            lastWidth !== width ||
-            lastHeight !== height) {
+            lastWidth !== frameWidth ||
+            lastHeight !== frameHeight) {
             // Destroy old textures
             for (const t of blurTextures) {
                 t.texture.destroy();
             }
             blurTextures = [];
+            downsampleBindGroups = [];
+            downsampleRenderPassDescriptor = [];
+            upsampleBindGroups = [];
+            // Invalidate frame cache since firstDst view has changed
+            cacheFrameTexture = null;
+            cacheFirstDownsampleBindGroup = null;
             for (let i = 0; i < length; i++) {
-                const w = Math.max(Math.trunc(width / 2 ** i), 1);
-                const h = Math.max(Math.trunc(height / 2 ** i), 1);
+                const w = Math.max(Math.trunc(frameWidth / 2 ** i), 1);
+                const h = Math.max(Math.trunc(frameHeight / 2 ** i), 1);
+                const texture = createTexture(w, h);
                 blurTextures.push({
-                    texture: createTexture(w, h),
+                    texture,
+                    view: texture.createView(),
                     width: w,
                     height: h,
                 });
             }
-            lastWidth = width;
-            lastHeight = height;
-            needsUpdate = true;
-        }
-        return needsUpdate;
-    }
-    const flipDownBuffer = createFlipBuffer(0); // create once
-    const flipUpBuffer = createFlipBuffer(1); // create once
-    return {
-        render: (frame, pass) => {
-            assert(!isContextLost(), RENDERING_EVENTS.ContextLost);
-            // Ensure textures and buffers are up-to-date
-            ensureBlurTextures(pass, processor.width, processor.height);
-            // Upload frame to the largest texture (level 0)
-            const largestTexture = blurTextures[0];
-            assert(largestTexture);
-            device.queue.copyExternalImageToTexture({ source: frame.frame, flipY: false }, { texture: largestTexture.texture }, { width: largestTexture.width, height: largestTexture.height });
-            // Downsampling passes
-            for (let i = 1; i < blurTextures.length; i++) {
+            // Pre-create downsample bind groups: blurTextures[i-1] → blurTextures[i]
+            for (let i = 2; i < blurTextures.length; i++) {
                 const src = blurTextures[i - 1];
-                assert(src);
                 const dst = blurTextures[i];
+                assert(src);
                 assert(dst);
-                const bindGroup = device.createBindGroup({
-                    layout: pipeline.getBindGroupLayout(0),
+                downsampleBindGroups.push(device.createBindGroup({
+                    layout: downsamplePipeline.getBindGroupLayout(0),
                     entries: [
                         { binding: 0, resource: sampler },
-                        { binding: 1, resource: src.texture.createView() },
-                        { binding: 2, resource: { buffer: flipDownBuffer } },
+                        { binding: 1, resource: src.view },
                     ],
-                });
-                const encoder = device.createCommandEncoder();
-                const passEncoder = encoder.beginRenderPass({
+                }));
+                downsampleRenderPassDescriptor.push({
+                    label: `downsample render pass ${i}`,
                     colorAttachments: [
-                        {
-                            view: dst.texture.createView(),
-                            loadOp: 'clear',
-                            storeOp: 'store',
-                            clearValue: RENDER_PASS_COLOR_ATTACHMENT_CLEAR_VALUE,
-                        },
+                        { loadOp: 'load', storeOp: 'store', view: dst.view },
                     ],
                 });
-                passEncoder.setPipeline(pipeline);
-                passEncoder.setBindGroup(0, bindGroup);
-                passEncoder.draw(6);
-                passEncoder.end();
-                device.queue.submit([encoder.finish()]);
             }
-            // Upsampling passes
+            // Pre-create upsample bind groups: blurTextures[i+1] → blurTextures[i]
+            // Filled in descending order so upsampleBindGroups[0] is the deepest pair.
             for (let i = blurTextures.length - 2; i >= 0; i--) {
                 const src = blurTextures[i + 1];
-                assert(src);
                 const dst = blurTextures[i];
+                assert(src);
                 assert(dst);
-                const bindGroup = device.createBindGroup({
-                    layout: pipeline.getBindGroupLayout(0),
+                upsampleBindGroups.push(device.createBindGroup({
+                    layout: upsamplePipeline.getBindGroupLayout(0),
                     entries: [
                         { binding: 0, resource: sampler },
-                        { binding: 1, resource: src.texture.createView() },
-                        { binding: 2, resource: { buffer: flipUpBuffer } },
+                        { binding: 1, resource: src.view },
                     ],
-                });
-                const encoder = device.createCommandEncoder();
-                const passEncoder = encoder.beginRenderPass({
+                }));
+                upsampleRenderPassDescriptor.push({
+                    label: `upsample render pass ${i}`,
                     colorAttachments: [
-                        {
-                            view: dst.texture.createView(),
-                            loadOp: 'clear',
-                            storeOp: 'store',
-                            clearValue: RENDER_PASS_COLOR_ATTACHMENT_CLEAR_VALUE,
-                        },
+                        { loadOp: 'load', storeOp: 'store', view: dst.view },
                     ],
                 });
-                passEncoder.setPipeline(pipeline);
-                passEncoder.setBindGroup(0, bindGroup);
-                passEncoder.draw(6);
-                passEncoder.end();
-                device.queue.submit([encoder.finish()]);
             }
-            // Final pass: draw to canvas
+            lastWidth = frameWidth;
+            lastHeight = frameHeight;
+        }
+    }
+    return {
+        render: (encoder, frame, pass) => {
+            assert(!isContextLost(), RENDERING_EVENTS.ContextLost);
+            // Ensure textures and bind groups are up-to-date
+            ensureBlurTextures(pass, width, height);
+            const firstDst = blurTextures[1];
+            assert(firstDst);
+            // Cache the first downsample bind group (frame → blurTextures[1])
+            if (frame !== cacheFrameTexture) {
+                cacheFirstDownsampleBindGroup = device.createBindGroup({
+                    layout: downsamplePipeline.getBindGroupLayout(0),
+                    entries: [
+                        { binding: 0, resource: sampler },
+                        { binding: 1, resource: frame.createView() },
+                    ],
+                });
+                cacheFirstDownsampleRenderPassDescriptor = {
+                    label: `downsample render pass 0`,
+                    colorAttachments: [
+                        { loadOp: 'load', storeOp: 'store', view: firstDst.view },
+                    ],
+                };
+                cacheFrameTexture = frame;
+            }
+            assert(cacheFirstDownsampleRenderPassDescriptor);
+            // First downsampling: frame → blurTextures[1]
+            const firstComputePass = encoder.beginRenderPass(cacheFirstDownsampleRenderPassDescriptor);
+            firstComputePass.setPipeline(downsamplePipeline);
+            firstComputePass.setBindGroup(0, cacheFirstDownsampleBindGroup);
+            firstComputePass.draw(3);
+            firstComputePass.end();
+            // Subsequent downsampling passes: blurTextures[i-1] → blurTextures[i]
+            for (let i = 2; i < blurTextures.length; i++) {
+                const dst = blurTextures[i];
+                assert(dst);
+                // downsampleBindGroups[i-1] covers blurTextures[i-1] → blurTextures[i]
+                const bindGroup = downsampleBindGroups[i - 2];
+                const renderPassDescriptor = downsampleRenderPassDescriptor[i - 2];
+                assert(bindGroup);
+                assert(renderPassDescriptor);
+                const pass = encoder.beginRenderPass(renderPassDescriptor);
+                pass.setPipeline(downsamplePipeline);
+                pass.setBindGroup(0, bindGroup);
+                pass.draw(3);
+                pass.end();
+            }
+            // Upsampling passes (deepest level first, up to blurTextures[0])
+            for (let i = 0; i < upsampleBindGroups.length; i++) {
+                const dstIdx = blurTextures.length - 2 - i;
+                const dst = blurTextures[dstIdx];
+                assert(dst);
+                const bindGroup = upsampleBindGroups[i];
+                const renderPassDescriptor = upsampleRenderPassDescriptor[i];
+                assert(bindGroup);
+                assert(renderPassDescriptor);
+                const pass = encoder.beginRenderPass(renderPassDescriptor);
+                pass.setPipeline(upsamplePipeline);
+                pass.setBindGroup(0, bindGroup);
+                pass.draw(3);
+                pass.end();
+            }
             const finalTexture = blurTextures[0];
             assert(finalTexture);
-            const finalBindGroup = device.createBindGroup({
-                layout: pipeline.getBindGroupLayout(0),
-                entries: [
-                    { binding: 0, resource: sampler },
-                    {
-                        binding: 1,
-                        resource: finalTexture.texture.createView(),
-                    },
-                    { binding: 2, resource: { buffer: flipUpBuffer } },
-                ],
-            });
-            const encoder = device.createCommandEncoder();
-            const passEncoder = encoder.beginRenderPass({
-                colorAttachments: [
-                    {
-                        view: context.getCurrentTexture().createView(),
-                        loadOp: 'clear',
-                        storeOp: 'store',
-                        clearValue: RENDER_PASS_COLOR_ATTACHMENT_CLEAR_VALUE,
-                    },
-                ],
-            });
-            passEncoder.setPipeline(pipeline);
-            passEncoder.setBindGroup(0, finalBindGroup);
-            passEncoder.draw(6);
-            passEncoder.end();
-            device.queue.submit([encoder.finish()]);
             return finalTexture.texture;
         },
         release: () => {
@@ -194,8 +196,12 @@ export const createBlurRenderer = (processor, device, presentationFormat, isCont
                 t.texture.destroy();
             }
             blurTextures = [];
-            flipDownBuffer.destroy();
-            flipUpBuffer.destroy();
+            downsampleRenderPassDescriptor = [];
+            upsampleRenderPassDescriptor = [];
+            downsampleBindGroups = [];
+            upsampleBindGroups = [];
+            cacheFrameTexture = null;
+            cacheFirstDownsampleBindGroup = null;
         },
     };
 };

package/dist/common/backends/webgpu/jointBilateralFilter.d.ts

@@ -0,0 +1,7 @@
+import type { RendererOptions } from '../../types/render';
+type JointBilateralFilterParams = Pick<RendererOptions, 'sigmaSpace' | 'sigmaRangeLo' | 'sigmaRangeHi' | 'backgroundThreshold' | 'foregroundThreshold'>;
+export declare const createJointBilateralFilterRenderer: (processor: OffscreenCanvas, device: GPUDevice, isContextLost: () => boolean, width: number, height: number, maskFormat: GPUTextureFormat) => {
+    render: (encoder: GPUCommandEncoder, maskTexture: GPUTexture, guideTexture: GPUTexture, params: JointBilateralFilterParams) => GPUTexture;
+    release: () => void;
+};
+export {};

package/dist/common/backends/webgpu/jointBilateralFilter.js

@@ -0,0 +1,162 @@
+import { makeShaderDataDefinitions, makeStructuredView } from 'webgpu-utils';
+import { assert } from '@pexip/utils';
+import { RENDERING_EVENTS } from '../../constants';
+import jointBilateralFilterHorizontalShader from './shaders/jointBilateralFilterHorizontal.wgsl';
+import jointBilateralFilterVerticalShader from './shaders/jointBilateralFilterVertical.wgsl';
+import { updateTextureStorageFormat } from './webgpuUtils';
+export const createJointBilateralFilterRenderer = (processor, device, isContextLost, width, height, maskFormat) => {
+    // Dispatch workgroups based on the shader's workgroup size (64x4)
+    const TILE_W = 32;
+    const TILE_ROWS = 4;
+    const TILE_H = 32;
+    const TILE_COLS = 4;
+    // Create intermediate texture for the filtered result
+    const tempTexture = device.createTexture({
+        label: 'Joint Bilateral Filtered temporary mask texture',
+        format: maskFormat,
+        size: [width, height],
+        usage: GPUTextureUsage.TEXTURE_BINDING | GPUTextureUsage.STORAGE_BINDING,
+    });
+    const outputTexture = device.createTexture({
+        label: 'Joint Bilateral Filtered Mask Texture',
+        format: maskFormat,
+        size: [width, height],
+        usage: GPUTextureUsage.TEXTURE_BINDING | GPUTextureUsage.STORAGE_BINDING,
+    });
+    const defs = makeShaderDataDefinitions(jointBilateralFilterHorizontalShader);
+    // @ts-expect-error This is what we get from the API
+    const uniforms = makeStructuredView(defs.uniforms.params);
+    // Create params buffer
+    const paramsBuffer = device.createBuffer({
+        size: uniforms.arrayBuffer.byteLength,
+        usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST,
+    });
+    const scaleFactor = width / processor.width;
+    const texelWidth = 1 / width;
+    const texelHeight = 1 / height;
+    uniforms.set({
+        width,
+        height,
+        texelWidth,
+        texelHeight,
+    });
+    const updateParams = (params) => {
+        uniforms.set({
+            ...params,
+            sigmaSpace: params.sigmaSpace * scaleFactor,
+        });
+        device.queue.writeBuffer(paramsBuffer, 0, uniforms.arrayBuffer);
+    };
+    // Generate shader code with appropriate format
+    const shaderHCode = updateTextureStorageFormat(jointBilateralFilterHorizontalShader, maskFormat);
+    const shaderVCode = updateTextureStorageFormat(jointBilateralFilterVerticalShader, maskFormat);
+    // Cache shader module and pipeline at init time
+    const shaderHModule = device.createShaderModule({
+        label: 'joint bilateral filter horizontal compute',
+        code: shaderHCode,
+    });
+    const shaderVModule = device.createShaderModule({
+        label: 'joint bilateral filter vertical compute',
+        code: shaderVCode,
+    });
+    const computeHPipeline = device.createComputePipeline({
+        label: 'joint bilateral filter horizontal pipeline',
+        layout: 'auto',
+        compute: { module: shaderHModule },
+    });
+    const computeVPipeline = device.createComputePipeline({
+        label: 'joint bilateral filter vertical pipeline',
+        layout: 'auto',
+        compute: { module: shaderVModule },
+    });
+    const uniformsHBindGroups = device.createBindGroup({
+        layout: computeHPipeline.getBindGroupLayout(1),
+        entries: [
+            { binding: 0, resource: { buffer: paramsBuffer } },
+            { binding: 1, resource: tempTexture.createView() },
+        ],
+    });
+    const uniformsVBindGroups = device.createBindGroup({
+        layout: computeVPipeline.getBindGroupLayout(1),
+        entries: [
+            { binding: 0, resource: { buffer: paramsBuffer } },
+            { binding: 1, resource: outputTexture.createView() },
+        ],
+    });
+    const compute = (label, encoder, pipeline, bindGroup0, bindGroup1, workgroupCountX, workgroupCountY) => {
+        const computePass = encoder.beginComputePass({ label });
+        computePass.setPipeline(pipeline);
+        computePass.setBindGroup(0, bindGroup0);
+        computePass.setBindGroup(1, bindGroup1);
+        computePass.dispatchWorkgroups(workgroupCountX, workgroupCountY);
+        computePass.end();
+    };
+    const wgCntXH = Math.ceil(width / TILE_W);
+    const wgCntYH = Math.ceil(height / TILE_ROWS);
+    const wgCntXV = Math.ceil(width / TILE_COLS);
+    const wgCntYV = Math.ceil(height / TILE_H);
+    let cacheMaskTexture = null;
+    let cacheGuideTexture = null;
+    let cacheBindGroupH = null;
+    let cacheBindGroupV = null;
+    const cache = {
+        sigmaSpace: -1,
+        sigmaRangeLo: -1,
+        sigmaRangeHi: -1,
+        backgroundThreshold: -1,
+        foregroundThreshold: -1,
+    };
+    return {
+        render: (encoder, maskTexture, guideTexture, params) => {
+            assert(!isContextLost(), RENDERING_EVENTS.ContextLost);
+            if (!(cache.sigmaSpace === params.sigmaSpace &&
+                cache.sigmaRangeLo === params.sigmaRangeLo &&
+                cache.sigmaRangeHi === params.sigmaRangeHi &&
+                cache.backgroundThreshold === params.backgroundThreshold &&
+                cache.foregroundThreshold === params.foregroundThreshold)) {
+                updateParams(params);
+                cache.sigmaSpace = params.sigmaSpace;
+                cache.sigmaRangeLo = params.sigmaRangeLo;
+                cache.sigmaRangeHi = params.sigmaRangeHi;
+                cache.backgroundThreshold = params.backgroundThreshold;
+                cache.foregroundThreshold = params.foregroundThreshold;
+            }
+            // Horizontal
+            if (cacheMaskTexture !== maskTexture ||
+                cacheGuideTexture !== guideTexture) {
+                cacheBindGroupH = device.createBindGroup({
+                    layout: computeHPipeline.getBindGroupLayout(0),
+                    entries: [
+                        { binding: 0, resource: maskTexture.createView() },
+                        { binding: 1, resource: guideTexture.createView() },
+                    ],
+                });
+                cacheMaskTexture = maskTexture;
+            }
+            // Horizontal pipeline
+            compute('Joint Bilateral Filter Horizontal pass', encoder, computeHPipeline, cacheBindGroupH, uniformsHBindGroups, wgCntXH, wgCntYH);
+            // Vertical
+            if (cacheGuideTexture !== guideTexture) {
+                cacheBindGroupV = device.createBindGroup({
+                    layout: computeVPipeline.getBindGroupLayout(0),
+                    entries: [
+                        { binding: 0, resource: tempTexture.createView() },
+                        { binding: 1, resource: guideTexture.createView() },
+                    ],
+                });
+                cacheGuideTexture = guideTexture;
+            }
+            compute('Joint Bilateral Filter Vertical pass', encoder, computeVPipeline, cacheBindGroupV, uniformsVBindGroups, wgCntXV, wgCntYV);
+            return outputTexture;
+        },
+        release: () => {
+            tempTexture.destroy();
+            outputTexture.destroy();
+            paramsBuffer.destroy();
+            cacheMaskTexture = null;
+            cacheGuideTexture = null;
+            cacheBindGroupH = null;
+            cacheBindGroupV = null;
+        },
+    };
+};