@pexip/media-processor 19.1.0 → 20.0.0

package/CHANGELOG.md

@@ -1,5 +1,35 @@
 # Changelog
 
+## 20.0.0
+
+### Major Changes
+
+- 2bfd9e1: Adds esm entry points.
+
+### Minor Changes
+
+- 4baa8a4: Implements dual filter blur wglsl shader.
+- 3555d9d: Adds `build:sourcemap` script.
+- 0416d6c: Introduces WebGPU option for video processing.
+- ed53b05: Upgrade Typescript
+- 5855473: Enables dynamic updates of processing width and height.
+
+### Patch Changes
+
+- d3909ed: Enable recommended linter rules
+- e76a15e: Migrate to biome linter
+- Updated dependencies [2bfd9e1]
+- Updated dependencies [4fe77de]
+- Updated dependencies [2bbff64]
+- Updated dependencies [4ea0492]
+- Updated dependencies [3555d9d]
+- Updated dependencies [1601985]
+- Updated dependencies [84956b8]
+- Updated dependencies [ed53b05]
+- Updated dependencies [d681fd3]
+- Updated dependencies [e76a15e]
+  - @pexip/utils@17.0.0
+
 ## 19.1.0
 
 ### Minor Changes

package/dist/common/backends/webgl/blur.js

@@ -64,8 +64,8 @@ export const createBlurRenderer = (processor, pass) => {
         const texture = createAndSetupTexture(gl);
         assert(texture);
         // Avoid fractional value
-        const width = Math.trunc(processor.width / Math.pow(2, idx));
-        const height = Math.trunc(processor.height / Math.pow(2, idx));
+        const width = Math.trunc(processor.width / 2 ** idx);
+        const height = Math.trunc(processor.height / 2 ** idx);
         gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, width, height, 0, gl.RGBA, gl.UNSIGNED_BYTE, null);
         const frameBuffer = gl.createFramebuffer();
         assert(frameBuffer);
@@ -99,7 +99,9 @@ export const createBlurRenderer = (processor, pass) => {
             // Adjust pass number accordingly
             const currentPass = pass + 1;
             if (currentPass > blurBuffers.length) {
-                generateBlurBuffers(currentPass - blurBuffers.length, blurBuffers.length).forEach(buffer => blurBuffers.push(buffer));
+                for (const buffer of generateBlurBuffers(currentPass - blurBuffers.length, blurBuffers.length)) {
+                    blurBuffers.push(buffer);
+                }
             }
             while (currentPass < blurBuffers.length) {
                 const buffer = blurBuffers.pop();
@@ -158,10 +160,10 @@ export const createBlurRenderer = (processor, pass) => {
             };
         },
         release: () => {
-            blurBuffers.forEach(({ frameBuffer, texture }) => {
+            for (const { frameBuffer, texture } of blurBuffers) {
                 gl.deleteTexture(texture);
                 gl.deleteFramebuffer(frameBuffer);
-            });
+            }
             gl.deleteBuffer(positionBuffer);
             gl.deleteBuffer(texCoordBuffer);
             gl.deleteVertexArray(vao);

package/dist/common/backends/webgl/renderer.d.ts

@@ -1,2 +1,2 @@
 import type { Renderer, RenderEventHandlers } from '../../types/render';
-export declare const createRenderer: (canvas: OffscreenCanvas | undefined, renderEventHandlers: RenderEventHandlers) => Renderer;
+export declare const createRenderer: (renderEventHandlers: RenderEventHandlers, canvas?: OffscreenCanvas) => Renderer;

package/dist/common/backends/webgl/renderer.js

@@ -1,27 +1,16 @@
 import { assert } from '@pexip/utils';
 import { PROCESSING_WIDTH, PROCESSING_HEIGHT, EDGE_BLUR_AMOUNT, BACKGROUND_BLUR_AMOUNT, RENDERING_EVENTS, } from '../../constants';
-import { toTextureInfo } from './webglUtils';
 import { createTextureRenderer } from './texture';
 import { createBlurRenderer } from './blur';
 import { createBlendRenderer } from './blender';
 import { createCanvasRenderer } from './canvas';
 import { createTextureToTextureRenderer } from './textureToTexture';
 import { createSmoothingMaskRenderer } from './smoothingMask';
-const createLazyProps = (stateBuilder) => {
-    const props = {};
-    return new Proxy(props, {
-        get(target, p, receiver) {
-            const prop = target[p];
-            if (!prop) {
-                target[p] = stateBuilder[p]?.(props);
-            }
-            return Reflect.get(target, p, receiver);
-        },
-    });
-};
-export const createRenderer = (canvas = new OffscreenCanvas(PROCESSING_WIDTH, PROCESSING_HEIGHT), renderEventHandlers) => {
+import { createLazyProps, handleWebGLContextLoss } from '../../utils';
+export const createRenderer = (renderEventHandlers, canvas = new OffscreenCanvas(PROCESSING_WIDTH, PROCESSING_HEIGHT)) => {
     const rendererCreator = {
         videoFrameRenderer: () => createTextureRenderer(canvas),
+        maskRenderer: () => createTextureRenderer(canvas),
         canvasRenderer: () => createCanvasRenderer(canvas),
         backgroundImageRenderer: () => createTextureRenderer(canvas),
         blendRenderer: () => createBlendRenderer(canvas),
@@ -29,17 +18,26 @@ export const createRenderer = (canvas = new OffscreenCanvas(PROCESSING_WIDTH, PR
         maskBlurRenderer: () => createBlurRenderer(canvas, EDGE_BLUR_AMOUNT),
         textureRenderer: () => createTextureToTextureRenderer(canvas),
         smoothingMaskRenderer: () => createSmoothingMaskRenderer(canvas),
-        release: (states) => () => Object.keys(states).forEach(key => key !== 'release' &&
-            states[key]?.release()),
+        release: (states) => () => {
+            for (const key in states) {
+                if (key !== 'release') {
+                    states[key]?.release();
+                }
+            }
+        },
     };
     let lazyProps = createLazyProps(rendererCreator);
     const props = {
         canvas,
         gl: null,
+        canvasContextEventsCleanup: () => undefined,
         // TODO: refactor to avoid repeated codes
         get videoFrameRenderer() {
             return lazyProps.videoFrameRenderer;
         },
+        get maskRenderer() {
+            return lazyProps.maskRenderer;
+        },
         get canvasRenderer() {
             return lazyProps.canvasRenderer;
         },
@@ -63,21 +61,12 @@ export const createRenderer = (canvas = new OffscreenCanvas(PROCESSING_WIDTH, PR
             return lazyProps.smoothingMaskRenderer;
         },
     };
-    const handleContextLost = (event) => {
-        event.preventDefault();
-        cleanup();
-        renderEventHandlers.contextLost(event.statusMessage);
-    };
-    const handleContextRestored = (event) => {
-        renderEventHandlers.contextRestored(event.statusMessage);
-    };
-    const handleContextCreationError = (event) => {
-        renderEventHandlers.contextCreationError(event.statusMessage);
-    };
     const init = () => {
-        canvas.addEventListener('webglcontextlost', handleContextLost);
-        canvas.addEventListener('webglcontextrestored', handleContextRestored);
-        canvas.addEventListener('webglcontextcreationerror', handleContextCreationError);
+        props.canvasContextEventsCleanup = handleWebGLContextLoss(canvas, {
+            contextLost: renderEventHandlers.contextLost,
+            contextRestored: renderEventHandlers.contextRestored,
+            contextCreationError: renderEventHandlers.contextCreationError,
+        }, cleanup);
         props.gl = canvas.getContext('webgl2', {
             premultipliedAlpha: false,
             powerPreference: 'high-performance',
@@ -86,8 +75,14 @@ export const createRenderer = (canvas = new OffscreenCanvas(PROCESSING_WIDTH, PR
     };
     const render = (mask, frame, options) => {
         assert(!props.gl?.isContextLost(), RENDERING_EVENTS.ContextLost);
+        const maskTexture = props.maskRenderer.render({
+            frame: mask,
+            width: mask.width,
+            height: mask.height,
+            timestamp: frame.timestamp,
+        });
         const smoothedMask = props.smoothingMaskRenderer.render({
-            mask: toTextureInfo(mask),
+            mask: maskTexture,
             prevMask: props.prevMask,
         }, options);
         const frameTexture = props.videoFrameRenderer.render(frame);
@@ -113,14 +108,12 @@ export const createRenderer = (canvas = new OffscreenCanvas(PROCESSING_WIDTH, PR
         props.prevMask = props.textureRenderer.render(smoothedMask);
     };
     const cleanup = () => {
+        props.canvasContextEventsCleanup();
         lazyProps.release();
         lazyProps = createLazyProps(rendererCreator);
         props.prevMask = undefined;
     };
     const release = () => {
-        canvas.removeEventListener('webglcontextlost', handleContextLost);
-        canvas.removeEventListener('webglcontextrestored', handleContextRestored);
-        canvas.removeEventListener('webglcontextcreationerror', handleContextCreationError);
         cleanup();
     };
     const isContextLost = () => {

package/dist/common/backends/webgl/smoothingMask.d.ts

@@ -5,7 +5,7 @@ import type { TextureInfo } from './types';
  */
 export declare const createSmoothingMaskRenderer: (processor: OffscreenCanvas) => {
     render: ({ mask, prevMask, }: {
-        prevMask?: TextureInfo | undefined;
+        prevMask?: TextureInfo;
         mask: TextureInfo;
     }, options: RendererOptions) => TextureInfo;
     release: () => void;

package/dist/common/backends/webgl/webglUtils.d.ts

@@ -61,7 +61,7 @@ export declare const createProgram: (gl: WebGLRenderingContext, shaders: [WebGLS
  *        A parallel array to opt_attribs letting you assign locations.
  */
 export declare const createProgramFromSources: (gl: WebGLRenderingContext, [vs, fs]: [GLShaderSource, GLShaderSource], attribs?: string[], locations?: number[]) => [WebGLProgram, Release];
-export declare const createAndSetupTexture: (gl: WebGL2RenderingContext) => WebGLTexture | null;
+export declare const createAndSetupTexture: (gl: WebGL2RenderingContext) => WebGLTexture;
 type Location<S extends string> = S extends `u_${string}` ? ReturnType<WebGL2RenderingContext['getUniformLocation']> : S extends `a_${string}` ? ReturnType<WebGL2RenderingContext['getAttribLocation']> : never;
 export declare const getLocations: <T extends Record<string, string>>(gl: WebGL2RenderingContext, program: WebGLProgram, locations: T) => { [Key in keyof T]: Location<T[Key]>; };
 export declare const createGLProgram: <A extends Record<string, string>, U extends Record<string, string>>(processCanvas: OffscreenCanvas, { attribs, uniforms, vertexSrc, fragmentSrc, }: {

package/dist/common/backends/webgl/webglUtils.js

@@ -37,7 +37,7 @@ function addLineNumbersWithError(src, log = '', lineOffset = 0) {
         if (!m[1] || m.index === undefined) {
             return [0, log];
         }
-        const lineNo = parseInt(m[1], 10);
+        const lineNo = Number.parseInt(m[1], 10);
         const next = matches[ndx + 1];
         const end = next ? next.index : log.length;
         const msg = log.substring(m.index, end);
@@ -93,10 +93,10 @@ export const createProgram = (gl, shaders, attribs, locations) => {
     if (!program) {
         throw new Error('Unable to create WebGL program');
     }
-    shaders.forEach(function (shader) {
+    for (const shader of shaders) {
         gl.attachShader(program, shader);
-    });
-    attribs?.forEach(function (attrib, ndx) {
+    }
+    attribs?.forEach((attrib, ndx) => {
         const locationIdx = locations?.[ndx] ?? ndx;
         gl.bindAttribLocation(program, locationIdx, attrib);
     });
@@ -118,10 +118,10 @@ export const createProgram = (gl, shaders, attribs, locations) => {
     return [
         program,
         () => {
-            shaders.forEach(shader => {
+            for (const shader of shaders) {
                 gl.detachShader(program, shader);
                 gl.deleteShader(shader);
-            });
+            }
             gl.deleteProgram(program);
         },
     ];
@@ -156,12 +156,12 @@ export const getLocations = (gl, program, locations) => Object.keys(locations).r
     if (!location) {
         return acc;
     }
-    return {
-        ...acc,
+    Object.assign(acc, {
         [key]: location.startsWith('u_')
             ? gl.getUniformLocation(program, location)
             : gl.getAttribLocation(program, location),
-    };
+    });
+    return acc;
 }, {});
 export const createGLProgram = (processCanvas, { attribs, uniforms, vertexSrc, fragmentSrc, }) => {
     const gl = processCanvas.getContext('webgl2', {
@@ -192,4 +192,3 @@ export const toTextureInfo = (mask) => ({
     width: mask.width,
     height: mask.height,
 });
-/* eslint-enable max-params --- this is utils */

package/dist/common/backends/webgpu/blender.d.ts

@@ -0,0 +1,8 @@
+/**
+ *
+ * Blend foreground and background according to the mask
+ */
+export declare const createBlendRenderer: (processor: OffscreenCanvas, device: GPUDevice, presentationFormat: GPUTextureFormat, isContextLost: () => boolean) => {
+    render: (frameTexture: GPUTexture, bgTexture: GPUTexture, maskTexture: GPUTexture) => void;
+    release: () => void;
+};

package/dist/common/backends/webgpu/blender.js

@@ -0,0 +1,78 @@
+import { assert } from '@pexip/utils';
+import { RENDERING_EVENTS } from '../../constants';
+import blenderShader from './shaders/blenderShader.wgsl';
+import { RENDER_PASS_COLOR_ATTACHMENT_CLEAR_VALUE } from './constants';
+/**
+ *
+ * Blend foreground and background according to the mask
+ */
+export const createBlendRenderer = (processor, device, presentationFormat, isContextLost) => {
+    const context = processor.getContext('webgpu');
+    if (!context) {
+        throw Error('WebGPU not supported on the canvas.');
+    }
+    context.configure({
+        device,
+        format: presentationFormat,
+        alphaMode: 'premultiplied',
+    });
+    return {
+        render: (frameTexture, bgTexture, maskTexture) => {
+            assert(!isContextLost(), RENDERING_EVENTS.ContextLost);
+            const pipeline = device.createRenderPipeline({
+                label: 'blender pipeline',
+                layout: 'auto',
+                vertex: {
+                    module: device.createShaderModule({
+                        label: 'blender vertex',
+                        code: blenderShader,
+                    }),
+                },
+                fragment: {
+                    module: device.createShaderModule({
+                        label: 'blender fragment',
+                        code: blenderShader,
+                    }),
+                    targets: [{ format: presentationFormat }],
+                },
+            });
+            const encoder = device.createCommandEncoder({
+                label: 'blender encoder',
+            });
+            const pass = encoder.beginRenderPass({
+                label: 'blender renderPass',
+                colorAttachments: [
+                    {
+                        view: context.getCurrentTexture().createView(),
+                        clearValue: RENDER_PASS_COLOR_ATTACHMENT_CLEAR_VALUE,
+                        loadOp: 'clear',
+                        storeOp: 'store',
+                    },
+                ],
+            });
+            pass.setPipeline(pipeline);
+            const sampler = device.createSampler({
+                addressModeU: 'repeat',
+                addressModeV: 'repeat',
+                magFilter: 'nearest',
+            });
+            const bindGroup = device.createBindGroup({
+                layout: pipeline.getBindGroupLayout(0),
+                entries: [
+                    { binding: 0, resource: sampler },
+                    { binding: 1, resource: bgTexture.createView() },
+                    { binding: 2, resource: frameTexture.createView() },
+                    { binding: 3, resource: maskTexture.createView() },
+                ],
+            });
+            pass.setBindGroup(0, bindGroup);
+            pass.draw(6); // call our vertex shader 6 times
+            pass.end();
+            const commandBuffer = encoder.finish();
+            device.queue.submit([commandBuffer]);
+        },
+        release: () => {
+            // Nothing
+        },
+    };
+};

package/dist/common/backends/webgpu/constants.d.ts

@@ -0,0 +1 @@
+export declare const RENDER_PASS_COLOR_ATTACHMENT_CLEAR_VALUE: number[];

package/dist/common/backends/webgpu/constants.js

@@ -0,0 +1 @@
+export const RENDER_PASS_COLOR_ATTACHMENT_CLEAR_VALUE = [0.0, 0.0, 0.0, 0];

package/dist/common/backends/webgpu/dualFilterBlur.d.ts

@@ -0,0 +1,5 @@
+import type { VideoFrameLike } from '../../types/media';
+export declare const createBlurRenderer: (processor: OffscreenCanvas, device: GPUDevice, presentationFormat: GPUTextureFormat, isContextLost: () => boolean) => {
+    render: (frame: VideoFrameLike, pass: number) => GPUTexture;
+    release: () => void;
+};

package/dist/common/backends/webgpu/dualFilterBlur.js

@@ -0,0 +1,201 @@
+import { assert } from '@pexip/utils';
+import { RENDERING_EVENTS } from '../../constants';
+import blurExample from './shaders/dualBlurShader.wgsl';
+import { clamping } from '../../utils';
+import { RENDER_PASS_COLOR_ATTACHMENT_CLEAR_VALUE } from './constants';
+const clampBlurAmount = clamping(1, 9);
+export const createBlurRenderer = (processor, device, presentationFormat, isContextLost) => {
+    const context = processor.getContext('webgpu');
+    if (!context) {
+        throw Error('WebGPU not supported on the canvas.');
+    }
+    context.configure({
+        device,
+        format: presentationFormat,
+        alphaMode: 'premultiplied',
+    });
+    // Helper to create a texture of given size
+    const createTexture = (width, height) => device.createTexture({
+        format: presentationFormat,
+        size: [width, height],
+        usage: GPUTextureUsage.TEXTURE_BINDING |
+            GPUTextureUsage.COPY_DST |
+            GPUTextureUsage.RENDER_ATTACHMENT,
+    });
+    // Helper to create a uniform buffer to flip downsampling vs upsampling
+    const createFlipBuffer = (value) => {
+        const buffer = device.createBuffer({
+            size: 4,
+            mappedAtCreation: true,
+            usage: GPUBufferUsage.UNIFORM,
+        });
+        new Uint32Array(buffer.getMappedRange())[0] = value;
+        buffer.unmap();
+        return buffer;
+    };
+    const sampler = device.createSampler({
+        magFilter: 'linear',
+        minFilter: 'linear',
+    });
+    const shaderModule = device.createShaderModule({ code: blurExample });
+    const pipeline = device.createRenderPipeline({
+        label: 'dual blur render pipeline',
+        layout: 'auto',
+        vertex: {
+            module: shaderModule,
+            entryPoint: 'vert_main',
+            buffers: [],
+        },
+        fragment: {
+            module: shaderModule,
+            entryPoint: 'frag_main',
+            targets: [{ format: presentationFormat }],
+        },
+        primitive: { topology: 'triangle-list' },
+    });
+    // --- State for persistent textures and buffers ---
+    let blurTextures = [];
+    let lastWidth = -1;
+    let lastHeight = -1;
+    // Helper to (re)generate textures if needed
+    function ensureBlurTextures(pass, width, height) {
+        const length = clampBlurAmount(pass + 1);
+        let needsUpdate = false;
+        if (blurTextures.length !== length ||
+            lastWidth !== width ||
+            lastHeight !== height) {
+            // Destroy old textures
+            for (const t of blurTextures) {
+                t.texture.destroy();
+            }
+            blurTextures = [];
+            for (let i = 0; i < length; i++) {
+                const w = Math.max(Math.trunc(width / 2 ** i), 1);
+                const h = Math.max(Math.trunc(height / 2 ** i), 1);
+                blurTextures.push({
+                    texture: createTexture(w, h),
+                    width: w,
+                    height: h,
+                });
+            }
+            lastWidth = width;
+            lastHeight = height;
+            needsUpdate = true;
+        }
+        return needsUpdate;
+    }
+    const flipDownBuffer = createFlipBuffer(0); // create once
+    const flipUpBuffer = createFlipBuffer(1); // create once
+    return {
+        render: (frame, pass) => {
+            assert(!isContextLost(), RENDERING_EVENTS.ContextLost);
+            // Ensure textures and buffers are up-to-date
+            ensureBlurTextures(pass, processor.width, processor.height);
+            // Upload frame to the largest texture (level 0)
+            const largestTexture = blurTextures[0];
+            assert(largestTexture);
+            device.queue.copyExternalImageToTexture({ source: frame.frame, flipY: false }, { texture: largestTexture.texture }, { width: largestTexture.width, height: largestTexture.height });
+            // Downsampling passes
+            for (let i = 1; i < blurTextures.length; i++) {
+                const src = blurTextures[i - 1];
+                assert(src);
+                const dst = blurTextures[i];
+                assert(dst);
+                const bindGroup = device.createBindGroup({
+                    layout: pipeline.getBindGroupLayout(0),
+                    entries: [
+                        { binding: 0, resource: sampler },
+                        { binding: 1, resource: src.texture.createView() },
+                        { binding: 2, resource: { buffer: flipDownBuffer } },
+                    ],
+                });
+                const encoder = device.createCommandEncoder();
+                const passEncoder = encoder.beginRenderPass({
+                    colorAttachments: [
+                        {
+                            view: dst.texture.createView(),
+                            loadOp: 'clear',
+                            storeOp: 'store',
+                            clearValue: RENDER_PASS_COLOR_ATTACHMENT_CLEAR_VALUE,
+                        },
+                    ],
+                });
+                passEncoder.setPipeline(pipeline);
+                passEncoder.setBindGroup(0, bindGroup);
+                passEncoder.draw(6);
+                passEncoder.end();
+                device.queue.submit([encoder.finish()]);
+            }
+            // Upsampling passes
+            for (let i = blurTextures.length - 2; i >= 0; i--) {
+                const src = blurTextures[i + 1];
+                assert(src);
+                const dst = blurTextures[i];
+                assert(dst);
+                const bindGroup = device.createBindGroup({
+                    layout: pipeline.getBindGroupLayout(0),
+                    entries: [
+                        { binding: 0, resource: sampler },
+                        { binding: 1, resource: src.texture.createView() },
+                        { binding: 2, resource: { buffer: flipUpBuffer } },
+                    ],
+                });
+                const encoder = device.createCommandEncoder();
+                const passEncoder = encoder.beginRenderPass({
+                    colorAttachments: [
+                        {
+                            view: dst.texture.createView(),
+                            loadOp: 'clear',
+                            storeOp: 'store',
+                            clearValue: RENDER_PASS_COLOR_ATTACHMENT_CLEAR_VALUE,
+                        },
+                    ],
+                });
+                passEncoder.setPipeline(pipeline);
+                passEncoder.setBindGroup(0, bindGroup);
+                passEncoder.draw(6);
+                passEncoder.end();
+                device.queue.submit([encoder.finish()]);
+            }
+            // Final pass: draw to canvas
+            const finalTexture = blurTextures[0];
+            assert(finalTexture);
+            const finalBindGroup = device.createBindGroup({
+                layout: pipeline.getBindGroupLayout(0),
+                entries: [
+                    { binding: 0, resource: sampler },
+                    {
+                        binding: 1,
+                        resource: finalTexture.texture.createView(),
+                    },
+                    { binding: 2, resource: { buffer: flipUpBuffer } },
+                ],
+            });
+            const encoder = device.createCommandEncoder();
+            const passEncoder = encoder.beginRenderPass({
+                colorAttachments: [
+                    {
+                        view: context.getCurrentTexture().createView(),
+                        loadOp: 'clear',
+                        storeOp: 'store',
+                        clearValue: RENDER_PASS_COLOR_ATTACHMENT_CLEAR_VALUE,
+                    },
+                ],
+            });
+            passEncoder.setPipeline(pipeline);
+            passEncoder.setBindGroup(0, finalBindGroup);
+            passEncoder.draw(6);
+            passEncoder.end();
+            device.queue.submit([encoder.finish()]);
+            return finalTexture.texture;
+        },
+        release: () => {
+            for (const t of blurTextures) {
+                t.texture.destroy();
+            }
+            blurTextures = [];
+            flipDownBuffer.destroy();
+            flipUpBuffer.destroy();
+        },
+    };
+};

package/dist/common/backends/webgpu/renderer.d.ts

@@ -0,0 +1,2 @@
+import type { Renderer, RenderEventHandlers } from '../../types/render';
+export declare const createRenderer: (renderEventHandlers: RenderEventHandlers, canvas?: OffscreenCanvas) => Renderer;