@pexip/media-processor 20.3.5 → 21.0.0

package/dist/common/backends/webgl/selectComponent.d.ts

@@ -0,0 +1,15 @@
+import type { RendererOptions } from '../../types/render';
+import type { TextureInfo } from './types';
+/**
+ * Based on Connected Component Labelling algorithm,
+ * find the center-most connected component in the mask and
+ * return a binary mask of just that component.
+ *
+ * @param processor - OffscreenCanvas for WebGL context
+ * @param width - target width of the downsampled mask to process (e.g. 32)
+ * @param height - target height of the downsampled mask to process (e.g. 18)
+ */
+export declare function createComponentSelector(processor: OffscreenCanvas, width: number, height: number): {
+    render: (softMask: TextureInfo, options: RendererOptions) => TextureInfo;
+    release: () => void;
+};

package/dist/common/backends/webgl/selectComponent.js

@@ -0,0 +1,102 @@
+import { assert } from '@pexip/utils';
+import { RENDERING_EVENTS } from '../../constants';
+import { createTextureToTextureRenderer } from './textureToTexture';
+import { scoreComponent, labelComponents, iou } from '../../utils';
+/**
+ * Based on Connected Component Labelling algorithm,
+ * find the center-most connected component in the mask and
+ * return a binary mask of just that component.
+ *
+ * @param processor - OffscreenCanvas for WebGL context
+ * @param width - target width of the downsampled mask to process (e.g. 32)
+ * @param height - target height of the downsampled mask to process (e.g. 18)
+ */
+export function createComponentSelector(processor, width, height) {
+    const gl = processor.getContext('webgl2', {
+        premultipliedAlpha: false,
+        powerPreference: 'high-performance',
+    });
+    const downSampler = createTextureToTextureRenderer(processor, true, false, width, height, false);
+    assert(gl, RENDERING_EVENTS.ContextCreationError);
+    const labels = new Int32Array(width * height);
+    const data = new Uint8Array(width * height);
+    const bin = new Uint8Array(width * height);
+    const bestMask = new Uint8Array(width * height);
+    const labelledMask = new Uint8Array(width * height);
+    let prevPrimaryScore = 0;
+    const outputTexture = gl.createTexture();
+    gl.bindTexture(gl.TEXTURE_2D, outputTexture);
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
+    const fbo = gl.createFramebuffer();
+    const config = {
+        // Frames persisted = -Math.log(1 + switchThreshold) / Math.log(decay)
+        // Width 0.3 switchThreshold and 0.95 decay, it persists about 6 frames
+        switchThreshold: 0.3,
+        decay: 0.95,
+        // Weights
+        area: 0.2,
+        height: 0.4,
+        center: 0.2,
+        iou: 0.2,
+    };
+    return {
+        render: (softMask, options) => {
+            const smallMask = downSampler.render(softMask);
+            // Read back single-channel 8-bit for simplicity here
+            gl.bindFramebuffer(gl.FRAMEBUFFER, fbo);
+            gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, smallMask.texture, 0);
+            gl.readPixels(0, 0, width, height, gl.RED, gl.UNSIGNED_BYTE, data);
+            gl.bindFramebuffer(gl.FRAMEBUFFER, null);
+            // Binarize using foregroundThreshold in 0..255 domain
+            const thresh = Math.max(0, Math.min(255, Math.floor(255 * options.foregroundThreshold)));
+            for (let i = 0; i < bin.length; i++) {
+                //@ts-expect-error will not be undefined
+                bin[i] = data[i] >= thresh ? 255 : 0;
+            }
+            let bestScore = -Infinity;
+            let bestLabel = 0;
+            const stats = labelComponents(bin, labels, width, height);
+            for (const [l, s] of stats.entries()) {
+                if (l === 0) {
+                    continue; // background
+                }
+                for (let i = 0; i < labels.length; i++) {
+                    labelledMask[i] = labels[i] === l ? 255 : 0;
+                }
+                const score = scoreComponent(s, iou(bestMask, labelledMask), {
+                    ...config,
+                    centerX: options.personCenter[0] * width,
+                    centerY: options.personCenter[1] * height,
+                }, width, height);
+                if (score > bestScore) {
+                    bestScore = score;
+                    bestLabel = l;
+                }
+            }
+            // Hysteresis: switch only if significantly better
+            const shouldSwitch = bestScore > prevPrimaryScore * (1.0 + config.switchThreshold);
+            if (shouldSwitch || prevPrimaryScore === 0) {
+                for (let i = 0; i < labels.length; i++) {
+                    bestMask[i] = labels[i] === bestLabel ? 255 : 0;
+                }
+                prevPrimaryScore = bestScore;
+            }
+            else {
+                prevPrimaryScore *= config.decay;
+            }
+            // Upload small filtered mask to GPU and return as TextureInfo (caller will upsample & combine)
+            gl.bindTexture(gl.TEXTURE_2D, outputTexture);
+            gl.texImage2D(gl.TEXTURE_2D, 0, gl.R8, width, height, 0, gl.RED, gl.UNSIGNED_BYTE, bestMask);
+            gl.bindTexture(gl.TEXTURE_2D, null);
+            return { texture: outputTexture, width, height };
+        },
+        release: () => {
+            gl.deleteTexture(outputTexture);
+            gl.deleteFramebuffer(fbo);
+            downSampler.release();
+        },
+    };
+}

package/dist/common/backends/webgl/smoothingMask.d.ts

@@ -3,9 +3,9 @@ import type { TextureInfo } from './types';
 /**
  * Smooth the mask edge
  */
-export declare const createSmoothingMaskRenderer: (processor: OffscreenCanvas) => {
-    render: ({ mask, prevMask, }: {
-        prevMask?: TextureInfo;
+export declare const createSmoothingMaskRenderer: (processor: OffscreenCanvas, width: number, height: number) => {
+    render: ({ mask, prevMask }: {
+        prevMask: TextureInfo;
         mask: TextureInfo;
     }, options: RendererOptions) => TextureInfo;
     release: () => void;

package/dist/common/backends/webgl/smoothingMask.js

@@ -1,14 +1,14 @@
 import { assert } from '@pexip/utils';
 import { m3 } from '../../m3';
 import { RENDERING_EVENTS } from '../../constants';
-import { createGLProgram, createAndSetupTexture } from './webglUtils';
-import { vertexShaderSource } from './shaders/vertexShader';
-import { smoothingMaskFragmentShaderSource } from './shaders/smoothingMaskFragmentShader';
+import { createGLProgram, createFrameBuffer, createMaskTexture, } from './webglUtils';
+import vertexSrc from './shaders/matrix.vert';
+import fragmentSrc from './shaders/smoothingMask.frag';
 /**
  * Smooth the mask edge
  */
-export const createSmoothingMaskRenderer = (processor) => {
-    const { gl, program, attribLocations, uniformLocations, release } = createGLProgram(processor, {
+export const createSmoothingMaskRenderer = (processor, width, height) => {
+    const { gl, program, attribLocations, uniformLocations, floatFullChannelSupported, floatSingleChannelSupported, release, } = createGLProgram(processor, {
         attribs: {
             position: 'a_position',
             texCoord: 'a_texCoord',
@@ -16,12 +16,13 @@ export const createSmoothingMaskRenderer = (processor) => {
         uniforms: {
             matrix: 'u_matrix',
             mask: 'u_mask',
-            maskCombineRatio: 'u_mask_combine_ratio',
-            foregroundThreshold: 'u_foreground_threshold',
             prevMask: 'u_mask_prev',
+            frame: 'u_frame',
+            prevFrame: 'u_frame_prev',
+            maskCombineRatio: 'u_mask_combine_ratio',
         },
-        vertexSrc: vertexShaderSource,
-        fragmentSrc: smoothingMaskFragmentShaderSource,
+        vertexSrc,
+        fragmentSrc,
     });
     const vao = gl.createVertexArray();
     gl.bindVertexArray(vao);
@@ -55,16 +56,13 @@ export const createSmoothingMaskRenderer = (processor) => {
     ]), gl.STATIC_DRAW);
     gl.enableVertexAttribArray(attribLocations.texCoord);
     gl.vertexAttribPointer(attribLocations.texCoord, 2, gl.FLOAT, false, 0, 0);
-    const outputTexture = createAndSetupTexture(gl);
-    gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, processor.width, processor.height, 0, gl.RGBA, gl.UNSIGNED_BYTE, null);
-    const frameBuffer = gl.createFramebuffer();
-    gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer);
-    gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, outputTexture, 0);
+    const outputTexture = createMaskTexture(gl, width, height, false, floatSingleChannelSupported, floatFullChannelSupported);
+    const frameBuffer = createFrameBuffer(gl, outputTexture);
     return {
-        render: ({ mask, prevMask, }, options) => {
+        render: ({ mask, prevMask }, options) => {
             assert(!gl.isContextLost(), RENDERING_EVENTS.ContextLost);
             gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer);
-            gl.viewport(0, 0, processor.width, processor.height);
+            gl.viewport(0, 0, mask.width, mask.height);
             gl.clearColor(0, 0, 0, 0);
             gl.clear(gl.COLOR_BUFFER_BIT);
             gl.useProgram(program);
@@ -75,19 +73,22 @@ export const createSmoothingMaskRenderer = (processor) => {
             gl.bindTexture(gl.TEXTURE_2D, mask.texture);
             gl.activeTexture(gl.TEXTURE1);
             gl.uniform1i(uniformLocations.prevMask, 1);
-            gl.bindTexture(gl.TEXTURE_2D, prevMask?.texture ?? mask.texture);
+            gl.bindTexture(gl.TEXTURE_2D, prevMask.texture);
             // No Flip Y Coordinate for drawing on the frame buffer
             // Project, position and scale to full-size of the canvas
             const matrix = m3.projection(1, 1, 1);
             gl.uniformMatrix3fv(uniformLocations.matrix, false, matrix);
             gl.uniform1f(uniformLocations.maskCombineRatio, options.maskCombineRatio);
-            gl.uniform1f(uniformLocations.foregroundThreshold, options.foregroundThreshold);
             gl.drawArrays(gl.TRIANGLES, 0, 6);
-            assert(outputTexture);
+            // Cleanup
+            gl.bindBuffer(gl.ARRAY_BUFFER, null);
+            gl.bindFramebuffer(gl.FRAMEBUFFER, null);
+            gl.bindTexture(gl.TEXTURE_2D, null);
+            gl.bindVertexArray(null);
             return {
                 texture: outputTexture,
-                width: processor.width,
-                height: processor.height,
+                width,
+                height,
             };
         },
         release: () => {

package/dist/common/backends/webgl/tentBlur.d.ts

@@ -0,0 +1,9 @@
+import type { TextureInfo } from './types';
+/**
+ * Create a tent blur renderer for small radius blurring of a mask.
+ * The blur is implemented as a separable filter.
+ */
+export declare const createTentBlurRenderer: (processor: OffscreenCanvas, width: number, height: number) => {
+    render: (mask: TextureInfo, radiusPx: number) => TextureInfo;
+    release: () => void;
+};

package/dist/common/backends/webgl/tentBlur.js

@@ -0,0 +1,118 @@
+import { assert } from '@pexip/utils';
+import { m3 } from '../../m3';
+import { RENDERING_EVENTS } from '../../constants';
+import { clamping } from '../../utils';
+import { createGLProgram, createMaskTexture, createFrameBuffer, } from './webglUtils';
+import vertexSrc from './shaders/matrix.vert';
+import fragmentSrc from './shaders/tentBlur.frag';
+const MAX_RADIUS_PX = 16;
+const clampRadiusPx = clamping(0, MAX_RADIUS_PX);
+/**
+ * Create a tent blur renderer for small radius blurring of a mask.
+ * The blur is implemented as a separable filter.
+ */
+export const createTentBlurRenderer = (processor, width, height) => {
+    const { gl, program, attribLocations, uniformLocations, floatSingleChannelSupported, floatFullChannelSupported, release, } = createGLProgram(processor, {
+        attribs: {
+            position: 'a_position',
+            texCoord: 'a_texCoord',
+        },
+        uniforms: {
+            matrix: 'u_matrix',
+            mask: 'u_mask',
+            texelSize: 'u_texel_size',
+            direction: 'u_direction',
+            radiusPx: 'u_radius_px',
+        },
+        vertexSrc,
+        fragmentSrc,
+    });
+    const vao = gl.createVertexArray();
+    gl.bindVertexArray(vao);
+    // Position
+    const positionBuffer = gl.createBuffer();
+    gl.enableVertexAttribArray(attribLocations.position);
+    gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
+    gl.vertexAttribPointer(attribLocations.position, 2, gl.FLOAT, false, 0, 0);
+    gl.bufferData(gl.ARRAY_BUFFER, 
+    // prettier-ignore
+    new Float32Array([
+        0, 0,
+        0, 1,
+        1, 0,
+        1, 0,
+        0, 1,
+        1, 1,
+    ]), gl.STATIC_DRAW);
+    // Texture coordinates for the rectangle
+    const texCoordBuffer = gl.createBuffer();
+    gl.bindBuffer(gl.ARRAY_BUFFER, texCoordBuffer);
+    gl.bufferData(gl.ARRAY_BUFFER, 
+    // prettier-ignore
+    new Float32Array([
+        0, 0,
+        0, 1,
+        1, 0,
+        1, 0,
+        0, 1,
+        1, 1,
+    ]), gl.STATIC_DRAW);
+    gl.enableVertexAttribArray(attribLocations.texCoord);
+    gl.vertexAttribPointer(attribLocations.texCoord, 2, gl.FLOAT, false, 0, 0);
+    const outputTexturePing = createMaskTexture(gl, width, height, false, floatSingleChannelSupported, floatFullChannelSupported);
+    const frameBufferPing = createFrameBuffer(gl, outputTexturePing);
+    const outputTexturePong = createMaskTexture(gl, width, height, false, floatSingleChannelSupported, floatFullChannelSupported);
+    const frameBufferPong = createFrameBuffer(gl, outputTexturePong);
+    const scaleFactor = processor.width / width;
+    return {
+        render: (mask, radiusPx) => {
+            assert(!gl.isContextLost(), RENDERING_EVENTS.ContextLost);
+            gl.clearColor(0, 0, 0, 0);
+            gl.clear(gl.COLOR_BUFFER_BIT);
+            gl.useProgram(program);
+            gl.bindVertexArray(vao);
+            gl.uniform2f(uniformLocations.texelSize, 1 / mask.width, 1 / mask.height);
+            gl.uniform1i(uniformLocations.radiusPx, clampRadiusPx(Math.floor(radiusPx / scaleFactor)));
+            const matrix = m3.projection(1, 1, 1);
+            // No Flip Y Coordinate for drawing on the frame buffer
+            // Project, position and scale to full-size of the canvas
+            gl.uniformMatrix3fv(uniformLocations.matrix, false, matrix);
+            let write = frameBufferPing;
+            let read = mask.texture;
+            // Ping-pong framebuffers for multiple passes
+            // Separable
+            for (let j = 0; j < 2; j++) {
+                // Textures
+                gl.activeTexture(gl.TEXTURE0);
+                gl.uniform1i(uniformLocations.mask, 0);
+                gl.bindTexture(gl.TEXTURE_2D, read);
+                // Horizontal then Vertical
+                gl.uniform2f(uniformLocations.direction, 1.0 - j, 0.0 + j);
+                gl.bindFramebuffer(gl.FRAMEBUFFER, write);
+                gl.viewport(0, 0, mask.width, mask.height);
+                gl.drawArrays(gl.TRIANGLES, 0, 6);
+                // Swap
+                [read, write] =
+                    write === frameBufferPing
+                        ? [outputTexturePing, frameBufferPong]
+                        : [outputTexturePong, frameBufferPing];
+            }
+            // Cleanup
+            gl.bindBuffer(gl.ARRAY_BUFFER, null);
+            gl.bindFramebuffer(gl.FRAMEBUFFER, null);
+            gl.bindTexture(gl.TEXTURE_2D, null);
+            gl.bindVertexArray(null);
+            return { texture: read, width, height };
+        },
+        release: () => {
+            gl.deleteBuffer(positionBuffer);
+            gl.deleteBuffer(texCoordBuffer);
+            gl.deleteTexture(outputTexturePing);
+            gl.deleteTexture(outputTexturePong);
+            gl.deleteFramebuffer(frameBufferPing);
+            gl.deleteFramebuffer(frameBufferPong);
+            gl.deleteVertexArray(vao);
+            release();
+        },
+    };
+};

package/dist/common/backends/webgl/texture.d.ts

@@ -3,7 +3,7 @@ import type { TextureInfo } from './types';
 /**
  * Render Video frame into texture
  */
-export declare const createTextureRenderer: (processor: OffscreenCanvas) => {
+export declare const createTextureRenderer: (processor: OffscreenCanvas, redOnly: boolean) => {
     render: (video: VideoFrameLike) => TextureInfo;
     release: () => void;
 };

package/dist/common/backends/webgl/texture.js

@@ -1,14 +1,14 @@
 import { assert } from '@pexip/utils';
 import { RENDERING_EVENTS } from '../../constants';
 import { m3 } from '../../m3';
-import { createAndSetupTexture, createGLProgram } from './webglUtils';
-import { vertexShaderSource } from './shaders/vertexShader';
-import { textureFragmentShaderSource } from './shaders/textureFragmentShader';
+import { createFrameBuffer, createGLProgram, createMaskTexture, createRGBAFrameTexture, } from './webglUtils';
+import vertexSrc from './shaders/matrix.vert';
+import fragmentSrc from './shaders/texture.frag';
 /**
  * Render Video frame into texture
  */
-export const createTextureRenderer = (processor) => {
-    const { gl, program, attribLocations, uniformLocations, release } = createGLProgram(processor, {
+export const createTextureRenderer = (processor, redOnly) => {
+    const { gl, program, attribLocations, uniformLocations, floatFullChannelSupported, floatSingleChannelSupported, release, } = createGLProgram(processor, {
         attribs: {
             position: 'a_position',
             texCoord: 'a_texCoord',
@@ -17,8 +17,8 @@ export const createTextureRenderer = (processor) => {
             matrix: 'u_matrix',
             texture: 'u_texture',
         },
-        vertexSrc: vertexShaderSource,
-        fragmentSrc: textureFragmentShaderSource,
+        vertexSrc,
+        fragmentSrc,
     });
     const vao = gl.createVertexArray();
     gl.bindVertexArray(vao);
@@ -53,12 +53,11 @@ export const createTextureRenderer = (processor) => {
     ]), gl.STATIC_DRAW);
     gl.enableVertexAttribArray(attribLocations.texCoord);
     gl.vertexAttribPointer(attribLocations.texCoord, 2, gl.FLOAT, false, 0, 0);
-    const frameTexture = createAndSetupTexture(gl);
-    const outputTexture = createAndSetupTexture(gl);
-    gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, processor.width, processor.height, 0, gl.RGBA, gl.UNSIGNED_BYTE, null);
-    const frameBuffer = gl.createFramebuffer();
-    gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer);
-    gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, outputTexture, 0);
+    const frameTexture = createRGBAFrameTexture(gl, processor.width, processor.height, true, floatFullChannelSupported);
+    const outputTexture = redOnly
+        ? createMaskTexture(gl, processor.width, processor.height, false, floatSingleChannelSupported, floatFullChannelSupported)
+        : createRGBAFrameTexture(gl, processor.width, processor.height, true, floatFullChannelSupported);
+    const frameBuffer = createFrameBuffer(gl, outputTexture);
     return {
         render: (video) => {
             assert(!gl.isContextLost(), RENDERING_EVENTS.ContextLost);
@@ -71,16 +70,19 @@ export const createTextureRenderer = (processor) => {
             gl.bindVertexArray(vao);
             // Texture Unit setup
             gl.uniform1i(uniformLocations.texture, 0);
-            // Project, position and scale to full-size of the canvas
+            // Project, position, and scale to full-size of the canvas
             const matrix = m3.projection(1, 1, 1);
             gl.uniformMatrix3fv(uniformLocations.matrix, false, matrix);
-            // Frame texture
+            // Upload the frame to texture
             gl.activeTexture(gl.TEXTURE0);
             gl.bindTexture(gl.TEXTURE_2D, frameTexture);
             gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, video.frame);
-            gl.generateMipmap(gl.TEXTURE_2D);
             gl.drawArrays(gl.TRIANGLES, 0, 6);
-            assert(outputTexture);
+            // Cleanup
+            gl.bindBuffer(gl.ARRAY_BUFFER, null);
+            gl.bindFramebuffer(gl.FRAMEBUFFER, null);
+            gl.bindTexture(gl.TEXTURE_2D, null);
+            gl.bindVertexArray(null);
             return {
                 texture: outputTexture,
                 width: processor.width,

package/dist/common/backends/webgl/textureToTexture.d.ts

@@ -2,7 +2,7 @@ import type { TextureInfo } from './types';
 /**
  * Render texture into another texture, i.e. clone
  */
-export declare const createTextureToTextureRenderer: (processor: OffscreenCanvas) => {
+export declare const createTextureToTextureRenderer: (processor: OffscreenCanvas, redOnly: boolean, linearSampling?: boolean, targetWidth?: number, targetHeight?: number, float?: boolean) => {
     render: (inputTexture: TextureInfo) => TextureInfo;
     release: () => void;
 };

package/dist/common/backends/webgl/textureToTexture.js

@@ -1,14 +1,14 @@
 import { assert } from '@pexip/utils';
 import { RENDERING_EVENTS } from '../../constants';
 import { m3 } from '../../m3';
-import { createAndSetupTexture, createGLProgram } from './webglUtils';
-import { vertexShaderSource } from './shaders/vertexShader';
-import { textureFragmentShaderSource } from './shaders/textureFragmentShader';
+import { createFrameBuffer, createGLProgram, createMaskTexture, createRGBAFrameTexture, } from './webglUtils';
+import vertexSrc from './shaders/matrix.vert';
+import fragmentSrc from './shaders/texture.frag';
 /**
  * Render texture into another texture, i.e. clone
  */
-export const createTextureToTextureRenderer = (processor) => {
-    const { gl, program, attribLocations, uniformLocations, release } = createGLProgram(processor, {
+export const createTextureToTextureRenderer = (processor, redOnly, linearSampling = false, targetWidth = processor.width, targetHeight = processor.height, float = true) => {
+    const { gl, program, attribLocations, uniformLocations, floatFullChannelSupported, floatSingleChannelSupported, release, } = createGLProgram(processor, {
         attribs: {
             position: 'a_position',
             texCoord: 'a_texCoord',
@@ -17,8 +17,8 @@ export const createTextureToTextureRenderer = (processor) => {
             matrix: 'u_matrix',
             texture: 'u_texture',
         },
-        vertexSrc: vertexShaderSource,
-        fragmentSrc: textureFragmentShaderSource,
+        vertexSrc,
+        fragmentSrc,
     });
     const vao = gl.createVertexArray();
     gl.bindVertexArray(vao);
@@ -53,17 +53,16 @@ export const createTextureToTextureRenderer = (processor) => {
     ]), gl.STATIC_DRAW);
     gl.enableVertexAttribArray(attribLocations.texCoord);
     gl.vertexAttribPointer(attribLocations.texCoord, 2, gl.FLOAT, false, 0, 0);
-    const outputTexture = createAndSetupTexture(gl);
-    gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, processor.width, processor.height, 0, gl.RGBA, gl.UNSIGNED_BYTE, null);
-    const frameBuffer = gl.createFramebuffer();
-    gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer);
-    gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, outputTexture, 0);
+    const outputTexture = redOnly
+        ? createMaskTexture(gl, targetWidth, targetHeight, linearSampling, float && floatSingleChannelSupported, float && floatFullChannelSupported)
+        : createRGBAFrameTexture(gl, targetWidth, targetHeight, linearSampling, float && floatFullChannelSupported);
+    const frameBuffer = createFrameBuffer(gl, outputTexture);
     return {
         render: (inputTexture) => {
             assert(!gl.isContextLost(), RENDERING_EVENTS.ContextLost);
             // Draw on the frame buffer
             gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer);
-            gl.viewport(0, 0, processor.width, processor.height);
+            gl.viewport(0, 0, targetWidth, targetHeight);
             gl.clearColor(0, 0, 0, 1);
             gl.clear(gl.COLOR_BUFFER_BIT);
             gl.useProgram(program);
@@ -76,11 +75,15 @@ export const createTextureToTextureRenderer = (processor) => {
             const matrix = m3.projection(1, 1, 1);
             gl.uniformMatrix3fv(uniformLocations.matrix, false, matrix);
             gl.drawArrays(gl.TRIANGLES, 0, 6);
-            assert(outputTexture);
+            // Cleanup
+            gl.bindBuffer(gl.ARRAY_BUFFER, null);
+            gl.bindFramebuffer(gl.FRAMEBUFFER, null);
+            gl.bindTexture(gl.TEXTURE_2D, null);
+            gl.bindVertexArray(null);
             return {
                 texture: outputTexture,
-                width: processor.width,
-                height: processor.height,
+                width: targetWidth,
+                height: targetHeight,
             };
         },
         release: () => {

package/dist/common/backends/webgl/webglUtils.d.ts

@@ -61,7 +61,11 @@ export declare const createProgram: (gl: WebGLRenderingContext, shaders: [WebGLS
  *        A parallel array to opt_attribs letting you assign locations.
  */
 export declare const createProgramFromSources: (gl: WebGLRenderingContext, [vs, fs]: [GLShaderSource, GLShaderSource], attribs?: string[], locations?: number[]) => [WebGLProgram, Release];
-export declare const createAndSetupTexture: (gl: WebGL2RenderingContext) => WebGLTexture;
+export declare const createAndSetupTexture: (gl: WebGL2RenderingContext, linear: boolean) => WebGLTexture;
+export declare const createRGBAFrameTexture: (gl: WebGL2RenderingContext, width: number, height: number, linear?: boolean, withFloat?: boolean) => WebGLTexture;
+export declare const createMaskTexture: (gl: WebGL2RenderingContext, width: number, height: number, linear?: boolean, withFloatSingle?: boolean, withFloatFull?: boolean) => WebGLTexture;
+export declare const createFrameBuffer: (gl: WebGL2RenderingContext, texture: WebGLTexture) => WebGLFramebuffer;
+export declare const support: (gl: WebGL2RenderingContext, internalFormat: GLint, format: GLenum, width?: number, height?: number) => boolean;
 type Location<S extends string> = S extends `u_${string}` ? ReturnType<WebGL2RenderingContext['getUniformLocation']> : S extends `a_${string}` ? ReturnType<WebGL2RenderingContext['getAttribLocation']> : never;
 export declare const getLocations: <T extends Record<string, string>>(gl: WebGL2RenderingContext, program: WebGLProgram, locations: T) => { [Key in keyof T]: Location<T[Key]>; };
 export declare const createGLProgram: <A extends Record<string, string>, U extends Record<string, string>>(processCanvas: OffscreenCanvas, { attribs, uniforms, vertexSrc, fragmentSrc, }: {
@@ -75,6 +79,8 @@ export declare const createGLProgram: <A extends Record<string, string>, U exten
     attribLocations: { [Key in keyof A]: Location<A[Key]>; };
     uniformLocations: { [Key_1 in keyof U]: Location<U[Key_1]>; };
     release: Release;
+    floatSingleChannelSupported: boolean;
+    floatFullChannelSupported: boolean;
 };
 export declare const toTextureInfo: (mask: MPMask) => TextureInfo;
 export {};

package/dist/common/backends/webgl/webglUtils.js

@@ -1,3 +1,5 @@
+import { assert } from '@pexip/utils';
+import { RENDERING_EVENTS } from '../../constants';
 /**
  * WebGL Shader type enum
  */
@@ -139,16 +141,69 @@ export const createProgramFromSources = (gl, [vs, fs], attribs, locations) => cr
     loadShader(gl, GL_SHADER_TYPE.VertexShader, vs),
     loadShader(gl, GL_SHADER_TYPE.FragmentShader, fs),
 ], attribs, locations);
-export const createAndSetupTexture = (gl) => {
+export const createAndSetupTexture = (gl, linear) => {
     const texture = gl.createTexture();
     gl.bindTexture(gl.TEXTURE_2D, texture);
-    // Set the parameters so we don't need mips and so we're not filtering
+    // Set the parameters so we don't need MIPS and so we're not filtering
     // and we don't repeat at the edges.
     gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
     gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, linear ? gl.LINEAR : gl.NEAREST);
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, linear ? gl.LINEAR : gl.NEAREST);
+    return texture;
+};
+export const createRGBAFrameTexture = (gl, width, height, linear = false, withFloat = false) => {
+    const texture = createAndSetupTexture(gl, linear);
+    if (withFloat) {
+        gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA16F, width, height, 0, gl.RGBA, gl.HALF_FLOAT, null);
+        gl.bindFramebuffer(gl.FRAMEBUFFER, null);
+        return texture;
+    }
+    gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, width, height, 0, gl.RGBA, gl.UNSIGNED_BYTE, null);
+    gl.bindTexture(gl.TEXTURE_2D, null);
+    return texture;
+};
+export const createMaskTexture = (gl, width, height, linear = false, withFloatSingle = false, withFloatFull = false) => {
+    const texture = createAndSetupTexture(gl, linear);
+    if (withFloatSingle) {
+        gl.texImage2D(gl.TEXTURE_2D, 0, gl.R16F, width, height, 0, gl.RED, gl.HALF_FLOAT, null);
+        gl.bindFramebuffer(gl.FRAMEBUFFER, null);
+        return texture;
+    }
+    if (withFloatFull) {
+        gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA16F, width, height, 0, gl.RGBA, gl.HALF_FLOAT, null);
+        gl.bindFramebuffer(gl.FRAMEBUFFER, null);
+        return texture;
+    }
+    gl.texImage2D(gl.TEXTURE_2D, 0, gl.R8, width, height, 0, gl.RED, gl.UNSIGNED_BYTE, null);
+    gl.bindTexture(gl.TEXTURE_2D, null);
+    return texture;
+};
+export const createFrameBuffer = (gl, texture) => {
+    const frameBuffer = gl.createFramebuffer();
+    gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer);
+    gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, texture, 0);
+    gl.bindFramebuffer(gl.FRAMEBUFFER, null);
+    return frameBuffer;
+};
+export const support = (gl, internalFormat, format, width = 2, height = 2) => {
+    // Create single-channel float16 texture
+    const tex = gl.createTexture();
+    gl.bindTexture(gl.TEXTURE_2D, tex);
+    gl.texImage2D(gl.TEXTURE_2D, 0, internalFormat, width, height, 0, format, gl.HALF_FLOAT, null);
     gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
     gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
-    return texture;
+    // Attach texture
+    const fb = createFrameBuffer(gl, tex);
+    gl.bindFramebuffer(gl.FRAMEBUFFER, fb);
+    // Check completeness
+    const complete = gl.checkFramebufferStatus(gl.FRAMEBUFFER) === gl.FRAMEBUFFER_COMPLETE;
+    // Clean up
+    gl.bindFramebuffer(gl.FRAMEBUFFER, null);
+    gl.deleteFramebuffer(fb);
+    gl.bindTexture(gl.TEXTURE_2D, null);
+    gl.deleteTexture(tex);
+    return complete;
 };
 export const getLocations = (gl, program, locations) => Object.keys(locations).reduce((acc, key) => {
     const k = key;
@@ -167,9 +222,7 @@ export const createGLProgram = (processCanvas, { attribs, uniforms, vertexSrc, f
         premultipliedAlpha: false,
         powerPreference: 'high-performance',
     });
-    if (!gl) {
-        throw new Error('WebGL2 is not supported');
-    }
+    assert(gl, RENDERING_EVENTS.ContextCreationError);
     gl.enable(gl.BLEND);
     gl.blendFunc(gl.SRC_ALPHA, gl.ONE_MINUS_SRC_ALPHA);
     assertShaderSource(vertexSrc);
@@ -178,12 +231,17 @@ export const createGLProgram = (processCanvas, { attribs, uniforms, vertexSrc, f
         vertexSrc,
         fragmentSrc,
     ]);
+    const hasFloat = !!gl.getExtension('EXT_color_buffer_float');
+    const floatSingleChannelSupported = hasFloat && support(gl, gl.R16F, gl.RED);
+    const floatFullChannelSupported = hasFloat && support(gl, gl.RGBA16F, gl.RGBA);
     return {
         gl,
         program,
         attribLocations: getLocations(gl, program, attribs),
         uniformLocations: getLocations(gl, program, uniforms),
         release,
+        floatSingleChannelSupported,
+        floatFullChannelSupported,
     };
 };
 export const toTextureInfo = (mask) => ({