@pexip/media-processor 20.3.4 → 21.0.0

package/api-docs/variables/SEGMENTATION_MODELS.mdx

@@ -0,0 +1,10 @@
+```ts
+const SEGMENTATION_MODELS: object;
+```
+
+## Type Declaration
+
+| Name | Type |
+| ------ | ------ |
+| <a id="mediapipeselfie"></a> `MediapipeSelfie` | `"selfie"` |
+| <a id="deeplabv3"></a> `DeepLabV3` | `"deeplabV3"` |

package/api-docs/variables/SIGMA_RANGE.mdx

@@ -0,0 +1,3 @@
+```ts
+const SIGMA_RANGE: [number, number];
+```

package/api-docs/variables/SIGMA_SPACE.mdx

@@ -0,0 +1,3 @@
+```ts
+const SIGMA_SPACE: 5 = 5;
+```

package/api-docs/variables/SILENT_THRESHOLD.mdx

@@ -0,0 +1,6 @@
+```ts
+const SILENT_THRESHOLD: number;
+```
+
+Default silent threshold
+At least one LSB 16-bit data (compare is on absolute value).

package/api-docs/variables/STRONG_EDGE_BLUR_AMOUNT.mdx

@@ -0,0 +1,3 @@
+```ts
+const STRONG_EDGE_BLUR_AMOUNT: 2 = 2;
+```

package/api-docs/variables/VOICE_PROBABILITY_THRESHOLD.mdx

@@ -0,0 +1,5 @@
+```ts
+const VOICE_PROBABILITY_THRESHOLD: 0.3 = 0.3;
+```
+
+Default Voice probability threshold

package/api-docs/variables/calculateMaxBlurPass.mdx

@@ -0,0 +1,13 @@
+```ts
+const calculateMaxBlurPass: (height) => number;
+```
+
+## Parameters
+
+| Parameter | Type |
+| ------ | ------ |
+| `height` | `number` |
+
+## Returns
+
+`number`

package/api-docs/variables/clamping.mdx

@@ -0,0 +1,26 @@
+```ts
+const clamping: (min, max) => (value) => number;
+```
+
+## Parameters
+
+| Parameter | Type |
+| ------ | ------ |
+| `min` | `number` |
+| `max` | `number` |
+
+## Returns
+
+```ts
+(value): number;
+```
+
+### Parameters
+
+| Parameter | Type |
+| ------ | ------ |
+| `value` | `number` |
+
+### Returns
+
+`number`

package/api-docs/variables/createLazyProps.mdx

@@ -0,0 +1,19 @@
+```ts
+const createLazyProps: <T>(stateBuilder) => T;
+```
+
+## Type Parameters
+
+| Type Parameter |
+| ------ |
+| `T` *extends* `Record`\<`string`, `unknown`\> |
+
+## Parameters
+
+| Parameter | Type |
+| ------ | ------ |
+| `stateBuilder` | `{ [K in keyof T]: (s: Partial<T>) => T[K] }` |
+
+## Returns
+
+`T`

package/api-docs/variables/createObjectUpdater.mdx

@@ -0,0 +1,31 @@
+```ts
+const createObjectUpdater: <T>(original) => (another) => void;
+```
+
+## Type Parameters
+
+| Type Parameter |
+| ------ |
+| `T` *extends* `{ [K in keyof T]: unknown }` |
+
+## Parameters
+
+| Parameter | Type |
+| ------ | ------ |
+| `original` | `T` |
+
+## Returns
+
+```ts
+(another): void;
+```
+
+### Parameters
+
+| Parameter | Type |
+| ------ | ------ |
+| `another` | `Partial`\<`T`\> |
+
+### Returns
+
+`void`

package/api-docs/variables/createRemoteImageBitmap.mdx

@@ -0,0 +1,21 @@
+```ts
+const createRemoteImageBitmap: (map?) => object;
+```
+
+## Parameters
+
+| Parameter | Type |
+| ------ | ------ |
+| `map?` | [`ImageMapIterable`](../type-aliases/ImageMapIterable.mdx) |
+
+## Returns
+
+`object`
+
+| Name | Type |
+| ------ | ------ |
+| `tryFetchingImage()` | (`uri`) => `Promise`\<`ImageBitmap` \| `null`\> |
+| `setImage()` | (`uri`, `image`) => `void` |
+| `getImage()` | (`uri`) => `ImageBitmap` \| `null` |
+| `removeImage()` | (`uri`) => `boolean` |
+| `clear()` | () => `void` |

package/api-docs/variables/getCanvasRenderingContext2D.mdx

@@ -0,0 +1,14 @@
+```ts
+const getCanvasRenderingContext2D: (canvas, options?) => OffscreenCanvasRenderingContext2D;
+```
+
+## Parameters
+
+| Parameter | Type |
+| ------ | ------ |
+| `canvas` | `OffscreenCanvas` |
+| `options?` | `CanvasRenderingContext2DOptions` |
+
+## Returns
+
+`OffscreenCanvasRenderingContext2D`

package/api-docs/variables/getErrorMessage.mdx

@@ -0,0 +1,13 @@
+```ts
+const getErrorMessage: (error) => string;
+```
+
+## Parameters
+
+| Parameter | Type |
+| ------ | ------ |
+| `error` | `unknown` |
+
+## Returns
+
+`string`

package/api-docs/variables/getImageSize.mdx

@@ -0,0 +1,13 @@
+```ts
+const getImageSize: (image) => Size;
+```
+
+## Parameters
+
+| Parameter | Type |
+| ------ | ------ |
+| `image` | `CanvasImageSource` |
+
+## Returns
+
+`Size`

package/api-docs/variables/handleWebGLContextLoss.mdx

@@ -0,0 +1,21 @@
+```ts
+const handleWebGLContextLoss: (canvas, eventHandlers, contextLostCb?) => () => void;
+```
+
+## Parameters
+
+| Parameter | Type |
+| ------ | ------ |
+| `canvas` | `OffscreenCanvas` |
+| `eventHandlers` | [`RenderEventHandlers`](../interfaces/RenderEventHandlers.mdx) |
+| `contextLostCb?` | () => `void` |
+
+## Returns
+
+```ts
+(): void;
+```
+
+### Returns
+
+`void`

package/api-docs/variables/isRenderingEvents.mdx

@@ -0,0 +1,13 @@
+```ts
+const isRenderingEvents: (t) => t is RenderingEvents;
+```
+
+## Parameters
+
+| Parameter | Type |
+| ------ | ------ |
+| `t` | `unknown` |
+
+## Returns
+
+`t is RenderingEvents`

package/api-docs/variables/isSegmentationModel.mdx

@@ -0,0 +1,13 @@
+```ts
+const isSegmentationModel: (t) => t is SegmentationModel;
+```
+
+## Parameters
+
+| Parameter | Type |
+| ------ | ------ |
+| `t` | `unknown` |
+
+## Returns
+
+`t is SegmentationModel`

package/api-docs/variables/resize.mdx

@@ -0,0 +1,30 @@
+```ts
+const resize: (fromWidth, fromHeight, toWidth, toHeight) => object;
+```
+
+Calculate the coordinates and size of the source and destination to fit the
+target aspect ratio.
+
+## Parameters
+
+| Parameter | Type | Description |
+| ------ | ------ | ------ |
+| `fromWidth` | `number` | The width of the source |
+| `fromHeight` | `number` | The height of the source |
+| `toWidth` | `number` | The width of the destination |
+| `toHeight` | `number` | The height of the destination |
+
+## Returns
+
+`object`
+
+| Name | Type |
+| ------ | ------ |
+| `sx` | `number` |
+| `sy` | `number` |
+| `sw` | `number` |
+| `sh` | `number` |
+| `dx` | `number` |
+| `dy` | `number` |
+| `dw` | `number` |
+| `dh` | `number` |

package/api-docs/variables/toRenderingEvents.mdx

@@ -0,0 +1,13 @@
+```ts
+const toRenderingEvents: (t) => RenderingEvents;
+```
+
+## Parameters
+
+| Parameter | Type |
+| ------ | ------ |
+| `t` | `unknown` |
+
+## Returns
+
+[`RenderingEvents`](../type-aliases/RenderingEvents.mdx)

package/api-docs/variables/urls.mdx

@@ -0,0 +1,10 @@
+```ts
+const urls: object;
+```
+
+## Type Declaration
+
+| Name | Type |
+| ------ | ------ |
+| <a id="denoise"></a> `denoise()` | () => `URL` |
+| <a id="mediaworker"></a> `mediaWorker()` | () => `URL` |

package/dist/common/backends/webgl/binaryMask.d.ts

@@ -0,0 +1,8 @@
+import type { TextureInfo } from './types';
+/**
+ * Create a binary mask renderer based on the provided threshold
+ */
+export declare const createBinaryMaskRenderer: (processor: OffscreenCanvas, width: number, height: number) => {
+    render: (mask: TextureInfo, threshold: number) => TextureInfo;
+    release: () => void;
+};

package/dist/common/backends/webgl/binaryMask.js

@@ -0,0 +1,95 @@
+import { assert } from '@pexip/utils';
+import { m3 } from '../../m3';
+import { RENDERING_EVENTS } from '../../constants';
+import { createGLProgram, createMaskTexture, createFrameBuffer, } from './webglUtils';
+import vertexSrc from './shaders/matrix.vert';
+import fragmentSrc from './shaders/binaryMask.frag';
+/**
+ * Create a binary mask renderer based on the provided threshold
+ */
+export const createBinaryMaskRenderer = (processor, width, height) => {
+    const { gl, program, attribLocations, uniformLocations, floatFullChannelSupported, floatSingleChannelSupported, release, } = createGLProgram(processor, {
+        attribs: {
+            position: 'a_position',
+            texCoord: 'a_texCoord',
+        },
+        uniforms: {
+            matrix: 'u_matrix',
+            mask: 'u_mask',
+            size: 'u_size',
+            threshold: 'u_threshold',
+        },
+        vertexSrc,
+        fragmentSrc,
+    });
+    const vao = gl.createVertexArray();
+    gl.bindVertexArray(vao);
+    // Position
+    const positionBuffer = gl.createBuffer();
+    gl.enableVertexAttribArray(attribLocations.position);
+    gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
+    gl.vertexAttribPointer(attribLocations.position, 2, gl.FLOAT, false, 0, 0);
+    gl.bufferData(gl.ARRAY_BUFFER, 
+    // prettier-ignore
+    new Float32Array([
+        0, 0,
+        0, 1,
+        1, 0,
+        1, 0,
+        0, 1,
+        1, 1,
+    ]), gl.STATIC_DRAW);
+    // Texture coordinates for the rectangle
+    const texCoordBuffer = gl.createBuffer();
+    gl.bindBuffer(gl.ARRAY_BUFFER, texCoordBuffer);
+    gl.bufferData(gl.ARRAY_BUFFER, 
+    // prettier-ignore
+    new Float32Array([
+        0, 0,
+        0, 1,
+        1, 0,
+        1, 0,
+        0, 1,
+        1, 1,
+    ]), gl.STATIC_DRAW);
+    gl.enableVertexAttribArray(attribLocations.texCoord);
+    gl.vertexAttribPointer(attribLocations.texCoord, 2, gl.FLOAT, false, 0, 0);
+    const outputTexture = createMaskTexture(gl, width, height, false, floatSingleChannelSupported, floatFullChannelSupported);
+    const frameBuffer = createFrameBuffer(gl, outputTexture);
+    return {
+        render: (mask, threshold) => {
+            assert(!gl.isContextLost(), RENDERING_EVENTS.ContextLost);
+            gl.clearColor(0, 0, 0, 0);
+            gl.clear(gl.COLOR_BUFFER_BIT);
+            gl.useProgram(program);
+            gl.bindVertexArray(vao);
+            gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer);
+            gl.viewport(0, 0, width, height);
+            // Textures
+            gl.activeTexture(gl.TEXTURE0);
+            gl.uniform1i(uniformLocations.mask, 0);
+            gl.bindTexture(gl.TEXTURE_2D, mask.texture);
+            gl.uniform2i(uniformLocations.size, mask.width, mask.height);
+            gl.uniform1f(uniformLocations.threshold, threshold);
+            // No Flip Y Coordinate for drawing on the frame buffer
+            // Project, position and scale to full-size of the canvas
+            const matrix = m3.projection(1, 1, 1);
+            gl.uniformMatrix3fv(uniformLocations.matrix, false, matrix);
+            gl.drawArrays(gl.TRIANGLES, 0, 6);
+            // Cleanup
+            gl.bindBuffer(gl.ARRAY_BUFFER, null);
+            gl.bindFramebuffer(gl.FRAMEBUFFER, null);
+            gl.bindTexture(gl.TEXTURE_2D, null);
+            gl.bindVertexArray(null);
+            return { texture: outputTexture, width, height };
+        },
+        release: () => {
+            gl.deleteBuffer(positionBuffer);
+            gl.deleteBuffer(texCoordBuffer);
+            gl.deleteTexture(outputTexture);
+            gl.deleteFramebuffer(frameBuffer);
+            gl.deleteVertexArray(vao);
+            release();
+        },
+    };
+};

package/dist/common/backends/webgl/blender.d.ts

@@ -1,3 +1,4 @@
+import type { RendererOptions } from '../../types/render';
 import type { TextureInfo } from './types';
 /**
  * Blend foreground and background according to the mask
@@ -7,6 +8,6 @@ export declare const createBlendRenderer: (processor: OffscreenCanvas) => {
         mask: TextureInfo;
         foreground: TextureInfo;
         background: TextureInfo;
-    }) => TextureInfo;
+    }, options: RendererOptions) => TextureInfo;
     release: () => void;
 };

package/dist/common/backends/webgl/blender.js

@@ -1,26 +1,27 @@
 import { assert } from '@pexip/utils';
 import { RENDERING_EVENTS } from '../../constants';
 import { m3 } from '../../m3';
-import { createGLProgram, createAndSetupTexture } from './webglUtils';
-import { vertexShaderSource } from './shaders/vertexShader';
-import { blendFragmentShaderSource } from './shaders/blendFragmentShader';
+import { createFrameBuffer, createGLProgram, createRGBAFrameTexture, } from './webglUtils';
+import vertexSrc from './shaders/matrix.vert';
+import fragmentSrc from './shaders/blend.frag';
 /**
  * Blend foreground and background according to the mask
  */
 export const createBlendRenderer = (processor) => {
-    const { gl, program, attribLocations, uniformLocations, release } = createGLProgram(processor, {
+    const { gl, program, attribLocations, floatFullChannelSupported, uniformLocations, release, } = createGLProgram(processor, {
         attribs: {
             position: 'a_position',
             texCoord: 'a_texCoord',
         },
         uniforms: {
             matrix: 'u_matrix',
+            mask: 'u_mask',
             frameBackground: 'u_frame_background',
             frameForeground: 'u_frame_foreground',
-            mask: 'u_mask',
+            confidenceThresholds: 'u_confidence_thresholds',
         },
-        vertexSrc: vertexShaderSource,
-        fragmentSrc: blendFragmentShaderSource,
+        vertexSrc,
+        fragmentSrc,
     });
     gl.enable(gl.BLEND);
     const vao = gl.createVertexArray();
@@ -55,18 +56,16 @@ export const createBlendRenderer = (processor) => {
     ]), gl.STATIC_DRAW);
     gl.enableVertexAttribArray(attribLocations.texCoord);
     gl.vertexAttribPointer(attribLocations.texCoord, 2, gl.FLOAT, false, 0, 0);
-    const outputTexture = createAndSetupTexture(gl);
-    gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, processor.width, processor.height, 0, gl.RGBA, gl.UNSIGNED_BYTE, null);
-    const frameBuffer = gl.createFramebuffer();
-    gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer);
-    gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, outputTexture, 0);
+    const outputTexture = createRGBAFrameTexture(gl, processor.width, processor.height, true, floatFullChannelSupported);
+    const frameBuffer = createFrameBuffer(gl, outputTexture);
     return {
-        render: ({ mask, foreground, background, }) => {
+        render: ({ mask, foreground, background, }, options) => {
             assert(!gl.isContextLost(), RENDERING_EVENTS.ContextLost);
+            gl.useProgram(program);
             gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer);
             gl.viewport(0, 0, processor.width, processor.height);
-            gl.useProgram(program);
             gl.bindVertexArray(vao);
+            gl.uniform2f(uniformLocations.confidenceThresholds, options.backgroundThreshold, options.foregroundThreshold);
             // Textures
             gl.activeTexture(gl.TEXTURE0);
             gl.uniform1i(uniformLocations.frameForeground, 0);
@@ -82,7 +81,11 @@ export const createBlendRenderer = (processor) => {
             const matrix = m3.projection(1, 1, 1);
             gl.uniformMatrix3fv(uniformLocations.matrix, false, matrix);
             gl.drawArrays(gl.TRIANGLES, 0, 6);
-            assert(outputTexture);
+            // Cleanup
+            gl.bindBuffer(gl.ARRAY_BUFFER, null);
+            gl.bindFramebuffer(gl.FRAMEBUFFER, null);
+            gl.bindTexture(gl.TEXTURE_2D, null);
+            gl.bindVertexArray(null);
             return {
                 texture: outputTexture,
                 width: processor.width,

package/dist/common/backends/webgl/blur.d.ts

@@ -2,7 +2,7 @@ import type { TextureInfo } from './types';
 /**
  * Blur provided frame
  */
-export declare const createBlurRenderer: (processor: OffscreenCanvas, pass: number) => {
+export declare const createBlurRenderer: (processor: OffscreenCanvas, pass: number, redOnly?: boolean) => {
     render: (frame: TextureInfo, pass: number) => TextureInfo;
     release: () => void;
 };

package/dist/common/backends/webgl/blur.js

@@ -2,15 +2,15 @@ import { assert } from '@pexip/utils';
 import { clamping } from '../../utils';
 import { RENDERING_EVENTS } from '../../constants';
 import { m3 } from '../../m3';
-import { createAndSetupTexture, createGLProgram } from './webglUtils';
-import { vertexShaderSource } from './shaders/vertexShader';
-import { dualFilterBlurFragmentShaderSource } from './shaders/dualFilterBlurFragmentShader';
+import { createFrameBuffer, createGLProgram, createMaskTexture, createRGBAFrameTexture, } from './webglUtils';
+import vertexSrc from './shaders/matrix.vert';
+import fragmentSrc from './shaders/dualFilterBlur.frag';
 const clampBlurAmount = clamping(1, 9);
 /**
  * Blur provided frame
  */
-export const createBlurRenderer = (processor, pass) => {
-    const { gl, program, attribLocations, uniformLocations, release } = createGLProgram(processor, {
+export const createBlurRenderer = (processor, pass, redOnly = false) => {
+    const { gl, program, attribLocations, uniformLocations, floatSingleChannelSupported, floatFullChannelSupported, release, } = createGLProgram(processor, {
         attribs: {
             position: 'a_position',
             texCoord: 'a_texCoord',
@@ -20,8 +20,8 @@ export const createBlurRenderer = (processor, pass) => {
             downsampling: 'u_downsampling',
             frame: 'u_frame',
         },
-        vertexSrc: vertexShaderSource,
-        fragmentSrc: dualFilterBlurFragmentShaderSource,
+        vertexSrc,
+        fragmentSrc,
     });
     const vao = gl.createVertexArray();
     gl.bindVertexArray(vao);
@@ -61,18 +61,13 @@ export const createBlurRenderer = (processor, pass) => {
         gl.viewport(0, 0, width, height);
     };
     const createFramebufferAndTexture = (idx) => {
-        const texture = createAndSetupTexture(gl);
-        assert(texture);
         // Avoid fractional value
         const width = Math.trunc(processor.width / 2 ** idx);
         const height = Math.trunc(processor.height / 2 ** idx);
-        gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, width, height, 0, gl.RGBA, gl.UNSIGNED_BYTE, null);
-        const frameBuffer = gl.createFramebuffer();
-        assert(frameBuffer);
-        gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer);
-        // Attach a texture to it.
-        const attachmentPoint = gl.COLOR_ATTACHMENT0;
-        gl.framebufferTexture2D(gl.FRAMEBUFFER, attachmentPoint, gl.TEXTURE_2D, texture, 0);
+        const texture = redOnly
+            ? createMaskTexture(gl, width, height, true, floatSingleChannelSupported, floatFullChannelSupported)
+            : createRGBAFrameTexture(gl, width, height, true, floatFullChannelSupported);
+        const frameBuffer = createFrameBuffer(gl, texture);
         return { texture, frameBuffer, width, height };
     };
     const generateBlurBuffers = (pass, offset = 0) => {
@@ -87,6 +82,13 @@ export const createBlurRenderer = (processor, pass) => {
     return {
         render: (frame, pass) => {
             assert(!gl.isContextLost(), RENDERING_EVENTS.ContextLost);
+            if (!pass) {
+                return {
+                    texture: frame.texture,
+                    width: frame.width,
+                    height: frame.height,
+                };
+            }
             gl.useProgram(program);
             gl.bindVertexArray(vao);
             // Frame texture
@@ -153,6 +155,11 @@ export const createBlurRenderer = (processor, pass) => {
                 // for the next draw, use the texture we just rendered to.
                 gl.bindTexture(gl.TEXTURE_2D, texture);
             }
+            // Cleanup
+            gl.bindBuffer(gl.ARRAY_BUFFER, null);
+            gl.bindFramebuffer(gl.FRAMEBUFFER, null);
+            gl.bindTexture(gl.TEXTURE_2D, null);
+            gl.bindVertexArray(null);
             return {
                 texture: blurBuffers[0]?.texture ?? frame.texture,
                 width: blurBuffers[0]?.width ?? frame.width,

package/dist/common/backends/webgl/canvas.js

@@ -2,8 +2,8 @@ import { assert } from '@pexip/utils';
 import { m3 } from '../../m3';
 import { RENDERING_EVENTS } from '../../constants';
 import { createGLProgram } from './webglUtils';
-import { vertexShaderSource } from './shaders/vertexShader';
-import { textureFragmentShaderSource } from './shaders/textureFragmentShader';
+import vertexSrc from './shaders/matrix.vert';
+import fragmentSrc from './shaders/texture.frag';
 /**
  * Render texture to the processing canvas framebuffer
  */
@@ -17,8 +17,8 @@ export const createCanvasRenderer = (processor) => {
             matrix: 'u_matrix',
             texture: 'u_texture',
         },
-        vertexSrc: vertexShaderSource,
-        fragmentSrc: textureFragmentShaderSource,
+        vertexSrc,
+        fragmentSrc,
     });
     const vao = gl.createVertexArray();
     gl.bindVertexArray(vao);
@@ -72,6 +72,11 @@ export const createCanvasRenderer = (processor) => {
             gl.activeTexture(gl.TEXTURE0);
             gl.bindTexture(gl.TEXTURE_2D, texture.texture);
             gl.drawArrays(gl.TRIANGLES, 0, 6);
+            // Cleanup
+            gl.bindBuffer(gl.ARRAY_BUFFER, null);
+            gl.bindFramebuffer(gl.FRAMEBUFFER, null);
+            gl.bindTexture(gl.TEXTURE_2D, null);
+            gl.bindVertexArray(null);
         },
         release: () => {
             gl.deleteBuffer(positionBuffer);

package/dist/common/backends/webgl/diffAbsMask.d.ts

@@ -0,0 +1,10 @@
+import type { TextureInfo } from './types';
+/**
+ * Create absulute difference renderer to caculate the difference between 2 textures
+ */
+export declare const createDiffAbsRenderer: (processor: OffscreenCanvas, width: number, height: number) => {
+    render: (textureA: TextureInfo, textureB: TextureInfo) => TextureInfo & {
+        frameBuffer: WebGLFramebuffer;
+    };
+    release: () => void;
+};