@pexip/media-processor 18.3.0 → 18.4.0

package/CHANGELOG.md

@@ -1,5 +1,17 @@
 # Changelog
 
+## 18.4.0
+
+### Minor Changes
+
+- 4bfce08: Decouple blender
+- 31e4c17: Update media-processor's Readme
+
+### Patch Changes
+
+- 36d5672: Remove offscreencanvas types
+  - @pexip/utils@16.12.0
+
 ## 18.3.0
 
 ## 18.2.0

package/README.md

@@ -6,7 +6,7 @@ An package for media data processing using Web API.
 
 `npm install @pexip/media-processor`
 
-## Examples
+## Usages
 
 ### Use Analyzer to get the MediaStream data
 
@@ -105,38 +105,36 @@ await audioGraph.release();
 ### Use background blur
 
 ```typescript
-const PROCESSING_WIDTH = 768;
-const PROCESSING_HEIGHT = 432;
-
+import {
+  createSegmenter,
+  createCanvasTransform,
+  createVideoTrackProcessor,
+  createVideoTrackProcessorWithFallback,
+} from '@pexip/media-processor';
+
+// Grab the user's camera stream
 const stream = await navigator.mediaDevices.getUserMedia({video: true});
 
-// Thus we can create a URL object to get the link to be used for the `gluePath`
-const selfieJs = new URL('@mediapipe/selfie_segmentation', import.meta);
-const segmenter = createMediapipeSegmenter('selfie_segmentation', {
-  modelType: selfieModel,
-  processingWidth: PROCESSING_WIDTH,
-  processingHeight: PROCESSING_HEIGHT,
-  gluePath: selfieJs.href,
-});
-
-const transformer = createCanvasTransform(segmenter, {
-  effects: 'blur',
-  width: mediaState.width,
-  height: mediaState.height,
-});
-
-const getTrackProcessor = () => {
-  // Feature detection if the browser has the `MediaStreamProcessor` API
-  if ('MediaStreamTrackProcessor' in window) {
-    return processAPI === 'Streams API'
-      ? createVideoTrackProcessor() // Using the latest Streams API
-      : createVideoTrackProcessorWithFallback(); // Using the fallback implementation
-  }
-  return createVideoTrackProcessorWithFallback(); // Using the fallback implementation
+// Setting the path to that `@mediapipe/tasks-vision` assets
+// It will be passed direct to
+// [FilesetResolver.forVisionTasks()](https://ai.google.dev/edge/api/mediapipe/js/tasks-vision.filesetresolver#filesetresolverforvisiontasks)
+const tasksVisionBasePath =
+  'A base path to specify the directory the Wasm files should be loaded from';
+
+const modelAsset = {
+  /**
+   * Path to mediapipe selfie segmentation model asset
+   */
+  path: 'A path to selfie segmentation model',
+  modelName: 'selfie' as const,
 };
 
-const videoProcessor = createVideoProcessor([transformer], getTrackProcessor());
+const segmenter = createSegmenter(tasksVisionBasePath, {modelAssets});
+// Create a processing transformer and set the effects to `blur`
+const transformer = createCanvasTransform(segmenter, {effects: 'blur'});
+const processor = createVideoProcessor([transformer]);
 
+// Start the processor
 await videoProcessor.open();
 
 // Passing the raw MediaStream to apply the effects

package/dist/common/backends/webgl/blender.d.ts

@@ -1,11 +1,12 @@
-import type { RendererOptions } from '../../types/render';
 import type { TextureInfo } from './types';
+/**
+ * Blend foreground and background according to the mask
+ */
 export declare const createBlendRenderer: (processor: OffscreenCanvas) => {
-    render: ({ mask, prevMask, foreground, background, }: {
-        prevMask?: TextureInfo | undefined;
+    render: ({ mask, foreground, background, }: {
         mask: TextureInfo;
         foreground: TextureInfo;
         background: TextureInfo;
-    }, options: RendererOptions) => TextureInfo;
+    }) => TextureInfo;
     release: () => void;
 };

package/dist/common/backends/webgl/blender.js

@@ -3,6 +3,9 @@ import { m3 } from '../../m3';
 import { createGLProgram, createAndSetupTexture } from './webglUtils';
 import { vertexShaderSource } from './shaders/vertexShader';
 import { blendFragmentShaderSource } from './shaders/blendFragmentShader';
+/**
+ * Blend foreground and background according to the mask
+ */
 export const createBlendRenderer = (processor) => {
     const { gl, program, attribLocations, uniformLocations, release } = createGLProgram(processor, {
         attribs: {
@@ -14,9 +17,6 @@ export const createBlendRenderer = (processor) => {
             frameBackground: 'u_frame_background',
             frameForeground: 'u_frame_foreground',
             mask: 'u_mask',
-            maskCombineRatio: 'u_mask_combine_ratio',
-            foregroundThreshold: 'u_foreground_threshold',
-            prevMask: 'u_mask_prev',
         },
         vertexSrc: vertexShaderSource,
         fragmentSrc: blendFragmentShaderSource,
@@ -60,7 +60,7 @@ export const createBlendRenderer = (processor) => {
     gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer);
     gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, outputTexture, 0);
     return {
-        render: ({ mask, prevMask, foreground, background, }, options) => {
+        render: ({ mask, foreground, background, }) => {
             gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer);
             gl.viewport(0, 0, processor.width, processor.height);
             gl.useProgram(program);
@@ -75,15 +75,10 @@ export const createBlendRenderer = (processor) => {
             gl.activeTexture(gl.TEXTURE2);
             gl.uniform1i(uniformLocations.mask, 2);
             gl.bindTexture(gl.TEXTURE_2D, mask.texture);
-            gl.activeTexture(gl.TEXTURE3);
-            gl.uniform1i(uniformLocations.prevMask, 3);
-            gl.bindTexture(gl.TEXTURE_2D, prevMask?.texture ?? mask.texture);
             // No Flip Y Coordinate for drawing on the frame buffer
             // Project, position and scale to full-size of the canvas
             const matrix = m3.projection(1, 1, 1);
             gl.uniformMatrix3fv(uniformLocations.matrix, false, matrix);
-            gl.uniform1f(uniformLocations.maskCombineRatio, options.maskCombineRatio);
-            gl.uniform1f(uniformLocations.foregroundThreshold, options.foregroundThreshold);
             gl.drawArrays(gl.TRIANGLES, 0, 6);
             assert(outputTexture);
             return {

package/dist/common/backends/webgl/blur.d.ts

@@ -1,4 +1,7 @@
 import type { TextureInfo } from './types';
+/**
+ * Blur provided frame
+ */
 export declare const createBlurRenderer: (processor: OffscreenCanvas, pass: number) => {
     render: (frame: TextureInfo, pass: number) => TextureInfo;
     release: () => void;

package/dist/common/backends/webgl/blur.js

@@ -4,6 +4,9 @@ import { createAndSetupTexture, createGLProgram } from './webglUtils';
 import { vertexShaderSource } from './shaders/vertexShader';
 import { dualFilterBlurFragmentShaderSource } from './shaders/dualFilterBlurFragmentShader';
 const clampBlurAmount = clamping(1, 9);
+/**
+ * Blur provided frame
+ */
 export const createBlurRenderer = (processor, pass) => {
     const { gl, program, attribLocations, uniformLocations, release } = createGLProgram(processor, {
         attribs: {

package/dist/common/backends/webgl/canvas.d.ts

@@ -1,4 +1,7 @@
 import type { TextureInfo } from './types';
+/**
+ * Render texture to the processing canvas framebuffer
+ */
 export declare const createCanvasRenderer: (processor: OffscreenCanvas) => {
     render: (texture: TextureInfo) => void;
     release: () => void;

package/dist/common/backends/webgl/canvas.js

@@ -2,6 +2,9 @@ import { m3 } from '../../m3';
 import { createGLProgram } from './webglUtils';
 import { vertexShaderSource } from './shaders/vertexShader';
 import { textureFragmentShaderSource } from './shaders/textureFragmentShader';
+/**
+ * Render texture to the processing canvas framebuffer
+ */
 export const createCanvasRenderer = (processor) => {
     const { gl, program, attribLocations, uniformLocations, release } = createGLProgram(processor, {
         attribs: {

package/dist/common/backends/webgl/renderer.js

@@ -5,6 +5,8 @@ import { createTextureRenderer } from './texture';
 import { createBlurRenderer } from './blur';
 import { createBlendRenderer } from './blender';
 import { createCanvasRenderer } from './canvas';
+import { createTextureToTextureRenderer } from './textureToTexture';
+import { createSmoothingMaskRenderer } from './smoothingMask';
 const createLazyProps = (stateBuilder) => {
     const props = {};
     return new Proxy(props, {
@@ -25,6 +27,8 @@ export const createRenderer = (canvas = new OffscreenCanvas(PROCESSING_WIDTH, PR
         blendRenderer: () => createBlendRenderer(canvas),
         blurRenderer: () => createBlurRenderer(canvas, props.backgroundBlurAmount),
         maskBlurRenderer: () => createBlurRenderer(canvas, props.edgeBlurAmount),
+        textureRenderer: () => createTextureToTextureRenderer(canvas),
+        smoothingMaskRenderer: () => createSmoothingMaskRenderer(canvas),
         release: (states) => () => Object.keys(states).forEach(key => key !== 'release' &&
             states[key]?.release()),
     };
@@ -58,6 +62,12 @@ export const createRenderer = (canvas = new OffscreenCanvas(PROCESSING_WIDTH, PR
         get maskBlurRenderer() {
             return lazyProps.maskBlurRenderer;
         },
+        get textureRenderer() {
+            return lazyProps.textureRenderer;
+        },
+        get smoothingMaskRenderer() {
+            return lazyProps.smoothingMaskRenderer;
+        },
     };
     const updateProps = createObjectUpdater(props);
     const update = options => {
@@ -81,6 +91,10 @@ export const createRenderer = (canvas = new OffscreenCanvas(PROCESSING_WIDTH, PR
         if (options) {
             updateProps(options);
         }
+        const smoothedMask = props.smoothingMaskRenderer.render({
+            mask: toTextureInfo(mask),
+            prevMask: props.prevMask,
+        }, props);
         const frameTexture = props.videoFrameRenderer.render(frame);
         let background = frameTexture;
         if (props.effects === 'blur' && props.backgroundBlurAmount) {
@@ -106,20 +120,18 @@ export const createRenderer = (canvas = new OffscreenCanvas(PROCESSING_WIDTH, PR
             }
         }
         const blendedFrameTexture = props.blendRenderer.render({
-            prevMask: props.prevMask && toTextureInfo(props.prevMask),
-            mask: toTextureInfo(mask),
+            mask: smoothedMask,
             foreground: frameTexture,
             background,
-        }, props);
+        });
         props.canvasRenderer.render(blendedFrameTexture);
-        props.prevMask?.close();
-        props.prevMask = mask.clone();
+        // Cloned the current smoothedMask so it can be fed back to the next round
+        props.prevMask = props.textureRenderer.render(smoothedMask);
     };
     const release = () => {
         lazyProps.release();
         lazyProps = createLazyProps(rendererCreator);
         props.backgroundImages.clear();
-        props.prevMask?.close();
         props.prevMask = undefined;
     };
     return { init, render, release, update };

package/dist/common/backends/webgl/shaders/blendFragmentShader.d.ts

@@ -1 +1 @@
-export declare const blendFragmentShaderSource = "#version 300 es\nprecision mediump float;\n\n// our textures\nuniform sampler2D u_mask;\nuniform sampler2D u_mask_prev;\nuniform sampler2D u_frame_foreground;\nuniform sampler2D u_frame_background;\n\n// control flags\nuniform float u_mask_combine_ratio;\nuniform float u_foreground_threshold;\n\nin vec2 v_texCoord;\n\n// we need to declare an output for the fragment shader\nout vec4 outColor;\n\n// Source: https://github.com/google/mediapipe/blob/8750c3dca1bc00fbe47acf501b7c9c61f3f3bd3f/mediapipe/calculators/image/segmentation_smoothing_calculator.cc#L377\nfloat SmoothingMask(sampler2D prevMask, sampler2D currMask, vec2 uv, float ratio) {\n    float prevMaskValue = texture(prevMask, uv).r;\n    float newMaskValue = texture(currMask, uv).r;\n\n    /*\n    * Assume p := newMaskValue\n    * H(p) := 1 + (p * log(p) + (1-p) * log(1-p)) / log(2)\n    * uncertainty alpha(p) =\n    *   Clamp(1 - (1 - H(p)) * (1 - H(p)), 0, 1) [squaring the\n    * uncertainty]\n    *\n    * The following polynomial approximates uncertainty alpha as a\n    * function of (p + 0.5):\n    */\n    const float c1 = 5.68842;\n    const float c2 = -0.748699;\n    const float c3 = -57.8051;\n    const float c4 = 291.309;\n    const float c5 = -624.717;\n    float t = newMaskValue - 0.5;\n    float x = t * t;\n\n    float uncertainty =\n    1.0 - min(1.0, x * (c1 + x * (c2 + x * (c3 + x * (c4 + x * c5)))));\n\n    float outputValue = newMaskValue + (prevMaskValue - newMaskValue) *\n                            (uncertainty * ratio);\n    return outputValue;\n}\n\nvoid main() {\n    vec2 halfpixel = 0.5 / vec2(textureSize(u_frame_background, 0));\n\n    vec4 background = texture(u_frame_background, v_texCoord);\n    vec4 foreground = texture(u_frame_foreground, v_texCoord);\n    float maskAlpha = SmoothingMask(u_mask_prev, u_mask, v_texCoord, u_mask_combine_ratio);\n    float filteredMask = maskAlpha;\n    if (maskAlpha < u_foreground_threshold) {\n        filteredMask = 0.0;\n    }\n\n    outColor = mix(background, foreground, filteredMask);\n}\n";
+export declare const blendFragmentShaderSource = "#version 300 es\nprecision mediump float;\n\n// our textures\nuniform sampler2D u_mask;\nuniform sampler2D u_frame_foreground;\nuniform sampler2D u_frame_background;\n\nin vec2 v_texCoord;\n\n// we need to declare an output for the fragment shader\nout vec4 outColor;\n\nvoid main() {\n    vec4 background = texture(u_frame_background, v_texCoord);\n    vec4 foreground = texture(u_frame_foreground, v_texCoord);\n    float filteredMask = texture(u_mask, v_texCoord).r;\n\n    outColor = mix(background, foreground, filteredMask);\n}\n";

package/dist/common/backends/webgl/shaders/blendFragmentShader.js

@@ -3,60 +3,18 @@ precision mediump float;
 
 // our textures
 uniform sampler2D u_mask;
-uniform sampler2D u_mask_prev;
 uniform sampler2D u_frame_foreground;
 uniform sampler2D u_frame_background;
 
-// control flags
-uniform float u_mask_combine_ratio;
-uniform float u_foreground_threshold;
-
 in vec2 v_texCoord;
 
 // we need to declare an output for the fragment shader
 out vec4 outColor;
 
-// Source: https://github.com/google/mediapipe/blob/8750c3dca1bc00fbe47acf501b7c9c61f3f3bd3f/mediapipe/calculators/image/segmentation_smoothing_calculator.cc#L377
-float SmoothingMask(sampler2D prevMask, sampler2D currMask, vec2 uv, float ratio) {
-    float prevMaskValue = texture(prevMask, uv).r;
-    float newMaskValue = texture(currMask, uv).r;
-
-    /*
-    * Assume p := newMaskValue
-    * H(p) := 1 + (p * log(p) + (1-p) * log(1-p)) / log(2)
-    * uncertainty alpha(p) =
-    *   Clamp(1 - (1 - H(p)) * (1 - H(p)), 0, 1) [squaring the
-    * uncertainty]
-    *
-    * The following polynomial approximates uncertainty alpha as a
-    * function of (p + 0.5):
-    */
-    const float c1 = 5.68842;
-    const float c2 = -0.748699;
-    const float c3 = -57.8051;
-    const float c4 = 291.309;
-    const float c5 = -624.717;
-    float t = newMaskValue - 0.5;
-    float x = t * t;
-
-    float uncertainty =
-    1.0 - min(1.0, x * (c1 + x * (c2 + x * (c3 + x * (c4 + x * c5)))));
-
-    float outputValue = newMaskValue + (prevMaskValue - newMaskValue) *
-                            (uncertainty * ratio);
-    return outputValue;
-}
-
 void main() {
-    vec2 halfpixel = 0.5 / vec2(textureSize(u_frame_background, 0));
-
     vec4 background = texture(u_frame_background, v_texCoord);
     vec4 foreground = texture(u_frame_foreground, v_texCoord);
-    float maskAlpha = SmoothingMask(u_mask_prev, u_mask, v_texCoord, u_mask_combine_ratio);
-    float filteredMask = maskAlpha;
-    if (maskAlpha < u_foreground_threshold) {
-        filteredMask = 0.0;
-    }
+    float filteredMask = texture(u_mask, v_texCoord).r;
 
     outColor = mix(background, foreground, filteredMask);
 }

package/dist/common/backends/webgl/shaders/smoothingMaskFragmentShader.d.ts

@@ -0,0 +1 @@
+export declare const smoothingMaskFragmentShaderSource = "#version 300 es\nprecision mediump float;\n\n// our textures\nuniform sampler2D u_mask;\nuniform sampler2D u_mask_prev;\n\n// control flags\nuniform float u_mask_combine_ratio;\nuniform float u_foreground_threshold;\n\nin vec2 v_texCoord;\n\n// we need to declare an output for the fragment shader\nout vec4 outColor;\n\n// Source: https://github.com/google/mediapipe/blob/8750c3dca1bc00fbe47acf501b7c9c61f3f3bd3f/mediapipe/calculators/image/segmentation_smoothing_calculator.cc#L377\nfloat SmoothingMask(sampler2D prevMask, sampler2D currMask, vec2 uv, float ratio) {\n    float prevMaskValue = texture(prevMask, uv).r;\n    float newMaskValue = texture(currMask, uv).r;\n\n    /*\n    * Assume p := newMaskValue\n    * H(p) := 1 + (p * log(p) + (1-p) * log(1-p)) / log(2)\n    * uncertainty alpha(p) =\n    *   Clamp(1 - (1 - H(p)) * (1 - H(p)), 0, 1) [squaring the\n    * uncertainty]\n    *\n    * The following polynomial approximates uncertainty alpha as a\n    * function of (p + 0.5):\n    */\n    const float c1 = 5.68842;\n    const float c2 = -0.748699;\n    const float c3 = -57.8051;\n    const float c4 = 291.309;\n    const float c5 = -624.717;\n    float t = newMaskValue - 0.5;\n    float x = t * t;\n\n    float uncertainty =\n    1.0 - min(1.0, x * (c1 + x * (c2 + x * (c3 + x * (c4 + x * c5)))));\n\n    float outputValue = newMaskValue + (prevMaskValue - newMaskValue) *\n                            (uncertainty * ratio);\n    return outputValue;\n}\n\nvoid main() {\n    float maskAlpha = SmoothingMask(u_mask_prev, u_mask, v_texCoord, u_mask_combine_ratio);\n    float backgroundMask = maskAlpha;\n    if (backgroundMask < u_foreground_threshold) {\n        backgroundMask = 0.0;\n    }\n\n    outColor = vec4(backgroundMask, backgroundMask, backgroundMask, backgroundMask);\n}\n";

package/dist/common/backends/webgl/shaders/smoothingMaskFragmentShader.js

@@ -0,0 +1,57 @@
+export const smoothingMaskFragmentShaderSource = `#version 300 es
+precision mediump float;
+
+// our textures
+uniform sampler2D u_mask;
+uniform sampler2D u_mask_prev;
+
+// control flags
+uniform float u_mask_combine_ratio;
+uniform float u_foreground_threshold;
+
+in vec2 v_texCoord;
+
+// we need to declare an output for the fragment shader
+out vec4 outColor;
+
+// Source: https://github.com/google/mediapipe/blob/8750c3dca1bc00fbe47acf501b7c9c61f3f3bd3f/mediapipe/calculators/image/segmentation_smoothing_calculator.cc#L377
+float SmoothingMask(sampler2D prevMask, sampler2D currMask, vec2 uv, float ratio) {
+    float prevMaskValue = texture(prevMask, uv).r;
+    float newMaskValue = texture(currMask, uv).r;
+
+    /*
+    * Assume p := newMaskValue
+    * H(p) := 1 + (p * log(p) + (1-p) * log(1-p)) / log(2)
+    * uncertainty alpha(p) =
+    *   Clamp(1 - (1 - H(p)) * (1 - H(p)), 0, 1) [squaring the
+    * uncertainty]
+    *
+    * The following polynomial approximates uncertainty alpha as a
+    * function of (p + 0.5):
+    */
+    const float c1 = 5.68842;
+    const float c2 = -0.748699;
+    const float c3 = -57.8051;
+    const float c4 = 291.309;
+    const float c5 = -624.717;
+    float t = newMaskValue - 0.5;
+    float x = t * t;
+
+    float uncertainty =
+    1.0 - min(1.0, x * (c1 + x * (c2 + x * (c3 + x * (c4 + x * c5)))));
+
+    float outputValue = newMaskValue + (prevMaskValue - newMaskValue) *
+                            (uncertainty * ratio);
+    return outputValue;
+}
+
+void main() {
+    float maskAlpha = SmoothingMask(u_mask_prev, u_mask, v_texCoord, u_mask_combine_ratio);
+    float backgroundMask = maskAlpha;
+    if (backgroundMask < u_foreground_threshold) {
+        backgroundMask = 0.0;
+    }
+
+    outColor = vec4(backgroundMask, backgroundMask, backgroundMask, backgroundMask);
+}
+`;

package/dist/common/backends/webgl/smoothingMask.d.ts

@@ -0,0 +1,12 @@
+import type { RendererOptions } from '../../types/render';
+import type { TextureInfo } from './types';
+/**
+ * Smooth the mask edge
+ */
+export declare const createSmoothingMaskRenderer: (processor: OffscreenCanvas) => {
+    render: ({ mask, prevMask, }: {
+        prevMask?: TextureInfo | undefined;
+        mask: TextureInfo;
+    }, options: RendererOptions) => TextureInfo;
+    release: () => void;
+};

package/dist/common/backends/webgl/smoothingMask.js

@@ -0,0 +1,100 @@
+import { assert } from '../../utils';
+import { m3 } from '../../m3';
+import { createGLProgram, createAndSetupTexture } from './webglUtils';
+import { vertexShaderSource } from './shaders/vertexShader';
+import { smoothingMaskFragmentShaderSource } from './shaders/smoothingMaskFragmentShader';
+/**
+ * Smooth the mask edge
+ */
+export const createSmoothingMaskRenderer = (processor) => {
+    const { gl, program, attribLocations, uniformLocations, release } = createGLProgram(processor, {
+        attribs: {
+            position: 'a_position',
+            texCoord: 'a_texCoord',
+        },
+        uniforms: {
+            matrix: 'u_matrix',
+            mask: 'u_mask',
+            maskCombineRatio: 'u_mask_combine_ratio',
+            foregroundThreshold: 'u_foreground_threshold',
+            prevMask: 'u_mask_prev',
+        },
+        vertexSrc: vertexShaderSource,
+        fragmentSrc: smoothingMaskFragmentShaderSource,
+    });
+    const vao = gl.createVertexArray();
+    gl.bindVertexArray(vao);
+    // Position
+    const positionBuffer = gl.createBuffer();
+    gl.enableVertexAttribArray(attribLocations.position);
+    gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
+    gl.vertexAttribPointer(attribLocations.position, 2, gl.FLOAT, false, 0, 0);
+    gl.bufferData(gl.ARRAY_BUFFER, 
+    // prettier-ignore
+    new Float32Array([
+        0, 0,
+        0, 1,
+        1, 0,
+        1, 0,
+        0, 1,
+        1, 1,
+    ]), gl.STATIC_DRAW);
+    // Texture coordinates for the rectangle
+    const texCoordBuffer = gl.createBuffer();
+    gl.bindBuffer(gl.ARRAY_BUFFER, texCoordBuffer);
+    gl.bufferData(gl.ARRAY_BUFFER, 
+    // prettier-ignore
+    new Float32Array([
+        0, 0,
+        0, 1,
+        1, 0,
+        1, 0,
+        0, 1,
+        1, 1,
+    ]), gl.STATIC_DRAW);
+    gl.enableVertexAttribArray(attribLocations.texCoord);
+    gl.vertexAttribPointer(attribLocations.texCoord, 2, gl.FLOAT, false, 0, 0);
+    const outputTexture = createAndSetupTexture(gl);
+    gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, processor.width, processor.height, 0, gl.RGBA, gl.UNSIGNED_BYTE, null);
+    const frameBuffer = gl.createFramebuffer();
+    gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer);
+    gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, outputTexture, 0);
+    return {
+        render: ({ mask, prevMask, }, options) => {
+            gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer);
+            gl.viewport(0, 0, processor.width, processor.height);
+            gl.clearColor(0, 0, 0, 0);
+            gl.clear(gl.COLOR_BUFFER_BIT);
+            gl.useProgram(program);
+            gl.bindVertexArray(vao);
+            // Textures
+            gl.activeTexture(gl.TEXTURE0);
+            gl.uniform1i(uniformLocations.mask, 0);
+            gl.bindTexture(gl.TEXTURE_2D, mask.texture);
+            gl.activeTexture(gl.TEXTURE1);
+            gl.uniform1i(uniformLocations.prevMask, 1);
+            gl.bindTexture(gl.TEXTURE_2D, prevMask?.texture ?? mask.texture);
+            // No Flip Y Coordinate for drawing on the frame buffer
+            // Project, position and scale to full-size of the canvas
+            const matrix = m3.projection(1, 1, 1);
+            gl.uniformMatrix3fv(uniformLocations.matrix, false, matrix);
+            gl.uniform1f(uniformLocations.maskCombineRatio, options.maskCombineRatio);
+            gl.uniform1f(uniformLocations.foregroundThreshold, options.foregroundThreshold);
+            gl.drawArrays(gl.TRIANGLES, 0, 6);
+            assert(outputTexture);
+            return {
+                texture: outputTexture,
+                width: processor.width,
+                height: processor.height,
+            };
+        },
+        release: () => {
+            gl.deleteBuffer(positionBuffer);
+            gl.deleteBuffer(texCoordBuffer);
+            gl.deleteTexture(outputTexture);
+            gl.deleteFramebuffer(frameBuffer);
+            gl.deleteVertexArray(vao);
+            release();
+        },
+    };
+};

package/dist/common/backends/webgl/texture.d.ts

@@ -1,5 +1,8 @@
 import type { VideoFrameLike } from '../../types/media';
 import type { TextureInfo } from './types';
+/**
+ * Render Video frame into texture
+ */
 export declare const createTextureRenderer: (processor: OffscreenCanvas) => {
     render: (video: VideoFrameLike) => TextureInfo;
     release: () => void;

package/dist/common/backends/webgl/texture.js

@@ -3,6 +3,9 @@ import { m3 } from '../../m3';
 import { createAndSetupTexture, createGLProgram } from './webglUtils';
 import { vertexShaderSource } from './shaders/vertexShader';
 import { textureFragmentShaderSource } from './shaders/textureFragmentShader';
+/**
+ * Render Video frame into texture
+ */
 export const createTextureRenderer = (processor) => {
     const { gl, program, attribLocations, uniformLocations, release } = createGLProgram(processor, {
         attribs: {

package/dist/common/backends/webgl/textureToTexture.d.ts

@@ -0,0 +1,8 @@
+import type { TextureInfo } from './types';
+/**
+ * Render texture into another texture, i.e. clone
+ */
+export declare const createTextureToTextureRenderer: (processor: OffscreenCanvas) => {
+    render: (inputTexture: TextureInfo) => TextureInfo;
+    release: () => void;
+};

package/dist/common/backends/webgl/textureToTexture.js

@@ -0,0 +1,92 @@
+import { assert } from '../../utils';
+import { m3 } from '../../m3';
+import { createAndSetupTexture, createGLProgram } from './webglUtils';
+import { vertexShaderSource } from './shaders/vertexShader';
+import { textureFragmentShaderSource } from './shaders/textureFragmentShader';
+/**
+ * Render texture into another texture, i.e. clone
+ */
+export const createTextureToTextureRenderer = (processor) => {
+    const { gl, program, attribLocations, uniformLocations, release } = createGLProgram(processor, {
+        attribs: {
+            position: 'a_position',
+            texCoord: 'a_texCoord',
+        },
+        uniforms: {
+            matrix: 'u_matrix',
+            texture: 'u_texture',
+        },
+        vertexSrc: vertexShaderSource,
+        fragmentSrc: textureFragmentShaderSource,
+    });
+    const vao = gl.createVertexArray();
+    gl.bindVertexArray(vao);
+    // Position
+    const positionBuffer = gl.createBuffer();
+    gl.enableVertexAttribArray(attribLocations.position);
+    gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
+    gl.vertexAttribPointer(attribLocations.position, 2, gl.FLOAT, false, 0, 0);
+    // Put a unit quad in the buffer
+    gl.bufferData(gl.ARRAY_BUFFER, 
+    // prettier-ignore
+    new Float32Array([
+        0, 0,
+        0, 1,
+        1, 0,
+        1, 0,
+        0, 1,
+        1, 1,
+    ]), gl.STATIC_DRAW);
+    // Texture coordinates for the rectangle
+    const texCoordBuffer = gl.createBuffer();
+    gl.bindBuffer(gl.ARRAY_BUFFER, texCoordBuffer);
+    gl.bufferData(gl.ARRAY_BUFFER, 
+    // prettier-ignore
+    new Float32Array([
+        0, 0,
+        0, 1,
+        1, 0,
+        1, 0,
+        0, 1,
+        1, 1,
+    ]), gl.STATIC_DRAW);
+    gl.enableVertexAttribArray(attribLocations.texCoord);
+    gl.vertexAttribPointer(attribLocations.texCoord, 2, gl.FLOAT, false, 0, 0);
+    const outputTexture = createAndSetupTexture(gl);
+    gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, processor.width, processor.height, 0, gl.RGBA, gl.UNSIGNED_BYTE, null);
+    const frameBuffer = gl.createFramebuffer();
+    gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer);
+    gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, outputTexture, 0);
+    return {
+        render: (inputTexture) => {
+            // Draw on the frame buffer
+            gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer);
+            gl.viewport(0, 0, processor.width, processor.height);
+            gl.clearColor(0, 0, 0, 1);
+            gl.clear(gl.COLOR_BUFFER_BIT);
+            gl.useProgram(program);
+            gl.bindVertexArray(vao);
+            // Texture Unit setup
+            gl.activeTexture(gl.TEXTURE0);
+            gl.uniform1i(uniformLocations.texture, 0);
+            gl.bindTexture(gl.TEXTURE_2D, inputTexture.texture);
+            // Project, position and scale to full-size of the canvas
+            const matrix = m3.projection(1, 1, 1);
+            gl.uniformMatrix3fv(uniformLocations.matrix, false, matrix);
+            gl.drawArrays(gl.TRIANGLES, 0, 6);
+            assert(outputTexture);
+            return {
+                texture: outputTexture,
+                width: processor.width,
+                height: processor.height,
+            };
+        },
+        release: () => {
+            gl.deleteBuffer(positionBuffer);
+            gl.deleteBuffer(texCoordBuffer);
+            gl.deleteTexture(outputTexture);
+            gl.deleteVertexArray(vao);
+            release();
+        },
+    };
+};