@pexip/media-processor 17.3.0 → 18.0.0

package/dist/common/backends/webgl/shaders/textureFragmentShader.js

@@ -0,0 +1,16 @@
+export const textureFragmentShaderSource = `#version 300 es
+precision mediump float;
+
+// Passed in from the vertex shader.
+in vec2 v_texCoord;
+
+// The texture.
+uniform sampler2D u_texture;
+
+// we need to declare an output for the fragment shader
+out vec4 outColor;
+
+void main() {
+  outColor = texture(u_texture, v_texCoord);
+}
+`;

package/dist/common/backends/webgl/shaders/vertexShader.d.ts

@@ -0,0 +1 @@
+export declare const vertexShaderSource = "#version 300 es\n// an attribute is an input (in) to a vertex shader.\n// It will receive data from a buffer\nin vec2 a_position;\nin vec2 a_texCoord;\n\n// Used to pass in the resolution of the canvas\nuniform mat3 u_matrix;\n\n// Used to pass the texture coordinates to the fragment shader\nout vec2 v_texCoord;\n\nvoid main() {\n    gl_Position = vec4((u_matrix * vec3(a_position, 1)).xy, 0, 1);\n\n    // pass the texCoord to the fragment shader\n    v_texCoord = a_texCoord;\n}\n";

package/dist/common/backends/webgl/shaders/vertexShader.js

@@ -0,0 +1,19 @@
+export const vertexShaderSource = `#version 300 es
+// an attribute is an input (in) to a vertex shader.
+// It will receive data from a buffer
+in vec2 a_position;
+in vec2 a_texCoord;
+
+// Used to pass in the resolution of the canvas
+uniform mat3 u_matrix;
+
+// Used to pass the texture coordinates to the fragment shader
+out vec2 v_texCoord;
+
+void main() {
+    gl_Position = vec4((u_matrix * vec3(a_position, 1)).xy, 0, 1);
+
+    // pass the texCoord to the fragment shader
+    v_texCoord = a_texCoord;
+}
+`;

package/dist/common/backends/webgl/texture.d.ts

@@ -0,0 +1,6 @@
+import type { VideoFrameLike } from '../../types/media';
+import type { TextureInfo } from './types';
+export declare const createTextureRenderer: (processor: OffscreenCanvas) => {
+    render: (video: VideoFrameLike) => TextureInfo;
+    release: () => void;
+};

package/dist/common/backends/webgl/texture.js

@@ -0,0 +1,94 @@
+import { assert } from '../../utils';
+import { m3 } from '../../m3';
+import { createAndSetupTexture, createGLProgram } from './webglUtils';
+import { vertexShaderSource } from './shaders/vertexShader';
+import { textureFragmentShaderSource } from './shaders/textureFragmentShader';
+export const createTextureRenderer = (processor) => {
+    const { gl, program, attribLocations, uniformLocations, release } = createGLProgram(processor, {
+        attribs: {
+            position: 'a_position',
+            texCoord: 'a_texCoord',
+        },
+        uniforms: {
+            matrix: 'u_matrix',
+            texture: 'u_texture',
+        },
+        vertexSrc: vertexShaderSource,
+        fragmentSrc: textureFragmentShaderSource,
+    });
+    const vao = gl.createVertexArray();
+    gl.bindVertexArray(vao);
+    // Position
+    const positionBuffer = gl.createBuffer();
+    gl.enableVertexAttribArray(attribLocations.position);
+    gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
+    gl.vertexAttribPointer(attribLocations.position, 2, gl.FLOAT, false, 0, 0);
+    // Put a unit quad in the buffer
+    gl.bufferData(gl.ARRAY_BUFFER, 
+    // prettier-ignore
+    new Float32Array([
+        0, 0,
+        0, 1,
+        1, 0,
+        1, 0,
+        0, 1,
+        1, 1,
+    ]), gl.STATIC_DRAW);
+    // Texture coordinates for the rectangle
+    const texCoordBuffer = gl.createBuffer();
+    gl.bindBuffer(gl.ARRAY_BUFFER, texCoordBuffer);
+    gl.bufferData(gl.ARRAY_BUFFER, 
+    // prettier-ignore
+    new Float32Array([
+        0, 0,
+        0, 1,
+        1, 0,
+        1, 0,
+        0, 1,
+        1, 1,
+    ]), gl.STATIC_DRAW);
+    gl.enableVertexAttribArray(attribLocations.texCoord);
+    gl.vertexAttribPointer(attribLocations.texCoord, 2, gl.FLOAT, false, 0, 0);
+    const frameTexture = createAndSetupTexture(gl);
+    const outputTexture = createAndSetupTexture(gl);
+    gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, processor.width, processor.height, 0, gl.RGBA, gl.UNSIGNED_BYTE, null);
+    const frameBuffer = gl.createFramebuffer();
+    gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer);
+    gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, outputTexture, 0);
+    return {
+        render: (video) => {
+            // Draw on the frame buffer
+            gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer);
+            gl.viewport(0, 0, processor.width, processor.height);
+            gl.clearColor(0, 0, 0, 1);
+            gl.clear(gl.COLOR_BUFFER_BIT);
+            gl.useProgram(program);
+            gl.bindVertexArray(vao);
+            // Texture Unit setup
+            gl.uniform1i(uniformLocations.texture, 0);
+            // Project, position and scale to full-size of the canvas
+            const matrix = m3.projection(1, 1, 1);
+            gl.uniformMatrix3fv(uniformLocations.matrix, false, matrix);
+            // Frame texture
+            gl.activeTexture(gl.TEXTURE0);
+            gl.bindTexture(gl.TEXTURE_2D, frameTexture);
+            gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, video.frame);
+            gl.generateMipmap(gl.TEXTURE_2D);
+            gl.drawArrays(gl.TRIANGLES, 0, 6);
+            assert(outputTexture);
+            return {
+                texture: outputTexture,
+                width: processor.width,
+                height: processor.height,
+            };
+        },
+        release: () => {
+            gl.deleteBuffer(positionBuffer);
+            gl.deleteBuffer(texCoordBuffer);
+            gl.deleteTexture(frameTexture);
+            gl.deleteTexture(outputTexture);
+            gl.deleteVertexArray(vao);
+            release();
+        },
+    };
+};

package/dist/common/backends/webgl/types.d.ts

@@ -0,0 +1,5 @@
+export interface TextureInfo {
+    texture: WebGLTexture;
+    width: number;
+    height: number;
+}

package/dist/common/backends/webgl/types.js

@@ -0,0 +1 @@
+export {};

package/dist/common/backends/webgl/webgl.mock.js

@@ -0,0 +1,8 @@
+"use strict";
+Object.defineProperty(self, 'WebGLRenderingContext', {
+    writable: true,
+    value: {
+        VERTEX_SHADER: 0x8b31,
+        FRAGMENT_SHADER: 0x8b30,
+    },
+});

package/dist/common/backends/webgl/webglUtils.d.ts

@@ -0,0 +1,80 @@
+import type { MPMask } from '../../types/segmentation';
+import type { TextureInfo } from './types';
+/**
+ * WebGL Shader type enum
+ */
+export declare const GL_SHADER_TYPE: {
+    readonly VertexShader: 35633;
+    readonly FragmentShader: 35632;
+};
+/**
+ * WebGL Shader name enum for error output
+ */
+export declare const GL_SHADER_NAME: {
+    readonly 35633: "VertexShader";
+    readonly 35632: "FragmentShader";
+};
+/**
+ * The type of WebGl shader:
+ * `WebGLRenderingContext.VERTEX_SHADER` | `WebGLRenderingContext.FRAGMENT_SHADER`
+ */
+export type GLShaderType = (typeof GL_SHADER_TYPE)[keyof typeof GL_SHADER_TYPE];
+/**
+ * Brand string for type ensuring the shader source is a GLSL ES 3.00 shader
+ */
+type GLShaderSource = string & {
+    __glShaderSource: true;
+};
+type Release = () => void;
+/**
+ * Ensure the shader source is WebGL2's shader language aka GLSL ES 3.00
+ */
+export declare const isShaderSource: (src: unknown) => src is GLShaderSource;
+/**
+ * Ensure the shader source is WebGL2's shader language aka GLSL ES 3.00
+ */
+export declare const assertShaderSource: (src: unknown) => asserts src is GLShaderSource;
+/**
+ * Load a shader
+ * @param gl - WebGLRenderingContext
+ * @param shaderType - The type of shader to load {@link GL_SHADER_TYPE}
+ * @param shaderSource - The source of the shader
+ */
+export declare const loadShader: (gl: WebGLRenderingContext, shaderType: GLShaderType, shaderSource: GLShaderSource) => WebGLShader;
+/**
+ * Create a WebGL program from 2 provided shaders
+ * @param gl - WebGLRenderingContext
+ * @param shaders - The shader tuple of `[vertexShader, fragmentShader]`
+ * @param attribs - An array of attribs names.
+ *        Locations will be assigned by index if not passed in.
+ * @param locations - The locations for the attribs.
+ *        A parallel array to opt_attribs letting you assign locations.
+ */
+export declare const createProgram: (gl: WebGLRenderingContext, shaders: [WebGLShader, WebGLShader], attribs?: string[], locations?: number[]) => [WebGLProgram, Release];
+/**
+ * Create a WebGL program from 2 sources
+ * @param gl - WebGLRenderingContext
+ * @param shaderSources - The shader source tuple of `[vertexShaderSource, fragmentShaderSource]`
+ * @param attribs - An array of attribs names.
+ *        Locations will be assigned by index if not passed in.
+ * @param locations - The locations for the attribs.
+ *        A parallel array to opt_attribs letting you assign locations.
+ */
+export declare const createProgramFromSources: (gl: WebGLRenderingContext, [vs, fs]: [GLShaderSource, GLShaderSource], attribs?: string[], locations?: number[]) => [WebGLProgram, Release];
+export declare const createAndSetupTexture: (gl: WebGL2RenderingContext) => WebGLTexture | null;
+type Location<S extends string> = S extends `u_${string}` ? ReturnType<WebGL2RenderingContext['getUniformLocation']> : S extends `a_${string}` ? ReturnType<WebGL2RenderingContext['getAttribLocation']> : never;
+export declare const getLocations: <T extends Record<string, string>>(gl: WebGL2RenderingContext, program: WebGLProgram, locations: T) => { [Key in keyof T]: Location<T[Key]>; };
+export declare const createGLProgram: <A extends Record<string, string>, U extends Record<string, string>>(processCanvas: OffscreenCanvas, { attribs, uniforms, vertexSrc, fragmentSrc, }: {
+    attribs: A;
+    uniforms: U;
+    vertexSrc: string;
+    fragmentSrc: string;
+}) => {
+    gl: WebGL2RenderingContext;
+    program: WebGLProgram;
+    attribLocations: { [Key in keyof A]: Location<A[Key]>; };
+    uniformLocations: { [Key_1 in keyof U]: Location<U[Key_1]>; };
+    release: Release;
+};
+export declare const toTextureInfo: (mask: MPMask) => TextureInfo;
+export {};

package/dist/common/backends/webgl/webglUtils.js

@@ -0,0 +1,194 @@
+/**
+ * WebGL Shader type enum
+ */
+export const GL_SHADER_TYPE = {
+    VertexShader: WebGLRenderingContext.VERTEX_SHADER,
+    FragmentShader: WebGLRenderingContext.FRAGMENT_SHADER,
+};
+/**
+ * WebGL Shader name enum for error output
+ */
+export const GL_SHADER_NAME = {
+    [GL_SHADER_TYPE.VertexShader]: 'VertexShader',
+    [GL_SHADER_TYPE.FragmentShader]: 'FragmentShader',
+};
+/**
+ * Ensure the shader source is WebGL2's shader language aka GLSL ES 3.00
+ */
+export const isShaderSource = (src) => {
+    if (typeof src === 'string') {
+        return src.startsWith('#version 300 es');
+    }
+    return false;
+};
+/**
+ * Ensure the shader source is WebGL2's shader language aka GLSL ES 3.00
+ */
+export const assertShaderSource = src => {
+    if (!isShaderSource(src)) {
+        throw new Error('Expecting GLSL ES 3.00 shader source');
+    }
+};
+const errorRE = /ERROR:\s*\d+:(\d+)/gi;
+function addLineNumbersWithError(src, log = '', lineOffset = 0) {
+    // Note: Error message formats are not defined by any spec so this may or may not work.
+    const matches = [...log.matchAll(errorRE)];
+    const lineNoToErrorMap = new Map(matches.map((m, ndx) => {
+        if (!m[1] || m.index === undefined) {
+            return [0, log];
+        }
+        const lineNo = parseInt(m[1], 10);
+        const next = matches[ndx + 1];
+        const end = next ? next.index : log.length;
+        const msg = log.substring(m.index, end);
+        return [lineNo - 1, msg];
+    }));
+    return src
+        .split('\n')
+        .map((line, lineNo) => {
+        const err = lineNoToErrorMap.get(lineNo);
+        return `${lineNo + 1 + lineOffset}: ${line}${err ? `\n\n^^^ ${err}` : ''}`;
+    })
+        .join('\n');
+}
+/**
+ * Load a shader
+ * @param gl - WebGLRenderingContext
+ * @param shaderType - The type of shader to load {@link GL_SHADER_TYPE}
+ * @param shaderSource - The source of the shader
+ */
+export const loadShader = (gl, shaderType, shaderSource) => {
+    // Create the shader object
+    const shader = gl.createShader(shaderType);
+    if (!shader) {
+        throw new Error(`Unable to create ${GL_SHADER_NAME[shaderType]}`);
+    }
+    // Load the shader source
+    gl.shaderSource(shader, shaderSource);
+    // Compile the shader
+    gl.compileShader(shader);
+    // Check the compile status
+    // https://developer.mozilla.org/en-US/docs/Web/API/WebGLRenderingContext/getShaderParameter#gl.compile_status
+    const compiled = gl.getShaderParameter(shader, gl.COMPILE_STATUS);
+    if (!compiled) {
+        // Something went wrong during compilation; get the error
+        const lastError = gl.getShaderInfoLog(shader);
+        gl.deleteShader(shader);
+        const message = `Error compiling ${GL_SHADER_NAME[shaderType]}: ${lastError ?? ''}\n${lastError ? addLineNumbersWithError(shaderSource, lastError) : 'no error message available'}`;
+        throw new Error(message);
+    }
+    return shader;
+};
+/**
+ * Create a WebGL program from 2 provided shaders
+ * @param gl - WebGLRenderingContext
+ * @param shaders - The shader tuple of `[vertexShader, fragmentShader]`
+ * @param attribs - An array of attribs names.
+ *        Locations will be assigned by index if not passed in.
+ * @param locations - The locations for the attribs.
+ *        A parallel array to opt_attribs letting you assign locations.
+ */
+export const createProgram = (gl, shaders, attribs, locations) => {
+    const program = gl.createProgram();
+    if (!program) {
+        throw new Error('Unable to create WebGL program');
+    }
+    shaders.forEach(function (shader) {
+        gl.attachShader(program, shader);
+    });
+    attribs?.forEach(function (attrib, ndx) {
+        const locationIdx = locations?.[ndx] ?? ndx;
+        gl.bindAttribLocation(program, locationIdx, attrib);
+    });
+    gl.linkProgram(program);
+    // Check the link status
+    const linked = gl.getProgramParameter(program, gl.LINK_STATUS);
+    if (!linked) {
+        // something went wrong with the link
+        const lastError = gl.getProgramInfoLog(program);
+        gl.deleteProgram(program);
+        throw new Error(`Error in program linking: ${lastError ?? ''}\n${shaders
+            .map(shader => {
+            const src = addLineNumbersWithError(gl.getShaderSource(shader) ?? '');
+            const shaderType = gl.getShaderParameter(shader, gl.SHADER_TYPE);
+            return `${GL_SHADER_NAME[shaderType]}:\n${src}`;
+        })
+            .join('\n')}`);
+    }
+    return [
+        program,
+        () => {
+            shaders.forEach(shader => {
+                gl.detachShader(program, shader);
+                gl.deleteShader(shader);
+            });
+            gl.deleteProgram(program);
+        },
+    ];
+};
+/**
+ * Create a WebGL program from 2 sources
+ * @param gl - WebGLRenderingContext
+ * @param shaderSources - The shader source tuple of `[vertexShaderSource, fragmentShaderSource]`
+ * @param attribs - An array of attribs names.
+ *        Locations will be assigned by index if not passed in.
+ * @param locations - The locations for the attribs.
+ *        A parallel array to opt_attribs letting you assign locations.
+ */
+export const createProgramFromSources = (gl, [vs, fs], attribs, locations) => createProgram(gl, [
+    loadShader(gl, GL_SHADER_TYPE.VertexShader, vs),
+    loadShader(gl, GL_SHADER_TYPE.FragmentShader, fs),
+], attribs, locations);
+export const createAndSetupTexture = (gl) => {
+    const texture = gl.createTexture();
+    gl.bindTexture(gl.TEXTURE_2D, texture);
+    // Set the parameters so we don't need mips and so we're not filtering
+    // and we don't repeat at the edges.
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
+    return texture;
+};
+export const getLocations = (gl, program, locations) => Object.keys(locations).reduce((acc, key) => {
+    const k = key;
+    const location = locations[k];
+    if (!location) {
+        return acc;
+    }
+    return {
+        ...acc,
+        [key]: location.startsWith('u_')
+            ? gl.getUniformLocation(program, location)
+            : gl.getAttribLocation(program, location),
+    };
+}, {});
+export const createGLProgram = (processCanvas, { attribs, uniforms, vertexSrc, fragmentSrc, }) => {
+    const gl = processCanvas.getContext('webgl2', {
+        premultipliedAlpha: false,
+    });
+    if (!gl) {
+        throw new Error('WebGL2 is not supported');
+    }
+    gl.enable(gl.BLEND);
+    gl.blendFunc(gl.SRC_ALPHA, gl.ONE_MINUS_SRC_ALPHA);
+    assertShaderSource(vertexSrc);
+    assertShaderSource(fragmentSrc);
+    const [program, release] = createProgramFromSources(gl, [
+        vertexSrc,
+        fragmentSrc,
+    ]);
+    return {
+        gl,
+        program,
+        attribLocations: getLocations(gl, program, attribs),
+        uniformLocations: getLocations(gl, program, uniforms),
+        release,
+    };
+};
+export const toTextureInfo = (mask) => ({
+    texture: mask.getAsWebGLTexture(),
+    width: mask.width,
+    height: mask.height,
+});
+/* eslint-enable max-params --- this is utils */

package/dist/common/canvasRenderUtils.d.ts

@@ -0,0 +1,62 @@
+import type { Segmentation } from './types/media';
+interface InternalCanvases {
+    drawImageDataCanvas?: OffscreenCanvas;
+    maskCanvas?: OffscreenCanvas;
+    blurredMaskCanvas?: OffscreenCanvas;
+    blurredCanvas?: OffscreenCanvas;
+    downScaledCanvas?: OffscreenCanvas;
+    backgroundImageCanvas?: OffscreenCanvas;
+    inputCanvas?: OffscreenCanvas;
+}
+export interface Point {
+    x: number;
+    y: number;
+}
+export interface Size {
+    width: number;
+    height: number;
+}
+export type Rect = Point & Size;
+/**
+ * Context Attributes
+ * @see {@link https://developer.mozilla.org/en-US/docs/Web/API/OffscreenCanvas/getContext}
+ */
+interface CanvasRenderingContext2DOptions {
+    alpha?: boolean;
+}
+export declare const getCanvasRenderingContext2D: (canvas: OffscreenCanvas, options?: CanvasRenderingContext2DOptions) => OffscreenCanvasRenderingContext2D;
+export declare const flipCanvasHorizontal: (canvas: OffscreenCanvas) => void;
+export declare const getImageSize: (image: CanvasImageSource) => Size;
+/**
+ * Create an OffscreenCanvas with provided width and height. When
+ * OffscreenCanvas is not available, a Canvas element is returned.
+ *
+ * @param width - canvas.width
+ * @param height - canvas.height
+ */
+export declare const createOffscreenCanvas: (width: number, height: number) => OffscreenCanvas;
+/**
+ * Compare the provided width and height to see if they are the same
+ *
+ * @param widthA - The width of A
+ * @param heightA - The height of A
+ * @param widthB - The width of B
+ * @param heightB - The height of B
+ */
+export declare const isEqualSize: (widthA: number, heightA: number, widthB: number, heightB: number) => boolean;
+export declare const clearCanvas: (canvas: OffscreenCanvas) => void;
+export declare const createCanvasRenderUtils: (processingWidth: number, processingHeight: number) => {
+    evaluateInput: (inputImage: CanvasImageSource) => OffscreenCanvas;
+    renderImageToCanvas: (image: CanvasImageSource, canvas: OffscreenCanvas, dw?: number, dh?: number, options?: CanvasRenderingContext2DOptions) => void;
+    drawBlurEffect: (canvas: OffscreenCanvas, inputImage: CanvasImageSource, segmentations: Segmentation | Segmentation[], foregroundThreshold?: number, backgroundBlurAmount?: number, edgeBlurAmount?: number, flipHorizontal?: boolean) => void;
+    drawOverlayEffect: (canvas: OffscreenCanvas, inputImage: CanvasImageSource, backgroundImage: CanvasImageSource | OffscreenCanvas, segmentations: Segmentation | Segmentation[], foregroundThreshold?: number, backgroundBlurAmount?: number, edgeBlurAmount?: number, flipHorizontal?: boolean) => void;
+    renderImageToOffScreenCanvas: (image: CanvasImageSource, canvasName: keyof InternalCanvases) => OffscreenCanvas;
+    renderImageDataToOffScreenCanvas: (image: ImageData, canvasName: keyof InternalCanvases) => OffscreenCanvas;
+    drawAndBlurImageOnOffScreenCanvas: ({ image, blurAmount, offscreenCanvasName, preserveOldDrawing, }: {
+        image: CanvasImageSource;
+        blurAmount: number;
+        offscreenCanvasName: keyof InternalCanvases;
+        preserveOldDrawing?: boolean | undefined;
+    }) => OffscreenCanvas;
+};
+export {};