@pexip/media-processor 16.7.1 → 17.1.0

package/dist/main/video/canvasTransform.js

@@ -0,0 +1,173 @@
+import { stopStreamTracks } from '../utils';
+import { FOREGROUND_THRESHOLD, BACKGROUND_BLUR_AMOUNT, EDGE_BLUR_AMOUNT, FLIP_HORIZONTAL, PROCESSING_WIDTH, PROCESSING_HEIGHT, } from './constants';
+import { createCanvas } from './utils';
+import { createCanvasRenderUtils } from './canvasRenderUtils';
+const createAssertInRange = (from, to) => (value) => {
+    if (value < from || value > to) {
+        throw new Error(`Invalid value (${value}) to range [${from}, ${to}]`);
+    }
+};
+const assertInRangeFrom0To1 = createAssertInRange(0, 1);
+const assertInRangeFrom0To20 = createAssertInRange(0, 20);
+const NOT_INITED_ERROR_MSG = 'Please call init() method first!';
+export const createTransform = (segmenter, { width = PROCESSING_WIDTH, height = PROCESSING_HEIGHT, foregroundThreshold = FOREGROUND_THRESHOLD, backgroundBlurAmount = BACKGROUND_BLUR_AMOUNT, edgeBlurAmount = EDGE_BLUR_AMOUNT, flipHorizontal = FLIP_HORIZONTAL, effects = 'none', selfManageSegmenter, bgImageUrl, } = {}) => {
+    const props = {
+        segmenter,
+        width,
+        height,
+        foregroundThreshold,
+        backgroundBlurAmount,
+        edgeBlurAmount,
+        flipHorizontal,
+        utils: createCanvasRenderUtils(width, height),
+        effects,
+        backgroundImage: undefined,
+        status: 'created',
+        backgroundImageUrl: bgImageUrl,
+    };
+    const processInput = async (input) => {
+        if (props.segmenter.status === 'created' ||
+            props.segmenter.status === 'closed') {
+            await props.segmenter.open();
+        }
+        if (props.segmenter.status === 'opening') {
+            return [];
+        }
+        return await props.segmenter.process(input);
+    };
+    const loadBackgroundImage = async (url) => {
+        if (props.backgroundImageUrl === url && props.backgroundImage) {
+            return;
+        }
+        props.backgroundImageUrl = url;
+        props.backgroundImage = await props.utils.loadBackgroundImage(url);
+    };
+    return {
+        get status() {
+            return props.status;
+        },
+        get width() {
+            return props.width;
+        },
+        get height() {
+            return props.height;
+        },
+        get foregroundThreshold() {
+            return props.foregroundThreshold;
+        },
+        get backgroundBlurAmount() {
+            return props.backgroundBlurAmount;
+        },
+        get edgeBlurAmount() {
+            return props.edgeBlurAmount;
+        },
+        get flipHorizontal() {
+            return props.flipHorizontal;
+        },
+        get effects() {
+            return props.effects;
+        },
+        get backgroundImage() {
+            return props.backgroundImage;
+        },
+        set foregroundThreshold(value) {
+            assertInRangeFrom0To1(value);
+            props.foregroundThreshold = value;
+        },
+        set backgroundBlurAmount(value) {
+            assertInRangeFrom0To20(value);
+            props.backgroundBlurAmount = value;
+        },
+        set edgeBlurAmount(value) {
+            assertInRangeFrom0To20(value);
+            props.edgeBlurAmount = value;
+        },
+        set flipHorizontal(value) {
+            props.flipHorizontal = value;
+        },
+        set effects(value) {
+            props.effects = value;
+        },
+        get backgroundImageUrl() {
+            return props.backgroundImageUrl;
+        },
+        set backgroundImage(canvas) {
+            props.backgroundImage = canvas;
+        },
+        loadBackgroundImage,
+        get segmenter() {
+            return props.segmenter;
+        },
+        set segmenter(value) {
+            if (value !== props.segmenter) {
+                props.segmenter = value;
+            }
+        },
+        init: async () => {
+            props.outputCanvas = createCanvas(width, height);
+            if (props.backgroundImageUrl) {
+                await loadBackgroundImage(props.backgroundImageUrl);
+            }
+            props.status = 'opened';
+        },
+        transform: async (videoFrame, controller) => {
+            if (!props.outputCanvas) {
+                throw new Error(NOT_INITED_ERROR_MSG);
+            }
+            switch (props.effects) {
+                case 'blur': {
+                    const image = await props.utils.renderImageToOffScreenCanvas(videoFrame, 'inputCanvas');
+                    const segmentations = await processInput(image);
+                    if (props.status === 'closed') {
+                        break;
+                    }
+                    if (props.outputCanvas.height !== height) {
+                        props.outputCanvas.width = width;
+                        props.outputCanvas.height = height;
+                    }
+                    await props.utils.drawBlurEffect(props.outputCanvas, image, segmentations, props.foregroundThreshold, props.backgroundBlurAmount, props.edgeBlurAmount, props.flipHorizontal);
+                    controller.enqueue(props.outputCanvas);
+                    break;
+                }
+                case 'overlay': {
+                    if (!props.backgroundImage) {
+                        throw new Error('Please call setBackgroundImage() method first');
+                    }
+                    const image = await props.utils.renderImageToOffScreenCanvas(videoFrame, 'inputCanvas');
+                    const segmentations = await processInput(image);
+                    if (props.status === 'closed') {
+                        break;
+                    }
+                    if (props.outputCanvas.height !== height) {
+                        props.outputCanvas.width = width;
+                        props.outputCanvas.height = height;
+                    }
+                    await props.utils.drawOverlayEffect(props.outputCanvas, image, props.backgroundImage, segmentations, props.foregroundThreshold, 0, // No blur for overlay
+                    props.edgeBlurAmount, props.flipHorizontal);
+                    controller.enqueue(props.outputCanvas);
+                    break;
+                }
+                case 'none': {
+                    controller.enqueue(videoFrame);
+                    break;
+                }
+            }
+            props.status = 'processing';
+        },
+        close: () => {
+            if (!selfManageSegmenter) {
+                segmenter.close();
+            }
+            props.outputCanvas = undefined;
+            stopStreamTracks(props.outputStream);
+            props.outputStream = undefined;
+            props.status = 'closed';
+        },
+        destroy: async () => {
+            if (!selfManageSegmenter) {
+                await segmenter.destroy();
+            }
+            props.status = 'destroyed';
+        },
+    };
+};

package/dist/main/video/constants.d.ts

@@ -0,0 +1,10 @@
+export declare const PROCESSING_WIDTH = 768;
+export declare const PROCESSING_HEIGHT = 432;
+export declare const FOREGROUND_THRESHOLD = 0.5;
+export declare const BACKGROUND_BLUR_AMOUNT = 3;
+export declare const EDGE_BLUR_AMOUNT = 3;
+export declare const FLIP_HORIZONTAL = false;
+export declare const FRAME_RATE = 20;
+export declare enum AbortReason {
+    Close = "close"
+}

package/dist/main/video/constants.js

@@ -0,0 +1,12 @@
+export const PROCESSING_WIDTH = 768;
+export const PROCESSING_HEIGHT = 432;
+// Rendering parameters
+export const FOREGROUND_THRESHOLD = 0.5;
+export const BACKGROUND_BLUR_AMOUNT = 3; // Between 1 and 20
+export const EDGE_BLUR_AMOUNT = 3; // Between 0 and 20
+export const FLIP_HORIZONTAL = false;
+export const FRAME_RATE = 20;
+export var AbortReason;
+(function (AbortReason) {
+    AbortReason["Close"] = "close";
+})(AbortReason || (AbortReason = {}));

package/dist/main/video/index.d.ts

@@ -0,0 +1,9 @@
+export * from './load';
+export * from './video';
+export * from './types';
+export { createTransform as createCanvasTransform } from './canvasTransform';
+export * from './videoStreamTrackProcessor';
+export { createFrameCallbackRequest } from './utils';
+export * from './segmenters';
+export * from './typeGuards';
+export * from './constants';

package/dist/main/video/index.js

@@ -0,0 +1,9 @@
+export * from './load';
+export * from './video';
+export * from './types';
+export { createTransform as createCanvasTransform } from './canvasTransform';
+export * from './videoStreamTrackProcessor';
+export { createFrameCallbackRequest } from './utils';
+export * from './segmenters';
+export * from './typeGuards';
+export * from './constants';

package/dist/main/video/load.d.ts

@@ -0,0 +1,17 @@
+export declare const loadScript: (path: string, id: string) => Promise<void>;
+export type WasmPaths = [string, string | undefined];
+export declare const loadWasms: (paths: WasmPaths[]) => Promise<void>;
+export declare const loadTfjsCore: (prodMode: boolean) => Promise<unknown>;
+export declare const loadTfjsBackendWebGl: () => Promise<{
+    default: typeof import("@tensorflow/tfjs-backend-webgl");
+    version_webgl: "4.2.0";
+    webgl: {
+        forceHalfFloat: typeof import("@tensorflow/tfjs-backend-webgl").forceHalfFloat;
+    };
+    forceHalfFloat(): void;
+    MathBackendWebGL: typeof import("@tensorflow/tfjs-backend-webgl").MathBackendWebGL;
+    setWebGLContext: typeof import("@tensorflow/tfjs-backend-webgl").setWebGLContext;
+    GPGPUContext: typeof import("@tensorflow/tfjs-backend-webgl").GPGPUContext;
+    gpgpu_util: typeof import("@tensorflow/tfjs-backend-webgl/dist/gpgpu_util");
+    webgl_util: typeof import("@tensorflow/tfjs-backend-webgl/dist/webgl_util");
+}>;

package/dist/main/video/load.js

@@ -0,0 +1,47 @@
+const policy = {
+    createScriptURL: (url) => {
+        if (new URL(url, document.baseURI).origin !== window.location.origin) {
+            throw new Error(`Trying to create script url not on same origin (${url} not on ${window.location.origin})`);
+        }
+        return url;
+    },
+};
+// Ambient definitions does not seem to be working, so we have to do this :(
+const sameOriginPolicy = typeof window.trustedTypes !== 'undefined' &&
+    window.trustedTypes.createPolicy
+    ? window.trustedTypes.createPolicy('same-origin', policy)
+    : policy;
+export const loadScript = (path, id) => new Promise((resolve, reject) => {
+    const scriptExist = document.getElementById(id);
+    if (scriptExist) {
+        return;
+    }
+    const removeScript = () => {
+        document.head.removeChild(script);
+    };
+    const script = document.createElement('script');
+    // Use a cast as typescript dom types do not support it yet.
+    script.src = sameOriginPolicy.createScriptURL(path);
+    script.id = id;
+    script.onload = () => {
+        removeScript();
+        resolve();
+    };
+    script.onerror = ev => {
+        removeScript();
+        reject(ev);
+    };
+    document.head.appendChild(script);
+});
+export const loadWasms = async (paths) => {
+    // TODO: map the wasm path
+    await Promise.all(paths.map(([path]) => loadScript(path, path)));
+};
+export const loadTfjsCore = async (prodMode) => {
+    const tfjs = await import('@tensorflow/tfjs-core');
+    if (prodMode) {
+        tfjs.enableProdMode();
+    }
+    return tfjs;
+};
+export const loadTfjsBackendWebGl = () => import('@tensorflow/tfjs-backend-webgl');

package/dist/main/video/segmenters/index.d.ts

@@ -0,0 +1 @@
+export { createSegmenter as createMediapipeSegmenter } from './mediapipe';

package/dist/main/video/segmenters/index.js

@@ -0,0 +1 @@
+export { createSegmenter as createMediapipeSegmenter } from './mediapipe';

package/dist/main/video/segmenters/mediapipe.d.ts

@@ -0,0 +1,13 @@
+import type { Segmenter, AsyncAssets } from '../types';
+export declare const SelfieSegmentationModelTypes: ['general', 'landscape'];
+export type SelfieSegmentationModelType = (typeof SelfieSegmentationModelTypes)[number];
+interface Options extends AsyncAssets {
+    modelType: SelfieSegmentationModelType;
+    processingWidth: number;
+    processingHeight: number;
+    gluePath: string;
+    selfieMode: boolean;
+    prodMode: boolean;
+}
+export declare const createSegmenter: (basePath?: string, { modelType, tfjsCoreLoaded, tfjsBackendLoaded, glueLoaded, processingWidth, processingHeight, gluePath, selfieMode, prodMode, }?: Partial<Options>) => Segmenter;
+export {};

package/dist/main/video/segmenters/mediapipe.js

@@ -0,0 +1,85 @@
+import { toSegmentation, createInputImageConvertor, ensure } from '../utils';
+import { createCanvasRenderUtils } from '../canvasRenderUtils';
+import { PROCESSING_WIDTH, PROCESSING_HEIGHT } from '../constants';
+import { loadScript, loadTfjsCore, loadTfjsBackendWebGl } from '../load';
+export const SelfieSegmentationModelTypes = [
+    'general',
+    'landscape',
+];
+export const createSegmenter = (basePath = '/', { modelType = 'general', tfjsCoreLoaded = false, tfjsBackendLoaded = false, glueLoaded = false, processingWidth = PROCESSING_WIDTH, processingHeight = PROCESSING_HEIGHT, gluePath = '', selfieMode = false, prodMode = true, } = {}) => {
+    const props = {
+        selfieMode,
+        segmentation: [],
+        renderUtils: createCanvasRenderUtils(processingWidth, processingHeight),
+        status: 'created',
+        tfjsCoreLoaded,
+        tfjsBackendLoaded,
+        glueLoaded,
+    };
+    const toInputImage = createInputImageConvertor(input => props.renderUtils.drawAndBlurImageOnOffScreenCanvas(input, 0, 'blurredCanvas'));
+    const segmentPerson = async (input) => {
+        const segmenter = ensure(props.segmenter);
+        props.status = 'processing';
+        const image = await toInputImage(input);
+        await segmenter.send({ image });
+        props.status = 'idle';
+    };
+    return {
+        get model() {
+            return 'mediapipeSelfie';
+        },
+        get width() {
+            return processingWidth;
+        },
+        get height() {
+            return processingHeight;
+        },
+        get status() {
+            return props.status;
+        },
+        open: async () => {
+            props.status = 'opening';
+            // Avoid side-effects from the modules
+            if (!props.tfjsCoreLoaded) {
+                await loadTfjsCore(prodMode);
+                props.tfjsCoreLoaded = true;
+            }
+            if (!props.tfjsBackendLoaded) {
+                await loadTfjsBackendWebGl();
+                props.tfjsBackendLoaded = true;
+            }
+            if (!props.glueLoaded) {
+                await loadScript(gluePath, gluePath);
+                props.glueLoaded = true;
+            }
+            props.segmenter = new SelfieSegmentation({
+                locateFile: (path) => `${basePath.replace(/\/+$/, '')}/${path}`,
+            });
+            const modelSelection = modelType === 'landscape' ? 1 : 0;
+            props.segmenter.setOptions({
+                modelSelection,
+                selfieMode: props.selfieMode,
+            });
+            props.segmenter.onResults(results => {
+                props.segmentation = [
+                    toSegmentation(results.segmentationMask, 'canvasimagesource', () => 'person'),
+                ];
+            });
+            await props.segmenter.initialize();
+            props.status = 'opened';
+        },
+        process: async (input) => {
+            await segmentPerson(input);
+            return props.segmentation;
+        },
+        close: () => {
+            props.segmenter?.reset();
+            props.status = 'closed';
+        },
+        destroy: async () => {
+            props.status = 'destroying';
+            await props.segmenter?.close();
+            props.status = 'destroyed';
+        },
+    };
+};

package/dist/main/video/transformer.d.ts

@@ -0,0 +1,10 @@
+/// <reference types="dom-webcodecs" />
+import type { InputFrame } from './types';
+export declare const nullTransformController: (controller: Pick<TransformStreamDefaultController<CanvasImageSource>, 'enqueue' | 'terminate'>) => TransformStreamDefaultController<CanvasImageSource>;
+export declare const adaptInputFrameTransformer: (transformer: Transformer<InputFrame, InputFrame>) => {
+    transform: (frame: VideoFrame, controller: TransformStreamDefaultController<VideoFrame>) => void | PromiseLike<void> | undefined;
+    flush?: TransformerFlushCallback<InputFrame> | undefined;
+    readableType?: undefined;
+    start?: TransformerStartCallback<InputFrame> | undefined;
+    writableType?: undefined;
+};

package/dist/main/video/transformer.js

@@ -0,0 +1,56 @@
+const wrapTransformController = (videoFrame, controller) => {
+    return {
+        get desiredSize() {
+            return controller.desiredSize;
+        },
+        enqueue: frame => {
+            if (!frame) {
+                return;
+            }
+            if (frame instanceof VideoFrame) {
+                return controller.enqueue(frame);
+            }
+            if (('OffscreenCanvas' in window &&
+                frame instanceof OffscreenCanvas) ||
+                frame instanceof HTMLCanvasElement ||
+                frame instanceof HTMLVideoElement ||
+                frame instanceof HTMLImageElement ||
+                frame instanceof ImageBitmap) {
+                const timestamp = videoFrame.timestamp ?? 0;
+                videoFrame.close();
+                return controller.enqueue(new VideoFrame(frame, { timestamp, alpha: 'discard' }));
+            }
+            throw new Error('Unexpected input frame');
+        },
+        error: reason => controller.error(reason),
+        terminate: () => controller.terminate(),
+    };
+};
+export const nullTransformController = (controller) => {
+    return {
+        get desiredSize() {
+            return 0;
+        },
+        enqueue: frame => {
+            if (('OffscreenCanvas' in window &&
+                frame instanceof OffscreenCanvas) ||
+                frame instanceof HTMLCanvasElement ||
+                frame instanceof HTMLVideoElement ||
+                frame instanceof HTMLImageElement ||
+                frame instanceof ImageBitmap) {
+                return controller.enqueue(frame);
+            }
+            throw new Error('Unexpected input frame');
+        },
+        error: reason => {
+            throw new Error(reason);
+        },
+        terminate: () => controller.terminate(),
+    };
+};
+export const adaptInputFrameTransformer = (transformer) => {
+    const transform = (frame, controller) => {
+        return transformer.transform?.(frame, wrapTransformController(frame, controller));
+    };
+    return { ...transformer, transform };
+};

package/dist/main/video/typeGuards.d.ts

@@ -0,0 +1,2 @@
+export declare const isRenderEffects: (t: unknown) => t is "blur" | "none" | "overlay";
+export declare const isSegmentationModel: (t: unknown) => t is "mediapipeSelfie";

package/dist/main/video/typeGuards.js

@@ -0,0 +1,13 @@
+import { RENDER_EFFECTS, SEG_MODELS } from './types';
+export const isRenderEffects = (t) => {
+    if (typeof t !== 'string') {
+        return false;
+    }
+    return RENDER_EFFECTS.some(effect => effect === t);
+};
+export const isSegmentationModel = (t) => {
+    if (typeof t !== 'string') {
+        return false;
+    }
+    return SEG_MODELS.some(model => model === t);
+};

package/dist/main/video/types.d.ts

@@ -0,0 +1,98 @@
+/// <reference types="dom-webcodecs" />
+import type { Tensor3D } from '@tensorflow/tfjs-core';
+import type { Canvas, Transform } from '../types';
+export declare const RENDER_EFFECTS: readonly ["none", "blur", "overlay"];
+export declare const SEG_MODELS: readonly ["mediapipeSelfie"];
+/**
+ * Interfaces from "tensorflow-models/body-segmentation" interfaces
+ */
+export type MaskUnderlyingType = 'canvasimagesource' | 'imagedata' | 'tensor';
+export interface Mask {
+    toCanvasImageSource(): Promise<CanvasImageSource>;
+    toImageData(): Promise<ImageData>;
+    toTensor(): Promise<Tensor3D>;
+    getUnderlyingType(): MaskUnderlyingType;
+}
+/**
+ * Interfaces from "tensorflow-models/body-segmentation" interfaces
+ */
+export interface Segmentation {
+    maskValueToLabel: (maskValue: number) => string;
+    mask: Mask;
+}
+export type Color = {
+    r: number;
+    g: number;
+    b: number;
+    a: number;
+};
+export type SegmentationModel = (typeof SEG_MODELS)[number];
+export type RenderEffects = (typeof RENDER_EFFECTS)[number];
+export type ProcessInputType = ImageData | HTMLVideoElement | HTMLImageElement | OffscreenCanvas | HTMLCanvasElement | ImageBitmap;
+export type InputFrame = CanvasImageSource | VideoFrame;
+export type ImageType = CanvasImageSource | ProcessInputType | VideoFrame;
+interface ProcessingSize {
+    /**
+     * Processing Width size
+     */
+    width: number;
+    /**
+     * Processing height size
+     */
+    height: number;
+}
+export interface RenderParams extends ProcessingSize {
+    /**
+     * Range in between 0 and 1
+     * @defaultValue `0.5`
+     */
+    foregroundThreshold: number;
+    /**
+     * Range in between 0 and 20
+     * @defaultValue `3`
+     */
+    backgroundBlurAmount: number;
+    /**
+     * Range in between 0 and 20
+     * @defaultValue `3`
+     */
+    edgeBlurAmount: number;
+    /**
+     * @defaultValue `false`
+     */
+    flipHorizontal: boolean;
+    /**
+     * @defaultValue `none`
+     */
+    effects: RenderEffects;
+    /**
+     * Background image used for background replacement
+     */
+    backgroundImage: Canvas | undefined;
+}
+export interface AsyncAssets {
+    tfjsCoreLoaded: boolean;
+    tfjsBackendLoaded: boolean;
+    glueLoaded: boolean;
+}
+export type ProcessStatus = 'created' | 'opened' | 'opening' | 'processing' | 'idle' | 'closed' | 'destroying' | 'destroyed';
+export interface Process {
+    status: ProcessStatus;
+    open(): Promise<void>;
+    close(): void;
+    destroy(): Promise<void>;
+}
+export interface Segmenter extends Process, ProcessingSize {
+    model: SegmentationModel;
+    process(input: ProcessInputType): Promise<Segmentation[]>;
+}
+export interface Detector<T> extends Process {
+    detect(input: ProcessInputType): Promise<T>;
+}
+export interface SegmentationParams extends RenderParams {
+    segmenter: Segmenter;
+    loadBackgroundImage(url: string): Promise<void>;
+    backgroundImageUrl?: string;
+}
+export type SegmentationTransform = Transform<InputFrame, InputFrame> & SegmentationParams & Omit<Process, 'open'>;
+export {};

package/dist/main/video/types.js

@@ -0,0 +1,20 @@
+export const RENDER_EFFECTS = [
+    /**
+     * Pass through, and thus there is no effects applied
+     */
+    'none',
+    /**
+     * Background blur effects
+     */
+    'blur',
+    /**
+     * Background overlay effects
+     */
+    'overlay',
+];
+export const SEG_MODELS = [
+    /**
+     * Mediapipe's Selfie Segmentation model
+     */
+    'mediapipeSelfie',
+];