@pexip/media-processor 17.4.0 → 18.1.0

package/dist/main/video/canvasTransform.js

@@ -1,16 +1,9 @@
-import { stopStreamTracks } from '../utils';
-import { FOREGROUND_THRESHOLD, BACKGROUND_BLUR_AMOUNT, EDGE_BLUR_AMOUNT, FLIP_HORIZONTAL, PROCESSING_WIDTH, PROCESSING_HEIGHT, } from './constants';
-import { createCanvas } from './utils';
-import { createCanvasRenderUtils } from './canvasRenderUtils';
-const createAssertInRange = (from, to) => (value) => {
-    if (value < from || value > to) {
-        throw new Error(`Invalid value (${value}) to range [${from}, ${to}]`);
-    }
-};
-const assertInRangeFrom0To1 = createAssertInRange(0, 1);
-const assertInRangeFrom0To20 = createAssertInRange(0, 20);
-const NOT_INITED_ERROR_MSG = 'Please call init() method first!';
-export const createTransform = (segmenter, { width = PROCESSING_WIDTH, height = PROCESSING_HEIGHT, foregroundThreshold = FOREGROUND_THRESHOLD, backgroundBlurAmount = BACKGROUND_BLUR_AMOUNT, edgeBlurAmount = EDGE_BLUR_AMOUNT, flipHorizontal = FLIP_HORIZONTAL, effects = 'none', selfManageSegmenter, bgImageUrl, } = {}) => {
+import { PROCESSING_HEIGHT, PROCESSING_WIDTH, FOREGROUND_THRESHOLD, BACKGROUND_BLUR_AMOUNT, EDGE_BLUR_AMOUNT, MASK_COMBINE_RATIO, } from '../../common/constants';
+import { createCanvasRenderUtils } from '../../common/canvasRenderUtils';
+import { clamping } from '../../common/utils';
+const clampInRangeFrom0To1 = clamping(0, 1);
+const clampInRangeFrom0To9 = clamping(0, 9);
+export const createTransform = (segmenter, { width = PROCESSING_WIDTH, height = PROCESSING_HEIGHT, foregroundThreshold = FOREGROUND_THRESHOLD, backgroundBlurAmount = BACKGROUND_BLUR_AMOUNT, edgeBlurAmount = EDGE_BLUR_AMOUNT, maskCombineRatio = MASK_COMBINE_RATIO, backgroundImageUrl = '', effects = 'none', selfManageSegmenter, ...options } = {}) => {
     const props = {
         segmenter,
         width,
@@ -18,40 +11,67 @@ export const createTransform = (segmenter, { width = PROCESSING_WIDTH, height =
         foregroundThreshold,
         backgroundBlurAmount,
         edgeBlurAmount,
-        flipHorizontal,
         utils: createCanvasRenderUtils(width, height),
         effects,
-        backgroundImage: undefined,
-        status: 'created',
-        backgroundImageUrl: bgImageUrl,
+        maskCombineRatio,
+        backgroundImage: options.backgroundImage,
+        status: 'new',
+        backgroundImageUrl,
+        outputCanvas: options.outputCanvas,
+        outputDelegate: options.outputDelegate,
+    };
+    const toVideoFrameLike = (input) => {
+        // Standardize the input
+        const image = input instanceof ImageData
+            ? props.utils.renderImageDataToOffScreenCanvas(input, 'drawImageDataCanvas')
+            : input;
+        // Pre-precessing the input to resize to the processing size
+        const canvas = props.utils.renderImageToOffScreenCanvas(image, 'inputCanvas');
+        if ('close' in input) {
+            input.close();
+        }
+        return {
+            frame: canvas.transferToImageBitmap(),
+            width,
+            height,
+            timestamp: performance.now(),
+        };
     };
     const processInput = async (input) => {
-        if (props.segmenter.status === 'created' ||
+        const renderOptions = {
+            foregroundThreshold: props.foregroundThreshold,
+            backgroundBlurAmount: props.backgroundBlurAmount,
+            edgeBlurAmount: props.edgeBlurAmount,
+            maskCombineRatio: props.maskCombineRatio,
+            backgroundImageUrl: props.backgroundImageUrl,
+            effects: props.effects,
+        };
+        if (props.segmenter.status === 'new' ||
             props.segmenter.status === 'closed') {
-            await props.segmenter.open();
+            await props.segmenter.open({
+                output: props.outputDelegate,
+                processingWidth: props.width,
+                processingHeight: props.height,
+                backgroundImage: props.backgroundImage,
+                renderOptions,
+            });
         }
         if (props.segmenter.status === 'opening') {
-            return [];
-        }
-        return await props.segmenter.process(input);
-    };
-    const loadBackgroundImage = async (url) => {
-        if (props.backgroundImageUrl === url && props.backgroundImage) {
-            return;
+            return undefined;
         }
-        props.backgroundImageUrl = url;
-        props.backgroundImage = await props.utils.loadBackgroundImage(url);
+        const inputFrame = toVideoFrameLike(input);
+        return props.segmenter.process(inputFrame, renderOptions);
     };
     return {
-        get status() {
-            return props.status;
-        },
         get width() {
             return props.width;
         },
         get height() {
             return props.height;
         },
+        get status() {
+            return props.status;
+        },
         get foregroundThreshold() {
             return props.foregroundThreshold;
         },
@@ -61,40 +81,59 @@ export const createTransform = (segmenter, { width = PROCESSING_WIDTH, height =
         get edgeBlurAmount() {
             return props.edgeBlurAmount;
         },
-        get flipHorizontal() {
-            return props.flipHorizontal;
-        },
         get effects() {
             return props.effects;
         },
-        get backgroundImage() {
-            return props.backgroundImage;
+        get maskCombineRatio() {
+            return props.maskCombineRatio;
         },
         set foregroundThreshold(value) {
-            assertInRangeFrom0To1(value);
-            props.foregroundThreshold = value;
+            props.foregroundThreshold = clampInRangeFrom0To1(value);
         },
         set backgroundBlurAmount(value) {
-            assertInRangeFrom0To20(value);
-            props.backgroundBlurAmount = value;
+            props.backgroundBlurAmount = clampInRangeFrom0To9(value);
         },
         set edgeBlurAmount(value) {
-            assertInRangeFrom0To20(value);
-            props.edgeBlurAmount = value;
-        },
-        set flipHorizontal(value) {
-            props.flipHorizontal = value;
+            props.edgeBlurAmount = clampInRangeFrom0To9(value);
         },
         set effects(value) {
             props.effects = value;
         },
+        set backgroundImageUrl(value) {
+            props.backgroundImageUrl = value;
+        },
         get backgroundImageUrl() {
             return props.backgroundImageUrl;
         },
-        set backgroundImage(canvas) {
-            props.backgroundImage = canvas;
+        set maskCombineRatio(value) {
+            props.maskCombineRatio = clampInRangeFrom0To1(value);
+        },
+        get outputCanvas() {
+            return props.outputCanvas;
+        },
+        get backgroundImage() {
+            return props.backgroundImage;
+        },
+        set backgroundImage(image) {
+            if (image &&
+                image.key !== props.backgroundImage?.key &&
+                image.image !== props.backgroundImage?.image) {
+                props.backgroundImage = image;
+                if (props.segmenter.status !== 'new' &&
+                    props.segmenter.status !== 'closed') {
+                    void props.segmenter.update({ backgroundImage: image });
+                }
+            }
+        },
+        set outputCanvas(canvas) {
+            props.outputCanvas = canvas;
+        },
+        get outputDelegate() {
+            return props.outputDelegate;
+        },
+        set outputDelegate(canvas) {
+            props.outputDelegate = canvas;
         },
-        loadBackgroundImage,
         get segmenter() {
             return props.segmenter;
         },
@@ -103,48 +142,27 @@ export const createTransform = (segmenter, { width = PROCESSING_WIDTH, height =
                 props.segmenter = value;
             }
         },
-        init: async () => {
-            props.outputCanvas = createCanvas(width, height);
-            if (props.backgroundImageUrl) {
-                await loadBackgroundImage(props.backgroundImageUrl);
-            }
+        init: () => {
             props.status = 'opened';
+            return Promise.resolve();
         },
         transform: async (videoFrame, controller) => {
-            if (!props.outputCanvas) {
-                throw new Error(NOT_INITED_ERROR_MSG);
-            }
             switch (props.effects) {
+                case 'overlay':
                 case 'blur': {
-                    const image = await props.utils.renderImageToOffScreenCanvas(videoFrame, 'inputCanvas');
-                    const segmentations = await processInput(image);
-                    if (props.status === 'closed') {
-                        break;
-                    }
-                    if (props.outputCanvas.height !== height) {
-                        props.outputCanvas.width = width;
-                        props.outputCanvas.height = height;
-                    }
-                    await props.utils.drawBlurEffect(props.outputCanvas, image, segmentations, props.foregroundThreshold, props.backgroundBlurAmount, props.edgeBlurAmount, props.flipHorizontal);
-                    controller.enqueue(props.outputCanvas);
-                    break;
-                }
-                case 'overlay': {
-                    if (!props.backgroundImage) {
-                        throw new Error('Please call setBackgroundImage() method first');
-                    }
-                    const image = await props.utils.renderImageToOffScreenCanvas(videoFrame, 'inputCanvas');
-                    const segmentations = await processInput(image);
-                    if (props.status === 'closed') {
+                    const result = await processInput(videoFrame);
+                    const processedFrame = props.outputCanvas ?? result?.frame;
+                    if (props.status === 'closed' ||
+                        // Avoid processing disposed frame
+                        !processedFrame ||
+                        ('displayWidth' in processedFrame &&
+                            processedFrame.displayWidth === 0) ||
+                        ('width' in processedFrame &&
+                            processedFrame.width === 0)) {
                         break;
                     }
-                    if (props.outputCanvas.height !== height) {
-                        props.outputCanvas.width = width;
-                        props.outputCanvas.height = height;
-                    }
-                    await props.utils.drawOverlayEffect(props.outputCanvas, image, props.backgroundImage, segmentations, props.foregroundThreshold, 0, // No blur for overlay
-                    props.edgeBlurAmount, props.flipHorizontal);
-                    controller.enqueue(props.outputCanvas);
+                    controller.enqueue(props.outputCanvas ?? result?.frame);
+                    result?.frame.close();
                     break;
                 }
                 case 'none': {
@@ -158,9 +176,6 @@ export const createTransform = (segmenter, { width = PROCESSING_WIDTH, height =
             if (!selfManageSegmenter) {
                 segmenter.close();
             }
-            props.outputCanvas = undefined;
-            stopStreamTracks(props.outputStream);
-            props.outputStream = undefined;
             props.status = 'closed';
         },
         destroy: async () => {

package/dist/main/video/constants.d.ts

@@ -1,9 +1,3 @@
-export declare const PROCESSING_WIDTH = 768;
-export declare const PROCESSING_HEIGHT = 432;
-export declare const FOREGROUND_THRESHOLD = 0.5;
-export declare const BACKGROUND_BLUR_AMOUNT = 3;
-export declare const EDGE_BLUR_AMOUNT = 3;
-export declare const FLIP_HORIZONTAL = false;
 export declare const FRAME_RATE = 20;
 export declare enum AbortReason {
     Close = "close"

package/dist/main/video/constants.js

@@ -1,10 +1,4 @@
-export const PROCESSING_WIDTH = 768;
-export const PROCESSING_HEIGHT = 432;
 // Rendering parameters
-export const FOREGROUND_THRESHOLD = 0.5;
-export const BACKGROUND_BLUR_AMOUNT = 3; // Between 1 and 20
-export const EDGE_BLUR_AMOUNT = 3; // Between 0 and 20
-export const FLIP_HORIZONTAL = false;
 export const FRAME_RATE = 20;
 export var AbortReason;
 (function (AbortReason) {

package/dist/main/video/index.d.ts

@@ -3,7 +3,7 @@ export * from './video';
 export * from './types';
 export { createTransform as createCanvasTransform } from './canvasTransform';
 export * from './videoStreamTrackProcessor';
-export { createFrameCallbackRequest } from './utils';
-export * from './segmenters';
+export { createFrameCallbackRequest, createCanvas } from './utils';
 export * from './typeGuards';
 export * from './constants';
+export * from './segmenters/mediapipe';

package/dist/main/video/index.js

@@ -3,7 +3,7 @@ export * from './video';
 export * from './types';
 export { createTransform as createCanvasTransform } from './canvasTransform';
 export * from './videoStreamTrackProcessor';
-export { createFrameCallbackRequest } from './utils';
-export * from './segmenters';
+export { createFrameCallbackRequest, createCanvas } from './utils';
 export * from './typeGuards';
 export * from './constants';
+export * from './segmenters/mediapipe';

package/dist/main/video/load.d.ts

@@ -1,17 +1,3 @@
 export declare const loadScript: (path: string, id: string) => Promise<void>;
 export type WasmPaths = [string, string | undefined];
 export declare const loadWasms: (paths: WasmPaths[]) => Promise<void>;
-export declare const loadTfjsCore: (prodMode: boolean) => Promise<unknown>;
-export declare const loadTfjsBackendWebGl: () => Promise<{
-    default: typeof import("@tensorflow/tfjs-backend-webgl");
-    version_webgl: "4.13.0";
-    webgl: {
-        forceHalfFloat: typeof import("@tensorflow/tfjs-backend-webgl").forceHalfFloat;
-    };
-    forceHalfFloat(): void;
-    MathBackendWebGL: typeof import("@tensorflow/tfjs-backend-webgl").MathBackendWebGL;
-    setWebGLContext: typeof import("@tensorflow/tfjs-backend-webgl").setWebGLContext;
-    GPGPUContext: typeof import("@tensorflow/tfjs-backend-webgl").GPGPUContext;
-    gpgpu_util: typeof import("@tensorflow/tfjs-backend-webgl/dist/gpgpu_util");
-    webgl_util: typeof import("@tensorflow/tfjs-backend-webgl/dist/webgl_util");
-}>;

package/dist/main/video/load.js

@@ -36,11 +36,3 @@ export const loadWasms = async (paths) => {
     // TODO: map the wasm path
     await Promise.all(paths.map(([path]) => loadScript(path, path)));
 };
-export const loadTfjsCore = async (prodMode) => {
-    const tfjs = await import('@tensorflow/tfjs-core');
-    if (prodMode) {
-        tfjs.enableProdMode();
-    }
-    return tfjs;
-};
-export const loadTfjsBackendWebGl = () => import('@tensorflow/tfjs-backend-webgl');

package/dist/main/video/segmenters/mediapipe.d.ts

@@ -1,13 +1,11 @@
-import type { Segmenter, AsyncAssets } from '../types';
-export declare const SelfieSegmentationModelTypes: ['general', 'landscape'];
-export type SelfieSegmentationModelType = (typeof SelfieSegmentationModelTypes)[number];
-interface Options extends AsyncAssets {
-    modelType: SelfieSegmentationModelType;
-    processingWidth: number;
-    processingHeight: number;
-    gluePath: string;
-    selfieMode: boolean;
-    prodMode: boolean;
+import type { Segmenter } from '../types';
+import type { SegmenterOptions, ImageSegmenterOptions, SegmentationModelAsset } from '../../../common/types/segmentation';
+interface Options extends ImageSegmenterOptions {
+    processingWidth?: number;
+    processingHeight?: number;
+    workerScriptUrl?: URL;
+    credentials?: RequestCredentials;
+    modelAsset: SegmentationModelAsset;
 }
-export declare const createSegmenter: (basePath?: string, { modelType, tfjsCoreLoaded, tfjsBackendLoaded, glueLoaded, processingWidth, processingHeight, gluePath, selfieMode, prodMode, }?: Partial<Options>) => Segmenter;
+export declare const createSegmenter: (basePath: SegmenterOptions['basePath'], { processingWidth, processingHeight, workerScriptUrl, credentials, ...options }: Options) => Segmenter;
 export {};

package/dist/main/video/segmenters/mediapipe.js

@@ -1,89 +1,139 @@
-import { toSegmentation, createInputImageConvertor, ensure } from '../utils';
-import { createCanvasRenderUtils } from '../canvasRenderUtils';
-import { PROCESSING_WIDTH, PROCESSING_HEIGHT } from '../constants';
-import { loadScript, loadTfjsCore, loadTfjsBackendWebGl } from '../load';
-export const SelfieSegmentationModelTypes = [
-    'general',
-    'landscape',
-];
-export const createSegmenter = (basePath = '/', { modelType = 'general', tfjsCoreLoaded = false, tfjsBackendLoaded = false, glueLoaded = false, processingWidth = PROCESSING_WIDTH, processingHeight = PROCESSING_HEIGHT, gluePath = '', selfieMode = false, prodMode = true, } = {}) => {
-    const props = {
-        selfieMode,
-        segmentation: [],
-        renderUtils: createCanvasRenderUtils(processingWidth, processingHeight),
-        status: 'created',
-        tfjsCoreLoaded,
-        tfjsBackendLoaded,
-        glueLoaded,
+import { PROCESS_STATUS, PROCESSING_WIDTH, PROCESSING_HEIGHT, } from '../../../common/constants';
+const cloneImageRecord = async (image) => {
+    const cloned = await createImageBitmap(image.image);
+    return { image: cloned, key: image.key };
+};
+export const createSegmenter = (basePath = '/', { processingWidth = PROCESSING_WIDTH, processingHeight = PROCESSING_HEIGHT, workerScriptUrl = new URL('../../../workers/mediaWorker.js', import.meta.url), credentials = 'same-origin', ...options }) => {
+    const props = { status: PROCESS_STATUS.New, modelAsset: options.modelAsset };
+    const initWorker = () => {
+        const mediaWorker = new Worker(workerScriptUrl, {
+            type: 'classic',
+            credentials,
+        });
+        mediaWorker.addEventListener('error', error => {
+            throw error.error;
+        });
+        mediaWorker.addEventListener('message', (event) => {
+            const { type } = event.data;
+            if (type === 'error') {
+                throw event.data.error;
+            }
+        });
+        return mediaWorker;
+    };
+    async function postMsg(expectedResponseType, message, transfer) {
+        if (!expectedResponseType) {
+            return props.mediaWorker?.postMessage(message, transfer);
+        }
+        return await new Promise((resolve, reject) => {
+            const handleMessage = (ev) => {
+                removeListener();
+                switch (ev.data.type) {
+                    case expectedResponseType: {
+                        // @ts-expect-error --- Call `resolve` is expecting to pass an argument or nothing
+                        resolve(ev.data.data);
+                        break;
+                    }
+                    case 'error': {
+                        reject(ev.data.error);
+                        break;
+                    }
+                    default: {
+                        throw new Error(`UnhandledEvent: ${ev.data.type}`);
+                    }
+                }
+            };
+            const handleError = (error) => {
+                removeListener();
+                reject(error.error);
+            };
+            const removeListener = () => {
+                props.mediaWorker?.removeEventListener('message', handleMessage);
+                props.mediaWorker?.removeEventListener('error', handleError);
+            };
+            props.mediaWorker?.addEventListener('message', handleMessage);
+            props.mediaWorker?.addEventListener('error', handleError);
+            props.mediaWorker?.postMessage(message, transfer);
+        });
+    }
+    const open = async (processorOptions) => {
+        props.status = PROCESS_STATUS.Opening;
+        props.mediaWorker = initWorker();
+        props.output = processorOptions?.output;
+        const backgroundImage = processorOptions?.backgroundImage &&
+            (await cloneImageRecord(processorOptions.backgroundImage));
+        await postMsg('opened', {
+            type: 'open',
+            data: {
+                basePath,
+                processingWidth,
+                processingHeight,
+                ...processorOptions,
+                imageSegmenterOptions: {
+                    ...options,
+                    ...processorOptions?.imageSegmenterOptions,
+                },
+                backgroundImage,
+            },
+        }, [processorOptions?.output, backgroundImage?.image].filter(Boolean));
+        props.status = PROCESS_STATUS.Opened;
+    };
+    const process = async (input, options) => {
+        props.status = PROCESS_STATUS.Processing;
+        const resultFrame = await postMsg('processed', {
+            type: 'process',
+            data: { videoFrame: input, renderOptions: options },
+        }, [input.frame]);
+        if (props.status === PROCESS_STATUS.Processing) {
+            props.status = PROCESS_STATUS.Idle;
+        }
+        return resultFrame;
+    };
+    const update = async (options) => {
+        const backgroundImage = options.backgroundImage &&
+            (await cloneImageRecord(options.backgroundImage));
+        const transfer = backgroundImage ? [backgroundImage.image] : [];
+        await postMsg(undefined, {
+            type: 'update',
+            data: {
+                ...options,
+                backgroundImage,
+            },
+        }, transfer);
     };
-    const toInputImage = createInputImageConvertor(input => props.renderUtils.drawAndBlurImageOnOffScreenCanvas({
-        image: input,
-        blurAmount: 0,
-        offscreenCanvasName: 'blurredCanvas',
-    }));
-    const segmentPerson = async (input) => {
-        const segmenter = ensure(props.segmenter);
-        props.status = 'processing';
-        const image = await toInputImage(input);
-        await segmenter.send({ image });
-        props.status = 'idle';
+    const close = () => {
+        props.status = PROCESS_STATUS.Closing;
+        void postMsg(undefined, { type: 'close' });
+        props.status = PROCESS_STATUS.Closed;
+    };
+    const destroy = async () => {
+        props.status = PROCESS_STATUS.Destroying;
+        await postMsg(undefined, { type: 'destroy' });
+        props.mediaWorker?.terminate();
+        props.mediaWorker = undefined;
+        props.status = PROCESS_STATUS.Destroyed;
+        return Promise.resolve();
     };
     return {
-        get model() {
-            return 'mediapipeSelfie';
+        get modelAsset() {
+            return props.modelAsset;
         },
-        get width() {
-            return processingWidth;
-        },
-        get height() {
-            return processingHeight;
+        set modelAsset(asset) {
+            props.modelAsset = asset;
         },
         get status() {
             return props.status;
         },
-        open: async () => {
-            props.status = 'opening';
-            // Avoid side-effects from the modules
-            if (!props.tfjsCoreLoaded) {
-                await loadTfjsCore(prodMode);
-                props.tfjsCoreLoaded = true;
-            }
-            if (!props.tfjsBackendLoaded) {
-                await loadTfjsBackendWebGl();
-                props.tfjsBackendLoaded = true;
-            }
-            if (!props.glueLoaded) {
-                await loadScript(gluePath, gluePath);
-                props.glueLoaded = true;
-            }
-            props.segmenter = new SelfieSegmentation({
-                locateFile: (path) => `${basePath.replace(/\/+$/, '')}/${path}`,
-            });
-            const modelSelection = modelType === 'landscape' ? 1 : 0;
-            props.segmenter.setOptions({
-                modelSelection,
-                selfieMode: props.selfieMode,
-            });
-            props.segmenter.onResults(results => {
-                props.segmentation = [
-                    toSegmentation(results.segmentationMask, 'canvasimagesource', () => 'person'),
-                ];
-            });
-            await props.segmenter.initialize();
-            props.status = 'opened';
-        },
-        process: async (input) => {
-            await segmentPerson(input);
-            return props.segmentation;
-        },
-        close: () => {
-            props.segmenter?.reset();
-            props.status = 'closed';
+        get width() {
+            return processingWidth;
         },
-        destroy: async () => {
-            props.status = 'destroying';
-            await props.segmenter?.close();
-            props.status = 'destroyed';
+        get height() {
+            return processingHeight;
         },
+        open,
+        update,
+        process,
+        close,
+        destroy,
     };
 };

package/dist/main/video/typeGuards.d.ts

@@ -1,2 +1 @@
 export declare const isRenderEffects: (t: unknown) => t is "blur" | "none" | "overlay";
-export declare const isSegmentationModel: (t: unknown) => t is "mediapipeSelfie";

package/dist/main/video/typeGuards.js

@@ -1,13 +1,7 @@
-import { RENDER_EFFECTS, SEG_MODELS } from './types';
+import { RENDER_EFFECT } from '../../common/constants';
 export const isRenderEffects = (t) => {
     if (typeof t !== 'string') {
         return false;
     }
-    return RENDER_EFFECTS.some(effect => effect === t);
-};
-export const isSegmentationModel = (t) => {
-    if (typeof t !== 'string') {
-        return false;
-    }
-    return SEG_MODELS.some(model => model === t);
+    return Object.values(RENDER_EFFECT).some(effect => effect === t);
 };