@pexip/media-processor 16.7.1 → 17.1.0

package/dist/main/utils.js

@@ -0,0 +1,364 @@
+export const hasCreateGain = (context) => typeof context.createGain !== 'undefined';
+export const hasAudioWorkletNode = () => typeof AudioWorkletNode !== 'undefined';
+export const hasAudioWorklet = () => typeof AudioWorklet !== 'undefined' && hasAudioWorkletNode();
+const stopTrack = (track) => track.stop();
+export const stopStreamTracks = (stream) => stream?.getTracks().forEach(stopTrack);
+/**
+ * A function to create `MediaStreamAudioSourceNode` using constructor or factory
+ * function depends on the browser supports
+ *
+ * @param context - @see {@link AudioContext}
+ * @param options - @see {@link MediaStreamAudioSourceOptions}
+ *
+ * @internal
+ */
+export const createMediaStreamAudioSourceNode = (context, options) => {
+    try {
+        const source = new MediaStreamAudioSourceNode(context, options);
+        return source;
+    }
+    catch {
+        return context.createMediaStreamSource(options.mediaStream);
+    }
+};
+/**
+ * A function to create `MediaStreamAudioSourceNode` using constructor or factory
+ * function depends on the browser supports
+ *
+ * @param context - @see {@link AudioContext}
+ * @param options - @see {@link MediaStreamAudioSourceOptions}
+ *
+ * @internal
+ */
+export const createMediaElementSourceNode = (context, options) => {
+    try {
+        const source = new MediaElementAudioSourceNode(context, options);
+        return source;
+    }
+    catch {
+        return context.createMediaElementSource(options.mediaElement);
+    }
+};
+/**
+ * A function to set AudioNodeOptions accordingly
+ */
+const setAudioNodeOptions = (node, options) => {
+    if (options?.channelCount) {
+        node.channelCount = options.channelCount;
+    }
+    if (options?.channelCountMode) {
+        node.channelCountMode = options.channelCountMode;
+    }
+    if (options?.channelInterpretation) {
+        node.channelInterpretation = options.channelInterpretation;
+    }
+};
+/**
+ * A function to create `AnalyserNode` using constructor or factory
+ * function depends on the browser supports
+ *
+ * @param audioContext - @see {@link AudioContext}
+ * @param options - @see {@link AnalyserOptions}
+ *
+ * @internal
+ */
+export const createAnalyserNode = (audioContext, options) => {
+    try {
+        const analyser = new AnalyserNode(audioContext, options);
+        return analyser;
+    }
+    catch {
+        const analyser = audioContext.createAnalyser();
+        options?.fftSize && (analyser.fftSize = options.fftSize);
+        options?.maxDecibels && (analyser.maxDecibels = options.maxDecibels);
+        options?.minDecibels && (analyser.minDecibels = options.minDecibels);
+        options?.smoothingTimeConstant &&
+            (analyser.smoothingTimeConstant = options.smoothingTimeConstant);
+        setAudioNodeOptions(analyser, options);
+        return analyser;
+    }
+};
+/**
+ * A function to create `GainNode` using constructor or factory
+ * function depends on the browser supports
+ *
+ * @param context - @see {@link AudioContext}
+ * @param options - @see {@link GainOptions}
+ *
+ * @internal
+ */
+export const createGainNode = (context, options) => {
+    try {
+        const volume = new GainNode(context, options);
+        return volume;
+    }
+    catch {
+        const volume = hasCreateGain(context)
+            ? context.createGain()
+            : context.createGainNode();
+        if (options?.gain) {
+            volume.gain.setValueAtTime(options.gain, context.currentTime);
+        }
+        setAudioNodeOptions(volume, options);
+        return volume;
+    }
+};
+/**
+ * A function to clone the Audio Track
+ *
+ * @param stream - Stream to be cloned
+ *
+ * @internal
+ */
+export const createMediaStreamAudioClone = (stream) => {
+    try {
+        const mediaStream = new MediaStream(stream.getAudioTracks().map(track => track.clone()));
+        return mediaStream;
+    }
+    catch {
+        return stream.clone();
+    }
+};
+/**
+ * A function to create `MediaStreamAudioDestinationNode` using constructor or
+ * factory function depends on the browser supports
+ *
+ * @param context - @see {@link AudioContext}
+ * @param options - @see {@link AudioNodeOptions}
+ *
+ * @internal
+ */
+export const createMediaStreamAudioDestinationNode = (context, options) => {
+    try {
+        const destination = new MediaStreamAudioDestinationNode(context, options);
+        return destination;
+    }
+    catch {
+        const destination = context.createMediaStreamDestination();
+        setAudioNodeOptions(destination, options);
+        return destination;
+    }
+};
+/**
+ * A function to create `DelayNode` using constructor or
+ * factory function depends on the browser supports
+ *
+ * @param context - @see {@link AudioContext}
+ * @param options - @see {@link DelayOptions}
+ *
+ * @internal
+ */
+export const createDelayNode = (context, options) => {
+    try {
+        const delay = new DelayNode(context, options);
+        return delay;
+    }
+    catch {
+        const delay = context.createDelay(options?.maxDelayTime);
+        if (options?.delayTime !== undefined) {
+            delay.delayTime.setValueAtTime(options?.delayTime, context.currentTime);
+        }
+        setAudioNodeOptions(delay, options);
+        return delay;
+    }
+};
+/**
+ * A function to create `ChannelSplitterNode` using constructor or
+ * factory function depends on the browser supports
+ *
+ * @param context - @see {@link AudioContext}
+ * @param options - @see {@link ChannelSplitterOptions}
+ *
+ * @internal
+ */
+export const createChannelSplitterNode = (context, options) => {
+    try {
+        const node = new ChannelSplitterNode(context, options);
+        return node;
+    }
+    catch {
+        const node = context.createChannelSplitter(options?.numberOfOutputs);
+        setAudioNodeOptions(node, options);
+        return node;
+    }
+};
+/**
+ * A function to create `ChannelMergerNode` using constructor or
+ * factory function depends on the browser supports
+ *
+ * @param context - @see {@link AudioContext}
+ * @param options - @see {@link ChannelSplitterOptions}
+ *
+ * @internal
+ */
+export const createChannelMergerNode = (context, options) => {
+    try {
+        const node = new ChannelMergerNode(context, options);
+        return node;
+    }
+    catch {
+        const node = context.createChannelMerger(options?.numberOfInputs);
+        setAudioNodeOptions(node, options);
+        return node;
+    }
+};
+/**
+ * Map mute value to gain value
+ *
+ * ```
+ * `true` -> 0
+ * `false` -> 1
+ * ```
+ */
+export const muteToGain = (mute) => (mute ? 0 : 1);
+/**
+ * Calculate the timeout based on the provided data and returns a timeout in
+ * milliseconds with compensation added
+ *
+ * @param targetTime - The target timeout after the compensation
+ * @param startTime - The start time of the last execution
+ * @param endTime - The end time of the last execution
+ */
+export const calculateNextTimeout = (targetTime, startTime, endTime) => Math.max(targetTime - Math.max(endTime - startTime, 0), 0);
+/**
+ * Convert a callback to an async callback with delay added
+ *
+ * @param callback - The callback to be delayed
+ * @param options - The options to inject dependences
+ *
+ * @example
+ *
+ * ```typescript
+ * const getRandom = () => Math.random();
+ * const [delayGetRandom, cancelDelayGetRandom] = createDelayedCallback(getRandom);
+ *
+ * // Delay 500 ms to get the random number
+ * const random = await delayGetRandom(500);
+ * ```
+ */
+export const createDelayedCallback = (callback, { setTimeout = window.setTimeout, clearTimeout = window.clearTimeout, } = {}) => {
+    const props = {
+        timeoutID: 0,
+    };
+    const cancelTimeout = () => {
+        if (props.timeoutID) {
+            clearTimeout(props.timeoutID);
+            props.timeoutID = 0;
+        }
+    };
+    const delayedCallback = async (delayMs, ...params) => {
+        // eslint-disable-next-line @typescript-eslint/no-invalid-void-type -- void
+        const resolved = await new Promise(resolve => {
+            cancelTimeout();
+            props.timeoutID = setTimeout(() => {
+                const result = callback(...params);
+                if (result instanceof Promise) {
+                    result
+                        .then(resolved => resolve(resolved))
+                        .catch(e => {
+                        throw e;
+                    });
+                }
+                else {
+                    resolve(result);
+                }
+            }, delayMs);
+            props.cancel = resolve;
+        });
+        return resolved;
+    };
+    const cancel = () => {
+        cancelTimeout();
+        props.cancel?.();
+    };
+    return [delayedCallback, cancel];
+};
+/**
+ * Convert the rate to milliseconds
+ */
+const rateToMs = (rate) => Math.ceil(1000 / rate);
+/**
+ * Create an async callback loop to be called recursively with delay based on
+ * the `frameRate`
+ *
+ * @param callback - The callback to be invoked
+ * @param frameRate - The rate to be expected to invoke the `callback`
+ */
+export const createAsyncCallbackLoop = (callback, frameRate, { setTimeout = window.setTimeout, clearTimeout = window.clearTimeout, now = () => performance.now(), } = {}) => {
+    const props = {
+        frameRate,
+        targetMs: rateToMs(frameRate),
+        prevCalledMs: 0,
+        timeoutID: 0,
+        stopped: false,
+    };
+    const [delayedCallback, cancel] = createDelayedCallback(callback, {
+        setTimeout,
+        clearTimeout,
+    });
+    const fork = async (...params) => {
+        if (props.stopped) {
+            return;
+        }
+        const currentMs = now();
+        const nextMs = calculateNextTimeout(props.targetMs, props.prevCalledMs, currentMs);
+        props.prevCalledMs = currentMs;
+        await delayedCallback(nextMs, ...params);
+        await fork(...params);
+    };
+    return {
+        start: async (...params) => {
+            props.prevCalledMs = now();
+            props.stopped = false;
+            await delayedCallback(0, ...params);
+            void fork(...params);
+        },
+        stop: () => {
+            props.stopped = true;
+            cancel();
+        },
+        get frameRate() {
+            return props.frameRate;
+        },
+        set frameRate(value) {
+            props.frameRate = value;
+            props.targetMs = rateToMs(value);
+        },
+    };
+};
+export const DEFAULT_THROTTLE_MS = 3000;
+/**
+ * A function to limit the provided callback being called too frequently, and
+ * assuming the function is called repeatably, and NOT for general purpose.
+ *
+ * @param callback - the callback to be called under the specified time
+ * @param throttleMs - the specified time for throttling
+ * @param clock - how to get the current time
+ */
+export const throttleProcess = (callback, throttleMs = DEFAULT_THROTTLE_MS, clock = performance) => {
+    let lastCall = 0;
+    return (...params) => {
+        const now = clock.now();
+        if (now - lastCall >= throttleMs) {
+            callback(...params);
+            lastCall = now;
+        }
+    };
+};
+/**
+ * Subscribe visibilitychange event
+ * @see {@link https://developer.mozilla.org/en-US/docs/Web/API/Document/visibilitychange_event}
+ *
+ * @param callback - A callback to be called with `document.hidden` when the event is trigger
+ */
+export const subscribeVisibilityChangeEvent = (callback) => {
+    const handleEvent = () => {
+        callback(document.hidden).catch(error => {
+            throw error;
+        });
+    };
+    document.addEventListener('visibilitychange', handleEvent);
+    return () => {
+        document.removeEventListener('visibilitychange', handleEvent);
+    };
+};

package/dist/main/video/canvasRenderUtils.d.ts

@@ -0,0 +1,22 @@
+/// <reference types="dom-webcodecs" />
+import type { Canvas } from '../types';
+import type { ProcessInputType, Segmentation, ImageType } from './types';
+interface InternalCanvases {
+    drawImageDataCanvas?: Canvas;
+    maskCanvas?: Canvas;
+    blurredMaskCanvas?: Canvas;
+    blurredCanvas?: Canvas;
+    backgroundImageCanvas?: Canvas;
+    inputCanvas?: Canvas;
+}
+export declare const createCanvasRenderUtils: (processingWidth: number, processingHeight: number) => {
+    evaluateInput: (inputImage: ProcessInputType | VideoFrame) => Promise<HTMLCanvasElement>;
+    renderImageToCanvas: (image: ImageType, canvas: Canvas, dw?: number, dh?: number, options?: CanvasRenderingContext2DSettings) => Promise<void>;
+    drawBlurEffect: (canvas: Canvas, inputImage: ImageType, segmentations: Segmentation | Segmentation[], foregroundThreshold?: number, backgroundBlurAmount?: number, edgeBlurAmount?: number, flipHorizontal?: boolean) => Promise<void>;
+    drawOverlayEffect: (canvas: Canvas, inputImage: ProcessInputType, backgroundImage: CanvasImageSource | OffscreenCanvas, segmentations: Segmentation | Segmentation[], foregroundThreshold?: number, backgroundBlurAmount?: number, edgeBlurAmount?: number, flipHorizontal?: boolean) => Promise<void>;
+    loadBackgroundImage: (url: string) => Promise<Canvas>;
+    renderImageToOffScreenCanvas: (image: ImageType, canvasName: keyof InternalCanvases) => Promise<HTMLCanvasElement>;
+    renderImageDataToOffScreenCanvas: (image: ImageData, canvasName: keyof InternalCanvases) => Canvas;
+    drawAndBlurImageOnOffScreenCanvas: (image: ImageType, blurAmount: number, offscreenCanvasName: keyof InternalCanvases) => Promise<Canvas>;
+};
+export {};

package/dist/main/video/canvasRenderUtils.js

@@ -0,0 +1,198 @@
+import { Tensor, browser } from '@tensorflow/tfjs-core/dist/base.js';
+import { fitDestinationSize } from '../process';
+import { createOffscreenCanvas, getImageSize, getCanvasRenderingContext2D, toBinaryMask, flipCanvasHorizontal, loadImage, } from './utils';
+export const createCanvasRenderUtils = (processingWidth, processingHeight) => {
+    const props = {};
+    const getImage = (imageName) => {
+        const image = props[imageName];
+        if (!image) {
+            const img = new Image();
+            props[imageName] = img;
+            return img;
+        }
+        return image;
+    };
+    const getCanvas = (canvasName) => {
+        const canvas = props[canvasName];
+        if (!canvas) {
+            const canvas = createOffscreenCanvas(processingWidth, processingHeight);
+            props[canvasName] = canvas;
+            return canvas;
+        }
+        return canvas;
+    };
+    const renderImageDataToOffScreenCanvas = (image, canvasName) => {
+        const canvas = getCanvas(canvasName);
+        const context = getCanvasRenderingContext2D(canvas);
+        context.putImageData(image, 0, 0);
+        return canvas;
+    };
+    /**
+     * Draw image on a 2D rendering context.
+     */
+    const drawImage = async (ctx, image, sx, sy, sw, sh, dx, dy, dw, dh) => {
+        if (image instanceof Tensor) {
+            const pixels = await browser.toPixels(image);
+            const { height, width } = getImageSize(image);
+            image = new ImageData(pixels, width, height);
+        }
+        const source = image instanceof ImageData
+            ? renderImageDataToOffScreenCanvas(image, 'drawImageDataCanvas')
+            : image;
+        if (sw === undefined || sh === undefined) {
+            ctx.drawImage(source, sx, sy);
+        }
+        else if (dx === undefined ||
+            dy === undefined ||
+            dw === undefined ||
+            dh === undefined) {
+            ctx.drawImage(source, sx, sy, sw, sh);
+        }
+        else {
+            ctx.drawImage(source, sx, sy, sw, sh, dx, dy, dw, dh);
+        }
+    };
+    const renderImageToCanvas = async (image, canvas, dw = processingWidth, dh = processingHeight, options = {}) => {
+        const { height, width } = getImageSize(image);
+        const rect = fitDestinationSize(width, height, dw, dh);
+        const ctx = getCanvasRenderingContext2D(canvas, options);
+        await drawImage(ctx, image, rect.x, rect.y, rect.width, rect.height);
+    };
+    const renderImageToOffScreenCanvas = async (image, canvasName) => {
+        const canvas = getCanvas(canvasName);
+        await renderImageToCanvas(image, canvas);
+        return canvas;
+    };
+    const drawWithCompositing = async (ctx, image, compositeOperation) => {
+        ctx.globalCompositeOperation = compositeOperation;
+        await drawImage(ctx, image, 0, 0);
+    };
+    // method copied from blur in https://codepen.io/zhaojun/pen/zZmRQe
+    const cpuBlur = async (canvas, image, blur) => {
+        const ctx = getCanvasRenderingContext2D(canvas);
+        let sum = 0;
+        const delta = 5;
+        const alphaLeft = 1 / (2 * Math.PI * delta * delta);
+        const step = blur < 3 ? 1 : 2;
+        for (let y = -blur; y <= blur; y += step) {
+            for (let x = -blur; x <= blur; x += step) {
+                const weight = alphaLeft *
+                    Math.exp(-(x * x + y * y) / (2 * delta * delta));
+                sum += weight;
+            }
+        }
+        for (let y = -blur; y <= blur; y += step) {
+            for (let x = -blur; x <= blur; x += step) {
+                ctx.globalAlpha =
+                    ((alphaLeft *
+                        Math.exp(-(x * x + y * y) / (2 * delta * delta))) /
+                        sum) *
+                        blur;
+                await drawImage(ctx, image, x, y);
+            }
+        }
+        ctx.globalAlpha = 1;
+    };
+    const drawAndBlurImageOnCanvas = async (image, blurAmount, canvas) => {
+        const { height, width } = getImageSize(image);
+        const ctx = getCanvasRenderingContext2D(canvas);
+        ctx.clearRect(0, 0, width, height);
+        if (blurAmount <= 0) {
+            return drawImage(ctx, image, 0, 0, width, height);
+        }
+        ctx.save();
+        if ('filter' in ctx) {
+            // Avoid the transparent edge by Gaussian blur
+            await drawImage(ctx, image, 0, 0, width, height);
+            ctx.filter = `blur(${blurAmount}px)`;
+            await drawImage(ctx, image, 0, 0, width, height);
+        }
+        else {
+            // Safari doesn't support filter
+            // https://developer.mozilla.org/en-US/docs/Web/API/CanvasRenderingContext2D/filter
+            await cpuBlur(canvas, image, blurAmount);
+        }
+        ctx.restore();
+    };
+    const drawAndBlurImageOnOffScreenCanvas = async (image, blurAmount, offscreenCanvasName) => {
+        const canvas = getCanvas(offscreenCanvasName);
+        if (blurAmount === 0) {
+            await renderImageToCanvas(image, canvas);
+        }
+        else {
+            await drawAndBlurImageOnCanvas(image, blurAmount, canvas);
+        }
+        return canvas;
+    };
+    const createPersonMask = async (segmentation, foregroundThreshold, edgeBlurAmount) => {
+        const backgroundMaskImage = await toBinaryMask(segmentation, { r: 0, g: 0, b: 0, a: 255 }, { r: 0, g: 0, b: 0, a: 0 }, false, foregroundThreshold);
+        if (!backgroundMaskImage) {
+            return getCanvas('maskCanvas');
+        }
+        const backgroundMask = renderImageDataToOffScreenCanvas(backgroundMaskImage, 'maskCanvas');
+        if (edgeBlurAmount === 0) {
+            return backgroundMask;
+        }
+        else {
+            return drawAndBlurImageOnOffScreenCanvas(backgroundMask, edgeBlurAmount, 'blurredMaskCanvas');
+        }
+    };
+    const loadImageElement = async (url, imageName) => {
+        const image = getImage(imageName);
+        await loadImage(image, url);
+        return image;
+    };
+    const loadAndDrawImageOnOffscreenCanvas = async (url, canvasName, imageName) => {
+        const image = await loadImageElement(url, imageName);
+        const canvas = getCanvas(canvasName);
+        const context = getCanvasRenderingContext2D(canvas);
+        const imageSize = getImageSize(image);
+        const rect = fitDestinationSize(imageSize.width, imageSize.height, processingWidth, processingHeight);
+        await drawImage(context, image, rect.x, rect.y, rect.width, rect.height);
+        return canvas;
+    };
+    const loadBackgroundImage = (url) => loadAndDrawImageOnOffscreenCanvas(url, 'backgroundImageCanvas', 'backgroundImage');
+    const drawBokehEffect = async (canvas, inputImage, backgroundImage, segmentations, foregroundThreshold = 0.5, backgroundBlurAmount = 3, edgeBlurAmount = 3, flipHorizontal = false) => {
+        const blurredImage = await drawAndBlurImageOnOffScreenCanvas(backgroundImage, backgroundBlurAmount, 'blurredCanvas');
+        const ctx = getCanvasRenderingContext2D(canvas);
+        if (Array.isArray(segmentations) && segmentations.length === 0) {
+            return drawImage(ctx, blurredImage, 0, 0);
+        }
+        const personMask = await createPersonMask(segmentations, foregroundThreshold, edgeBlurAmount);
+        ctx.save();
+        if (flipHorizontal) {
+            flipCanvasHorizontal(canvas);
+        }
+        // draw the original image on the final canvas
+        const { height, width } = getImageSize(inputImage);
+        await drawImage(ctx, inputImage, 0, 0, width, height);
+        // "destination-in" - "The existing canvas content is kept where both the
+        // new shape and existing canvas content overlap. Everything else is made
+        // transparent."
+        // crop what's not the person using the mask from the original image
+        await drawWithCompositing(ctx, personMask, 'destination-in');
+        // "destination-over" - "The existing canvas content is kept where both the
+        // new shape and existing canvas content overlap. Everything else is made
+        // transparent."
+        // draw the blurred background on top of the original image where it doesn't
+        // overlap.
+        await drawWithCompositing(ctx, blurredImage, 'destination-over');
+        ctx.restore();
+    };
+    const drawBlurEffect = (canvas, inputImage, segmentations, foregroundThreshold = 0.5, backgroundBlurAmount = 3, edgeBlurAmount = 3, flipHorizontal = false) => drawBokehEffect(canvas, inputImage, inputImage, segmentations, foregroundThreshold, backgroundBlurAmount, edgeBlurAmount, flipHorizontal);
+    const drawOverlayEffect = (canvas, inputImage, backgroundImage, segmentations, foregroundThreshold = 0.5, backgroundBlurAmount = 0, edgeBlurAmount = 3, flipHorizontal = false) => drawBokehEffect(canvas, inputImage, backgroundImage, segmentations, foregroundThreshold, backgroundBlurAmount, edgeBlurAmount, flipHorizontal);
+    const evaluateInput = async (inputImage) => {
+        const image = await renderImageToOffScreenCanvas(inputImage, 'inputCanvas');
+        return image;
+    };
+    return {
+        evaluateInput,
+        renderImageToCanvas,
+        drawBlurEffect,
+        drawOverlayEffect,
+        loadBackgroundImage,
+        renderImageToOffScreenCanvas,
+        renderImageDataToOffScreenCanvas,
+        drawAndBlurImageOnOffScreenCanvas,
+    };
+};

package/dist/main/video/canvasTransform.d.ts

@@ -0,0 +1,8 @@
+import type { Segmenter, RenderParams, SegmentationTransform } from './types';
+type Params = Omit<RenderParams, 'frameRate'>;
+interface Options extends Omit<Params, 'backgroundImage'> {
+    selfManageSegmenter?: boolean;
+    bgImageUrl?: string;
+}
+export declare const createTransform: (segmenter: Segmenter, { width, height, foregroundThreshold, backgroundBlurAmount, edgeBlurAmount, flipHorizontal, effects, selfManageSegmenter, bgImageUrl, }?: Partial<Options>) => SegmentationTransform;
+export {};