@pexip/media-processor 21.0.0 → 22.0.0

package/dist/main/video/canvasTransform.d.ts

@@ -1,17 +0,0 @@
-import type { ImageRecord } from '../../common/types/media';
-import type { RendererOptions } from '../../common/types/render';
-import type { Segmenter, SegmentationTransform } from './types';
-interface Options extends RendererOptions {
-    selfManageSegmenter?: boolean;
-    outputCanvas?: HTMLCanvasElement;
-    outputDelegate?: OffscreenCanvas;
-    backgroundImage?: ImageRecord;
-    width?: number;
-    height?: number;
-}
-export declare const createTransform: (segmenter: Segmenter, { width, height, foregroundThreshold, backgroundThreshold, backgroundBlurAmount, edgeBlurAmount, maskCombineRatio, backgroundImageUrl, sigmaSpace, sigmaRangeLo, sigmaRangeHi, effects, backend, selfManageSegmenter, personCenter, excludeBystanders, morphPass, morphErodeRadiusPx, morphDilateRadiusPx, lightWrapIntensity, downSampleFactor, lightWrapTightness, lightWrapBlurAmount, lightWrapEdgeBand, ...options }?: Partial<Options>) => SegmentationTransform & {
-    outputCanvas: HTMLCanvasElement | undefined;
-    outputDelegate: OffscreenCanvas | undefined;
-    backgroundImage?: ImageRecord;
-};
-export {};

package/dist/main/video/canvasTransform.js

@@ -1,341 +0,0 @@
-import { PROCESSING_HEIGHT, PROCESSING_WIDTH, FOREGROUND_THRESHOLD, BACKGROUND_THRESHOLD, BACKGROUND_BLUR_AMOUNT, EDGE_BLUR_AMOUNT, MASK_COMBINE_RATIO, SIGMA_RANGE, SIGMA_SPACE, PERSON_CENTER, EXCLUDE_BYSTANDERS, MORPH_PASS, MORPH_DILATE_RADIUS_PX, MORPH_ERODE_RADIUS_PX, LIGHT_WRAP_INTENSITY, DOWN_SAMPLE_FACTOR, LIGHT_WRAP_TIGHTNESS, LIGHT_WRAP_BLUR_AMOUNT, LIGHT_WRAP_EDGE_BAND, } from '../../common/constants';
-import { createCanvasRenderUtils } from '../../common/canvasRenderUtils';
-import { clamping, createObjectUpdater, nearestPowerOfTwo, } from '../../common/utils';
-import { assert } from '@pexip/utils';
-const clampInRangeFrom0To1 = clamping(0, 1);
-const clampInRangeFrom0To9 = clamping(0, 9);
-export const createTransform = (segmenter, { width = PROCESSING_WIDTH, height = PROCESSING_HEIGHT, foregroundThreshold = FOREGROUND_THRESHOLD, backgroundThreshold = BACKGROUND_THRESHOLD, backgroundBlurAmount = BACKGROUND_BLUR_AMOUNT, edgeBlurAmount = EDGE_BLUR_AMOUNT, maskCombineRatio = MASK_COMBINE_RATIO, backgroundImageUrl = '', sigmaSpace = SIGMA_SPACE, sigmaRangeLo = SIGMA_RANGE[0], sigmaRangeHi = SIGMA_RANGE[1], effects = 'none', backend = 'webgl', selfManageSegmenter, personCenter = PERSON_CENTER, excludeBystanders = EXCLUDE_BYSTANDERS, morphPass = MORPH_PASS, morphErodeRadiusPx = MORPH_ERODE_RADIUS_PX, morphDilateRadiusPx = MORPH_DILATE_RADIUS_PX, lightWrapIntensity = LIGHT_WRAP_INTENSITY, downSampleFactor = DOWN_SAMPLE_FACTOR, lightWrapTightness = LIGHT_WRAP_TIGHTNESS, lightWrapBlurAmount = LIGHT_WRAP_BLUR_AMOUNT, lightWrapEdgeBand = LIGHT_WRAP_EDGE_BAND, ...options } = {}) => {
-    const props = {
-        segmenter,
-        processingWidth: width,
-        processingHeight: height,
-        backgroundImage: options.backgroundImage,
-        status: 'new',
-        outputCanvas: options.outputCanvas,
-        outputDelegate: options.outputDelegate,
-        unsubscribe: null,
-        utils: undefined,
-        renderOptions: {
-            foregroundThreshold,
-            backgroundThreshold,
-            backgroundBlurAmount,
-            edgeBlurAmount,
-            backgroundImageUrl,
-            effects,
-            backend,
-            maskCombineRatio,
-            sigmaRangeHi,
-            sigmaRangeLo,
-            sigmaSpace,
-            personCenter,
-            excludeBystanders,
-            morphPass,
-            morphErodeRadiusPx,
-            morphDilateRadiusPx,
-            lightWrapIntensity,
-            downSampleFactor,
-            lightWrapTightness,
-            lightWrapBlurAmount,
-            lightWrapEdgeBand,
-        },
-    };
-    const updateProps = createObjectUpdater(props);
-    const handleContextRestored = () => {
-        assert(props.utils);
-        props.utils.reset();
-    };
-    const toVideoFrameLike = (input) => {
-        // Standardize the input
-        assert(props.utils);
-        const image = input instanceof ImageData
-            ? props.utils.renderImageDataToOffScreenCanvas(input, 'drawImageDataCanvas')
-            : input;
-        // Pre-precessing the input to resize to the processing size
-        const canvas = props.utils.renderImageToOffScreenCanvas(image, 'inputCanvas');
-        if ('close' in input) {
-            input.close();
-        }
-        return {
-            frame: canvas.transferToImageBitmap(),
-            width: props.processingWidth,
-            height: props.processingHeight,
-            timestamp: performance.now(),
-        };
-    };
-    const processInput = async (input) => {
-        if (props.segmenter.status === 'new' ||
-            props.segmenter.status === 'closed') {
-            await props.segmenter.open({
-                output: props.outputDelegate,
-                processingWidth: props.processingWidth,
-                processingHeight: props.processingHeight,
-                backgroundImage: props.backgroundImage,
-                renderOptions: props.renderOptions,
-            });
-        }
-        if (!props.utils) {
-            props.utils = createCanvasRenderUtils(props.processingWidth, props.processingHeight);
-        }
-        if (props.segmenter.status === 'opening') {
-            return undefined;
-        }
-        const inputFrame = toVideoFrameLike(input);
-        return props.segmenter.process(inputFrame, props.renderOptions);
-    };
-    const canUpdate = () => props.segmenter.status !== 'new' &&
-        props.segmenter.status !== 'closed' &&
-        props.segmenter.status !== 'closing';
-    const updateProcessingDimensions = (width, height) => {
-        let newDimensions = false;
-        if (width && props.processingWidth !== width) {
-            props.processingWidth = width;
-            newDimensions = true;
-        }
-        if (height && props.processingHeight !== height) {
-            props.processingHeight = height;
-            newDimensions = true;
-        }
-        if (newDimensions) {
-            props.utils = createCanvasRenderUtils(props.processingWidth, props.processingHeight);
-        }
-    };
-    return {
-        get width() {
-            return props.processingWidth;
-        },
-        get height() {
-            return props.processingHeight;
-        },
-        get status() {
-            return props.status;
-        },
-        get foregroundThreshold() {
-            return props.renderOptions.foregroundThreshold;
-        },
-        get backgroundThreshold() {
-            return props.renderOptions.backgroundThreshold;
-        },
-        get backgroundBlurAmount() {
-            return props.renderOptions.backgroundBlurAmount;
-        },
-        get edgeBlurAmount() {
-            return props.renderOptions.edgeBlurAmount;
-        },
-        get effects() {
-            return props.renderOptions.effects;
-        },
-        get maskCombineRatio() {
-            return props.renderOptions.maskCombineRatio;
-        },
-        get backend() {
-            return props.renderOptions.backend;
-        },
-        get sigmaSpace() {
-            return props.renderOptions.sigmaSpace;
-        },
-        get sigmaRangeLo() {
-            return props.renderOptions.sigmaRangeLo;
-        },
-        get sigmaRangeHi() {
-            return props.renderOptions.sigmaRangeHi;
-        },
-        get personCenter() {
-            return props.renderOptions.personCenter;
-        },
-        get excludeBystanders() {
-            return props.renderOptions.excludeBystanders;
-        },
-        get morphPass() {
-            return props.renderOptions.morphPass;
-        },
-        get morphErodeRadiusPx() {
-            return props.renderOptions.morphErodeRadiusPx;
-        },
-        get morphDilateRadiusPx() {
-            return props.renderOptions.morphDilateRadiusPx;
-        },
-        get lightWrapIntensity() {
-            return props.renderOptions.lightWrapIntensity;
-        },
-        get lightWrapTightness() {
-            return props.renderOptions.lightWrapTightness;
-        },
-        get lightWrapBlurAmount() {
-            return props.renderOptions.lightWrapBlurAmount;
-        },
-        get lightWrapEdgeBand() {
-            return props.renderOptions.lightWrapEdgeBand;
-        },
-        get downSampleFactor() {
-            return props.renderOptions.downSampleFactor;
-        },
-        set morphPass(value) {
-            props.renderOptions.morphPass = value;
-        },
-        set morphErodeRadiusPx(value) {
-            props.renderOptions.morphErodeRadiusPx = value;
-        },
-        set morphDilateRadiusPx(value) {
-            props.renderOptions.morphDilateRadiusPx = value;
-        },
-        set personCenter(value) {
-            props.renderOptions.personCenter = value;
-        },
-        set excludeBystanders(value) {
-            props.renderOptions.excludeBystanders = value;
-        },
-        set foregroundThreshold(value) {
-            assert(value >= props.renderOptions.backgroundThreshold);
-            props.renderOptions.foregroundThreshold =
-                clampInRangeFrom0To1(value);
-        },
-        set backgroundThreshold(value) {
-            assert(value <= props.renderOptions.foregroundThreshold);
-            props.renderOptions.backgroundThreshold =
-                clampInRangeFrom0To1(value);
-        },
-        set backgroundBlurAmount(value) {
-            props.renderOptions.backgroundBlurAmount =
-                clampInRangeFrom0To9(value);
-        },
-        set edgeBlurAmount(value) {
-            props.renderOptions.edgeBlurAmount = clampInRangeFrom0To9(value);
-        },
-        set effects(value) {
-            props.renderOptions.effects = value;
-        },
-        set backgroundImageUrl(value) {
-            props.renderOptions.backgroundImageUrl = value;
-        },
-        get backgroundImageUrl() {
-            return props.renderOptions.backgroundImageUrl;
-        },
-        set maskCombineRatio(value) {
-            props.renderOptions.maskCombineRatio = clampInRangeFrom0To1(value);
-        },
-        set sigmaRangeLo(value) {
-            props.renderOptions.sigmaRangeLo = value;
-        },
-        set sigmaRangeHi(value) {
-            props.renderOptions.sigmaRangeHi = value;
-        },
-        set sigmaSpace(value) {
-            props.renderOptions.sigmaSpace = value;
-        },
-        set lightWrapIntensity(value) {
-            props.renderOptions.lightWrapIntensity = value;
-        },
-        set lightWrapTightness(value) {
-            props.renderOptions.lightWrapTightness = value;
-        },
-        set lightWrapBlurAmount(value) {
-            props.renderOptions.lightWrapBlurAmount = value;
-        },
-        set lightWrapEdgeBand(value) {
-            props.renderOptions.lightWrapEdgeBand = value;
-        },
-        set downSampleFactor(value) {
-            props.renderOptions.downSampleFactor = nearestPowerOfTwo(value, 0, 2);
-        },
-        get outputCanvas() {
-            return props.outputCanvas;
-        },
-        get backgroundImage() {
-            return props.backgroundImage;
-        },
-        set backgroundImage(image) {
-            if (image &&
-                image.key !== props.backgroundImage?.key &&
-                image.image !== props.backgroundImage?.image) {
-                props.backgroundImage = image;
-                if (canUpdate()) {
-                    void props.segmenter.update({ backgroundImage: image });
-                }
-            }
-        },
-        set outputCanvas(canvas) {
-            props.outputCanvas = canvas;
-        },
-        get outputDelegate() {
-            return props.outputDelegate;
-        },
-        set outputDelegate(canvas) {
-            props.outputDelegate = canvas;
-        },
-        get segmenter() {
-            return props.segmenter;
-        },
-        set segmenter(value) {
-            if (value !== props.segmenter) {
-                props.segmenter = value;
-            }
-        },
-        set backend(value) {
-            if (value === props.renderOptions.backend) {
-                return;
-            }
-            props.renderOptions.backend = value;
-            if (canUpdate()) {
-                segmenter.update({
-                    renderOptions: props.renderOptions,
-                });
-            }
-        },
-        init: (options = {}) => {
-            updateProps(options);
-            props.status = 'opened';
-            props.unsubscribe = segmenter.subscribe('ContextRestored', handleContextRestored);
-            return Promise.resolve();
-        },
-        update: async (options) => {
-            updateProcessingDimensions(options.processingWidth, options.processingHeight);
-            // update props needs to happen after dimension comparison
-            updateProps(options);
-            if (canUpdate()) {
-                await segmenter.update(options);
-            }
-        },
-        transform: async (videoFrame, controller) => {
-            switch (props.renderOptions.effects) {
-                case 'overlay':
-                case 'blur': {
-                    const result = await processInput(videoFrame);
-                    const processedFrame = props.outputCanvas ?? result?.frame;
-                    if (props.status === 'closed' ||
-                        // Avoid processing disposed frame
-                        !processedFrame ||
-                        ('displayWidth' in processedFrame &&
-                            processedFrame.displayWidth === 0) ||
-                        ('width' in processedFrame &&
-                            processedFrame.width === 0)) {
-                        break;
-                    }
-                    if ('close' in videoFrame) {
-                        videoFrame.close();
-                    }
-                    controller.enqueue(props.outputCanvas ?? result?.frame);
-                    result?.frame.close();
-                    break;
-                }
-                case 'none': {
-                    controller.enqueue(videoFrame);
-                    break;
-                }
-            }
-            props.status = 'processing';
-        },
-        close: () => {
-            if (!selfManageSegmenter) {
-                segmenter.close();
-            }
-            props.unsubscribe?.();
-            props.status = 'closed';
-        },
-        destroy: async () => {
-            if (!selfManageSegmenter) {
-                await segmenter.destroy();
-            }
-            props.status = 'destroyed';
-        },
-    };
-};

package/dist/main/video/segmenters/mediapipe.d.ts

@@ -1,22 +0,0 @@
-import type { Segmenter } from '../types';
-import type { Delegate, SegmenterOptions, ImageSegmenterOptions, SegmentationModelAsset } from '../../../common/types/segmentation';
-interface Options extends Omit<ImageSegmenterOptions, 'delegate'> {
-    /**
-     * The URL for the worker script.
-     */
-    workerScriptUrl?: URL;
-    /**
-     * Request Credentials for the worker.
-     *
-     * @defaultValue 'same-origin'
-     */
-    credentials?: RequestCredentials;
-    /**
-     * @see `SegmentationModelAsset`
-     */
-    modelAsset: SegmentationModelAsset;
-    delegate?: () => Delegate;
-    log?: (message: string, meta?: unknown) => void;
-}
-export declare const createSegmenter: (basePath?: SegmenterOptions["basePath"], { workerScriptUrl, credentials, delegate, log, ...options }?: Options) => Segmenter;
-export {};

package/dist/main/video/transformer.d.ts

@@ -1,9 +0,0 @@
-import type { InputFrame } from './types';
-export declare const nullTransformController: (controller: Pick<TransformStreamDefaultController<CanvasImageSource>, "enqueue" | "terminate">) => TransformStreamDefaultController<CanvasImageSource>;
-export declare const adaptInputFrameTransformer: (transformer: Transformer<InputFrame, InputFrame>) => {
-    transform: (frame: VideoFrame, controller: TransformStreamDefaultController<VideoFrame>) => void | PromiseLike<void> | undefined;
-    flush?: TransformerFlushCallback<InputFrame> | undefined;
-    readableType?: undefined;
-    start?: TransformerStartCallback<InputFrame> | undefined;
-    writableType?: undefined;
-};

package/dist/main/video/transformer.js

@@ -1,68 +0,0 @@
-const wrapTransformController = (videoFrame, controller) => {
-    return {
-        get desiredSize() {
-            return controller.desiredSize;
-        },
-        enqueue: frame => {
-            if (!frame) {
-                return;
-            }
-            if (frame instanceof VideoFrame) {
-                return controller.enqueue(frame);
-            }
-            if (('OffscreenCanvas' in window &&
-                frame instanceof OffscreenCanvas) ||
-                frame instanceof HTMLCanvasElement ||
-                frame instanceof HTMLVideoElement ||
-                frame instanceof HTMLImageElement ||
-                frame instanceof ImageBitmap) {
-                const timestamp = videoFrame.timestamp ?? 0;
-                try {
-                    // Constructing a new VideoFrame with the modified frame
-                    // could throw an error
-                    const newFrame = new VideoFrame(frame, {
-                        timestamp,
-                        alpha: 'discard',
-                    });
-                    controller.enqueue(newFrame);
-                    videoFrame.close();
-                }
-                catch {
-                    controller.enqueue(videoFrame);
-                }
-                return;
-            }
-            throw new Error('Unexpected input frame');
-        },
-        error: reason => controller.error(reason),
-        terminate: () => controller.terminate(),
-    };
-};
-export const nullTransformController = (controller) => {
-    return {
-        get desiredSize() {
-            return 0;
-        },
-        enqueue: frame => {
-            if (('OffscreenCanvas' in window &&
-                frame instanceof OffscreenCanvas) ||
-                frame instanceof HTMLCanvasElement ||
-                frame instanceof HTMLVideoElement ||
-                frame instanceof HTMLImageElement ||
-                frame instanceof ImageBitmap) {
-                return controller.enqueue(frame);
-            }
-            throw new Error('Unexpected input frame');
-        },
-        error: reason => {
-            throw new Error(reason);
-        },
-        terminate: () => controller.terminate(),
-    };
-};
-export const adaptInputFrameTransformer = (transformer) => {
-    const transform = (frame, controller) => {
-        return transformer.transform?.(frame, wrapTransformController(frame, controller));
-    };
-    return { ...transformer, transform };
-};

package/dist/main/video/videoStreamTrackProcessor.d.ts

@@ -1,20 +0,0 @@
-import type { InputFrame } from './types';
-type Track = MediaStreamVideoTrack | MediaStreamVideoTrackGenerator;
-interface Options {
-    signal?: AbortSignal;
-}
-export type ProcessVideoTrack = (track: MediaStreamVideoTrack, transformers: Transformer<InputFrame, InputFrame>[], options?: Options) => Promise<Track>;
-/**
- * Video track processor using MediaStreamTrackProcessor API
- */
-export declare const createVideoTrackProcessor: () => ProcessVideoTrack;
-interface FallbackOptions {
-    width?: number;
-    height?: number;
-    frameRate?: number;
-}
-/**
- * Video track processor using Canvas[captureStream] API
- */
-export declare const createVideoTrackProcessorWithFallback: ({ width, height, frameRate, }?: FallbackOptions) => ProcessVideoTrack;
-export {};

package/dist/main/video/videoStreamTrackProcessor.js

@@ -1,86 +0,0 @@
-import { assert } from '@pexip/utils';
-import { createMediaStreamTrackProcessor } from '../processor';
-import { createMediaStreamTrackGenerator } from '../generator';
-import { createStreamTransformer } from '../transformer';
-import { stopStreamTracks } from '../utils';
-import { PROCESSING_HEIGHT, PROCESSING_WIDTH } from '../../common/constants';
-import { getImageSize } from '../../common/canvasRenderUtils';
-import { adaptInputFrameTransformer, nullTransformController, } from './transformer';
-import { toVideoElement, playVideo, createFrameCallbackRequest, createCanvas, getCanvasRenderingContext2D, } from './utils';
-import { AbortReason, FRAME_RATE } from './constants';
-/**
- * Video track processor using MediaStreamTrackProcessor API
- */
-export const createVideoTrackProcessor = () => (track, transformers, { signal } = {}) => {
-    if (!transformers.length) {
-        return Promise.resolve(track);
-    }
-    const processor = createMediaStreamTrackProcessor({ track });
-    const trackGenerator = createMediaStreamTrackGenerator({ kind: 'video' });
-    let readable = processor.readable;
-    for (const transformer of transformers) {
-        readable = readable.pipeThrough(createStreamTransformer(adaptInputFrameTransformer(transformer)), { signal });
-    }
-    // Promise returned from `ReadableStream['pipeTo']` is only resolved
-    // when the streaming is finished
-    readable
-        .pipeTo(trackGenerator.writable, {
-        signal,
-    })
-        .catch(error => {
-        // Ignore abort error
-        if (signal &&
-            !(signal.aborted &&
-                (signal.reason === AbortReason.Close ||
-                    // AbortSignal['reason'] is only supported from Chromium v98 or Firefox v97
-                    !('reason' in AbortSignal.prototype)))) {
-            throw error;
-        }
-    });
-    return Promise.resolve(trackGenerator);
-};
-/**
- * Video track processor using Canvas[captureStream] API
- */
-export const createVideoTrackProcessorWithFallback = ({ width = PROCESSING_WIDTH, height = PROCESSING_HEIGHT, frameRate = FRAME_RATE, } = {}) => async (track, transformers, { signal } = {}) => {
-    const [transformer] = transformers;
-    if (!transformer?.transform) {
-        return track;
-    }
-    assert(transformers.length <= 1, 'Multi-transformer is NOT supported');
-    const outputCanvas = createCanvas(width, height);
-    const ctx = getCanvasRenderingContext2D(outputCanvas, { alpha: false });
-    const render = (input) => {
-        assert(transformer.transform);
-        return Promise.resolve(transformer.transform(input, nullTransformController({
-            enqueue: frame => {
-                if (!frame) {
-                    return;
-                }
-                const frameSize = getImageSize(frame);
-                if (frameSize.height !== outputCanvas.height ||
-                    frameSize.width !== outputCanvas.width) {
-                    outputCanvas.height = frameSize.height;
-                    outputCanvas.width = frameSize.width;
-                }
-                ctx.drawImage(frame, 0, 0, outputCanvas.width, outputCanvas.height);
-            },
-            terminate: () => {
-                stop();
-            },
-        })));
-    };
-    const runner = createFrameCallbackRequest(render, frameRate);
-    const videoElement = toVideoElement(new MediaStream([track]), width, height);
-    await playVideo(videoElement);
-    await runner.start(videoElement);
-    const stream = outputCanvas.captureStream(frameRate);
-    const [trackGenerated] = stream.getVideoTracks();
-    assert(trackGenerated, 'Canvas captureStream returns no video track');
-    const stop = () => {
-        stopStreamTracks(stream);
-        runner.stop();
-    };
-    signal?.addEventListener('abort', stop);
-    return trackGenerated;
-};