@pexip/media-processor 17.4.0 → 18.0.0

package/dist/main/video/types.d.ts

@@ -1,35 +1,16 @@
-import type { Tensor3D } from '@tensorflow/tfjs-core';
-import type { Canvas, Transform } from '../types';
-export declare const RENDER_EFFECTS: readonly ["none", "blur", "overlay"];
-export declare const SEG_MODELS: readonly ["mediapipeSelfie"];
-/**
- * Interfaces from "tensorflow-models/body-segmentation" interfaces
- */
-export type MaskUnderlyingType = 'canvasimagesource' | 'imagedata' | 'tensor';
-export interface Mask {
-    toCanvasImageSource(): Promise<CanvasImageSource>;
-    toImageData(): Promise<ImageData>;
-    toTensor(): Promise<Tensor3D>;
-    getUnderlyingType(): MaskUnderlyingType;
-}
-/**
- * Interfaces from "tensorflow-models/body-segmentation" interfaces
- */
-export interface Segmentation {
-    maskValueToLabel: (maskValue: number) => string;
-    mask: Mask;
-}
+import type { ProcessStatus } from '../../common/constants';
+import type { ProcessInputType, VideoFrameLike } from '../../common/types/media';
+import type { RendererOptions } from '../../common/types/render';
+import type { SegmentationModelAsset } from '../../common/types/segmentation';
+import type { ProcessorOptions } from '../../common/types/processor';
+import type { Transform } from '../types';
 export type Color = {
     r: number;
     g: number;
     b: number;
     a: number;
 };
-export type SegmentationModel = (typeof SEG_MODELS)[number];
-export type RenderEffects = (typeof RENDER_EFFECTS)[number];
-export type ProcessInputType = ImageData | HTMLVideoElement | HTMLImageElement | OffscreenCanvas | HTMLCanvasElement | ImageBitmap;
-export type InputFrame = CanvasImageSource | VideoFrame;
-export type ImageType = CanvasImageSource | ProcessInputType | VideoFrame;
+export type InputFrame = HTMLCanvasElement | OffscreenCanvas | VideoFrame | ImageBitmap;
 interface ProcessingSize {
     /**
      * Processing Width size
@@ -40,58 +21,35 @@ interface ProcessingSize {
      */
     height: number;
 }
-export interface RenderParams extends ProcessingSize {
-    /**
-     * Range in between 0 and 1
-     * @defaultValue `0.5`
-     */
-    foregroundThreshold: number;
-    /**
-     * Range in between 0 and 20
-     * @defaultValue `3`
-     */
-    backgroundBlurAmount: number;
-    /**
-     * Range in between 0 and 20
-     * @defaultValue `3`
-     */
-    edgeBlurAmount: number;
-    /**
-     * @defaultValue `false`
-     */
-    flipHorizontal: boolean;
-    /**
-     * @defaultValue `none`
-     */
-    effects: RenderEffects;
-    /**
-     * Background image used for background replacement
-     */
-    backgroundImage: Canvas | undefined;
-}
 export interface AsyncAssets {
     tfjsCoreLoaded: boolean;
     tfjsBackendLoaded: boolean;
     glueLoaded: boolean;
 }
-export type ProcessStatus = 'created' | 'opened' | 'opening' | 'processing' | 'idle' | 'closed' | 'destroying' | 'destroyed';
 export interface Process {
     status: ProcessStatus;
     open(): Promise<void>;
     close(): void;
     destroy(): Promise<void>;
 }
+export interface BackgroundImage {
+    key: string;
+    image: Blob;
+}
 export interface Segmenter extends Process, ProcessingSize {
-    model: SegmentationModel;
-    process(input: ProcessInputType): Promise<Segmentation[]>;
+    modelAsset: SegmentationModelAsset;
+    backgroundImage?: BackgroundImage;
+    open(options?: ProcessorOptions): Promise<void>;
+    update(options: ProcessorOptions): Promise<void>;
+    process(input: VideoFrameLike, options?: RendererOptions): Promise<VideoFrameLike | undefined>;
 }
 export interface Detector<T> extends Process {
     detect(input: ProcessInputType): Promise<T>;
 }
-export interface SegmentationParams extends RenderParams {
+export interface SegmentationParams extends RendererOptions {
     segmenter: Segmenter;
-    loadBackgroundImage(url: string): Promise<void>;
-    backgroundImageUrl?: string;
+    width: number;
+    height: number;
 }
 export type SegmentationTransform = Transform<InputFrame, InputFrame> & SegmentationParams & Omit<Process, 'open'>;
 export {};

package/dist/main/video/types.js

@@ -1,20 +1 @@
-export const RENDER_EFFECTS = [
-    /**
-     * Pass through, and thus there is no effects applied
-     */
-    'none',
-    /**
-     * Background blur effects
-     */
-    'blur',
-    /**
-     * Background overlay effects
-     */
-    'overlay',
-];
-export const SEG_MODELS = [
-    /**
-     * Mediapipe's Selfie Segmentation model
-     */
-    'mediapipeSelfie',
-];
+export {};

package/dist/main/video/utils.d.ts

@@ -1,11 +1,11 @@
-import type { Tensor3D } from '@tensorflow/tfjs-core';
 /**
  * We support several ways to get image inputs.
  */
 export type InputImage = HTMLVideoElement | HTMLImageElement | HTMLCanvasElement;
-import type { Size, Rect, Frame, Canvas, CanvasContext, Callback } from '../types';
+import type { Frame, Canvas, CanvasContext, Callback } from '../types';
 import { subscribeVisibilityChangeEvent } from '../utils';
-import type { MaskUnderlyingType, Segmentation, Color, ImageType, ProcessInputType } from './types';
+import type { ImageType, ProcessInputType, Rect } from '../../common/types/media';
+import type { Color } from './types';
 /**
  * Draw the source onto the provided canvas according to the destination rect
  *
@@ -62,18 +62,11 @@ export declare const playVideo: (video: HTMLVideoElement) => Promise<void>;
 export declare const loadImage: (image: HTMLImageElement, src: string) => Promise<void>;
 export declare const toNumber: (value: number | SVGAnimatedLength) => number;
 export declare const getCanvasRenderingContext2D: (canvas: Canvas, options?: CanvasRenderingContext2DSettings) => CanvasRenderingContext2D;
-export declare const getImageSize: (image: VideoFrame | CanvasImageSource | OffscreenCanvas | ImageData | SVGImageElement | Tensor3D) => Size;
-export declare const toHTMLCanvasElementLossy: (image: ImageData | SVGImageElement | VideoFrame | OffscreenCanvas | Tensor3D) => Promise<HTMLCanvasElement>;
-export declare const toImageDataLossy: (image: CanvasImageSource | Tensor3D) => Promise<ImageData>;
-export declare const toTensorLossy: (image: CanvasImageSource | ImageData) => Promise<Tensor3D>;
-/**
- * Construct a Segmentation object with tensorflow-models/shared utils
- */
-export declare const toSegmentation: (data: CanvasImageSource | ImageData | Tensor3D, type: MaskUnderlyingType, maskValueToLabel: (value: number) => string) => Segmentation;
+export declare const hasVideoFrameSupport: () => boolean;
+export declare const isVideoFrame: (t: unknown) => t is VideoFrame;
 export declare const flipCanvasHorizontal: (canvas: Canvas) => void;
 export declare const drawStroke: (bytes: Uint8ClampedArray, row: number, column: number, width: number, radius: number, color?: Color) => void;
 export declare const isSegmentationBoundary: (data: Uint8ClampedArray, row: number, column: number, width: number, isForegroundId: boolean[], alphaCutoff: number, radius?: number) => boolean;
-export declare const toBinaryMask: (segmentation: Segmentation | Segmentation[], foreground?: Color, background?: Color, drawContour?: boolean, foregroundThreshold?: number, foregroundMaskValues?: number[]) => Promise<ImageData | null>;
 type Draw = (input: ImageType) => Promise<Canvas>;
 export declare const createInputImageConvertor: (draw: Draw) => (input: ProcessInputType) => Promise<InputImage>;
 export declare const ensure: <T>(prop: T, message?: string) => NonNullable<T>;

package/dist/main/video/utils.js

@@ -1,4 +1,3 @@
-import { Tensor, browser } from '@tensorflow/tfjs-core/dist/base.js';
 import { createAsyncCallbackLoop, subscribeVisibilityChangeEvent, } from '../utils';
 /**
  * Draw the source onto the provided canvas according to the destination rect
@@ -165,99 +164,12 @@ export const getCanvasRenderingContext2D = (canvas, options) => {
     }
     return context;
 };
-export const getImageSize = (image) => {
-    if (image instanceof Tensor) {
-        const [height = 0, width = 0] = image.shape.slice(0, 2);
-        return { height, width };
+export const hasVideoFrameSupport = () => 'VideoFrame' in window;
+export const isVideoFrame = (t) => {
+    if (hasVideoFrameSupport() && t instanceof VideoFrame) {
+        return true;
     }
-    if ('VideoFrame' in window && image instanceof VideoFrame) {
-        return { height: image.displayHeight, width: image.displayWidth };
-    }
-    if ('offsetHeight' in image &&
-        image.offsetHeight !== 0 &&
-        'offsetWidth' in image &&
-        image.offsetWidth !== 0) {
-        return { height: image.offsetHeight, width: image.offsetWidth };
-    }
-    if ('height' in image &&
-        image.height !== 0 &&
-        'width' in image &&
-        image.width !== 0) {
-        return { height: toNumber(image.height), width: toNumber(image.width) };
-    }
-    throw new Error('Unknown input image');
-};
-export const toHTMLCanvasElementLossy = async (image) => {
-    const { width, height } = getImageSize(image);
-    const canvas = createCanvas(width, height);
-    if (image instanceof Tensor) {
-        await browser.toPixels(image, canvas);
-        return canvas;
-    }
-    const context = getCanvasRenderingContext2D(canvas);
-    if (image instanceof ImageData) {
-        context.putImageData(image, 0, 0);
-    }
-    else {
-        context.drawImage(image, 0, 0);
-    }
-    return canvas;
-};
-export const toImageDataLossy = async (image) => {
-    const { width, height } = getImageSize(image);
-    if (image instanceof Tensor) {
-        return new ImageData(await browser.toPixels(image), width, height);
-    }
-    const canvas = createOffscreenCanvas(width, height);
-    const context = getCanvasRenderingContext2D(canvas);
-    context.drawImage(image, 0, 0);
-    return context.getImageData(0, 0, canvas.width, canvas.height);
-};
-export const toTensorLossy = async (image) => {
-    const pixelsInput = image instanceof SVGImageElement ||
-        image instanceof OffscreenCanvas ||
-        image instanceof VideoFrame
-        ? await toHTMLCanvasElementLossy(image)
-        : image;
-    return browser.fromPixels(pixelsInput, 4);
-};
-/**
- * Construct a Segmentation object with tensorflow-models/shared utils
- */
-export const toSegmentation = (data, type, maskValueToLabel) => {
-    const mask = {
-        toCanvasImageSource: async () => {
-            if (data instanceof HTMLCanvasElement) {
-                return data;
-            }
-            if (data instanceof HTMLVideoElement ||
-                data instanceof ImageBitmap ||
-                data instanceof HTMLImageElement) {
-                const canvas = createCanvas(data.width, data.height);
-                const context = canvas.getContext('2d');
-                context?.drawImage(data, 0, 0, data.width, data.height);
-                return canvas;
-            }
-            return await toHTMLCanvasElementLossy(data);
-        },
-        toImageData: async () => {
-            if (data instanceof ImageData) {
-                return data;
-            }
-            return await toImageDataLossy(data);
-        },
-        toTensor: async () => {
-            if (data instanceof Tensor) {
-                return data;
-            }
-            return await toTensorLossy(data);
-        },
-        getUnderlyingType: () => type,
-    };
-    return {
-        maskValueToLabel,
-        mask,
-    };
+    return false;
 };
 export const flipCanvasHorizontal = (canvas) => {
     const ctx = getCanvasRenderingContext2D(canvas);
@@ -300,65 +212,6 @@ export const isSegmentationBoundary = (data, row, column, width, isForegroundId,
     }
     return numberBackgroundPixels > 0;
 };
-export const toBinaryMask = async (segmentation, foreground = {
-    r: 0,
-    g: 0,
-    b: 0,
-    a: 0,
-}, background = {
-    r: 0,
-    g: 0,
-    b: 0,
-    a: 255,
-}, drawContour = false, foregroundThreshold = 0.5, foregroundMaskValues = Array.from({ length: 256 }, (_, i) => i)) => {
-    const segmentations = !Array.isArray(segmentation)
-        ? [segmentation]
-        : segmentation;
-    if (segmentations.length === 0) {
-        return null;
-    }
-    const masks = await Promise.all(segmentations.map(segmentation => segmentation.mask.toImageData()));
-    const [imageData] = masks;
-    if (!imageData) {
-        return null;
-    }
-    const { width, height } = imageData;
-    const bytes = new Uint8ClampedArray(width * height * 4);
-    const alphaCutoff = Math.round(255 * foregroundThreshold);
-    const isForegroundId = new Array(256).fill(false);
-    foregroundMaskValues.forEach(id => (isForegroundId[id] = true));
-    for (let i = 0; i < height; i++) {
-        for (let j = 0; j < width; j++) {
-            const n = i * width + j;
-            bytes[4 * n + 0] = background.r;
-            bytes[4 * n + 1] = background.g;
-            bytes[4 * n + 2] = background.b;
-            bytes[4 * n + 3] = background.a;
-            for (const mask of masks) {
-                const maskForegroundColor = mask.data[4 * n];
-                const maskAlphaColor = mask.data[4 * n + 3];
-                if (maskForegroundColor !== undefined &&
-                    isForegroundId[maskForegroundColor] &&
-                    maskAlphaColor !== undefined &&
-                    maskAlphaColor >= alphaCutoff) {
-                    bytes[4 * n] = foreground.r;
-                    bytes[4 * n + 1] = foreground.g;
-                    bytes[4 * n + 2] = foreground.b;
-                    bytes[4 * n + 3] = foreground.a;
-                    if (drawContour &&
-                        i - 1 >= 0 &&
-                        i + 1 < height &&
-                        j - 1 >= 0 &&
-                        j + 1 < width &&
-                        isSegmentationBoundary(mask.data, i, j, width, isForegroundId, alphaCutoff)) {
-                        drawStroke(bytes, i, j, width, 1);
-                    }
-                }
-            }
-        }
-    }
-    return new ImageData(bytes, width, height);
-};
 export const createInputImageConvertor = (draw) => async (input) => {
     return (input instanceof ImageData || input instanceof ImageBitmap
         ? await draw(input)

package/dist/main/video/video.js

@@ -34,6 +34,7 @@ export const createVideoProcessor = (transformers, processTrack) => {
         stopStreamTracks(props.outputStream);
         props.outputStream = undefined;
         props.processing = false;
+        transformers.forEach(transformer => transformer.close());
     };
     const destroy = async () => {
         close();

package/dist/main/video/videoStreamTrackProcessor.js

@@ -2,9 +2,11 @@ import { createMediaStreamTrackProcessor } from '../processor';
 import { createMediaStreamTrackGenerator } from '../generator';
 import { createStreamTransformer } from '../transformer';
 import { stopStreamTracks } from '../utils';
+import { PROCESSING_HEIGHT, PROCESSING_WIDTH } from '../../common/constants';
+import { getImageSize } from '../../common/canvasRenderUtils';
 import { adaptInputFrameTransformer, nullTransformController, } from './transformer';
-import { toVideoElement, playVideo, createFrameCallbackRequest, createCanvas, getImageSize, getCanvasRenderingContext2D, } from './utils';
-import { AbortReason, PROCESSING_WIDTH, PROCESSING_HEIGHT, FRAME_RATE, } from './constants';
+import { toVideoElement, playVideo, createFrameCallbackRequest, createCanvas, getCanvasRenderingContext2D, } from './utils';
+import { AbortReason, FRAME_RATE } from './constants';
 export const createVideoTrackProcessor = () => (track, transformers, { signal } = {}) => {
     if (!transformers.length) {
         return Promise.resolve(track);
@@ -53,11 +55,12 @@ export const createVideoTrackProcessorWithFallback = ({ width = PROCESSING_WIDTH
                     return;
                 }
                 const frameSize = getImageSize(frame);
-                if (frameSize.height !== outputCanvas.height) {
+                if (frameSize.height !== outputCanvas.height ||
+                    frameSize.width !== outputCanvas.width) {
                     outputCanvas.height = frameSize.height;
                     outputCanvas.width = frameSize.width;
                 }
-                ctx.drawImage(frame, 0, 0, frameSize.width, frameSize.height);
+                ctx.drawImage(frame, 0, 0, outputCanvas.width, outputCanvas.height);
             },
             terminate: () => {
                 stop();

package/dist/workers/mediaWorker.d.ts

@@ -0,0 +1 @@
+export {};