@pexip/media-processor 21.0.0 → 22.0.0

package/dist/main/video/{segmenters/mediapipe.js → segmenter.js}

@@ -1,9 +1,6 @@
 import { assert } from '@pexip/utils';
-import { PROCESS_STATUS, RENDERING_EVENTS } from '../../../common/constants';
-const cloneImageRecord = async (image) => {
-    const cloned = await createImageBitmap(image.image);
-    return { image: cloned, key: image.key };
-};
+import { PROCESS_STATUS, RENDERING_EVENTS } from '../../common/constants';
+import { cloneImageRecord } from '../../common/utils';
 export const createSegmenter = (
 /**
  * A base path to specify the directory the Wasm files should be loaded
@@ -12,7 +9,7 @@ export const createSegmenter = (
  * @see {@link
  * https://ai.google.dev/edge/api/mediapipe/js/tasks-vision.filesetresolver#filesetresolverforvisiontasks | FileResolver}
  */
-basePath = '/', { workerScriptUrl = new URL('../../../workers/mediaWorker.js', import.meta.url), credentials = 'same-origin', delegate, log, ...options } = { modelAsset: { modelName: 'selfie', path: '' } }) => {
+basePath = '/', { workerScriptUrl = new URL('../../workers/mediaWorker.js', import.meta.url), credentials = 'same-origin', delegate, log, stats, ...options } = { modelAsset: { modelName: 'selfie', path: '' } }) => {
     const props = {
         status: PROCESS_STATUS.New,
         modelAsset: options.modelAsset,
@@ -28,24 +25,35 @@ basePath = '/', { workerScriptUrl = new URL('../../../workers/mediaWorker.js', i
             error.preventDefault();
         });
         mediaWorker.addEventListener('message', (event) => {
-            const { type } = event.data;
-            if (type === 'debug') {
-                log?.(event.data.data.message, event.data.data.context);
-            }
-            if (type === 'event') {
-                log?.(event.data.data.type, event.data.data.message);
-                for (const cb of props.eventHandlers.get(event.data.data.type) ?? []) {
-                    cb();
+            switch (event.data.type) {
+                case 'debug': {
+                    log?.(event.data.data.message, event.data.data.context);
+                    break;
                 }
-                switch (event.data.data.type) {
-                    case RENDERING_EVENTS.ProcessorRestarted:
-                        break;
-                    case RENDERING_EVENTS.ContextRestored:
-                        void handleContextRestored();
-                        break;
-                    default:
-                        props.status = PROCESS_STATUS.Error;
-                        break;
+                case 'event': {
+                    log?.(event.data.data.type, event.data.data.message);
+                    for (const cb of props.eventHandlers.get(event.data.data.type) ?? []) {
+                        cb();
+                    }
+                    switch (event.data.data.type) {
+                        case RENDERING_EVENTS.ProcessorRestarted:
+                            break;
+                        case RENDERING_EVENTS.ContextRestored:
+                            void handleContextRestored();
+                            break;
+                        default:
+                            props.status = PROCESS_STATUS.Error;
+                            break;
+                    }
+                    break;
+                }
+                case 'stats': {
+                    stats?.(event.data.data);
+                    break;
+                }
+                default: {
+                    // Do nothing
+                    break;
                 }
             }
         });
@@ -72,7 +80,8 @@ basePath = '/', { workerScriptUrl = new URL('../../../workers/mediaWorker.js', i
     };
     async function postMsg(expectedResponseType, message, transfer) {
         if (!expectedResponseType) {
-            return props.mediaWorker?.postMessage(message, transfer ?? []);
+            props.mediaWorker?.postMessage(message, transfer ?? []);
+            return;
         }
         return await new Promise((resolve, reject) => {
             const handleMessage = (ev) => {
@@ -83,8 +92,10 @@ basePath = '/', { workerScriptUrl = new URL('../../../workers/mediaWorker.js', i
                         resolve(ev.data.data);
                         break;
                     }
-                    case 'debug': {
-                        // Nothing to handle
+                    case 'debug':
+                    case 'stats': {
+                        // Nothing to handle -- these are continues
+                        // messages handled by the global event listener
                         break;
                     }
                     case 'event': {
@@ -113,17 +124,16 @@ basePath = '/', { workerScriptUrl = new URL('../../../workers/mediaWorker.js', i
             props.mediaWorker?.postMessage(message, transfer ?? []);
         });
     }
-    const open = async ({ processingWidth, processingHeight, output, imageSegmenterOptions, ...restOptions }) => {
+    const open = async ({ imageSegmenterOptions, ...restOptions }) => {
         props.status = PROCESS_STATUS.Opening;
-        props.mediaWorker = initWorker();
-        props.output = output;
+        if (!props.mediaWorker) {
+            props.mediaWorker = initWorker();
+        }
         const backgroundImage = restOptions?.backgroundImage &&
             (await cloneImageRecord(restOptions.backgroundImage));
         try {
             const processorSettings = {
                 basePath,
-                processingWidth,
-                processingHeight,
                 ...restOptions,
                 imageSegmenterOptions: {
                     ...options,
@@ -133,43 +143,66 @@ basePath = '/', { workerScriptUrl = new URL('../../../workers/mediaWorker.js', i
                 backgroundImage,
             };
             log?.('Init processor with config', { config: processorSettings });
-            await postMsg('opened', {
+            const result = await postMsg('opened', {
                 type: 'open',
                 data: processorSettings,
-            }, [output, backgroundImage?.image].filter(Boolean));
+            }, backgroundImage?.image ? [backgroundImage.image] : []);
             props.status = PROCESS_STATUS.Opened;
+            return result;
         }
         catch (error) {
             props.status = PROCESS_STATUS.Error;
             log?.('Failed to open process', {
                 error: error,
             });
+            return {};
         }
     };
-    const process = async (input, options) => {
+    async function process(input, options) {
         if (props.status === PROCESS_STATUS.Error ||
             props.status === PROCESS_STATUS.Updating) {
             return input;
         }
         props.status = PROCESS_STATUS.Processing;
-        try {
-            const resultFrame = await postMsg('processed', {
-                type: 'process',
-                data: { videoFrame: input, renderOptions: options },
-            }, [input.frame]);
-            if (props.status === PROCESS_STATUS.Processing) {
+        if (input instanceof MediaStreamTrack) {
+            try {
+                // Clone the track and keep the original in main
+                const clonedTrack = input.clone();
+                const output = await postMsg('processed', {
+                    type: 'process',
+                    data: { track: clonedTrack, renderOptions: options },
+                }, [clonedTrack]);
+                assert(output);
                 props.status = PROCESS_STATUS.Idle;
+                return output;
+            }
+            catch (error) {
+                props.status = PROCESS_STATUS.Error;
+                log?.('Error during the processing, return original track instead.', { error });
+                return input;
             }
-            return resultFrame;
         }
-        catch (error) {
-            props.status = PROCESS_STATUS.Error;
-            log?.('Error during the processing, return original instead.', {
-                error,
-            });
-            return input;
+        else {
+            try {
+                await postMsg('processed', {
+                    type: 'process',
+                    data: {
+                        readable: input.readable,
+                        writable: input.writable,
+                        renderOptions: options,
+                    },
+                }, [input.readable, input.writable]);
+                props.status = PROCESS_STATUS.Idle;
+            }
+            catch (error) {
+                props.status = PROCESS_STATUS.Error;
+                log?.('Error during the processing, return original ReadWritableStream instead.', {
+                    error,
+                });
+                return input;
+            }
         }
-    };
+    }
     const update = async (options) => {
         try {
             if (props.status === PROCESS_STATUS.Processing ||
@@ -183,7 +216,8 @@ basePath = '/', { workerScriptUrl = new URL('../../../workers/mediaWorker.js', i
                             props.mediaWorker?.removeEventListener('message', handleMessage);
                             resolve(undefined);
                         }
-                        else {
+                        else if (ev.data.type !== 'debug' &&
+                            ev.data.type !== 'stats') {
                             reject(new Error(`UnhandledEvent: ${ev.data.type}`));
                         }
                     };
@@ -196,7 +230,7 @@ basePath = '/', { workerScriptUrl = new URL('../../../workers/mediaWorker.js', i
                 (await cloneImageRecord(options.backgroundImage));
             log?.('Update processor config', { config: options });
             const transfer = backgroundImage ? [backgroundImage.image] : [];
-            await postMsg('updated', {
+            const changed = await postMsg('updated', {
                 type: 'update',
                 data: {
                     ...options,
@@ -204,25 +238,30 @@ basePath = '/', { workerScriptUrl = new URL('../../../workers/mediaWorker.js', i
                 },
             }, transfer);
             props.status = PROCESS_STATUS.Updated;
+            return changed;
         }
         catch (error) {
             props.status = PROCESS_STATUS.Error;
             log?.('Failed to update process', {
                 error: error,
             });
+            return {};
         }
     };
+    const mute = (muted) => {
+        postMsg(undefined, { type: 'mute', data: muted });
+    };
     const close = () => {
         props.status = PROCESS_STATUS.Closing;
         log?.('Close processor');
-        void postMsg(undefined, { type: 'close' });
-        props.eventHandlers.clear();
+        postMsg(undefined, { type: 'close' });
         props.status = PROCESS_STATUS.Closed;
     };
     const destroy = async () => {
         props.status = PROCESS_STATUS.Destroying;
         log?.('Destroy processor');
-        await postMsg(undefined, { type: 'destroy' });
+        props.eventHandlers.clear();
+        postMsg(undefined, { type: 'destroy' });
         props.mediaWorker?.terminate();
         props.mediaWorker = undefined;
         props.status = PROCESS_STATUS.Destroyed;
@@ -251,6 +290,7 @@ basePath = '/', { workerScriptUrl = new URL('../../../workers/mediaWorker.js', i
         open,
         update,
         process,
+        mute,
         close,
         destroy,
     };

package/dist/main/video/typeGuards.d.ts

@@ -1,2 +1,4 @@
 import { type RenderEffects } from '../../common/constants';
 export declare const isRenderEffects: (t: unknown) => t is RenderEffects;
+export declare const isMediaStreamVideoTrack: (t: MediaStreamTrack) => t is MediaStreamVideoTrack;
+export declare function assertMediaStreamVideoTrack(t: MediaStreamTrack): asserts t is MediaStreamVideoTrack;

package/dist/main/video/typeGuards.js

@@ -1,7 +1,14 @@
 import { RENDER_EFFECT } from '../../common/constants';
+const EFFECTS = Object.values(RENDER_EFFECT);
 export const isRenderEffects = (t) => {
     if (typeof t !== 'string') {
         return false;
     }
-    return Object.values(RENDER_EFFECT).some(effect => effect === t);
+    return EFFECTS.some(effect => effect === t);
 };
+export const isMediaStreamVideoTrack = (t) => t.kind === 'video';
+export function assertMediaStreamVideoTrack(t) {
+    if (!isMediaStreamVideoTrack(t)) {
+        throw new TypeError('Not a video track', { cause: t });
+    }
+}

package/dist/main/video/types.d.ts

@@ -1,9 +1,8 @@
 import type { ProcessStatus, RenderingEvents } from '../../common/constants';
-import type { ProcessInputType, VideoFrameLike } from '../../common/types/media';
+import type { ReadWritableStream } from '../../common/types/media';
 import type { RendererOptions } from '../../common/types/render';
 import type { SegmentationModelAsset } from '../../common/types/segmentation';
 import type { ProcessorOptions } from '../../common/types/processor';
-import type { Transform } from '../types';
 export type Color = {
     r: number;
     g: number;
@@ -16,9 +15,9 @@ export interface AsyncAssets {
     tfjsBackendLoaded: boolean;
     glueLoaded: boolean;
 }
-export interface Process {
+export interface Process<T> {
     status: ProcessStatus;
-    open(): Promise<void>;
+    open(): Promise<T>;
     close(): void;
     destroy(): Promise<void>;
 }
@@ -28,21 +27,42 @@ export interface BackgroundImage {
 }
 type Unsubscribe = () => void;
 type Callback<T> = (args: T) => void;
-export interface Segmenter extends Process {
+export interface Segmenter extends Process<Partial<RendererOptions>> {
     modelAsset: SegmentationModelAsset;
     backgroundImage?: BackgroundImage;
-    open(options?: ProcessorOptions): Promise<void>;
-    update(options: ProcessorOptions): Promise<void>;
-    process(input: VideoFrameLike, options?: RendererOptions): Promise<VideoFrameLike | undefined>;
+    /**
+     * Initialize the Segmenter as well as the underlying worker.
+     * It is expected to be called once.
+     */
+    open(options?: ProcessorOptions): Promise<Partial<RendererOptions>>;
+    /**
+     * Update the processor options
+     */
+    update(options: ProcessorOptions): Promise<Partial<RendererOptions>>;
+    /**
+     * Mute/unmute the track in the worker
+     */
+    mute(muted: boolean): void;
+    /**
+     * Process provided input
+     */
+    process(input: MediaStreamTrack, options?: RendererOptions): Promise<MediaStreamTrack>;
+    process(input: ReadWritableStream<VideoFrame>, options?: RendererOptions): Promise<ReadWritableStream<VideoFrame> | undefined>;
+    process(input: MediaStreamTrack | ReadWritableStream<VideoFrame>, options?: RendererOptions): Promise<MediaStreamTrack | ReadWritableStream<VideoFrame> | undefined>;
+    /**
+     * Subscribe rendering events, @see RenderingEvents
+     */
     subscribe(event: RenderingEvents, callback: Callback<void>): Unsubscribe;
 }
-export interface Detector<T> extends Process {
-    detect(input: ProcessInputType): Promise<T>;
+export interface InputProcessAdaptor {
+    /**
+     * Adapt the provided input track based on browser capabilities.
+     */
+    adapt(track: MediaStreamTrack): MediaStreamTrack | ReadWritableStream<VideoFrame>;
+    /**
+     * The generated track from the `adapt` function, and only available
+     * after calling the `adapt` method otherwise `undefined`
+     */
+    outputTrack?: MediaStreamTrack;
 }
-export interface SegmentationParams extends RendererOptions {
-    segmenter: Segmenter;
-    width: number;
-    height: number;
-}
-export type SegmentationTransform = Transform<InputFrame, InputFrame> & SegmentationParams & Pick<Segmenter, 'update'> & Omit<Process, 'open'>;
 export {};

package/dist/main/video/utils.d.ts

@@ -58,13 +58,14 @@ export declare const toVideoElement: (input: MediaStream, width: number, height:
 export declare const playVideo: (video: HTMLVideoElement, { setTimeout, clearTimeout }?: {
     setTimeout?: (((handler: TimerHandler, timeout?: number, ...arguments: any[]) => number) & typeof globalThis.setTimeout) | undefined;
     clearTimeout?: (((id: number | undefined) => void) & typeof globalThis.clearTimeout) | undefined;
-}) => Promise<void>;
+}) => Promise<[void, void]> | undefined;
 /**
  * Load image with provided image element
  */
 export declare const loadImage: (image: HTMLImageElement, src: string) => Promise<void>;
 export declare const toNumber: (value: number | SVGAnimatedLength) => number;
-export declare const getCanvasRenderingContext2D: (canvas: Canvas, options?: CanvasRenderingContext2DSettings) => CanvasRenderingContext2D;
+export declare function getCanvasRenderingContext2D(canvas: HTMLCanvasElement, options?: CanvasRenderingContext2DSettings): CanvasRenderingContext2D;
+export declare function getCanvasRenderingContext2D(canvas: OffscreenCanvas, options?: CanvasRenderingContext2DSettings): OffscreenCanvasRenderingContext2D;
 export declare const hasVideoFrameSupport: () => boolean;
 export declare const isVideoFrame: (t: unknown) => t is VideoFrame;
 export declare const flipCanvasHorizontal: (canvas: Canvas) => void;
@@ -89,8 +90,8 @@ interface FrameCallbackRequestOptions {
  * @param frameRate - A fallback frame rate when we are not able to get the rate
  * from API
  */
-export declare const createFrameCallbackRequest: (callback: Callback<Promise<void>, [ProcessInputType]>, frameRate: number, { subscribeVisibilityChange, }?: FrameCallbackRequestOptions) => {
-    start: (input: ProcessInputType) => Promise<void>;
+export declare const createFrameCallbackRequest: <T>(callback: Callback<Promise<void>, [T]>, frameRate: number, { subscribeVisibilityChange, }?: FrameCallbackRequestOptions) => {
+    start: (input: T) => Promise<void>;
     stop: () => void;
     frameRate: number;
 };

package/dist/main/video/utils.js

@@ -126,52 +126,57 @@ var HTMLMediaElementReadyState;
     HTMLMediaElementReadyState[HTMLMediaElementReadyState["HaveFutureData"] = 3] = "HaveFutureData";
     HTMLMediaElementReadyState[HTMLMediaElementReadyState["HaveEnoughData"] = 4] = "HaveEnoughData";
 })(HTMLMediaElementReadyState || (HTMLMediaElementReadyState = {}));
-export const playVideo = async (video, { setTimeout = window.setTimeout, clearTimeout = window.clearTimeout } = {}) => {
+export const playVideo = (video, { setTimeout = window.setTimeout, clearTimeout = window.clearTimeout } = {}) => {
     if (video.autoplay) {
         return;
     }
-    if (video.readyState <
-        HTMLMediaElementReadyState.HaveCurrentData) {
-        // Not enough data, wait for the data and then play
-        await new Promise((resolve, reject) => {
-            const clearTimeoutTimer = () => {
-                clearTimeout(timeoutId);
-                timeoutId = 0;
-            };
-            const unsubscribe = () => {
-                video.removeEventListener('loadeddata', waitForEvent);
-                video.removeEventListener('error', handleError);
-                video.removeEventListener('stalled', handleStalled);
-                video.removeEventListener('ended', handleEnded);
-                clearTimeoutTimer();
-            };
-            const handleEnded = () => {
-                unsubscribe();
-                reject(new Error('VideoEnded', { cause: video }));
-            };
-            const handleStalled = () => {
-                unsubscribe();
-                reject(new Error('VideoStalled', { cause: video }));
-            };
-            const handleError = (error) => {
-                unsubscribe();
-                reject(error.error);
-            };
-            const waitForEvent = () => {
-                unsubscribe();
+    return Promise.all([
+        video.play(),
+        new Promise((resolve, reject) => {
+            if (video.readyState <
+                HTMLMediaElementReadyState.HaveCurrentData) {
+                // Not enough data, wait for the data and then play
+                const clearTimeoutTimer = () => {
+                    clearTimeout(timeoutId);
+                    timeoutId = 0;
+                };
+                const unsubscribe = () => {
+                    video.removeEventListener('loadeddata', waitForEvent);
+                    video.removeEventListener('error', handleError);
+                    video.removeEventListener('stalled', handleStalled);
+                    video.removeEventListener('ended', handleEnded);
+                    clearTimeoutTimer();
+                };
+                const handleEnded = () => {
+                    unsubscribe();
+                    reject(new Error('VideoEnded', { cause: video }));
+                };
+                const handleStalled = () => {
+                    unsubscribe();
+                    reject(new Error('VideoStalled', { cause: video }));
+                };
+                const handleError = (error) => {
+                    unsubscribe();
+                    reject(error.error);
+                };
+                const waitForEvent = () => {
+                    unsubscribe();
+                    resolve();
+                };
+                // Use setTimeout to prevent a race to `loadeddata` event
+                video.addEventListener('loadeddata', waitForEvent);
+                video.addEventListener('error', handleError);
+                video.addEventListener('stalled', handleStalled);
+                video.addEventListener('ended', handleEnded);
+                let timeoutId = setTimeout(() => {
+                    reject(new Error('PlayVideoTimeout', { cause: video }));
+                }, PLAY_VIDEO_TIMEOUT);
+            }
+            else {
                 resolve();
-            };
-            // Use setTimeout to prevent a race to `loadeddata` event
-            video.addEventListener('loadeddata', waitForEvent);
-            video.addEventListener('error', handleError);
-            video.addEventListener('stalled', handleStalled);
-            video.addEventListener('ended', handleEnded);
-            let timeoutId = setTimeout(() => {
-                reject(new Error('PlayVideoTimeout', { cause: video }));
-            }, PLAY_VIDEO_TIMEOUT);
-        });
-    }
-    return await video.play();
+            }
+        }),
+    ]);
 };
 /**
  * Load image with provided image element
@@ -184,13 +189,13 @@ export const loadImage = async (image, src) => {
     });
 };
 export const toNumber = (value) => value instanceof SVGAnimatedLength ? value.baseVal.value : value;
-export const getCanvasRenderingContext2D = (canvas, options) => {
+export function getCanvasRenderingContext2D(canvas, options) {
     const context = canvas.getContext('2d', options);
     if (!context) {
         throw new Error('Cannot get CanvasRenderingContext2D');
     }
     return context;
-};
+}
 export const hasVideoFrameSupport = () => 'VideoFrame' in window;
 export const isVideoFrame = (t) => {
     if (hasVideoFrameSupport() && t instanceof VideoFrame) {

package/dist/main/video/video.d.ts

@@ -1,13 +1,23 @@
-import type { Transform } from '../types';
-import type { InputFrame } from './types';
-import type { ProcessVideoTrack } from './videoStreamTrackProcessor';
-import type { ProcessorOptions } from '../../common';
+import type { ProcessorOptions, ProcessStatus, RendererOptions } from '../../common';
+import type { Segmenter, InputProcessAdaptor } from './types';
+export interface ProcessorUpdateOptions extends Omit<ProcessorOptions, 'renderOptions'> {
+    renderOptions?: Partial<RendererOptions>;
+}
 export interface VideoProcessor {
-    open(options?: ProcessorOptions): Promise<void>;
+    readonly status: ProcessStatus;
+    /**
+     * Initialize the video processor and related
+     */
+    open(options?: ProcessorUpdateOptions): Promise<void>;
+    /**
+     * Update the video processor with provided options
+     */
+    update(options: ProcessorUpdateOptions): Promise<void>;
+    mute(muted: boolean): void;
     /**
      * Process the video and return a MediaStream
      */
-    process: (source: MediaStream) => Promise<MediaStream>;
+    process: (source: MediaStreamTrack) => Promise<MediaStreamTrack>;
     /**
      * Stop the process of video segmentation
      */
@@ -16,5 +26,17 @@ export interface VideoProcessor {
      * Destroy the segmentation wasm instance
      */
     destroy: () => Promise<void>;
+    /**
+     * Current renderer options
+     */
+    readonly renderOptions: RendererOptions;
+    /**
+     * Current processing width
+     */
+    readonly processingWidth: number;
+    /**
+     * Current processing height
+     */
+    readonly processingHeight: number;
 }
-export declare const createVideoProcessor: (transformers: Transform<InputFrame, InputFrame>[], processTrack: ProcessVideoTrack) => VideoProcessor;
+export declare const createVideoProcessor: (segmenter: Segmenter, inputProcessAdaptor: InputProcessAdaptor) => VideoProcessor;