@pexip/media-processor 18.5.0 → 19.1.0

package/dist/main/video/segmenters/mediapipe.js

@@ -1,4 +1,5 @@
-import { PROCESS_STATUS, PROCESSING_WIDTH, PROCESSING_HEIGHT, } from '../../../common/constants';
+import { assert } from '@pexip/utils';
+import { PROCESS_STATUS, PROCESSING_WIDTH, PROCESSING_HEIGHT, RENDERING_EVENTS, } from '../../../common/constants';
 const cloneImageRecord = async (image) => {
     const cloned = await createImageBitmap(image.image);
     return { image: cloned, key: image.key };
@@ -11,49 +12,98 @@ export const createSegmenter = (
  * @see {@link
  * https://ai.google.dev/edge/api/mediapipe/js/tasks-vision.filesetresolver#filesetresolverforvisiontasks | FileResolver}
  */
-basePath = '/', { processingWidth = PROCESSING_WIDTH, processingHeight = PROCESSING_HEIGHT, workerScriptUrl = new URL('../../../workers/mediaWorker.js', import.meta.url), credentials = 'same-origin', delegate, ...options }) => {
-    const props = { status: PROCESS_STATUS.New, modelAsset: options.modelAsset };
+basePath = '/', { processingWidth = PROCESSING_WIDTH, processingHeight = PROCESSING_HEIGHT, workerScriptUrl = new URL('../../../workers/mediaWorker.js', import.meta.url), credentials = 'same-origin', delegate, log, ...options }) => {
+    const props = {
+        status: PROCESS_STATUS.New,
+        modelAsset: options.modelAsset,
+        eventHandlers: new Map(),
+    };
     const initWorker = () => {
         const mediaWorker = new Worker(workerScriptUrl, {
             type: 'classic',
             credentials,
         });
         mediaWorker.addEventListener('error', error => {
-            throw error.error;
+            log?.(`${error.message}`, error);
+            error.preventDefault();
         });
         mediaWorker.addEventListener('message', (event) => {
             const { type } = event.data;
-            if (type === 'error') {
-                throw event.data.error;
+            if (type === 'debug') {
+                log?.(event.data.data.message, event.data.data.context);
+            }
+            if (type === 'event') {
+                log?.(event.data.data.type, event.data.data.message);
+                props.eventHandlers
+                    .get(event.data.data.type)
+                    ?.forEach(cb => {
+                    cb();
+                });
+                switch (event.data.data.type) {
+                    case RENDERING_EVENTS.ProcessorRestarted:
+                        break;
+                    case RENDERING_EVENTS.ContextRestored:
+                        void handleContextRestored();
+                        break;
+                    default:
+                        props.status = PROCESS_STATUS.Error;
+                        break;
+                }
             }
         });
         return mediaWorker;
     };
+    const handleContextRestored = async () => {
+        try {
+            await postMsg('updated', {
+                type: 'update',
+                data: { restart: true },
+            });
+            props.status = PROCESS_STATUS.Updated;
+            props.eventHandlers.get('ProcessorRestarted')?.forEach(cb => {
+                cb();
+            });
+        }
+        catch (err) {
+            log?.('Failed to restart process', {
+                error: err,
+            });
+            props.status = PROCESS_STATUS.Error;
+        }
+    };
     async function postMsg(expectedResponseType, message, transfer) {
         if (!expectedResponseType) {
             return props.mediaWorker?.postMessage(message, transfer ?? []);
         }
         return await new Promise((resolve, reject) => {
             const handleMessage = (ev) => {
-                removeListener();
                 switch (ev.data.type) {
                     case expectedResponseType: {
+                        removeListener();
                         // @ts-expect-error --- Call `resolve` is expecting to pass an argument or nothing
                         resolve(ev.data.data);
                         break;
                     }
-                    case 'error': {
-                        reject(ev.data.error);
+                    case 'debug': {
+                        // Nothing to handle
+                        break;
+                    }
+                    case 'event': {
+                        removeListener();
+                        reject(new Error(ev.data.data.type, {
+                            cause: ev.data.data.message,
+                        }));
                         break;
                     }
                     default: {
-                        throw new Error(`UnhandledEvent: ${ev.data.type}`);
+                        removeListener();
+                        reject(new Error(`UnhandledEvent: ${ev.data.type}`));
                     }
                 }
             };
             const handleError = (error) => {
                 removeListener();
-                reject(error.error);
+                reject(error.message);
             };
             const removeListener = () => {
                 props.mediaWorker?.removeEventListener('message', handleMessage);
@@ -70,9 +120,8 @@ basePath = '/', { processingWidth = PROCESSING_WIDTH, processingHeight = PROCESS
         props.output = processorOptions?.output;
         const backgroundImage = processorOptions?.backgroundImage &&
             (await cloneImageRecord(processorOptions.backgroundImage));
-        await postMsg('opened', {
-            type: 'open',
-            data: {
+        try {
+            const processorSettings = {
                 basePath,
                 processingWidth,
                 processingHeight,
@@ -83,40 +132,75 @@ basePath = '/', { processingWidth = PROCESSING_WIDTH, processingHeight = PROCESS
                     ...processorOptions?.imageSegmenterOptions,
                 },
                 backgroundImage,
-            },
-        }, [processorOptions?.output, backgroundImage?.image].filter(Boolean));
-        props.status = PROCESS_STATUS.Opened;
+            };
+            log?.('Init processor with config', { config: processorSettings });
+            await postMsg('opened', {
+                type: 'open',
+                data: processorSettings,
+            }, [processorOptions?.output, backgroundImage?.image].filter(Boolean));
+            props.status = PROCESS_STATUS.Opened;
+        }
+        catch (error) {
+            props.status = PROCESS_STATUS.Error;
+            log?.('Failed to open process', {
+                error: error,
+            });
+        }
     };
     const process = async (input, options) => {
+        if (props.status === PROCESS_STATUS.Error) {
+            return input;
+        }
         props.status = PROCESS_STATUS.Processing;
-        const resultFrame = await postMsg('processed', {
-            type: 'process',
-            data: { videoFrame: input, renderOptions: options },
-        }, [input.frame]);
-        if (props.status === PROCESS_STATUS.Processing) {
-            props.status = PROCESS_STATUS.Idle;
+        try {
+            const resultFrame = await postMsg('processed', {
+                type: 'process',
+                data: { videoFrame: input, renderOptions: options },
+            }, [input.frame]);
+            if (props.status === PROCESS_STATUS.Processing) {
+                props.status = PROCESS_STATUS.Idle;
+            }
+            return resultFrame;
+        }
+        catch (error) {
+            props.status = PROCESS_STATUS.Error;
+            log?.('Error during the processing, return original instead.', {
+                error,
+            });
+            return input;
         }
-        return resultFrame;
     };
     const update = async (options) => {
-        const backgroundImage = options.backgroundImage &&
-            (await cloneImageRecord(options.backgroundImage));
-        const transfer = backgroundImage ? [backgroundImage.image] : [];
-        await postMsg(undefined, {
-            type: 'update',
-            data: {
-                ...options,
-                backgroundImage,
-            },
-        }, transfer);
+        try {
+            const backgroundImage = options.backgroundImage &&
+                (await cloneImageRecord(options.backgroundImage));
+            log?.('Update processor config', { config: options });
+            const transfer = backgroundImage ? [backgroundImage.image] : [];
+            await postMsg('updated', {
+                type: 'update',
+                data: {
+                    ...options,
+                    backgroundImage,
+                },
+            }, transfer);
+        }
+        catch (error) {
+            props.status = PROCESS_STATUS.Error;
+            log?.('Failed to update process', {
+                error: error,
+            });
+        }
     };
     const close = () => {
         props.status = PROCESS_STATUS.Closing;
+        log?.('Close processor');
         void postMsg(undefined, { type: 'close' });
+        props.eventHandlers.clear();
         props.status = PROCESS_STATUS.Closed;
     };
     const destroy = async () => {
         props.status = PROCESS_STATUS.Destroying;
+        log?.('Destroy processor');
         await postMsg(undefined, { type: 'destroy' });
         props.mediaWorker?.terminate();
         props.mediaWorker = undefined;
@@ -139,6 +223,16 @@ basePath = '/', { processingWidth = PROCESSING_WIDTH, processingHeight = PROCESS
         get height() {
             return processingHeight;
         },
+        subscribe: (event, callback) => {
+            let handlers = props.eventHandlers.get(event);
+            if (!handlers) {
+                props.eventHandlers.set(event, new Set());
+                handlers = props.eventHandlers.get(event);
+            }
+            assert(handlers);
+            handlers.add(callback);
+            return () => handlers?.delete(callback);
+        },
         open,
         update,
         process,

package/dist/main/video/transformer.js

@@ -17,8 +17,20 @@ const wrapTransformController = (videoFrame, controller) => {
                 frame instanceof HTMLImageElement ||
                 frame instanceof ImageBitmap) {
                 const timestamp = videoFrame.timestamp ?? 0;
-                videoFrame.close();
-                return controller.enqueue(new VideoFrame(frame, { timestamp, alpha: 'discard' }));
+                try {
+                    // Constructing a new VideoFrame with the modified frame
+                    // could throw an error
+                    const newFrame = new VideoFrame(frame, {
+                        timestamp,
+                        alpha: 'discard',
+                    });
+                    controller.enqueue(newFrame);
+                    videoFrame.close();
+                }
+                catch {
+                    controller.enqueue(videoFrame);
+                }
+                return;
             }
             throw new Error('Unexpected input frame');
         },

package/dist/main/video/types.d.ts

@@ -1,4 +1,4 @@
-import type { ProcessStatus } from '../../common/constants';
+import type { ProcessStatus, RenderingEvents } from '../../common/constants';
 import type { ProcessInputType, VideoFrameLike } from '../../common/types/media';
 import type { RendererOptions } from '../../common/types/render';
 import type { SegmentationModelAsset } from '../../common/types/segmentation';
@@ -36,12 +36,15 @@ export interface BackgroundImage {
     key: string;
     image: Blob;
 }
+type Unsubscribe = () => void;
+type Callback<T> = (args: T) => void;
 export interface Segmenter extends Process, ProcessingSize {
     modelAsset: SegmentationModelAsset;
     backgroundImage?: BackgroundImage;
     open(options?: ProcessorOptions): Promise<void>;
     update(options: ProcessorOptions): Promise<void>;
     process(input: VideoFrameLike, options?: RendererOptions): Promise<VideoFrameLike | undefined>;
+    subscribe(event: RenderingEvents, callback: Callback<void>): Unsubscribe;
 }
 export interface Detector<T> extends Process {
     detect(input: ProcessInputType): Promise<T>;

package/dist/main/video/utils.d.ts

@@ -55,7 +55,10 @@ export declare const createOffscreenCanvas: (width: number, height: number) => H
 export declare const createVideoElement: (width: number, height: number) => HTMLVideoElement;
 export declare const setVideoElementSrc: (video: HTMLVideoElement, src: MediaProvider) => void;
 export declare const toVideoElement: (input: MediaStream, width: number, height: number) => HTMLVideoElement;
-export declare const playVideo: (video: HTMLVideoElement) => Promise<void>;
+export declare const playVideo: (video: HTMLVideoElement, { setTimeout, clearTimeout }?: {
+    setTimeout?: (((handler: TimerHandler, timeout?: number | undefined, ...arguments: any[]) => number) & typeof setTimeout) | undefined;
+    clearTimeout?: (((id: number | undefined) => void) & typeof clearTimeout) | undefined;
+}) => Promise<void>;
 /**
  * Load image with provided image element
  */

package/dist/main/video/utils.js

@@ -1,4 +1,5 @@
 import { createAsyncCallbackLoop, subscribeVisibilityChangeEvent, } from '../utils';
+import { PLAY_VIDEO_TIMEOUT } from './constants';
 /**
  * Draw the source onto the provided canvas according to the destination rect
  *
@@ -113,6 +114,10 @@ export const toVideoElement = (input, width, height) => {
     setVideoElementSrc(video, input);
     return video;
 };
+/**
+ * The HTMLMediaElement.readyState property indicates the readiness state of the media
+ * @see {@link https://developer.mozilla.org/en-US/docs/Web/API/HTMLMediaElement/readyState}
+ */
 var HTMLMediaElementReadyState;
 (function (HTMLMediaElementReadyState) {
     HTMLMediaElementReadyState[HTMLMediaElementReadyState["HaveNothing"] = 0] = "HaveNothing";
@@ -121,28 +126,50 @@ var HTMLMediaElementReadyState;
     HTMLMediaElementReadyState[HTMLMediaElementReadyState["HaveFutureData"] = 3] = "HaveFutureData";
     HTMLMediaElementReadyState[HTMLMediaElementReadyState["HaveEnoughData"] = 4] = "HaveEnoughData";
 })(HTMLMediaElementReadyState || (HTMLMediaElementReadyState = {}));
-export const playVideo = async (video) => {
+export const playVideo = async (video, { setTimeout = window.setTimeout, clearTimeout = window.clearTimeout } = {}) => {
     if (video.autoplay) {
         return;
     }
     if (video.readyState <
         HTMLMediaElementReadyState.HaveCurrentData) {
         // Not enough data, wait for the data and then play
-        await Promise.race([
-            new Promise(resolve => {
-                const waitForEvent = () => {
-                    video.removeEventListener('loadeddata', waitForEvent);
-                    resolve();
-                };
-                video.addEventListener('loadeddata', waitForEvent);
-            }),
+        await new Promise((resolve, reject) => {
+            const clearTimeoutTimer = () => {
+                clearTimeout(timeoutId);
+                timeoutId = 0;
+            };
+            const unsubscribe = () => {
+                video.removeEventListener('loadeddata', waitForEvent);
+                video.removeEventListener('error', handleError);
+                video.removeEventListener('stalled', handleStalled);
+                video.removeEventListener('ended', handleEnded);
+                clearTimeoutTimer();
+            };
+            const handleEnded = () => {
+                unsubscribe();
+                reject(new Error('VideoEnded', { cause: video }));
+            };
+            const handleStalled = () => {
+                unsubscribe();
+                reject(new Error('VideoStalled', { cause: video }));
+            };
+            const handleError = (error) => {
+                unsubscribe();
+                reject(error.error);
+            };
+            const waitForEvent = () => {
+                unsubscribe();
+                resolve();
+            };
             // Use setTimeout to prevent a race to `loadeddata` event
-            new Promise(resolve => {
-                setTimeout(() => {
-                    resolve();
-                }, 3000);
-            }),
-        ]);
+            video.addEventListener('loadeddata', waitForEvent);
+            video.addEventListener('error', handleError);
+            video.addEventListener('stalled', handleStalled);
+            video.addEventListener('ended', handleEnded);
+            let timeoutId = setTimeout(() => {
+                reject(new Error('PlayVideoTimeout', { cause: video }));
+            }, PLAY_VIDEO_TIMEOUT);
+        });
     }
     return await video.play();
 };

package/dist/main/video/videoStreamTrackProcessor.js

@@ -1,3 +1,4 @@
+import { assert } from '@pexip/utils';
 import { createMediaStreamTrackProcessor } from '../processor';
 import { createMediaStreamTrackGenerator } from '../generator';
 import { createStreamTransformer } from '../transformer';
@@ -46,16 +47,12 @@ export const createVideoTrackProcessorWithFallback = ({ width = PROCESSING_WIDTH
     if (!transformer?.transform) {
         return track;
     }
-    if (transformers.length > 1) {
-        throw new Error('Multi-transformer is NOT supported');
-    }
+    assert(transformers.length <= 1, 'Multi-transformer is NOT supported');
     const outputCanvas = createCanvas(width, height);
     const ctx = getCanvasRenderingContext2D(outputCanvas, { alpha: false });
     const render = (input) => {
-        if (!transformer.transform) {
-            throw new Error('Transform is undefined');
-        }
-        return transformer.transform(input, nullTransformController({
+        assert(transformer.transform);
+        return Promise.resolve(transformer.transform(input, nullTransformController({
             enqueue: frame => {
                 if (!frame) {
                     return;
@@ -71,7 +68,7 @@ export const createVideoTrackProcessorWithFallback = ({ width = PROCESSING_WIDTH
             terminate: () => {
                 stop();
             },
-        }));
+        })));
     };
     const runner = createFrameCallbackRequest(render, frameRate);
     const videoElement = toVideoElement(new MediaStream([track]), width, height);
@@ -79,9 +76,7 @@ export const createVideoTrackProcessorWithFallback = ({ width = PROCESSING_WIDTH
     await runner.start(videoElement);
     const stream = outputCanvas.captureStream(frameRate);
     const [trackGenerated] = stream.getVideoTracks();
-    if (!trackGenerated) {
-        throw new Error('Canvas captureStream returns no video track');
-    }
+    assert(trackGenerated, 'Canvas captureStream returns no video track');
     const stop = () => {
         stopStreamTracks(stream);
         runner.stop();