@pexip/media-processor 19.1.0 → 20.0.0

package/dist/main/video/segmenters/mediapipe.d.ts

@@ -1,16 +1,6 @@
 import type { Segmenter } from '../types';
 import type { Delegate, SegmenterOptions, ImageSegmenterOptions, SegmentationModelAsset } from '../../../common/types/segmentation';
 interface Options extends Omit<ImageSegmenterOptions, 'delegate'> {
-    /**
-     * Processing width for the processing canvas.
-     * @defaultValue 768
-     */
-    processingWidth?: number;
-    /**
-     * Processing height for the processing canvas.
-     * @defaultValue 432
-     */
-    processingHeight?: number;
     /**
      * The URL for the worker script.
      */
@@ -28,5 +18,5 @@ interface Options extends Omit<ImageSegmenterOptions, 'delegate'> {
     delegate?: () => Delegate;
     log?: (message: string, meta?: unknown) => void;
 }
-export declare const createSegmenter: (basePath: SegmenterOptions['basePath'], { processingWidth, processingHeight, workerScriptUrl, credentials, delegate, log, ...options }: Options) => Segmenter;
+export declare const createSegmenter: (basePath?: SegmenterOptions["basePath"], { workerScriptUrl, credentials, delegate, log, ...options }?: Options) => Segmenter;
 export {};

package/dist/main/video/segmenters/mediapipe.js

@@ -1,5 +1,5 @@
 import { assert } from '@pexip/utils';
-import { PROCESS_STATUS, PROCESSING_WIDTH, PROCESSING_HEIGHT, RENDERING_EVENTS, } from '../../../common/constants';
+import { PROCESS_STATUS, RENDERING_EVENTS } from '../../../common/constants';
 const cloneImageRecord = async (image) => {
     const cloned = await createImageBitmap(image.image);
     return { image: cloned, key: image.key };
@@ -12,7 +12,7 @@ export const createSegmenter = (
  * @see {@link
  * https://ai.google.dev/edge/api/mediapipe/js/tasks-vision.filesetresolver#filesetresolverforvisiontasks | FileResolver}
  */
-basePath = '/', { processingWidth = PROCESSING_WIDTH, processingHeight = PROCESSING_HEIGHT, workerScriptUrl = new URL('../../../workers/mediaWorker.js', import.meta.url), credentials = 'same-origin', delegate, log, ...options }) => {
+basePath = '/', { workerScriptUrl = new URL('../../../workers/mediaWorker.js', import.meta.url), credentials = 'same-origin', delegate, log, ...options } = { modelAsset: { modelName: 'selfie', path: '' } }) => {
     const props = {
         status: PROCESS_STATUS.New,
         modelAsset: options.modelAsset,
@@ -34,11 +34,9 @@ basePath = '/', { processingWidth = PROCESSING_WIDTH, processingHeight = PROCESS
             }
             if (type === 'event') {
                 log?.(event.data.data.type, event.data.data.message);
-                props.eventHandlers
-                    .get(event.data.data.type)
-                    ?.forEach(cb => {
+                for (const cb of props.eventHandlers.get(event.data.data.type) ?? []) {
                     cb();
-                });
+                }
                 switch (event.data.data.type) {
                     case RENDERING_EVENTS.ProcessorRestarted:
                         break;
@@ -55,14 +53,15 @@ basePath = '/', { processingWidth = PROCESSING_WIDTH, processingHeight = PROCESS
     };
     const handleContextRestored = async () => {
         try {
+            props.status = PROCESS_STATUS.Updating;
             await postMsg('updated', {
                 type: 'update',
                 data: { restart: true },
             });
             props.status = PROCESS_STATUS.Updated;
-            props.eventHandlers.get('ProcessorRestarted')?.forEach(cb => {
+            for (const cb of props.eventHandlers.get('ContextRestored') ?? []) {
                 cb();
-            });
+            }
         }
         catch (err) {
             log?.('Failed to restart process', {
@@ -114,22 +113,22 @@ basePath = '/', { processingWidth = PROCESSING_WIDTH, processingHeight = PROCESS
             props.mediaWorker?.postMessage(message, transfer ?? []);
         });
     }
-    const open = async (processorOptions) => {
+    const open = async ({ processingWidth, processingHeight, output, imageSegmenterOptions, ...restOptions }) => {
         props.status = PROCESS_STATUS.Opening;
         props.mediaWorker = initWorker();
-        props.output = processorOptions?.output;
-        const backgroundImage = processorOptions?.backgroundImage &&
-            (await cloneImageRecord(processorOptions.backgroundImage));
+        props.output = output;
+        const backgroundImage = restOptions?.backgroundImage &&
+            (await cloneImageRecord(restOptions.backgroundImage));
         try {
             const processorSettings = {
                 basePath,
                 processingWidth,
                 processingHeight,
-                ...processorOptions,
+                ...restOptions,
                 imageSegmenterOptions: {
                     ...options,
                     delegate: delegate?.(),
-                    ...processorOptions?.imageSegmenterOptions,
+                    ...imageSegmenterOptions,
                 },
                 backgroundImage,
             };
@@ -137,7 +136,7 @@ basePath = '/', { processingWidth = PROCESSING_WIDTH, processingHeight = PROCESS
             await postMsg('opened', {
                 type: 'open',
                 data: processorSettings,
-            }, [processorOptions?.output, backgroundImage?.image].filter(Boolean));
+            }, [output, backgroundImage?.image].filter(Boolean));
             props.status = PROCESS_STATUS.Opened;
         }
         catch (error) {
@@ -148,7 +147,8 @@ basePath = '/', { processingWidth = PROCESSING_WIDTH, processingHeight = PROCESS
         }
     };
     const process = async (input, options) => {
-        if (props.status === PROCESS_STATUS.Error) {
+        if (props.status === PROCESS_STATUS.Error ||
+            props.status === PROCESS_STATUS.Updating) {
             return input;
         }
         props.status = PROCESS_STATUS.Processing;
@@ -172,6 +172,26 @@ basePath = '/', { processingWidth = PROCESSING_WIDTH, processingHeight = PROCESS
     };
     const update = async (options) => {
         try {
+            if (props.status === PROCESS_STATUS.Processing ||
+                props.status === PROCESS_STATUS.Opening ||
+                props.status === PROCESS_STATUS.Updating) {
+                const waitForStatus = new Promise((resolve, reject) => {
+                    const handleMessage = (ev) => {
+                        if (ev.data.type === 'processed' ||
+                            ev.data.type === 'opened' ||
+                            ev.data.type === 'updated') {
+                            props.mediaWorker?.removeEventListener('message', handleMessage);
+                            resolve(undefined);
+                        }
+                        else {
+                            reject(new Error(`UnhandledEvent: ${ev.data.type}`));
+                        }
+                    };
+                    props.mediaWorker?.addEventListener('message', handleMessage);
+                });
+                await waitForStatus;
+            }
+            props.status = PROCESS_STATUS.Updating;
             const backgroundImage = options.backgroundImage &&
                 (await cloneImageRecord(options.backgroundImage));
             log?.('Update processor config', { config: options });
@@ -183,6 +203,7 @@ basePath = '/', { processingWidth = PROCESSING_WIDTH, processingHeight = PROCESS
                     backgroundImage,
                 },
             }, transfer);
+            props.status = PROCESS_STATUS.Updated;
         }
         catch (error) {
             props.status = PROCESS_STATUS.Error;
@@ -217,12 +238,6 @@ basePath = '/', { processingWidth = PROCESSING_WIDTH, processingHeight = PROCESS
         get status() {
             return props.status;
         },
-        get width() {
-            return processingWidth;
-        },
-        get height() {
-            return processingHeight;
-        },
         subscribe: (event, callback) => {
             let handlers = props.eventHandlers.get(event);
             if (!handlers) {

package/dist/main/video/transformer.d.ts

@@ -1,5 +1,5 @@
 import type { InputFrame } from './types';
-export declare const nullTransformController: (controller: Pick<TransformStreamDefaultController<CanvasImageSource>, 'enqueue' | 'terminate'>) => TransformStreamDefaultController<CanvasImageSource>;
+export declare const nullTransformController: (controller: Pick<TransformStreamDefaultController<CanvasImageSource>, "enqueue" | "terminate">) => TransformStreamDefaultController<CanvasImageSource>;
 export declare const adaptInputFrameTransformer: (transformer: Transformer<InputFrame, InputFrame>) => {
     transform: (frame: VideoFrame, controller: TransformStreamDefaultController<VideoFrame>) => void | PromiseLike<void> | undefined;
     flush?: TransformerFlushCallback<InputFrame> | undefined;

package/dist/main/video/typeGuards.d.ts

@@ -1 +1,2 @@
-export declare const isRenderEffects: (t: unknown) => t is "blur" | "none" | "overlay";
+import { type RenderEffects } from '../../common/constants';
+export declare const isRenderEffects: (t: unknown) => t is RenderEffects;

package/dist/main/video/types.d.ts

@@ -11,16 +11,6 @@ export type Color = {
     a: number;
 };
 export type InputFrame = HTMLCanvasElement | OffscreenCanvas | VideoFrame | ImageBitmap;
-interface ProcessingSize {
-    /**
-     * Processing Width size
-     */
-    width: number;
-    /**
-     * Processing height size
-     */
-    height: number;
-}
 export interface AsyncAssets {
     tfjsCoreLoaded: boolean;
     tfjsBackendLoaded: boolean;
@@ -38,7 +28,7 @@ export interface BackgroundImage {
 }
 type Unsubscribe = () => void;
 type Callback<T> = (args: T) => void;
-export interface Segmenter extends Process, ProcessingSize {
+export interface Segmenter extends Process {
     modelAsset: SegmentationModelAsset;
     backgroundImage?: BackgroundImage;
     open(options?: ProcessorOptions): Promise<void>;
@@ -54,5 +44,5 @@ export interface SegmentationParams extends RendererOptions {
     width: number;
     height: number;
 }
-export type SegmentationTransform = Transform<InputFrame, InputFrame> & SegmentationParams & Omit<Process, 'open'>;
+export type SegmentationTransform = Transform<InputFrame, InputFrame> & SegmentationParams & Pick<Segmenter, 'update'> & Omit<Process, 'open'>;
 export {};

package/dist/main/video/utils.d.ts

@@ -56,8 +56,8 @@ export declare const createVideoElement: (width: number, height: number) => HTML
 export declare const setVideoElementSrc: (video: HTMLVideoElement, src: MediaProvider) => void;
 export declare const toVideoElement: (input: MediaStream, width: number, height: number) => HTMLVideoElement;
 export declare const playVideo: (video: HTMLVideoElement, { setTimeout, clearTimeout }?: {
-    setTimeout?: (((handler: TimerHandler, timeout?: number | undefined, ...arguments: any[]) => number) & typeof setTimeout) | undefined;
-    clearTimeout?: (((id: number | undefined) => void) & typeof clearTimeout) | undefined;
+    setTimeout?: (((handler: TimerHandler, timeout?: number, ...arguments: any[]) => number) & typeof globalThis.setTimeout) | undefined;
+    clearTimeout?: (((id: number | undefined) => void) & typeof globalThis.clearTimeout) | undefined;
 }) => Promise<void>;
 /**
  * Load image with provided image element

package/dist/main/video/utils.js

@@ -81,9 +81,9 @@ export const setVideoElementSrc = (video, src) => {
         }
     };
     if (src instanceof MediaStream) {
-        src.getVideoTracks().forEach(track => {
+        for (const track of src.getVideoTracks()) {
             track.addEventListener('ended', revokeObjectURL);
-        });
+        }
     }
     if ('MediaSource' in window && src instanceof MediaSource) {
         src.addEventListener('sourceended', revokeObjectURL);

package/dist/main/video/video.d.ts

@@ -1,8 +1,9 @@
 import type { Transform } from '../types';
 import type { InputFrame } from './types';
 import type { ProcessVideoTrack } from './videoStreamTrackProcessor';
+import type { ProcessorOptions } from '../../common';
 export interface VideoProcessor {
-    open(): Promise<void>;
+    open(options?: ProcessorOptions): Promise<void>;
     /**
      * Process the video and return a MediaStream
      */
@@ -16,4 +17,4 @@ export interface VideoProcessor {
      */
     destroy: () => Promise<void>;
 }
-export declare const createVideoProcessor: (transformers: Array<Transform<InputFrame, InputFrame>>, processTrack: ProcessVideoTrack) => VideoProcessor;
+export declare const createVideoProcessor: (transformers: Transform<InputFrame, InputFrame>[], processTrack: ProcessVideoTrack) => VideoProcessor;

package/dist/main/video/video.js

@@ -4,8 +4,8 @@ export const createVideoProcessor = (transformers, processTrack) => {
     const props = {
         processing: false,
     };
-    const open = async () => {
-        await Promise.all(transformers.map(transformer => transformer.init()));
+    const open = async (options) => {
+        await Promise.all(transformers.map(transformer => transformer.init(options)));
     };
     const process = async (source) => {
         const [track] = source.getVideoTracks();
@@ -34,7 +34,9 @@ export const createVideoProcessor = (transformers, processTrack) => {
         stopStreamTracks(props.outputStream);
         props.outputStream = undefined;
         props.processing = false;
-        transformers.forEach(transformer => transformer.close());
+        for (const transformer of transformers) {
+            transformer.close();
+        }
     };
     const destroy = async () => {
         close();

package/dist/main/video/videoStreamTrackProcessor.d.ts

@@ -3,7 +3,7 @@ type Track = MediaStreamVideoTrack | MediaStreamVideoTrackGenerator;
 interface Options {
     signal?: AbortSignal;
 }
-export type ProcessVideoTrack = (track: MediaStreamVideoTrack, transformers: Array<Transformer<InputFrame, InputFrame>>, options?: Options) => Promise<Track>;
+export type ProcessVideoTrack = (track: MediaStreamVideoTrack, transformers: Transformer<InputFrame, InputFrame>[], options?: Options) => Promise<Track>;
 /**
  * Video track processor using MediaStreamTrackProcessor API
  */

package/dist/main/video/videoStreamTrackProcessor.js

@@ -18,9 +18,9 @@ export const createVideoTrackProcessor = () => (track, transformers, { signal }
     const processor = createMediaStreamTrackProcessor({ track });
     const trackGenerator = createMediaStreamTrackGenerator({ kind: 'video' });
     let readable = processor.readable;
-    transformers.forEach(transformer => {
+    for (const transformer of transformers) {
         readable = readable.pipeThrough(createStreamTransformer(adaptInputFrameTransformer(transformer)), { signal });
-    });
+    }
     // Promise returned from `ReadableStream['pipeTo']` is only resolved
     // when the streaming is finished
     readable