@pexip/media-processor 21.0.0 → 22.1.0

package/dist/main/video/inputProcessAdaptor.js

@@ -0,0 +1,198 @@
+import { assert } from '@pexip/utils';
+import { createMediaStreamTrackProcessor } from '../processor';
+import { createMediaStreamTrackGenerator } from '../generator';
+import { playVideo, getCanvasRenderingContext2D } from './utils';
+import { MediaStreamTrackTransferable } from '../utils';
+import { FRAME_RATE } from './constants';
+/**
+ * An implementation of `InputProcessAdaptor` which uses `MediaStreamTrackGenerator` and
+ * `MediaStreamTrackProcessor` APIs to create `ReadableStream` & `WritableStream`
+ */
+export const createReadWritableStreamInputProcessAdaptor = () => {
+    let outputTrack;
+    return {
+        adapt(track) {
+            assert(track.kind === 'video');
+            const processor = createMediaStreamTrackProcessor({
+                track: track,
+            });
+            outputTrack = createMediaStreamTrackGenerator({
+                kind: 'video',
+            });
+            return {
+                readable: processor.readable,
+                writable: outputTrack.writable,
+            };
+        },
+        get outputTrack() {
+            return outputTrack;
+        },
+    };
+};
+/**
+ * Wait for the next frame using the best available API based on
+ * page visibility:
+ *
+ * - Visible + rVFC supported → `requestVideoFrameCallback`
+ * - Visible + no rVFC        → `requestAnimationFrame`
+ * - Hidden                   → `setTimeout` at the given frame rate
+ */
+const waitForNextFrame = (video, msPerFrame) => {
+    if (document.hidden) {
+        return new Promise(r => setTimeout(r, msPerFrame));
+    }
+    if ('requestVideoFrameCallback' in HTMLVideoElement.prototype) {
+        return new Promise(r => video.requestVideoFrameCallback(() => r()));
+    }
+    return new Promise(r => requestAnimationFrame(() => r()));
+};
+/**
+ * An implementation of `InputProcessAdaptor` which uses `HTMLCanvasElement` or
+ * `OffscreenCanvas` API to create `ReadableStream` & `WritableStream`
+ */
+export const createReadWritableStreamFallbackInputProcessAdaptor = ({ frameRate, }) => {
+    let outputTrack;
+    return {
+        adapt(track) {
+            assert(track.kind === 'video');
+            const video = document.createElement('video');
+            let readableCanvas;
+            let readableCtx;
+            let timestamp = 0;
+            const fps = track.getSettings().frameRate || frameRate;
+            const timeMsPerFrame = 1000 / fps;
+            const releaseElements = () => {
+                video.width = 0;
+                video.height = 0;
+                if (readableCanvas) {
+                    readableCanvas.width = 0;
+                    readableCanvas.height = 0;
+                }
+            };
+            const readable = new ReadableStream({
+                async start(controller) {
+                    const cleanup = () => {
+                        controller.close();
+                        releaseElements();
+                    };
+                    track.addEventListener('ended', cleanup, {
+                        once: true,
+                    });
+                    video.srcObject = new MediaStream([track]);
+                    video.muted = true;
+                    video.playsInline = true;
+                    await playVideo(video);
+                    readableCanvas = new OffscreenCanvas(video.videoWidth, video.videoHeight);
+                    readableCtx = getCanvasRenderingContext2D(readableCanvas, {
+                        alpha: false,
+                    });
+                    timestamp = performance.now();
+                },
+                cancel() {
+                    releaseElements();
+                },
+                async pull(controller) {
+                    if (track.readyState === 'ended') {
+                        releaseElements();
+                        return controller.close();
+                    }
+                    // Wait a bit if too fast, otherwise go ahead for the next frame
+                    const diffMs = timeMsPerFrame - (performance.now() - timestamp);
+                    if (diffMs) {
+                        await waitForNextFrame(video, diffMs);
+                        // @ts-expect-error  --- sth is wrong with type narrowing
+                        if (track.readyState === 'ended') {
+                            releaseElements();
+                            return controller.close();
+                        }
+                    }
+                    if (readableCanvas.width !== video.videoWidth) {
+                        readableCanvas.width = video.videoWidth;
+                    }
+                    if (readableCanvas.height !== video.videoHeight) {
+                        readableCanvas.height = video.videoHeight;
+                    }
+                    timestamp = performance.now();
+                    readableCtx.drawImage(video, 0, 0);
+                    controller.enqueue(new VideoFrame(readableCanvas, {
+                        // VideoFrame timestamp is in microseconds
+                        // https://developer.mozilla.org/en-US/docs/Web/API/VideoFrame/timestamp
+                        timestamp: timestamp * 1000,
+                    }));
+                },
+            });
+            // captureStream API is only available in HTMLCanvasElement
+            const writableCanvas = document.createElement('canvas');
+            const writableCtx = getCanvasRenderingContext2D(writableCanvas, {
+                alpha: false,
+                desynchronized: true,
+            });
+            const [capturedVideoTrack] = writableCanvas
+                .captureStream(frameRate)
+                .getVideoTracks();
+            outputTrack = capturedVideoTrack;
+            const release = () => {
+                capturedVideoTrack?.stop();
+                writableCanvas.width = 0;
+                writableCanvas.height = 0;
+            };
+            const writable = new WritableStream({
+                abort() {
+                    release();
+                },
+                close() {
+                    release();
+                },
+                write(frame) {
+                    if (writableCanvas.width !== frame.displayWidth) {
+                        writableCanvas.width = frame.displayWidth;
+                    }
+                    if (writableCanvas.height !== frame.displayHeight) {
+                        writableCanvas.height = frame.displayHeight;
+                    }
+                    writableCtx.drawImage(frame, 0, 0, writableCanvas.width, writableCanvas.height);
+                    frame.close();
+                },
+            });
+            return {
+                readable,
+                writable,
+            };
+        },
+        get outputTrack() {
+            return outputTrack;
+        },
+    };
+};
+/**
+ * An implementation of `InputProcessAdaptor` which adapts `MediaStreamTrack`
+ * directly as specified from the spec https://w3c.github.io/mediacapture-transform
+ */
+export const createMediaStreamTrackInputProcessAdaptor = () => {
+    return {
+        adapt(track) {
+            return track;
+        },
+    };
+};
+/**
+ * Get the supported InputProcessAdaptor based on feature detection
+ * and preference provided
+ *
+ * @param tryNativeAPI - Whether or not to try to use the Native MediaStreamTrackProcessor APIs
+ * @param targetFrameRate - The target frame rate to use when fallback implementation of
+ * video frame extraction rate
+ */
+export const getDefaultInputProcessAdaptor = (tryNativeAPI = true, targetFrameRate = FRAME_RATE) => {
+    if (tryNativeAPI) {
+        if (MediaStreamTrackTransferable) {
+            return createMediaStreamTrackInputProcessAdaptor();
+        }
+        else if ('MediaStreamTrackProcessor' in window) {
+            return createReadWritableStreamInputProcessAdaptor();
+        }
+    }
+    return createReadWritableStreamFallbackInputProcessAdaptor({
+        frameRate: targetFrameRate,
+    });
+};

package/dist/main/video/segmenter.d.ts

@@ -0,0 +1,42 @@
+import type { Segmenter } from './types';
+import type { Delegate, SegmenterOptions, ImageSegmenterOptions, SegmentationModelAsset } from '../../common/types/segmentation';
+import type { FrameTransformStats } from '../../common/types/messageEvents';
+type LogMethod = (meta: unknown, message?: string) => void;
+export type Logger = {
+    debug: LogMethod;
+    info: LogMethod;
+    warn: LogMethod;
+    error: LogMethod;
+};
+interface Options extends Omit<ImageSegmenterOptions, 'delegate'> {
+    /**
+     * The URL for the worker script.
+     */
+    workerScriptUrl?: URL;
+    /**
+     * Request Credentials for the worker.
+     *
+     * @defaultValue 'same-origin'
+     */
+    credentials?: RequestCredentials;
+    /**
+     * @see `SegmentationModelAsset`
+     */
+    modelAsset: SegmentationModelAsset;
+    delegate?: () => Delegate;
+    log?: Logger;
+    /**
+     * Stats callback handler, e.g. benchmask
+     */
+    stats?: (stats: FrameTransformStats[]) => void;
+}
+export declare const createSegmenter: (
+/**
+ * A base path to specify the directory the Wasm files should be loaded
+ * from. If not provided, it will be loaded from the host's root directory.
+ *
+ * @see {@link
+ * https://ai.google.dev/edge/api/mediapipe/js/tasks-vision.filesetresolver#filesetresolverforvisiontasks | FileResolver}
+ */
+basePath?: SegmenterOptions["basePath"], { workerScriptUrl, credentials, delegate, log, stats, ...options }?: Options) => Segmenter;
+export {};

package/dist/main/video/{segmenters/mediapipe.js → segmenter.js}

@@ -1,9 +1,6 @@
 import { assert } from '@pexip/utils';
-import { PROCESS_STATUS, RENDERING_EVENTS } from '../../../common/constants';
-const cloneImageRecord = async (image) => {
-    const cloned = await createImageBitmap(image.image);
-    return { image: cloned, key: image.key };
-};
+import { PROCESS_STATUS, RENDERING_EVENTS } from '../../common/constants';
+import { cloneImageRecord } from '../../common/utils';
 export const createSegmenter = (
 /**
  * A base path to specify the directory the Wasm files should be loaded
@@ -12,7 +9,7 @@ export const createSegmenter = (
  * @see {@link
  * https://ai.google.dev/edge/api/mediapipe/js/tasks-vision.filesetresolver#filesetresolverforvisiontasks | FileResolver}
  */
-basePath = '/', { workerScriptUrl = new URL('../../../workers/mediaWorker.js', import.meta.url), credentials = 'same-origin', delegate, log, ...options } = { modelAsset: { modelName: 'selfie', path: '' } }) => {
+basePath = '/', { workerScriptUrl = new URL('../../workers/mediaWorker.js', import.meta.url), credentials = 'same-origin', delegate, log, stats, ...options } = { modelAsset: { modelName: 'selfie', path: '' } }) => {
     const props = {
         status: PROCESS_STATUS.New,
         modelAsset: options.modelAsset,
@@ -24,28 +21,53 @@ basePath = '/', { workerScriptUrl = new URL('../../../workers/mediaWorker.js', i
             credentials,
         });
         mediaWorker.addEventListener('error', error => {
-            log?.(`${error.message}`, error);
+            log?.error({ event: error, error: error.error }, `${error.message}`);
             error.preventDefault();
         });
         mediaWorker.addEventListener('message', (event) => {
-            const { type } = event.data;
-            if (type === 'debug') {
-                log?.(event.data.data.message, event.data.data.context);
-            }
-            if (type === 'event') {
-                log?.(event.data.data.type, event.data.data.message);
-                for (const cb of props.eventHandlers.get(event.data.data.type) ?? []) {
-                    cb();
+            switch (event.data.type) {
+                case 'debug': {
+                    log?.debug(event.data.data.context, event.data.data.message);
+                    break;
                 }
-                switch (event.data.data.type) {
-                    case RENDERING_EVENTS.ProcessorRestarted:
-                        break;
-                    case RENDERING_EVENTS.ContextRestored:
-                        void handleContextRestored();
-                        break;
-                    default:
-                        props.status = PROCESS_STATUS.Error;
-                        break;
+                case 'event': {
+                    switch (event.data.data.type) {
+                        case 'Error':
+                        case 'ContextCreationError':
+                        case 'SamplingError':
+                            log?.error(event.data.data, event.data.data.message);
+                            break;
+                        case 'ContextLost':
+                        case 'ContextRestored':
+                        case 'ProcessorRestarted':
+                            log?.info(event.data.data, event.data.data.message);
+                            break;
+                        default:
+                            log?.debug(event.data.data, event.data.data.message);
+                            break;
+                    }
+                    for (const cb of props.eventHandlers.get(event.data.data.type) ?? []) {
+                        cb();
+                    }
+                    switch (event.data.data.type) {
+                        case RENDERING_EVENTS.ProcessorRestarted:
+                            break;
+                        case RENDERING_EVENTS.ContextRestored:
+                            void handleContextRestored();
+                            break;
+                        default:
+                            props.status = PROCESS_STATUS.Error;
+                            break;
+                    }
+                    break;
+                }
+                case 'stats': {
+                    stats?.(event.data.data);
+                    break;
+                }
+                default: {
+                    // Do nothing
+                    break;
                 }
             }
         });
@@ -64,15 +86,14 @@ basePath = '/', { workerScriptUrl = new URL('../../../workers/mediaWorker.js', i
             }
         }
         catch (err) {
-            log?.('Failed to restart process', {
-                error: err,
-            });
+            log?.error({ error: err }, 'Failed to restart process');
             props.status = PROCESS_STATUS.Error;
         }
     };
     async function postMsg(expectedResponseType, message, transfer) {
         if (!expectedResponseType) {
-            return props.mediaWorker?.postMessage(message, transfer ?? []);
+            props.mediaWorker?.postMessage(message, transfer ?? []);
+            return;
         }
         return await new Promise((resolve, reject) => {
             const handleMessage = (ev) => {
@@ -83,26 +104,34 @@ basePath = '/', { workerScriptUrl = new URL('../../../workers/mediaWorker.js', i
                         resolve(ev.data.data);
                         break;
                     }
-                    case 'debug': {
-                        // Nothing to handle
+                    case 'debug':
+                    case 'stats': {
+                        // Nothing to handle -- these are continues
+                        // messages handled by the global event listener
                         break;
                     }
                     case 'event': {
                         removeListener();
-                        reject(new Error(ev.data.data.type, {
-                            cause: ev.data.data.message,
-                        }));
+                        reject(ev.data.data.error ??
+                            new Error(ev.data.data.type, {
+                                cause: ev.data.data.message,
+                            }));
                         break;
                     }
                     default: {
-                        removeListener();
-                        reject(new Error(`UnhandledEvent: ${ev.data.type}`));
+                        // Ignore messages that don't match the expected
+                        // response type. Other concurrent requests (e.g.
+                        // a `close` arriving while a `processed` is in
+                        // flight) post their own responses on the same
+                        // worker; rejecting here would cross-poison
+                        // listeners and surface false errors.
+                        break;
                     }
                 }
             };
             const handleError = (error) => {
                 removeListener();
-                reject(error.message);
+                reject(error.error ?? new Error(error.message));
             };
             const removeListener = () => {
                 props.mediaWorker?.removeEventListener('message', handleMessage);
@@ -113,17 +142,16 @@ basePath = '/', { workerScriptUrl = new URL('../../../workers/mediaWorker.js', i
             props.mediaWorker?.postMessage(message, transfer ?? []);
         });
     }
-    const open = async ({ processingWidth, processingHeight, output, imageSegmenterOptions, ...restOptions }) => {
+    const open = async ({ imageSegmenterOptions, ...restOptions }) => {
         props.status = PROCESS_STATUS.Opening;
-        props.mediaWorker = initWorker();
-        props.output = output;
+        if (!props.mediaWorker) {
+            props.mediaWorker = initWorker();
+        }
         const backgroundImage = restOptions?.backgroundImage &&
             (await cloneImageRecord(restOptions.backgroundImage));
         try {
             const processorSettings = {
                 basePath,
-                processingWidth,
-                processingHeight,
                 ...restOptions,
                 imageSegmenterOptions: {
                     ...options,
@@ -132,44 +160,65 @@ basePath = '/', { workerScriptUrl = new URL('../../../workers/mediaWorker.js', i
                 },
                 backgroundImage,
             };
-            log?.('Init processor with config', { config: processorSettings });
-            await postMsg('opened', {
+            log?.debug({
+                config: processorSettings,
+            }, 'Init processor with config');
+            const result = await postMsg('opened', {
                 type: 'open',
                 data: processorSettings,
-            }, [output, backgroundImage?.image].filter(Boolean));
+            }, backgroundImage?.image ? [backgroundImage.image] : []);
             props.status = PROCESS_STATUS.Opened;
+            return result;
         }
         catch (error) {
             props.status = PROCESS_STATUS.Error;
-            log?.('Failed to open process', {
-                error: error,
-            });
+            log?.warn({ error: error }, 'Failed to open process');
+            return {};
         }
     };
-    const process = async (input, options) => {
+    async function process(input, options) {
         if (props.status === PROCESS_STATUS.Error ||
             props.status === PROCESS_STATUS.Updating) {
             return input;
         }
         props.status = PROCESS_STATUS.Processing;
-        try {
-            const resultFrame = await postMsg('processed', {
-                type: 'process',
-                data: { videoFrame: input, renderOptions: options },
-            }, [input.frame]);
-            if (props.status === PROCESS_STATUS.Processing) {
+        if (input instanceof MediaStreamTrack) {
+            try {
+                // Clone the track and keep the original in main
+                const clonedTrack = input.clone();
+                const output = await postMsg('processed', {
+                    type: 'process',
+                    data: { track: clonedTrack, renderOptions: options },
+                }, [clonedTrack]);
+                assert(output);
                 props.status = PROCESS_STATUS.Idle;
+                return output;
+            }
+            catch (error) {
+                props.status = PROCESS_STATUS.Error;
+                log?.warn({ error }, 'Error during the processing, return original track instead.');
+                return input;
             }
-            return resultFrame;
         }
-        catch (error) {
-            props.status = PROCESS_STATUS.Error;
-            log?.('Error during the processing, return original instead.', {
-                error,
-            });
-            return input;
+        else {
+            try {
+                await postMsg('processed', {
+                    type: 'process',
+                    data: {
+                        readable: input.readable,
+                        writable: input.writable,
+                        renderOptions: options,
+                    },
+                }, [input.readable, input.writable]);
+                props.status = PROCESS_STATUS.Idle;
+            }
+            catch (error) {
+                props.status = PROCESS_STATUS.Error;
+                log?.warn({ error }, 'Error during the processing, return original ReadWritableStream instead.');
+                return input;
+            }
         }
-    };
+    }
     const update = async (options) => {
         try {
             if (props.status === PROCESS_STATUS.Processing ||
@@ -179,11 +228,13 @@ basePath = '/', { workerScriptUrl = new URL('../../../workers/mediaWorker.js', i
                     const handleMessage = (ev) => {
                         if (ev.data.type === 'processed' ||
                             ev.data.type === 'opened' ||
-                            ev.data.type === 'updated') {
+                            ev.data.type === 'updated' ||
+                            ev.data.type === 'closed') {
                             props.mediaWorker?.removeEventListener('message', handleMessage);
                             resolve(undefined);
                         }
-                        else {
+                        else if (ev.data.type !== 'debug' &&
+                            ev.data.type !== 'stats') {
                             reject(new Error(`UnhandledEvent: ${ev.data.type}`));
                         }
                     };
@@ -194,9 +245,9 @@ basePath = '/', { workerScriptUrl = new URL('../../../workers/mediaWorker.js', i
             props.status = PROCESS_STATUS.Updating;
             const backgroundImage = options.backgroundImage &&
                 (await cloneImageRecord(options.backgroundImage));
-            log?.('Update processor config', { config: options });
+            log?.debug({ config: options }, 'Update processor config');
             const transfer = backgroundImage ? [backgroundImage.image] : [];
-            await postMsg('updated', {
+            const changed = await postMsg('updated', {
                 type: 'update',
                 data: {
                     ...options,
@@ -204,25 +255,28 @@ basePath = '/', { workerScriptUrl = new URL('../../../workers/mediaWorker.js', i
                 },
             }, transfer);
             props.status = PROCESS_STATUS.Updated;
+            return changed;
         }
         catch (error) {
             props.status = PROCESS_STATUS.Error;
-            log?.('Failed to update process', {
-                error: error,
-            });
+            log?.warn({ error: error }, 'Failed to update process');
+            return {};
         }
     };
-    const close = () => {
+    const mute = (muted) => {
+        postMsg(undefined, { type: 'mute', data: muted });
+    };
+    const close = async () => {
         props.status = PROCESS_STATUS.Closing;
-        log?.('Close processor');
-        void postMsg(undefined, { type: 'close' });
-        props.eventHandlers.clear();
+        log?.info('Close processor');
+        await postMsg('closed', { type: 'close' });
         props.status = PROCESS_STATUS.Closed;
     };
     const destroy = async () => {
         props.status = PROCESS_STATUS.Destroying;
-        log?.('Destroy processor');
-        await postMsg(undefined, { type: 'destroy' });
+        log?.info('Destroy processor');
+        props.eventHandlers.clear();
+        postMsg(undefined, { type: 'destroy' });
         props.mediaWorker?.terminate();
         props.mediaWorker = undefined;
         props.status = PROCESS_STATUS.Destroyed;
@@ -251,6 +305,7 @@ basePath = '/', { workerScriptUrl = new URL('../../../workers/mediaWorker.js', i
         open,
         update,
         process,
+        mute,
         close,
         destroy,
     };

package/dist/main/video/typeGuards.d.ts

@@ -1,2 +1,4 @@
 import { type RenderEffects } from '../../common/constants';
 export declare const isRenderEffects: (t: unknown) => t is RenderEffects;
+export declare const isMediaStreamVideoTrack: (t: MediaStreamTrack) => t is MediaStreamVideoTrack;
+export declare function assertMediaStreamVideoTrack(t: MediaStreamTrack): asserts t is MediaStreamVideoTrack;

package/dist/main/video/typeGuards.js

@@ -1,7 +1,14 @@
 import { RENDER_EFFECT } from '../../common/constants';
+const EFFECTS = Object.values(RENDER_EFFECT);
 export const isRenderEffects = (t) => {
     if (typeof t !== 'string') {
         return false;
     }
-    return Object.values(RENDER_EFFECT).some(effect => effect === t);
+    return EFFECTS.some(effect => effect === t);
 };
+export const isMediaStreamVideoTrack = (t) => t.kind === 'video';
+export function assertMediaStreamVideoTrack(t) {
+    if (!isMediaStreamVideoTrack(t)) {
+        throw new TypeError('Not a video track', { cause: t });
+    }
+}