@pexip/media-processor 17.4.0 → 18.1.0

package/dist/common/types/segmentation.d.ts

@@ -0,0 +1,99 @@
+/// <reference types="offscreencanvas" />
+export declare const SEGMENTATION_MODELS: {
+    readonly MediapipeSelfie: "selfie";
+    readonly DeepLabV3: "deeplabV3";
+};
+export type SegmentationModel = (typeof SEGMENTATION_MODELS)[keyof typeof SEGMENTATION_MODELS];
+export interface SegmentationModelAsset {
+    path: string;
+    modelName: SegmentationModel;
+}
+export interface ImageSegmenterOptions {
+    /**
+     * The canvas element to bind textures to. This has to be set for GPU
+     * processing. The task will initialize a WebGL context and throw an error if
+     * this fails (e.g. if you have already initialized a different type of
+     * context).
+     */
+    canvas?: OffscreenCanvas;
+    modelAsset?: SegmentationModelAsset;
+    /** Overrides the default backend to use for the provided model. */
+    delegate?: 'CPU' | 'GPU' | undefined;
+}
+export interface SegmenterOptions {
+    basePath?: string;
+    output?: OffscreenCanvas | null;
+    processingWidth?: number;
+    processingHeight?: number;
+    imageSegmenterOptions?: ImageSegmenterOptions;
+}
+export interface SegmentationSmoothingConfig {
+    combineWithPreviousRatio: number;
+}
+/**
+ * Wrapper for a mask produced by a Segmentation Task.
+ */
+export interface MPMask {
+    /** Returns the canvas element that the mask is bound to. */
+    readonly canvas: HTMLCanvasElement | OffscreenCanvas | undefined;
+    /** Returns the width of the mask. */
+    readonly width: number;
+    /** Returns the height of the mask. */
+    readonly height: number;
+    /**
+     * Returns whether this `MPMask` contains a mask of type `Uint8Array`.
+     */
+    hasUint8Array(): boolean;
+    /**
+     * Returns whether this `MPMask` contains a mask of type `Float32Array`.
+     */
+    hasFloat32Array(): boolean;
+    /**
+     * Returns whether this `MPMask` contains a mask of type `WebGLTexture`.
+     */
+    hasWebGLTexture(): boolean;
+    /**
+     * Returns the underlying mask as a Uint8Array`. Note that this involves an
+     * expensive GPU to CPU transfer if the current mask is only available as a
+     * `WebGLTexture`.
+     *
+     * @return The current data as a Uint8Array.
+     */
+    getAsUint8Array(): Uint8Array;
+    /**
+     * Returns the underlying mask as a single channel `Float32Array`. Note that
+     * this involves an expensive GPU to CPU transfer if the current mask is
+     * only available as a `WebGLTexture`.
+     *
+     * @return The current mask as a Float32Array.
+     */
+    getAsFloat32Array(): Float32Array;
+    /**
+     * Returns the underlying mask as a `WebGLTexture` object. Note that this
+     * involves a CPU to GPU transfer if the current mask is only available as
+     * a CPU array. The returned texture is bound to the current canvas (see
+     * `.canvas`).
+     *
+     * @return The current mask as a WebGLTexture.
+     */
+    getAsWebGLTexture(): WebGLTexture;
+    /**
+     * Creates a copy of the resources stored in this `MPMask`. You can
+     * invoke this method to extend the lifetime of a mask returned by a
+     * MediaPipe Task. Note that performance critical applications should aim to
+     * only use the `MPMask` within the MediaPipe Task callback so that
+     * copies can be avoided.
+     */
+    clone(): MPMask;
+    /**
+     * Frees up any resources owned by this `MPMask` instance.
+     *
+     * Note that this method does not free masks that are owned by the C++
+     * Task, as these are freed automatically once you leave the MediaPipe
+     * callback. Additionally, some shared state is freed only once you invoke
+     * the Task's `close()` method.
+     *
+     * @export
+     */
+    close(): void;
+}

package/dist/common/types/segmentation.js

@@ -0,0 +1,4 @@
+export const SEGMENTATION_MODELS = {
+    MediapipeSelfie: 'selfie',
+    DeepLabV3: 'deeplabV3',
+};

package/dist/common/types/utils.d.ts

@@ -0,0 +1,19 @@
+export type ExtractMessageEventType<T extends {
+    type: string;
+}> = T extends {
+    type: infer O;
+} ? O : never;
+export type IncludeMessageEventDataType<E extends {
+    type: string;
+}, I> = E extends {
+    type: infer A;
+    data: infer D;
+} ? A extends I ? D : never : E extends {
+    type: infer B;
+} ? B extends I ? void : never : never;
+/**
+ * From https://github.com/Microsoft/TypeScript/issues/26223#issuecomment-674500430
+ */
+export type TupleOf<T, N extends number> = N extends N ? number extends N ? T[] : _TupleOf<T, N, []> : never;
+type _TupleOf<T, N extends number, R extends unknown[]> = R['length'] extends N ? R : _TupleOf<T, N, [T, ...R]>;
+export {};

package/dist/common/types/utils.js

@@ -0,0 +1 @@
+export {};

package/dist/common/utils.d.ts

@@ -0,0 +1,35 @@
+import type { SegmentationModel } from './types/segmentation';
+import type { ImageMapIterable } from './types/media';
+type Assert = <T>(t: T, message?: string) => asserts t is NonNullable<T>;
+export declare const assert: Assert;
+export declare const isSegmentationModel: (t: unknown) => t is SegmentationModel;
+export declare const createObjectUpdater: <T extends { [K in keyof T]: unknown; }>(original: T) => (another: Partial<T>) => void;
+export declare const createRemoteImageBitmap: (map?: ImageMapIterable) => {
+    tryFetchingImage: (uri: string) => Promise<ImageBitmap | null>;
+    setImage: (uri: string, image: ImageBitmap) => void;
+    getImage: (uri: string) => ImageBitmap | null;
+    removeImage: (uri: string) => boolean;
+    clear: () => void;
+};
+export declare const clamping: (min: number, max: number) => (value: number) => number;
+export declare const calculateMaxBlurPass: (height: number) => number;
+/**
+ * Calculate the coordinates and size of the source and destination to fit the
+ * target aspect ratio.
+ *
+ * @param fromWidth - The width of the source
+ * @param fromHeight - The height of the source
+ * @param toWidth - The width of the destination
+ * @param toHeight - The height of the destination
+ */
+export declare const resize: (fromWidth: number, fromHeight: number, toWidth: number, toHeight: number) => {
+    sx: number;
+    sy: number;
+    sw: number;
+    sh: number;
+    dx: number;
+    dy: number;
+    dw: number;
+    dh: number;
+};
+export {};

package/dist/common/utils.js

@@ -0,0 +1,129 @@
+import { SEGMENTATION_MODELS } from './types/segmentation';
+export const assert = (t, message = 'AssertError') => {
+    if (t === undefined || t === null) {
+        throw new Error(message, { cause: t });
+    }
+    return t;
+};
+export const isSegmentationModel = (t) => {
+    if (typeof t === 'string' &&
+        Object.values(SEGMENTATION_MODELS).includes(t)) {
+        return true;
+    }
+    return false;
+};
+export const createObjectUpdater = (original) => {
+    const keys = new Set(Object.keys(original));
+    return (another) => {
+        for (const key in another) {
+            if (keys.has(key) && typeof original[key] === typeof another[key]) {
+                original[key] = another[key];
+            }
+        }
+    };
+};
+export const createRemoteImageBitmap = (map) => {
+    const cache = new Map(map);
+    const fetchingCache = new Set();
+    const fetchImage = async (uri) => {
+        const response = await fetch(uri);
+        const blob = await response.blob();
+        const image = await createImageBitmap(blob);
+        return image;
+    };
+    const tryFetchingImage = async (uri) => {
+        const image = cache.get(uri);
+        if (!image) {
+            if (fetchingCache.has(uri)) {
+                return null;
+            }
+            fetchingCache.add(uri);
+            const image = await fetchImage(uri);
+            cache.set(uri, image);
+            fetchingCache.delete(uri);
+            return image;
+        }
+        return image;
+    };
+    const removeImage = (uri) => {
+        const image = cache.get(uri);
+        image?.close();
+        return cache.delete(uri);
+    };
+    return {
+        tryFetchingImage,
+        setImage: (uri, image) => {
+            if (cache.has(uri)) {
+                removeImage(uri);
+            }
+            cache.set(uri, image);
+        },
+        getImage: (uri) => {
+            return cache.get(uri) ?? null;
+        },
+        removeImage,
+        clear: () => {
+            cache.forEach(img => img.close());
+            cache.clear();
+        },
+    };
+};
+export const clamping = (min, max) => (value) => Math.min(Math.max(value, min), max);
+export const calculateMaxBlurPass = (height) => Math.trunc(Math.log2(height));
+/**
+ * Calculate the coordinates and size of the source and destination to fit the
+ * target aspect ratio.
+ *
+ * @param fromWidth - The width of the source
+ * @param fromHeight - The height of the source
+ * @param toWidth - The width of the destination
+ * @param toHeight - The height of the destination
+ */
+export const resize = (fromWidth, fromHeight, toWidth, toHeight) => {
+    if (!fromWidth ||
+        !fromHeight ||
+        !toWidth ||
+        !toHeight ||
+        fromWidth * toHeight === toWidth * fromHeight) {
+        return {
+            sx: 0,
+            sy: 0,
+            sw: fromWidth,
+            sh: fromHeight,
+            dx: 0,
+            dy: 0,
+            dw: toWidth,
+            dh: toHeight,
+        };
+    }
+    const originalAspectRatio = fromWidth / fromHeight;
+    const targetAspectRatio = toWidth / toHeight;
+    if (originalAspectRatio > targetAspectRatio) {
+        const sw = Math.ceil(fromWidth * targetAspectRatio) / originalAspectRatio;
+        const delta = Math.abs(fromWidth - sw);
+        const sx = Math.ceil(delta / 2);
+        return {
+            sx,
+            sy: 0,
+            sw,
+            sh: fromHeight,
+            dx: 0,
+            dy: 0,
+            dw: toWidth,
+            dh: toHeight,
+        };
+    }
+    const sh = Math.ceil(fromHeight * originalAspectRatio) / targetAspectRatio;
+    const delta = Math.abs(fromHeight - sh);
+    const sy = Math.ceil(delta / 2);
+    return {
+        sx: 0,
+        sy,
+        sw: fromWidth,
+        sh,
+        dx: 0,
+        dy: 0,
+        dw: toWidth,
+        dh: toHeight,
+    };
+};

package/dist/main/benchUtils.d.ts

@@ -1,12 +1,13 @@
 import type { Clock } from './types';
+type FPS = number;
+type Timestamp = number;
 export interface Benchmark {
     begin(): void;
     end(): void;
-    calculateFps(): number;
+    calculateFps(): [Timestamp, FPS];
 }
 interface BenchmarkOptions {
     calculationThresholdMS?: number;
 }
 export declare const createBenchmark: (clock?: Clock, { calculationThresholdMS }?: BenchmarkOptions) => Benchmark;
-export declare const calculateFps: (time: number) => number;
 export {};

package/dist/main/benchUtils.js

@@ -18,17 +18,18 @@ export const createBenchmark = (clock = performance, { calculationThresholdMS =
         },
         calculateFps: () => {
             const calculateTime = clock.now();
-            if (calculateTime - props.lastCalculateTime >=
-                calculationThresholdMS) {
-                const result = props.count === 0 ? 0 : props.sumDelta / props.count;
+            const lastDuration = calculateTime - props.lastCalculateTime;
+            if (lastDuration >= calculationThresholdMS) {
+                const result = props.count === 0
+                    ? 0
+                    : (lastDuration * props.count) / props.sumDelta;
                 props.sumDelta = 0;
                 props.count = 0;
                 props.lastCalculateTime = calculateTime;
                 props.lastResult = result;
-                return result;
+                return [calculateTime, result];
             }
-            return props.lastResult;
+            return [props.lastCalculateTime, props.lastResult];
         },
     };
 };
-export const calculateFps = (time) => time === 0 ? time : 1000.0 / time;

package/dist/main/index.d.ts

@@ -14,4 +14,6 @@ export { createAsyncCallbackLoop } from './utils';
 export * from './benchUtils';
 export declare const urls: {
     denoise: () => URL;
+    mediaWorker: () => URL;
 };
+export * from '../common/index';

package/dist/main/index.js

@@ -14,4 +14,6 @@ export { createAsyncCallbackLoop } from './utils';
 export * from './benchUtils';
 export const urls = {
     denoise: () => new URL('../worklets/denoise.worklet.js', import.meta.url),
+    mediaWorker: () => new URL('../workers/mediaWorker.js', import.meta.url),
 };
+export * from '../common/index';

package/dist/main/process.d.ts

@@ -1,5 +1,5 @@
 import type { Queue } from '@pexip/utils';
-import type { StatsOptions, AudioSamples, AudioStats, Rect, Clock, IsVoice, ThrottleOptions } from './types';
+import type { StatsOptions, AudioSamples, AudioStats, Clock, IsVoice, ThrottleOptions } from './types';
 /**
  * Default silent threshold
  * At least one LSB 16-bit data (compare is on absolute value).
@@ -191,16 +191,6 @@ export declare const isVoiceActivity: ({ volumeThreshold, VADTimeThreshold, cloc
  * @param heightB - The height of B
  */
 export declare const isEqualSize: (widthA: number, heightA: number, widthB: number, heightB: number) => boolean;
-/**
- * Convert the source size to destination size when necessary based on the
- * height
- *
- * @param sw - Source width
- * @param sh - Source height
- * @param dw - destination width
- * @param dh - destination height
- */
-export declare const fitDestinationSize: (sw: number, sh: number, dw: number, dh: number) => Rect;
 /**
  * A function to check provided time series data is considered as voice activity
  *

package/dist/main/process.js

@@ -274,26 +274,6 @@ export const isVoiceActivity = ({ volumeThreshold = 0.05, VADTimeThreshold = 500
  * @param heightB - The height of B
  */
 export const isEqualSize = (widthA, heightA, widthB, heightB) => widthA === widthB && heightA === heightB;
-/**
- * Convert the source size to destination size when necessary based on the
- * height
- *
- * @param sw - Source width
- * @param sh - Source height
- * @param dw - destination width
- * @param dh - destination height
- */
-export const fitDestinationSize = (sw, sh, dw, dh) => {
-    if (!sw || !sh || !dw || !dh) {
-        return { x: 0, y: 0, width: 0, height: 0 };
-    }
-    if (isEqualSize(sw, sh, dw, dh)) {
-        return { x: 0, y: 0, width: sw, height: sh };
-    }
-    const height = Math.floor(dw * (sh / sw));
-    const y = Math.floor((dh - height) / 2);
-    return { x: 0, y, width: dw, height };
-};
 /**
  * A function to check provided time series data is considered as voice activity
  *