@pexip/media-processor 20.3.5 → 22.0.0

package/dist/common/utils.d.ts

@@ -1,14 +1,16 @@
 import type { SegmentationModel } from './types/segmentation';
-import type { ImageMapIterable } from './types/media';
+import type { ImageMapIterable, ImageRecord } from './types/media';
 import type { RenderingEvents } from './constants';
 import type { RenderEventHandlers } from './types/render';
+import type { Stats, SelectionOptions } from './types/utils';
 export declare const isSegmentationModel: (t: unknown) => t is SegmentationModel;
 export declare const isRenderingEvents: (t: unknown) => t is RenderingEvents;
 export declare const toRenderingEvents: (t: unknown) => RenderingEvents;
 export declare const getErrorMessage: (error: unknown) => string;
 export declare const createObjectUpdater: <T extends { [K in keyof T]: unknown; }>(original: T) => (another: Partial<T>) => void;
+export declare const ImageFetchError = "ImageFetchError";
 export declare const createRemoteImageBitmap: (map?: ImageMapIterable) => {
-    tryFetchingImage: (uri: string) => Promise<ImageBitmap | null>;
+    tryFetchingImage: (uri: string, abortSignal: AbortSignal) => Promise<ImageBitmap | null | undefined>;
     setImage: (uri: string, image: ImageBitmap) => void;
     getImage: (uri: string) => ImageBitmap | null;
     removeImage: (uri: string) => boolean;
@@ -16,6 +18,12 @@ export declare const createRemoteImageBitmap: (map?: ImageMapIterable) => {
 };
 export declare const clamping: (min: number, max: number) => (value: number) => number;
 export declare const calculateMaxBlurPass: (height: number) => number;
+/**
+ * Coerce any positive number to the nearest power of two within [minPow, maxPow]
+ * Example: nearestPowerOfTwo(13) => 16; nearestPowerOfTwo(3.2) => 4
+ */
+export declare function nearestPowerOfTwo(n: number, minPow?: number, // 2^0 = 1
+maxPow?: number): number;
 /**
  * Calculate the coordinates and size of the source and destination to fit the
  * target aspect ratio.
@@ -37,3 +45,22 @@ export declare const resize: (fromWidth: number, fromHeight: number, toWidth: nu
 };
 export declare const createLazyProps: <T extends Record<string, unknown>>(stateBuilder: { [K in keyof T]: (s: Partial<T>) => T[K]; }) => T;
 export declare const handleWebGLContextLoss: (canvas: OffscreenCanvas, eventHandlers: RenderEventHandlers, contextLostCb?: () => void) => () => void;
+/**
+ * Intersection over Union (IoU), how much two masks overlap as a fraction of either's total.
+ * 1.0 = perfect match, 0 = no overlap.
+ * Used for stable, robust tracking of the main person between frames,
+ * especially with bystander removal and temporal instance switching.
+ */
+export declare function iou(a: Uint8Array, b: Uint8Array): number;
+export declare function labelComponents(input: Uint8Array, labels: Int32Array, width: number, height: number): Map<number, Stats>;
+export declare function scoreComponent(stats: Stats, overlap: number, weights: SelectionOptions, width: number, height: number): number;
+export declare const cloneImageRecord: (image: ImageRecord) => Promise<{
+    image: ImageBitmap;
+    key: string;
+}>;
+/**
+ * Compare provided ImageRecord
+ *
+ * @returns true if the two records have the same key and image, false otherwise
+ */
+export declare const compareImageRecords: (a: ImageRecord, b: ImageRecord | undefined) => boolean;

package/dist/common/utils.js

@@ -31,26 +31,52 @@ export const createObjectUpdater = (original) => {
         }
     };
 };
+export const ImageFetchError = 'ImageFetchError';
+const isLikelyImageContentType = (ct) => {
+    if (!ct) {
+        return false;
+    }
+    // handle e.g. "image/png; charset=binary"
+    return ct.toLowerCase().startsWith('image/');
+};
 export const createRemoteImageBitmap = (map) => {
     const cache = new Map(map);
     const fetchingCache = new Set();
-    const fetchImage = async (uri) => {
-        const response = await fetch(uri);
-        const blob = await response.blob();
-        const image = await createImageBitmap(blob);
-        return image;
+    const fetchImage = async (uri, abortSignal) => {
+        const response = await fetch(uri, { signal: abortSignal });
+        if (response.ok &&
+            isLikelyImageContentType(response.headers.get('content-type'))) {
+            const blob = await response.blob();
+            const image = await createImageBitmap(blob);
+            return image;
+        }
     };
-    const tryFetchingImage = async (uri) => {
+    const tryFetchingImage = async (uri, abortSignal) => {
         const image = cache.get(uri);
         if (!image) {
             if (fetchingCache.has(uri)) {
                 return null;
             }
             fetchingCache.add(uri);
-            const image = await fetchImage(uri);
-            cache.set(uri, image);
-            fetchingCache.delete(uri);
-            return image;
+            try {
+                const image = await fetchImage(uri, abortSignal);
+                if (image) {
+                    cache.set(uri, image);
+                    return image;
+                }
+            }
+            catch (error) {
+                if (error instanceof Error) {
+                    if (error.name !== 'AbortError') {
+                        throw new Error(ImageFetchError, { cause: error });
+                    }
+                    return;
+                }
+                throw error;
+            }
+            finally {
+                fetchingCache.delete(uri);
+            }
         }
         return image;
     };
@@ -81,6 +107,18 @@ export const createRemoteImageBitmap = (map) => {
 };
 export const clamping = (min, max) => (value) => Math.min(Math.max(value, min), max);
 export const calculateMaxBlurPass = (height) => Math.trunc(Math.log2(height));
+/**
+ * Coerce any positive number to the nearest power of two within [minPow, maxPow]
+ * Example: nearestPowerOfTwo(13) => 16; nearestPowerOfTwo(3.2) => 4
+ */
+export function nearestPowerOfTwo(n, minPow = 0, // 2^0 = 1
+maxPow = 4) {
+    if (n <= 0) {
+        return 1;
+    }
+    const k = clamping(minPow, maxPow)(calculateMaxBlurPass(n));
+    return 2 ** k;
+}
 /**
  * Calculate the coordinates and size of the source and destination to fit the
  * target aspect ratio.
@@ -171,3 +209,161 @@ export const handleWebGLContextLoss = (canvas, eventHandlers, contextLostCb) =>
         canvas.removeEventListener('webglcontextcreationerror', handleContextCreationError);
     };
 };
+/**
+ * Intersection over Union (IoU), how much two masks overlap as a fraction of either's total.
+ * 1.0 = perfect match, 0 = no overlap.
+ * Used for stable, robust tracking of the main person between frames,
+ * especially with bystander removal and temporal instance switching.
+ */
+export function iou(a, b) {
+    let inter = 0;
+    let union = 0;
+    for (let i = 0; i < a.length; i++) {
+        // @ts-expect-error TypeArray will not be undefined
+        const av = a[i] >= 128;
+        // @ts-expect-error TypeArray will not be undefined
+        const bv = b[i] >= 128;
+        if (av && bv) {
+            inter++;
+        }
+        if (av || bv) {
+            union++;
+        }
+    }
+    return union ? inter / union : 0;
+}
+export function labelComponents(input, labels, width, height) {
+    // 8-connected component labeling with union-find
+    let nextLabel = 1;
+    const parent = [0];
+    function find(x) {
+        if (parent[x] === undefined) {
+            return x;
+        }
+        if (parent[x] !== x) {
+            parent[x] = find(parent[x]);
+        }
+        return parent[x];
+    }
+    function unite(a, b) {
+        if (!a || !b) {
+            return;
+        }
+        a = find(a);
+        b = find(b);
+        if (a !== b) {
+            parent[b] = a; // Or vice versa, as long as consistent
+        }
+    }
+    // First pass: assign and union with neighbors (left, top, top-left, top-right)
+    for (let y = 0; y < height; y++) {
+        for (let x = 0; x < width; x++) {
+            const i = y * width + x;
+            //@ts-expect-error will not be undefined
+            if (input[i] < 128) {
+                labels[i] = 0;
+                continue;
+            }
+            const left = x > 0 ? labels[i - 1] : 0;
+            const top = y > 0 ? labels[i - width] : 0;
+            const topLeft = x > 0 && y > 0 ? labels[i - width - 1] : 0;
+            const topRight = x < width - 1 && y > 0 ? labels[i - width + 1] : 0;
+            // Pick an existing neighbor label if present, else new
+            let assign = left || top || topLeft || topRight;
+            if (!assign) {
+                assign = nextLabel++;
+                parent[assign] = assign;
+            }
+            labels[i] = assign;
+            // Union all differing neighbors with the assigned label
+            if (left && left !== assign) {
+                unite(assign, left);
+            }
+            if (top && top !== assign) {
+                unite(assign, top);
+            }
+            if (topLeft && topLeft !== assign) {
+                unite(assign, topLeft);
+            }
+            if (topRight && topRight !== assign) {
+                unite(assign, topRight);
+            }
+        }
+    }
+    // Second pass: compress labels to root ids
+    for (let i = 0; i < labels.length; i++) {
+        const l = labels[i];
+        labels[i] = l ? find(l) : 0;
+    }
+    // Compute component stats for selection
+    const stats = new Map();
+    for (let y = 0; y < height; y++) {
+        for (let x = 0; x < width; x++) {
+            const l = labels[y * width + x];
+            if (!l) {
+                continue;
+            }
+            const s = stats.get(l) ?? {
+                sumX: 0,
+                sumY: 0,
+                count: 0,
+                minX: width,
+                minY: height,
+                maxX: -1,
+                maxY: -1,
+            };
+            s.sumX += x;
+            s.sumY += y;
+            s.count++;
+            if (x < s.minX) {
+                s.minX = x;
+            }
+            if (x > s.maxX) {
+                s.maxX = x;
+            }
+            if (y < s.minY) {
+                s.minY = y;
+            }
+            if (y > s.maxY) {
+                s.maxY = y;
+            }
+            stats.set(l, s);
+        }
+    }
+    return stats;
+}
+export function scoreComponent(stats, overlap, weights, width, height) {
+    const area = stats.count;
+    const heightPx = stats.maxY - stats.minY + 1;
+    // const widthPx = stats.maxX - stats.minX + 1;
+    const mx = stats.sumX / stats.count;
+    const my = stats.sumY / stats.count;
+    const d2 = (mx - weights.centerX) ** 2 + (my - weights.centerY) ** 2;
+    const centerScore = 1.0 - Math.min(1.0, Math.sqrt(d2) / Math.max(width, height)); // normalize
+    // Normalize area/height by frame size to [0..1]
+    const areaN = area / (width * height);
+    const heightN = heightPx / height;
+    const score = weights.iou * overlap +
+        weights.area * areaN +
+        weights.height * heightN +
+        weights.center * centerScore;
+    return score;
+}
+export const cloneImageRecord = async (image) => {
+    const cloned = await createImageBitmap(image.image);
+    return { image: cloned, key: image.key };
+};
+/**
+ * Compare provided ImageRecord
+ *
+ * @returns true if the two records have the same key and image, false otherwise
+ */
+export const compareImageRecords = (a, b) => {
+    if (!b) {
+        return false;
+    }
+    if (a.key !== b.key || a.image !== b.image) {
+        return false;
+    }
+    return true;
+};

package/dist/main/audio.js

@@ -341,8 +341,8 @@ function createBaseAudioNode(name, create, release) {
  * @param shouldResetEnabled - Whether or not to enable the cloned track
  */
 export const createStreamSourceGraphNode = (mediaStream, shouldResetEnabled = true) => {
-    let stream = undefined;
-    let unsubscribeUnmutes = undefined;
+    let stream;
+    let unsubscribeUnmutes;
     return createBaseAudioNode('source', context => {
         stream = createMediaStreamAudioClone(mediaStream);
         // Enabled the track for the cloned stream
@@ -448,7 +448,7 @@ export const createAnalyzerGraphNode = (options) => {
  * Create a MediaStreamAudioDestinationNodeInit
  */
 export const createStreamDestinationGraphNode = (options) => {
-    let node = undefined;
+    let node;
     return createBaseAudioNode('destination', context => {
         node = createMediaStreamAudioDestinationNode(context, options);
         return [node, node];

package/dist/main/benchUtils.d.ts

@@ -2,12 +2,44 @@ import type { Clock } from './types';
 type FPS = number;
 type Timestamp = number;
 export interface Benchmark {
-    begin(): void;
-    end(): void;
+    begin(timestamp?: number): void;
+    end(timestamp?: number): void;
+    reset(): void;
     calculateFps(): [Timestamp, FPS];
 }
 interface BenchmarkOptions {
     calculationThresholdMS?: number;
+    warmupCount?: number;
+    averageThreshold?: number;
+    bufferSize?: number;
 }
-export declare const createBenchmark: (clock?: Clock, { calculationThresholdMS }?: BenchmarkOptions) => Benchmark;
+/**
+ * Creates a benchmarking utility for measuring frame durations and calculating frames per second (FPS).
+ *
+ * @remarks
+ * This function is designed for performance measurement in animation or game loops.
+ * It records frame start/end times, ignores an initial warmup period, utilizes a circular buffer
+ * to average recent frame durations, and computes a smoothed FPS value on demand.
+ * The calculation may only occur if sufficient samples have been gathered and the minimum interval between
+ * calculations has elapsed.
+ *
+ * @param clock - An object with a `now()` method returning the current timestamp (in milliseconds). Defaults to `window.performance`.
+ * @param options - Optional configuration options for the benchmark utility.
+ * @param options.bufferSize - Size of the circular buffer used to store recent frame durations. Defaults to 1024.
+ * @param options.calculationThresholdMS - Minimum interval in milliseconds between FPS recalculations. Defaults to 1000.
+ * @param options.warmupCount - Number of initial frames to ignore before starting to record durations. Defaults to 100.
+ * @param options.averageThreshold - Minimum number of samples in the buffer required before calculating FPS. Defaults to 100.
+ *
+ * @returns An object with methods to start/end frame measurements, reset the benchmark, and calculate current FPS.
+ */
+export declare const createBenchmark: (clock?: Clock, { bufferSize, calculationThresholdMS, warmupCount, averageThreshold, }?: BenchmarkOptions) => Benchmark;
+export declare const createWindowedStats: (windowSize?: number) => {
+    add(value: number): void;
+    readonly mean: number;
+    readonly variance: number;
+    readonly stddev: number;
+    readonly stdError: number;
+    readonly count: number;
+    clear(): void;
+};
 export {};

package/dist/main/benchUtils.js

@@ -1,35 +1,169 @@
-export const createBenchmark = (clock = performance, { calculationThresholdMS = 1000 } = {}) => {
+import { createCircularBuffer } from '@pexip/utils';
+/**
+ * Creates a benchmarking utility for measuring frame durations and calculating frames per second (FPS).
+ *
+ * @remarks
+ * This function is designed for performance measurement in animation or game loops.
+ * It records frame start/end times, ignores an initial warmup period, utilizes a circular buffer
+ * to average recent frame durations, and computes a smoothed FPS value on demand.
+ * The calculation may only occur if sufficient samples have been gathered and the minimum interval between
+ * calculations has elapsed.
+ *
+ * @param clock - An object with a `now()` method returning the current timestamp (in milliseconds). Defaults to `window.performance`.
+ * @param options - Optional configuration options for the benchmark utility.
+ * @param options.bufferSize - Size of the circular buffer used to store recent frame durations. Defaults to 1024.
+ * @param options.calculationThresholdMS - Minimum interval in milliseconds between FPS recalculations. Defaults to 1000.
+ * @param options.warmupCount - Number of initial frames to ignore before starting to record durations. Defaults to 100.
+ * @param options.averageThreshold - Minimum number of samples in the buffer required before calculating FPS. Defaults to 100.
+ *
+ * @returns An object with methods to start/end frame measurements, reset the benchmark, and calculate current FPS.
+ */
+export const createBenchmark = (clock = performance, { bufferSize = 2 ** 10, calculationThresholdMS = 1000, warmupCount = 100, averageThreshold = 100, } = {}) => {
     const props = {
+        buffer: createCircularBuffer(bufferSize, Float32Array),
         beginTime: 0,
         endTime: 0,
-        sumDelta: 0,
-        count: 0,
         lastCalculateTime: 0,
         lastResult: 0,
+        warmupCount: 0,
+        sum: 0,
     };
     return {
-        begin: () => {
-            props.beginTime = clock.now();
+        begin: (timestamp = clock.now()) => {
+            if (props.warmupCount < warmupCount) {
+                return;
+            }
+            props.beginTime = timestamp;
+        },
+        end: (timestamp = clock.now()) => {
+            if (props.warmupCount < warmupCount) {
+                props.warmupCount += 1;
+                return;
+            }
+            props.endTime = timestamp;
+            if (props.buffer.isFull()) {
+                const oldest = props.buffer.peek() ?? 0;
+                props.sum -= oldest;
+            }
+            const elapsed = props.endTime - props.beginTime;
+            props.buffer.push(elapsed);
+            props.sum += elapsed;
         },
-        end: () => {
-            props.endTime = clock.now();
-            props.sumDelta += props.endTime - props.beginTime;
-            ++props.count;
+        reset: () => {
+            props.buffer.clear();
+            props.warmupCount = 0;
+            props.beginTime = 0;
+            props.endTime = 0;
+            props.lastCalculateTime = 0;
         },
         calculateFps: () => {
+            if (props.warmupCount < warmupCount) {
+                return [props.lastCalculateTime, props.lastResult];
+            }
             const calculateTime = clock.now();
             const lastDuration = calculateTime - props.lastCalculateTime;
-            if (lastDuration >= calculationThresholdMS) {
-                const result = props.count === 0
-                    ? 0
-                    : (lastDuration * props.count) / props.sumDelta;
-                props.sumDelta = 0;
-                props.count = 0;
+            if (props.lastCalculateTime &&
+                lastDuration >= calculationThresholdMS &&
+                props.buffer.size() > averageThreshold) {
+                const result = (lastDuration * props.buffer.size()) / props.sum;
                 props.lastCalculateTime = calculateTime;
                 props.lastResult = result;
                 return [calculateTime, result];
             }
+            props.lastCalculateTime = calculateTime;
             return [props.lastCalculateTime, props.lastResult];
         },
     };
 };
+function mean(sum, count) {
+    if (count > 0) {
+        return sum / count;
+    }
+    return 0;
+}
+function variance(count, mean, sumSquare) {
+    if (count < 2) {
+        return 0;
+    }
+    return sumSquare / count - mean * mean;
+}
+function stddev(variance) {
+    return Math.sqrt(variance);
+}
+function stdError(count, stddev) {
+    if (count === 0) {
+        return 0;
+    }
+    return stddev / Math.sqrt(count);
+}
+export const createWindowedStats = (windowSize = 2 ** 10) => {
+    const props = {
+        buffer: createCircularBuffer(windowSize, Float32Array),
+        sum: 0,
+        sumSqr: 0,
+    };
+    const cache = {
+        mean: 0,
+        variance: 0,
+        stddev: 0,
+        stdError: 0,
+    };
+    const dirty = {
+        mean: false,
+        variance: false,
+        stddev: false,
+        stdError: false,
+    };
+    return {
+        add(value) {
+            if (props.buffer.isFull()) {
+                const oldest = props.buffer.peek() ?? 0;
+                props.sum -= oldest;
+                props.sumSqr -= oldest * oldest;
+            }
+            props.buffer.push(value);
+            props.sum += value;
+            props.sumSqr += value * value;
+            dirty.mean = true;
+            dirty.variance = true;
+            dirty.stddev = true;
+            dirty.stdError = true;
+        },
+        get mean() {
+            if (dirty.mean) {
+                cache.mean = mean(props.sum, this.count);
+                dirty.mean = false;
+            }
+            return cache.mean;
+        },
+        get variance() {
+            if (dirty.variance) {
+                cache.variance = variance(this.count, this.mean, props.sumSqr);
+                dirty.variance = false;
+            }
+            return cache.variance;
+        },
+        get stddev() {
+            if (dirty.stddev) {
+                cache.stddev = stddev(this.variance);
+                dirty.stddev = false;
+            }
+            return cache.stddev;
+        },
+        get stdError() {
+            if (dirty.stdError) {
+                cache.stdError = stdError(this.count, this.stddev);
+                dirty.stdError = false;
+            }
+            return cache.stdError;
+        },
+        get count() {
+            return props.buffer.size();
+        },
+        clear() {
+            props.buffer.clear();
+            props.sum = 0;
+            props.sumSqr = 0;
+        },
+    };
+};

package/dist/main/index.d.ts

@@ -10,7 +10,7 @@ export * from './types';
 export * from './math';
 export * from './process';
 export * from './typeGuards';
-export { createAsyncCallbackLoop } from './utils';
+export { createAsyncCallbackLoop, MediaStreamTrackTransferable } from './utils';
 export * from './benchUtils';
 export declare const urls: {
     denoise: () => URL;

package/dist/main/index.js

@@ -10,7 +10,7 @@ export * from './types';
 export * from './math';
 export * from './process';
 export * from './typeGuards';
-export { createAsyncCallbackLoop } from './utils';
+export { createAsyncCallbackLoop, MediaStreamTrackTransferable } from './utils';
 export * from './benchUtils';
 export const urls = {
     denoise: () => new URL('../worklets/denoise.worklet.js', import.meta.url),