@pexip/media-processor 22.2.0 → 22.3.0

package/CHANGELOG.md

@@ -1,5 +1,25 @@
 # Changelog
 
+## 22.3.0
+
+### Minor Changes
+
+- 2f3c6b7: Add pure CPU processing capability
+
+### Patch Changes
+
+- b666166: Shares one WebGPU adapter and device across the renderers, and
+  destroys the device when the processor is destroyed. One of each was requested
+  for every renderer, so each rebuild of it, on a change of the processing
+  dimensions or the backend or on a recovery from a lost context, left the
+  memory of the pipelines and the shader modules of the previous one allocated.
+  Rebuilding a renderer no longer waits for a device either.
+- 53af4ef: Relinquishes the WebGL context of the processing canvases when they
+  are replaced. A context is bound to its canvas for life, so restarting the
+  video processing repeatedly, e.g. by changing the effect, could exhaust the
+  contexts available to the page, at which point the browser drops the oldest
+  ones and the video feed stops.
+
 ## 22.2.0
 
 ### Minor Changes

package/README.md

@@ -178,6 +178,40 @@ You can do it with chrome, [See here][profiling web audio].
 - No `fetch` API in the `AudioWorkletGlobalScope`
 - No `TextEncoder/Decoder` APIs in the `AudioWorkletGlobalScope`
 
+## CPU pipeline (LiteRT + Canvas2D)
+
+By default the media processor uses MediaPipe with a WebGL or WebGPU delegate
+for GPU-accelerated background segmentation. On devices without GPU support, or
+when you explicitly want a pure-CPU path, you can switch to the LiteRT inference
+pipeline.
+
+Provide `litertjsCoreBasePath` instead of (or in addition to) `basePath` when
+opening the processor. Its presence is the switch that selects the LiteRT
+pipeline:
+
+```typescript
+processor.open({
+  litertjsCoreBasePath: urls.litertjsCoreBasePath, // path to @litertjs/core/wasm/
+  imageSegmenterOptions: {
+    modelAsset: {
+      path: urls.selfieModelPath, // path to a .tflite selfie model
+      modelName: 'selfie',
+    },
+  },
+});
+```
+
+The `@litertjs/core` WASM files are bundled inside `@pexip/media-assets` under
+`@litertjs/core/wasm/`. Point `litertjsCoreBasePath` at this directory relative
+to your web root.
+
+### Supported models
+
+| Pipeline            | Models                | Renderer      |
+| ------------------- | --------------------- | ------------- |
+| MediaPipe (default) | `selfie`, `deeplabV3` | WebGL, WebGPU |
+| LiteRT CPU          | `selfie`              | Canvas2D      |
+
 ## References
 
 - [MDN - Web Audio API][web audio api]

package/api-docs/README.mdx

@@ -178,6 +178,40 @@ You can do it with chrome, [See here][profiling web audio].
 - No `fetch` API in the `AudioWorkletGlobalScope`
 - No `TextEncoder/Decoder` APIs in the `AudioWorkletGlobalScope`
 
+## CPU pipeline (LiteRT + Canvas2D)
+
+By default the media processor uses MediaPipe with a WebGL or WebGPU delegate
+for GPU-accelerated background segmentation. On devices without GPU support, or
+when you explicitly want a pure-CPU path, you can switch to the LiteRT inference
+pipeline.
+
+Provide `litertjsCoreBasePath` instead of (or in addition to) `basePath` when
+opening the processor. Its presence is the switch that selects the LiteRT
+pipeline:
+
+```typescript
+processor.open({
+  litertjsCoreBasePath: urls.litertjsCoreBasePath, // path to @litertjs/core/wasm/
+  imageSegmenterOptions: {
+    modelAsset: {
+      path: urls.selfieModelPath, // path to a .tflite selfie model
+      modelName: 'selfie',
+    },
+  },
+});
+```
+
+The `@litertjs/core` WASM files are bundled inside `@pexip/media-assets` under
+`@litertjs/core/wasm/`. Point `litertjsCoreBasePath` at this directory relative
+to your web root.
+
+### Supported models
+
+| Pipeline            | Models                | Renderer      |
+| ------------------- | --------------------- | ------------- |
+| MediaPipe (default) | `selfie`, `deeplabV3` | WebGL, WebGPU |
+| LiteRT CPU          | `selfie`              | Canvas2D      |
+
 ## References
 
 - [MDN - Web Audio API][web audio api]
@@ -234,6 +268,7 @@ A library for media analysis using Web APIs.
 | [Stats](interfaces/Stats.mdx) | - |
 | [Weights](interfaces/Weights.mdx) | - |
 | [SelectionOptions](interfaces/SelectionOptions.mdx) | - |
+| [TensorMetadata](interfaces/TensorMetadata.mdx) | - |
 | [Benchmark](interfaces/Benchmark.mdx) | - |
 | [Point](interfaces/Point.mdx) | Interface for Point consist of coordinates x and y |
 | [Size](interfaces/Size.mdx) | - |
@@ -284,6 +319,7 @@ A library for media analysis using Web APIs.
 | [IncludeMessageEventDataType](type-aliases/IncludeMessageEventDataType.mdx) | - |
 | [TupleOf](type-aliases/TupleOf.mdx) | From https://github.com/Microsoft/TypeScript/issues/26223#issuecomment-674500430 |
 | [OptionalKeys](type-aliases/OptionalKeys.mdx) | - |
+| [TensorLayout](type-aliases/TensorLayout.mdx) | - |
 | [Canvas](type-aliases/Canvas.mdx) | - |
 | [Unsubscribe](type-aliases/Unsubscribe.mdx) | Unsubscribe the subscription |
 | [AudioBufferFloats](type-aliases/AudioBufferFloats.mdx) | Same as [AudioBuffer](https://developer.mozilla.org/en-US/docs/Web/API/AudioBuffer) Or the return from [AnalyserNode.getFloatFrequencyData()](https://developer.mozilla.org/en-US/docs/Web/API/AnalyserNode/getFloatFrequencyData) |
@@ -372,6 +408,7 @@ A library for media analysis using Web APIs.
 | [cloneImageRecord](variables/cloneImageRecord.mdx) | - |
 | [compareImageRecords](variables/compareImageRecords.mdx) | Compare provided ImageRecord |
 | [getCanUseWebGL](variables/getCanUseWebGL.mdx) | - |
+| [getTensorMetadata](variables/getTensorMetadata.mdx) | - |
 | [urls](variables/urls.mdx) | - |
 | [SILENT\_THRESHOLD](variables/SILENT_THRESHOLD.mdx) | Default silent threshold At least one LSB 16-bit data (compare is on absolute value). |
 | [MONO\_THRESHOLD](variables/MONO_THRESHOLD.mdx) | Default mono detection threshold Data must be identical within one LSB 16-bit to be identified as mono. |

package/api-docs/interfaces/ProcessorEvent.mdx

@@ -6,3 +6,4 @@
 | <a id="error"></a> `error?` | `Error` |
 | <a id="message"></a> `message?` | `string` |
 | <a id="track"></a> `track?` | `MediaStreamTrack` |
+| <a id="recoverable"></a> `recoverable?` | `boolean` |

package/api-docs/interfaces/ProcessorOptions.mdx

@@ -4,12 +4,13 @@
 
 ## Properties
 
-| Property | Type | Inherited from |
-| ------ | ------ | ------ |
-| <a id="renderoptions"></a> `renderOptions?` | [`RendererOptions`](RendererOptions.mdx) | - |
-| <a id="backgroundimage"></a> `backgroundImage?` | [`ImageRecord`](ImageRecord.mdx) | - |
-| <a id="restart"></a> `restart?` | `boolean` | - |
-| <a id="basepath"></a> `basePath?` | `string` | [`SegmenterOptions`](SegmenterOptions.mdx).[`basePath`](SegmenterOptions.mdx#basepath) |
-| <a id="processingwidth"></a> `processingWidth?` | `number` | [`SegmenterOptions`](SegmenterOptions.mdx).[`processingWidth`](SegmenterOptions.mdx#processingwidth) |
-| <a id="processingheight"></a> `processingHeight?` | `number` | [`SegmenterOptions`](SegmenterOptions.mdx).[`processingHeight`](SegmenterOptions.mdx#processingheight) |
-| <a id="imagesegmenteroptions"></a> `imageSegmenterOptions?` | [`ImageSegmenterOptions`](ImageSegmenterOptions.mdx) | [`SegmenterOptions`](SegmenterOptions.mdx).[`imageSegmenterOptions`](SegmenterOptions.mdx#imagesegmenteroptions) |
+| Property | Type | Description | Inherited from |
+| ------ | ------ | ------ | ------ |
+| <a id="renderoptions"></a> `renderOptions?` | [`RendererOptions`](RendererOptions.mdx) | - | - |
+| <a id="backgroundimage"></a> `backgroundImage?` | [`ImageRecord`](ImageRecord.mdx) | - | - |
+| <a id="restart"></a> `restart?` | `boolean` | - | - |
+| <a id="basepath"></a> `basePath?` | `string` | MediaPipe fileset base URL. | [`SegmenterOptions`](SegmenterOptions.mdx).[`basePath`](SegmenterOptions.mdx#basepath) |
+| <a id="litertjscorebasepath"></a> `litertjsCoreBasePath?` | `string` | LiteRT WASM base URL. Providing this selects the CPU inference pipeline (LiteRT + Canvas2D) instead of the default MediaPipe GPU pipeline. | [`SegmenterOptions`](SegmenterOptions.mdx).[`litertjsCoreBasePath`](SegmenterOptions.mdx#litertjscorebasepath) |
+| <a id="processingwidth"></a> `processingWidth?` | `number` | - | [`SegmenterOptions`](SegmenterOptions.mdx).[`processingWidth`](SegmenterOptions.mdx#processingwidth) |
+| <a id="processingheight"></a> `processingHeight?` | `number` | - | [`SegmenterOptions`](SegmenterOptions.mdx).[`processingHeight`](SegmenterOptions.mdx#processingheight) |
+| <a id="imagesegmenteroptions"></a> `imageSegmenterOptions?` | [`ImageSegmenterOptions`](ImageSegmenterOptions.mdx) | - | [`SegmenterOptions`](SegmenterOptions.mdx).[`imageSegmenterOptions`](SegmenterOptions.mdx#imagesegmenteroptions) |

package/api-docs/interfaces/ProcessorUpdateOptions.mdx

@@ -6,12 +6,13 @@
 
 ## Properties
 
-| Property | Type | Inherited from |
-| ------ | ------ | ------ |
-| <a id="backgroundimage"></a> `backgroundImage?` | [`ImageRecord`](ImageRecord.mdx) | [`ProcessorOptions`](ProcessorOptions.mdx).[`backgroundImage`](ProcessorOptions.mdx#backgroundimage) |
-| <a id="restart"></a> `restart?` | `boolean` | [`ProcessorOptions`](ProcessorOptions.mdx).[`restart`](ProcessorOptions.mdx#restart) |
-| <a id="basepath"></a> `basePath?` | `string` | [`SegmenterOptions`](SegmenterOptions.mdx).[`basePath`](SegmenterOptions.mdx#basepath) |
-| <a id="processingwidth"></a> `processingWidth?` | `number` | [`SegmenterOptions`](SegmenterOptions.mdx).[`processingWidth`](SegmenterOptions.mdx#processingwidth) |
-| <a id="processingheight"></a> `processingHeight?` | `number` | [`SegmenterOptions`](SegmenterOptions.mdx).[`processingHeight`](SegmenterOptions.mdx#processingheight) |
-| <a id="imagesegmenteroptions"></a> `imageSegmenterOptions?` | [`ImageSegmenterOptions`](ImageSegmenterOptions.mdx) | [`SegmenterOptions`](SegmenterOptions.mdx).[`imageSegmenterOptions`](SegmenterOptions.mdx#imagesegmenteroptions) |
-| <a id="renderoptions"></a> `renderOptions?` | `Partial`\<[`RendererOptions`](RendererOptions.mdx)\> | - |
+| Property | Type | Description | Inherited from |
+| ------ | ------ | ------ | ------ |
+| <a id="backgroundimage"></a> `backgroundImage?` | [`ImageRecord`](ImageRecord.mdx) | - | [`ProcessorOptions`](ProcessorOptions.mdx).[`backgroundImage`](ProcessorOptions.mdx#backgroundimage) |
+| <a id="restart"></a> `restart?` | `boolean` | - | [`ProcessorOptions`](ProcessorOptions.mdx).[`restart`](ProcessorOptions.mdx#restart) |
+| <a id="basepath"></a> `basePath?` | `string` | MediaPipe fileset base URL. | [`SegmenterOptions`](SegmenterOptions.mdx).[`basePath`](SegmenterOptions.mdx#basepath) |
+| <a id="litertjscorebasepath"></a> `litertjsCoreBasePath?` | `string` | LiteRT WASM base URL. Providing this selects the CPU inference pipeline (LiteRT + Canvas2D) instead of the default MediaPipe GPU pipeline. | [`SegmenterOptions`](SegmenterOptions.mdx).[`litertjsCoreBasePath`](SegmenterOptions.mdx#litertjscorebasepath) |
+| <a id="processingwidth"></a> `processingWidth?` | `number` | - | [`SegmenterOptions`](SegmenterOptions.mdx).[`processingWidth`](SegmenterOptions.mdx#processingwidth) |
+| <a id="processingheight"></a> `processingHeight?` | `number` | - | [`SegmenterOptions`](SegmenterOptions.mdx).[`processingHeight`](SegmenterOptions.mdx#processingheight) |
+| <a id="imagesegmenteroptions"></a> `imageSegmenterOptions?` | [`ImageSegmenterOptions`](ImageSegmenterOptions.mdx) | - | [`SegmenterOptions`](SegmenterOptions.mdx).[`imageSegmenterOptions`](SegmenterOptions.mdx#imagesegmenteroptions) |
+| <a id="renderoptions"></a> `renderOptions?` | `Partial`\<[`RendererOptions`](RendererOptions.mdx)\> | - | - |

package/api-docs/interfaces/SegmenterOptions.mdx

@@ -4,9 +4,10 @@
 
 ## Properties
 
-| Property | Type |
-| ------ | ------ |
-| <a id="basepath"></a> `basePath?` | `string` |
-| <a id="processingwidth"></a> `processingWidth?` | `number` |
-| <a id="processingheight"></a> `processingHeight?` | `number` |
-| <a id="imagesegmenteroptions"></a> `imageSegmenterOptions?` | [`ImageSegmenterOptions`](ImageSegmenterOptions.mdx) |
+| Property | Type | Description |
+| ------ | ------ | ------ |
+| <a id="basepath"></a> `basePath?` | `string` | MediaPipe fileset base URL. |
+| <a id="litertjscorebasepath"></a> `litertjsCoreBasePath?` | `string` | LiteRT WASM base URL. Providing this selects the CPU inference pipeline (LiteRT + Canvas2D) instead of the default MediaPipe GPU pipeline. |
+| <a id="processingwidth"></a> `processingWidth?` | `number` | - |
+| <a id="processingheight"></a> `processingHeight?` | `number` | - |
+| <a id="imagesegmenteroptions"></a> `imageSegmenterOptions?` | [`ImageSegmenterOptions`](ImageSegmenterOptions.mdx) | - |

package/api-docs/interfaces/TensorMetadata.mdx

@@ -0,0 +1,11 @@
+## Properties
+
+| Property | Type |
+| ------ | ------ |
+| <a id="name"></a> `name` | `string` |
+| <a id="shape"></a> `shape` | readonly `number[]` |
+| <a id="width"></a> `width` | `number` |
+| <a id="height"></a> `height` | `number` |
+| <a id="layout"></a> `layout` | [`TensorLayout`](../type-aliases/TensorLayout.mdx) |
+| <a id="dtype"></a> `dtype` | `"float32" \| "int32" \| "uint8"` |
+| <a id="channels"></a> `channels` | `number` |

package/api-docs/type-aliases/TensorLayout.mdx

@@ -0,0 +1,3 @@
+```ts
+type TensorLayout = "nhwc" | "nchw";
+```

package/api-docs/variables/getTensorMetadata.mdx

@@ -0,0 +1,14 @@
+```ts
+const getTensorMetadata: (details, supportedChannels?) => TensorMetadata;
+```
+
+## Parameters
+
+| Parameter | Type |
+| ------ | ------ |
+| `details` | `TensorDetails` |
+| `supportedChannels?` | readonly `number[]` |
+
+## Returns
+
+[`TensorMetadata`](../interfaces/TensorMetadata.mdx)

package/dist/common/backends/canvas2d/postprocessor.d.ts

@@ -0,0 +1,30 @@
+import type { TypedArray, TensorDetails, Tensor } from '../../inferencer';
+import type { TensorMetadata } from '../../utils';
+/**
+ * Extract person confidence into an 8-bit mask.
+ *
+ * For two-channel outputs, background is category 0 and person is category 1.
+ *
+ * @example
+ * ```ts
+ * extractPersonConfidence(new Float32Array([0.1, 0.9]), mask, metadata);
+ * ```
+ */
+export declare const extractPersonConfidence: (source: TypedArray, target: Uint8ClampedArray, metadata: TensorMetadata) => Uint8ClampedArray;
+/**
+ * Create a reusable converter from LiteRT selfie output to a mask canvas.
+ *
+ * @example
+ * ```ts
+ * const output = createSelfieOutputProcessor(details, 256, 144);
+ * const maskCanvas = await output.process(inferenceOutputs);
+ * ```
+ */
+export declare const createSelfieOutputProcessor: (output: TensorDetails, createCanvas?: (width: number, height: number) => OffscreenCanvas, createImageData?: (width: number, height: number) => ImageData) => {
+    metadata: TensorMetadata;
+    process: (inferenceOutputs: Record<string, Tensor>) => Promise<OffscreenCanvas>;
+    release: () => void;
+    readonly canvas: OffscreenCanvas | undefined;
+    readonly maskBuffer: Uint8ClampedArray<ArrayBufferLike> | undefined;
+    readonly imageData: ImageData | undefined;
+};

package/dist/common/backends/canvas2d/postprocessor.js

@@ -0,0 +1,88 @@
+import { getTensorMetadata } from '../../utils';
+import { createOffscreenCanvas, getCanvasRenderingContext2D, } from '../../canvasRenderUtils';
+/**
+ * Extract person confidence into an 8-bit mask.
+ *
+ * For two-channel outputs, background is category 0 and person is category 1.
+ *
+ * @example
+ * ```ts
+ * extractPersonConfidence(new Float32Array([0.1, 0.9]), mask, metadata);
+ * ```
+ */
+export const extractPersonConfidence = (source, target, metadata) => {
+    const pixelCount = metadata.width * metadata.height;
+    if (target.length !== pixelCount ||
+        source.length !== pixelCount * metadata.channels) {
+        throw new Error('Output tensor and mask buffer sizes do not match');
+    }
+    const isFloat = source instanceof Float32Array;
+    for (let pixelIndex = 0; pixelIndex < pixelCount; pixelIndex++) {
+        const sourceIndex = metadata.layout === 'nhwc'
+            ? pixelIndex * metadata.channels + metadata.channels - 1
+            : (metadata.channels - 1) * pixelCount + pixelIndex;
+        const confidence = source[sourceIndex] ?? 0;
+        target[pixelIndex] = isFloat
+            ? Math.round(Math.min(1, Math.max(0, confidence)) * 255)
+            : confidence;
+    }
+    return target;
+};
+/**
+ * Create a reusable converter from LiteRT selfie output to a mask canvas.
+ *
+ * @example
+ * ```ts
+ * const output = createSelfieOutputProcessor(details, 256, 144);
+ * const maskCanvas = await output.process(inferenceOutputs);
+ * ```
+ */
+export const createSelfieOutputProcessor = (output, createCanvas = (width, height) => createOffscreenCanvas(width, height), createImageData = (width, height) => new ImageData(width, height)) => {
+    const metadata = getTensorMetadata(output, [1, 2]);
+    let canvas = createCanvas(metadata.width, metadata.height);
+    let context = getCanvasRenderingContext2D(canvas);
+    let maskBuffer = new Uint8ClampedArray(metadata.width * metadata.height);
+    let imageData = createImageData(metadata.width, metadata.height);
+    let rgbaBuffer = imageData.data;
+    const process = async (inferenceOutputs) => {
+        if (!canvas || !context || !maskBuffer || !rgbaBuffer || !imageData) {
+            throw new Error('The output processor has been released');
+        }
+        const output = inferenceOutputs[metadata.name];
+        if (!output) {
+            throw new Error(`Missing model output: ${metadata.name}`);
+        }
+        const values = await output.data();
+        extractPersonConfidence(values, maskBuffer, metadata);
+        for (let pixelIndex = 0; pixelIndex < maskBuffer.length; pixelIndex++) {
+            rgbaBuffer[pixelIndex * 4 + 3] = maskBuffer[pixelIndex] ?? 0;
+        }
+        context.putImageData(imageData, 0, 0);
+        return canvas;
+    };
+    const release = () => {
+        context = undefined;
+        maskBuffer = undefined;
+        rgbaBuffer = undefined;
+        imageData = undefined;
+        if (canvas) {
+            canvas.width = 0;
+            canvas.height = 0;
+        }
+        canvas = undefined;
+    };
+    return {
+        metadata,
+        process,
+        release,
+        get canvas() {
+            return canvas;
+        },
+        get maskBuffer() {
+            return maskBuffer;
+        },
+        get imageData() {
+            return imageData;
+        },
+    };
+};

package/dist/common/backends/canvas2d/preprocessor.d.ts

@@ -0,0 +1,56 @@
+import { Tensor, type TensorDetails } from '../../inferencer';
+import { type TensorLayout } from '../../utils';
+export { getTensorMetadata } from '../../utils';
+/**
+ * Convert packed RGBA pixels to normalized model RGB input.
+ *
+ * Float inputs use the selfie model metadata normalization `(value - 0) / 255`.
+ * Uint8 inputs retain their original channel values.
+ *
+ * @example
+ * ```ts
+ * convertRgbaToRgbTensor(
+ *   new Uint8Array([255, 128, 0, 255]),
+ *   new Float32Array(3),
+ *   'nhwc',
+ * );
+ * ```
+ */
+export declare const convertRgbaToRgbTensor: (rgba: Uint8Array, target: Float32Array | Uint8Array, layout: TensorLayout) => Float32Array | Uint8Array;
+/**
+ * Calculate a centered crop that keeps model input and output masks aligned.
+ *
+ * @example
+ * ```ts
+ * getModelResizeCoordinates(1280, 720, 256, 256);
+ * ```
+ */
+export declare const getModelResizeCoordinates: (sourceWidth: number, sourceHeight: number, modelWidth: number, modelHeight: number, processingWidth?: number, processingHeight?: number) => {
+    sx: number;
+    sy: number;
+    sw: number;
+    sh: number;
+    dx: number;
+    dy: number;
+    dw: number;
+    dh: number;
+};
+/**
+ * Create a reusable Canvas2D preprocessor for a LiteRT model input.
+ *
+ * @example
+ * ```ts
+ * const preprocessor = createCanvas2dPreprocessor(inputDetails);
+ * const input = await preprocessor.process(videoFrame);
+ * ```
+ */
+export declare const createCanvas2dPreprocessor: (details: TensorDetails, processingWidth?: number, processingHeight?: number) => {
+    metadata: import("../..").TensorMetadata;
+    process: (frame: VideoFrame) => Promise<{
+        [x: string]: Tensor;
+    }>;
+    release: () => void;
+    readonly canvas: OffscreenCanvas | undefined;
+    readonly pixelBuffer: Uint8Array<ArrayBufferLike> | undefined;
+    readonly tensorStorage: Float32Array<ArrayBuffer> | Uint8Array<ArrayBuffer>;
+};

package/dist/common/backends/canvas2d/preprocessor.js

@@ -0,0 +1,139 @@
+import { Tensor } from '../../inferencer';
+import { createOffscreenCanvas, getCanvasRenderingContext2D, } from '../../canvasRenderUtils';
+import { getImageSize } from '../../canvasRenderUtils';
+import { resize, getTensorMetadata } from '../../utils';
+export { getTensorMetadata } from '../../utils';
+/**
+ * Convert packed RGBA pixels to normalized model RGB input.
+ *
+ * Float inputs use the selfie model metadata normalization `(value - 0) / 255`.
+ * Uint8 inputs retain their original channel values.
+ *
+ * @example
+ * ```ts
+ * convertRgbaToRgbTensor(
+ *   new Uint8Array([255, 128, 0, 255]),
+ *   new Float32Array(3),
+ *   'nhwc',
+ * );
+ * ```
+ */
+export const convertRgbaToRgbTensor = (rgba, target, layout) => {
+    const pixelCount = rgba.length / 4;
+    if (!Number.isInteger(pixelCount) || target.length !== pixelCount * 3) {
+        throw new Error('RGBA and RGB tensor buffer sizes do not match');
+    }
+    const normalize = target instanceof Float32Array;
+    for (let pixelIndex = 0; pixelIndex < pixelCount; pixelIndex++) {
+        const rgbaOffset = pixelIndex * 4;
+        for (let channelIndex = 0; channelIndex < 3; channelIndex++) {
+            const targetIndex = layout === 'nhwc'
+                ? pixelIndex * 3 + channelIndex
+                : channelIndex * pixelCount + pixelIndex;
+            const value = rgba[rgbaOffset + channelIndex] ?? 0;
+            target[targetIndex] = normalize ? value / 255 : value;
+        }
+    }
+    return target;
+};
+/**
+ * Calculate a centered crop that keeps model input and output masks aligned.
+ *
+ * @example
+ * ```ts
+ * getModelResizeCoordinates(1280, 720, 256, 256);
+ * ```
+ */
+export const getModelResizeCoordinates = (sourceWidth, sourceHeight, modelWidth, modelHeight, processingWidth = modelWidth, processingHeight = modelHeight) => {
+    if (sourceWidth <= 0 ||
+        sourceHeight <= 0 ||
+        modelWidth <= 0 ||
+        modelHeight <= 0 ||
+        processingWidth <= 0 ||
+        processingHeight <= 0) {
+        throw new Error('Source, processing, and model dimensions must be positive');
+    }
+    const sourceCrop = resize(sourceWidth, sourceHeight, processingWidth, processingHeight);
+    const scale = Math.min(modelWidth / processingWidth, modelHeight / processingHeight);
+    const destinationWidth = processingWidth * scale;
+    const destinationHeight = processingHeight * scale;
+    return {
+        sx: sourceCrop.sx,
+        sy: sourceCrop.sy,
+        sw: sourceCrop.sw,
+        sh: sourceCrop.sh,
+        dx: (modelWidth - destinationWidth) / 2,
+        dy: (modelHeight - destinationHeight) / 2,
+        dw: destinationWidth,
+        dh: destinationHeight,
+    };
+};
+/**
+ * Create a reusable Canvas2D preprocessor for a LiteRT model input.
+ *
+ * @example
+ * ```ts
+ * const preprocessor = createCanvas2dPreprocessor(inputDetails);
+ * const input = await preprocessor.process(videoFrame);
+ * ```
+ */
+export const createCanvas2dPreprocessor = (details, processingWidth, processingHeight) => {
+    const metadata = getTensorMetadata(details);
+    let canvas = createOffscreenCanvas(metadata.width, metadata.height);
+    let context = getCanvasRenderingContext2D(canvas, { alpha: false });
+    let pixelBuffer = new Uint8Array(metadata.width * metadata.height * 4);
+    let tensorStorage = metadata.dtype === 'float32'
+        ? new Float32Array(metadata.width * metadata.height * 3)
+        : new Uint8Array(metadata.width * metadata.height * 3);
+    let tensor;
+    const process = async (frame) => {
+        if (!canvas || !context || !pixelBuffer) {
+            throw new Error('The preprocessor has been released');
+        }
+        const sourceSize = getImageSize(frame);
+        const coordinates = getModelResizeCoordinates(sourceSize.width, sourceSize.height, metadata.width, metadata.height, processingWidth ?? metadata.width, processingHeight ?? metadata.height);
+        context.clearRect(0, 0, metadata.width, metadata.height);
+        context.drawImage(frame, coordinates.sx, coordinates.sy, coordinates.sw, coordinates.sh, coordinates.dx, coordinates.dy, coordinates.dw, coordinates.dh);
+        const resizedFrame = new VideoFrame(canvas, {
+            // VideoFrame timestamp is in microseconds
+            // https://developer.mozilla.org/en-US/docs/Web/API/VideoFrame/timestamp
+            timestamp: frame.timestamp ?? performance.now() * 1000,
+        });
+        try {
+            await resizedFrame.copyTo(pixelBuffer, { format: 'RGBA' });
+        }
+        finally {
+            resizedFrame.close();
+        }
+        convertRgbaToRgbTensor(pixelBuffer, tensorStorage, metadata.layout);
+        tensor?.delete();
+        tensor = new Tensor(tensorStorage, Array.from(metadata.shape));
+        return { [metadata.name]: tensor };
+    };
+    const release = () => {
+        tensor?.delete();
+        tensor = undefined;
+        tensorStorage = new Uint8Array();
+        pixelBuffer = undefined;
+        context = undefined;
+        if (canvas) {
+            canvas.width = 0;
+            canvas.height = 0;
+        }
+        canvas = undefined;
+    };
+    return {
+        metadata,
+        process,
+        release,
+        get canvas() {
+            return canvas;
+        },
+        get pixelBuffer() {
+            return pixelBuffer;
+        },
+        get tensorStorage() {
+            return tensorStorage;
+        },
+    };
+};

package/dist/common/backends/webgpu/renderer.js

@@ -272,6 +272,7 @@ export const createRenderer = (adapter, renderEventHandlers, canvas = new Offscr
     };
     const release = () => {
         lazyProps.release();
+        props.device = undefined;
         props.deviceLost = false;
         props.prevMask2 = undefined;
         props.prevMask4 = undefined;

package/dist/common/backends/webgpu/webgpuUtils.d.ts

@@ -1,6 +1,41 @@
 export declare const hasR16FloatTextureStorage: (adapter: GPUAdapter) => boolean;
 export declare const hasF16: (adapter: GPUAdapter) => boolean;
+export declare class GPUDeviceRequestInvalidatedError extends Error {
+    constructor();
+}
+/**
+ * Get the GPU adapter, requesting it only once.
+ *
+ * @example
+ * ```ts
+ * const adapter = await getGPUAdapter();
+ * ```
+ */
+export declare const getGPUAdapter: () => Promise<GPUAdapter | null>;
+/**
+ * Get the GPU device, requesting it only once.
+ *
+ * A device owns the memory of the pipelines and the shader modules created from
+ * it, and neither of those can be freed on their own, so it is shared by every
+ * renderer instead of being requested per renderer.
+ *
+ * @param adapter - The adapter to request the device from
+ *
+ * @example
+ * ```ts
+ * const device = await getGPUDevice(adapter);
+ * ```
+ */
 export declare const getGPUDevice: (adapter: GPUAdapter) => Promise<GPUDevice>;
+/**
+ * Destroy the shared GPU device.
+ *
+ * @example
+ * ```ts
+ * destroyGPUDevice();
+ * ```
+ */
+export declare const destroyGPUDevice: () => void;
 /**
  * Check if provided format is supported
  *