@pexip/media-processor 22.1.0 → 22.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. package/CHANGELOG.md +51 -0
  2. package/README.md +34 -0
  3. package/api-docs/README.mdx +38 -0
  4. package/api-docs/functions/createBenchmark.mdx +7 -6
  5. package/api-docs/interfaces/Benchmark.mdx +3 -5
  6. package/api-docs/interfaces/ProcessorEvent.mdx +1 -0
  7. package/api-docs/interfaces/ProcessorOptions.mdx +10 -9
  8. package/api-docs/interfaces/ProcessorUpdateOptions.mdx +10 -9
  9. package/api-docs/interfaces/RendererOptions.mdx +1 -1
  10. package/api-docs/interfaces/SegmenterOptions.mdx +7 -6
  11. package/api-docs/interfaces/TensorMetadata.mdx +11 -0
  12. package/api-docs/type-aliases/TensorLayout.mdx +3 -0
  13. package/api-docs/variables/RENDER_BACKEND.mdx +1 -0
  14. package/api-docs/variables/getCanUseWebGL.mdx +7 -0
  15. package/api-docs/variables/getTensorMetadata.mdx +14 -0
  16. package/dist/common/backends/canvas2d/postprocessor.d.ts +30 -0
  17. package/dist/common/backends/canvas2d/postprocessor.js +88 -0
  18. package/dist/common/backends/canvas2d/preprocessor.d.ts +56 -0
  19. package/dist/common/backends/canvas2d/preprocessor.js +139 -0
  20. package/dist/common/backends/canvas2d/renderer.d.ts +2 -0
  21. package/dist/common/backends/canvas2d/renderer.js +36 -0
  22. package/dist/common/backends/webgl/blur.d.ts +1 -1
  23. package/dist/common/backends/webgl/blur.js +11 -25
  24. package/dist/common/backends/webgl/renderer.js +4 -4
  25. package/dist/common/backends/webgpu/dualFilterBlur.js +93 -114
  26. package/dist/common/backends/webgpu/renderer.js +1 -0
  27. package/dist/common/backends/webgpu/webgpuUtils.d.ts +35 -0
  28. package/dist/common/backends/webgpu/webgpuUtils.js +84 -2
  29. package/dist/common/canvasRenderUtils.d.ts +23 -4
  30. package/dist/common/canvasRenderUtils.js +48 -11
  31. package/dist/common/constants.d.ts +1 -0
  32. package/dist/common/constants.js +1 -0
  33. package/dist/common/inferencer.d.ts +27 -0
  34. package/dist/common/inferencer.js +106 -0
  35. package/dist/common/tsconfig.tsbuildinfo +1 -1
  36. package/dist/common/types/processor.d.ts +1 -0
  37. package/dist/common/types/segmentation.d.ts +8 -0
  38. package/dist/common/utils.d.ts +13 -0
  39. package/dist/common/utils.js +56 -0
  40. package/dist/main/audio.js +20 -2
  41. package/dist/main/benchUtils.d.ts +28 -18
  42. package/dist/main/benchUtils.js +66 -56
  43. package/dist/main/tsconfig.tsbuildinfo +1 -1
  44. package/dist/main/utils.d.ts +9 -0
  45. package/dist/main/utils.js +12 -0
  46. package/dist/main/video/segmenter.d.ts +4 -2
  47. package/dist/main/video/segmenter.js +36 -11
  48. package/dist/main/video/video.js +13 -1
  49. package/dist/workers/mediaWorker.js +23 -23
  50. package/dist/workers/tsconfig.tsbuildinfo +1 -1
  51. package/package.json +7 -10
@@ -0,0 +1,2 @@
1
+ import type { Renderer, RenderEventHandlers } from '../../types/render';
2
+ export declare const createRenderer: (renderEventHandlers: RenderEventHandlers, canvas?: OffscreenCanvas) => Renderer;
@@ -0,0 +1,36 @@
1
+ import { PROCESSING_HEIGHT, PROCESSING_WIDTH } from '../../constants';
2
+ import { createCanvasRenderUtils } from '../../canvasRenderUtils';
3
+ export const createRenderer = (renderEventHandlers, canvas = new OffscreenCanvas(PROCESSING_WIDTH, PROCESSING_HEIGHT)) => {
4
+ const utils = createCanvasRenderUtils(canvas.width, canvas.height);
5
+ let ctx = null;
6
+ const init = () => {
7
+ ctx = canvas.getContext('2d');
8
+ if (!ctx) {
9
+ renderEventHandlers.contextCreationError('Failed to create 2D rendering context');
10
+ }
11
+ };
12
+ const render = (mask, frame, options) => {
13
+ if (!ctx) {
14
+ return;
15
+ }
16
+ ctx.clearRect(0, 0, canvas.width, canvas.height);
17
+ const inputImage = utils.evaluateInput(frame);
18
+ if (options.effects === 'blur' && options.backgroundBlurAmount) {
19
+ utils.drawBlurEffect(canvas, inputImage, mask, options.foregroundThreshold, options.backgroundBlurAmount, options.edgeBlurAmount);
20
+ }
21
+ else if (options?.effects === 'overlay' && options.backgroundImage) {
22
+ utils.drawOverlayEffect(canvas, inputImage, options.backgroundImage, mask, options.foregroundThreshold, 0, options.edgeBlurAmount);
23
+ }
24
+ };
25
+ const cleanup = () => {
26
+ utils.reset();
27
+ ctx = null;
28
+ };
29
+ const release = () => {
30
+ cleanup();
31
+ };
32
+ const isContextLost = () => {
33
+ return !ctx || ctx.isContextLost();
34
+ };
35
+ return { init, render, release, isContextLost };
36
+ };
@@ -2,7 +2,7 @@ import type { TextureInfo } from './types';
2
2
  /**
3
3
  * Blur provided frame
4
4
  */
5
- export declare const createBlurRenderer: (processor: OffscreenCanvas, pass: number, redOnly?: boolean) => {
5
+ export declare const createBlurRenderer: (processor: OffscreenCanvas, redOnly?: boolean) => {
6
6
  render: (frame: TextureInfo, pass: number) => TextureInfo;
7
7
  release: () => void;
8
8
  };
@@ -5,11 +5,12 @@ import { m3 } from '../../m3';
5
5
  import { createFrameBuffer, createGLProgram, createMaskTexture, createRGBAFrameTexture, } from './webglUtils';
6
6
  import vertexSrc from './shaders/matrix.vert';
7
7
  import fragmentSrc from './shaders/dualFilterBlur.frag';
8
- const clampBlurAmount = clamping(1, 9);
8
+ const MAX_BLUR_LEVEL = 9;
9
+ const clampBlurAmount = clamping(1, MAX_BLUR_LEVEL);
9
10
  /**
10
11
  * Blur provided frame
11
12
  */
12
- export const createBlurRenderer = (processor, pass, redOnly = false) => {
13
+ export const createBlurRenderer = (processor, redOnly = false) => {
13
14
  const { gl, program, attribLocations, uniformLocations, floatSingleChannelSupported, floatFullChannelSupported, release, } = createGLProgram(processor, {
14
15
  attribs: {
15
16
  position: 'a_position',
@@ -62,23 +63,19 @@ export const createBlurRenderer = (processor, pass, redOnly = false) => {
62
63
  };
63
64
  const createFramebufferAndTexture = (idx) => {
64
65
  // Avoid fractional value
65
- const width = Math.trunc(processor.width / 2 ** idx);
66
- const height = Math.trunc(processor.height / 2 ** idx);
66
+ const width = Math.max(Math.trunc(processor.width / 2 ** idx), 1);
67
+ const height = Math.max(Math.trunc(processor.height / 2 ** idx), 1);
67
68
  const texture = redOnly
68
69
  ? createMaskTexture(gl, width, height, true, floatSingleChannelSupported, floatFullChannelSupported)
69
70
  : createRGBAFrameTexture(gl, width, height, true, floatFullChannelSupported);
70
71
  const frameBuffer = createFrameBuffer(gl, texture);
71
72
  return { texture, frameBuffer, width, height };
72
73
  };
73
- const generateBlurBuffers = (pass, offset = 0) => {
74
- const length = Math.max(clampBlurAmount(offset + pass) - offset, 0);
75
- return Array.from({
76
- length,
77
- }).map((_, currentIdx) => createFramebufferAndTexture(offset + currentIdx));
78
- };
79
- // Generate multiple FrameBuffers, textures and the widths and heights are of size
74
+ // Allocate all levels upfront, textures and the widths and heights are of size
80
75
  // of half of the previous one. E.g. 640x480 -> 320x240 -> 160x120 -> 80x60
81
- const blurBuffers = generateBlurBuffers(pass + 1);
76
+ const blurBuffers = Array.from({
77
+ length: MAX_BLUR_LEVEL,
78
+ }).map((_, currentIdx) => createFramebufferAndTexture(currentIdx));
82
79
  return {
83
80
  render: (frame, pass) => {
84
81
  assert(!gl.isContextLost(), RENDERING_EVENTS.ContextLost);
@@ -98,18 +95,7 @@ export const createBlurRenderer = (processor, pass, redOnly = false) => {
98
95
  // Dual filter blur
99
96
  // Downsampling
100
97
  gl.uniform1i(uniformLocations.downsampling, 1);
101
- // Adjust pass number accordingly
102
- const currentPass = pass + 1;
103
- if (currentPass > blurBuffers.length) {
104
- for (const buffer of generateBlurBuffers(currentPass - blurBuffers.length, blurBuffers.length)) {
105
- blurBuffers.push(buffer);
106
- }
107
- }
108
- while (currentPass < blurBuffers.length) {
109
- const buffer = blurBuffers.pop();
110
- gl.deleteTexture(buffer?.texture ?? null);
111
- gl.deleteFramebuffer(buffer?.frameBuffer ?? null);
112
- }
98
+ const currentPass = clampBlurAmount(pass + 1);
113
99
  // Skip the first pass
114
100
  for (let i = 1; i < currentPass; i++) {
115
101
  const { frameBuffer = null, texture = null, width = processor.width, height = processor.height, } = blurBuffers[i] ?? {};
@@ -134,7 +120,7 @@ export const createBlurRenderer = (processor, pass, redOnly = false) => {
134
120
  // Upsampling
135
121
  gl.uniform1i(uniformLocations.downsampling, 0);
136
122
  // Skip the first pass
137
- for (let i = pass - 1; i >= 0; i--) {
123
+ for (let i = currentPass - 2; i >= 0; i--) {
138
124
  const { frameBuffer = null, texture = null, width = processor.width, height = processor.height, } = blurBuffers[i] ?? {};
139
125
  // console.log(`Drawing upsampling pass ${i}, ${width}x${height}`);
140
126
  // Setup to draw to the frame buffer
@@ -1,5 +1,5 @@
1
1
  import { assert } from '@pexip/utils';
2
- import { BACKGROUND_BLUR_AMOUNT, PROCESSING_HEIGHT, PROCESSING_WIDTH, RENDERING_EVENTS, STRONG_EDGE_BLUR_AMOUNT, } from '../../constants';
2
+ import { PROCESSING_HEIGHT, PROCESSING_WIDTH, RENDERING_EVENTS, STRONG_EDGE_BLUR_AMOUNT, } from '../../constants';
3
3
  import { createLazyProps, handleWebGLContextLoss } from '../../utils';
4
4
  import { createBinaryMaskRenderer } from './binaryMask';
5
5
  import { createBlendRenderer } from './blender';
@@ -25,7 +25,7 @@ export const createRenderer = (renderEventHandlers, canvas = new OffscreenCanvas
25
25
  const rendererCreator = {
26
26
  backgroundImageRenderer: () => createTextureRenderer(canvas, false),
27
27
  blendRenderer: () => createBlendRenderer(canvas),
28
- blurRenderer: () => createBlurRenderer(canvas, BACKGROUND_BLUR_AMOUNT),
28
+ blurRenderer: () => createBlurRenderer(canvas),
29
29
  canvasRenderer: () => createCanvasRenderer(canvas),
30
30
  frameDownsampler2: () => createTextureToTextureRenderer(canvas, false, true, halfWidth, halfHeight),
31
31
  frameDownsampler4: () => createTextureToTextureRenderer(canvas, false, true, quarterWidth, quarterHeight),
@@ -34,9 +34,9 @@ export const createRenderer = (renderEventHandlers, canvas = new OffscreenCanvas
34
34
  jointBilateralFilterRenderer2: () => createJointBilateralFilterRenderer(canvas, halfWidth, halfHeight),
35
35
  jointBilateralFilterRenderer4: () => createJointBilateralFilterRenderer(canvas, quarterWidth, quarterHeight),
36
36
  jointBilateralFilterRenderer: () => createJointBilateralFilterRenderer(canvas, canvas.width, canvas.height),
37
- lightWrapBlurRenderer: () => createBlurRenderer(canvas, BACKGROUND_BLUR_AMOUNT),
37
+ lightWrapBlurRenderer: () => createBlurRenderer(canvas),
38
38
  maskBlurRenderer: () => createTentBlurRenderer(canvas, canvas.width, canvas.height),
39
- strongMaskBlurRenderer: () => createBlurRenderer(canvas, STRONG_EDGE_BLUR_AMOUNT),
39
+ strongMaskBlurRenderer: () => createBlurRenderer(canvas),
40
40
  maskDownsampler2: () => createTextureToTextureRenderer(canvas, true, false, halfWidth, halfHeight),
41
41
  maskDownsampler4: () => createTextureToTextureRenderer(canvas, true, false, quarterWidth, quarterHeight),
42
42
  maskRenderer: () => createTextureRenderer(canvas, true),
@@ -2,7 +2,8 @@ import { assert } from '@pexip/utils';
2
2
  import { RENDERING_EVENTS } from '../../constants';
3
3
  import blurComputeShader from './shaders/dualBlurComputeShader.wgsl';
4
4
  import { clamping } from '../../utils';
5
- const clampBlurAmount = clamping(1, 9);
5
+ const MAX_BLUR_LEVEL = 9;
6
+ const clampBlurAmount = clamping(1, MAX_BLUR_LEVEL);
6
7
  export const createBlurRenderer = (device, width, height, textureFormat, isContextLost) => {
7
8
  const shaderModule = device.createShaderModule({
8
9
  label: 'dual blur compute shader',
@@ -34,103 +35,80 @@ export const createBlurRenderer = (device, width, height, textureFormat, isConte
34
35
  addressModeU: 'clamp-to-edge',
35
36
  addressModeV: 'clamp-to-edge',
36
37
  });
37
- // Helper to create a texture of given size
38
- const createTexture = (w, h) => device.createTexture({
39
- label: 'dual filter blur texture',
40
- format: textureFormat,
41
- size: [w, h],
42
- usage: GPUTextureUsage.TEXTURE_BINDING |
43
- GPUTextureUsage.RENDER_ATTACHMENT,
44
- });
45
- // --- State for persistent textures and bind groups ---
46
- let blurTextures = [];
47
- let lastWidth = -1;
48
- let lastHeight = -1;
49
- let downsampleRenderPassDescriptor = [];
50
- let upsampleRenderPassDescriptor = [];
51
- let downsampleBindGroups = [];
52
- let upsampleBindGroups = [];
38
+ // Allocate all levels upfront at max blur depth
39
+ const blurTextures = [];
40
+ for (let i = 0; i < MAX_BLUR_LEVEL; i++) {
41
+ const w = Math.max(Math.trunc(width / 2 ** i), 1);
42
+ const h = Math.max(Math.trunc(height / 2 ** i), 1);
43
+ const texture = device.createTexture({
44
+ label: `dual filter blur texture level ${i}`,
45
+ format: textureFormat,
46
+ size: [w, h],
47
+ usage: GPUTextureUsage.TEXTURE_BINDING |
48
+ GPUTextureUsage.RENDER_ATTACHMENT,
49
+ });
50
+ blurTextures.push({
51
+ texture,
52
+ view: texture.createView(),
53
+ width: w,
54
+ height: h,
55
+ });
56
+ }
57
+ // Pre-create all downsample bind groups: blurTextures[i-1] → blurTextures[i]
58
+ const downsampleBindGroups = [];
59
+ const downsampleRenderPassDescriptors = [];
60
+ for (let i = 2; i < MAX_BLUR_LEVEL; i++) {
61
+ const src = blurTextures[i - 1];
62
+ const dst = blurTextures[i];
63
+ assert(src);
64
+ assert(dst);
65
+ downsampleBindGroups.push(device.createBindGroup({
66
+ layout: downsamplePipeline.getBindGroupLayout(0),
67
+ entries: [
68
+ { binding: 0, resource: sampler },
69
+ { binding: 1, resource: src.view },
70
+ ],
71
+ }));
72
+ downsampleRenderPassDescriptors.push({
73
+ label: `downsample render pass ${i}`,
74
+ colorAttachments: [
75
+ { loadOp: 'load', storeOp: 'store', view: dst.view },
76
+ ],
77
+ });
78
+ }
79
+ // Pre-create all upsample bind groups: blurTextures[i+1] → blurTextures[i]
80
+ // Index 0 = deepest pair (MAX_BLUR_LEVELS-1 → MAX_BLUR_LEVEL-2)
81
+ const upsampleBindGroups = [];
82
+ const upsampleRenderPassDescriptors = [];
83
+ for (let i = MAX_BLUR_LEVEL - 2; i >= 0; i--) {
84
+ const src = blurTextures[i + 1];
85
+ const dst = blurTextures[i];
86
+ assert(src);
87
+ assert(dst);
88
+ upsampleBindGroups.push(device.createBindGroup({
89
+ layout: upsamplePipeline.getBindGroupLayout(0),
90
+ entries: [
91
+ { binding: 0, resource: sampler },
92
+ { binding: 1, resource: src.view },
93
+ ],
94
+ }));
95
+ upsampleRenderPassDescriptors.push({
96
+ label: `upsample render pass ${i}`,
97
+ colorAttachments: [
98
+ { loadOp: 'load', storeOp: 'store', view: dst.view },
99
+ ],
100
+ });
101
+ }
53
102
  let cacheFrameTexture = null;
54
103
  let cacheFirstDownsampleBindGroup = null;
55
104
  let cacheFirstDownsampleRenderPassDescriptor = null;
56
- // Helper to (re)generate textures and bind groups if needed
57
- function ensureBlurTextures(pass, frameWidth, frameHeight) {
58
- const length = clampBlurAmount(pass + 1);
59
- if (blurTextures.length !== length ||
60
- lastWidth !== frameWidth ||
61
- lastHeight !== frameHeight) {
62
- // Destroy old textures
63
- for (const t of blurTextures) {
64
- t.texture.destroy();
65
- }
66
- blurTextures = [];
67
- downsampleBindGroups = [];
68
- downsampleRenderPassDescriptor = [];
69
- upsampleBindGroups = [];
70
- // Invalidate frame cache since firstDst view has changed
71
- cacheFrameTexture = null;
72
- cacheFirstDownsampleBindGroup = null;
73
- for (let i = 0; i < length; i++) {
74
- const w = Math.max(Math.trunc(frameWidth / 2 ** i), 1);
75
- const h = Math.max(Math.trunc(frameHeight / 2 ** i), 1);
76
- const texture = createTexture(w, h);
77
- blurTextures.push({
78
- texture,
79
- view: texture.createView(),
80
- width: w,
81
- height: h,
82
- });
83
- }
84
- // Pre-create downsample bind groups: blurTextures[i-1] → blurTextures[i]
85
- for (let i = 2; i < blurTextures.length; i++) {
86
- const src = blurTextures[i - 1];
87
- const dst = blurTextures[i];
88
- assert(src);
89
- assert(dst);
90
- downsampleBindGroups.push(device.createBindGroup({
91
- layout: downsamplePipeline.getBindGroupLayout(0),
92
- entries: [
93
- { binding: 0, resource: sampler },
94
- { binding: 1, resource: src.view },
95
- ],
96
- }));
97
- downsampleRenderPassDescriptor.push({
98
- label: `downsample render pass ${i}`,
99
- colorAttachments: [
100
- { loadOp: 'load', storeOp: 'store', view: dst.view },
101
- ],
102
- });
103
- }
104
- // Pre-create upsample bind groups: blurTextures[i+1] → blurTextures[i]
105
- // Filled in descending order so upsampleBindGroups[0] is the deepest pair.
106
- for (let i = blurTextures.length - 2; i >= 0; i--) {
107
- const src = blurTextures[i + 1];
108
- const dst = blurTextures[i];
109
- assert(src);
110
- assert(dst);
111
- upsampleBindGroups.push(device.createBindGroup({
112
- layout: upsamplePipeline.getBindGroupLayout(0),
113
- entries: [
114
- { binding: 0, resource: sampler },
115
- { binding: 1, resource: src.view },
116
- ],
117
- }));
118
- upsampleRenderPassDescriptor.push({
119
- label: `upsample render pass ${i}`,
120
- colorAttachments: [
121
- { loadOp: 'load', storeOp: 'store', view: dst.view },
122
- ],
123
- });
124
- }
125
- lastWidth = frameWidth;
126
- lastHeight = frameHeight;
127
- }
128
- }
129
105
  return {
130
106
  render: (encoder, frame, pass) => {
131
107
  assert(!isContextLost(), RENDERING_EVENTS.ContextLost);
132
- // Ensure textures and bind groups are up-to-date
133
- ensureBlurTextures(pass, width, height);
108
+ if (!pass) {
109
+ return frame;
110
+ }
111
+ const length = clampBlurAmount(pass + 1);
134
112
  const firstDst = blurTextures[1];
135
113
  assert(firstDst);
136
114
  // Cache the first downsample bind group (frame → blurTextures[1])
@@ -158,34 +136,34 @@ export const createBlurRenderer = (device, width, height, textureFormat, isConte
158
136
  firstComputePass.draw(3);
159
137
  firstComputePass.end();
160
138
  // Subsequent downsampling passes: blurTextures[i-1] → blurTextures[i]
161
- for (let i = 2; i < blurTextures.length; i++) {
139
+ for (let i = 2; i < length; i++) {
162
140
  const dst = blurTextures[i];
163
141
  assert(dst);
164
142
  // downsampleBindGroups[i-1] covers blurTextures[i-1] → blurTextures[i]
165
143
  const bindGroup = downsampleBindGroups[i - 2];
166
- const renderPassDescriptor = downsampleRenderPassDescriptor[i - 2];
144
+ const renderPassDescriptor = downsampleRenderPassDescriptors[i - 2];
167
145
  assert(bindGroup);
168
146
  assert(renderPassDescriptor);
169
- const pass = encoder.beginRenderPass(renderPassDescriptor);
170
- pass.setPipeline(downsamplePipeline);
171
- pass.setBindGroup(0, bindGroup);
172
- pass.draw(3);
173
- pass.end();
147
+ const renderPass = encoder.beginRenderPass(renderPassDescriptor);
148
+ renderPass.setPipeline(downsamplePipeline);
149
+ renderPass.setBindGroup(0, bindGroup);
150
+ renderPass.draw(3);
151
+ renderPass.end();
174
152
  }
175
153
  // Upsampling passes (deepest level first, up to blurTextures[0])
176
- for (let i = 0; i < upsampleBindGroups.length; i++) {
177
- const dstIdx = blurTextures.length - 2 - i;
178
- const dst = blurTextures[dstIdx];
179
- assert(dst);
154
+ // upsampleBindGroups are indexed from deepest (MAX-2 MAX-1) at [0]
155
+ // We need to start from the pair ending at `length-1` and go up to 0
156
+ const upsampleStartIdx = MAX_BLUR_LEVEL - 1 - (length - 1);
157
+ for (let i = upsampleStartIdx; i < upsampleBindGroups.length; i++) {
180
158
  const bindGroup = upsampleBindGroups[i];
181
- const renderPassDescriptor = upsampleRenderPassDescriptor[i];
159
+ const renderPassDescriptor = upsampleRenderPassDescriptors[i];
182
160
  assert(bindGroup);
183
161
  assert(renderPassDescriptor);
184
- const pass = encoder.beginRenderPass(renderPassDescriptor);
185
- pass.setPipeline(upsamplePipeline);
186
- pass.setBindGroup(0, bindGroup);
187
- pass.draw(3);
188
- pass.end();
162
+ const renderPass = encoder.beginRenderPass(renderPassDescriptor);
163
+ renderPass.setPipeline(upsamplePipeline);
164
+ renderPass.setBindGroup(0, bindGroup);
165
+ renderPass.draw(3);
166
+ renderPass.end();
189
167
  }
190
168
  const finalTexture = blurTextures[0];
191
169
  assert(finalTexture);
@@ -195,13 +173,14 @@ export const createBlurRenderer = (device, width, height, textureFormat, isConte
195
173
  for (const t of blurTextures) {
196
174
  t.texture.destroy();
197
175
  }
198
- blurTextures = [];
199
- downsampleRenderPassDescriptor = [];
200
- upsampleRenderPassDescriptor = [];
201
- downsampleBindGroups = [];
202
- upsampleBindGroups = [];
176
+ blurTextures.length = 0;
177
+ downsampleBindGroups.length = 0;
178
+ downsampleRenderPassDescriptors.length = 0;
179
+ upsampleBindGroups.length = 0;
180
+ upsampleRenderPassDescriptors.length = 0;
203
181
  cacheFrameTexture = null;
204
182
  cacheFirstDownsampleBindGroup = null;
183
+ cacheFirstDownsampleRenderPassDescriptor = null;
205
184
  },
206
185
  };
207
186
  };
@@ -272,6 +272,7 @@ export const createRenderer = (adapter, renderEventHandlers, canvas = new Offscr
272
272
  };
273
273
  const release = () => {
274
274
  lazyProps.release();
275
+ props.device = undefined;
275
276
  props.deviceLost = false;
276
277
  props.prevMask2 = undefined;
277
278
  props.prevMask4 = undefined;
@@ -1,6 +1,41 @@
1
1
  export declare const hasR16FloatTextureStorage: (adapter: GPUAdapter) => boolean;
2
2
  export declare const hasF16: (adapter: GPUAdapter) => boolean;
3
+ export declare class GPUDeviceRequestInvalidatedError extends Error {
4
+ constructor();
5
+ }
6
+ /**
7
+ * Get the GPU adapter, requesting it only once.
8
+ *
9
+ * @example
10
+ * ```ts
11
+ * const adapter = await getGPUAdapter();
12
+ * ```
13
+ */
14
+ export declare const getGPUAdapter: () => Promise<GPUAdapter | null>;
15
+ /**
16
+ * Get the GPU device, requesting it only once.
17
+ *
18
+ * A device owns the memory of the pipelines and the shader modules created from
19
+ * it, and neither of those can be freed on their own, so it is shared by every
20
+ * renderer instead of being requested per renderer.
21
+ *
22
+ * @param adapter - The adapter to request the device from
23
+ *
24
+ * @example
25
+ * ```ts
26
+ * const device = await getGPUDevice(adapter);
27
+ * ```
28
+ */
3
29
  export declare const getGPUDevice: (adapter: GPUAdapter) => Promise<GPUDevice>;
30
+ /**
31
+ * Destroy the shared GPU device.
32
+ *
33
+ * @example
34
+ * ```ts
35
+ * destroyGPUDevice();
36
+ * ```
37
+ */
38
+ export declare const destroyGPUDevice: () => void;
4
39
  /**
5
40
  * Check if provided format is supported
6
41
  *
@@ -1,6 +1,36 @@
1
1
  export const hasR16FloatTextureStorage = (adapter) => adapter.features.has('texture-formats-tier1');
2
2
  export const hasF16 = (adapter) => adapter.features.has('shader-f16');
3
- export const getGPUDevice = async (adapter) => {
3
+ let pendingAdapter;
4
+ let pendingDevice;
5
+ let currentDevice;
6
+ let generation = 0;
7
+ export class GPUDeviceRequestInvalidatedError extends Error {
8
+ constructor() {
9
+ super('GPU device request was invalidated');
10
+ this.name = 'GPUDeviceRequestInvalidatedError';
11
+ }
12
+ }
13
+ const reset = () => {
14
+ currentDevice = undefined;
15
+ pendingDevice = undefined;
16
+ // An adapter is consumed by a device request and cannot be reused after a loss
17
+ pendingAdapter = undefined;
18
+ };
19
+ /**
20
+ * Get the GPU adapter, requesting it only once.
21
+ *
22
+ * @example
23
+ * ```ts
24
+ * const adapter = await getGPUAdapter();
25
+ * ```
26
+ */
27
+ export const getGPUAdapter = async () => {
28
+ pendingAdapter ?? (pendingAdapter = navigator.gpu.requestAdapter({
29
+ powerPreference: 'high-performance',
30
+ }));
31
+ return pendingAdapter;
32
+ };
33
+ const requestGPUDevice = async (adapter, requested) => {
4
34
  const requiredFeatures = [];
5
35
  if (hasR16FloatTextureStorage(adapter)) {
6
36
  // https://gpuweb.github.io/gpuweb/#plain-color-formats
@@ -14,9 +44,61 @@ export const getGPUDevice = async (adapter) => {
14
44
  // https://developer.chrome.com/blog/new-in-webgpu-120
15
45
  requiredFeatures.push('shader-f16');
16
46
  }
17
- const device = await adapter?.requestDevice({ requiredFeatures });
47
+ const device = await adapter.requestDevice({ requiredFeatures });
48
+ if (generation !== requested) {
49
+ device.destroy();
50
+ throw new GPUDeviceRequestInvalidatedError();
51
+ }
52
+ currentDevice = device;
53
+ void device.lost.then(() => {
54
+ // A lost device cannot be used again, so the next one is requested
55
+ if (currentDevice === device) {
56
+ reset();
57
+ }
58
+ });
18
59
  return device;
19
60
  };
61
+ /**
62
+ * Get the GPU device, requesting it only once.
63
+ *
64
+ * A device owns the memory of the pipelines and the shader modules created from
65
+ * it, and neither of those can be freed on their own, so it is shared by every
66
+ * renderer instead of being requested per renderer.
67
+ *
68
+ * @param adapter - The adapter to request the device from
69
+ *
70
+ * @example
71
+ * ```ts
72
+ * const device = await getGPUDevice(adapter);
73
+ * ```
74
+ */
75
+ export const getGPUDevice = async (adapter) => {
76
+ if (!pendingDevice) {
77
+ const request = requestGPUDevice(adapter, generation);
78
+ pendingDevice = request;
79
+ void request.catch(() => {
80
+ // Do not clear a newer request started after invalidation.
81
+ if (pendingDevice === request) {
82
+ pendingDevice = undefined;
83
+ }
84
+ });
85
+ }
86
+ return pendingDevice;
87
+ };
88
+ /**
89
+ * Destroy the shared GPU device.
90
+ *
91
+ * @example
92
+ * ```ts
93
+ * destroyGPUDevice();
94
+ * ```
95
+ */
96
+ export const destroyGPUDevice = () => {
97
+ // Invalidate an unresolved request before clearing the cache.
98
+ generation += 1;
99
+ currentDevice?.destroy();
100
+ reset();
101
+ };
20
102
  /**
21
103
  * Check if provided format is supported
22
104
  *
@@ -1,4 +1,23 @@
1
- import type { Segmentation } from './types/media';
1
+ /**
2
+ * Create a reusable converter that transforms a single-channel mask into
3
+ * RGBA ImageData with values written into the alpha channel.
4
+ * The returned function reuses the same ImageData buffer across calls.
5
+ *
6
+ * @example
7
+ * ```ts
8
+ * const toAlphaMask = createAlphaMaskConverter();
9
+ * const imageData = toAlphaMask(mask);
10
+ * ctx.putImageData(imageData, 0, 0);
11
+ * ```
12
+ */
13
+ export declare const createAlphaMaskConverter: () => (mask: {
14
+ getAsUint8Array(): Uint8Array;
15
+ width: number;
16
+ height: number;
17
+ }) => ImageData;
18
+ export declare const createCpuMaskRenderer: (canvas: OffscreenCanvas) => {
19
+ render: (imageData: ImageData) => void;
20
+ };
2
21
  declare const INTERNAL_CANVAS_KEYS: {
3
22
  readonly drawImageDataCanvas: "drawImageDataCanvas";
4
23
  readonly maskCanvas: "maskCanvas";
@@ -52,12 +71,12 @@ export declare const clearCanvas: (canvas: OffscreenCanvas) => void;
52
71
  export declare const createCanvasRenderUtils: (processingWidth: number, processingHeight: number) => {
53
72
  evaluateInput: (inputImage: CanvasImageSource) => OffscreenCanvas;
54
73
  renderImageToCanvas: (image: CanvasImageSource, canvas: OffscreenCanvas, dw?: number, dh?: number, options?: CanvasRenderingContext2DOptions) => void;
55
- drawBlurEffect: (canvas: OffscreenCanvas, inputImage: CanvasImageSource, segmentations: Segmentation | Segmentation[], foregroundThreshold?: number, backgroundBlurAmount?: number, edgeBlurAmount?: number, flipHorizontal?: boolean) => void;
56
- drawOverlayEffect: (canvas: OffscreenCanvas, inputImage: CanvasImageSource, backgroundImage: CanvasImageSource | OffscreenCanvas, segmentations: Segmentation | Segmentation[], foregroundThreshold?: number, backgroundBlurAmount?: number, edgeBlurAmount?: number, flipHorizontal?: boolean) => void;
74
+ drawBlurEffect: (canvas: OffscreenCanvas, inputImage: CanvasImageSource, mask: CanvasImageSource | OffscreenCanvas, foregroundThreshold?: number, backgroundBlurAmount?: number, edgeBlurAmount?: number, flipHorizontal?: boolean) => void;
75
+ drawOverlayEffect: (canvas: OffscreenCanvas, inputImage: CanvasImageSource, backgroundImage: CanvasImageSource | OffscreenCanvas, mask: CanvasImageSource | OffscreenCanvas, foregroundThreshold?: number, backgroundBlurAmount?: number, edgeBlurAmount?: number, flipHorizontal?: boolean) => void;
57
76
  renderImageToOffScreenCanvas: (image: CanvasImageSource, canvasName: keyof InternalCanvases) => OffscreenCanvas;
58
77
  renderImageDataToOffScreenCanvas: (image: ImageData, canvasName: keyof InternalCanvases) => OffscreenCanvas;
59
78
  drawAndBlurImageOnOffScreenCanvas: ({ image, blurAmount, offscreenCanvasName, preserveOldDrawing, }: {
60
- image: CanvasImageSource;
79
+ image: CanvasImageSource | OffscreenCanvas;
61
80
  blurAmount: number;
62
81
  offscreenCanvasName: keyof InternalCanvases;
63
82
  preserveOldDrawing?: boolean;