@pexip/media-processor 22.1.0 → 22.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +51 -0
- package/README.md +34 -0
- package/api-docs/README.mdx +38 -0
- package/api-docs/functions/createBenchmark.mdx +7 -6
- package/api-docs/interfaces/Benchmark.mdx +3 -5
- package/api-docs/interfaces/ProcessorEvent.mdx +1 -0
- package/api-docs/interfaces/ProcessorOptions.mdx +10 -9
- package/api-docs/interfaces/ProcessorUpdateOptions.mdx +10 -9
- package/api-docs/interfaces/RendererOptions.mdx +1 -1
- package/api-docs/interfaces/SegmenterOptions.mdx +7 -6
- package/api-docs/interfaces/TensorMetadata.mdx +11 -0
- package/api-docs/type-aliases/TensorLayout.mdx +3 -0
- package/api-docs/variables/RENDER_BACKEND.mdx +1 -0
- package/api-docs/variables/getCanUseWebGL.mdx +7 -0
- package/api-docs/variables/getTensorMetadata.mdx +14 -0
- package/dist/common/backends/canvas2d/postprocessor.d.ts +30 -0
- package/dist/common/backends/canvas2d/postprocessor.js +88 -0
- package/dist/common/backends/canvas2d/preprocessor.d.ts +56 -0
- package/dist/common/backends/canvas2d/preprocessor.js +139 -0
- package/dist/common/backends/canvas2d/renderer.d.ts +2 -0
- package/dist/common/backends/canvas2d/renderer.js +36 -0
- package/dist/common/backends/webgl/blur.d.ts +1 -1
- package/dist/common/backends/webgl/blur.js +11 -25
- package/dist/common/backends/webgl/renderer.js +4 -4
- package/dist/common/backends/webgpu/dualFilterBlur.js +93 -114
- package/dist/common/backends/webgpu/renderer.js +1 -0
- package/dist/common/backends/webgpu/webgpuUtils.d.ts +35 -0
- package/dist/common/backends/webgpu/webgpuUtils.js +84 -2
- package/dist/common/canvasRenderUtils.d.ts +23 -4
- package/dist/common/canvasRenderUtils.js +48 -11
- package/dist/common/constants.d.ts +1 -0
- package/dist/common/constants.js +1 -0
- package/dist/common/inferencer.d.ts +27 -0
- package/dist/common/inferencer.js +106 -0
- package/dist/common/tsconfig.tsbuildinfo +1 -1
- package/dist/common/types/processor.d.ts +1 -0
- package/dist/common/types/segmentation.d.ts +8 -0
- package/dist/common/utils.d.ts +13 -0
- package/dist/common/utils.js +56 -0
- package/dist/main/audio.js +20 -2
- package/dist/main/benchUtils.d.ts +28 -18
- package/dist/main/benchUtils.js +66 -56
- package/dist/main/tsconfig.tsbuildinfo +1 -1
- package/dist/main/utils.d.ts +9 -0
- package/dist/main/utils.js +12 -0
- package/dist/main/video/segmenter.d.ts +4 -2
- package/dist/main/video/segmenter.js +36 -11
- package/dist/main/video/video.js +13 -1
- package/dist/workers/mediaWorker.js +23 -23
- package/dist/workers/tsconfig.tsbuildinfo +1 -1
- package/package.json +7 -10
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { PROCESSING_HEIGHT, PROCESSING_WIDTH } from '../../constants';
|
|
2
|
+
import { createCanvasRenderUtils } from '../../canvasRenderUtils';
|
|
3
|
+
export const createRenderer = (renderEventHandlers, canvas = new OffscreenCanvas(PROCESSING_WIDTH, PROCESSING_HEIGHT)) => {
|
|
4
|
+
const utils = createCanvasRenderUtils(canvas.width, canvas.height);
|
|
5
|
+
let ctx = null;
|
|
6
|
+
const init = () => {
|
|
7
|
+
ctx = canvas.getContext('2d');
|
|
8
|
+
if (!ctx) {
|
|
9
|
+
renderEventHandlers.contextCreationError('Failed to create 2D rendering context');
|
|
10
|
+
}
|
|
11
|
+
};
|
|
12
|
+
const render = (mask, frame, options) => {
|
|
13
|
+
if (!ctx) {
|
|
14
|
+
return;
|
|
15
|
+
}
|
|
16
|
+
ctx.clearRect(0, 0, canvas.width, canvas.height);
|
|
17
|
+
const inputImage = utils.evaluateInput(frame);
|
|
18
|
+
if (options.effects === 'blur' && options.backgroundBlurAmount) {
|
|
19
|
+
utils.drawBlurEffect(canvas, inputImage, mask, options.foregroundThreshold, options.backgroundBlurAmount, options.edgeBlurAmount);
|
|
20
|
+
}
|
|
21
|
+
else if (options?.effects === 'overlay' && options.backgroundImage) {
|
|
22
|
+
utils.drawOverlayEffect(canvas, inputImage, options.backgroundImage, mask, options.foregroundThreshold, 0, options.edgeBlurAmount);
|
|
23
|
+
}
|
|
24
|
+
};
|
|
25
|
+
const cleanup = () => {
|
|
26
|
+
utils.reset();
|
|
27
|
+
ctx = null;
|
|
28
|
+
};
|
|
29
|
+
const release = () => {
|
|
30
|
+
cleanup();
|
|
31
|
+
};
|
|
32
|
+
const isContextLost = () => {
|
|
33
|
+
return !ctx || ctx.isContextLost();
|
|
34
|
+
};
|
|
35
|
+
return { init, render, release, isContextLost };
|
|
36
|
+
};
|
|
@@ -2,7 +2,7 @@ import type { TextureInfo } from './types';
|
|
|
2
2
|
/**
|
|
3
3
|
* Blur provided frame
|
|
4
4
|
*/
|
|
5
|
-
export declare const createBlurRenderer: (processor: OffscreenCanvas,
|
|
5
|
+
export declare const createBlurRenderer: (processor: OffscreenCanvas, redOnly?: boolean) => {
|
|
6
6
|
render: (frame: TextureInfo, pass: number) => TextureInfo;
|
|
7
7
|
release: () => void;
|
|
8
8
|
};
|
|
@@ -5,11 +5,12 @@ import { m3 } from '../../m3';
|
|
|
5
5
|
import { createFrameBuffer, createGLProgram, createMaskTexture, createRGBAFrameTexture, } from './webglUtils';
|
|
6
6
|
import vertexSrc from './shaders/matrix.vert';
|
|
7
7
|
import fragmentSrc from './shaders/dualFilterBlur.frag';
|
|
8
|
-
const
|
|
8
|
+
const MAX_BLUR_LEVEL = 9;
|
|
9
|
+
const clampBlurAmount = clamping(1, MAX_BLUR_LEVEL);
|
|
9
10
|
/**
|
|
10
11
|
* Blur provided frame
|
|
11
12
|
*/
|
|
12
|
-
export const createBlurRenderer = (processor,
|
|
13
|
+
export const createBlurRenderer = (processor, redOnly = false) => {
|
|
13
14
|
const { gl, program, attribLocations, uniformLocations, floatSingleChannelSupported, floatFullChannelSupported, release, } = createGLProgram(processor, {
|
|
14
15
|
attribs: {
|
|
15
16
|
position: 'a_position',
|
|
@@ -62,23 +63,19 @@ export const createBlurRenderer = (processor, pass, redOnly = false) => {
|
|
|
62
63
|
};
|
|
63
64
|
const createFramebufferAndTexture = (idx) => {
|
|
64
65
|
// Avoid fractional value
|
|
65
|
-
const width = Math.trunc(processor.width / 2 ** idx);
|
|
66
|
-
const height = Math.trunc(processor.height / 2 ** idx);
|
|
66
|
+
const width = Math.max(Math.trunc(processor.width / 2 ** idx), 1);
|
|
67
|
+
const height = Math.max(Math.trunc(processor.height / 2 ** idx), 1);
|
|
67
68
|
const texture = redOnly
|
|
68
69
|
? createMaskTexture(gl, width, height, true, floatSingleChannelSupported, floatFullChannelSupported)
|
|
69
70
|
: createRGBAFrameTexture(gl, width, height, true, floatFullChannelSupported);
|
|
70
71
|
const frameBuffer = createFrameBuffer(gl, texture);
|
|
71
72
|
return { texture, frameBuffer, width, height };
|
|
72
73
|
};
|
|
73
|
-
|
|
74
|
-
const length = Math.max(clampBlurAmount(offset + pass) - offset, 0);
|
|
75
|
-
return Array.from({
|
|
76
|
-
length,
|
|
77
|
-
}).map((_, currentIdx) => createFramebufferAndTexture(offset + currentIdx));
|
|
78
|
-
};
|
|
79
|
-
// Generate multiple FrameBuffers, textures and the widths and heights are of size
|
|
74
|
+
// Allocate all levels upfront, textures and the widths and heights are of size
|
|
80
75
|
// of half of the previous one. E.g. 640x480 -> 320x240 -> 160x120 -> 80x60
|
|
81
|
-
const blurBuffers =
|
|
76
|
+
const blurBuffers = Array.from({
|
|
77
|
+
length: MAX_BLUR_LEVEL,
|
|
78
|
+
}).map((_, currentIdx) => createFramebufferAndTexture(currentIdx));
|
|
82
79
|
return {
|
|
83
80
|
render: (frame, pass) => {
|
|
84
81
|
assert(!gl.isContextLost(), RENDERING_EVENTS.ContextLost);
|
|
@@ -98,18 +95,7 @@ export const createBlurRenderer = (processor, pass, redOnly = false) => {
|
|
|
98
95
|
// Dual filter blur
|
|
99
96
|
// Downsampling
|
|
100
97
|
gl.uniform1i(uniformLocations.downsampling, 1);
|
|
101
|
-
|
|
102
|
-
const currentPass = pass + 1;
|
|
103
|
-
if (currentPass > blurBuffers.length) {
|
|
104
|
-
for (const buffer of generateBlurBuffers(currentPass - blurBuffers.length, blurBuffers.length)) {
|
|
105
|
-
blurBuffers.push(buffer);
|
|
106
|
-
}
|
|
107
|
-
}
|
|
108
|
-
while (currentPass < blurBuffers.length) {
|
|
109
|
-
const buffer = blurBuffers.pop();
|
|
110
|
-
gl.deleteTexture(buffer?.texture ?? null);
|
|
111
|
-
gl.deleteFramebuffer(buffer?.frameBuffer ?? null);
|
|
112
|
-
}
|
|
98
|
+
const currentPass = clampBlurAmount(pass + 1);
|
|
113
99
|
// Skip the first pass
|
|
114
100
|
for (let i = 1; i < currentPass; i++) {
|
|
115
101
|
const { frameBuffer = null, texture = null, width = processor.width, height = processor.height, } = blurBuffers[i] ?? {};
|
|
@@ -134,7 +120,7 @@ export const createBlurRenderer = (processor, pass, redOnly = false) => {
|
|
|
134
120
|
// Upsampling
|
|
135
121
|
gl.uniform1i(uniformLocations.downsampling, 0);
|
|
136
122
|
// Skip the first pass
|
|
137
|
-
for (let i =
|
|
123
|
+
for (let i = currentPass - 2; i >= 0; i--) {
|
|
138
124
|
const { frameBuffer = null, texture = null, width = processor.width, height = processor.height, } = blurBuffers[i] ?? {};
|
|
139
125
|
// console.log(`Drawing upsampling pass ${i}, ${width}x${height}`);
|
|
140
126
|
// Setup to draw to the frame buffer
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { assert } from '@pexip/utils';
|
|
2
|
-
import {
|
|
2
|
+
import { PROCESSING_HEIGHT, PROCESSING_WIDTH, RENDERING_EVENTS, STRONG_EDGE_BLUR_AMOUNT, } from '../../constants';
|
|
3
3
|
import { createLazyProps, handleWebGLContextLoss } from '../../utils';
|
|
4
4
|
import { createBinaryMaskRenderer } from './binaryMask';
|
|
5
5
|
import { createBlendRenderer } from './blender';
|
|
@@ -25,7 +25,7 @@ export const createRenderer = (renderEventHandlers, canvas = new OffscreenCanvas
|
|
|
25
25
|
const rendererCreator = {
|
|
26
26
|
backgroundImageRenderer: () => createTextureRenderer(canvas, false),
|
|
27
27
|
blendRenderer: () => createBlendRenderer(canvas),
|
|
28
|
-
blurRenderer: () => createBlurRenderer(canvas
|
|
28
|
+
blurRenderer: () => createBlurRenderer(canvas),
|
|
29
29
|
canvasRenderer: () => createCanvasRenderer(canvas),
|
|
30
30
|
frameDownsampler2: () => createTextureToTextureRenderer(canvas, false, true, halfWidth, halfHeight),
|
|
31
31
|
frameDownsampler4: () => createTextureToTextureRenderer(canvas, false, true, quarterWidth, quarterHeight),
|
|
@@ -34,9 +34,9 @@ export const createRenderer = (renderEventHandlers, canvas = new OffscreenCanvas
|
|
|
34
34
|
jointBilateralFilterRenderer2: () => createJointBilateralFilterRenderer(canvas, halfWidth, halfHeight),
|
|
35
35
|
jointBilateralFilterRenderer4: () => createJointBilateralFilterRenderer(canvas, quarterWidth, quarterHeight),
|
|
36
36
|
jointBilateralFilterRenderer: () => createJointBilateralFilterRenderer(canvas, canvas.width, canvas.height),
|
|
37
|
-
lightWrapBlurRenderer: () => createBlurRenderer(canvas
|
|
37
|
+
lightWrapBlurRenderer: () => createBlurRenderer(canvas),
|
|
38
38
|
maskBlurRenderer: () => createTentBlurRenderer(canvas, canvas.width, canvas.height),
|
|
39
|
-
strongMaskBlurRenderer: () => createBlurRenderer(canvas
|
|
39
|
+
strongMaskBlurRenderer: () => createBlurRenderer(canvas),
|
|
40
40
|
maskDownsampler2: () => createTextureToTextureRenderer(canvas, true, false, halfWidth, halfHeight),
|
|
41
41
|
maskDownsampler4: () => createTextureToTextureRenderer(canvas, true, false, quarterWidth, quarterHeight),
|
|
42
42
|
maskRenderer: () => createTextureRenderer(canvas, true),
|
|
@@ -2,7 +2,8 @@ import { assert } from '@pexip/utils';
|
|
|
2
2
|
import { RENDERING_EVENTS } from '../../constants';
|
|
3
3
|
import blurComputeShader from './shaders/dualBlurComputeShader.wgsl';
|
|
4
4
|
import { clamping } from '../../utils';
|
|
5
|
-
const
|
|
5
|
+
const MAX_BLUR_LEVEL = 9;
|
|
6
|
+
const clampBlurAmount = clamping(1, MAX_BLUR_LEVEL);
|
|
6
7
|
export const createBlurRenderer = (device, width, height, textureFormat, isContextLost) => {
|
|
7
8
|
const shaderModule = device.createShaderModule({
|
|
8
9
|
label: 'dual blur compute shader',
|
|
@@ -34,103 +35,80 @@ export const createBlurRenderer = (device, width, height, textureFormat, isConte
|
|
|
34
35
|
addressModeU: 'clamp-to-edge',
|
|
35
36
|
addressModeV: 'clamp-to-edge',
|
|
36
37
|
});
|
|
37
|
-
//
|
|
38
|
-
const
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
38
|
+
// Allocate all levels upfront at max blur depth
|
|
39
|
+
const blurTextures = [];
|
|
40
|
+
for (let i = 0; i < MAX_BLUR_LEVEL; i++) {
|
|
41
|
+
const w = Math.max(Math.trunc(width / 2 ** i), 1);
|
|
42
|
+
const h = Math.max(Math.trunc(height / 2 ** i), 1);
|
|
43
|
+
const texture = device.createTexture({
|
|
44
|
+
label: `dual filter blur texture level ${i}`,
|
|
45
|
+
format: textureFormat,
|
|
46
|
+
size: [w, h],
|
|
47
|
+
usage: GPUTextureUsage.TEXTURE_BINDING |
|
|
48
|
+
GPUTextureUsage.RENDER_ATTACHMENT,
|
|
49
|
+
});
|
|
50
|
+
blurTextures.push({
|
|
51
|
+
texture,
|
|
52
|
+
view: texture.createView(),
|
|
53
|
+
width: w,
|
|
54
|
+
height: h,
|
|
55
|
+
});
|
|
56
|
+
}
|
|
57
|
+
// Pre-create all downsample bind groups: blurTextures[i-1] → blurTextures[i]
|
|
58
|
+
const downsampleBindGroups = [];
|
|
59
|
+
const downsampleRenderPassDescriptors = [];
|
|
60
|
+
for (let i = 2; i < MAX_BLUR_LEVEL; i++) {
|
|
61
|
+
const src = blurTextures[i - 1];
|
|
62
|
+
const dst = blurTextures[i];
|
|
63
|
+
assert(src);
|
|
64
|
+
assert(dst);
|
|
65
|
+
downsampleBindGroups.push(device.createBindGroup({
|
|
66
|
+
layout: downsamplePipeline.getBindGroupLayout(0),
|
|
67
|
+
entries: [
|
|
68
|
+
{ binding: 0, resource: sampler },
|
|
69
|
+
{ binding: 1, resource: src.view },
|
|
70
|
+
],
|
|
71
|
+
}));
|
|
72
|
+
downsampleRenderPassDescriptors.push({
|
|
73
|
+
label: `downsample render pass ${i}`,
|
|
74
|
+
colorAttachments: [
|
|
75
|
+
{ loadOp: 'load', storeOp: 'store', view: dst.view },
|
|
76
|
+
],
|
|
77
|
+
});
|
|
78
|
+
}
|
|
79
|
+
// Pre-create all upsample bind groups: blurTextures[i+1] → blurTextures[i]
|
|
80
|
+
// Index 0 = deepest pair (MAX_BLUR_LEVELS-1 → MAX_BLUR_LEVEL-2)
|
|
81
|
+
const upsampleBindGroups = [];
|
|
82
|
+
const upsampleRenderPassDescriptors = [];
|
|
83
|
+
for (let i = MAX_BLUR_LEVEL - 2; i >= 0; i--) {
|
|
84
|
+
const src = blurTextures[i + 1];
|
|
85
|
+
const dst = blurTextures[i];
|
|
86
|
+
assert(src);
|
|
87
|
+
assert(dst);
|
|
88
|
+
upsampleBindGroups.push(device.createBindGroup({
|
|
89
|
+
layout: upsamplePipeline.getBindGroupLayout(0),
|
|
90
|
+
entries: [
|
|
91
|
+
{ binding: 0, resource: sampler },
|
|
92
|
+
{ binding: 1, resource: src.view },
|
|
93
|
+
],
|
|
94
|
+
}));
|
|
95
|
+
upsampleRenderPassDescriptors.push({
|
|
96
|
+
label: `upsample render pass ${i}`,
|
|
97
|
+
colorAttachments: [
|
|
98
|
+
{ loadOp: 'load', storeOp: 'store', view: dst.view },
|
|
99
|
+
],
|
|
100
|
+
});
|
|
101
|
+
}
|
|
53
102
|
let cacheFrameTexture = null;
|
|
54
103
|
let cacheFirstDownsampleBindGroup = null;
|
|
55
104
|
let cacheFirstDownsampleRenderPassDescriptor = null;
|
|
56
|
-
// Helper to (re)generate textures and bind groups if needed
|
|
57
|
-
function ensureBlurTextures(pass, frameWidth, frameHeight) {
|
|
58
|
-
const length = clampBlurAmount(pass + 1);
|
|
59
|
-
if (blurTextures.length !== length ||
|
|
60
|
-
lastWidth !== frameWidth ||
|
|
61
|
-
lastHeight !== frameHeight) {
|
|
62
|
-
// Destroy old textures
|
|
63
|
-
for (const t of blurTextures) {
|
|
64
|
-
t.texture.destroy();
|
|
65
|
-
}
|
|
66
|
-
blurTextures = [];
|
|
67
|
-
downsampleBindGroups = [];
|
|
68
|
-
downsampleRenderPassDescriptor = [];
|
|
69
|
-
upsampleBindGroups = [];
|
|
70
|
-
// Invalidate frame cache since firstDst view has changed
|
|
71
|
-
cacheFrameTexture = null;
|
|
72
|
-
cacheFirstDownsampleBindGroup = null;
|
|
73
|
-
for (let i = 0; i < length; i++) {
|
|
74
|
-
const w = Math.max(Math.trunc(frameWidth / 2 ** i), 1);
|
|
75
|
-
const h = Math.max(Math.trunc(frameHeight / 2 ** i), 1);
|
|
76
|
-
const texture = createTexture(w, h);
|
|
77
|
-
blurTextures.push({
|
|
78
|
-
texture,
|
|
79
|
-
view: texture.createView(),
|
|
80
|
-
width: w,
|
|
81
|
-
height: h,
|
|
82
|
-
});
|
|
83
|
-
}
|
|
84
|
-
// Pre-create downsample bind groups: blurTextures[i-1] → blurTextures[i]
|
|
85
|
-
for (let i = 2; i < blurTextures.length; i++) {
|
|
86
|
-
const src = blurTextures[i - 1];
|
|
87
|
-
const dst = blurTextures[i];
|
|
88
|
-
assert(src);
|
|
89
|
-
assert(dst);
|
|
90
|
-
downsampleBindGroups.push(device.createBindGroup({
|
|
91
|
-
layout: downsamplePipeline.getBindGroupLayout(0),
|
|
92
|
-
entries: [
|
|
93
|
-
{ binding: 0, resource: sampler },
|
|
94
|
-
{ binding: 1, resource: src.view },
|
|
95
|
-
],
|
|
96
|
-
}));
|
|
97
|
-
downsampleRenderPassDescriptor.push({
|
|
98
|
-
label: `downsample render pass ${i}`,
|
|
99
|
-
colorAttachments: [
|
|
100
|
-
{ loadOp: 'load', storeOp: 'store', view: dst.view },
|
|
101
|
-
],
|
|
102
|
-
});
|
|
103
|
-
}
|
|
104
|
-
// Pre-create upsample bind groups: blurTextures[i+1] → blurTextures[i]
|
|
105
|
-
// Filled in descending order so upsampleBindGroups[0] is the deepest pair.
|
|
106
|
-
for (let i = blurTextures.length - 2; i >= 0; i--) {
|
|
107
|
-
const src = blurTextures[i + 1];
|
|
108
|
-
const dst = blurTextures[i];
|
|
109
|
-
assert(src);
|
|
110
|
-
assert(dst);
|
|
111
|
-
upsampleBindGroups.push(device.createBindGroup({
|
|
112
|
-
layout: upsamplePipeline.getBindGroupLayout(0),
|
|
113
|
-
entries: [
|
|
114
|
-
{ binding: 0, resource: sampler },
|
|
115
|
-
{ binding: 1, resource: src.view },
|
|
116
|
-
],
|
|
117
|
-
}));
|
|
118
|
-
upsampleRenderPassDescriptor.push({
|
|
119
|
-
label: `upsample render pass ${i}`,
|
|
120
|
-
colorAttachments: [
|
|
121
|
-
{ loadOp: 'load', storeOp: 'store', view: dst.view },
|
|
122
|
-
],
|
|
123
|
-
});
|
|
124
|
-
}
|
|
125
|
-
lastWidth = frameWidth;
|
|
126
|
-
lastHeight = frameHeight;
|
|
127
|
-
}
|
|
128
|
-
}
|
|
129
105
|
return {
|
|
130
106
|
render: (encoder, frame, pass) => {
|
|
131
107
|
assert(!isContextLost(), RENDERING_EVENTS.ContextLost);
|
|
132
|
-
|
|
133
|
-
|
|
108
|
+
if (!pass) {
|
|
109
|
+
return frame;
|
|
110
|
+
}
|
|
111
|
+
const length = clampBlurAmount(pass + 1);
|
|
134
112
|
const firstDst = blurTextures[1];
|
|
135
113
|
assert(firstDst);
|
|
136
114
|
// Cache the first downsample bind group (frame → blurTextures[1])
|
|
@@ -158,34 +136,34 @@ export const createBlurRenderer = (device, width, height, textureFormat, isConte
|
|
|
158
136
|
firstComputePass.draw(3);
|
|
159
137
|
firstComputePass.end();
|
|
160
138
|
// Subsequent downsampling passes: blurTextures[i-1] → blurTextures[i]
|
|
161
|
-
for (let i = 2; i <
|
|
139
|
+
for (let i = 2; i < length; i++) {
|
|
162
140
|
const dst = blurTextures[i];
|
|
163
141
|
assert(dst);
|
|
164
142
|
// downsampleBindGroups[i-1] covers blurTextures[i-1] → blurTextures[i]
|
|
165
143
|
const bindGroup = downsampleBindGroups[i - 2];
|
|
166
|
-
const renderPassDescriptor =
|
|
144
|
+
const renderPassDescriptor = downsampleRenderPassDescriptors[i - 2];
|
|
167
145
|
assert(bindGroup);
|
|
168
146
|
assert(renderPassDescriptor);
|
|
169
|
-
const
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
147
|
+
const renderPass = encoder.beginRenderPass(renderPassDescriptor);
|
|
148
|
+
renderPass.setPipeline(downsamplePipeline);
|
|
149
|
+
renderPass.setBindGroup(0, bindGroup);
|
|
150
|
+
renderPass.draw(3);
|
|
151
|
+
renderPass.end();
|
|
174
152
|
}
|
|
175
153
|
// Upsampling passes (deepest level first, up to blurTextures[0])
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
154
|
+
// upsampleBindGroups are indexed from deepest (MAX-2 → MAX-1) at [0]
|
|
155
|
+
// We need to start from the pair ending at `length-1` and go up to 0
|
|
156
|
+
const upsampleStartIdx = MAX_BLUR_LEVEL - 1 - (length - 1);
|
|
157
|
+
for (let i = upsampleStartIdx; i < upsampleBindGroups.length; i++) {
|
|
180
158
|
const bindGroup = upsampleBindGroups[i];
|
|
181
|
-
const renderPassDescriptor =
|
|
159
|
+
const renderPassDescriptor = upsampleRenderPassDescriptors[i];
|
|
182
160
|
assert(bindGroup);
|
|
183
161
|
assert(renderPassDescriptor);
|
|
184
|
-
const
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
162
|
+
const renderPass = encoder.beginRenderPass(renderPassDescriptor);
|
|
163
|
+
renderPass.setPipeline(upsamplePipeline);
|
|
164
|
+
renderPass.setBindGroup(0, bindGroup);
|
|
165
|
+
renderPass.draw(3);
|
|
166
|
+
renderPass.end();
|
|
189
167
|
}
|
|
190
168
|
const finalTexture = blurTextures[0];
|
|
191
169
|
assert(finalTexture);
|
|
@@ -195,13 +173,14 @@ export const createBlurRenderer = (device, width, height, textureFormat, isConte
|
|
|
195
173
|
for (const t of blurTextures) {
|
|
196
174
|
t.texture.destroy();
|
|
197
175
|
}
|
|
198
|
-
blurTextures =
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
176
|
+
blurTextures.length = 0;
|
|
177
|
+
downsampleBindGroups.length = 0;
|
|
178
|
+
downsampleRenderPassDescriptors.length = 0;
|
|
179
|
+
upsampleBindGroups.length = 0;
|
|
180
|
+
upsampleRenderPassDescriptors.length = 0;
|
|
203
181
|
cacheFrameTexture = null;
|
|
204
182
|
cacheFirstDownsampleBindGroup = null;
|
|
183
|
+
cacheFirstDownsampleRenderPassDescriptor = null;
|
|
205
184
|
},
|
|
206
185
|
};
|
|
207
186
|
};
|
|
@@ -272,6 +272,7 @@ export const createRenderer = (adapter, renderEventHandlers, canvas = new Offscr
|
|
|
272
272
|
};
|
|
273
273
|
const release = () => {
|
|
274
274
|
lazyProps.release();
|
|
275
|
+
props.device = undefined;
|
|
275
276
|
props.deviceLost = false;
|
|
276
277
|
props.prevMask2 = undefined;
|
|
277
278
|
props.prevMask4 = undefined;
|
|
@@ -1,6 +1,41 @@
|
|
|
1
1
|
export declare const hasR16FloatTextureStorage: (adapter: GPUAdapter) => boolean;
|
|
2
2
|
export declare const hasF16: (adapter: GPUAdapter) => boolean;
|
|
3
|
+
export declare class GPUDeviceRequestInvalidatedError extends Error {
|
|
4
|
+
constructor();
|
|
5
|
+
}
|
|
6
|
+
/**
|
|
7
|
+
* Get the GPU adapter, requesting it only once.
|
|
8
|
+
*
|
|
9
|
+
* @example
|
|
10
|
+
* ```ts
|
|
11
|
+
* const adapter = await getGPUAdapter();
|
|
12
|
+
* ```
|
|
13
|
+
*/
|
|
14
|
+
export declare const getGPUAdapter: () => Promise<GPUAdapter | null>;
|
|
15
|
+
/**
|
|
16
|
+
* Get the GPU device, requesting it only once.
|
|
17
|
+
*
|
|
18
|
+
* A device owns the memory of the pipelines and the shader modules created from
|
|
19
|
+
* it, and neither of those can be freed on their own, so it is shared by every
|
|
20
|
+
* renderer instead of being requested per renderer.
|
|
21
|
+
*
|
|
22
|
+
* @param adapter - The adapter to request the device from
|
|
23
|
+
*
|
|
24
|
+
* @example
|
|
25
|
+
* ```ts
|
|
26
|
+
* const device = await getGPUDevice(adapter);
|
|
27
|
+
* ```
|
|
28
|
+
*/
|
|
3
29
|
export declare const getGPUDevice: (adapter: GPUAdapter) => Promise<GPUDevice>;
|
|
30
|
+
/**
|
|
31
|
+
* Destroy the shared GPU device.
|
|
32
|
+
*
|
|
33
|
+
* @example
|
|
34
|
+
* ```ts
|
|
35
|
+
* destroyGPUDevice();
|
|
36
|
+
* ```
|
|
37
|
+
*/
|
|
38
|
+
export declare const destroyGPUDevice: () => void;
|
|
4
39
|
/**
|
|
5
40
|
* Check if provided format is supported
|
|
6
41
|
*
|
|
@@ -1,6 +1,36 @@
|
|
|
1
1
|
export const hasR16FloatTextureStorage = (adapter) => adapter.features.has('texture-formats-tier1');
|
|
2
2
|
export const hasF16 = (adapter) => adapter.features.has('shader-f16');
|
|
3
|
-
|
|
3
|
+
let pendingAdapter;
|
|
4
|
+
let pendingDevice;
|
|
5
|
+
let currentDevice;
|
|
6
|
+
let generation = 0;
|
|
7
|
+
export class GPUDeviceRequestInvalidatedError extends Error {
|
|
8
|
+
constructor() {
|
|
9
|
+
super('GPU device request was invalidated');
|
|
10
|
+
this.name = 'GPUDeviceRequestInvalidatedError';
|
|
11
|
+
}
|
|
12
|
+
}
|
|
13
|
+
const reset = () => {
|
|
14
|
+
currentDevice = undefined;
|
|
15
|
+
pendingDevice = undefined;
|
|
16
|
+
// An adapter is consumed by a device request and cannot be reused after a loss
|
|
17
|
+
pendingAdapter = undefined;
|
|
18
|
+
};
|
|
19
|
+
/**
|
|
20
|
+
* Get the GPU adapter, requesting it only once.
|
|
21
|
+
*
|
|
22
|
+
* @example
|
|
23
|
+
* ```ts
|
|
24
|
+
* const adapter = await getGPUAdapter();
|
|
25
|
+
* ```
|
|
26
|
+
*/
|
|
27
|
+
export const getGPUAdapter = async () => {
|
|
28
|
+
pendingAdapter ?? (pendingAdapter = navigator.gpu.requestAdapter({
|
|
29
|
+
powerPreference: 'high-performance',
|
|
30
|
+
}));
|
|
31
|
+
return pendingAdapter;
|
|
32
|
+
};
|
|
33
|
+
const requestGPUDevice = async (adapter, requested) => {
|
|
4
34
|
const requiredFeatures = [];
|
|
5
35
|
if (hasR16FloatTextureStorage(adapter)) {
|
|
6
36
|
// https://gpuweb.github.io/gpuweb/#plain-color-formats
|
|
@@ -14,9 +44,61 @@ export const getGPUDevice = async (adapter) => {
|
|
|
14
44
|
// https://developer.chrome.com/blog/new-in-webgpu-120
|
|
15
45
|
requiredFeatures.push('shader-f16');
|
|
16
46
|
}
|
|
17
|
-
const device = await adapter
|
|
47
|
+
const device = await adapter.requestDevice({ requiredFeatures });
|
|
48
|
+
if (generation !== requested) {
|
|
49
|
+
device.destroy();
|
|
50
|
+
throw new GPUDeviceRequestInvalidatedError();
|
|
51
|
+
}
|
|
52
|
+
currentDevice = device;
|
|
53
|
+
void device.lost.then(() => {
|
|
54
|
+
// A lost device cannot be used again, so the next one is requested
|
|
55
|
+
if (currentDevice === device) {
|
|
56
|
+
reset();
|
|
57
|
+
}
|
|
58
|
+
});
|
|
18
59
|
return device;
|
|
19
60
|
};
|
|
61
|
+
/**
|
|
62
|
+
* Get the GPU device, requesting it only once.
|
|
63
|
+
*
|
|
64
|
+
* A device owns the memory of the pipelines and the shader modules created from
|
|
65
|
+
* it, and neither of those can be freed on their own, so it is shared by every
|
|
66
|
+
* renderer instead of being requested per renderer.
|
|
67
|
+
*
|
|
68
|
+
* @param adapter - The adapter to request the device from
|
|
69
|
+
*
|
|
70
|
+
* @example
|
|
71
|
+
* ```ts
|
|
72
|
+
* const device = await getGPUDevice(adapter);
|
|
73
|
+
* ```
|
|
74
|
+
*/
|
|
75
|
+
export const getGPUDevice = async (adapter) => {
|
|
76
|
+
if (!pendingDevice) {
|
|
77
|
+
const request = requestGPUDevice(adapter, generation);
|
|
78
|
+
pendingDevice = request;
|
|
79
|
+
void request.catch(() => {
|
|
80
|
+
// Do not clear a newer request started after invalidation.
|
|
81
|
+
if (pendingDevice === request) {
|
|
82
|
+
pendingDevice = undefined;
|
|
83
|
+
}
|
|
84
|
+
});
|
|
85
|
+
}
|
|
86
|
+
return pendingDevice;
|
|
87
|
+
};
|
|
88
|
+
/**
|
|
89
|
+
* Destroy the shared GPU device.
|
|
90
|
+
*
|
|
91
|
+
* @example
|
|
92
|
+
* ```ts
|
|
93
|
+
* destroyGPUDevice();
|
|
94
|
+
* ```
|
|
95
|
+
*/
|
|
96
|
+
export const destroyGPUDevice = () => {
|
|
97
|
+
// Invalidate an unresolved request before clearing the cache.
|
|
98
|
+
generation += 1;
|
|
99
|
+
currentDevice?.destroy();
|
|
100
|
+
reset();
|
|
101
|
+
};
|
|
20
102
|
/**
|
|
21
103
|
* Check if provided format is supported
|
|
22
104
|
*
|
|
@@ -1,4 +1,23 @@
|
|
|
1
|
-
|
|
1
|
+
/**
|
|
2
|
+
* Create a reusable converter that transforms a single-channel mask into
|
|
3
|
+
* RGBA ImageData with values written into the alpha channel.
|
|
4
|
+
* The returned function reuses the same ImageData buffer across calls.
|
|
5
|
+
*
|
|
6
|
+
* @example
|
|
7
|
+
* ```ts
|
|
8
|
+
* const toAlphaMask = createAlphaMaskConverter();
|
|
9
|
+
* const imageData = toAlphaMask(mask);
|
|
10
|
+
* ctx.putImageData(imageData, 0, 0);
|
|
11
|
+
* ```
|
|
12
|
+
*/
|
|
13
|
+
export declare const createAlphaMaskConverter: () => (mask: {
|
|
14
|
+
getAsUint8Array(): Uint8Array;
|
|
15
|
+
width: number;
|
|
16
|
+
height: number;
|
|
17
|
+
}) => ImageData;
|
|
18
|
+
export declare const createCpuMaskRenderer: (canvas: OffscreenCanvas) => {
|
|
19
|
+
render: (imageData: ImageData) => void;
|
|
20
|
+
};
|
|
2
21
|
declare const INTERNAL_CANVAS_KEYS: {
|
|
3
22
|
readonly drawImageDataCanvas: "drawImageDataCanvas";
|
|
4
23
|
readonly maskCanvas: "maskCanvas";
|
|
@@ -52,12 +71,12 @@ export declare const clearCanvas: (canvas: OffscreenCanvas) => void;
|
|
|
52
71
|
export declare const createCanvasRenderUtils: (processingWidth: number, processingHeight: number) => {
|
|
53
72
|
evaluateInput: (inputImage: CanvasImageSource) => OffscreenCanvas;
|
|
54
73
|
renderImageToCanvas: (image: CanvasImageSource, canvas: OffscreenCanvas, dw?: number, dh?: number, options?: CanvasRenderingContext2DOptions) => void;
|
|
55
|
-
drawBlurEffect: (canvas: OffscreenCanvas, inputImage: CanvasImageSource,
|
|
56
|
-
drawOverlayEffect: (canvas: OffscreenCanvas, inputImage: CanvasImageSource, backgroundImage: CanvasImageSource | OffscreenCanvas,
|
|
74
|
+
drawBlurEffect: (canvas: OffscreenCanvas, inputImage: CanvasImageSource, mask: CanvasImageSource | OffscreenCanvas, foregroundThreshold?: number, backgroundBlurAmount?: number, edgeBlurAmount?: number, flipHorizontal?: boolean) => void;
|
|
75
|
+
drawOverlayEffect: (canvas: OffscreenCanvas, inputImage: CanvasImageSource, backgroundImage: CanvasImageSource | OffscreenCanvas, mask: CanvasImageSource | OffscreenCanvas, foregroundThreshold?: number, backgroundBlurAmount?: number, edgeBlurAmount?: number, flipHorizontal?: boolean) => void;
|
|
57
76
|
renderImageToOffScreenCanvas: (image: CanvasImageSource, canvasName: keyof InternalCanvases) => OffscreenCanvas;
|
|
58
77
|
renderImageDataToOffScreenCanvas: (image: ImageData, canvasName: keyof InternalCanvases) => OffscreenCanvas;
|
|
59
78
|
drawAndBlurImageOnOffScreenCanvas: ({ image, blurAmount, offscreenCanvasName, preserveOldDrawing, }: {
|
|
60
|
-
image: CanvasImageSource;
|
|
79
|
+
image: CanvasImageSource | OffscreenCanvas;
|
|
61
80
|
blurAmount: number;
|
|
62
81
|
offscreenCanvasName: keyof InternalCanvases;
|
|
63
82
|
preserveOldDrawing?: boolean;
|