@stream-io/video-filters-web 0.8.6 → 0.8.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/dist/esm/index.js +10 -0
- package/dist/esm/index.js.map +1 -0
- package/dist/esm/src/BaseVideoProcessor.js +107 -0
- package/dist/esm/src/BaseVideoProcessor.js.map +1 -0
- package/dist/esm/src/FallbackGenerator.js +58 -0
- package/dist/esm/src/FallbackGenerator.js.map +1 -0
- package/dist/esm/src/FallbackProcessor.js +89 -0
- package/dist/esm/src/FallbackProcessor.js.map +1 -0
- package/dist/esm/src/FullScreenBlur.js +41 -0
- package/dist/esm/src/FullScreenBlur.js.map +1 -0
- package/dist/esm/src/FullScreenBlurRenderer.js +276 -0
- package/dist/esm/src/FullScreenBlurRenderer.js.map +1 -0
- package/dist/esm/src/VirtualBackground.js +143 -0
- package/dist/esm/src/VirtualBackground.js.map +1 -0
- package/dist/esm/src/WebGLRenderer.js +723 -0
- package/dist/esm/src/WebGLRenderer.js.map +1 -0
- package/dist/esm/src/compatibility.js +48 -0
- package/dist/esm/src/compatibility.js.map +1 -0
- package/dist/esm/src/legacy/createRenderer.js +33 -0
- package/dist/esm/src/legacy/createRenderer.js.map +1 -0
- package/dist/esm/src/legacy/helpers/webglHelper.js +94 -0
- package/dist/esm/src/legacy/helpers/webglHelper.js.map +1 -0
- package/dist/esm/src/legacy/segmentation.js +14 -0
- package/dist/esm/src/legacy/segmentation.js.map +1 -0
- package/dist/esm/src/legacy/tflite-simd.js +728 -0
- package/dist/esm/src/legacy/tflite-simd.js.map +1 -0
- package/dist/esm/src/legacy/tflite.js +31 -0
- package/dist/esm/src/legacy/tflite.js.map +1 -0
- package/dist/esm/src/legacy/webgl2/backgroundBlurStage.js +202 -0
- package/dist/esm/src/legacy/webgl2/backgroundBlurStage.js.map +1 -0
- package/dist/esm/src/legacy/webgl2/backgroundImageStage.js +157 -0
- package/dist/esm/src/legacy/webgl2/backgroundImageStage.js.map +1 -0
- package/dist/esm/src/legacy/webgl2/jointBilateralFilterStage.js +113 -0
- package/dist/esm/src/legacy/webgl2/jointBilateralFilterStage.js.map +1 -0
- package/dist/esm/src/legacy/webgl2/resizingStage.js +57 -0
- package/dist/esm/src/legacy/webgl2/resizingStage.js.map +1 -0
- package/dist/esm/src/legacy/webgl2/softmaxStage.js +51 -0
- package/dist/esm/src/legacy/webgl2/softmaxStage.js.map +1 -0
- package/dist/esm/src/legacy/webgl2/webgl2Pipeline.js +105 -0
- package/dist/esm/src/legacy/webgl2/webgl2Pipeline.js.map +1 -0
- package/dist/esm/src/mediapipe.js +16 -0
- package/dist/esm/src/mediapipe.js.map +1 -0
- package/dist/esm/src/types.js +11 -0
- package/dist/esm/src/types.js.map +1 -0
- package/dist/esm/src/version.js +5 -0
- package/dist/esm/src/version.js.map +1 -0
- package/dist/index.cjs.js +1 -1
- package/package.json +3 -3
- package/dist/index.es.js +0 -3022
- package/dist/index.es.js.map +0 -1
package/dist/index.es.js
DELETED
|
@@ -1,3022 +0,0 @@
|
|
|
1
|
-
import { simd } from 'wasm-feature-detect';
|
|
2
|
-
import { WorkerTimer } from '@stream-io/worker-timer';
|
|
3
|
-
import { FilesetResolver, ImageSegmenter } from '@mediapipe/tasks-vision';
|
|
4
|
-
|
|
5
|
-
/**
|
|
6
|
-
* Checks if the current platform is a mobile device.
|
|
7
|
-
*
|
|
8
|
-
* See:
|
|
9
|
-
* https://developer.mozilla.org/en-US/docs/Web/HTTP/Browser_detection_using_the_user_agent
|
|
10
|
-
*/
|
|
11
|
-
const isMobile = () => /Mobi/i.test(navigator.userAgent);
|
|
12
|
-
/**
|
|
13
|
-
* Checks whether the current browser is Safari.
|
|
14
|
-
*/
|
|
15
|
-
const isSafari = () => /^((?!chrome|android).)*safari/i.test(navigator.userAgent);
|
|
16
|
-
/**
|
|
17
|
-
* Runs a check to see if the current platform supports
|
|
18
|
-
* the necessary APIs required for the video filters.
|
|
19
|
-
*/
|
|
20
|
-
const isPlatformSupported = async ({ forceMobileSupport = false, forceSafariSupport = false, } = {}) => typeof document !== 'undefined' &&
|
|
21
|
-
typeof window !== 'undefined' &&
|
|
22
|
-
typeof navigator !== 'undefined' &&
|
|
23
|
-
// we don't support mobile devices yet due to performance issues
|
|
24
|
-
(forceMobileSupport || !isMobile()) &&
|
|
25
|
-
// Safari has issues with timer throttling, causing low FPS when the tab goes to the background
|
|
26
|
-
(forceSafariSupport || !isSafari()) &&
|
|
27
|
-
typeof WebAssembly !== 'undefined' &&
|
|
28
|
-
!!window.WebGL2RenderingContext && // WebGL2 is required for the video filters
|
|
29
|
-
!!document.createElement('canvas').getContext('webgl2') &&
|
|
30
|
-
(await simd()); // SIMD is required for the wasm module
|
|
31
|
-
/**
|
|
32
|
-
* Runs a check to see if the current platform supports
|
|
33
|
-
* the necessary APIs required for the MediaPipe-based video filters.
|
|
34
|
-
*/
|
|
35
|
-
const isMediaPipePlatformSupported = async ({ forceMobileSupport = false, forceSafariSupport = false, } = {}) => typeof document !== 'undefined' &&
|
|
36
|
-
typeof window !== 'undefined' &&
|
|
37
|
-
typeof navigator !== 'undefined' &&
|
|
38
|
-
// we don't support mobile devices yet due to performance issues
|
|
39
|
-
(forceMobileSupport || !isMobile()) &&
|
|
40
|
-
// Safari has issues with timer throttling, causing low FPS when the tab goes to the background
|
|
41
|
-
(forceSafariSupport || !isSafari()) &&
|
|
42
|
-
typeof WebAssembly !== 'undefined' &&
|
|
43
|
-
typeof OffscreenCanvas !== 'undefined' && // OffscreenCanvas is required for efficient rendering
|
|
44
|
-
!!window.WebGL2RenderingContext && // WebGL2 is required for the video filters
|
|
45
|
-
!!new OffscreenCanvas(1, 1).getContext('webgl2') &&
|
|
46
|
-
typeof VideoFrame !== 'undefined' && // VideoFrame API is required for frame processing
|
|
47
|
-
typeof createImageBitmap !== 'undefined'; // createImageBitmap is required for background image processing
|
|
48
|
-
|
|
49
|
-
/**
|
|
50
|
-
* Use it along with boyswan.glsl-literal VSCode extension
|
|
51
|
-
* to get GLSL syntax highlighting.
|
|
52
|
-
* https://marketplace.visualstudio.com/items?itemName=boyswan.glsl-literal
|
|
53
|
-
*
|
|
54
|
-
* On VSCode OSS, boyswan.glsl-literal requires slevesque.shader extension
|
|
55
|
-
* to be installed as well.
|
|
56
|
-
* https://marketplace.visualstudio.com/items?itemName=slevesque.shader
|
|
57
|
-
*/
|
|
58
|
-
const glsl = String.raw;
|
|
59
|
-
function createPipelineStageProgram(gl, vertexShader, fragmentShader, positionBuffer, texCoordBuffer) {
|
|
60
|
-
const program = createProgram(gl, vertexShader, fragmentShader);
|
|
61
|
-
const positionAttributeLocation = gl.getAttribLocation(program, 'a_position');
|
|
62
|
-
gl.enableVertexAttribArray(positionAttributeLocation);
|
|
63
|
-
gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
|
|
64
|
-
gl.vertexAttribPointer(positionAttributeLocation, 2, gl.FLOAT, false, 0, 0);
|
|
65
|
-
const texCoordAttributeLocation = gl.getAttribLocation(program, 'a_texCoord');
|
|
66
|
-
gl.enableVertexAttribArray(texCoordAttributeLocation);
|
|
67
|
-
gl.bindBuffer(gl.ARRAY_BUFFER, texCoordBuffer);
|
|
68
|
-
gl.vertexAttribPointer(texCoordAttributeLocation, 2, gl.FLOAT, false, 0, 0);
|
|
69
|
-
return program;
|
|
70
|
-
}
|
|
71
|
-
function createProgram(gl, vertexShader, fragmentShader) {
|
|
72
|
-
const program = gl.createProgram();
|
|
73
|
-
gl.attachShader(program, vertexShader);
|
|
74
|
-
gl.attachShader(program, fragmentShader);
|
|
75
|
-
gl.linkProgram(program);
|
|
76
|
-
if (!gl.getProgramParameter(program, gl.LINK_STATUS)) {
|
|
77
|
-
throw new Error(`Could not link WebGL program: ${gl.getProgramInfoLog(program)}`);
|
|
78
|
-
}
|
|
79
|
-
return program;
|
|
80
|
-
}
|
|
81
|
-
function compileShader(gl, shaderType, shaderSource) {
|
|
82
|
-
const shader = gl.createShader(shaderType);
|
|
83
|
-
gl.shaderSource(shader, shaderSource);
|
|
84
|
-
gl.compileShader(shader);
|
|
85
|
-
if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) {
|
|
86
|
-
throw new Error(`Could not compile shader: ${gl.getShaderInfoLog(shader)}`);
|
|
87
|
-
}
|
|
88
|
-
return shader;
|
|
89
|
-
}
|
|
90
|
-
function createTexture(gl, internalformat, width, height, minFilter = gl.NEAREST, magFilter = gl.NEAREST) {
|
|
91
|
-
const texture = gl.createTexture();
|
|
92
|
-
gl.bindTexture(gl.TEXTURE_2D, texture);
|
|
93
|
-
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
|
|
94
|
-
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
|
|
95
|
-
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, minFilter);
|
|
96
|
-
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, magFilter);
|
|
97
|
-
gl.texStorage2D(gl.TEXTURE_2D, 1, internalformat, width, height);
|
|
98
|
-
return texture;
|
|
99
|
-
}
|
|
100
|
-
async function readPixelsAsync(gl, x, y, width, height, format, type, dest) {
|
|
101
|
-
const buf = gl.createBuffer();
|
|
102
|
-
gl.bindBuffer(gl.PIXEL_PACK_BUFFER, buf);
|
|
103
|
-
gl.bufferData(gl.PIXEL_PACK_BUFFER, dest.byteLength, gl.STREAM_READ);
|
|
104
|
-
gl.readPixels(x, y, width, height, format, type, 0);
|
|
105
|
-
gl.bindBuffer(gl.PIXEL_PACK_BUFFER, null);
|
|
106
|
-
await getBufferSubDataAsync(gl, gl.PIXEL_PACK_BUFFER, buf, 0, dest);
|
|
107
|
-
gl.deleteBuffer(buf);
|
|
108
|
-
return dest;
|
|
109
|
-
}
|
|
110
|
-
async function getBufferSubDataAsync(gl, target, buffer, srcByteOffset, dstBuffer, dstOffset, length) {
|
|
111
|
-
const sync = gl.fenceSync(gl.SYNC_GPU_COMMANDS_COMPLETE, 0);
|
|
112
|
-
gl.flush();
|
|
113
|
-
if (!sync)
|
|
114
|
-
return;
|
|
115
|
-
const res = await clientWaitAsync(gl, sync);
|
|
116
|
-
gl.deleteSync(sync);
|
|
117
|
-
if (res !== gl.WAIT_FAILED) {
|
|
118
|
-
gl.bindBuffer(target, buffer);
|
|
119
|
-
gl.getBufferSubData(target, srcByteOffset, dstBuffer, dstOffset, length);
|
|
120
|
-
gl.bindBuffer(target, null);
|
|
121
|
-
}
|
|
122
|
-
}
|
|
123
|
-
function clientWaitAsync(gl, sync) {
|
|
124
|
-
return new Promise((resolve) => {
|
|
125
|
-
function test() {
|
|
126
|
-
const res = gl.clientWaitSync(sync, 0, 0);
|
|
127
|
-
if (res === gl.WAIT_FAILED) {
|
|
128
|
-
resolve(res);
|
|
129
|
-
return;
|
|
130
|
-
}
|
|
131
|
-
if (res === gl.TIMEOUT_EXPIRED) {
|
|
132
|
-
setTimeout(test);
|
|
133
|
-
return;
|
|
134
|
-
}
|
|
135
|
-
resolve(res);
|
|
136
|
-
}
|
|
137
|
-
setTimeout(test);
|
|
138
|
-
});
|
|
139
|
-
}
|
|
140
|
-
|
|
141
|
-
function buildBackgroundBlurStage(gl, vertexShader, positionBuffer, texCoordBuffer, personMaskTexture, canvas, blurLevel) {
|
|
142
|
-
const blurPass = buildBlurPass(gl, vertexShader, positionBuffer, texCoordBuffer, personMaskTexture, canvas, blurLevel);
|
|
143
|
-
const blendPass = buildBlendPass(gl, positionBuffer, texCoordBuffer, canvas);
|
|
144
|
-
function render() {
|
|
145
|
-
blurPass.render();
|
|
146
|
-
blendPass.render();
|
|
147
|
-
}
|
|
148
|
-
function updateCoverage(coverage) {
|
|
149
|
-
blendPass.updateCoverage(coverage);
|
|
150
|
-
}
|
|
151
|
-
function cleanUp() {
|
|
152
|
-
blendPass.cleanUp();
|
|
153
|
-
blurPass.cleanUp();
|
|
154
|
-
}
|
|
155
|
-
return {
|
|
156
|
-
render,
|
|
157
|
-
updateCoverage,
|
|
158
|
-
cleanUp,
|
|
159
|
-
};
|
|
160
|
-
}
|
|
161
|
-
function buildBlurPass(gl, vertexShader, positionBuffer, texCoordBuffer, personMaskTexture, canvas, blurLevel) {
|
|
162
|
-
const sigma = typeof blurLevel === 'number'
|
|
163
|
-
? blurLevel
|
|
164
|
-
: blurLevel === 'low'
|
|
165
|
-
? 2
|
|
166
|
-
: blurLevel === 'medium'
|
|
167
|
-
? 4
|
|
168
|
-
: 6;
|
|
169
|
-
const windowSize = Math.max(1, Math.floor(sigma * 3));
|
|
170
|
-
const offset = new Array(windowSize).fill(0).map((v, index) => index);
|
|
171
|
-
const variance = sigma ** 2;
|
|
172
|
-
const weights = offset.map((x) => {
|
|
173
|
-
const m = sigma * Math.sqrt(2 * Math.PI);
|
|
174
|
-
const e = Math.exp(-(x ** 2) / (2 * variance));
|
|
175
|
-
return e / m;
|
|
176
|
-
});
|
|
177
|
-
const fragmentShaderSource = glsl `#version 300 es
|
|
178
|
-
|
|
179
|
-
precision highp float;
|
|
180
|
-
|
|
181
|
-
uniform sampler2D u_inputFrame;
|
|
182
|
-
uniform sampler2D u_personMask;
|
|
183
|
-
uniform vec2 u_texelSize;
|
|
184
|
-
|
|
185
|
-
in vec2 v_texCoord;
|
|
186
|
-
out vec4 outColor;
|
|
187
|
-
|
|
188
|
-
const float offset[${windowSize}] = float[](${offset.map((i) => i.toFixed(10)).join(', ')});
|
|
189
|
-
const float weight[${windowSize}] = float[](${weights.map((i) => i.toFixed(10)).join(', ')});
|
|
190
|
-
|
|
191
|
-
void main() {
|
|
192
|
-
vec4 centerColor = texture(u_inputFrame, v_texCoord);
|
|
193
|
-
float personMask = texture(u_personMask, v_texCoord).a;
|
|
194
|
-
|
|
195
|
-
vec4 frameColor = centerColor * weight[0] * (1.0 - personMask);
|
|
196
|
-
|
|
197
|
-
for (int i = 1; i < ${windowSize}; i++) {
|
|
198
|
-
vec2 offset = vec2(offset[i]) * u_texelSize;
|
|
199
|
-
|
|
200
|
-
vec2 texCoord = v_texCoord + offset;
|
|
201
|
-
frameColor += texture(u_inputFrame, texCoord)
|
|
202
|
-
* weight[i]
|
|
203
|
-
* (1.0 - texture(u_personMask, texCoord).a);
|
|
204
|
-
|
|
205
|
-
texCoord = v_texCoord - offset;
|
|
206
|
-
frameColor += texture(u_inputFrame, texCoord)
|
|
207
|
-
* weight[i]
|
|
208
|
-
* (1.0 - texture(u_personMask, texCoord).a);
|
|
209
|
-
}
|
|
210
|
-
outColor = vec4(frameColor.rgb + (1.0 - frameColor.a) * centerColor.rgb, 1.0);
|
|
211
|
-
}
|
|
212
|
-
`;
|
|
213
|
-
const scale = 0.5;
|
|
214
|
-
const outputWidth = canvas.width * scale;
|
|
215
|
-
const outputHeight = canvas.height * scale;
|
|
216
|
-
const texelWidth = 1 / outputWidth;
|
|
217
|
-
const texelHeight = 1 / outputHeight;
|
|
218
|
-
const fragmentShader = compileShader(gl, gl.FRAGMENT_SHADER, fragmentShaderSource);
|
|
219
|
-
const program = createPipelineStageProgram(gl, vertexShader, fragmentShader, positionBuffer, texCoordBuffer);
|
|
220
|
-
const inputFrameLocation = gl.getUniformLocation(program, 'u_inputFrame');
|
|
221
|
-
const personMaskLocation = gl.getUniformLocation(program, 'u_personMask');
|
|
222
|
-
const texelSizeLocation = gl.getUniformLocation(program, 'u_texelSize');
|
|
223
|
-
const texture1 = createTexture(gl, gl.RGBA8, outputWidth, outputHeight, gl.NEAREST,
|
|
224
|
-
// @ts-expect-error types are incomplete
|
|
225
|
-
gl.LINEAR);
|
|
226
|
-
const texture2 = createTexture(gl, gl.RGBA8, outputWidth, outputHeight, gl.NEAREST,
|
|
227
|
-
// @ts-expect-error types are incomplete
|
|
228
|
-
gl.LINEAR);
|
|
229
|
-
const frameBuffer1 = gl.createFramebuffer();
|
|
230
|
-
gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer1);
|
|
231
|
-
gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, texture1, 0);
|
|
232
|
-
const frameBuffer2 = gl.createFramebuffer();
|
|
233
|
-
gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer2);
|
|
234
|
-
gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, texture2, 0);
|
|
235
|
-
gl.useProgram(program);
|
|
236
|
-
gl.uniform1i(personMaskLocation, 1);
|
|
237
|
-
function render() {
|
|
238
|
-
gl.viewport(0, 0, outputWidth, outputHeight);
|
|
239
|
-
gl.useProgram(program);
|
|
240
|
-
gl.uniform1i(inputFrameLocation, 0);
|
|
241
|
-
gl.activeTexture(gl.TEXTURE1);
|
|
242
|
-
gl.bindTexture(gl.TEXTURE_2D, personMaskTexture);
|
|
243
|
-
for (let i = 0; i < 3; i++) {
|
|
244
|
-
gl.uniform2f(texelSizeLocation, 0, texelHeight);
|
|
245
|
-
gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer1);
|
|
246
|
-
gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4);
|
|
247
|
-
gl.activeTexture(gl.TEXTURE2);
|
|
248
|
-
gl.bindTexture(gl.TEXTURE_2D, texture1);
|
|
249
|
-
gl.uniform1i(inputFrameLocation, 2);
|
|
250
|
-
gl.uniform2f(texelSizeLocation, texelWidth, 0);
|
|
251
|
-
gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer2);
|
|
252
|
-
gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4);
|
|
253
|
-
gl.bindTexture(gl.TEXTURE_2D, texture2);
|
|
254
|
-
}
|
|
255
|
-
}
|
|
256
|
-
function cleanUp() {
|
|
257
|
-
gl.deleteFramebuffer(frameBuffer2);
|
|
258
|
-
gl.deleteFramebuffer(frameBuffer1);
|
|
259
|
-
gl.deleteTexture(texture2);
|
|
260
|
-
gl.deleteTexture(texture1);
|
|
261
|
-
gl.deleteProgram(program);
|
|
262
|
-
gl.deleteShader(fragmentShader);
|
|
263
|
-
}
|
|
264
|
-
return {
|
|
265
|
-
render,
|
|
266
|
-
cleanUp,
|
|
267
|
-
};
|
|
268
|
-
}
|
|
269
|
-
function buildBlendPass(gl, positionBuffer, texCoordBuffer, canvas) {
|
|
270
|
-
const vertexShaderSource = glsl `#version 300 es
|
|
271
|
-
|
|
272
|
-
in vec2 a_position;
|
|
273
|
-
in vec2 a_texCoord;
|
|
274
|
-
|
|
275
|
-
out vec2 v_texCoord;
|
|
276
|
-
|
|
277
|
-
void main() {
|
|
278
|
-
// Flipping Y is required when rendering to canvas
|
|
279
|
-
gl_Position = vec4(a_position * vec2(1.0, -1.0), 0.0, 1.0);
|
|
280
|
-
v_texCoord = a_texCoord;
|
|
281
|
-
}
|
|
282
|
-
`;
|
|
283
|
-
const fragmentShaderSource = glsl `#version 300 es
|
|
284
|
-
|
|
285
|
-
precision highp float;
|
|
286
|
-
|
|
287
|
-
uniform sampler2D u_inputFrame;
|
|
288
|
-
uniform sampler2D u_personMask;
|
|
289
|
-
uniform sampler2D u_blurredInputFrame;
|
|
290
|
-
uniform vec2 u_coverage;
|
|
291
|
-
|
|
292
|
-
in vec2 v_texCoord;
|
|
293
|
-
|
|
294
|
-
out vec4 outColor;
|
|
295
|
-
|
|
296
|
-
void main() {
|
|
297
|
-
vec3 color = texture(u_inputFrame, v_texCoord).rgb;
|
|
298
|
-
vec3 blurredColor = texture(u_blurredInputFrame, v_texCoord).rgb;
|
|
299
|
-
float personMask = texture(u_personMask, v_texCoord).a;
|
|
300
|
-
personMask = smoothstep(u_coverage.x, u_coverage.y, personMask);
|
|
301
|
-
outColor = vec4(mix(blurredColor, color, personMask), 1.0);
|
|
302
|
-
}
|
|
303
|
-
`;
|
|
304
|
-
const { width: outputWidth, height: outputHeight } = canvas;
|
|
305
|
-
const vertexShader = compileShader(gl, gl.VERTEX_SHADER, vertexShaderSource);
|
|
306
|
-
const fragmentShader = compileShader(gl, gl.FRAGMENT_SHADER, fragmentShaderSource);
|
|
307
|
-
const program = createPipelineStageProgram(gl, vertexShader, fragmentShader, positionBuffer, texCoordBuffer);
|
|
308
|
-
const inputFrameLocation = gl.getUniformLocation(program, 'u_inputFrame');
|
|
309
|
-
const personMaskLocation = gl.getUniformLocation(program, 'u_personMask');
|
|
310
|
-
const blurredInputFrame = gl.getUniformLocation(program, 'u_blurredInputFrame');
|
|
311
|
-
const coverageLocation = gl.getUniformLocation(program, 'u_coverage');
|
|
312
|
-
gl.useProgram(program);
|
|
313
|
-
gl.uniform1i(inputFrameLocation, 0);
|
|
314
|
-
gl.uniform1i(personMaskLocation, 1);
|
|
315
|
-
gl.uniform1i(blurredInputFrame, 2);
|
|
316
|
-
gl.uniform2f(coverageLocation, 0, 1);
|
|
317
|
-
function render() {
|
|
318
|
-
gl.viewport(0, 0, outputWidth, outputHeight);
|
|
319
|
-
gl.useProgram(program);
|
|
320
|
-
gl.bindFramebuffer(gl.FRAMEBUFFER, null);
|
|
321
|
-
gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4);
|
|
322
|
-
}
|
|
323
|
-
function updateCoverage(coverage) {
|
|
324
|
-
gl.useProgram(program);
|
|
325
|
-
gl.uniform2f(coverageLocation, coverage[0], coverage[1]);
|
|
326
|
-
}
|
|
327
|
-
function cleanUp() {
|
|
328
|
-
gl.deleteProgram(program);
|
|
329
|
-
gl.deleteShader(fragmentShader);
|
|
330
|
-
gl.deleteShader(vertexShader);
|
|
331
|
-
}
|
|
332
|
-
return {
|
|
333
|
-
render,
|
|
334
|
-
updateCoverage,
|
|
335
|
-
cleanUp,
|
|
336
|
-
};
|
|
337
|
-
}
|
|
338
|
-
|
|
339
|
-
function buildBackgroundImageStage(gl, positionBuffer, texCoordBuffer, personMaskTexture, backgroundImage, canvas) {
|
|
340
|
-
const vertexShaderSource = glsl `#version 300 es
|
|
341
|
-
|
|
342
|
-
uniform vec2 u_backgroundScale;
|
|
343
|
-
uniform vec2 u_backgroundOffset;
|
|
344
|
-
|
|
345
|
-
in vec2 a_position;
|
|
346
|
-
in vec2 a_texCoord;
|
|
347
|
-
|
|
348
|
-
out vec2 v_texCoord;
|
|
349
|
-
out vec2 v_backgroundCoord;
|
|
350
|
-
|
|
351
|
-
void main() {
|
|
352
|
-
// Flipping Y is required when rendering to canvas
|
|
353
|
-
gl_Position = vec4(a_position * vec2(1.0, -1.0), 0.0, 1.0);
|
|
354
|
-
v_texCoord = a_texCoord;
|
|
355
|
-
v_backgroundCoord = a_texCoord * u_backgroundScale + u_backgroundOffset;
|
|
356
|
-
}
|
|
357
|
-
`;
|
|
358
|
-
const fragmentShaderSource = glsl `#version 300 es
|
|
359
|
-
|
|
360
|
-
precision highp float;
|
|
361
|
-
|
|
362
|
-
uniform sampler2D u_inputFrame;
|
|
363
|
-
uniform sampler2D u_personMask;
|
|
364
|
-
uniform sampler2D u_background;
|
|
365
|
-
uniform vec2 u_coverage;
|
|
366
|
-
uniform float u_lightWrapping;
|
|
367
|
-
uniform float u_blendMode;
|
|
368
|
-
|
|
369
|
-
in vec2 v_texCoord;
|
|
370
|
-
in vec2 v_backgroundCoord;
|
|
371
|
-
|
|
372
|
-
out vec4 outColor;
|
|
373
|
-
|
|
374
|
-
vec3 screen(vec3 a, vec3 b) {
|
|
375
|
-
return 1.0 - (1.0 - a) * (1.0 - b);
|
|
376
|
-
}
|
|
377
|
-
|
|
378
|
-
vec3 linearDodge(vec3 a, vec3 b) {
|
|
379
|
-
return a + b;
|
|
380
|
-
}
|
|
381
|
-
|
|
382
|
-
void main() {
|
|
383
|
-
vec3 frameColor = texture(u_inputFrame, v_texCoord).rgb;
|
|
384
|
-
vec3 backgroundColor = texture(u_background, v_backgroundCoord).rgb;
|
|
385
|
-
float personMask = texture(u_personMask, v_texCoord).a;
|
|
386
|
-
float lightWrapMask = 1.0 - max(0.0, personMask - u_coverage.y) / (1.0 - u_coverage.y);
|
|
387
|
-
vec3 lightWrap = u_lightWrapping * lightWrapMask * backgroundColor;
|
|
388
|
-
|
|
389
|
-
frameColor = u_blendMode * linearDodge(frameColor, lightWrap)
|
|
390
|
-
+ (1.0 - u_blendMode) * screen(frameColor, lightWrap);
|
|
391
|
-
personMask = smoothstep(u_coverage.x, u_coverage.y, personMask);
|
|
392
|
-
outColor = vec4(frameColor * personMask + backgroundColor * (1.0 - personMask), 1.0);
|
|
393
|
-
}
|
|
394
|
-
`;
|
|
395
|
-
const { width: outputWidth, height: outputHeight } = canvas;
|
|
396
|
-
const outputRatio = outputWidth / outputHeight;
|
|
397
|
-
const vertexShader = compileShader(gl, gl.VERTEX_SHADER, vertexShaderSource);
|
|
398
|
-
const fragmentShader = compileShader(gl, gl.FRAGMENT_SHADER, fragmentShaderSource);
|
|
399
|
-
const program = createPipelineStageProgram(gl, vertexShader, fragmentShader, positionBuffer, texCoordBuffer);
|
|
400
|
-
const backgroundScaleLocation = gl.getUniformLocation(program, 'u_backgroundScale');
|
|
401
|
-
const backgroundOffsetLocation = gl.getUniformLocation(program, 'u_backgroundOffset');
|
|
402
|
-
const inputFrameLocation = gl.getUniformLocation(program, 'u_inputFrame');
|
|
403
|
-
const personMaskLocation = gl.getUniformLocation(program, 'u_personMask');
|
|
404
|
-
const backgroundLocation = gl.getUniformLocation(program, 'u_background');
|
|
405
|
-
const coverageLocation = gl.getUniformLocation(program, 'u_coverage');
|
|
406
|
-
const lightWrappingLocation = gl.getUniformLocation(program, 'u_lightWrapping');
|
|
407
|
-
const blendModeLocation = gl.getUniformLocation(program, 'u_blendMode');
|
|
408
|
-
gl.useProgram(program);
|
|
409
|
-
gl.uniform2f(backgroundScaleLocation, 1, 1);
|
|
410
|
-
gl.uniform2f(backgroundOffsetLocation, 0, 0);
|
|
411
|
-
gl.uniform1i(inputFrameLocation, 0);
|
|
412
|
-
gl.uniform1i(personMaskLocation, 1);
|
|
413
|
-
gl.uniform2f(coverageLocation, 0, 1);
|
|
414
|
-
gl.uniform1f(lightWrappingLocation, 0);
|
|
415
|
-
gl.uniform1f(blendModeLocation, 0);
|
|
416
|
-
let backgroundTexture = null;
|
|
417
|
-
// TODO Find a better to handle background being loaded
|
|
418
|
-
if (backgroundImage?.complete) {
|
|
419
|
-
updateBackgroundImage(backgroundImage);
|
|
420
|
-
}
|
|
421
|
-
else if (backgroundImage) {
|
|
422
|
-
backgroundImage.onload = () => {
|
|
423
|
-
updateBackgroundImage(backgroundImage);
|
|
424
|
-
};
|
|
425
|
-
}
|
|
426
|
-
function render() {
|
|
427
|
-
gl.viewport(0, 0, outputWidth, outputHeight);
|
|
428
|
-
gl.useProgram(program);
|
|
429
|
-
gl.activeTexture(gl.TEXTURE1);
|
|
430
|
-
gl.bindTexture(gl.TEXTURE_2D, personMaskTexture);
|
|
431
|
-
if (backgroundTexture !== null) {
|
|
432
|
-
gl.activeTexture(gl.TEXTURE2);
|
|
433
|
-
gl.bindTexture(gl.TEXTURE_2D, backgroundTexture);
|
|
434
|
-
// TODO Handle correctly the background not loaded yet
|
|
435
|
-
gl.uniform1i(backgroundLocation, 2);
|
|
436
|
-
}
|
|
437
|
-
gl.bindFramebuffer(gl.FRAMEBUFFER, null);
|
|
438
|
-
gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4);
|
|
439
|
-
}
|
|
440
|
-
function updateBackgroundImage(bgImage) {
|
|
441
|
-
backgroundTexture = createTexture(gl, gl.RGBA8, bgImage.naturalWidth, bgImage.naturalHeight,
|
|
442
|
-
// @ts-expect-error types are incomplete
|
|
443
|
-
gl.LINEAR, gl.LINEAR);
|
|
444
|
-
gl.texSubImage2D(gl.TEXTURE_2D, 0, 0, 0, bgImage.naturalWidth, bgImage.naturalHeight, gl.RGBA, gl.UNSIGNED_BYTE, bgImage);
|
|
445
|
-
let xOffset = 0;
|
|
446
|
-
let yOffset = 0;
|
|
447
|
-
let backgroundWidth = bgImage.naturalWidth;
|
|
448
|
-
let backgroundHeight = bgImage.naturalHeight;
|
|
449
|
-
const backgroundRatio = backgroundWidth / backgroundHeight;
|
|
450
|
-
if (backgroundRatio < outputRatio) {
|
|
451
|
-
backgroundHeight = backgroundWidth / outputRatio;
|
|
452
|
-
yOffset = (bgImage.naturalHeight - backgroundHeight) / 2;
|
|
453
|
-
}
|
|
454
|
-
else {
|
|
455
|
-
backgroundWidth = backgroundHeight * outputRatio;
|
|
456
|
-
xOffset = (bgImage.naturalWidth - backgroundWidth) / 2;
|
|
457
|
-
}
|
|
458
|
-
const xScale = backgroundWidth / bgImage.naturalWidth;
|
|
459
|
-
const yScale = backgroundHeight / bgImage.naturalHeight;
|
|
460
|
-
xOffset /= bgImage.naturalWidth;
|
|
461
|
-
yOffset /= bgImage.naturalHeight;
|
|
462
|
-
gl.uniform2f(backgroundScaleLocation, xScale, yScale);
|
|
463
|
-
gl.uniform2f(backgroundOffsetLocation, xOffset, yOffset);
|
|
464
|
-
}
|
|
465
|
-
function updateCoverage(coverage) {
|
|
466
|
-
gl.useProgram(program);
|
|
467
|
-
gl.uniform2f(coverageLocation, coverage[0], coverage[1]);
|
|
468
|
-
}
|
|
469
|
-
function updateLightWrapping(lightWrapping) {
|
|
470
|
-
gl.useProgram(program);
|
|
471
|
-
gl.uniform1f(lightWrappingLocation, lightWrapping);
|
|
472
|
-
}
|
|
473
|
-
function updateBlendMode(blendMode) {
|
|
474
|
-
gl.useProgram(program);
|
|
475
|
-
gl.uniform1f(blendModeLocation, blendMode === 'screen' ? 0 : 1);
|
|
476
|
-
}
|
|
477
|
-
function cleanUp() {
|
|
478
|
-
gl.deleteTexture(backgroundTexture);
|
|
479
|
-
gl.deleteProgram(program);
|
|
480
|
-
gl.deleteShader(fragmentShader);
|
|
481
|
-
gl.deleteShader(vertexShader);
|
|
482
|
-
}
|
|
483
|
-
return {
|
|
484
|
-
render,
|
|
485
|
-
updateCoverage,
|
|
486
|
-
updateLightWrapping,
|
|
487
|
-
updateBlendMode,
|
|
488
|
-
cleanUp,
|
|
489
|
-
};
|
|
490
|
-
}
|
|
491
|
-
|
|
492
|
-
function buildJointBilateralFilterStage(gl, vertexShader, positionBuffer, texCoordBuffer, inputTexture, outputTexture, canvas, segmentationConfig) {
|
|
493
|
-
const fragmentShaderSource = glsl `#version 300 es
|
|
494
|
-
|
|
495
|
-
precision highp float;
|
|
496
|
-
|
|
497
|
-
uniform sampler2D u_inputFrame;
|
|
498
|
-
uniform sampler2D u_segmentationMask;
|
|
499
|
-
uniform vec2 u_texelSize;
|
|
500
|
-
uniform float u_step;
|
|
501
|
-
uniform float u_radius;
|
|
502
|
-
uniform float u_offset;
|
|
503
|
-
uniform float u_sigmaTexel;
|
|
504
|
-
uniform float u_sigmaColor;
|
|
505
|
-
|
|
506
|
-
in vec2 v_texCoord;
|
|
507
|
-
out vec4 outColor;
|
|
508
|
-
|
|
509
|
-
float gaussian(float x, float sigma) {
|
|
510
|
-
float coeff = -0.5 / (sigma * sigma * 4.0 + 1.0e-6);
|
|
511
|
-
return exp((x * x) * coeff);
|
|
512
|
-
}
|
|
513
|
-
|
|
514
|
-
void main() {
|
|
515
|
-
vec2 centerCoord = v_texCoord;
|
|
516
|
-
vec3 centerColor = texture(u_inputFrame, centerCoord).rgb;
|
|
517
|
-
float newVal = 0.0;
|
|
518
|
-
|
|
519
|
-
float spaceWeight = 0.0;
|
|
520
|
-
float colorWeight = 0.0;
|
|
521
|
-
float totalWeight = 0.0;
|
|
522
|
-
|
|
523
|
-
// Subsample kernel space.
|
|
524
|
-
for (float i = -u_radius + u_offset; i <= u_radius; i += u_step) {
|
|
525
|
-
for (float j = -u_radius + u_offset; j <= u_radius; j += u_step) {
|
|
526
|
-
vec2 shift = vec2(j, i) * u_texelSize;
|
|
527
|
-
vec2 coord = vec2(centerCoord + shift);
|
|
528
|
-
vec3 frameColor = texture(u_inputFrame, coord).rgb;
|
|
529
|
-
float outVal = texture(u_segmentationMask, coord).a;
|
|
530
|
-
|
|
531
|
-
spaceWeight = gaussian(distance(centerCoord, coord), u_sigmaTexel);
|
|
532
|
-
colorWeight = gaussian(distance(centerColor, frameColor), u_sigmaColor);
|
|
533
|
-
totalWeight += spaceWeight * colorWeight;
|
|
534
|
-
|
|
535
|
-
newVal += spaceWeight * colorWeight * outVal;
|
|
536
|
-
}
|
|
537
|
-
}
|
|
538
|
-
newVal /= totalWeight;
|
|
539
|
-
|
|
540
|
-
outColor = vec4(vec3(0.0), newVal);
|
|
541
|
-
}
|
|
542
|
-
`;
|
|
543
|
-
const { width: segmentationWidth, height: segmentationHeight } = segmentationConfig;
|
|
544
|
-
const { width: outputWidth, height: outputHeight } = canvas;
|
|
545
|
-
const texelWidth = 1 / outputWidth;
|
|
546
|
-
const texelHeight = 1 / outputHeight;
|
|
547
|
-
const fragmentShader = compileShader(gl, gl.FRAGMENT_SHADER, fragmentShaderSource);
|
|
548
|
-
const program = createPipelineStageProgram(gl, vertexShader, fragmentShader, positionBuffer, texCoordBuffer);
|
|
549
|
-
const inputFrameLocation = gl.getUniformLocation(program, 'u_inputFrame');
|
|
550
|
-
const segmentationMaskLocation = gl.getUniformLocation(program, 'u_segmentationMask');
|
|
551
|
-
const texelSizeLocation = gl.getUniformLocation(program, 'u_texelSize');
|
|
552
|
-
const stepLocation = gl.getUniformLocation(program, 'u_step');
|
|
553
|
-
const radiusLocation = gl.getUniformLocation(program, 'u_radius');
|
|
554
|
-
const offsetLocation = gl.getUniformLocation(program, 'u_offset');
|
|
555
|
-
const sigmaTexelLocation = gl.getUniformLocation(program, 'u_sigmaTexel');
|
|
556
|
-
const sigmaColorLocation = gl.getUniformLocation(program, 'u_sigmaColor');
|
|
557
|
-
const frameBuffer = gl.createFramebuffer();
|
|
558
|
-
gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer);
|
|
559
|
-
gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, outputTexture, 0);
|
|
560
|
-
gl.useProgram(program);
|
|
561
|
-
gl.uniform1i(inputFrameLocation, 0);
|
|
562
|
-
gl.uniform1i(segmentationMaskLocation, 1);
|
|
563
|
-
gl.uniform2f(texelSizeLocation, texelWidth, texelHeight);
|
|
564
|
-
// Ensures default values are configured to prevent infinite
|
|
565
|
-
// loop in fragment shader
|
|
566
|
-
updateSigmaSpace(0);
|
|
567
|
-
updateSigmaColor(0);
|
|
568
|
-
function render() {
|
|
569
|
-
gl.viewport(0, 0, outputWidth, outputHeight);
|
|
570
|
-
gl.useProgram(program);
|
|
571
|
-
gl.activeTexture(gl.TEXTURE1);
|
|
572
|
-
gl.bindTexture(gl.TEXTURE_2D, inputTexture);
|
|
573
|
-
gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer);
|
|
574
|
-
gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4);
|
|
575
|
-
}
|
|
576
|
-
function updateSigmaSpace(sigmaSpace) {
|
|
577
|
-
sigmaSpace *= Math.max(outputWidth / segmentationWidth, outputHeight / segmentationHeight);
|
|
578
|
-
const kSparsityFactor = 0.66; // Higher is sparser.
|
|
579
|
-
const step = Math.max(1, Math.sqrt(sigmaSpace) * kSparsityFactor);
|
|
580
|
-
const radius = sigmaSpace;
|
|
581
|
-
const offset = step > 1 ? step * 0.5 : 0;
|
|
582
|
-
const sigmaTexel = Math.max(texelWidth, texelHeight) * sigmaSpace;
|
|
583
|
-
gl.useProgram(program);
|
|
584
|
-
gl.uniform1f(stepLocation, step);
|
|
585
|
-
gl.uniform1f(radiusLocation, radius);
|
|
586
|
-
gl.uniform1f(offsetLocation, offset);
|
|
587
|
-
gl.uniform1f(sigmaTexelLocation, sigmaTexel);
|
|
588
|
-
}
|
|
589
|
-
function updateSigmaColor(sigmaColor) {
|
|
590
|
-
gl.useProgram(program);
|
|
591
|
-
gl.uniform1f(sigmaColorLocation, sigmaColor);
|
|
592
|
-
}
|
|
593
|
-
function cleanUp() {
|
|
594
|
-
gl.deleteFramebuffer(frameBuffer);
|
|
595
|
-
gl.deleteProgram(program);
|
|
596
|
-
gl.deleteShader(fragmentShader);
|
|
597
|
-
}
|
|
598
|
-
return { render, updateSigmaSpace, updateSigmaColor, cleanUp };
|
|
599
|
-
}
|
|
600
|
-
|
|
601
|
-
function buildResizingStage(gl, vertexShader, positionBuffer, texCoordBuffer, tflite, segmentationConfig, onError) {
|
|
602
|
-
const fragmentShaderSource = glsl `#version 300 es
|
|
603
|
-
|
|
604
|
-
precision highp float;
|
|
605
|
-
uniform sampler2D u_inputFrame;
|
|
606
|
-
in vec2 v_texCoord;
|
|
607
|
-
out vec4 outColor;
|
|
608
|
-
|
|
609
|
-
void main() {
|
|
610
|
-
outColor = texture(u_inputFrame, v_texCoord);
|
|
611
|
-
}
|
|
612
|
-
`;
|
|
613
|
-
// TFLite memory will be accessed as float32
|
|
614
|
-
const tfliteInputMemoryOffset = tflite._getInputMemoryOffset() / 4;
|
|
615
|
-
const { width: outputWidth, height: outputHeight } = segmentationConfig;
|
|
616
|
-
const outputPixelCount = outputWidth * outputHeight;
|
|
617
|
-
const fragmentShader = compileShader(gl, gl.FRAGMENT_SHADER, fragmentShaderSource);
|
|
618
|
-
const program = createPipelineStageProgram(gl, vertexShader, fragmentShader, positionBuffer, texCoordBuffer);
|
|
619
|
-
const inputFrameLocation = gl.getUniformLocation(program, 'u_inputFrame');
|
|
620
|
-
const outputTexture = createTexture(gl, gl.RGBA8, outputWidth, outputHeight);
|
|
621
|
-
const frameBuffer = gl.createFramebuffer();
|
|
622
|
-
gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer);
|
|
623
|
-
gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, outputTexture, 0);
|
|
624
|
-
const outputPixels = new Uint8Array(outputPixelCount * 4);
|
|
625
|
-
gl.useProgram(program);
|
|
626
|
-
gl.uniform1i(inputFrameLocation, 0);
|
|
627
|
-
function render() {
|
|
628
|
-
gl.viewport(0, 0, outputWidth, outputHeight);
|
|
629
|
-
gl.useProgram(program);
|
|
630
|
-
gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer);
|
|
631
|
-
gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4);
|
|
632
|
-
// Downloads pixels asynchronously from GPU while rendering the current frame.
|
|
633
|
-
// The pixels will be available in the next frame render which results
|
|
634
|
-
// in offsets in the segmentation output but increases the frame rate.
|
|
635
|
-
readPixelsAsync(gl, 0, 0, outputWidth, outputHeight, gl.RGBA, gl.UNSIGNED_BYTE, outputPixels).catch((error) => {
|
|
636
|
-
});
|
|
637
|
-
for (let i = 0; i < outputPixelCount; i++) {
|
|
638
|
-
const tfliteIndex = tfliteInputMemoryOffset + i * 3;
|
|
639
|
-
const outputIndex = i * 4;
|
|
640
|
-
tflite.HEAPF32[tfliteIndex] = outputPixels[outputIndex] / 255;
|
|
641
|
-
tflite.HEAPF32[tfliteIndex + 1] = outputPixels[outputIndex + 1] / 255;
|
|
642
|
-
tflite.HEAPF32[tfliteIndex + 2] = outputPixels[outputIndex + 2] / 255;
|
|
643
|
-
}
|
|
644
|
-
}
|
|
645
|
-
function cleanUp() {
|
|
646
|
-
gl.deleteFramebuffer(frameBuffer);
|
|
647
|
-
gl.deleteTexture(outputTexture);
|
|
648
|
-
gl.deleteProgram(program);
|
|
649
|
-
gl.deleteShader(fragmentShader);
|
|
650
|
-
}
|
|
651
|
-
return { render, cleanUp };
|
|
652
|
-
}
|
|
653
|
-
|
|
654
|
-
function buildSoftmaxStage(gl, vertexShader, positionBuffer, texCoordBuffer, tflite, outputTexture, segmentationConfig) {
|
|
655
|
-
const fragmentShaderSource = glsl `#version 300 es
|
|
656
|
-
|
|
657
|
-
precision highp float;
|
|
658
|
-
|
|
659
|
-
uniform sampler2D u_inputSegmentation;
|
|
660
|
-
in vec2 v_texCoord;
|
|
661
|
-
out vec4 outColor;
|
|
662
|
-
|
|
663
|
-
void main() {
|
|
664
|
-
vec2 segmentation = texture(u_inputSegmentation, v_texCoord).rg;
|
|
665
|
-
float shift = max(segmentation.r, segmentation.g);
|
|
666
|
-
float backgroundExp = exp(segmentation.r - shift);
|
|
667
|
-
float personExp = exp(segmentation.g - shift);
|
|
668
|
-
outColor = vec4(vec3(0.0), personExp / (backgroundExp + personExp));
|
|
669
|
-
}
|
|
670
|
-
`;
|
|
671
|
-
// TFLite memory will be accessed as float32
|
|
672
|
-
const tfliteOutputMemoryOffset = tflite._getOutputMemoryOffset() / 4;
|
|
673
|
-
const { width: segmentationWidth, height: segmentationHeight } = segmentationConfig;
|
|
674
|
-
const fragmentShader = compileShader(gl, gl.FRAGMENT_SHADER, fragmentShaderSource);
|
|
675
|
-
const program = createPipelineStageProgram(gl, vertexShader, fragmentShader, positionBuffer, texCoordBuffer);
|
|
676
|
-
const inputLocation = gl.getUniformLocation(program, 'u_inputSegmentation');
|
|
677
|
-
const inputTexture = createTexture(gl, gl.RG32F, segmentationWidth, segmentationHeight);
|
|
678
|
-
const frameBuffer = gl.createFramebuffer();
|
|
679
|
-
gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer);
|
|
680
|
-
gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, outputTexture, 0);
|
|
681
|
-
gl.useProgram(program);
|
|
682
|
-
gl.uniform1i(inputLocation, 1);
|
|
683
|
-
function render() {
|
|
684
|
-
gl.viewport(0, 0, segmentationWidth, segmentationHeight);
|
|
685
|
-
gl.useProgram(program);
|
|
686
|
-
gl.activeTexture(gl.TEXTURE1);
|
|
687
|
-
gl.bindTexture(gl.TEXTURE_2D, inputTexture);
|
|
688
|
-
gl.texSubImage2D(gl.TEXTURE_2D, 0, 0, 0, segmentationWidth, segmentationHeight, gl.RG, gl.FLOAT, tflite.HEAPF32, tfliteOutputMemoryOffset);
|
|
689
|
-
gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer);
|
|
690
|
-
gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4);
|
|
691
|
-
}
|
|
692
|
-
function cleanUp() {
|
|
693
|
-
gl.deleteFramebuffer(frameBuffer);
|
|
694
|
-
gl.deleteTexture(inputTexture);
|
|
695
|
-
gl.deleteProgram(program);
|
|
696
|
-
gl.deleteShader(fragmentShader);
|
|
697
|
-
}
|
|
698
|
-
return { render, cleanUp };
|
|
699
|
-
}
|
|
700
|
-
|
|
701
|
-
function buildWebGL2Pipeline(videoSource, backgroundImage, blurLevel, backgroundFilter, canvas, tflite, segmentationConfig, onError) {
|
|
702
|
-
const gl = canvas.getContext('webgl2');
|
|
703
|
-
if (!gl)
|
|
704
|
-
throw new Error('WebGL2 is not supported');
|
|
705
|
-
if (gl.isContextLost())
|
|
706
|
-
throw new Error('WebGL2 context was lost');
|
|
707
|
-
const { width: frameWidth, height: frameHeight } = videoSource;
|
|
708
|
-
const { width: segmentationWidth, height: segmentationHeight } = segmentationConfig;
|
|
709
|
-
const vertexShaderSource = glsl `#version 300 es
|
|
710
|
-
|
|
711
|
-
in vec2 a_position;
|
|
712
|
-
in vec2 a_texCoord;
|
|
713
|
-
out vec2 v_texCoord;
|
|
714
|
-
|
|
715
|
-
void main() {
|
|
716
|
-
gl_Position = vec4(a_position, 0.0, 1.0);
|
|
717
|
-
v_texCoord = a_texCoord;
|
|
718
|
-
}
|
|
719
|
-
`;
|
|
720
|
-
const vertexShader = compileShader(gl, gl.VERTEX_SHADER, vertexShaderSource);
|
|
721
|
-
const vertexArray = gl.createVertexArray();
|
|
722
|
-
gl.bindVertexArray(vertexArray);
|
|
723
|
-
const positionBuffer = gl.createBuffer();
|
|
724
|
-
gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
|
|
725
|
-
gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([-1, -1, 1.0, -1, -1, 1.0, 1.0, 1.0]), gl.STATIC_DRAW);
|
|
726
|
-
const texCoordBuffer = gl.createBuffer();
|
|
727
|
-
gl.bindBuffer(gl.ARRAY_BUFFER, texCoordBuffer);
|
|
728
|
-
gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([0.0, 0.0, 1.0, 0.0, 0.0, 1.0, 1.0, 1.0]), gl.STATIC_DRAW);
|
|
729
|
-
// We don't use texStorage2D here because texImage2D seems faster
|
|
730
|
-
// to upload video texture than texSubImage2D even though the latter
|
|
731
|
-
// is supposed to be the recommended way:
|
|
732
|
-
// https://developer.mozilla.org/en-US/docs/Web/API/WebGL_API/WebGL_best_practices#use_texstorage_to_create_textures
|
|
733
|
-
const inputFrameTexture = gl.createTexture();
|
|
734
|
-
gl.bindTexture(gl.TEXTURE_2D, inputFrameTexture);
|
|
735
|
-
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
|
|
736
|
-
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
|
|
737
|
-
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
|
|
738
|
-
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
|
|
739
|
-
// TODO Rename segmentation and person mask to be more specific
|
|
740
|
-
const segmentationTexture = createTexture(gl, gl.RGBA8, segmentationWidth, segmentationHeight);
|
|
741
|
-
const personMaskTexture = createTexture(gl, gl.RGBA8, frameWidth, frameHeight);
|
|
742
|
-
const resizingStage = buildResizingStage(gl, vertexShader, positionBuffer, texCoordBuffer, tflite, segmentationConfig);
|
|
743
|
-
const loadSegmentationStage = buildSoftmaxStage(gl, vertexShader, positionBuffer, texCoordBuffer, tflite, segmentationTexture, segmentationConfig);
|
|
744
|
-
const jointBilateralFilterStage = buildJointBilateralFilterStage(gl, vertexShader, positionBuffer, texCoordBuffer, segmentationTexture, personMaskTexture, canvas, segmentationConfig);
|
|
745
|
-
const backgroundStage = backgroundFilter === 'blur'
|
|
746
|
-
? buildBackgroundBlurStage(gl, vertexShader, positionBuffer, texCoordBuffer, personMaskTexture, canvas, blurLevel || 'high')
|
|
747
|
-
: buildBackgroundImageStage(gl, positionBuffer, texCoordBuffer, personMaskTexture, backgroundImage, canvas);
|
|
748
|
-
function render() {
|
|
749
|
-
gl.activeTexture(gl.TEXTURE0);
|
|
750
|
-
gl.bindTexture(gl.TEXTURE_2D, inputFrameTexture);
|
|
751
|
-
// texImage2D seems faster than texSubImage2D to upload
|
|
752
|
-
// video texture
|
|
753
|
-
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, videoSource);
|
|
754
|
-
gl.bindVertexArray(vertexArray);
|
|
755
|
-
resizingStage.render();
|
|
756
|
-
tflite._runInference();
|
|
757
|
-
loadSegmentationStage.render();
|
|
758
|
-
jointBilateralFilterStage.render();
|
|
759
|
-
backgroundStage.render();
|
|
760
|
-
}
|
|
761
|
-
function updatePostProcessingConfig() {
|
|
762
|
-
jointBilateralFilterStage.updateSigmaSpace(1);
|
|
763
|
-
jointBilateralFilterStage.updateSigmaColor(0.1);
|
|
764
|
-
if (backgroundFilter === 'image') {
|
|
765
|
-
const backgroundImageStage = backgroundStage;
|
|
766
|
-
backgroundImageStage.updateCoverage([0.5, 0.75]);
|
|
767
|
-
backgroundImageStage.updateLightWrapping(0.3);
|
|
768
|
-
backgroundImageStage.updateBlendMode('screen');
|
|
769
|
-
}
|
|
770
|
-
else if (backgroundFilter === 'blur') {
|
|
771
|
-
const backgroundBlurStage = backgroundStage;
|
|
772
|
-
backgroundBlurStage.updateCoverage([0.5, 0.75]);
|
|
773
|
-
}
|
|
774
|
-
else {
|
|
775
|
-
// TODO Handle no background in a separate pipeline path
|
|
776
|
-
const backgroundImageStage = backgroundStage;
|
|
777
|
-
backgroundImageStage.updateCoverage([0, 0.9999]);
|
|
778
|
-
backgroundImageStage.updateLightWrapping(0);
|
|
779
|
-
}
|
|
780
|
-
}
|
|
781
|
-
function cleanUp() {
|
|
782
|
-
backgroundStage.cleanUp();
|
|
783
|
-
jointBilateralFilterStage.cleanUp();
|
|
784
|
-
loadSegmentationStage.cleanUp();
|
|
785
|
-
resizingStage.cleanUp();
|
|
786
|
-
gl.deleteTexture(personMaskTexture);
|
|
787
|
-
gl.deleteTexture(segmentationTexture);
|
|
788
|
-
gl.deleteTexture(inputFrameTexture);
|
|
789
|
-
gl.deleteBuffer(texCoordBuffer);
|
|
790
|
-
gl.deleteBuffer(positionBuffer);
|
|
791
|
-
gl.deleteVertexArray(vertexArray);
|
|
792
|
-
gl.deleteShader(vertexShader);
|
|
793
|
-
}
|
|
794
|
-
return { render, updatePostProcessingConfig, cleanUp };
|
|
795
|
-
}
|
|
796
|
-
|
|
797
|
-
var SegmentationLevel;
|
|
798
|
-
(function (SegmentationLevel) {
|
|
799
|
-
SegmentationLevel["LOW"] = "low";
|
|
800
|
-
SegmentationLevel["HIGH"] = "high";
|
|
801
|
-
})(SegmentationLevel || (SegmentationLevel = {}));
|
|
802
|
-
const getSegmentationParams = (level) => {
|
|
803
|
-
if (level === SegmentationLevel.HIGH) {
|
|
804
|
-
return { width: 256, height: 144 };
|
|
805
|
-
}
|
|
806
|
-
return { width: 160, height: 96 };
|
|
807
|
-
};
|
|
808
|
-
|
|
809
|
-
function createRenderer(tflite, videoSource, targetCanvas, options, onError) {
|
|
810
|
-
const { backgroundFilter, backgroundImage, backgroundBlurLevel, segmentationLevel = SegmentationLevel.HIGH, fps = 30, } = options;
|
|
811
|
-
if (backgroundFilter === 'image' && !backgroundImage) {
|
|
812
|
-
throw new Error(`backgroundImage element is required when backgroundFilter is image`);
|
|
813
|
-
}
|
|
814
|
-
const pipeline = buildWebGL2Pipeline(videoSource, backgroundImage, backgroundBlurLevel, backgroundFilter, targetCanvas, tflite, getSegmentationParams(segmentationLevel));
|
|
815
|
-
const timers = new WorkerTimer({ useWorker: true });
|
|
816
|
-
const id = timers.setInterval(() => {
|
|
817
|
-
try {
|
|
818
|
-
pipeline.render();
|
|
819
|
-
if (backgroundFilter === 'image') {
|
|
820
|
-
pipeline.updatePostProcessingConfig();
|
|
821
|
-
}
|
|
822
|
-
}
|
|
823
|
-
catch (error) {
|
|
824
|
-
onError?.(error);
|
|
825
|
-
}
|
|
826
|
-
}, Math.floor(1000 / (fps <= 0 ? 30 : fps)));
|
|
827
|
-
return {
|
|
828
|
-
dispose: () => {
|
|
829
|
-
pipeline.cleanUp();
|
|
830
|
-
timers.clearInterval(id);
|
|
831
|
-
timers.destroy();
|
|
832
|
-
},
|
|
833
|
-
};
|
|
834
|
-
}
|
|
835
|
-
|
|
836
|
-
const createTFLiteSIMDModule = (__Module) => {
|
|
837
|
-
__Module = __Module || {};
|
|
838
|
-
|
|
839
|
-
var _scriptDir =
|
|
840
|
-
typeof document !== 'undefined' && document.currentScript
|
|
841
|
-
? document.currentScript.src
|
|
842
|
-
: undefined;
|
|
843
|
-
|
|
844
|
-
var Module = typeof __Module != 'undefined' ? __Module : {};
|
|
845
|
-
var readyPromiseResolve, readyPromiseReject;
|
|
846
|
-
Module['ready'] = new Promise(function (resolve, reject) {
|
|
847
|
-
readyPromiseResolve = resolve;
|
|
848
|
-
readyPromiseReject = reject;
|
|
849
|
-
});
|
|
850
|
-
var moduleOverrides = Object.assign({}, Module);
|
|
851
|
-
var thisProgram = './this.program';
|
|
852
|
-
var quit_ = (status, toThrow) => {
|
|
853
|
-
throw toThrow;
|
|
854
|
-
};
|
|
855
|
-
var ENVIRONMENT_IS_WEB = true;
|
|
856
|
-
var scriptDirectory = '';
|
|
857
|
-
|
|
858
|
-
function locateFile(path) {
|
|
859
|
-
if (Module['locateFile']) {
|
|
860
|
-
return Module['locateFile'](path, scriptDirectory);
|
|
861
|
-
}
|
|
862
|
-
return scriptDirectory + path;
|
|
863
|
-
}
|
|
864
|
-
|
|
865
|
-
var readBinary;
|
|
866
|
-
{
|
|
867
|
-
if (typeof document != 'undefined' && document.currentScript) {
|
|
868
|
-
scriptDirectory = document.currentScript.src;
|
|
869
|
-
}
|
|
870
|
-
if (_scriptDir) {
|
|
871
|
-
scriptDirectory = _scriptDir;
|
|
872
|
-
}
|
|
873
|
-
if (scriptDirectory.indexOf('blob:') !== 0) {
|
|
874
|
-
scriptDirectory = scriptDirectory.substr(
|
|
875
|
-
0,
|
|
876
|
-
scriptDirectory.replace(/[?#].*/, '').lastIndexOf('/') + 1,
|
|
877
|
-
);
|
|
878
|
-
} else {
|
|
879
|
-
scriptDirectory = '';
|
|
880
|
-
}
|
|
881
|
-
}
|
|
882
|
-
var out = Module['print'] || console.log.bind(console);
|
|
883
|
-
var err = Module['printErr'] || console.warn.bind(console);
|
|
884
|
-
Object.assign(Module, moduleOverrides);
|
|
885
|
-
moduleOverrides = null;
|
|
886
|
-
if (Module['arguments']) Module['arguments'];
|
|
887
|
-
if (Module['thisProgram']) thisProgram = Module['thisProgram'];
|
|
888
|
-
if (Module['quit']) quit_ = Module['quit'];
|
|
889
|
-
var wasmBinary;
|
|
890
|
-
if (Module['wasmBinary']) wasmBinary = Module['wasmBinary'];
|
|
891
|
-
Module['noExitRuntime'] || true;
|
|
892
|
-
if (typeof WebAssembly != 'object') {
|
|
893
|
-
abort('no native wasm support detected');
|
|
894
|
-
}
|
|
895
|
-
var wasmMemory;
|
|
896
|
-
var ABORT = false;
|
|
897
|
-
|
|
898
|
-
var UTF8Decoder =
|
|
899
|
-
typeof TextDecoder != 'undefined' ? new TextDecoder('utf8') : undefined;
|
|
900
|
-
|
|
901
|
-
function UTF8ArrayToString(heapOrArray, idx, maxBytesToRead) {
|
|
902
|
-
var endIdx = idx + maxBytesToRead;
|
|
903
|
-
var endPtr = idx;
|
|
904
|
-
while (heapOrArray[endPtr] && !(endPtr >= endIdx)) ++endPtr;
|
|
905
|
-
if (endPtr - idx > 16 && heapOrArray.buffer && UTF8Decoder) {
|
|
906
|
-
return UTF8Decoder.decode(heapOrArray.subarray(idx, endPtr));
|
|
907
|
-
}
|
|
908
|
-
var str = '';
|
|
909
|
-
while (idx < endPtr) {
|
|
910
|
-
var u0 = heapOrArray[idx++];
|
|
911
|
-
if (!(u0 & 128)) {
|
|
912
|
-
str += String.fromCharCode(u0);
|
|
913
|
-
continue;
|
|
914
|
-
}
|
|
915
|
-
var u1 = heapOrArray[idx++] & 63;
|
|
916
|
-
if ((u0 & 224) == 192) {
|
|
917
|
-
str += String.fromCharCode(((u0 & 31) << 6) | u1);
|
|
918
|
-
continue;
|
|
919
|
-
}
|
|
920
|
-
var u2 = heapOrArray[idx++] & 63;
|
|
921
|
-
if ((u0 & 240) == 224) {
|
|
922
|
-
u0 = ((u0 & 15) << 12) | (u1 << 6) | u2;
|
|
923
|
-
} else {
|
|
924
|
-
u0 =
|
|
925
|
-
((u0 & 7) << 18) | (u1 << 12) | (u2 << 6) | (heapOrArray[idx++] & 63);
|
|
926
|
-
}
|
|
927
|
-
if (u0 < 65536) {
|
|
928
|
-
str += String.fromCharCode(u0);
|
|
929
|
-
} else {
|
|
930
|
-
var ch = u0 - 65536;
|
|
931
|
-
str += String.fromCharCode(55296 | (ch >> 10), 56320 | (ch & 1023));
|
|
932
|
-
}
|
|
933
|
-
}
|
|
934
|
-
return str;
|
|
935
|
-
}
|
|
936
|
-
|
|
937
|
-
var buffer, HEAP8, HEAPU8, HEAPU32;
|
|
938
|
-
|
|
939
|
-
function updateGlobalBufferAndViews(buf) {
|
|
940
|
-
buffer = buf;
|
|
941
|
-
Module['HEAP8'] = HEAP8 = new Int8Array(buf);
|
|
942
|
-
Module['HEAP16'] = new Int16Array(buf);
|
|
943
|
-
Module['HEAP32'] = new Int32Array(buf);
|
|
944
|
-
Module['HEAPU8'] = HEAPU8 = new Uint8Array(buf);
|
|
945
|
-
Module['HEAPU16'] = new Uint16Array(buf);
|
|
946
|
-
Module['HEAPU32'] = HEAPU32 = new Uint32Array(buf);
|
|
947
|
-
Module['HEAPF32'] = new Float32Array(buf);
|
|
948
|
-
Module['HEAPF64'] = new Float64Array(buf);
|
|
949
|
-
}
|
|
950
|
-
|
|
951
|
-
Module['INITIAL_MEMORY'] || 16777216;
|
|
952
|
-
var __ATPRERUN__ = [];
|
|
953
|
-
var __ATINIT__ = [];
|
|
954
|
-
var __ATPOSTRUN__ = [];
|
|
955
|
-
|
|
956
|
-
function preRun() {
|
|
957
|
-
if (Module['preRun']) {
|
|
958
|
-
if (typeof Module['preRun'] == 'function')
|
|
959
|
-
Module['preRun'] = [Module['preRun']];
|
|
960
|
-
while (Module['preRun'].length) {
|
|
961
|
-
addOnPreRun(Module['preRun'].shift());
|
|
962
|
-
}
|
|
963
|
-
}
|
|
964
|
-
callRuntimeCallbacks(__ATPRERUN__);
|
|
965
|
-
}
|
|
966
|
-
|
|
967
|
-
function initRuntime() {
|
|
968
|
-
callRuntimeCallbacks(__ATINIT__);
|
|
969
|
-
}
|
|
970
|
-
|
|
971
|
-
function postRun() {
|
|
972
|
-
if (Module['postRun']) {
|
|
973
|
-
if (typeof Module['postRun'] == 'function')
|
|
974
|
-
Module['postRun'] = [Module['postRun']];
|
|
975
|
-
while (Module['postRun'].length) {
|
|
976
|
-
addOnPostRun(Module['postRun'].shift());
|
|
977
|
-
}
|
|
978
|
-
}
|
|
979
|
-
callRuntimeCallbacks(__ATPOSTRUN__);
|
|
980
|
-
}
|
|
981
|
-
|
|
982
|
-
function addOnPreRun(cb) {
|
|
983
|
-
__ATPRERUN__.unshift(cb);
|
|
984
|
-
}
|
|
985
|
-
|
|
986
|
-
function addOnInit(cb) {
|
|
987
|
-
__ATINIT__.unshift(cb);
|
|
988
|
-
}
|
|
989
|
-
|
|
990
|
-
function addOnPostRun(cb) {
|
|
991
|
-
__ATPOSTRUN__.unshift(cb);
|
|
992
|
-
}
|
|
993
|
-
|
|
994
|
-
var runDependencies = 0;
|
|
995
|
-
var dependenciesFulfilled = null;
|
|
996
|
-
|
|
997
|
-
function addRunDependency(id) {
|
|
998
|
-
runDependencies++;
|
|
999
|
-
if (Module['monitorRunDependencies']) {
|
|
1000
|
-
Module['monitorRunDependencies'](runDependencies);
|
|
1001
|
-
}
|
|
1002
|
-
}
|
|
1003
|
-
|
|
1004
|
-
function removeRunDependency(id) {
|
|
1005
|
-
runDependencies--;
|
|
1006
|
-
if (Module['monitorRunDependencies']) {
|
|
1007
|
-
Module['monitorRunDependencies'](runDependencies);
|
|
1008
|
-
}
|
|
1009
|
-
if (runDependencies == 0) {
|
|
1010
|
-
if (dependenciesFulfilled) {
|
|
1011
|
-
var callback = dependenciesFulfilled;
|
|
1012
|
-
dependenciesFulfilled = null;
|
|
1013
|
-
callback();
|
|
1014
|
-
}
|
|
1015
|
-
}
|
|
1016
|
-
}
|
|
1017
|
-
|
|
1018
|
-
function abort(what) {
|
|
1019
|
-
{
|
|
1020
|
-
if (Module['onAbort']) {
|
|
1021
|
-
Module['onAbort'](what);
|
|
1022
|
-
}
|
|
1023
|
-
}
|
|
1024
|
-
what = 'Aborted(' + what + ')';
|
|
1025
|
-
err(what);
|
|
1026
|
-
ABORT = true;
|
|
1027
|
-
what += '. Build with -sASSERTIONS for more info.';
|
|
1028
|
-
var e = new WebAssembly.RuntimeError(what);
|
|
1029
|
-
readyPromiseReject(e);
|
|
1030
|
-
throw e;
|
|
1031
|
-
}
|
|
1032
|
-
|
|
1033
|
-
var dataURIPrefix = 'data:application/octet-stream;base64,';
|
|
1034
|
-
|
|
1035
|
-
function isDataURI(filename) {
|
|
1036
|
-
return filename.startsWith(dataURIPrefix);
|
|
1037
|
-
}
|
|
1038
|
-
|
|
1039
|
-
var wasmBinaryFile;
|
|
1040
|
-
wasmBinaryFile = 'tflite-simd.wasm';
|
|
1041
|
-
if (!isDataURI(wasmBinaryFile)) {
|
|
1042
|
-
wasmBinaryFile = locateFile(wasmBinaryFile);
|
|
1043
|
-
}
|
|
1044
|
-
|
|
1045
|
-
function getBinary(file) {
|
|
1046
|
-
try {
|
|
1047
|
-
if (file == wasmBinaryFile && wasmBinary) {
|
|
1048
|
-
return new Uint8Array(wasmBinary);
|
|
1049
|
-
}
|
|
1050
|
-
if (readBinary) ;
|
|
1051
|
-
throw 'both async and sync fetching of the wasm failed';
|
|
1052
|
-
// eslint-disable-next-line @typescript-eslint/no-shadow
|
|
1053
|
-
} catch (err) {
|
|
1054
|
-
abort(err);
|
|
1055
|
-
}
|
|
1056
|
-
}
|
|
1057
|
-
|
|
1058
|
-
function getBinaryPromise() {
|
|
1059
|
-
if (!wasmBinary && (ENVIRONMENT_IS_WEB)) {
|
|
1060
|
-
if (typeof fetch == 'function') {
|
|
1061
|
-
return fetch(wasmBinaryFile, { credentials: 'same-origin' })
|
|
1062
|
-
.then(function (response) {
|
|
1063
|
-
if (!response['ok']) {
|
|
1064
|
-
throw (
|
|
1065
|
-
"failed to load wasm binary file at '" + wasmBinaryFile + "'"
|
|
1066
|
-
);
|
|
1067
|
-
}
|
|
1068
|
-
return response['arrayBuffer']();
|
|
1069
|
-
})
|
|
1070
|
-
.catch(function () {
|
|
1071
|
-
return getBinary(wasmBinaryFile);
|
|
1072
|
-
});
|
|
1073
|
-
}
|
|
1074
|
-
}
|
|
1075
|
-
return Promise.resolve().then(function () {
|
|
1076
|
-
return getBinary(wasmBinaryFile);
|
|
1077
|
-
});
|
|
1078
|
-
}
|
|
1079
|
-
|
|
1080
|
-
function createWasm() {
|
|
1081
|
-
var info = {
|
|
1082
|
-
env: asmLibraryArg,
|
|
1083
|
-
wasi_snapshot_preview1: asmLibraryArg,
|
|
1084
|
-
};
|
|
1085
|
-
|
|
1086
|
-
function receiveInstance(instance, module) {
|
|
1087
|
-
var exports = instance.exports;
|
|
1088
|
-
Module['asm'] = exports;
|
|
1089
|
-
wasmMemory = Module['asm']['memory'];
|
|
1090
|
-
updateGlobalBufferAndViews(wasmMemory.buffer);
|
|
1091
|
-
Module['asm']['__indirect_function_table'];
|
|
1092
|
-
addOnInit(Module['asm']['__wasm_call_ctors']);
|
|
1093
|
-
removeRunDependency();
|
|
1094
|
-
}
|
|
1095
|
-
|
|
1096
|
-
addRunDependency();
|
|
1097
|
-
|
|
1098
|
-
function receiveInstantiationResult(result) {
|
|
1099
|
-
receiveInstance(result['instance']);
|
|
1100
|
-
}
|
|
1101
|
-
|
|
1102
|
-
function instantiateArrayBuffer(receiver) {
|
|
1103
|
-
return getBinaryPromise()
|
|
1104
|
-
.then(function (binary) {
|
|
1105
|
-
return WebAssembly.instantiate(binary, info);
|
|
1106
|
-
})
|
|
1107
|
-
.then(function (instance) {
|
|
1108
|
-
return instance;
|
|
1109
|
-
})
|
|
1110
|
-
.then(receiver, function (reason) {
|
|
1111
|
-
err('failed to asynchronously prepare wasm: ' + reason);
|
|
1112
|
-
abort(reason);
|
|
1113
|
-
});
|
|
1114
|
-
}
|
|
1115
|
-
|
|
1116
|
-
function instantiateAsync() {
|
|
1117
|
-
if (
|
|
1118
|
-
!wasmBinary &&
|
|
1119
|
-
typeof WebAssembly.instantiateStreaming == 'function' &&
|
|
1120
|
-
!isDataURI(wasmBinaryFile) &&
|
|
1121
|
-
typeof fetch == 'function'
|
|
1122
|
-
) {
|
|
1123
|
-
return fetch(wasmBinaryFile, { credentials: 'same-origin' }).then(
|
|
1124
|
-
function (response) {
|
|
1125
|
-
var result = WebAssembly.instantiateStreaming(response, info);
|
|
1126
|
-
return result.then(receiveInstantiationResult, function (reason) {
|
|
1127
|
-
err('wasm streaming compile failed: ' + reason);
|
|
1128
|
-
err('falling back to ArrayBuffer instantiation');
|
|
1129
|
-
return instantiateArrayBuffer(receiveInstantiationResult);
|
|
1130
|
-
});
|
|
1131
|
-
},
|
|
1132
|
-
);
|
|
1133
|
-
} else {
|
|
1134
|
-
return instantiateArrayBuffer(receiveInstantiationResult);
|
|
1135
|
-
}
|
|
1136
|
-
}
|
|
1137
|
-
|
|
1138
|
-
if (Module['instantiateWasm']) {
|
|
1139
|
-
try {
|
|
1140
|
-
var exports = Module['instantiateWasm'](info, receiveInstance);
|
|
1141
|
-
return exports;
|
|
1142
|
-
} catch (e) {
|
|
1143
|
-
err('Module.instantiateWasm callback failed with error: ' + e);
|
|
1144
|
-
readyPromiseReject(e);
|
|
1145
|
-
}
|
|
1146
|
-
}
|
|
1147
|
-
instantiateAsync().catch(readyPromiseReject);
|
|
1148
|
-
return {};
|
|
1149
|
-
}
|
|
1150
|
-
|
|
1151
|
-
function ExitStatus(status) {
|
|
1152
|
-
this.name = 'ExitStatus';
|
|
1153
|
-
this.message = 'Program terminated with exit(' + status + ')';
|
|
1154
|
-
this.status = status;
|
|
1155
|
-
}
|
|
1156
|
-
|
|
1157
|
-
function callRuntimeCallbacks(callbacks) {
|
|
1158
|
-
while (callbacks.length > 0) {
|
|
1159
|
-
callbacks.shift()(Module);
|
|
1160
|
-
}
|
|
1161
|
-
}
|
|
1162
|
-
|
|
1163
|
-
function __dlinit(main_dso_handle) {}
|
|
1164
|
-
|
|
1165
|
-
var dlopenMissingError =
|
|
1166
|
-
'To use dlopen, you need enable dynamic linking, see https://github.com/emscripten-core/emscripten/wiki/Linking';
|
|
1167
|
-
|
|
1168
|
-
function __dlopen_js(filename, flag) {
|
|
1169
|
-
abort(dlopenMissingError);
|
|
1170
|
-
}
|
|
1171
|
-
|
|
1172
|
-
function __dlsym_js(handle, symbol) {
|
|
1173
|
-
abort(dlopenMissingError);
|
|
1174
|
-
}
|
|
1175
|
-
|
|
1176
|
-
var nowIsMonotonic = true;
|
|
1177
|
-
|
|
1178
|
-
function __emscripten_get_now_is_monotonic() {
|
|
1179
|
-
return nowIsMonotonic;
|
|
1180
|
-
}
|
|
1181
|
-
|
|
1182
|
-
function __mmap_js(len, prot, flags, fd, off, allocated) {
|
|
1183
|
-
return -52;
|
|
1184
|
-
}
|
|
1185
|
-
|
|
1186
|
-
function __munmap_js(addr, len, prot, flags, fd, offset) {}
|
|
1187
|
-
|
|
1188
|
-
function _abort() {
|
|
1189
|
-
abort('');
|
|
1190
|
-
}
|
|
1191
|
-
|
|
1192
|
-
function _emscripten_date_now() {
|
|
1193
|
-
return Date.now();
|
|
1194
|
-
}
|
|
1195
|
-
|
|
1196
|
-
function getHeapMax() {
|
|
1197
|
-
return 2147483648;
|
|
1198
|
-
}
|
|
1199
|
-
|
|
1200
|
-
function _emscripten_get_heap_max() {
|
|
1201
|
-
return getHeapMax();
|
|
1202
|
-
}
|
|
1203
|
-
|
|
1204
|
-
var _emscripten_get_now;
|
|
1205
|
-
_emscripten_get_now = () => performance.now();
|
|
1206
|
-
|
|
1207
|
-
function _emscripten_memcpy_big(dest, src, num) {
|
|
1208
|
-
HEAPU8.copyWithin(dest, src, src + num);
|
|
1209
|
-
}
|
|
1210
|
-
|
|
1211
|
-
function emscripten_realloc_buffer(size) {
|
|
1212
|
-
try {
|
|
1213
|
-
wasmMemory.grow((size - buffer.byteLength + 65535) >>> 16);
|
|
1214
|
-
updateGlobalBufferAndViews(wasmMemory.buffer);
|
|
1215
|
-
return 1;
|
|
1216
|
-
} catch (e) {}
|
|
1217
|
-
}
|
|
1218
|
-
|
|
1219
|
-
function _emscripten_resize_heap(requestedSize) {
|
|
1220
|
-
var oldSize = HEAPU8.length;
|
|
1221
|
-
requestedSize = requestedSize >>> 0;
|
|
1222
|
-
var maxHeapSize = getHeapMax();
|
|
1223
|
-
if (requestedSize > maxHeapSize) {
|
|
1224
|
-
return false;
|
|
1225
|
-
}
|
|
1226
|
-
let alignUp = (x, multiple) => x + ((multiple - (x % multiple)) % multiple);
|
|
1227
|
-
for (var cutDown = 1; cutDown <= 4; cutDown *= 2) {
|
|
1228
|
-
var overGrownHeapSize = oldSize * (1 + 0.2 / cutDown);
|
|
1229
|
-
overGrownHeapSize = Math.min(
|
|
1230
|
-
overGrownHeapSize,
|
|
1231
|
-
requestedSize + 100663296,
|
|
1232
|
-
);
|
|
1233
|
-
var newSize = Math.min(
|
|
1234
|
-
maxHeapSize,
|
|
1235
|
-
alignUp(Math.max(requestedSize, overGrownHeapSize), 65536),
|
|
1236
|
-
);
|
|
1237
|
-
var replacement = emscripten_realloc_buffer(newSize);
|
|
1238
|
-
if (replacement) {
|
|
1239
|
-
return true;
|
|
1240
|
-
}
|
|
1241
|
-
}
|
|
1242
|
-
return false;
|
|
1243
|
-
}
|
|
1244
|
-
|
|
1245
|
-
var ENV = {};
|
|
1246
|
-
|
|
1247
|
-
function getExecutableName() {
|
|
1248
|
-
return thisProgram || './this.program';
|
|
1249
|
-
}
|
|
1250
|
-
|
|
1251
|
-
function getEnvStrings() {
|
|
1252
|
-
if (!getEnvStrings.strings) {
|
|
1253
|
-
var lang =
|
|
1254
|
-
(
|
|
1255
|
-
(typeof navigator == 'object' &&
|
|
1256
|
-
navigator.languages &&
|
|
1257
|
-
navigator.languages[0]) ||
|
|
1258
|
-
'C'
|
|
1259
|
-
).replace('-', '_') + '.UTF-8';
|
|
1260
|
-
var env = {
|
|
1261
|
-
USER: 'web_user',
|
|
1262
|
-
LOGNAME: 'web_user',
|
|
1263
|
-
PATH: '/',
|
|
1264
|
-
PWD: '/',
|
|
1265
|
-
HOME: '/home/web_user',
|
|
1266
|
-
LANG: lang,
|
|
1267
|
-
_: getExecutableName(),
|
|
1268
|
-
};
|
|
1269
|
-
for (var x in ENV) {
|
|
1270
|
-
if (ENV[x] === undefined) delete env[x];
|
|
1271
|
-
else env[x] = ENV[x];
|
|
1272
|
-
}
|
|
1273
|
-
var strings = [];
|
|
1274
|
-
for (var x in env) {
|
|
1275
|
-
strings.push(x + '=' + env[x]);
|
|
1276
|
-
}
|
|
1277
|
-
getEnvStrings.strings = strings;
|
|
1278
|
-
}
|
|
1279
|
-
return getEnvStrings.strings;
|
|
1280
|
-
}
|
|
1281
|
-
|
|
1282
|
-
// eslint-disable-next-line @typescript-eslint/no-shadow
|
|
1283
|
-
function writeAsciiToMemory(str, buffer, dontAddNull) {
|
|
1284
|
-
for (var i = 0; i < str.length; ++i) {
|
|
1285
|
-
HEAP8[buffer++ >> 0] = str.charCodeAt(i);
|
|
1286
|
-
}
|
|
1287
|
-
HEAP8[buffer >> 0] = 0;
|
|
1288
|
-
}
|
|
1289
|
-
|
|
1290
|
-
function _environ_get(__environ, environ_buf) {
|
|
1291
|
-
var bufSize = 0;
|
|
1292
|
-
getEnvStrings().forEach(function (string, i) {
|
|
1293
|
-
var ptr = environ_buf + bufSize;
|
|
1294
|
-
HEAPU32[(__environ + i * 4) >> 2] = ptr;
|
|
1295
|
-
writeAsciiToMemory(string, ptr);
|
|
1296
|
-
bufSize += string.length + 1;
|
|
1297
|
-
});
|
|
1298
|
-
return 0;
|
|
1299
|
-
}
|
|
1300
|
-
|
|
1301
|
-
function _environ_sizes_get(penviron_count, penviron_buf_size) {
|
|
1302
|
-
var strings = getEnvStrings();
|
|
1303
|
-
HEAPU32[penviron_count >> 2] = strings.length;
|
|
1304
|
-
var bufSize = 0;
|
|
1305
|
-
strings.forEach(function (string) {
|
|
1306
|
-
bufSize += string.length + 1;
|
|
1307
|
-
});
|
|
1308
|
-
HEAPU32[penviron_buf_size >> 2] = bufSize;
|
|
1309
|
-
return 0;
|
|
1310
|
-
}
|
|
1311
|
-
|
|
1312
|
-
function _proc_exit(code) {
|
|
1313
|
-
quit_(code, new ExitStatus(code));
|
|
1314
|
-
}
|
|
1315
|
-
|
|
1316
|
-
function exitJS(status, implicit) {
|
|
1317
|
-
_proc_exit(status);
|
|
1318
|
-
}
|
|
1319
|
-
|
|
1320
|
-
var _exit = exitJS;
|
|
1321
|
-
|
|
1322
|
-
function _fd_close(fd) {
|
|
1323
|
-
return 52;
|
|
1324
|
-
}
|
|
1325
|
-
|
|
1326
|
-
function _fd_seek(fd, offset_low, offset_high, whence, newOffset) {
|
|
1327
|
-
return 70;
|
|
1328
|
-
}
|
|
1329
|
-
|
|
1330
|
-
var printCharBuffers = [null, [], []];
|
|
1331
|
-
|
|
1332
|
-
function printChar(stream, curr) {
|
|
1333
|
-
// eslint-disable-next-line @typescript-eslint/no-shadow
|
|
1334
|
-
var buffer = printCharBuffers[stream];
|
|
1335
|
-
if (curr === 0 || curr === 10) {
|
|
1336
|
-
(stream === 1 ? out : err)(UTF8ArrayToString(buffer, 0));
|
|
1337
|
-
buffer.length = 0;
|
|
1338
|
-
} else {
|
|
1339
|
-
buffer.push(curr);
|
|
1340
|
-
}
|
|
1341
|
-
}
|
|
1342
|
-
|
|
1343
|
-
function _fd_write(fd, iov, iovcnt, pnum) {
|
|
1344
|
-
var num = 0;
|
|
1345
|
-
for (var i = 0; i < iovcnt; i++) {
|
|
1346
|
-
var ptr = HEAPU32[iov >> 2];
|
|
1347
|
-
var len = HEAPU32[(iov + 4) >> 2];
|
|
1348
|
-
iov += 8;
|
|
1349
|
-
for (var j = 0; j < len; j++) {
|
|
1350
|
-
printChar(fd, HEAPU8[ptr + j]);
|
|
1351
|
-
}
|
|
1352
|
-
num += len;
|
|
1353
|
-
}
|
|
1354
|
-
HEAPU32[pnum >> 2] = num;
|
|
1355
|
-
return 0;
|
|
1356
|
-
}
|
|
1357
|
-
|
|
1358
|
-
function getRandomDevice() {
|
|
1359
|
-
if (
|
|
1360
|
-
typeof crypto == 'object' &&
|
|
1361
|
-
typeof crypto['getRandomValues'] == 'function'
|
|
1362
|
-
) {
|
|
1363
|
-
var randomBuffer = new Uint8Array(1);
|
|
1364
|
-
return () => {
|
|
1365
|
-
crypto.getRandomValues(randomBuffer);
|
|
1366
|
-
return randomBuffer[0];
|
|
1367
|
-
};
|
|
1368
|
-
} else return () => abort('randomDevice');
|
|
1369
|
-
}
|
|
1370
|
-
|
|
1371
|
-
// eslint-disable-next-line @typescript-eslint/no-shadow
|
|
1372
|
-
function _getentropy(buffer, size) {
|
|
1373
|
-
if (!_getentropy.randomDevice) {
|
|
1374
|
-
_getentropy.randomDevice = getRandomDevice();
|
|
1375
|
-
}
|
|
1376
|
-
for (var i = 0; i < size; i++) {
|
|
1377
|
-
HEAP8[(buffer + i) >> 0] = _getentropy.randomDevice();
|
|
1378
|
-
}
|
|
1379
|
-
return 0;
|
|
1380
|
-
}
|
|
1381
|
-
|
|
1382
|
-
var asmLibraryArg = {
|
|
1383
|
-
_dlinit: __dlinit,
|
|
1384
|
-
_dlopen_js: __dlopen_js,
|
|
1385
|
-
_dlsym_js: __dlsym_js,
|
|
1386
|
-
_emscripten_get_now_is_monotonic: __emscripten_get_now_is_monotonic,
|
|
1387
|
-
_mmap_js: __mmap_js,
|
|
1388
|
-
_munmap_js: __munmap_js,
|
|
1389
|
-
abort: _abort,
|
|
1390
|
-
emscripten_date_now: _emscripten_date_now,
|
|
1391
|
-
emscripten_get_heap_max: _emscripten_get_heap_max,
|
|
1392
|
-
emscripten_get_now: _emscripten_get_now,
|
|
1393
|
-
emscripten_memcpy_big: _emscripten_memcpy_big,
|
|
1394
|
-
emscripten_resize_heap: _emscripten_resize_heap,
|
|
1395
|
-
environ_get: _environ_get,
|
|
1396
|
-
environ_sizes_get: _environ_sizes_get,
|
|
1397
|
-
exit: _exit,
|
|
1398
|
-
fd_close: _fd_close,
|
|
1399
|
-
fd_seek: _fd_seek,
|
|
1400
|
-
fd_write: _fd_write,
|
|
1401
|
-
getentropy: _getentropy,
|
|
1402
|
-
};
|
|
1403
|
-
createWasm();
|
|
1404
|
-
(Module['___wasm_call_ctors'] = function () {
|
|
1405
|
-
return (Module['___wasm_call_ctors'] =
|
|
1406
|
-
Module['asm']['__wasm_call_ctors']).apply(null, arguments);
|
|
1407
|
-
});
|
|
1408
|
-
(Module['_getModelBufferMemoryOffset'] =
|
|
1409
|
-
function () {
|
|
1410
|
-
return (Module[
|
|
1411
|
-
'_getModelBufferMemoryOffset'
|
|
1412
|
-
] =
|
|
1413
|
-
Module['asm']['getModelBufferMemoryOffset']).apply(null, arguments);
|
|
1414
|
-
});
|
|
1415
|
-
(Module['_getInputMemoryOffset'] = function () {
|
|
1416
|
-
return (Module['_getInputMemoryOffset'] =
|
|
1417
|
-
Module['asm']['getInputMemoryOffset']).apply(null, arguments);
|
|
1418
|
-
});
|
|
1419
|
-
(Module['_getInputHeight'] = function () {
|
|
1420
|
-
return (Module['_getInputHeight'] =
|
|
1421
|
-
Module['asm']['getInputHeight']).apply(null, arguments);
|
|
1422
|
-
});
|
|
1423
|
-
(Module['_getInputWidth'] = function () {
|
|
1424
|
-
return (Module['_getInputWidth'] =
|
|
1425
|
-
Module['asm']['getInputWidth']).apply(null, arguments);
|
|
1426
|
-
});
|
|
1427
|
-
(Module['_getInputChannelCount'] = function () {
|
|
1428
|
-
return (Module['_getInputChannelCount'] =
|
|
1429
|
-
Module['asm']['getInputChannelCount']).apply(null, arguments);
|
|
1430
|
-
});
|
|
1431
|
-
(Module['_getOutputMemoryOffset'] = function () {
|
|
1432
|
-
return (Module['_getOutputMemoryOffset'] =
|
|
1433
|
-
Module['asm']['getOutputMemoryOffset']).apply(null, arguments);
|
|
1434
|
-
});
|
|
1435
|
-
(Module['_getOutputHeight'] = function () {
|
|
1436
|
-
return (Module['_getOutputHeight'] =
|
|
1437
|
-
Module['asm']['getOutputHeight']).apply(null, arguments);
|
|
1438
|
-
});
|
|
1439
|
-
(Module['_getOutputWidth'] = function () {
|
|
1440
|
-
return (Module['_getOutputWidth'] =
|
|
1441
|
-
Module['asm']['getOutputWidth']).apply(null, arguments);
|
|
1442
|
-
});
|
|
1443
|
-
(Module['_getOutputChannelCount'] = function () {
|
|
1444
|
-
return (Module['_getOutputChannelCount'] =
|
|
1445
|
-
Module['asm']['getOutputChannelCount']).apply(null, arguments);
|
|
1446
|
-
});
|
|
1447
|
-
(Module['_loadModel'] = function () {
|
|
1448
|
-
return (Module['_loadModel'] =
|
|
1449
|
-
Module['asm']['loadModel']).apply(null, arguments);
|
|
1450
|
-
});
|
|
1451
|
-
(Module['_runInference'] = function () {
|
|
1452
|
-
return (Module['_runInference'] =
|
|
1453
|
-
Module['asm']['runInference']).apply(null, arguments);
|
|
1454
|
-
});
|
|
1455
|
-
(Module['_malloc'] = function () {
|
|
1456
|
-
return (Module['_malloc'] = Module['asm']['malloc']).apply(
|
|
1457
|
-
null,
|
|
1458
|
-
arguments,
|
|
1459
|
-
);
|
|
1460
|
-
});
|
|
1461
|
-
(Module['___errno_location'] = function () {
|
|
1462
|
-
return (Module['___errno_location'] =
|
|
1463
|
-
Module['asm']['__errno_location']).apply(null, arguments);
|
|
1464
|
-
});
|
|
1465
|
-
(Module['___dl_seterr'] = function () {
|
|
1466
|
-
return (Module['___dl_seterr'] =
|
|
1467
|
-
Module['asm']['__dl_seterr']).apply(null, arguments);
|
|
1468
|
-
});
|
|
1469
|
-
(Module['stackSave'] = function () {
|
|
1470
|
-
return (Module['stackSave'] = Module['asm']['stackSave']).apply(
|
|
1471
|
-
null,
|
|
1472
|
-
arguments,
|
|
1473
|
-
);
|
|
1474
|
-
});
|
|
1475
|
-
(Module['stackRestore'] = function () {
|
|
1476
|
-
return (Module['stackRestore'] =
|
|
1477
|
-
Module['asm']['stackRestore']).apply(null, arguments);
|
|
1478
|
-
});
|
|
1479
|
-
(Module['stackAlloc'] = function () {
|
|
1480
|
-
return (Module['stackAlloc'] =
|
|
1481
|
-
Module['asm']['stackAlloc']).apply(null, arguments);
|
|
1482
|
-
});
|
|
1483
|
-
(Module['dynCall_jjj'] = function () {
|
|
1484
|
-
return (Module['dynCall_jjj'] =
|
|
1485
|
-
Module['asm']['dynCall_jjj']).apply(null, arguments);
|
|
1486
|
-
});
|
|
1487
|
-
(Module['dynCall_jiii'] = function () {
|
|
1488
|
-
return (Module['dynCall_jiii'] =
|
|
1489
|
-
Module['asm']['dynCall_jiii']).apply(null, arguments);
|
|
1490
|
-
});
|
|
1491
|
-
(Module['dynCall_iiiijj'] = function () {
|
|
1492
|
-
return (Module['dynCall_iiiijj'] =
|
|
1493
|
-
Module['asm']['dynCall_iiiijj']).apply(null, arguments);
|
|
1494
|
-
});
|
|
1495
|
-
(Module['dynCall_viijj'] = function () {
|
|
1496
|
-
return (Module['dynCall_viijj'] =
|
|
1497
|
-
Module['asm']['dynCall_viijj']).apply(null, arguments);
|
|
1498
|
-
});
|
|
1499
|
-
(Module['dynCall_viiijjj'] = function () {
|
|
1500
|
-
return (Module['dynCall_viiijjj'] =
|
|
1501
|
-
Module['asm']['dynCall_viiijjj']).apply(null, arguments);
|
|
1502
|
-
});
|
|
1503
|
-
(Module['dynCall_iijjiiii'] = function () {
|
|
1504
|
-
return (Module['dynCall_iijjiiii'] =
|
|
1505
|
-
Module['asm']['dynCall_iijjiiii']).apply(null, arguments);
|
|
1506
|
-
});
|
|
1507
|
-
(Module['dynCall_jiji'] = function () {
|
|
1508
|
-
return (Module['dynCall_jiji'] =
|
|
1509
|
-
Module['asm']['dynCall_jiji']).apply(null, arguments);
|
|
1510
|
-
});
|
|
1511
|
-
var calledRun;
|
|
1512
|
-
dependenciesFulfilled = function runCaller() {
|
|
1513
|
-
if (!calledRun) run();
|
|
1514
|
-
if (!calledRun) dependenciesFulfilled = runCaller;
|
|
1515
|
-
};
|
|
1516
|
-
|
|
1517
|
-
function run(args) {
|
|
1518
|
-
if (runDependencies > 0) {
|
|
1519
|
-
return;
|
|
1520
|
-
}
|
|
1521
|
-
preRun();
|
|
1522
|
-
if (runDependencies > 0) {
|
|
1523
|
-
return;
|
|
1524
|
-
}
|
|
1525
|
-
|
|
1526
|
-
function doRun() {
|
|
1527
|
-
if (calledRun) return;
|
|
1528
|
-
calledRun = true;
|
|
1529
|
-
Module['calledRun'] = true;
|
|
1530
|
-
if (ABORT) return;
|
|
1531
|
-
initRuntime();
|
|
1532
|
-
readyPromiseResolve(Module);
|
|
1533
|
-
if (Module['onRuntimeInitialized']) Module['onRuntimeInitialized']();
|
|
1534
|
-
postRun();
|
|
1535
|
-
}
|
|
1536
|
-
|
|
1537
|
-
if (Module['setStatus']) {
|
|
1538
|
-
Module['setStatus']('Running...');
|
|
1539
|
-
setTimeout(function () {
|
|
1540
|
-
setTimeout(function () {
|
|
1541
|
-
Module['setStatus']('');
|
|
1542
|
-
}, 1);
|
|
1543
|
-
doRun();
|
|
1544
|
-
}, 1);
|
|
1545
|
-
} else {
|
|
1546
|
-
doRun();
|
|
1547
|
-
}
|
|
1548
|
-
}
|
|
1549
|
-
|
|
1550
|
-
if (Module['preInit']) {
|
|
1551
|
-
if (typeof Module['preInit'] == 'function')
|
|
1552
|
-
Module['preInit'] = [Module['preInit']];
|
|
1553
|
-
while (Module['preInit'].length > 0) {
|
|
1554
|
-
Module['preInit'].pop()();
|
|
1555
|
-
}
|
|
1556
|
-
}
|
|
1557
|
-
run();
|
|
1558
|
-
|
|
1559
|
-
return __Module.ready;
|
|
1560
|
-
};
|
|
1561
|
-
|
|
1562
|
-
const version = "0.8.6";
|
|
1563
|
-
const packageName = "@stream-io/video-filters-web";
|
|
1564
|
-
|
|
1565
|
-
// @ts-expect-error - module is not declared
|
|
1566
|
-
// This is a WebAssembly module compiled from the TensorFlow Lite C++ library.
|
|
1567
|
-
const createTFLite = createTFLiteSIMDModule;
|
|
1568
|
-
const loadTFLite = async (options = {}) => {
|
|
1569
|
-
const { basePath = `https://unpkg.com/${packageName}@${version}/tf`, tfFilePath = `${basePath}/tflite/tflite-simd.wasm`, modelFilePath = `${basePath}/models/segm_full_v679.tflite`, } = options;
|
|
1570
|
-
const [tfLite, model] = await Promise.all([
|
|
1571
|
-
createTFLite({ locateFile: () => tfFilePath }),
|
|
1572
|
-
fetchModel(modelFilePath),
|
|
1573
|
-
]);
|
|
1574
|
-
const modelBufferOffset = tfLite._getModelBufferMemoryOffset();
|
|
1575
|
-
tfLite.HEAPU8.set(new Uint8Array(model), modelBufferOffset);
|
|
1576
|
-
tfLite._loadModel(model.byteLength);
|
|
1577
|
-
return tfLite;
|
|
1578
|
-
};
|
|
1579
|
-
let lastModelFilePath$1 = '';
|
|
1580
|
-
let modelFileCache$1;
|
|
1581
|
-
const fetchModel = async (modelFilePath) => {
|
|
1582
|
-
const model = modelFilePath === lastModelFilePath$1 && modelFileCache$1
|
|
1583
|
-
? modelFileCache$1
|
|
1584
|
-
: await fetch(modelFilePath).then((r) => r.arrayBuffer());
|
|
1585
|
-
// Cache the model file for future use.
|
|
1586
|
-
modelFileCache$1 = model;
|
|
1587
|
-
lastModelFilePath$1 = modelFilePath;
|
|
1588
|
-
return model;
|
|
1589
|
-
};
|
|
1590
|
-
|
|
1591
|
-
let lastModelFilePath = '';
|
|
1592
|
-
let modelFileCache;
|
|
1593
|
-
const loadMediaPipe = async (options = {}) => {
|
|
1594
|
-
const { basePath = `https://unpkg.com/${packageName}@${version}/mediapipe`, modelPath = `${basePath}/models/selfie_segmenter_landscape.tflite`, } = options;
|
|
1595
|
-
const model = modelPath === lastModelFilePath && modelFileCache
|
|
1596
|
-
? modelFileCache
|
|
1597
|
-
: await fetch(modelPath).then((r) => r.arrayBuffer());
|
|
1598
|
-
modelFileCache = model;
|
|
1599
|
-
lastModelFilePath = modelPath;
|
|
1600
|
-
return model;
|
|
1601
|
-
};
|
|
1602
|
-
|
|
1603
|
-
/**
|
|
1604
|
-
* Maps blur level to blur strength values.
|
|
1605
|
-
*/
|
|
1606
|
-
const BACKGROUND_BLUR_MAP = {
|
|
1607
|
-
low: 3,
|
|
1608
|
-
medium: 5,
|
|
1609
|
-
high: 7,
|
|
1610
|
-
};
|
|
1611
|
-
|
|
1612
|
-
/**
|
|
1613
|
-
* Fallback video processor for browsers that do not support MediaStreamTrackGenerator.
|
|
1614
|
-
*
|
|
1615
|
-
* Produces a video MediaStreamTrack sourced from a canvas and exposes
|
|
1616
|
-
* a WritableStream<VideoFrame> on track.writable for writing frames.
|
|
1617
|
-
*/
|
|
1618
|
-
class FallbackGenerator {
|
|
1619
|
-
constructor({ kind, signalTarget }) {
|
|
1620
|
-
if (kind !== 'video') {
|
|
1621
|
-
throw new Error('Only video tracks are supported');
|
|
1622
|
-
}
|
|
1623
|
-
const canvas = document.createElement('canvas');
|
|
1624
|
-
const ctx = canvas.getContext('2d', { desynchronized: true });
|
|
1625
|
-
if (!ctx) {
|
|
1626
|
-
throw new Error('Failed to get 2D context from canvas');
|
|
1627
|
-
}
|
|
1628
|
-
const mediaStream = canvas.captureStream();
|
|
1629
|
-
const track = mediaStream.getVideoTracks()[0];
|
|
1630
|
-
const height = signalTarget?.getSettings().height;
|
|
1631
|
-
const width = signalTarget?.getSettings().width;
|
|
1632
|
-
if (height && width) {
|
|
1633
|
-
canvas.height = height;
|
|
1634
|
-
canvas.width = width;
|
|
1635
|
-
}
|
|
1636
|
-
if (!track) {
|
|
1637
|
-
throw new Error('Failed to create canvas track');
|
|
1638
|
-
}
|
|
1639
|
-
if (signalTarget) {
|
|
1640
|
-
signalTarget.addEventListener('ended', () => {
|
|
1641
|
-
track.stop();
|
|
1642
|
-
});
|
|
1643
|
-
}
|
|
1644
|
-
track.writable = new WritableStream({
|
|
1645
|
-
write: (frame) => {
|
|
1646
|
-
if (canvas.width !== frame.displayWidth ||
|
|
1647
|
-
canvas.height !== frame.displayHeight) {
|
|
1648
|
-
canvas.width = frame.displayWidth;
|
|
1649
|
-
canvas.height = frame.displayHeight;
|
|
1650
|
-
}
|
|
1651
|
-
ctx.drawImage(frame, 0, 0, canvas.width, canvas.height);
|
|
1652
|
-
frame.close();
|
|
1653
|
-
},
|
|
1654
|
-
abort: () => {
|
|
1655
|
-
track.stop();
|
|
1656
|
-
},
|
|
1657
|
-
close: () => {
|
|
1658
|
-
track.stop();
|
|
1659
|
-
},
|
|
1660
|
-
});
|
|
1661
|
-
return track;
|
|
1662
|
-
}
|
|
1663
|
-
}
|
|
1664
|
-
const TrackGenerator = typeof MediaStreamTrackGenerator !== 'undefined'
|
|
1665
|
-
? MediaStreamTrackGenerator
|
|
1666
|
-
: FallbackGenerator;
|
|
1667
|
-
|
|
1668
|
-
/**
|
|
1669
|
-
* Fallback implementation for browsers without MediaStreamTrackGenerator.
|
|
1670
|
-
*
|
|
1671
|
-
* Produces a video MediaStreamTrack sourced from a canvas and exposes a
|
|
1672
|
-
* WritableStream<VideoFrame> on track.writable. Written frames are drawn
|
|
1673
|
-
* into the canvas and update the underlying track automatically.
|
|
1674
|
-
*/
|
|
1675
|
-
class FallbackProcessor {
|
|
1676
|
-
readable;
|
|
1677
|
-
workerTimer;
|
|
1678
|
-
video;
|
|
1679
|
-
constructor({ track }) {
|
|
1680
|
-
if (!track)
|
|
1681
|
-
throw new Error('MediaStreamTrack is required');
|
|
1682
|
-
if (track.kind !== 'video') {
|
|
1683
|
-
throw new Error('MediaStreamTrack must be video');
|
|
1684
|
-
}
|
|
1685
|
-
let running = true;
|
|
1686
|
-
this.video = document.createElement('video');
|
|
1687
|
-
this.video.muted = true;
|
|
1688
|
-
this.video.playsInline = true;
|
|
1689
|
-
this.video.srcObject = new MediaStream([track]);
|
|
1690
|
-
const canvas = new OffscreenCanvas(1, 1);
|
|
1691
|
-
const ctx = canvas.getContext('2d');
|
|
1692
|
-
if (!ctx)
|
|
1693
|
-
throw new Error('Failed to get 2D context from OffscreenCanvas');
|
|
1694
|
-
let timestamp = 0;
|
|
1695
|
-
const frameRate = track.getSettings().frameRate || 30;
|
|
1696
|
-
let frameDuration = 1000 / frameRate;
|
|
1697
|
-
this.workerTimer = new WorkerTimer({ useWorker: true });
|
|
1698
|
-
this.readable = new ReadableStream({
|
|
1699
|
-
start: async () => {
|
|
1700
|
-
await Promise.all([
|
|
1701
|
-
this.video.play(),
|
|
1702
|
-
new Promise((r) => this.video.addEventListener('loadeddata', r, { once: true })),
|
|
1703
|
-
]);
|
|
1704
|
-
frameDuration = 1000 / (track.getSettings().frameRate || 30);
|
|
1705
|
-
timestamp = performance.now();
|
|
1706
|
-
},
|
|
1707
|
-
pull: async (controller) => {
|
|
1708
|
-
if (!running) {
|
|
1709
|
-
controller.close();
|
|
1710
|
-
this.close();
|
|
1711
|
-
return;
|
|
1712
|
-
}
|
|
1713
|
-
const delta = performance.now() - timestamp;
|
|
1714
|
-
if (delta <= frameDuration) {
|
|
1715
|
-
await new Promise((r) => this.workerTimer.setTimeout(r, frameDuration - delta));
|
|
1716
|
-
}
|
|
1717
|
-
timestamp = performance.now();
|
|
1718
|
-
if (canvas.width !== this.video.videoWidth ||
|
|
1719
|
-
canvas.height !== this.video.videoHeight) {
|
|
1720
|
-
canvas.width = this.video.videoWidth;
|
|
1721
|
-
canvas.height = this.video.videoHeight;
|
|
1722
|
-
}
|
|
1723
|
-
ctx.drawImage(this.video, 0, 0);
|
|
1724
|
-
try {
|
|
1725
|
-
const frame = new VideoFrame(canvas, {
|
|
1726
|
-
timestamp: Math.round(this.video.currentTime * 1000000),
|
|
1727
|
-
});
|
|
1728
|
-
controller.enqueue(frame);
|
|
1729
|
-
}
|
|
1730
|
-
catch (err) {
|
|
1731
|
-
running = false;
|
|
1732
|
-
controller.error(err);
|
|
1733
|
-
this.close();
|
|
1734
|
-
}
|
|
1735
|
-
},
|
|
1736
|
-
cancel: () => {
|
|
1737
|
-
running = false;
|
|
1738
|
-
this.close();
|
|
1739
|
-
},
|
|
1740
|
-
});
|
|
1741
|
-
}
|
|
1742
|
-
close = () => {
|
|
1743
|
-
this.video.pause();
|
|
1744
|
-
this.video.srcObject = null;
|
|
1745
|
-
this.video.src = '';
|
|
1746
|
-
this.workerTimer.destroy();
|
|
1747
|
-
};
|
|
1748
|
-
}
|
|
1749
|
-
const TrackProcessor = typeof MediaStreamTrackProcessor !== 'undefined'
|
|
1750
|
-
? MediaStreamTrackProcessor
|
|
1751
|
-
: FallbackProcessor;
|
|
1752
|
-
|
|
1753
|
-
/**
|
|
1754
|
-
* Base class for real-time video filters.
|
|
1755
|
-
*
|
|
1756
|
-
* It sets up the full pipeline that reads frames from the input track,
|
|
1757
|
-
* processes them, and outputs a new track with your effect applied. Subclasses
|
|
1758
|
-
* only need to implement `initialize` (run once before processing starts) and
|
|
1759
|
-
* `transform` (called for every frame).
|
|
1760
|
-
*
|
|
1761
|
-
* Everything else—canvas setup, performance tracking, error handling, and
|
|
1762
|
-
* clean shutdown is handled for you. Calling `start()` returns a processed
|
|
1763
|
-
* `MediaStreamTrack` ready to use.
|
|
1764
|
-
*/
|
|
1765
|
-
class BaseVideoProcessor {
|
|
1766
|
-
track;
|
|
1767
|
-
processor;
|
|
1768
|
-
generator;
|
|
1769
|
-
hooks;
|
|
1770
|
-
abortController = new AbortController();
|
|
1771
|
-
canvas;
|
|
1772
|
-
frames = 0;
|
|
1773
|
-
delayTotal = 0;
|
|
1774
|
-
lastStatsTime = 0;
|
|
1775
|
-
/**
|
|
1776
|
-
* Constructs a new instance.
|
|
1777
|
-
*/
|
|
1778
|
-
constructor(track, hooks = {}) {
|
|
1779
|
-
this.track = track;
|
|
1780
|
-
this.processor = new TrackProcessor({ track });
|
|
1781
|
-
this.generator = new TrackGenerator({
|
|
1782
|
-
kind: 'video',
|
|
1783
|
-
signalTarget: track,
|
|
1784
|
-
});
|
|
1785
|
-
this.hooks = hooks;
|
|
1786
|
-
}
|
|
1787
|
-
async start() {
|
|
1788
|
-
const { readable } = this.processor;
|
|
1789
|
-
const { writable } = this.generator;
|
|
1790
|
-
const { width = 1280, height = 720 } = this.track.getSettings();
|
|
1791
|
-
this.canvas = new OffscreenCanvas(width, height);
|
|
1792
|
-
await this.initialize();
|
|
1793
|
-
const transformStream = new TransformStream({
|
|
1794
|
-
transform: async (frame, controller) => {
|
|
1795
|
-
try {
|
|
1796
|
-
if (this.abortController.signal.aborted)
|
|
1797
|
-
return frame.close();
|
|
1798
|
-
if (this.canvas.width !== frame.displayWidth ||
|
|
1799
|
-
this.canvas.height !== frame.displayHeight) {
|
|
1800
|
-
this.canvas.width = frame.displayWidth;
|
|
1801
|
-
this.canvas.height = frame.displayHeight;
|
|
1802
|
-
}
|
|
1803
|
-
const processed = await this.transform(frame);
|
|
1804
|
-
controller.enqueue(processed);
|
|
1805
|
-
}
|
|
1806
|
-
catch (e) {
|
|
1807
|
-
this.hooks.onError?.(e);
|
|
1808
|
-
}
|
|
1809
|
-
finally {
|
|
1810
|
-
frame.close();
|
|
1811
|
-
}
|
|
1812
|
-
},
|
|
1813
|
-
flush: () => this.onFlush(),
|
|
1814
|
-
});
|
|
1815
|
-
readable
|
|
1816
|
-
.pipeThrough(transformStream, { signal: this.abortController.signal })
|
|
1817
|
-
.pipeTo(writable, { signal: this.abortController.signal })
|
|
1818
|
-
.catch((e) => {
|
|
1819
|
-
if (e.name !== 'AbortError' && e.name !== 'InvalidStateError') {
|
|
1820
|
-
console.error(`[${this.processorName}] Error processing track:`, e);
|
|
1821
|
-
this.hooks.onError?.(e);
|
|
1822
|
-
}
|
|
1823
|
-
});
|
|
1824
|
-
return this.generator;
|
|
1825
|
-
}
|
|
1826
|
-
stop() {
|
|
1827
|
-
this.abortController.abort();
|
|
1828
|
-
this.generator.stop();
|
|
1829
|
-
this.onStop();
|
|
1830
|
-
}
|
|
1831
|
-
updateStats(delay) {
|
|
1832
|
-
this.frames++;
|
|
1833
|
-
this.delayTotal += delay;
|
|
1834
|
-
const now = performance.now();
|
|
1835
|
-
if (this.lastStatsTime === 0) {
|
|
1836
|
-
this.lastStatsTime = now;
|
|
1837
|
-
return;
|
|
1838
|
-
}
|
|
1839
|
-
if (now - this.lastStatsTime >= 1000) {
|
|
1840
|
-
const avgDelay = Math.round((this.delayTotal / this.frames) * 100) / 100;
|
|
1841
|
-
const fps = Math.round((1000 * this.frames) / (now - this.lastStatsTime));
|
|
1842
|
-
this.hooks.onStats?.({ delay: avgDelay, fps, timestamp: now });
|
|
1843
|
-
this.frames = 0;
|
|
1844
|
-
this.delayTotal = 0;
|
|
1845
|
-
this.lastStatsTime = now;
|
|
1846
|
-
}
|
|
1847
|
-
}
|
|
1848
|
-
onFlush() { }
|
|
1849
|
-
onStop() { }
|
|
1850
|
-
get processorName() {
|
|
1851
|
-
return 'base-processor';
|
|
1852
|
-
}
|
|
1853
|
-
}
|
|
1854
|
-
|
|
1855
|
-
class WebGLRenderer {
|
|
1856
|
-
canvas;
|
|
1857
|
-
gl;
|
|
1858
|
-
stateUpdateProgram;
|
|
1859
|
-
maskRefineProgram;
|
|
1860
|
-
blurProgram;
|
|
1861
|
-
blendProgram;
|
|
1862
|
-
stateUpdateLocations;
|
|
1863
|
-
maskRefineLocations;
|
|
1864
|
-
blurLocations;
|
|
1865
|
-
blendLocations;
|
|
1866
|
-
positionBuffer;
|
|
1867
|
-
texCoordBuffer;
|
|
1868
|
-
storedStateTextures;
|
|
1869
|
-
fbo;
|
|
1870
|
-
refineFbo;
|
|
1871
|
-
refinedMaskTexture;
|
|
1872
|
-
frameTexture;
|
|
1873
|
-
blurTexture1;
|
|
1874
|
-
blurTexture2;
|
|
1875
|
-
blurFbo1;
|
|
1876
|
-
blurFbo2;
|
|
1877
|
-
running = false;
|
|
1878
|
-
static DEFAULT_BG_COLOR = [33, 150, 243, 255];
|
|
1879
|
-
currentStateIndex = 0;
|
|
1880
|
-
backgroundRenderInfo = null;
|
|
1881
|
-
activeBackgroundSourceIdentifier = null;
|
|
1882
|
-
constructor(canvas) {
|
|
1883
|
-
this.canvas = canvas;
|
|
1884
|
-
const gl = this.canvas.getContext('webgl2', {
|
|
1885
|
-
alpha: false,
|
|
1886
|
-
antialias: false,
|
|
1887
|
-
desynchronized: true,
|
|
1888
|
-
});
|
|
1889
|
-
if (!gl)
|
|
1890
|
-
throw new Error('WebGL2 not supported');
|
|
1891
|
-
this.gl = gl;
|
|
1892
|
-
const stateUpdateVertexShaderSource = `attribute vec2 a_position; attribute vec2 a_texCoord; varying vec2 v_texCoord; void main() { gl_Position = vec4(a_position, 0.0, 1.0); v_texCoord = a_texCoord; }`;
|
|
1893
|
-
const stateUpdateFragmentShaderSource = `
|
|
1894
|
-
precision mediump float;
|
|
1895
|
-
varying vec2 v_texCoord;
|
|
1896
|
-
uniform sampler2D u_categoryTexture;
|
|
1897
|
-
uniform sampler2D u_confidenceTexture;
|
|
1898
|
-
uniform sampler2D u_prevStateTexture;
|
|
1899
|
-
uniform float u_smoothingFactor;
|
|
1900
|
-
uniform float u_smoothstepMin;
|
|
1901
|
-
uniform float u_smoothstepMax;
|
|
1902
|
-
uniform int u_selfieModel;
|
|
1903
|
-
|
|
1904
|
-
void main() {
|
|
1905
|
-
vec2 prevCoord = vec2(v_texCoord.x, 1.0 - v_texCoord.y);
|
|
1906
|
-
float categoryValue = texture2D(u_categoryTexture, v_texCoord).r;
|
|
1907
|
-
float confidenceValue = texture2D(u_confidenceTexture, v_texCoord).r;
|
|
1908
|
-
|
|
1909
|
-
if (u_selfieModel == 1) {
|
|
1910
|
-
categoryValue = 1.0 - categoryValue;
|
|
1911
|
-
confidenceValue = 1.0 - confidenceValue;
|
|
1912
|
-
}
|
|
1913
|
-
|
|
1914
|
-
if (categoryValue > 0.0) {
|
|
1915
|
-
categoryValue = 1.0;
|
|
1916
|
-
confidenceValue = 1.0 - confidenceValue;
|
|
1917
|
-
}
|
|
1918
|
-
|
|
1919
|
-
float nonLinearConfidence = smoothstep(u_smoothstepMin, u_smoothstepMax, confidenceValue);
|
|
1920
|
-
float prevCategoryValue = texture2D(u_prevStateTexture, prevCoord).r;
|
|
1921
|
-
float alpha = u_smoothingFactor * nonLinearConfidence;
|
|
1922
|
-
float newCategoryValue = alpha * categoryValue + (1.0 - alpha) * prevCategoryValue;
|
|
1923
|
-
|
|
1924
|
-
gl_FragColor = vec4(newCategoryValue, 0.0, 0.0, 0.0);
|
|
1925
|
-
}
|
|
1926
|
-
`;
|
|
1927
|
-
this.stateUpdateProgram = this.createAndLinkProgram(stateUpdateVertexShaderSource, stateUpdateFragmentShaderSource);
|
|
1928
|
-
this.stateUpdateLocations = {
|
|
1929
|
-
position: gl.getAttribLocation(this.stateUpdateProgram, 'a_position'),
|
|
1930
|
-
texCoord: gl.getAttribLocation(this.stateUpdateProgram, 'a_texCoord'),
|
|
1931
|
-
categoryTexture: gl.getUniformLocation(this.stateUpdateProgram, 'u_categoryTexture'),
|
|
1932
|
-
confidenceTexture: gl.getUniformLocation(this.stateUpdateProgram, 'u_confidenceTexture'),
|
|
1933
|
-
prevStateTexture: gl.getUniformLocation(this.stateUpdateProgram, 'u_prevStateTexture'),
|
|
1934
|
-
smoothingFactor: gl.getUniformLocation(this.stateUpdateProgram, 'u_smoothingFactor'),
|
|
1935
|
-
smoothstepMin: gl.getUniformLocation(this.stateUpdateProgram, 'u_smoothstepMin'),
|
|
1936
|
-
smoothstepMax: gl.getUniformLocation(this.stateUpdateProgram, 'u_smoothstepMax'),
|
|
1937
|
-
selfieModel: gl.getUniformLocation(this.stateUpdateProgram, 'u_selfieModel'),
|
|
1938
|
-
};
|
|
1939
|
-
const maskRefineVertexShaderSource = stateUpdateVertexShaderSource;
|
|
1940
|
-
const maskRefineFragmentShaderSource = `
|
|
1941
|
-
precision mediump float;
|
|
1942
|
-
varying vec2 v_texCoord;
|
|
1943
|
-
|
|
1944
|
-
uniform sampler2D u_maskTexture;
|
|
1945
|
-
uniform sampler2D u_frameTexture;
|
|
1946
|
-
uniform vec2 u_texelSize;
|
|
1947
|
-
uniform float u_sigmaSpatial;
|
|
1948
|
-
uniform float u_sigmaRange;
|
|
1949
|
-
|
|
1950
|
-
void main() {
|
|
1951
|
-
vec2 flippedCoord = v_texCoord;
|
|
1952
|
-
vec3 centerPixelColor = texture2D(u_frameTexture, v_texCoord).rgb;
|
|
1953
|
-
float totalWeight = 0.0;
|
|
1954
|
-
float weightedMaskSum = 0.0;
|
|
1955
|
-
|
|
1956
|
-
for (int offsetX = -2; offsetX <= 2; offsetX++) {
|
|
1957
|
-
for (int offsetY = -2; offsetY <= 2; offsetY++) {
|
|
1958
|
-
vec2 shift = vec2(float(offsetX), float(offsetY)) * u_texelSize;
|
|
1959
|
-
vec2 frameCoord = v_texCoord + shift;
|
|
1960
|
-
vec2 maskCoord = flippedCoord + shift;
|
|
1961
|
-
|
|
1962
|
-
vec3 neighborPixelColor = texture2D(u_frameTexture, frameCoord).rgb;
|
|
1963
|
-
float neighborMaskValue = texture2D(u_maskTexture, maskCoord).r;
|
|
1964
|
-
|
|
1965
|
-
float spatialWeight = exp(-dot(shift, shift) / (2.0 * u_sigmaSpatial * u_sigmaSpatial));
|
|
1966
|
-
vec3 colorDifference = neighborPixelColor - centerPixelColor;
|
|
1967
|
-
float rangeWeight = exp(-(dot(colorDifference, colorDifference)) / (2.0 * u_sigmaRange * u_sigmaRange));
|
|
1968
|
-
|
|
1969
|
-
float combinedWeight = spatialWeight * rangeWeight;
|
|
1970
|
-
weightedMaskSum += neighborMaskValue * combinedWeight;
|
|
1971
|
-
totalWeight += combinedWeight;
|
|
1972
|
-
}
|
|
1973
|
-
}
|
|
1974
|
-
|
|
1975
|
-
float refinedMaskValue = weightedMaskSum / max(totalWeight, 1e-6);
|
|
1976
|
-
gl_FragColor = vec4(refinedMaskValue, refinedMaskValue, refinedMaskValue, 1.0);
|
|
1977
|
-
}
|
|
1978
|
-
`;
|
|
1979
|
-
this.maskRefineProgram = this.createAndLinkProgram(maskRefineVertexShaderSource, maskRefineFragmentShaderSource);
|
|
1980
|
-
this.maskRefineLocations = {
|
|
1981
|
-
position: gl.getAttribLocation(this.maskRefineProgram, 'a_position'),
|
|
1982
|
-
texCoord: gl.getAttribLocation(this.maskRefineProgram, 'a_texCoord'),
|
|
1983
|
-
maskTexture: gl.getUniformLocation(this.maskRefineProgram, 'u_maskTexture'),
|
|
1984
|
-
frameTexture: gl.getUniformLocation(this.maskRefineProgram, 'u_frameTexture'),
|
|
1985
|
-
texelSize: gl.getUniformLocation(this.maskRefineProgram, 'u_texelSize'),
|
|
1986
|
-
sigmaSpatial: gl.getUniformLocation(this.maskRefineProgram, 'u_sigmaSpatial'),
|
|
1987
|
-
sigmaRange: gl.getUniformLocation(this.maskRefineProgram, 'u_sigmaRange'),
|
|
1988
|
-
};
|
|
1989
|
-
const blurVertexShaderSource = stateUpdateVertexShaderSource;
|
|
1990
|
-
const blurFragmentShaderSource = `
|
|
1991
|
-
precision highp float;
|
|
1992
|
-
varying vec2 v_texCoord;
|
|
1993
|
-
|
|
1994
|
-
uniform sampler2D u_image;
|
|
1995
|
-
uniform sampler2D u_personMask;
|
|
1996
|
-
uniform vec2 u_texelSize;
|
|
1997
|
-
uniform float u_sigma;
|
|
1998
|
-
uniform float u_radiusScale;
|
|
1999
|
-
uniform vec2 u_direction;
|
|
2000
|
-
|
|
2001
|
-
const int KERNEL_RADIUS = 10;
|
|
2002
|
-
|
|
2003
|
-
float gauss(float x, float s) {
|
|
2004
|
-
return exp(-(x * x) / (2.0 * s * s));
|
|
2005
|
-
}
|
|
2006
|
-
|
|
2007
|
-
void main() {
|
|
2008
|
-
vec2 maskCoord = u_direction.y > 0.5 ? vec2(v_texCoord.x, 1.0 - v_texCoord.y) : v_texCoord;
|
|
2009
|
-
float mCenter = texture2D(u_personMask, maskCoord).r;
|
|
2010
|
-
float wCenter = gauss(0.0, u_sigma);
|
|
2011
|
-
vec4 accum = texture2D(u_image, v_texCoord) * wCenter * (1.0 - mCenter);
|
|
2012
|
-
float weightSum = wCenter * (1.0 - mCenter);
|
|
2013
|
-
|
|
2014
|
-
for (int i = 1; i <= KERNEL_RADIUS; i++) {
|
|
2015
|
-
float f = float(i);
|
|
2016
|
-
float offset = f * u_radiusScale;
|
|
2017
|
-
float w = gauss(offset, u_sigma);
|
|
2018
|
-
vec2 texOffset = u_direction * offset * u_texelSize;
|
|
2019
|
-
|
|
2020
|
-
vec2 uvPlus = v_texCoord + texOffset;
|
|
2021
|
-
vec2 maskCoordPlus = u_direction.y > 0.5 ? vec2(uvPlus.x, 1.0 - uvPlus.y) : uvPlus;
|
|
2022
|
-
float mPlus = texture2D(u_personMask, maskCoordPlus).r;
|
|
2023
|
-
accum += texture2D(u_image, uvPlus) * w * (1.0 - mPlus);
|
|
2024
|
-
weightSum += w * (1.0 - mPlus);
|
|
2025
|
-
|
|
2026
|
-
vec2 uvMinus = v_texCoord - texOffset;
|
|
2027
|
-
vec2 maskCoordMinus = u_direction.y > 0.5 ? vec2(uvMinus.x, 1.0 - uvMinus.y) : uvMinus;
|
|
2028
|
-
float mMinus = texture2D(u_personMask, maskCoordMinus).r;
|
|
2029
|
-
accum += texture2D(u_image, uvMinus) * w * (1.0 - mMinus);
|
|
2030
|
-
weightSum += w * (1.0 - mMinus);
|
|
2031
|
-
}
|
|
2032
|
-
|
|
2033
|
-
vec4 blurred = accum / max(weightSum, 1e-6);
|
|
2034
|
-
gl_FragColor = blurred;
|
|
2035
|
-
}
|
|
2036
|
-
`;
|
|
2037
|
-
this.blurProgram = this.createAndLinkProgram(blurVertexShaderSource, blurFragmentShaderSource);
|
|
2038
|
-
this.blurLocations = {
|
|
2039
|
-
position: gl.getAttribLocation(this.blurProgram, 'a_position'),
|
|
2040
|
-
texCoord: gl.getAttribLocation(this.blurProgram, 'a_texCoord'),
|
|
2041
|
-
image: gl.getUniformLocation(this.blurProgram, 'u_image'),
|
|
2042
|
-
personMask: gl.getUniformLocation(this.blurProgram, 'u_personMask'),
|
|
2043
|
-
texelSize: gl.getUniformLocation(this.blurProgram, 'u_texelSize'),
|
|
2044
|
-
sigma: gl.getUniformLocation(this.blurProgram, 'u_sigma'),
|
|
2045
|
-
radiusScale: gl.getUniformLocation(this.blurProgram, 'u_radiusScale'),
|
|
2046
|
-
direction: gl.getUniformLocation(this.blurProgram, 'u_direction'),
|
|
2047
|
-
};
|
|
2048
|
-
const blendVertexShaderSource = stateUpdateVertexShaderSource;
|
|
2049
|
-
const blendFragmentShaderSource = `
|
|
2050
|
-
precision mediump float;
|
|
2051
|
-
varying vec2 v_texCoord;
|
|
2052
|
-
|
|
2053
|
-
uniform sampler2D u_frameTexture;
|
|
2054
|
-
uniform sampler2D u_currentStateTexture;
|
|
2055
|
-
uniform sampler2D u_backgroundTexture;
|
|
2056
|
-
uniform vec2 u_bgImageDimensions;
|
|
2057
|
-
uniform vec2 u_canvasDimensions;
|
|
2058
|
-
uniform float u_borderSmooth;
|
|
2059
|
-
uniform float u_bgBlur;
|
|
2060
|
-
uniform float u_bgBlurRadius;
|
|
2061
|
-
uniform int u_enabled;
|
|
2062
|
-
|
|
2063
|
-
vec4 getMixedFragColor(vec2 bgTexCoord, vec2 categoryCoord, vec2 offset) {
|
|
2064
|
-
vec4 backgroundColor = texture2D(u_backgroundTexture, bgTexCoord + offset);
|
|
2065
|
-
vec4 frameColor = texture2D(u_frameTexture, v_texCoord + offset);
|
|
2066
|
-
float categoryValue = texture2D(u_currentStateTexture, categoryCoord + offset).r;
|
|
2067
|
-
return mix(backgroundColor, frameColor, categoryValue);
|
|
2068
|
-
}
|
|
2069
|
-
|
|
2070
|
-
void main() {
|
|
2071
|
-
if (u_enabled == 0) {
|
|
2072
|
-
gl_FragColor = texture2D(u_frameTexture, v_texCoord);
|
|
2073
|
-
return;
|
|
2074
|
-
}
|
|
2075
|
-
|
|
2076
|
-
vec2 categoryCoord = v_texCoord;
|
|
2077
|
-
float categoryValue = texture2D(u_currentStateTexture, categoryCoord).r;
|
|
2078
|
-
|
|
2079
|
-
float canvasAspect = u_canvasDimensions.x / u_canvasDimensions.y;
|
|
2080
|
-
float bgAspect = u_bgImageDimensions.x / u_bgImageDimensions.y;
|
|
2081
|
-
|
|
2082
|
-
vec2 bgTexCoord = v_texCoord;
|
|
2083
|
-
float scaleX = 1.0;
|
|
2084
|
-
float scaleY = 1.0;
|
|
2085
|
-
float offsetX = 0.0;
|
|
2086
|
-
float offsetY = 0.0;
|
|
2087
|
-
|
|
2088
|
-
if (canvasAspect < bgAspect) {
|
|
2089
|
-
scaleY = 1.0;
|
|
2090
|
-
scaleX = bgAspect / canvasAspect;
|
|
2091
|
-
offsetX = (1.0 - scaleX) / 2.0;
|
|
2092
|
-
} else {
|
|
2093
|
-
scaleX = 1.0;
|
|
2094
|
-
scaleY = canvasAspect / bgAspect;
|
|
2095
|
-
offsetY = (1.0 - scaleY) / 2.0;
|
|
2096
|
-
}
|
|
2097
|
-
|
|
2098
|
-
bgTexCoord = vec2((v_texCoord.x - offsetX) / scaleX, (v_texCoord.y - offsetY) / scaleY);
|
|
2099
|
-
gl_FragColor = getMixedFragColor(bgTexCoord, categoryCoord, vec2(0.0, 0.0));
|
|
2100
|
-
}`;
|
|
2101
|
-
this.blendProgram = this.createAndLinkProgram(blendVertexShaderSource, blendFragmentShaderSource);
|
|
2102
|
-
this.blendLocations = {
|
|
2103
|
-
position: gl.getAttribLocation(this.blendProgram, 'a_position'),
|
|
2104
|
-
texCoord: gl.getAttribLocation(this.blendProgram, 'a_texCoord'),
|
|
2105
|
-
frameTexture: gl.getUniformLocation(this.blendProgram, 'u_frameTexture'),
|
|
2106
|
-
currentStateTexture: gl.getUniformLocation(this.blendProgram, 'u_currentStateTexture'),
|
|
2107
|
-
backgroundTexture: gl.getUniformLocation(this.blendProgram, 'u_backgroundTexture'),
|
|
2108
|
-
bgImageDimensions: gl.getUniformLocation(this.blendProgram, 'u_bgImageDimensions'),
|
|
2109
|
-
canvasDimensions: gl.getUniformLocation(this.blendProgram, 'u_canvasDimensions'),
|
|
2110
|
-
borderSmooth: gl.getUniformLocation(this.blendProgram, 'u_borderSmooth'),
|
|
2111
|
-
bgBlur: gl.getUniformLocation(this.blendProgram, 'u_bgBlur'),
|
|
2112
|
-
bgBlurRadius: gl.getUniformLocation(this.blendProgram, 'u_bgBlurRadius'),
|
|
2113
|
-
enabled: gl.getUniformLocation(this.blendProgram, 'u_enabled'),
|
|
2114
|
-
};
|
|
2115
|
-
this.positionBuffer = gl.createBuffer();
|
|
2116
|
-
gl.bindBuffer(gl.ARRAY_BUFFER, this.positionBuffer);
|
|
2117
|
-
gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([-1, -1, 1, -1, -1, 1, -1, 1, 1, -1, 1, 1]), gl.STATIC_DRAW);
|
|
2118
|
-
this.texCoordBuffer = gl.createBuffer();
|
|
2119
|
-
gl.bindBuffer(gl.ARRAY_BUFFER, this.texCoordBuffer);
|
|
2120
|
-
gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([0, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 0]), gl.STATIC_DRAW);
|
|
2121
|
-
this.storedStateTextures = Array.from({ length: 2 }, () => {
|
|
2122
|
-
const tex = gl.createTexture();
|
|
2123
|
-
gl.bindTexture(gl.TEXTURE_2D, tex);
|
|
2124
|
-
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, 1, 1, 0, gl.RGBA, gl.UNSIGNED_BYTE, new Uint8Array([0, 0, 0, 255]));
|
|
2125
|
-
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
|
|
2126
|
-
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
|
|
2127
|
-
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
|
|
2128
|
-
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR);
|
|
2129
|
-
return tex;
|
|
2130
|
-
});
|
|
2131
|
-
gl.bindTexture(gl.TEXTURE_2D, null);
|
|
2132
|
-
this.fbo = gl.createFramebuffer();
|
|
2133
|
-
this.refineFbo = gl.createFramebuffer();
|
|
2134
|
-
const refinedTex = gl.createTexture();
|
|
2135
|
-
this.frameTexture = gl.createTexture();
|
|
2136
|
-
if (!refinedTex)
|
|
2137
|
-
throw new Error('Failed to create refined mask texture');
|
|
2138
|
-
gl.bindTexture(gl.TEXTURE_2D, refinedTex);
|
|
2139
|
-
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, 1, 1, 0, gl.RGBA, gl.UNSIGNED_BYTE, null);
|
|
2140
|
-
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
|
|
2141
|
-
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
|
|
2142
|
-
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
|
|
2143
|
-
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR);
|
|
2144
|
-
gl.bindTexture(gl.TEXTURE_2D, null);
|
|
2145
|
-
this.refinedMaskTexture = refinedTex;
|
|
2146
|
-
const mkColorTex = () => {
|
|
2147
|
-
const t = gl.createTexture();
|
|
2148
|
-
if (!t)
|
|
2149
|
-
throw new Error('Failed to create blur texture');
|
|
2150
|
-
gl.bindTexture(gl.TEXTURE_2D, t);
|
|
2151
|
-
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, 1, 1, 0, gl.RGBA, gl.UNSIGNED_BYTE, null);
|
|
2152
|
-
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
|
|
2153
|
-
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
|
|
2154
|
-
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
|
|
2155
|
-
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR);
|
|
2156
|
-
gl.bindTexture(gl.TEXTURE_2D, null);
|
|
2157
|
-
return t;
|
|
2158
|
-
};
|
|
2159
|
-
this.blurTexture1 = mkColorTex();
|
|
2160
|
-
this.blurTexture2 = mkColorTex();
|
|
2161
|
-
const mkFbo = (tex) => {
|
|
2162
|
-
const fb = gl.createFramebuffer();
|
|
2163
|
-
if (!fb || !tex)
|
|
2164
|
-
throw new Error('Failed to create blur FBO');
|
|
2165
|
-
gl.bindFramebuffer(gl.FRAMEBUFFER, fb);
|
|
2166
|
-
gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, tex, 0);
|
|
2167
|
-
gl.bindFramebuffer(gl.FRAMEBUFFER, null);
|
|
2168
|
-
return fb;
|
|
2169
|
-
};
|
|
2170
|
-
this.blurFbo1 = mkFbo(this.blurTexture1);
|
|
2171
|
-
this.blurFbo2 = mkFbo(this.blurTexture2);
|
|
2172
|
-
this.running = true;
|
|
2173
|
-
}
|
|
2174
|
-
createAndLinkProgram(vsSource, fsSource) {
|
|
2175
|
-
const vs = this.createShader(this.gl.VERTEX_SHADER, vsSource);
|
|
2176
|
-
const fs = this.createShader(this.gl.FRAGMENT_SHADER, fsSource);
|
|
2177
|
-
const prog = this.gl.createProgram();
|
|
2178
|
-
if (!prog)
|
|
2179
|
-
throw new Error('Failed to create program');
|
|
2180
|
-
this.gl.attachShader(prog, vs);
|
|
2181
|
-
this.gl.attachShader(prog, fs);
|
|
2182
|
-
this.gl.linkProgram(prog);
|
|
2183
|
-
if (!this.gl.getProgramParameter(prog, this.gl.LINK_STATUS)) {
|
|
2184
|
-
console.error('Program link error:', this.gl.getProgramInfoLog(prog));
|
|
2185
|
-
this.gl.deleteProgram(prog);
|
|
2186
|
-
throw new Error('Link fail');
|
|
2187
|
-
}
|
|
2188
|
-
this.gl.detachShader(prog, vs);
|
|
2189
|
-
this.gl.detachShader(prog, fs);
|
|
2190
|
-
this.gl.deleteShader(vs);
|
|
2191
|
-
this.gl.deleteShader(fs);
|
|
2192
|
-
return prog;
|
|
2193
|
-
}
|
|
2194
|
-
createShader(type, source) {
|
|
2195
|
-
const shader = this.gl.createShader(type);
|
|
2196
|
-
if (!shader)
|
|
2197
|
-
throw new Error(`Failed to create shader type: ${type}`);
|
|
2198
|
-
this.gl.shaderSource(shader, source);
|
|
2199
|
-
this.gl.compileShader(shader);
|
|
2200
|
-
if (!this.gl.getShaderParameter(shader, this.gl.COMPILE_STATUS)) {
|
|
2201
|
-
console.error('Shader compile error:', this.gl.getShaderInfoLog(shader));
|
|
2202
|
-
this.gl.deleteShader(shader);
|
|
2203
|
-
throw new Error('Failed to compile shader');
|
|
2204
|
-
}
|
|
2205
|
-
return shader;
|
|
2206
|
-
}
|
|
2207
|
-
createColorTexture(r, g, b, a) {
|
|
2208
|
-
const texture = this.gl.createTexture();
|
|
2209
|
-
if (!texture)
|
|
2210
|
-
throw new Error('Failed to create texture for color');
|
|
2211
|
-
this.gl.bindTexture(this.gl.TEXTURE_2D, texture);
|
|
2212
|
-
const pixel = new Uint8Array([r, g, b, a]);
|
|
2213
|
-
this.gl.texImage2D(this.gl.TEXTURE_2D, 0, this.gl.RGBA, 1, 1, 0, this.gl.RGBA, this.gl.UNSIGNED_BYTE, pixel);
|
|
2214
|
-
this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_WRAP_S, this.gl.CLAMP_TO_EDGE);
|
|
2215
|
-
this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_WRAP_T, this.gl.CLAMP_TO_EDGE);
|
|
2216
|
-
this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_MIN_FILTER, this.gl.NEAREST);
|
|
2217
|
-
this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_MAG_FILTER, this.gl.NEAREST);
|
|
2218
|
-
this.gl.bindTexture(this.gl.TEXTURE_2D, null);
|
|
2219
|
-
return { texture, color: [r, g, b, a] };
|
|
2220
|
-
}
|
|
2221
|
-
updateBackgroundIfNeeded(newSource) {
|
|
2222
|
-
const gl = this.gl;
|
|
2223
|
-
let newIdentifier;
|
|
2224
|
-
if (!newSource) {
|
|
2225
|
-
const [r, g, b, a] = WebGLRenderer.DEFAULT_BG_COLOR;
|
|
2226
|
-
newIdentifier = `color(${r},${g},${b},${a})`;
|
|
2227
|
-
}
|
|
2228
|
-
else {
|
|
2229
|
-
newIdentifier = newSource.url;
|
|
2230
|
-
}
|
|
2231
|
-
if (newIdentifier === this.activeBackgroundSourceIdentifier &&
|
|
2232
|
-
this.backgroundRenderInfo) {
|
|
2233
|
-
return;
|
|
2234
|
-
}
|
|
2235
|
-
if (this.backgroundRenderInfo) {
|
|
2236
|
-
gl.deleteTexture(this.backgroundRenderInfo.texture);
|
|
2237
|
-
this.backgroundRenderInfo = null;
|
|
2238
|
-
}
|
|
2239
|
-
this.activeBackgroundSourceIdentifier = newIdentifier;
|
|
2240
|
-
if (!newSource) {
|
|
2241
|
-
const [r, g, b, a] = WebGLRenderer.DEFAULT_BG_COLOR;
|
|
2242
|
-
const colorTexData = this.createColorTexture(r, g, b, a);
|
|
2243
|
-
this.backgroundRenderInfo = {
|
|
2244
|
-
type: 'color',
|
|
2245
|
-
texture: colorTexData.texture,
|
|
2246
|
-
color: colorTexData.color,
|
|
2247
|
-
};
|
|
2248
|
-
this.activeBackgroundSourceIdentifier = `color(${r},${g},${b},${a})`;
|
|
2249
|
-
}
|
|
2250
|
-
else {
|
|
2251
|
-
if (newSource.type === 'image') {
|
|
2252
|
-
const { media, url } = newSource;
|
|
2253
|
-
const texture = this.gl.createTexture();
|
|
2254
|
-
if (!texture) {
|
|
2255
|
-
throw new Error('Failed to create texture object for image.');
|
|
2256
|
-
}
|
|
2257
|
-
this.gl.bindTexture(this.gl.TEXTURE_2D, texture);
|
|
2258
|
-
this.gl.texImage2D(this.gl.TEXTURE_2D, 0, this.gl.RGBA, this.gl.RGBA, this.gl.UNSIGNED_BYTE, media);
|
|
2259
|
-
this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_WRAP_S, this.gl.CLAMP_TO_EDGE);
|
|
2260
|
-
this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_WRAP_T, this.gl.CLAMP_TO_EDGE);
|
|
2261
|
-
this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_MIN_FILTER, this.gl.LINEAR);
|
|
2262
|
-
this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_MAG_FILTER, this.gl.LINEAR);
|
|
2263
|
-
this.gl.bindTexture(this.gl.TEXTURE_2D, null);
|
|
2264
|
-
this.backgroundRenderInfo = {
|
|
2265
|
-
type: 'image',
|
|
2266
|
-
texture,
|
|
2267
|
-
width: media.width,
|
|
2268
|
-
height: media.height,
|
|
2269
|
-
url,
|
|
2270
|
-
};
|
|
2271
|
-
}
|
|
2272
|
-
else if (newSource.type === 'video') {
|
|
2273
|
-
const { media, url } = newSource;
|
|
2274
|
-
const canvas = new OffscreenCanvas(1, 1);
|
|
2275
|
-
const ctx = canvas.getContext('2d');
|
|
2276
|
-
const writer = new WritableStream({
|
|
2277
|
-
write(videoFrame) {
|
|
2278
|
-
canvas.width = videoFrame.codedWidth;
|
|
2279
|
-
canvas.height = videoFrame.codedHeight;
|
|
2280
|
-
ctx?.drawImage(videoFrame, 0, 0);
|
|
2281
|
-
videoFrame.close();
|
|
2282
|
-
},
|
|
2283
|
-
close() {
|
|
2284
|
-
console.log('[virtual-background] video background close');
|
|
2285
|
-
},
|
|
2286
|
-
});
|
|
2287
|
-
media.pipeTo(writer).catch((err) => {
|
|
2288
|
-
console.error('media.pipeTo(writer) error', err);
|
|
2289
|
-
});
|
|
2290
|
-
const texture = this.gl.createTexture();
|
|
2291
|
-
if (!texture)
|
|
2292
|
-
throw new Error('Failed to create texture for video');
|
|
2293
|
-
this.gl.bindTexture(this.gl.TEXTURE_2D, texture);
|
|
2294
|
-
this.gl.texImage2D(this.gl.TEXTURE_2D, 0, this.gl.RGBA, 1, 1, 0, this.gl.RGBA, this.gl.UNSIGNED_BYTE, null);
|
|
2295
|
-
this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_WRAP_S, this.gl.CLAMP_TO_EDGE);
|
|
2296
|
-
this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_WRAP_T, this.gl.CLAMP_TO_EDGE);
|
|
2297
|
-
this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_MIN_FILTER, this.gl.LINEAR);
|
|
2298
|
-
this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_MAG_FILTER, this.gl.LINEAR);
|
|
2299
|
-
this.gl.bindTexture(this.gl.TEXTURE_2D, null);
|
|
2300
|
-
this.backgroundRenderInfo = {
|
|
2301
|
-
type: 'video',
|
|
2302
|
-
texture,
|
|
2303
|
-
url,
|
|
2304
|
-
media,
|
|
2305
|
-
canvas,
|
|
2306
|
-
};
|
|
2307
|
-
}
|
|
2308
|
-
}
|
|
2309
|
-
if (!this.backgroundRenderInfo) {
|
|
2310
|
-
console.error('Critical: backgroundRenderInfo is null after processing new source. Setting default color.');
|
|
2311
|
-
const [r, g, b, a] = WebGLRenderer.DEFAULT_BG_COLOR;
|
|
2312
|
-
const colorTexData = this.createColorTexture(r, g, b, a);
|
|
2313
|
-
this.backgroundRenderInfo = {
|
|
2314
|
-
type: 'color',
|
|
2315
|
-
texture: colorTexData.texture,
|
|
2316
|
-
color: colorTexData.color,
|
|
2317
|
-
};
|
|
2318
|
-
this.activeBackgroundSourceIdentifier = `color(${r},${g},${b},${a})`;
|
|
2319
|
-
}
|
|
2320
|
-
}
|
|
2321
|
-
render(videoFrame, options, categoryTexture, confidenceTexture) {
|
|
2322
|
-
if (!this.running)
|
|
2323
|
-
return;
|
|
2324
|
-
const { gl, fbo, frameTexture, storedStateTextures, stateUpdateProgram, stateUpdateLocations, refineFbo, refinedMaskTexture, maskRefineProgram, maskRefineLocations, blendProgram, blendLocations, blurFbo1, blurFbo2, blurTexture1, blurTexture2, } = this;
|
|
2325
|
-
const { displayWidth: width, displayHeight: height } = videoFrame;
|
|
2326
|
-
if (this.canvas.width !== width || this.canvas.height !== height) {
|
|
2327
|
-
this.canvas.width = width;
|
|
2328
|
-
this.canvas.height = height;
|
|
2329
|
-
}
|
|
2330
|
-
if (!categoryTexture || !confidenceTexture) {
|
|
2331
|
-
gl.viewport(0, 0, gl.drawingBufferWidth, gl.drawingBufferHeight);
|
|
2332
|
-
gl.useProgram(blendProgram);
|
|
2333
|
-
const frame = gl.createTexture();
|
|
2334
|
-
gl.activeTexture(gl.TEXTURE0);
|
|
2335
|
-
gl.bindTexture(gl.TEXTURE_2D, frame);
|
|
2336
|
-
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, videoFrame);
|
|
2337
|
-
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
|
|
2338
|
-
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
|
|
2339
|
-
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
|
|
2340
|
-
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
|
|
2341
|
-
gl.uniform1i(blendLocations.frameTexture, 0);
|
|
2342
|
-
gl.uniform1i(blendLocations.enabled, 0);
|
|
2343
|
-
gl.enableVertexAttribArray(blendLocations.position);
|
|
2344
|
-
gl.bindBuffer(gl.ARRAY_BUFFER, this.positionBuffer);
|
|
2345
|
-
gl.vertexAttribPointer(blendLocations.position, 2, gl.FLOAT, false, 0, 0);
|
|
2346
|
-
gl.enableVertexAttribArray(blendLocations.texCoord);
|
|
2347
|
-
gl.bindBuffer(gl.ARRAY_BUFFER, this.texCoordBuffer);
|
|
2348
|
-
gl.vertexAttribPointer(blendLocations.texCoord, 2, gl.FLOAT, false, 0, 0);
|
|
2349
|
-
gl.drawArrays(gl.TRIANGLES, 0, 6);
|
|
2350
|
-
gl.deleteTexture(frame);
|
|
2351
|
-
gl.activeTexture(gl.TEXTURE0);
|
|
2352
|
-
gl.bindTexture(gl.TEXTURE_2D, null);
|
|
2353
|
-
return;
|
|
2354
|
-
}
|
|
2355
|
-
const readStateIndex = this.currentStateIndex;
|
|
2356
|
-
const writeStateIndex = (this.currentStateIndex + 1) % 2;
|
|
2357
|
-
const prevStateTexture = storedStateTextures[readStateIndex];
|
|
2358
|
-
const newStateTexture = storedStateTextures[writeStateIndex];
|
|
2359
|
-
this.updateBackgroundIfNeeded(options.backgroundSource);
|
|
2360
|
-
gl.bindFramebuffer(gl.FRAMEBUFFER, fbo);
|
|
2361
|
-
gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, newStateTexture, 0);
|
|
2362
|
-
gl.bindTexture(gl.TEXTURE_2D, newStateTexture);
|
|
2363
|
-
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, width, height, 0, gl.RGBA, gl.UNSIGNED_BYTE, null);
|
|
2364
|
-
gl.viewport(0, 0, width, height);
|
|
2365
|
-
gl.useProgram(stateUpdateProgram);
|
|
2366
|
-
gl.activeTexture(gl.TEXTURE0);
|
|
2367
|
-
gl.bindTexture(gl.TEXTURE_2D, categoryTexture);
|
|
2368
|
-
gl.uniform1i(stateUpdateLocations.categoryTexture, 0);
|
|
2369
|
-
gl.activeTexture(gl.TEXTURE1);
|
|
2370
|
-
gl.bindTexture(gl.TEXTURE_2D, confidenceTexture);
|
|
2371
|
-
gl.uniform1i(stateUpdateLocations.confidenceTexture, 1);
|
|
2372
|
-
gl.activeTexture(gl.TEXTURE2);
|
|
2373
|
-
gl.bindTexture(gl.TEXTURE_2D, prevStateTexture);
|
|
2374
|
-
gl.uniform1i(stateUpdateLocations.prevStateTexture, 2);
|
|
2375
|
-
gl.uniform1f(stateUpdateLocations.smoothingFactor, options.segmentationOptions?.smoothingFactor ?? 0.8);
|
|
2376
|
-
gl.uniform1f(stateUpdateLocations.smoothstepMin, options.segmentationOptions?.smoothstepMin ?? 0.6);
|
|
2377
|
-
gl.uniform1f(stateUpdateLocations.smoothstepMax, options.segmentationOptions?.smoothstepMax ?? 0.9);
|
|
2378
|
-
gl.uniform1i(stateUpdateLocations.selfieModel, options.isSelfieMode ? 1 : 0);
|
|
2379
|
-
gl.enableVertexAttribArray(stateUpdateLocations.position);
|
|
2380
|
-
gl.bindBuffer(gl.ARRAY_BUFFER, this.positionBuffer);
|
|
2381
|
-
gl.vertexAttribPointer(stateUpdateLocations.position, 2, gl.FLOAT, false, 0, 0);
|
|
2382
|
-
gl.enableVertexAttribArray(stateUpdateLocations.texCoord);
|
|
2383
|
-
gl.bindBuffer(gl.ARRAY_BUFFER, this.texCoordBuffer);
|
|
2384
|
-
gl.vertexAttribPointer(stateUpdateLocations.texCoord, 2, gl.FLOAT, false, 0, 0);
|
|
2385
|
-
gl.drawArrays(gl.TRIANGLES, 0, 6);
|
|
2386
|
-
gl.bindFramebuffer(gl.FRAMEBUFFER, refineFbo);
|
|
2387
|
-
gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, refinedMaskTexture, 0);
|
|
2388
|
-
gl.bindTexture(gl.TEXTURE_2D, refinedMaskTexture);
|
|
2389
|
-
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, width, height, 0, gl.RGBA, gl.UNSIGNED_BYTE, null);
|
|
2390
|
-
gl.viewport(0, 0, width, height);
|
|
2391
|
-
gl.useProgram(maskRefineProgram);
|
|
2392
|
-
gl.enableVertexAttribArray(maskRefineLocations.position);
|
|
2393
|
-
gl.bindBuffer(gl.ARRAY_BUFFER, this.positionBuffer);
|
|
2394
|
-
gl.vertexAttribPointer(maskRefineLocations.position, 2, gl.FLOAT, false, 0, 0);
|
|
2395
|
-
gl.enableVertexAttribArray(maskRefineLocations.texCoord);
|
|
2396
|
-
gl.bindBuffer(gl.ARRAY_BUFFER, this.texCoordBuffer);
|
|
2397
|
-
gl.vertexAttribPointer(maskRefineLocations.texCoord, 2, gl.FLOAT, false, 0, 0);
|
|
2398
|
-
gl.activeTexture(gl.TEXTURE0);
|
|
2399
|
-
gl.bindTexture(gl.TEXTURE_2D, newStateTexture);
|
|
2400
|
-
gl.uniform1i(maskRefineLocations.maskTexture, 0);
|
|
2401
|
-
gl.activeTexture(gl.TEXTURE1);
|
|
2402
|
-
gl.bindTexture(gl.TEXTURE_2D, frameTexture);
|
|
2403
|
-
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, videoFrame);
|
|
2404
|
-
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
|
|
2405
|
-
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
|
|
2406
|
-
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
|
|
2407
|
-
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
|
|
2408
|
-
gl.uniform1i(maskRefineLocations.frameTexture, 1);
|
|
2409
|
-
gl.uniform2f(maskRefineLocations.texelSize, 1.0 / width, 1.0 / height);
|
|
2410
|
-
gl.uniform1f(maskRefineLocations.sigmaSpatial, 2.0);
|
|
2411
|
-
gl.uniform1f(maskRefineLocations.sigmaRange, 0.1);
|
|
2412
|
-
gl.drawArrays(gl.TRIANGLES, 0, 6);
|
|
2413
|
-
gl.disableVertexAttribArray(maskRefineLocations.position);
|
|
2414
|
-
gl.disableVertexAttribArray(maskRefineLocations.texCoord);
|
|
2415
|
-
let backgroundTexToUse;
|
|
2416
|
-
let bgWToSend = width;
|
|
2417
|
-
let bgHToSend = height;
|
|
2418
|
-
if (options.bgBlur > 0 && options.bgBlurRadius > 0) {
|
|
2419
|
-
const downscale = 0.5;
|
|
2420
|
-
const blurW = Math.floor(width * downscale);
|
|
2421
|
-
const blurH = Math.floor(height * downscale);
|
|
2422
|
-
gl.bindTexture(gl.TEXTURE_2D, blurTexture1);
|
|
2423
|
-
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, blurW, blurH, 0, gl.RGBA, gl.UNSIGNED_BYTE, null);
|
|
2424
|
-
gl.bindTexture(gl.TEXTURE_2D, blurTexture2);
|
|
2425
|
-
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, blurW, blurH, 0, gl.RGBA, gl.UNSIGNED_BYTE, null);
|
|
2426
|
-
const KERNEL_RADIUS = 10.0;
|
|
2427
|
-
const radiusScale = Math.max(0.0, options.bgBlurRadius) / KERNEL_RADIUS;
|
|
2428
|
-
gl.useProgram(this.blurProgram);
|
|
2429
|
-
gl.enableVertexAttribArray(this.blurLocations.position);
|
|
2430
|
-
gl.bindBuffer(gl.ARRAY_BUFFER, this.positionBuffer);
|
|
2431
|
-
gl.vertexAttribPointer(this.blurLocations.position, 2, gl.FLOAT, false, 0, 0);
|
|
2432
|
-
gl.enableVertexAttribArray(this.blurLocations.texCoord);
|
|
2433
|
-
gl.bindBuffer(gl.ARRAY_BUFFER, this.texCoordBuffer);
|
|
2434
|
-
gl.vertexAttribPointer(this.blurLocations.texCoord, 2, gl.FLOAT, false, 0, 0);
|
|
2435
|
-
gl.activeTexture(gl.TEXTURE1);
|
|
2436
|
-
gl.bindTexture(gl.TEXTURE_2D, refinedMaskTexture);
|
|
2437
|
-
gl.uniform1i(this.blurLocations.personMask, 1);
|
|
2438
|
-
gl.uniform1f(this.blurLocations.sigma, options.bgBlur * 0.7);
|
|
2439
|
-
gl.uniform1f(this.blurLocations.radiusScale, radiusScale);
|
|
2440
|
-
const blurPasses = [
|
|
2441
|
-
{
|
|
2442
|
-
direction: [1.0, 0.0],
|
|
2443
|
-
input: frameTexture,
|
|
2444
|
-
output: blurFbo1,
|
|
2445
|
-
texelSize: [1.0 / width, 1.0 / height],
|
|
2446
|
-
},
|
|
2447
|
-
{
|
|
2448
|
-
direction: [0.0, 1.0],
|
|
2449
|
-
input: blurTexture1,
|
|
2450
|
-
output: blurFbo2,
|
|
2451
|
-
texelSize: [1.0 / blurW, 1.0 / blurH],
|
|
2452
|
-
},
|
|
2453
|
-
{
|
|
2454
|
-
direction: [1.0, 0.0],
|
|
2455
|
-
input: blurTexture2,
|
|
2456
|
-
output: blurFbo1,
|
|
2457
|
-
texelSize: [1.0 / blurW, 1.0 / blurH],
|
|
2458
|
-
},
|
|
2459
|
-
{
|
|
2460
|
-
direction: [0.0, 1.0],
|
|
2461
|
-
input: blurTexture1,
|
|
2462
|
-
output: blurFbo2,
|
|
2463
|
-
texelSize: [1.0 / blurW, 1.0 / blurH],
|
|
2464
|
-
},
|
|
2465
|
-
];
|
|
2466
|
-
for (const pass of blurPasses) {
|
|
2467
|
-
gl.bindFramebuffer(gl.FRAMEBUFFER, pass.output);
|
|
2468
|
-
gl.viewport(0, 0, blurW, blurH);
|
|
2469
|
-
gl.activeTexture(gl.TEXTURE0);
|
|
2470
|
-
gl.bindTexture(gl.TEXTURE_2D, pass.input);
|
|
2471
|
-
gl.uniform1i(this.blurLocations.image, 0);
|
|
2472
|
-
gl.uniform2f(this.blurLocations.texelSize, pass.texelSize[0], pass.texelSize[1]);
|
|
2473
|
-
gl.uniform2f(this.blurLocations.direction, pass.direction[0], pass.direction[1]);
|
|
2474
|
-
gl.drawArrays(gl.TRIANGLES, 0, 6);
|
|
2475
|
-
}
|
|
2476
|
-
backgroundTexToUse = blurTexture2;
|
|
2477
|
-
bgWToSend = blurW;
|
|
2478
|
-
bgHToSend = blurH;
|
|
2479
|
-
}
|
|
2480
|
-
else if (options.backgroundSource && this.backgroundRenderInfo) {
|
|
2481
|
-
backgroundTexToUse = this.backgroundRenderInfo.texture;
|
|
2482
|
-
if (this.backgroundRenderInfo.type === 'video') {
|
|
2483
|
-
const { canvas } = this.backgroundRenderInfo;
|
|
2484
|
-
bgWToSend = canvas.width || width;
|
|
2485
|
-
bgHToSend = canvas.height || height;
|
|
2486
|
-
}
|
|
2487
|
-
else if (this.backgroundRenderInfo.type === 'image') {
|
|
2488
|
-
bgWToSend = this.backgroundRenderInfo.width;
|
|
2489
|
-
bgHToSend = this.backgroundRenderInfo.height;
|
|
2490
|
-
}
|
|
2491
|
-
else {
|
|
2492
|
-
bgWToSend = width;
|
|
2493
|
-
bgHToSend = height;
|
|
2494
|
-
}
|
|
2495
|
-
}
|
|
2496
|
-
else {
|
|
2497
|
-
backgroundTexToUse = this.backgroundRenderInfo?.texture ?? null;
|
|
2498
|
-
}
|
|
2499
|
-
gl.bindFramebuffer(gl.FRAMEBUFFER, null);
|
|
2500
|
-
gl.viewport(0, 0, gl.drawingBufferWidth, gl.drawingBufferHeight);
|
|
2501
|
-
gl.useProgram(blendProgram);
|
|
2502
|
-
gl.activeTexture(gl.TEXTURE0);
|
|
2503
|
-
gl.bindTexture(gl.TEXTURE_2D, frameTexture);
|
|
2504
|
-
gl.uniform1i(blendLocations.frameTexture, 0);
|
|
2505
|
-
gl.uniform1f(blendLocations.borderSmooth, 0);
|
|
2506
|
-
gl.uniform1f(blendLocations.bgBlur, options.bgBlur);
|
|
2507
|
-
gl.uniform1f(blendLocations.bgBlurRadius, options.bgBlurRadius);
|
|
2508
|
-
gl.uniform1i(blendLocations.enabled, 1);
|
|
2509
|
-
gl.activeTexture(gl.TEXTURE1);
|
|
2510
|
-
gl.bindTexture(gl.TEXTURE_2D, refinedMaskTexture);
|
|
2511
|
-
gl.uniform1i(blendLocations.currentStateTexture, 1);
|
|
2512
|
-
if (backgroundTexToUse) {
|
|
2513
|
-
gl.activeTexture(gl.TEXTURE2);
|
|
2514
|
-
gl.bindTexture(gl.TEXTURE_2D, backgroundTexToUse);
|
|
2515
|
-
gl.uniform1i(blendLocations.backgroundTexture, 2);
|
|
2516
|
-
gl.uniform2f(blendLocations.bgImageDimensions, bgWToSend, bgHToSend);
|
|
2517
|
-
gl.uniform2f(blendLocations.canvasDimensions, width, height);
|
|
2518
|
-
}
|
|
2519
|
-
else {
|
|
2520
|
-
gl.uniform2f(blendLocations.bgImageDimensions, width, height);
|
|
2521
|
-
gl.uniform2f(blendLocations.canvasDimensions, width, height);
|
|
2522
|
-
}
|
|
2523
|
-
gl.enableVertexAttribArray(blendLocations.position);
|
|
2524
|
-
gl.bindBuffer(gl.ARRAY_BUFFER, this.positionBuffer);
|
|
2525
|
-
gl.vertexAttribPointer(blendLocations.position, 2, gl.FLOAT, false, 0, 0);
|
|
2526
|
-
gl.enableVertexAttribArray(blendLocations.texCoord);
|
|
2527
|
-
gl.bindBuffer(gl.ARRAY_BUFFER, this.texCoordBuffer);
|
|
2528
|
-
gl.vertexAttribPointer(blendLocations.texCoord, 2, gl.FLOAT, false, 0, 0);
|
|
2529
|
-
gl.drawArrays(gl.TRIANGLES, 0, 6);
|
|
2530
|
-
for (let i = 0; i < 3; ++i) {
|
|
2531
|
-
gl.activeTexture(gl.TEXTURE0 + i);
|
|
2532
|
-
gl.bindTexture(gl.TEXTURE_2D, null);
|
|
2533
|
-
}
|
|
2534
|
-
this.currentStateIndex = writeStateIndex;
|
|
2535
|
-
}
|
|
2536
|
-
close() {
|
|
2537
|
-
if (!this.running)
|
|
2538
|
-
return;
|
|
2539
|
-
this.running = false;
|
|
2540
|
-
const { gl, fbo, refineFbo, refinedMaskTexture, blurFbo1, blurFbo2 } = this;
|
|
2541
|
-
gl.clearColor(0, 0, 0, 0);
|
|
2542
|
-
gl.clear(gl.COLOR_BUFFER_BIT);
|
|
2543
|
-
gl.bindFramebuffer(gl.FRAMEBUFFER, null);
|
|
2544
|
-
if (fbo)
|
|
2545
|
-
gl.deleteFramebuffer(fbo);
|
|
2546
|
-
if (refineFbo)
|
|
2547
|
-
gl.deleteFramebuffer(refineFbo);
|
|
2548
|
-
if (blurFbo1)
|
|
2549
|
-
gl.deleteFramebuffer(blurFbo1);
|
|
2550
|
-
if (blurFbo2)
|
|
2551
|
-
gl.deleteFramebuffer(blurFbo2);
|
|
2552
|
-
gl.deleteProgram(this.stateUpdateProgram);
|
|
2553
|
-
gl.deleteProgram(this.maskRefineProgram);
|
|
2554
|
-
gl.deleteProgram(this.blurProgram);
|
|
2555
|
-
gl.deleteProgram(this.blendProgram);
|
|
2556
|
-
if (this.positionBuffer)
|
|
2557
|
-
gl.deleteBuffer(this.positionBuffer);
|
|
2558
|
-
if (this.texCoordBuffer)
|
|
2559
|
-
gl.deleteBuffer(this.texCoordBuffer);
|
|
2560
|
-
if (refinedMaskTexture)
|
|
2561
|
-
gl.deleteTexture(refinedMaskTexture);
|
|
2562
|
-
if (this.blurTexture1)
|
|
2563
|
-
gl.deleteTexture(this.blurTexture1);
|
|
2564
|
-
if (this.blurTexture2)
|
|
2565
|
-
gl.deleteTexture(this.blurTexture2);
|
|
2566
|
-
this.storedStateTextures.forEach((t) => t && gl.deleteTexture(t));
|
|
2567
|
-
this.storedStateTextures.splice(0, this.storedStateTextures.length);
|
|
2568
|
-
if (this.backgroundRenderInfo?.texture) {
|
|
2569
|
-
gl.deleteTexture(this.backgroundRenderInfo.texture);
|
|
2570
|
-
this.backgroundRenderInfo = null;
|
|
2571
|
-
}
|
|
2572
|
-
this.activeBackgroundSourceIdentifier = null;
|
|
2573
|
-
}
|
|
2574
|
-
}
|
|
2575
|
-
|
|
2576
|
-
/**
|
|
2577
|
-
* Wraps a video MediaStreamTrack in a real-time processing pipeline.
|
|
2578
|
-
* Incoming frames are processed through a transformer and re-emitted
|
|
2579
|
-
* on a new MediaStreamVideoTrack for downstream consumption.
|
|
2580
|
-
*/
|
|
2581
|
-
class VirtualBackground extends BaseVideoProcessor {
|
|
2582
|
-
options;
|
|
2583
|
-
segmenter = null;
|
|
2584
|
-
isSegmenterReady = false;
|
|
2585
|
-
webGlRenderer;
|
|
2586
|
-
opts;
|
|
2587
|
-
latestCategoryMask = undefined;
|
|
2588
|
-
latestConfidenceMask = undefined;
|
|
2589
|
-
lastFrameTime = -1;
|
|
2590
|
-
constructor(track, options = {}, hooks = {}) {
|
|
2591
|
-
super(track, hooks);
|
|
2592
|
-
this.options = options;
|
|
2593
|
-
}
|
|
2594
|
-
async initialize() {
|
|
2595
|
-
this.webGlRenderer = new WebGLRenderer(this.canvas);
|
|
2596
|
-
await this.initializeSegmenter();
|
|
2597
|
-
}
|
|
2598
|
-
async initializeSegmenter() {
|
|
2599
|
-
try {
|
|
2600
|
-
this.opts = await this.initializeSegmenterOptions();
|
|
2601
|
-
const basePath = this.options.basePath ||
|
|
2602
|
-
`https://unpkg.com/${packageName}@${version}/mediapipe`;
|
|
2603
|
-
const model = this.options.modelPath ||
|
|
2604
|
-
`${basePath}/models/selfie_segmenter_landscape.tflite`;
|
|
2605
|
-
const fileset = await FilesetResolver.forVisionTasks(`${basePath}/wasm`);
|
|
2606
|
-
this.segmenter = await ImageSegmenter.createFromOptions(fileset, {
|
|
2607
|
-
baseOptions: { modelAssetPath: model, delegate: 'GPU' },
|
|
2608
|
-
runningMode: 'VIDEO',
|
|
2609
|
-
outputCategoryMask: true,
|
|
2610
|
-
outputConfidenceMasks: true,
|
|
2611
|
-
canvas: this.canvas,
|
|
2612
|
-
});
|
|
2613
|
-
this.isSegmenterReady = true;
|
|
2614
|
-
}
|
|
2615
|
-
catch (error) {
|
|
2616
|
-
this.isSegmenterReady = false;
|
|
2617
|
-
this.hooks.onError?.(error);
|
|
2618
|
-
throw error;
|
|
2619
|
-
}
|
|
2620
|
-
}
|
|
2621
|
-
async transform(frame) {
|
|
2622
|
-
const currentTime = frame.timestamp;
|
|
2623
|
-
const hasNewFrame = currentTime - this.lastFrameTime > 1_000;
|
|
2624
|
-
this.lastFrameTime = currentTime;
|
|
2625
|
-
if (hasNewFrame && this.isSegmenterReady && this.segmenter) {
|
|
2626
|
-
const start = performance.now();
|
|
2627
|
-
await this.runSegmentation(frame);
|
|
2628
|
-
this.webGlRenderer.render(frame, this.opts, this.latestCategoryMask, this.latestConfidenceMask);
|
|
2629
|
-
this.updateStats(performance.now() - start);
|
|
2630
|
-
}
|
|
2631
|
-
return new VideoFrame(this.canvas, { timestamp: frame.timestamp });
|
|
2632
|
-
}
|
|
2633
|
-
async runSegmentation(frame) {
|
|
2634
|
-
if (!this.segmenter)
|
|
2635
|
-
return;
|
|
2636
|
-
return new Promise((resolve) => {
|
|
2637
|
-
const timestamp = Math.floor(performance.now());
|
|
2638
|
-
this.segmenter.segmentForVideo(frame, timestamp, (result) => {
|
|
2639
|
-
try {
|
|
2640
|
-
this.latestCategoryMask = result.categoryMask?.getAsWebGLTexture();
|
|
2641
|
-
this.latestConfidenceMask =
|
|
2642
|
-
result.confidenceMasks?.[0]?.getAsWebGLTexture();
|
|
2643
|
-
}
|
|
2644
|
-
catch (err) {
|
|
2645
|
-
console.error('[virtual-background] segmentation error:', err);
|
|
2646
|
-
this.hooks.onError?.(err);
|
|
2647
|
-
}
|
|
2648
|
-
finally {
|
|
2649
|
-
result.close();
|
|
2650
|
-
resolve();
|
|
2651
|
-
}
|
|
2652
|
-
});
|
|
2653
|
-
});
|
|
2654
|
-
}
|
|
2655
|
-
async initializeSegmenterOptions() {
|
|
2656
|
-
const isSelfieMode = this.options.modelPath
|
|
2657
|
-
? this.options.modelPath.includes('selfie_segmenter')
|
|
2658
|
-
: true;
|
|
2659
|
-
if (this.options.backgroundFilter === 'image') {
|
|
2660
|
-
const source = await this.loadBackground(this.options.backgroundImage);
|
|
2661
|
-
return {
|
|
2662
|
-
backgroundSource: source,
|
|
2663
|
-
bgBlur: 0,
|
|
2664
|
-
bgBlurRadius: 0,
|
|
2665
|
-
isSelfieMode,
|
|
2666
|
-
segmentationOptions: this.options.segmentationOptions,
|
|
2667
|
-
};
|
|
2668
|
-
}
|
|
2669
|
-
const blurLevel = this.options.backgroundBlurLevel;
|
|
2670
|
-
const strength = typeof blurLevel === 'string'
|
|
2671
|
-
? BACKGROUND_BLUR_MAP[blurLevel]
|
|
2672
|
-
: Math.round(blurLevel ?? 5);
|
|
2673
|
-
return {
|
|
2674
|
-
backgroundSource: undefined,
|
|
2675
|
-
bgBlur: Math.min(strength * 1.5, 20),
|
|
2676
|
-
bgBlurRadius: Math.min(strength, 10),
|
|
2677
|
-
isSelfieMode,
|
|
2678
|
-
segmentationOptions: this.options.segmentationOptions,
|
|
2679
|
-
};
|
|
2680
|
-
}
|
|
2681
|
-
async loadBackground(url) {
|
|
2682
|
-
if (!url)
|
|
2683
|
-
return null;
|
|
2684
|
-
const result = await fetch(url, { signal: this.abortController.signal });
|
|
2685
|
-
if (!result.ok) {
|
|
2686
|
-
throw new Error(`[virtual-background] Failed to load background image: ${result.status} ${result.statusText}`);
|
|
2687
|
-
}
|
|
2688
|
-
return {
|
|
2689
|
-
type: 'image',
|
|
2690
|
-
media: await createImageBitmap(await result.blob()),
|
|
2691
|
-
url,
|
|
2692
|
-
};
|
|
2693
|
-
}
|
|
2694
|
-
onFlush() {
|
|
2695
|
-
this.destroySegmenter();
|
|
2696
|
-
}
|
|
2697
|
-
onStop() {
|
|
2698
|
-
this.webGlRenderer?.close();
|
|
2699
|
-
this.destroySegmenter();
|
|
2700
|
-
}
|
|
2701
|
-
destroySegmenter() {
|
|
2702
|
-
this.segmenter?.close();
|
|
2703
|
-
this.segmenter = null;
|
|
2704
|
-
this.isSegmenterReady = false;
|
|
2705
|
-
}
|
|
2706
|
-
get processorName() {
|
|
2707
|
-
return 'background-processor';
|
|
2708
|
-
}
|
|
2709
|
-
}
|
|
2710
|
-
|
|
2711
|
-
/**
|
|
2712
|
-
* Simple WebGL renderer for full-screen Gaussian blur.
|
|
2713
|
-
* Uses a two-pass separable Gaussian blur (horizontal then vertical).
|
|
2714
|
-
* Optimized for moderation use cases by blurring at reduced resolution (15% scale)
|
|
2715
|
-
* and upscaling back to full resolution for output.
|
|
2716
|
-
*/
|
|
2717
|
-
class FullScreenBlurRenderer {
|
|
2718
|
-
canvas;
|
|
2719
|
-
gl;
|
|
2720
|
-
blurProgramHandle;
|
|
2721
|
-
blurLocations;
|
|
2722
|
-
passthroughProgramHandle;
|
|
2723
|
-
passthroughLocations;
|
|
2724
|
-
positionBuffer;
|
|
2725
|
-
texCoordBuffer;
|
|
2726
|
-
pingTexture;
|
|
2727
|
-
pongTexture;
|
|
2728
|
-
pingFbo;
|
|
2729
|
-
pongFbo;
|
|
2730
|
-
inputTexture = null;
|
|
2731
|
-
isRunning = false;
|
|
2732
|
-
targetWidth = 0;
|
|
2733
|
-
targetHeight = 0;
|
|
2734
|
-
weightCache = new Map();
|
|
2735
|
-
constructor(canvas) {
|
|
2736
|
-
this.canvas = canvas;
|
|
2737
|
-
const gl = canvas.getContext('webgl2', {
|
|
2738
|
-
alpha: false,
|
|
2739
|
-
antialias: false,
|
|
2740
|
-
desynchronized: true,
|
|
2741
|
-
});
|
|
2742
|
-
if (!gl)
|
|
2743
|
-
throw new Error('WebGL2 not supported');
|
|
2744
|
-
this.gl = gl;
|
|
2745
|
-
const vertexShaderSource = `#version 300 es
|
|
2746
|
-
precision highp float;
|
|
2747
|
-
in vec2 a_position;
|
|
2748
|
-
in vec2 a_texCoord;
|
|
2749
|
-
out vec2 v_texCoord;
|
|
2750
|
-
void main() {
|
|
2751
|
-
v_texCoord = a_texCoord;
|
|
2752
|
-
gl_Position = vec4(a_position, 0.0, 1.0);
|
|
2753
|
-
}
|
|
2754
|
-
`;
|
|
2755
|
-
const fragmentShaderSource = `#version 300 es
|
|
2756
|
-
precision highp float;
|
|
2757
|
-
in vec2 v_texCoord;
|
|
2758
|
-
out vec4 outColor;
|
|
2759
|
-
uniform sampler2D u_image;
|
|
2760
|
-
uniform vec2 u_texelSize;
|
|
2761
|
-
uniform vec2 u_direction;
|
|
2762
|
-
uniform float u_weights[25];
|
|
2763
|
-
void main() {
|
|
2764
|
-
vec4 color = vec4(0.0);
|
|
2765
|
-
for (int i = -12; i <= 12; i++) {
|
|
2766
|
-
float w = u_weights[i + 12];
|
|
2767
|
-
if (w == 0.0) continue;
|
|
2768
|
-
vec2 offset = float(i) * u_direction * u_texelSize;
|
|
2769
|
-
color += w * texture(u_image, v_texCoord + offset);
|
|
2770
|
-
}
|
|
2771
|
-
outColor = color;
|
|
2772
|
-
}
|
|
2773
|
-
`;
|
|
2774
|
-
this.blurProgramHandle = this.createAndLinkProgram(vertexShaderSource, fragmentShaderSource);
|
|
2775
|
-
const passthroughFragmentShaderSource = `#version 300 es
|
|
2776
|
-
precision highp float;
|
|
2777
|
-
in vec2 v_texCoord;
|
|
2778
|
-
out vec4 outColor;
|
|
2779
|
-
uniform sampler2D u_image;
|
|
2780
|
-
void main() {
|
|
2781
|
-
outColor = texture(u_image, v_texCoord);
|
|
2782
|
-
}
|
|
2783
|
-
`;
|
|
2784
|
-
this.passthroughProgramHandle = this.createAndLinkProgram(vertexShaderSource, passthroughFragmentShaderSource);
|
|
2785
|
-
const blurProgram = this.blurProgramHandle;
|
|
2786
|
-
const passthroughProgram = this.passthroughProgramHandle;
|
|
2787
|
-
this.blurLocations = {
|
|
2788
|
-
positionLocation: gl.getAttribLocation(blurProgram, 'a_position'),
|
|
2789
|
-
texCoordLocation: gl.getAttribLocation(blurProgram, 'a_texCoord'),
|
|
2790
|
-
imageLocation: gl.getUniformLocation(blurProgram, 'u_image'),
|
|
2791
|
-
texelSizeLocation: gl.getUniformLocation(blurProgram, 'u_texelSize'),
|
|
2792
|
-
directionLocation: gl.getUniformLocation(blurProgram, 'u_direction'),
|
|
2793
|
-
weightsLocation: gl.getUniformLocation(blurProgram, 'u_weights'),
|
|
2794
|
-
};
|
|
2795
|
-
this.passthroughLocations = {
|
|
2796
|
-
positionLocation: gl.getAttribLocation(passthroughProgram, 'a_position'),
|
|
2797
|
-
texCoordLocation: gl.getAttribLocation(passthroughProgram, 'a_texCoord'),
|
|
2798
|
-
imageLocation: gl.getUniformLocation(passthroughProgram, 'u_image'),
|
|
2799
|
-
};
|
|
2800
|
-
this.positionBuffer = gl.createBuffer();
|
|
2801
|
-
gl.bindBuffer(gl.ARRAY_BUFFER, this.positionBuffer);
|
|
2802
|
-
gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([-1, -1, 1, -1, -1, 1, -1, 1, 1, -1, 1, 1]), gl.STATIC_DRAW);
|
|
2803
|
-
this.texCoordBuffer = gl.createBuffer();
|
|
2804
|
-
gl.bindBuffer(gl.ARRAY_BUFFER, this.texCoordBuffer);
|
|
2805
|
-
gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([0, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 0]), gl.STATIC_DRAW);
|
|
2806
|
-
const createTexture2D = () => {
|
|
2807
|
-
const tex = gl.createTexture();
|
|
2808
|
-
if (!tex)
|
|
2809
|
-
throw new Error('Failed to create texture');
|
|
2810
|
-
gl.bindTexture(gl.TEXTURE_2D, tex);
|
|
2811
|
-
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
|
|
2812
|
-
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
|
|
2813
|
-
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
|
|
2814
|
-
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR);
|
|
2815
|
-
gl.bindTexture(gl.TEXTURE_2D, null);
|
|
2816
|
-
return tex;
|
|
2817
|
-
};
|
|
2818
|
-
const createFramebufferForTexture = (tex) => {
|
|
2819
|
-
const fb = gl.createFramebuffer();
|
|
2820
|
-
if (!fb)
|
|
2821
|
-
throw new Error('Failed to create framebuffer');
|
|
2822
|
-
gl.bindFramebuffer(gl.FRAMEBUFFER, fb);
|
|
2823
|
-
gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, tex, 0);
|
|
2824
|
-
gl.bindFramebuffer(gl.FRAMEBUFFER, null);
|
|
2825
|
-
return fb;
|
|
2826
|
-
};
|
|
2827
|
-
this.pingTexture = createTexture2D();
|
|
2828
|
-
this.pongTexture = createTexture2D();
|
|
2829
|
-
this.pingFbo = createFramebufferForTexture(this.pingTexture);
|
|
2830
|
-
this.pongFbo = createFramebufferForTexture(this.pongTexture);
|
|
2831
|
-
this.inputTexture = createTexture2D();
|
|
2832
|
-
gl.useProgram(blurProgram);
|
|
2833
|
-
gl.bindBuffer(gl.ARRAY_BUFFER, this.positionBuffer);
|
|
2834
|
-
gl.enableVertexAttribArray(this.blurLocations.positionLocation);
|
|
2835
|
-
gl.vertexAttribPointer(this.blurLocations.positionLocation, 2, gl.FLOAT, false, 0, 0);
|
|
2836
|
-
gl.bindBuffer(gl.ARRAY_BUFFER, this.texCoordBuffer);
|
|
2837
|
-
gl.enableVertexAttribArray(this.blurLocations.texCoordLocation);
|
|
2838
|
-
gl.vertexAttribPointer(this.blurLocations.texCoordLocation, 2, gl.FLOAT, false, 0, 0);
|
|
2839
|
-
if (this.blurLocations.imageLocation) {
|
|
2840
|
-
gl.uniform1i(this.blurLocations.imageLocation, 0);
|
|
2841
|
-
}
|
|
2842
|
-
this.isRunning = true;
|
|
2843
|
-
}
|
|
2844
|
-
createAndLinkProgram(vsSource, fsSource) {
|
|
2845
|
-
const gl = this.gl;
|
|
2846
|
-
const vs = this.createShader(gl.VERTEX_SHADER, vsSource);
|
|
2847
|
-
const fs = this.createShader(gl.FRAGMENT_SHADER, fsSource);
|
|
2848
|
-
const prog = gl.createProgram();
|
|
2849
|
-
if (!prog)
|
|
2850
|
-
throw new Error('Failed to create program');
|
|
2851
|
-
gl.attachShader(prog, vs);
|
|
2852
|
-
gl.attachShader(prog, fs);
|
|
2853
|
-
gl.linkProgram(prog);
|
|
2854
|
-
if (!gl.getProgramParameter(prog, gl.LINK_STATUS)) {
|
|
2855
|
-
throw new Error('Shader link failed: ' + gl.getProgramInfoLog(prog));
|
|
2856
|
-
}
|
|
2857
|
-
gl.deleteShader(vs);
|
|
2858
|
-
gl.deleteShader(fs);
|
|
2859
|
-
return prog;
|
|
2860
|
-
}
|
|
2861
|
-
createShader(type, source) {
|
|
2862
|
-
const gl = this.gl;
|
|
2863
|
-
const shader = gl.createShader(type);
|
|
2864
|
-
if (!shader)
|
|
2865
|
-
throw new Error('Failed to create shader');
|
|
2866
|
-
gl.shaderSource(shader, source);
|
|
2867
|
-
gl.compileShader(shader);
|
|
2868
|
-
if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) {
|
|
2869
|
-
throw new Error('Shader compile failed: ' + gl.getShaderInfoLog(shader));
|
|
2870
|
-
}
|
|
2871
|
-
return shader;
|
|
2872
|
-
}
|
|
2873
|
-
getGaussianWeights(radius) {
|
|
2874
|
-
const r = Math.max(0, Math.min(radius | 0, 12));
|
|
2875
|
-
const cached = this.weightCache.get(r);
|
|
2876
|
-
if (cached)
|
|
2877
|
-
return cached;
|
|
2878
|
-
const weights = new Float32Array(25);
|
|
2879
|
-
if (r === 0) {
|
|
2880
|
-
weights[12] = 1.0;
|
|
2881
|
-
this.weightCache.set(r, weights);
|
|
2882
|
-
return weights;
|
|
2883
|
-
}
|
|
2884
|
-
const sigma = r * 0.6;
|
|
2885
|
-
let sum = 0;
|
|
2886
|
-
for (let i = -r; i <= r; i++) {
|
|
2887
|
-
const w = Math.exp(-(i * i) / (2 * sigma * sigma));
|
|
2888
|
-
weights[i + 12] = w;
|
|
2889
|
-
sum += w;
|
|
2890
|
-
}
|
|
2891
|
-
for (let i = -r; i <= r; i++) {
|
|
2892
|
-
weights[i + 12] /= sum;
|
|
2893
|
-
}
|
|
2894
|
-
this.weightCache.set(r, weights);
|
|
2895
|
-
return weights;
|
|
2896
|
-
}
|
|
2897
|
-
render(frame, radius) {
|
|
2898
|
-
if (!this.isRunning)
|
|
2899
|
-
return;
|
|
2900
|
-
const gl = this.gl;
|
|
2901
|
-
const width = frame.displayWidth;
|
|
2902
|
-
const height = frame.displayHeight;
|
|
2903
|
-
if (!width || !height)
|
|
2904
|
-
return;
|
|
2905
|
-
if (this.canvas.width !== width || this.canvas.height !== height) {
|
|
2906
|
-
this.canvas.width = width;
|
|
2907
|
-
this.canvas.height = height;
|
|
2908
|
-
}
|
|
2909
|
-
const scale = 0.15;
|
|
2910
|
-
const scaledWidth = Math.max(1, Math.floor(width * scale));
|
|
2911
|
-
const scaledHeight = Math.max(1, Math.floor(height * scale));
|
|
2912
|
-
if (scaledWidth !== this.targetWidth ||
|
|
2913
|
-
scaledHeight !== this.targetHeight) {
|
|
2914
|
-
this.targetWidth = scaledWidth;
|
|
2915
|
-
this.targetHeight = scaledHeight;
|
|
2916
|
-
gl.bindTexture(gl.TEXTURE_2D, this.pingTexture);
|
|
2917
|
-
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, scaledWidth, scaledHeight, 0, gl.RGBA, gl.UNSIGNED_BYTE, null);
|
|
2918
|
-
gl.bindTexture(gl.TEXTURE_2D, this.pongTexture);
|
|
2919
|
-
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, scaledWidth, scaledHeight, 0, gl.RGBA, gl.UNSIGNED_BYTE, null);
|
|
2920
|
-
gl.bindTexture(gl.TEXTURE_2D, null);
|
|
2921
|
-
}
|
|
2922
|
-
gl.activeTexture(gl.TEXTURE0);
|
|
2923
|
-
gl.bindTexture(gl.TEXTURE_2D, this.inputTexture);
|
|
2924
|
-
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, frame);
|
|
2925
|
-
gl.useProgram(this.blurProgramHandle);
|
|
2926
|
-
if (this.blurLocations.texelSizeLocation) {
|
|
2927
|
-
gl.uniform2f(this.blurLocations.texelSizeLocation, 1.0 / scaledWidth, 1.0 / scaledHeight);
|
|
2928
|
-
}
|
|
2929
|
-
const weights = this.getGaussianWeights(radius);
|
|
2930
|
-
if (this.blurLocations.weightsLocation) {
|
|
2931
|
-
gl.uniform1fv(this.blurLocations.weightsLocation, weights);
|
|
2932
|
-
}
|
|
2933
|
-
gl.viewport(0, 0, scaledWidth, scaledHeight);
|
|
2934
|
-
gl.bindFramebuffer(gl.FRAMEBUFFER, this.pingFbo);
|
|
2935
|
-
gl.bindTexture(gl.TEXTURE_2D, this.inputTexture);
|
|
2936
|
-
if (this.blurLocations.directionLocation) {
|
|
2937
|
-
gl.uniform2f(this.blurLocations.directionLocation, 1.0, 0.0);
|
|
2938
|
-
}
|
|
2939
|
-
gl.drawArrays(gl.TRIANGLES, 0, 6);
|
|
2940
|
-
gl.bindFramebuffer(gl.FRAMEBUFFER, this.pongFbo);
|
|
2941
|
-
gl.bindTexture(gl.TEXTURE_2D, this.pingTexture);
|
|
2942
|
-
if (this.blurLocations.directionLocation) {
|
|
2943
|
-
gl.uniform2f(this.blurLocations.directionLocation, 0.0, 1.0);
|
|
2944
|
-
}
|
|
2945
|
-
gl.drawArrays(gl.TRIANGLES, 0, 6);
|
|
2946
|
-
gl.bindFramebuffer(gl.FRAMEBUFFER, null);
|
|
2947
|
-
gl.viewport(0, 0, width, height);
|
|
2948
|
-
gl.useProgram(this.passthroughProgramHandle);
|
|
2949
|
-
gl.bindBuffer(gl.ARRAY_BUFFER, this.positionBuffer);
|
|
2950
|
-
gl.enableVertexAttribArray(this.passthroughLocations.positionLocation);
|
|
2951
|
-
gl.vertexAttribPointer(this.passthroughLocations.positionLocation, 2, gl.FLOAT, false, 0, 0);
|
|
2952
|
-
gl.bindBuffer(gl.ARRAY_BUFFER, this.texCoordBuffer);
|
|
2953
|
-
gl.enableVertexAttribArray(this.passthroughLocations.texCoordLocation);
|
|
2954
|
-
gl.vertexAttribPointer(this.passthroughLocations.texCoordLocation, 2, gl.FLOAT, false, 0, 0);
|
|
2955
|
-
gl.bindTexture(gl.TEXTURE_2D, this.pongTexture);
|
|
2956
|
-
if (this.passthroughLocations.imageLocation) {
|
|
2957
|
-
gl.uniform1i(this.passthroughLocations.imageLocation, 0);
|
|
2958
|
-
}
|
|
2959
|
-
gl.drawArrays(gl.TRIANGLES, 0, 6);
|
|
2960
|
-
}
|
|
2961
|
-
close() {
|
|
2962
|
-
if (!this.isRunning)
|
|
2963
|
-
return;
|
|
2964
|
-
this.isRunning = false;
|
|
2965
|
-
const gl = this.gl;
|
|
2966
|
-
if (this.pingFbo)
|
|
2967
|
-
gl.deleteFramebuffer(this.pingFbo);
|
|
2968
|
-
if (this.pongFbo)
|
|
2969
|
-
gl.deleteFramebuffer(this.pongFbo);
|
|
2970
|
-
if (this.pingTexture)
|
|
2971
|
-
gl.deleteTexture(this.pingTexture);
|
|
2972
|
-
if (this.pongTexture)
|
|
2973
|
-
gl.deleteTexture(this.pongTexture);
|
|
2974
|
-
if (this.inputTexture)
|
|
2975
|
-
gl.deleteTexture(this.inputTexture);
|
|
2976
|
-
if (this.positionBuffer)
|
|
2977
|
-
gl.deleteBuffer(this.positionBuffer);
|
|
2978
|
-
if (this.texCoordBuffer)
|
|
2979
|
-
gl.deleteBuffer(this.texCoordBuffer);
|
|
2980
|
-
gl.deleteProgram(this.blurProgramHandle);
|
|
2981
|
-
gl.deleteProgram(this.passthroughProgramHandle);
|
|
2982
|
-
}
|
|
2983
|
-
}
|
|
2984
|
-
|
|
2985
|
-
/**
|
|
2986
|
-
* A video filter that applies a full-screen blur to each frame.
|
|
2987
|
-
*
|
|
2988
|
-
* It uses a WebGL renderer to blur the incoming camera track and outputs
|
|
2989
|
-
* a new track with the effect applied. Setup and frame handling are managed
|
|
2990
|
-
* by the base processor.
|
|
2991
|
-
*/
|
|
2992
|
-
class FullScreenBlur extends BaseVideoProcessor {
|
|
2993
|
-
blurRenderer;
|
|
2994
|
-
blurRadius;
|
|
2995
|
-
/**
|
|
2996
|
-
* Creates a new full-screen blur processor for the given video track.
|
|
2997
|
-
*
|
|
2998
|
-
* @param track - The input camera track to blur.
|
|
2999
|
-
* @param options - Optional settings such as the blur radius.
|
|
3000
|
-
* @param hooks - Optional callbacks for stats and error reporting.
|
|
3001
|
-
*/
|
|
3002
|
-
constructor(track, options = {}, hooks = {}) {
|
|
3003
|
-
super(track, hooks);
|
|
3004
|
-
this.blurRadius = options.blurRadius ?? 6;
|
|
3005
|
-
}
|
|
3006
|
-
async initialize() {
|
|
3007
|
-
this.blurRenderer = new FullScreenBlurRenderer(this.canvas);
|
|
3008
|
-
}
|
|
3009
|
-
async transform(frame) {
|
|
3010
|
-
this.blurRenderer.render(frame, this.blurRadius);
|
|
3011
|
-
return new VideoFrame(this.canvas, { timestamp: frame.timestamp });
|
|
3012
|
-
}
|
|
3013
|
-
onStop() {
|
|
3014
|
-
this.blurRenderer?.close();
|
|
3015
|
-
}
|
|
3016
|
-
get processorName() {
|
|
3017
|
-
return 'fullscreen-blur';
|
|
3018
|
-
}
|
|
3019
|
-
}
|
|
3020
|
-
|
|
3021
|
-
export { BACKGROUND_BLUR_MAP, BaseVideoProcessor, FullScreenBlur, SegmentationLevel, VirtualBackground, createRenderer, isMediaPipePlatformSupported, isPlatformSupported, loadMediaPipe, loadTFLite };
|
|
3022
|
-
//# sourceMappingURL=index.es.js.map
|