@stream-io/video-filters-web 0.8.6 → 0.8.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. package/CHANGELOG.md +6 -0
  2. package/dist/esm/index.js +10 -0
  3. package/dist/esm/index.js.map +1 -0
  4. package/dist/esm/src/BaseVideoProcessor.js +107 -0
  5. package/dist/esm/src/BaseVideoProcessor.js.map +1 -0
  6. package/dist/esm/src/FallbackGenerator.js +58 -0
  7. package/dist/esm/src/FallbackGenerator.js.map +1 -0
  8. package/dist/esm/src/FallbackProcessor.js +89 -0
  9. package/dist/esm/src/FallbackProcessor.js.map +1 -0
  10. package/dist/esm/src/FullScreenBlur.js +41 -0
  11. package/dist/esm/src/FullScreenBlur.js.map +1 -0
  12. package/dist/esm/src/FullScreenBlurRenderer.js +276 -0
  13. package/dist/esm/src/FullScreenBlurRenderer.js.map +1 -0
  14. package/dist/esm/src/VirtualBackground.js +143 -0
  15. package/dist/esm/src/VirtualBackground.js.map +1 -0
  16. package/dist/esm/src/WebGLRenderer.js +723 -0
  17. package/dist/esm/src/WebGLRenderer.js.map +1 -0
  18. package/dist/esm/src/compatibility.js +48 -0
  19. package/dist/esm/src/compatibility.js.map +1 -0
  20. package/dist/esm/src/legacy/createRenderer.js +33 -0
  21. package/dist/esm/src/legacy/createRenderer.js.map +1 -0
  22. package/dist/esm/src/legacy/helpers/webglHelper.js +94 -0
  23. package/dist/esm/src/legacy/helpers/webglHelper.js.map +1 -0
  24. package/dist/esm/src/legacy/segmentation.js +14 -0
  25. package/dist/esm/src/legacy/segmentation.js.map +1 -0
  26. package/dist/esm/src/legacy/tflite-simd.js +728 -0
  27. package/dist/esm/src/legacy/tflite-simd.js.map +1 -0
  28. package/dist/esm/src/legacy/tflite.js +31 -0
  29. package/dist/esm/src/legacy/tflite.js.map +1 -0
  30. package/dist/esm/src/legacy/webgl2/backgroundBlurStage.js +202 -0
  31. package/dist/esm/src/legacy/webgl2/backgroundBlurStage.js.map +1 -0
  32. package/dist/esm/src/legacy/webgl2/backgroundImageStage.js +157 -0
  33. package/dist/esm/src/legacy/webgl2/backgroundImageStage.js.map +1 -0
  34. package/dist/esm/src/legacy/webgl2/jointBilateralFilterStage.js +113 -0
  35. package/dist/esm/src/legacy/webgl2/jointBilateralFilterStage.js.map +1 -0
  36. package/dist/esm/src/legacy/webgl2/resizingStage.js +57 -0
  37. package/dist/esm/src/legacy/webgl2/resizingStage.js.map +1 -0
  38. package/dist/esm/src/legacy/webgl2/softmaxStage.js +51 -0
  39. package/dist/esm/src/legacy/webgl2/softmaxStage.js.map +1 -0
  40. package/dist/esm/src/legacy/webgl2/webgl2Pipeline.js +105 -0
  41. package/dist/esm/src/legacy/webgl2/webgl2Pipeline.js.map +1 -0
  42. package/dist/esm/src/mediapipe.js +16 -0
  43. package/dist/esm/src/mediapipe.js.map +1 -0
  44. package/dist/esm/src/types.js +11 -0
  45. package/dist/esm/src/types.js.map +1 -0
  46. package/dist/esm/src/version.js +5 -0
  47. package/dist/esm/src/version.js.map +1 -0
  48. package/dist/index.cjs.js +1 -1
  49. package/package.json +3 -3
  50. package/dist/index.es.js +0 -3022
  51. package/dist/index.es.js.map +0 -1
package/dist/index.es.js DELETED
@@ -1,3022 +0,0 @@
1
- import { simd } from 'wasm-feature-detect';
2
- import { WorkerTimer } from '@stream-io/worker-timer';
3
- import { FilesetResolver, ImageSegmenter } from '@mediapipe/tasks-vision';
4
-
5
- /**
6
- * Checks if the current platform is a mobile device.
7
- *
8
- * See:
9
- * https://developer.mozilla.org/en-US/docs/Web/HTTP/Browser_detection_using_the_user_agent
10
- */
11
- const isMobile = () => /Mobi/i.test(navigator.userAgent);
12
- /**
13
- * Checks whether the current browser is Safari.
14
- */
15
- const isSafari = () => /^((?!chrome|android).)*safari/i.test(navigator.userAgent);
16
- /**
17
- * Runs a check to see if the current platform supports
18
- * the necessary APIs required for the video filters.
19
- */
20
- const isPlatformSupported = async ({ forceMobileSupport = false, forceSafariSupport = false, } = {}) => typeof document !== 'undefined' &&
21
- typeof window !== 'undefined' &&
22
- typeof navigator !== 'undefined' &&
23
- // we don't support mobile devices yet due to performance issues
24
- (forceMobileSupport || !isMobile()) &&
25
- // Safari has issues with timer throttling, causing low FPS when the tab goes to the background
26
- (forceSafariSupport || !isSafari()) &&
27
- typeof WebAssembly !== 'undefined' &&
28
- !!window.WebGL2RenderingContext && // WebGL2 is required for the video filters
29
- !!document.createElement('canvas').getContext('webgl2') &&
30
- (await simd()); // SIMD is required for the wasm module
31
- /**
32
- * Runs a check to see if the current platform supports
33
- * the necessary APIs required for the MediaPipe-based video filters.
34
- */
35
- const isMediaPipePlatformSupported = async ({ forceMobileSupport = false, forceSafariSupport = false, } = {}) => typeof document !== 'undefined' &&
36
- typeof window !== 'undefined' &&
37
- typeof navigator !== 'undefined' &&
38
- // we don't support mobile devices yet due to performance issues
39
- (forceMobileSupport || !isMobile()) &&
40
- // Safari has issues with timer throttling, causing low FPS when the tab goes to the background
41
- (forceSafariSupport || !isSafari()) &&
42
- typeof WebAssembly !== 'undefined' &&
43
- typeof OffscreenCanvas !== 'undefined' && // OffscreenCanvas is required for efficient rendering
44
- !!window.WebGL2RenderingContext && // WebGL2 is required for the video filters
45
- !!new OffscreenCanvas(1, 1).getContext('webgl2') &&
46
- typeof VideoFrame !== 'undefined' && // VideoFrame API is required for frame processing
47
- typeof createImageBitmap !== 'undefined'; // createImageBitmap is required for background image processing
48
-
49
- /**
50
- * Use it along with boyswan.glsl-literal VSCode extension
51
- * to get GLSL syntax highlighting.
52
- * https://marketplace.visualstudio.com/items?itemName=boyswan.glsl-literal
53
- *
54
- * On VSCode OSS, boyswan.glsl-literal requires slevesque.shader extension
55
- * to be installed as well.
56
- * https://marketplace.visualstudio.com/items?itemName=slevesque.shader
57
- */
58
- const glsl = String.raw;
59
- function createPipelineStageProgram(gl, vertexShader, fragmentShader, positionBuffer, texCoordBuffer) {
60
- const program = createProgram(gl, vertexShader, fragmentShader);
61
- const positionAttributeLocation = gl.getAttribLocation(program, 'a_position');
62
- gl.enableVertexAttribArray(positionAttributeLocation);
63
- gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
64
- gl.vertexAttribPointer(positionAttributeLocation, 2, gl.FLOAT, false, 0, 0);
65
- const texCoordAttributeLocation = gl.getAttribLocation(program, 'a_texCoord');
66
- gl.enableVertexAttribArray(texCoordAttributeLocation);
67
- gl.bindBuffer(gl.ARRAY_BUFFER, texCoordBuffer);
68
- gl.vertexAttribPointer(texCoordAttributeLocation, 2, gl.FLOAT, false, 0, 0);
69
- return program;
70
- }
71
- function createProgram(gl, vertexShader, fragmentShader) {
72
- const program = gl.createProgram();
73
- gl.attachShader(program, vertexShader);
74
- gl.attachShader(program, fragmentShader);
75
- gl.linkProgram(program);
76
- if (!gl.getProgramParameter(program, gl.LINK_STATUS)) {
77
- throw new Error(`Could not link WebGL program: ${gl.getProgramInfoLog(program)}`);
78
- }
79
- return program;
80
- }
81
- function compileShader(gl, shaderType, shaderSource) {
82
- const shader = gl.createShader(shaderType);
83
- gl.shaderSource(shader, shaderSource);
84
- gl.compileShader(shader);
85
- if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) {
86
- throw new Error(`Could not compile shader: ${gl.getShaderInfoLog(shader)}`);
87
- }
88
- return shader;
89
- }
90
- function createTexture(gl, internalformat, width, height, minFilter = gl.NEAREST, magFilter = gl.NEAREST) {
91
- const texture = gl.createTexture();
92
- gl.bindTexture(gl.TEXTURE_2D, texture);
93
- gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
94
- gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
95
- gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, minFilter);
96
- gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, magFilter);
97
- gl.texStorage2D(gl.TEXTURE_2D, 1, internalformat, width, height);
98
- return texture;
99
- }
100
- async function readPixelsAsync(gl, x, y, width, height, format, type, dest) {
101
- const buf = gl.createBuffer();
102
- gl.bindBuffer(gl.PIXEL_PACK_BUFFER, buf);
103
- gl.bufferData(gl.PIXEL_PACK_BUFFER, dest.byteLength, gl.STREAM_READ);
104
- gl.readPixels(x, y, width, height, format, type, 0);
105
- gl.bindBuffer(gl.PIXEL_PACK_BUFFER, null);
106
- await getBufferSubDataAsync(gl, gl.PIXEL_PACK_BUFFER, buf, 0, dest);
107
- gl.deleteBuffer(buf);
108
- return dest;
109
- }
110
- async function getBufferSubDataAsync(gl, target, buffer, srcByteOffset, dstBuffer, dstOffset, length) {
111
- const sync = gl.fenceSync(gl.SYNC_GPU_COMMANDS_COMPLETE, 0);
112
- gl.flush();
113
- if (!sync)
114
- return;
115
- const res = await clientWaitAsync(gl, sync);
116
- gl.deleteSync(sync);
117
- if (res !== gl.WAIT_FAILED) {
118
- gl.bindBuffer(target, buffer);
119
- gl.getBufferSubData(target, srcByteOffset, dstBuffer, dstOffset, length);
120
- gl.bindBuffer(target, null);
121
- }
122
- }
123
- function clientWaitAsync(gl, sync) {
124
- return new Promise((resolve) => {
125
- function test() {
126
- const res = gl.clientWaitSync(sync, 0, 0);
127
- if (res === gl.WAIT_FAILED) {
128
- resolve(res);
129
- return;
130
- }
131
- if (res === gl.TIMEOUT_EXPIRED) {
132
- setTimeout(test);
133
- return;
134
- }
135
- resolve(res);
136
- }
137
- setTimeout(test);
138
- });
139
- }
140
-
141
- function buildBackgroundBlurStage(gl, vertexShader, positionBuffer, texCoordBuffer, personMaskTexture, canvas, blurLevel) {
142
- const blurPass = buildBlurPass(gl, vertexShader, positionBuffer, texCoordBuffer, personMaskTexture, canvas, blurLevel);
143
- const blendPass = buildBlendPass(gl, positionBuffer, texCoordBuffer, canvas);
144
- function render() {
145
- blurPass.render();
146
- blendPass.render();
147
- }
148
- function updateCoverage(coverage) {
149
- blendPass.updateCoverage(coverage);
150
- }
151
- function cleanUp() {
152
- blendPass.cleanUp();
153
- blurPass.cleanUp();
154
- }
155
- return {
156
- render,
157
- updateCoverage,
158
- cleanUp,
159
- };
160
- }
161
- function buildBlurPass(gl, vertexShader, positionBuffer, texCoordBuffer, personMaskTexture, canvas, blurLevel) {
162
- const sigma = typeof blurLevel === 'number'
163
- ? blurLevel
164
- : blurLevel === 'low'
165
- ? 2
166
- : blurLevel === 'medium'
167
- ? 4
168
- : 6;
169
- const windowSize = Math.max(1, Math.floor(sigma * 3));
170
- const offset = new Array(windowSize).fill(0).map((v, index) => index);
171
- const variance = sigma ** 2;
172
- const weights = offset.map((x) => {
173
- const m = sigma * Math.sqrt(2 * Math.PI);
174
- const e = Math.exp(-(x ** 2) / (2 * variance));
175
- return e / m;
176
- });
177
- const fragmentShaderSource = glsl `#version 300 es
178
-
179
- precision highp float;
180
-
181
- uniform sampler2D u_inputFrame;
182
- uniform sampler2D u_personMask;
183
- uniform vec2 u_texelSize;
184
-
185
- in vec2 v_texCoord;
186
- out vec4 outColor;
187
-
188
- const float offset[${windowSize}] = float[](${offset.map((i) => i.toFixed(10)).join(', ')});
189
- const float weight[${windowSize}] = float[](${weights.map((i) => i.toFixed(10)).join(', ')});
190
-
191
- void main() {
192
- vec4 centerColor = texture(u_inputFrame, v_texCoord);
193
- float personMask = texture(u_personMask, v_texCoord).a;
194
-
195
- vec4 frameColor = centerColor * weight[0] * (1.0 - personMask);
196
-
197
- for (int i = 1; i < ${windowSize}; i++) {
198
- vec2 offset = vec2(offset[i]) * u_texelSize;
199
-
200
- vec2 texCoord = v_texCoord + offset;
201
- frameColor += texture(u_inputFrame, texCoord)
202
- * weight[i]
203
- * (1.0 - texture(u_personMask, texCoord).a);
204
-
205
- texCoord = v_texCoord - offset;
206
- frameColor += texture(u_inputFrame, texCoord)
207
- * weight[i]
208
- * (1.0 - texture(u_personMask, texCoord).a);
209
- }
210
- outColor = vec4(frameColor.rgb + (1.0 - frameColor.a) * centerColor.rgb, 1.0);
211
- }
212
- `;
213
- const scale = 0.5;
214
- const outputWidth = canvas.width * scale;
215
- const outputHeight = canvas.height * scale;
216
- const texelWidth = 1 / outputWidth;
217
- const texelHeight = 1 / outputHeight;
218
- const fragmentShader = compileShader(gl, gl.FRAGMENT_SHADER, fragmentShaderSource);
219
- const program = createPipelineStageProgram(gl, vertexShader, fragmentShader, positionBuffer, texCoordBuffer);
220
- const inputFrameLocation = gl.getUniformLocation(program, 'u_inputFrame');
221
- const personMaskLocation = gl.getUniformLocation(program, 'u_personMask');
222
- const texelSizeLocation = gl.getUniformLocation(program, 'u_texelSize');
223
- const texture1 = createTexture(gl, gl.RGBA8, outputWidth, outputHeight, gl.NEAREST,
224
- // @ts-expect-error types are incomplete
225
- gl.LINEAR);
226
- const texture2 = createTexture(gl, gl.RGBA8, outputWidth, outputHeight, gl.NEAREST,
227
- // @ts-expect-error types are incomplete
228
- gl.LINEAR);
229
- const frameBuffer1 = gl.createFramebuffer();
230
- gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer1);
231
- gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, texture1, 0);
232
- const frameBuffer2 = gl.createFramebuffer();
233
- gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer2);
234
- gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, texture2, 0);
235
- gl.useProgram(program);
236
- gl.uniform1i(personMaskLocation, 1);
237
- function render() {
238
- gl.viewport(0, 0, outputWidth, outputHeight);
239
- gl.useProgram(program);
240
- gl.uniform1i(inputFrameLocation, 0);
241
- gl.activeTexture(gl.TEXTURE1);
242
- gl.bindTexture(gl.TEXTURE_2D, personMaskTexture);
243
- for (let i = 0; i < 3; i++) {
244
- gl.uniform2f(texelSizeLocation, 0, texelHeight);
245
- gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer1);
246
- gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4);
247
- gl.activeTexture(gl.TEXTURE2);
248
- gl.bindTexture(gl.TEXTURE_2D, texture1);
249
- gl.uniform1i(inputFrameLocation, 2);
250
- gl.uniform2f(texelSizeLocation, texelWidth, 0);
251
- gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer2);
252
- gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4);
253
- gl.bindTexture(gl.TEXTURE_2D, texture2);
254
- }
255
- }
256
- function cleanUp() {
257
- gl.deleteFramebuffer(frameBuffer2);
258
- gl.deleteFramebuffer(frameBuffer1);
259
- gl.deleteTexture(texture2);
260
- gl.deleteTexture(texture1);
261
- gl.deleteProgram(program);
262
- gl.deleteShader(fragmentShader);
263
- }
264
- return {
265
- render,
266
- cleanUp,
267
- };
268
- }
269
- function buildBlendPass(gl, positionBuffer, texCoordBuffer, canvas) {
270
- const vertexShaderSource = glsl `#version 300 es
271
-
272
- in vec2 a_position;
273
- in vec2 a_texCoord;
274
-
275
- out vec2 v_texCoord;
276
-
277
- void main() {
278
- // Flipping Y is required when rendering to canvas
279
- gl_Position = vec4(a_position * vec2(1.0, -1.0), 0.0, 1.0);
280
- v_texCoord = a_texCoord;
281
- }
282
- `;
283
- const fragmentShaderSource = glsl `#version 300 es
284
-
285
- precision highp float;
286
-
287
- uniform sampler2D u_inputFrame;
288
- uniform sampler2D u_personMask;
289
- uniform sampler2D u_blurredInputFrame;
290
- uniform vec2 u_coverage;
291
-
292
- in vec2 v_texCoord;
293
-
294
- out vec4 outColor;
295
-
296
- void main() {
297
- vec3 color = texture(u_inputFrame, v_texCoord).rgb;
298
- vec3 blurredColor = texture(u_blurredInputFrame, v_texCoord).rgb;
299
- float personMask = texture(u_personMask, v_texCoord).a;
300
- personMask = smoothstep(u_coverage.x, u_coverage.y, personMask);
301
- outColor = vec4(mix(blurredColor, color, personMask), 1.0);
302
- }
303
- `;
304
- const { width: outputWidth, height: outputHeight } = canvas;
305
- const vertexShader = compileShader(gl, gl.VERTEX_SHADER, vertexShaderSource);
306
- const fragmentShader = compileShader(gl, gl.FRAGMENT_SHADER, fragmentShaderSource);
307
- const program = createPipelineStageProgram(gl, vertexShader, fragmentShader, positionBuffer, texCoordBuffer);
308
- const inputFrameLocation = gl.getUniformLocation(program, 'u_inputFrame');
309
- const personMaskLocation = gl.getUniformLocation(program, 'u_personMask');
310
- const blurredInputFrame = gl.getUniformLocation(program, 'u_blurredInputFrame');
311
- const coverageLocation = gl.getUniformLocation(program, 'u_coverage');
312
- gl.useProgram(program);
313
- gl.uniform1i(inputFrameLocation, 0);
314
- gl.uniform1i(personMaskLocation, 1);
315
- gl.uniform1i(blurredInputFrame, 2);
316
- gl.uniform2f(coverageLocation, 0, 1);
317
- function render() {
318
- gl.viewport(0, 0, outputWidth, outputHeight);
319
- gl.useProgram(program);
320
- gl.bindFramebuffer(gl.FRAMEBUFFER, null);
321
- gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4);
322
- }
323
- function updateCoverage(coverage) {
324
- gl.useProgram(program);
325
- gl.uniform2f(coverageLocation, coverage[0], coverage[1]);
326
- }
327
- function cleanUp() {
328
- gl.deleteProgram(program);
329
- gl.deleteShader(fragmentShader);
330
- gl.deleteShader(vertexShader);
331
- }
332
- return {
333
- render,
334
- updateCoverage,
335
- cleanUp,
336
- };
337
- }
338
-
339
- function buildBackgroundImageStage(gl, positionBuffer, texCoordBuffer, personMaskTexture, backgroundImage, canvas) {
340
- const vertexShaderSource = glsl `#version 300 es
341
-
342
- uniform vec2 u_backgroundScale;
343
- uniform vec2 u_backgroundOffset;
344
-
345
- in vec2 a_position;
346
- in vec2 a_texCoord;
347
-
348
- out vec2 v_texCoord;
349
- out vec2 v_backgroundCoord;
350
-
351
- void main() {
352
- // Flipping Y is required when rendering to canvas
353
- gl_Position = vec4(a_position * vec2(1.0, -1.0), 0.0, 1.0);
354
- v_texCoord = a_texCoord;
355
- v_backgroundCoord = a_texCoord * u_backgroundScale + u_backgroundOffset;
356
- }
357
- `;
358
- const fragmentShaderSource = glsl `#version 300 es
359
-
360
- precision highp float;
361
-
362
- uniform sampler2D u_inputFrame;
363
- uniform sampler2D u_personMask;
364
- uniform sampler2D u_background;
365
- uniform vec2 u_coverage;
366
- uniform float u_lightWrapping;
367
- uniform float u_blendMode;
368
-
369
- in vec2 v_texCoord;
370
- in vec2 v_backgroundCoord;
371
-
372
- out vec4 outColor;
373
-
374
- vec3 screen(vec3 a, vec3 b) {
375
- return 1.0 - (1.0 - a) * (1.0 - b);
376
- }
377
-
378
- vec3 linearDodge(vec3 a, vec3 b) {
379
- return a + b;
380
- }
381
-
382
- void main() {
383
- vec3 frameColor = texture(u_inputFrame, v_texCoord).rgb;
384
- vec3 backgroundColor = texture(u_background, v_backgroundCoord).rgb;
385
- float personMask = texture(u_personMask, v_texCoord).a;
386
- float lightWrapMask = 1.0 - max(0.0, personMask - u_coverage.y) / (1.0 - u_coverage.y);
387
- vec3 lightWrap = u_lightWrapping * lightWrapMask * backgroundColor;
388
-
389
- frameColor = u_blendMode * linearDodge(frameColor, lightWrap)
390
- + (1.0 - u_blendMode) * screen(frameColor, lightWrap);
391
- personMask = smoothstep(u_coverage.x, u_coverage.y, personMask);
392
- outColor = vec4(frameColor * personMask + backgroundColor * (1.0 - personMask), 1.0);
393
- }
394
- `;
395
- const { width: outputWidth, height: outputHeight } = canvas;
396
- const outputRatio = outputWidth / outputHeight;
397
- const vertexShader = compileShader(gl, gl.VERTEX_SHADER, vertexShaderSource);
398
- const fragmentShader = compileShader(gl, gl.FRAGMENT_SHADER, fragmentShaderSource);
399
- const program = createPipelineStageProgram(gl, vertexShader, fragmentShader, positionBuffer, texCoordBuffer);
400
- const backgroundScaleLocation = gl.getUniformLocation(program, 'u_backgroundScale');
401
- const backgroundOffsetLocation = gl.getUniformLocation(program, 'u_backgroundOffset');
402
- const inputFrameLocation = gl.getUniformLocation(program, 'u_inputFrame');
403
- const personMaskLocation = gl.getUniformLocation(program, 'u_personMask');
404
- const backgroundLocation = gl.getUniformLocation(program, 'u_background');
405
- const coverageLocation = gl.getUniformLocation(program, 'u_coverage');
406
- const lightWrappingLocation = gl.getUniformLocation(program, 'u_lightWrapping');
407
- const blendModeLocation = gl.getUniformLocation(program, 'u_blendMode');
408
- gl.useProgram(program);
409
- gl.uniform2f(backgroundScaleLocation, 1, 1);
410
- gl.uniform2f(backgroundOffsetLocation, 0, 0);
411
- gl.uniform1i(inputFrameLocation, 0);
412
- gl.uniform1i(personMaskLocation, 1);
413
- gl.uniform2f(coverageLocation, 0, 1);
414
- gl.uniform1f(lightWrappingLocation, 0);
415
- gl.uniform1f(blendModeLocation, 0);
416
- let backgroundTexture = null;
417
- // TODO Find a better to handle background being loaded
418
- if (backgroundImage?.complete) {
419
- updateBackgroundImage(backgroundImage);
420
- }
421
- else if (backgroundImage) {
422
- backgroundImage.onload = () => {
423
- updateBackgroundImage(backgroundImage);
424
- };
425
- }
426
- function render() {
427
- gl.viewport(0, 0, outputWidth, outputHeight);
428
- gl.useProgram(program);
429
- gl.activeTexture(gl.TEXTURE1);
430
- gl.bindTexture(gl.TEXTURE_2D, personMaskTexture);
431
- if (backgroundTexture !== null) {
432
- gl.activeTexture(gl.TEXTURE2);
433
- gl.bindTexture(gl.TEXTURE_2D, backgroundTexture);
434
- // TODO Handle correctly the background not loaded yet
435
- gl.uniform1i(backgroundLocation, 2);
436
- }
437
- gl.bindFramebuffer(gl.FRAMEBUFFER, null);
438
- gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4);
439
- }
440
- function updateBackgroundImage(bgImage) {
441
- backgroundTexture = createTexture(gl, gl.RGBA8, bgImage.naturalWidth, bgImage.naturalHeight,
442
- // @ts-expect-error types are incomplete
443
- gl.LINEAR, gl.LINEAR);
444
- gl.texSubImage2D(gl.TEXTURE_2D, 0, 0, 0, bgImage.naturalWidth, bgImage.naturalHeight, gl.RGBA, gl.UNSIGNED_BYTE, bgImage);
445
- let xOffset = 0;
446
- let yOffset = 0;
447
- let backgroundWidth = bgImage.naturalWidth;
448
- let backgroundHeight = bgImage.naturalHeight;
449
- const backgroundRatio = backgroundWidth / backgroundHeight;
450
- if (backgroundRatio < outputRatio) {
451
- backgroundHeight = backgroundWidth / outputRatio;
452
- yOffset = (bgImage.naturalHeight - backgroundHeight) / 2;
453
- }
454
- else {
455
- backgroundWidth = backgroundHeight * outputRatio;
456
- xOffset = (bgImage.naturalWidth - backgroundWidth) / 2;
457
- }
458
- const xScale = backgroundWidth / bgImage.naturalWidth;
459
- const yScale = backgroundHeight / bgImage.naturalHeight;
460
- xOffset /= bgImage.naturalWidth;
461
- yOffset /= bgImage.naturalHeight;
462
- gl.uniform2f(backgroundScaleLocation, xScale, yScale);
463
- gl.uniform2f(backgroundOffsetLocation, xOffset, yOffset);
464
- }
465
- function updateCoverage(coverage) {
466
- gl.useProgram(program);
467
- gl.uniform2f(coverageLocation, coverage[0], coverage[1]);
468
- }
469
- function updateLightWrapping(lightWrapping) {
470
- gl.useProgram(program);
471
- gl.uniform1f(lightWrappingLocation, lightWrapping);
472
- }
473
- function updateBlendMode(blendMode) {
474
- gl.useProgram(program);
475
- gl.uniform1f(blendModeLocation, blendMode === 'screen' ? 0 : 1);
476
- }
477
- function cleanUp() {
478
- gl.deleteTexture(backgroundTexture);
479
- gl.deleteProgram(program);
480
- gl.deleteShader(fragmentShader);
481
- gl.deleteShader(vertexShader);
482
- }
483
- return {
484
- render,
485
- updateCoverage,
486
- updateLightWrapping,
487
- updateBlendMode,
488
- cleanUp,
489
- };
490
- }
491
-
492
- function buildJointBilateralFilterStage(gl, vertexShader, positionBuffer, texCoordBuffer, inputTexture, outputTexture, canvas, segmentationConfig) {
493
- const fragmentShaderSource = glsl `#version 300 es
494
-
495
- precision highp float;
496
-
497
- uniform sampler2D u_inputFrame;
498
- uniform sampler2D u_segmentationMask;
499
- uniform vec2 u_texelSize;
500
- uniform float u_step;
501
- uniform float u_radius;
502
- uniform float u_offset;
503
- uniform float u_sigmaTexel;
504
- uniform float u_sigmaColor;
505
-
506
- in vec2 v_texCoord;
507
- out vec4 outColor;
508
-
509
- float gaussian(float x, float sigma) {
510
- float coeff = -0.5 / (sigma * sigma * 4.0 + 1.0e-6);
511
- return exp((x * x) * coeff);
512
- }
513
-
514
- void main() {
515
- vec2 centerCoord = v_texCoord;
516
- vec3 centerColor = texture(u_inputFrame, centerCoord).rgb;
517
- float newVal = 0.0;
518
-
519
- float spaceWeight = 0.0;
520
- float colorWeight = 0.0;
521
- float totalWeight = 0.0;
522
-
523
- // Subsample kernel space.
524
- for (float i = -u_radius + u_offset; i <= u_radius; i += u_step) {
525
- for (float j = -u_radius + u_offset; j <= u_radius; j += u_step) {
526
- vec2 shift = vec2(j, i) * u_texelSize;
527
- vec2 coord = vec2(centerCoord + shift);
528
- vec3 frameColor = texture(u_inputFrame, coord).rgb;
529
- float outVal = texture(u_segmentationMask, coord).a;
530
-
531
- spaceWeight = gaussian(distance(centerCoord, coord), u_sigmaTexel);
532
- colorWeight = gaussian(distance(centerColor, frameColor), u_sigmaColor);
533
- totalWeight += spaceWeight * colorWeight;
534
-
535
- newVal += spaceWeight * colorWeight * outVal;
536
- }
537
- }
538
- newVal /= totalWeight;
539
-
540
- outColor = vec4(vec3(0.0), newVal);
541
- }
542
- `;
543
- const { width: segmentationWidth, height: segmentationHeight } = segmentationConfig;
544
- const { width: outputWidth, height: outputHeight } = canvas;
545
- const texelWidth = 1 / outputWidth;
546
- const texelHeight = 1 / outputHeight;
547
- const fragmentShader = compileShader(gl, gl.FRAGMENT_SHADER, fragmentShaderSource);
548
- const program = createPipelineStageProgram(gl, vertexShader, fragmentShader, positionBuffer, texCoordBuffer);
549
- const inputFrameLocation = gl.getUniformLocation(program, 'u_inputFrame');
550
- const segmentationMaskLocation = gl.getUniformLocation(program, 'u_segmentationMask');
551
- const texelSizeLocation = gl.getUniformLocation(program, 'u_texelSize');
552
- const stepLocation = gl.getUniformLocation(program, 'u_step');
553
- const radiusLocation = gl.getUniformLocation(program, 'u_radius');
554
- const offsetLocation = gl.getUniformLocation(program, 'u_offset');
555
- const sigmaTexelLocation = gl.getUniformLocation(program, 'u_sigmaTexel');
556
- const sigmaColorLocation = gl.getUniformLocation(program, 'u_sigmaColor');
557
- const frameBuffer = gl.createFramebuffer();
558
- gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer);
559
- gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, outputTexture, 0);
560
- gl.useProgram(program);
561
- gl.uniform1i(inputFrameLocation, 0);
562
- gl.uniform1i(segmentationMaskLocation, 1);
563
- gl.uniform2f(texelSizeLocation, texelWidth, texelHeight);
564
- // Ensures default values are configured to prevent infinite
565
- // loop in fragment shader
566
- updateSigmaSpace(0);
567
- updateSigmaColor(0);
568
- function render() {
569
- gl.viewport(0, 0, outputWidth, outputHeight);
570
- gl.useProgram(program);
571
- gl.activeTexture(gl.TEXTURE1);
572
- gl.bindTexture(gl.TEXTURE_2D, inputTexture);
573
- gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer);
574
- gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4);
575
- }
576
- function updateSigmaSpace(sigmaSpace) {
577
- sigmaSpace *= Math.max(outputWidth / segmentationWidth, outputHeight / segmentationHeight);
578
- const kSparsityFactor = 0.66; // Higher is sparser.
579
- const step = Math.max(1, Math.sqrt(sigmaSpace) * kSparsityFactor);
580
- const radius = sigmaSpace;
581
- const offset = step > 1 ? step * 0.5 : 0;
582
- const sigmaTexel = Math.max(texelWidth, texelHeight) * sigmaSpace;
583
- gl.useProgram(program);
584
- gl.uniform1f(stepLocation, step);
585
- gl.uniform1f(radiusLocation, radius);
586
- gl.uniform1f(offsetLocation, offset);
587
- gl.uniform1f(sigmaTexelLocation, sigmaTexel);
588
- }
589
- function updateSigmaColor(sigmaColor) {
590
- gl.useProgram(program);
591
- gl.uniform1f(sigmaColorLocation, sigmaColor);
592
- }
593
- function cleanUp() {
594
- gl.deleteFramebuffer(frameBuffer);
595
- gl.deleteProgram(program);
596
- gl.deleteShader(fragmentShader);
597
- }
598
- return { render, updateSigmaSpace, updateSigmaColor, cleanUp };
599
- }
600
-
601
- function buildResizingStage(gl, vertexShader, positionBuffer, texCoordBuffer, tflite, segmentationConfig, onError) {
602
- const fragmentShaderSource = glsl `#version 300 es
603
-
604
- precision highp float;
605
- uniform sampler2D u_inputFrame;
606
- in vec2 v_texCoord;
607
- out vec4 outColor;
608
-
609
- void main() {
610
- outColor = texture(u_inputFrame, v_texCoord);
611
- }
612
- `;
613
- // TFLite memory will be accessed as float32
614
- const tfliteInputMemoryOffset = tflite._getInputMemoryOffset() / 4;
615
- const { width: outputWidth, height: outputHeight } = segmentationConfig;
616
- const outputPixelCount = outputWidth * outputHeight;
617
- const fragmentShader = compileShader(gl, gl.FRAGMENT_SHADER, fragmentShaderSource);
618
- const program = createPipelineStageProgram(gl, vertexShader, fragmentShader, positionBuffer, texCoordBuffer);
619
- const inputFrameLocation = gl.getUniformLocation(program, 'u_inputFrame');
620
- const outputTexture = createTexture(gl, gl.RGBA8, outputWidth, outputHeight);
621
- const frameBuffer = gl.createFramebuffer();
622
- gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer);
623
- gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, outputTexture, 0);
624
- const outputPixels = new Uint8Array(outputPixelCount * 4);
625
- gl.useProgram(program);
626
- gl.uniform1i(inputFrameLocation, 0);
627
- function render() {
628
- gl.viewport(0, 0, outputWidth, outputHeight);
629
- gl.useProgram(program);
630
- gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer);
631
- gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4);
632
- // Downloads pixels asynchronously from GPU while rendering the current frame.
633
- // The pixels will be available in the next frame render which results
634
- // in offsets in the segmentation output but increases the frame rate.
635
- readPixelsAsync(gl, 0, 0, outputWidth, outputHeight, gl.RGBA, gl.UNSIGNED_BYTE, outputPixels).catch((error) => {
636
- });
637
- for (let i = 0; i < outputPixelCount; i++) {
638
- const tfliteIndex = tfliteInputMemoryOffset + i * 3;
639
- const outputIndex = i * 4;
640
- tflite.HEAPF32[tfliteIndex] = outputPixels[outputIndex] / 255;
641
- tflite.HEAPF32[tfliteIndex + 1] = outputPixels[outputIndex + 1] / 255;
642
- tflite.HEAPF32[tfliteIndex + 2] = outputPixels[outputIndex + 2] / 255;
643
- }
644
- }
645
- function cleanUp() {
646
- gl.deleteFramebuffer(frameBuffer);
647
- gl.deleteTexture(outputTexture);
648
- gl.deleteProgram(program);
649
- gl.deleteShader(fragmentShader);
650
- }
651
- return { render, cleanUp };
652
- }
653
-
654
- function buildSoftmaxStage(gl, vertexShader, positionBuffer, texCoordBuffer, tflite, outputTexture, segmentationConfig) {
655
- const fragmentShaderSource = glsl `#version 300 es
656
-
657
- precision highp float;
658
-
659
- uniform sampler2D u_inputSegmentation;
660
- in vec2 v_texCoord;
661
- out vec4 outColor;
662
-
663
- void main() {
664
- vec2 segmentation = texture(u_inputSegmentation, v_texCoord).rg;
665
- float shift = max(segmentation.r, segmentation.g);
666
- float backgroundExp = exp(segmentation.r - shift);
667
- float personExp = exp(segmentation.g - shift);
668
- outColor = vec4(vec3(0.0), personExp / (backgroundExp + personExp));
669
- }
670
- `;
671
- // TFLite memory will be accessed as float32
672
- const tfliteOutputMemoryOffset = tflite._getOutputMemoryOffset() / 4;
673
- const { width: segmentationWidth, height: segmentationHeight } = segmentationConfig;
674
- const fragmentShader = compileShader(gl, gl.FRAGMENT_SHADER, fragmentShaderSource);
675
- const program = createPipelineStageProgram(gl, vertexShader, fragmentShader, positionBuffer, texCoordBuffer);
676
- const inputLocation = gl.getUniformLocation(program, 'u_inputSegmentation');
677
- const inputTexture = createTexture(gl, gl.RG32F, segmentationWidth, segmentationHeight);
678
- const frameBuffer = gl.createFramebuffer();
679
- gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer);
680
- gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, outputTexture, 0);
681
- gl.useProgram(program);
682
- gl.uniform1i(inputLocation, 1);
683
- function render() {
684
- gl.viewport(0, 0, segmentationWidth, segmentationHeight);
685
- gl.useProgram(program);
686
- gl.activeTexture(gl.TEXTURE1);
687
- gl.bindTexture(gl.TEXTURE_2D, inputTexture);
688
- gl.texSubImage2D(gl.TEXTURE_2D, 0, 0, 0, segmentationWidth, segmentationHeight, gl.RG, gl.FLOAT, tflite.HEAPF32, tfliteOutputMemoryOffset);
689
- gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer);
690
- gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4);
691
- }
692
- function cleanUp() {
693
- gl.deleteFramebuffer(frameBuffer);
694
- gl.deleteTexture(inputTexture);
695
- gl.deleteProgram(program);
696
- gl.deleteShader(fragmentShader);
697
- }
698
- return { render, cleanUp };
699
- }
700
-
701
- function buildWebGL2Pipeline(videoSource, backgroundImage, blurLevel, backgroundFilter, canvas, tflite, segmentationConfig, onError) {
702
- const gl = canvas.getContext('webgl2');
703
- if (!gl)
704
- throw new Error('WebGL2 is not supported');
705
- if (gl.isContextLost())
706
- throw new Error('WebGL2 context was lost');
707
- const { width: frameWidth, height: frameHeight } = videoSource;
708
- const { width: segmentationWidth, height: segmentationHeight } = segmentationConfig;
709
- const vertexShaderSource = glsl `#version 300 es
710
-
711
- in vec2 a_position;
712
- in vec2 a_texCoord;
713
- out vec2 v_texCoord;
714
-
715
- void main() {
716
- gl_Position = vec4(a_position, 0.0, 1.0);
717
- v_texCoord = a_texCoord;
718
- }
719
- `;
720
- const vertexShader = compileShader(gl, gl.VERTEX_SHADER, vertexShaderSource);
721
- const vertexArray = gl.createVertexArray();
722
- gl.bindVertexArray(vertexArray);
723
- const positionBuffer = gl.createBuffer();
724
- gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
725
- gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([-1, -1, 1.0, -1, -1, 1.0, 1.0, 1.0]), gl.STATIC_DRAW);
726
- const texCoordBuffer = gl.createBuffer();
727
- gl.bindBuffer(gl.ARRAY_BUFFER, texCoordBuffer);
728
- gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([0.0, 0.0, 1.0, 0.0, 0.0, 1.0, 1.0, 1.0]), gl.STATIC_DRAW);
729
- // We don't use texStorage2D here because texImage2D seems faster
730
- // to upload video texture than texSubImage2D even though the latter
731
- // is supposed to be the recommended way:
732
- // https://developer.mozilla.org/en-US/docs/Web/API/WebGL_API/WebGL_best_practices#use_texstorage_to_create_textures
733
- const inputFrameTexture = gl.createTexture();
734
- gl.bindTexture(gl.TEXTURE_2D, inputFrameTexture);
735
- gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
736
- gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
737
- gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
738
- gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
739
- // TODO Rename segmentation and person mask to be more specific
740
- const segmentationTexture = createTexture(gl, gl.RGBA8, segmentationWidth, segmentationHeight);
741
- const personMaskTexture = createTexture(gl, gl.RGBA8, frameWidth, frameHeight);
742
- const resizingStage = buildResizingStage(gl, vertexShader, positionBuffer, texCoordBuffer, tflite, segmentationConfig);
743
- const loadSegmentationStage = buildSoftmaxStage(gl, vertexShader, positionBuffer, texCoordBuffer, tflite, segmentationTexture, segmentationConfig);
744
- const jointBilateralFilterStage = buildJointBilateralFilterStage(gl, vertexShader, positionBuffer, texCoordBuffer, segmentationTexture, personMaskTexture, canvas, segmentationConfig);
745
- const backgroundStage = backgroundFilter === 'blur'
746
- ? buildBackgroundBlurStage(gl, vertexShader, positionBuffer, texCoordBuffer, personMaskTexture, canvas, blurLevel || 'high')
747
- : buildBackgroundImageStage(gl, positionBuffer, texCoordBuffer, personMaskTexture, backgroundImage, canvas);
748
- function render() {
749
- gl.activeTexture(gl.TEXTURE0);
750
- gl.bindTexture(gl.TEXTURE_2D, inputFrameTexture);
751
- // texImage2D seems faster than texSubImage2D to upload
752
- // video texture
753
- gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, videoSource);
754
- gl.bindVertexArray(vertexArray);
755
- resizingStage.render();
756
- tflite._runInference();
757
- loadSegmentationStage.render();
758
- jointBilateralFilterStage.render();
759
- backgroundStage.render();
760
- }
761
- function updatePostProcessingConfig() {
762
- jointBilateralFilterStage.updateSigmaSpace(1);
763
- jointBilateralFilterStage.updateSigmaColor(0.1);
764
- if (backgroundFilter === 'image') {
765
- const backgroundImageStage = backgroundStage;
766
- backgroundImageStage.updateCoverage([0.5, 0.75]);
767
- backgroundImageStage.updateLightWrapping(0.3);
768
- backgroundImageStage.updateBlendMode('screen');
769
- }
770
- else if (backgroundFilter === 'blur') {
771
- const backgroundBlurStage = backgroundStage;
772
- backgroundBlurStage.updateCoverage([0.5, 0.75]);
773
- }
774
- else {
775
- // TODO Handle no background in a separate pipeline path
776
- const backgroundImageStage = backgroundStage;
777
- backgroundImageStage.updateCoverage([0, 0.9999]);
778
- backgroundImageStage.updateLightWrapping(0);
779
- }
780
- }
781
- function cleanUp() {
782
- backgroundStage.cleanUp();
783
- jointBilateralFilterStage.cleanUp();
784
- loadSegmentationStage.cleanUp();
785
- resizingStage.cleanUp();
786
- gl.deleteTexture(personMaskTexture);
787
- gl.deleteTexture(segmentationTexture);
788
- gl.deleteTexture(inputFrameTexture);
789
- gl.deleteBuffer(texCoordBuffer);
790
- gl.deleteBuffer(positionBuffer);
791
- gl.deleteVertexArray(vertexArray);
792
- gl.deleteShader(vertexShader);
793
- }
794
- return { render, updatePostProcessingConfig, cleanUp };
795
- }
796
-
797
- var SegmentationLevel;
798
- (function (SegmentationLevel) {
799
- SegmentationLevel["LOW"] = "low";
800
- SegmentationLevel["HIGH"] = "high";
801
- })(SegmentationLevel || (SegmentationLevel = {}));
802
- const getSegmentationParams = (level) => {
803
- if (level === SegmentationLevel.HIGH) {
804
- return { width: 256, height: 144 };
805
- }
806
- return { width: 160, height: 96 };
807
- };
808
-
809
- function createRenderer(tflite, videoSource, targetCanvas, options, onError) {
810
- const { backgroundFilter, backgroundImage, backgroundBlurLevel, segmentationLevel = SegmentationLevel.HIGH, fps = 30, } = options;
811
- if (backgroundFilter === 'image' && !backgroundImage) {
812
- throw new Error(`backgroundImage element is required when backgroundFilter is image`);
813
- }
814
- const pipeline = buildWebGL2Pipeline(videoSource, backgroundImage, backgroundBlurLevel, backgroundFilter, targetCanvas, tflite, getSegmentationParams(segmentationLevel));
815
- const timers = new WorkerTimer({ useWorker: true });
816
- const id = timers.setInterval(() => {
817
- try {
818
- pipeline.render();
819
- if (backgroundFilter === 'image') {
820
- pipeline.updatePostProcessingConfig();
821
- }
822
- }
823
- catch (error) {
824
- onError?.(error);
825
- }
826
- }, Math.floor(1000 / (fps <= 0 ? 30 : fps)));
827
- return {
828
- dispose: () => {
829
- pipeline.cleanUp();
830
- timers.clearInterval(id);
831
- timers.destroy();
832
- },
833
- };
834
- }
835
-
836
- const createTFLiteSIMDModule = (__Module) => {
837
- __Module = __Module || {};
838
-
839
- var _scriptDir =
840
- typeof document !== 'undefined' && document.currentScript
841
- ? document.currentScript.src
842
- : undefined;
843
-
844
- var Module = typeof __Module != 'undefined' ? __Module : {};
845
- var readyPromiseResolve, readyPromiseReject;
846
- Module['ready'] = new Promise(function (resolve, reject) {
847
- readyPromiseResolve = resolve;
848
- readyPromiseReject = reject;
849
- });
850
- var moduleOverrides = Object.assign({}, Module);
851
- var thisProgram = './this.program';
852
- var quit_ = (status, toThrow) => {
853
- throw toThrow;
854
- };
855
- var ENVIRONMENT_IS_WEB = true;
856
- var scriptDirectory = '';
857
-
858
- function locateFile(path) {
859
- if (Module['locateFile']) {
860
- return Module['locateFile'](path, scriptDirectory);
861
- }
862
- return scriptDirectory + path;
863
- }
864
-
865
- var readBinary;
866
- {
867
- if (typeof document != 'undefined' && document.currentScript) {
868
- scriptDirectory = document.currentScript.src;
869
- }
870
- if (_scriptDir) {
871
- scriptDirectory = _scriptDir;
872
- }
873
- if (scriptDirectory.indexOf('blob:') !== 0) {
874
- scriptDirectory = scriptDirectory.substr(
875
- 0,
876
- scriptDirectory.replace(/[?#].*/, '').lastIndexOf('/') + 1,
877
- );
878
- } else {
879
- scriptDirectory = '';
880
- }
881
- }
882
- var out = Module['print'] || console.log.bind(console);
883
- var err = Module['printErr'] || console.warn.bind(console);
884
- Object.assign(Module, moduleOverrides);
885
- moduleOverrides = null;
886
- if (Module['arguments']) Module['arguments'];
887
- if (Module['thisProgram']) thisProgram = Module['thisProgram'];
888
- if (Module['quit']) quit_ = Module['quit'];
889
- var wasmBinary;
890
- if (Module['wasmBinary']) wasmBinary = Module['wasmBinary'];
891
- Module['noExitRuntime'] || true;
892
- if (typeof WebAssembly != 'object') {
893
- abort('no native wasm support detected');
894
- }
895
- var wasmMemory;
896
- var ABORT = false;
897
-
898
- var UTF8Decoder =
899
- typeof TextDecoder != 'undefined' ? new TextDecoder('utf8') : undefined;
900
-
901
- function UTF8ArrayToString(heapOrArray, idx, maxBytesToRead) {
902
- var endIdx = idx + maxBytesToRead;
903
- var endPtr = idx;
904
- while (heapOrArray[endPtr] && !(endPtr >= endIdx)) ++endPtr;
905
- if (endPtr - idx > 16 && heapOrArray.buffer && UTF8Decoder) {
906
- return UTF8Decoder.decode(heapOrArray.subarray(idx, endPtr));
907
- }
908
- var str = '';
909
- while (idx < endPtr) {
910
- var u0 = heapOrArray[idx++];
911
- if (!(u0 & 128)) {
912
- str += String.fromCharCode(u0);
913
- continue;
914
- }
915
- var u1 = heapOrArray[idx++] & 63;
916
- if ((u0 & 224) == 192) {
917
- str += String.fromCharCode(((u0 & 31) << 6) | u1);
918
- continue;
919
- }
920
- var u2 = heapOrArray[idx++] & 63;
921
- if ((u0 & 240) == 224) {
922
- u0 = ((u0 & 15) << 12) | (u1 << 6) | u2;
923
- } else {
924
- u0 =
925
- ((u0 & 7) << 18) | (u1 << 12) | (u2 << 6) | (heapOrArray[idx++] & 63);
926
- }
927
- if (u0 < 65536) {
928
- str += String.fromCharCode(u0);
929
- } else {
930
- var ch = u0 - 65536;
931
- str += String.fromCharCode(55296 | (ch >> 10), 56320 | (ch & 1023));
932
- }
933
- }
934
- return str;
935
- }
936
-
937
- var buffer, HEAP8, HEAPU8, HEAPU32;
938
-
939
- function updateGlobalBufferAndViews(buf) {
940
- buffer = buf;
941
- Module['HEAP8'] = HEAP8 = new Int8Array(buf);
942
- Module['HEAP16'] = new Int16Array(buf);
943
- Module['HEAP32'] = new Int32Array(buf);
944
- Module['HEAPU8'] = HEAPU8 = new Uint8Array(buf);
945
- Module['HEAPU16'] = new Uint16Array(buf);
946
- Module['HEAPU32'] = HEAPU32 = new Uint32Array(buf);
947
- Module['HEAPF32'] = new Float32Array(buf);
948
- Module['HEAPF64'] = new Float64Array(buf);
949
- }
950
-
951
- Module['INITIAL_MEMORY'] || 16777216;
952
- var __ATPRERUN__ = [];
953
- var __ATINIT__ = [];
954
- var __ATPOSTRUN__ = [];
955
-
956
- function preRun() {
957
- if (Module['preRun']) {
958
- if (typeof Module['preRun'] == 'function')
959
- Module['preRun'] = [Module['preRun']];
960
- while (Module['preRun'].length) {
961
- addOnPreRun(Module['preRun'].shift());
962
- }
963
- }
964
- callRuntimeCallbacks(__ATPRERUN__);
965
- }
966
-
967
- function initRuntime() {
968
- callRuntimeCallbacks(__ATINIT__);
969
- }
970
-
971
- function postRun() {
972
- if (Module['postRun']) {
973
- if (typeof Module['postRun'] == 'function')
974
- Module['postRun'] = [Module['postRun']];
975
- while (Module['postRun'].length) {
976
- addOnPostRun(Module['postRun'].shift());
977
- }
978
- }
979
- callRuntimeCallbacks(__ATPOSTRUN__);
980
- }
981
-
982
- function addOnPreRun(cb) {
983
- __ATPRERUN__.unshift(cb);
984
- }
985
-
986
- function addOnInit(cb) {
987
- __ATINIT__.unshift(cb);
988
- }
989
-
990
- function addOnPostRun(cb) {
991
- __ATPOSTRUN__.unshift(cb);
992
- }
993
-
994
- var runDependencies = 0;
995
- var dependenciesFulfilled = null;
996
-
997
- function addRunDependency(id) {
998
- runDependencies++;
999
- if (Module['monitorRunDependencies']) {
1000
- Module['monitorRunDependencies'](runDependencies);
1001
- }
1002
- }
1003
-
1004
- function removeRunDependency(id) {
1005
- runDependencies--;
1006
- if (Module['monitorRunDependencies']) {
1007
- Module['monitorRunDependencies'](runDependencies);
1008
- }
1009
- if (runDependencies == 0) {
1010
- if (dependenciesFulfilled) {
1011
- var callback = dependenciesFulfilled;
1012
- dependenciesFulfilled = null;
1013
- callback();
1014
- }
1015
- }
1016
- }
1017
-
1018
- function abort(what) {
1019
- {
1020
- if (Module['onAbort']) {
1021
- Module['onAbort'](what);
1022
- }
1023
- }
1024
- what = 'Aborted(' + what + ')';
1025
- err(what);
1026
- ABORT = true;
1027
- what += '. Build with -sASSERTIONS for more info.';
1028
- var e = new WebAssembly.RuntimeError(what);
1029
- readyPromiseReject(e);
1030
- throw e;
1031
- }
1032
-
1033
- var dataURIPrefix = 'data:application/octet-stream;base64,';
1034
-
1035
- function isDataURI(filename) {
1036
- return filename.startsWith(dataURIPrefix);
1037
- }
1038
-
1039
- var wasmBinaryFile;
1040
- wasmBinaryFile = 'tflite-simd.wasm';
1041
- if (!isDataURI(wasmBinaryFile)) {
1042
- wasmBinaryFile = locateFile(wasmBinaryFile);
1043
- }
1044
-
1045
- function getBinary(file) {
1046
- try {
1047
- if (file == wasmBinaryFile && wasmBinary) {
1048
- return new Uint8Array(wasmBinary);
1049
- }
1050
- if (readBinary) ;
1051
- throw 'both async and sync fetching of the wasm failed';
1052
- // eslint-disable-next-line @typescript-eslint/no-shadow
1053
- } catch (err) {
1054
- abort(err);
1055
- }
1056
- }
1057
-
1058
- function getBinaryPromise() {
1059
- if (!wasmBinary && (ENVIRONMENT_IS_WEB)) {
1060
- if (typeof fetch == 'function') {
1061
- return fetch(wasmBinaryFile, { credentials: 'same-origin' })
1062
- .then(function (response) {
1063
- if (!response['ok']) {
1064
- throw (
1065
- "failed to load wasm binary file at '" + wasmBinaryFile + "'"
1066
- );
1067
- }
1068
- return response['arrayBuffer']();
1069
- })
1070
- .catch(function () {
1071
- return getBinary(wasmBinaryFile);
1072
- });
1073
- }
1074
- }
1075
- return Promise.resolve().then(function () {
1076
- return getBinary(wasmBinaryFile);
1077
- });
1078
- }
1079
-
1080
- function createWasm() {
1081
- var info = {
1082
- env: asmLibraryArg,
1083
- wasi_snapshot_preview1: asmLibraryArg,
1084
- };
1085
-
1086
- function receiveInstance(instance, module) {
1087
- var exports = instance.exports;
1088
- Module['asm'] = exports;
1089
- wasmMemory = Module['asm']['memory'];
1090
- updateGlobalBufferAndViews(wasmMemory.buffer);
1091
- Module['asm']['__indirect_function_table'];
1092
- addOnInit(Module['asm']['__wasm_call_ctors']);
1093
- removeRunDependency();
1094
- }
1095
-
1096
- addRunDependency();
1097
-
1098
- function receiveInstantiationResult(result) {
1099
- receiveInstance(result['instance']);
1100
- }
1101
-
1102
- function instantiateArrayBuffer(receiver) {
1103
- return getBinaryPromise()
1104
- .then(function (binary) {
1105
- return WebAssembly.instantiate(binary, info);
1106
- })
1107
- .then(function (instance) {
1108
- return instance;
1109
- })
1110
- .then(receiver, function (reason) {
1111
- err('failed to asynchronously prepare wasm: ' + reason);
1112
- abort(reason);
1113
- });
1114
- }
1115
-
1116
- function instantiateAsync() {
1117
- if (
1118
- !wasmBinary &&
1119
- typeof WebAssembly.instantiateStreaming == 'function' &&
1120
- !isDataURI(wasmBinaryFile) &&
1121
- typeof fetch == 'function'
1122
- ) {
1123
- return fetch(wasmBinaryFile, { credentials: 'same-origin' }).then(
1124
- function (response) {
1125
- var result = WebAssembly.instantiateStreaming(response, info);
1126
- return result.then(receiveInstantiationResult, function (reason) {
1127
- err('wasm streaming compile failed: ' + reason);
1128
- err('falling back to ArrayBuffer instantiation');
1129
- return instantiateArrayBuffer(receiveInstantiationResult);
1130
- });
1131
- },
1132
- );
1133
- } else {
1134
- return instantiateArrayBuffer(receiveInstantiationResult);
1135
- }
1136
- }
1137
-
1138
- if (Module['instantiateWasm']) {
1139
- try {
1140
- var exports = Module['instantiateWasm'](info, receiveInstance);
1141
- return exports;
1142
- } catch (e) {
1143
- err('Module.instantiateWasm callback failed with error: ' + e);
1144
- readyPromiseReject(e);
1145
- }
1146
- }
1147
- instantiateAsync().catch(readyPromiseReject);
1148
- return {};
1149
- }
1150
-
1151
- function ExitStatus(status) {
1152
- this.name = 'ExitStatus';
1153
- this.message = 'Program terminated with exit(' + status + ')';
1154
- this.status = status;
1155
- }
1156
-
1157
- function callRuntimeCallbacks(callbacks) {
1158
- while (callbacks.length > 0) {
1159
- callbacks.shift()(Module);
1160
- }
1161
- }
1162
-
1163
- function __dlinit(main_dso_handle) {}
1164
-
1165
- var dlopenMissingError =
1166
- 'To use dlopen, you need enable dynamic linking, see https://github.com/emscripten-core/emscripten/wiki/Linking';
1167
-
1168
- function __dlopen_js(filename, flag) {
1169
- abort(dlopenMissingError);
1170
- }
1171
-
1172
- function __dlsym_js(handle, symbol) {
1173
- abort(dlopenMissingError);
1174
- }
1175
-
1176
- var nowIsMonotonic = true;
1177
-
1178
- function __emscripten_get_now_is_monotonic() {
1179
- return nowIsMonotonic;
1180
- }
1181
-
1182
- function __mmap_js(len, prot, flags, fd, off, allocated) {
1183
- return -52;
1184
- }
1185
-
1186
- function __munmap_js(addr, len, prot, flags, fd, offset) {}
1187
-
1188
- function _abort() {
1189
- abort('');
1190
- }
1191
-
1192
- function _emscripten_date_now() {
1193
- return Date.now();
1194
- }
1195
-
1196
- function getHeapMax() {
1197
- return 2147483648;
1198
- }
1199
-
1200
- function _emscripten_get_heap_max() {
1201
- return getHeapMax();
1202
- }
1203
-
1204
- var _emscripten_get_now;
1205
- _emscripten_get_now = () => performance.now();
1206
-
1207
- function _emscripten_memcpy_big(dest, src, num) {
1208
- HEAPU8.copyWithin(dest, src, src + num);
1209
- }
1210
-
1211
- function emscripten_realloc_buffer(size) {
1212
- try {
1213
- wasmMemory.grow((size - buffer.byteLength + 65535) >>> 16);
1214
- updateGlobalBufferAndViews(wasmMemory.buffer);
1215
- return 1;
1216
- } catch (e) {}
1217
- }
1218
-
1219
- function _emscripten_resize_heap(requestedSize) {
1220
- var oldSize = HEAPU8.length;
1221
- requestedSize = requestedSize >>> 0;
1222
- var maxHeapSize = getHeapMax();
1223
- if (requestedSize > maxHeapSize) {
1224
- return false;
1225
- }
1226
- let alignUp = (x, multiple) => x + ((multiple - (x % multiple)) % multiple);
1227
- for (var cutDown = 1; cutDown <= 4; cutDown *= 2) {
1228
- var overGrownHeapSize = oldSize * (1 + 0.2 / cutDown);
1229
- overGrownHeapSize = Math.min(
1230
- overGrownHeapSize,
1231
- requestedSize + 100663296,
1232
- );
1233
- var newSize = Math.min(
1234
- maxHeapSize,
1235
- alignUp(Math.max(requestedSize, overGrownHeapSize), 65536),
1236
- );
1237
- var replacement = emscripten_realloc_buffer(newSize);
1238
- if (replacement) {
1239
- return true;
1240
- }
1241
- }
1242
- return false;
1243
- }
1244
-
1245
- var ENV = {};
1246
-
1247
- function getExecutableName() {
1248
- return thisProgram || './this.program';
1249
- }
1250
-
1251
- function getEnvStrings() {
1252
- if (!getEnvStrings.strings) {
1253
- var lang =
1254
- (
1255
- (typeof navigator == 'object' &&
1256
- navigator.languages &&
1257
- navigator.languages[0]) ||
1258
- 'C'
1259
- ).replace('-', '_') + '.UTF-8';
1260
- var env = {
1261
- USER: 'web_user',
1262
- LOGNAME: 'web_user',
1263
- PATH: '/',
1264
- PWD: '/',
1265
- HOME: '/home/web_user',
1266
- LANG: lang,
1267
- _: getExecutableName(),
1268
- };
1269
- for (var x in ENV) {
1270
- if (ENV[x] === undefined) delete env[x];
1271
- else env[x] = ENV[x];
1272
- }
1273
- var strings = [];
1274
- for (var x in env) {
1275
- strings.push(x + '=' + env[x]);
1276
- }
1277
- getEnvStrings.strings = strings;
1278
- }
1279
- return getEnvStrings.strings;
1280
- }
1281
-
1282
- // eslint-disable-next-line @typescript-eslint/no-shadow
1283
- function writeAsciiToMemory(str, buffer, dontAddNull) {
1284
- for (var i = 0; i < str.length; ++i) {
1285
- HEAP8[buffer++ >> 0] = str.charCodeAt(i);
1286
- }
1287
- HEAP8[buffer >> 0] = 0;
1288
- }
1289
-
1290
- function _environ_get(__environ, environ_buf) {
1291
- var bufSize = 0;
1292
- getEnvStrings().forEach(function (string, i) {
1293
- var ptr = environ_buf + bufSize;
1294
- HEAPU32[(__environ + i * 4) >> 2] = ptr;
1295
- writeAsciiToMemory(string, ptr);
1296
- bufSize += string.length + 1;
1297
- });
1298
- return 0;
1299
- }
1300
-
1301
- function _environ_sizes_get(penviron_count, penviron_buf_size) {
1302
- var strings = getEnvStrings();
1303
- HEAPU32[penviron_count >> 2] = strings.length;
1304
- var bufSize = 0;
1305
- strings.forEach(function (string) {
1306
- bufSize += string.length + 1;
1307
- });
1308
- HEAPU32[penviron_buf_size >> 2] = bufSize;
1309
- return 0;
1310
- }
1311
-
1312
- function _proc_exit(code) {
1313
- quit_(code, new ExitStatus(code));
1314
- }
1315
-
1316
- function exitJS(status, implicit) {
1317
- _proc_exit(status);
1318
- }
1319
-
1320
- var _exit = exitJS;
1321
-
1322
- function _fd_close(fd) {
1323
- return 52;
1324
- }
1325
-
1326
- function _fd_seek(fd, offset_low, offset_high, whence, newOffset) {
1327
- return 70;
1328
- }
1329
-
1330
- var printCharBuffers = [null, [], []];
1331
-
1332
- function printChar(stream, curr) {
1333
- // eslint-disable-next-line @typescript-eslint/no-shadow
1334
- var buffer = printCharBuffers[stream];
1335
- if (curr === 0 || curr === 10) {
1336
- (stream === 1 ? out : err)(UTF8ArrayToString(buffer, 0));
1337
- buffer.length = 0;
1338
- } else {
1339
- buffer.push(curr);
1340
- }
1341
- }
1342
-
1343
- function _fd_write(fd, iov, iovcnt, pnum) {
1344
- var num = 0;
1345
- for (var i = 0; i < iovcnt; i++) {
1346
- var ptr = HEAPU32[iov >> 2];
1347
- var len = HEAPU32[(iov + 4) >> 2];
1348
- iov += 8;
1349
- for (var j = 0; j < len; j++) {
1350
- printChar(fd, HEAPU8[ptr + j]);
1351
- }
1352
- num += len;
1353
- }
1354
- HEAPU32[pnum >> 2] = num;
1355
- return 0;
1356
- }
1357
-
1358
- function getRandomDevice() {
1359
- if (
1360
- typeof crypto == 'object' &&
1361
- typeof crypto['getRandomValues'] == 'function'
1362
- ) {
1363
- var randomBuffer = new Uint8Array(1);
1364
- return () => {
1365
- crypto.getRandomValues(randomBuffer);
1366
- return randomBuffer[0];
1367
- };
1368
- } else return () => abort('randomDevice');
1369
- }
1370
-
1371
- // eslint-disable-next-line @typescript-eslint/no-shadow
1372
- function _getentropy(buffer, size) {
1373
- if (!_getentropy.randomDevice) {
1374
- _getentropy.randomDevice = getRandomDevice();
1375
- }
1376
- for (var i = 0; i < size; i++) {
1377
- HEAP8[(buffer + i) >> 0] = _getentropy.randomDevice();
1378
- }
1379
- return 0;
1380
- }
1381
-
1382
- var asmLibraryArg = {
1383
- _dlinit: __dlinit,
1384
- _dlopen_js: __dlopen_js,
1385
- _dlsym_js: __dlsym_js,
1386
- _emscripten_get_now_is_monotonic: __emscripten_get_now_is_monotonic,
1387
- _mmap_js: __mmap_js,
1388
- _munmap_js: __munmap_js,
1389
- abort: _abort,
1390
- emscripten_date_now: _emscripten_date_now,
1391
- emscripten_get_heap_max: _emscripten_get_heap_max,
1392
- emscripten_get_now: _emscripten_get_now,
1393
- emscripten_memcpy_big: _emscripten_memcpy_big,
1394
- emscripten_resize_heap: _emscripten_resize_heap,
1395
- environ_get: _environ_get,
1396
- environ_sizes_get: _environ_sizes_get,
1397
- exit: _exit,
1398
- fd_close: _fd_close,
1399
- fd_seek: _fd_seek,
1400
- fd_write: _fd_write,
1401
- getentropy: _getentropy,
1402
- };
1403
- createWasm();
1404
- (Module['___wasm_call_ctors'] = function () {
1405
- return (Module['___wasm_call_ctors'] =
1406
- Module['asm']['__wasm_call_ctors']).apply(null, arguments);
1407
- });
1408
- (Module['_getModelBufferMemoryOffset'] =
1409
- function () {
1410
- return (Module[
1411
- '_getModelBufferMemoryOffset'
1412
- ] =
1413
- Module['asm']['getModelBufferMemoryOffset']).apply(null, arguments);
1414
- });
1415
- (Module['_getInputMemoryOffset'] = function () {
1416
- return (Module['_getInputMemoryOffset'] =
1417
- Module['asm']['getInputMemoryOffset']).apply(null, arguments);
1418
- });
1419
- (Module['_getInputHeight'] = function () {
1420
- return (Module['_getInputHeight'] =
1421
- Module['asm']['getInputHeight']).apply(null, arguments);
1422
- });
1423
- (Module['_getInputWidth'] = function () {
1424
- return (Module['_getInputWidth'] =
1425
- Module['asm']['getInputWidth']).apply(null, arguments);
1426
- });
1427
- (Module['_getInputChannelCount'] = function () {
1428
- return (Module['_getInputChannelCount'] =
1429
- Module['asm']['getInputChannelCount']).apply(null, arguments);
1430
- });
1431
- (Module['_getOutputMemoryOffset'] = function () {
1432
- return (Module['_getOutputMemoryOffset'] =
1433
- Module['asm']['getOutputMemoryOffset']).apply(null, arguments);
1434
- });
1435
- (Module['_getOutputHeight'] = function () {
1436
- return (Module['_getOutputHeight'] =
1437
- Module['asm']['getOutputHeight']).apply(null, arguments);
1438
- });
1439
- (Module['_getOutputWidth'] = function () {
1440
- return (Module['_getOutputWidth'] =
1441
- Module['asm']['getOutputWidth']).apply(null, arguments);
1442
- });
1443
- (Module['_getOutputChannelCount'] = function () {
1444
- return (Module['_getOutputChannelCount'] =
1445
- Module['asm']['getOutputChannelCount']).apply(null, arguments);
1446
- });
1447
- (Module['_loadModel'] = function () {
1448
- return (Module['_loadModel'] =
1449
- Module['asm']['loadModel']).apply(null, arguments);
1450
- });
1451
- (Module['_runInference'] = function () {
1452
- return (Module['_runInference'] =
1453
- Module['asm']['runInference']).apply(null, arguments);
1454
- });
1455
- (Module['_malloc'] = function () {
1456
- return (Module['_malloc'] = Module['asm']['malloc']).apply(
1457
- null,
1458
- arguments,
1459
- );
1460
- });
1461
- (Module['___errno_location'] = function () {
1462
- return (Module['___errno_location'] =
1463
- Module['asm']['__errno_location']).apply(null, arguments);
1464
- });
1465
- (Module['___dl_seterr'] = function () {
1466
- return (Module['___dl_seterr'] =
1467
- Module['asm']['__dl_seterr']).apply(null, arguments);
1468
- });
1469
- (Module['stackSave'] = function () {
1470
- return (Module['stackSave'] = Module['asm']['stackSave']).apply(
1471
- null,
1472
- arguments,
1473
- );
1474
- });
1475
- (Module['stackRestore'] = function () {
1476
- return (Module['stackRestore'] =
1477
- Module['asm']['stackRestore']).apply(null, arguments);
1478
- });
1479
- (Module['stackAlloc'] = function () {
1480
- return (Module['stackAlloc'] =
1481
- Module['asm']['stackAlloc']).apply(null, arguments);
1482
- });
1483
- (Module['dynCall_jjj'] = function () {
1484
- return (Module['dynCall_jjj'] =
1485
- Module['asm']['dynCall_jjj']).apply(null, arguments);
1486
- });
1487
- (Module['dynCall_jiii'] = function () {
1488
- return (Module['dynCall_jiii'] =
1489
- Module['asm']['dynCall_jiii']).apply(null, arguments);
1490
- });
1491
- (Module['dynCall_iiiijj'] = function () {
1492
- return (Module['dynCall_iiiijj'] =
1493
- Module['asm']['dynCall_iiiijj']).apply(null, arguments);
1494
- });
1495
- (Module['dynCall_viijj'] = function () {
1496
- return (Module['dynCall_viijj'] =
1497
- Module['asm']['dynCall_viijj']).apply(null, arguments);
1498
- });
1499
- (Module['dynCall_viiijjj'] = function () {
1500
- return (Module['dynCall_viiijjj'] =
1501
- Module['asm']['dynCall_viiijjj']).apply(null, arguments);
1502
- });
1503
- (Module['dynCall_iijjiiii'] = function () {
1504
- return (Module['dynCall_iijjiiii'] =
1505
- Module['asm']['dynCall_iijjiiii']).apply(null, arguments);
1506
- });
1507
- (Module['dynCall_jiji'] = function () {
1508
- return (Module['dynCall_jiji'] =
1509
- Module['asm']['dynCall_jiji']).apply(null, arguments);
1510
- });
1511
- var calledRun;
1512
- dependenciesFulfilled = function runCaller() {
1513
- if (!calledRun) run();
1514
- if (!calledRun) dependenciesFulfilled = runCaller;
1515
- };
1516
-
1517
- function run(args) {
1518
- if (runDependencies > 0) {
1519
- return;
1520
- }
1521
- preRun();
1522
- if (runDependencies > 0) {
1523
- return;
1524
- }
1525
-
1526
- function doRun() {
1527
- if (calledRun) return;
1528
- calledRun = true;
1529
- Module['calledRun'] = true;
1530
- if (ABORT) return;
1531
- initRuntime();
1532
- readyPromiseResolve(Module);
1533
- if (Module['onRuntimeInitialized']) Module['onRuntimeInitialized']();
1534
- postRun();
1535
- }
1536
-
1537
- if (Module['setStatus']) {
1538
- Module['setStatus']('Running...');
1539
- setTimeout(function () {
1540
- setTimeout(function () {
1541
- Module['setStatus']('');
1542
- }, 1);
1543
- doRun();
1544
- }, 1);
1545
- } else {
1546
- doRun();
1547
- }
1548
- }
1549
-
1550
- if (Module['preInit']) {
1551
- if (typeof Module['preInit'] == 'function')
1552
- Module['preInit'] = [Module['preInit']];
1553
- while (Module['preInit'].length > 0) {
1554
- Module['preInit'].pop()();
1555
- }
1556
- }
1557
- run();
1558
-
1559
- return __Module.ready;
1560
- };
1561
-
1562
- const version = "0.8.6";
1563
- const packageName = "@stream-io/video-filters-web";
1564
-
1565
- // @ts-expect-error - module is not declared
1566
- // This is a WebAssembly module compiled from the TensorFlow Lite C++ library.
1567
- const createTFLite = createTFLiteSIMDModule;
1568
- const loadTFLite = async (options = {}) => {
1569
- const { basePath = `https://unpkg.com/${packageName}@${version}/tf`, tfFilePath = `${basePath}/tflite/tflite-simd.wasm`, modelFilePath = `${basePath}/models/segm_full_v679.tflite`, } = options;
1570
- const [tfLite, model] = await Promise.all([
1571
- createTFLite({ locateFile: () => tfFilePath }),
1572
- fetchModel(modelFilePath),
1573
- ]);
1574
- const modelBufferOffset = tfLite._getModelBufferMemoryOffset();
1575
- tfLite.HEAPU8.set(new Uint8Array(model), modelBufferOffset);
1576
- tfLite._loadModel(model.byteLength);
1577
- return tfLite;
1578
- };
1579
- let lastModelFilePath$1 = '';
1580
- let modelFileCache$1;
1581
- const fetchModel = async (modelFilePath) => {
1582
- const model = modelFilePath === lastModelFilePath$1 && modelFileCache$1
1583
- ? modelFileCache$1
1584
- : await fetch(modelFilePath).then((r) => r.arrayBuffer());
1585
- // Cache the model file for future use.
1586
- modelFileCache$1 = model;
1587
- lastModelFilePath$1 = modelFilePath;
1588
- return model;
1589
- };
1590
-
1591
- let lastModelFilePath = '';
1592
- let modelFileCache;
1593
- const loadMediaPipe = async (options = {}) => {
1594
- const { basePath = `https://unpkg.com/${packageName}@${version}/mediapipe`, modelPath = `${basePath}/models/selfie_segmenter_landscape.tflite`, } = options;
1595
- const model = modelPath === lastModelFilePath && modelFileCache
1596
- ? modelFileCache
1597
- : await fetch(modelPath).then((r) => r.arrayBuffer());
1598
- modelFileCache = model;
1599
- lastModelFilePath = modelPath;
1600
- return model;
1601
- };
1602
-
1603
- /**
1604
- * Maps blur level to blur strength values.
1605
- */
1606
- const BACKGROUND_BLUR_MAP = {
1607
- low: 3,
1608
- medium: 5,
1609
- high: 7,
1610
- };
1611
-
1612
- /**
1613
- * Fallback video processor for browsers that do not support MediaStreamTrackGenerator.
1614
- *
1615
- * Produces a video MediaStreamTrack sourced from a canvas and exposes
1616
- * a WritableStream<VideoFrame> on track.writable for writing frames.
1617
- */
1618
- class FallbackGenerator {
1619
- constructor({ kind, signalTarget }) {
1620
- if (kind !== 'video') {
1621
- throw new Error('Only video tracks are supported');
1622
- }
1623
- const canvas = document.createElement('canvas');
1624
- const ctx = canvas.getContext('2d', { desynchronized: true });
1625
- if (!ctx) {
1626
- throw new Error('Failed to get 2D context from canvas');
1627
- }
1628
- const mediaStream = canvas.captureStream();
1629
- const track = mediaStream.getVideoTracks()[0];
1630
- const height = signalTarget?.getSettings().height;
1631
- const width = signalTarget?.getSettings().width;
1632
- if (height && width) {
1633
- canvas.height = height;
1634
- canvas.width = width;
1635
- }
1636
- if (!track) {
1637
- throw new Error('Failed to create canvas track');
1638
- }
1639
- if (signalTarget) {
1640
- signalTarget.addEventListener('ended', () => {
1641
- track.stop();
1642
- });
1643
- }
1644
- track.writable = new WritableStream({
1645
- write: (frame) => {
1646
- if (canvas.width !== frame.displayWidth ||
1647
- canvas.height !== frame.displayHeight) {
1648
- canvas.width = frame.displayWidth;
1649
- canvas.height = frame.displayHeight;
1650
- }
1651
- ctx.drawImage(frame, 0, 0, canvas.width, canvas.height);
1652
- frame.close();
1653
- },
1654
- abort: () => {
1655
- track.stop();
1656
- },
1657
- close: () => {
1658
- track.stop();
1659
- },
1660
- });
1661
- return track;
1662
- }
1663
- }
1664
- const TrackGenerator = typeof MediaStreamTrackGenerator !== 'undefined'
1665
- ? MediaStreamTrackGenerator
1666
- : FallbackGenerator;
1667
-
1668
- /**
1669
- * Fallback implementation for browsers without MediaStreamTrackGenerator.
1670
- *
1671
- * Produces a video MediaStreamTrack sourced from a canvas and exposes a
1672
- * WritableStream<VideoFrame> on track.writable. Written frames are drawn
1673
- * into the canvas and update the underlying track automatically.
1674
- */
1675
- class FallbackProcessor {
1676
- readable;
1677
- workerTimer;
1678
- video;
1679
- constructor({ track }) {
1680
- if (!track)
1681
- throw new Error('MediaStreamTrack is required');
1682
- if (track.kind !== 'video') {
1683
- throw new Error('MediaStreamTrack must be video');
1684
- }
1685
- let running = true;
1686
- this.video = document.createElement('video');
1687
- this.video.muted = true;
1688
- this.video.playsInline = true;
1689
- this.video.srcObject = new MediaStream([track]);
1690
- const canvas = new OffscreenCanvas(1, 1);
1691
- const ctx = canvas.getContext('2d');
1692
- if (!ctx)
1693
- throw new Error('Failed to get 2D context from OffscreenCanvas');
1694
- let timestamp = 0;
1695
- const frameRate = track.getSettings().frameRate || 30;
1696
- let frameDuration = 1000 / frameRate;
1697
- this.workerTimer = new WorkerTimer({ useWorker: true });
1698
- this.readable = new ReadableStream({
1699
- start: async () => {
1700
- await Promise.all([
1701
- this.video.play(),
1702
- new Promise((r) => this.video.addEventListener('loadeddata', r, { once: true })),
1703
- ]);
1704
- frameDuration = 1000 / (track.getSettings().frameRate || 30);
1705
- timestamp = performance.now();
1706
- },
1707
- pull: async (controller) => {
1708
- if (!running) {
1709
- controller.close();
1710
- this.close();
1711
- return;
1712
- }
1713
- const delta = performance.now() - timestamp;
1714
- if (delta <= frameDuration) {
1715
- await new Promise((r) => this.workerTimer.setTimeout(r, frameDuration - delta));
1716
- }
1717
- timestamp = performance.now();
1718
- if (canvas.width !== this.video.videoWidth ||
1719
- canvas.height !== this.video.videoHeight) {
1720
- canvas.width = this.video.videoWidth;
1721
- canvas.height = this.video.videoHeight;
1722
- }
1723
- ctx.drawImage(this.video, 0, 0);
1724
- try {
1725
- const frame = new VideoFrame(canvas, {
1726
- timestamp: Math.round(this.video.currentTime * 1000000),
1727
- });
1728
- controller.enqueue(frame);
1729
- }
1730
- catch (err) {
1731
- running = false;
1732
- controller.error(err);
1733
- this.close();
1734
- }
1735
- },
1736
- cancel: () => {
1737
- running = false;
1738
- this.close();
1739
- },
1740
- });
1741
- }
1742
- close = () => {
1743
- this.video.pause();
1744
- this.video.srcObject = null;
1745
- this.video.src = '';
1746
- this.workerTimer.destroy();
1747
- };
1748
- }
1749
- const TrackProcessor = typeof MediaStreamTrackProcessor !== 'undefined'
1750
- ? MediaStreamTrackProcessor
1751
- : FallbackProcessor;
1752
-
1753
- /**
1754
- * Base class for real-time video filters.
1755
- *
1756
- * It sets up the full pipeline that reads frames from the input track,
1757
- * processes them, and outputs a new track with your effect applied. Subclasses
1758
- * only need to implement `initialize` (run once before processing starts) and
1759
- * `transform` (called for every frame).
1760
- *
1761
- * Everything else—canvas setup, performance tracking, error handling, and
1762
- * clean shutdown is handled for you. Calling `start()` returns a processed
1763
- * `MediaStreamTrack` ready to use.
1764
- */
1765
- class BaseVideoProcessor {
1766
- track;
1767
- processor;
1768
- generator;
1769
- hooks;
1770
- abortController = new AbortController();
1771
- canvas;
1772
- frames = 0;
1773
- delayTotal = 0;
1774
- lastStatsTime = 0;
1775
- /**
1776
- * Constructs a new instance.
1777
- */
1778
- constructor(track, hooks = {}) {
1779
- this.track = track;
1780
- this.processor = new TrackProcessor({ track });
1781
- this.generator = new TrackGenerator({
1782
- kind: 'video',
1783
- signalTarget: track,
1784
- });
1785
- this.hooks = hooks;
1786
- }
1787
- async start() {
1788
- const { readable } = this.processor;
1789
- const { writable } = this.generator;
1790
- const { width = 1280, height = 720 } = this.track.getSettings();
1791
- this.canvas = new OffscreenCanvas(width, height);
1792
- await this.initialize();
1793
- const transformStream = new TransformStream({
1794
- transform: async (frame, controller) => {
1795
- try {
1796
- if (this.abortController.signal.aborted)
1797
- return frame.close();
1798
- if (this.canvas.width !== frame.displayWidth ||
1799
- this.canvas.height !== frame.displayHeight) {
1800
- this.canvas.width = frame.displayWidth;
1801
- this.canvas.height = frame.displayHeight;
1802
- }
1803
- const processed = await this.transform(frame);
1804
- controller.enqueue(processed);
1805
- }
1806
- catch (e) {
1807
- this.hooks.onError?.(e);
1808
- }
1809
- finally {
1810
- frame.close();
1811
- }
1812
- },
1813
- flush: () => this.onFlush(),
1814
- });
1815
- readable
1816
- .pipeThrough(transformStream, { signal: this.abortController.signal })
1817
- .pipeTo(writable, { signal: this.abortController.signal })
1818
- .catch((e) => {
1819
- if (e.name !== 'AbortError' && e.name !== 'InvalidStateError') {
1820
- console.error(`[${this.processorName}] Error processing track:`, e);
1821
- this.hooks.onError?.(e);
1822
- }
1823
- });
1824
- return this.generator;
1825
- }
1826
- stop() {
1827
- this.abortController.abort();
1828
- this.generator.stop();
1829
- this.onStop();
1830
- }
1831
- updateStats(delay) {
1832
- this.frames++;
1833
- this.delayTotal += delay;
1834
- const now = performance.now();
1835
- if (this.lastStatsTime === 0) {
1836
- this.lastStatsTime = now;
1837
- return;
1838
- }
1839
- if (now - this.lastStatsTime >= 1000) {
1840
- const avgDelay = Math.round((this.delayTotal / this.frames) * 100) / 100;
1841
- const fps = Math.round((1000 * this.frames) / (now - this.lastStatsTime));
1842
- this.hooks.onStats?.({ delay: avgDelay, fps, timestamp: now });
1843
- this.frames = 0;
1844
- this.delayTotal = 0;
1845
- this.lastStatsTime = now;
1846
- }
1847
- }
1848
- onFlush() { }
1849
- onStop() { }
1850
- get processorName() {
1851
- return 'base-processor';
1852
- }
1853
- }
1854
-
1855
- class WebGLRenderer {
1856
- canvas;
1857
- gl;
1858
- stateUpdateProgram;
1859
- maskRefineProgram;
1860
- blurProgram;
1861
- blendProgram;
1862
- stateUpdateLocations;
1863
- maskRefineLocations;
1864
- blurLocations;
1865
- blendLocations;
1866
- positionBuffer;
1867
- texCoordBuffer;
1868
- storedStateTextures;
1869
- fbo;
1870
- refineFbo;
1871
- refinedMaskTexture;
1872
- frameTexture;
1873
- blurTexture1;
1874
- blurTexture2;
1875
- blurFbo1;
1876
- blurFbo2;
1877
- running = false;
1878
- static DEFAULT_BG_COLOR = [33, 150, 243, 255];
1879
- currentStateIndex = 0;
1880
- backgroundRenderInfo = null;
1881
- activeBackgroundSourceIdentifier = null;
1882
- constructor(canvas) {
1883
- this.canvas = canvas;
1884
- const gl = this.canvas.getContext('webgl2', {
1885
- alpha: false,
1886
- antialias: false,
1887
- desynchronized: true,
1888
- });
1889
- if (!gl)
1890
- throw new Error('WebGL2 not supported');
1891
- this.gl = gl;
1892
- const stateUpdateVertexShaderSource = `attribute vec2 a_position; attribute vec2 a_texCoord; varying vec2 v_texCoord; void main() { gl_Position = vec4(a_position, 0.0, 1.0); v_texCoord = a_texCoord; }`;
1893
- const stateUpdateFragmentShaderSource = `
1894
- precision mediump float;
1895
- varying vec2 v_texCoord;
1896
- uniform sampler2D u_categoryTexture;
1897
- uniform sampler2D u_confidenceTexture;
1898
- uniform sampler2D u_prevStateTexture;
1899
- uniform float u_smoothingFactor;
1900
- uniform float u_smoothstepMin;
1901
- uniform float u_smoothstepMax;
1902
- uniform int u_selfieModel;
1903
-
1904
- void main() {
1905
- vec2 prevCoord = vec2(v_texCoord.x, 1.0 - v_texCoord.y);
1906
- float categoryValue = texture2D(u_categoryTexture, v_texCoord).r;
1907
- float confidenceValue = texture2D(u_confidenceTexture, v_texCoord).r;
1908
-
1909
- if (u_selfieModel == 1) {
1910
- categoryValue = 1.0 - categoryValue;
1911
- confidenceValue = 1.0 - confidenceValue;
1912
- }
1913
-
1914
- if (categoryValue > 0.0) {
1915
- categoryValue = 1.0;
1916
- confidenceValue = 1.0 - confidenceValue;
1917
- }
1918
-
1919
- float nonLinearConfidence = smoothstep(u_smoothstepMin, u_smoothstepMax, confidenceValue);
1920
- float prevCategoryValue = texture2D(u_prevStateTexture, prevCoord).r;
1921
- float alpha = u_smoothingFactor * nonLinearConfidence;
1922
- float newCategoryValue = alpha * categoryValue + (1.0 - alpha) * prevCategoryValue;
1923
-
1924
- gl_FragColor = vec4(newCategoryValue, 0.0, 0.0, 0.0);
1925
- }
1926
- `;
1927
- this.stateUpdateProgram = this.createAndLinkProgram(stateUpdateVertexShaderSource, stateUpdateFragmentShaderSource);
1928
- this.stateUpdateLocations = {
1929
- position: gl.getAttribLocation(this.stateUpdateProgram, 'a_position'),
1930
- texCoord: gl.getAttribLocation(this.stateUpdateProgram, 'a_texCoord'),
1931
- categoryTexture: gl.getUniformLocation(this.stateUpdateProgram, 'u_categoryTexture'),
1932
- confidenceTexture: gl.getUniformLocation(this.stateUpdateProgram, 'u_confidenceTexture'),
1933
- prevStateTexture: gl.getUniformLocation(this.stateUpdateProgram, 'u_prevStateTexture'),
1934
- smoothingFactor: gl.getUniformLocation(this.stateUpdateProgram, 'u_smoothingFactor'),
1935
- smoothstepMin: gl.getUniformLocation(this.stateUpdateProgram, 'u_smoothstepMin'),
1936
- smoothstepMax: gl.getUniformLocation(this.stateUpdateProgram, 'u_smoothstepMax'),
1937
- selfieModel: gl.getUniformLocation(this.stateUpdateProgram, 'u_selfieModel'),
1938
- };
1939
- const maskRefineVertexShaderSource = stateUpdateVertexShaderSource;
1940
- const maskRefineFragmentShaderSource = `
1941
- precision mediump float;
1942
- varying vec2 v_texCoord;
1943
-
1944
- uniform sampler2D u_maskTexture;
1945
- uniform sampler2D u_frameTexture;
1946
- uniform vec2 u_texelSize;
1947
- uniform float u_sigmaSpatial;
1948
- uniform float u_sigmaRange;
1949
-
1950
- void main() {
1951
- vec2 flippedCoord = v_texCoord;
1952
- vec3 centerPixelColor = texture2D(u_frameTexture, v_texCoord).rgb;
1953
- float totalWeight = 0.0;
1954
- float weightedMaskSum = 0.0;
1955
-
1956
- for (int offsetX = -2; offsetX <= 2; offsetX++) {
1957
- for (int offsetY = -2; offsetY <= 2; offsetY++) {
1958
- vec2 shift = vec2(float(offsetX), float(offsetY)) * u_texelSize;
1959
- vec2 frameCoord = v_texCoord + shift;
1960
- vec2 maskCoord = flippedCoord + shift;
1961
-
1962
- vec3 neighborPixelColor = texture2D(u_frameTexture, frameCoord).rgb;
1963
- float neighborMaskValue = texture2D(u_maskTexture, maskCoord).r;
1964
-
1965
- float spatialWeight = exp(-dot(shift, shift) / (2.0 * u_sigmaSpatial * u_sigmaSpatial));
1966
- vec3 colorDifference = neighborPixelColor - centerPixelColor;
1967
- float rangeWeight = exp(-(dot(colorDifference, colorDifference)) / (2.0 * u_sigmaRange * u_sigmaRange));
1968
-
1969
- float combinedWeight = spatialWeight * rangeWeight;
1970
- weightedMaskSum += neighborMaskValue * combinedWeight;
1971
- totalWeight += combinedWeight;
1972
- }
1973
- }
1974
-
1975
- float refinedMaskValue = weightedMaskSum / max(totalWeight, 1e-6);
1976
- gl_FragColor = vec4(refinedMaskValue, refinedMaskValue, refinedMaskValue, 1.0);
1977
- }
1978
- `;
1979
- this.maskRefineProgram = this.createAndLinkProgram(maskRefineVertexShaderSource, maskRefineFragmentShaderSource);
1980
- this.maskRefineLocations = {
1981
- position: gl.getAttribLocation(this.maskRefineProgram, 'a_position'),
1982
- texCoord: gl.getAttribLocation(this.maskRefineProgram, 'a_texCoord'),
1983
- maskTexture: gl.getUniformLocation(this.maskRefineProgram, 'u_maskTexture'),
1984
- frameTexture: gl.getUniformLocation(this.maskRefineProgram, 'u_frameTexture'),
1985
- texelSize: gl.getUniformLocation(this.maskRefineProgram, 'u_texelSize'),
1986
- sigmaSpatial: gl.getUniformLocation(this.maskRefineProgram, 'u_sigmaSpatial'),
1987
- sigmaRange: gl.getUniformLocation(this.maskRefineProgram, 'u_sigmaRange'),
1988
- };
1989
- const blurVertexShaderSource = stateUpdateVertexShaderSource;
1990
- const blurFragmentShaderSource = `
1991
- precision highp float;
1992
- varying vec2 v_texCoord;
1993
-
1994
- uniform sampler2D u_image;
1995
- uniform sampler2D u_personMask;
1996
- uniform vec2 u_texelSize;
1997
- uniform float u_sigma;
1998
- uniform float u_radiusScale;
1999
- uniform vec2 u_direction;
2000
-
2001
- const int KERNEL_RADIUS = 10;
2002
-
2003
- float gauss(float x, float s) {
2004
- return exp(-(x * x) / (2.0 * s * s));
2005
- }
2006
-
2007
- void main() {
2008
- vec2 maskCoord = u_direction.y > 0.5 ? vec2(v_texCoord.x, 1.0 - v_texCoord.y) : v_texCoord;
2009
- float mCenter = texture2D(u_personMask, maskCoord).r;
2010
- float wCenter = gauss(0.0, u_sigma);
2011
- vec4 accum = texture2D(u_image, v_texCoord) * wCenter * (1.0 - mCenter);
2012
- float weightSum = wCenter * (1.0 - mCenter);
2013
-
2014
- for (int i = 1; i <= KERNEL_RADIUS; i++) {
2015
- float f = float(i);
2016
- float offset = f * u_radiusScale;
2017
- float w = gauss(offset, u_sigma);
2018
- vec2 texOffset = u_direction * offset * u_texelSize;
2019
-
2020
- vec2 uvPlus = v_texCoord + texOffset;
2021
- vec2 maskCoordPlus = u_direction.y > 0.5 ? vec2(uvPlus.x, 1.0 - uvPlus.y) : uvPlus;
2022
- float mPlus = texture2D(u_personMask, maskCoordPlus).r;
2023
- accum += texture2D(u_image, uvPlus) * w * (1.0 - mPlus);
2024
- weightSum += w * (1.0 - mPlus);
2025
-
2026
- vec2 uvMinus = v_texCoord - texOffset;
2027
- vec2 maskCoordMinus = u_direction.y > 0.5 ? vec2(uvMinus.x, 1.0 - uvMinus.y) : uvMinus;
2028
- float mMinus = texture2D(u_personMask, maskCoordMinus).r;
2029
- accum += texture2D(u_image, uvMinus) * w * (1.0 - mMinus);
2030
- weightSum += w * (1.0 - mMinus);
2031
- }
2032
-
2033
- vec4 blurred = accum / max(weightSum, 1e-6);
2034
- gl_FragColor = blurred;
2035
- }
2036
- `;
2037
- this.blurProgram = this.createAndLinkProgram(blurVertexShaderSource, blurFragmentShaderSource);
2038
- this.blurLocations = {
2039
- position: gl.getAttribLocation(this.blurProgram, 'a_position'),
2040
- texCoord: gl.getAttribLocation(this.blurProgram, 'a_texCoord'),
2041
- image: gl.getUniformLocation(this.blurProgram, 'u_image'),
2042
- personMask: gl.getUniformLocation(this.blurProgram, 'u_personMask'),
2043
- texelSize: gl.getUniformLocation(this.blurProgram, 'u_texelSize'),
2044
- sigma: gl.getUniformLocation(this.blurProgram, 'u_sigma'),
2045
- radiusScale: gl.getUniformLocation(this.blurProgram, 'u_radiusScale'),
2046
- direction: gl.getUniformLocation(this.blurProgram, 'u_direction'),
2047
- };
2048
- const blendVertexShaderSource = stateUpdateVertexShaderSource;
2049
- const blendFragmentShaderSource = `
2050
- precision mediump float;
2051
- varying vec2 v_texCoord;
2052
-
2053
- uniform sampler2D u_frameTexture;
2054
- uniform sampler2D u_currentStateTexture;
2055
- uniform sampler2D u_backgroundTexture;
2056
- uniform vec2 u_bgImageDimensions;
2057
- uniform vec2 u_canvasDimensions;
2058
- uniform float u_borderSmooth;
2059
- uniform float u_bgBlur;
2060
- uniform float u_bgBlurRadius;
2061
- uniform int u_enabled;
2062
-
2063
- vec4 getMixedFragColor(vec2 bgTexCoord, vec2 categoryCoord, vec2 offset) {
2064
- vec4 backgroundColor = texture2D(u_backgroundTexture, bgTexCoord + offset);
2065
- vec4 frameColor = texture2D(u_frameTexture, v_texCoord + offset);
2066
- float categoryValue = texture2D(u_currentStateTexture, categoryCoord + offset).r;
2067
- return mix(backgroundColor, frameColor, categoryValue);
2068
- }
2069
-
2070
- void main() {
2071
- if (u_enabled == 0) {
2072
- gl_FragColor = texture2D(u_frameTexture, v_texCoord);
2073
- return;
2074
- }
2075
-
2076
- vec2 categoryCoord = v_texCoord;
2077
- float categoryValue = texture2D(u_currentStateTexture, categoryCoord).r;
2078
-
2079
- float canvasAspect = u_canvasDimensions.x / u_canvasDimensions.y;
2080
- float bgAspect = u_bgImageDimensions.x / u_bgImageDimensions.y;
2081
-
2082
- vec2 bgTexCoord = v_texCoord;
2083
- float scaleX = 1.0;
2084
- float scaleY = 1.0;
2085
- float offsetX = 0.0;
2086
- float offsetY = 0.0;
2087
-
2088
- if (canvasAspect < bgAspect) {
2089
- scaleY = 1.0;
2090
- scaleX = bgAspect / canvasAspect;
2091
- offsetX = (1.0 - scaleX) / 2.0;
2092
- } else {
2093
- scaleX = 1.0;
2094
- scaleY = canvasAspect / bgAspect;
2095
- offsetY = (1.0 - scaleY) / 2.0;
2096
- }
2097
-
2098
- bgTexCoord = vec2((v_texCoord.x - offsetX) / scaleX, (v_texCoord.y - offsetY) / scaleY);
2099
- gl_FragColor = getMixedFragColor(bgTexCoord, categoryCoord, vec2(0.0, 0.0));
2100
- }`;
2101
- this.blendProgram = this.createAndLinkProgram(blendVertexShaderSource, blendFragmentShaderSource);
2102
- this.blendLocations = {
2103
- position: gl.getAttribLocation(this.blendProgram, 'a_position'),
2104
- texCoord: gl.getAttribLocation(this.blendProgram, 'a_texCoord'),
2105
- frameTexture: gl.getUniformLocation(this.blendProgram, 'u_frameTexture'),
2106
- currentStateTexture: gl.getUniformLocation(this.blendProgram, 'u_currentStateTexture'),
2107
- backgroundTexture: gl.getUniformLocation(this.blendProgram, 'u_backgroundTexture'),
2108
- bgImageDimensions: gl.getUniformLocation(this.blendProgram, 'u_bgImageDimensions'),
2109
- canvasDimensions: gl.getUniformLocation(this.blendProgram, 'u_canvasDimensions'),
2110
- borderSmooth: gl.getUniformLocation(this.blendProgram, 'u_borderSmooth'),
2111
- bgBlur: gl.getUniformLocation(this.blendProgram, 'u_bgBlur'),
2112
- bgBlurRadius: gl.getUniformLocation(this.blendProgram, 'u_bgBlurRadius'),
2113
- enabled: gl.getUniformLocation(this.blendProgram, 'u_enabled'),
2114
- };
2115
- this.positionBuffer = gl.createBuffer();
2116
- gl.bindBuffer(gl.ARRAY_BUFFER, this.positionBuffer);
2117
- gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([-1, -1, 1, -1, -1, 1, -1, 1, 1, -1, 1, 1]), gl.STATIC_DRAW);
2118
- this.texCoordBuffer = gl.createBuffer();
2119
- gl.bindBuffer(gl.ARRAY_BUFFER, this.texCoordBuffer);
2120
- gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([0, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 0]), gl.STATIC_DRAW);
2121
- this.storedStateTextures = Array.from({ length: 2 }, () => {
2122
- const tex = gl.createTexture();
2123
- gl.bindTexture(gl.TEXTURE_2D, tex);
2124
- gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, 1, 1, 0, gl.RGBA, gl.UNSIGNED_BYTE, new Uint8Array([0, 0, 0, 255]));
2125
- gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
2126
- gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
2127
- gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
2128
- gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR);
2129
- return tex;
2130
- });
2131
- gl.bindTexture(gl.TEXTURE_2D, null);
2132
- this.fbo = gl.createFramebuffer();
2133
- this.refineFbo = gl.createFramebuffer();
2134
- const refinedTex = gl.createTexture();
2135
- this.frameTexture = gl.createTexture();
2136
- if (!refinedTex)
2137
- throw new Error('Failed to create refined mask texture');
2138
- gl.bindTexture(gl.TEXTURE_2D, refinedTex);
2139
- gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, 1, 1, 0, gl.RGBA, gl.UNSIGNED_BYTE, null);
2140
- gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
2141
- gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
2142
- gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
2143
- gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR);
2144
- gl.bindTexture(gl.TEXTURE_2D, null);
2145
- this.refinedMaskTexture = refinedTex;
2146
- const mkColorTex = () => {
2147
- const t = gl.createTexture();
2148
- if (!t)
2149
- throw new Error('Failed to create blur texture');
2150
- gl.bindTexture(gl.TEXTURE_2D, t);
2151
- gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, 1, 1, 0, gl.RGBA, gl.UNSIGNED_BYTE, null);
2152
- gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
2153
- gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
2154
- gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
2155
- gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR);
2156
- gl.bindTexture(gl.TEXTURE_2D, null);
2157
- return t;
2158
- };
2159
- this.blurTexture1 = mkColorTex();
2160
- this.blurTexture2 = mkColorTex();
2161
- const mkFbo = (tex) => {
2162
- const fb = gl.createFramebuffer();
2163
- if (!fb || !tex)
2164
- throw new Error('Failed to create blur FBO');
2165
- gl.bindFramebuffer(gl.FRAMEBUFFER, fb);
2166
- gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, tex, 0);
2167
- gl.bindFramebuffer(gl.FRAMEBUFFER, null);
2168
- return fb;
2169
- };
2170
- this.blurFbo1 = mkFbo(this.blurTexture1);
2171
- this.blurFbo2 = mkFbo(this.blurTexture2);
2172
- this.running = true;
2173
- }
2174
- createAndLinkProgram(vsSource, fsSource) {
2175
- const vs = this.createShader(this.gl.VERTEX_SHADER, vsSource);
2176
- const fs = this.createShader(this.gl.FRAGMENT_SHADER, fsSource);
2177
- const prog = this.gl.createProgram();
2178
- if (!prog)
2179
- throw new Error('Failed to create program');
2180
- this.gl.attachShader(prog, vs);
2181
- this.gl.attachShader(prog, fs);
2182
- this.gl.linkProgram(prog);
2183
- if (!this.gl.getProgramParameter(prog, this.gl.LINK_STATUS)) {
2184
- console.error('Program link error:', this.gl.getProgramInfoLog(prog));
2185
- this.gl.deleteProgram(prog);
2186
- throw new Error('Link fail');
2187
- }
2188
- this.gl.detachShader(prog, vs);
2189
- this.gl.detachShader(prog, fs);
2190
- this.gl.deleteShader(vs);
2191
- this.gl.deleteShader(fs);
2192
- return prog;
2193
- }
2194
- createShader(type, source) {
2195
- const shader = this.gl.createShader(type);
2196
- if (!shader)
2197
- throw new Error(`Failed to create shader type: ${type}`);
2198
- this.gl.shaderSource(shader, source);
2199
- this.gl.compileShader(shader);
2200
- if (!this.gl.getShaderParameter(shader, this.gl.COMPILE_STATUS)) {
2201
- console.error('Shader compile error:', this.gl.getShaderInfoLog(shader));
2202
- this.gl.deleteShader(shader);
2203
- throw new Error('Failed to compile shader');
2204
- }
2205
- return shader;
2206
- }
2207
- createColorTexture(r, g, b, a) {
2208
- const texture = this.gl.createTexture();
2209
- if (!texture)
2210
- throw new Error('Failed to create texture for color');
2211
- this.gl.bindTexture(this.gl.TEXTURE_2D, texture);
2212
- const pixel = new Uint8Array([r, g, b, a]);
2213
- this.gl.texImage2D(this.gl.TEXTURE_2D, 0, this.gl.RGBA, 1, 1, 0, this.gl.RGBA, this.gl.UNSIGNED_BYTE, pixel);
2214
- this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_WRAP_S, this.gl.CLAMP_TO_EDGE);
2215
- this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_WRAP_T, this.gl.CLAMP_TO_EDGE);
2216
- this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_MIN_FILTER, this.gl.NEAREST);
2217
- this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_MAG_FILTER, this.gl.NEAREST);
2218
- this.gl.bindTexture(this.gl.TEXTURE_2D, null);
2219
- return { texture, color: [r, g, b, a] };
2220
- }
2221
- updateBackgroundIfNeeded(newSource) {
2222
- const gl = this.gl;
2223
- let newIdentifier;
2224
- if (!newSource) {
2225
- const [r, g, b, a] = WebGLRenderer.DEFAULT_BG_COLOR;
2226
- newIdentifier = `color(${r},${g},${b},${a})`;
2227
- }
2228
- else {
2229
- newIdentifier = newSource.url;
2230
- }
2231
- if (newIdentifier === this.activeBackgroundSourceIdentifier &&
2232
- this.backgroundRenderInfo) {
2233
- return;
2234
- }
2235
- if (this.backgroundRenderInfo) {
2236
- gl.deleteTexture(this.backgroundRenderInfo.texture);
2237
- this.backgroundRenderInfo = null;
2238
- }
2239
- this.activeBackgroundSourceIdentifier = newIdentifier;
2240
- if (!newSource) {
2241
- const [r, g, b, a] = WebGLRenderer.DEFAULT_BG_COLOR;
2242
- const colorTexData = this.createColorTexture(r, g, b, a);
2243
- this.backgroundRenderInfo = {
2244
- type: 'color',
2245
- texture: colorTexData.texture,
2246
- color: colorTexData.color,
2247
- };
2248
- this.activeBackgroundSourceIdentifier = `color(${r},${g},${b},${a})`;
2249
- }
2250
- else {
2251
- if (newSource.type === 'image') {
2252
- const { media, url } = newSource;
2253
- const texture = this.gl.createTexture();
2254
- if (!texture) {
2255
- throw new Error('Failed to create texture object for image.');
2256
- }
2257
- this.gl.bindTexture(this.gl.TEXTURE_2D, texture);
2258
- this.gl.texImage2D(this.gl.TEXTURE_2D, 0, this.gl.RGBA, this.gl.RGBA, this.gl.UNSIGNED_BYTE, media);
2259
- this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_WRAP_S, this.gl.CLAMP_TO_EDGE);
2260
- this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_WRAP_T, this.gl.CLAMP_TO_EDGE);
2261
- this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_MIN_FILTER, this.gl.LINEAR);
2262
- this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_MAG_FILTER, this.gl.LINEAR);
2263
- this.gl.bindTexture(this.gl.TEXTURE_2D, null);
2264
- this.backgroundRenderInfo = {
2265
- type: 'image',
2266
- texture,
2267
- width: media.width,
2268
- height: media.height,
2269
- url,
2270
- };
2271
- }
2272
- else if (newSource.type === 'video') {
2273
- const { media, url } = newSource;
2274
- const canvas = new OffscreenCanvas(1, 1);
2275
- const ctx = canvas.getContext('2d');
2276
- const writer = new WritableStream({
2277
- write(videoFrame) {
2278
- canvas.width = videoFrame.codedWidth;
2279
- canvas.height = videoFrame.codedHeight;
2280
- ctx?.drawImage(videoFrame, 0, 0);
2281
- videoFrame.close();
2282
- },
2283
- close() {
2284
- console.log('[virtual-background] video background close');
2285
- },
2286
- });
2287
- media.pipeTo(writer).catch((err) => {
2288
- console.error('media.pipeTo(writer) error', err);
2289
- });
2290
- const texture = this.gl.createTexture();
2291
- if (!texture)
2292
- throw new Error('Failed to create texture for video');
2293
- this.gl.bindTexture(this.gl.TEXTURE_2D, texture);
2294
- this.gl.texImage2D(this.gl.TEXTURE_2D, 0, this.gl.RGBA, 1, 1, 0, this.gl.RGBA, this.gl.UNSIGNED_BYTE, null);
2295
- this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_WRAP_S, this.gl.CLAMP_TO_EDGE);
2296
- this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_WRAP_T, this.gl.CLAMP_TO_EDGE);
2297
- this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_MIN_FILTER, this.gl.LINEAR);
2298
- this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_MAG_FILTER, this.gl.LINEAR);
2299
- this.gl.bindTexture(this.gl.TEXTURE_2D, null);
2300
- this.backgroundRenderInfo = {
2301
- type: 'video',
2302
- texture,
2303
- url,
2304
- media,
2305
- canvas,
2306
- };
2307
- }
2308
- }
2309
- if (!this.backgroundRenderInfo) {
2310
- console.error('Critical: backgroundRenderInfo is null after processing new source. Setting default color.');
2311
- const [r, g, b, a] = WebGLRenderer.DEFAULT_BG_COLOR;
2312
- const colorTexData = this.createColorTexture(r, g, b, a);
2313
- this.backgroundRenderInfo = {
2314
- type: 'color',
2315
- texture: colorTexData.texture,
2316
- color: colorTexData.color,
2317
- };
2318
- this.activeBackgroundSourceIdentifier = `color(${r},${g},${b},${a})`;
2319
- }
2320
- }
2321
- render(videoFrame, options, categoryTexture, confidenceTexture) {
2322
- if (!this.running)
2323
- return;
2324
- const { gl, fbo, frameTexture, storedStateTextures, stateUpdateProgram, stateUpdateLocations, refineFbo, refinedMaskTexture, maskRefineProgram, maskRefineLocations, blendProgram, blendLocations, blurFbo1, blurFbo2, blurTexture1, blurTexture2, } = this;
2325
- const { displayWidth: width, displayHeight: height } = videoFrame;
2326
- if (this.canvas.width !== width || this.canvas.height !== height) {
2327
- this.canvas.width = width;
2328
- this.canvas.height = height;
2329
- }
2330
- if (!categoryTexture || !confidenceTexture) {
2331
- gl.viewport(0, 0, gl.drawingBufferWidth, gl.drawingBufferHeight);
2332
- gl.useProgram(blendProgram);
2333
- const frame = gl.createTexture();
2334
- gl.activeTexture(gl.TEXTURE0);
2335
- gl.bindTexture(gl.TEXTURE_2D, frame);
2336
- gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, videoFrame);
2337
- gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
2338
- gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
2339
- gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
2340
- gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
2341
- gl.uniform1i(blendLocations.frameTexture, 0);
2342
- gl.uniform1i(blendLocations.enabled, 0);
2343
- gl.enableVertexAttribArray(blendLocations.position);
2344
- gl.bindBuffer(gl.ARRAY_BUFFER, this.positionBuffer);
2345
- gl.vertexAttribPointer(blendLocations.position, 2, gl.FLOAT, false, 0, 0);
2346
- gl.enableVertexAttribArray(blendLocations.texCoord);
2347
- gl.bindBuffer(gl.ARRAY_BUFFER, this.texCoordBuffer);
2348
- gl.vertexAttribPointer(blendLocations.texCoord, 2, gl.FLOAT, false, 0, 0);
2349
- gl.drawArrays(gl.TRIANGLES, 0, 6);
2350
- gl.deleteTexture(frame);
2351
- gl.activeTexture(gl.TEXTURE0);
2352
- gl.bindTexture(gl.TEXTURE_2D, null);
2353
- return;
2354
- }
2355
- const readStateIndex = this.currentStateIndex;
2356
- const writeStateIndex = (this.currentStateIndex + 1) % 2;
2357
- const prevStateTexture = storedStateTextures[readStateIndex];
2358
- const newStateTexture = storedStateTextures[writeStateIndex];
2359
- this.updateBackgroundIfNeeded(options.backgroundSource);
2360
- gl.bindFramebuffer(gl.FRAMEBUFFER, fbo);
2361
- gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, newStateTexture, 0);
2362
- gl.bindTexture(gl.TEXTURE_2D, newStateTexture);
2363
- gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, width, height, 0, gl.RGBA, gl.UNSIGNED_BYTE, null);
2364
- gl.viewport(0, 0, width, height);
2365
- gl.useProgram(stateUpdateProgram);
2366
- gl.activeTexture(gl.TEXTURE0);
2367
- gl.bindTexture(gl.TEXTURE_2D, categoryTexture);
2368
- gl.uniform1i(stateUpdateLocations.categoryTexture, 0);
2369
- gl.activeTexture(gl.TEXTURE1);
2370
- gl.bindTexture(gl.TEXTURE_2D, confidenceTexture);
2371
- gl.uniform1i(stateUpdateLocations.confidenceTexture, 1);
2372
- gl.activeTexture(gl.TEXTURE2);
2373
- gl.bindTexture(gl.TEXTURE_2D, prevStateTexture);
2374
- gl.uniform1i(stateUpdateLocations.prevStateTexture, 2);
2375
- gl.uniform1f(stateUpdateLocations.smoothingFactor, options.segmentationOptions?.smoothingFactor ?? 0.8);
2376
- gl.uniform1f(stateUpdateLocations.smoothstepMin, options.segmentationOptions?.smoothstepMin ?? 0.6);
2377
- gl.uniform1f(stateUpdateLocations.smoothstepMax, options.segmentationOptions?.smoothstepMax ?? 0.9);
2378
- gl.uniform1i(stateUpdateLocations.selfieModel, options.isSelfieMode ? 1 : 0);
2379
- gl.enableVertexAttribArray(stateUpdateLocations.position);
2380
- gl.bindBuffer(gl.ARRAY_BUFFER, this.positionBuffer);
2381
- gl.vertexAttribPointer(stateUpdateLocations.position, 2, gl.FLOAT, false, 0, 0);
2382
- gl.enableVertexAttribArray(stateUpdateLocations.texCoord);
2383
- gl.bindBuffer(gl.ARRAY_BUFFER, this.texCoordBuffer);
2384
- gl.vertexAttribPointer(stateUpdateLocations.texCoord, 2, gl.FLOAT, false, 0, 0);
2385
- gl.drawArrays(gl.TRIANGLES, 0, 6);
2386
- gl.bindFramebuffer(gl.FRAMEBUFFER, refineFbo);
2387
- gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, refinedMaskTexture, 0);
2388
- gl.bindTexture(gl.TEXTURE_2D, refinedMaskTexture);
2389
- gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, width, height, 0, gl.RGBA, gl.UNSIGNED_BYTE, null);
2390
- gl.viewport(0, 0, width, height);
2391
- gl.useProgram(maskRefineProgram);
2392
- gl.enableVertexAttribArray(maskRefineLocations.position);
2393
- gl.bindBuffer(gl.ARRAY_BUFFER, this.positionBuffer);
2394
- gl.vertexAttribPointer(maskRefineLocations.position, 2, gl.FLOAT, false, 0, 0);
2395
- gl.enableVertexAttribArray(maskRefineLocations.texCoord);
2396
- gl.bindBuffer(gl.ARRAY_BUFFER, this.texCoordBuffer);
2397
- gl.vertexAttribPointer(maskRefineLocations.texCoord, 2, gl.FLOAT, false, 0, 0);
2398
- gl.activeTexture(gl.TEXTURE0);
2399
- gl.bindTexture(gl.TEXTURE_2D, newStateTexture);
2400
- gl.uniform1i(maskRefineLocations.maskTexture, 0);
2401
- gl.activeTexture(gl.TEXTURE1);
2402
- gl.bindTexture(gl.TEXTURE_2D, frameTexture);
2403
- gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, videoFrame);
2404
- gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
2405
- gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
2406
- gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
2407
- gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
2408
- gl.uniform1i(maskRefineLocations.frameTexture, 1);
2409
- gl.uniform2f(maskRefineLocations.texelSize, 1.0 / width, 1.0 / height);
2410
- gl.uniform1f(maskRefineLocations.sigmaSpatial, 2.0);
2411
- gl.uniform1f(maskRefineLocations.sigmaRange, 0.1);
2412
- gl.drawArrays(gl.TRIANGLES, 0, 6);
2413
- gl.disableVertexAttribArray(maskRefineLocations.position);
2414
- gl.disableVertexAttribArray(maskRefineLocations.texCoord);
2415
- let backgroundTexToUse;
2416
- let bgWToSend = width;
2417
- let bgHToSend = height;
2418
- if (options.bgBlur > 0 && options.bgBlurRadius > 0) {
2419
- const downscale = 0.5;
2420
- const blurW = Math.floor(width * downscale);
2421
- const blurH = Math.floor(height * downscale);
2422
- gl.bindTexture(gl.TEXTURE_2D, blurTexture1);
2423
- gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, blurW, blurH, 0, gl.RGBA, gl.UNSIGNED_BYTE, null);
2424
- gl.bindTexture(gl.TEXTURE_2D, blurTexture2);
2425
- gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, blurW, blurH, 0, gl.RGBA, gl.UNSIGNED_BYTE, null);
2426
- const KERNEL_RADIUS = 10.0;
2427
- const radiusScale = Math.max(0.0, options.bgBlurRadius) / KERNEL_RADIUS;
2428
- gl.useProgram(this.blurProgram);
2429
- gl.enableVertexAttribArray(this.blurLocations.position);
2430
- gl.bindBuffer(gl.ARRAY_BUFFER, this.positionBuffer);
2431
- gl.vertexAttribPointer(this.blurLocations.position, 2, gl.FLOAT, false, 0, 0);
2432
- gl.enableVertexAttribArray(this.blurLocations.texCoord);
2433
- gl.bindBuffer(gl.ARRAY_BUFFER, this.texCoordBuffer);
2434
- gl.vertexAttribPointer(this.blurLocations.texCoord, 2, gl.FLOAT, false, 0, 0);
2435
- gl.activeTexture(gl.TEXTURE1);
2436
- gl.bindTexture(gl.TEXTURE_2D, refinedMaskTexture);
2437
- gl.uniform1i(this.blurLocations.personMask, 1);
2438
- gl.uniform1f(this.blurLocations.sigma, options.bgBlur * 0.7);
2439
- gl.uniform1f(this.blurLocations.radiusScale, radiusScale);
2440
- const blurPasses = [
2441
- {
2442
- direction: [1.0, 0.0],
2443
- input: frameTexture,
2444
- output: blurFbo1,
2445
- texelSize: [1.0 / width, 1.0 / height],
2446
- },
2447
- {
2448
- direction: [0.0, 1.0],
2449
- input: blurTexture1,
2450
- output: blurFbo2,
2451
- texelSize: [1.0 / blurW, 1.0 / blurH],
2452
- },
2453
- {
2454
- direction: [1.0, 0.0],
2455
- input: blurTexture2,
2456
- output: blurFbo1,
2457
- texelSize: [1.0 / blurW, 1.0 / blurH],
2458
- },
2459
- {
2460
- direction: [0.0, 1.0],
2461
- input: blurTexture1,
2462
- output: blurFbo2,
2463
- texelSize: [1.0 / blurW, 1.0 / blurH],
2464
- },
2465
- ];
2466
- for (const pass of blurPasses) {
2467
- gl.bindFramebuffer(gl.FRAMEBUFFER, pass.output);
2468
- gl.viewport(0, 0, blurW, blurH);
2469
- gl.activeTexture(gl.TEXTURE0);
2470
- gl.bindTexture(gl.TEXTURE_2D, pass.input);
2471
- gl.uniform1i(this.blurLocations.image, 0);
2472
- gl.uniform2f(this.blurLocations.texelSize, pass.texelSize[0], pass.texelSize[1]);
2473
- gl.uniform2f(this.blurLocations.direction, pass.direction[0], pass.direction[1]);
2474
- gl.drawArrays(gl.TRIANGLES, 0, 6);
2475
- }
2476
- backgroundTexToUse = blurTexture2;
2477
- bgWToSend = blurW;
2478
- bgHToSend = blurH;
2479
- }
2480
- else if (options.backgroundSource && this.backgroundRenderInfo) {
2481
- backgroundTexToUse = this.backgroundRenderInfo.texture;
2482
- if (this.backgroundRenderInfo.type === 'video') {
2483
- const { canvas } = this.backgroundRenderInfo;
2484
- bgWToSend = canvas.width || width;
2485
- bgHToSend = canvas.height || height;
2486
- }
2487
- else if (this.backgroundRenderInfo.type === 'image') {
2488
- bgWToSend = this.backgroundRenderInfo.width;
2489
- bgHToSend = this.backgroundRenderInfo.height;
2490
- }
2491
- else {
2492
- bgWToSend = width;
2493
- bgHToSend = height;
2494
- }
2495
- }
2496
- else {
2497
- backgroundTexToUse = this.backgroundRenderInfo?.texture ?? null;
2498
- }
2499
- gl.bindFramebuffer(gl.FRAMEBUFFER, null);
2500
- gl.viewport(0, 0, gl.drawingBufferWidth, gl.drawingBufferHeight);
2501
- gl.useProgram(blendProgram);
2502
- gl.activeTexture(gl.TEXTURE0);
2503
- gl.bindTexture(gl.TEXTURE_2D, frameTexture);
2504
- gl.uniform1i(blendLocations.frameTexture, 0);
2505
- gl.uniform1f(blendLocations.borderSmooth, 0);
2506
- gl.uniform1f(blendLocations.bgBlur, options.bgBlur);
2507
- gl.uniform1f(blendLocations.bgBlurRadius, options.bgBlurRadius);
2508
- gl.uniform1i(blendLocations.enabled, 1);
2509
- gl.activeTexture(gl.TEXTURE1);
2510
- gl.bindTexture(gl.TEXTURE_2D, refinedMaskTexture);
2511
- gl.uniform1i(blendLocations.currentStateTexture, 1);
2512
- if (backgroundTexToUse) {
2513
- gl.activeTexture(gl.TEXTURE2);
2514
- gl.bindTexture(gl.TEXTURE_2D, backgroundTexToUse);
2515
- gl.uniform1i(blendLocations.backgroundTexture, 2);
2516
- gl.uniform2f(blendLocations.bgImageDimensions, bgWToSend, bgHToSend);
2517
- gl.uniform2f(blendLocations.canvasDimensions, width, height);
2518
- }
2519
- else {
2520
- gl.uniform2f(blendLocations.bgImageDimensions, width, height);
2521
- gl.uniform2f(blendLocations.canvasDimensions, width, height);
2522
- }
2523
- gl.enableVertexAttribArray(blendLocations.position);
2524
- gl.bindBuffer(gl.ARRAY_BUFFER, this.positionBuffer);
2525
- gl.vertexAttribPointer(blendLocations.position, 2, gl.FLOAT, false, 0, 0);
2526
- gl.enableVertexAttribArray(blendLocations.texCoord);
2527
- gl.bindBuffer(gl.ARRAY_BUFFER, this.texCoordBuffer);
2528
- gl.vertexAttribPointer(blendLocations.texCoord, 2, gl.FLOAT, false, 0, 0);
2529
- gl.drawArrays(gl.TRIANGLES, 0, 6);
2530
- for (let i = 0; i < 3; ++i) {
2531
- gl.activeTexture(gl.TEXTURE0 + i);
2532
- gl.bindTexture(gl.TEXTURE_2D, null);
2533
- }
2534
- this.currentStateIndex = writeStateIndex;
2535
- }
2536
- close() {
2537
- if (!this.running)
2538
- return;
2539
- this.running = false;
2540
- const { gl, fbo, refineFbo, refinedMaskTexture, blurFbo1, blurFbo2 } = this;
2541
- gl.clearColor(0, 0, 0, 0);
2542
- gl.clear(gl.COLOR_BUFFER_BIT);
2543
- gl.bindFramebuffer(gl.FRAMEBUFFER, null);
2544
- if (fbo)
2545
- gl.deleteFramebuffer(fbo);
2546
- if (refineFbo)
2547
- gl.deleteFramebuffer(refineFbo);
2548
- if (blurFbo1)
2549
- gl.deleteFramebuffer(blurFbo1);
2550
- if (blurFbo2)
2551
- gl.deleteFramebuffer(blurFbo2);
2552
- gl.deleteProgram(this.stateUpdateProgram);
2553
- gl.deleteProgram(this.maskRefineProgram);
2554
- gl.deleteProgram(this.blurProgram);
2555
- gl.deleteProgram(this.blendProgram);
2556
- if (this.positionBuffer)
2557
- gl.deleteBuffer(this.positionBuffer);
2558
- if (this.texCoordBuffer)
2559
- gl.deleteBuffer(this.texCoordBuffer);
2560
- if (refinedMaskTexture)
2561
- gl.deleteTexture(refinedMaskTexture);
2562
- if (this.blurTexture1)
2563
- gl.deleteTexture(this.blurTexture1);
2564
- if (this.blurTexture2)
2565
- gl.deleteTexture(this.blurTexture2);
2566
- this.storedStateTextures.forEach((t) => t && gl.deleteTexture(t));
2567
- this.storedStateTextures.splice(0, this.storedStateTextures.length);
2568
- if (this.backgroundRenderInfo?.texture) {
2569
- gl.deleteTexture(this.backgroundRenderInfo.texture);
2570
- this.backgroundRenderInfo = null;
2571
- }
2572
- this.activeBackgroundSourceIdentifier = null;
2573
- }
2574
- }
2575
-
2576
- /**
2577
- * Wraps a video MediaStreamTrack in a real-time processing pipeline.
2578
- * Incoming frames are processed through a transformer and re-emitted
2579
- * on a new MediaStreamVideoTrack for downstream consumption.
2580
- */
2581
- class VirtualBackground extends BaseVideoProcessor {
2582
- options;
2583
- segmenter = null;
2584
- isSegmenterReady = false;
2585
- webGlRenderer;
2586
- opts;
2587
- latestCategoryMask = undefined;
2588
- latestConfidenceMask = undefined;
2589
- lastFrameTime = -1;
2590
- constructor(track, options = {}, hooks = {}) {
2591
- super(track, hooks);
2592
- this.options = options;
2593
- }
2594
- async initialize() {
2595
- this.webGlRenderer = new WebGLRenderer(this.canvas);
2596
- await this.initializeSegmenter();
2597
- }
2598
- async initializeSegmenter() {
2599
- try {
2600
- this.opts = await this.initializeSegmenterOptions();
2601
- const basePath = this.options.basePath ||
2602
- `https://unpkg.com/${packageName}@${version}/mediapipe`;
2603
- const model = this.options.modelPath ||
2604
- `${basePath}/models/selfie_segmenter_landscape.tflite`;
2605
- const fileset = await FilesetResolver.forVisionTasks(`${basePath}/wasm`);
2606
- this.segmenter = await ImageSegmenter.createFromOptions(fileset, {
2607
- baseOptions: { modelAssetPath: model, delegate: 'GPU' },
2608
- runningMode: 'VIDEO',
2609
- outputCategoryMask: true,
2610
- outputConfidenceMasks: true,
2611
- canvas: this.canvas,
2612
- });
2613
- this.isSegmenterReady = true;
2614
- }
2615
- catch (error) {
2616
- this.isSegmenterReady = false;
2617
- this.hooks.onError?.(error);
2618
- throw error;
2619
- }
2620
- }
2621
- async transform(frame) {
2622
- const currentTime = frame.timestamp;
2623
- const hasNewFrame = currentTime - this.lastFrameTime > 1_000;
2624
- this.lastFrameTime = currentTime;
2625
- if (hasNewFrame && this.isSegmenterReady && this.segmenter) {
2626
- const start = performance.now();
2627
- await this.runSegmentation(frame);
2628
- this.webGlRenderer.render(frame, this.opts, this.latestCategoryMask, this.latestConfidenceMask);
2629
- this.updateStats(performance.now() - start);
2630
- }
2631
- return new VideoFrame(this.canvas, { timestamp: frame.timestamp });
2632
- }
2633
- async runSegmentation(frame) {
2634
- if (!this.segmenter)
2635
- return;
2636
- return new Promise((resolve) => {
2637
- const timestamp = Math.floor(performance.now());
2638
- this.segmenter.segmentForVideo(frame, timestamp, (result) => {
2639
- try {
2640
- this.latestCategoryMask = result.categoryMask?.getAsWebGLTexture();
2641
- this.latestConfidenceMask =
2642
- result.confidenceMasks?.[0]?.getAsWebGLTexture();
2643
- }
2644
- catch (err) {
2645
- console.error('[virtual-background] segmentation error:', err);
2646
- this.hooks.onError?.(err);
2647
- }
2648
- finally {
2649
- result.close();
2650
- resolve();
2651
- }
2652
- });
2653
- });
2654
- }
2655
- async initializeSegmenterOptions() {
2656
- const isSelfieMode = this.options.modelPath
2657
- ? this.options.modelPath.includes('selfie_segmenter')
2658
- : true;
2659
- if (this.options.backgroundFilter === 'image') {
2660
- const source = await this.loadBackground(this.options.backgroundImage);
2661
- return {
2662
- backgroundSource: source,
2663
- bgBlur: 0,
2664
- bgBlurRadius: 0,
2665
- isSelfieMode,
2666
- segmentationOptions: this.options.segmentationOptions,
2667
- };
2668
- }
2669
- const blurLevel = this.options.backgroundBlurLevel;
2670
- const strength = typeof blurLevel === 'string'
2671
- ? BACKGROUND_BLUR_MAP[blurLevel]
2672
- : Math.round(blurLevel ?? 5);
2673
- return {
2674
- backgroundSource: undefined,
2675
- bgBlur: Math.min(strength * 1.5, 20),
2676
- bgBlurRadius: Math.min(strength, 10),
2677
- isSelfieMode,
2678
- segmentationOptions: this.options.segmentationOptions,
2679
- };
2680
- }
2681
- async loadBackground(url) {
2682
- if (!url)
2683
- return null;
2684
- const result = await fetch(url, { signal: this.abortController.signal });
2685
- if (!result.ok) {
2686
- throw new Error(`[virtual-background] Failed to load background image: ${result.status} ${result.statusText}`);
2687
- }
2688
- return {
2689
- type: 'image',
2690
- media: await createImageBitmap(await result.blob()),
2691
- url,
2692
- };
2693
- }
2694
- onFlush() {
2695
- this.destroySegmenter();
2696
- }
2697
- onStop() {
2698
- this.webGlRenderer?.close();
2699
- this.destroySegmenter();
2700
- }
2701
- destroySegmenter() {
2702
- this.segmenter?.close();
2703
- this.segmenter = null;
2704
- this.isSegmenterReady = false;
2705
- }
2706
- get processorName() {
2707
- return 'background-processor';
2708
- }
2709
- }
2710
-
2711
- /**
2712
- * Simple WebGL renderer for full-screen Gaussian blur.
2713
- * Uses a two-pass separable Gaussian blur (horizontal then vertical).
2714
- * Optimized for moderation use cases by blurring at reduced resolution (15% scale)
2715
- * and upscaling back to full resolution for output.
2716
- */
2717
- class FullScreenBlurRenderer {
2718
- canvas;
2719
- gl;
2720
- blurProgramHandle;
2721
- blurLocations;
2722
- passthroughProgramHandle;
2723
- passthroughLocations;
2724
- positionBuffer;
2725
- texCoordBuffer;
2726
- pingTexture;
2727
- pongTexture;
2728
- pingFbo;
2729
- pongFbo;
2730
- inputTexture = null;
2731
- isRunning = false;
2732
- targetWidth = 0;
2733
- targetHeight = 0;
2734
- weightCache = new Map();
2735
- constructor(canvas) {
2736
- this.canvas = canvas;
2737
- const gl = canvas.getContext('webgl2', {
2738
- alpha: false,
2739
- antialias: false,
2740
- desynchronized: true,
2741
- });
2742
- if (!gl)
2743
- throw new Error('WebGL2 not supported');
2744
- this.gl = gl;
2745
- const vertexShaderSource = `#version 300 es
2746
- precision highp float;
2747
- in vec2 a_position;
2748
- in vec2 a_texCoord;
2749
- out vec2 v_texCoord;
2750
- void main() {
2751
- v_texCoord = a_texCoord;
2752
- gl_Position = vec4(a_position, 0.0, 1.0);
2753
- }
2754
- `;
2755
- const fragmentShaderSource = `#version 300 es
2756
- precision highp float;
2757
- in vec2 v_texCoord;
2758
- out vec4 outColor;
2759
- uniform sampler2D u_image;
2760
- uniform vec2 u_texelSize;
2761
- uniform vec2 u_direction;
2762
- uniform float u_weights[25];
2763
- void main() {
2764
- vec4 color = vec4(0.0);
2765
- for (int i = -12; i <= 12; i++) {
2766
- float w = u_weights[i + 12];
2767
- if (w == 0.0) continue;
2768
- vec2 offset = float(i) * u_direction * u_texelSize;
2769
- color += w * texture(u_image, v_texCoord + offset);
2770
- }
2771
- outColor = color;
2772
- }
2773
- `;
2774
- this.blurProgramHandle = this.createAndLinkProgram(vertexShaderSource, fragmentShaderSource);
2775
- const passthroughFragmentShaderSource = `#version 300 es
2776
- precision highp float;
2777
- in vec2 v_texCoord;
2778
- out vec4 outColor;
2779
- uniform sampler2D u_image;
2780
- void main() {
2781
- outColor = texture(u_image, v_texCoord);
2782
- }
2783
- `;
2784
- this.passthroughProgramHandle = this.createAndLinkProgram(vertexShaderSource, passthroughFragmentShaderSource);
2785
- const blurProgram = this.blurProgramHandle;
2786
- const passthroughProgram = this.passthroughProgramHandle;
2787
- this.blurLocations = {
2788
- positionLocation: gl.getAttribLocation(blurProgram, 'a_position'),
2789
- texCoordLocation: gl.getAttribLocation(blurProgram, 'a_texCoord'),
2790
- imageLocation: gl.getUniformLocation(blurProgram, 'u_image'),
2791
- texelSizeLocation: gl.getUniformLocation(blurProgram, 'u_texelSize'),
2792
- directionLocation: gl.getUniformLocation(blurProgram, 'u_direction'),
2793
- weightsLocation: gl.getUniformLocation(blurProgram, 'u_weights'),
2794
- };
2795
- this.passthroughLocations = {
2796
- positionLocation: gl.getAttribLocation(passthroughProgram, 'a_position'),
2797
- texCoordLocation: gl.getAttribLocation(passthroughProgram, 'a_texCoord'),
2798
- imageLocation: gl.getUniformLocation(passthroughProgram, 'u_image'),
2799
- };
2800
- this.positionBuffer = gl.createBuffer();
2801
- gl.bindBuffer(gl.ARRAY_BUFFER, this.positionBuffer);
2802
- gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([-1, -1, 1, -1, -1, 1, -1, 1, 1, -1, 1, 1]), gl.STATIC_DRAW);
2803
- this.texCoordBuffer = gl.createBuffer();
2804
- gl.bindBuffer(gl.ARRAY_BUFFER, this.texCoordBuffer);
2805
- gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([0, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 0]), gl.STATIC_DRAW);
2806
- const createTexture2D = () => {
2807
- const tex = gl.createTexture();
2808
- if (!tex)
2809
- throw new Error('Failed to create texture');
2810
- gl.bindTexture(gl.TEXTURE_2D, tex);
2811
- gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
2812
- gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
2813
- gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
2814
- gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR);
2815
- gl.bindTexture(gl.TEXTURE_2D, null);
2816
- return tex;
2817
- };
2818
- const createFramebufferForTexture = (tex) => {
2819
- const fb = gl.createFramebuffer();
2820
- if (!fb)
2821
- throw new Error('Failed to create framebuffer');
2822
- gl.bindFramebuffer(gl.FRAMEBUFFER, fb);
2823
- gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, tex, 0);
2824
- gl.bindFramebuffer(gl.FRAMEBUFFER, null);
2825
- return fb;
2826
- };
2827
- this.pingTexture = createTexture2D();
2828
- this.pongTexture = createTexture2D();
2829
- this.pingFbo = createFramebufferForTexture(this.pingTexture);
2830
- this.pongFbo = createFramebufferForTexture(this.pongTexture);
2831
- this.inputTexture = createTexture2D();
2832
- gl.useProgram(blurProgram);
2833
- gl.bindBuffer(gl.ARRAY_BUFFER, this.positionBuffer);
2834
- gl.enableVertexAttribArray(this.blurLocations.positionLocation);
2835
- gl.vertexAttribPointer(this.blurLocations.positionLocation, 2, gl.FLOAT, false, 0, 0);
2836
- gl.bindBuffer(gl.ARRAY_BUFFER, this.texCoordBuffer);
2837
- gl.enableVertexAttribArray(this.blurLocations.texCoordLocation);
2838
- gl.vertexAttribPointer(this.blurLocations.texCoordLocation, 2, gl.FLOAT, false, 0, 0);
2839
- if (this.blurLocations.imageLocation) {
2840
- gl.uniform1i(this.blurLocations.imageLocation, 0);
2841
- }
2842
- this.isRunning = true;
2843
- }
2844
- createAndLinkProgram(vsSource, fsSource) {
2845
- const gl = this.gl;
2846
- const vs = this.createShader(gl.VERTEX_SHADER, vsSource);
2847
- const fs = this.createShader(gl.FRAGMENT_SHADER, fsSource);
2848
- const prog = gl.createProgram();
2849
- if (!prog)
2850
- throw new Error('Failed to create program');
2851
- gl.attachShader(prog, vs);
2852
- gl.attachShader(prog, fs);
2853
- gl.linkProgram(prog);
2854
- if (!gl.getProgramParameter(prog, gl.LINK_STATUS)) {
2855
- throw new Error('Shader link failed: ' + gl.getProgramInfoLog(prog));
2856
- }
2857
- gl.deleteShader(vs);
2858
- gl.deleteShader(fs);
2859
- return prog;
2860
- }
2861
- createShader(type, source) {
2862
- const gl = this.gl;
2863
- const shader = gl.createShader(type);
2864
- if (!shader)
2865
- throw new Error('Failed to create shader');
2866
- gl.shaderSource(shader, source);
2867
- gl.compileShader(shader);
2868
- if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) {
2869
- throw new Error('Shader compile failed: ' + gl.getShaderInfoLog(shader));
2870
- }
2871
- return shader;
2872
- }
2873
- getGaussianWeights(radius) {
2874
- const r = Math.max(0, Math.min(radius | 0, 12));
2875
- const cached = this.weightCache.get(r);
2876
- if (cached)
2877
- return cached;
2878
- const weights = new Float32Array(25);
2879
- if (r === 0) {
2880
- weights[12] = 1.0;
2881
- this.weightCache.set(r, weights);
2882
- return weights;
2883
- }
2884
- const sigma = r * 0.6;
2885
- let sum = 0;
2886
- for (let i = -r; i <= r; i++) {
2887
- const w = Math.exp(-(i * i) / (2 * sigma * sigma));
2888
- weights[i + 12] = w;
2889
- sum += w;
2890
- }
2891
- for (let i = -r; i <= r; i++) {
2892
- weights[i + 12] /= sum;
2893
- }
2894
- this.weightCache.set(r, weights);
2895
- return weights;
2896
- }
2897
- render(frame, radius) {
2898
- if (!this.isRunning)
2899
- return;
2900
- const gl = this.gl;
2901
- const width = frame.displayWidth;
2902
- const height = frame.displayHeight;
2903
- if (!width || !height)
2904
- return;
2905
- if (this.canvas.width !== width || this.canvas.height !== height) {
2906
- this.canvas.width = width;
2907
- this.canvas.height = height;
2908
- }
2909
- const scale = 0.15;
2910
- const scaledWidth = Math.max(1, Math.floor(width * scale));
2911
- const scaledHeight = Math.max(1, Math.floor(height * scale));
2912
- if (scaledWidth !== this.targetWidth ||
2913
- scaledHeight !== this.targetHeight) {
2914
- this.targetWidth = scaledWidth;
2915
- this.targetHeight = scaledHeight;
2916
- gl.bindTexture(gl.TEXTURE_2D, this.pingTexture);
2917
- gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, scaledWidth, scaledHeight, 0, gl.RGBA, gl.UNSIGNED_BYTE, null);
2918
- gl.bindTexture(gl.TEXTURE_2D, this.pongTexture);
2919
- gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, scaledWidth, scaledHeight, 0, gl.RGBA, gl.UNSIGNED_BYTE, null);
2920
- gl.bindTexture(gl.TEXTURE_2D, null);
2921
- }
2922
- gl.activeTexture(gl.TEXTURE0);
2923
- gl.bindTexture(gl.TEXTURE_2D, this.inputTexture);
2924
- gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, frame);
2925
- gl.useProgram(this.blurProgramHandle);
2926
- if (this.blurLocations.texelSizeLocation) {
2927
- gl.uniform2f(this.blurLocations.texelSizeLocation, 1.0 / scaledWidth, 1.0 / scaledHeight);
2928
- }
2929
- const weights = this.getGaussianWeights(radius);
2930
- if (this.blurLocations.weightsLocation) {
2931
- gl.uniform1fv(this.blurLocations.weightsLocation, weights);
2932
- }
2933
- gl.viewport(0, 0, scaledWidth, scaledHeight);
2934
- gl.bindFramebuffer(gl.FRAMEBUFFER, this.pingFbo);
2935
- gl.bindTexture(gl.TEXTURE_2D, this.inputTexture);
2936
- if (this.blurLocations.directionLocation) {
2937
- gl.uniform2f(this.blurLocations.directionLocation, 1.0, 0.0);
2938
- }
2939
- gl.drawArrays(gl.TRIANGLES, 0, 6);
2940
- gl.bindFramebuffer(gl.FRAMEBUFFER, this.pongFbo);
2941
- gl.bindTexture(gl.TEXTURE_2D, this.pingTexture);
2942
- if (this.blurLocations.directionLocation) {
2943
- gl.uniform2f(this.blurLocations.directionLocation, 0.0, 1.0);
2944
- }
2945
- gl.drawArrays(gl.TRIANGLES, 0, 6);
2946
- gl.bindFramebuffer(gl.FRAMEBUFFER, null);
2947
- gl.viewport(0, 0, width, height);
2948
- gl.useProgram(this.passthroughProgramHandle);
2949
- gl.bindBuffer(gl.ARRAY_BUFFER, this.positionBuffer);
2950
- gl.enableVertexAttribArray(this.passthroughLocations.positionLocation);
2951
- gl.vertexAttribPointer(this.passthroughLocations.positionLocation, 2, gl.FLOAT, false, 0, 0);
2952
- gl.bindBuffer(gl.ARRAY_BUFFER, this.texCoordBuffer);
2953
- gl.enableVertexAttribArray(this.passthroughLocations.texCoordLocation);
2954
- gl.vertexAttribPointer(this.passthroughLocations.texCoordLocation, 2, gl.FLOAT, false, 0, 0);
2955
- gl.bindTexture(gl.TEXTURE_2D, this.pongTexture);
2956
- if (this.passthroughLocations.imageLocation) {
2957
- gl.uniform1i(this.passthroughLocations.imageLocation, 0);
2958
- }
2959
- gl.drawArrays(gl.TRIANGLES, 0, 6);
2960
- }
2961
- close() {
2962
- if (!this.isRunning)
2963
- return;
2964
- this.isRunning = false;
2965
- const gl = this.gl;
2966
- if (this.pingFbo)
2967
- gl.deleteFramebuffer(this.pingFbo);
2968
- if (this.pongFbo)
2969
- gl.deleteFramebuffer(this.pongFbo);
2970
- if (this.pingTexture)
2971
- gl.deleteTexture(this.pingTexture);
2972
- if (this.pongTexture)
2973
- gl.deleteTexture(this.pongTexture);
2974
- if (this.inputTexture)
2975
- gl.deleteTexture(this.inputTexture);
2976
- if (this.positionBuffer)
2977
- gl.deleteBuffer(this.positionBuffer);
2978
- if (this.texCoordBuffer)
2979
- gl.deleteBuffer(this.texCoordBuffer);
2980
- gl.deleteProgram(this.blurProgramHandle);
2981
- gl.deleteProgram(this.passthroughProgramHandle);
2982
- }
2983
- }
2984
-
2985
- /**
2986
- * A video filter that applies a full-screen blur to each frame.
2987
- *
2988
- * It uses a WebGL renderer to blur the incoming camera track and outputs
2989
- * a new track with the effect applied. Setup and frame handling are managed
2990
- * by the base processor.
2991
- */
2992
- class FullScreenBlur extends BaseVideoProcessor {
2993
- blurRenderer;
2994
- blurRadius;
2995
- /**
2996
- * Creates a new full-screen blur processor for the given video track.
2997
- *
2998
- * @param track - The input camera track to blur.
2999
- * @param options - Optional settings such as the blur radius.
3000
- * @param hooks - Optional callbacks for stats and error reporting.
3001
- */
3002
- constructor(track, options = {}, hooks = {}) {
3003
- super(track, hooks);
3004
- this.blurRadius = options.blurRadius ?? 6;
3005
- }
3006
- async initialize() {
3007
- this.blurRenderer = new FullScreenBlurRenderer(this.canvas);
3008
- }
3009
- async transform(frame) {
3010
- this.blurRenderer.render(frame, this.blurRadius);
3011
- return new VideoFrame(this.canvas, { timestamp: frame.timestamp });
3012
- }
3013
- onStop() {
3014
- this.blurRenderer?.close();
3015
- }
3016
- get processorName() {
3017
- return 'fullscreen-blur';
3018
- }
3019
- }
3020
-
3021
- export { BACKGROUND_BLUR_MAP, BaseVideoProcessor, FullScreenBlur, SegmentationLevel, VirtualBackground, createRenderer, isMediaPipePlatformSupported, isPlatformSupported, loadMediaPipe, loadTFLite };
3022
- //# sourceMappingURL=index.es.js.map