@pexip/media-processor 20.3.5 → 22.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (201) hide show
  1. package/CHANGELOG.md +109 -0
  2. package/README.md +17 -10
  3. package/api-docs/README.mdx +69 -23
  4. package/api-docs/functions/assertMediaStreamVideoTrack.mdx +13 -0
  5. package/api-docs/functions/createBenchmark.mdx +17 -5
  6. package/api-docs/functions/createFrameCallbackRequest.mdx +8 -2
  7. package/api-docs/functions/createMediaStreamTrackInputProcessAdaptor.mdx +10 -0
  8. package/api-docs/functions/createReadWritableStreamFallbackInputProcessAdaptor.mdx +17 -0
  9. package/api-docs/functions/createReadWritableStreamInputProcessAdaptor.mdx +10 -0
  10. package/api-docs/functions/createVideoProcessor.mdx +3 -3
  11. package/api-docs/functions/createWindowedStats.mdx +23 -0
  12. package/api-docs/functions/getDefaultInputProcessAdaptor.mdx +17 -0
  13. package/api-docs/functions/iou.mdx +19 -0
  14. package/api-docs/functions/isMediaStreamVideoTrack.mdx +13 -0
  15. package/api-docs/functions/labelComponents.mdx +20 -0
  16. package/api-docs/functions/nearestPowerOfTwo.mdx +21 -0
  17. package/api-docs/functions/playVideo.mdx +16 -0
  18. package/api-docs/functions/scoreComponent.mdx +22 -0
  19. package/api-docs/functions/setVideoElementSrc.mdx +14 -0
  20. package/api-docs/interfaces/Analyzer.mdx +4 -0
  21. package/api-docs/interfaces/Benchmark.mdx +26 -2
  22. package/api-docs/interfaces/FrameTransformStats.mdx +9 -0
  23. package/api-docs/interfaces/Gain.mdx +4 -0
  24. package/api-docs/interfaces/ImageSegmenterOptions.mdx +0 -2
  25. package/api-docs/interfaces/InputProcessAdaptor.mdx +28 -0
  26. package/api-docs/interfaces/Process.mdx +8 -3
  27. package/api-docs/interfaces/ProcessEventMute.mdx +6 -0
  28. package/api-docs/interfaces/ProcessWorkerEventOpened.mdx +7 -0
  29. package/api-docs/interfaces/ProcessWorkerEventStats.mdx +6 -0
  30. package/api-docs/interfaces/ProcessWorkerEventUpdated.mdx +7 -0
  31. package/api-docs/interfaces/ProcessorEvent.mdx +1 -1
  32. package/api-docs/interfaces/ProcessorOptions.mdx +0 -1
  33. package/api-docs/interfaces/ProcessorProcessOptions.mdx +3 -1
  34. package/api-docs/interfaces/ProcessorUpdateOptions.mdx +15 -0
  35. package/api-docs/interfaces/ReadWritableStream.mdx +12 -0
  36. package/api-docs/interfaces/Renderer.mdx +2 -2
  37. package/api-docs/interfaces/RendererOptions.mdx +25 -12
  38. package/api-docs/interfaces/Segmenter.mdx +89 -10
  39. package/api-docs/interfaces/SegmenterOptions.mdx +0 -1
  40. package/api-docs/interfaces/SelectionOptions.mdx +15 -0
  41. package/api-docs/interfaces/Stats.mdx +11 -0
  42. package/api-docs/interfaces/VideoProcessor.mdx +49 -5
  43. package/api-docs/interfaces/Weights.mdx +12 -0
  44. package/api-docs/type-aliases/OptionalKeys.mdx +10 -0
  45. package/api-docs/type-aliases/ProcessEvents.mdx +1 -0
  46. package/api-docs/type-aliases/ProcessInputType.mdx +3 -1
  47. package/api-docs/type-aliases/ProcessWorkerEvents.mdx +6 -4
  48. package/api-docs/type-aliases/ProcessorWorkerEvents.mdx +1 -1
  49. package/api-docs/type-aliases/Struct.mdx +9 -0
  50. package/api-docs/variables/BACKGROUND_THRESHOLD.mdx +3 -0
  51. package/api-docs/variables/DOWN_SAMPLE_FACTOR.mdx +3 -0
  52. package/api-docs/variables/EDGE_BLUR_AMOUNT.mdx +1 -1
  53. package/api-docs/variables/EXCLUDE_BYSTANDERS.mdx +3 -0
  54. package/api-docs/variables/FOREGROUND_THRESHOLD.mdx +1 -1
  55. package/api-docs/variables/FRAME_RATE.mdx +1 -1
  56. package/api-docs/variables/ImageFetchError.mdx +3 -0
  57. package/api-docs/variables/LIGHT_WRAP_BLUR_AMOUNT.mdx +3 -0
  58. package/api-docs/variables/LIGHT_WRAP_EDGE_BAND.mdx +3 -0
  59. package/api-docs/variables/LIGHT_WRAP_INTENSITY.mdx +3 -0
  60. package/api-docs/variables/LIGHT_WRAP_TIGHTNESS.mdx +3 -0
  61. package/api-docs/variables/MASK_COMBINE_RATIO.mdx +1 -1
  62. package/api-docs/variables/MORPH_DILATE_RADIUS_PX.mdx +3 -0
  63. package/api-docs/variables/MORPH_ERODE_RADIUS_PX.mdx +3 -0
  64. package/api-docs/variables/MORPH_PASS.mdx +3 -0
  65. package/api-docs/variables/MediaStreamTrackTransferable.mdx +5 -0
  66. package/api-docs/variables/PERSON_CENTER.mdx +3 -0
  67. package/api-docs/variables/RENDERER_OPTIONS_STRUCT.mdx +3 -0
  68. package/api-docs/variables/SIGMA_RANGE.mdx +3 -0
  69. package/api-docs/variables/SIGMA_SPACE.mdx +3 -0
  70. package/api-docs/variables/STRONG_EDGE_BLUR_AMOUNT.mdx +3 -0
  71. package/api-docs/variables/cloneImageRecord.mdx +19 -0
  72. package/api-docs/variables/compareImageRecords.mdx +18 -0
  73. package/api-docs/variables/createRemoteImageBitmap.mdx +1 -1
  74. package/api-docs/variables/getCanvasRenderingContext2D.mdx +14 -0
  75. package/api-docs/variables/getImageSize.mdx +13 -0
  76. package/api-docs/variables/isRenderOptionsKey.mdx +13 -0
  77. package/api-docs/variables/isValidGpuAPI.mdx +13 -0
  78. package/dist/common/backends/webgl/binaryMask.d.ts +8 -0
  79. package/dist/common/backends/webgl/binaryMask.js +95 -0
  80. package/dist/common/backends/webgl/blender.d.ts +2 -1
  81. package/dist/common/backends/webgl/blender.js +18 -15
  82. package/dist/common/backends/webgl/blur.d.ts +1 -1
  83. package/dist/common/backends/webgl/blur.js +23 -16
  84. package/dist/common/backends/webgl/canvas.js +9 -4
  85. package/dist/common/backends/webgl/diffAbsMask.d.ts +10 -0
  86. package/dist/common/backends/webgl/diffAbsMask.js +103 -0
  87. package/dist/common/backends/webgl/geodesicReconstruction.d.ts +13 -0
  88. package/dist/common/backends/webgl/geodesicReconstruction.js +176 -0
  89. package/dist/common/backends/webgl/jointBilateralFilter.d.ts +12 -0
  90. package/dist/common/backends/webgl/jointBilateralFilter.js +108 -0
  91. package/dist/common/backends/webgl/morphErode.d.ts +13 -0
  92. package/dist/common/backends/webgl/morphErode.js +121 -0
  93. package/dist/common/backends/webgl/renderer.js +194 -44
  94. package/dist/common/backends/webgl/replace.d.ts +17 -0
  95. package/dist/common/backends/webgl/replace.js +126 -0
  96. package/dist/common/backends/webgl/selectComponent.d.ts +15 -0
  97. package/dist/common/backends/webgl/selectComponent.js +102 -0
  98. package/dist/common/backends/webgl/smoothingMask.d.ts +3 -3
  99. package/dist/common/backends/webgl/smoothingMask.js +22 -21
  100. package/dist/common/backends/webgl/tentBlur.d.ts +9 -0
  101. package/dist/common/backends/webgl/tentBlur.js +118 -0
  102. package/dist/common/backends/webgl/texture.d.ts +2 -3
  103. package/dist/common/backends/webgl/texture.js +21 -19
  104. package/dist/common/backends/webgl/textureToTexture.d.ts +1 -1
  105. package/dist/common/backends/webgl/textureToTexture.js +19 -16
  106. package/dist/common/backends/webgl/webglUtils.d.ts +7 -1
  107. package/dist/common/backends/webgl/webglUtils.js +72 -10
  108. package/dist/common/backends/webgpu/blender.d.ts +6 -1
  109. package/dist/common/backends/webgpu/blender.js +96 -51
  110. package/dist/common/backends/webgpu/constants.js +1 -1
  111. package/dist/common/backends/webgpu/dualFilterBlur.d.ts +2 -3
  112. package/dist/common/backends/webgpu/dualFilterBlur.js +146 -140
  113. package/dist/common/backends/webgpu/jointBilateralFilter.d.ts +7 -0
  114. package/dist/common/backends/webgpu/jointBilateralFilter.js +162 -0
  115. package/dist/common/backends/webgpu/renderer.js +205 -43
  116. package/dist/common/backends/webgpu/replace.d.ts +11 -0
  117. package/dist/common/backends/webgpu/replace.js +175 -0
  118. package/dist/common/backends/webgpu/resize.d.ts +4 -0
  119. package/dist/common/backends/webgpu/resize.js +76 -0
  120. package/dist/common/backends/webgpu/smoothingMask.d.ts +2 -2
  121. package/dist/common/backends/webgpu/smoothingMask.js +74 -40
  122. package/dist/common/backends/webgpu/tentBlur.d.ts +5 -0
  123. package/dist/common/backends/webgpu/tentBlur.js +131 -0
  124. package/dist/common/backends/webgpu/texture.d.ts +2 -3
  125. package/dist/common/backends/webgpu/texture.js +29 -20
  126. package/dist/common/backends/webgpu/textureToTexture.d.ts +1 -1
  127. package/dist/common/backends/webgpu/textureToTexture.js +28 -15
  128. package/dist/common/backends/webgpu/types.d.ts +2 -0
  129. package/dist/common/backends/webgpu/webgpuUtils.d.ts +24 -4
  130. package/dist/common/backends/webgpu/webgpuUtils.js +60 -11
  131. package/dist/common/constants.d.ts +22 -5
  132. package/dist/common/constants.js +23 -5
  133. package/dist/common/index.d.ts +2 -0
  134. package/dist/common/index.js +2 -0
  135. package/dist/common/tsconfig.tsbuildinfo +1 -1
  136. package/dist/common/typeGuards.d.ts +2 -0
  137. package/dist/common/typeGuards.js +7 -0
  138. package/dist/common/types/media.d.ts +5 -1
  139. package/dist/common/types/messageEvents.d.ts +31 -4
  140. package/dist/common/types/processor.d.ts +6 -4
  141. package/dist/common/types/render.d.ts +147 -15
  142. package/dist/common/types/render.js +23 -1
  143. package/dist/common/types/segmentation.d.ts +0 -12
  144. package/dist/common/types/utils.d.ts +21 -0
  145. package/dist/common/utils.d.ts +29 -2
  146. package/dist/common/utils.js +206 -10
  147. package/dist/main/audio.js +3 -3
  148. package/dist/main/benchUtils.d.ts +35 -3
  149. package/dist/main/benchUtils.js +149 -15
  150. package/dist/main/index.d.ts +1 -1
  151. package/dist/main/index.js +1 -1
  152. package/dist/main/tsconfig.tsbuildinfo +1 -1
  153. package/dist/main/types.d.ts +0 -6
  154. package/dist/main/utils.d.ts +4 -0
  155. package/dist/main/utils.js +31 -0
  156. package/dist/main/video/constants.d.ts +1 -4
  157. package/dist/main/video/constants.js +1 -5
  158. package/dist/main/video/index.d.ts +3 -4
  159. package/dist/main/video/index.js +3 -4
  160. package/dist/main/video/inputProcessAdaptor.d.ts +30 -0
  161. package/dist/main/video/inputProcessAdaptor.js +198 -0
  162. package/dist/main/video/segmenter.d.ts +35 -0
  163. package/dist/main/video/{segmenters/mediapipe.js → segmenter.js} +93 -53
  164. package/dist/main/video/typeGuards.d.ts +2 -0
  165. package/dist/main/video/typeGuards.js +8 -1
  166. package/dist/main/video/types.d.ts +36 -16
  167. package/dist/main/video/utils.d.ts +5 -4
  168. package/dist/main/video/utils.js +49 -44
  169. package/dist/main/video/video.d.ts +29 -7
  170. package/dist/main/video/video.js +323 -28
  171. package/dist/workers/mediaWorker.js +35 -162
  172. package/dist/workers/tsconfig.tsbuildinfo +1 -1
  173. package/dist/worklets/tsconfig.tsbuildinfo +1 -1
  174. package/package.json +13 -12
  175. package/api-docs/functions/createCanvasTransform.mdx +0 -14
  176. package/api-docs/functions/createVideoTrackProcessor.mdx +0 -9
  177. package/api-docs/functions/createVideoTrackProcessorWithFallback.mdx +0 -15
  178. package/api-docs/interfaces/Detector.mdx +0 -81
  179. package/api-docs/interfaces/SegmentationParams.mdx +0 -18
  180. package/api-docs/interfaces/Transform.mdx +0 -62
  181. package/api-docs/type-aliases/ProcessVideoTrack.mdx +0 -15
  182. package/api-docs/type-aliases/SegmentationTransform.mdx +0 -3
  183. package/dist/common/backends/webgl/shaders/blendFragmentShader.d.ts +0 -1
  184. package/dist/common/backends/webgl/shaders/blendFragmentShader.js +0 -21
  185. package/dist/common/backends/webgl/shaders/dualFilterBlurFragmentShader.d.ts +0 -1
  186. package/dist/common/backends/webgl/shaders/dualFilterBlurFragmentShader.js +0 -45
  187. package/dist/common/backends/webgl/shaders/smoothingMaskFragmentShader.d.ts +0 -1
  188. package/dist/common/backends/webgl/shaders/smoothingMaskFragmentShader.js +0 -57
  189. package/dist/common/backends/webgl/shaders/textureFragmentShader.d.ts +0 -1
  190. package/dist/common/backends/webgl/shaders/textureFragmentShader.js +0 -16
  191. package/dist/common/backends/webgl/shaders/vertexShader.d.ts +0 -1
  192. package/dist/common/backends/webgl/shaders/vertexShader.js +0 -19
  193. package/dist/main/transformer.d.ts +0 -1
  194. package/dist/main/transformer.js +0 -4
  195. package/dist/main/video/canvasTransform.d.ts +0 -17
  196. package/dist/main/video/canvasTransform.js +0 -253
  197. package/dist/main/video/segmenters/mediapipe.d.ts +0 -22
  198. package/dist/main/video/transformer.d.ts +0 -9
  199. package/dist/main/video/transformer.js +0 -68
  200. package/dist/main/video/videoStreamTrackProcessor.d.ts +0 -20
  201. package/dist/main/video/videoStreamTrackProcessor.js +0 -86
@@ -0,0 +1,103 @@
1
+ import { assert } from '@pexip/utils';
2
+ import { m3 } from '../../m3';
3
+ import { RENDERING_EVENTS } from '../../constants';
4
+ import { createGLProgram, createMaskTexture, createFrameBuffer, } from './webglUtils';
5
+ import vertexSrc from './shaders/matrix.vert';
6
+ import fragmentSrc from './shaders/diffAbs.frag';
7
+ /**
8
+ * Create absulute difference renderer to caculate the difference between 2 textures
9
+ */
10
+ export const createDiffAbsRenderer = (processor, width, height) => {
11
+ const { gl, program, attribLocations, uniformLocations, floatSingleChannelSupported, floatFullChannelSupported, release, } = createGLProgram(processor, {
12
+ attribs: {
13
+ position: 'a_position',
14
+ texCoord: 'a_texCoord',
15
+ },
16
+ uniforms: {
17
+ matrix: 'u_matrix',
18
+ texA: 'u_tex_a',
19
+ texB: 'u_tex_b',
20
+ size: 'u_size',
21
+ },
22
+ vertexSrc,
23
+ fragmentSrc,
24
+ });
25
+ const vao = gl.createVertexArray();
26
+ gl.bindVertexArray(vao);
27
+ // Position
28
+ const positionBuffer = gl.createBuffer();
29
+ gl.enableVertexAttribArray(attribLocations.position);
30
+ gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
31
+ gl.vertexAttribPointer(attribLocations.position, 2, gl.FLOAT, false, 0, 0);
32
+ gl.bufferData(gl.ARRAY_BUFFER,
33
+ // prettier-ignore
34
+ new Float32Array([
35
+ 0, 0,
36
+ 0, 1,
37
+ 1, 0,
38
+ 1, 0,
39
+ 0, 1,
40
+ 1, 1,
41
+ ]), gl.STATIC_DRAW);
42
+ // Texture coordinates for the rectangle
43
+ const texCoordBuffer = gl.createBuffer();
44
+ gl.bindBuffer(gl.ARRAY_BUFFER, texCoordBuffer);
45
+ gl.bufferData(gl.ARRAY_BUFFER,
46
+ // prettier-ignore
47
+ new Float32Array([
48
+ 0, 0,
49
+ 0, 1,
50
+ 1, 0,
51
+ 1, 0,
52
+ 0, 1,
53
+ 1, 1,
54
+ ]), gl.STATIC_DRAW);
55
+ gl.enableVertexAttribArray(attribLocations.texCoord);
56
+ gl.vertexAttribPointer(attribLocations.texCoord, 2, gl.FLOAT, false, 0, 0);
57
+ const outputTexturePing = createMaskTexture(gl, width, height, true, floatSingleChannelSupported, floatFullChannelSupported);
58
+ const frameBufferPing = createFrameBuffer(gl, outputTexturePing);
59
+ return {
60
+ render: (textureA, textureB) => {
61
+ assert(!gl.isContextLost(), RENDERING_EVENTS.ContextLost);
62
+ assert(textureA.width === textureB.width &&
63
+ textureA.height === textureB.height);
64
+ gl.useProgram(program);
65
+ gl.bindVertexArray(vao);
66
+ gl.uniform2i(uniformLocations.size, textureA.width, textureA.height);
67
+ gl.activeTexture(gl.TEXTURE0);
68
+ gl.uniform1i(uniformLocations.texA, 0);
69
+ gl.bindTexture(gl.TEXTURE_2D, textureA.texture);
70
+ gl.activeTexture(gl.TEXTURE1);
71
+ gl.uniform1i(uniformLocations.texB, 1);
72
+ gl.bindTexture(gl.TEXTURE_2D, textureB.texture);
73
+ // No Flip Y Coordinate for drawing on the frame buffer
74
+ // Project, position and scale to full-size of the canvas
75
+ const matrix = m3.projection(1, 1, 1);
76
+ gl.uniformMatrix3fv(uniformLocations.matrix, false, matrix);
77
+ gl.bindFramebuffer(gl.FRAMEBUFFER, frameBufferPing);
78
+ gl.viewport(0, 0, width, height);
79
+ gl.clearColor(0, 0, 0, 0);
80
+ gl.clear(gl.COLOR_BUFFER_BIT);
81
+ gl.drawArrays(gl.TRIANGLES, 0, 6);
82
+ // Cleanup
83
+ gl.bindBuffer(gl.ARRAY_BUFFER, null);
84
+ gl.bindFramebuffer(gl.FRAMEBUFFER, null);
85
+ gl.bindTexture(gl.TEXTURE_2D, null);
86
+ gl.bindVertexArray(null);
87
+ return {
88
+ texture: outputTexturePing,
89
+ frameBuffer: frameBufferPing,
90
+ width,
91
+ height,
92
+ };
93
+ },
94
+ release: () => {
95
+ gl.deleteBuffer(positionBuffer);
96
+ gl.deleteBuffer(texCoordBuffer);
97
+ gl.deleteTexture(outputTexturePing);
98
+ gl.deleteFramebuffer(frameBufferPing);
99
+ gl.deleteVertexArray(vao);
100
+ release();
101
+ },
102
+ };
103
+ };
@@ -0,0 +1,13 @@
1
+ import type { RendererOptions } from '../../types/render';
2
+ import type { TextureInfo } from './types';
3
+ /**
4
+ * A Geodesic Reconstruction Renderer that performs geodesic dilation of
5
+ * a mask constrained by a reference mask.
6
+ */
7
+ export declare const createGeodesicReconstructionRenderer: (processor: OffscreenCanvas, width: number, height: number) => {
8
+ render: ({ mask, maskRef }: {
9
+ mask: TextureInfo;
10
+ maskRef: TextureInfo;
11
+ }, options: Pick<RendererOptions, "morphDilateRadiusPx" | "morphPass">) => TextureInfo;
12
+ release: () => void;
13
+ };
@@ -0,0 +1,176 @@
1
+ import { assert } from '@pexip/utils';
2
+ import { m3 } from '../../m3';
3
+ import { RENDERING_EVENTS } from '../../constants';
4
+ import { clamping } from '../../utils';
5
+ import { createGLProgram, createMaskTexture, createFrameBuffer, } from './webglUtils';
6
+ import vertexSrc from './shaders/matrix.vert';
7
+ import fragmentSrc from './shaders/dilateClipToMask.frag';
8
+ import { createDiffAbsRenderer } from './diffAbsMask';
9
+ // Convergence threshold used when testing for geodesic reconstruction stability.
10
+ const CONVERGENCE_MAX_ITERATIONS = 64;
11
+ const MAX_RADIUS_PX = 64;
12
+ const clampRadiusPx = clamping(0, MAX_RADIUS_PX);
13
+ /**
14
+ * A Geodesic Reconstruction Renderer that performs geodesic dilation of
15
+ * a mask constrained by a reference mask.
16
+ */
17
+ export const createGeodesicReconstructionRenderer = (processor, width, height) => {
18
+ const { gl, program, attribLocations, uniformLocations, floatSingleChannelSupported, floatFullChannelSupported, release, } = createGLProgram(processor, {
19
+ attribs: {
20
+ position: 'a_position',
21
+ texCoord: 'a_texCoord',
22
+ },
23
+ uniforms: {
24
+ matrix: 'u_matrix',
25
+ mask: 'u_mask',
26
+ maskRef: 'u_ref_mask',
27
+ texelSize: 'u_texel_size',
28
+ radiusPx: 'u_radius_px',
29
+ direction: 'u_direction',
30
+ },
31
+ vertexSrc,
32
+ fragmentSrc,
33
+ });
34
+ const vao = gl.createVertexArray();
35
+ gl.bindVertexArray(vao);
36
+ // Position
37
+ const positionBuffer = gl.createBuffer();
38
+ gl.enableVertexAttribArray(attribLocations.position);
39
+ gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
40
+ gl.vertexAttribPointer(attribLocations.position, 2, gl.FLOAT, false, 0, 0);
41
+ gl.bufferData(gl.ARRAY_BUFFER,
42
+ // prettier-ignore
43
+ new Float32Array([
44
+ 0, 0,
45
+ 0, 1,
46
+ 1, 0,
47
+ 1, 0,
48
+ 0, 1,
49
+ 1, 1,
50
+ ]), gl.STATIC_DRAW);
51
+ // Texture coordinates for the rectangle
52
+ const texCoordBuffer = gl.createBuffer();
53
+ gl.bindBuffer(gl.ARRAY_BUFFER, texCoordBuffer);
54
+ gl.bufferData(gl.ARRAY_BUFFER,
55
+ // prettier-ignore
56
+ new Float32Array([
57
+ 0, 0,
58
+ 0, 1,
59
+ 1, 0,
60
+ 1, 0,
61
+ 0, 1,
62
+ 1, 1,
63
+ ]), gl.STATIC_DRAW);
64
+ gl.enableVertexAttribArray(attribLocations.texCoord);
65
+ gl.vertexAttribPointer(attribLocations.texCoord, 2, gl.FLOAT, false, 0, 0);
66
+ const scaleFactor = processor.width / width;
67
+ const MIN_CCL_WIDTH = 32;
68
+ const MIN_CCL_HEIGHT = 18;
69
+ const convergenceWidth = Math.max(Math.floor(width / 4), MIN_CCL_WIDTH);
70
+ const convergenceHeight = Math.max(Math.floor(height / 4), MIN_CCL_HEIGHT);
71
+ const CONVERGENCE_EPSILON = convergenceWidth === MIN_CCL_WIDTH ? 1e-8 : 1e-4;
72
+ const convergenceSize = convergenceWidth * convergenceHeight;
73
+ const diffRenderer = createDiffAbsRenderer(processor, convergenceWidth, convergenceHeight);
74
+ const supportFloat = floatSingleChannelSupported || floatFullChannelSupported;
75
+ const readPixelsFormat = supportFloat ? gl.RGBA : gl.RED;
76
+ const readPixelsType = supportFloat ? gl.FLOAT : gl.UNSIGNED_BYTE;
77
+ const convergenceDiff = supportFloat
78
+ ? new Float32Array(convergenceSize * 4)
79
+ : new Uint8Array(convergenceSize);
80
+ const outputTexturePing = createMaskTexture(gl, width, height, false, floatSingleChannelSupported, floatFullChannelSupported);
81
+ const outputTexturePong = createMaskTexture(gl, width, height, false, floatSingleChannelSupported, floatFullChannelSupported);
82
+ const frameBufferPing = createFrameBuffer(gl, outputTexturePing);
83
+ const frameBufferPong = createFrameBuffer(gl, outputTexturePong);
84
+ return {
85
+ render: ({ mask, maskRef }, options) => {
86
+ assert(!gl.isContextLost(), RENDERING_EVENTS.ContextLost);
87
+ let write = frameBufferPing;
88
+ let read = mask.texture;
89
+ const dilate = () => {
90
+ gl.useProgram(program);
91
+ gl.bindVertexArray(vao);
92
+ gl.uniform1i(uniformLocations.radiusPx, clampRadiusPx(Math.floor(options.morphDilateRadiusPx / scaleFactor)));
93
+ gl.uniform2f(uniformLocations.texelSize, 1 / mask.width, 1 / mask.height);
94
+ // Reference Mask
95
+ gl.activeTexture(gl.TEXTURE1);
96
+ gl.uniform1i(uniformLocations.maskRef, 1);
97
+ gl.bindTexture(gl.TEXTURE_2D, maskRef.texture);
98
+ // No Flip Y Coordinate for drawing on the frame buffer
99
+ // Project, position and scale to full-size of the canvas
100
+ const matrix = m3.projection(1, 1, 1);
101
+ gl.uniformMatrix3fv(uniformLocations.matrix, false, matrix);
102
+ // Separable
103
+ for (let j = 0; j < 2; j++) {
104
+ // Textures
105
+ gl.activeTexture(gl.TEXTURE0);
106
+ gl.uniform1i(uniformLocations.mask, 0);
107
+ gl.bindTexture(gl.TEXTURE_2D, read);
108
+ // Horizontal then Vertical
109
+ gl.uniform2f(uniformLocations.direction, 1.0 - j, 0.0 + j);
110
+ gl.bindFramebuffer(gl.FRAMEBUFFER, write);
111
+ gl.viewport(0, 0, width, height);
112
+ gl.drawArrays(gl.TRIANGLES, 0, 6);
113
+ // Swap
114
+ [read, write] =
115
+ write === frameBufferPing
116
+ ? [outputTexturePing, frameBufferPong]
117
+ : [outputTexturePong, frameBufferPing];
118
+ }
119
+ };
120
+ for (let i = 0; i < options.morphPass; i++) {
121
+ dilate();
122
+ }
123
+ let convergent = false;
124
+ let count = 0;
125
+ while (!convergent) {
126
+ dilate();
127
+ const diff = diffRenderer.render({ texture: outputTexturePing, width, height }, { texture: outputTexturePong, width, height });
128
+ gl.bindFramebuffer(gl.FRAMEBUFFER, diff.frameBuffer);
129
+ gl.readPixels(0, 0, convergenceWidth, convergenceHeight, readPixelsFormat, readPixelsType, convergenceDiff);
130
+ let sum = 0;
131
+ let meanDiff = 0;
132
+ if (supportFloat) {
133
+ for (let i = 0, j = 0; i < convergenceSize; ++i, j += 4) {
134
+ //@ts-expect-error will not be undefined
135
+ sum += convergenceDiff[j];
136
+ }
137
+ meanDiff = sum / convergenceSize;
138
+ }
139
+ else {
140
+ for (let i = 0; i < convergenceSize; ++i) {
141
+ //@ts-expect-error will not be undefined
142
+ sum += convergenceDiff[i];
143
+ }
144
+ meanDiff = sum / convergenceSize / 255.0;
145
+ }
146
+ gl.bindFramebuffer(gl.FRAMEBUFFER, null);
147
+ if (meanDiff < CONVERGENCE_EPSILON ||
148
+ count >= CONVERGENCE_MAX_ITERATIONS) {
149
+ // console.log(
150
+ // `Geodesic reconstruction converged in ${count} iterations with ${meanDiff}`,
151
+ // {convergenceWidth, convergenceHeight, width, height},
152
+ // );
153
+ convergent = true;
154
+ }
155
+ count++;
156
+ }
157
+ // Cleanup
158
+ gl.bindBuffer(gl.ARRAY_BUFFER, null);
159
+ gl.bindFramebuffer(gl.FRAMEBUFFER, null);
160
+ gl.bindTexture(gl.TEXTURE_2D, null);
161
+ gl.bindVertexArray(null);
162
+ return { texture: read, width, height };
163
+ },
164
+ release: () => {
165
+ gl.deleteBuffer(positionBuffer);
166
+ gl.deleteBuffer(texCoordBuffer);
167
+ gl.deleteTexture(outputTexturePing);
168
+ gl.deleteTexture(outputTexturePong);
169
+ gl.deleteFramebuffer(frameBufferPing);
170
+ gl.deleteFramebuffer(frameBufferPong);
171
+ gl.deleteVertexArray(vao);
172
+ diffRenderer.release();
173
+ release();
174
+ },
175
+ };
176
+ };
@@ -0,0 +1,12 @@
1
+ import type { RendererOptions } from '../../types/render';
2
+ import type { TextureInfo } from './types';
3
+ /**
4
+ * Joint Bilateral Filter
5
+ */
6
+ export declare const createJointBilateralFilterRenderer: (processor: OffscreenCanvas, width: number, height: number) => {
7
+ render: ({ input, guide }: {
8
+ input: TextureInfo;
9
+ guide: TextureInfo;
10
+ }, options: RendererOptions) => TextureInfo;
11
+ release: () => void;
12
+ };
@@ -0,0 +1,108 @@
1
+ import { assert } from '@pexip/utils';
2
+ import { RENDERING_EVENTS } from '../../constants';
3
+ import { m3 } from '../../m3';
4
+ import { createGLProgram, createMaskTexture, createFrameBuffer, } from './webglUtils';
5
+ import vertexSrc from './shaders/matrix.vert';
6
+ import fragmentSrc from './shaders/jointBilateralFilter.frag';
7
+ /**
8
+ * Joint Bilateral Filter
9
+ */
10
+ export const createJointBilateralFilterRenderer = (processor, width, height) => {
11
+ const { gl, program, attribLocations, uniformLocations, release } = createGLProgram(processor, {
12
+ attribs: {
13
+ position: 'a_position',
14
+ texCoord: 'a_texCoord',
15
+ },
16
+ uniforms: {
17
+ frameGuide: 'u_frame_guide',
18
+ frameInput: 'u_frame_input',
19
+ matrix: 'u_matrix',
20
+ texelSize: 'u_texel_size',
21
+ sigmaSpace: 'u_sigma_space',
22
+ sigmaRangeLo: 'u_sigma_range_lo',
23
+ sigmaRangeHi: 'u_sigma_range_hi',
24
+ foregroundThreshold: 'u_foreground_threshold',
25
+ backgroundThreshold: 'u_background_threshold',
26
+ },
27
+ vertexSrc,
28
+ fragmentSrc,
29
+ });
30
+ gl.enable(gl.BLEND);
31
+ const vao = gl.createVertexArray();
32
+ gl.bindVertexArray(vao);
33
+ // Position
34
+ const positionBuffer = gl.createBuffer();
35
+ gl.enableVertexAttribArray(attribLocations.position);
36
+ gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
37
+ gl.vertexAttribPointer(attribLocations.position, 2, gl.FLOAT, false, 0, 0);
38
+ gl.bufferData(gl.ARRAY_BUFFER,
39
+ // prettier-ignore
40
+ new Float32Array([
41
+ 0, 0,
42
+ 0, 1,
43
+ 1, 0,
44
+ 1, 0,
45
+ 0, 1,
46
+ 1, 1,
47
+ ]), gl.STATIC_DRAW);
48
+ // Texture coordinates for the rectangle
49
+ const texCoordBuffer = gl.createBuffer();
50
+ gl.bindBuffer(gl.ARRAY_BUFFER, texCoordBuffer);
51
+ gl.bufferData(gl.ARRAY_BUFFER,
52
+ // prettier-ignore
53
+ new Float32Array([
54
+ 0, 0,
55
+ 0, 1,
56
+ 1, 0,
57
+ 1, 0,
58
+ 0, 1,
59
+ 1, 1,
60
+ ]), gl.STATIC_DRAW);
61
+ gl.enableVertexAttribArray(attribLocations.texCoord);
62
+ gl.vertexAttribPointer(attribLocations.texCoord, 2, gl.FLOAT, false, 0, 0);
63
+ const outputTexture = createMaskTexture(gl, width, height);
64
+ const frameBuffer = createFrameBuffer(gl, outputTexture);
65
+ const scaleFactor = processor.width / width;
66
+ return {
67
+ render: ({ input, guide }, options) => {
68
+ assert(!gl.isContextLost(), RENDERING_EVENTS.ContextLost);
69
+ assert(options.foregroundThreshold >= options.backgroundThreshold, 'foregroundThreshold must be >= backgroundThreshold');
70
+ gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer);
71
+ gl.viewport(0, 0, width, height);
72
+ gl.useProgram(program);
73
+ gl.bindVertexArray(vao);
74
+ // Textures
75
+ gl.activeTexture(gl.TEXTURE0);
76
+ gl.uniform1i(uniformLocations.frameInput, 0);
77
+ gl.bindTexture(gl.TEXTURE_2D, input.texture);
78
+ gl.activeTexture(gl.TEXTURE1);
79
+ gl.uniform1i(uniformLocations.frameGuide, 1);
80
+ gl.bindTexture(gl.TEXTURE_2D, guide.texture);
81
+ gl.uniform2f(uniformLocations.texelSize, 1.0 / input.width, 1.0 / input.height);
82
+ gl.uniform1f(uniformLocations.sigmaSpace, options.sigmaSpace / scaleFactor);
83
+ gl.uniform1f(uniformLocations.sigmaRangeLo, options.sigmaRangeLo);
84
+ gl.uniform1f(uniformLocations.sigmaRangeHi, options.sigmaRangeHi);
85
+ gl.uniform1f(uniformLocations.backgroundThreshold, options.backgroundThreshold);
86
+ gl.uniform1f(uniformLocations.foregroundThreshold, options.foregroundThreshold);
87
+ // No Flip Y Coordinate for drawing on the frame buffer
88
+ // Project, position and scale to full-size of the canvas
89
+ const matrix = m3.projection(1, 1, 1);
90
+ gl.uniformMatrix3fv(uniformLocations.matrix, false, matrix);
91
+ gl.drawArrays(gl.TRIANGLES, 0, 6);
92
+ // Cleanup
93
+ gl.bindBuffer(gl.ARRAY_BUFFER, null);
94
+ gl.bindFramebuffer(gl.FRAMEBUFFER, null);
95
+ gl.bindTexture(gl.TEXTURE_2D, null);
96
+ gl.bindVertexArray(null);
97
+ return { texture: outputTexture, width, height };
98
+ },
99
+ release: () => {
100
+ gl.deleteBuffer(positionBuffer);
101
+ gl.deleteBuffer(texCoordBuffer);
102
+ gl.deleteTexture(outputTexture);
103
+ gl.deleteFramebuffer(frameBuffer);
104
+ gl.deleteVertexArray(vao);
105
+ release();
106
+ },
107
+ };
108
+ };
@@ -0,0 +1,13 @@
1
+ import type { RendererOptions } from '../../types/render';
2
+ import type { TextureInfo } from './types';
3
+ /**
4
+ * Create a morphological erosion renderer that performs erosion of a mask with a
5
+ * given radius and number of passes.
6
+ * The erosion is implemented as a separable filter.
7
+ */
8
+ export declare const createMorphErodeRenderer: (processor: OffscreenCanvas, width: number, height: number) => {
9
+ render: ({ mask }: {
10
+ mask: TextureInfo;
11
+ }, options: Pick<RendererOptions, "morphPass" | "morphErodeRadiusPx">) => TextureInfo;
12
+ release: () => void;
13
+ };
@@ -0,0 +1,121 @@
1
+ import { assert } from '@pexip/utils';
2
+ import { m3 } from '../../m3';
3
+ import { RENDERING_EVENTS } from '../../constants';
4
+ import { clamping } from '../../utils';
5
+ import { createGLProgram, createMaskTexture, createFrameBuffer, } from './webglUtils';
6
+ import vertexSrc from './shaders/matrix.vert';
7
+ import fragmentSrc from './shaders/erode.frag';
8
+ const MAX_RADIUS_PX = 64;
9
+ const clampRadiusPx = clamping(0, MAX_RADIUS_PX);
10
+ /**
11
+ * Create a morphological erosion renderer that performs erosion of a mask with a
12
+ * given radius and number of passes.
13
+ * The erosion is implemented as a separable filter.
14
+ */
15
+ export const createMorphErodeRenderer = (processor, width, height) => {
16
+ const { gl, program, attribLocations, uniformLocations, floatSingleChannelSupported, floatFullChannelSupported, release, } = createGLProgram(processor, {
17
+ attribs: {
18
+ position: 'a_position',
19
+ texCoord: 'a_texCoord',
20
+ },
21
+ uniforms: {
22
+ matrix: 'u_matrix',
23
+ mask: 'u_mask',
24
+ texelSize: 'u_texel_size',
25
+ radiusPx: 'u_radius_px',
26
+ direction: 'u_direction',
27
+ },
28
+ vertexSrc,
29
+ fragmentSrc,
30
+ });
31
+ const vao = gl.createVertexArray();
32
+ gl.bindVertexArray(vao);
33
+ // Position
34
+ const positionBuffer = gl.createBuffer();
35
+ gl.enableVertexAttribArray(attribLocations.position);
36
+ gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
37
+ gl.vertexAttribPointer(attribLocations.position, 2, gl.FLOAT, false, 0, 0);
38
+ gl.bufferData(gl.ARRAY_BUFFER,
39
+ // prettier-ignore
40
+ new Float32Array([
41
+ 0, 0,
42
+ 0, 1,
43
+ 1, 0,
44
+ 1, 0,
45
+ 0, 1,
46
+ 1, 1,
47
+ ]), gl.STATIC_DRAW);
48
+ // Texture coordinates for the rectangle
49
+ const texCoordBuffer = gl.createBuffer();
50
+ gl.bindBuffer(gl.ARRAY_BUFFER, texCoordBuffer);
51
+ gl.bufferData(gl.ARRAY_BUFFER,
52
+ // prettier-ignore
53
+ new Float32Array([
54
+ 0, 0,
55
+ 0, 1,
56
+ 1, 0,
57
+ 1, 0,
58
+ 0, 1,
59
+ 1, 1,
60
+ ]), gl.STATIC_DRAW);
61
+ gl.enableVertexAttribArray(attribLocations.texCoord);
62
+ gl.vertexAttribPointer(attribLocations.texCoord, 2, gl.FLOAT, false, 0, 0);
63
+ const outputTexturePing = createMaskTexture(gl, width, height, false, floatSingleChannelSupported, floatFullChannelSupported);
64
+ const frameBufferPing = createFrameBuffer(gl, outputTexturePing);
65
+ const outputTexturePong = createMaskTexture(gl, width, height, false, floatSingleChannelSupported, floatFullChannelSupported);
66
+ const frameBufferPong = createFrameBuffer(gl, outputTexturePong);
67
+ const scaleFactor = processor.width / width;
68
+ return {
69
+ render: ({ mask }, options) => {
70
+ assert(!gl.isContextLost(), RENDERING_EVENTS.ContextLost);
71
+ gl.clearColor(0, 0, 0, 0);
72
+ gl.clear(gl.COLOR_BUFFER_BIT);
73
+ gl.useProgram(program);
74
+ gl.bindVertexArray(vao);
75
+ gl.uniform2f(uniformLocations.texelSize, 1 / mask.width, 1 / mask.height);
76
+ gl.uniform1i(uniformLocations.radiusPx, clampRadiusPx(Math.floor(options.morphErodeRadiusPx / scaleFactor)));
77
+ const matrix = m3.projection(1, 1, 1);
78
+ // No Flip Y Coordinate for drawing on the frame buffer
79
+ // Project, position and scale to full-size of the canvas
80
+ gl.uniformMatrix3fv(uniformLocations.matrix, false, matrix);
81
+ let write = frameBufferPing;
82
+ let read = mask.texture;
83
+ // Ping-pong framebuffers for multiple passes
84
+ for (let i = 0; i < options.morphPass; i++) {
85
+ // Separable
86
+ for (let j = 0; j < 2; j++) {
87
+ // Textures
88
+ gl.activeTexture(gl.TEXTURE0);
89
+ gl.uniform1i(uniformLocations.mask, 0);
90
+ gl.bindTexture(gl.TEXTURE_2D, read);
91
+ // Horizontal then Vertical
92
+ gl.uniform2f(uniformLocations.direction, 1.0 - j, 0.0 + j);
93
+ gl.bindFramebuffer(gl.FRAMEBUFFER, write);
94
+ gl.viewport(0, 0, mask.width, mask.height);
95
+ gl.drawArrays(gl.TRIANGLES, 0, 6);
96
+ // Swap
97
+ [read, write] =
98
+ write === frameBufferPing
99
+ ? [outputTexturePing, frameBufferPong]
100
+ : [outputTexturePong, frameBufferPing];
101
+ }
102
+ }
103
+ // Cleanup
104
+ gl.bindBuffer(gl.ARRAY_BUFFER, null);
105
+ gl.bindFramebuffer(gl.FRAMEBUFFER, null);
106
+ gl.bindTexture(gl.TEXTURE_2D, null);
107
+ gl.bindVertexArray(null);
108
+ return { texture: read, width, height };
109
+ },
110
+ release: () => {
111
+ gl.deleteBuffer(positionBuffer);
112
+ gl.deleteBuffer(texCoordBuffer);
113
+ gl.deleteTexture(outputTexturePing);
114
+ gl.deleteTexture(outputTexturePong);
115
+ gl.deleteFramebuffer(frameBufferPing);
116
+ gl.deleteFramebuffer(frameBufferPong);
117
+ gl.deleteVertexArray(vao);
118
+ release();
119
+ },
120
+ };
121
+ };