@pexip/media-processor 20.3.5 → 22.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (201) hide show
  1. package/CHANGELOG.md +109 -0
  2. package/README.md +17 -10
  3. package/api-docs/README.mdx +69 -23
  4. package/api-docs/functions/assertMediaStreamVideoTrack.mdx +13 -0
  5. package/api-docs/functions/createBenchmark.mdx +17 -5
  6. package/api-docs/functions/createFrameCallbackRequest.mdx +8 -2
  7. package/api-docs/functions/createMediaStreamTrackInputProcessAdaptor.mdx +10 -0
  8. package/api-docs/functions/createReadWritableStreamFallbackInputProcessAdaptor.mdx +17 -0
  9. package/api-docs/functions/createReadWritableStreamInputProcessAdaptor.mdx +10 -0
  10. package/api-docs/functions/createVideoProcessor.mdx +3 -3
  11. package/api-docs/functions/createWindowedStats.mdx +23 -0
  12. package/api-docs/functions/getDefaultInputProcessAdaptor.mdx +17 -0
  13. package/api-docs/functions/iou.mdx +19 -0
  14. package/api-docs/functions/isMediaStreamVideoTrack.mdx +13 -0
  15. package/api-docs/functions/labelComponents.mdx +20 -0
  16. package/api-docs/functions/nearestPowerOfTwo.mdx +21 -0
  17. package/api-docs/functions/playVideo.mdx +16 -0
  18. package/api-docs/functions/scoreComponent.mdx +22 -0
  19. package/api-docs/functions/setVideoElementSrc.mdx +14 -0
  20. package/api-docs/interfaces/Analyzer.mdx +4 -0
  21. package/api-docs/interfaces/Benchmark.mdx +26 -2
  22. package/api-docs/interfaces/FrameTransformStats.mdx +9 -0
  23. package/api-docs/interfaces/Gain.mdx +4 -0
  24. package/api-docs/interfaces/ImageSegmenterOptions.mdx +0 -2
  25. package/api-docs/interfaces/InputProcessAdaptor.mdx +28 -0
  26. package/api-docs/interfaces/Process.mdx +8 -3
  27. package/api-docs/interfaces/ProcessEventMute.mdx +6 -0
  28. package/api-docs/interfaces/ProcessWorkerEventOpened.mdx +7 -0
  29. package/api-docs/interfaces/ProcessWorkerEventStats.mdx +6 -0
  30. package/api-docs/interfaces/ProcessWorkerEventUpdated.mdx +7 -0
  31. package/api-docs/interfaces/ProcessorEvent.mdx +1 -1
  32. package/api-docs/interfaces/ProcessorOptions.mdx +0 -1
  33. package/api-docs/interfaces/ProcessorProcessOptions.mdx +3 -1
  34. package/api-docs/interfaces/ProcessorUpdateOptions.mdx +15 -0
  35. package/api-docs/interfaces/ReadWritableStream.mdx +12 -0
  36. package/api-docs/interfaces/Renderer.mdx +2 -2
  37. package/api-docs/interfaces/RendererOptions.mdx +25 -12
  38. package/api-docs/interfaces/Segmenter.mdx +89 -10
  39. package/api-docs/interfaces/SegmenterOptions.mdx +0 -1
  40. package/api-docs/interfaces/SelectionOptions.mdx +15 -0
  41. package/api-docs/interfaces/Stats.mdx +11 -0
  42. package/api-docs/interfaces/VideoProcessor.mdx +49 -5
  43. package/api-docs/interfaces/Weights.mdx +12 -0
  44. package/api-docs/type-aliases/OptionalKeys.mdx +10 -0
  45. package/api-docs/type-aliases/ProcessEvents.mdx +1 -0
  46. package/api-docs/type-aliases/ProcessInputType.mdx +3 -1
  47. package/api-docs/type-aliases/ProcessWorkerEvents.mdx +6 -4
  48. package/api-docs/type-aliases/ProcessorWorkerEvents.mdx +1 -1
  49. package/api-docs/type-aliases/Struct.mdx +9 -0
  50. package/api-docs/variables/BACKGROUND_THRESHOLD.mdx +3 -0
  51. package/api-docs/variables/DOWN_SAMPLE_FACTOR.mdx +3 -0
  52. package/api-docs/variables/EDGE_BLUR_AMOUNT.mdx +1 -1
  53. package/api-docs/variables/EXCLUDE_BYSTANDERS.mdx +3 -0
  54. package/api-docs/variables/FOREGROUND_THRESHOLD.mdx +1 -1
  55. package/api-docs/variables/FRAME_RATE.mdx +1 -1
  56. package/api-docs/variables/ImageFetchError.mdx +3 -0
  57. package/api-docs/variables/LIGHT_WRAP_BLUR_AMOUNT.mdx +3 -0
  58. package/api-docs/variables/LIGHT_WRAP_EDGE_BAND.mdx +3 -0
  59. package/api-docs/variables/LIGHT_WRAP_INTENSITY.mdx +3 -0
  60. package/api-docs/variables/LIGHT_WRAP_TIGHTNESS.mdx +3 -0
  61. package/api-docs/variables/MASK_COMBINE_RATIO.mdx +1 -1
  62. package/api-docs/variables/MORPH_DILATE_RADIUS_PX.mdx +3 -0
  63. package/api-docs/variables/MORPH_ERODE_RADIUS_PX.mdx +3 -0
  64. package/api-docs/variables/MORPH_PASS.mdx +3 -0
  65. package/api-docs/variables/MediaStreamTrackTransferable.mdx +5 -0
  66. package/api-docs/variables/PERSON_CENTER.mdx +3 -0
  67. package/api-docs/variables/RENDERER_OPTIONS_STRUCT.mdx +3 -0
  68. package/api-docs/variables/SIGMA_RANGE.mdx +3 -0
  69. package/api-docs/variables/SIGMA_SPACE.mdx +3 -0
  70. package/api-docs/variables/STRONG_EDGE_BLUR_AMOUNT.mdx +3 -0
  71. package/api-docs/variables/cloneImageRecord.mdx +19 -0
  72. package/api-docs/variables/compareImageRecords.mdx +18 -0
  73. package/api-docs/variables/createRemoteImageBitmap.mdx +1 -1
  74. package/api-docs/variables/getCanvasRenderingContext2D.mdx +14 -0
  75. package/api-docs/variables/getImageSize.mdx +13 -0
  76. package/api-docs/variables/isRenderOptionsKey.mdx +13 -0
  77. package/api-docs/variables/isValidGpuAPI.mdx +13 -0
  78. package/dist/common/backends/webgl/binaryMask.d.ts +8 -0
  79. package/dist/common/backends/webgl/binaryMask.js +95 -0
  80. package/dist/common/backends/webgl/blender.d.ts +2 -1
  81. package/dist/common/backends/webgl/blender.js +18 -15
  82. package/dist/common/backends/webgl/blur.d.ts +1 -1
  83. package/dist/common/backends/webgl/blur.js +23 -16
  84. package/dist/common/backends/webgl/canvas.js +9 -4
  85. package/dist/common/backends/webgl/diffAbsMask.d.ts +10 -0
  86. package/dist/common/backends/webgl/diffAbsMask.js +103 -0
  87. package/dist/common/backends/webgl/geodesicReconstruction.d.ts +13 -0
  88. package/dist/common/backends/webgl/geodesicReconstruction.js +176 -0
  89. package/dist/common/backends/webgl/jointBilateralFilter.d.ts +12 -0
  90. package/dist/common/backends/webgl/jointBilateralFilter.js +108 -0
  91. package/dist/common/backends/webgl/morphErode.d.ts +13 -0
  92. package/dist/common/backends/webgl/morphErode.js +121 -0
  93. package/dist/common/backends/webgl/renderer.js +194 -44
  94. package/dist/common/backends/webgl/replace.d.ts +17 -0
  95. package/dist/common/backends/webgl/replace.js +126 -0
  96. package/dist/common/backends/webgl/selectComponent.d.ts +15 -0
  97. package/dist/common/backends/webgl/selectComponent.js +102 -0
  98. package/dist/common/backends/webgl/smoothingMask.d.ts +3 -3
  99. package/dist/common/backends/webgl/smoothingMask.js +22 -21
  100. package/dist/common/backends/webgl/tentBlur.d.ts +9 -0
  101. package/dist/common/backends/webgl/tentBlur.js +118 -0
  102. package/dist/common/backends/webgl/texture.d.ts +2 -3
  103. package/dist/common/backends/webgl/texture.js +21 -19
  104. package/dist/common/backends/webgl/textureToTexture.d.ts +1 -1
  105. package/dist/common/backends/webgl/textureToTexture.js +19 -16
  106. package/dist/common/backends/webgl/webglUtils.d.ts +7 -1
  107. package/dist/common/backends/webgl/webglUtils.js +72 -10
  108. package/dist/common/backends/webgpu/blender.d.ts +6 -1
  109. package/dist/common/backends/webgpu/blender.js +96 -51
  110. package/dist/common/backends/webgpu/constants.js +1 -1
  111. package/dist/common/backends/webgpu/dualFilterBlur.d.ts +2 -3
  112. package/dist/common/backends/webgpu/dualFilterBlur.js +146 -140
  113. package/dist/common/backends/webgpu/jointBilateralFilter.d.ts +7 -0
  114. package/dist/common/backends/webgpu/jointBilateralFilter.js +162 -0
  115. package/dist/common/backends/webgpu/renderer.js +205 -43
  116. package/dist/common/backends/webgpu/replace.d.ts +11 -0
  117. package/dist/common/backends/webgpu/replace.js +175 -0
  118. package/dist/common/backends/webgpu/resize.d.ts +4 -0
  119. package/dist/common/backends/webgpu/resize.js +76 -0
  120. package/dist/common/backends/webgpu/smoothingMask.d.ts +2 -2
  121. package/dist/common/backends/webgpu/smoothingMask.js +74 -40
  122. package/dist/common/backends/webgpu/tentBlur.d.ts +5 -0
  123. package/dist/common/backends/webgpu/tentBlur.js +131 -0
  124. package/dist/common/backends/webgpu/texture.d.ts +2 -3
  125. package/dist/common/backends/webgpu/texture.js +29 -20
  126. package/dist/common/backends/webgpu/textureToTexture.d.ts +1 -1
  127. package/dist/common/backends/webgpu/textureToTexture.js +28 -15
  128. package/dist/common/backends/webgpu/types.d.ts +2 -0
  129. package/dist/common/backends/webgpu/webgpuUtils.d.ts +24 -4
  130. package/dist/common/backends/webgpu/webgpuUtils.js +60 -11
  131. package/dist/common/constants.d.ts +22 -5
  132. package/dist/common/constants.js +23 -5
  133. package/dist/common/index.d.ts +2 -0
  134. package/dist/common/index.js +2 -0
  135. package/dist/common/tsconfig.tsbuildinfo +1 -1
  136. package/dist/common/typeGuards.d.ts +2 -0
  137. package/dist/common/typeGuards.js +7 -0
  138. package/dist/common/types/media.d.ts +5 -1
  139. package/dist/common/types/messageEvents.d.ts +31 -4
  140. package/dist/common/types/processor.d.ts +6 -4
  141. package/dist/common/types/render.d.ts +147 -15
  142. package/dist/common/types/render.js +23 -1
  143. package/dist/common/types/segmentation.d.ts +0 -12
  144. package/dist/common/types/utils.d.ts +21 -0
  145. package/dist/common/utils.d.ts +29 -2
  146. package/dist/common/utils.js +206 -10
  147. package/dist/main/audio.js +3 -3
  148. package/dist/main/benchUtils.d.ts +35 -3
  149. package/dist/main/benchUtils.js +149 -15
  150. package/dist/main/index.d.ts +1 -1
  151. package/dist/main/index.js +1 -1
  152. package/dist/main/tsconfig.tsbuildinfo +1 -1
  153. package/dist/main/types.d.ts +0 -6
  154. package/dist/main/utils.d.ts +4 -0
  155. package/dist/main/utils.js +31 -0
  156. package/dist/main/video/constants.d.ts +1 -4
  157. package/dist/main/video/constants.js +1 -5
  158. package/dist/main/video/index.d.ts +3 -4
  159. package/dist/main/video/index.js +3 -4
  160. package/dist/main/video/inputProcessAdaptor.d.ts +30 -0
  161. package/dist/main/video/inputProcessAdaptor.js +198 -0
  162. package/dist/main/video/segmenter.d.ts +35 -0
  163. package/dist/main/video/{segmenters/mediapipe.js → segmenter.js} +93 -53
  164. package/dist/main/video/typeGuards.d.ts +2 -0
  165. package/dist/main/video/typeGuards.js +8 -1
  166. package/dist/main/video/types.d.ts +36 -16
  167. package/dist/main/video/utils.d.ts +5 -4
  168. package/dist/main/video/utils.js +49 -44
  169. package/dist/main/video/video.d.ts +29 -7
  170. package/dist/main/video/video.js +323 -28
  171. package/dist/workers/mediaWorker.js +35 -162
  172. package/dist/workers/tsconfig.tsbuildinfo +1 -1
  173. package/dist/worklets/tsconfig.tsbuildinfo +1 -1
  174. package/package.json +13 -12
  175. package/api-docs/functions/createCanvasTransform.mdx +0 -14
  176. package/api-docs/functions/createVideoTrackProcessor.mdx +0 -9
  177. package/api-docs/functions/createVideoTrackProcessorWithFallback.mdx +0 -15
  178. package/api-docs/interfaces/Detector.mdx +0 -81
  179. package/api-docs/interfaces/SegmentationParams.mdx +0 -18
  180. package/api-docs/interfaces/Transform.mdx +0 -62
  181. package/api-docs/type-aliases/ProcessVideoTrack.mdx +0 -15
  182. package/api-docs/type-aliases/SegmentationTransform.mdx +0 -3
  183. package/dist/common/backends/webgl/shaders/blendFragmentShader.d.ts +0 -1
  184. package/dist/common/backends/webgl/shaders/blendFragmentShader.js +0 -21
  185. package/dist/common/backends/webgl/shaders/dualFilterBlurFragmentShader.d.ts +0 -1
  186. package/dist/common/backends/webgl/shaders/dualFilterBlurFragmentShader.js +0 -45
  187. package/dist/common/backends/webgl/shaders/smoothingMaskFragmentShader.d.ts +0 -1
  188. package/dist/common/backends/webgl/shaders/smoothingMaskFragmentShader.js +0 -57
  189. package/dist/common/backends/webgl/shaders/textureFragmentShader.d.ts +0 -1
  190. package/dist/common/backends/webgl/shaders/textureFragmentShader.js +0 -16
  191. package/dist/common/backends/webgl/shaders/vertexShader.d.ts +0 -1
  192. package/dist/common/backends/webgl/shaders/vertexShader.js +0 -19
  193. package/dist/main/transformer.d.ts +0 -1
  194. package/dist/main/transformer.js +0 -4
  195. package/dist/main/video/canvasTransform.d.ts +0 -17
  196. package/dist/main/video/canvasTransform.js +0 -253
  197. package/dist/main/video/segmenters/mediapipe.d.ts +0 -22
  198. package/dist/main/video/transformer.d.ts +0 -9
  199. package/dist/main/video/transformer.js +0 -68
  200. package/dist/main/video/videoStreamTrackProcessor.d.ts +0 -20
  201. package/dist/main/video/videoStreamTrackProcessor.js +0 -86
@@ -0,0 +1,16 @@
1
+ ```ts
2
+ function playVideo(video, __namedParameters): Promise<[void, void]> | undefined;
3
+ ```
4
+
5
+ ## Parameters
6
+
7
+ | Parameter | Type |
8
+ | ------ | ------ |
9
+ | `video` | `HTMLVideoElement` |
10
+ | `__namedParameters` | \{ `setTimeout?`: (`handler`, `timeout?`, ...`arguments`) => `number` & (`handler`, `timeout?`, ...`arguments`) => `number`; `clearTimeout?`: (`id`) => `void` & (`id`) => `void`; \} |
11
+ | `__namedParameters.setTimeout?` | (`handler`, `timeout?`, ...`arguments`) => `number` & (`handler`, `timeout?`, ...`arguments`) => `number` |
12
+ | `__namedParameters.clearTimeout?` | (`id`) => `void` & (`id`) => `void` |
13
+
14
+ ## Returns
15
+
16
+ `Promise`\<\[`void`, `void`\]\> \| `undefined`
@@ -0,0 +1,22 @@
1
+ ```ts
2
+ function scoreComponent(
3
+ stats,
4
+ overlap,
5
+ weights,
6
+ width,
7
+ height): number;
8
+ ```
9
+
10
+ ## Parameters
11
+
12
+ | Parameter | Type |
13
+ | ------ | ------ |
14
+ | `stats` | [`Stats`](../interfaces/Stats.mdx) |
15
+ | `overlap` | `number` |
16
+ | `weights` | [`SelectionOptions`](../interfaces/SelectionOptions.mdx) |
17
+ | `width` | `number` |
18
+ | `height` | `number` |
19
+
20
+ ## Returns
21
+
22
+ `number`
@@ -0,0 +1,14 @@
1
+ ```ts
2
+ function setVideoElementSrc(video, src): void;
3
+ ```
4
+
5
+ ## Parameters
6
+
7
+ | Parameter | Type |
8
+ | ------ | ------ |
9
+ | `video` | `HTMLVideoElement` |
10
+ | `src` | `MediaProvider` |
11
+
12
+ ## Returns
13
+
14
+ `void`
@@ -163,6 +163,8 @@ connect(
163
163
  input?): AudioNode;
164
164
  ```
165
165
 
166
+ The **`connect()`** method of the AudioNode interface lets you connect one of the node's outputs to a target, which may be either another AudioNode (thereby directing the sound data to the specified node) or an AudioParam, so that the node's output data is automatically used to change the value of that parameter over time.
167
+
166
168
  [MDN Reference](https://developer.mozilla.org/docs/Web/API/AudioNode/connect)
167
169
 
168
170
  ##### Parameters
@@ -216,6 +218,8 @@ BaseAudioNode.connect
216
218
  disconnect(): void;
217
219
  ```
218
220
 
221
+ The **`disconnect()`** method of the AudioNode interface lets you disconnect one or more nodes from the node on which the method is called.
222
+
219
223
  [MDN Reference](https://developer.mozilla.org/docs/Web/API/AudioNode/disconnect)
220
224
 
221
225
  ##### Returns
@@ -3,9 +3,15 @@
3
3
  ### begin()
4
4
 
5
5
  ```ts
6
- begin(): void;
6
+ begin(timestamp?): void;
7
7
  ```
8
8
 
9
+ #### Parameters
10
+
11
+ | Parameter | Type |
12
+ | ------ | ------ |
13
+ | `timestamp?` | `number` |
14
+
9
15
  #### Returns
10
16
 
11
17
  `void`
@@ -15,7 +21,25 @@ begin(): void;
15
21
  ### end()
16
22
 
17
23
  ```ts
18
- end(): void;
24
+ end(timestamp?): void;
25
+ ```
26
+
27
+ #### Parameters
28
+
29
+ | Parameter | Type |
30
+ | ------ | ------ |
31
+ | `timestamp?` | `number` |
32
+
33
+ #### Returns
34
+
35
+ `void`
36
+
37
+ ***
38
+
39
+ ### reset()
40
+
41
+ ```ts
42
+ reset(): void;
19
43
  ```
20
44
 
21
45
  #### Returns
@@ -0,0 +1,9 @@
1
+ Frame transform statistic, i.e. the timing data for processing each frame
2
+
3
+ ## Properties
4
+
5
+ | Property | Type | Description |
6
+ | ------ | ------ | ------ |
7
+ | <a id="start"></a> `start` | `number` | Start time of the transform |
8
+ | <a id="end"></a> `end` | `number` | End time of the transform |
9
+ | <a id="segmentation"></a> `segmentation?` | `number` | Segmentation inference end time |
@@ -22,6 +22,8 @@ connect(
22
22
  input?): AudioNode;
23
23
  ```
24
24
 
25
+ The **`connect()`** method of the AudioNode interface lets you connect one of the node's outputs to a target, which may be either another AudioNode (thereby directing the sound data to the specified node) or an AudioParam, so that the node's output data is automatically used to change the value of that parameter over time.
26
+
25
27
  [MDN Reference](https://developer.mozilla.org/docs/Web/API/AudioNode/connect)
26
28
 
27
29
  ##### Parameters
@@ -75,6 +77,8 @@ BaseAudioNode.connect
75
77
  disconnect(): void;
76
78
  ```
77
79
 
80
+ The **`disconnect()`** method of the AudioNode interface lets you disconnect one or more nodes from the node on which the method is called.
81
+
78
82
  [MDN Reference](https://developer.mozilla.org/docs/Web/API/AudioNode/disconnect)
79
83
 
80
84
  ##### Returns
@@ -2,7 +2,5 @@
2
2
 
3
3
  | Property | Type | Description |
4
4
  | ------ | ------ | ------ |
5
- | <a id="canvas"></a> `canvas?` | `OffscreenCanvas` | The canvas element to bind textures to. This has to be set for GPU processing. The task will initialize a WebGL or WebGPU context and throw an error if this fails (e.g. if you have already initialized a different type of context). |
6
- | <a id="segmentercanvas"></a> `segmenterCanvas?` | `OffscreenCanvas` | The Google TaskVision segmentation model only works with WebGl so we need a separate canvas for this. |
7
5
  | <a id="modelasset"></a> `modelAsset?` | [`SegmentationModelAsset`](SegmentationModelAsset.mdx) | - |
8
6
  | <a id="delegate"></a> `delegate?` | [`Delegate`](../type-aliases/Delegate.mdx) | Overrides the default backend to use for the provided model. |
@@ -0,0 +1,28 @@
1
+ ## Properties
2
+
3
+ | Property | Type | Description |
4
+ | ------ | ------ | ------ |
5
+ | <a id="outputtrack"></a> `outputTrack?` | `MediaStreamTrack` | The generated track from the `adapt` function, and only available after calling the `adapt` method otherwise `undefined` |
6
+
7
+ ## Methods
8
+
9
+ ### adapt()
10
+
11
+ ```ts
12
+ adapt(track):
13
+ | MediaStreamTrack
14
+ | ReadWritableStream<VideoFrame>;
15
+ ```
16
+
17
+ Adapt the provided input track based on browser capabilities.
18
+
19
+ #### Parameters
20
+
21
+ | Parameter | Type |
22
+ | ------ | ------ |
23
+ | `track` | `MediaStreamTrack` |
24
+
25
+ #### Returns
26
+
27
+ \| `MediaStreamTrack`
28
+ \| [`ReadWritableStream`](ReadWritableStream.mdx)\<`VideoFrame`\>
@@ -1,7 +1,12 @@
1
1
  ## Extended by
2
2
 
3
3
  - [`Segmenter`](Segmenter.mdx)
4
- - [`Detector`](Detector.mdx)
4
+
5
+ ## Type Parameters
6
+
7
+ | Type Parameter |
8
+ | ------ |
9
+ | `T` |
5
10
 
6
11
  ## Properties
7
12
 
@@ -14,12 +19,12 @@
14
19
  ### open()
15
20
 
16
21
  ```ts
17
- open(): Promise<void>;
22
+ open(): Promise<T>;
18
23
  ```
19
24
 
20
25
  #### Returns
21
26
 
22
- `Promise`\<`void`\>
27
+ `Promise`\<`T`\>
23
28
 
24
29
  ***
25
30
 
@@ -0,0 +1,6 @@
1
+ ## Properties
2
+
3
+ | Property | Type |
4
+ | ------ | ------ |
5
+ | <a id="type"></a> `type` | `"mute"` |
6
+ | <a id="data"></a> `data` | `boolean` |
@@ -1,5 +1,12 @@
1
+ ## Type Parameters
2
+
3
+ | Type Parameter |
4
+ | ------ |
5
+ | `T` |
6
+
1
7
  ## Properties
2
8
 
3
9
  | Property | Type |
4
10
  | ------ | ------ |
5
11
  | <a id="type"></a> `type` | `"opened"` |
12
+ | <a id="data"></a> `data` | `T` |
@@ -0,0 +1,6 @@
1
+ ## Properties
2
+
3
+ | Property | Type |
4
+ | ------ | ------ |
5
+ | <a id="type"></a> `type` | `"stats"` |
6
+ | <a id="data"></a> `data` | [`FrameTransformStats`](FrameTransformStats.mdx)[] |
@@ -1,5 +1,12 @@
1
+ ## Type Parameters
2
+
3
+ | Type Parameter |
4
+ | ------ |
5
+ | `T` |
6
+
1
7
  ## Properties
2
8
 
3
9
  | Property | Type |
4
10
  | ------ | ------ |
5
11
  | <a id="type"></a> `type` | `"updated"` |
12
+ | <a id="data"></a> `data` | `T` |
@@ -4,4 +4,4 @@
4
4
  | ------ | ------ |
5
5
  | <a id="type"></a> `type` | \| `"Error"` \| `"ContextCreationError"` \| `"ContextLost"` \| `"ContextRestored"` \| `"ProcessorRestarted"` \| `"SamplingError"` |
6
6
  | <a id="message"></a> `message?` | `string` |
7
- | <a id="frame"></a> `frame?` | [`VideoFrameLike`](VideoFrameLike.mdx) |
7
+ | <a id="track"></a> `track?` | `MediaStreamTrack` |
@@ -10,7 +10,6 @@
10
10
  | <a id="backgroundimage"></a> `backgroundImage?` | [`ImageRecord`](ImageRecord.mdx) | - |
11
11
  | <a id="restart"></a> `restart?` | `boolean` | - |
12
12
  | <a id="basepath"></a> `basePath?` | `string` | [`SegmenterOptions`](SegmenterOptions.mdx).[`basePath`](SegmenterOptions.mdx#basepath) |
13
- | <a id="output"></a> `output?` | `OffscreenCanvas` \| `null` | [`SegmenterOptions`](SegmenterOptions.mdx).[`output`](SegmenterOptions.mdx#output) |
14
13
  | <a id="processingwidth"></a> `processingWidth?` | `number` | [`SegmenterOptions`](SegmenterOptions.mdx).[`processingWidth`](SegmenterOptions.mdx#processingwidth) |
15
14
  | <a id="processingheight"></a> `processingHeight?` | `number` | [`SegmenterOptions`](SegmenterOptions.mdx).[`processingHeight`](SegmenterOptions.mdx#processingheight) |
16
15
  | <a id="imagesegmenteroptions"></a> `imageSegmenterOptions?` | [`ImageSegmenterOptions`](ImageSegmenterOptions.mdx) | [`SegmenterOptions`](SegmenterOptions.mdx).[`imageSegmenterOptions`](SegmenterOptions.mdx#imagesegmenteroptions) |
@@ -2,5 +2,7 @@
2
2
 
3
3
  | Property | Type |
4
4
  | ------ | ------ |
5
- | <a id="videoframe"></a> `videoFrame` | [`VideoFrameLike`](VideoFrameLike.mdx) |
5
+ | <a id="track"></a> `track?` | `MediaStreamTrack` |
6
+ | <a id="readable"></a> `readable?` | `ReadableStream`\<`VideoFrame`\> |
7
+ | <a id="writable"></a> `writable?` | `WritableStream`\<`VideoFrame`\> |
6
8
  | <a id="renderoptions"></a> `renderOptions?` | [`RendererOptions`](RendererOptions.mdx) |
@@ -0,0 +1,15 @@
1
+ ## Extends
2
+
3
+ - `Omit`\<[`ProcessorOptions`](ProcessorOptions.mdx), `"renderOptions"`\>
4
+
5
+ ## Properties
6
+
7
+ | Property | Type | Inherited from |
8
+ | ------ | ------ | ------ |
9
+ | <a id="backgroundimage"></a> `backgroundImage?` | [`ImageRecord`](ImageRecord.mdx) | [`ProcessorOptions`](ProcessorOptions.mdx).[`backgroundImage`](ProcessorOptions.mdx#backgroundimage) |
10
+ | <a id="restart"></a> `restart?` | `boolean` | [`ProcessorOptions`](ProcessorOptions.mdx).[`restart`](ProcessorOptions.mdx#restart) |
11
+ | <a id="basepath"></a> `basePath?` | `string` | [`SegmenterOptions`](SegmenterOptions.mdx).[`basePath`](SegmenterOptions.mdx#basepath) |
12
+ | <a id="processingwidth"></a> `processingWidth?` | `number` | [`SegmenterOptions`](SegmenterOptions.mdx).[`processingWidth`](SegmenterOptions.mdx#processingwidth) |
13
+ | <a id="processingheight"></a> `processingHeight?` | `number` | [`SegmenterOptions`](SegmenterOptions.mdx).[`processingHeight`](SegmenterOptions.mdx#processingheight) |
14
+ | <a id="imagesegmenteroptions"></a> `imageSegmenterOptions?` | [`ImageSegmenterOptions`](ImageSegmenterOptions.mdx) | [`SegmenterOptions`](SegmenterOptions.mdx).[`imageSegmenterOptions`](SegmenterOptions.mdx#imagesegmenteroptions) |
15
+ | <a id="renderoptions"></a> `renderOptions?` | `Partial`\<[`RendererOptions`](RendererOptions.mdx)\> | - |
@@ -0,0 +1,12 @@
1
+ ## Type Parameters
2
+
3
+ | Type Parameter |
4
+ | ------ |
5
+ | `T` |
6
+
7
+ ## Properties
8
+
9
+ | Property | Type |
10
+ | ------ | ------ |
11
+ | <a id="readable"></a> `readable` | `ReadableStream`\<`T`\> |
12
+ | <a id="writable"></a> `writable` | `WritableStream`\<`T`\> |
@@ -31,8 +31,8 @@ render(
31
31
 
32
32
  | Parameter | Type |
33
33
  | ------ | ------ |
34
- | `mask` | `ImageBitmap` |
35
- | `frame` | [`VideoFrameLike`](VideoFrameLike.mdx) |
34
+ | `mask` | `OffscreenCanvas` |
35
+ | `frame` | `VideoFrame` |
36
36
  | `options` | `Options` |
37
37
 
38
38
  #### Returns
@@ -1,15 +1,28 @@
1
- ## Extended by
2
-
3
- - [`SegmentationParams`](SegmentationParams.mdx)
1
+ Options controlling the rendering and effects for person/background segmentation.
2
+ All values are stable per-frame and can be safely updated in real-time.
4
3
 
5
4
  ## Properties
6
5
 
7
- | Property | Type | Default value | Description |
8
- | ------ | ------ | ------ | ------ |
9
- | <a id="backgroundimageurl"></a> `backgroundImageUrl` | `string` | `undefined` | - |
10
- | <a id="foregroundthreshold"></a> `foregroundThreshold` | `number` | `undefined` | Confident score threshold to be considered as the foreground between 0 and 1 |
11
- | <a id="backgroundbluramount"></a> `backgroundBlurAmount` | `number` | `undefined` | Between 0 and 10 |
12
- | <a id="edgebluramount"></a> `edgeBlurAmount` | `number` | `3` | Range in between 0 and 20 |
13
- | <a id="maskcombineratio"></a> `maskCombineRatio` | `number` | `undefined` | The ratio used for smoothing the mask between 0 and 1 |
14
- | <a id="effects"></a> `effects` | `"blur"` \| `"none"` \| `"overlay"` | `undefined` | Effects |
15
- | <a id="backend"></a> `backend` | `"webgl"` \| `"webgpu"` | `undefined` | Backend to render background effects |
6
+ | Property | Type | Description |
7
+ | ------ | ------ | ------ |
8
+ | <a id="backgroundimageurl"></a> `backgroundImageUrl` | `string` | URL or Data URI for the virtual background image (used in "overlay" effect). |
9
+ | <a id="backgroundbluramount"></a> `backgroundBlurAmount` | `number` | Amount of background blur (radius in px): [0,9]. Recommend: 3–6 for portraits. Higher may be slow and over-blurred. **Default** `3` |
10
+ | <a id="edgebluramount"></a> `edgeBlurAmount` | `number` | Edge blur for mask boundary antialiasing (radius in px): [0,9]. Small values (1–5) smooth mask stair-steps; large values soften edges more visibly. **Default** `4` |
11
+ | <a id="effects"></a> `effects` | `"blur"` \| `"none"` \| `"overlay"` | Effects to apply: "none", "blur", or "overlay". "blur" = background blur. "overlay" = replace BG with backgroundImageUrl. |
12
+ | <a id="backend"></a> `backend` | `"webgl"` \| `"webgpu"` | Backend to render background effects |
13
+ | <a id="foregroundthreshold"></a> `foregroundThreshold` | `number` | [0,1] Upper mask confidence: all mask pixels >= this are considered definite foreground. Adjust to make mask edges tighter or more permissive for foreground inclusion. Example: 0.8 ("mask > 0.8 = sure foreground"). Must always be >= backgroundThreshold. **Default** `0.85` |
14
+ | <a id="backgroundthreshold"></a> `backgroundThreshold` | `number` | [0,1] Lower mask confidence: all mask pixels <= this are considered definite background. Adjust to make mask edges tighter or more permissive for background inclusion. Example: 0.3 ("mask < 0.3 = sure background"). **Default** `0.35` |
15
+ | <a id="sigmaspace"></a> `sigmaSpace` | `number` | Bilateral filter: sigma in pixel space for mask edge snapping. Controls the neighborhood/pixel distance—higher is larger window (5–10 typical). **Default** `5` |
16
+ | <a id="sigmarangelo"></a> `sigmaRangeLo` | `number` | Bilateral filter: minimum sigma in color/luma (for mask interiors), [0,1]. Controls how little color difference is needed to cross-mask regions away from edge. Lower = sharper edge preservation in mask interiors. Example: 0.2 **Default** `0.2` |
17
+ | <a id="sigmarangehi"></a> `sigmaRangeHi` | `number` | Bilateral filter: maximum sigma in color/luma (for mask uncertain zones), [0,1]. Controls how much color similarity is required at uncertain/edge mask locations. Higher = more mask smoothing at edges. Example: 0.8 **Default** `0.8` |
18
+ | <a id="maskcombineratio"></a> `maskCombineRatio` | `number` | Ratio for mask smoothing (how much uncertainty enables temporal blend), [0,1]. Higher = more frame-to-frame blending, more stability but more trailing ghosts. Typical: 0.6–0.85. **Default** `0.8` |
19
+ | <a id="excludebystanders"></a> `excludeBystanders` | `boolean` | **`Experimental`** Whether to exclude bystanders/connected people from the output mask. The main person instance (closest to personCenter OR largest blob) is kept in the output mask. All other bystanders/connected people are suppressed (bystander removal). **Default** `false (no bystander removal, all detected people are included in the mask). @experimental This feature is experimental and only works in WebGL backend.` |
20
+ | <a id="personcenter"></a> `personCenter` | \[`number`, `number`\] | **`Experimental`** Person center: normalized (x, y) in [0,1] image coordinates for main subject selection. Used by the CCL/bias filter to select the main person blob. **Default** `[0.5, 0.42] (= center, slightly above vertical midpoint). @experimental This feature is experimental and only works in WebGL backend.` |
21
+ | <a id="morpheroderadiuspx"></a> `morphErodeRadiusPx` | `number` | **`Experimental`** Morphological erosion: radius in pixels [1,12]. Sets kernel size for bridge-breaking. Larger values break thicker bridges but may erode fine details. **Default** `5 @experimental This feature is experimental and only works in WebGL backend.` |
22
+ | <a id="morphpass"></a> `morphPass` | `number` | **`Experimental`** Morphological pass: number of separable passes [1,10]. More passes increase the effective span. **Default** `4 @experimental This feature is experimental and only works in WebGL backend.` |
23
+ | <a id="morphdilateradiuspx"></a> `morphDilateRadiusPx` | `number` | **`Experimental`** Geodesic dilation: radius in pixels [1,12]. Must match erosion for good reconstruction. **Default** `6 @experimental This feature is experimental and only works in WebGL backend.` |
24
+ | <a id="lightwrapintensity"></a> `lightWrapIntensity` | `number` | [0,1] Light wrap amount (halo intensity) at mask edge. Wraps blurred background color just outside the subject edge onto the subject boundary for realism. **Default** `0.8` |
25
+ | <a id="lightwraptightness"></a> `lightWrapTightness` | `number` | [0.5,4.0] Controls power/tightness of the wrap band falloff. Higher = tighter/narrower wrap. **Default** `1.2` |
26
+ | <a id="lightwrapedgeband"></a> `lightWrapEdgeBand` | `number` | Width of the wrap band (feather size, in normalized mask units) between the backgroundThreshold and foregroundThreshold in [0.0, (foregroundThreshold - backgroundThreshold) / 2] Higher = wider soft halo. **Default** `0.25` |
27
+ | <a id="lightwrapbluramount"></a> `lightWrapBlurAmount` | `number` | Blur amount used for background color in the wrap band (units=blur passes). **Default** `1` |
28
+ | <a id="downsamplefactor"></a> `downSampleFactor` | `number` | Downsampling factor for mask/blurring, must be [2,4]. 2 = half resolution, 4 = quarter resolution for faster filtering. Use for performance tuning vs. quality. **Default** `2` |
@@ -1,6 +1,6 @@
1
1
  ## Extends
2
2
 
3
- - [`Process`](Process.mdx)
3
+ - [`Process`](Process.mdx)\<`Partial`\<[`RendererOptions`](RendererOptions.mdx)\>\>
4
4
 
5
5
  ## Properties
6
6
 
@@ -47,9 +47,12 @@ destroy(): Promise<void>;
47
47
  ### open()
48
48
 
49
49
  ```ts
50
- open(options?): Promise<void>;
50
+ open(options?): Promise<Partial<RendererOptions>>;
51
51
  ```
52
52
 
53
+ Initialize the Segmenter as well as the underlying worker.
54
+ It is expected to be called once.
55
+
53
56
  #### Parameters
54
57
 
55
58
  | Parameter | Type |
@@ -58,7 +61,7 @@ open(options?): Promise<void>;
58
61
 
59
62
  #### Returns
60
63
 
61
- `Promise`\<`void`\>
64
+ `Promise`\<`Partial`\<[`RendererOptions`](RendererOptions.mdx)\>\>
62
65
 
63
66
  #### Overrides
64
67
 
@@ -69,9 +72,11 @@ open(options?): Promise<void>;
69
72
  ### update()
70
73
 
71
74
  ```ts
72
- update(options): Promise<void>;
75
+ update(options): Promise<Partial<RendererOptions>>;
73
76
  ```
74
77
 
78
+ Update the processor options
79
+
75
80
  #### Parameters
76
81
 
77
82
  | Parameter | Type |
@@ -80,26 +85,94 @@ update(options): Promise<void>;
80
85
 
81
86
  #### Returns
82
87
 
83
- `Promise`\<`void`\>
88
+ `Promise`\<`Partial`\<[`RendererOptions`](RendererOptions.mdx)\>\>
84
89
 
85
90
  ***
86
91
 
87
- ### process()
92
+ ### mute()
88
93
 
89
94
  ```ts
90
- process(input, options?): Promise<VideoFrameLike | undefined>;
95
+ mute(muted): void;
91
96
  ```
92
97
 
98
+ Mute/unmute the track in the worker
99
+
93
100
  #### Parameters
94
101
 
95
102
  | Parameter | Type |
96
103
  | ------ | ------ |
97
- | `input` | [`VideoFrameLike`](VideoFrameLike.mdx) |
98
- | `options?` | [`RendererOptions`](RendererOptions.mdx) |
104
+ | `muted` | `boolean` |
99
105
 
100
106
  #### Returns
101
107
 
102
- `Promise`\<[`VideoFrameLike`](VideoFrameLike.mdx) \| `undefined`\>
108
+ `void`
109
+
110
+ ***
111
+
112
+ ### process()
113
+
114
+ #### Call Signature
115
+
116
+ ```ts
117
+ process(input, options?): Promise<MediaStreamTrack>;
118
+ ```
119
+
120
+ Process provided input
121
+
122
+ ##### Parameters
123
+
124
+ | Parameter | Type |
125
+ | ------ | ------ |
126
+ | `input` | `MediaStreamTrack` |
127
+ | `options?` | [`RendererOptions`](RendererOptions.mdx) |
128
+
129
+ ##### Returns
130
+
131
+ `Promise`\<`MediaStreamTrack`\>
132
+
133
+ #### Call Signature
134
+
135
+ ```ts
136
+ process(input, options?): Promise<
137
+ | ReadWritableStream<VideoFrame>
138
+ | undefined>;
139
+ ```
140
+
141
+ ##### Parameters
142
+
143
+ | Parameter | Type |
144
+ | ------ | ------ |
145
+ | `input` | [`ReadWritableStream`](ReadWritableStream.mdx)\<`VideoFrame`\> |
146
+ | `options?` | [`RendererOptions`](RendererOptions.mdx) |
147
+
148
+ ##### Returns
149
+
150
+ `Promise`\<
151
+ \| [`ReadWritableStream`](ReadWritableStream.mdx)\<`VideoFrame`\>
152
+ \| `undefined`\>
153
+
154
+ #### Call Signature
155
+
156
+ ```ts
157
+ process(input, options?): Promise<
158
+ | MediaStreamTrack
159
+ | ReadWritableStream<VideoFrame>
160
+ | undefined>;
161
+ ```
162
+
163
+ ##### Parameters
164
+
165
+ | Parameter | Type |
166
+ | ------ | ------ |
167
+ | `input` | \| `MediaStreamTrack` \| [`ReadWritableStream`](ReadWritableStream.mdx)\<`VideoFrame`\> |
168
+ | `options?` | [`RendererOptions`](RendererOptions.mdx) |
169
+
170
+ ##### Returns
171
+
172
+ `Promise`\<
173
+ \| `MediaStreamTrack`
174
+ \| [`ReadWritableStream`](ReadWritableStream.mdx)\<`VideoFrame`\>
175
+ \| `undefined`\>
103
176
 
104
177
  ***
105
178
 
@@ -109,6 +182,8 @@ process(input, options?): Promise<VideoFrameLike | undefined>;
109
182
  subscribe(event, callback): Unsubscribe;
110
183
  ```
111
184
 
185
+ Subscribe rendering events,
186
+
112
187
  #### Parameters
113
188
 
114
189
  | Parameter | Type |
@@ -119,3 +194,7 @@ subscribe(event, callback): Unsubscribe;
119
194
  #### Returns
120
195
 
121
196
  `Unsubscribe`
197
+
198
+ #### See
199
+
200
+ RenderingEvents
@@ -7,7 +7,6 @@
7
7
  | Property | Type |
8
8
  | ------ | ------ |
9
9
  | <a id="basepath"></a> `basePath?` | `string` |
10
- | <a id="output"></a> `output?` | `OffscreenCanvas` \| `null` |
11
10
  | <a id="processingwidth"></a> `processingWidth?` | `number` |
12
11
  | <a id="processingheight"></a> `processingHeight?` | `number` |
13
12
  | <a id="imagesegmenteroptions"></a> `imageSegmenterOptions?` | [`ImageSegmenterOptions`](ImageSegmenterOptions.mdx) |
@@ -0,0 +1,15 @@
1
+ ## Extends
2
+
3
+ - [`Weights`](Weights.mdx)
4
+
5
+ ## Properties
6
+
7
+ | Property | Type | Inherited from |
8
+ | ------ | ------ | ------ |
9
+ | <a id="iou"></a> `iou` | `number` | [`Weights`](Weights.mdx).[`iou`](Weights.mdx#iou) |
10
+ | <a id="area"></a> `area` | `number` | [`Weights`](Weights.mdx).[`area`](Weights.mdx#area) |
11
+ | <a id="height"></a> `height` | `number` | [`Weights`](Weights.mdx).[`height`](Weights.mdx#height) |
12
+ | <a id="center"></a> `center` | `number` | [`Weights`](Weights.mdx).[`center`](Weights.mdx#center) |
13
+ | <a id="centerx"></a> `centerX` | `number` | - |
14
+ | <a id="centery"></a> `centerY` | `number` | - |
15
+ | <a id="decay"></a> `decay` | `number` | - |
@@ -0,0 +1,11 @@
1
+ ## Properties
2
+
3
+ | Property | Type |
4
+ | ------ | ------ |
5
+ | <a id="sumx"></a> `sumX` | `number` |
6
+ | <a id="sumy"></a> `sumY` | `number` |
7
+ | <a id="count"></a> `count` | `number` |
8
+ | <a id="minx"></a> `minX` | `number` |
9
+ | <a id="maxx"></a> `maxX` | `number` |
10
+ | <a id="miny"></a> `minY` | `number` |
11
+ | <a id="maxy"></a> `maxY` | `number` |
@@ -1,9 +1,13 @@
1
1
  ## Properties
2
2
 
3
- | Property | Type | Description |
4
- | ------ | ------ | ------ |
5
- | <a id="process"></a> `process` | (`source`) => `Promise`\<`MediaStream`\> | Process the video and return a MediaStream |
6
- | <a id="destroy"></a> `destroy` | () => `Promise`\<`void`\> | Destroy the segmentation wasm instance |
3
+ | Property | Modifier | Type | Description |
4
+ | ------ | ------ | ------ | ------ |
5
+ | <a id="status"></a> `status` | `readonly` | \| `"closed"` \| `"closing"` \| `"Error"` \| `"destroyed"` \| `"destroying"` \| `"idle"` \| `"new"` \| `"opened"` \| `"opening"` \| `"processing"` \| `"updated"` \| `"updating"` | - |
6
+ | <a id="process"></a> `process` | `public` | (`source`) => `Promise`\<`MediaStreamTrack`\> | Process the video and return a MediaStream |
7
+ | <a id="destroy"></a> `destroy` | `public` | () => `Promise`\<`void`\> | Destroy the segmentation wasm instance |
8
+ | <a id="renderoptions"></a> `renderOptions` | `readonly` | [`RendererOptions`](RendererOptions.mdx) | Current renderer options |
9
+ | <a id="processingwidth"></a> `processingWidth` | `readonly` | `number` | Current processing width |
10
+ | <a id="processingheight"></a> `processingHeight` | `readonly` | `number` | Current processing height |
7
11
 
8
12
  ## Methods
9
13
 
@@ -13,11 +17,33 @@
13
17
  open(options?): Promise<void>;
14
18
  ```
15
19
 
20
+ Initialize the video processor and related
21
+
22
+ #### Parameters
23
+
24
+ | Parameter | Type |
25
+ | ------ | ------ |
26
+ | `options?` | [`ProcessorUpdateOptions`](ProcessorUpdateOptions.mdx) |
27
+
28
+ #### Returns
29
+
30
+ `Promise`\<`void`\>
31
+
32
+ ***
33
+
34
+ ### update()
35
+
36
+ ```ts
37
+ update(options): Promise<void>;
38
+ ```
39
+
40
+ Update the video processor with provided options
41
+
16
42
  #### Parameters
17
43
 
18
44
  | Parameter | Type |
19
45
  | ------ | ------ |
20
- | `options?` | [`ProcessorOptions`](ProcessorOptions.mdx) |
46
+ | `options` | [`ProcessorUpdateOptions`](ProcessorUpdateOptions.mdx) |
21
47
 
22
48
  #### Returns
23
49
 
@@ -25,6 +51,24 @@ open(options?): Promise<void>;
25
51
 
26
52
  ***
27
53
 
54
+ ### mute()
55
+
56
+ ```ts
57
+ mute(muted): void;
58
+ ```
59
+
60
+ #### Parameters
61
+
62
+ | Parameter | Type |
63
+ | ------ | ------ |
64
+ | `muted` | `boolean` |
65
+
66
+ #### Returns
67
+
68
+ `void`
69
+
70
+ ***
71
+
28
72
  ### close()
29
73
 
30
74
  ```ts