@pexip/media-processor 21.0.0 → 22.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (115) hide show
  1. package/CHANGELOG.md +70 -0
  2. package/README.md +17 -10
  3. package/api-docs/README.mdx +37 -18
  4. package/api-docs/functions/assertMediaStreamVideoTrack.mdx +13 -0
  5. package/api-docs/functions/createFrameCallbackRequest.mdx +8 -2
  6. package/api-docs/functions/createMediaStreamTrackInputProcessAdaptor.mdx +10 -0
  7. package/api-docs/functions/createReadWritableStreamFallbackInputProcessAdaptor.mdx +17 -0
  8. package/api-docs/functions/createReadWritableStreamInputProcessAdaptor.mdx +10 -0
  9. package/api-docs/functions/createVideoProcessor.mdx +3 -3
  10. package/api-docs/functions/getDefaultInputProcessAdaptor.mdx +17 -0
  11. package/api-docs/functions/isMediaStreamVideoTrack.mdx +13 -0
  12. package/api-docs/functions/playVideo.mdx +2 -2
  13. package/api-docs/interfaces/Analyzer.mdx +4 -0
  14. package/api-docs/interfaces/Benchmark.mdx +14 -2
  15. package/api-docs/interfaces/FrameTransformStats.mdx +9 -0
  16. package/api-docs/interfaces/Gain.mdx +4 -0
  17. package/api-docs/interfaces/ImageSegmenterOptions.mdx +0 -2
  18. package/api-docs/interfaces/InputProcessAdaptor.mdx +28 -0
  19. package/api-docs/interfaces/Process.mdx +8 -3
  20. package/api-docs/interfaces/ProcessEventMute.mdx +6 -0
  21. package/api-docs/interfaces/ProcessWorkerEventOpened.mdx +7 -0
  22. package/api-docs/interfaces/ProcessWorkerEventStats.mdx +6 -0
  23. package/api-docs/interfaces/ProcessWorkerEventUpdated.mdx +7 -0
  24. package/api-docs/interfaces/ProcessorEvent.mdx +1 -1
  25. package/api-docs/interfaces/ProcessorOptions.mdx +0 -1
  26. package/api-docs/interfaces/ProcessorProcessOptions.mdx +3 -1
  27. package/api-docs/interfaces/ProcessorUpdateOptions.mdx +15 -0
  28. package/api-docs/interfaces/ReadWritableStream.mdx +12 -0
  29. package/api-docs/interfaces/Renderer.mdx +2 -2
  30. package/api-docs/interfaces/RendererOptions.mdx +0 -4
  31. package/api-docs/interfaces/Segmenter.mdx +89 -10
  32. package/api-docs/interfaces/SegmenterOptions.mdx +0 -1
  33. package/api-docs/interfaces/VideoProcessor.mdx +49 -5
  34. package/api-docs/type-aliases/ProcessEvents.mdx +1 -0
  35. package/api-docs/type-aliases/ProcessInputType.mdx +3 -1
  36. package/api-docs/type-aliases/ProcessWorkerEvents.mdx +6 -4
  37. package/api-docs/type-aliases/ProcessorWorkerEvents.mdx +1 -1
  38. package/api-docs/type-aliases/Struct.mdx +9 -0
  39. package/api-docs/variables/FRAME_RATE.mdx +1 -1
  40. package/api-docs/variables/ImageFetchError.mdx +3 -0
  41. package/api-docs/variables/MediaStreamTrackTransferable.mdx +5 -0
  42. package/api-docs/variables/RENDERER_OPTIONS_STRUCT.mdx +3 -0
  43. package/api-docs/variables/cloneImageRecord.mdx +19 -0
  44. package/api-docs/variables/compareImageRecords.mdx +18 -0
  45. package/api-docs/variables/createRemoteImageBitmap.mdx +1 -1
  46. package/api-docs/variables/isRenderOptionsKey.mdx +13 -0
  47. package/api-docs/variables/isValidGpuAPI.mdx +13 -0
  48. package/dist/common/backends/webgl/renderer.js +2 -12
  49. package/dist/common/backends/webgl/texture.d.ts +1 -2
  50. package/dist/common/backends/webgl/texture.js +2 -2
  51. package/dist/common/backends/webgl/webglUtils.js +8 -4
  52. package/dist/common/backends/webgpu/renderer.js +2 -12
  53. package/dist/common/backends/webgpu/tentBlur.js +1 -1
  54. package/dist/common/backends/webgpu/texture.d.ts +1 -2
  55. package/dist/common/backends/webgpu/texture.js +6 -2
  56. package/dist/common/constants.d.ts +3 -0
  57. package/dist/common/constants.js +4 -0
  58. package/dist/common/index.d.ts +1 -0
  59. package/dist/common/index.js +1 -0
  60. package/dist/common/tsconfig.tsbuildinfo +1 -1
  61. package/dist/common/typeGuards.d.ts +2 -0
  62. package/dist/common/typeGuards.js +7 -0
  63. package/dist/common/types/media.d.ts +5 -1
  64. package/dist/common/types/messageEvents.d.ts +31 -4
  65. package/dist/common/types/processor.d.ts +6 -4
  66. package/dist/common/types/render.d.ts +5 -2
  67. package/dist/common/types/render.js +23 -1
  68. package/dist/common/types/segmentation.d.ts +0 -12
  69. package/dist/common/utils.d.ts +13 -2
  70. package/dist/common/utils.js +54 -10
  71. package/dist/main/audio.js +3 -3
  72. package/dist/main/benchUtils.d.ts +2 -2
  73. package/dist/main/benchUtils.js +4 -4
  74. package/dist/main/index.d.ts +1 -1
  75. package/dist/main/index.js +1 -1
  76. package/dist/main/tsconfig.tsbuildinfo +1 -1
  77. package/dist/main/types.d.ts +0 -6
  78. package/dist/main/utils.d.ts +4 -0
  79. package/dist/main/utils.js +31 -0
  80. package/dist/main/video/constants.d.ts +1 -4
  81. package/dist/main/video/constants.js +1 -5
  82. package/dist/main/video/index.d.ts +2 -3
  83. package/dist/main/video/index.js +2 -3
  84. package/dist/main/video/inputProcessAdaptor.d.ts +30 -0
  85. package/dist/main/video/inputProcessAdaptor.js +198 -0
  86. package/dist/main/video/segmenter.d.ts +35 -0
  87. package/dist/main/video/{segmenters/mediapipe.js → segmenter.js} +93 -53
  88. package/dist/main/video/typeGuards.d.ts +2 -0
  89. package/dist/main/video/typeGuards.js +8 -1
  90. package/dist/main/video/types.d.ts +36 -16
  91. package/dist/main/video/utils.d.ts +5 -4
  92. package/dist/main/video/utils.js +49 -44
  93. package/dist/main/video/video.d.ts +29 -7
  94. package/dist/main/video/video.js +323 -28
  95. package/dist/workers/mediaWorker.js +23 -23
  96. package/dist/workers/tsconfig.tsbuildinfo +1 -1
  97. package/dist/worklets/tsconfig.tsbuildinfo +1 -1
  98. package/package.json +7 -7
  99. package/api-docs/functions/createCanvasTransform.mdx +0 -14
  100. package/api-docs/functions/createVideoTrackProcessor.mdx +0 -9
  101. package/api-docs/functions/createVideoTrackProcessorWithFallback.mdx +0 -15
  102. package/api-docs/interfaces/Detector.mdx +0 -81
  103. package/api-docs/interfaces/SegmentationParams.mdx +0 -35
  104. package/api-docs/interfaces/Transform.mdx +0 -62
  105. package/api-docs/type-aliases/ProcessVideoTrack.mdx +0 -15
  106. package/api-docs/type-aliases/SegmentationTransform.mdx +0 -3
  107. package/dist/main/transformer.d.ts +0 -1
  108. package/dist/main/transformer.js +0 -4
  109. package/dist/main/video/canvasTransform.d.ts +0 -17
  110. package/dist/main/video/canvasTransform.js +0 -341
  111. package/dist/main/video/segmenters/mediapipe.d.ts +0 -22
  112. package/dist/main/video/transformer.d.ts +0 -9
  113. package/dist/main/video/transformer.js +0 -68
  114. package/dist/main/video/videoStreamTrackProcessor.d.ts +0 -20
  115. package/dist/main/video/videoStreamTrackProcessor.js +0 -86
@@ -0,0 +1,15 @@
1
+ ## Extends
2
+
3
+ - `Omit`\<[`ProcessorOptions`](ProcessorOptions.mdx), `"renderOptions"`\>
4
+
5
+ ## Properties
6
+
7
+ | Property | Type | Inherited from |
8
+ | ------ | ------ | ------ |
9
+ | <a id="backgroundimage"></a> `backgroundImage?` | [`ImageRecord`](ImageRecord.mdx) | [`ProcessorOptions`](ProcessorOptions.mdx).[`backgroundImage`](ProcessorOptions.mdx#backgroundimage) |
10
+ | <a id="restart"></a> `restart?` | `boolean` | [`ProcessorOptions`](ProcessorOptions.mdx).[`restart`](ProcessorOptions.mdx#restart) |
11
+ | <a id="basepath"></a> `basePath?` | `string` | [`SegmenterOptions`](SegmenterOptions.mdx).[`basePath`](SegmenterOptions.mdx#basepath) |
12
+ | <a id="processingwidth"></a> `processingWidth?` | `number` | [`SegmenterOptions`](SegmenterOptions.mdx).[`processingWidth`](SegmenterOptions.mdx#processingwidth) |
13
+ | <a id="processingheight"></a> `processingHeight?` | `number` | [`SegmenterOptions`](SegmenterOptions.mdx).[`processingHeight`](SegmenterOptions.mdx#processingheight) |
14
+ | <a id="imagesegmenteroptions"></a> `imageSegmenterOptions?` | [`ImageSegmenterOptions`](ImageSegmenterOptions.mdx) | [`SegmenterOptions`](SegmenterOptions.mdx).[`imageSegmenterOptions`](SegmenterOptions.mdx#imagesegmenteroptions) |
15
+ | <a id="renderoptions"></a> `renderOptions?` | `Partial`\<[`RendererOptions`](RendererOptions.mdx)\> | - |
@@ -0,0 +1,12 @@
1
+ ## Type Parameters
2
+
3
+ | Type Parameter |
4
+ | ------ |
5
+ | `T` |
6
+
7
+ ## Properties
8
+
9
+ | Property | Type |
10
+ | ------ | ------ |
11
+ | <a id="readable"></a> `readable` | `ReadableStream`\<`T`\> |
12
+ | <a id="writable"></a> `writable` | `WritableStream`\<`T`\> |
@@ -31,8 +31,8 @@ render(
31
31
 
32
32
  | Parameter | Type |
33
33
  | ------ | ------ |
34
- | `mask` | `ImageBitmap` |
35
- | `frame` | [`VideoFrameLike`](VideoFrameLike.mdx) |
34
+ | `mask` | `OffscreenCanvas` |
35
+ | `frame` | `VideoFrame` |
36
36
  | `options` | `Options` |
37
37
 
38
38
  #### Returns
@@ -1,10 +1,6 @@
1
1
  Options controlling the rendering and effects for person/background segmentation.
2
2
  All values are stable per-frame and can be safely updated in real-time.
3
3
 
4
- ## Extended by
5
-
6
- - [`SegmentationParams`](SegmentationParams.mdx)
7
-
8
4
  ## Properties
9
5
 
10
6
  | Property | Type | Description |
@@ -1,6 +1,6 @@
1
1
  ## Extends
2
2
 
3
- - [`Process`](Process.mdx)
3
+ - [`Process`](Process.mdx)\<`Partial`\<[`RendererOptions`](RendererOptions.mdx)\>\>
4
4
 
5
5
  ## Properties
6
6
 
@@ -47,9 +47,12 @@ destroy(): Promise<void>;
47
47
  ### open()
48
48
 
49
49
  ```ts
50
- open(options?): Promise<void>;
50
+ open(options?): Promise<Partial<RendererOptions>>;
51
51
  ```
52
52
 
53
+ Initialize the Segmenter as well as the underlying worker.
54
+ It is expected to be called once.
55
+
53
56
  #### Parameters
54
57
 
55
58
  | Parameter | Type |
@@ -58,7 +61,7 @@ open(options?): Promise<void>;
58
61
 
59
62
  #### Returns
60
63
 
61
- `Promise`\<`void`\>
64
+ `Promise`\<`Partial`\<[`RendererOptions`](RendererOptions.mdx)\>\>
62
65
 
63
66
  #### Overrides
64
67
 
@@ -69,9 +72,11 @@ open(options?): Promise<void>;
69
72
  ### update()
70
73
 
71
74
  ```ts
72
- update(options): Promise<void>;
75
+ update(options): Promise<Partial<RendererOptions>>;
73
76
  ```
74
77
 
78
+ Update the processor options
79
+
75
80
  #### Parameters
76
81
 
77
82
  | Parameter | Type |
@@ -80,26 +85,94 @@ update(options): Promise<void>;
80
85
 
81
86
  #### Returns
82
87
 
83
- `Promise`\<`void`\>
88
+ `Promise`\<`Partial`\<[`RendererOptions`](RendererOptions.mdx)\>\>
84
89
 
85
90
  ***
86
91
 
87
- ### process()
92
+ ### mute()
88
93
 
89
94
  ```ts
90
- process(input, options?): Promise<VideoFrameLike | undefined>;
95
+ mute(muted): void;
91
96
  ```
92
97
 
98
+ Mute/unmute the track in the worker
99
+
93
100
  #### Parameters
94
101
 
95
102
  | Parameter | Type |
96
103
  | ------ | ------ |
97
- | `input` | [`VideoFrameLike`](VideoFrameLike.mdx) |
98
- | `options?` | [`RendererOptions`](RendererOptions.mdx) |
104
+ | `muted` | `boolean` |
99
105
 
100
106
  #### Returns
101
107
 
102
- `Promise`\<[`VideoFrameLike`](VideoFrameLike.mdx) \| `undefined`\>
108
+ `void`
109
+
110
+ ***
111
+
112
+ ### process()
113
+
114
+ #### Call Signature
115
+
116
+ ```ts
117
+ process(input, options?): Promise<MediaStreamTrack>;
118
+ ```
119
+
120
+ Process provided input
121
+
122
+ ##### Parameters
123
+
124
+ | Parameter | Type |
125
+ | ------ | ------ |
126
+ | `input` | `MediaStreamTrack` |
127
+ | `options?` | [`RendererOptions`](RendererOptions.mdx) |
128
+
129
+ ##### Returns
130
+
131
+ `Promise`\<`MediaStreamTrack`\>
132
+
133
+ #### Call Signature
134
+
135
+ ```ts
136
+ process(input, options?): Promise<
137
+ | ReadWritableStream<VideoFrame>
138
+ | undefined>;
139
+ ```
140
+
141
+ ##### Parameters
142
+
143
+ | Parameter | Type |
144
+ | ------ | ------ |
145
+ | `input` | [`ReadWritableStream`](ReadWritableStream.mdx)\<`VideoFrame`\> |
146
+ | `options?` | [`RendererOptions`](RendererOptions.mdx) |
147
+
148
+ ##### Returns
149
+
150
+ `Promise`\<
151
+ \| [`ReadWritableStream`](ReadWritableStream.mdx)\<`VideoFrame`\>
152
+ \| `undefined`\>
153
+
154
+ #### Call Signature
155
+
156
+ ```ts
157
+ process(input, options?): Promise<
158
+ | MediaStreamTrack
159
+ | ReadWritableStream<VideoFrame>
160
+ | undefined>;
161
+ ```
162
+
163
+ ##### Parameters
164
+
165
+ | Parameter | Type |
166
+ | ------ | ------ |
167
+ | `input` | \| `MediaStreamTrack` \| [`ReadWritableStream`](ReadWritableStream.mdx)\<`VideoFrame`\> |
168
+ | `options?` | [`RendererOptions`](RendererOptions.mdx) |
169
+
170
+ ##### Returns
171
+
172
+ `Promise`\<
173
+ \| `MediaStreamTrack`
174
+ \| [`ReadWritableStream`](ReadWritableStream.mdx)\<`VideoFrame`\>
175
+ \| `undefined`\>
103
176
 
104
177
  ***
105
178
 
@@ -109,6 +182,8 @@ process(input, options?): Promise<VideoFrameLike | undefined>;
109
182
  subscribe(event, callback): Unsubscribe;
110
183
  ```
111
184
 
185
+ Subscribe rendering events,
186
+
112
187
  #### Parameters
113
188
 
114
189
  | Parameter | Type |
@@ -119,3 +194,7 @@ subscribe(event, callback): Unsubscribe;
119
194
  #### Returns
120
195
 
121
196
  `Unsubscribe`
197
+
198
+ #### See
199
+
200
+ RenderingEvents
@@ -7,7 +7,6 @@
7
7
  | Property | Type |
8
8
  | ------ | ------ |
9
9
  | <a id="basepath"></a> `basePath?` | `string` |
10
- | <a id="output"></a> `output?` | `OffscreenCanvas` \| `null` |
11
10
  | <a id="processingwidth"></a> `processingWidth?` | `number` |
12
11
  | <a id="processingheight"></a> `processingHeight?` | `number` |
13
12
  | <a id="imagesegmenteroptions"></a> `imageSegmenterOptions?` | [`ImageSegmenterOptions`](ImageSegmenterOptions.mdx) |
@@ -1,9 +1,13 @@
1
1
  ## Properties
2
2
 
3
- | Property | Type | Description |
4
- | ------ | ------ | ------ |
5
- | <a id="process"></a> `process` | (`source`) => `Promise`\<`MediaStream`\> | Process the video and return a MediaStream |
6
- | <a id="destroy"></a> `destroy` | () => `Promise`\<`void`\> | Destroy the segmentation wasm instance |
3
+ | Property | Modifier | Type | Description |
4
+ | ------ | ------ | ------ | ------ |
5
+ | <a id="status"></a> `status` | `readonly` | \| `"closed"` \| `"closing"` \| `"Error"` \| `"destroyed"` \| `"destroying"` \| `"idle"` \| `"new"` \| `"opened"` \| `"opening"` \| `"processing"` \| `"updated"` \| `"updating"` | - |
6
+ | <a id="process"></a> `process` | `public` | (`source`) => `Promise`\<`MediaStreamTrack`\> | Process the video and return a MediaStream |
7
+ | <a id="destroy"></a> `destroy` | `public` | () => `Promise`\<`void`\> | Destroy the segmentation wasm instance |
8
+ | <a id="renderoptions"></a> `renderOptions` | `readonly` | [`RendererOptions`](RendererOptions.mdx) | Current renderer options |
9
+ | <a id="processingwidth"></a> `processingWidth` | `readonly` | `number` | Current processing width |
10
+ | <a id="processingheight"></a> `processingHeight` | `readonly` | `number` | Current processing height |
7
11
 
8
12
  ## Methods
9
13
 
@@ -13,11 +17,33 @@
13
17
  open(options?): Promise<void>;
14
18
  ```
15
19
 
20
+ Initialize the video processor and related
21
+
22
+ #### Parameters
23
+
24
+ | Parameter | Type |
25
+ | ------ | ------ |
26
+ | `options?` | [`ProcessorUpdateOptions`](ProcessorUpdateOptions.mdx) |
27
+
28
+ #### Returns
29
+
30
+ `Promise`\<`void`\>
31
+
32
+ ***
33
+
34
+ ### update()
35
+
36
+ ```ts
37
+ update(options): Promise<void>;
38
+ ```
39
+
40
+ Update the video processor with provided options
41
+
16
42
  #### Parameters
17
43
 
18
44
  | Parameter | Type |
19
45
  | ------ | ------ |
20
- | `options?` | [`ProcessorOptions`](ProcessorOptions.mdx) |
46
+ | `options` | [`ProcessorUpdateOptions`](ProcessorUpdateOptions.mdx) |
21
47
 
22
48
  #### Returns
23
49
 
@@ -25,6 +51,24 @@ open(options?): Promise<void>;
25
51
 
26
52
  ***
27
53
 
54
+ ### mute()
55
+
56
+ ```ts
57
+ mute(muted): void;
58
+ ```
59
+
60
+ #### Parameters
61
+
62
+ | Parameter | Type |
63
+ | ------ | ------ |
64
+ | `muted` | `boolean` |
65
+
66
+ #### Returns
67
+
68
+ `void`
69
+
70
+ ***
71
+
28
72
  ### close()
29
73
 
30
74
  ```ts
@@ -4,6 +4,7 @@ type ProcessEvents<T, I> =
4
4
  | ProcessEventProcess<I>
5
5
  | ProcessEventUpdate<T>
6
6
  | ProcessEventClose
7
+ | ProcessEventMute
7
8
  | ProcessEventDestroy;
8
9
  ```
9
10
 
@@ -1,3 +1,5 @@
1
1
  ```ts
2
- type ProcessInputType = VideoFrame | ImageData | OffscreenCanvas | ImageBitmap;
2
+ type ProcessInputType =
3
+ | MediaStreamTrack
4
+ | ReadWritableStream<VideoFrame>;
3
5
  ```
@@ -1,10 +1,11 @@
1
1
  ```ts
2
- type ProcessWorkerEvents<T, E, D> =
2
+ type ProcessWorkerEvents<T, E, D, U> =
3
3
  | ProcessWorkerEventProcessed<T>
4
- | ProcessWorkerEventOpened
5
- | ProcessWorkerEventUpdated
4
+ | ProcessWorkerEventOpened<U>
5
+ | ProcessWorkerEventUpdated<U>
6
6
  | ProcessWorkerEvent<E>
7
- | ProcessWorkerEventDebug<D>;
7
+ | ProcessWorkerEventDebug<D>
8
+ | ProcessWorkerEventStats;
8
9
  ```
9
10
 
10
11
  Message events returned from the processor
@@ -16,3 +17,4 @@ Message events returned from the processor
16
17
  | `T` |
17
18
  | `E` |
18
19
  | `D` |
20
+ | `U` |
@@ -1,3 +1,3 @@
1
1
  ```ts
2
- type ProcessorWorkerEvents = ProcessWorkerEvents<VideoFrameLike, ProcessorEvent, ProcessWorkerDebugMessage>;
2
+ type ProcessorWorkerEvents = ProcessWorkerEvents<MediaStreamTrack, ProcessorEvent, ProcessWorkerDebugMessage, Partial<RendererOptions>>;
3
3
  ```
@@ -0,0 +1,9 @@
1
+ ```ts
2
+ type Struct<Type> = { [Key in keyof Type]: "number" | "boolean" | "string" | "object" };
3
+ ```
4
+
5
+ ## Type Parameters
6
+
7
+ | Type Parameter |
8
+ | ------ |
9
+ | `Type` |
@@ -1,3 +1,3 @@
1
1
  ```ts
2
- const FRAME_RATE: 20 = 20;
2
+ const FRAME_RATE: 30 = 30;
3
3
  ```
@@ -0,0 +1,3 @@
1
+ ```ts
2
+ const ImageFetchError: "ImageFetchError" = "ImageFetchError";
3
+ ```
@@ -0,0 +1,5 @@
1
+ ```ts
2
+ const MediaStreamTrackTransferable: boolean;
3
+ ```
4
+
5
+ Result of the feature detection if `MediaStreamTrack` is `Transferable`
@@ -0,0 +1,3 @@
1
+ ```ts
2
+ const RENDERER_OPTIONS_STRUCT: Struct<RendererOptions>;
3
+ ```
@@ -0,0 +1,19 @@
1
+ ```ts
2
+ const cloneImageRecord: (image) => Promise<{
3
+ image: ImageBitmap;
4
+ key: string;
5
+ }>;
6
+ ```
7
+
8
+ ## Parameters
9
+
10
+ | Parameter | Type |
11
+ | ------ | ------ |
12
+ | `image` | [`ImageRecord`](../interfaces/ImageRecord.mdx) |
13
+
14
+ ## Returns
15
+
16
+ `Promise`\<\{
17
+ `image`: `ImageBitmap`;
18
+ `key`: `string`;
19
+ \}\>
@@ -0,0 +1,18 @@
1
+ ```ts
2
+ const compareImageRecords: (a, b) => boolean;
3
+ ```
4
+
5
+ Compare provided ImageRecord
6
+
7
+ ## Parameters
8
+
9
+ | Parameter | Type |
10
+ | ------ | ------ |
11
+ | `a` | [`ImageRecord`](../interfaces/ImageRecord.mdx) |
12
+ | `b` | [`ImageRecord`](../interfaces/ImageRecord.mdx) \| `undefined` |
13
+
14
+ ## Returns
15
+
16
+ `boolean`
17
+
18
+ true if the two records have the same key and image, false otherwise
@@ -14,7 +14,7 @@ const createRemoteImageBitmap: (map?) => object;
14
14
 
15
15
  | Name | Type |
16
16
  | ------ | ------ |
17
- | `tryFetchingImage()` | (`uri`) => `Promise`\<`ImageBitmap` \| `null`\> |
17
+ | `tryFetchingImage()` | (`uri`, `abortSignal`) => `Promise`\<`ImageBitmap` \| `null` \| `undefined`\> |
18
18
  | `setImage()` | (`uri`, `image`) => `void` |
19
19
  | `getImage()` | (`uri`) => `ImageBitmap` \| `null` |
20
20
  | `removeImage()` | (`uri`) => `boolean` |
@@ -0,0 +1,13 @@
1
+ ```ts
2
+ const isRenderOptionsKey: (k) => k is keyof RendererOptions;
3
+ ```
4
+
5
+ ## Parameters
6
+
7
+ | Parameter | Type |
8
+ | ------ | ------ |
9
+ | `k` | `unknown` |
10
+
11
+ ## Returns
12
+
13
+ `k is keyof RendererOptions`
@@ -0,0 +1,13 @@
1
+ ```ts
2
+ const isValidGpuAPI: (t) => t is RenderBackend;
3
+ ```
4
+
5
+ ## Parameters
6
+
7
+ | Parameter | Type |
8
+ | ------ | ------ |
9
+ | `t` | `unknown` |
10
+
11
+ ## Returns
12
+
13
+ `t is RenderBackend`
@@ -178,12 +178,7 @@ export const createRenderer = (renderEventHandlers, canvas = new OffscreenCanvas
178
178
  assert(!props.gl?.isContextLost(), RENDERING_EVENTS.ContextLost);
179
179
  // 1. Upload current frame and raw mask
180
180
  const frameTexture = props.videoFrameRenderer.render(frame);
181
- const maskTexture = props.maskRenderer.render({
182
- frame: mask,
183
- width: mask.width,
184
- height: mask.height,
185
- timestamp: frame.timestamp,
186
- });
181
+ const maskTexture = props.maskRenderer.render(mask);
187
182
  // 2. Joint Bilateral Filter
188
183
  let smoothedMask;
189
184
  let compositeMaskInput;
@@ -240,12 +235,7 @@ export const createRenderer = (renderEventHandlers, canvas = new OffscreenCanvas
240
235
  }, options);
241
236
  }
242
237
  else if (options?.effects === 'overlay' && options.backgroundImage) {
243
- const background = props.backgroundImageRenderer.render({
244
- frame: options.backgroundImage,
245
- timestamp: performance.now(),
246
- width: options.backgroundImage.width,
247
- height: options.backgroundImage.height,
248
- });
238
+ const background = props.backgroundImageRenderer.render(options.backgroundImage);
249
239
  outputFrameTexture = props.replaceRenderer.render({
250
240
  mask: compositeMaskInput,
251
241
  foreground: frameTexture,
@@ -1,9 +1,8 @@
1
- import type { VideoFrameLike } from '../../types/media';
2
1
  import type { TextureInfo } from './types';
3
2
  /**
4
3
  * Render Video frame into texture
5
4
  */
6
5
  export declare const createTextureRenderer: (processor: OffscreenCanvas, redOnly: boolean) => {
7
- render: (video: VideoFrameLike) => TextureInfo;
6
+ render: (frame: VideoFrame | ImageBitmap | OffscreenCanvas) => TextureInfo;
8
7
  release: () => void;
9
8
  };
@@ -59,7 +59,7 @@ export const createTextureRenderer = (processor, redOnly) => {
59
59
  : createRGBAFrameTexture(gl, processor.width, processor.height, true, floatFullChannelSupported);
60
60
  const frameBuffer = createFrameBuffer(gl, outputTexture);
61
61
  return {
62
- render: (video) => {
62
+ render: (frame) => {
63
63
  assert(!gl.isContextLost(), RENDERING_EVENTS.ContextLost);
64
64
  // Draw on the frame buffer
65
65
  gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer);
@@ -76,7 +76,7 @@ export const createTextureRenderer = (processor, redOnly) => {
76
76
  // Upload the frame to texture
77
77
  gl.activeTexture(gl.TEXTURE0);
78
78
  gl.bindTexture(gl.TEXTURE_2D, frameTexture);
79
- gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, video.frame);
79
+ gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, frame);
80
80
  gl.drawArrays(gl.TRIANGLES, 0, 6);
81
81
  // Cleanup
82
82
  gl.bindBuffer(gl.ARRAY_BUFFER, null);
@@ -92,6 +92,7 @@ export const loadShader = (gl, shaderType, shaderSource) => {
92
92
  */
93
93
  export const createProgram = (gl, shaders, attribs, locations) => {
94
94
  const program = gl.createProgram();
95
+ let released = false;
95
96
  if (!program) {
96
97
  throw new Error('Unable to create WebGL program');
97
98
  }
@@ -120,11 +121,14 @@ export const createProgram = (gl, shaders, attribs, locations) => {
120
121
  return [
121
122
  program,
122
123
  () => {
123
- for (const shader of shaders) {
124
- gl.detachShader(program, shader);
125
- gl.deleteShader(shader);
124
+ if (!released) {
125
+ released = true;
126
+ for (const shader of shaders) {
127
+ gl.detachShader(program, shader);
128
+ gl.deleteShader(shader);
129
+ }
130
+ gl.deleteProgram(program);
126
131
  }
127
- gl.deleteProgram(program);
128
132
  },
129
133
  ];
130
134
  };
@@ -195,12 +195,7 @@ export const createRenderer = (adapter, renderEventHandlers, canvas = new Offscr
195
195
  label: 'frame upload pass',
196
196
  });
197
197
  const frameTexture = props.videoFrameRenderer.render(frame, frameUploadPass);
198
- const maskTexture = props.maskRenderer.render({
199
- frame: mask,
200
- width: mask.width,
201
- height: mask.height,
202
- timestamp: frame.timestamp,
203
- }, frameUploadPass);
198
+ const maskTexture = props.maskRenderer.render(mask, frameUploadPass);
204
199
  frameUploadPass.end();
205
200
  let smoothedMask;
206
201
  let compositeInputMask;
@@ -253,12 +248,7 @@ export const createRenderer = (adapter, renderEventHandlers, canvas = new Offscr
253
248
  const bgImgUploadPass = encoder.beginComputePass({
254
249
  label: 'background image upload pass',
255
250
  });
256
- const bgTexture = props.backgroundImageRenderer.render({
257
- frame: options.backgroundImage,
258
- width: options.backgroundImage.width,
259
- height: options.backgroundImage.height,
260
- timestamp: frame.timestamp,
261
- }, bgImgUploadPass);
251
+ const bgTexture = props.backgroundImageRenderer.render(options.backgroundImage, bgImgUploadPass);
262
252
  bgImgUploadPass.end();
263
253
  const blurredBackgroundImageTexture = props.blurRenderer.render(encoder, bgTexture, options.backgroundBlurAmount);
264
254
  props.replaceRenderer.render(encoder, {
@@ -17,13 +17,13 @@ export const createTentBlurRenderer = (device, isContextLost, width, height, tex
17
17
  workgroupSizeX: WORKGROUP_SIZE_X,
18
18
  workgroupSizeY: WORKGROUP_SIZE_Y,
19
19
  });
20
- shaderHCode = enableF16InShader(shaderHCode);
21
20
  let shaderVCode = updateTextureStorageFormat(tentBlurVShader, textureFormat);
22
21
  shaderVCode = syncWorkgroupSizeWithShader(shaderVCode, {
23
22
  workgroupSizeX: WORKGROUP_SIZE_X,
24
23
  workgroupSizeY: WORKGROUP_SIZE_Y,
25
24
  });
26
25
  if (supportF16) {
26
+ shaderHCode = enableF16InShader(shaderHCode);
27
27
  shaderVCode = enableF16InShader(shaderVCode);
28
28
  }
29
29
  const defs = makeShaderDataDefinitions(shaderHCode);
@@ -1,5 +1,4 @@
1
- import type { VideoFrameLike } from '../../types/media';
2
1
  export declare const createTextureRenderer: (device: GPUDevice, isContextLost: () => boolean, textureFormat: GPUTextureFormat, width: number, height: number, filterMode: GPUFilterMode) => {
3
- render: (frame: VideoFrameLike, pass: GPUComputePassEncoder) => GPUTexture;
2
+ render: (frame: VideoFrame | ImageBitmap | OffscreenCanvas, pass: GPUComputePassEncoder) => GPUTexture;
4
3
  release: () => void;
5
4
  };
@@ -25,8 +25,12 @@ export const createTextureRenderer = (device, isContextLost, textureFormat, widt
25
25
  return {
26
26
  render: (frame, pass) => {
27
27
  assert(!isContextLost(), RENDERING_EVENTS.ContextLost);
28
- const texture = getOrCreateTexture(frame.width, frame.height);
29
- device.queue.copyExternalImageToTexture({ source: frame.frame, flipY: false }, { texture }, { width: frame.width, height: frame.height });
28
+ const width = frame instanceof VideoFrame ? frame.displayWidth : frame.width;
29
+ const height = frame instanceof VideoFrame
30
+ ? frame.displayHeight
31
+ : frame.height;
32
+ const texture = getOrCreateTexture(width, height);
33
+ device.queue.copyExternalImageToTexture({ source: frame, flipY: false }, { texture }, { width, height });
30
34
  return resizeRenderer.render(pass, texture);
31
35
  },
32
36
  release: () => {
@@ -75,4 +75,7 @@ export declare const RENDERING_EVENTS: {
75
75
  readonly SamplingError: "SamplingError";
76
76
  };
77
77
  export type RenderingEvents = ExtractValueTypes<typeof RENDERING_EVENTS>;
78
+ export declare enum AbortReason {
79
+ Close = "close"
80
+ }
78
81
  export {};
@@ -70,3 +70,7 @@ export const RENDERING_EVENTS = {
70
70
  */
71
71
  SamplingError: 'SamplingError',
72
72
  };
73
+ export var AbortReason;
74
+ (function (AbortReason) {
75
+ AbortReason["Close"] = "close";
76
+ })(AbortReason || (AbortReason = {}));
@@ -6,4 +6,5 @@ export * from './types/render';
6
6
  export * from './types/processor';
7
7
  export * from './constants';
8
8
  export * from './utils';
9
+ export * from './typeGuards';
9
10
  export { getCanvasRenderingContext2D, getImageSize } from './canvasRenderUtils';
@@ -6,4 +6,5 @@ export * from './types/render';
6
6
  export * from './types/processor';
7
7
  export * from './constants';
8
8
  export * from './utils';
9
+ export * from './typeGuards';
9
10
  export { getCanvasRenderingContext2D, getImageSize } from './canvasRenderUtils';