@pexip/media-processor 21.0.0 → 22.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +70 -0
- package/README.md +17 -10
- package/api-docs/README.mdx +37 -18
- package/api-docs/functions/assertMediaStreamVideoTrack.mdx +13 -0
- package/api-docs/functions/createFrameCallbackRequest.mdx +8 -2
- package/api-docs/functions/createMediaStreamTrackInputProcessAdaptor.mdx +10 -0
- package/api-docs/functions/createReadWritableStreamFallbackInputProcessAdaptor.mdx +17 -0
- package/api-docs/functions/createReadWritableStreamInputProcessAdaptor.mdx +10 -0
- package/api-docs/functions/createVideoProcessor.mdx +3 -3
- package/api-docs/functions/getDefaultInputProcessAdaptor.mdx +17 -0
- package/api-docs/functions/isMediaStreamVideoTrack.mdx +13 -0
- package/api-docs/functions/playVideo.mdx +2 -2
- package/api-docs/interfaces/Analyzer.mdx +4 -0
- package/api-docs/interfaces/Benchmark.mdx +14 -2
- package/api-docs/interfaces/FrameTransformStats.mdx +9 -0
- package/api-docs/interfaces/Gain.mdx +4 -0
- package/api-docs/interfaces/ImageSegmenterOptions.mdx +0 -2
- package/api-docs/interfaces/InputProcessAdaptor.mdx +28 -0
- package/api-docs/interfaces/Process.mdx +8 -3
- package/api-docs/interfaces/ProcessEventMute.mdx +6 -0
- package/api-docs/interfaces/ProcessWorkerEventOpened.mdx +7 -0
- package/api-docs/interfaces/ProcessWorkerEventStats.mdx +6 -0
- package/api-docs/interfaces/ProcessWorkerEventUpdated.mdx +7 -0
- package/api-docs/interfaces/ProcessorEvent.mdx +1 -1
- package/api-docs/interfaces/ProcessorOptions.mdx +0 -1
- package/api-docs/interfaces/ProcessorProcessOptions.mdx +3 -1
- package/api-docs/interfaces/ProcessorUpdateOptions.mdx +15 -0
- package/api-docs/interfaces/ReadWritableStream.mdx +12 -0
- package/api-docs/interfaces/Renderer.mdx +2 -2
- package/api-docs/interfaces/RendererOptions.mdx +0 -4
- package/api-docs/interfaces/Segmenter.mdx +89 -10
- package/api-docs/interfaces/SegmenterOptions.mdx +0 -1
- package/api-docs/interfaces/VideoProcessor.mdx +49 -5
- package/api-docs/type-aliases/ProcessEvents.mdx +1 -0
- package/api-docs/type-aliases/ProcessInputType.mdx +3 -1
- package/api-docs/type-aliases/ProcessWorkerEvents.mdx +6 -4
- package/api-docs/type-aliases/ProcessorWorkerEvents.mdx +1 -1
- package/api-docs/type-aliases/Struct.mdx +9 -0
- package/api-docs/variables/FRAME_RATE.mdx +1 -1
- package/api-docs/variables/ImageFetchError.mdx +3 -0
- package/api-docs/variables/MediaStreamTrackTransferable.mdx +5 -0
- package/api-docs/variables/RENDERER_OPTIONS_STRUCT.mdx +3 -0
- package/api-docs/variables/cloneImageRecord.mdx +19 -0
- package/api-docs/variables/compareImageRecords.mdx +18 -0
- package/api-docs/variables/createRemoteImageBitmap.mdx +1 -1
- package/api-docs/variables/isRenderOptionsKey.mdx +13 -0
- package/api-docs/variables/isValidGpuAPI.mdx +13 -0
- package/dist/common/backends/webgl/renderer.js +2 -12
- package/dist/common/backends/webgl/texture.d.ts +1 -2
- package/dist/common/backends/webgl/texture.js +2 -2
- package/dist/common/backends/webgl/webglUtils.js +8 -4
- package/dist/common/backends/webgpu/renderer.js +2 -12
- package/dist/common/backends/webgpu/tentBlur.js +1 -1
- package/dist/common/backends/webgpu/texture.d.ts +1 -2
- package/dist/common/backends/webgpu/texture.js +6 -2
- package/dist/common/constants.d.ts +3 -0
- package/dist/common/constants.js +4 -0
- package/dist/common/index.d.ts +1 -0
- package/dist/common/index.js +1 -0
- package/dist/common/tsconfig.tsbuildinfo +1 -1
- package/dist/common/typeGuards.d.ts +2 -0
- package/dist/common/typeGuards.js +7 -0
- package/dist/common/types/media.d.ts +5 -1
- package/dist/common/types/messageEvents.d.ts +31 -4
- package/dist/common/types/processor.d.ts +6 -4
- package/dist/common/types/render.d.ts +5 -2
- package/dist/common/types/render.js +23 -1
- package/dist/common/types/segmentation.d.ts +0 -12
- package/dist/common/utils.d.ts +13 -2
- package/dist/common/utils.js +54 -10
- package/dist/main/audio.js +3 -3
- package/dist/main/benchUtils.d.ts +2 -2
- package/dist/main/benchUtils.js +4 -4
- package/dist/main/index.d.ts +1 -1
- package/dist/main/index.js +1 -1
- package/dist/main/tsconfig.tsbuildinfo +1 -1
- package/dist/main/types.d.ts +0 -6
- package/dist/main/utils.d.ts +4 -0
- package/dist/main/utils.js +31 -0
- package/dist/main/video/constants.d.ts +1 -4
- package/dist/main/video/constants.js +1 -5
- package/dist/main/video/index.d.ts +2 -3
- package/dist/main/video/index.js +2 -3
- package/dist/main/video/inputProcessAdaptor.d.ts +30 -0
- package/dist/main/video/inputProcessAdaptor.js +198 -0
- package/dist/main/video/segmenter.d.ts +35 -0
- package/dist/main/video/{segmenters/mediapipe.js → segmenter.js} +93 -53
- package/dist/main/video/typeGuards.d.ts +2 -0
- package/dist/main/video/typeGuards.js +8 -1
- package/dist/main/video/types.d.ts +36 -16
- package/dist/main/video/utils.d.ts +5 -4
- package/dist/main/video/utils.js +49 -44
- package/dist/main/video/video.d.ts +29 -7
- package/dist/main/video/video.js +323 -28
- package/dist/workers/mediaWorker.js +23 -23
- package/dist/workers/tsconfig.tsbuildinfo +1 -1
- package/dist/worklets/tsconfig.tsbuildinfo +1 -1
- package/package.json +7 -7
- package/api-docs/functions/createCanvasTransform.mdx +0 -14
- package/api-docs/functions/createVideoTrackProcessor.mdx +0 -9
- package/api-docs/functions/createVideoTrackProcessorWithFallback.mdx +0 -15
- package/api-docs/interfaces/Detector.mdx +0 -81
- package/api-docs/interfaces/SegmentationParams.mdx +0 -35
- package/api-docs/interfaces/Transform.mdx +0 -62
- package/api-docs/type-aliases/ProcessVideoTrack.mdx +0 -15
- package/api-docs/type-aliases/SegmentationTransform.mdx +0 -3
- package/dist/main/transformer.d.ts +0 -1
- package/dist/main/transformer.js +0 -4
- package/dist/main/video/canvasTransform.d.ts +0 -17
- package/dist/main/video/canvasTransform.js +0 -341
- package/dist/main/video/segmenters/mediapipe.d.ts +0 -22
- package/dist/main/video/transformer.d.ts +0 -9
- package/dist/main/video/transformer.js +0 -68
- package/dist/main/video/videoStreamTrackProcessor.d.ts +0 -20
- package/dist/main/video/videoStreamTrackProcessor.js +0 -86
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
## Extends
|
|
2
|
+
|
|
3
|
+
- `Omit`\<[`ProcessorOptions`](ProcessorOptions.mdx), `"renderOptions"`\>
|
|
4
|
+
|
|
5
|
+
## Properties
|
|
6
|
+
|
|
7
|
+
| Property | Type | Inherited from |
|
|
8
|
+
| ------ | ------ | ------ |
|
|
9
|
+
| <a id="backgroundimage"></a> `backgroundImage?` | [`ImageRecord`](ImageRecord.mdx) | [`ProcessorOptions`](ProcessorOptions.mdx).[`backgroundImage`](ProcessorOptions.mdx#backgroundimage) |
|
|
10
|
+
| <a id="restart"></a> `restart?` | `boolean` | [`ProcessorOptions`](ProcessorOptions.mdx).[`restart`](ProcessorOptions.mdx#restart) |
|
|
11
|
+
| <a id="basepath"></a> `basePath?` | `string` | [`SegmenterOptions`](SegmenterOptions.mdx).[`basePath`](SegmenterOptions.mdx#basepath) |
|
|
12
|
+
| <a id="processingwidth"></a> `processingWidth?` | `number` | [`SegmenterOptions`](SegmenterOptions.mdx).[`processingWidth`](SegmenterOptions.mdx#processingwidth) |
|
|
13
|
+
| <a id="processingheight"></a> `processingHeight?` | `number` | [`SegmenterOptions`](SegmenterOptions.mdx).[`processingHeight`](SegmenterOptions.mdx#processingheight) |
|
|
14
|
+
| <a id="imagesegmenteroptions"></a> `imageSegmenterOptions?` | [`ImageSegmenterOptions`](ImageSegmenterOptions.mdx) | [`SegmenterOptions`](SegmenterOptions.mdx).[`imageSegmenterOptions`](SegmenterOptions.mdx#imagesegmenteroptions) |
|
|
15
|
+
| <a id="renderoptions"></a> `renderOptions?` | `Partial`\<[`RendererOptions`](RendererOptions.mdx)\> | - |
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
## Type Parameters
|
|
2
|
+
|
|
3
|
+
| Type Parameter |
|
|
4
|
+
| ------ |
|
|
5
|
+
| `T` |
|
|
6
|
+
|
|
7
|
+
## Properties
|
|
8
|
+
|
|
9
|
+
| Property | Type |
|
|
10
|
+
| ------ | ------ |
|
|
11
|
+
| <a id="readable"></a> `readable` | `ReadableStream`\<`T`\> |
|
|
12
|
+
| <a id="writable"></a> `writable` | `WritableStream`\<`T`\> |
|
|
@@ -1,10 +1,6 @@
|
|
|
1
1
|
Options controlling the rendering and effects for person/background segmentation.
|
|
2
2
|
All values are stable per-frame and can be safely updated in real-time.
|
|
3
3
|
|
|
4
|
-
## Extended by
|
|
5
|
-
|
|
6
|
-
- [`SegmentationParams`](SegmentationParams.mdx)
|
|
7
|
-
|
|
8
4
|
## Properties
|
|
9
5
|
|
|
10
6
|
| Property | Type | Description |
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
## Extends
|
|
2
2
|
|
|
3
|
-
- [`Process`](Process.mdx)
|
|
3
|
+
- [`Process`](Process.mdx)\<`Partial`\<[`RendererOptions`](RendererOptions.mdx)\>\>
|
|
4
4
|
|
|
5
5
|
## Properties
|
|
6
6
|
|
|
@@ -47,9 +47,12 @@ destroy(): Promise<void>;
|
|
|
47
47
|
### open()
|
|
48
48
|
|
|
49
49
|
```ts
|
|
50
|
-
open(options?): Promise<
|
|
50
|
+
open(options?): Promise<Partial<RendererOptions>>;
|
|
51
51
|
```
|
|
52
52
|
|
|
53
|
+
Initialize the Segmenter as well as the underlying worker.
|
|
54
|
+
It is expected to be called once.
|
|
55
|
+
|
|
53
56
|
#### Parameters
|
|
54
57
|
|
|
55
58
|
| Parameter | Type |
|
|
@@ -58,7 +61,7 @@ open(options?): Promise<void>;
|
|
|
58
61
|
|
|
59
62
|
#### Returns
|
|
60
63
|
|
|
61
|
-
`Promise`\<`
|
|
64
|
+
`Promise`\<`Partial`\<[`RendererOptions`](RendererOptions.mdx)\>\>
|
|
62
65
|
|
|
63
66
|
#### Overrides
|
|
64
67
|
|
|
@@ -69,9 +72,11 @@ open(options?): Promise<void>;
|
|
|
69
72
|
### update()
|
|
70
73
|
|
|
71
74
|
```ts
|
|
72
|
-
update(options): Promise<
|
|
75
|
+
update(options): Promise<Partial<RendererOptions>>;
|
|
73
76
|
```
|
|
74
77
|
|
|
78
|
+
Update the processor options
|
|
79
|
+
|
|
75
80
|
#### Parameters
|
|
76
81
|
|
|
77
82
|
| Parameter | Type |
|
|
@@ -80,26 +85,94 @@ update(options): Promise<void>;
|
|
|
80
85
|
|
|
81
86
|
#### Returns
|
|
82
87
|
|
|
83
|
-
`Promise`\<`
|
|
88
|
+
`Promise`\<`Partial`\<[`RendererOptions`](RendererOptions.mdx)\>\>
|
|
84
89
|
|
|
85
90
|
***
|
|
86
91
|
|
|
87
|
-
###
|
|
92
|
+
### mute()
|
|
88
93
|
|
|
89
94
|
```ts
|
|
90
|
-
|
|
95
|
+
mute(muted): void;
|
|
91
96
|
```
|
|
92
97
|
|
|
98
|
+
Mute/unmute the track in the worker
|
|
99
|
+
|
|
93
100
|
#### Parameters
|
|
94
101
|
|
|
95
102
|
| Parameter | Type |
|
|
96
103
|
| ------ | ------ |
|
|
97
|
-
| `
|
|
98
|
-
| `options?` | [`RendererOptions`](RendererOptions.mdx) |
|
|
104
|
+
| `muted` | `boolean` |
|
|
99
105
|
|
|
100
106
|
#### Returns
|
|
101
107
|
|
|
102
|
-
`
|
|
108
|
+
`void`
|
|
109
|
+
|
|
110
|
+
***
|
|
111
|
+
|
|
112
|
+
### process()
|
|
113
|
+
|
|
114
|
+
#### Call Signature
|
|
115
|
+
|
|
116
|
+
```ts
|
|
117
|
+
process(input, options?): Promise<MediaStreamTrack>;
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
Process provided input
|
|
121
|
+
|
|
122
|
+
##### Parameters
|
|
123
|
+
|
|
124
|
+
| Parameter | Type |
|
|
125
|
+
| ------ | ------ |
|
|
126
|
+
| `input` | `MediaStreamTrack` |
|
|
127
|
+
| `options?` | [`RendererOptions`](RendererOptions.mdx) |
|
|
128
|
+
|
|
129
|
+
##### Returns
|
|
130
|
+
|
|
131
|
+
`Promise`\<`MediaStreamTrack`\>
|
|
132
|
+
|
|
133
|
+
#### Call Signature
|
|
134
|
+
|
|
135
|
+
```ts
|
|
136
|
+
process(input, options?): Promise<
|
|
137
|
+
| ReadWritableStream<VideoFrame>
|
|
138
|
+
| undefined>;
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
##### Parameters
|
|
142
|
+
|
|
143
|
+
| Parameter | Type |
|
|
144
|
+
| ------ | ------ |
|
|
145
|
+
| `input` | [`ReadWritableStream`](ReadWritableStream.mdx)\<`VideoFrame`\> |
|
|
146
|
+
| `options?` | [`RendererOptions`](RendererOptions.mdx) |
|
|
147
|
+
|
|
148
|
+
##### Returns
|
|
149
|
+
|
|
150
|
+
`Promise`\<
|
|
151
|
+
\| [`ReadWritableStream`](ReadWritableStream.mdx)\<`VideoFrame`\>
|
|
152
|
+
\| `undefined`\>
|
|
153
|
+
|
|
154
|
+
#### Call Signature
|
|
155
|
+
|
|
156
|
+
```ts
|
|
157
|
+
process(input, options?): Promise<
|
|
158
|
+
| MediaStreamTrack
|
|
159
|
+
| ReadWritableStream<VideoFrame>
|
|
160
|
+
| undefined>;
|
|
161
|
+
```
|
|
162
|
+
|
|
163
|
+
##### Parameters
|
|
164
|
+
|
|
165
|
+
| Parameter | Type |
|
|
166
|
+
| ------ | ------ |
|
|
167
|
+
| `input` | \| `MediaStreamTrack` \| [`ReadWritableStream`](ReadWritableStream.mdx)\<`VideoFrame`\> |
|
|
168
|
+
| `options?` | [`RendererOptions`](RendererOptions.mdx) |
|
|
169
|
+
|
|
170
|
+
##### Returns
|
|
171
|
+
|
|
172
|
+
`Promise`\<
|
|
173
|
+
\| `MediaStreamTrack`
|
|
174
|
+
\| [`ReadWritableStream`](ReadWritableStream.mdx)\<`VideoFrame`\>
|
|
175
|
+
\| `undefined`\>
|
|
103
176
|
|
|
104
177
|
***
|
|
105
178
|
|
|
@@ -109,6 +182,8 @@ process(input, options?): Promise<VideoFrameLike | undefined>;
|
|
|
109
182
|
subscribe(event, callback): Unsubscribe;
|
|
110
183
|
```
|
|
111
184
|
|
|
185
|
+
Subscribe rendering events,
|
|
186
|
+
|
|
112
187
|
#### Parameters
|
|
113
188
|
|
|
114
189
|
| Parameter | Type |
|
|
@@ -119,3 +194,7 @@ subscribe(event, callback): Unsubscribe;
|
|
|
119
194
|
#### Returns
|
|
120
195
|
|
|
121
196
|
`Unsubscribe`
|
|
197
|
+
|
|
198
|
+
#### See
|
|
199
|
+
|
|
200
|
+
RenderingEvents
|
|
@@ -7,7 +7,6 @@
|
|
|
7
7
|
| Property | Type |
|
|
8
8
|
| ------ | ------ |
|
|
9
9
|
| <a id="basepath"></a> `basePath?` | `string` |
|
|
10
|
-
| <a id="output"></a> `output?` | `OffscreenCanvas` \| `null` |
|
|
11
10
|
| <a id="processingwidth"></a> `processingWidth?` | `number` |
|
|
12
11
|
| <a id="processingheight"></a> `processingHeight?` | `number` |
|
|
13
12
|
| <a id="imagesegmenteroptions"></a> `imageSegmenterOptions?` | [`ImageSegmenterOptions`](ImageSegmenterOptions.mdx) |
|
|
@@ -1,9 +1,13 @@
|
|
|
1
1
|
## Properties
|
|
2
2
|
|
|
3
|
-
| Property | Type | Description |
|
|
4
|
-
| ------ | ------ | ------ |
|
|
5
|
-
| <a id="
|
|
6
|
-
| <a id="
|
|
3
|
+
| Property | Modifier | Type | Description |
|
|
4
|
+
| ------ | ------ | ------ | ------ |
|
|
5
|
+
| <a id="status"></a> `status` | `readonly` | \| `"closed"` \| `"closing"` \| `"Error"` \| `"destroyed"` \| `"destroying"` \| `"idle"` \| `"new"` \| `"opened"` \| `"opening"` \| `"processing"` \| `"updated"` \| `"updating"` | - |
|
|
6
|
+
| <a id="process"></a> `process` | `public` | (`source`) => `Promise`\<`MediaStreamTrack`\> | Process the video and return a MediaStream |
|
|
7
|
+
| <a id="destroy"></a> `destroy` | `public` | () => `Promise`\<`void`\> | Destroy the segmentation wasm instance |
|
|
8
|
+
| <a id="renderoptions"></a> `renderOptions` | `readonly` | [`RendererOptions`](RendererOptions.mdx) | Current renderer options |
|
|
9
|
+
| <a id="processingwidth"></a> `processingWidth` | `readonly` | `number` | Current processing width |
|
|
10
|
+
| <a id="processingheight"></a> `processingHeight` | `readonly` | `number` | Current processing height |
|
|
7
11
|
|
|
8
12
|
## Methods
|
|
9
13
|
|
|
@@ -13,11 +17,33 @@
|
|
|
13
17
|
open(options?): Promise<void>;
|
|
14
18
|
```
|
|
15
19
|
|
|
20
|
+
Initialize the video processor and related
|
|
21
|
+
|
|
22
|
+
#### Parameters
|
|
23
|
+
|
|
24
|
+
| Parameter | Type |
|
|
25
|
+
| ------ | ------ |
|
|
26
|
+
| `options?` | [`ProcessorUpdateOptions`](ProcessorUpdateOptions.mdx) |
|
|
27
|
+
|
|
28
|
+
#### Returns
|
|
29
|
+
|
|
30
|
+
`Promise`\<`void`\>
|
|
31
|
+
|
|
32
|
+
***
|
|
33
|
+
|
|
34
|
+
### update()
|
|
35
|
+
|
|
36
|
+
```ts
|
|
37
|
+
update(options): Promise<void>;
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
Update the video processor with provided options
|
|
41
|
+
|
|
16
42
|
#### Parameters
|
|
17
43
|
|
|
18
44
|
| Parameter | Type |
|
|
19
45
|
| ------ | ------ |
|
|
20
|
-
| `options
|
|
46
|
+
| `options` | [`ProcessorUpdateOptions`](ProcessorUpdateOptions.mdx) |
|
|
21
47
|
|
|
22
48
|
#### Returns
|
|
23
49
|
|
|
@@ -25,6 +51,24 @@ open(options?): Promise<void>;
|
|
|
25
51
|
|
|
26
52
|
***
|
|
27
53
|
|
|
54
|
+
### mute()
|
|
55
|
+
|
|
56
|
+
```ts
|
|
57
|
+
mute(muted): void;
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
#### Parameters
|
|
61
|
+
|
|
62
|
+
| Parameter | Type |
|
|
63
|
+
| ------ | ------ |
|
|
64
|
+
| `muted` | `boolean` |
|
|
65
|
+
|
|
66
|
+
#### Returns
|
|
67
|
+
|
|
68
|
+
`void`
|
|
69
|
+
|
|
70
|
+
***
|
|
71
|
+
|
|
28
72
|
### close()
|
|
29
73
|
|
|
30
74
|
```ts
|
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
```ts
|
|
2
|
-
type ProcessWorkerEvents<T, E, D> =
|
|
2
|
+
type ProcessWorkerEvents<T, E, D, U> =
|
|
3
3
|
| ProcessWorkerEventProcessed<T>
|
|
4
|
-
| ProcessWorkerEventOpened
|
|
5
|
-
| ProcessWorkerEventUpdated
|
|
4
|
+
| ProcessWorkerEventOpened<U>
|
|
5
|
+
| ProcessWorkerEventUpdated<U>
|
|
6
6
|
| ProcessWorkerEvent<E>
|
|
7
|
-
| ProcessWorkerEventDebug<D
|
|
7
|
+
| ProcessWorkerEventDebug<D>
|
|
8
|
+
| ProcessWorkerEventStats;
|
|
8
9
|
```
|
|
9
10
|
|
|
10
11
|
Message events returned from the processor
|
|
@@ -16,3 +17,4 @@ Message events returned from the processor
|
|
|
16
17
|
| `T` |
|
|
17
18
|
| `E` |
|
|
18
19
|
| `D` |
|
|
20
|
+
| `U` |
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
```ts
|
|
2
|
+
const cloneImageRecord: (image) => Promise<{
|
|
3
|
+
image: ImageBitmap;
|
|
4
|
+
key: string;
|
|
5
|
+
}>;
|
|
6
|
+
```
|
|
7
|
+
|
|
8
|
+
## Parameters
|
|
9
|
+
|
|
10
|
+
| Parameter | Type |
|
|
11
|
+
| ------ | ------ |
|
|
12
|
+
| `image` | [`ImageRecord`](../interfaces/ImageRecord.mdx) |
|
|
13
|
+
|
|
14
|
+
## Returns
|
|
15
|
+
|
|
16
|
+
`Promise`\<\{
|
|
17
|
+
`image`: `ImageBitmap`;
|
|
18
|
+
`key`: `string`;
|
|
19
|
+
\}\>
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
```ts
|
|
2
|
+
const compareImageRecords: (a, b) => boolean;
|
|
3
|
+
```
|
|
4
|
+
|
|
5
|
+
Compare provided ImageRecord
|
|
6
|
+
|
|
7
|
+
## Parameters
|
|
8
|
+
|
|
9
|
+
| Parameter | Type |
|
|
10
|
+
| ------ | ------ |
|
|
11
|
+
| `a` | [`ImageRecord`](../interfaces/ImageRecord.mdx) |
|
|
12
|
+
| `b` | [`ImageRecord`](../interfaces/ImageRecord.mdx) \| `undefined` |
|
|
13
|
+
|
|
14
|
+
## Returns
|
|
15
|
+
|
|
16
|
+
`boolean`
|
|
17
|
+
|
|
18
|
+
true if the two records have the same key and image, false otherwise
|
|
@@ -14,7 +14,7 @@ const createRemoteImageBitmap: (map?) => object;
|
|
|
14
14
|
|
|
15
15
|
| Name | Type |
|
|
16
16
|
| ------ | ------ |
|
|
17
|
-
| `tryFetchingImage()` | (`uri`) => `Promise`\<`ImageBitmap` \| `null`\> |
|
|
17
|
+
| `tryFetchingImage()` | (`uri`, `abortSignal`) => `Promise`\<`ImageBitmap` \| `null` \| `undefined`\> |
|
|
18
18
|
| `setImage()` | (`uri`, `image`) => `void` |
|
|
19
19
|
| `getImage()` | (`uri`) => `ImageBitmap` \| `null` |
|
|
20
20
|
| `removeImage()` | (`uri`) => `boolean` |
|
|
@@ -178,12 +178,7 @@ export const createRenderer = (renderEventHandlers, canvas = new OffscreenCanvas
|
|
|
178
178
|
assert(!props.gl?.isContextLost(), RENDERING_EVENTS.ContextLost);
|
|
179
179
|
// 1. Upload current frame and raw mask
|
|
180
180
|
const frameTexture = props.videoFrameRenderer.render(frame);
|
|
181
|
-
const maskTexture = props.maskRenderer.render(
|
|
182
|
-
frame: mask,
|
|
183
|
-
width: mask.width,
|
|
184
|
-
height: mask.height,
|
|
185
|
-
timestamp: frame.timestamp,
|
|
186
|
-
});
|
|
181
|
+
const maskTexture = props.maskRenderer.render(mask);
|
|
187
182
|
// 2. Joint Bilateral Filter
|
|
188
183
|
let smoothedMask;
|
|
189
184
|
let compositeMaskInput;
|
|
@@ -240,12 +235,7 @@ export const createRenderer = (renderEventHandlers, canvas = new OffscreenCanvas
|
|
|
240
235
|
}, options);
|
|
241
236
|
}
|
|
242
237
|
else if (options?.effects === 'overlay' && options.backgroundImage) {
|
|
243
|
-
const background = props.backgroundImageRenderer.render(
|
|
244
|
-
frame: options.backgroundImage,
|
|
245
|
-
timestamp: performance.now(),
|
|
246
|
-
width: options.backgroundImage.width,
|
|
247
|
-
height: options.backgroundImage.height,
|
|
248
|
-
});
|
|
238
|
+
const background = props.backgroundImageRenderer.render(options.backgroundImage);
|
|
249
239
|
outputFrameTexture = props.replaceRenderer.render({
|
|
250
240
|
mask: compositeMaskInput,
|
|
251
241
|
foreground: frameTexture,
|
|
@@ -1,9 +1,8 @@
|
|
|
1
|
-
import type { VideoFrameLike } from '../../types/media';
|
|
2
1
|
import type { TextureInfo } from './types';
|
|
3
2
|
/**
|
|
4
3
|
* Render Video frame into texture
|
|
5
4
|
*/
|
|
6
5
|
export declare const createTextureRenderer: (processor: OffscreenCanvas, redOnly: boolean) => {
|
|
7
|
-
render: (
|
|
6
|
+
render: (frame: VideoFrame | ImageBitmap | OffscreenCanvas) => TextureInfo;
|
|
8
7
|
release: () => void;
|
|
9
8
|
};
|
|
@@ -59,7 +59,7 @@ export const createTextureRenderer = (processor, redOnly) => {
|
|
|
59
59
|
: createRGBAFrameTexture(gl, processor.width, processor.height, true, floatFullChannelSupported);
|
|
60
60
|
const frameBuffer = createFrameBuffer(gl, outputTexture);
|
|
61
61
|
return {
|
|
62
|
-
render: (
|
|
62
|
+
render: (frame) => {
|
|
63
63
|
assert(!gl.isContextLost(), RENDERING_EVENTS.ContextLost);
|
|
64
64
|
// Draw on the frame buffer
|
|
65
65
|
gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer);
|
|
@@ -76,7 +76,7 @@ export const createTextureRenderer = (processor, redOnly) => {
|
|
|
76
76
|
// Upload the frame to texture
|
|
77
77
|
gl.activeTexture(gl.TEXTURE0);
|
|
78
78
|
gl.bindTexture(gl.TEXTURE_2D, frameTexture);
|
|
79
|
-
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE,
|
|
79
|
+
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, frame);
|
|
80
80
|
gl.drawArrays(gl.TRIANGLES, 0, 6);
|
|
81
81
|
// Cleanup
|
|
82
82
|
gl.bindBuffer(gl.ARRAY_BUFFER, null);
|
|
@@ -92,6 +92,7 @@ export const loadShader = (gl, shaderType, shaderSource) => {
|
|
|
92
92
|
*/
|
|
93
93
|
export const createProgram = (gl, shaders, attribs, locations) => {
|
|
94
94
|
const program = gl.createProgram();
|
|
95
|
+
let released = false;
|
|
95
96
|
if (!program) {
|
|
96
97
|
throw new Error('Unable to create WebGL program');
|
|
97
98
|
}
|
|
@@ -120,11 +121,14 @@ export const createProgram = (gl, shaders, attribs, locations) => {
|
|
|
120
121
|
return [
|
|
121
122
|
program,
|
|
122
123
|
() => {
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
124
|
+
if (!released) {
|
|
125
|
+
released = true;
|
|
126
|
+
for (const shader of shaders) {
|
|
127
|
+
gl.detachShader(program, shader);
|
|
128
|
+
gl.deleteShader(shader);
|
|
129
|
+
}
|
|
130
|
+
gl.deleteProgram(program);
|
|
126
131
|
}
|
|
127
|
-
gl.deleteProgram(program);
|
|
128
132
|
},
|
|
129
133
|
];
|
|
130
134
|
};
|
|
@@ -195,12 +195,7 @@ export const createRenderer = (adapter, renderEventHandlers, canvas = new Offscr
|
|
|
195
195
|
label: 'frame upload pass',
|
|
196
196
|
});
|
|
197
197
|
const frameTexture = props.videoFrameRenderer.render(frame, frameUploadPass);
|
|
198
|
-
const maskTexture = props.maskRenderer.render(
|
|
199
|
-
frame: mask,
|
|
200
|
-
width: mask.width,
|
|
201
|
-
height: mask.height,
|
|
202
|
-
timestamp: frame.timestamp,
|
|
203
|
-
}, frameUploadPass);
|
|
198
|
+
const maskTexture = props.maskRenderer.render(mask, frameUploadPass);
|
|
204
199
|
frameUploadPass.end();
|
|
205
200
|
let smoothedMask;
|
|
206
201
|
let compositeInputMask;
|
|
@@ -253,12 +248,7 @@ export const createRenderer = (adapter, renderEventHandlers, canvas = new Offscr
|
|
|
253
248
|
const bgImgUploadPass = encoder.beginComputePass({
|
|
254
249
|
label: 'background image upload pass',
|
|
255
250
|
});
|
|
256
|
-
const bgTexture = props.backgroundImageRenderer.render(
|
|
257
|
-
frame: options.backgroundImage,
|
|
258
|
-
width: options.backgroundImage.width,
|
|
259
|
-
height: options.backgroundImage.height,
|
|
260
|
-
timestamp: frame.timestamp,
|
|
261
|
-
}, bgImgUploadPass);
|
|
251
|
+
const bgTexture = props.backgroundImageRenderer.render(options.backgroundImage, bgImgUploadPass);
|
|
262
252
|
bgImgUploadPass.end();
|
|
263
253
|
const blurredBackgroundImageTexture = props.blurRenderer.render(encoder, bgTexture, options.backgroundBlurAmount);
|
|
264
254
|
props.replaceRenderer.render(encoder, {
|
|
@@ -17,13 +17,13 @@ export const createTentBlurRenderer = (device, isContextLost, width, height, tex
|
|
|
17
17
|
workgroupSizeX: WORKGROUP_SIZE_X,
|
|
18
18
|
workgroupSizeY: WORKGROUP_SIZE_Y,
|
|
19
19
|
});
|
|
20
|
-
shaderHCode = enableF16InShader(shaderHCode);
|
|
21
20
|
let shaderVCode = updateTextureStorageFormat(tentBlurVShader, textureFormat);
|
|
22
21
|
shaderVCode = syncWorkgroupSizeWithShader(shaderVCode, {
|
|
23
22
|
workgroupSizeX: WORKGROUP_SIZE_X,
|
|
24
23
|
workgroupSizeY: WORKGROUP_SIZE_Y,
|
|
25
24
|
});
|
|
26
25
|
if (supportF16) {
|
|
26
|
+
shaderHCode = enableF16InShader(shaderHCode);
|
|
27
27
|
shaderVCode = enableF16InShader(shaderVCode);
|
|
28
28
|
}
|
|
29
29
|
const defs = makeShaderDataDefinitions(shaderHCode);
|
|
@@ -1,5 +1,4 @@
|
|
|
1
|
-
import type { VideoFrameLike } from '../../types/media';
|
|
2
1
|
export declare const createTextureRenderer: (device: GPUDevice, isContextLost: () => boolean, textureFormat: GPUTextureFormat, width: number, height: number, filterMode: GPUFilterMode) => {
|
|
3
|
-
render: (frame:
|
|
2
|
+
render: (frame: VideoFrame | ImageBitmap | OffscreenCanvas, pass: GPUComputePassEncoder) => GPUTexture;
|
|
4
3
|
release: () => void;
|
|
5
4
|
};
|
|
@@ -25,8 +25,12 @@ export const createTextureRenderer = (device, isContextLost, textureFormat, widt
|
|
|
25
25
|
return {
|
|
26
26
|
render: (frame, pass) => {
|
|
27
27
|
assert(!isContextLost(), RENDERING_EVENTS.ContextLost);
|
|
28
|
-
const
|
|
29
|
-
|
|
28
|
+
const width = frame instanceof VideoFrame ? frame.displayWidth : frame.width;
|
|
29
|
+
const height = frame instanceof VideoFrame
|
|
30
|
+
? frame.displayHeight
|
|
31
|
+
: frame.height;
|
|
32
|
+
const texture = getOrCreateTexture(width, height);
|
|
33
|
+
device.queue.copyExternalImageToTexture({ source: frame, flipY: false }, { texture }, { width, height });
|
|
30
34
|
return resizeRenderer.render(pass, texture);
|
|
31
35
|
},
|
|
32
36
|
release: () => {
|
package/dist/common/constants.js
CHANGED
package/dist/common/index.d.ts
CHANGED
package/dist/common/index.js
CHANGED