@snovakow/greenscreen 0.0.0-stage → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +238 -2
  3. package/dist/greenscreen.d.ts +32 -0
  4. package/dist/greenscreen.d.ts.map +1 -0
  5. package/dist/greenscreen.global.js +315 -0
  6. package/dist/greenscreen.global.js.map +1 -0
  7. package/dist/greenscreen.js +250 -0
  8. package/dist/greenscreen.js.map +1 -0
  9. package/dist/greenscreen.min.js +315 -0
  10. package/dist/greenscreen.min.js.map +1 -0
  11. package/dist/index.d.ts +6 -0
  12. package/dist/index.d.ts.map +1 -0
  13. package/dist/index.js +3 -0
  14. package/dist/index.js.map +1 -0
  15. package/dist/pack.d.ts +7 -0
  16. package/dist/pack.d.ts.map +1 -0
  17. package/dist/pack.js +63 -0
  18. package/dist/pack.js.map +1 -0
  19. package/dist/react.d.ts +26 -0
  20. package/dist/react.d.ts.map +1 -0
  21. package/dist/react.js +78 -0
  22. package/dist/react.js.map +1 -0
  23. package/dist/renderer.d.ts +19 -0
  24. package/dist/renderer.d.ts.map +1 -0
  25. package/dist/renderer.js +25 -0
  26. package/dist/renderer.js.map +1 -0
  27. package/dist/settings.d.ts +36 -0
  28. package/dist/settings.d.ts.map +1 -0
  29. package/dist/settings.js +112 -0
  30. package/dist/settings.js.map +1 -0
  31. package/dist/shader.glsl.d.ts +3 -0
  32. package/dist/shader.glsl.d.ts.map +1 -0
  33. package/dist/shader.glsl.js +179 -0
  34. package/dist/shader.glsl.js.map +1 -0
  35. package/dist/shader.wgsl.d.ts +2 -0
  36. package/dist/shader.wgsl.d.ts.map +1 -0
  37. package/dist/shader.wgsl.js +160 -0
  38. package/dist/shader.wgsl.js.map +1 -0
  39. package/dist/source.d.ts +16 -0
  40. package/dist/source.d.ts.map +1 -0
  41. package/dist/source.js +103 -0
  42. package/dist/source.js.map +1 -0
  43. package/dist/three-webgpu.d.ts +20 -0
  44. package/dist/three-webgpu.d.ts.map +1 -0
  45. package/dist/three-webgpu.js +178 -0
  46. package/dist/three-webgpu.js.map +1 -0
  47. package/dist/three.d.ts +27 -0
  48. package/dist/three.d.ts.map +1 -0
  49. package/dist/three.js +81 -0
  50. package/dist/three.js.map +1 -0
  51. package/dist/types.d.ts +4 -0
  52. package/dist/types.d.ts.map +1 -0
  53. package/dist/types.js +4 -0
  54. package/dist/types.js.map +1 -0
  55. package/dist/webgl.d.ts +13 -0
  56. package/dist/webgl.d.ts.map +1 -0
  57. package/dist/webgl.js +183 -0
  58. package/dist/webgl.js.map +1 -0
  59. package/dist/webgpu.d.ts +19 -0
  60. package/dist/webgpu.d.ts.map +1 -0
  61. package/dist/webgpu.js +303 -0
  62. package/dist/webgpu.js.map +1 -0
  63. package/package.json +103 -4
  64. package/src/global.ts +11 -0
  65. package/src/greenscreen.ts +286 -0
  66. package/src/index.ts +5 -0
  67. package/src/pack.ts +73 -0
  68. package/src/react.tsx +126 -0
  69. package/src/renderer.ts +53 -0
  70. package/src/settings.ts +177 -0
  71. package/src/shader.glsl.ts +179 -0
  72. package/src/shader.wgsl.ts +159 -0
  73. package/src/source.ts +105 -0
  74. package/src/three-webgpu.ts +261 -0
  75. package/src/three.ts +114 -0
  76. package/src/types.ts +10 -0
  77. package/src/webgl.ts +191 -0
  78. package/src/webgpu.ts +312 -0
package/src/webgpu.ts ADDED
@@ -0,0 +1,312 @@
1
+ import { PACKED_LENGTH } from './pack.ts'
2
+ import { contextOf, tooLarge } from './renderer.ts'
3
+ import type { Context2D, Renderer, RendererEvents } from './renderer.ts'
4
+ import { SHADER } from './shader.wgsl.ts'
5
+ import type { Canvas, Source } from './types.ts'
6
+
7
+ // The WebGPU usage flags, as the spec numbers them, rather than the browser's globals, which TypeScript doesn't declare
8
+ const TEXTURE_COPY_DST = 0x02
9
+ const TEXTURE_BINDING = 0x04
10
+ const TEXTURE_RENDER_ATTACHMENT = 0x10
11
+ const BUFFER_COPY_DST = 0x08
12
+ const BUFFER_UNIFORM = 0x40
13
+
14
+ // One device for every keyer on the page, as a page can have many, and WebGPU, unlike WebGL, doesn't need a context of
15
+ // its own for each. It's asked for once, and asked for again only after it's lost
16
+ let shared: Promise<GPUDevice> | null = null
17
+ // The device once it's there, for a keyer made after it to start with at once
18
+ let current: GPUDevice | null = null
19
+
20
+ // Whether 'auto' takes WebGPU, found once, and again after the device is lost. known is what it came to, for a keyer to
21
+ // choose by at once, and null until then
22
+ let preferred: Promise<boolean> | null = null
23
+ let known: boolean | null = null
24
+
25
+ // The page's device, from the adapter given or the browser's own. It's asked for with the largest texture the adapter
26
+ // has, as a device takes pictures of only 8192 pixels a side unless more is asked for
27
+ export function requestDevice(given?: GPUAdapter): Promise<GPUDevice> {
28
+ if (shared) return shared
29
+ const attempt = (async () => {
30
+ const adapter = given ?? (await requestAdapter())
31
+ const device = await adapter.requestDevice({
32
+ requiredLimits: { maxTextureDimension2D: adapter.limits.maxTextureDimension2D },
33
+ })
34
+ void device.lost.then(() => {
35
+ if (shared !== attempt) return
36
+ shared = null
37
+ current = null
38
+ preferred = null
39
+ known = null
40
+ })
41
+ return device
42
+ })()
43
+ shared = attempt
44
+ // Before anything that waits for it goes on, as this waits first
45
+ void attempt.then(
46
+ (device) => {
47
+ if (shared === attempt) current = device
48
+ },
49
+ () => {},
50
+ )
51
+ return attempt
52
+ }
53
+
54
+ // The device, where it has been had already and not lost since
55
+ export function currentDevice(): GPUDevice | null {
56
+ return current
57
+ }
58
+
59
+ async function requestAdapter(): Promise<GPUAdapter> {
60
+ const gpu = typeof navigator === 'undefined' ? undefined : navigator.gpu
61
+ if (!gpu) throw new Error('Greenscreen: WebGPU is not available here')
62
+ const adapter = await gpu.requestAdapter()
63
+ if (!adapter) throw new Error('Greenscreen: WebGPU has no adapter here')
64
+ return adapter
65
+ }
66
+
67
+ // Whether 'auto' takes WebGPU: where the browser has an adapter for it on the GPU, and a device from that adapter. A
68
+ // fallback adapter, which runs WebGPU on the CPU, doesn't count, as WebGL2 is then likely on the GPU, and faster
69
+ export function prefersWebGPU(): Promise<boolean> {
70
+ if (preferred) return preferred
71
+ const attempt = (async () => {
72
+ try {
73
+ const adapter = await requestAdapter()
74
+ if (adapter.info.isFallbackAdapter) return false
75
+ await requestDevice(adapter)
76
+ return true
77
+ } catch {
78
+ return false
79
+ }
80
+ })()
81
+ preferred = attempt
82
+ void attempt.then((webgpu) => {
83
+ if (preferred === attempt) known = webgpu
84
+ })
85
+ return attempt
86
+ }
87
+
88
+ // What prefersWebGPU() comes to, where that's known already: once it has resolved, or at once where there's no WebGPU
89
+ export function knownPreference(): boolean | null {
90
+ if (typeof navigator === 'undefined' || !navigator.gpu) return false
91
+ return known
92
+ }
93
+
94
+ const pipelines = new WeakMap<GPUDevice, Map<GPUTextureFormat, GPURenderPipeline>>()
95
+
96
+ // The keying pipeline for the device and canvas format, made once. It has no sampler: the shader reads the texture's
97
+ // pixels itself, and blends them as OBS's sampler would
98
+ export function pipelineFor(device: GPUDevice, format: GPUTextureFormat): GPURenderPipeline {
99
+ let byFormat = pipelines.get(device)
100
+ if (!byFormat) {
101
+ byFormat = new Map()
102
+ pipelines.set(device, byFormat)
103
+ }
104
+ let made = byFormat.get(format)
105
+ if (!made) {
106
+ const module = device.createShaderModule({ label: 'Greenscreen', code: SHADER })
107
+ made = device.createRenderPipeline({
108
+ label: 'Greenscreen',
109
+ layout: 'auto',
110
+ vertex: { module, entryPoint: 'vertexMain' },
111
+ fragment: { module, entryPoint: 'fragmentMain', targets: [{ format }] },
112
+ primitive: { topology: 'triangle-list' },
113
+ })
114
+ byFormat.set(format, made)
115
+ }
116
+ return made
117
+ }
118
+
119
+ export function preferredFormat(): GPUTextureFormat {
120
+ return navigator.gpu.getPreferredCanvasFormat()
121
+ }
122
+
123
+ // WebGPU keys into a canvas of its own, off the page, and each picture is copied from there into the keyer's canvas,
124
+ // which shows it through a 2D context. A WebGPU canvas on the page gives drawImage nothing once the browser has drawn
125
+ // the page, where a 2D canvas keeps its picture until it's drawn again, on the page or off it
126
+ export class WebGPURenderer implements Renderer {
127
+ readonly backend = 'webgpu'
128
+ readonly #canvas: Canvas
129
+ readonly #output: Context2D
130
+ readonly #surface: OffscreenCanvas
131
+ readonly #context: GPUCanvasContext
132
+ readonly #events: RendererEvents
133
+ #device!: GPUDevice
134
+ #pipeline!: GPURenderPipeline
135
+ #uniforms!: GPUBuffer
136
+ #texture: GPUTexture | null = null
137
+ #bindGroup: GPUBindGroup | null = null
138
+ #videos: Context2D | null = null
139
+ #lost = false
140
+ #disposed = false
141
+
142
+ constructor(canvas: Canvas, device: GPUDevice, events: RendererEvents) {
143
+ // WebGPU first, so a canvas WebGPU can't draw for is left as it was
144
+ const surface = new OffscreenCanvas(1, 1)
145
+ const context = contextOf(surface, 'webgpu')
146
+ if (!context) throw new Error('Greenscreen: WebGPU is not available on this canvas')
147
+ const output = contextOf(canvas, '2d')
148
+ if (!output) throw new Error('Greenscreen: this canvas has a context of another kind already')
149
+ this.#canvas = canvas
150
+ this.#output = output
151
+ this.#surface = surface
152
+ this.#context = context
153
+ this.#events = events
154
+ this.#build(device)
155
+ }
156
+
157
+ get lost() {
158
+ return this.#lost
159
+ }
160
+
161
+ // The surface set up for the device, premultiplied as the shader's output is
162
+ #build(device: GPUDevice) {
163
+ const format = preferredFormat()
164
+ this.#device = device
165
+ this.#context.configure({ device, format, alphaMode: 'premultiplied', usage: TEXTURE_RENDER_ATTACHMENT })
166
+ this.#pipeline = pipelineFor(device, format)
167
+ this.#uniforms = device.createBuffer({
168
+ label: 'Greenscreen settings',
169
+ size: PACKED_LENGTH * 4,
170
+ usage: BUFFER_UNIFORM | BUFFER_COPY_DST,
171
+ })
172
+ this.#texture = null
173
+ this.#bindGroup = null
174
+ void device.lost.then(() => this.#onLost(device))
175
+ }
176
+
177
+ // Lost with the device, it asks for the next one and starts again with it
178
+ async #onLost(device: GPUDevice) {
179
+ if (this.#disposed || device !== this.#device) return
180
+ this.#lost = true
181
+ this.#texture = null
182
+ this.#bindGroup = null
183
+ this.#events.lost()
184
+ try {
185
+ const next = await requestDevice()
186
+ if (this.#disposed) return
187
+ this.#build(next)
188
+ this.#lost = false
189
+ this.#events.restored()
190
+ } catch {
191
+ // No device to be had: it stays lost
192
+ }
193
+ }
194
+
195
+ // The picture as it comes, top row first and not premultiplied, in a texture of its size. copyExternalImageToTexture
196
+ // takes every kind of source, a video by way of a 2D canvas. A picture larger than the device's textures throws, as
197
+ // a texture of its size would be made without an error thrown, and never drawn
198
+ upload(source: Source, width: number, height: number) {
199
+ if (this.#lost) return
200
+ const device = this.#device
201
+ const limit = device.limits.maxTextureDimension2D
202
+ if (width > limit || height > limit) throw tooLarge(width, height, 'WebGPU', limit)
203
+ let texture = this.#texture
204
+ if (!texture || texture.width !== width || texture.height !== height) {
205
+ texture?.destroy()
206
+ texture = device.createTexture({
207
+ label: 'Greenscreen picture',
208
+ size: { width, height },
209
+ format: 'rgba8unorm',
210
+ usage: TEXTURE_BINDING | TEXTURE_COPY_DST | TEXTURE_RENDER_ATTACHMENT,
211
+ })
212
+ this.#texture = texture
213
+ this.#bindGroup = device.createBindGroup({
214
+ layout: this.#pipeline.getBindGroupLayout(0),
215
+ entries: [
216
+ { binding: 0, resource: texture.createView() },
217
+ { binding: 1, resource: { buffer: this.#uniforms } },
218
+ ],
219
+ })
220
+ }
221
+ device.queue.copyExternalImageToTexture(
222
+ { source: this.#asDrawn(source, width, height), flipY: false },
223
+ { texture, colorSpace: 'srgb', premultipliedAlpha: false },
224
+ { width, height },
225
+ )
226
+ }
227
+
228
+ // A video comes in through a 2D canvas. WebGPU's own copy converts a video's colours from the video's colour space
229
+ // to sRGB, which moves those of a BT.601 video, as cameras' and recordings often are, by as much as 20 levels in 255:
230
+ // the keyer would key colours a little apart from those WebGL keys, and from those a 2D canvas reads, as an
231
+ // eyedropper does. A 2D canvas reads a video as WebGL does. It also reads a camera's first frame, which WebGPU's copy
232
+ // can't. A VideoFrame of RGB pixels, as one made from a canvas is, has nothing to convert, and comes as it is
233
+ #asDrawn(source: Source, width: number, height: number): Source {
234
+ const video = 'videoWidth' in source || ('displayWidth' in source && !/^(RGB|BGR)/.test(source.format ?? ''))
235
+ if (!video) return source
236
+ let context = this.#videos
237
+ if (!context) {
238
+ context = contextOf(stagingCanvas(), '2d')
239
+ if (!context) return source
240
+ this.#videos = context
241
+ }
242
+ const canvas = context.canvas
243
+ if (canvas.width !== width || canvas.height !== height) {
244
+ canvas.width = width
245
+ canvas.height = height
246
+ }
247
+ context.globalCompositeOperation = 'copy'
248
+ context.drawImage(source, 0, 0, width, height)
249
+ return canvas
250
+ }
251
+
252
+ // The surface's texture is taken just before it's drawn over whole, and the picture is copied across, alpha and all,
253
+ // into the keyer's canvas
254
+ draw(settings: Float32Array<ArrayBuffer>) {
255
+ const texture = this.#texture
256
+ const bindGroup = this.#bindGroup
257
+ if (this.#lost || !texture || !bindGroup) return
258
+ const { width, height } = texture
259
+ for (const canvas of [this.#surface, this.#canvas]) {
260
+ if (canvas.width !== width || canvas.height !== height) {
261
+ canvas.width = width
262
+ canvas.height = height
263
+ }
264
+ }
265
+ const device = this.#device
266
+ device.queue.writeBuffer(this.#uniforms, 0, settings, 0, settings.length)
267
+ const encoder = device.createCommandEncoder()
268
+ const pass = encoder.beginRenderPass({
269
+ colorAttachments: [
270
+ {
271
+ view: this.#context.getCurrentTexture().createView(),
272
+ loadOp: 'clear',
273
+ clearValue: [0, 0, 0, 0],
274
+ storeOp: 'store',
275
+ },
276
+ ],
277
+ })
278
+ pass.setPipeline(this.#pipeline)
279
+ pass.setBindGroup(0, bindGroup)
280
+ pass.draw(3)
281
+ pass.end()
282
+ device.queue.submit([encoder.finish()])
283
+ const output = this.#output
284
+ output.globalCompositeOperation = 'copy'
285
+ output.drawImage(this.#surface, 0, 0)
286
+ }
287
+
288
+ // The keyer's canvas shows its picture through a 2D context, which clears whatever its composite operation
289
+ clear() {
290
+ const canvas = this.#canvas
291
+ this.#output.clearRect(0, 0, canvas.width, canvas.height)
292
+ }
293
+
294
+ dispose() {
295
+ this.#disposed = true
296
+ this.#lost = true
297
+ this.#texture?.destroy()
298
+ this.#uniforms.destroy()
299
+ this.#texture = null
300
+ this.#bindGroup = null
301
+ this.#videos = null
302
+ this.#context.unconfigure()
303
+ }
304
+ }
305
+
306
+ // The canvas a video is drawn into on its way in: an element, off the page, where there's a document. WebGPU's copy
307
+ // from a 2D OffscreenCanvas comes out empty on SwiftShader, Chrome's software renderer, where one from an element
308
+ // doesn't. A worker has only OffscreenCanvas
309
+ function stagingCanvas(): Canvas {
310
+ if (typeof document !== 'undefined') return document.createElement('canvas')
311
+ return new OffscreenCanvas(1, 1)
312
+ }