@snovakow/greenscreen 0.0.0-stage → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +238 -2
- package/dist/greenscreen.d.ts +32 -0
- package/dist/greenscreen.d.ts.map +1 -0
- package/dist/greenscreen.global.js +315 -0
- package/dist/greenscreen.global.js.map +1 -0
- package/dist/greenscreen.js +250 -0
- package/dist/greenscreen.js.map +1 -0
- package/dist/greenscreen.min.js +315 -0
- package/dist/greenscreen.min.js.map +1 -0
- package/dist/index.d.ts +6 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +3 -0
- package/dist/index.js.map +1 -0
- package/dist/pack.d.ts +7 -0
- package/dist/pack.d.ts.map +1 -0
- package/dist/pack.js +63 -0
- package/dist/pack.js.map +1 -0
- package/dist/react.d.ts +26 -0
- package/dist/react.d.ts.map +1 -0
- package/dist/react.js +78 -0
- package/dist/react.js.map +1 -0
- package/dist/renderer.d.ts +19 -0
- package/dist/renderer.d.ts.map +1 -0
- package/dist/renderer.js +25 -0
- package/dist/renderer.js.map +1 -0
- package/dist/settings.d.ts +36 -0
- package/dist/settings.d.ts.map +1 -0
- package/dist/settings.js +112 -0
- package/dist/settings.js.map +1 -0
- package/dist/shader.glsl.d.ts +3 -0
- package/dist/shader.glsl.d.ts.map +1 -0
- package/dist/shader.glsl.js +179 -0
- package/dist/shader.glsl.js.map +1 -0
- package/dist/shader.wgsl.d.ts +2 -0
- package/dist/shader.wgsl.d.ts.map +1 -0
- package/dist/shader.wgsl.js +160 -0
- package/dist/shader.wgsl.js.map +1 -0
- package/dist/source.d.ts +16 -0
- package/dist/source.d.ts.map +1 -0
- package/dist/source.js +103 -0
- package/dist/source.js.map +1 -0
- package/dist/three-webgpu.d.ts +20 -0
- package/dist/three-webgpu.d.ts.map +1 -0
- package/dist/three-webgpu.js +178 -0
- package/dist/three-webgpu.js.map +1 -0
- package/dist/three.d.ts +27 -0
- package/dist/three.d.ts.map +1 -0
- package/dist/three.js +81 -0
- package/dist/three.js.map +1 -0
- package/dist/types.d.ts +4 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +4 -0
- package/dist/types.js.map +1 -0
- package/dist/webgl.d.ts +13 -0
- package/dist/webgl.d.ts.map +1 -0
- package/dist/webgl.js +183 -0
- package/dist/webgl.js.map +1 -0
- package/dist/webgpu.d.ts +19 -0
- package/dist/webgpu.d.ts.map +1 -0
- package/dist/webgpu.js +303 -0
- package/dist/webgpu.js.map +1 -0
- package/package.json +103 -4
- package/src/global.ts +11 -0
- package/src/greenscreen.ts +286 -0
- package/src/index.ts +5 -0
- package/src/pack.ts +73 -0
- package/src/react.tsx +126 -0
- package/src/renderer.ts +53 -0
- package/src/settings.ts +177 -0
- package/src/shader.glsl.ts +179 -0
- package/src/shader.wgsl.ts +159 -0
- package/src/source.ts +105 -0
- package/src/three-webgpu.ts +261 -0
- package/src/three.ts +114 -0
- package/src/types.ts +10 -0
- package/src/webgl.ts +191 -0
- package/src/webgpu.ts +312 -0
package/src/webgpu.ts
ADDED
|
@@ -0,0 +1,312 @@
|
|
|
1
|
+
import { PACKED_LENGTH } from './pack.ts'
|
|
2
|
+
import { contextOf, tooLarge } from './renderer.ts'
|
|
3
|
+
import type { Context2D, Renderer, RendererEvents } from './renderer.ts'
|
|
4
|
+
import { SHADER } from './shader.wgsl.ts'
|
|
5
|
+
import type { Canvas, Source } from './types.ts'
|
|
6
|
+
|
|
7
|
+
// The WebGPU usage flags, as the spec numbers them, rather than the browser's globals, which TypeScript doesn't declare
|
|
8
|
+
const TEXTURE_COPY_DST = 0x02
|
|
9
|
+
const TEXTURE_BINDING = 0x04
|
|
10
|
+
const TEXTURE_RENDER_ATTACHMENT = 0x10
|
|
11
|
+
const BUFFER_COPY_DST = 0x08
|
|
12
|
+
const BUFFER_UNIFORM = 0x40
|
|
13
|
+
|
|
14
|
+
// One device for every keyer on the page, as a page can have many, and WebGPU, unlike WebGL, doesn't need a context of
|
|
15
|
+
// its own for each. It's asked for once, and asked for again only after it's lost
|
|
16
|
+
let shared: Promise<GPUDevice> | null = null
|
|
17
|
+
// The device once it's there, for a keyer made after it to start with at once
|
|
18
|
+
let current: GPUDevice | null = null
|
|
19
|
+
|
|
20
|
+
// Whether 'auto' takes WebGPU, found once, and again after the device is lost. known is what it came to, for a keyer to
|
|
21
|
+
// choose by at once, and null until then
|
|
22
|
+
let preferred: Promise<boolean> | null = null
|
|
23
|
+
let known: boolean | null = null
|
|
24
|
+
|
|
25
|
+
// The page's device, from the adapter given or the browser's own. It's asked for with the largest texture the adapter
|
|
26
|
+
// has, as a device takes pictures of only 8192 pixels a side unless more is asked for
|
|
27
|
+
export function requestDevice(given?: GPUAdapter): Promise<GPUDevice> {
|
|
28
|
+
if (shared) return shared
|
|
29
|
+
const attempt = (async () => {
|
|
30
|
+
const adapter = given ?? (await requestAdapter())
|
|
31
|
+
const device = await adapter.requestDevice({
|
|
32
|
+
requiredLimits: { maxTextureDimension2D: adapter.limits.maxTextureDimension2D },
|
|
33
|
+
})
|
|
34
|
+
void device.lost.then(() => {
|
|
35
|
+
if (shared !== attempt) return
|
|
36
|
+
shared = null
|
|
37
|
+
current = null
|
|
38
|
+
preferred = null
|
|
39
|
+
known = null
|
|
40
|
+
})
|
|
41
|
+
return device
|
|
42
|
+
})()
|
|
43
|
+
shared = attempt
|
|
44
|
+
// Before anything that waits for it goes on, as this waits first
|
|
45
|
+
void attempt.then(
|
|
46
|
+
(device) => {
|
|
47
|
+
if (shared === attempt) current = device
|
|
48
|
+
},
|
|
49
|
+
() => {},
|
|
50
|
+
)
|
|
51
|
+
return attempt
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
// The device, where it has been had already and not lost since
|
|
55
|
+
export function currentDevice(): GPUDevice | null {
|
|
56
|
+
return current
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
async function requestAdapter(): Promise<GPUAdapter> {
|
|
60
|
+
const gpu = typeof navigator === 'undefined' ? undefined : navigator.gpu
|
|
61
|
+
if (!gpu) throw new Error('Greenscreen: WebGPU is not available here')
|
|
62
|
+
const adapter = await gpu.requestAdapter()
|
|
63
|
+
if (!adapter) throw new Error('Greenscreen: WebGPU has no adapter here')
|
|
64
|
+
return adapter
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
// Whether 'auto' takes WebGPU: where the browser has an adapter for it on the GPU, and a device from that adapter. A
|
|
68
|
+
// fallback adapter, which runs WebGPU on the CPU, doesn't count, as WebGL2 is then likely on the GPU, and faster
|
|
69
|
+
export function prefersWebGPU(): Promise<boolean> {
|
|
70
|
+
if (preferred) return preferred
|
|
71
|
+
const attempt = (async () => {
|
|
72
|
+
try {
|
|
73
|
+
const adapter = await requestAdapter()
|
|
74
|
+
if (adapter.info.isFallbackAdapter) return false
|
|
75
|
+
await requestDevice(adapter)
|
|
76
|
+
return true
|
|
77
|
+
} catch {
|
|
78
|
+
return false
|
|
79
|
+
}
|
|
80
|
+
})()
|
|
81
|
+
preferred = attempt
|
|
82
|
+
void attempt.then((webgpu) => {
|
|
83
|
+
if (preferred === attempt) known = webgpu
|
|
84
|
+
})
|
|
85
|
+
return attempt
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
// What prefersWebGPU() comes to, where that's known already: once it has resolved, or at once where there's no WebGPU
|
|
89
|
+
export function knownPreference(): boolean | null {
|
|
90
|
+
if (typeof navigator === 'undefined' || !navigator.gpu) return false
|
|
91
|
+
return known
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
const pipelines = new WeakMap<GPUDevice, Map<GPUTextureFormat, GPURenderPipeline>>()
|
|
95
|
+
|
|
96
|
+
// The keying pipeline for the device and canvas format, made once. It has no sampler: the shader reads the texture's
|
|
97
|
+
// pixels itself, and blends them as OBS's sampler would
|
|
98
|
+
export function pipelineFor(device: GPUDevice, format: GPUTextureFormat): GPURenderPipeline {
|
|
99
|
+
let byFormat = pipelines.get(device)
|
|
100
|
+
if (!byFormat) {
|
|
101
|
+
byFormat = new Map()
|
|
102
|
+
pipelines.set(device, byFormat)
|
|
103
|
+
}
|
|
104
|
+
let made = byFormat.get(format)
|
|
105
|
+
if (!made) {
|
|
106
|
+
const module = device.createShaderModule({ label: 'Greenscreen', code: SHADER })
|
|
107
|
+
made = device.createRenderPipeline({
|
|
108
|
+
label: 'Greenscreen',
|
|
109
|
+
layout: 'auto',
|
|
110
|
+
vertex: { module, entryPoint: 'vertexMain' },
|
|
111
|
+
fragment: { module, entryPoint: 'fragmentMain', targets: [{ format }] },
|
|
112
|
+
primitive: { topology: 'triangle-list' },
|
|
113
|
+
})
|
|
114
|
+
byFormat.set(format, made)
|
|
115
|
+
}
|
|
116
|
+
return made
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
export function preferredFormat(): GPUTextureFormat {
|
|
120
|
+
return navigator.gpu.getPreferredCanvasFormat()
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
// WebGPU keys into a canvas of its own, off the page, and each picture is copied from there into the keyer's canvas,
|
|
124
|
+
// which shows it through a 2D context. A WebGPU canvas on the page gives drawImage nothing once the browser has drawn
|
|
125
|
+
// the page, where a 2D canvas keeps its picture until it's drawn again, on the page or off it
|
|
126
|
+
export class WebGPURenderer implements Renderer {
|
|
127
|
+
readonly backend = 'webgpu'
|
|
128
|
+
readonly #canvas: Canvas
|
|
129
|
+
readonly #output: Context2D
|
|
130
|
+
readonly #surface: OffscreenCanvas
|
|
131
|
+
readonly #context: GPUCanvasContext
|
|
132
|
+
readonly #events: RendererEvents
|
|
133
|
+
#device!: GPUDevice
|
|
134
|
+
#pipeline!: GPURenderPipeline
|
|
135
|
+
#uniforms!: GPUBuffer
|
|
136
|
+
#texture: GPUTexture | null = null
|
|
137
|
+
#bindGroup: GPUBindGroup | null = null
|
|
138
|
+
#videos: Context2D | null = null
|
|
139
|
+
#lost = false
|
|
140
|
+
#disposed = false
|
|
141
|
+
|
|
142
|
+
constructor(canvas: Canvas, device: GPUDevice, events: RendererEvents) {
|
|
143
|
+
// WebGPU first, so a canvas WebGPU can't draw for is left as it was
|
|
144
|
+
const surface = new OffscreenCanvas(1, 1)
|
|
145
|
+
const context = contextOf(surface, 'webgpu')
|
|
146
|
+
if (!context) throw new Error('Greenscreen: WebGPU is not available on this canvas')
|
|
147
|
+
const output = contextOf(canvas, '2d')
|
|
148
|
+
if (!output) throw new Error('Greenscreen: this canvas has a context of another kind already')
|
|
149
|
+
this.#canvas = canvas
|
|
150
|
+
this.#output = output
|
|
151
|
+
this.#surface = surface
|
|
152
|
+
this.#context = context
|
|
153
|
+
this.#events = events
|
|
154
|
+
this.#build(device)
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
get lost() {
|
|
158
|
+
return this.#lost
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
// The surface set up for the device, premultiplied as the shader's output is
|
|
162
|
+
#build(device: GPUDevice) {
|
|
163
|
+
const format = preferredFormat()
|
|
164
|
+
this.#device = device
|
|
165
|
+
this.#context.configure({ device, format, alphaMode: 'premultiplied', usage: TEXTURE_RENDER_ATTACHMENT })
|
|
166
|
+
this.#pipeline = pipelineFor(device, format)
|
|
167
|
+
this.#uniforms = device.createBuffer({
|
|
168
|
+
label: 'Greenscreen settings',
|
|
169
|
+
size: PACKED_LENGTH * 4,
|
|
170
|
+
usage: BUFFER_UNIFORM | BUFFER_COPY_DST,
|
|
171
|
+
})
|
|
172
|
+
this.#texture = null
|
|
173
|
+
this.#bindGroup = null
|
|
174
|
+
void device.lost.then(() => this.#onLost(device))
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
// Lost with the device, it asks for the next one and starts again with it
|
|
178
|
+
async #onLost(device: GPUDevice) {
|
|
179
|
+
if (this.#disposed || device !== this.#device) return
|
|
180
|
+
this.#lost = true
|
|
181
|
+
this.#texture = null
|
|
182
|
+
this.#bindGroup = null
|
|
183
|
+
this.#events.lost()
|
|
184
|
+
try {
|
|
185
|
+
const next = await requestDevice()
|
|
186
|
+
if (this.#disposed) return
|
|
187
|
+
this.#build(next)
|
|
188
|
+
this.#lost = false
|
|
189
|
+
this.#events.restored()
|
|
190
|
+
} catch {
|
|
191
|
+
// No device to be had: it stays lost
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
// The picture as it comes, top row first and not premultiplied, in a texture of its size. copyExternalImageToTexture
|
|
196
|
+
// takes every kind of source, a video by way of a 2D canvas. A picture larger than the device's textures throws, as
|
|
197
|
+
// a texture of its size would be made without an error thrown, and never drawn
|
|
198
|
+
upload(source: Source, width: number, height: number) {
|
|
199
|
+
if (this.#lost) return
|
|
200
|
+
const device = this.#device
|
|
201
|
+
const limit = device.limits.maxTextureDimension2D
|
|
202
|
+
if (width > limit || height > limit) throw tooLarge(width, height, 'WebGPU', limit)
|
|
203
|
+
let texture = this.#texture
|
|
204
|
+
if (!texture || texture.width !== width || texture.height !== height) {
|
|
205
|
+
texture?.destroy()
|
|
206
|
+
texture = device.createTexture({
|
|
207
|
+
label: 'Greenscreen picture',
|
|
208
|
+
size: { width, height },
|
|
209
|
+
format: 'rgba8unorm',
|
|
210
|
+
usage: TEXTURE_BINDING | TEXTURE_COPY_DST | TEXTURE_RENDER_ATTACHMENT,
|
|
211
|
+
})
|
|
212
|
+
this.#texture = texture
|
|
213
|
+
this.#bindGroup = device.createBindGroup({
|
|
214
|
+
layout: this.#pipeline.getBindGroupLayout(0),
|
|
215
|
+
entries: [
|
|
216
|
+
{ binding: 0, resource: texture.createView() },
|
|
217
|
+
{ binding: 1, resource: { buffer: this.#uniforms } },
|
|
218
|
+
],
|
|
219
|
+
})
|
|
220
|
+
}
|
|
221
|
+
device.queue.copyExternalImageToTexture(
|
|
222
|
+
{ source: this.#asDrawn(source, width, height), flipY: false },
|
|
223
|
+
{ texture, colorSpace: 'srgb', premultipliedAlpha: false },
|
|
224
|
+
{ width, height },
|
|
225
|
+
)
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
// A video comes in through a 2D canvas. WebGPU's own copy converts a video's colours from the video's colour space
|
|
229
|
+
// to sRGB, which moves those of a BT.601 video, as cameras' and recordings often are, by as much as 20 levels in 255:
|
|
230
|
+
// the keyer would key colours a little apart from those WebGL keys, and from those a 2D canvas reads, as an
|
|
231
|
+
// eyedropper does. A 2D canvas reads a video as WebGL does. It also reads a camera's first frame, which WebGPU's copy
|
|
232
|
+
// can't. A VideoFrame of RGB pixels, as one made from a canvas is, has nothing to convert, and comes as it is
|
|
233
|
+
#asDrawn(source: Source, width: number, height: number): Source {
|
|
234
|
+
const video = 'videoWidth' in source || ('displayWidth' in source && !/^(RGB|BGR)/.test(source.format ?? ''))
|
|
235
|
+
if (!video) return source
|
|
236
|
+
let context = this.#videos
|
|
237
|
+
if (!context) {
|
|
238
|
+
context = contextOf(stagingCanvas(), '2d')
|
|
239
|
+
if (!context) return source
|
|
240
|
+
this.#videos = context
|
|
241
|
+
}
|
|
242
|
+
const canvas = context.canvas
|
|
243
|
+
if (canvas.width !== width || canvas.height !== height) {
|
|
244
|
+
canvas.width = width
|
|
245
|
+
canvas.height = height
|
|
246
|
+
}
|
|
247
|
+
context.globalCompositeOperation = 'copy'
|
|
248
|
+
context.drawImage(source, 0, 0, width, height)
|
|
249
|
+
return canvas
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
// The surface's texture is taken just before it's drawn over whole, and the picture is copied across, alpha and all,
|
|
253
|
+
// into the keyer's canvas
|
|
254
|
+
draw(settings: Float32Array<ArrayBuffer>) {
|
|
255
|
+
const texture = this.#texture
|
|
256
|
+
const bindGroup = this.#bindGroup
|
|
257
|
+
if (this.#lost || !texture || !bindGroup) return
|
|
258
|
+
const { width, height } = texture
|
|
259
|
+
for (const canvas of [this.#surface, this.#canvas]) {
|
|
260
|
+
if (canvas.width !== width || canvas.height !== height) {
|
|
261
|
+
canvas.width = width
|
|
262
|
+
canvas.height = height
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
const device = this.#device
|
|
266
|
+
device.queue.writeBuffer(this.#uniforms, 0, settings, 0, settings.length)
|
|
267
|
+
const encoder = device.createCommandEncoder()
|
|
268
|
+
const pass = encoder.beginRenderPass({
|
|
269
|
+
colorAttachments: [
|
|
270
|
+
{
|
|
271
|
+
view: this.#context.getCurrentTexture().createView(),
|
|
272
|
+
loadOp: 'clear',
|
|
273
|
+
clearValue: [0, 0, 0, 0],
|
|
274
|
+
storeOp: 'store',
|
|
275
|
+
},
|
|
276
|
+
],
|
|
277
|
+
})
|
|
278
|
+
pass.setPipeline(this.#pipeline)
|
|
279
|
+
pass.setBindGroup(0, bindGroup)
|
|
280
|
+
pass.draw(3)
|
|
281
|
+
pass.end()
|
|
282
|
+
device.queue.submit([encoder.finish()])
|
|
283
|
+
const output = this.#output
|
|
284
|
+
output.globalCompositeOperation = 'copy'
|
|
285
|
+
output.drawImage(this.#surface, 0, 0)
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
// The keyer's canvas shows its picture through a 2D context, which clears whatever its composite operation
|
|
289
|
+
clear() {
|
|
290
|
+
const canvas = this.#canvas
|
|
291
|
+
this.#output.clearRect(0, 0, canvas.width, canvas.height)
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
dispose() {
|
|
295
|
+
this.#disposed = true
|
|
296
|
+
this.#lost = true
|
|
297
|
+
this.#texture?.destroy()
|
|
298
|
+
this.#uniforms.destroy()
|
|
299
|
+
this.#texture = null
|
|
300
|
+
this.#bindGroup = null
|
|
301
|
+
this.#videos = null
|
|
302
|
+
this.#context.unconfigure()
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
// The canvas a video is drawn into on its way in: an element, off the page, where there's a document. WebGPU's copy
|
|
307
|
+
// from a 2D OffscreenCanvas comes out empty on SwiftShader, Chrome's software renderer, where one from an element
|
|
308
|
+
// doesn't. A worker has only OffscreenCanvas
|
|
309
|
+
function stagingCanvas(): Canvas {
|
|
310
|
+
if (typeof document !== 'undefined') return document.createElement('canvas')
|
|
311
|
+
return new OffscreenCanvas(1, 1)
|
|
312
|
+
}
|