@snovakow/greenscreen 0.0.0-stage → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +238 -2
  3. package/dist/greenscreen.d.ts +32 -0
  4. package/dist/greenscreen.d.ts.map +1 -0
  5. package/dist/greenscreen.global.js +315 -0
  6. package/dist/greenscreen.global.js.map +1 -0
  7. package/dist/greenscreen.js +250 -0
  8. package/dist/greenscreen.js.map +1 -0
  9. package/dist/greenscreen.min.js +315 -0
  10. package/dist/greenscreen.min.js.map +1 -0
  11. package/dist/index.d.ts +6 -0
  12. package/dist/index.d.ts.map +1 -0
  13. package/dist/index.js +3 -0
  14. package/dist/index.js.map +1 -0
  15. package/dist/pack.d.ts +7 -0
  16. package/dist/pack.d.ts.map +1 -0
  17. package/dist/pack.js +63 -0
  18. package/dist/pack.js.map +1 -0
  19. package/dist/react.d.ts +26 -0
  20. package/dist/react.d.ts.map +1 -0
  21. package/dist/react.js +78 -0
  22. package/dist/react.js.map +1 -0
  23. package/dist/renderer.d.ts +19 -0
  24. package/dist/renderer.d.ts.map +1 -0
  25. package/dist/renderer.js +25 -0
  26. package/dist/renderer.js.map +1 -0
  27. package/dist/settings.d.ts +36 -0
  28. package/dist/settings.d.ts.map +1 -0
  29. package/dist/settings.js +112 -0
  30. package/dist/settings.js.map +1 -0
  31. package/dist/shader.glsl.d.ts +3 -0
  32. package/dist/shader.glsl.d.ts.map +1 -0
  33. package/dist/shader.glsl.js +179 -0
  34. package/dist/shader.glsl.js.map +1 -0
  35. package/dist/shader.wgsl.d.ts +2 -0
  36. package/dist/shader.wgsl.d.ts.map +1 -0
  37. package/dist/shader.wgsl.js +160 -0
  38. package/dist/shader.wgsl.js.map +1 -0
  39. package/dist/source.d.ts +16 -0
  40. package/dist/source.d.ts.map +1 -0
  41. package/dist/source.js +103 -0
  42. package/dist/source.js.map +1 -0
  43. package/dist/three-webgpu.d.ts +20 -0
  44. package/dist/three-webgpu.d.ts.map +1 -0
  45. package/dist/three-webgpu.js +178 -0
  46. package/dist/three-webgpu.js.map +1 -0
  47. package/dist/three.d.ts +27 -0
  48. package/dist/three.d.ts.map +1 -0
  49. package/dist/three.js +81 -0
  50. package/dist/three.js.map +1 -0
  51. package/dist/types.d.ts +4 -0
  52. package/dist/types.d.ts.map +1 -0
  53. package/dist/types.js +4 -0
  54. package/dist/types.js.map +1 -0
  55. package/dist/webgl.d.ts +13 -0
  56. package/dist/webgl.d.ts.map +1 -0
  57. package/dist/webgl.js +183 -0
  58. package/dist/webgl.js.map +1 -0
  59. package/dist/webgpu.d.ts +19 -0
  60. package/dist/webgpu.d.ts.map +1 -0
  61. package/dist/webgpu.js +303 -0
  62. package/dist/webgpu.js.map +1 -0
  63. package/package.json +103 -4
  64. package/src/global.ts +11 -0
  65. package/src/greenscreen.ts +286 -0
  66. package/src/index.ts +5 -0
  67. package/src/pack.ts +73 -0
  68. package/src/react.tsx +126 -0
  69. package/src/renderer.ts +53 -0
  70. package/src/settings.ts +177 -0
  71. package/src/shader.glsl.ts +179 -0
  72. package/src/shader.wgsl.ts +159 -0
  73. package/src/source.ts +105 -0
  74. package/src/three-webgpu.ts +261 -0
  75. package/src/three.ts +114 -0
  76. package/src/types.ts +10 -0
  77. package/src/webgl.ts +191 -0
  78. package/src/webgpu.ts +312 -0
@@ -0,0 +1,261 @@
1
+ import {
2
+ clamp,
3
+ dot,
4
+ float,
5
+ floor,
6
+ Fn,
7
+ fwidth,
8
+ If,
9
+ int,
10
+ ivec2,
11
+ max,
12
+ min,
13
+ mix,
14
+ pow,
15
+ select,
16
+ texture,
17
+ textureLoad,
18
+ textureSize,
19
+ uniform,
20
+ uv,
21
+ vec2,
22
+ vec3,
23
+ vec4,
24
+ } from 'three/tsl'
25
+ import { ColorManagement, NodeMaterial, SRGBTransfer, Vector4 } from 'three/webgpu'
26
+ import type { Node, Texture } from 'three/webgpu'
27
+ import { CB, CR, pack, PACKED_LENGTH } from './pack.ts'
28
+ import { DEFAULT_SETTINGS, mergeSettings } from './settings.ts'
29
+ import type { Settings } from './settings.ts'
30
+
31
+ // The keyer as a node material, for three.js's WebGPURenderer, on its WebGPU and WebGL2 backends alike: OBS's chroma
32
+ // key filter and the add-ons, in the same steps as the GLSL in shader.glsl.ts, in TSL, from the same packed settings.
33
+ // It keys its map as OBS's filter would key it drawn at the mesh's size on screen, and shows it as
34
+ // MeshBasicNodeMaterial would, keyed, then mirrored, cut and coloured by the add-ons. It doesn't apply the texture's
35
+ // offset, repeat or rotation, and it reads the map's own pixels and blends them itself, as OBS's sampler does, so it
36
+ // uses no mipmaps. Like every node material, it gives the renderer linear colour, which the renderer encodes for the
37
+ // screen itself
38
+
39
+ export interface GreenscreenNodeMaterialParameters {
40
+ // The picture to key: a VideoTexture, a Texture of an image or canvas, or any other 8-bit texture, in sRGB or with no
41
+ // colour space. Its colour space is read when it's given, and when the settings are set
42
+ map?: Texture | null
43
+ settings?: Readonly<Partial<Settings>>
44
+ }
45
+
46
+ type Vec4Uniform = Node<'vec4'> & { value: Vector4 }
47
+ type Vec2Node = Node<'vec2'>
48
+ type Vec3Node = Node<'vec3'>
49
+ type Vec4Node = Node<'vec4'>
50
+ type FloatNode = Node<'float'>
51
+
52
+ interface Uniforms {
53
+ key: Vec4Uniform
54
+ matte: Vec4Uniform
55
+ correction: Vec4Uniform
56
+ extras: Vec4Uniform
57
+ flags: Vec4Uniform
58
+ corners01: Vec4Uniform
59
+ corners23: Vec4Uniform
60
+ switches: Vec4Uniform
61
+ }
62
+
63
+ const NAMES = ['key', 'matte', 'correction', 'extras', 'flags', 'corners01', 'corners23', 'switches'] as const
64
+
65
+ const LUMA = [0.2126, 0.7152, 0.0722] as const
66
+ const SCALE_B = 1.8556
67
+ const SCALE_R = 1.5748
68
+
69
+ const luma = (c: Vec3Node) => dot(c, vec3(...LUMA))
70
+
71
+ // hueSpill's: an sRGB colour's Pb and Pr, and the colour again from its luma and those
72
+ const chromaOf = (encoded: Vec3Node) => {
73
+ const y = luma(encoded)
74
+ return vec2(encoded.z.sub(y).div(SCALE_B), encoded.x.sub(y).div(SCALE_R))
75
+ }
76
+
77
+ const fromLumaChroma = (y: FloatNode, c: Vec2Node) => {
78
+ const r = y.add(c.y.mul(SCALE_R))
79
+ const b = y.add(c.x.mul(SCALE_B))
80
+ return vec3(r, y.sub(r.mul(LUMA[0])).sub(b.mul(LUMA[2])).div(LUMA[1]), b)
81
+ }
82
+
83
+ const channelToLinear = (c: FloatNode) => select(c.lessThanEqual(0.04045), c.div(12.92), pow(c.add(0.055).div(1.055), 2.4))
84
+ const channelToEncoded = (c: FloatNode) =>
85
+ select(c.lessThanEqual(0.0031308), c.mul(12.92), pow(c, 1 / 2.4).mul(1.055).sub(0.055))
86
+ const toLinear = (c: Vec3Node) => vec3(channelToLinear(c.x), channelToLinear(c.y), channelToLinear(c.z))
87
+ const toEncoded = (c: Vec3Node) => vec3(channelToEncoded(c.x), channelToEncoded(c.y), channelToEncoded(c.z))
88
+
89
+ const cutSide = (a: Vec2Node, b: Vec2Node, p: Vec2Node, size: Vec2Node) => {
90
+ const side = b.sub(a).mul(size)
91
+ const len = side.length()
92
+ const q = p.sub(a).mul(size)
93
+ return select(len.equal(0), float(1e6), q.y.mul(side.x).sub(q.x.mul(side.y)).div(max(len, 1e-12)))
94
+ }
95
+
96
+ function keyer(map: Texture, u: Uniforms) {
97
+ return Fn(() => {
98
+ // Where this pixel is on the output, 0 to 1 across and down, and output pixels to the picture's size
99
+ const out = vec2(uv().x, float(1).sub(uv().y)).toVar()
100
+ const size = float(1).div(max(fwidth(out), vec2(1e-6))).toVar()
101
+
102
+ // Where it is on the picture, mirrored by flip, and the picture's size in pixels
103
+ const p = vec2(select(u.flags.x.greaterThan(0.5), float(1).sub(out.x), out.x), out.y).toVar()
104
+ // oxlint-disable-next-line typescript/no-unsafe-type-assertion -- @types/three gives textureSize an untyped node
105
+ const pixels = vec2(textureSize(texture(map), int(0)) as unknown as Node<'uvec2'>).toVar()
106
+ const last = pixels.sub(1).toVar()
107
+
108
+ // A pixel, at a column and a row from the top in whole numbers, clamped to the picture's edges and premultiplied in
109
+ // linear light, and the picture at a point between pixels, as OBS's sampler gives it
110
+ const pixelAt = (at: Vec2Node): Vec4Node => {
111
+ const clamped = clamp(at, vec2(0, 0), last).toVar()
112
+ const row = select(u.flags.y.greaterThan(0.5), last.y.sub(clamped.y), clamped.y)
113
+ const texel = textureLoad(map, ivec2(vec2(clamped.x, row)), int(0)).toVar()
114
+ const rgb = select(u.flags.w.greaterThan(0.5), texel.rgb, toLinear(texel.rgb))
115
+ return vec4(rgb.mul(texel.a), texel.a)
116
+ }
117
+ const sampleAt = (q: Vec2Node): Vec4Node => {
118
+ const f = q.mul(pixels).sub(0.5).toVar()
119
+ const corner = floor(f).toVar()
120
+ const w = f.sub(corner).toVar()
121
+ const above = mix(pixelAt(corner), pixelAt(corner.add(vec2(1, 0))), w.x)
122
+ const below = mix(pixelAt(corner.add(vec2(0, 1))), pixelAt(corner.add(vec2(1, 1))), w.x)
123
+ return mix(above, below, w.y)
124
+ }
125
+ const keyDistance = (encoded: Vec3Node) =>
126
+ vec2(dot(encoded, vec3(CB[0], CB[1], CB[2])).add(CB[3]), dot(encoded, vec3(CR[0], CR[1], CR[2])).add(CR[3]))
127
+ .sub(u.key.xy)
128
+ .length()
129
+
130
+ const sampled = sampleAt(p).toVar()
131
+ const alpha = sampled.a.toVar()
132
+ const rgb = select(alpha.greaterThan(0), sampled.rgb.div(alpha), vec3(0)).toVar()
133
+
134
+ // OBS's filter
135
+ If(u.key.w.greaterThan(0.5), () => {
136
+ const texel = vec2(1, 1).div(pixels).toVar()
137
+ const across = vec2(texel.x, texel.y.mul(0.5)).toVar()
138
+ const up = vec2(texel.x.mul(0.5), texel.y.negate()).toVar()
139
+ const around = keyDistance(toEncoded(sampleAt(p.sub(across)).rgb))
140
+ .add(keyDistance(toEncoded(sampleAt(p.add(across)).rgb)))
141
+ .add(keyDistance(toEncoded(sampleAt(p.sub(up)).rgb)))
142
+ .add(keyDistance(toEncoded(sampleAt(p.add(up)).rgb)))
143
+ const beyond = around.mul(2).add(keyDistance(toEncoded(rgb))).div(9).sub(u.matte.x).toVar()
144
+ alpha.mulAssign(u.matte.w)
145
+ alpha.mulAssign(pow(clamp(beyond.div(u.matte.y), 0, 1), 1.5))
146
+ const kept = pow(clamp(beyond.div(u.matte.z), 0, 1), 1.5).toVar()
147
+ If(u.switches.y.greaterThan(0.5), () => {
148
+ // The add-on, hueSpill
149
+ const encoded = toEncoded(rgb).toVar()
150
+ const toward = u.key.xy.sub(vec2(CB[3], CR[3])).mul(u.key.z).toVar()
151
+ const c = chromaOf(encoded).toVar()
152
+ c.subAssign(toward.mul(max(dot(c, toward), 0)).mul(float(1).sub(kept)))
153
+ rgb.assign(toLinear(clamp(fromLumaChroma(luma(encoded), c), 0, 1)))
154
+ }).Else(() => {
155
+ rgb.assign(mix(vec3(luma(rgb)), rgb, kept))
156
+ })
157
+ rgb.assign(pow(max(rgb, vec3(0)), vec3(u.correction.x)).mul(u.correction.y).add(u.correction.z))
158
+ })
159
+ rgb.assign(clamp(rgb, 0, 1))
160
+
161
+ // The add-ons
162
+ rgb.mulAssign(u.extras.xyz)
163
+ rgb.assign(clamp(mix(vec3(luma(rgb)), rgb, u.extras.w), 0, 1))
164
+ If(u.switches.x.greaterThan(0.5), () => {
165
+ const c0 = u.corners01.xy
166
+ const c1 = u.corners01.zw
167
+ const c2 = u.corners23.xy
168
+ const c3 = u.corners23.zw
169
+ const d = min(
170
+ min(cutSide(c0, c1, p, size), cutSide(c1, c2, p, size)),
171
+ min(cutSide(c2, c3, p, size), cutSide(c3, c0, p, size)),
172
+ )
173
+ alpha.mulAssign(clamp(d.add(0.5), 0, 1))
174
+ })
175
+
176
+ // Straight, and linear: the material premultiplies it, and the renderer encodes it
177
+ return vec4(rgb, alpha)
178
+ })()
179
+ }
180
+
181
+ export class GreenscreenNodeMaterial extends NodeMaterial {
182
+ static get type(): string {
183
+ return 'GreenscreenNodeMaterial'
184
+ }
185
+
186
+ readonly isGreenscreenNodeMaterial = true
187
+ #settings: Readonly<Settings> = DEFAULT_SETTINGS
188
+ readonly #packed = new Float32Array(PACKED_LENGTH)
189
+ readonly #uniforms: Uniforms = {
190
+ key: uniform(new Vector4()),
191
+ matte: uniform(new Vector4()),
192
+ correction: uniform(new Vector4()),
193
+ extras: uniform(new Vector4()),
194
+ flags: uniform(new Vector4()),
195
+ corners01: uniform(new Vector4()),
196
+ corners23: uniform(new Vector4()),
197
+ switches: uniform(new Vector4()),
198
+ }
199
+ #map: Texture | null = null
200
+
201
+ constructor({ map = null, settings = {} }: GreenscreenNodeMaterialParameters = {}) {
202
+ super()
203
+ this.name = 'GreenscreenNodeMaterial'
204
+ this.transparent = true
205
+ this.premultipliedAlpha = true
206
+ this.depthWrite = false
207
+ this.lights = false
208
+ this.map = map
209
+ this.set(settings)
210
+ }
211
+
212
+ get map(): Texture | null {
213
+ return this.#map
214
+ }
215
+
216
+ // A new map makes a new shader, as each is built for its texture
217
+ set map(map: Texture | null) {
218
+ if (map === this.#map && this.fragmentNode) return
219
+ this.#map = map
220
+ this.fragmentNode = map ? keyer(map, this.#uniforms) : vec4(0)
221
+ this.needsUpdate = true
222
+ this.#write()
223
+ }
224
+
225
+ get settings(): Readonly<Settings> {
226
+ return this.#settings
227
+ }
228
+
229
+ set(patch: Readonly<Partial<Settings>>): void {
230
+ this.#settings = mergeSettings(this.#settings, patch)
231
+ this.#write()
232
+ }
233
+
234
+ reset(): void {
235
+ this.set(DEFAULT_SETTINGS)
236
+ }
237
+
238
+ // The packed settings into the uniforms, with the flags the material sets itself: rows bottom up in uv, linear out,
239
+ // and linear in from a texture in sRGB, which the GPU decodes as it's read
240
+ #write() {
241
+ const packed = pack(this.#settings, this.#packed)
242
+ const map = this.#map
243
+ packed[17] = 1
244
+ packed[18] = 1
245
+ packed[19] = map && ColorManagement.getTransfer(map.colorSpace) === SRGBTransfer ? 1 : 0
246
+ const u = this.#uniforms
247
+ NAMES.forEach((name, i) => u[name].value.fromArray(packed, i * 4))
248
+ }
249
+
250
+ // A clone keys the same map with the same settings, in uniforms of its own. NodeMaterial's copy would hand it the
251
+ // source's shader, which reads the source's uniforms
252
+ copy(source: this): this {
253
+ super.copy(source)
254
+ if (#settings in source) this.#settings = source.#settings
255
+ this.#map = source.map ?? null
256
+ this.fragmentNode = this.#map ? keyer(this.#map, this.#uniforms) : vec4(0)
257
+ this.needsUpdate = true
258
+ this.#write()
259
+ return this
260
+ }
261
+ }
package/src/three.ts ADDED
@@ -0,0 +1,114 @@
1
+ import { ColorManagement, GLSL3, RawShaderMaterial, SRGBTransfer } from 'three'
2
+ import type { IUniform, Texture, WebGLRenderer } from 'three'
3
+ import { pack, PACKED_LENGTH } from './pack.ts'
4
+ import { DEFAULT_SETTINGS, mergeSettings } from './settings.ts'
5
+ import type { Settings } from './settings.ts'
6
+ import { FRAGMENT } from './shader.glsl.ts'
7
+
8
+ // The keyer as a three.js material, for WebGLRenderer: the core's own GLSL, drawn on whatever mesh it's given. It keys
9
+ // the texture in map as OBS's filter would key it drawn at the mesh's size on screen, and shows it as MeshBasicMaterial
10
+ // would show that map on the mesh, keyed, then mirrored, cut and coloured by the add-ons. It doesn't apply the
11
+ // texture's offset, repeat or rotation. It reads the map's own pixels and blends them itself, as OBS's sampler does, so
12
+ // it uses no mipmaps. The cut's corners, and OBS's samples round each pixel, are on the picture as it shows on the
13
+ // mesh, with y down
14
+
15
+ export interface GreenscreenMaterialParameters {
16
+ // The picture to key: a VideoTexture, a Texture of an image or canvas, or any other 8-bit texture. It may be in
17
+ // sRGB or have no colour space; either way it's keyed as OBS keys a picture of those sRGB values. It must not be
18
+ // premultiplied
19
+ map?: Texture | null
20
+ settings?: Readonly<Partial<Settings>>
21
+ }
22
+
23
+ const VERTEX = /* glsl */ `precision highp float;
24
+ in vec3 position;
25
+ in vec2 uv;
26
+ uniform mat4 modelViewMatrix;
27
+ uniform mat4 projectionMatrix;
28
+ out vec2 vUv;
29
+ void main() {
30
+ vUv = vec2(uv.x, 1.0 - uv.y);
31
+ gl_Position = projectionMatrix * modelViewMatrix * vec4(position, 1.0);
32
+ }
33
+ `
34
+
35
+ interface Uniforms {
36
+ [uniform: string]: IUniform
37
+ image: IUniform<Texture | null>
38
+ settings: IUniform<Float32Array>
39
+ }
40
+
41
+ export class GreenscreenMaterial extends RawShaderMaterial {
42
+ readonly isGreenscreenMaterial = true
43
+ declare uniforms: Uniforms
44
+ #settings: Readonly<Settings> = DEFAULT_SETTINGS
45
+
46
+ constructor({ map = null, settings = {} }: GreenscreenMaterialParameters = {}) {
47
+ super({
48
+ name: 'GreenscreenMaterial',
49
+ glslVersion: GLSL3,
50
+ vertexShader: VERTEX,
51
+ fragmentShader: FRAGMENT,
52
+ uniforms: { image: { value: map }, settings: { value: new Float32Array(PACKED_LENGTH) } },
53
+ // Its output is premultiplied, and blends as that is
54
+ transparent: true,
55
+ premultipliedAlpha: true,
56
+ depthWrite: false,
57
+ })
58
+ this.set(settings)
59
+ }
60
+
61
+ get map(): Texture | null {
62
+ return this.uniforms.image.value
63
+ }
64
+
65
+ set map(texture: Texture | null) {
66
+ this.uniforms.image.value = texture
67
+ }
68
+
69
+ // The settings it keys with, as a frozen object
70
+ get settings(): Readonly<Settings> {
71
+ return this.#settings
72
+ }
73
+
74
+ // Changes the settings named, leaving the rest. The values are copied
75
+ set(patch: Readonly<Partial<Settings>>): void {
76
+ this.#settings = mergeSettings(this.#settings, patch)
77
+ pack(this.#settings, this.uniforms.settings.value)
78
+ this.uniformsNeedUpdate = true
79
+ }
80
+
81
+ reset(): void {
82
+ this.set(DEFAULT_SETTINGS)
83
+ }
84
+
85
+ // What the shader can't know from the settings: the texture's rows run bottom up in uv, as MeshBasicMaterial
86
+ // samples them; whether the texture gives linear values, as an 8-bit sRGB texture other than a video's does in
87
+ // three.js; and whether what it draws into wants linear values, as anything but the screen does
88
+ onBeforeRender(renderer: WebGLRenderer): void {
89
+ const packed = this.uniforms.settings.value
90
+ const target = renderer.getRenderTarget()
91
+ const space =
92
+ target === null
93
+ ? renderer.outputColorSpace
94
+ : 'isXRRenderTarget' in target && target.isXRRenderTarget === true
95
+ ? target.texture.colorSpace
96
+ : ColorManagement.workingColorSpace
97
+ const map = this.map
98
+ packed[17] = 1
99
+ packed[18] = ColorManagement.getTransfer(space) === SRGBTransfer ? 0 : 1
100
+ packed[19] =
101
+ map && !('isVideoTexture' in map && map.isVideoTexture === true) && ColorManagement.getTransfer(map.colorSpace) === SRGBTransfer
102
+ ? 1
103
+ : 0
104
+ this.uniformsNeedUpdate = true
105
+ }
106
+
107
+ // A clone keys with the same settings, in a settings array of its own
108
+ copy(source: this): this {
109
+ super.copy(source)
110
+ this.uniforms.settings.value = Float32Array.from(source.uniforms.settings.value)
111
+ if (#settings in source) this.#settings = source.#settings
112
+ return this
113
+ }
114
+ }
package/src/types.ts ADDED
@@ -0,0 +1,10 @@
1
+ // The public types the keyer's API is written in, kept apart from the modules that use them so that the declarations a
2
+ // project reads name nothing newer than they must
3
+
4
+ // Anything the browser can upload as a texture: a video, an image, a canvas, an ImageBitmap, ImageData or a VideoFrame
5
+ export type Source = TexImageSource
6
+
7
+ // The graphics API a keyer draws with
8
+ export type BackendName = 'webgl' | 'webgpu'
9
+
10
+ export type Canvas = HTMLCanvasElement | OffscreenCanvas
package/src/webgl.ts ADDED
@@ -0,0 +1,191 @@
1
+ import { contextOf, tooLarge } from './renderer.ts'
2
+ import type { Renderer, RendererEvents } from './renderer.ts'
3
+ import { FRAGMENT, VERTEX } from './shader.glsl.ts'
4
+ import type { Canvas, Source } from './types.ts'
5
+
6
+ // Drawn with blending premultiplied, as the shader's output is, and kept between draws, so the canvas can be copied
7
+ // from at any time and not just in the task that drew it
8
+ const ATTRIBUTES: WebGLContextAttributes = {
9
+ alpha: true,
10
+ premultipliedAlpha: true,
11
+ antialias: false,
12
+ depth: false,
13
+ stencil: false,
14
+ preserveDrawingBuffer: true,
15
+ }
16
+
17
+ export class WebGLRenderer implements Renderer {
18
+ readonly backend = 'webgl'
19
+ readonly #canvas: Canvas
20
+ readonly #gl: WebGL2RenderingContext
21
+ readonly #events: RendererEvents
22
+ #program: WebGLProgram | null = null
23
+ #settings: WebGLUniformLocation | null = null
24
+ #texture: WebGLTexture | null = null
25
+ #width = 0
26
+ #height = 0
27
+ // The most pixels a side it can take and draw
28
+ #limit = 0
29
+ #lost = false
30
+
31
+ constructor(canvas: Canvas, events: RendererEvents) {
32
+ const gl = contextOf(canvas, 'webgl2', ATTRIBUTES)
33
+ if (!gl) throw new Error('Greenscreen: WebGL2 is not available on this canvas')
34
+ this.#canvas = canvas
35
+ this.#gl = gl
36
+ this.#events = events
37
+ canvas.addEventListener('webglcontextlost', this.#onLost)
38
+ canvas.addEventListener('webglcontextrestored', this.#onRestored)
39
+ this.#build()
40
+ }
41
+
42
+ get lost() {
43
+ return this.#lost
44
+ }
45
+
46
+ // The program and the texture, as at first and again once a lost context is back. The picture is uploaded as it
47
+ // comes: top row first, not premultiplied, in sRGB
48
+ #build() {
49
+ const gl = this.#gl
50
+ const program = gl.createProgram()
51
+ const vertex = compile(gl, gl.VERTEX_SHADER, VERTEX)
52
+ const fragment = compile(gl, gl.FRAGMENT_SHADER, `#version 300 es\n${FRAGMENT}`)
53
+ gl.attachShader(program, vertex)
54
+ gl.attachShader(program, fragment)
55
+ gl.linkProgram(program)
56
+ if (!gl.getProgramParameter(program, gl.LINK_STATUS) && !gl.isContextLost()) {
57
+ throw new Error(`Greenscreen: the shader didn't link: ${gl.getProgramInfoLog(program)}`)
58
+ }
59
+ gl.deleteShader(vertex)
60
+ gl.deleteShader(fragment)
61
+ gl.useProgram(program)
62
+ gl.uniform1i(gl.getUniformLocation(program, 'image'), 0)
63
+ this.#program = program
64
+ this.#settings = gl.getUniformLocation(program, 'settings')
65
+
66
+ const texture = gl.createTexture()
67
+ gl.activeTexture(gl.TEXTURE0)
68
+ gl.bindTexture(gl.TEXTURE_2D, texture)
69
+ gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR)
70
+ gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR)
71
+ gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE)
72
+ gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE)
73
+ gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, false)
74
+ gl.pixelStorei(gl.UNPACK_PREMULTIPLY_ALPHA_WEBGL, false)
75
+ gl.pixelStorei(gl.UNPACK_COLORSPACE_CONVERSION_WEBGL, gl.BROWSER_DEFAULT_WEBGL)
76
+ this.#texture = texture
77
+ this.#width = 0
78
+ this.#height = 0
79
+ // The widest picture it can take and draw: its largest texture, and its largest viewport, as the canvas is sized to
80
+ // the picture. A lost context says neither, and draws nothing until it's back
81
+ const texture2D: unknown = gl.getParameter(gl.MAX_TEXTURE_SIZE)
82
+ const viewport: unknown = gl.getParameter(gl.MAX_VIEWPORT_DIMS)
83
+ const sides = [texture2D, ...(viewport instanceof Int32Array ? viewport : [])]
84
+ this.#limit = Math.min(...sides.filter((side) => typeof side === 'number'))
85
+ }
86
+
87
+ // A picture larger than the context's textures throws, as it would otherwise be drawn black
88
+ upload(source: Source, width: number, height: number) {
89
+ if (this.#lost) return
90
+ const limit = this.#limit
91
+ if (width > limit || height > limit) throw tooLarge(width, height, 'WebGL2', limit)
92
+ const gl = this.#gl
93
+ gl.activeTexture(gl.TEXTURE0)
94
+ gl.bindTexture(gl.TEXTURE_2D, this.#texture)
95
+ gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA8, gl.RGBA, gl.UNSIGNED_BYTE, this.#straight(source, width, height))
96
+ this.#width = width
97
+ this.#height = height
98
+ }
99
+
100
+ // WebGL uploads an ImageBitmap as it was made, ignoring the unpacking flags, so one made premultiplied, as they are
101
+ // by default, would arrive premultiplied. Drawn into a canvas first, it arrives straight, as everything else does
102
+ #bitmaps: OffscreenCanvasRenderingContext2D | null = null
103
+
104
+ #straight(source: Source, width: number, height: number): Source {
105
+ if (typeof ImageBitmap === 'undefined' || !(source instanceof ImageBitmap) || typeof OffscreenCanvas === 'undefined') {
106
+ return source
107
+ }
108
+ let context = this.#bitmaps
109
+ if (!context) {
110
+ context = new OffscreenCanvas(width, height).getContext('2d')
111
+ if (!context) return source
112
+ this.#bitmaps = context
113
+ }
114
+ const canvas = context.canvas
115
+ if (canvas.width !== width || canvas.height !== height) {
116
+ canvas.width = width
117
+ canvas.height = height
118
+ } else {
119
+ context.clearRect(0, 0, width, height)
120
+ }
121
+ context.drawImage(source, 0, 0)
122
+ return canvas
123
+ }
124
+
125
+ // Every pixel is drawn over, unblended, so there's nothing to clear first
126
+ draw(settings: Float32Array<ArrayBuffer>) {
127
+ if (this.#lost || !this.#width || !this.#height) return
128
+ const gl = this.#gl
129
+ const canvas = this.#canvas
130
+ if (canvas.width !== this.#width || canvas.height !== this.#height) {
131
+ canvas.width = this.#width
132
+ canvas.height = this.#height
133
+ }
134
+ gl.viewport(0, 0, this.#width, this.#height)
135
+ gl.useProgram(this.#program)
136
+ gl.uniform4fv(this.#settings, settings)
137
+ gl.activeTexture(gl.TEXTURE0)
138
+ gl.bindTexture(gl.TEXTURE_2D, this.#texture)
139
+ gl.drawArrays(gl.TRIANGLES, 0, 3)
140
+ }
141
+
142
+ // The drawing buffer is kept between draws, so it stays clear until the next
143
+ clear() {
144
+ if (this.#lost) return
145
+ const gl = this.#gl
146
+ gl.clearColor(0, 0, 0, 0)
147
+ gl.clear(gl.COLOR_BUFFER_BIT)
148
+ }
149
+
150
+ // Lost, its default is prevented, which lets the browser give it back
151
+ #onLost = (event: Event) => {
152
+ event.preventDefault()
153
+ this.#lost = true
154
+ this.#program = null
155
+ this.#texture = null
156
+ this.#width = 0
157
+ this.#height = 0
158
+ this.#events.lost()
159
+ }
160
+
161
+ #onRestored = () => {
162
+ this.#lost = false
163
+ this.#build()
164
+ this.#events.restored()
165
+ }
166
+
167
+ // Loses the context on purpose, as that frees it at once rather than when it's collected, and a page can only have
168
+ // so many
169
+ dispose() {
170
+ const gl = this.#gl
171
+ this.#canvas.removeEventListener('webglcontextlost', this.#onLost)
172
+ this.#canvas.removeEventListener('webglcontextrestored', this.#onRestored)
173
+ if (!this.#lost) {
174
+ gl.deleteTexture(this.#texture)
175
+ gl.deleteProgram(this.#program)
176
+ }
177
+ this.#lost = true
178
+ gl.getExtension('WEBGL_lose_context')?.loseContext()
179
+ }
180
+ }
181
+
182
+ function compile(gl: WebGL2RenderingContext, type: number, source: string) {
183
+ const shader = gl.createShader(type)
184
+ if (!shader) throw new Error('Greenscreen: no shader could be made')
185
+ gl.shaderSource(shader, source)
186
+ gl.compileShader(shader)
187
+ if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS) && !gl.isContextLost()) {
188
+ throw new Error(`Greenscreen: the shader didn't compile: ${gl.getShaderInfoLog(shader)}`)
189
+ }
190
+ return shader
191
+ }