@snovakow/greenscreen 0.0.0-stage → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +239 -2
- package/dist/greenscreen.d.ts +32 -0
- package/dist/greenscreen.d.ts.map +1 -0
- package/dist/greenscreen.global.js +350 -0
- package/dist/greenscreen.global.js.map +1 -0
- package/dist/greenscreen.js +250 -0
- package/dist/greenscreen.js.map +1 -0
- package/dist/greenscreen.min.js +350 -0
- package/dist/greenscreen.min.js.map +1 -0
- package/dist/index.d.ts +6 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +3 -0
- package/dist/index.js.map +1 -0
- package/dist/pack.d.ts +7 -0
- package/dist/pack.d.ts.map +1 -0
- package/dist/pack.js +63 -0
- package/dist/pack.js.map +1 -0
- package/dist/react.d.ts +26 -0
- package/dist/react.d.ts.map +1 -0
- package/dist/react.js +78 -0
- package/dist/react.js.map +1 -0
- package/dist/renderer.d.ts +19 -0
- package/dist/renderer.d.ts.map +1 -0
- package/dist/renderer.js +25 -0
- package/dist/renderer.js.map +1 -0
- package/dist/settings.d.ts +36 -0
- package/dist/settings.d.ts.map +1 -0
- package/dist/settings.js +112 -0
- package/dist/settings.js.map +1 -0
- package/dist/shader.glsl.d.ts +3 -0
- package/dist/shader.glsl.d.ts.map +1 -0
- package/dist/shader.glsl.js +199 -0
- package/dist/shader.glsl.js.map +1 -0
- package/dist/shader.wgsl.d.ts +2 -0
- package/dist/shader.wgsl.d.ts.map +1 -0
- package/dist/shader.wgsl.js +177 -0
- package/dist/shader.wgsl.js.map +1 -0
- package/dist/source.d.ts +16 -0
- package/dist/source.d.ts.map +1 -0
- package/dist/source.js +103 -0
- package/dist/source.js.map +1 -0
- package/dist/three-webgpu.d.ts +20 -0
- package/dist/three-webgpu.d.ts.map +1 -0
- package/dist/three-webgpu.js +193 -0
- package/dist/three-webgpu.js.map +1 -0
- package/dist/three.d.ts +27 -0
- package/dist/three.d.ts.map +1 -0
- package/dist/three.js +81 -0
- package/dist/three.js.map +1 -0
- package/dist/types.d.ts +4 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +4 -0
- package/dist/types.js.map +1 -0
- package/dist/webgl.d.ts +13 -0
- package/dist/webgl.d.ts.map +1 -0
- package/dist/webgl.js +183 -0
- package/dist/webgl.js.map +1 -0
- package/dist/webgpu.d.ts +19 -0
- package/dist/webgpu.d.ts.map +1 -0
- package/dist/webgpu.js +303 -0
- package/dist/webgpu.js.map +1 -0
- package/package.json +103 -4
- package/src/global.ts +11 -0
- package/src/greenscreen.ts +286 -0
- package/src/index.ts +5 -0
- package/src/pack.ts +73 -0
- package/src/react.tsx +126 -0
- package/src/renderer.ts +53 -0
- package/src/settings.ts +179 -0
- package/src/shader.glsl.ts +199 -0
- package/src/shader.wgsl.ts +176 -0
- package/src/source.ts +105 -0
- package/src/three-webgpu.ts +279 -0
- package/src/three.ts +114 -0
- package/src/types.ts +10 -0
- package/src/webgl.ts +191 -0
- package/src/webgpu.ts +312 -0
|
@@ -0,0 +1,279 @@
|
|
|
1
|
+
import {
|
|
2
|
+
clamp,
|
|
3
|
+
dot,
|
|
4
|
+
float,
|
|
5
|
+
floor,
|
|
6
|
+
Fn,
|
|
7
|
+
fwidth,
|
|
8
|
+
If,
|
|
9
|
+
int,
|
|
10
|
+
ivec2,
|
|
11
|
+
max,
|
|
12
|
+
min,
|
|
13
|
+
mix,
|
|
14
|
+
pow,
|
|
15
|
+
select,
|
|
16
|
+
texture,
|
|
17
|
+
textureLoad,
|
|
18
|
+
textureSize,
|
|
19
|
+
uniform,
|
|
20
|
+
uv,
|
|
21
|
+
vec2,
|
|
22
|
+
vec3,
|
|
23
|
+
vec4,
|
|
24
|
+
} from 'three/tsl'
|
|
25
|
+
import { ColorManagement, NodeMaterial, SRGBTransfer, Vector4 } from 'three/webgpu'
|
|
26
|
+
import type { Node, Texture } from 'three/webgpu'
|
|
27
|
+
import { CB, CR, pack, PACKED_LENGTH } from './pack.ts'
|
|
28
|
+
import { DEFAULT_SETTINGS, mergeSettings } from './settings.ts'
|
|
29
|
+
import type { Settings } from './settings.ts'
|
|
30
|
+
|
|
31
|
+
// The keyer as a node material, for three.js's WebGPURenderer, on its WebGPU and WebGL2 backends alike: OBS's chroma
|
|
32
|
+
// key filter and the add-ons, in the same steps as the GLSL in shader.glsl.ts, in TSL, from the same packed settings.
|
|
33
|
+
// It keys its map as OBS's filter would key it drawn at the mesh's size on screen, and shows it as
|
|
34
|
+
// MeshBasicNodeMaterial would, keyed, with the add-ons' colour, mirror and cut. It doesn't apply the texture's
|
|
35
|
+
// offset, repeat or rotation, and it reads the map's own pixels and blends them itself, as OBS's sampler does, so it
|
|
36
|
+
// uses no mipmaps. Like every node material, it gives the renderer linear colour, which the renderer encodes for the
|
|
37
|
+
// screen itself
|
|
38
|
+
|
|
39
|
+
export interface GreenscreenNodeMaterialParameters {
|
|
40
|
+
// The picture to key: a VideoTexture, a Texture of an image or canvas, or any other 8-bit texture, in sRGB or with no
|
|
41
|
+
// colour space. Its colour space is read when it's given, and when the settings are set
|
|
42
|
+
map?: Texture | null
|
|
43
|
+
settings?: Readonly<Partial<Settings>>
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
type Vec4Uniform = Node<'vec4'> & { value: Vector4 }
|
|
47
|
+
type Vec2Node = Node<'vec2'>
|
|
48
|
+
type Vec3Node = Node<'vec3'>
|
|
49
|
+
type Vec4Node = Node<'vec4'>
|
|
50
|
+
type FloatNode = Node<'float'>
|
|
51
|
+
|
|
52
|
+
interface Uniforms {
|
|
53
|
+
key: Vec4Uniform
|
|
54
|
+
matte: Vec4Uniform
|
|
55
|
+
correction: Vec4Uniform
|
|
56
|
+
extras: Vec4Uniform
|
|
57
|
+
flags: Vec4Uniform
|
|
58
|
+
corners01: Vec4Uniform
|
|
59
|
+
corners23: Vec4Uniform
|
|
60
|
+
switches: Vec4Uniform
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
const NAMES = ['key', 'matte', 'correction', 'extras', 'flags', 'corners01', 'corners23', 'switches'] as const
|
|
64
|
+
|
|
65
|
+
const LUMA = [0.2126, 0.7152, 0.0722] as const
|
|
66
|
+
const SCALE_B = 1.8556
|
|
67
|
+
const SCALE_R = 1.5748
|
|
68
|
+
|
|
69
|
+
const luma = (c: Vec3Node) => dot(c, vec3(...LUMA))
|
|
70
|
+
|
|
71
|
+
// hueSpill's: an sRGB colour's Pb and Pr, and the colour again from its luma and those
|
|
72
|
+
const chromaOf = (encoded: Vec3Node) => {
|
|
73
|
+
const y = luma(encoded)
|
|
74
|
+
return vec2(encoded.z.sub(y).div(SCALE_B), encoded.x.sub(y).div(SCALE_R))
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
const fromLumaChroma = (y: FloatNode, c: Vec2Node) => {
|
|
78
|
+
const r = y.add(c.y.mul(SCALE_R))
|
|
79
|
+
const b = y.add(c.x.mul(SCALE_B))
|
|
80
|
+
return vec3(r, y.sub(r.mul(LUMA[0])).sub(b.mul(LUMA[2])).div(LUMA[1]), b)
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
// The add-on saturation, as in shader.glsl.ts: HSB's saturation scaled, keeping the hue and the brightest channel
|
|
84
|
+
const saturated = (rgb: Vec3Node, s: FloatNode) => {
|
|
85
|
+
const top = max(max(rgb.x, rgb.y), rgb.z)
|
|
86
|
+
const spread = top.sub(min(min(rgb.x, rgb.y), rgb.z))
|
|
87
|
+
return vec3(top).sub(vec3(top).sub(rgb).mul(min(s, top.div(max(spread, 1e-12)))))
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
const channelToLinear = (c: FloatNode) => select(c.lessThanEqual(0.04045), c.div(12.92), pow(c.add(0.055).div(1.055), 2.4))
|
|
91
|
+
const channelToEncoded = (c: FloatNode) =>
|
|
92
|
+
select(c.lessThanEqual(0.0031308), c.mul(12.92), pow(c, 1 / 2.4).mul(1.055).sub(0.055))
|
|
93
|
+
const toLinear = (c: Vec3Node) => vec3(channelToLinear(c.x), channelToLinear(c.y), channelToLinear(c.z))
|
|
94
|
+
const toEncoded = (c: Vec3Node) => vec3(channelToEncoded(c.x), channelToEncoded(c.y), channelToEncoded(c.z))
|
|
95
|
+
|
|
96
|
+
const cutSide = (a: Vec2Node, b: Vec2Node, p: Vec2Node, size: Vec2Node) => {
|
|
97
|
+
const side = b.sub(a).mul(size)
|
|
98
|
+
const len = side.length()
|
|
99
|
+
const q = p.sub(a).mul(size)
|
|
100
|
+
return select(len.equal(0), float(1e6), q.y.mul(side.x).sub(q.x.mul(side.y)).div(max(len, 1e-12)))
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
function keyer(map: Texture, u: Uniforms) {
|
|
104
|
+
return Fn(() => {
|
|
105
|
+
// Where this pixel is on the output, 0 to 1 across and down, and output pixels to the picture's size
|
|
106
|
+
const out = vec2(uv().x, float(1).sub(uv().y)).toVar()
|
|
107
|
+
const size = float(1).div(max(fwidth(out), vec2(1e-6))).toVar()
|
|
108
|
+
|
|
109
|
+
// Where it is on the picture, mirrored by flip, and the picture's size in pixels
|
|
110
|
+
const p = vec2(select(u.flags.x.greaterThan(0.5), float(1).sub(out.x), out.x), out.y).toVar()
|
|
111
|
+
// oxlint-disable-next-line typescript/no-unsafe-type-assertion -- @types/three gives textureSize an untyped node
|
|
112
|
+
const pixels = vec2(textureSize(texture(map), int(0)) as unknown as Node<'uvec2'>).toVar()
|
|
113
|
+
const last = pixels.sub(1).toVar()
|
|
114
|
+
|
|
115
|
+
// A pixel, at a column and a row from the top in whole numbers, clamped to the picture's edges and premultiplied in
|
|
116
|
+
// linear light, and the picture at a point between pixels, as OBS's sampler gives it
|
|
117
|
+
const pixelAt = (at: Vec2Node): Vec4Node => {
|
|
118
|
+
const clamped = clamp(at, vec2(0, 0), last).toVar()
|
|
119
|
+
const row = select(u.flags.y.greaterThan(0.5), last.y.sub(clamped.y), clamped.y)
|
|
120
|
+
const texel = textureLoad(map, ivec2(vec2(clamped.x, row)), int(0)).toVar()
|
|
121
|
+
const rgb = select(u.flags.w.greaterThan(0.5), texel.rgb, toLinear(texel.rgb))
|
|
122
|
+
return vec4(rgb.mul(texel.a), texel.a)
|
|
123
|
+
}
|
|
124
|
+
const sampleAt = (q: Vec2Node): Vec4Node => {
|
|
125
|
+
const f = q.mul(pixels).sub(0.5).toVar()
|
|
126
|
+
const corner = floor(f).toVar()
|
|
127
|
+
const w = f.sub(corner).toVar()
|
|
128
|
+
const above = mix(pixelAt(corner), pixelAt(corner.add(vec2(1, 0))), w.x)
|
|
129
|
+
const below = mix(pixelAt(corner.add(vec2(0, 1))), pixelAt(corner.add(vec2(1, 1))), w.x)
|
|
130
|
+
return mix(above, below, w.y)
|
|
131
|
+
}
|
|
132
|
+
const keyDistance = (encoded: Vec3Node) =>
|
|
133
|
+
vec2(dot(encoded, vec3(CB[0], CB[1], CB[2])).add(CB[3]), dot(encoded, vec3(CR[0], CR[1], CR[2])).add(CR[3]))
|
|
134
|
+
.sub(u.key.xy)
|
|
135
|
+
.length()
|
|
136
|
+
|
|
137
|
+
const sampled = sampleAt(p).toVar()
|
|
138
|
+
const alpha = sampled.a.toVar()
|
|
139
|
+
const rgb = select(alpha.greaterThan(0), sampled.rgb.div(alpha), vec3(0)).toVar()
|
|
140
|
+
|
|
141
|
+
// OBS's filter: the key's distance, and from it the alpha and how much of the colour spill keeps
|
|
142
|
+
const keyed = u.key.w.greaterThan(0.5)
|
|
143
|
+
const kept = float(1).toVar()
|
|
144
|
+
If(keyed, () => {
|
|
145
|
+
const texel = vec2(1, 1).div(pixels).toVar()
|
|
146
|
+
const across = vec2(texel.x, texel.y.mul(0.5)).toVar()
|
|
147
|
+
const up = vec2(texel.x.mul(0.5), texel.y.negate()).toVar()
|
|
148
|
+
const around = keyDistance(toEncoded(sampleAt(p.sub(across)).rgb))
|
|
149
|
+
.add(keyDistance(toEncoded(sampleAt(p.add(across)).rgb)))
|
|
150
|
+
.add(keyDistance(toEncoded(sampleAt(p.sub(up)).rgb)))
|
|
151
|
+
.add(keyDistance(toEncoded(sampleAt(p.add(up)).rgb)))
|
|
152
|
+
const beyond = around.mul(2).add(keyDistance(toEncoded(rgb))).div(9).sub(u.matte.x).toVar()
|
|
153
|
+
alpha.mulAssign(u.matte.w)
|
|
154
|
+
alpha.mulAssign(pow(clamp(beyond.div(u.matte.y), 0, 1), 1.5))
|
|
155
|
+
kept.assign(pow(clamp(beyond.div(u.matte.z), 0, 1), 1.5))
|
|
156
|
+
})
|
|
157
|
+
|
|
158
|
+
// The add-ons tint and saturation. Spill desaturates to the tinted colour's luma, as it was before saturation
|
|
159
|
+
rgb.mulAssign(u.extras.xyz)
|
|
160
|
+
const grey = luma(rgb).toVar()
|
|
161
|
+
If(u.extras.w.notEqual(1), () => {
|
|
162
|
+
rgb.assign(saturated(rgb, u.extras.w))
|
|
163
|
+
})
|
|
164
|
+
|
|
165
|
+
// The rest of OBS's filter: spill, then gamma, contrast and brightness
|
|
166
|
+
If(keyed, () => {
|
|
167
|
+
If(u.switches.y.greaterThan(0.5), () => {
|
|
168
|
+
// The add-on, hueSpill
|
|
169
|
+
const encoded = toEncoded(rgb).toVar()
|
|
170
|
+
const toward = u.key.xy.sub(vec2(CB[3], CR[3])).mul(u.key.z).toVar()
|
|
171
|
+
const c = chromaOf(encoded).toVar()
|
|
172
|
+
c.subAssign(toward.mul(max(dot(c, toward), 0)).mul(float(1).sub(kept)))
|
|
173
|
+
rgb.assign(toLinear(clamp(fromLumaChroma(luma(encoded), c), 0, 1)))
|
|
174
|
+
}).Else(() => {
|
|
175
|
+
rgb.assign(mix(vec3(grey), rgb, kept))
|
|
176
|
+
})
|
|
177
|
+
rgb.assign(pow(max(rgb, vec3(0)), vec3(u.correction.x)).mul(u.correction.y).add(u.correction.z))
|
|
178
|
+
})
|
|
179
|
+
rgb.assign(clamp(rgb, 0, 1))
|
|
180
|
+
|
|
181
|
+
// The add-on crop
|
|
182
|
+
If(u.switches.x.greaterThan(0.5), () => {
|
|
183
|
+
const c0 = u.corners01.xy
|
|
184
|
+
const c1 = u.corners01.zw
|
|
185
|
+
const c2 = u.corners23.xy
|
|
186
|
+
const c3 = u.corners23.zw
|
|
187
|
+
const d = min(
|
|
188
|
+
min(cutSide(c0, c1, p, size), cutSide(c1, c2, p, size)),
|
|
189
|
+
min(cutSide(c2, c3, p, size), cutSide(c3, c0, p, size)),
|
|
190
|
+
)
|
|
191
|
+
alpha.mulAssign(clamp(d.add(0.5), 0, 1))
|
|
192
|
+
})
|
|
193
|
+
|
|
194
|
+
// Straight, and linear: the material premultiplies it, and the renderer encodes it
|
|
195
|
+
return vec4(rgb, alpha)
|
|
196
|
+
})()
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
export class GreenscreenNodeMaterial extends NodeMaterial {
|
|
200
|
+
static get type(): string {
|
|
201
|
+
return 'GreenscreenNodeMaterial'
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
readonly isGreenscreenNodeMaterial = true
|
|
205
|
+
#settings: Readonly<Settings> = DEFAULT_SETTINGS
|
|
206
|
+
readonly #packed = new Float32Array(PACKED_LENGTH)
|
|
207
|
+
readonly #uniforms: Uniforms = {
|
|
208
|
+
key: uniform(new Vector4()),
|
|
209
|
+
matte: uniform(new Vector4()),
|
|
210
|
+
correction: uniform(new Vector4()),
|
|
211
|
+
extras: uniform(new Vector4()),
|
|
212
|
+
flags: uniform(new Vector4()),
|
|
213
|
+
corners01: uniform(new Vector4()),
|
|
214
|
+
corners23: uniform(new Vector4()),
|
|
215
|
+
switches: uniform(new Vector4()),
|
|
216
|
+
}
|
|
217
|
+
#map: Texture | null = null
|
|
218
|
+
|
|
219
|
+
constructor({ map = null, settings = {} }: GreenscreenNodeMaterialParameters = {}) {
|
|
220
|
+
super()
|
|
221
|
+
this.name = 'GreenscreenNodeMaterial'
|
|
222
|
+
this.transparent = true
|
|
223
|
+
this.premultipliedAlpha = true
|
|
224
|
+
this.depthWrite = false
|
|
225
|
+
this.lights = false
|
|
226
|
+
this.map = map
|
|
227
|
+
this.set(settings)
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
get map(): Texture | null {
|
|
231
|
+
return this.#map
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
// A new map makes a new shader, as each is built for its texture
|
|
235
|
+
set map(map: Texture | null) {
|
|
236
|
+
if (map === this.#map && this.fragmentNode) return
|
|
237
|
+
this.#map = map
|
|
238
|
+
this.fragmentNode = map ? keyer(map, this.#uniforms) : vec4(0)
|
|
239
|
+
this.needsUpdate = true
|
|
240
|
+
this.#write()
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
get settings(): Readonly<Settings> {
|
|
244
|
+
return this.#settings
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
set(patch: Readonly<Partial<Settings>>): void {
|
|
248
|
+
this.#settings = mergeSettings(this.#settings, patch)
|
|
249
|
+
this.#write()
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
reset(): void {
|
|
253
|
+
this.set(DEFAULT_SETTINGS)
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
// The packed settings into the uniforms, with the flags the material sets itself: rows bottom up in uv, linear out,
|
|
257
|
+
// and linear in from a texture in sRGB, which the GPU decodes as it's read
|
|
258
|
+
#write() {
|
|
259
|
+
const packed = pack(this.#settings, this.#packed)
|
|
260
|
+
const map = this.#map
|
|
261
|
+
packed[17] = 1
|
|
262
|
+
packed[18] = 1
|
|
263
|
+
packed[19] = map && ColorManagement.getTransfer(map.colorSpace) === SRGBTransfer ? 1 : 0
|
|
264
|
+
const u = this.#uniforms
|
|
265
|
+
NAMES.forEach((name, i) => u[name].value.fromArray(packed, i * 4))
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
// A clone keys the same map with the same settings, in uniforms of its own. NodeMaterial's copy would hand it the
|
|
269
|
+
// source's shader, which reads the source's uniforms
|
|
270
|
+
copy(source: this): this {
|
|
271
|
+
super.copy(source)
|
|
272
|
+
if (#settings in source) this.#settings = source.#settings
|
|
273
|
+
this.#map = source.map ?? null
|
|
274
|
+
this.fragmentNode = this.#map ? keyer(this.#map, this.#uniforms) : vec4(0)
|
|
275
|
+
this.needsUpdate = true
|
|
276
|
+
this.#write()
|
|
277
|
+
return this
|
|
278
|
+
}
|
|
279
|
+
}
|
package/src/three.ts
ADDED
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
import { ColorManagement, GLSL3, RawShaderMaterial, SRGBTransfer } from 'three'
|
|
2
|
+
import type { IUniform, Texture, WebGLRenderer } from 'three'
|
|
3
|
+
import { pack, PACKED_LENGTH } from './pack.ts'
|
|
4
|
+
import { DEFAULT_SETTINGS, mergeSettings } from './settings.ts'
|
|
5
|
+
import type { Settings } from './settings.ts'
|
|
6
|
+
import { FRAGMENT } from './shader.glsl.ts'
|
|
7
|
+
|
|
8
|
+
// The keyer as a three.js material, for WebGLRenderer: the core's own GLSL, drawn on whatever mesh it's given. It keys
|
|
9
|
+
// the texture in map as OBS's filter would key it drawn at the mesh's size on screen, and shows it as MeshBasicMaterial
|
|
10
|
+
// would show that map on the mesh, keyed, with the add-ons' colour, mirror and cut. It doesn't apply the
|
|
11
|
+
// texture's offset, repeat or rotation. It reads the map's own pixels and blends them itself, as OBS's sampler does, so
|
|
12
|
+
// it uses no mipmaps. The cut's corners, and OBS's samples round each pixel, are on the picture as it shows on the
|
|
13
|
+
// mesh, with y down
|
|
14
|
+
|
|
15
|
+
export interface GreenscreenMaterialParameters {
|
|
16
|
+
// The picture to key: a VideoTexture, a Texture of an image or canvas, or any other 8-bit texture. It may be in
|
|
17
|
+
// sRGB or have no colour space; either way it's keyed as OBS keys a picture of those sRGB values. It must not be
|
|
18
|
+
// premultiplied
|
|
19
|
+
map?: Texture | null
|
|
20
|
+
settings?: Readonly<Partial<Settings>>
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
const VERTEX = /* glsl */ `precision highp float;
|
|
24
|
+
in vec3 position;
|
|
25
|
+
in vec2 uv;
|
|
26
|
+
uniform mat4 modelViewMatrix;
|
|
27
|
+
uniform mat4 projectionMatrix;
|
|
28
|
+
out vec2 vUv;
|
|
29
|
+
void main() {
|
|
30
|
+
vUv = vec2(uv.x, 1.0 - uv.y);
|
|
31
|
+
gl_Position = projectionMatrix * modelViewMatrix * vec4(position, 1.0);
|
|
32
|
+
}
|
|
33
|
+
`
|
|
34
|
+
|
|
35
|
+
interface Uniforms {
|
|
36
|
+
[uniform: string]: IUniform
|
|
37
|
+
image: IUniform<Texture | null>
|
|
38
|
+
settings: IUniform<Float32Array>
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export class GreenscreenMaterial extends RawShaderMaterial {
|
|
42
|
+
readonly isGreenscreenMaterial = true
|
|
43
|
+
declare uniforms: Uniforms
|
|
44
|
+
#settings: Readonly<Settings> = DEFAULT_SETTINGS
|
|
45
|
+
|
|
46
|
+
constructor({ map = null, settings = {} }: GreenscreenMaterialParameters = {}) {
|
|
47
|
+
super({
|
|
48
|
+
name: 'GreenscreenMaterial',
|
|
49
|
+
glslVersion: GLSL3,
|
|
50
|
+
vertexShader: VERTEX,
|
|
51
|
+
fragmentShader: FRAGMENT,
|
|
52
|
+
uniforms: { image: { value: map }, settings: { value: new Float32Array(PACKED_LENGTH) } },
|
|
53
|
+
// Its output is premultiplied, and blends as that is
|
|
54
|
+
transparent: true,
|
|
55
|
+
premultipliedAlpha: true,
|
|
56
|
+
depthWrite: false,
|
|
57
|
+
})
|
|
58
|
+
this.set(settings)
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
get map(): Texture | null {
|
|
62
|
+
return this.uniforms.image.value
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
set map(texture: Texture | null) {
|
|
66
|
+
this.uniforms.image.value = texture
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
// The settings it keys with, as a frozen object
|
|
70
|
+
get settings(): Readonly<Settings> {
|
|
71
|
+
return this.#settings
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
// Changes the settings named, leaving the rest. The values are copied
|
|
75
|
+
set(patch: Readonly<Partial<Settings>>): void {
|
|
76
|
+
this.#settings = mergeSettings(this.#settings, patch)
|
|
77
|
+
pack(this.#settings, this.uniforms.settings.value)
|
|
78
|
+
this.uniformsNeedUpdate = true
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
reset(): void {
|
|
82
|
+
this.set(DEFAULT_SETTINGS)
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
// What the shader can't know from the settings: the texture's rows run bottom up in uv, as MeshBasicMaterial
|
|
86
|
+
// samples them; whether the texture gives linear values, as an 8-bit sRGB texture other than a video's does in
|
|
87
|
+
// three.js; and whether what it draws into wants linear values, as anything but the screen does
|
|
88
|
+
onBeforeRender(renderer: WebGLRenderer): void {
|
|
89
|
+
const packed = this.uniforms.settings.value
|
|
90
|
+
const target = renderer.getRenderTarget()
|
|
91
|
+
const space =
|
|
92
|
+
target === null
|
|
93
|
+
? renderer.outputColorSpace
|
|
94
|
+
: 'isXRRenderTarget' in target && target.isXRRenderTarget === true
|
|
95
|
+
? target.texture.colorSpace
|
|
96
|
+
: ColorManagement.workingColorSpace
|
|
97
|
+
const map = this.map
|
|
98
|
+
packed[17] = 1
|
|
99
|
+
packed[18] = ColorManagement.getTransfer(space) === SRGBTransfer ? 0 : 1
|
|
100
|
+
packed[19] =
|
|
101
|
+
map && !('isVideoTexture' in map && map.isVideoTexture === true) && ColorManagement.getTransfer(map.colorSpace) === SRGBTransfer
|
|
102
|
+
? 1
|
|
103
|
+
: 0
|
|
104
|
+
this.uniformsNeedUpdate = true
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
// A clone keys with the same settings, in a settings array of its own
|
|
108
|
+
copy(source: this): this {
|
|
109
|
+
super.copy(source)
|
|
110
|
+
this.uniforms.settings.value = Float32Array.from(source.uniforms.settings.value)
|
|
111
|
+
if (#settings in source) this.#settings = source.#settings
|
|
112
|
+
return this
|
|
113
|
+
}
|
|
114
|
+
}
|
package/src/types.ts
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
// The public types the keyer's API is written in, kept apart from the modules that use them so that the declarations a
|
|
2
|
+
// project reads name nothing newer than they must
|
|
3
|
+
|
|
4
|
+
// Anything the browser can upload as a texture: a video, an image, a canvas, an ImageBitmap, ImageData or a VideoFrame
|
|
5
|
+
export type Source = TexImageSource
|
|
6
|
+
|
|
7
|
+
// The graphics API a keyer draws with
|
|
8
|
+
export type BackendName = 'webgl' | 'webgpu'
|
|
9
|
+
|
|
10
|
+
export type Canvas = HTMLCanvasElement | OffscreenCanvas
|
package/src/webgl.ts
ADDED
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
import { contextOf, tooLarge } from './renderer.ts'
|
|
2
|
+
import type { Renderer, RendererEvents } from './renderer.ts'
|
|
3
|
+
import { FRAGMENT, VERTEX } from './shader.glsl.ts'
|
|
4
|
+
import type { Canvas, Source } from './types.ts'
|
|
5
|
+
|
|
6
|
+
// Drawn with blending premultiplied, as the shader's output is, and kept between draws, so the canvas can be copied
|
|
7
|
+
// from at any time and not just in the task that drew it
|
|
8
|
+
const ATTRIBUTES: WebGLContextAttributes = {
|
|
9
|
+
alpha: true,
|
|
10
|
+
premultipliedAlpha: true,
|
|
11
|
+
antialias: false,
|
|
12
|
+
depth: false,
|
|
13
|
+
stencil: false,
|
|
14
|
+
preserveDrawingBuffer: true,
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export class WebGLRenderer implements Renderer {
|
|
18
|
+
readonly backend = 'webgl'
|
|
19
|
+
readonly #canvas: Canvas
|
|
20
|
+
readonly #gl: WebGL2RenderingContext
|
|
21
|
+
readonly #events: RendererEvents
|
|
22
|
+
#program: WebGLProgram | null = null
|
|
23
|
+
#settings: WebGLUniformLocation | null = null
|
|
24
|
+
#texture: WebGLTexture | null = null
|
|
25
|
+
#width = 0
|
|
26
|
+
#height = 0
|
|
27
|
+
// The most pixels a side it can take and draw
|
|
28
|
+
#limit = 0
|
|
29
|
+
#lost = false
|
|
30
|
+
|
|
31
|
+
constructor(canvas: Canvas, events: RendererEvents) {
|
|
32
|
+
const gl = contextOf(canvas, 'webgl2', ATTRIBUTES)
|
|
33
|
+
if (!gl) throw new Error('Greenscreen: WebGL2 is not available on this canvas')
|
|
34
|
+
this.#canvas = canvas
|
|
35
|
+
this.#gl = gl
|
|
36
|
+
this.#events = events
|
|
37
|
+
canvas.addEventListener('webglcontextlost', this.#onLost)
|
|
38
|
+
canvas.addEventListener('webglcontextrestored', this.#onRestored)
|
|
39
|
+
this.#build()
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
get lost() {
|
|
43
|
+
return this.#lost
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
// The program and the texture, as at first and again once a lost context is back. The picture is uploaded as it
|
|
47
|
+
// comes: top row first, not premultiplied, in sRGB
|
|
48
|
+
#build() {
|
|
49
|
+
const gl = this.#gl
|
|
50
|
+
const program = gl.createProgram()
|
|
51
|
+
const vertex = compile(gl, gl.VERTEX_SHADER, VERTEX)
|
|
52
|
+
const fragment = compile(gl, gl.FRAGMENT_SHADER, `#version 300 es\n${FRAGMENT}`)
|
|
53
|
+
gl.attachShader(program, vertex)
|
|
54
|
+
gl.attachShader(program, fragment)
|
|
55
|
+
gl.linkProgram(program)
|
|
56
|
+
if (!gl.getProgramParameter(program, gl.LINK_STATUS) && !gl.isContextLost()) {
|
|
57
|
+
throw new Error(`Greenscreen: the shader didn't link: ${gl.getProgramInfoLog(program)}`)
|
|
58
|
+
}
|
|
59
|
+
gl.deleteShader(vertex)
|
|
60
|
+
gl.deleteShader(fragment)
|
|
61
|
+
gl.useProgram(program)
|
|
62
|
+
gl.uniform1i(gl.getUniformLocation(program, 'image'), 0)
|
|
63
|
+
this.#program = program
|
|
64
|
+
this.#settings = gl.getUniformLocation(program, 'settings')
|
|
65
|
+
|
|
66
|
+
const texture = gl.createTexture()
|
|
67
|
+
gl.activeTexture(gl.TEXTURE0)
|
|
68
|
+
gl.bindTexture(gl.TEXTURE_2D, texture)
|
|
69
|
+
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR)
|
|
70
|
+
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR)
|
|
71
|
+
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE)
|
|
72
|
+
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE)
|
|
73
|
+
gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, false)
|
|
74
|
+
gl.pixelStorei(gl.UNPACK_PREMULTIPLY_ALPHA_WEBGL, false)
|
|
75
|
+
gl.pixelStorei(gl.UNPACK_COLORSPACE_CONVERSION_WEBGL, gl.BROWSER_DEFAULT_WEBGL)
|
|
76
|
+
this.#texture = texture
|
|
77
|
+
this.#width = 0
|
|
78
|
+
this.#height = 0
|
|
79
|
+
// The widest picture it can take and draw: its largest texture, and its largest viewport, as the canvas is sized to
|
|
80
|
+
// the picture. A lost context says neither, and draws nothing until it's back
|
|
81
|
+
const texture2D: unknown = gl.getParameter(gl.MAX_TEXTURE_SIZE)
|
|
82
|
+
const viewport: unknown = gl.getParameter(gl.MAX_VIEWPORT_DIMS)
|
|
83
|
+
const sides = [texture2D, ...(viewport instanceof Int32Array ? viewport : [])]
|
|
84
|
+
this.#limit = Math.min(...sides.filter((side) => typeof side === 'number'))
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
// A picture larger than the context's textures throws, as it would otherwise be drawn black
|
|
88
|
+
upload(source: Source, width: number, height: number) {
|
|
89
|
+
if (this.#lost) return
|
|
90
|
+
const limit = this.#limit
|
|
91
|
+
if (width > limit || height > limit) throw tooLarge(width, height, 'WebGL2', limit)
|
|
92
|
+
const gl = this.#gl
|
|
93
|
+
gl.activeTexture(gl.TEXTURE0)
|
|
94
|
+
gl.bindTexture(gl.TEXTURE_2D, this.#texture)
|
|
95
|
+
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA8, gl.RGBA, gl.UNSIGNED_BYTE, this.#straight(source, width, height))
|
|
96
|
+
this.#width = width
|
|
97
|
+
this.#height = height
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
// WebGL uploads an ImageBitmap as it was made, ignoring the unpacking flags, so one made premultiplied, as they are
|
|
101
|
+
// by default, would arrive premultiplied. Drawn into a canvas first, it arrives straight, as everything else does
|
|
102
|
+
#bitmaps: OffscreenCanvasRenderingContext2D | null = null
|
|
103
|
+
|
|
104
|
+
#straight(source: Source, width: number, height: number): Source {
|
|
105
|
+
if (typeof ImageBitmap === 'undefined' || !(source instanceof ImageBitmap) || typeof OffscreenCanvas === 'undefined') {
|
|
106
|
+
return source
|
|
107
|
+
}
|
|
108
|
+
let context = this.#bitmaps
|
|
109
|
+
if (!context) {
|
|
110
|
+
context = new OffscreenCanvas(width, height).getContext('2d')
|
|
111
|
+
if (!context) return source
|
|
112
|
+
this.#bitmaps = context
|
|
113
|
+
}
|
|
114
|
+
const canvas = context.canvas
|
|
115
|
+
if (canvas.width !== width || canvas.height !== height) {
|
|
116
|
+
canvas.width = width
|
|
117
|
+
canvas.height = height
|
|
118
|
+
} else {
|
|
119
|
+
context.clearRect(0, 0, width, height)
|
|
120
|
+
}
|
|
121
|
+
context.drawImage(source, 0, 0)
|
|
122
|
+
return canvas
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
// Every pixel is drawn over, unblended, so there's nothing to clear first
|
|
126
|
+
draw(settings: Float32Array<ArrayBuffer>) {
|
|
127
|
+
if (this.#lost || !this.#width || !this.#height) return
|
|
128
|
+
const gl = this.#gl
|
|
129
|
+
const canvas = this.#canvas
|
|
130
|
+
if (canvas.width !== this.#width || canvas.height !== this.#height) {
|
|
131
|
+
canvas.width = this.#width
|
|
132
|
+
canvas.height = this.#height
|
|
133
|
+
}
|
|
134
|
+
gl.viewport(0, 0, this.#width, this.#height)
|
|
135
|
+
gl.useProgram(this.#program)
|
|
136
|
+
gl.uniform4fv(this.#settings, settings)
|
|
137
|
+
gl.activeTexture(gl.TEXTURE0)
|
|
138
|
+
gl.bindTexture(gl.TEXTURE_2D, this.#texture)
|
|
139
|
+
gl.drawArrays(gl.TRIANGLES, 0, 3)
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
// The drawing buffer is kept between draws, so it stays clear until the next
|
|
143
|
+
clear() {
|
|
144
|
+
if (this.#lost) return
|
|
145
|
+
const gl = this.#gl
|
|
146
|
+
gl.clearColor(0, 0, 0, 0)
|
|
147
|
+
gl.clear(gl.COLOR_BUFFER_BIT)
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
// Lost, its default is prevented, which lets the browser give it back
|
|
151
|
+
#onLost = (event: Event) => {
|
|
152
|
+
event.preventDefault()
|
|
153
|
+
this.#lost = true
|
|
154
|
+
this.#program = null
|
|
155
|
+
this.#texture = null
|
|
156
|
+
this.#width = 0
|
|
157
|
+
this.#height = 0
|
|
158
|
+
this.#events.lost()
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
#onRestored = () => {
|
|
162
|
+
this.#lost = false
|
|
163
|
+
this.#build()
|
|
164
|
+
this.#events.restored()
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
// Loses the context on purpose, as that frees it at once rather than when it's collected, and a page can only have
|
|
168
|
+
// so many
|
|
169
|
+
dispose() {
|
|
170
|
+
const gl = this.#gl
|
|
171
|
+
this.#canvas.removeEventListener('webglcontextlost', this.#onLost)
|
|
172
|
+
this.#canvas.removeEventListener('webglcontextrestored', this.#onRestored)
|
|
173
|
+
if (!this.#lost) {
|
|
174
|
+
gl.deleteTexture(this.#texture)
|
|
175
|
+
gl.deleteProgram(this.#program)
|
|
176
|
+
}
|
|
177
|
+
this.#lost = true
|
|
178
|
+
gl.getExtension('WEBGL_lose_context')?.loseContext()
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
function compile(gl: WebGL2RenderingContext, type: number, source: string) {
|
|
183
|
+
const shader = gl.createShader(type)
|
|
184
|
+
if (!shader) throw new Error('Greenscreen: no shader could be made')
|
|
185
|
+
gl.shaderSource(shader, source)
|
|
186
|
+
gl.compileShader(shader)
|
|
187
|
+
if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS) && !gl.isContextLost()) {
|
|
188
|
+
throw new Error(`Greenscreen: the shader didn't compile: ${gl.getShaderInfoLog(shader)}`)
|
|
189
|
+
}
|
|
190
|
+
return shader
|
|
191
|
+
}
|