@snovakow/greenscreen 0.0.0-stage → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +239 -2
- package/dist/greenscreen.d.ts +32 -0
- package/dist/greenscreen.d.ts.map +1 -0
- package/dist/greenscreen.global.js +350 -0
- package/dist/greenscreen.global.js.map +1 -0
- package/dist/greenscreen.js +250 -0
- package/dist/greenscreen.js.map +1 -0
- package/dist/greenscreen.min.js +350 -0
- package/dist/greenscreen.min.js.map +1 -0
- package/dist/index.d.ts +6 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +3 -0
- package/dist/index.js.map +1 -0
- package/dist/pack.d.ts +7 -0
- package/dist/pack.d.ts.map +1 -0
- package/dist/pack.js +63 -0
- package/dist/pack.js.map +1 -0
- package/dist/react.d.ts +26 -0
- package/dist/react.d.ts.map +1 -0
- package/dist/react.js +78 -0
- package/dist/react.js.map +1 -0
- package/dist/renderer.d.ts +19 -0
- package/dist/renderer.d.ts.map +1 -0
- package/dist/renderer.js +25 -0
- package/dist/renderer.js.map +1 -0
- package/dist/settings.d.ts +36 -0
- package/dist/settings.d.ts.map +1 -0
- package/dist/settings.js +112 -0
- package/dist/settings.js.map +1 -0
- package/dist/shader.glsl.d.ts +3 -0
- package/dist/shader.glsl.d.ts.map +1 -0
- package/dist/shader.glsl.js +199 -0
- package/dist/shader.glsl.js.map +1 -0
- package/dist/shader.wgsl.d.ts +2 -0
- package/dist/shader.wgsl.d.ts.map +1 -0
- package/dist/shader.wgsl.js +177 -0
- package/dist/shader.wgsl.js.map +1 -0
- package/dist/source.d.ts +16 -0
- package/dist/source.d.ts.map +1 -0
- package/dist/source.js +103 -0
- package/dist/source.js.map +1 -0
- package/dist/three-webgpu.d.ts +20 -0
- package/dist/three-webgpu.d.ts.map +1 -0
- package/dist/three-webgpu.js +193 -0
- package/dist/three-webgpu.js.map +1 -0
- package/dist/three.d.ts +27 -0
- package/dist/three.d.ts.map +1 -0
- package/dist/three.js +81 -0
- package/dist/three.js.map +1 -0
- package/dist/types.d.ts +4 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +4 -0
- package/dist/types.js.map +1 -0
- package/dist/webgl.d.ts +13 -0
- package/dist/webgl.d.ts.map +1 -0
- package/dist/webgl.js +183 -0
- package/dist/webgl.js.map +1 -0
- package/dist/webgpu.d.ts +19 -0
- package/dist/webgpu.d.ts.map +1 -0
- package/dist/webgpu.js +303 -0
- package/dist/webgpu.js.map +1 -0
- package/package.json +103 -4
- package/src/global.ts +11 -0
- package/src/greenscreen.ts +286 -0
- package/src/index.ts +5 -0
- package/src/pack.ts +73 -0
- package/src/react.tsx +126 -0
- package/src/renderer.ts +53 -0
- package/src/settings.ts +179 -0
- package/src/shader.glsl.ts +199 -0
- package/src/shader.wgsl.ts +176 -0
- package/src/source.ts +105 -0
- package/src/three-webgpu.ts +279 -0
- package/src/three.ts +114 -0
- package/src/types.ts +10 -0
- package/src/webgl.ts +191 -0
- package/src/webgpu.ts +312 -0
package/src/settings.ts
ADDED
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
// What the keyer does to a picture, and how that's packed into the numbers each of its shaders reads
|
|
2
|
+
|
|
3
|
+
// A point on the picture: 0 to 1 across it from the left, and 0 to 1 down it from the top
|
|
4
|
+
export interface Point {
|
|
5
|
+
x: number
|
|
6
|
+
y: number
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
// The keyer keys as OBS Studio's chroma key filter does, the filter's second version, and takes its settings in OBS's
|
|
10
|
+
// units, ranges and defaults, so a filter's numbers can be copied from OBS as they are. The add-ons are the library's
|
|
11
|
+
// own, and not in OBS. At their defaults they change nothing, and the picture is keyed exactly as OBS keys it
|
|
12
|
+
// https://github.com/obsproject/obs-studio/blob/master/plugins/obs-filters/chroma-key-filter.c
|
|
13
|
+
export interface Settings {
|
|
14
|
+
// OBS's chroma key filter
|
|
15
|
+
|
|
16
|
+
// Whether the filter runs, as its eye in OBS's list of filters. Off, the picture passes through it as it came, and
|
|
17
|
+
// only the add-ons apply
|
|
18
|
+
key: boolean
|
|
19
|
+
// OBS's Key Color, as 0xrrggbb. KEY_COLORS has OBS's Green, Blue and Magenta. Any other colour is OBS's Custom Color
|
|
20
|
+
keyColor: number
|
|
21
|
+
// OBS's Similarity, 1 to 1000: how near the key a colour's chroma must be to be taken out
|
|
22
|
+
similarity: number
|
|
23
|
+
// OBS's Smoothness, 1 to 1000: how far past similarity a colour fades back in
|
|
24
|
+
smoothness: number
|
|
25
|
+
// OBS's Key Color Spill Reduction, 1 to 1000: how far past similarity a colour kept is desaturated, which takes the
|
|
26
|
+
// key's cast off edges and hair
|
|
27
|
+
spill: number
|
|
28
|
+
// OBS's Opacity, 0 to 1
|
|
29
|
+
opacity: number
|
|
30
|
+
// OBS's Contrast, −4 to 4: a factor from 1/5 to 5, 0 for no change
|
|
31
|
+
contrast: number
|
|
32
|
+
// OBS's Brightness, −1 to 1, added: 0 for no change
|
|
33
|
+
brightness: number
|
|
34
|
+
// OBS's Gamma, −1 to 1: an exponent from 2 to 1/2, 0 for no change
|
|
35
|
+
gamma: number
|
|
36
|
+
|
|
37
|
+
// Add-ons: the library's own, not OBS's. Flip and crop act on the picture OBS's filter gives. Tint, saturation and
|
|
38
|
+
// hueSpill act inside the filter, between its steps or in place of one
|
|
39
|
+
|
|
40
|
+
// Spill takes out only the key's hue, keeping each colour's luma and its other hues, where OBS's spill desaturates
|
|
41
|
+
// the whole colour. It reaches as far as spill does, and takes as much of the key's hue out as OBS would of the colour
|
|
42
|
+
hueSpill: boolean
|
|
43
|
+
// Mirrors the picture left to right
|
|
44
|
+
flip: boolean
|
|
45
|
+
// Cuts the picture down to the shape its corners make
|
|
46
|
+
crop: boolean
|
|
47
|
+
// The cut's four corners, clockwise from the top left, in the picture's own coordinates: not mirrored by flip. The
|
|
48
|
+
// shape they make must be convex. They're kept while crop is off
|
|
49
|
+
corners: readonly Readonly<Point>[]
|
|
50
|
+
// Multiplies the picture's colour, as 0xrrggbb: white for no change. It acts once the key's distance is measured, so
|
|
51
|
+
// it doesn't change what's keyed, and before spill and gamma, contrast and brightness
|
|
52
|
+
tint: number
|
|
53
|
+
// Scales the colour's HSB saturation, keeping its hue and its brightest channel, after tint and before spill: 0 for
|
|
54
|
+
// the grey of that channel, 1 for no change, up to 2. No colour goes past full saturation
|
|
55
|
+
saturation: number
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
export type NumericSetting = 'similarity' | 'smoothness' | 'spill' | 'opacity' | 'contrast' | 'brightness' | 'gamma' | 'saturation'
|
|
59
|
+
|
|
60
|
+
export interface Range {
|
|
61
|
+
readonly min: number
|
|
62
|
+
readonly max: number
|
|
63
|
+
// The step OBS's slider moves in, or the add-on's. Values between steps are kept as they are, except similarity's,
|
|
64
|
+
// smoothness's and spill's, which OBS keeps as whole numbers, and which are rounded to them
|
|
65
|
+
readonly step: number
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
// Each number's limits, for a control panel to use: OBS's sliders', and the add-on's. Values outside them are clamped
|
|
69
|
+
export const RANGES: Readonly<Record<NumericSetting, Range>> = Object.freeze({
|
|
70
|
+
similarity: { min: 1, max: 1000, step: 1 },
|
|
71
|
+
smoothness: { min: 1, max: 1000, step: 1 },
|
|
72
|
+
spill: { min: 1, max: 1000, step: 1 },
|
|
73
|
+
opacity: { min: 0, max: 1, step: 0.0001 },
|
|
74
|
+
contrast: { min: -4, max: 4, step: 0.01 },
|
|
75
|
+
brightness: { min: -1, max: 1, step: 0.0001 },
|
|
76
|
+
gamma: { min: -1, max: 1, step: 0.01 },
|
|
77
|
+
saturation: { min: 0, max: 2, step: 0.01 },
|
|
78
|
+
})
|
|
79
|
+
|
|
80
|
+
// OBS's Green, Blue and Magenta. OBS writes its blue as 0xFF9900, as it keeps colours as 0xbbggrr
|
|
81
|
+
export const KEY_COLORS: Readonly<{ green: number; blue: number; magenta: number }> = Object.freeze({
|
|
82
|
+
green: 0x00ff00,
|
|
83
|
+
blue: 0x0099ff,
|
|
84
|
+
magenta: 0xff00ff,
|
|
85
|
+
})
|
|
86
|
+
|
|
87
|
+
// The whole picture, which is where the cut starts
|
|
88
|
+
const WHOLE: readonly Readonly<Point>[] = Object.freeze([
|
|
89
|
+
Object.freeze({ x: 0, y: 0 }),
|
|
90
|
+
Object.freeze({ x: 1, y: 0 }),
|
|
91
|
+
Object.freeze({ x: 1, y: 1 }),
|
|
92
|
+
Object.freeze({ x: 0, y: 1 }),
|
|
93
|
+
])
|
|
94
|
+
|
|
95
|
+
// OBS's defaults for its filter, with the add-ons changing nothing
|
|
96
|
+
export const DEFAULT_SETTINGS: Readonly<Settings> = Object.freeze({
|
|
97
|
+
key: true,
|
|
98
|
+
keyColor: KEY_COLORS.green,
|
|
99
|
+
similarity: 400,
|
|
100
|
+
smoothness: 80,
|
|
101
|
+
spill: 100,
|
|
102
|
+
opacity: 1,
|
|
103
|
+
contrast: 0,
|
|
104
|
+
brightness: 0,
|
|
105
|
+
gamma: 0,
|
|
106
|
+
hueSpill: false,
|
|
107
|
+
flip: false,
|
|
108
|
+
crop: false,
|
|
109
|
+
corners: WHOLE,
|
|
110
|
+
tint: 0xffffff,
|
|
111
|
+
saturation: 1,
|
|
112
|
+
})
|
|
113
|
+
|
|
114
|
+
const NUMBERS: readonly NumericSetting[] = [
|
|
115
|
+
'similarity',
|
|
116
|
+
'smoothness',
|
|
117
|
+
'spill',
|
|
118
|
+
'opacity',
|
|
119
|
+
'contrast',
|
|
120
|
+
'brightness',
|
|
121
|
+
'gamma',
|
|
122
|
+
'saturation',
|
|
123
|
+
]
|
|
124
|
+
|
|
125
|
+
// The numbers OBS keeps as whole numbers, as its sliders for them move in whole steps and it stores them as integers
|
|
126
|
+
const WHOLE_NUMBERS: ReadonlySet<NumericSetting> = new Set(['similarity', 'smoothness', 'spill'])
|
|
127
|
+
|
|
128
|
+
// The settings with the patch's changes made, as a new frozen object. A value that's undefined leaves the setting as
|
|
129
|
+
// it was, and a name that isn't a setting is passed over, so an app's own object of settings can be given as it is.
|
|
130
|
+
// A value of the wrong kind throws, so a mistake shows where it's made
|
|
131
|
+
export function mergeSettings(base: Readonly<Settings>, patch: Readonly<Partial<Settings>>): Readonly<Settings> {
|
|
132
|
+
const next: Settings = { ...base }
|
|
133
|
+
if (patch.key !== undefined) next.key = flag('key', patch.key)
|
|
134
|
+
if (patch.hueSpill !== undefined) next.hueSpill = flag('hueSpill', patch.hueSpill)
|
|
135
|
+
if (patch.flip !== undefined) next.flip = flag('flip', patch.flip)
|
|
136
|
+
if (patch.crop !== undefined) next.crop = flag('crop', patch.crop)
|
|
137
|
+
if (patch.keyColor !== undefined) next.keyColor = colour('keyColor', patch.keyColor)
|
|
138
|
+
if (patch.tint !== undefined) next.tint = colour('tint', patch.tint)
|
|
139
|
+
if (patch.corners !== undefined) next.corners = corners(patch.corners)
|
|
140
|
+
for (const name of NUMBERS) {
|
|
141
|
+
const value = patch[name]
|
|
142
|
+
if (value !== undefined) next[name] = number(name, value)
|
|
143
|
+
}
|
|
144
|
+
return Object.freeze(next)
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
const clamp = (x: number, low: number, high: number) => Math.min(Math.max(x, low), high)
|
|
148
|
+
|
|
149
|
+
function flag(name: string, value: unknown): boolean {
|
|
150
|
+
if (typeof value !== 'boolean') throw new TypeError(`Greenscreen: ${name} must be true or false`)
|
|
151
|
+
return value
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
function colour(name: string, value: unknown): number {
|
|
155
|
+
if (typeof value !== 'number' || !Number.isInteger(value) || value < 0 || value > 0xffffff) {
|
|
156
|
+
throw new TypeError(`Greenscreen: ${name} must be a colour from 0x000000 to 0xffffff`)
|
|
157
|
+
}
|
|
158
|
+
return value
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
function number(name: NumericSetting, value: unknown): number {
|
|
162
|
+
if (typeof value !== 'number' || !Number.isFinite(value)) throw new TypeError(`Greenscreen: ${name} must be a number`)
|
|
163
|
+
const clamped = clamp(value, RANGES[name].min, RANGES[name].max)
|
|
164
|
+
return WHOLE_NUMBERS.has(name) ? Math.round(clamped) : clamped
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
function corners(value: unknown): readonly Readonly<Point>[] {
|
|
168
|
+
if (!Array.isArray(value) || value.length !== 4) throw new TypeError('Greenscreen: corners must be four points')
|
|
169
|
+
return Object.freeze(
|
|
170
|
+
value.map((point: unknown) => {
|
|
171
|
+
const x: unknown = typeof point === 'object' && point !== null && 'x' in point ? point.x : undefined
|
|
172
|
+
const y: unknown = typeof point === 'object' && point !== null && 'y' in point ? point.y : undefined
|
|
173
|
+
if (typeof x !== 'number' || typeof y !== 'number' || !Number.isFinite(x) || !Number.isFinite(y)) {
|
|
174
|
+
throw new TypeError('Greenscreen: each corner must have a number x and y')
|
|
175
|
+
}
|
|
176
|
+
return Object.freeze({ x: clamp(x, 0, 1), y: clamp(y, 0, 1) })
|
|
177
|
+
}),
|
|
178
|
+
)
|
|
179
|
+
}
|
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
// The keyer in GLSL ES 3.00. The fragment shader is shared by the WebGL2 renderer and the three.js material, each
|
|
2
|
+
// giving it vUv, where the pixel is on the output: 0 to 1 across and down. pack.ts says what's in settings.
|
|
3
|
+
//
|
|
4
|
+
// First it keys as OBS Studio's chroma key filter does, the filter's second version, in the same steps and order:
|
|
5
|
+
// https://github.com/obsproject/obs-studio/blob/master/plugins/obs-filters/data/chroma_key_filter_v2.effect
|
|
6
|
+
// - The picture is sampled as OBS holds a source: premultiplied, in linear light, and blended between pixels in
|
|
7
|
+
// linear light, as OBS's sampler blends its texture. The pixel's own colour is then divided by its alpha.
|
|
8
|
+
// - How far a colour's chroma is from the key's is measured on its sRGB values. The measure is blurred: four samples
|
|
9
|
+
// round the pixel, a pixel across and half a pixel down, and half a pixel across and a pixel up, each either way,
|
|
10
|
+
// count twice each, and the pixel itself once, of nine.
|
|
11
|
+
// - Past similarity the alpha fades in over smoothness, and the colour is desaturated over spill, each on a curve of
|
|
12
|
+
// power 1.5. Opacity multiplies the alpha, and gamma, contrast and brightness the colour, in linear light. The
|
|
13
|
+
// colour is clamped, as OBS's target of 8 bits a channel holds it.
|
|
14
|
+
// The add-ons aren't OBS's. Tint and saturation act inside the filter, once the key's distance is measured, which they
|
|
15
|
+
// don't change, and before spill and the colour step: the tint multiplies, OBS's spill takes the tinted colour's luma
|
|
16
|
+
// to desaturate to, and saturation scales the colour's HSB saturation. They act with the key off too. hueSpill, where
|
|
17
|
+
// it's on, takes the key's hue out of the colour in place of OBS's desaturation, by as much, keeping the colour's luma
|
|
18
|
+
// and other hues, on its sRGB values. Crop cuts, and flip mirrors the result. The output is the colour multiplied by
|
|
19
|
+
// its alpha
|
|
20
|
+
export const FRAGMENT: string = /* glsl */ `precision highp float;
|
|
21
|
+
precision highp int;
|
|
22
|
+
precision highp sampler2D;
|
|
23
|
+
|
|
24
|
+
uniform sampler2D image;
|
|
25
|
+
uniform vec4 settings[8];
|
|
26
|
+
|
|
27
|
+
in vec2 vUv;
|
|
28
|
+
out vec4 fragColor;
|
|
29
|
+
|
|
30
|
+
// A colour's Cb and Cr from its sRGB values, as OBS's filter has them: BT.709's, in studio range
|
|
31
|
+
const vec3 CB = vec3(-0.100644, -0.338572, 0.439216);
|
|
32
|
+
const vec3 CR = vec3(0.439216, -0.398942, -0.040274);
|
|
33
|
+
const float CHROMA_OFFSET = 0.501961;
|
|
34
|
+
// BT.709's luma, which OBS desaturates to
|
|
35
|
+
const vec3 LUMA = vec3(0.2126, 0.7152, 0.0722);
|
|
36
|
+
// The scales that take B' − Y' and R' − Y' to BT.709's Pb and Pr, for hueSpill
|
|
37
|
+
const vec2 CHROMA_SCALE = vec2(1.8556, 1.5748);
|
|
38
|
+
|
|
39
|
+
float toLinear(float c) {
|
|
40
|
+
return c <= 0.04045 ? c / 12.92 : pow((c + 0.055) / 1.055, 2.4);
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
float toEncoded(float c) {
|
|
44
|
+
return c <= 0.0031308 ? 12.92 * c : 1.055 * pow(c, 1.0 / 2.4) - 0.055;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
vec3 toLinear(vec3 c) {
|
|
48
|
+
return vec3(toLinear(c.r), toLinear(c.g), toLinear(c.b));
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
vec3 toEncoded(vec3 c) {
|
|
52
|
+
return vec3(toEncoded(c.r), toEncoded(c.g), toEncoded(c.b));
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
// The picture's pixel at a column, and a row from the top, clamped to the picture's edges as OBS's sampler clamps.
|
|
56
|
+
// It's premultiplied, in linear light, as OBS holds a source, whichever way the texture holds it
|
|
57
|
+
vec4 pixelAt(ivec2 at, ivec2 size) {
|
|
58
|
+
at = clamp(at, ivec2(0), size - 1);
|
|
59
|
+
if (settings[4].y > 0.5) at.y = size.y - 1 - at.y;
|
|
60
|
+
vec4 texel = texelFetch(image, at, 0);
|
|
61
|
+
vec3 rgb = settings[4].w > 0.5 ? texel.rgb : toLinear(texel.rgb);
|
|
62
|
+
return vec4(rgb * texel.a, texel.a);
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
// The picture at q, 0 to 1 across and down it, as OBS's linear sampler gives it: the four pixels round q, blended
|
|
66
|
+
vec4 sampleAt(vec2 q, ivec2 size) {
|
|
67
|
+
vec2 f = q * vec2(size) - 0.5;
|
|
68
|
+
vec2 corner = floor(f);
|
|
69
|
+
vec2 w = f - corner;
|
|
70
|
+
ivec2 i = ivec2(corner);
|
|
71
|
+
vec4 above = mix(pixelAt(i, size), pixelAt(i + ivec2(1, 0), size), w.x);
|
|
72
|
+
vec4 below = mix(pixelAt(i + ivec2(0, 1), size), pixelAt(i + ivec2(1, 1), size), w.x);
|
|
73
|
+
return mix(above, below, w.y);
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
// How far a colour's chroma is from the key's, from its sRGB values
|
|
77
|
+
float keyDistance(vec3 encoded) {
|
|
78
|
+
return distance(vec2(dot(encoded, CB), dot(encoded, CR)) + CHROMA_OFFSET, settings[0].xy);
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
// hueSpill's: an sRGB colour's Pb and Pr, and the colour again from its luma and those
|
|
82
|
+
vec2 chromaOf(vec3 encoded) {
|
|
83
|
+
float y = dot(encoded, LUMA);
|
|
84
|
+
return vec2(encoded.b - y, encoded.r - y) / CHROMA_SCALE;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
vec3 fromLumaChroma(float y, vec2 c) {
|
|
88
|
+
float r = y + c.y * CHROMA_SCALE.y;
|
|
89
|
+
float b = y + c.x * CHROMA_SCALE.x;
|
|
90
|
+
return vec3(r, (y - LUMA.r * r - LUMA.b * b) / LUMA.g, b);
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
// The add-on saturation: the colour's HSB saturation scaled by s, keeping its hue and its brightest channel. Each
|
|
94
|
+
// channel's distance below the brightest scales by s, but no further than takes the dimmest to 0, as saturated as a
|
|
95
|
+
// colour can be
|
|
96
|
+
vec3 saturated(vec3 rgb, float s) {
|
|
97
|
+
float top = max(max(rgb.r, rgb.g), rgb.b);
|
|
98
|
+
float spread = top - min(min(rgb.r, rgb.g), rgb.b);
|
|
99
|
+
return top - (top - rgb) * min(s, top / max(spread, 1e-12));
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
// How far inside the cut's side from a to b the point p is, in output pixels. The points are in the picture's
|
|
103
|
+
// coordinates, and size scales them to output pixels once they're subtracted, so a short side between two corners
|
|
104
|
+
// close together keeps its direction. Two corners together leave no side, and a triangle
|
|
105
|
+
float cutSide(vec2 a, vec2 b, vec2 p, vec2 size) {
|
|
106
|
+
vec2 side = (b - a) * size;
|
|
107
|
+
float len = length(side);
|
|
108
|
+
if (len == 0.0) return 1.0e6;
|
|
109
|
+
vec2 q = (p - a) * size;
|
|
110
|
+
return (q.y * side.x - q.x * side.y) / len;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
// 1 inside the cut, 0 outside, and between across the output pixel a side passes through, so its edges are smooth at
|
|
114
|
+
// any size. The corners must make a convex shape, which a point is inside where it is inside all of its sides. They
|
|
115
|
+
// go clockwise, which puts the inside of each side on its right
|
|
116
|
+
float cutMask(vec2 p, vec2 size) {
|
|
117
|
+
vec2 c0 = settings[5].xy;
|
|
118
|
+
vec2 c1 = settings[5].zw;
|
|
119
|
+
vec2 c2 = settings[6].xy;
|
|
120
|
+
vec2 c3 = settings[6].zw;
|
|
121
|
+
float d = min(
|
|
122
|
+
min(cutSide(c0, c1, p, size), cutSide(c1, c2, p, size)),
|
|
123
|
+
min(cutSide(c2, c3, p, size), cutSide(c3, c0, p, size))
|
|
124
|
+
);
|
|
125
|
+
return clamp(d + 0.5, 0.0, 1.0);
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
void main() {
|
|
129
|
+
// Output pixels to the picture's width and height, for the cut's soft edge
|
|
130
|
+
vec2 size = 1.0 / max(fwidth(vUv), vec2(1e-6));
|
|
131
|
+
vec4 key = settings[0];
|
|
132
|
+
vec4 matte = settings[1];
|
|
133
|
+
vec4 correction = settings[2];
|
|
134
|
+
vec4 extras = settings[3];
|
|
135
|
+
vec4 flags = settings[4];
|
|
136
|
+
|
|
137
|
+
// Where this pixel is on the picture, mirrored by flip
|
|
138
|
+
vec2 p = vec2(flags.x > 0.5 ? 1.0 - vUv.x : vUv.x, vUv.y);
|
|
139
|
+
ivec2 pixels = textureSize(image, 0);
|
|
140
|
+
|
|
141
|
+
vec4 sampled = sampleAt(p, pixels);
|
|
142
|
+
float alpha = sampled.a;
|
|
143
|
+
vec3 rgb = alpha > 0.0 ? sampled.rgb / alpha : vec3(0.0);
|
|
144
|
+
|
|
145
|
+
// OBS's filter: the key's distance, and from it the alpha and how much of the colour spill keeps
|
|
146
|
+
bool keyed = key.w > 0.5;
|
|
147
|
+
float kept = 1.0;
|
|
148
|
+
if (keyed) {
|
|
149
|
+
vec2 texel = 1.0 / vec2(pixels);
|
|
150
|
+
vec2 across = vec2(texel.x, 0.5 * texel.y);
|
|
151
|
+
vec2 up = vec2(0.5 * texel.x, -texel.y);
|
|
152
|
+
float around = keyDistance(toEncoded(sampleAt(p - across, pixels).rgb))
|
|
153
|
+
+ keyDistance(toEncoded(sampleAt(p + across, pixels).rgb))
|
|
154
|
+
+ keyDistance(toEncoded(sampleAt(p - up, pixels).rgb))
|
|
155
|
+
+ keyDistance(toEncoded(sampleAt(p + up, pixels).rgb));
|
|
156
|
+
float beyond = (2.0 * around + keyDistance(toEncoded(rgb))) / 9.0 - matte.x;
|
|
157
|
+
alpha *= matte.w;
|
|
158
|
+
alpha *= pow(clamp(beyond / matte.y, 0.0, 1.0), 1.5);
|
|
159
|
+
kept = pow(clamp(beyond / matte.z, 0.0, 1.0), 1.5);
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
// The add-ons tint and saturation. Spill desaturates to the tinted colour's luma, as it was before saturation
|
|
163
|
+
rgb *= extras.rgb;
|
|
164
|
+
float grey = dot(rgb, LUMA);
|
|
165
|
+
if (extras.w != 1.0) rgb = saturated(rgb, extras.w);
|
|
166
|
+
|
|
167
|
+
// The rest of OBS's filter: spill, then gamma, contrast and brightness
|
|
168
|
+
if (keyed) {
|
|
169
|
+
if (settings[7].y > 0.5) {
|
|
170
|
+
// The add-on hueSpill: only the part of the colour's chroma that points the key's way comes out
|
|
171
|
+
vec3 encoded = toEncoded(rgb);
|
|
172
|
+
vec2 toward = (key.xy - CHROMA_OFFSET) * key.z;
|
|
173
|
+
vec2 c = chromaOf(encoded);
|
|
174
|
+
c -= toward * max(dot(c, toward), 0.0) * (1.0 - kept);
|
|
175
|
+
rgb = toLinear(clamp(fromLumaChroma(dot(encoded, LUMA), c), 0.0, 1.0));
|
|
176
|
+
} else {
|
|
177
|
+
rgb = mix(vec3(grey), rgb, kept);
|
|
178
|
+
}
|
|
179
|
+
rgb = pow(max(rgb, 0.0), vec3(correction.x)) * correction.y + correction.z;
|
|
180
|
+
}
|
|
181
|
+
rgb = clamp(rgb, 0.0, 1.0);
|
|
182
|
+
|
|
183
|
+
// The add-on crop
|
|
184
|
+
if (settings[7].x > 0.5) alpha *= cutMask(p, size);
|
|
185
|
+
|
|
186
|
+
if (flags.z < 0.5) rgb = toEncoded(rgb);
|
|
187
|
+
fragColor = vec4(rgb * alpha, alpha);
|
|
188
|
+
}
|
|
189
|
+
`
|
|
190
|
+
|
|
191
|
+
// One triangle over the whole canvas, its corners made from their index, so there's nothing to upload
|
|
192
|
+
export const VERTEX: string = /* glsl */ `#version 300 es
|
|
193
|
+
out vec2 vUv;
|
|
194
|
+
void main() {
|
|
195
|
+
vec2 position = vec2(float((gl_VertexID & 1) << 2) - 1.0, float((gl_VertexID & 2) << 1) - 1.0);
|
|
196
|
+
vUv = vec2(position.x + 1.0, 1.0 - position.y) * 0.5;
|
|
197
|
+
gl_Position = vec4(position, 0.0, 1.0);
|
|
198
|
+
}
|
|
199
|
+
`
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
// The keyer in WGSL, for the WebGPU renderer: OBS's chroma key filter and the add-ons, in the same steps as the GLSL
|
|
2
|
+
// in shader.glsl.ts, which says what they are. Its texture always holds the picture's sRGB values with the top row
|
|
3
|
+
// first, so it reads none of the renderer flags that the GLSL takes from three.js
|
|
4
|
+
export const SHADER: string = /* wgsl */ `
|
|
5
|
+
struct Settings {
|
|
6
|
+
key: vec4f,
|
|
7
|
+
matte: vec4f,
|
|
8
|
+
correction: vec4f,
|
|
9
|
+
extras: vec4f,
|
|
10
|
+
flags: vec4f,
|
|
11
|
+
corners: array<vec4f, 2>,
|
|
12
|
+
switches: vec4f,
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
@group(0) @binding(0) var image: texture_2d<f32>;
|
|
16
|
+
@group(0) @binding(1) var<uniform> settings: Settings;
|
|
17
|
+
|
|
18
|
+
struct Varyings {
|
|
19
|
+
@builtin(position) position: vec4f,
|
|
20
|
+
@location(0) uv: vec2f,
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
// One triangle over the whole canvas, its corners made from their index
|
|
24
|
+
@vertex
|
|
25
|
+
fn vertexMain(@builtin(vertex_index) index: u32) -> Varyings {
|
|
26
|
+
let position = vec2f(f32((index & 1u) << 2u) - 1.0, f32((index & 2u) << 1u) - 1.0);
|
|
27
|
+
var result: Varyings;
|
|
28
|
+
result.position = vec4f(position, 0.0, 1.0);
|
|
29
|
+
result.uv = vec2f(position.x + 1.0, 1.0 - position.y) * 0.5;
|
|
30
|
+
return result;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
const CB = vec3f(-0.100644, -0.338572, 0.439216);
|
|
34
|
+
const CR = vec3f(0.439216, -0.398942, -0.040274);
|
|
35
|
+
const CHROMA_OFFSET = 0.501961;
|
|
36
|
+
const LUMA = vec3f(0.2126, 0.7152, 0.0722);
|
|
37
|
+
const CHROMA_SCALE = vec2f(1.8556, 1.5748);
|
|
38
|
+
|
|
39
|
+
fn channelToLinear(c: f32) -> f32 {
|
|
40
|
+
return select(pow((c + 0.055) / 1.055, 2.4), c / 12.92, c <= 0.04045);
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
fn channelToEncoded(c: f32) -> f32 {
|
|
44
|
+
return select(1.055 * pow(c, 1.0 / 2.4) - 0.055, 12.92 * c, c <= 0.0031308);
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
fn toLinear(c: vec3f) -> vec3f {
|
|
48
|
+
return vec3f(channelToLinear(c.r), channelToLinear(c.g), channelToLinear(c.b));
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
fn toEncoded(c: vec3f) -> vec3f {
|
|
52
|
+
return vec3f(channelToEncoded(c.r), channelToEncoded(c.g), channelToEncoded(c.b));
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
fn pixelAt(at: vec2i, size: vec2i) -> vec4f {
|
|
56
|
+
let texel = textureLoad(image, clamp(at, vec2i(0), size - 1), 0);
|
|
57
|
+
return vec4f(toLinear(texel.rgb) * texel.a, texel.a);
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
fn sampleAt(q: vec2f, size: vec2i) -> vec4f {
|
|
61
|
+
let f = q * vec2f(size) - 0.5;
|
|
62
|
+
let corner = floor(f);
|
|
63
|
+
let w = f - corner;
|
|
64
|
+
let i = vec2i(corner);
|
|
65
|
+
let above = mix(pixelAt(i, size), pixelAt(i + vec2i(1, 0), size), w.x);
|
|
66
|
+
let below = mix(pixelAt(i + vec2i(0, 1), size), pixelAt(i + vec2i(1, 1), size), w.x);
|
|
67
|
+
return mix(above, below, w.y);
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
fn keyDistance(encoded: vec3f) -> f32 {
|
|
71
|
+
return distance(vec2f(dot(encoded, CB), dot(encoded, CR)) + CHROMA_OFFSET, settings.key.xy);
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
fn chromaOf(encoded: vec3f) -> vec2f {
|
|
75
|
+
let y = dot(encoded, LUMA);
|
|
76
|
+
return vec2f(encoded.b - y, encoded.r - y) / CHROMA_SCALE;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
fn fromLumaChroma(y: f32, c: vec2f) -> vec3f {
|
|
80
|
+
let r = y + c.y * CHROMA_SCALE.y;
|
|
81
|
+
let b = y + c.x * CHROMA_SCALE.x;
|
|
82
|
+
return vec3f(r, (y - LUMA.r * r - LUMA.b * b) / LUMA.g, b);
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
fn saturated(rgb: vec3f, s: f32) -> vec3f {
|
|
86
|
+
let top = max(max(rgb.r, rgb.g), rgb.b);
|
|
87
|
+
let spread = top - min(min(rgb.r, rgb.g), rgb.b);
|
|
88
|
+
return top - (top - rgb) * min(s, top / max(spread, 1e-12));
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
fn cutSide(a: vec2f, b: vec2f, p: vec2f, size: vec2f) -> f32 {
|
|
92
|
+
let side = (b - a) * size;
|
|
93
|
+
let len = length(side);
|
|
94
|
+
if (len == 0.0) {
|
|
95
|
+
return 1.0e6;
|
|
96
|
+
}
|
|
97
|
+
let q = (p - a) * size;
|
|
98
|
+
return (q.y * side.x - q.x * side.y) / len;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
fn cutMask(p: vec2f, size: vec2f) -> f32 {
|
|
102
|
+
let c0 = settings.corners[0].xy;
|
|
103
|
+
let c1 = settings.corners[0].zw;
|
|
104
|
+
let c2 = settings.corners[1].xy;
|
|
105
|
+
let c3 = settings.corners[1].zw;
|
|
106
|
+
let d = min(
|
|
107
|
+
min(cutSide(c0, c1, p, size), cutSide(c1, c2, p, size)),
|
|
108
|
+
min(cutSide(c2, c3, p, size), cutSide(c3, c0, p, size))
|
|
109
|
+
);
|
|
110
|
+
return clamp(d + 0.5, 0.0, 1.0);
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
@fragment
|
|
114
|
+
fn fragmentMain(input: Varyings) -> @location(0) vec4f {
|
|
115
|
+
// Taken first, while every pixel takes the same way, as a derivative must be
|
|
116
|
+
let size = 1.0 / max(fwidth(input.uv), vec2f(1e-6));
|
|
117
|
+
let key = settings.key;
|
|
118
|
+
let matte = settings.matte;
|
|
119
|
+
let correction = settings.correction;
|
|
120
|
+
let extras = settings.extras;
|
|
121
|
+
|
|
122
|
+
let p = vec2f(select(input.uv.x, 1.0 - input.uv.x, settings.flags.x > 0.5), input.uv.y);
|
|
123
|
+
let pixels = vec2i(textureDimensions(image));
|
|
124
|
+
|
|
125
|
+
let sampled = sampleAt(p, pixels);
|
|
126
|
+
var alpha = sampled.a;
|
|
127
|
+
var rgb = select(vec3f(0.0), sampled.rgb / alpha, alpha > 0.0);
|
|
128
|
+
|
|
129
|
+
// OBS's filter: the key's distance, and from it the alpha and how much of the colour spill keeps
|
|
130
|
+
let keyed = key.w > 0.5;
|
|
131
|
+
var kept = 1.0;
|
|
132
|
+
if (keyed) {
|
|
133
|
+
let texel = 1.0 / vec2f(pixels);
|
|
134
|
+
let across = vec2f(texel.x, 0.5 * texel.y);
|
|
135
|
+
let up = vec2f(0.5 * texel.x, -texel.y);
|
|
136
|
+
let around = keyDistance(toEncoded(sampleAt(p - across, pixels).rgb))
|
|
137
|
+
+ keyDistance(toEncoded(sampleAt(p + across, pixels).rgb))
|
|
138
|
+
+ keyDistance(toEncoded(sampleAt(p - up, pixels).rgb))
|
|
139
|
+
+ keyDistance(toEncoded(sampleAt(p + up, pixels).rgb));
|
|
140
|
+
let beyond = (2.0 * around + keyDistance(toEncoded(rgb))) / 9.0 - matte.x;
|
|
141
|
+
alpha *= matte.w;
|
|
142
|
+
alpha *= pow(clamp(beyond / matte.y, 0.0, 1.0), 1.5);
|
|
143
|
+
kept = pow(clamp(beyond / matte.z, 0.0, 1.0), 1.5);
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
// The add-ons tint and saturation. Spill desaturates to the tinted colour's luma, as it was before saturation
|
|
147
|
+
rgb *= extras.rgb;
|
|
148
|
+
let grey = dot(rgb, LUMA);
|
|
149
|
+
if (extras.w != 1.0) {
|
|
150
|
+
rgb = saturated(rgb, extras.w);
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
// The rest of OBS's filter: spill, then gamma, contrast and brightness
|
|
154
|
+
if (keyed) {
|
|
155
|
+
if (settings.switches.y > 0.5) {
|
|
156
|
+
// The add-on, hueSpill
|
|
157
|
+
let encoded = toEncoded(rgb);
|
|
158
|
+
let toward = (key.xy - CHROMA_OFFSET) * key.z;
|
|
159
|
+
var c = chromaOf(encoded);
|
|
160
|
+
c -= toward * max(dot(c, toward), 0.0) * (1.0 - kept);
|
|
161
|
+
rgb = toLinear(clamp(fromLumaChroma(dot(encoded, LUMA), c), vec3f(0.0), vec3f(1.0)));
|
|
162
|
+
} else {
|
|
163
|
+
rgb = mix(vec3f(grey), rgb, kept);
|
|
164
|
+
}
|
|
165
|
+
rgb = pow(max(rgb, vec3f(0.0)), vec3f(correction.x)) * correction.y + correction.z;
|
|
166
|
+
}
|
|
167
|
+
rgb = clamp(rgb, vec3f(0.0), vec3f(1.0));
|
|
168
|
+
|
|
169
|
+
// The add-on crop
|
|
170
|
+
if (settings.switches.x > 0.5) {
|
|
171
|
+
alpha *= cutMask(p, size);
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
return vec4f(toEncoded(rgb) * alpha, alpha);
|
|
175
|
+
}
|
|
176
|
+
`
|
package/src/source.ts
ADDED
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
import type { Source } from './types.ts'
|
|
2
|
+
|
|
3
|
+
// How a keyer tells when its source has a new picture to take. A video's frames are watched for. An image is new when
|
|
4
|
+
// its source changes. A canvas can be drawn on at any time with nothing to say so, so it's taken every time. The rest
|
|
5
|
+
// never change. Each kind is told apart by what it has rather than its class, so an element from another window, as a
|
|
6
|
+
// pop-out's would be, is still known
|
|
7
|
+
|
|
8
|
+
// How long a video goes without a frame callback before its clock is watched instead: callbacks come only while the
|
|
9
|
+
// page is drawing, and some browsers have none
|
|
10
|
+
const STALLED = 100
|
|
11
|
+
|
|
12
|
+
// The source's own size in pixels
|
|
13
|
+
export function sizeOf(source: Source): [number, number] {
|
|
14
|
+
if ('videoWidth' in source) return [source.videoWidth, source.videoHeight]
|
|
15
|
+
if ('naturalWidth' in source) return [source.naturalWidth, source.naturalHeight]
|
|
16
|
+
if ('displayWidth' in source) return [source.displayWidth, source.displayHeight]
|
|
17
|
+
return [source.width, source.height]
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
// What a keyer given no onError does with a picture it can't key
|
|
21
|
+
export function warnUnkeyable(error: unknown): void {
|
|
22
|
+
console.warn('Greenscreen: the picture could not be keyed', error)
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
// One source as a keyer follows it: whether it has a picture yet, and whether it has a new one since it was last taken
|
|
26
|
+
export class Feed {
|
|
27
|
+
readonly source: Source
|
|
28
|
+
#fresh = true
|
|
29
|
+
#time = -1
|
|
30
|
+
#src = ''
|
|
31
|
+
#handle = 0
|
|
32
|
+
#lastFrame = -Infinity
|
|
33
|
+
// The picture that last failed, by its src where it has one, so each failure is reported once. Null once one is taken
|
|
34
|
+
#failedOn: string | null = null
|
|
35
|
+
#disposed = false
|
|
36
|
+
|
|
37
|
+
constructor(source: Source) {
|
|
38
|
+
this.source = source
|
|
39
|
+
this.#watch()
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
#watch() {
|
|
43
|
+
const video = this.source
|
|
44
|
+
if (!('requestVideoFrameCallback' in video) || this.#disposed) return
|
|
45
|
+
this.#handle = video.requestVideoFrameCallback(() => {
|
|
46
|
+
this.#fresh = true
|
|
47
|
+
this.#lastFrame = performance.now()
|
|
48
|
+
this.#watch()
|
|
49
|
+
})
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
// Whether there's a picture to take: a video with a frame, an image that has loaded, anything else with a size
|
|
53
|
+
hasPicture(): boolean {
|
|
54
|
+
const source = this.source
|
|
55
|
+
if ('videoWidth' in source) return source.readyState >= 2 && source.videoWidth > 0 && source.videoHeight > 0
|
|
56
|
+
if ('naturalWidth' in source) return source.complete && source.naturalWidth > 0
|
|
57
|
+
const [width, height] = sizeOf(source)
|
|
58
|
+
return width > 0 && height > 0
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
// Whether the picture has changed since it was last taken
|
|
62
|
+
changed(): boolean {
|
|
63
|
+
if (this.#fresh) return true
|
|
64
|
+
const source = this.source
|
|
65
|
+
if ('videoWidth' in source) return performance.now() - this.#lastFrame > STALLED && source.currentTime !== this.#time
|
|
66
|
+
if ('naturalWidth' in source) return source.currentSrc !== this.#src
|
|
67
|
+
return 'getContext' in source
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
size(): [number, number] {
|
|
71
|
+
return sizeOf(this.source)
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
// The picture has been taken
|
|
75
|
+
taken() {
|
|
76
|
+
this.#fresh = false
|
|
77
|
+
this.#failedOn = null
|
|
78
|
+
const source = this.source
|
|
79
|
+
if ('videoWidth' in source) this.#time = source.currentTime
|
|
80
|
+
else if ('naturalWidth' in source) this.#src = source.currentSrc
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
// Taking the picture failed, as it does for one larger than the GPU's textures, or from another site without CORS.
|
|
84
|
+
// It's tried again only when the picture changes. Returns whether this failure is new: the first since a picture was
|
|
85
|
+
// last taken, or of an image or video since given another src, so each is reported once however often it's tried
|
|
86
|
+
failed(): boolean {
|
|
87
|
+
const source = this.source
|
|
88
|
+
const what = 'currentSrc' in source ? source.currentSrc : ''
|
|
89
|
+
const first = this.#failedOn !== what
|
|
90
|
+
this.taken()
|
|
91
|
+
this.#failedOn = what
|
|
92
|
+
return first
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
// Take the picture again on the next render, as its pixels have changed
|
|
96
|
+
invalidate() {
|
|
97
|
+
this.#fresh = true
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
dispose() {
|
|
101
|
+
this.#disposed = true
|
|
102
|
+
const video = this.source
|
|
103
|
+
if (this.#handle && 'cancelVideoFrameCallback' in video) video.cancelVideoFrameCallback(this.#handle)
|
|
104
|
+
}
|
|
105
|
+
}
|