@snovakow/greenscreen 0.0.0-stage → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +238 -2
  3. package/dist/greenscreen.d.ts +32 -0
  4. package/dist/greenscreen.d.ts.map +1 -0
  5. package/dist/greenscreen.global.js +315 -0
  6. package/dist/greenscreen.global.js.map +1 -0
  7. package/dist/greenscreen.js +250 -0
  8. package/dist/greenscreen.js.map +1 -0
  9. package/dist/greenscreen.min.js +315 -0
  10. package/dist/greenscreen.min.js.map +1 -0
  11. package/dist/index.d.ts +6 -0
  12. package/dist/index.d.ts.map +1 -0
  13. package/dist/index.js +3 -0
  14. package/dist/index.js.map +1 -0
  15. package/dist/pack.d.ts +7 -0
  16. package/dist/pack.d.ts.map +1 -0
  17. package/dist/pack.js +63 -0
  18. package/dist/pack.js.map +1 -0
  19. package/dist/react.d.ts +26 -0
  20. package/dist/react.d.ts.map +1 -0
  21. package/dist/react.js +78 -0
  22. package/dist/react.js.map +1 -0
  23. package/dist/renderer.d.ts +19 -0
  24. package/dist/renderer.d.ts.map +1 -0
  25. package/dist/renderer.js +25 -0
  26. package/dist/renderer.js.map +1 -0
  27. package/dist/settings.d.ts +36 -0
  28. package/dist/settings.d.ts.map +1 -0
  29. package/dist/settings.js +112 -0
  30. package/dist/settings.js.map +1 -0
  31. package/dist/shader.glsl.d.ts +3 -0
  32. package/dist/shader.glsl.d.ts.map +1 -0
  33. package/dist/shader.glsl.js +179 -0
  34. package/dist/shader.glsl.js.map +1 -0
  35. package/dist/shader.wgsl.d.ts +2 -0
  36. package/dist/shader.wgsl.d.ts.map +1 -0
  37. package/dist/shader.wgsl.js +160 -0
  38. package/dist/shader.wgsl.js.map +1 -0
  39. package/dist/source.d.ts +16 -0
  40. package/dist/source.d.ts.map +1 -0
  41. package/dist/source.js +103 -0
  42. package/dist/source.js.map +1 -0
  43. package/dist/three-webgpu.d.ts +20 -0
  44. package/dist/three-webgpu.d.ts.map +1 -0
  45. package/dist/three-webgpu.js +178 -0
  46. package/dist/three-webgpu.js.map +1 -0
  47. package/dist/three.d.ts +27 -0
  48. package/dist/three.d.ts.map +1 -0
  49. package/dist/three.js +81 -0
  50. package/dist/three.js.map +1 -0
  51. package/dist/types.d.ts +4 -0
  52. package/dist/types.d.ts.map +1 -0
  53. package/dist/types.js +4 -0
  54. package/dist/types.js.map +1 -0
  55. package/dist/webgl.d.ts +13 -0
  56. package/dist/webgl.d.ts.map +1 -0
  57. package/dist/webgl.js +183 -0
  58. package/dist/webgl.js.map +1 -0
  59. package/dist/webgpu.d.ts +19 -0
  60. package/dist/webgpu.d.ts.map +1 -0
  61. package/dist/webgpu.js +303 -0
  62. package/dist/webgpu.js.map +1 -0
  63. package/package.json +103 -4
  64. package/src/global.ts +11 -0
  65. package/src/greenscreen.ts +286 -0
  66. package/src/index.ts +5 -0
  67. package/src/pack.ts +73 -0
  68. package/src/react.tsx +126 -0
  69. package/src/renderer.ts +53 -0
  70. package/src/settings.ts +177 -0
  71. package/src/shader.glsl.ts +179 -0
  72. package/src/shader.wgsl.ts +159 -0
  73. package/src/source.ts +105 -0
  74. package/src/three-webgpu.ts +261 -0
  75. package/src/three.ts +114 -0
  76. package/src/types.ts +10 -0
  77. package/src/webgl.ts +191 -0
  78. package/src/webgpu.ts +312 -0
@@ -0,0 +1,177 @@
1
+ // What the keyer does to a picture, and how that's packed into the numbers each of its shaders reads
2
+
3
+ // A point on the picture: 0 to 1 across it from the left, and 0 to 1 down it from the top
4
+ export interface Point {
5
+ x: number
6
+ y: number
7
+ }
8
+
9
+ // The keyer keys as OBS Studio's chroma key filter does, the filter's second version, and takes its settings in OBS's
10
+ // units, ranges and defaults, so a filter's numbers can be copied from OBS as they are. The add-ons are the library's
11
+ // own, and not in OBS. At their defaults they change nothing, and the picture is keyed exactly as OBS keys it
12
+ // https://github.com/obsproject/obs-studio/blob/master/plugins/obs-filters/chroma-key-filter.c
13
+ export interface Settings {
14
+ // OBS's chroma key filter
15
+
16
+ // Whether the filter runs, as its eye in OBS's list of filters. Off, the picture passes through it as it came, and
17
+ // only the add-ons apply
18
+ key: boolean
19
+ // OBS's Key Color, as 0xrrggbb. KEY_COLORS has OBS's Green, Blue and Magenta. Any other colour is OBS's Custom Color
20
+ keyColor: number
21
+ // OBS's Similarity, 1 to 1000: how near the key a colour's chroma must be to be taken out
22
+ similarity: number
23
+ // OBS's Smoothness, 1 to 1000: how far past similarity a colour fades back in
24
+ smoothness: number
25
+ // OBS's Key Color Spill Reduction, 1 to 1000: how far past similarity a colour kept is desaturated, which takes the
26
+ // key's cast off edges and hair
27
+ spill: number
28
+ // OBS's Opacity, 0 to 1
29
+ opacity: number
30
+ // OBS's Contrast, −4 to 4: a factor from 1/5 to 5, 0 for no change
31
+ contrast: number
32
+ // OBS's Brightness, −1 to 1, added: 0 for no change
33
+ brightness: number
34
+ // OBS's Gamma, −1 to 1: an exponent from 2 to 1/2, 0 for no change
35
+ gamma: number
36
+
37
+ // Add-ons: the library's own, not OBS's. They act on the picture OBS's filter gives, except hueSpill, which acts
38
+ // inside the filter
39
+
40
+ // Spill takes out only the key's hue, keeping each colour's luma and its other hues, where OBS's spill desaturates
41
+ // the whole colour. It reaches as far as spill does, and takes as much of the key's hue out as OBS would of the colour
42
+ hueSpill: boolean
43
+ // Mirrors the picture left to right
44
+ flip: boolean
45
+ // Cuts the picture down to the shape its corners make
46
+ crop: boolean
47
+ // The cut's four corners, clockwise from the top left, in the picture's own coordinates: not mirrored by flip. The
48
+ // shape they make must be convex. They're kept while crop is off
49
+ corners: readonly Readonly<Point>[]
50
+ // Multiplies the picture's colour, as 0xrrggbb: white for no change
51
+ tint: number
52
+ // 0 for grey, 1 for no change, up to 2
53
+ saturation: number
54
+ }
55
+
56
+ export type NumericSetting = 'similarity' | 'smoothness' | 'spill' | 'opacity' | 'contrast' | 'brightness' | 'gamma' | 'saturation'
57
+
58
+ export interface Range {
59
+ readonly min: number
60
+ readonly max: number
61
+ // The step OBS's slider moves in, or the add-on's. Values between steps are kept as they are, except similarity's,
62
+ // smoothness's and spill's, which OBS keeps as whole numbers, and which are rounded to them
63
+ readonly step: number
64
+ }
65
+
66
+ // Each number's limits, for a control panel to use: OBS's sliders', and the add-on's. Values outside them are clamped
67
+ export const RANGES: Readonly<Record<NumericSetting, Range>> = Object.freeze({
68
+ similarity: { min: 1, max: 1000, step: 1 },
69
+ smoothness: { min: 1, max: 1000, step: 1 },
70
+ spill: { min: 1, max: 1000, step: 1 },
71
+ opacity: { min: 0, max: 1, step: 0.0001 },
72
+ contrast: { min: -4, max: 4, step: 0.01 },
73
+ brightness: { min: -1, max: 1, step: 0.0001 },
74
+ gamma: { min: -1, max: 1, step: 0.01 },
75
+ saturation: { min: 0, max: 2, step: 0.01 },
76
+ })
77
+
78
+ // OBS's Green, Blue and Magenta. OBS writes its blue as 0xFF9900, as it keeps colours as 0xbbggrr
79
+ export const KEY_COLORS: Readonly<{ green: number; blue: number; magenta: number }> = Object.freeze({
80
+ green: 0x00ff00,
81
+ blue: 0x0099ff,
82
+ magenta: 0xff00ff,
83
+ })
84
+
85
+ // The whole picture, which is where the cut starts
86
+ const WHOLE: readonly Readonly<Point>[] = Object.freeze([
87
+ Object.freeze({ x: 0, y: 0 }),
88
+ Object.freeze({ x: 1, y: 0 }),
89
+ Object.freeze({ x: 1, y: 1 }),
90
+ Object.freeze({ x: 0, y: 1 }),
91
+ ])
92
+
93
+ // OBS's defaults for its filter, with the add-ons changing nothing
94
+ export const DEFAULT_SETTINGS: Readonly<Settings> = Object.freeze({
95
+ key: true,
96
+ keyColor: KEY_COLORS.green,
97
+ similarity: 400,
98
+ smoothness: 80,
99
+ spill: 100,
100
+ opacity: 1,
101
+ contrast: 0,
102
+ brightness: 0,
103
+ gamma: 0,
104
+ hueSpill: false,
105
+ flip: false,
106
+ crop: false,
107
+ corners: WHOLE,
108
+ tint: 0xffffff,
109
+ saturation: 1,
110
+ })
111
+
112
+ const NUMBERS: readonly NumericSetting[] = [
113
+ 'similarity',
114
+ 'smoothness',
115
+ 'spill',
116
+ 'opacity',
117
+ 'contrast',
118
+ 'brightness',
119
+ 'gamma',
120
+ 'saturation',
121
+ ]
122
+
123
+ // The numbers OBS keeps as whole numbers, as its sliders for them move in whole steps and it stores them as integers
124
+ const WHOLE_NUMBERS: ReadonlySet<NumericSetting> = new Set(['similarity', 'smoothness', 'spill'])
125
+
126
+ // The settings with the patch's changes made, as a new frozen object. A value that's undefined leaves the setting as
127
+ // it was, and a name that isn't a setting is passed over, so an app's own object of settings can be given as it is.
128
+ // A value of the wrong kind throws, so a mistake shows where it's made
129
+ export function mergeSettings(base: Readonly<Settings>, patch: Readonly<Partial<Settings>>): Readonly<Settings> {
130
+ const next: Settings = { ...base }
131
+ if (patch.key !== undefined) next.key = flag('key', patch.key)
132
+ if (patch.hueSpill !== undefined) next.hueSpill = flag('hueSpill', patch.hueSpill)
133
+ if (patch.flip !== undefined) next.flip = flag('flip', patch.flip)
134
+ if (patch.crop !== undefined) next.crop = flag('crop', patch.crop)
135
+ if (patch.keyColor !== undefined) next.keyColor = colour('keyColor', patch.keyColor)
136
+ if (patch.tint !== undefined) next.tint = colour('tint', patch.tint)
137
+ if (patch.corners !== undefined) next.corners = corners(patch.corners)
138
+ for (const name of NUMBERS) {
139
+ const value = patch[name]
140
+ if (value !== undefined) next[name] = number(name, value)
141
+ }
142
+ return Object.freeze(next)
143
+ }
144
+
145
+ const clamp = (x: number, low: number, high: number) => Math.min(Math.max(x, low), high)
146
+
147
+ function flag(name: string, value: unknown): boolean {
148
+ if (typeof value !== 'boolean') throw new TypeError(`Greenscreen: ${name} must be true or false`)
149
+ return value
150
+ }
151
+
152
+ function colour(name: string, value: unknown): number {
153
+ if (typeof value !== 'number' || !Number.isInteger(value) || value < 0 || value > 0xffffff) {
154
+ throw new TypeError(`Greenscreen: ${name} must be a colour from 0x000000 to 0xffffff`)
155
+ }
156
+ return value
157
+ }
158
+
159
+ function number(name: NumericSetting, value: unknown): number {
160
+ if (typeof value !== 'number' || !Number.isFinite(value)) throw new TypeError(`Greenscreen: ${name} must be a number`)
161
+ const clamped = clamp(value, RANGES[name].min, RANGES[name].max)
162
+ return WHOLE_NUMBERS.has(name) ? Math.round(clamped) : clamped
163
+ }
164
+
165
+ function corners(value: unknown): readonly Readonly<Point>[] {
166
+ if (!Array.isArray(value) || value.length !== 4) throw new TypeError('Greenscreen: corners must be four points')
167
+ return Object.freeze(
168
+ value.map((point: unknown) => {
169
+ const x: unknown = typeof point === 'object' && point !== null && 'x' in point ? point.x : undefined
170
+ const y: unknown = typeof point === 'object' && point !== null && 'y' in point ? point.y : undefined
171
+ if (typeof x !== 'number' || typeof y !== 'number' || !Number.isFinite(x) || !Number.isFinite(y)) {
172
+ throw new TypeError('Greenscreen: each corner must have a number x and y')
173
+ }
174
+ return Object.freeze({ x: clamp(x, 0, 1), y: clamp(y, 0, 1) })
175
+ }),
176
+ )
177
+ }
@@ -0,0 +1,179 @@
1
+ // The keyer in GLSL ES 3.00. The fragment shader is shared by the WebGL2 renderer and the three.js material, each
2
+ // giving it vUv, where the pixel is on the output: 0 to 1 across and down. pack.ts says what's in settings.
3
+ //
4
+ // First it keys as OBS Studio's chroma key filter does, the filter's second version, in the same steps and order:
5
+ // https://github.com/obsproject/obs-studio/blob/master/plugins/obs-filters/data/chroma_key_filter_v2.effect
6
+ // - The picture is sampled as OBS holds a source: premultiplied, in linear light, and blended between pixels in
7
+ // linear light, as OBS's sampler blends its texture. The pixel's own colour is then divided by its alpha.
8
+ // - How far a colour's chroma is from the key's is measured on its sRGB values. The measure is blurred: four samples
9
+ // round the pixel, a pixel across and half a pixel down, and half a pixel across and a pixel up, each either way,
10
+ // count twice each, and the pixel itself once, of nine.
11
+ // - Past similarity the alpha fades in over smoothness, and the colour is desaturated over spill, each on a curve of
12
+ // power 1.5. Opacity multiplies the alpha, and gamma, contrast and brightness the colour, in linear light. The
13
+ // colour is clamped, as OBS's target of 8 bits a channel holds it.
14
+ // Then the add-ons, which aren't OBS's: the tint multiplies, saturation mixes with luma, and crop cuts. Flip mirrors
15
+ // the result. One add-on acts inside the filter: hueSpill, where it's on, takes the key's hue out of the colour in place
16
+ // of OBS's desaturation, by as much, keeping the colour's luma and other hues, on its sRGB values. The output is the
17
+ // colour multiplied by its alpha
18
+ export const FRAGMENT: string = /* glsl */ `precision highp float;
19
+ precision highp int;
20
+ precision highp sampler2D;
21
+
22
+ uniform sampler2D image;
23
+ uniform vec4 settings[8];
24
+
25
+ in vec2 vUv;
26
+ out vec4 fragColor;
27
+
28
+ // A colour's Cb and Cr from its sRGB values, as OBS's filter has them: BT.709's, in studio range
29
+ const vec3 CB = vec3(-0.100644, -0.338572, 0.439216);
30
+ const vec3 CR = vec3(0.439216, -0.398942, -0.040274);
31
+ const float CHROMA_OFFSET = 0.501961;
32
+ // BT.709's luma, which OBS desaturates to
33
+ const vec3 LUMA = vec3(0.2126, 0.7152, 0.0722);
34
+ // The scales that take B' − Y' and R' − Y' to BT.709's Pb and Pr, for hueSpill
35
+ const vec2 CHROMA_SCALE = vec2(1.8556, 1.5748);
36
+
37
+ float toLinear(float c) {
38
+ return c <= 0.04045 ? c / 12.92 : pow((c + 0.055) / 1.055, 2.4);
39
+ }
40
+
41
+ float toEncoded(float c) {
42
+ return c <= 0.0031308 ? 12.92 * c : 1.055 * pow(c, 1.0 / 2.4) - 0.055;
43
+ }
44
+
45
+ vec3 toLinear(vec3 c) {
46
+ return vec3(toLinear(c.r), toLinear(c.g), toLinear(c.b));
47
+ }
48
+
49
+ vec3 toEncoded(vec3 c) {
50
+ return vec3(toEncoded(c.r), toEncoded(c.g), toEncoded(c.b));
51
+ }
52
+
53
+ // The picture's pixel at a column, and a row from the top, clamped to the picture's edges as OBS's sampler clamps.
54
+ // It's premultiplied, in linear light, as OBS holds a source, whichever way the texture holds it
55
+ vec4 pixelAt(ivec2 at, ivec2 size) {
56
+ at = clamp(at, ivec2(0), size - 1);
57
+ if (settings[4].y > 0.5) at.y = size.y - 1 - at.y;
58
+ vec4 texel = texelFetch(image, at, 0);
59
+ vec3 rgb = settings[4].w > 0.5 ? texel.rgb : toLinear(texel.rgb);
60
+ return vec4(rgb * texel.a, texel.a);
61
+ }
62
+
63
+ // The picture at q, 0 to 1 across and down it, as OBS's linear sampler gives it: the four pixels round q, blended
64
+ vec4 sampleAt(vec2 q, ivec2 size) {
65
+ vec2 f = q * vec2(size) - 0.5;
66
+ vec2 corner = floor(f);
67
+ vec2 w = f - corner;
68
+ ivec2 i = ivec2(corner);
69
+ vec4 above = mix(pixelAt(i, size), pixelAt(i + ivec2(1, 0), size), w.x);
70
+ vec4 below = mix(pixelAt(i + ivec2(0, 1), size), pixelAt(i + ivec2(1, 1), size), w.x);
71
+ return mix(above, below, w.y);
72
+ }
73
+
74
+ // How far a colour's chroma is from the key's, from its sRGB values
75
+ float keyDistance(vec3 encoded) {
76
+ return distance(vec2(dot(encoded, CB), dot(encoded, CR)) + CHROMA_OFFSET, settings[0].xy);
77
+ }
78
+
79
+ // hueSpill's: an sRGB colour's Pb and Pr, and the colour again from its luma and those
80
+ vec2 chromaOf(vec3 encoded) {
81
+ float y = dot(encoded, LUMA);
82
+ return vec2(encoded.b - y, encoded.r - y) / CHROMA_SCALE;
83
+ }
84
+
85
+ vec3 fromLumaChroma(float y, vec2 c) {
86
+ float r = y + c.y * CHROMA_SCALE.y;
87
+ float b = y + c.x * CHROMA_SCALE.x;
88
+ return vec3(r, (y - LUMA.r * r - LUMA.b * b) / LUMA.g, b);
89
+ }
90
+
91
+ // How far inside the cut's side from a to b the point p is, in output pixels. The points are in the picture's
92
+ // coordinates, and size scales them to output pixels once they're subtracted, so a short side between two corners
93
+ // close together keeps its direction. Two corners together leave no side, and a triangle
94
+ float cutSide(vec2 a, vec2 b, vec2 p, vec2 size) {
95
+ vec2 side = (b - a) * size;
96
+ float len = length(side);
97
+ if (len == 0.0) return 1.0e6;
98
+ vec2 q = (p - a) * size;
99
+ return (q.y * side.x - q.x * side.y) / len;
100
+ }
101
+
102
+ // 1 inside the cut, 0 outside, and between across the output pixel a side passes through, so its edges are smooth at
103
+ // any size. The corners must make a convex shape, which a point is inside where it is inside all of its sides. They
104
+ // go clockwise, which puts the inside of each side on its right
105
+ float cutMask(vec2 p, vec2 size) {
106
+ vec2 c0 = settings[5].xy;
107
+ vec2 c1 = settings[5].zw;
108
+ vec2 c2 = settings[6].xy;
109
+ vec2 c3 = settings[6].zw;
110
+ float d = min(
111
+ min(cutSide(c0, c1, p, size), cutSide(c1, c2, p, size)),
112
+ min(cutSide(c2, c3, p, size), cutSide(c3, c0, p, size))
113
+ );
114
+ return clamp(d + 0.5, 0.0, 1.0);
115
+ }
116
+
117
+ void main() {
118
+ // Output pixels to the picture's width and height, for the cut's soft edge
119
+ vec2 size = 1.0 / max(fwidth(vUv), vec2(1e-6));
120
+ vec4 key = settings[0];
121
+ vec4 matte = settings[1];
122
+ vec4 correction = settings[2];
123
+ vec4 extras = settings[3];
124
+ vec4 flags = settings[4];
125
+
126
+ // Where this pixel is on the picture, mirrored by flip
127
+ vec2 p = vec2(flags.x > 0.5 ? 1.0 - vUv.x : vUv.x, vUv.y);
128
+ ivec2 pixels = textureSize(image, 0);
129
+
130
+ vec4 sampled = sampleAt(p, pixels);
131
+ float alpha = sampled.a;
132
+ vec3 rgb = alpha > 0.0 ? sampled.rgb / alpha : vec3(0.0);
133
+
134
+ // OBS's filter
135
+ if (key.w > 0.5) {
136
+ vec2 texel = 1.0 / vec2(pixels);
137
+ vec2 across = vec2(texel.x, 0.5 * texel.y);
138
+ vec2 up = vec2(0.5 * texel.x, -texel.y);
139
+ float around = keyDistance(toEncoded(sampleAt(p - across, pixels).rgb))
140
+ + keyDistance(toEncoded(sampleAt(p + across, pixels).rgb))
141
+ + keyDistance(toEncoded(sampleAt(p - up, pixels).rgb))
142
+ + keyDistance(toEncoded(sampleAt(p + up, pixels).rgb));
143
+ float beyond = (2.0 * around + keyDistance(toEncoded(rgb))) / 9.0 - matte.x;
144
+ alpha *= matte.w;
145
+ alpha *= pow(clamp(beyond / matte.y, 0.0, 1.0), 1.5);
146
+ float kept = pow(clamp(beyond / matte.z, 0.0, 1.0), 1.5);
147
+ if (settings[7].y > 0.5) {
148
+ // The add-on: only the part of the colour's chroma that points the key's way comes out
149
+ vec3 encoded = toEncoded(rgb);
150
+ vec2 toward = (key.xy - CHROMA_OFFSET) * key.z;
151
+ vec2 c = chromaOf(encoded);
152
+ c -= toward * max(dot(c, toward), 0.0) * (1.0 - kept);
153
+ rgb = toLinear(clamp(fromLumaChroma(dot(encoded, LUMA), c), 0.0, 1.0));
154
+ } else {
155
+ rgb = mix(vec3(dot(rgb, LUMA)), rgb, kept);
156
+ }
157
+ rgb = pow(max(rgb, 0.0), vec3(correction.x)) * correction.y + correction.z;
158
+ }
159
+ rgb = clamp(rgb, 0.0, 1.0);
160
+
161
+ // The add-ons
162
+ rgb *= extras.rgb;
163
+ rgb = clamp(mix(vec3(dot(rgb, LUMA)), rgb, extras.w), 0.0, 1.0);
164
+ if (settings[7].x > 0.5) alpha *= cutMask(p, size);
165
+
166
+ if (flags.z < 0.5) rgb = toEncoded(rgb);
167
+ fragColor = vec4(rgb * alpha, alpha);
168
+ }
169
+ `
170
+
171
+ // One triangle over the whole canvas, its corners made from their index, so there's nothing to upload
172
+ export const VERTEX: string = /* glsl */ `#version 300 es
173
+ out vec2 vUv;
174
+ void main() {
175
+ vec2 position = vec2(float((gl_VertexID & 1) << 2) - 1.0, float((gl_VertexID & 2) << 1) - 1.0);
176
+ vUv = vec2(position.x + 1.0, 1.0 - position.y) * 0.5;
177
+ gl_Position = vec4(position, 0.0, 1.0);
178
+ }
179
+ `
@@ -0,0 +1,159 @@
1
+ // The keyer in WGSL, for the WebGPU renderer: OBS's chroma key filter and then the add-ons, in the same steps as the
2
+ // GLSL in shader.glsl.ts, which says what they are. Its texture always holds the picture's sRGB values with the top row
3
+ // first, so it reads none of the renderer flags that the GLSL takes from three.js
4
+ export const SHADER: string = /* wgsl */ `
5
+ struct Settings {
6
+ key: vec4f,
7
+ matte: vec4f,
8
+ correction: vec4f,
9
+ extras: vec4f,
10
+ flags: vec4f,
11
+ corners: array<vec4f, 2>,
12
+ switches: vec4f,
13
+ }
14
+
15
+ @group(0) @binding(0) var image: texture_2d<f32>;
16
+ @group(0) @binding(1) var<uniform> settings: Settings;
17
+
18
+ struct Varyings {
19
+ @builtin(position) position: vec4f,
20
+ @location(0) uv: vec2f,
21
+ }
22
+
23
+ // One triangle over the whole canvas, its corners made from their index
24
+ @vertex
25
+ fn vertexMain(@builtin(vertex_index) index: u32) -> Varyings {
26
+ let position = vec2f(f32((index & 1u) << 2u) - 1.0, f32((index & 2u) << 1u) - 1.0);
27
+ var result: Varyings;
28
+ result.position = vec4f(position, 0.0, 1.0);
29
+ result.uv = vec2f(position.x + 1.0, 1.0 - position.y) * 0.5;
30
+ return result;
31
+ }
32
+
33
+ const CB = vec3f(-0.100644, -0.338572, 0.439216);
34
+ const CR = vec3f(0.439216, -0.398942, -0.040274);
35
+ const CHROMA_OFFSET = 0.501961;
36
+ const LUMA = vec3f(0.2126, 0.7152, 0.0722);
37
+ const CHROMA_SCALE = vec2f(1.8556, 1.5748);
38
+
39
+ fn channelToLinear(c: f32) -> f32 {
40
+ return select(pow((c + 0.055) / 1.055, 2.4), c / 12.92, c <= 0.04045);
41
+ }
42
+
43
+ fn channelToEncoded(c: f32) -> f32 {
44
+ return select(1.055 * pow(c, 1.0 / 2.4) - 0.055, 12.92 * c, c <= 0.0031308);
45
+ }
46
+
47
+ fn toLinear(c: vec3f) -> vec3f {
48
+ return vec3f(channelToLinear(c.r), channelToLinear(c.g), channelToLinear(c.b));
49
+ }
50
+
51
+ fn toEncoded(c: vec3f) -> vec3f {
52
+ return vec3f(channelToEncoded(c.r), channelToEncoded(c.g), channelToEncoded(c.b));
53
+ }
54
+
55
+ fn pixelAt(at: vec2i, size: vec2i) -> vec4f {
56
+ let texel = textureLoad(image, clamp(at, vec2i(0), size - 1), 0);
57
+ return vec4f(toLinear(texel.rgb) * texel.a, texel.a);
58
+ }
59
+
60
+ fn sampleAt(q: vec2f, size: vec2i) -> vec4f {
61
+ let f = q * vec2f(size) - 0.5;
62
+ let corner = floor(f);
63
+ let w = f - corner;
64
+ let i = vec2i(corner);
65
+ let above = mix(pixelAt(i, size), pixelAt(i + vec2i(1, 0), size), w.x);
66
+ let below = mix(pixelAt(i + vec2i(0, 1), size), pixelAt(i + vec2i(1, 1), size), w.x);
67
+ return mix(above, below, w.y);
68
+ }
69
+
70
+ fn keyDistance(encoded: vec3f) -> f32 {
71
+ return distance(vec2f(dot(encoded, CB), dot(encoded, CR)) + CHROMA_OFFSET, settings.key.xy);
72
+ }
73
+
74
+ fn chromaOf(encoded: vec3f) -> vec2f {
75
+ let y = dot(encoded, LUMA);
76
+ return vec2f(encoded.b - y, encoded.r - y) / CHROMA_SCALE;
77
+ }
78
+
79
+ fn fromLumaChroma(y: f32, c: vec2f) -> vec3f {
80
+ let r = y + c.y * CHROMA_SCALE.y;
81
+ let b = y + c.x * CHROMA_SCALE.x;
82
+ return vec3f(r, (y - LUMA.r * r - LUMA.b * b) / LUMA.g, b);
83
+ }
84
+
85
+ fn cutSide(a: vec2f, b: vec2f, p: vec2f, size: vec2f) -> f32 {
86
+ let side = (b - a) * size;
87
+ let len = length(side);
88
+ if (len == 0.0) {
89
+ return 1.0e6;
90
+ }
91
+ let q = (p - a) * size;
92
+ return (q.y * side.x - q.x * side.y) / len;
93
+ }
94
+
95
+ fn cutMask(p: vec2f, size: vec2f) -> f32 {
96
+ let c0 = settings.corners[0].xy;
97
+ let c1 = settings.corners[0].zw;
98
+ let c2 = settings.corners[1].xy;
99
+ let c3 = settings.corners[1].zw;
100
+ let d = min(
101
+ min(cutSide(c0, c1, p, size), cutSide(c1, c2, p, size)),
102
+ min(cutSide(c2, c3, p, size), cutSide(c3, c0, p, size))
103
+ );
104
+ return clamp(d + 0.5, 0.0, 1.0);
105
+ }
106
+
107
+ @fragment
108
+ fn fragmentMain(input: Varyings) -> @location(0) vec4f {
109
+ // Taken first, while every pixel takes the same way, as a derivative must be
110
+ let size = 1.0 / max(fwidth(input.uv), vec2f(1e-6));
111
+ let key = settings.key;
112
+ let matte = settings.matte;
113
+ let correction = settings.correction;
114
+ let extras = settings.extras;
115
+
116
+ let p = vec2f(select(input.uv.x, 1.0 - input.uv.x, settings.flags.x > 0.5), input.uv.y);
117
+ let pixels = vec2i(textureDimensions(image));
118
+
119
+ let sampled = sampleAt(p, pixels);
120
+ var alpha = sampled.a;
121
+ var rgb = select(vec3f(0.0), sampled.rgb / alpha, alpha > 0.0);
122
+
123
+ // OBS's filter
124
+ if (key.w > 0.5) {
125
+ let texel = 1.0 / vec2f(pixels);
126
+ let across = vec2f(texel.x, 0.5 * texel.y);
127
+ let up = vec2f(0.5 * texel.x, -texel.y);
128
+ let around = keyDistance(toEncoded(sampleAt(p - across, pixels).rgb))
129
+ + keyDistance(toEncoded(sampleAt(p + across, pixels).rgb))
130
+ + keyDistance(toEncoded(sampleAt(p - up, pixels).rgb))
131
+ + keyDistance(toEncoded(sampleAt(p + up, pixels).rgb));
132
+ let beyond = (2.0 * around + keyDistance(toEncoded(rgb))) / 9.0 - matte.x;
133
+ alpha *= matte.w;
134
+ alpha *= pow(clamp(beyond / matte.y, 0.0, 1.0), 1.5);
135
+ let kept = pow(clamp(beyond / matte.z, 0.0, 1.0), 1.5);
136
+ if (settings.switches.y > 0.5) {
137
+ // The add-on, hueSpill
138
+ let encoded = toEncoded(rgb);
139
+ let toward = (key.xy - CHROMA_OFFSET) * key.z;
140
+ var c = chromaOf(encoded);
141
+ c -= toward * max(dot(c, toward), 0.0) * (1.0 - kept);
142
+ rgb = toLinear(clamp(fromLumaChroma(dot(encoded, LUMA), c), vec3f(0.0), vec3f(1.0)));
143
+ } else {
144
+ rgb = mix(vec3f(dot(rgb, LUMA)), rgb, kept);
145
+ }
146
+ rgb = pow(max(rgb, vec3f(0.0)), vec3f(correction.x)) * correction.y + correction.z;
147
+ }
148
+ rgb = clamp(rgb, vec3f(0.0), vec3f(1.0));
149
+
150
+ // The add-ons
151
+ rgb *= extras.rgb;
152
+ rgb = clamp(mix(vec3f(dot(rgb, LUMA)), rgb, extras.w), vec3f(0.0), vec3f(1.0));
153
+ if (settings.switches.x > 0.5) {
154
+ alpha *= cutMask(p, size);
155
+ }
156
+
157
+ return vec4f(toEncoded(rgb) * alpha, alpha);
158
+ }
159
+ `
package/src/source.ts ADDED
@@ -0,0 +1,105 @@
1
+ import type { Source } from './types.ts'
2
+
3
+ // How a keyer tells when its source has a new picture to take. A video's frames are watched for. An image is new when
4
+ // its source changes. A canvas can be drawn on at any time with nothing to say so, so it's taken every time. The rest
5
+ // never change. Each kind is told apart by what it has rather than its class, so an element from another window, as a
6
+ // pop-out's would be, is still known
7
+
8
+ // How long a video goes without a frame callback before its clock is watched instead: callbacks come only while the
9
+ // page is drawing, and some browsers have none
10
+ const STALLED = 100
11
+
12
+ // The source's own size in pixels
13
+ export function sizeOf(source: Source): [number, number] {
14
+ if ('videoWidth' in source) return [source.videoWidth, source.videoHeight]
15
+ if ('naturalWidth' in source) return [source.naturalWidth, source.naturalHeight]
16
+ if ('displayWidth' in source) return [source.displayWidth, source.displayHeight]
17
+ return [source.width, source.height]
18
+ }
19
+
20
+ // What a keyer given no onError does with a picture it can't key
21
+ export function warnUnkeyable(error: unknown): void {
22
+ console.warn('Greenscreen: the picture could not be keyed', error)
23
+ }
24
+
25
+ // One source as a keyer follows it: whether it has a picture yet, and whether it has a new one since it was last taken
26
+ export class Feed {
27
+ readonly source: Source
28
+ #fresh = true
29
+ #time = -1
30
+ #src = ''
31
+ #handle = 0
32
+ #lastFrame = -Infinity
33
+ // The picture that last failed, by its src where it has one, so each failure is reported once. Null once one is taken
34
+ #failedOn: string | null = null
35
+ #disposed = false
36
+
37
+ constructor(source: Source) {
38
+ this.source = source
39
+ this.#watch()
40
+ }
41
+
42
+ #watch() {
43
+ const video = this.source
44
+ if (!('requestVideoFrameCallback' in video) || this.#disposed) return
45
+ this.#handle = video.requestVideoFrameCallback(() => {
46
+ this.#fresh = true
47
+ this.#lastFrame = performance.now()
48
+ this.#watch()
49
+ })
50
+ }
51
+
52
+ // Whether there's a picture to take: a video with a frame, an image that has loaded, anything else with a size
53
+ hasPicture(): boolean {
54
+ const source = this.source
55
+ if ('videoWidth' in source) return source.readyState >= 2 && source.videoWidth > 0 && source.videoHeight > 0
56
+ if ('naturalWidth' in source) return source.complete && source.naturalWidth > 0
57
+ const [width, height] = sizeOf(source)
58
+ return width > 0 && height > 0
59
+ }
60
+
61
+ // Whether the picture has changed since it was last taken
62
+ changed(): boolean {
63
+ if (this.#fresh) return true
64
+ const source = this.source
65
+ if ('videoWidth' in source) return performance.now() - this.#lastFrame > STALLED && source.currentTime !== this.#time
66
+ if ('naturalWidth' in source) return source.currentSrc !== this.#src
67
+ return 'getContext' in source
68
+ }
69
+
70
+ size(): [number, number] {
71
+ return sizeOf(this.source)
72
+ }
73
+
74
+ // The picture has been taken
75
+ taken() {
76
+ this.#fresh = false
77
+ this.#failedOn = null
78
+ const source = this.source
79
+ if ('videoWidth' in source) this.#time = source.currentTime
80
+ else if ('naturalWidth' in source) this.#src = source.currentSrc
81
+ }
82
+
83
+ // Taking the picture failed, as it does for one larger than the GPU's textures, or from another site without CORS.
84
+ // It's tried again only when the picture changes. Returns whether this failure is new: the first since a picture was
85
+ // last taken, or of an image or video since given another src, so each is reported once however often it's tried
86
+ failed(): boolean {
87
+ const source = this.source
88
+ const what = 'currentSrc' in source ? source.currentSrc : ''
89
+ const first = this.#failedOn !== what
90
+ this.taken()
91
+ this.#failedOn = what
92
+ return first
93
+ }
94
+
95
+ // Take the picture again on the next render, as its pixels have changed
96
+ invalidate() {
97
+ this.#fresh = true
98
+ }
99
+
100
+ dispose() {
101
+ this.#disposed = true
102
+ const video = this.source
103
+ if (this.#handle && 'cancelVideoFrameCallback' in video) video.cancelVideoFrameCallback(this.#handle)
104
+ }
105
+ }