@ume-group/contracts 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +37 -0
- package/dist/adserving.d.ts +150 -0
- package/dist/adserving.d.ts.map +1 -0
- package/dist/adserving.js +8 -0
- package/dist/campaigns.d.ts +37 -0
- package/dist/campaigns.d.ts.map +1 -0
- package/dist/campaigns.js +8 -0
- package/dist/gausst.d.ts +236 -0
- package/dist/gausst.d.ts.map +1 -0
- package/dist/gausst.js +307 -0
- package/dist/gausst.test.d.ts +2 -0
- package/dist/gausst.test.d.ts.map +1 -0
- package/dist/gausst.test.js +71 -0
- package/dist/index.d.ts +1531 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +1112 -0
- package/dist/layer2/index.d.ts +9 -0
- package/dist/layer2/index.d.ts.map +1 -0
- package/dist/layer2/index.js +10 -0
- package/dist/layer2/shaders.d.ts +185 -0
- package/dist/layer2/shaders.d.ts.map +1 -0
- package/dist/layer2/shaders.js +604 -0
- package/dist/layer2/webcam-utils.d.ts +113 -0
- package/dist/layer2/webcam-utils.d.ts.map +1 -0
- package/dist/layer2/webcam-utils.js +147 -0
- package/dist/layer2/webcam-utils.test.d.ts +2 -0
- package/dist/layer2/webcam-utils.test.d.ts.map +1 -0
- package/dist/layer2/webcam-utils.test.js +18 -0
- package/dist/layer2.d.ts +558 -0
- package/dist/layer2.d.ts.map +1 -0
- package/dist/layer2.js +376 -0
- package/dist/layer2.test.d.ts +2 -0
- package/dist/layer2.test.d.ts.map +1 -0
- package/dist/layer2.test.js +65 -0
- package/dist/perspective.d.ts +28 -0
- package/dist/perspective.d.ts.map +1 -0
- package/dist/perspective.js +157 -0
- package/dist/segmentation/MediaPipeSegmenter.d.ts +201 -0
- package/dist/segmentation/MediaPipeSegmenter.d.ts.map +1 -0
- package/dist/segmentation/MediaPipeSegmenter.js +434 -0
- package/dist/segmentation/index.d.ts +5 -0
- package/dist/segmentation/index.d.ts.map +1 -0
- package/dist/segmentation/index.js +4 -0
- package/dist/webcam/GarbageMatteDragManager.d.ts +63 -0
- package/dist/webcam/GarbageMatteDragManager.d.ts.map +1 -0
- package/dist/webcam/GarbageMatteDragManager.js +183 -0
- package/dist/webcam/WebcamStreamManager.d.ts +103 -0
- package/dist/webcam/WebcamStreamManager.d.ts.map +1 -0
- package/dist/webcam/WebcamStreamManager.js +356 -0
- package/dist/webcam/index.d.ts +5 -0
- package/dist/webcam/index.d.ts.map +1 -0
- package/dist/webcam/index.js +2 -0
- package/openapi/admetise.yaml +632 -0
- package/openapi/includu.yaml +621 -0
- package/openapi/integration.yaml +372 -0
- package/openapi/shared/schemas.yaml +227 -0
- package/package.json +53 -0
|
@@ -0,0 +1,604 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared WebGL shaders for Layer 2 webcam compositing
|
|
3
|
+
*
|
|
4
|
+
* These shaders are used by both editor and player for consistent rendering.
|
|
5
|
+
* The fragment shader supports multiple modes:
|
|
6
|
+
* - AI segmentation (MediaPipe)
|
|
7
|
+
* - Chroma key (green/blue/custom with HSV window option)
|
|
8
|
+
* - Procedural silhouette (fallback/placeholder)
|
|
9
|
+
* - Full view mode (no masking)
|
|
10
|
+
*
|
|
11
|
+
* ArchiMate: Application Component (Shared Rendering Logic)
|
|
12
|
+
*/
|
|
13
|
+
/**
|
|
14
|
+
* Simple vertex shader for webcam plane rendering
|
|
15
|
+
*/
|
|
16
|
+
export const webcamVertexShader = `
|
|
17
|
+
varying vec2 vUv;
|
|
18
|
+
void main() {
|
|
19
|
+
vUv = uv;
|
|
20
|
+
gl_Position = projectionMatrix * modelViewMatrix * vec4(position, 1.0);
|
|
21
|
+
}
|
|
22
|
+
`;
|
|
23
|
+
/**
|
|
24
|
+
* Fragment shader for webcam compositing with background removal
|
|
25
|
+
*
|
|
26
|
+
* Supports:
|
|
27
|
+
* - AI segmentation mask
|
|
28
|
+
* - Chroma key with HSV or YCbCr algorithms
|
|
29
|
+
* - Garbage matte (rectangle crop with feathering)
|
|
30
|
+
* - Color correction (brightness, contrast, saturation, temperature)
|
|
31
|
+
* - Pre-multiplied alpha for correct edge blending
|
|
32
|
+
*
|
|
33
|
+
* Uniforms:
|
|
34
|
+
* - webcamTexture: The live webcam VideoTexture
|
|
35
|
+
* - segmentationMask: AI-generated mask (white = person)
|
|
36
|
+
* - hasTexture: Whether webcam texture is available
|
|
37
|
+
* - useAIMask: Enable AI segmentation mode
|
|
38
|
+
* - maskThreshold: AI mask threshold (0.0 - 1.0)
|
|
39
|
+
* - aspectRatio: Plane aspect ratio for silhouette proportions (always >= 1.0)
|
|
40
|
+
* - videoAspectRatio: Actual video W/H (may be < 1.0 for portrait), used for cover-crop UV
|
|
41
|
+
* - useChromaKey: Enable chroma key mode
|
|
42
|
+
* - chromaKeyColor: Key color in RGB (0.0 - 1.0)
|
|
43
|
+
* - chromaKeyTolerance: Color tolerance (0.0 - 1.0)
|
|
44
|
+
* - chromaKeySpill: Spill suppression amount (0.0 - 1.0)
|
|
45
|
+
* - chromaKeyEdge: Edge softness (0.0 - 1.0)
|
|
46
|
+
* - setupMode: Show silhouette overlay for positioning (editor only)
|
|
47
|
+
* - useHsvWindow: Enable HSV window mode for advanced keying
|
|
48
|
+
* - hsvMinHue/hsvMaxHue: Hue range (0-360)
|
|
49
|
+
* - hsvMinSat/hsvMaxSat: Saturation range (0-1)
|
|
50
|
+
* - hsvMinVal/hsvMaxVal: Value range (0-1)
|
|
51
|
+
* - useGarbageMatte: Enable garbage matte cropping
|
|
52
|
+
* - garbageMatteIsRect: True = rectangle, false = polygon texture
|
|
53
|
+
* - garbageMatteRect: Rectangle bounds (x1, y1, x2, y2)
|
|
54
|
+
* - garbageMatteInvert: Invert mask
|
|
55
|
+
* - garbageMatteFeather: Edge softness in UV units
|
|
56
|
+
* - garbageMatteFeatherEdges: Per-edge feather (top, right, bottom, left)
|
|
57
|
+
* - usePerEdgeFeather: Enable per-edge feathering
|
|
58
|
+
* - ccBrightness/ccContrast/ccSaturation/ccTemperature: Color correction
|
|
59
|
+
* - fullViewMode: Show entire webcam without masking
|
|
60
|
+
* - darkOpacity: Opacity for areas outside silhouette (editor preview)
|
|
61
|
+
* - placeholderColor/placeholderOpacity: Editor placeholder when no webcam
|
|
62
|
+
*/
|
|
63
|
+
export const webcamFragmentShader = `
|
|
64
|
+
uniform sampler2D webcamTexture;
|
|
65
|
+
uniform sampler2D segmentationMask;
|
|
66
|
+
uniform float darkOpacity;
|
|
67
|
+
uniform bool hasTexture;
|
|
68
|
+
uniform bool useAIMask;
|
|
69
|
+
uniform float maskThreshold;
|
|
70
|
+
uniform vec3 placeholderColor;
|
|
71
|
+
uniform float placeholderOpacity;
|
|
72
|
+
uniform float aspectRatio;
|
|
73
|
+
uniform float videoAspectRatio;
|
|
74
|
+
// Chroma key uniforms
|
|
75
|
+
uniform bool useChromaKey;
|
|
76
|
+
uniform vec3 chromaKeyColor;
|
|
77
|
+
uniform float chromaKeyTolerance;
|
|
78
|
+
uniform float chromaKeySpill;
|
|
79
|
+
uniform float chromaKeyEdge;
|
|
80
|
+
uniform bool setupMode;
|
|
81
|
+
// HSV window uniforms
|
|
82
|
+
uniform bool useHsvWindow;
|
|
83
|
+
uniform float hsvMinHue;
|
|
84
|
+
uniform float hsvMaxHue;
|
|
85
|
+
uniform float hsvMinSat;
|
|
86
|
+
uniform float hsvMaxSat;
|
|
87
|
+
uniform float hsvMinVal;
|
|
88
|
+
uniform float hsvMaxVal;
|
|
89
|
+
// Garbage matte uniforms
|
|
90
|
+
uniform bool useGarbageMatte;
|
|
91
|
+
uniform bool garbageMatteIsRect;
|
|
92
|
+
uniform vec4 garbageMatteRect;
|
|
93
|
+
uniform sampler2D garbageMatteMask;
|
|
94
|
+
uniform bool garbageMatteInvert;
|
|
95
|
+
uniform float garbageMatteFeather;
|
|
96
|
+
uniform vec4 garbageMatteFeatherEdges;
|
|
97
|
+
uniform bool usePerEdgeFeather;
|
|
98
|
+
// Color correction uniforms
|
|
99
|
+
uniform float ccBrightness;
|
|
100
|
+
uniform float ccContrast;
|
|
101
|
+
uniform float ccSaturation;
|
|
102
|
+
uniform float ccTemperature;
|
|
103
|
+
// Full view mode
|
|
104
|
+
uniform bool fullViewMode;
|
|
105
|
+
// Debug: visualize the AI mask as a color overlay
|
|
106
|
+
uniform bool debugShowMask;
|
|
107
|
+
// Silhouette scale for generous capture (participantHeight / captureHeight)
|
|
108
|
+
uniform float silhouetteScale;
|
|
109
|
+
|
|
110
|
+
varying vec2 vUv;
|
|
111
|
+
|
|
112
|
+
// Check if point is inside an ellipse (for head)
|
|
113
|
+
bool inEllipse(vec2 p, vec2 center, vec2 radius) {
|
|
114
|
+
vec2 d = (p - center) / radius;
|
|
115
|
+
return dot(d, d) <= 1.0;
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
// Check if point is inside a rectangle
|
|
119
|
+
bool inRect(vec2 p, vec2 center, vec2 halfSize) {
|
|
120
|
+
vec2 d = abs(p - center);
|
|
121
|
+
return d.x <= halfSize.x && d.y <= halfSize.y;
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
// Convert RGB to HSV (Hue 0-6, Saturation 0-1, Value 0-1)
|
|
125
|
+
// Based on legacy Gausst chroma key implementation
|
|
126
|
+
vec3 rgb2hsv(vec3 c) {
|
|
127
|
+
float minVal = min(min(c.r, c.g), c.b);
|
|
128
|
+
float maxVal = max(max(c.r, c.g), c.b);
|
|
129
|
+
float delta = maxVal - minVal;
|
|
130
|
+
|
|
131
|
+
float h = 0.0;
|
|
132
|
+
float s = (maxVal > 0.0) ? delta / maxVal : 0.0;
|
|
133
|
+
float v = maxVal;
|
|
134
|
+
|
|
135
|
+
if (delta > 0.001) {
|
|
136
|
+
if (c.r == maxVal) {
|
|
137
|
+
if (minVal == c.g) {
|
|
138
|
+
h = 5.0 + (1.0 - c.b / c.r);
|
|
139
|
+
} else {
|
|
140
|
+
h = c.g / c.r;
|
|
141
|
+
}
|
|
142
|
+
} else if (c.g == maxVal) {
|
|
143
|
+
if (minVal == c.r) {
|
|
144
|
+
h = 2.0 + (1.0 - c.b / c.g);
|
|
145
|
+
} else {
|
|
146
|
+
h = 1.0 + c.r / c.g;
|
|
147
|
+
}
|
|
148
|
+
} else {
|
|
149
|
+
if (minVal == c.r) {
|
|
150
|
+
h = 3.0 + (1.0 - c.g / c.b);
|
|
151
|
+
} else {
|
|
152
|
+
h = 4.0 + c.r / c.b;
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
return vec3(h, s, v);
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
// Convert RGB to HSV with 0-360 degree hue for Gausst window mode
|
|
161
|
+
vec3 rgb2hsv360(vec3 c) {
|
|
162
|
+
vec3 hsv = rgb2hsv(c);
|
|
163
|
+
hsv.x = hsv.x * 60.0;
|
|
164
|
+
return hsv;
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
// Garbage matte mask function
|
|
168
|
+
// Returns 1.0 = keep pixel, 0.0 = discard pixel
|
|
169
|
+
float getGarbageMatteMask(vec2 uv) {
|
|
170
|
+
if (!useGarbageMatte) return 1.0;
|
|
171
|
+
|
|
172
|
+
float mask;
|
|
173
|
+
if (garbageMatteIsRect) {
|
|
174
|
+
float x1 = garbageMatteRect.x;
|
|
175
|
+
float y1 = garbageMatteRect.y;
|
|
176
|
+
float x2 = garbageMatteRect.z;
|
|
177
|
+
float y2 = garbageMatteRect.w;
|
|
178
|
+
|
|
179
|
+
bool isFullFrame = x1 < 0.01 && y1 < 0.01 && x2 > 0.99 && y2 > 0.99;
|
|
180
|
+
|
|
181
|
+
bool hasFeather = usePerEdgeFeather
|
|
182
|
+
? (garbageMatteFeatherEdges.x > 0.0 || garbageMatteFeatherEdges.y > 0.0 ||
|
|
183
|
+
garbageMatteFeatherEdges.z > 0.0 || garbageMatteFeatherEdges.w > 0.0)
|
|
184
|
+
: (garbageMatteFeather > 0.0);
|
|
185
|
+
|
|
186
|
+
if (hasFeather && !isFullFrame) {
|
|
187
|
+
float dLeft = x1 - uv.x;
|
|
188
|
+
float dRight = uv.x - x2;
|
|
189
|
+
float dTop = y1 - uv.y;
|
|
190
|
+
float dBottom = uv.y - y2;
|
|
191
|
+
|
|
192
|
+
float featherTop = usePerEdgeFeather ? garbageMatteFeatherEdges.x : garbageMatteFeather;
|
|
193
|
+
float featherRight = usePerEdgeFeather ? garbageMatteFeatherEdges.y : garbageMatteFeather;
|
|
194
|
+
float featherBottom = usePerEdgeFeather ? garbageMatteFeatherEdges.z : garbageMatteFeather;
|
|
195
|
+
float featherLeft = usePerEdgeFeather ? garbageMatteFeatherEdges.w : garbageMatteFeather;
|
|
196
|
+
|
|
197
|
+
float maskTop = featherTop > 0.0 ? 1.0 - smoothstep(-featherTop, featherTop, dTop) : (dTop < 0.0 ? 1.0 : 0.0);
|
|
198
|
+
float maskRight = featherRight > 0.0 ? 1.0 - smoothstep(-featherRight, featherRight, dRight) : (dRight < 0.0 ? 1.0 : 0.0);
|
|
199
|
+
float maskBottom = featherBottom > 0.0 ? 1.0 - smoothstep(-featherBottom, featherBottom, dBottom) : (dBottom < 0.0 ? 1.0 : 0.0);
|
|
200
|
+
float maskLeft = featherLeft > 0.0 ? 1.0 - smoothstep(-featherLeft, featherLeft, dLeft) : (dLeft < 0.0 ? 1.0 : 0.0);
|
|
201
|
+
|
|
202
|
+
// Use min() to keep edge feathering independent without corner rounding
|
|
203
|
+
// (multiplication would cause 0.5 * 0.5 = 0.25 at corners, creating rounded effect)
|
|
204
|
+
mask = min(min(maskTop, maskBottom), min(maskLeft, maskRight));
|
|
205
|
+
} else {
|
|
206
|
+
bool inside = uv.x >= x1 && uv.x <= x2 && uv.y >= y1 && uv.y <= y2;
|
|
207
|
+
mask = inside ? 1.0 : 0.0;
|
|
208
|
+
}
|
|
209
|
+
} else {
|
|
210
|
+
mask = texture2D(garbageMatteMask, uv).r;
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
return garbageMatteInvert ? 1.0 - mask : mask;
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
// HSV window-based chroma key (Gausst algorithm)
|
|
217
|
+
float hsvWindowMask(vec3 color, float minH, float maxH, float minS, float maxS, float minV, float maxV, float edgeSoftness) {
|
|
218
|
+
vec3 hsv = rgb2hsv360(color);
|
|
219
|
+
|
|
220
|
+
bool hueInRange;
|
|
221
|
+
if (minH <= maxH) {
|
|
222
|
+
hueInRange = hsv.x >= minH && hsv.x <= maxH;
|
|
223
|
+
} else {
|
|
224
|
+
hueInRange = hsv.x >= minH || hsv.x <= maxH;
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
bool satInRange = hsv.y >= minS && hsv.y <= maxS;
|
|
228
|
+
bool valInRange = hsv.z >= minV && hsv.z <= maxV;
|
|
229
|
+
|
|
230
|
+
if (hueInRange && satInRange && valInRange) {
|
|
231
|
+
float hueCenter = (minH + maxH) / 2.0;
|
|
232
|
+
float hueDist = abs(hsv.x - hueCenter) / ((maxH - minH) / 2.0 + 0.001);
|
|
233
|
+
float satCenter = (minS + maxS) / 2.0;
|
|
234
|
+
float satDist = abs(hsv.y - satCenter) / ((maxS - minS) / 2.0 + 0.001);
|
|
235
|
+
float valCenter = (minV + maxV) / 2.0;
|
|
236
|
+
float valDist = abs(hsv.z - valCenter) / ((maxV - minV) / 2.0 + 0.001);
|
|
237
|
+
|
|
238
|
+
float dist = max(max(hueDist, satDist), valDist);
|
|
239
|
+
return smoothstep(1.0 - edgeSoftness * 2.0, 1.0, dist);
|
|
240
|
+
} else {
|
|
241
|
+
return 1.0;
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
// Chroma key mask calculation
|
|
246
|
+
// HSV window acts as an additional refinement filter on top of the normal
|
|
247
|
+
// chroma key. With full range (0-360, 0-1, 0-1), result is identical to
|
|
248
|
+
// normal chroma key. Narrowing the HSV window restricts which pixels get keyed.
|
|
249
|
+
float chromaKeyMask(vec3 color, vec3 keyColor, float tolerance, float edgeSoftness) {
|
|
250
|
+
bool isGreenScreen = keyColor.g > 0.5 && keyColor.g > keyColor.r * 1.5 && keyColor.g > keyColor.b * 1.5;
|
|
251
|
+
bool isBlueScreen = keyColor.b > 0.5 && keyColor.b > keyColor.r * 1.5 && keyColor.b > keyColor.g * 1.5;
|
|
252
|
+
|
|
253
|
+
float normalMask;
|
|
254
|
+
|
|
255
|
+
if (isGreenScreen || isBlueScreen) {
|
|
256
|
+
float Y = 0.299 * color.r + 0.587 * color.g + 0.114 * color.b;
|
|
257
|
+
float Cb = 0.564 * (color.b - Y);
|
|
258
|
+
float Cr = 0.713 * (color.r - Y);
|
|
259
|
+
|
|
260
|
+
float keyY = 0.299 * keyColor.r + 0.587 * keyColor.g + 0.114 * keyColor.b;
|
|
261
|
+
float keyCb = 0.564 * (keyColor.b - keyY);
|
|
262
|
+
float keyCr = 0.713 * (keyColor.r - keyY);
|
|
263
|
+
|
|
264
|
+
float dist = distance(vec2(Cb, Cr), vec2(keyCb, keyCr));
|
|
265
|
+
|
|
266
|
+
float scaledTolerance = tolerance * 0.5;
|
|
267
|
+
float softRange = edgeSoftness * 0.3;
|
|
268
|
+
normalMask = smoothstep(scaledTolerance - softRange, scaledTolerance + softRange, dist);
|
|
269
|
+
} else {
|
|
270
|
+
vec3 colorHSV = rgb2hsv(color);
|
|
271
|
+
vec3 keyHSV = rgb2hsv(keyColor);
|
|
272
|
+
|
|
273
|
+
float hueDiff = abs(colorHSV.x - keyHSV.x);
|
|
274
|
+
if (hueDiff > 3.0) hueDiff = 6.0 - hueDiff;
|
|
275
|
+
hueDiff /= 3.0;
|
|
276
|
+
|
|
277
|
+
float satDiff = abs(colorHSV.y - keyHSV.y);
|
|
278
|
+
float valDiff = abs(colorHSV.z - keyHSV.z);
|
|
279
|
+
|
|
280
|
+
float dist = hueDiff * 0.7 + satDiff * 0.2 + valDiff * 0.1;
|
|
281
|
+
|
|
282
|
+
if (colorHSV.y < 0.15 && keyHSV.y < 0.15) {
|
|
283
|
+
dist = valDiff * 0.8 + satDiff * 0.2;
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
float scaledTolerance = tolerance * 0.4;
|
|
287
|
+
float softRange = edgeSoftness * 0.2;
|
|
288
|
+
normalMask = smoothstep(scaledTolerance - softRange, scaledTolerance + softRange, dist);
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
if (useHsvWindow) {
|
|
292
|
+
// HSV window restricts keying: pixels outside the window stay opaque
|
|
293
|
+
float hsvMask = hsvWindowMask(color, hsvMinHue, hsvMaxHue, hsvMinSat, hsvMaxSat, hsvMinVal, hsvMaxVal, edgeSoftness);
|
|
294
|
+
return max(normalMask, hsvMask);
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
return normalMask;
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
// Spill suppression - works with any key color
|
|
301
|
+
vec3 suppressSpill(vec3 color, vec3 keyColor, float spillAmount) {
|
|
302
|
+
if (spillAmount <= 0.0) return color;
|
|
303
|
+
|
|
304
|
+
float keyMax = max(keyColor.r, max(keyColor.g, keyColor.b));
|
|
305
|
+
if (keyMax < 0.01) return color;
|
|
306
|
+
|
|
307
|
+
vec3 keyDir = keyColor / keyMax;
|
|
308
|
+
float colorDot = dot(color, keyDir);
|
|
309
|
+
float keyDot = dot(keyDir, keyDir);
|
|
310
|
+
vec3 neutral = color - keyDir * max(0.0, (colorDot / keyDot - length(color) * 0.3));
|
|
311
|
+
float keyInfluence = dot(normalize(color + 0.001), normalize(keyColor + 0.001));
|
|
312
|
+
keyInfluence = max(0.0, keyInfluence - 0.5) * 2.0;
|
|
313
|
+
|
|
314
|
+
return mix(color, neutral, spillAmount * keyInfluence);
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
// Color correction
|
|
318
|
+
vec3 applyColorCorrection(vec3 color) {
|
|
319
|
+
if (ccBrightness == 0.0 && ccContrast == 0.0 && ccSaturation == 0.0 && ccTemperature == 0.0) {
|
|
320
|
+
return color;
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
vec3 result = color;
|
|
324
|
+
|
|
325
|
+
// Brightness
|
|
326
|
+
result += ccBrightness * 0.5;
|
|
327
|
+
|
|
328
|
+
// Contrast
|
|
329
|
+
float contrastFactor = 1.0 + ccContrast;
|
|
330
|
+
result = (result - 0.5) * contrastFactor + 0.5;
|
|
331
|
+
|
|
332
|
+
// Saturation
|
|
333
|
+
float luminance = dot(result, vec3(0.299, 0.587, 0.114));
|
|
334
|
+
vec3 grayscale = vec3(luminance);
|
|
335
|
+
float satFactor = 1.0 + ccSaturation;
|
|
336
|
+
result = mix(grayscale, result, satFactor);
|
|
337
|
+
|
|
338
|
+
// Temperature
|
|
339
|
+
if (ccTemperature != 0.0) {
|
|
340
|
+
float temp = ccTemperature * 0.2;
|
|
341
|
+
result.r += temp;
|
|
342
|
+
result.g += temp * 0.5;
|
|
343
|
+
result.b -= temp;
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
return clamp(result, 0.0, 1.0);
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
// Procedural silhouette check
|
|
350
|
+
bool inSilhouette(vec2 uv, float aspectRatioFactor) {
|
|
351
|
+
float fullPersonHeight = 0.85;
|
|
352
|
+
float personHeight = fullPersonHeight * silhouetteScale;
|
|
353
|
+
float bottomMargin = (1.0 - fullPersonHeight) / 2.0;
|
|
354
|
+
|
|
355
|
+
float feetY = bottomMargin;
|
|
356
|
+
float legHeight = personHeight * 0.45;
|
|
357
|
+
float hipY = feetY + legHeight;
|
|
358
|
+
float torsoHeight = personHeight * 0.35;
|
|
359
|
+
float shoulderY = hipY + torsoHeight;
|
|
360
|
+
float neckLength = personHeight * 0.03;
|
|
361
|
+
float neckY = shoulderY + neckLength;
|
|
362
|
+
float headRadius = personHeight * 0.09;
|
|
363
|
+
float headY = neckY + headRadius;
|
|
364
|
+
|
|
365
|
+
float torsoWidth = personHeight * 0.22 * aspectRatioFactor;
|
|
366
|
+
float legWidth = personHeight * 0.08 * aspectRatioFactor;
|
|
367
|
+
float legGap = personHeight * 0.02 * aspectRatioFactor;
|
|
368
|
+
float armWidth = personHeight * 0.05 * aspectRatioFactor;
|
|
369
|
+
float armLength = personHeight * 0.35;
|
|
370
|
+
|
|
371
|
+
float cx = 0.5;
|
|
372
|
+
|
|
373
|
+
float headRadiusX = headRadius * aspectRatioFactor;
|
|
374
|
+
float headRadiusY = headRadius;
|
|
375
|
+
if (inEllipse(uv, vec2(cx, headY), vec2(headRadiusX, headRadiusY))) return true;
|
|
376
|
+
|
|
377
|
+
float neckWidth = headRadiusX * 0.5;
|
|
378
|
+
if (inRect(uv, vec2(cx, neckY - neckLength/2.0), vec2(neckWidth, neckLength/2.0))) return true;
|
|
379
|
+
|
|
380
|
+
if (inRect(uv, vec2(cx, hipY + torsoHeight/2.0), vec2(torsoWidth/2.0, torsoHeight/2.0))) return true;
|
|
381
|
+
|
|
382
|
+
float armX = cx - torsoWidth/2.0 - armWidth/2.0;
|
|
383
|
+
if (inRect(uv, vec2(armX, shoulderY - armLength/2.0), vec2(armWidth/2.0, armLength/2.0))) return true;
|
|
384
|
+
|
|
385
|
+
armX = cx + torsoWidth/2.0 + armWidth/2.0;
|
|
386
|
+
if (inRect(uv, vec2(armX, shoulderY - armLength/2.0), vec2(armWidth/2.0, armLength/2.0))) return true;
|
|
387
|
+
|
|
388
|
+
float legX = cx - legGap - legWidth/2.0;
|
|
389
|
+
if (inRect(uv, vec2(legX, feetY + legHeight/2.0), vec2(legWidth/2.0, legHeight/2.0))) return true;
|
|
390
|
+
|
|
391
|
+
legX = cx + legGap + legWidth/2.0;
|
|
392
|
+
if (inRect(uv, vec2(legX, feetY + legHeight/2.0), vec2(legWidth/2.0, legHeight/2.0))) return true;
|
|
393
|
+
|
|
394
|
+
return false;
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
// Cover-crop UV: maps plane UV to texture UV when video AR differs from plane AR.
|
|
398
|
+
// Acts like CSS object-fit:cover — fills the plane, crops excess, no stretching.
|
|
399
|
+
// When video and plane have the same AR, returns uv unchanged.
|
|
400
|
+
vec2 getCoverUV(vec2 uv) {
|
|
401
|
+
float planeAR = aspectRatio;
|
|
402
|
+
float vidAR = videoAspectRatio;
|
|
403
|
+
if (vidAR <= 0.0 || planeAR <= 0.0 || abs(vidAR - planeAR) < 0.01) {
|
|
404
|
+
return uv;
|
|
405
|
+
}
|
|
406
|
+
if (vidAR < planeAR) {
|
|
407
|
+
// Video is narrower/taller than plane — crop top/bottom
|
|
408
|
+
float scale = vidAR / planeAR;
|
|
409
|
+
return vec2(uv.x, uv.y * scale + (1.0 - scale) * 0.5);
|
|
410
|
+
} else {
|
|
411
|
+
// Video is wider than plane — crop sides
|
|
412
|
+
float scale = planeAR / vidAR;
|
|
413
|
+
return vec2(uv.x * scale + (1.0 - scale) * 0.5, uv.y);
|
|
414
|
+
}
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
void main() {
|
|
418
|
+
// Positioning mode: setup modal open AND neither AI nor chroma key active
|
|
419
|
+
// In this mode, show full silhouette guide WITHOUT garbage matte
|
|
420
|
+
// setupMode is set by component when webcam setup modal is open
|
|
421
|
+
bool isPositioningMode = setupMode && !useAIMask && !useChromaKey && !fullViewMode;
|
|
422
|
+
|
|
423
|
+
// Get garbage matte mask - skip in positioning mode to show full silhouette
|
|
424
|
+
float garbageMask = isPositioningMode ? 1.0 : getGarbageMatteMask(vUv);
|
|
425
|
+
|
|
426
|
+
if (garbageMask < 0.001) {
|
|
427
|
+
discard;
|
|
428
|
+
}
|
|
429
|
+
|
|
430
|
+
float aspectRatioFactor = 1.0 / aspectRatio;
|
|
431
|
+
|
|
432
|
+
// Cover-crop UV for texture sampling when video AR differs from plane AR
|
|
433
|
+
// (e.g., portrait video on landscape plane). Silhouette/matte use vUv (plane space).
|
|
434
|
+
vec2 texUV = getCoverUV(vUv);
|
|
435
|
+
vec4 texColor = texture2D(webcamTexture, texUV);
|
|
436
|
+
|
|
437
|
+
// Debug: visualize AI mask as color overlay (red=person, blue=background)
|
|
438
|
+
if (debugShowMask && useAIMask) {
|
|
439
|
+
float maskVal = texture2D(segmentationMask, texUV).r;
|
|
440
|
+
gl_FragColor = vec4(maskVal, 0.0, 1.0 - maskVal, 1.0);
|
|
441
|
+
return;
|
|
442
|
+
}
|
|
443
|
+
|
|
444
|
+
// Full view mode - MUST be checked FIRST before any processing
|
|
445
|
+
// Used during eyedropper color picking to show raw webcam
|
|
446
|
+
if (fullViewMode) {
|
|
447
|
+
float fvGarbageMask = getGarbageMatteMask(vUv);
|
|
448
|
+
if (fvGarbageMask < 0.5) {
|
|
449
|
+
discard;
|
|
450
|
+
}
|
|
451
|
+
if (hasTexture) {
|
|
452
|
+
gl_FragColor = vec4(texColor.rgb, 1.0);
|
|
453
|
+
} else {
|
|
454
|
+
gl_FragColor = vec4(placeholderColor, placeholderOpacity);
|
|
455
|
+
}
|
|
456
|
+
return;
|
|
457
|
+
}
|
|
458
|
+
|
|
459
|
+
// Chroma key mode
|
|
460
|
+
if (useChromaKey && hasTexture) {
|
|
461
|
+
float mask = chromaKeyMask(texColor.rgb, chromaKeyColor, chromaKeyTolerance, chromaKeyEdge);
|
|
462
|
+
|
|
463
|
+
bool insideSilhouette = inSilhouette(vUv, aspectRatioFactor);
|
|
464
|
+
|
|
465
|
+
if (setupMode) {
|
|
466
|
+
// Setup mode: show silhouette overlay for positioning
|
|
467
|
+
vec3 cleanColor = suppressSpill(texColor.rgb, chromaKeyColor, chromaKeySpill);
|
|
468
|
+
cleanColor = applyColorCorrection(cleanColor);
|
|
469
|
+
|
|
470
|
+
if (mask < 0.01) {
|
|
471
|
+
if (insideSilhouette) {
|
|
472
|
+
vec3 cyan = vec3(0.0, 0.8, 0.8);
|
|
473
|
+
gl_FragColor = vec4(cyan * 0.3, 0.3);
|
|
474
|
+
} else {
|
|
475
|
+
gl_FragColor = vec4(0.0, 0.0, 0.0, 0.4);
|
|
476
|
+
}
|
|
477
|
+
} else {
|
|
478
|
+
if (insideSilhouette) {
|
|
479
|
+
gl_FragColor = vec4(cleanColor, 1.0);
|
|
480
|
+
} else {
|
|
481
|
+
vec3 warningTint = mix(cleanColor, vec3(1.0, 0.5, 0.0), 0.2);
|
|
482
|
+
gl_FragColor = vec4(warningTint * mask, mask);
|
|
483
|
+
}
|
|
484
|
+
}
|
|
485
|
+
return;
|
|
486
|
+
}
|
|
487
|
+
|
|
488
|
+
// Normal chroma key compositing
|
|
489
|
+
if (mask < 0.01) {
|
|
490
|
+
discard;
|
|
491
|
+
} else if (mask > 0.99) {
|
|
492
|
+
vec3 cleanColor = suppressSpill(texColor.rgb, chromaKeyColor, chromaKeySpill);
|
|
493
|
+
cleanColor = applyColorCorrection(cleanColor);
|
|
494
|
+
float finalAlpha = garbageMask;
|
|
495
|
+
gl_FragColor = vec4(cleanColor * finalAlpha, finalAlpha);
|
|
496
|
+
} else {
|
|
497
|
+
vec3 cleanColor = suppressSpill(texColor.rgb, chromaKeyColor, chromaKeySpill);
|
|
498
|
+
cleanColor = applyColorCorrection(cleanColor);
|
|
499
|
+
float finalAlpha = mask * garbageMask;
|
|
500
|
+
gl_FragColor = vec4(cleanColor * finalAlpha, finalAlpha);
|
|
501
|
+
}
|
|
502
|
+
return;
|
|
503
|
+
}
|
|
504
|
+
|
|
505
|
+
// Apply color correction for non-chroma-key modes
|
|
506
|
+
texColor.rgb = applyColorCorrection(texColor.rgb);
|
|
507
|
+
|
|
508
|
+
// AI mask or silhouette fallback
|
|
509
|
+
bool inside;
|
|
510
|
+
|
|
511
|
+
if (useAIMask && hasTexture) {
|
|
512
|
+
float maskValue = texture2D(segmentationMask, texUV).r;
|
|
513
|
+
|
|
514
|
+
if (useHsvWindow) {
|
|
515
|
+
// HSV refines AI mask: background-colored pixels get reduced confidence
|
|
516
|
+
// hsvMask: 0 = matches background color, 1 = doesn't match
|
|
517
|
+
float hsvMask = hsvWindowMask(texColor.rgb, hsvMinHue, hsvMaxHue,
|
|
518
|
+
hsvMinSat, hsvMaxSat, hsvMinVal, hsvMaxVal, 0.1);
|
|
519
|
+
// Background pixels need ~3.3x higher AI confidence to survive
|
|
520
|
+
float refinedMask = maskValue * (0.3 + 0.7 * hsvMask);
|
|
521
|
+
inside = refinedMask > maskThreshold;
|
|
522
|
+
} else {
|
|
523
|
+
inside = maskValue > maskThreshold;
|
|
524
|
+
}
|
|
525
|
+
} else {
|
|
526
|
+
inside = inSilhouette(vUv, aspectRatioFactor);
|
|
527
|
+
}
|
|
528
|
+
|
|
529
|
+
// isPositioningMode is defined at top of main() - shows full silhouette guide
|
|
530
|
+
|
|
531
|
+
if (hasTexture) {
|
|
532
|
+
if (inside) {
|
|
533
|
+
gl_FragColor = vec4(texColor.rgb * garbageMask, garbageMask);
|
|
534
|
+
} else {
|
|
535
|
+
if (useAIMask) {
|
|
536
|
+
discard;
|
|
537
|
+
} else if (isPositioningMode) {
|
|
538
|
+
// Positioning mode: much darker outside for clear guide visibility
|
|
539
|
+
// Use 75% opacity dark overlay so silhouette boundary is obvious
|
|
540
|
+
float positioningAlpha = 0.75 * garbageMask;
|
|
541
|
+
gl_FragColor = vec4(0.0, 0.0, 0.0, positioningAlpha);
|
|
542
|
+
} else {
|
|
543
|
+
float finalAlpha = darkOpacity * garbageMask;
|
|
544
|
+
gl_FragColor = vec4(0.0, 0.0, 0.0, finalAlpha);
|
|
545
|
+
}
|
|
546
|
+
}
|
|
547
|
+
} else {
|
|
548
|
+
if (inside) {
|
|
549
|
+
float alpha = (placeholderOpacity + 0.15) * garbageMask;
|
|
550
|
+
gl_FragColor = vec4(placeholderColor * alpha, alpha);
|
|
551
|
+
} else {
|
|
552
|
+
float alpha = placeholderOpacity * garbageMask;
|
|
553
|
+
gl_FragColor = vec4(placeholderColor * 0.5 * alpha, alpha);
|
|
554
|
+
}
|
|
555
|
+
}
|
|
556
|
+
}
|
|
557
|
+
`;
|
|
558
|
+
/**
|
|
559
|
+
* Get default uniform values for player mode (no editor-specific features)
|
|
560
|
+
*/
|
|
561
|
+
export function getDefaultPlayerUniforms() {
|
|
562
|
+
return {
|
|
563
|
+
hasTexture: { value: false },
|
|
564
|
+
useAIMask: { value: false },
|
|
565
|
+
maskThreshold: { value: 0.5 },
|
|
566
|
+
darkOpacity: { value: 0.0 }, // No dark overlay in player
|
|
567
|
+
placeholderOpacity: { value: 0.0 }, // No placeholder in player
|
|
568
|
+
aspectRatio: { value: 16 / 9 },
|
|
569
|
+
useChromaKey: { value: false },
|
|
570
|
+
chromaKeyTolerance: { value: 0.35 },
|
|
571
|
+
chromaKeySpill: { value: 0.25 },
|
|
572
|
+
chromaKeyEdge: { value: 0.08 },
|
|
573
|
+
setupMode: { value: false }, // Never in setup mode in player
|
|
574
|
+
useHsvWindow: { value: false },
|
|
575
|
+
hsvMinHue: { value: 60 },
|
|
576
|
+
hsvMaxHue: { value: 180 },
|
|
577
|
+
hsvMinSat: { value: 0.2 },
|
|
578
|
+
hsvMaxSat: { value: 1.0 },
|
|
579
|
+
hsvMinVal: { value: 0.2 },
|
|
580
|
+
hsvMaxVal: { value: 1.0 },
|
|
581
|
+
useGarbageMatte: { value: false },
|
|
582
|
+
garbageMatteIsRect: { value: true },
|
|
583
|
+
garbageMatteInvert: { value: false },
|
|
584
|
+
garbageMatteFeather: { value: 0 },
|
|
585
|
+
usePerEdgeFeather: { value: false },
|
|
586
|
+
ccBrightness: { value: 0 },
|
|
587
|
+
ccContrast: { value: 0 },
|
|
588
|
+
ccSaturation: { value: 0 },
|
|
589
|
+
ccTemperature: { value: 0 },
|
|
590
|
+
fullViewMode: { value: false },
|
|
591
|
+
debugShowMask: { value: false },
|
|
592
|
+
silhouetteScale: { value: 1.0 }
|
|
593
|
+
};
|
|
594
|
+
}
|
|
595
|
+
/**
|
|
596
|
+
* Get default uniform values for editor mode (includes editor-specific features)
|
|
597
|
+
*/
|
|
598
|
+
export function getDefaultEditorUniforms() {
|
|
599
|
+
return {
|
|
600
|
+
...getDefaultPlayerUniforms(),
|
|
601
|
+
darkOpacity: { value: 0.5 }, // Show dark overlay outside silhouette
|
|
602
|
+
placeholderOpacity: { value: 0.25 } // Show placeholder when no webcam
|
|
603
|
+
};
|
|
604
|
+
}
|