@livekit/track-processors 0.6.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +31 -9
- package/dist/index.js +312 -99
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +311 -98
- package/dist/index.mjs.map +1 -1
- package/dist/src/ProcessorWrapper.d.ts +12 -7
- package/dist/src/index.d.ts +51 -5
- package/dist/src/logger.d.ts +29 -0
- package/dist/src/transformers/BackgroundTransformer.d.ts +9 -4
- package/dist/src/transformers/VideoTransformer.d.ts +1 -1
- package/dist/src/transformers/types.d.ts +11 -8
- package/dist/src/webgl/index.d.ts +1 -0
- package/dist/src/webgl/shader-programs/compositeShader.d.ts +1 -0
- package/package.json +3 -3
- package/src/ProcessorWrapper.ts +63 -23
- package/src/index.ts +206 -42
- package/src/logger.ts +74 -0
- package/src/transformers/BackgroundTransformer.ts +40 -23
- package/src/transformers/VideoTransformer.ts +1 -1
- package/src/transformers/types.ts +11 -8
- package/src/webgl/index.ts +24 -11
- package/src/webgl/shader-programs/compositeShader.ts +18 -13
- package/src/webgl/utils.ts +6 -2
package/dist/index.js
CHANGED
|
@@ -26,12 +26,68 @@ function createCanvas(width, height) {
|
|
|
26
26
|
return canvas;
|
|
27
27
|
}
|
|
28
28
|
|
|
29
|
+
// src/logger.ts
|
|
30
|
+
var _livekitclient = require('livekit-client');
|
|
31
|
+
var LogLevel = /* @__PURE__ */ ((LogLevel2) => {
|
|
32
|
+
LogLevel2[LogLevel2["trace"] = 0] = "trace";
|
|
33
|
+
LogLevel2[LogLevel2["debug"] = 1] = "debug";
|
|
34
|
+
LogLevel2[LogLevel2["info"] = 2] = "info";
|
|
35
|
+
LogLevel2[LogLevel2["warn"] = 3] = "warn";
|
|
36
|
+
LogLevel2[LogLevel2["error"] = 4] = "error";
|
|
37
|
+
LogLevel2[LogLevel2["silent"] = 5] = "silent";
|
|
38
|
+
return LogLevel2;
|
|
39
|
+
})(LogLevel || {});
|
|
40
|
+
var LoggerNames = /* @__PURE__ */ ((LoggerNames2) => {
|
|
41
|
+
LoggerNames2["ProcessorWrapper"] = "livekit-processor-wrapper";
|
|
42
|
+
LoggerNames2["BackgroundProcessor"] = "livekit-background-processor";
|
|
43
|
+
LoggerNames2["WebGl"] = "livekit-track-processor-web-gl";
|
|
44
|
+
return LoggerNames2;
|
|
45
|
+
})(LoggerNames || {});
|
|
46
|
+
var livekitLogger = getLogger("livekit");
|
|
47
|
+
var livekitLoggers = Object.values(LoggerNames).map((name) => getLogger(name));
|
|
48
|
+
livekitLogger.setDefaultLevel(2 /* info */);
|
|
49
|
+
function getLogger(name) {
|
|
50
|
+
return _livekitclient.getLogger.call(void 0, name);
|
|
51
|
+
}
|
|
52
|
+
function setLogLevel(level, loggerName) {
|
|
53
|
+
if (loggerName) {
|
|
54
|
+
getLogger(loggerName).setLevel(level);
|
|
55
|
+
} else {
|
|
56
|
+
for (const logger of livekitLoggers) {
|
|
57
|
+
logger.setLevel(level);
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
function setLogExtension(extension, logger) {
|
|
62
|
+
const loggers = logger ? [logger] : livekitLoggers;
|
|
63
|
+
loggers.forEach((logR) => {
|
|
64
|
+
const originalFactory = logR.methodFactory;
|
|
65
|
+
logR.methodFactory = (methodName, configLevel, loggerName) => {
|
|
66
|
+
const rawMethod = originalFactory(methodName, configLevel, loggerName);
|
|
67
|
+
const logLevel = LogLevel[methodName];
|
|
68
|
+
const needLog = logLevel >= configLevel && logLevel < 5 /* silent */;
|
|
69
|
+
return (msg, context) => {
|
|
70
|
+
if (context)
|
|
71
|
+
rawMethod(msg, context);
|
|
72
|
+
else
|
|
73
|
+
rawMethod(msg);
|
|
74
|
+
if (needLog) {
|
|
75
|
+
extension(logLevel, msg, context);
|
|
76
|
+
}
|
|
77
|
+
};
|
|
78
|
+
};
|
|
79
|
+
logR.setLevel(logR.getLevel());
|
|
80
|
+
});
|
|
81
|
+
}
|
|
82
|
+
|
|
29
83
|
// src/ProcessorWrapper.ts
|
|
30
84
|
var ProcessorWrapper = class _ProcessorWrapper {
|
|
31
85
|
constructor(transformer, name, options = {}) {
|
|
32
86
|
// For tracking whether we're using the stream API fallback
|
|
33
87
|
this.useStreamFallback = false;
|
|
34
88
|
this.processingEnabled = false;
|
|
89
|
+
this.log = getLogger("livekit-processor-wrapper" /* ProcessorWrapper */);
|
|
90
|
+
this.lifecycleState = "idle";
|
|
35
91
|
var _a;
|
|
36
92
|
this.name = name;
|
|
37
93
|
this.transformer = transformer;
|
|
@@ -92,6 +148,8 @@ var ProcessorWrapper = class _ProcessorWrapper {
|
|
|
92
148
|
}
|
|
93
149
|
}
|
|
94
150
|
async init(opts) {
|
|
151
|
+
this.log.debug("Init called");
|
|
152
|
+
this.lifecycleState = "initializing";
|
|
95
153
|
await this.setup(opts);
|
|
96
154
|
if (!this.canvas) {
|
|
97
155
|
throw new TypeError("Expected canvas to be defined after setup");
|
|
@@ -105,6 +163,7 @@ var ProcessorWrapper = class _ProcessorWrapper {
|
|
|
105
163
|
} else {
|
|
106
164
|
this.initStreamProcessorPath();
|
|
107
165
|
}
|
|
166
|
+
this.lifecycleState = "running";
|
|
108
167
|
}
|
|
109
168
|
initStreamProcessorPath() {
|
|
110
169
|
if (!this.processor || !this.trackGenerator) {
|
|
@@ -114,14 +173,15 @@ var ProcessorWrapper = class _ProcessorWrapper {
|
|
|
114
173
|
}
|
|
115
174
|
const readableStream = this.processor.readable;
|
|
116
175
|
const pipedStream = readableStream.pipeThrough(this.transformer.transformer);
|
|
117
|
-
|
|
118
|
-
this.symbol = symbol;
|
|
119
|
-
pipedStream.pipeTo(this.trackGenerator.writable).then(() => this.destroy(symbol)).catch((e) => {
|
|
176
|
+
pipedStream.pipeTo(this.trackGenerator.writable).then(() => this.handleMediaExhausted()).catch((e) => {
|
|
120
177
|
if (e instanceof DOMException && e.name === "AbortError") {
|
|
121
|
-
|
|
178
|
+
this.log.log("stream processor path aborted");
|
|
179
|
+
} else if (e instanceof DOMException && e.name === "InvalidStateError" && e.message === "Stream closed") {
|
|
180
|
+
this.log.log("stream processor underlying stream closed");
|
|
181
|
+
this.handleMediaExhausted();
|
|
122
182
|
} else {
|
|
123
|
-
|
|
124
|
-
this.destroy(
|
|
183
|
+
this.log.error("error when trying to pipe", e);
|
|
184
|
+
this.destroy();
|
|
125
185
|
}
|
|
126
186
|
});
|
|
127
187
|
this.processedTrack = this.trackGenerator;
|
|
@@ -154,7 +214,7 @@ var ProcessorWrapper = class _ProcessorWrapper {
|
|
|
154
214
|
try {
|
|
155
215
|
this.transformer.transform(frame, controller);
|
|
156
216
|
} catch (e) {
|
|
157
|
-
|
|
217
|
+
this.log.error("Error in transform:", e);
|
|
158
218
|
frame.close();
|
|
159
219
|
}
|
|
160
220
|
};
|
|
@@ -162,6 +222,7 @@ var ProcessorWrapper = class _ProcessorWrapper {
|
|
|
162
222
|
}
|
|
163
223
|
startRenderLoop() {
|
|
164
224
|
if (!this.sourceDummy || !(this.sourceDummy instanceof HTMLVideoElement)) {
|
|
225
|
+
this.handleMediaExhausted();
|
|
165
226
|
return;
|
|
166
227
|
}
|
|
167
228
|
let lastVideoTimestamp = -1;
|
|
@@ -175,10 +236,11 @@ var ProcessorWrapper = class _ProcessorWrapper {
|
|
|
175
236
|
let lastFpsLog = 0;
|
|
176
237
|
const renderLoop = () => {
|
|
177
238
|
if (!this.processingEnabled || !this.sourceDummy || !(this.sourceDummy instanceof HTMLVideoElement)) {
|
|
239
|
+
this.handleMediaExhausted();
|
|
178
240
|
return;
|
|
179
241
|
}
|
|
180
242
|
if (this.sourceDummy.paused) {
|
|
181
|
-
|
|
243
|
+
this.log.warn("Video is paused, trying to play");
|
|
182
244
|
this.sourceDummy.play();
|
|
183
245
|
return;
|
|
184
246
|
}
|
|
@@ -198,7 +260,7 @@ var ProcessorWrapper = class _ProcessorWrapper {
|
|
|
198
260
|
estimatedVideoFps = 1e3 / avgFrameTime;
|
|
199
261
|
const isDevelopment = typeof window !== "undefined" && window.location.hostname === "localhost" || window.location.hostname === "127.0.0.1";
|
|
200
262
|
if (isDevelopment && now - lastFpsLog > 5e3) {
|
|
201
|
-
|
|
263
|
+
this.log.debug(
|
|
202
264
|
`[${this.name}] Estimated video FPS: ${estimatedVideoFps.toFixed(
|
|
203
265
|
1
|
|
204
266
|
)}, Processing at: ${(frameCount / 5).toFixed(1)} FPS`
|
|
@@ -225,7 +287,7 @@ var ProcessorWrapper = class _ProcessorWrapper {
|
|
|
225
287
|
}
|
|
226
288
|
}
|
|
227
289
|
} catch (e) {
|
|
228
|
-
|
|
290
|
+
this.log.error("Error in render loop:", e);
|
|
229
291
|
}
|
|
230
292
|
}
|
|
231
293
|
this.animationFrameId = requestAnimationFrame(renderLoop);
|
|
@@ -233,7 +295,8 @@ var ProcessorWrapper = class _ProcessorWrapper {
|
|
|
233
295
|
this.animationFrameId = requestAnimationFrame(renderLoop);
|
|
234
296
|
}
|
|
235
297
|
async restart(opts) {
|
|
236
|
-
|
|
298
|
+
this.log.debug("Restart called");
|
|
299
|
+
await this.destroy({ willProcessorRestart: true });
|
|
237
300
|
await this.init(opts);
|
|
238
301
|
}
|
|
239
302
|
async restartTransformer(...options) {
|
|
@@ -242,11 +305,18 @@ var ProcessorWrapper = class _ProcessorWrapper {
|
|
|
242
305
|
async updateTransformerOptions(...options) {
|
|
243
306
|
await this.transformer.update(options[0]);
|
|
244
307
|
}
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
308
|
+
/** Called if the media pipeline no longer can read frames to process from the source media */
|
|
309
|
+
async handleMediaExhausted() {
|
|
310
|
+
this.log.debug("Media was exhausted from source");
|
|
311
|
+
if (this.lifecycleState !== "running") {
|
|
248
312
|
return;
|
|
249
313
|
}
|
|
314
|
+
this.lifecycleState = "media-exhausted";
|
|
315
|
+
await this.cleanup();
|
|
316
|
+
}
|
|
317
|
+
/** Tears down the media stack logic initialized in initStreamProcessorPath / initFallbackPath */
|
|
318
|
+
async cleanup() {
|
|
319
|
+
var _a, _b, _c, _d;
|
|
250
320
|
if (this.useStreamFallback) {
|
|
251
321
|
this.processingEnabled = false;
|
|
252
322
|
if (this.animationFrameId) {
|
|
@@ -261,7 +331,20 @@ var ProcessorWrapper = class _ProcessorWrapper {
|
|
|
261
331
|
await ((_c = (_b = this.processor) == null ? void 0 : _b.writableControl) == null ? void 0 : _c.close());
|
|
262
332
|
(_d = this.trackGenerator) == null ? void 0 : _d.stop();
|
|
263
333
|
}
|
|
264
|
-
|
|
334
|
+
}
|
|
335
|
+
async destroy(transformerDestroyOptions = { willProcessorRestart: false }) {
|
|
336
|
+
this.log.debug(`Destroy called - lifecycleState=${this.lifecycleState}, transformerDestroyOptions=${JSON.stringify(transformerDestroyOptions)}`);
|
|
337
|
+
switch (this.lifecycleState) {
|
|
338
|
+
case "running":
|
|
339
|
+
case "media-exhausted":
|
|
340
|
+
this.lifecycleState = "destroying";
|
|
341
|
+
await this.cleanup();
|
|
342
|
+
await this.transformer.destroy(transformerDestroyOptions);
|
|
343
|
+
this.lifecycleState = "destroyed";
|
|
344
|
+
break;
|
|
345
|
+
default:
|
|
346
|
+
break;
|
|
347
|
+
}
|
|
265
348
|
}
|
|
266
349
|
};
|
|
267
350
|
|
|
@@ -274,6 +357,7 @@ var dependencies = {
|
|
|
274
357
|
};
|
|
275
358
|
|
|
276
359
|
// src/webgl/utils.ts
|
|
360
|
+
var log = getLogger("livekit-track-processor-web-gl" /* WebGl */);
|
|
277
361
|
function initTexture(gl, texIndex) {
|
|
278
362
|
const texRef = gl.TEXTURE0 + texIndex;
|
|
279
363
|
gl.activeTexture(texRef);
|
|
@@ -291,7 +375,7 @@ function createShader(gl, type, source) {
|
|
|
291
375
|
gl.shaderSource(shader, source);
|
|
292
376
|
gl.compileShader(shader);
|
|
293
377
|
if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) {
|
|
294
|
-
|
|
378
|
+
log.error("Shader compile failed:", gl.getShaderInfoLog(shader));
|
|
295
379
|
gl.deleteShader(shader);
|
|
296
380
|
throw new Error("Shader compile failed");
|
|
297
381
|
}
|
|
@@ -303,7 +387,7 @@ function createProgram(gl, vs, fs) {
|
|
|
303
387
|
gl.attachShader(program, fs);
|
|
304
388
|
gl.linkProgram(program);
|
|
305
389
|
if (!gl.getProgramParameter(program, gl.LINK_STATUS)) {
|
|
306
|
-
|
|
390
|
+
log.error("Program link failed:", gl.getProgramInfoLog(program));
|
|
307
391
|
throw new Error("Program link failed");
|
|
308
392
|
}
|
|
309
393
|
return program;
|
|
@@ -507,29 +591,33 @@ var compositeFragmentShader = glsl`#version 300 es
|
|
|
507
591
|
precision mediump float;
|
|
508
592
|
in vec2 texCoords;
|
|
509
593
|
uniform sampler2D background;
|
|
594
|
+
uniform bool disableBackground;
|
|
510
595
|
uniform sampler2D frame;
|
|
511
596
|
uniform sampler2D mask;
|
|
512
597
|
out vec4 fragColor;
|
|
513
598
|
|
|
514
599
|
void main() {
|
|
515
|
-
|
|
516
600
|
vec4 frameTex = texture(frame, texCoords);
|
|
517
|
-
vec4 bgTex = texture(background, texCoords);
|
|
518
601
|
|
|
519
|
-
|
|
602
|
+
if (disableBackground) {
|
|
603
|
+
fragColor = frameTex;
|
|
604
|
+
} else {
|
|
605
|
+
vec4 bgTex = texture(background, texCoords);
|
|
606
|
+
|
|
607
|
+
float maskVal = texture(mask, texCoords).r;
|
|
608
|
+
|
|
609
|
+
// Compute screen-space gradient to detect edge sharpness
|
|
610
|
+
float grad = length(vec2(dFdx(maskVal), dFdy(maskVal)));
|
|
520
611
|
|
|
521
|
-
|
|
522
|
-
float grad = length(vec2(dFdx(maskVal), dFdy(maskVal)));
|
|
612
|
+
float edgeSoftness = 2.0; // higher = softer
|
|
523
613
|
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
// Create a smooth edge around binary transition
|
|
527
|
-
float smoothAlpha = smoothstep(0.5 - grad * edgeSoftness, 0.5 + grad * edgeSoftness, maskVal);
|
|
614
|
+
// Create a smooth edge around binary transition
|
|
615
|
+
float smoothAlpha = smoothstep(0.5 - grad * edgeSoftness, 0.5 + grad * edgeSoftness, maskVal);
|
|
528
616
|
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
617
|
+
// Optional: preserve frame alpha, or override as fully opaque
|
|
618
|
+
vec4 blended = mix(bgTex, vec4(frameTex.rgb, 1.0), 1.0 - smoothAlpha);
|
|
619
|
+
fragColor = blended;
|
|
620
|
+
}
|
|
533
621
|
|
|
534
622
|
}
|
|
535
623
|
`;
|
|
@@ -544,6 +632,7 @@ function createCompositeProgram(gl) {
|
|
|
544
632
|
mask: gl.getUniformLocation(compositeProgram, "mask"),
|
|
545
633
|
frame: gl.getUniformLocation(compositeProgram, "frame"),
|
|
546
634
|
background: gl.getUniformLocation(compositeProgram, "background"),
|
|
635
|
+
disableBackground: gl.getUniformLocation(compositeProgram, "disableBackground"),
|
|
547
636
|
stepWidth: gl.getUniformLocation(compositeProgram, "u_stepWidth")
|
|
548
637
|
};
|
|
549
638
|
return {
|
|
@@ -613,6 +702,7 @@ function applyDownsampling(gl, inputTexture, downSampler, vertexBuffer, width, h
|
|
|
613
702
|
}
|
|
614
703
|
|
|
615
704
|
// src/webgl/index.ts
|
|
705
|
+
var log2 = getLogger("livekit-track-processor-web-gl" /* WebGl */);
|
|
616
706
|
var setupWebGL = (canvas) => {
|
|
617
707
|
const gl = canvas.getContext("webgl2", {
|
|
618
708
|
antialias: true,
|
|
@@ -622,7 +712,7 @@ var setupWebGL = (canvas) => {
|
|
|
622
712
|
let maskBlurRadius = 8;
|
|
623
713
|
const downsampleFactor = 4;
|
|
624
714
|
if (!gl) {
|
|
625
|
-
|
|
715
|
+
log2.error("Failed to create WebGL context");
|
|
626
716
|
return void 0;
|
|
627
717
|
}
|
|
628
718
|
gl.enable(gl.BLEND);
|
|
@@ -633,7 +723,8 @@ var setupWebGL = (canvas) => {
|
|
|
633
723
|
const {
|
|
634
724
|
mask: maskTextureLocation,
|
|
635
725
|
frame: frameTextureLocation,
|
|
636
|
-
background: bgTextureLocation
|
|
726
|
+
background: bgTextureLocation,
|
|
727
|
+
disableBackground: disableBackgroundLocation
|
|
637
728
|
} = composite.uniformLocations;
|
|
638
729
|
const blur = createBlurProgram(gl);
|
|
639
730
|
const blurProgram = blur.program;
|
|
@@ -672,11 +763,13 @@ var setupWebGL = (canvas) => {
|
|
|
672
763
|
createFramebuffer(gl, finalMaskTextures[0], canvas.width, canvas.height),
|
|
673
764
|
createFramebuffer(gl, finalMaskTextures[1], canvas.width, canvas.height)
|
|
674
765
|
];
|
|
766
|
+
let backgroundImageDisabled = false;
|
|
675
767
|
gl.useProgram(compositeProgram);
|
|
768
|
+
gl.uniform1i(disableBackgroundLocation, backgroundImageDisabled ? 1 : 0);
|
|
676
769
|
gl.uniform1i(bgTextureLocation, 0);
|
|
677
770
|
gl.uniform1i(frameTextureLocation, 1);
|
|
678
771
|
gl.uniform1i(maskTextureLocation, 2);
|
|
679
|
-
let customBackgroundImage =
|
|
772
|
+
let customBackgroundImage = null;
|
|
680
773
|
function renderFrame(frame) {
|
|
681
774
|
if (frame.codedWidth === 0 || finalMaskTextures.length === 0) {
|
|
682
775
|
return;
|
|
@@ -708,7 +801,7 @@ var setupWebGL = (canvas) => {
|
|
|
708
801
|
bgBlurFrameBuffers,
|
|
709
802
|
bgBlurTextures
|
|
710
803
|
);
|
|
711
|
-
} else {
|
|
804
|
+
} else if (customBackgroundImage) {
|
|
712
805
|
gl.activeTexture(gl.TEXTURE0);
|
|
713
806
|
gl.bindTexture(gl.TEXTURE_2D, bgTexture);
|
|
714
807
|
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, customBackgroundImage);
|
|
@@ -724,6 +817,7 @@ var setupWebGL = (canvas) => {
|
|
|
724
817
|
gl.activeTexture(gl.TEXTURE0);
|
|
725
818
|
gl.bindTexture(gl.TEXTURE_2D, backgroundTexture);
|
|
726
819
|
gl.uniform1i(bgTextureLocation, 0);
|
|
820
|
+
gl.uniform1i(disableBackgroundLocation, backgroundImageDisabled ? 1 : 0);
|
|
727
821
|
gl.activeTexture(gl.TEXTURE1);
|
|
728
822
|
gl.bindTexture(gl.TEXTURE_2D, frameTexture);
|
|
729
823
|
gl.uniform1i(frameTextureLocation, 1);
|
|
@@ -733,26 +827,30 @@ var setupWebGL = (canvas) => {
|
|
|
733
827
|
gl.drawArrays(gl.TRIANGLES, 0, 6);
|
|
734
828
|
}
|
|
735
829
|
async function setBackgroundImage(image) {
|
|
736
|
-
customBackgroundImage =
|
|
830
|
+
customBackgroundImage = null;
|
|
737
831
|
if (image) {
|
|
832
|
+
customBackgroundImage = getEmptyImageData();
|
|
738
833
|
try {
|
|
739
834
|
const croppedImage = await resizeImageToCover(image, canvas.width, canvas.height);
|
|
740
835
|
customBackgroundImage = croppedImage;
|
|
741
836
|
} catch (error) {
|
|
742
|
-
|
|
837
|
+
log2.error(
|
|
743
838
|
"Error processing background image, falling back to black background:",
|
|
744
839
|
error
|
|
745
840
|
);
|
|
746
841
|
}
|
|
842
|
+
gl.activeTexture(gl.TEXTURE0);
|
|
843
|
+
gl.bindTexture(gl.TEXTURE_2D, bgTexture);
|
|
844
|
+
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, customBackgroundImage);
|
|
747
845
|
}
|
|
748
|
-
gl.activeTexture(gl.TEXTURE0);
|
|
749
|
-
gl.bindTexture(gl.TEXTURE_2D, bgTexture);
|
|
750
|
-
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, customBackgroundImage);
|
|
751
846
|
}
|
|
752
847
|
function setBlurRadius(radius) {
|
|
753
848
|
blurRadius = radius ? Math.max(1, Math.floor(radius / downsampleFactor)) : null;
|
|
754
849
|
setBackgroundImage(null);
|
|
755
850
|
}
|
|
851
|
+
function setBackgroundDisabled(disabled) {
|
|
852
|
+
backgroundImageDisabled = disabled;
|
|
853
|
+
}
|
|
756
854
|
function updateMask(mask) {
|
|
757
855
|
const tempFramebuffers = [tempMaskFrameBuffer, finalMaskFrameBuffers[writeMaskIndex]];
|
|
758
856
|
const tempTextures = [tempMaskTexture, finalMaskTextures[writeMaskIndex]];
|
|
@@ -804,13 +902,13 @@ var setupWebGL = (canvas) => {
|
|
|
804
902
|
if (customBackgroundImage instanceof ImageBitmap) {
|
|
805
903
|
customBackgroundImage.close();
|
|
806
904
|
}
|
|
807
|
-
customBackgroundImage =
|
|
905
|
+
customBackgroundImage = null;
|
|
808
906
|
}
|
|
809
907
|
bgBlurTextures = [];
|
|
810
908
|
bgBlurFrameBuffers = [];
|
|
811
909
|
finalMaskTextures = [];
|
|
812
910
|
}
|
|
813
|
-
return { renderFrame, updateMask, setBackgroundImage, setBlurRadius, cleanup };
|
|
911
|
+
return { renderFrame, updateMask, setBackgroundImage, setBlurRadius, setBackgroundDisabled, cleanup };
|
|
814
912
|
};
|
|
815
913
|
|
|
816
914
|
// src/transformers/VideoTransformer.ts
|
|
@@ -860,9 +958,10 @@ var VideoTransformer = class {
|
|
|
860
958
|
var BackgroundProcessor = class extends VideoTransformer {
|
|
861
959
|
constructor(opts) {
|
|
862
960
|
super();
|
|
863
|
-
this.
|
|
961
|
+
this.backgroundImageAndPath = null;
|
|
864
962
|
this.segmentationTimeMs = 0;
|
|
865
963
|
this.isFirstFrame = true;
|
|
964
|
+
this.log = getLogger("livekit-processor-wrapper" /* ProcessorWrapper */);
|
|
866
965
|
this.options = opts;
|
|
867
966
|
this.update(opts);
|
|
868
967
|
}
|
|
@@ -870,7 +969,7 @@ var BackgroundProcessor = class extends VideoTransformer {
|
|
|
870
969
|
return typeof OffscreenCanvas !== "undefined" && typeof VideoFrame !== "undefined" && typeof createImageBitmap !== "undefined" && !!document.createElement("canvas").getContext("webgl2");
|
|
871
970
|
}
|
|
872
971
|
async init({ outputCanvas, inputElement: inputVideo }) {
|
|
873
|
-
var _a, _b, _c, _d, _e, _f;
|
|
972
|
+
var _a, _b, _c, _d, _e, _f, _g, _h;
|
|
874
973
|
await super.init({ outputCanvas, inputElement: inputVideo });
|
|
875
974
|
const fileSet = await vision.FilesetResolver.forVisionTasks(
|
|
876
975
|
(_b = (_a = this.options.assetPaths) == null ? void 0 : _a.tasksVisionFileSet) != null ? _b : `https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@${dependencies["@mediapipe/tasks-vision"]}/wasm`
|
|
@@ -886,43 +985,53 @@ var BackgroundProcessor = class extends VideoTransformer {
|
|
|
886
985
|
outputCategoryMask: true,
|
|
887
986
|
outputConfidenceMasks: false
|
|
888
987
|
});
|
|
889
|
-
if ((
|
|
890
|
-
await this.
|
|
891
|
-
(err) =>
|
|
988
|
+
if ((_e = this.options) == null ? void 0 : _e.imagePath) {
|
|
989
|
+
await this.loadAndSetBackground(this.options.imagePath).catch(
|
|
990
|
+
(err) => this.log.error("Error while loading processor background image: ", err)
|
|
892
991
|
);
|
|
893
992
|
}
|
|
894
|
-
if (this.options.blurRadius) {
|
|
993
|
+
if (typeof this.options.blurRadius === "number") {
|
|
895
994
|
(_f = this.gl) == null ? void 0 : _f.setBlurRadius(this.options.blurRadius);
|
|
896
995
|
}
|
|
996
|
+
(_h = this.gl) == null ? void 0 : _h.setBackgroundDisabled((_g = this.options.backgroundDisabled) != null ? _g : false);
|
|
897
997
|
}
|
|
898
|
-
async destroy() {
|
|
998
|
+
async destroy(options) {
|
|
899
999
|
var _a;
|
|
900
1000
|
await super.destroy();
|
|
901
1001
|
await ((_a = this.imageSegmenter) == null ? void 0 : _a.close());
|
|
902
|
-
this.
|
|
903
|
-
|
|
1002
|
+
this.backgroundImageAndPath = null;
|
|
1003
|
+
if (!(options == null ? void 0 : options.willProcessorRestart)) {
|
|
1004
|
+
this.isFirstFrame = true;
|
|
1005
|
+
}
|
|
904
1006
|
}
|
|
905
|
-
async
|
|
906
|
-
var _a;
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
1007
|
+
async loadAndSetBackground(path) {
|
|
1008
|
+
var _a, _b;
|
|
1009
|
+
if (!this.backgroundImageAndPath || ((_a = this.backgroundImageAndPath) == null ? void 0 : _a.path) !== path) {
|
|
1010
|
+
const img = new Image();
|
|
1011
|
+
await new Promise((resolve, reject) => {
|
|
1012
|
+
img.crossOrigin = "Anonymous";
|
|
1013
|
+
img.onload = () => resolve(img);
|
|
1014
|
+
img.onerror = (err) => reject(err);
|
|
1015
|
+
img.src = path;
|
|
1016
|
+
});
|
|
1017
|
+
const imageData = await createImageBitmap(img);
|
|
1018
|
+
this.backgroundImageAndPath = { imageData, path };
|
|
1019
|
+
}
|
|
1020
|
+
(_b = this.gl) == null ? void 0 : _b.setBackgroundImage(this.backgroundImageAndPath.imageData);
|
|
916
1021
|
}
|
|
917
1022
|
async transform(frame, controller) {
|
|
918
|
-
var _a, _b;
|
|
1023
|
+
var _a, _b, _c, _d;
|
|
919
1024
|
let enqueuedFrame = false;
|
|
920
1025
|
try {
|
|
921
1026
|
if (!(frame instanceof VideoFrame) || frame.codedWidth === 0 || frame.codedHeight === 0) {
|
|
922
|
-
|
|
1027
|
+
this.log.debug("empty frame detected, ignoring");
|
|
923
1028
|
return;
|
|
924
1029
|
}
|
|
925
|
-
|
|
1030
|
+
let skipProcessingFrame = (_b = (_a = this.isDisabled) != null ? _a : this.options.backgroundDisabled) != null ? _b : false;
|
|
1031
|
+
if (typeof this.options.blurRadius !== "number" && typeof this.options.imagePath !== "string") {
|
|
1032
|
+
skipProcessingFrame = true;
|
|
1033
|
+
}
|
|
1034
|
+
if (skipProcessingFrame) {
|
|
926
1035
|
controller.enqueue(frame);
|
|
927
1036
|
enqueuedFrame = true;
|
|
928
1037
|
return;
|
|
@@ -973,13 +1082,13 @@ var BackgroundProcessor = class extends VideoTransformer {
|
|
|
973
1082
|
segmentationTimeMs: this.segmentationTimeMs,
|
|
974
1083
|
filterTimeMs
|
|
975
1084
|
};
|
|
976
|
-
(
|
|
1085
|
+
(_d = (_c = this.options).onFrameProcessed) == null ? void 0 : _d.call(_c, stats);
|
|
977
1086
|
} else {
|
|
978
1087
|
controller.enqueue(frame);
|
|
979
1088
|
}
|
|
980
1089
|
await segmentationPromise;
|
|
981
1090
|
} catch (e) {
|
|
982
|
-
|
|
1091
|
+
this.log.error("Error while processing frame: ", e);
|
|
983
1092
|
} finally {
|
|
984
1093
|
if (!enqueuedFrame) {
|
|
985
1094
|
frame.close();
|
|
@@ -987,14 +1096,15 @@ var BackgroundProcessor = class extends VideoTransformer {
|
|
|
987
1096
|
}
|
|
988
1097
|
}
|
|
989
1098
|
async update(opts) {
|
|
990
|
-
var _a, _b, _c;
|
|
1099
|
+
var _a, _b, _c, _d, _e;
|
|
991
1100
|
this.options = { ...this.options, ...opts };
|
|
992
1101
|
(_b = this.gl) == null ? void 0 : _b.setBlurRadius((_a = opts.blurRadius) != null ? _a : null);
|
|
993
1102
|
if (opts.imagePath) {
|
|
994
|
-
await this.
|
|
1103
|
+
await this.loadAndSetBackground(opts.imagePath);
|
|
995
1104
|
} else {
|
|
996
1105
|
(_c = this.gl) == null ? void 0 : _c.setBackgroundImage(null);
|
|
997
1106
|
}
|
|
1107
|
+
(_e = this.gl) == null ? void 0 : _e.setBackgroundDisabled((_d = opts.backgroundDisabled) != null ? _d : false);
|
|
998
1108
|
}
|
|
999
1109
|
async drawFrame(frame) {
|
|
1000
1110
|
var _a;
|
|
@@ -1011,9 +1121,136 @@ var BackgroundProcessor = class extends VideoTransformer {
|
|
|
1011
1121
|
};
|
|
1012
1122
|
|
|
1013
1123
|
// src/index.ts
|
|
1124
|
+
var DEFAULT_BLUR_RADIUS = 10;
|
|
1014
1125
|
var supportsBackgroundProcessors = () => BackgroundProcessor.isSupported && ProcessorWrapper.isSupported;
|
|
1015
1126
|
var supportsModernBackgroundProcessors = () => BackgroundProcessor.isSupported && ProcessorWrapper.hasModernApiSupport;
|
|
1016
|
-
var
|
|
1127
|
+
var BackgroundProcessorWrapper = class extends ProcessorWrapper {
|
|
1128
|
+
get mode() {
|
|
1129
|
+
const options = this.transformer.options;
|
|
1130
|
+
if (options.backgroundDisabled) {
|
|
1131
|
+
return "disabled";
|
|
1132
|
+
}
|
|
1133
|
+
if (typeof options.imagePath === "string" && typeof options.blurRadius === "undefined") {
|
|
1134
|
+
return "virtual-background";
|
|
1135
|
+
}
|
|
1136
|
+
if (typeof options.imagePath === "undefined") {
|
|
1137
|
+
return "background-blur";
|
|
1138
|
+
}
|
|
1139
|
+
return "legacy";
|
|
1140
|
+
}
|
|
1141
|
+
async switchTo(options) {
|
|
1142
|
+
var _a;
|
|
1143
|
+
switch (options.mode) {
|
|
1144
|
+
case "background-blur":
|
|
1145
|
+
await this.updateTransformerOptions({
|
|
1146
|
+
imagePath: void 0,
|
|
1147
|
+
blurRadius: (_a = options.blurRadius) != null ? _a : DEFAULT_BLUR_RADIUS,
|
|
1148
|
+
backgroundDisabled: false
|
|
1149
|
+
});
|
|
1150
|
+
break;
|
|
1151
|
+
case "virtual-background":
|
|
1152
|
+
await this.updateTransformerOptions({
|
|
1153
|
+
imagePath: options.imagePath,
|
|
1154
|
+
blurRadius: void 0,
|
|
1155
|
+
backgroundDisabled: false
|
|
1156
|
+
});
|
|
1157
|
+
break;
|
|
1158
|
+
case "disabled":
|
|
1159
|
+
await this.updateTransformerOptions({ imagePath: void 0, backgroundDisabled: true });
|
|
1160
|
+
break;
|
|
1161
|
+
}
|
|
1162
|
+
}
|
|
1163
|
+
};
|
|
1164
|
+
var BackgroundProcessor2 = (options, name = "background-processor") => {
|
|
1165
|
+
const isTransformerSupported = BackgroundProcessor.isSupported;
|
|
1166
|
+
const isProcessorSupported = ProcessorWrapper.isSupported;
|
|
1167
|
+
if (!isTransformerSupported) {
|
|
1168
|
+
throw new Error("Background transformer is not supported in this browser");
|
|
1169
|
+
}
|
|
1170
|
+
if (!isProcessorSupported) {
|
|
1171
|
+
throw new Error(
|
|
1172
|
+
"Neither MediaStreamTrackProcessor nor canvas.captureStream() fallback is supported in this browser"
|
|
1173
|
+
);
|
|
1174
|
+
}
|
|
1175
|
+
let transformer, processorOpts;
|
|
1176
|
+
switch (options.mode) {
|
|
1177
|
+
case "background-blur": {
|
|
1178
|
+
const {
|
|
1179
|
+
// eslint-disable-next-line no-unused-vars
|
|
1180
|
+
mode,
|
|
1181
|
+
blurRadius = DEFAULT_BLUR_RADIUS,
|
|
1182
|
+
segmenterOptions,
|
|
1183
|
+
assetPaths,
|
|
1184
|
+
onFrameProcessed,
|
|
1185
|
+
...rest
|
|
1186
|
+
} = options;
|
|
1187
|
+
processorOpts = rest;
|
|
1188
|
+
transformer = new BackgroundProcessor({
|
|
1189
|
+
blurRadius,
|
|
1190
|
+
segmenterOptions,
|
|
1191
|
+
assetPaths,
|
|
1192
|
+
onFrameProcessed
|
|
1193
|
+
});
|
|
1194
|
+
break;
|
|
1195
|
+
}
|
|
1196
|
+
case "virtual-background": {
|
|
1197
|
+
const {
|
|
1198
|
+
// eslint-disable-next-line no-unused-vars
|
|
1199
|
+
mode,
|
|
1200
|
+
imagePath,
|
|
1201
|
+
segmenterOptions,
|
|
1202
|
+
assetPaths,
|
|
1203
|
+
onFrameProcessed,
|
|
1204
|
+
...rest
|
|
1205
|
+
} = options;
|
|
1206
|
+
processorOpts = rest;
|
|
1207
|
+
transformer = new BackgroundProcessor({
|
|
1208
|
+
imagePath,
|
|
1209
|
+
segmenterOptions,
|
|
1210
|
+
assetPaths,
|
|
1211
|
+
onFrameProcessed
|
|
1212
|
+
});
|
|
1213
|
+
break;
|
|
1214
|
+
}
|
|
1215
|
+
case "disabled": {
|
|
1216
|
+
const {
|
|
1217
|
+
segmenterOptions,
|
|
1218
|
+
assetPaths,
|
|
1219
|
+
onFrameProcessed,
|
|
1220
|
+
...rest
|
|
1221
|
+
} = options;
|
|
1222
|
+
processorOpts = rest;
|
|
1223
|
+
transformer = new BackgroundProcessor({
|
|
1224
|
+
segmenterOptions,
|
|
1225
|
+
assetPaths,
|
|
1226
|
+
onFrameProcessed
|
|
1227
|
+
});
|
|
1228
|
+
break;
|
|
1229
|
+
}
|
|
1230
|
+
default: {
|
|
1231
|
+
const {
|
|
1232
|
+
blurRadius,
|
|
1233
|
+
imagePath,
|
|
1234
|
+
segmenterOptions,
|
|
1235
|
+
assetPaths,
|
|
1236
|
+
onFrameProcessed,
|
|
1237
|
+
...rest
|
|
1238
|
+
} = options;
|
|
1239
|
+
processorOpts = rest;
|
|
1240
|
+
transformer = new BackgroundProcessor({
|
|
1241
|
+
blurRadius,
|
|
1242
|
+
imagePath,
|
|
1243
|
+
segmenterOptions,
|
|
1244
|
+
assetPaths,
|
|
1245
|
+
onFrameProcessed
|
|
1246
|
+
});
|
|
1247
|
+
break;
|
|
1248
|
+
}
|
|
1249
|
+
}
|
|
1250
|
+
const processor = new BackgroundProcessorWrapper(transformer, name, processorOpts);
|
|
1251
|
+
return processor;
|
|
1252
|
+
};
|
|
1253
|
+
var BackgroundBlur = (blurRadius = DEFAULT_BLUR_RADIUS, segmenterOptions, onFrameProcessed, processorOptions) => {
|
|
1017
1254
|
return BackgroundProcessor2(
|
|
1018
1255
|
{
|
|
1019
1256
|
blurRadius,
|
|
@@ -1035,35 +1272,6 @@ var VirtualBackground = (imagePath, segmenterOptions, onFrameProcessed, processo
|
|
|
1035
1272
|
"virtual-background"
|
|
1036
1273
|
);
|
|
1037
1274
|
};
|
|
1038
|
-
var BackgroundProcessor2 = (options, name = "background-processor") => {
|
|
1039
|
-
const isTransformerSupported = BackgroundProcessor.isSupported;
|
|
1040
|
-
const isProcessorSupported = ProcessorWrapper.isSupported;
|
|
1041
|
-
if (!isTransformerSupported) {
|
|
1042
|
-
throw new Error("Background transformer is not supported in this browser");
|
|
1043
|
-
}
|
|
1044
|
-
if (!isProcessorSupported) {
|
|
1045
|
-
throw new Error(
|
|
1046
|
-
"Neither MediaStreamTrackProcessor nor canvas.captureStream() fallback is supported in this browser"
|
|
1047
|
-
);
|
|
1048
|
-
}
|
|
1049
|
-
const {
|
|
1050
|
-
blurRadius,
|
|
1051
|
-
imagePath,
|
|
1052
|
-
segmenterOptions,
|
|
1053
|
-
assetPaths,
|
|
1054
|
-
onFrameProcessed,
|
|
1055
|
-
...processorOpts
|
|
1056
|
-
} = options;
|
|
1057
|
-
const transformer = new BackgroundProcessor({
|
|
1058
|
-
blurRadius,
|
|
1059
|
-
imagePath,
|
|
1060
|
-
segmenterOptions,
|
|
1061
|
-
assetPaths,
|
|
1062
|
-
onFrameProcessed
|
|
1063
|
-
});
|
|
1064
|
-
const processor = new ProcessorWrapper(transformer, name, processorOpts);
|
|
1065
|
-
return processor;
|
|
1066
|
-
};
|
|
1067
1275
|
|
|
1068
1276
|
|
|
1069
1277
|
|
|
@@ -1073,5 +1281,10 @@ var BackgroundProcessor2 = (options, name = "background-processor") => {
|
|
|
1073
1281
|
|
|
1074
1282
|
|
|
1075
1283
|
|
|
1076
|
-
|
|
1284
|
+
|
|
1285
|
+
|
|
1286
|
+
|
|
1287
|
+
|
|
1288
|
+
|
|
1289
|
+
exports.BackgroundBlur = BackgroundBlur; exports.BackgroundProcessor = BackgroundProcessor2; exports.BackgroundTransformer = BackgroundProcessor; exports.LogLevel = LogLevel; exports.LoggerNames = LoggerNames; exports.ProcessorWrapper = ProcessorWrapper; exports.VideoTransformer = VideoTransformer; exports.VirtualBackground = VirtualBackground; exports.getLogger = getLogger; exports.setLogExtension = setLogExtension; exports.setLogLevel = setLogLevel; exports.supportsBackgroundProcessors = supportsBackgroundProcessors; exports.supportsModernBackgroundProcessors = supportsModernBackgroundProcessors;
|
|
1077
1290
|
//# sourceMappingURL=index.js.map
|