@livekit/track-processors 0.6.1 → 0.7.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +31 -9
- package/dist/index.js +297 -84
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +296 -83
- package/dist/index.mjs.map +1 -1
- package/dist/src/ProcessorWrapper.d.ts +12 -7
- package/dist/src/index.d.ts +56 -5
- package/dist/src/logger.d.ts +29 -0
- package/dist/src/transformers/BackgroundTransformer.d.ts +4 -2
- package/dist/src/transformers/VideoTransformer.d.ts +1 -1
- package/dist/src/transformers/types.d.ts +11 -8
- package/dist/src/webgl/index.d.ts +1 -0
- package/dist/src/webgl/shader-programs/compositeShader.d.ts +1 -0
- package/package.json +3 -4
- package/src/ProcessorWrapper.ts +66 -24
- package/src/index.ts +211 -42
- package/src/logger.ts +74 -0
- package/src/transformers/BackgroundTransformer.ts +22 -8
- package/src/transformers/VideoTransformer.ts +1 -1
- package/src/transformers/types.ts +11 -8
- package/src/webgl/index.ts +24 -11
- package/src/webgl/shader-programs/compositeShader.ts +18 -13
- package/src/webgl/utils.ts +6 -2
package/dist/index.js
CHANGED
|
@@ -26,12 +26,68 @@ function createCanvas(width, height) {
|
|
|
26
26
|
return canvas;
|
|
27
27
|
}
|
|
28
28
|
|
|
29
|
+
// src/logger.ts
|
|
30
|
+
var _livekitclient = require('livekit-client');
|
|
31
|
+
var LogLevel = /* @__PURE__ */ ((LogLevel2) => {
|
|
32
|
+
LogLevel2[LogLevel2["trace"] = 0] = "trace";
|
|
33
|
+
LogLevel2[LogLevel2["debug"] = 1] = "debug";
|
|
34
|
+
LogLevel2[LogLevel2["info"] = 2] = "info";
|
|
35
|
+
LogLevel2[LogLevel2["warn"] = 3] = "warn";
|
|
36
|
+
LogLevel2[LogLevel2["error"] = 4] = "error";
|
|
37
|
+
LogLevel2[LogLevel2["silent"] = 5] = "silent";
|
|
38
|
+
return LogLevel2;
|
|
39
|
+
})(LogLevel || {});
|
|
40
|
+
var LoggerNames = /* @__PURE__ */ ((LoggerNames2) => {
|
|
41
|
+
LoggerNames2["ProcessorWrapper"] = "livekit-processor-wrapper";
|
|
42
|
+
LoggerNames2["BackgroundProcessor"] = "livekit-background-processor";
|
|
43
|
+
LoggerNames2["WebGl"] = "livekit-track-processor-web-gl";
|
|
44
|
+
return LoggerNames2;
|
|
45
|
+
})(LoggerNames || {});
|
|
46
|
+
var livekitLogger = getLogger("livekit");
|
|
47
|
+
var livekitLoggers = Object.values(LoggerNames).map((name) => getLogger(name));
|
|
48
|
+
livekitLogger.setDefaultLevel(2 /* info */);
|
|
49
|
+
function getLogger(name) {
|
|
50
|
+
return _livekitclient.getLogger.call(void 0, name);
|
|
51
|
+
}
|
|
52
|
+
function setLogLevel(level, loggerName) {
|
|
53
|
+
if (loggerName) {
|
|
54
|
+
getLogger(loggerName).setLevel(level);
|
|
55
|
+
} else {
|
|
56
|
+
for (const logger of livekitLoggers) {
|
|
57
|
+
logger.setLevel(level);
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
function setLogExtension(extension, logger) {
|
|
62
|
+
const loggers = logger ? [logger] : livekitLoggers;
|
|
63
|
+
loggers.forEach((logR) => {
|
|
64
|
+
const originalFactory = logR.methodFactory;
|
|
65
|
+
logR.methodFactory = (methodName, configLevel, loggerName) => {
|
|
66
|
+
const rawMethod = originalFactory(methodName, configLevel, loggerName);
|
|
67
|
+
const logLevel = LogLevel[methodName];
|
|
68
|
+
const needLog = logLevel >= configLevel && logLevel < 5 /* silent */;
|
|
69
|
+
return (msg, context) => {
|
|
70
|
+
if (context)
|
|
71
|
+
rawMethod(msg, context);
|
|
72
|
+
else
|
|
73
|
+
rawMethod(msg);
|
|
74
|
+
if (needLog) {
|
|
75
|
+
extension(logLevel, msg, context);
|
|
76
|
+
}
|
|
77
|
+
};
|
|
78
|
+
};
|
|
79
|
+
logR.setLevel(logR.getLevel());
|
|
80
|
+
});
|
|
81
|
+
}
|
|
82
|
+
|
|
29
83
|
// src/ProcessorWrapper.ts
|
|
30
84
|
var ProcessorWrapper = class _ProcessorWrapper {
|
|
31
85
|
constructor(transformer, name, options = {}) {
|
|
32
86
|
// For tracking whether we're using the stream API fallback
|
|
33
87
|
this.useStreamFallback = false;
|
|
34
88
|
this.processingEnabled = false;
|
|
89
|
+
this.log = getLogger("livekit-processor-wrapper" /* ProcessorWrapper */);
|
|
90
|
+
this.lifecycleState = "idle";
|
|
35
91
|
var _a;
|
|
36
92
|
this.name = name;
|
|
37
93
|
this.transformer = transformer;
|
|
@@ -92,6 +148,8 @@ var ProcessorWrapper = class _ProcessorWrapper {
|
|
|
92
148
|
}
|
|
93
149
|
}
|
|
94
150
|
async init(opts) {
|
|
151
|
+
this.log.debug("Init called");
|
|
152
|
+
this.lifecycleState = "initializing";
|
|
95
153
|
await this.setup(opts);
|
|
96
154
|
if (!this.canvas) {
|
|
97
155
|
throw new TypeError("Expected canvas to be defined after setup");
|
|
@@ -105,6 +163,7 @@ var ProcessorWrapper = class _ProcessorWrapper {
|
|
|
105
163
|
} else {
|
|
106
164
|
this.initStreamProcessorPath();
|
|
107
165
|
}
|
|
166
|
+
this.lifecycleState = "running";
|
|
108
167
|
}
|
|
109
168
|
initStreamProcessorPath() {
|
|
110
169
|
if (!this.processor || !this.trackGenerator) {
|
|
@@ -114,14 +173,15 @@ var ProcessorWrapper = class _ProcessorWrapper {
|
|
|
114
173
|
}
|
|
115
174
|
const readableStream = this.processor.readable;
|
|
116
175
|
const pipedStream = readableStream.pipeThrough(this.transformer.transformer);
|
|
117
|
-
|
|
118
|
-
this.symbol = symbol;
|
|
119
|
-
pipedStream.pipeTo(this.trackGenerator.writable).then(() => this.destroy(symbol)).catch((e) => {
|
|
176
|
+
pipedStream.pipeTo(this.trackGenerator.writable).then(() => this.handleMediaExhausted()).catch((e) => {
|
|
120
177
|
if (e instanceof DOMException && e.name === "AbortError") {
|
|
121
|
-
|
|
178
|
+
this.log.log("stream processor path aborted");
|
|
179
|
+
} else if (e instanceof DOMException && e.name === "InvalidStateError" && e.message === "Stream closed") {
|
|
180
|
+
this.log.log("stream processor underlying stream closed");
|
|
181
|
+
this.handleMediaExhausted();
|
|
122
182
|
} else {
|
|
123
|
-
|
|
124
|
-
this.destroy(
|
|
183
|
+
this.log.error("error when trying to pipe", e);
|
|
184
|
+
this.destroy();
|
|
125
185
|
}
|
|
126
186
|
});
|
|
127
187
|
this.processedTrack = this.trackGenerator;
|
|
@@ -154,7 +214,7 @@ var ProcessorWrapper = class _ProcessorWrapper {
|
|
|
154
214
|
try {
|
|
155
215
|
this.transformer.transform(frame, controller);
|
|
156
216
|
} catch (e) {
|
|
157
|
-
|
|
217
|
+
this.log.error("Error in transform:", e);
|
|
158
218
|
frame.close();
|
|
159
219
|
}
|
|
160
220
|
};
|
|
@@ -162,6 +222,7 @@ var ProcessorWrapper = class _ProcessorWrapper {
|
|
|
162
222
|
}
|
|
163
223
|
startRenderLoop() {
|
|
164
224
|
if (!this.sourceDummy || !(this.sourceDummy instanceof HTMLVideoElement)) {
|
|
225
|
+
this.handleMediaExhausted();
|
|
165
226
|
return;
|
|
166
227
|
}
|
|
167
228
|
let lastVideoTimestamp = -1;
|
|
@@ -175,11 +236,14 @@ var ProcessorWrapper = class _ProcessorWrapper {
|
|
|
175
236
|
let lastFpsLog = 0;
|
|
176
237
|
const renderLoop = () => {
|
|
177
238
|
if (!this.processingEnabled || !this.sourceDummy || !(this.sourceDummy instanceof HTMLVideoElement)) {
|
|
239
|
+
this.handleMediaExhausted();
|
|
178
240
|
return;
|
|
179
241
|
}
|
|
180
242
|
if (this.sourceDummy.paused) {
|
|
181
|
-
|
|
182
|
-
this.sourceDummy.play()
|
|
243
|
+
this.log.warn("Video is paused, trying to play");
|
|
244
|
+
this.sourceDummy.play().then(() => {
|
|
245
|
+
this.animationFrameId = requestAnimationFrame(renderLoop);
|
|
246
|
+
});
|
|
183
247
|
return;
|
|
184
248
|
}
|
|
185
249
|
const videoTime = videoElement.currentTime;
|
|
@@ -198,7 +262,7 @@ var ProcessorWrapper = class _ProcessorWrapper {
|
|
|
198
262
|
estimatedVideoFps = 1e3 / avgFrameTime;
|
|
199
263
|
const isDevelopment = typeof window !== "undefined" && window.location.hostname === "localhost" || window.location.hostname === "127.0.0.1";
|
|
200
264
|
if (isDevelopment && now - lastFpsLog > 5e3) {
|
|
201
|
-
|
|
265
|
+
this.log.debug(
|
|
202
266
|
`[${this.name}] Estimated video FPS: ${estimatedVideoFps.toFixed(
|
|
203
267
|
1
|
|
204
268
|
)}, Processing at: ${(frameCount / 5).toFixed(1)} FPS`
|
|
@@ -225,7 +289,7 @@ var ProcessorWrapper = class _ProcessorWrapper {
|
|
|
225
289
|
}
|
|
226
290
|
}
|
|
227
291
|
} catch (e) {
|
|
228
|
-
|
|
292
|
+
this.log.error("Error in render loop:", e);
|
|
229
293
|
}
|
|
230
294
|
}
|
|
231
295
|
this.animationFrameId = requestAnimationFrame(renderLoop);
|
|
@@ -233,7 +297,8 @@ var ProcessorWrapper = class _ProcessorWrapper {
|
|
|
233
297
|
this.animationFrameId = requestAnimationFrame(renderLoop);
|
|
234
298
|
}
|
|
235
299
|
async restart(opts) {
|
|
236
|
-
|
|
300
|
+
this.log.debug("Restart called");
|
|
301
|
+
await this.destroy({ willProcessorRestart: true });
|
|
237
302
|
await this.init(opts);
|
|
238
303
|
}
|
|
239
304
|
async restartTransformer(...options) {
|
|
@@ -242,11 +307,18 @@ var ProcessorWrapper = class _ProcessorWrapper {
|
|
|
242
307
|
async updateTransformerOptions(...options) {
|
|
243
308
|
await this.transformer.update(options[0]);
|
|
244
309
|
}
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
310
|
+
/** Called if the media pipeline no longer can read frames to process from the source media */
|
|
311
|
+
async handleMediaExhausted() {
|
|
312
|
+
this.log.debug("Media was exhausted from source");
|
|
313
|
+
if (this.lifecycleState !== "running") {
|
|
248
314
|
return;
|
|
249
315
|
}
|
|
316
|
+
this.lifecycleState = "media-exhausted";
|
|
317
|
+
await this.cleanup();
|
|
318
|
+
}
|
|
319
|
+
/** Tears down the media stack logic initialized in initStreamProcessorPath / initFallbackPath */
|
|
320
|
+
async cleanup() {
|
|
321
|
+
var _a, _b, _c, _d;
|
|
250
322
|
if (this.useStreamFallback) {
|
|
251
323
|
this.processingEnabled = false;
|
|
252
324
|
if (this.animationFrameId) {
|
|
@@ -261,7 +333,20 @@ var ProcessorWrapper = class _ProcessorWrapper {
|
|
|
261
333
|
await ((_c = (_b = this.processor) == null ? void 0 : _b.writableControl) == null ? void 0 : _c.close());
|
|
262
334
|
(_d = this.trackGenerator) == null ? void 0 : _d.stop();
|
|
263
335
|
}
|
|
264
|
-
|
|
336
|
+
}
|
|
337
|
+
async destroy(transformerDestroyOptions = { willProcessorRestart: false }) {
|
|
338
|
+
this.log.debug(`Destroy called - lifecycleState=${this.lifecycleState}, transformerDestroyOptions=${JSON.stringify(transformerDestroyOptions)}`);
|
|
339
|
+
switch (this.lifecycleState) {
|
|
340
|
+
case "running":
|
|
341
|
+
case "media-exhausted":
|
|
342
|
+
this.lifecycleState = "destroying";
|
|
343
|
+
await this.cleanup();
|
|
344
|
+
await this.transformer.destroy(transformerDestroyOptions);
|
|
345
|
+
this.lifecycleState = "destroyed";
|
|
346
|
+
break;
|
|
347
|
+
default:
|
|
348
|
+
break;
|
|
349
|
+
}
|
|
265
350
|
}
|
|
266
351
|
};
|
|
267
352
|
|
|
@@ -274,6 +359,7 @@ var dependencies = {
|
|
|
274
359
|
};
|
|
275
360
|
|
|
276
361
|
// src/webgl/utils.ts
|
|
362
|
+
var log = getLogger("livekit-track-processor-web-gl" /* WebGl */);
|
|
277
363
|
function initTexture(gl, texIndex) {
|
|
278
364
|
const texRef = gl.TEXTURE0 + texIndex;
|
|
279
365
|
gl.activeTexture(texRef);
|
|
@@ -291,7 +377,7 @@ function createShader(gl, type, source) {
|
|
|
291
377
|
gl.shaderSource(shader, source);
|
|
292
378
|
gl.compileShader(shader);
|
|
293
379
|
if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) {
|
|
294
|
-
|
|
380
|
+
log.error("Shader compile failed:", gl.getShaderInfoLog(shader));
|
|
295
381
|
gl.deleteShader(shader);
|
|
296
382
|
throw new Error("Shader compile failed");
|
|
297
383
|
}
|
|
@@ -303,7 +389,7 @@ function createProgram(gl, vs, fs) {
|
|
|
303
389
|
gl.attachShader(program, fs);
|
|
304
390
|
gl.linkProgram(program);
|
|
305
391
|
if (!gl.getProgramParameter(program, gl.LINK_STATUS)) {
|
|
306
|
-
|
|
392
|
+
log.error("Program link failed:", gl.getProgramInfoLog(program));
|
|
307
393
|
throw new Error("Program link failed");
|
|
308
394
|
}
|
|
309
395
|
return program;
|
|
@@ -507,29 +593,33 @@ var compositeFragmentShader = glsl`#version 300 es
|
|
|
507
593
|
precision mediump float;
|
|
508
594
|
in vec2 texCoords;
|
|
509
595
|
uniform sampler2D background;
|
|
596
|
+
uniform bool disableBackground;
|
|
510
597
|
uniform sampler2D frame;
|
|
511
598
|
uniform sampler2D mask;
|
|
512
599
|
out vec4 fragColor;
|
|
513
600
|
|
|
514
601
|
void main() {
|
|
515
|
-
|
|
516
602
|
vec4 frameTex = texture(frame, texCoords);
|
|
517
|
-
vec4 bgTex = texture(background, texCoords);
|
|
518
603
|
|
|
519
|
-
|
|
604
|
+
if (disableBackground) {
|
|
605
|
+
fragColor = frameTex;
|
|
606
|
+
} else {
|
|
607
|
+
vec4 bgTex = texture(background, texCoords);
|
|
608
|
+
|
|
609
|
+
float maskVal = texture(mask, texCoords).r;
|
|
520
610
|
|
|
521
|
-
|
|
522
|
-
|
|
611
|
+
// Compute screen-space gradient to detect edge sharpness
|
|
612
|
+
float grad = length(vec2(dFdx(maskVal), dFdy(maskVal)));
|
|
523
613
|
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
// Create a smooth edge around binary transition
|
|
527
|
-
float smoothAlpha = smoothstep(0.5 - grad * edgeSoftness, 0.5 + grad * edgeSoftness, maskVal);
|
|
614
|
+
float edgeSoftness = 2.0; // higher = softer
|
|
528
615
|
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
616
|
+
// Create a smooth edge around binary transition
|
|
617
|
+
float smoothAlpha = smoothstep(0.5 - grad * edgeSoftness, 0.5 + grad * edgeSoftness, maskVal);
|
|
618
|
+
|
|
619
|
+
// Optional: preserve frame alpha, or override as fully opaque
|
|
620
|
+
vec4 blended = mix(bgTex, vec4(frameTex.rgb, 1.0), 1.0 - smoothAlpha);
|
|
621
|
+
fragColor = blended;
|
|
622
|
+
}
|
|
533
623
|
|
|
534
624
|
}
|
|
535
625
|
`;
|
|
@@ -544,6 +634,7 @@ function createCompositeProgram(gl) {
|
|
|
544
634
|
mask: gl.getUniformLocation(compositeProgram, "mask"),
|
|
545
635
|
frame: gl.getUniformLocation(compositeProgram, "frame"),
|
|
546
636
|
background: gl.getUniformLocation(compositeProgram, "background"),
|
|
637
|
+
disableBackground: gl.getUniformLocation(compositeProgram, "disableBackground"),
|
|
547
638
|
stepWidth: gl.getUniformLocation(compositeProgram, "u_stepWidth")
|
|
548
639
|
};
|
|
549
640
|
return {
|
|
@@ -613,6 +704,7 @@ function applyDownsampling(gl, inputTexture, downSampler, vertexBuffer, width, h
|
|
|
613
704
|
}
|
|
614
705
|
|
|
615
706
|
// src/webgl/index.ts
|
|
707
|
+
var log2 = getLogger("livekit-track-processor-web-gl" /* WebGl */);
|
|
616
708
|
var setupWebGL = (canvas) => {
|
|
617
709
|
const gl = canvas.getContext("webgl2", {
|
|
618
710
|
antialias: true,
|
|
@@ -622,7 +714,7 @@ var setupWebGL = (canvas) => {
|
|
|
622
714
|
let maskBlurRadius = 8;
|
|
623
715
|
const downsampleFactor = 4;
|
|
624
716
|
if (!gl) {
|
|
625
|
-
|
|
717
|
+
log2.error("Failed to create WebGL context");
|
|
626
718
|
return void 0;
|
|
627
719
|
}
|
|
628
720
|
gl.enable(gl.BLEND);
|
|
@@ -633,7 +725,8 @@ var setupWebGL = (canvas) => {
|
|
|
633
725
|
const {
|
|
634
726
|
mask: maskTextureLocation,
|
|
635
727
|
frame: frameTextureLocation,
|
|
636
|
-
background: bgTextureLocation
|
|
728
|
+
background: bgTextureLocation,
|
|
729
|
+
disableBackground: disableBackgroundLocation
|
|
637
730
|
} = composite.uniformLocations;
|
|
638
731
|
const blur = createBlurProgram(gl);
|
|
639
732
|
const blurProgram = blur.program;
|
|
@@ -672,11 +765,13 @@ var setupWebGL = (canvas) => {
|
|
|
672
765
|
createFramebuffer(gl, finalMaskTextures[0], canvas.width, canvas.height),
|
|
673
766
|
createFramebuffer(gl, finalMaskTextures[1], canvas.width, canvas.height)
|
|
674
767
|
];
|
|
768
|
+
let backgroundImageDisabled = false;
|
|
675
769
|
gl.useProgram(compositeProgram);
|
|
770
|
+
gl.uniform1i(disableBackgroundLocation, backgroundImageDisabled ? 1 : 0);
|
|
676
771
|
gl.uniform1i(bgTextureLocation, 0);
|
|
677
772
|
gl.uniform1i(frameTextureLocation, 1);
|
|
678
773
|
gl.uniform1i(maskTextureLocation, 2);
|
|
679
|
-
let customBackgroundImage =
|
|
774
|
+
let customBackgroundImage = null;
|
|
680
775
|
function renderFrame(frame) {
|
|
681
776
|
if (frame.codedWidth === 0 || finalMaskTextures.length === 0) {
|
|
682
777
|
return;
|
|
@@ -708,7 +803,7 @@ var setupWebGL = (canvas) => {
|
|
|
708
803
|
bgBlurFrameBuffers,
|
|
709
804
|
bgBlurTextures
|
|
710
805
|
);
|
|
711
|
-
} else {
|
|
806
|
+
} else if (customBackgroundImage) {
|
|
712
807
|
gl.activeTexture(gl.TEXTURE0);
|
|
713
808
|
gl.bindTexture(gl.TEXTURE_2D, bgTexture);
|
|
714
809
|
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, customBackgroundImage);
|
|
@@ -724,6 +819,7 @@ var setupWebGL = (canvas) => {
|
|
|
724
819
|
gl.activeTexture(gl.TEXTURE0);
|
|
725
820
|
gl.bindTexture(gl.TEXTURE_2D, backgroundTexture);
|
|
726
821
|
gl.uniform1i(bgTextureLocation, 0);
|
|
822
|
+
gl.uniform1i(disableBackgroundLocation, backgroundImageDisabled ? 1 : 0);
|
|
727
823
|
gl.activeTexture(gl.TEXTURE1);
|
|
728
824
|
gl.bindTexture(gl.TEXTURE_2D, frameTexture);
|
|
729
825
|
gl.uniform1i(frameTextureLocation, 1);
|
|
@@ -733,26 +829,30 @@ var setupWebGL = (canvas) => {
|
|
|
733
829
|
gl.drawArrays(gl.TRIANGLES, 0, 6);
|
|
734
830
|
}
|
|
735
831
|
async function setBackgroundImage(image) {
|
|
736
|
-
customBackgroundImage =
|
|
832
|
+
customBackgroundImage = null;
|
|
737
833
|
if (image) {
|
|
834
|
+
customBackgroundImage = getEmptyImageData();
|
|
738
835
|
try {
|
|
739
836
|
const croppedImage = await resizeImageToCover(image, canvas.width, canvas.height);
|
|
740
837
|
customBackgroundImage = croppedImage;
|
|
741
838
|
} catch (error) {
|
|
742
|
-
|
|
839
|
+
log2.error(
|
|
743
840
|
"Error processing background image, falling back to black background:",
|
|
744
841
|
error
|
|
745
842
|
);
|
|
746
843
|
}
|
|
844
|
+
gl.activeTexture(gl.TEXTURE0);
|
|
845
|
+
gl.bindTexture(gl.TEXTURE_2D, bgTexture);
|
|
846
|
+
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, customBackgroundImage);
|
|
747
847
|
}
|
|
748
|
-
gl.activeTexture(gl.TEXTURE0);
|
|
749
|
-
gl.bindTexture(gl.TEXTURE_2D, bgTexture);
|
|
750
|
-
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, customBackgroundImage);
|
|
751
848
|
}
|
|
752
849
|
function setBlurRadius(radius) {
|
|
753
850
|
blurRadius = radius ? Math.max(1, Math.floor(radius / downsampleFactor)) : null;
|
|
754
851
|
setBackgroundImage(null);
|
|
755
852
|
}
|
|
853
|
+
function setBackgroundDisabled(disabled) {
|
|
854
|
+
backgroundImageDisabled = disabled;
|
|
855
|
+
}
|
|
756
856
|
function updateMask(mask) {
|
|
757
857
|
const tempFramebuffers = [tempMaskFrameBuffer, finalMaskFrameBuffers[writeMaskIndex]];
|
|
758
858
|
const tempTextures = [tempMaskTexture, finalMaskTextures[writeMaskIndex]];
|
|
@@ -804,13 +904,13 @@ var setupWebGL = (canvas) => {
|
|
|
804
904
|
if (customBackgroundImage instanceof ImageBitmap) {
|
|
805
905
|
customBackgroundImage.close();
|
|
806
906
|
}
|
|
807
|
-
customBackgroundImage =
|
|
907
|
+
customBackgroundImage = null;
|
|
808
908
|
}
|
|
809
909
|
bgBlurTextures = [];
|
|
810
910
|
bgBlurFrameBuffers = [];
|
|
811
911
|
finalMaskTextures = [];
|
|
812
912
|
}
|
|
813
|
-
return { renderFrame, updateMask, setBackgroundImage, setBlurRadius, cleanup };
|
|
913
|
+
return { renderFrame, updateMask, setBackgroundImage, setBlurRadius, setBackgroundDisabled, cleanup };
|
|
814
914
|
};
|
|
815
915
|
|
|
816
916
|
// src/transformers/VideoTransformer.ts
|
|
@@ -863,6 +963,7 @@ var BackgroundProcessor = class extends VideoTransformer {
|
|
|
863
963
|
this.backgroundImageAndPath = null;
|
|
864
964
|
this.segmentationTimeMs = 0;
|
|
865
965
|
this.isFirstFrame = true;
|
|
966
|
+
this.log = getLogger("livekit-processor-wrapper" /* ProcessorWrapper */);
|
|
866
967
|
this.options = opts;
|
|
867
968
|
this.update(opts);
|
|
868
969
|
}
|
|
@@ -870,7 +971,7 @@ var BackgroundProcessor = class extends VideoTransformer {
|
|
|
870
971
|
return typeof OffscreenCanvas !== "undefined" && typeof VideoFrame !== "undefined" && typeof createImageBitmap !== "undefined" && !!document.createElement("canvas").getContext("webgl2");
|
|
871
972
|
}
|
|
872
973
|
async init({ outputCanvas, inputElement: inputVideo }) {
|
|
873
|
-
var _a, _b, _c, _d, _e, _f;
|
|
974
|
+
var _a, _b, _c, _d, _e, _f, _g, _h;
|
|
874
975
|
await super.init({ outputCanvas, inputElement: inputVideo });
|
|
875
976
|
const fileSet = await vision.FilesetResolver.forVisionTasks(
|
|
876
977
|
(_b = (_a = this.options.assetPaths) == null ? void 0 : _a.tasksVisionFileSet) != null ? _b : `https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@${dependencies["@mediapipe/tasks-vision"]}/wasm`
|
|
@@ -888,19 +989,22 @@ var BackgroundProcessor = class extends VideoTransformer {
|
|
|
888
989
|
});
|
|
889
990
|
if ((_e = this.options) == null ? void 0 : _e.imagePath) {
|
|
890
991
|
await this.loadAndSetBackground(this.options.imagePath).catch(
|
|
891
|
-
(err) =>
|
|
992
|
+
(err) => this.log.error("Error while loading processor background image: ", err)
|
|
892
993
|
);
|
|
893
994
|
}
|
|
894
|
-
if (this.options.blurRadius) {
|
|
995
|
+
if (typeof this.options.blurRadius === "number") {
|
|
895
996
|
(_f = this.gl) == null ? void 0 : _f.setBlurRadius(this.options.blurRadius);
|
|
896
997
|
}
|
|
998
|
+
(_h = this.gl) == null ? void 0 : _h.setBackgroundDisabled((_g = this.options.backgroundDisabled) != null ? _g : false);
|
|
897
999
|
}
|
|
898
|
-
async destroy() {
|
|
1000
|
+
async destroy(options) {
|
|
899
1001
|
var _a;
|
|
900
1002
|
await super.destroy();
|
|
901
1003
|
await ((_a = this.imageSegmenter) == null ? void 0 : _a.close());
|
|
902
1004
|
this.backgroundImageAndPath = null;
|
|
903
|
-
|
|
1005
|
+
if (!(options == null ? void 0 : options.willProcessorRestart)) {
|
|
1006
|
+
this.isFirstFrame = true;
|
|
1007
|
+
}
|
|
904
1008
|
}
|
|
905
1009
|
async loadAndSetBackground(path) {
|
|
906
1010
|
var _a, _b;
|
|
@@ -918,14 +1022,18 @@ var BackgroundProcessor = class extends VideoTransformer {
|
|
|
918
1022
|
(_b = this.gl) == null ? void 0 : _b.setBackgroundImage(this.backgroundImageAndPath.imageData);
|
|
919
1023
|
}
|
|
920
1024
|
async transform(frame, controller) {
|
|
921
|
-
var _a, _b;
|
|
1025
|
+
var _a, _b, _c, _d;
|
|
922
1026
|
let enqueuedFrame = false;
|
|
923
1027
|
try {
|
|
924
1028
|
if (!(frame instanceof VideoFrame) || frame.codedWidth === 0 || frame.codedHeight === 0) {
|
|
925
|
-
|
|
1029
|
+
this.log.debug("empty frame detected, ignoring");
|
|
926
1030
|
return;
|
|
927
1031
|
}
|
|
928
|
-
|
|
1032
|
+
let skipProcessingFrame = (_b = (_a = this.isDisabled) != null ? _a : this.options.backgroundDisabled) != null ? _b : false;
|
|
1033
|
+
if (typeof this.options.blurRadius !== "number" && typeof this.options.imagePath !== "string") {
|
|
1034
|
+
skipProcessingFrame = true;
|
|
1035
|
+
}
|
|
1036
|
+
if (skipProcessingFrame) {
|
|
929
1037
|
controller.enqueue(frame);
|
|
930
1038
|
enqueuedFrame = true;
|
|
931
1039
|
return;
|
|
@@ -976,13 +1084,13 @@ var BackgroundProcessor = class extends VideoTransformer {
|
|
|
976
1084
|
segmentationTimeMs: this.segmentationTimeMs,
|
|
977
1085
|
filterTimeMs
|
|
978
1086
|
};
|
|
979
|
-
(
|
|
1087
|
+
(_d = (_c = this.options).onFrameProcessed) == null ? void 0 : _d.call(_c, stats);
|
|
980
1088
|
} else {
|
|
981
1089
|
controller.enqueue(frame);
|
|
982
1090
|
}
|
|
983
1091
|
await segmentationPromise;
|
|
984
1092
|
} catch (e) {
|
|
985
|
-
|
|
1093
|
+
this.log.error("Error while processing frame: ", e);
|
|
986
1094
|
} finally {
|
|
987
1095
|
if (!enqueuedFrame) {
|
|
988
1096
|
frame.close();
|
|
@@ -990,7 +1098,7 @@ var BackgroundProcessor = class extends VideoTransformer {
|
|
|
990
1098
|
}
|
|
991
1099
|
}
|
|
992
1100
|
async update(opts) {
|
|
993
|
-
var _a, _b, _c;
|
|
1101
|
+
var _a, _b, _c, _d, _e;
|
|
994
1102
|
this.options = { ...this.options, ...opts };
|
|
995
1103
|
(_b = this.gl) == null ? void 0 : _b.setBlurRadius((_a = opts.blurRadius) != null ? _a : null);
|
|
996
1104
|
if (opts.imagePath) {
|
|
@@ -998,6 +1106,7 @@ var BackgroundProcessor = class extends VideoTransformer {
|
|
|
998
1106
|
} else {
|
|
999
1107
|
(_c = this.gl) == null ? void 0 : _c.setBackgroundImage(null);
|
|
1000
1108
|
}
|
|
1109
|
+
(_e = this.gl) == null ? void 0 : _e.setBackgroundDisabled((_d = opts.backgroundDisabled) != null ? _d : false);
|
|
1001
1110
|
}
|
|
1002
1111
|
async drawFrame(frame) {
|
|
1003
1112
|
var _a;
|
|
@@ -1014,9 +1123,136 @@ var BackgroundProcessor = class extends VideoTransformer {
|
|
|
1014
1123
|
};
|
|
1015
1124
|
|
|
1016
1125
|
// src/index.ts
|
|
1126
|
+
var DEFAULT_BLUR_RADIUS = 10;
|
|
1017
1127
|
var supportsBackgroundProcessors = () => BackgroundProcessor.isSupported && ProcessorWrapper.isSupported;
|
|
1018
1128
|
var supportsModernBackgroundProcessors = () => BackgroundProcessor.isSupported && ProcessorWrapper.hasModernApiSupport;
|
|
1019
|
-
var
|
|
1129
|
+
var BackgroundProcessorWrapper = class extends ProcessorWrapper {
|
|
1130
|
+
get mode() {
|
|
1131
|
+
const options = this.transformer.options;
|
|
1132
|
+
if (options.backgroundDisabled) {
|
|
1133
|
+
return "disabled";
|
|
1134
|
+
}
|
|
1135
|
+
if (typeof options.imagePath === "string" && typeof options.blurRadius === "undefined") {
|
|
1136
|
+
return "virtual-background";
|
|
1137
|
+
}
|
|
1138
|
+
if (typeof options.imagePath === "undefined") {
|
|
1139
|
+
return "background-blur";
|
|
1140
|
+
}
|
|
1141
|
+
return "legacy";
|
|
1142
|
+
}
|
|
1143
|
+
async switchTo(options) {
|
|
1144
|
+
var _a;
|
|
1145
|
+
switch (options.mode) {
|
|
1146
|
+
case "background-blur":
|
|
1147
|
+
await this.updateTransformerOptions({
|
|
1148
|
+
imagePath: void 0,
|
|
1149
|
+
blurRadius: (_a = options.blurRadius) != null ? _a : DEFAULT_BLUR_RADIUS,
|
|
1150
|
+
backgroundDisabled: false
|
|
1151
|
+
});
|
|
1152
|
+
break;
|
|
1153
|
+
case "virtual-background":
|
|
1154
|
+
await this.updateTransformerOptions({
|
|
1155
|
+
imagePath: options.imagePath,
|
|
1156
|
+
blurRadius: void 0,
|
|
1157
|
+
backgroundDisabled: false
|
|
1158
|
+
});
|
|
1159
|
+
break;
|
|
1160
|
+
case "disabled":
|
|
1161
|
+
await this.updateTransformerOptions({ imagePath: void 0, backgroundDisabled: true });
|
|
1162
|
+
break;
|
|
1163
|
+
}
|
|
1164
|
+
}
|
|
1165
|
+
};
|
|
1166
|
+
var BackgroundProcessor2 = (options, name = "background-processor") => {
|
|
1167
|
+
const isTransformerSupported = BackgroundProcessor.isSupported;
|
|
1168
|
+
const isProcessorSupported = ProcessorWrapper.isSupported;
|
|
1169
|
+
if (!isTransformerSupported) {
|
|
1170
|
+
throw new Error("Background transformer is not supported in this browser");
|
|
1171
|
+
}
|
|
1172
|
+
if (!isProcessorSupported) {
|
|
1173
|
+
throw new Error(
|
|
1174
|
+
"Neither MediaStreamTrackProcessor nor canvas.captureStream() fallback is supported in this browser"
|
|
1175
|
+
);
|
|
1176
|
+
}
|
|
1177
|
+
let transformer, processorOpts;
|
|
1178
|
+
switch (options.mode) {
|
|
1179
|
+
case "background-blur": {
|
|
1180
|
+
const {
|
|
1181
|
+
// eslint-disable-next-line no-unused-vars
|
|
1182
|
+
mode,
|
|
1183
|
+
blurRadius = DEFAULT_BLUR_RADIUS,
|
|
1184
|
+
segmenterOptions,
|
|
1185
|
+
assetPaths,
|
|
1186
|
+
onFrameProcessed,
|
|
1187
|
+
...rest
|
|
1188
|
+
} = options;
|
|
1189
|
+
processorOpts = rest;
|
|
1190
|
+
transformer = new BackgroundProcessor({
|
|
1191
|
+
blurRadius,
|
|
1192
|
+
segmenterOptions,
|
|
1193
|
+
assetPaths,
|
|
1194
|
+
onFrameProcessed
|
|
1195
|
+
});
|
|
1196
|
+
break;
|
|
1197
|
+
}
|
|
1198
|
+
case "virtual-background": {
|
|
1199
|
+
const {
|
|
1200
|
+
// eslint-disable-next-line no-unused-vars
|
|
1201
|
+
mode,
|
|
1202
|
+
imagePath,
|
|
1203
|
+
segmenterOptions,
|
|
1204
|
+
assetPaths,
|
|
1205
|
+
onFrameProcessed,
|
|
1206
|
+
...rest
|
|
1207
|
+
} = options;
|
|
1208
|
+
processorOpts = rest;
|
|
1209
|
+
transformer = new BackgroundProcessor({
|
|
1210
|
+
imagePath,
|
|
1211
|
+
segmenterOptions,
|
|
1212
|
+
assetPaths,
|
|
1213
|
+
onFrameProcessed
|
|
1214
|
+
});
|
|
1215
|
+
break;
|
|
1216
|
+
}
|
|
1217
|
+
case "disabled": {
|
|
1218
|
+
const {
|
|
1219
|
+
segmenterOptions,
|
|
1220
|
+
assetPaths,
|
|
1221
|
+
onFrameProcessed,
|
|
1222
|
+
...rest
|
|
1223
|
+
} = options;
|
|
1224
|
+
processorOpts = rest;
|
|
1225
|
+
transformer = new BackgroundProcessor({
|
|
1226
|
+
segmenterOptions,
|
|
1227
|
+
assetPaths,
|
|
1228
|
+
onFrameProcessed
|
|
1229
|
+
});
|
|
1230
|
+
break;
|
|
1231
|
+
}
|
|
1232
|
+
default: {
|
|
1233
|
+
const {
|
|
1234
|
+
blurRadius,
|
|
1235
|
+
imagePath,
|
|
1236
|
+
segmenterOptions,
|
|
1237
|
+
assetPaths,
|
|
1238
|
+
onFrameProcessed,
|
|
1239
|
+
...rest
|
|
1240
|
+
} = options;
|
|
1241
|
+
processorOpts = rest;
|
|
1242
|
+
transformer = new BackgroundProcessor({
|
|
1243
|
+
blurRadius,
|
|
1244
|
+
imagePath,
|
|
1245
|
+
segmenterOptions,
|
|
1246
|
+
assetPaths,
|
|
1247
|
+
onFrameProcessed
|
|
1248
|
+
});
|
|
1249
|
+
break;
|
|
1250
|
+
}
|
|
1251
|
+
}
|
|
1252
|
+
const processor = new BackgroundProcessorWrapper(transformer, name, processorOpts);
|
|
1253
|
+
return processor;
|
|
1254
|
+
};
|
|
1255
|
+
var BackgroundBlur = (blurRadius = DEFAULT_BLUR_RADIUS, segmenterOptions, onFrameProcessed, processorOptions) => {
|
|
1020
1256
|
return BackgroundProcessor2(
|
|
1021
1257
|
{
|
|
1022
1258
|
blurRadius,
|
|
@@ -1038,35 +1274,6 @@ var VirtualBackground = (imagePath, segmenterOptions, onFrameProcessed, processo
|
|
|
1038
1274
|
"virtual-background"
|
|
1039
1275
|
);
|
|
1040
1276
|
};
|
|
1041
|
-
var BackgroundProcessor2 = (options, name = "background-processor") => {
|
|
1042
|
-
const isTransformerSupported = BackgroundProcessor.isSupported;
|
|
1043
|
-
const isProcessorSupported = ProcessorWrapper.isSupported;
|
|
1044
|
-
if (!isTransformerSupported) {
|
|
1045
|
-
throw new Error("Background transformer is not supported in this browser");
|
|
1046
|
-
}
|
|
1047
|
-
if (!isProcessorSupported) {
|
|
1048
|
-
throw new Error(
|
|
1049
|
-
"Neither MediaStreamTrackProcessor nor canvas.captureStream() fallback is supported in this browser"
|
|
1050
|
-
);
|
|
1051
|
-
}
|
|
1052
|
-
const {
|
|
1053
|
-
blurRadius,
|
|
1054
|
-
imagePath,
|
|
1055
|
-
segmenterOptions,
|
|
1056
|
-
assetPaths,
|
|
1057
|
-
onFrameProcessed,
|
|
1058
|
-
...processorOpts
|
|
1059
|
-
} = options;
|
|
1060
|
-
const transformer = new BackgroundProcessor({
|
|
1061
|
-
blurRadius,
|
|
1062
|
-
imagePath,
|
|
1063
|
-
segmenterOptions,
|
|
1064
|
-
assetPaths,
|
|
1065
|
-
onFrameProcessed
|
|
1066
|
-
});
|
|
1067
|
-
const processor = new ProcessorWrapper(transformer, name, processorOpts);
|
|
1068
|
-
return processor;
|
|
1069
|
-
};
|
|
1070
1277
|
|
|
1071
1278
|
|
|
1072
1279
|
|
|
@@ -1076,5 +1283,11 @@ var BackgroundProcessor2 = (options, name = "background-processor") => {
|
|
|
1076
1283
|
|
|
1077
1284
|
|
|
1078
1285
|
|
|
1079
|
-
|
|
1286
|
+
|
|
1287
|
+
|
|
1288
|
+
|
|
1289
|
+
|
|
1290
|
+
|
|
1291
|
+
|
|
1292
|
+
exports.BackgroundBlur = BackgroundBlur; exports.BackgroundProcessor = BackgroundProcessor2; exports.BackgroundProcessorWrapper = BackgroundProcessorWrapper; exports.BackgroundTransformer = BackgroundProcessor; exports.LogLevel = LogLevel; exports.LoggerNames = LoggerNames; exports.ProcessorWrapper = ProcessorWrapper; exports.VideoTransformer = VideoTransformer; exports.VirtualBackground = VirtualBackground; exports.getLogger = getLogger; exports.setLogExtension = setLogExtension; exports.setLogLevel = setLogLevel; exports.supportsBackgroundProcessors = supportsBackgroundProcessors; exports.supportsModernBackgroundProcessors = supportsModernBackgroundProcessors;
|
|
1080
1293
|
//# sourceMappingURL=index.js.map
|