@livekit/track-processors 0.2.4 → 0.2.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +12 -12
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +12 -12
- package/dist/index.mjs.map +1 -1
- package/dist/src/index.d.ts +1 -0
- package/package.json +3 -3
- package/src/ProcessorPipeline.ts +2 -2
- package/src/index.ts +2 -0
- package/src/transformers/BackgroundTransformer.ts +7 -7
package/dist/index.js
CHANGED
|
@@ -28,7 +28,7 @@ var ProcessorPipeline = class {
|
|
|
28
28
|
this.transformers = transformers;
|
|
29
29
|
}
|
|
30
30
|
async init(opts) {
|
|
31
|
-
var _a, _b;
|
|
31
|
+
var _a, _b, _c, _d;
|
|
32
32
|
this.source = opts.track;
|
|
33
33
|
const origConstraints = this.source.getConstraints();
|
|
34
34
|
await this.source.applyConstraints(__spreadProps(__spreadValues({}, origConstraints), {
|
|
@@ -41,8 +41,8 @@ var ProcessorPipeline = class {
|
|
|
41
41
|
this.sourceSettings = this.source.getSettings();
|
|
42
42
|
this.sourceDummy = opts.element;
|
|
43
43
|
if (this.sourceDummy instanceof HTMLVideoElement) {
|
|
44
|
-
this.sourceDummy.height = this.sourceSettings.height;
|
|
45
|
-
this.sourceDummy.width = this.sourceSettings.width;
|
|
44
|
+
this.sourceDummy.height = (_a = this.sourceSettings.height) != null ? _a : 300;
|
|
45
|
+
this.sourceDummy.width = (_b = this.sourceSettings.width) != null ? _b : 300;
|
|
46
46
|
}
|
|
47
47
|
if (!(this.sourceDummy instanceof HTMLVideoElement)) {
|
|
48
48
|
throw TypeError("Currently only video transformers are supported");
|
|
@@ -53,8 +53,8 @@ var ProcessorPipeline = class {
|
|
|
53
53
|
signalTarget: this.source
|
|
54
54
|
});
|
|
55
55
|
this.canvas = new OffscreenCanvas(
|
|
56
|
-
(
|
|
57
|
-
(
|
|
56
|
+
(_c = this.sourceSettings.width) != null ? _c : 300,
|
|
57
|
+
(_d = this.sourceSettings.height) != null ? _d : 300
|
|
58
58
|
);
|
|
59
59
|
let readableStream = this.processor.readable;
|
|
60
60
|
for (const transformer of this.transformers) {
|
|
@@ -201,10 +201,10 @@ var BackgroundProcessor = class extends VideoTransformer {
|
|
|
201
201
|
this.ctx.globalCompositeOperation = "copy";
|
|
202
202
|
const bitmap = await maskToBitmap(
|
|
203
203
|
this.segmentationResults.categoryMask,
|
|
204
|
-
this.
|
|
205
|
-
this.
|
|
204
|
+
this.segmentationResults.categoryMask.width,
|
|
205
|
+
this.segmentationResults.categoryMask.height
|
|
206
206
|
);
|
|
207
|
-
this.ctx.drawImage(bitmap, 0, 0);
|
|
207
|
+
this.ctx.drawImage(bitmap, 0, 0, this.canvas.width, this.canvas.height);
|
|
208
208
|
this.ctx.filter = "none";
|
|
209
209
|
this.ctx.globalCompositeOperation = "source-in";
|
|
210
210
|
if (this.backgroundImage) {
|
|
@@ -236,18 +236,18 @@ var BackgroundProcessor = class extends VideoTransformer {
|
|
|
236
236
|
this.ctx.globalCompositeOperation = "copy";
|
|
237
237
|
const bitmap = await maskToBitmap(
|
|
238
238
|
this.segmentationResults.categoryMask,
|
|
239
|
-
this.
|
|
240
|
-
this.
|
|
239
|
+
this.segmentationResults.categoryMask.width,
|
|
240
|
+
this.segmentationResults.categoryMask.height
|
|
241
241
|
);
|
|
242
242
|
this.ctx.filter = "blur(3px)";
|
|
243
243
|
this.ctx.globalCompositeOperation = "copy";
|
|
244
|
-
this.ctx.drawImage(bitmap, 0, 0);
|
|
244
|
+
this.ctx.drawImage(bitmap, 0, 0, this.canvas.width, this.canvas.height);
|
|
245
245
|
this.ctx.filter = "none";
|
|
246
246
|
this.ctx.globalCompositeOperation = "source-out";
|
|
247
247
|
this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);
|
|
248
248
|
this.ctx.globalCompositeOperation = "destination-over";
|
|
249
249
|
this.ctx.filter = `blur(${this.blurRadius}px)`;
|
|
250
|
-
this.ctx.drawImage(frame, 0, 0);
|
|
250
|
+
this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);
|
|
251
251
|
this.ctx.restore();
|
|
252
252
|
}
|
|
253
253
|
};
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/ProcessorPipeline.ts","../src/transformers/BackgroundTransformer.ts","../package.json","../src/transformers/VideoTransformer.ts","../src/transformers/DummyTransformer.ts","../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;AAGA,IAAqB,oBAArB,MAA6E;AAAA,EAC3E,WAAW,cAAc;AACvB,WACE,OAAO,8BAA8B,eACrC,OAAO,8BAA8B;AAAA,EAEzC;AAAA,EAoBA,YAAY,cAA4C,MAAc;AACpE,SAAK,OAAO;AACZ,SAAK,eAAe;AAAA,EACtB;AAAA,EAEA,MAAM,KAAK,MAAoC;AAlCjD;AAmCI,SAAK,SAAS,KAAK;AACnB,UAAM,kBAAkB,KAAK,OAAO,eAAe;AACnD,UAAM,KAAK,OAAO,iBAAiB,iCAC9B,kBAD8B;AAAA;AAAA;AAAA;AAAA;AAAA,MAMjC,YAAY;AAAA,IACd,EAAC;AACD,SAAK,iBAAiB,KAAK,OAAO,YAAY;AAC9C,SAAK,cAAc,KAAK;AACxB,QAAI,KAAK,uBAAuB,kBAAkB;AAChD,WAAK,YAAY,SAAS,KAAK,eAAe;AAC9C,WAAK,YAAY,QAAQ,KAAK,eAAe;AAAA,IAC/C;AACA,QAAI,EAAE,KAAK,uBAAuB,mBAAmB;AACnD,YAAM,UAAU,iDAAiD;AAAA,IACnE;AAEA,SAAK,YAAY,IAAI,0BAA0B,EAAE,OAAO,KAAK,OAAO,CAAC;AAErE,SAAK,iBAAiB,IAAI,0BAA0B;AAAA,MAClD,MAAM;AAAA,MACN,cAAc,KAAK;AAAA,IACrB,CAAC;AAED,SAAK,SAAS,IAAI;AAAA,OAChB,UAAK,eAAe,UAApB,YAA6B;AAAA,OAC7B,UAAK,eAAe,WAApB,YAA8B;AAAA,IAChC;AAEA,QAAI,iBAAiB,KAAK,UAAU;AACpC,eAAW,eAAe,KAAK,cAAc;AAC3C,YAAM,YAAY,KAAK;AAAA,QACrB,cAAc,KAAK;AAAA,QACnB,cAAc,KAAK;AAAA,MACrB,CAAC;AACD,uBAAiB,eAAe,YAAY,YAAa,WAAY;AAAA,IACvE;AACA,mBACG,OAAO,KAAK,eAAe,QAAQ,EACnC,MAAM,CAAC,MAAM,QAAQ,MAAM,6BAA6B,CAAC,CAAC,EAC1D,QAAQ,MAAM,KAAK,QAAQ,CAAC;AAC/B,SAAK,iBAAiB,KAAK;AAAA,EAC7B;AAAA,EAEA,MAAM,UAAU;AAlFlB;AAmFI,eAAW,eAAe,KAAK,cAAc;AAC3C,YAAM,YAAY,QAAQ;AAAA,IAC5B;AACA,eAAK,mBAAL,mBAAqB;AAAA,EACvB;AACF;;;ACxFA,YAAY,YAAY;;;ACwBtB,mBAAgB;AAAA,EACd,uBAAuB;AAAA,EACvB,2BAA2B;AAC7B;;;ACzBF,IAA8B,mBAA9B,MAAgF;AAAA,EAAhF;AASE,SAAU,aAAuB;AAAA;AAAA,EAEjC,MAAM,KAAK;AAAA,IACT;AAAA,IACA,cAAc;AAAA,EAChB,GAA+C;AAhBjD;AAiBI,QAAI,EAAE,sBAAsB,mBAAmB;AAC7C,YAAM,UAAU,qDAAqD;AAAA,IACvE;AACA,SAAK,cAAc,IAAI,gBAAgB;AAAA,MACrC,WAAW,CAAC,OAAO,eAAe,KAAK,UAAU,OAAO,UAAU;AAAA,IACpE,CAAC;AACD,SAAK,SAAS,gBAAgB;AAC9B,QAAI,cAAc;AAChB,WAAK,QAAM,UAAK,WAAL,mBAAa,WAAW,UAAS;AAAA,IAC9C;AACA,SAAK,aAAa;AAClB,SAAK,aAAa;AAAA,EACpB;AAAA,EAEA,MAAM,UAAU;AACd,SAAK,aAAa;AAClB,SAAK,SAAS;AACd,SAAK,MAAM;AAAA,EACb;AAMF;;;AF5BA,IAAqB,sBAArB,cAAiD,iBAAiB;AAAA,EAehE,YAAY,MAAyB;AACnC,UAAM;AAPR,2BAAsC;AAQpC,SAAK,UAAU;AACf,QAAI,KAAK,YAAY;AACnB,WAAK,aAAa,KAAK;AAAA,IACzB,WAAW,KAAK,WAAW;AACzB,WAAK,eAAe,KAAK,SAAS;AAAA,IACpC;AAEA,SAAK,QAAQ;AAAA,EACf;AAAA,EAxBA,WAAW,cAAc;AACvB,WAAO,OAAO,oBAAoB;AAAA,EACpC;AAAA,EAwBA,MAAM,KAAK,EAAE,cAAc,cAAc,WAAW,GAAgC;AAClF,UAAM,MAAM,KAAK,EAAE,cAAc,cAAc,WAAW,CAAC;AAE3D,UAAM,UAAU,MAAa,uBAAgB;AAAA,MAC3C,wDAAwD,aAAa,yBAAyB,CAAC;AAAA,IACjG;AAEA,SAAK,iBAAiB,MAAa,sBAAe,kBAAkB,SAAS;AAAA,MAC3E,aAAa;AAAA,QACX,gBACE;AAAA,QACF,UAAU;AAAA,SACP,KAAK,QAAQ;AAAA,MAElB,aAAa;AAAA,MACb,oBAAoB;AAAA,MACpB,uBAAuB;AAAA,IACzB,CAAC;AAAA,EAGH;AAAA,EAEA,MAAM,UAAU;AA9DlB;AA+DI,UAAM,MAAM,QAAQ;AACpB,YAAM,UAAK,mBAAL,mBAAqB;AAC3B,SAAK,kBAAkB;AAAA,EACzB;AAAA,EAEA,MAAM,eAAe,MAAc;AACjC,UAAM,MAAM,IAAI,MAAM;AAEtB,UAAM,IAAI,QAAQ,CAAC,SAAS,WAAW;AACrC,UAAI,cAAc;AAClB,UAAI,SAAS,MAAM,QAAQ,GAAG;AAC9B,UAAI,UAAU,CAAC,QAAQ,OAAO,GAAG;AACjC,UAAI,MAAM;AAAA,IACZ,CAAC;AACD,UAAM,YAAY,MAAM,kBAAkB,GAAG;AAC7C,SAAK,kBAAkB;AAAA,EACzB;AAAA,EAEA,MAAM,UAAU,OAAmB,YAA0D;AAjF/F;AAkFI,QAAI;AACF,UAAI,KAAK,YAAY;AACnB,mBAAW,QAAQ,KAAK;AACxB;AAAA,MACF;AACA,UAAI,CAAC,KAAK,QAAQ;AAChB,cAAM,UAAU,sCAAsC;AAAA,MACxD;AACA,UAAI,cAAc,YAAY,IAAI;AAClC,iBAAK,mBAAL,mBAAqB;AAAA,QACnB,KAAK;AAAA,QACL;AAAA,QACA,CAAC,WAAY,KAAK,sBAAsB;AAAA;AAG1C,UAAI,KAAK,YAAY;AACnB,cAAM,KAAK,eAAe,KAAK;AAAA,MACjC,OAAO;AACL,cAAM,KAAK,sBAAsB,KAAK;AAAA,MACxC;AACA,YAAM,WAAW,IAAI,WAAW,KAAK,QAAQ;AAAA,QAC3C,WAAW,MAAM,aAAa,KAAK,IAAI;AAAA,MACzC,CAAC;AACD,iBAAW,QAAQ,QAAQ;AAAA,IAC7B,UAAE;AACA,YAAM,MAAM;AAAA,IACd;AAAA,EACF;AAAA,EAEA,MAAM,sBAAsB,OAAmB;AA/GjD;AAgHI,QAAI,CAAC,KAAK,UAAU,CAAC,KAAK,OAAO,CAAC,KAAK,uBAAuB,CAAC,KAAK;AAAY;AAGhF,SAAI,UAAK,wBAAL,mBAA0B,cAAc;AAC1C,WAAK,IAAI,SAAS;AAClB,WAAK,IAAI,2BAA2B;AACpC,YAAM,SAAS,MAAM;AAAA,QACnB,KAAK,oBAAoB;AAAA,QACzB,KAAK,WAAW;AAAA,QAChB,KAAK,WAAW;AAAA,MAClB;AACA,WAAK,IAAI,UAAU,QAAQ,GAAG,CAAC;AAC/B,WAAK,IAAI,SAAS;AAClB,WAAK,IAAI,2BAA2B;AACpC,UAAI,KAAK,iBAAiB;AACxB,aAAK,IAAI;AAAA,UACP,KAAK;AAAA,UACL;AAAA,UACA;AAAA,UACA,KAAK,gBAAgB;AAAA,UACrB,KAAK,gBAAgB;AAAA,UACrB;AAAA,UACA;AAAA,UACA,KAAK,OAAO;AAAA,UACZ,KAAK,OAAO;AAAA,QACd;AAAA,MACF,OAAO;AACL,aAAK,IAAI,YAAY;AACrB,aAAK,IAAI,SAAS,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AAAA,MAC/D;AAEA,WAAK,IAAI,2BAA2B;AAAA,IACtC;AACA,SAAK,IAAI,UAAU,OAAO,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AAAA,EAEvE;AAAA,EAEA,MAAM,eAAe,OAAmB;AArJ1C;AAsJI,QACE,CAAC,KAAK,OACN,CAAC,KAAK,UACN,GAAC,gBAAK,wBAAL,mBAA0B,iBAA1B,mBAAwC,WACzC,CAAC,KAAK,YACN;AACA;AAAA,IACF;AAEA,SAAK,IAAI,KAAK;AACd,SAAK,IAAI,2BAA2B;AAEpC,UAAM,SAAS,MAAM;AAAA,MACnB,KAAK,oBAAoB;AAAA,MACzB,KAAK,WAAW;AAAA,MAChB,KAAK,WAAW;AAAA,IAClB;AAEA,SAAK,IAAI,SAAS;AAClB,SAAK,IAAI,2BAA2B;AACpC,SAAK,IAAI,UAAU,QAAQ,GAAG,CAAC;AAC/B,SAAK,IAAI,SAAS;AAClB,SAAK,IAAI,2BAA2B;AACpC,SAAK,IAAI,UAAU,OAAO,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AACrE,SAAK,IAAI,2BAA2B;AACpC,SAAK,IAAI,SAAS,QAAQ,KAAK,UAAU;AACzC,SAAK,IAAI,UAAU,OAAO,GAAG,CAAC;AAC9B,SAAK,IAAI,QAAQ;AAAA,EACnB;AACF;AAEA,SAAS,aACP,MACA,YACA,aACsB;AACtB,QAAM,YAA+B,IAAI,kBAAkB,aAAa,cAAc,CAAC;AACvF,QAAM,SAAS,KAAK,gBAAgB;AACpC,WAAS,IAAI,GAAG,IAAI,OAAO,QAAQ,KAAK,GAAG;AACzC,cAAU,IAAI,CAAC,IAAI,OAAO,CAAC;AAC3B,cAAU,IAAI,IAAI,CAAC,IAAI,OAAO,CAAC;AAC/B,cAAU,IAAI,IAAI,CAAC,IAAI,OAAO,CAAC;AAC/B,cAAU,IAAI,IAAI,CAAC,IAAI,OAAO,CAAC;AAAA,EACjC;AACA,QAAM,UAAU,IAAI,UAAU,WAAW,YAAY,WAAW;AAEhE,SAAO,kBAAkB,OAAO;AAClC;;;AGnMA,IAAqB,mBAArB,cAA8C,iBAAiB;AAAA,EAC7D,MAAM,UAAU,OAAmB,YAA0D;AAC3F,eAAW,QAAQ,KAAK;AAAA,EAC1B;AAAA,EAEA,MAAM,UAAU;AAAA,EAEhB;AACF;;;ACNO,IAAM,iBAAiB,CAC5B,aAAqB,IACrB,qBACG;AACH,QAAM,sBAAsB,kBAAkB,eAAe,oBAAsB;AACnF,MAAI,CAAC,qBAAqB;AACxB,UAAM,IAAI,MAAM,2CAA2C;AAAA,EAC7D;AACA,QAAM,WAAW,IAAI;AAAA,IACnB,CAAC,IAAI,oBAAsB,EAAE,YAAY,iBAAiB,CAAC,CAAC;AAAA,IAC5D;AAAA,EACF;AACA,SAAO;AACT;AAEO,IAAM,oBAAoB,CAAC,WAAmB,qBAA4C;AAC/F,QAAM,sBAAsB,kBAAkB,eAAe,oBAAsB;AACnF,MAAI,CAAC,qBAAqB;AACxB,UAAM,IAAI,MAAM,2CAA2C;AAAA,EAC7D;AACA,QAAM,WAAW,IAAI;AAAA,IACnB,CAAC,IAAI,oBAAsB,EAAE,WAAW,iBAAiB,CAAC,CAAC;AAAA,IAC3D;AAAA,EACF;AACA,SAAO;AACT;AAEO,IAAM,QAAQ,MAAM;AACzB,QAAM,sBAAsB,kBAAkB,eAAe,oBAAsB;AACnF,MAAI,CAAC,qBAAqB;AACxB,UAAM,IAAI,MAAM,2CAA2C;AAAA,EAC7D;AACA,QAAM,WAAW,IAAI,kBAAkB,CAAC,IAAI,iBAAiB,CAAC,GAAG,OAAO;AACxE,SAAO;AACT","sourcesContent":["import type { ProcessorOptions, Track, TrackProcessor } from 'livekit-client';\nimport { VideoTrackTransformer } from './transformers';\n\nexport default class ProcessorPipeline implements TrackProcessor<Track.Kind> {\n static get isSupported() {\n return (\n typeof MediaStreamTrackGenerator !== 'undefined' &&\n typeof MediaStreamTrackProcessor !== 'undefined'\n );\n }\n\n name: string;\n\n source?: MediaStreamVideoTrack;\n\n sourceSettings?: MediaTrackSettings;\n\n processor?: MediaStreamTrackProcessor<VideoFrame>;\n\n trackGenerator?: MediaStreamTrackGenerator<VideoFrame>;\n\n canvas?: OffscreenCanvas;\n\n sourceDummy?: HTMLMediaElement;\n\n processedTrack?: MediaStreamTrack;\n\n transformers: Array<VideoTrackTransformer>;\n\n constructor(transformers: Array<VideoTrackTransformer>, name: string) {\n this.name = name;\n this.transformers = transformers;\n }\n\n async init(opts: ProcessorOptions<Track.Kind>) {\n this.source = opts.track as MediaStreamVideoTrack;\n const origConstraints = this.source.getConstraints();\n await this.source.applyConstraints({\n ...origConstraints,\n // @ts-expect-error when a mediastream track is resized and/or cropped, the `VideoFrame` will have a coded height/width of the original video size\n // this leads to a shift of the underlying video as the frame itself is being rendered with the coded size\n // but image segmentation is based on the display dimensions (-> the cropped version)\n // in order to prevent this, we force the resize mode to \"none\"\n resizeMode: 'none',\n });\n this.sourceSettings = this.source.getSettings();\n this.sourceDummy = opts.element;\n if (this.sourceDummy instanceof HTMLVideoElement) {\n this.sourceDummy.height = this.sourceSettings.height!;\n this.sourceDummy.width = this.sourceSettings.width!;\n }\n if (!(this.sourceDummy instanceof HTMLVideoElement)) {\n throw TypeError('Currently only video transformers are supported');\n }\n // TODO explore if we can do all the processing work in a webworker\n this.processor = new MediaStreamTrackProcessor({ track: this.source });\n\n this.trackGenerator = new MediaStreamTrackGenerator({\n kind: 'video',\n signalTarget: this.source,\n });\n\n this.canvas = new OffscreenCanvas(\n this.sourceSettings.width ?? 300,\n this.sourceSettings.height ?? 300,\n );\n\n let readableStream = this.processor.readable;\n for (const transformer of this.transformers) {\n await transformer.init({\n outputCanvas: this.canvas,\n inputElement: this.sourceDummy!,\n });\n readableStream = readableStream.pipeThrough(transformer!.transformer!);\n }\n readableStream\n .pipeTo(this.trackGenerator.writable)\n .catch((e) => console.error('error when trying to pipe', e))\n .finally(() => this.destroy());\n this.processedTrack = this.trackGenerator as MediaStreamVideoTrack;\n }\n\n async destroy() {\n for (const transformer of this.transformers) {\n await transformer.destroy();\n }\n this.trackGenerator?.stop();\n }\n}\n","import * as vision from '@mediapipe/tasks-vision';\nimport { dependencies } from '../../package.json';\nimport VideoTransformer from './VideoTransformer';\nimport { VideoTransformerInitOptions } from './types';\n\nexport type SegmenterBaseOptions = Partial<vision.ImageSegmenterOptions['baseOptions']>;\n\nexport type BackgroundOptions = {\n blurRadius?: number;\n imagePath?: string;\n segmenterOptions?: SegmenterBaseOptions;\n};\n\nexport default class BackgroundProcessor extends VideoTransformer {\n static get isSupported() {\n return typeof OffscreenCanvas !== 'undefined';\n }\n\n imageSegmenter?: vision.ImageSegmenter;\n\n segmentationResults: vision.ImageSegmenterResult | undefined;\n\n backgroundImage: ImageBitmap | null = null;\n\n blurRadius?: number;\n\n options: BackgroundOptions;\n\n constructor(opts: BackgroundOptions) {\n super();\n this.options = opts;\n if (opts.blurRadius) {\n this.blurRadius = opts.blurRadius;\n } else if (opts.imagePath) {\n this.loadBackground(opts.imagePath);\n }\n\n this.options.segmenterOptions;\n }\n\n async init({ outputCanvas, inputElement: inputVideo }: VideoTransformerInitOptions) {\n await super.init({ outputCanvas, inputElement: inputVideo });\n\n const fileSet = await vision.FilesetResolver.forVisionTasks(\n `https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@${dependencies['@mediapipe/tasks-vision']}/wasm`,\n );\n\n this.imageSegmenter = await vision.ImageSegmenter.createFromOptions(fileSet, {\n baseOptions: {\n modelAssetPath:\n 'https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_segmenter/float16/latest/selfie_segmenter.tflite',\n delegate: 'GPU',\n ...this.options.segmenterOptions,\n },\n runningMode: 'VIDEO',\n outputCategoryMask: true,\n outputConfidenceMasks: false,\n });\n\n // this.loadBackground(opts.backgroundUrl).catch((e) => console.error(e));\n }\n\n async destroy() {\n await super.destroy();\n await this.imageSegmenter?.close();\n this.backgroundImage = null;\n }\n\n async loadBackground(path: string) {\n const img = new Image();\n\n await new Promise((resolve, reject) => {\n img.crossOrigin = 'Anonymous';\n img.onload = () => resolve(img);\n img.onerror = (err) => reject(err);\n img.src = path;\n });\n const imageData = await createImageBitmap(img);\n this.backgroundImage = imageData;\n }\n\n async transform(frame: VideoFrame, controller: TransformStreamDefaultController<VideoFrame>) {\n try {\n if (this.isDisabled) {\n controller.enqueue(frame);\n return;\n }\n if (!this.canvas) {\n throw TypeError('Canvas needs to be initialized first');\n }\n let startTimeMs = performance.now();\n this.imageSegmenter?.segmentForVideo(\n this.inputVideo!,\n startTimeMs,\n (result) => (this.segmentationResults = result),\n );\n\n if (this.blurRadius) {\n await this.blurBackground(frame);\n } else {\n await this.drawVirtualBackground(frame);\n }\n const newFrame = new VideoFrame(this.canvas, {\n timestamp: frame.timestamp || Date.now(),\n });\n controller.enqueue(newFrame);\n } finally {\n frame.close();\n }\n }\n\n async drawVirtualBackground(frame: VideoFrame) {\n if (!this.canvas || !this.ctx || !this.segmentationResults || !this.inputVideo) return;\n // this.ctx.save();\n // this.ctx.clearRect(0, 0, this.canvas.width, this.canvas.height);\n if (this.segmentationResults?.categoryMask) {\n this.ctx.filter = 'blur(3px)';\n this.ctx.globalCompositeOperation = 'copy';\n const bitmap = await maskToBitmap(\n this.segmentationResults.categoryMask,\n this.inputVideo.videoWidth,\n this.inputVideo.videoHeight,\n );\n this.ctx.drawImage(bitmap, 0, 0);\n this.ctx.filter = 'none';\n this.ctx.globalCompositeOperation = 'source-in';\n if (this.backgroundImage) {\n this.ctx.drawImage(\n this.backgroundImage,\n 0,\n 0,\n this.backgroundImage.width,\n this.backgroundImage.height,\n 0,\n 0,\n this.canvas.width,\n this.canvas.height,\n );\n } else {\n this.ctx.fillStyle = '#00FF00';\n this.ctx.fillRect(0, 0, this.canvas.width, this.canvas.height);\n }\n\n this.ctx.globalCompositeOperation = 'destination-over';\n }\n this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);\n // this.ctx.restore();\n }\n\n async blurBackground(frame: VideoFrame) {\n if (\n !this.ctx ||\n !this.canvas ||\n !this.segmentationResults?.categoryMask?.canvas ||\n !this.inputVideo\n ) {\n return;\n }\n\n this.ctx.save();\n this.ctx.globalCompositeOperation = 'copy';\n\n const bitmap = await maskToBitmap(\n this.segmentationResults.categoryMask,\n this.inputVideo.videoWidth,\n this.inputVideo.videoHeight,\n );\n\n this.ctx.filter = 'blur(3px)';\n this.ctx.globalCompositeOperation = 'copy';\n this.ctx.drawImage(bitmap, 0, 0);\n this.ctx.filter = 'none';\n this.ctx.globalCompositeOperation = 'source-out';\n this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);\n this.ctx.globalCompositeOperation = 'destination-over';\n this.ctx.filter = `blur(${this.blurRadius}px)`;\n this.ctx.drawImage(frame, 0, 0);\n this.ctx.restore();\n }\n}\n\nfunction maskToBitmap(\n mask: vision.MPMask,\n videoWidth: number,\n videoHeight: number,\n): Promise<ImageBitmap> {\n const dataArray: Uint8ClampedArray = new Uint8ClampedArray(videoWidth * videoHeight * 4);\n const result = mask.getAsUint8Array();\n for (let i = 0; i < result.length; i += 1) {\n dataArray[i * 4] = result[i];\n dataArray[i * 4 + 1] = result[i];\n dataArray[i * 4 + 2] = result[i];\n dataArray[i * 4 + 3] = result[i];\n }\n const dataNew = new ImageData(dataArray, videoWidth, videoHeight);\n\n return createImageBitmap(dataNew);\n}\n","{\n \"name\": \"@livekit/track-processors\",\n \"version\": \"0.2.4\",\n \"description\": \"LiveKit track processors\",\n \"main\": \"dist/index.js\",\n \"module\": \"dist/index.mjs\",\n \"source\": \"src/index.ts\",\n \"types\": \"dist/src/index.d.ts\",\n \"repository\": \"git@github.com:livekit/livekit-track-processors.git\",\n \"author\": \"Lukas Seiler\",\n \"license\": \"Apache-2.0\",\n \"scripts\": {\n \"build\": \"tsup --onSuccess \\\"tsc --declaration --emitDeclarationOnly\\\"\",\n \"build-docs\": \"yarn exec typedoc\",\n \"build-sample\": \"cd example && webpack && cp styles.css index.html dist/\",\n \"lint\": \"eslint src\",\n \"test\": \"jest\",\n \"sample\": \"vite serve example --port 8080 --open\",\n \"prepublish\": \"yarn build\"\n },\n \"files\": [\n \"dist\",\n \"src\"\n ],\n \"dependencies\": {\n \"@mediapipe/holistic\": \"0.5.1675471629\",\n \"@mediapipe/tasks-vision\": \"0.10.1\"\n },\n \"peerDependencies\": {\n \"livekit-client\": \"^1.12.0\"\n },\n \"devDependencies\": {\n \"@trivago/prettier-plugin-sort-imports\": \"^4.1.1\",\n \"@types/dom-mediacapture-transform\": \"^0.1.5\",\n \"@types/jest\": \"^27.0.3\",\n \"@types/offscreencanvas\": \"^2019.7.0\",\n \"@typescript-eslint/eslint-plugin\": \"^4.31.2\",\n \"@webpack-cli/serve\": \"^1.5.2\",\n \"eslint\": \"8.39.0\",\n \"eslint-config-airbnb-typescript\": \"17.0.0\",\n \"eslint-config-prettier\": \"8.8.0\",\n \"eslint-plugin-ecmascript-compat\": \"^3.0.0\",\n \"eslint-plugin-import\": \"2.27.5\",\n \"jest\": \"^27.4.3\",\n \"livekit-client\": \"^0.16.6\",\n \"prettier\": \"^2.8.8\",\n \"ts-jest\": \"^27.0.7\",\n \"ts-loader\": \"^8.1.0\",\n \"ts-proto\": \"^1.85.0\",\n \"tsup\": \"^7.1.0\",\n \"typedoc\": \"^0.20.35\",\n \"typedoc-plugin-no-inherit\": \"1.3.0\",\n \"typescript\": \"^5.0.4\",\n \"vite\": \"^4.3.8\",\n \"webpack\": \"^5.53.0\",\n \"webpack-cli\": \"^4.8.0\",\n \"webpack-dev-server\": \"^4.2.1\"\n }\n}\n","import { VideoTrackTransformer, VideoTransformerInitOptions } from './types';\n\nexport default abstract class VideoTransformer implements VideoTrackTransformer {\n transformer?: TransformStream;\n\n canvas?: OffscreenCanvas;\n\n ctx?: OffscreenCanvasRenderingContext2D;\n\n inputVideo?: HTMLVideoElement;\n\n protected isDisabled?: Boolean = false;\n\n async init({\n outputCanvas,\n inputElement: inputVideo,\n }: VideoTransformerInitOptions): Promise<void> {\n if (!(inputVideo instanceof HTMLVideoElement)) {\n throw TypeError('Video transformer needs a HTMLVideoElement as input');\n }\n this.transformer = new TransformStream({\n transform: (frame, controller) => this.transform(frame, controller),\n });\n this.canvas = outputCanvas || null;\n if (outputCanvas) {\n this.ctx = this.canvas?.getContext('2d') || undefined;\n }\n this.inputVideo = inputVideo;\n this.isDisabled = false;\n }\n\n async destroy() {\n this.isDisabled = true;\n this.canvas = undefined;\n this.ctx = undefined;\n }\n\n abstract transform(\n frame: VideoFrame,\n controller: TransformStreamDefaultController<VideoFrame>,\n ): void;\n}\n","import VideoTransformer from './VideoTransformer';\n\nexport default class DummyTransformer extends VideoTransformer {\n async transform(frame: VideoFrame, controller: TransformStreamDefaultController<VideoFrame>) {\n controller.enqueue(frame);\n }\n\n async destroy() {\n // nothing to do\n }\n}\n","import ProcessorPipeline from './ProcessorPipeline';\nimport BackgroundTransformer, { SegmenterBaseOptions } from './transformers/BackgroundTransformer';\nimport DummyTransformer from './transformers/DummyTransformer';\n\nexport const BackgroundBlur = (\n blurRadius: number = 10,\n segmenterOptions?: SegmenterBaseOptions,\n) => {\n const isPipelineSupported = ProcessorPipeline.isSupported && BackgroundTransformer.isSupported;\n if (!isPipelineSupported) {\n throw new Error('pipeline is not supported in this browser');\n }\n const pipeline = new ProcessorPipeline(\n [new BackgroundTransformer({ blurRadius, segmenterOptions })],\n 'background-blur',\n );\n return pipeline;\n};\n\nexport const VirtualBackground = (imagePath: string, segmenterOptions?: SegmenterBaseOptions) => {\n const isPipelineSupported = ProcessorPipeline.isSupported && BackgroundTransformer.isSupported;\n if (!isPipelineSupported) {\n throw new Error('pipeline is not supported in this browser');\n }\n const pipeline = new ProcessorPipeline(\n [new BackgroundTransformer({ imagePath, segmenterOptions })],\n 'virtual-background',\n );\n return pipeline;\n};\n\nexport const Dummy = () => {\n const isPipelineSupported = ProcessorPipeline.isSupported && BackgroundTransformer.isSupported;\n if (!isPipelineSupported) {\n throw new Error('pipeline is not supported in this browser');\n }\n const pipeline = new ProcessorPipeline([new DummyTransformer()], 'dummy');\n return pipeline;\n};\n"]}
|
|
1
|
+
{"version":3,"sources":["../src/ProcessorPipeline.ts","../src/transformers/BackgroundTransformer.ts","../package.json","../src/transformers/VideoTransformer.ts","../src/transformers/DummyTransformer.ts","../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;AAGA,IAAqB,oBAArB,MAA6E;AAAA,EAC3E,WAAW,cAAc;AACvB,WACE,OAAO,8BAA8B,eACrC,OAAO,8BAA8B;AAAA,EAEzC;AAAA,EAoBA,YAAY,cAA4C,MAAc;AACpE,SAAK,OAAO;AACZ,SAAK,eAAe;AAAA,EACtB;AAAA,EAEA,MAAM,KAAK,MAAoC;AAlCjD;AAmCI,SAAK,SAAS,KAAK;AACnB,UAAM,kBAAkB,KAAK,OAAO,eAAe;AACnD,UAAM,KAAK,OAAO,iBAAiB,iCAC9B,kBAD8B;AAAA;AAAA;AAAA;AAAA;AAAA,MAMjC,YAAY;AAAA,IACd,EAAC;AACD,SAAK,iBAAiB,KAAK,OAAO,YAAY;AAC9C,SAAK,cAAc,KAAK;AACxB,QAAI,KAAK,uBAAuB,kBAAkB;AAChD,WAAK,YAAY,UAAS,UAAK,eAAe,WAApB,YAA8B;AACxD,WAAK,YAAY,SAAQ,UAAK,eAAe,UAApB,YAA6B;AAAA,IACxD;AACA,QAAI,EAAE,KAAK,uBAAuB,mBAAmB;AACnD,YAAM,UAAU,iDAAiD;AAAA,IACnE;AAEA,SAAK,YAAY,IAAI,0BAA0B,EAAE,OAAO,KAAK,OAAO,CAAC;AAErE,SAAK,iBAAiB,IAAI,0BAA0B;AAAA,MAClD,MAAM;AAAA,MACN,cAAc,KAAK;AAAA,IACrB,CAAC;AAED,SAAK,SAAS,IAAI;AAAA,OAChB,UAAK,eAAe,UAApB,YAA6B;AAAA,OAC7B,UAAK,eAAe,WAApB,YAA8B;AAAA,IAChC;AAEA,QAAI,iBAAiB,KAAK,UAAU;AACpC,eAAW,eAAe,KAAK,cAAc;AAC3C,YAAM,YAAY,KAAK;AAAA,QACrB,cAAc,KAAK;AAAA,QACnB,cAAc,KAAK;AAAA,MACrB,CAAC;AACD,uBAAiB,eAAe,YAAY,YAAa,WAAY;AAAA,IACvE;AACA,mBACG,OAAO,KAAK,eAAe,QAAQ,EACnC,MAAM,CAAC,MAAM,QAAQ,MAAM,6BAA6B,CAAC,CAAC,EAC1D,QAAQ,MAAM,KAAK,QAAQ,CAAC;AAC/B,SAAK,iBAAiB,KAAK;AAAA,EAC7B;AAAA,EAEA,MAAM,UAAU;AAlFlB;AAmFI,eAAW,eAAe,KAAK,cAAc;AAC3C,YAAM,YAAY,QAAQ;AAAA,IAC5B;AACA,eAAK,mBAAL,mBAAqB;AAAA,EACvB;AACF;;;ACxFA,YAAY,YAAY;;;ACwBtB,mBAAgB;AAAA,EACd,uBAAuB;AAAA,EACvB,2BAA2B;AAC7B;;;ACzBF,IAA8B,mBAA9B,MAAgF;AAAA,EAAhF;AASE,SAAU,aAAuB;AAAA;AAAA,EAEjC,MAAM,KAAK;AAAA,IACT;AAAA,IACA,cAAc;AAAA,EAChB,GAA+C;AAhBjD;AAiBI,QAAI,EAAE,sBAAsB,mBAAmB;AAC7C,YAAM,UAAU,qDAAqD;AAAA,IACvE;AACA,SAAK,cAAc,IAAI,gBAAgB;AAAA,MACrC,WAAW,CAAC,OAAO,eAAe,KAAK,UAAU,OAAO,UAAU;AAAA,IACpE,CAAC;AACD,SAAK,SAAS,gBAAgB;AAC9B,QAAI,cAAc;AAChB,WAAK,QAAM,UAAK,WAAL,mBAAa,WAAW,UAAS;AAAA,IAC9C;AACA,SAAK,aAAa;AAClB,SAAK,aAAa;AAAA,EACpB;AAAA,EAEA,MAAM,UAAU;AACd,SAAK,aAAa;AAClB,SAAK,SAAS;AACd,SAAK,MAAM;AAAA,EACb;AAMF;;;AF5BA,IAAqB,sBAArB,cAAiD,iBAAiB;AAAA,EAehE,YAAY,MAAyB;AACnC,UAAM;AAPR,2BAAsC;AAQpC,SAAK,UAAU;AACf,QAAI,KAAK,YAAY;AACnB,WAAK,aAAa,KAAK;AAAA,IACzB,WAAW,KAAK,WAAW;AACzB,WAAK,eAAe,KAAK,SAAS;AAAA,IACpC;AAEA,SAAK,QAAQ;AAAA,EACf;AAAA,EAxBA,WAAW,cAAc;AACvB,WAAO,OAAO,oBAAoB;AAAA,EACpC;AAAA,EAwBA,MAAM,KAAK,EAAE,cAAc,cAAc,WAAW,GAAgC;AAClF,UAAM,MAAM,KAAK,EAAE,cAAc,cAAc,WAAW,CAAC;AAE3D,UAAM,UAAU,MAAa,uBAAgB;AAAA,MAC3C,wDAAwD,aAAa,yBAAyB,CAAC;AAAA,IACjG;AAEA,SAAK,iBAAiB,MAAa,sBAAe,kBAAkB,SAAS;AAAA,MAC3E,aAAa;AAAA,QACX,gBACE;AAAA,QACF,UAAU;AAAA,SACP,KAAK,QAAQ;AAAA,MAElB,aAAa;AAAA,MACb,oBAAoB;AAAA,MACpB,uBAAuB;AAAA,IACzB,CAAC;AAAA,EAGH;AAAA,EAEA,MAAM,UAAU;AA9DlB;AA+DI,UAAM,MAAM,QAAQ;AACpB,YAAM,UAAK,mBAAL,mBAAqB;AAC3B,SAAK,kBAAkB;AAAA,EACzB;AAAA,EAEA,MAAM,eAAe,MAAc;AACjC,UAAM,MAAM,IAAI,MAAM;AAEtB,UAAM,IAAI,QAAQ,CAAC,SAAS,WAAW;AACrC,UAAI,cAAc;AAClB,UAAI,SAAS,MAAM,QAAQ,GAAG;AAC9B,UAAI,UAAU,CAAC,QAAQ,OAAO,GAAG;AACjC,UAAI,MAAM;AAAA,IACZ,CAAC;AACD,UAAM,YAAY,MAAM,kBAAkB,GAAG;AAC7C,SAAK,kBAAkB;AAAA,EACzB;AAAA,EAEA,MAAM,UAAU,OAAmB,YAA0D;AAjF/F;AAkFI,QAAI;AACF,UAAI,KAAK,YAAY;AACnB,mBAAW,QAAQ,KAAK;AACxB;AAAA,MACF;AACA,UAAI,CAAC,KAAK,QAAQ;AAChB,cAAM,UAAU,sCAAsC;AAAA,MACxD;AACA,UAAI,cAAc,YAAY,IAAI;AAClC,iBAAK,mBAAL,mBAAqB;AAAA,QACnB,KAAK;AAAA,QACL;AAAA,QACA,CAAC,WAAY,KAAK,sBAAsB;AAAA;AAG1C,UAAI,KAAK,YAAY;AACnB,cAAM,KAAK,eAAe,KAAK;AAAA,MACjC,OAAO;AACL,cAAM,KAAK,sBAAsB,KAAK;AAAA,MACxC;AACA,YAAM,WAAW,IAAI,WAAW,KAAK,QAAQ;AAAA,QAC3C,WAAW,MAAM,aAAa,KAAK,IAAI;AAAA,MACzC,CAAC;AACD,iBAAW,QAAQ,QAAQ;AAAA,IAC7B,UAAE;AACA,YAAM,MAAM;AAAA,IACd;AAAA,EACF;AAAA,EAEA,MAAM,sBAAsB,OAAmB;AA/GjD;AAgHI,QAAI,CAAC,KAAK,UAAU,CAAC,KAAK,OAAO,CAAC,KAAK,uBAAuB,CAAC,KAAK;AAAY;AAGhF,SAAI,UAAK,wBAAL,mBAA0B,cAAc;AAC1C,WAAK,IAAI,SAAS;AAClB,WAAK,IAAI,2BAA2B;AACpC,YAAM,SAAS,MAAM;AAAA,QACnB,KAAK,oBAAoB;AAAA,QACzB,KAAK,oBAAoB,aAAa;AAAA,QACtC,KAAK,oBAAoB,aAAa;AAAA,MACxC;AACA,WAAK,IAAI,UAAU,QAAQ,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AACtE,WAAK,IAAI,SAAS;AAClB,WAAK,IAAI,2BAA2B;AACpC,UAAI,KAAK,iBAAiB;AACxB,aAAK,IAAI;AAAA,UACP,KAAK;AAAA,UACL;AAAA,UACA;AAAA,UACA,KAAK,gBAAgB;AAAA,UACrB,KAAK,gBAAgB;AAAA,UACrB;AAAA,UACA;AAAA,UACA,KAAK,OAAO;AAAA,UACZ,KAAK,OAAO;AAAA,QACd;AAAA,MACF,OAAO;AACL,aAAK,IAAI,YAAY;AACrB,aAAK,IAAI,SAAS,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AAAA,MAC/D;AAEA,WAAK,IAAI,2BAA2B;AAAA,IACtC;AACA,SAAK,IAAI,UAAU,OAAO,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AAAA,EAEvE;AAAA,EAEA,MAAM,eAAe,OAAmB;AArJ1C;AAsJI,QACE,CAAC,KAAK,OACN,CAAC,KAAK,UACN,GAAC,gBAAK,wBAAL,mBAA0B,iBAA1B,mBAAwC,WACzC,CAAC,KAAK,YACN;AACA;AAAA,IACF;AAEA,SAAK,IAAI,KAAK;AACd,SAAK,IAAI,2BAA2B;AAEpC,UAAM,SAAS,MAAM;AAAA,MACnB,KAAK,oBAAoB;AAAA,MACzB,KAAK,oBAAoB,aAAa;AAAA,MACtC,KAAK,oBAAoB,aAAa;AAAA,IACxC;AAEA,SAAK,IAAI,SAAS;AAClB,SAAK,IAAI,2BAA2B;AACpC,SAAK,IAAI,UAAU,QAAQ,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AACtE,SAAK,IAAI,SAAS;AAClB,SAAK,IAAI,2BAA2B;AACpC,SAAK,IAAI,UAAU,OAAO,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AACrE,SAAK,IAAI,2BAA2B;AACpC,SAAK,IAAI,SAAS,QAAQ,KAAK,UAAU;AACzC,SAAK,IAAI,UAAU,OAAO,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AACrE,SAAK,IAAI,QAAQ;AAAA,EACnB;AACF;AAEA,SAAS,aACP,MACA,YACA,aACsB;AACtB,QAAM,YAA+B,IAAI,kBAAkB,aAAa,cAAc,CAAC;AACvF,QAAM,SAAS,KAAK,gBAAgB;AACpC,WAAS,IAAI,GAAG,IAAI,OAAO,QAAQ,KAAK,GAAG;AACzC,cAAU,IAAI,CAAC,IAAI,OAAO,CAAC;AAC3B,cAAU,IAAI,IAAI,CAAC,IAAI,OAAO,CAAC;AAC/B,cAAU,IAAI,IAAI,CAAC,IAAI,OAAO,CAAC;AAC/B,cAAU,IAAI,IAAI,CAAC,IAAI,OAAO,CAAC;AAAA,EACjC;AACA,QAAM,UAAU,IAAI,UAAU,WAAW,YAAY,WAAW;AAEhE,SAAO,kBAAkB,OAAO;AAClC;;;AGnMA,IAAqB,mBAArB,cAA8C,iBAAiB;AAAA,EAC7D,MAAM,UAAU,OAAmB,YAA0D;AAC3F,eAAW,QAAQ,KAAK;AAAA,EAC1B;AAAA,EAEA,MAAM,UAAU;AAAA,EAEhB;AACF;;;ACJO,IAAM,iBAAiB,CAC5B,aAAqB,IACrB,qBACG;AACH,QAAM,sBAAsB,kBAAkB,eAAe,oBAAsB;AACnF,MAAI,CAAC,qBAAqB;AACxB,UAAM,IAAI,MAAM,2CAA2C;AAAA,EAC7D;AACA,QAAM,WAAW,IAAI;AAAA,IACnB,CAAC,IAAI,oBAAsB,EAAE,YAAY,iBAAiB,CAAC,CAAC;AAAA,IAC5D;AAAA,EACF;AACA,SAAO;AACT;AAEO,IAAM,oBAAoB,CAAC,WAAmB,qBAA4C;AAC/F,QAAM,sBAAsB,kBAAkB,eAAe,oBAAsB;AACnF,MAAI,CAAC,qBAAqB;AACxB,UAAM,IAAI,MAAM,2CAA2C;AAAA,EAC7D;AACA,QAAM,WAAW,IAAI;AAAA,IACnB,CAAC,IAAI,oBAAsB,EAAE,WAAW,iBAAiB,CAAC,CAAC;AAAA,IAC3D;AAAA,EACF;AACA,SAAO;AACT;AAEO,IAAM,QAAQ,MAAM;AACzB,QAAM,sBAAsB,kBAAkB,eAAe,oBAAsB;AACnF,MAAI,CAAC,qBAAqB;AACxB,UAAM,IAAI,MAAM,2CAA2C;AAAA,EAC7D;AACA,QAAM,WAAW,IAAI,kBAAkB,CAAC,IAAI,iBAAiB,CAAC,GAAG,OAAO;AACxE,SAAO;AACT","sourcesContent":["import type { ProcessorOptions, Track, TrackProcessor } from 'livekit-client';\nimport { VideoTrackTransformer } from './transformers';\n\nexport default class ProcessorPipeline implements TrackProcessor<Track.Kind> {\n static get isSupported() {\n return (\n typeof MediaStreamTrackGenerator !== 'undefined' &&\n typeof MediaStreamTrackProcessor !== 'undefined'\n );\n }\n\n name: string;\n\n source?: MediaStreamVideoTrack;\n\n sourceSettings?: MediaTrackSettings;\n\n processor?: MediaStreamTrackProcessor<VideoFrame>;\n\n trackGenerator?: MediaStreamTrackGenerator<VideoFrame>;\n\n canvas?: OffscreenCanvas;\n\n sourceDummy?: HTMLMediaElement;\n\n processedTrack?: MediaStreamTrack;\n\n transformers: Array<VideoTrackTransformer>;\n\n constructor(transformers: Array<VideoTrackTransformer>, name: string) {\n this.name = name;\n this.transformers = transformers;\n }\n\n async init(opts: ProcessorOptions<Track.Kind>) {\n this.source = opts.track as MediaStreamVideoTrack;\n const origConstraints = this.source.getConstraints();\n await this.source.applyConstraints({\n ...origConstraints,\n // @ts-expect-error when a mediastream track is resized and/or cropped, the `VideoFrame` will have a coded height/width of the original video size\n // this leads to a shift of the underlying video as the frame itself is being rendered with the coded size\n // but image segmentation is based on the display dimensions (-> the cropped version)\n // in order to prevent this, we force the resize mode to \"none\"\n resizeMode: 'none',\n });\n this.sourceSettings = this.source.getSettings();\n this.sourceDummy = opts.element;\n if (this.sourceDummy instanceof HTMLVideoElement) {\n this.sourceDummy.height = this.sourceSettings.height ?? 300;\n this.sourceDummy.width = this.sourceSettings.width ?? 300;\n }\n if (!(this.sourceDummy instanceof HTMLVideoElement)) {\n throw TypeError('Currently only video transformers are supported');\n }\n // TODO explore if we can do all the processing work in a webworker\n this.processor = new MediaStreamTrackProcessor({ track: this.source });\n\n this.trackGenerator = new MediaStreamTrackGenerator({\n kind: 'video',\n signalTarget: this.source,\n });\n\n this.canvas = new OffscreenCanvas(\n this.sourceSettings.width ?? 300,\n this.sourceSettings.height ?? 300,\n );\n\n let readableStream = this.processor.readable;\n for (const transformer of this.transformers) {\n await transformer.init({\n outputCanvas: this.canvas,\n inputElement: this.sourceDummy!,\n });\n readableStream = readableStream.pipeThrough(transformer!.transformer!);\n }\n readableStream\n .pipeTo(this.trackGenerator.writable)\n .catch((e) => console.error('error when trying to pipe', e))\n .finally(() => this.destroy());\n this.processedTrack = this.trackGenerator as MediaStreamVideoTrack;\n }\n\n async destroy() {\n for (const transformer of this.transformers) {\n await transformer.destroy();\n }\n this.trackGenerator?.stop();\n }\n}\n","import * as vision from '@mediapipe/tasks-vision';\nimport { dependencies } from '../../package.json';\nimport VideoTransformer from './VideoTransformer';\nimport { VideoTransformerInitOptions } from './types';\n\nexport type SegmenterBaseOptions = Partial<vision.ImageSegmenterOptions['baseOptions']>;\n\nexport type BackgroundOptions = {\n blurRadius?: number;\n imagePath?: string;\n segmenterOptions?: SegmenterBaseOptions;\n};\n\nexport default class BackgroundProcessor extends VideoTransformer {\n static get isSupported() {\n return typeof OffscreenCanvas !== 'undefined';\n }\n\n imageSegmenter?: vision.ImageSegmenter;\n\n segmentationResults: vision.ImageSegmenterResult | undefined;\n\n backgroundImage: ImageBitmap | null = null;\n\n blurRadius?: number;\n\n options: BackgroundOptions;\n\n constructor(opts: BackgroundOptions) {\n super();\n this.options = opts;\n if (opts.blurRadius) {\n this.blurRadius = opts.blurRadius;\n } else if (opts.imagePath) {\n this.loadBackground(opts.imagePath);\n }\n\n this.options.segmenterOptions;\n }\n\n async init({ outputCanvas, inputElement: inputVideo }: VideoTransformerInitOptions) {\n await super.init({ outputCanvas, inputElement: inputVideo });\n\n const fileSet = await vision.FilesetResolver.forVisionTasks(\n `https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@${dependencies['@mediapipe/tasks-vision']}/wasm`,\n );\n\n this.imageSegmenter = await vision.ImageSegmenter.createFromOptions(fileSet, {\n baseOptions: {\n modelAssetPath:\n 'https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_segmenter/float16/latest/selfie_segmenter.tflite',\n delegate: 'GPU',\n ...this.options.segmenterOptions,\n },\n runningMode: 'VIDEO',\n outputCategoryMask: true,\n outputConfidenceMasks: false,\n });\n\n // this.loadBackground(opts.backgroundUrl).catch((e) => console.error(e));\n }\n\n async destroy() {\n await super.destroy();\n await this.imageSegmenter?.close();\n this.backgroundImage = null;\n }\n\n async loadBackground(path: string) {\n const img = new Image();\n\n await new Promise((resolve, reject) => {\n img.crossOrigin = 'Anonymous';\n img.onload = () => resolve(img);\n img.onerror = (err) => reject(err);\n img.src = path;\n });\n const imageData = await createImageBitmap(img);\n this.backgroundImage = imageData;\n }\n\n async transform(frame: VideoFrame, controller: TransformStreamDefaultController<VideoFrame>) {\n try {\n if (this.isDisabled) {\n controller.enqueue(frame);\n return;\n }\n if (!this.canvas) {\n throw TypeError('Canvas needs to be initialized first');\n }\n let startTimeMs = performance.now();\n this.imageSegmenter?.segmentForVideo(\n this.inputVideo!,\n startTimeMs,\n (result) => (this.segmentationResults = result),\n );\n\n if (this.blurRadius) {\n await this.blurBackground(frame);\n } else {\n await this.drawVirtualBackground(frame);\n }\n const newFrame = new VideoFrame(this.canvas, {\n timestamp: frame.timestamp || Date.now(),\n });\n controller.enqueue(newFrame);\n } finally {\n frame.close();\n }\n }\n\n async drawVirtualBackground(frame: VideoFrame) {\n if (!this.canvas || !this.ctx || !this.segmentationResults || !this.inputVideo) return;\n // this.ctx.save();\n // this.ctx.clearRect(0, 0, this.canvas.width, this.canvas.height);\n if (this.segmentationResults?.categoryMask) {\n this.ctx.filter = 'blur(3px)';\n this.ctx.globalCompositeOperation = 'copy';\n const bitmap = await maskToBitmap(\n this.segmentationResults.categoryMask,\n this.segmentationResults.categoryMask.width,\n this.segmentationResults.categoryMask.height,\n );\n this.ctx.drawImage(bitmap, 0, 0, this.canvas.width, this.canvas.height);\n this.ctx.filter = 'none';\n this.ctx.globalCompositeOperation = 'source-in';\n if (this.backgroundImage) {\n this.ctx.drawImage(\n this.backgroundImage,\n 0,\n 0,\n this.backgroundImage.width,\n this.backgroundImage.height,\n 0,\n 0,\n this.canvas.width,\n this.canvas.height,\n );\n } else {\n this.ctx.fillStyle = '#00FF00';\n this.ctx.fillRect(0, 0, this.canvas.width, this.canvas.height);\n }\n\n this.ctx.globalCompositeOperation = 'destination-over';\n }\n this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);\n // this.ctx.restore();\n }\n\n async blurBackground(frame: VideoFrame) {\n if (\n !this.ctx ||\n !this.canvas ||\n !this.segmentationResults?.categoryMask?.canvas ||\n !this.inputVideo\n ) {\n return;\n }\n\n this.ctx.save();\n this.ctx.globalCompositeOperation = 'copy';\n\n const bitmap = await maskToBitmap(\n this.segmentationResults.categoryMask,\n this.segmentationResults.categoryMask.width,\n this.segmentationResults.categoryMask.height,\n );\n\n this.ctx.filter = 'blur(3px)';\n this.ctx.globalCompositeOperation = 'copy';\n this.ctx.drawImage(bitmap, 0, 0, this.canvas.width, this.canvas.height);\n this.ctx.filter = 'none';\n this.ctx.globalCompositeOperation = 'source-out';\n this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);\n this.ctx.globalCompositeOperation = 'destination-over';\n this.ctx.filter = `blur(${this.blurRadius}px)`;\n this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);\n this.ctx.restore();\n }\n}\n\nfunction maskToBitmap(\n mask: vision.MPMask,\n videoWidth: number,\n videoHeight: number,\n): Promise<ImageBitmap> {\n const dataArray: Uint8ClampedArray = new Uint8ClampedArray(videoWidth * videoHeight * 4);\n const result = mask.getAsUint8Array();\n for (let i = 0; i < result.length; i += 1) {\n dataArray[i * 4] = result[i];\n dataArray[i * 4 + 1] = result[i];\n dataArray[i * 4 + 2] = result[i];\n dataArray[i * 4 + 3] = result[i];\n }\n const dataNew = new ImageData(dataArray, videoWidth, videoHeight);\n\n return createImageBitmap(dataNew);\n}\n","{\n \"name\": \"@livekit/track-processors\",\n \"version\": \"0.2.5\",\n \"description\": \"LiveKit track processors\",\n \"main\": \"dist/index.js\",\n \"module\": \"dist/index.mjs\",\n \"source\": \"src/index.ts\",\n \"types\": \"dist/src/index.d.ts\",\n \"repository\": \"git@github.com:livekit/livekit-track-processors.git\",\n \"author\": \"Lukas Seiler\",\n \"license\": \"Apache-2.0\",\n \"scripts\": {\n \"build\": \"tsup --onSuccess \\\"tsc --declaration --emitDeclarationOnly\\\"\",\n \"build-docs\": \"yarn exec typedoc\",\n \"build-sample\": \"cd example && vite build\",\n \"lint\": \"eslint src\",\n \"test\": \"jest\",\n \"sample\": \"vite serve example --port 8080 --open\",\n \"prepublish\": \"yarn build\"\n },\n \"files\": [\n \"dist\",\n \"src\"\n ],\n \"dependencies\": {\n \"@mediapipe/holistic\": \"0.5.1675471629\",\n \"@mediapipe/tasks-vision\": \"0.10.1\"\n },\n \"peerDependencies\": {\n \"livekit-client\": \"^1.12.0\"\n },\n \"devDependencies\": {\n \"@trivago/prettier-plugin-sort-imports\": \"^4.1.1\",\n \"@types/dom-mediacapture-transform\": \"^0.1.5\",\n \"@types/jest\": \"^27.0.3\",\n \"@types/offscreencanvas\": \"^2019.7.0\",\n \"@typescript-eslint/eslint-plugin\": \"^4.31.2\",\n \"@webpack-cli/serve\": \"^1.5.2\",\n \"eslint\": \"8.39.0\",\n \"eslint-config-airbnb-typescript\": \"17.0.0\",\n \"eslint-config-prettier\": \"8.8.0\",\n \"eslint-plugin-ecmascript-compat\": \"^3.0.0\",\n \"eslint-plugin-import\": \"2.27.5\",\n \"jest\": \"^27.4.3\",\n \"livekit-client\": \"^1.12.0\",\n \"prettier\": \"^2.8.8\",\n \"ts-jest\": \"^27.0.7\",\n \"ts-loader\": \"^8.1.0\",\n \"ts-proto\": \"^1.85.0\",\n \"tsup\": \"^7.1.0\",\n \"typedoc\": \"^0.20.35\",\n \"typedoc-plugin-no-inherit\": \"1.3.0\",\n \"typescript\": \"^5.0.4\",\n \"vite\": \"^4.3.8\",\n \"webpack\": \"^5.53.0\",\n \"webpack-cli\": \"^4.8.0\",\n \"webpack-dev-server\": \"^4.2.1\"\n }\n}\n","import { VideoTrackTransformer, VideoTransformerInitOptions } from './types';\n\nexport default abstract class VideoTransformer implements VideoTrackTransformer {\n transformer?: TransformStream;\n\n canvas?: OffscreenCanvas;\n\n ctx?: OffscreenCanvasRenderingContext2D;\n\n inputVideo?: HTMLVideoElement;\n\n protected isDisabled?: Boolean = false;\n\n async init({\n outputCanvas,\n inputElement: inputVideo,\n }: VideoTransformerInitOptions): Promise<void> {\n if (!(inputVideo instanceof HTMLVideoElement)) {\n throw TypeError('Video transformer needs a HTMLVideoElement as input');\n }\n this.transformer = new TransformStream({\n transform: (frame, controller) => this.transform(frame, controller),\n });\n this.canvas = outputCanvas || null;\n if (outputCanvas) {\n this.ctx = this.canvas?.getContext('2d') || undefined;\n }\n this.inputVideo = inputVideo;\n this.isDisabled = false;\n }\n\n async destroy() {\n this.isDisabled = true;\n this.canvas = undefined;\n this.ctx = undefined;\n }\n\n abstract transform(\n frame: VideoFrame,\n controller: TransformStreamDefaultController<VideoFrame>,\n ): void;\n}\n","import VideoTransformer from './VideoTransformer';\n\nexport default class DummyTransformer extends VideoTransformer {\n async transform(frame: VideoFrame, controller: TransformStreamDefaultController<VideoFrame>) {\n controller.enqueue(frame);\n }\n\n async destroy() {\n // nothing to do\n }\n}\n","import ProcessorPipeline from './ProcessorPipeline';\nimport BackgroundTransformer, { SegmenterBaseOptions } from './transformers/BackgroundTransformer';\nimport DummyTransformer from './transformers/DummyTransformer';\n\nexport * from './transformers/types';\n\nexport const BackgroundBlur = (\n blurRadius: number = 10,\n segmenterOptions?: SegmenterBaseOptions,\n) => {\n const isPipelineSupported = ProcessorPipeline.isSupported && BackgroundTransformer.isSupported;\n if (!isPipelineSupported) {\n throw new Error('pipeline is not supported in this browser');\n }\n const pipeline = new ProcessorPipeline(\n [new BackgroundTransformer({ blurRadius, segmenterOptions })],\n 'background-blur',\n );\n return pipeline;\n};\n\nexport const VirtualBackground = (imagePath: string, segmenterOptions?: SegmenterBaseOptions) => {\n const isPipelineSupported = ProcessorPipeline.isSupported && BackgroundTransformer.isSupported;\n if (!isPipelineSupported) {\n throw new Error('pipeline is not supported in this browser');\n }\n const pipeline = new ProcessorPipeline(\n [new BackgroundTransformer({ imagePath, segmenterOptions })],\n 'virtual-background',\n );\n return pipeline;\n};\n\nexport const Dummy = () => {\n const isPipelineSupported = ProcessorPipeline.isSupported && BackgroundTransformer.isSupported;\n if (!isPipelineSupported) {\n throw new Error('pipeline is not supported in this browser');\n }\n const pipeline = new ProcessorPipeline([new DummyTransformer()], 'dummy');\n return pipeline;\n};\n"]}
|
package/dist/index.mjs
CHANGED
|
@@ -28,7 +28,7 @@ var ProcessorPipeline = class {
|
|
|
28
28
|
this.transformers = transformers;
|
|
29
29
|
}
|
|
30
30
|
async init(opts) {
|
|
31
|
-
var _a, _b;
|
|
31
|
+
var _a, _b, _c, _d;
|
|
32
32
|
this.source = opts.track;
|
|
33
33
|
const origConstraints = this.source.getConstraints();
|
|
34
34
|
await this.source.applyConstraints(__spreadProps(__spreadValues({}, origConstraints), {
|
|
@@ -41,8 +41,8 @@ var ProcessorPipeline = class {
|
|
|
41
41
|
this.sourceSettings = this.source.getSettings();
|
|
42
42
|
this.sourceDummy = opts.element;
|
|
43
43
|
if (this.sourceDummy instanceof HTMLVideoElement) {
|
|
44
|
-
this.sourceDummy.height = this.sourceSettings.height;
|
|
45
|
-
this.sourceDummy.width = this.sourceSettings.width;
|
|
44
|
+
this.sourceDummy.height = (_a = this.sourceSettings.height) != null ? _a : 300;
|
|
45
|
+
this.sourceDummy.width = (_b = this.sourceSettings.width) != null ? _b : 300;
|
|
46
46
|
}
|
|
47
47
|
if (!(this.sourceDummy instanceof HTMLVideoElement)) {
|
|
48
48
|
throw TypeError("Currently only video transformers are supported");
|
|
@@ -53,8 +53,8 @@ var ProcessorPipeline = class {
|
|
|
53
53
|
signalTarget: this.source
|
|
54
54
|
});
|
|
55
55
|
this.canvas = new OffscreenCanvas(
|
|
56
|
-
(
|
|
57
|
-
(
|
|
56
|
+
(_c = this.sourceSettings.width) != null ? _c : 300,
|
|
57
|
+
(_d = this.sourceSettings.height) != null ? _d : 300
|
|
58
58
|
);
|
|
59
59
|
let readableStream = this.processor.readable;
|
|
60
60
|
for (const transformer of this.transformers) {
|
|
@@ -201,10 +201,10 @@ var BackgroundProcessor = class extends VideoTransformer {
|
|
|
201
201
|
this.ctx.globalCompositeOperation = "copy";
|
|
202
202
|
const bitmap = await maskToBitmap(
|
|
203
203
|
this.segmentationResults.categoryMask,
|
|
204
|
-
this.
|
|
205
|
-
this.
|
|
204
|
+
this.segmentationResults.categoryMask.width,
|
|
205
|
+
this.segmentationResults.categoryMask.height
|
|
206
206
|
);
|
|
207
|
-
this.ctx.drawImage(bitmap, 0, 0);
|
|
207
|
+
this.ctx.drawImage(bitmap, 0, 0, this.canvas.width, this.canvas.height);
|
|
208
208
|
this.ctx.filter = "none";
|
|
209
209
|
this.ctx.globalCompositeOperation = "source-in";
|
|
210
210
|
if (this.backgroundImage) {
|
|
@@ -236,18 +236,18 @@ var BackgroundProcessor = class extends VideoTransformer {
|
|
|
236
236
|
this.ctx.globalCompositeOperation = "copy";
|
|
237
237
|
const bitmap = await maskToBitmap(
|
|
238
238
|
this.segmentationResults.categoryMask,
|
|
239
|
-
this.
|
|
240
|
-
this.
|
|
239
|
+
this.segmentationResults.categoryMask.width,
|
|
240
|
+
this.segmentationResults.categoryMask.height
|
|
241
241
|
);
|
|
242
242
|
this.ctx.filter = "blur(3px)";
|
|
243
243
|
this.ctx.globalCompositeOperation = "copy";
|
|
244
|
-
this.ctx.drawImage(bitmap, 0, 0);
|
|
244
|
+
this.ctx.drawImage(bitmap, 0, 0, this.canvas.width, this.canvas.height);
|
|
245
245
|
this.ctx.filter = "none";
|
|
246
246
|
this.ctx.globalCompositeOperation = "source-out";
|
|
247
247
|
this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);
|
|
248
248
|
this.ctx.globalCompositeOperation = "destination-over";
|
|
249
249
|
this.ctx.filter = `blur(${this.blurRadius}px)`;
|
|
250
|
-
this.ctx.drawImage(frame, 0, 0);
|
|
250
|
+
this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);
|
|
251
251
|
this.ctx.restore();
|
|
252
252
|
}
|
|
253
253
|
};
|
package/dist/index.mjs.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/ProcessorPipeline.ts","../src/transformers/BackgroundTransformer.ts","../package.json","../src/transformers/VideoTransformer.ts","../src/transformers/DummyTransformer.ts","../src/index.ts"],"sourcesContent":["import type { ProcessorOptions, Track, TrackProcessor } from 'livekit-client';\nimport { VideoTrackTransformer } from './transformers';\n\nexport default class ProcessorPipeline implements TrackProcessor<Track.Kind> {\n static get isSupported() {\n return (\n typeof MediaStreamTrackGenerator !== 'undefined' &&\n typeof MediaStreamTrackProcessor !== 'undefined'\n );\n }\n\n name: string;\n\n source?: MediaStreamVideoTrack;\n\n sourceSettings?: MediaTrackSettings;\n\n processor?: MediaStreamTrackProcessor<VideoFrame>;\n\n trackGenerator?: MediaStreamTrackGenerator<VideoFrame>;\n\n canvas?: OffscreenCanvas;\n\n sourceDummy?: HTMLMediaElement;\n\n processedTrack?: MediaStreamTrack;\n\n transformers: Array<VideoTrackTransformer>;\n\n constructor(transformers: Array<VideoTrackTransformer>, name: string) {\n this.name = name;\n this.transformers = transformers;\n }\n\n async init(opts: ProcessorOptions<Track.Kind>) {\n this.source = opts.track as MediaStreamVideoTrack;\n const origConstraints = this.source.getConstraints();\n await this.source.applyConstraints({\n ...origConstraints,\n // @ts-expect-error when a mediastream track is resized and/or cropped, the `VideoFrame` will have a coded height/width of the original video size\n // this leads to a shift of the underlying video as the frame itself is being rendered with the coded size\n // but image segmentation is based on the display dimensions (-> the cropped version)\n // in order to prevent this, we force the resize mode to \"none\"\n resizeMode: 'none',\n });\n this.sourceSettings = this.source.getSettings();\n this.sourceDummy = opts.element;\n if (this.sourceDummy instanceof HTMLVideoElement) {\n this.sourceDummy.height = this.sourceSettings.height!;\n this.sourceDummy.width = this.sourceSettings.width!;\n }\n if (!(this.sourceDummy instanceof HTMLVideoElement)) {\n throw TypeError('Currently only video transformers are supported');\n }\n // TODO explore if we can do all the processing work in a webworker\n this.processor = new MediaStreamTrackProcessor({ track: this.source });\n\n this.trackGenerator = new MediaStreamTrackGenerator({\n kind: 'video',\n signalTarget: this.source,\n });\n\n this.canvas = new OffscreenCanvas(\n this.sourceSettings.width ?? 300,\n this.sourceSettings.height ?? 300,\n );\n\n let readableStream = this.processor.readable;\n for (const transformer of this.transformers) {\n await transformer.init({\n outputCanvas: this.canvas,\n inputElement: this.sourceDummy!,\n });\n readableStream = readableStream.pipeThrough(transformer!.transformer!);\n }\n readableStream\n .pipeTo(this.trackGenerator.writable)\n .catch((e) => console.error('error when trying to pipe', e))\n .finally(() => this.destroy());\n this.processedTrack = this.trackGenerator as MediaStreamVideoTrack;\n }\n\n async destroy() {\n for (const transformer of this.transformers) {\n await transformer.destroy();\n }\n this.trackGenerator?.stop();\n }\n}\n","import * as vision from '@mediapipe/tasks-vision';\nimport { dependencies } from '../../package.json';\nimport VideoTransformer from './VideoTransformer';\nimport { VideoTransformerInitOptions } from './types';\n\nexport type SegmenterBaseOptions = Partial<vision.ImageSegmenterOptions['baseOptions']>;\n\nexport type BackgroundOptions = {\n blurRadius?: number;\n imagePath?: string;\n segmenterOptions?: SegmenterBaseOptions;\n};\n\nexport default class BackgroundProcessor extends VideoTransformer {\n static get isSupported() {\n return typeof OffscreenCanvas !== 'undefined';\n }\n\n imageSegmenter?: vision.ImageSegmenter;\n\n segmentationResults: vision.ImageSegmenterResult | undefined;\n\n backgroundImage: ImageBitmap | null = null;\n\n blurRadius?: number;\n\n options: BackgroundOptions;\n\n constructor(opts: BackgroundOptions) {\n super();\n this.options = opts;\n if (opts.blurRadius) {\n this.blurRadius = opts.blurRadius;\n } else if (opts.imagePath) {\n this.loadBackground(opts.imagePath);\n }\n\n this.options.segmenterOptions;\n }\n\n async init({ outputCanvas, inputElement: inputVideo }: VideoTransformerInitOptions) {\n await super.init({ outputCanvas, inputElement: inputVideo });\n\n const fileSet = await vision.FilesetResolver.forVisionTasks(\n `https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@${dependencies['@mediapipe/tasks-vision']}/wasm`,\n );\n\n this.imageSegmenter = await vision.ImageSegmenter.createFromOptions(fileSet, {\n baseOptions: {\n modelAssetPath:\n 'https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_segmenter/float16/latest/selfie_segmenter.tflite',\n delegate: 'GPU',\n ...this.options.segmenterOptions,\n },\n runningMode: 'VIDEO',\n outputCategoryMask: true,\n outputConfidenceMasks: false,\n });\n\n // this.loadBackground(opts.backgroundUrl).catch((e) => console.error(e));\n }\n\n async destroy() {\n await super.destroy();\n await this.imageSegmenter?.close();\n this.backgroundImage = null;\n }\n\n async loadBackground(path: string) {\n const img = new Image();\n\n await new Promise((resolve, reject) => {\n img.crossOrigin = 'Anonymous';\n img.onload = () => resolve(img);\n img.onerror = (err) => reject(err);\n img.src = path;\n });\n const imageData = await createImageBitmap(img);\n this.backgroundImage = imageData;\n }\n\n async transform(frame: VideoFrame, controller: TransformStreamDefaultController<VideoFrame>) {\n try {\n if (this.isDisabled) {\n controller.enqueue(frame);\n return;\n }\n if (!this.canvas) {\n throw TypeError('Canvas needs to be initialized first');\n }\n let startTimeMs = performance.now();\n this.imageSegmenter?.segmentForVideo(\n this.inputVideo!,\n startTimeMs,\n (result) => (this.segmentationResults = result),\n );\n\n if (this.blurRadius) {\n await this.blurBackground(frame);\n } else {\n await this.drawVirtualBackground(frame);\n }\n const newFrame = new VideoFrame(this.canvas, {\n timestamp: frame.timestamp || Date.now(),\n });\n controller.enqueue(newFrame);\n } finally {\n frame.close();\n }\n }\n\n async drawVirtualBackground(frame: VideoFrame) {\n if (!this.canvas || !this.ctx || !this.segmentationResults || !this.inputVideo) return;\n // this.ctx.save();\n // this.ctx.clearRect(0, 0, this.canvas.width, this.canvas.height);\n if (this.segmentationResults?.categoryMask) {\n this.ctx.filter = 'blur(3px)';\n this.ctx.globalCompositeOperation = 'copy';\n const bitmap = await maskToBitmap(\n this.segmentationResults.categoryMask,\n this.inputVideo.videoWidth,\n this.inputVideo.videoHeight,\n );\n this.ctx.drawImage(bitmap, 0, 0);\n this.ctx.filter = 'none';\n this.ctx.globalCompositeOperation = 'source-in';\n if (this.backgroundImage) {\n this.ctx.drawImage(\n this.backgroundImage,\n 0,\n 0,\n this.backgroundImage.width,\n this.backgroundImage.height,\n 0,\n 0,\n this.canvas.width,\n this.canvas.height,\n );\n } else {\n this.ctx.fillStyle = '#00FF00';\n this.ctx.fillRect(0, 0, this.canvas.width, this.canvas.height);\n }\n\n this.ctx.globalCompositeOperation = 'destination-over';\n }\n this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);\n // this.ctx.restore();\n }\n\n async blurBackground(frame: VideoFrame) {\n if (\n !this.ctx ||\n !this.canvas ||\n !this.segmentationResults?.categoryMask?.canvas ||\n !this.inputVideo\n ) {\n return;\n }\n\n this.ctx.save();\n this.ctx.globalCompositeOperation = 'copy';\n\n const bitmap = await maskToBitmap(\n this.segmentationResults.categoryMask,\n this.inputVideo.videoWidth,\n this.inputVideo.videoHeight,\n );\n\n this.ctx.filter = 'blur(3px)';\n this.ctx.globalCompositeOperation = 'copy';\n this.ctx.drawImage(bitmap, 0, 0);\n this.ctx.filter = 'none';\n this.ctx.globalCompositeOperation = 'source-out';\n this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);\n this.ctx.globalCompositeOperation = 'destination-over';\n this.ctx.filter = `blur(${this.blurRadius}px)`;\n this.ctx.drawImage(frame, 0, 0);\n this.ctx.restore();\n }\n}\n\nfunction maskToBitmap(\n mask: vision.MPMask,\n videoWidth: number,\n videoHeight: number,\n): Promise<ImageBitmap> {\n const dataArray: Uint8ClampedArray = new Uint8ClampedArray(videoWidth * videoHeight * 4);\n const result = mask.getAsUint8Array();\n for (let i = 0; i < result.length; i += 1) {\n dataArray[i * 4] = result[i];\n dataArray[i * 4 + 1] = result[i];\n dataArray[i * 4 + 2] = result[i];\n dataArray[i * 4 + 3] = result[i];\n }\n const dataNew = new ImageData(dataArray, videoWidth, videoHeight);\n\n return createImageBitmap(dataNew);\n}\n","{\n \"name\": \"@livekit/track-processors\",\n \"version\": \"0.2.4\",\n \"description\": \"LiveKit track processors\",\n \"main\": \"dist/index.js\",\n \"module\": \"dist/index.mjs\",\n \"source\": \"src/index.ts\",\n \"types\": \"dist/src/index.d.ts\",\n \"repository\": \"git@github.com:livekit/livekit-track-processors.git\",\n \"author\": \"Lukas Seiler\",\n \"license\": \"Apache-2.0\",\n \"scripts\": {\n \"build\": \"tsup --onSuccess \\\"tsc --declaration --emitDeclarationOnly\\\"\",\n \"build-docs\": \"yarn exec typedoc\",\n \"build-sample\": \"cd example && webpack && cp styles.css index.html dist/\",\n \"lint\": \"eslint src\",\n \"test\": \"jest\",\n \"sample\": \"vite serve example --port 8080 --open\",\n \"prepublish\": \"yarn build\"\n },\n \"files\": [\n \"dist\",\n \"src\"\n ],\n \"dependencies\": {\n \"@mediapipe/holistic\": \"0.5.1675471629\",\n \"@mediapipe/tasks-vision\": \"0.10.1\"\n },\n \"peerDependencies\": {\n \"livekit-client\": \"^1.12.0\"\n },\n \"devDependencies\": {\n \"@trivago/prettier-plugin-sort-imports\": \"^4.1.1\",\n \"@types/dom-mediacapture-transform\": \"^0.1.5\",\n \"@types/jest\": \"^27.0.3\",\n \"@types/offscreencanvas\": \"^2019.7.0\",\n \"@typescript-eslint/eslint-plugin\": \"^4.31.2\",\n \"@webpack-cli/serve\": \"^1.5.2\",\n \"eslint\": \"8.39.0\",\n \"eslint-config-airbnb-typescript\": \"17.0.0\",\n \"eslint-config-prettier\": \"8.8.0\",\n \"eslint-plugin-ecmascript-compat\": \"^3.0.0\",\n \"eslint-plugin-import\": \"2.27.5\",\n \"jest\": \"^27.4.3\",\n \"livekit-client\": \"^0.16.6\",\n \"prettier\": \"^2.8.8\",\n \"ts-jest\": \"^27.0.7\",\n \"ts-loader\": \"^8.1.0\",\n \"ts-proto\": \"^1.85.0\",\n \"tsup\": \"^7.1.0\",\n \"typedoc\": \"^0.20.35\",\n \"typedoc-plugin-no-inherit\": \"1.3.0\",\n \"typescript\": \"^5.0.4\",\n \"vite\": \"^4.3.8\",\n \"webpack\": \"^5.53.0\",\n \"webpack-cli\": \"^4.8.0\",\n \"webpack-dev-server\": \"^4.2.1\"\n }\n}\n","import { VideoTrackTransformer, VideoTransformerInitOptions } from './types';\n\nexport default abstract class VideoTransformer implements VideoTrackTransformer {\n transformer?: TransformStream;\n\n canvas?: OffscreenCanvas;\n\n ctx?: OffscreenCanvasRenderingContext2D;\n\n inputVideo?: HTMLVideoElement;\n\n protected isDisabled?: Boolean = false;\n\n async init({\n outputCanvas,\n inputElement: inputVideo,\n }: VideoTransformerInitOptions): Promise<void> {\n if (!(inputVideo instanceof HTMLVideoElement)) {\n throw TypeError('Video transformer needs a HTMLVideoElement as input');\n }\n this.transformer = new TransformStream({\n transform: (frame, controller) => this.transform(frame, controller),\n });\n this.canvas = outputCanvas || null;\n if (outputCanvas) {\n this.ctx = this.canvas?.getContext('2d') || undefined;\n }\n this.inputVideo = inputVideo;\n this.isDisabled = false;\n }\n\n async destroy() {\n this.isDisabled = true;\n this.canvas = undefined;\n this.ctx = undefined;\n }\n\n abstract transform(\n frame: VideoFrame,\n controller: TransformStreamDefaultController<VideoFrame>,\n ): void;\n}\n","import VideoTransformer from './VideoTransformer';\n\nexport default class DummyTransformer extends VideoTransformer {\n async transform(frame: VideoFrame, controller: TransformStreamDefaultController<VideoFrame>) {\n controller.enqueue(frame);\n }\n\n async destroy() {\n // nothing to do\n }\n}\n","import ProcessorPipeline from './ProcessorPipeline';\nimport BackgroundTransformer, { SegmenterBaseOptions } from './transformers/BackgroundTransformer';\nimport DummyTransformer from './transformers/DummyTransformer';\n\nexport const BackgroundBlur = (\n blurRadius: number = 10,\n segmenterOptions?: SegmenterBaseOptions,\n) => {\n const isPipelineSupported = ProcessorPipeline.isSupported && BackgroundTransformer.isSupported;\n if (!isPipelineSupported) {\n throw new Error('pipeline is not supported in this browser');\n }\n const pipeline = new ProcessorPipeline(\n [new BackgroundTransformer({ blurRadius, segmenterOptions })],\n 'background-blur',\n );\n return pipeline;\n};\n\nexport const VirtualBackground = (imagePath: string, segmenterOptions?: SegmenterBaseOptions) => {\n const isPipelineSupported = ProcessorPipeline.isSupported && BackgroundTransformer.isSupported;\n if (!isPipelineSupported) {\n throw new Error('pipeline is not supported in this browser');\n }\n const pipeline = new ProcessorPipeline(\n [new BackgroundTransformer({ imagePath, segmenterOptions })],\n 'virtual-background',\n );\n return pipeline;\n};\n\nexport const Dummy = () => {\n const isPipelineSupported = ProcessorPipeline.isSupported && BackgroundTransformer.isSupported;\n if (!isPipelineSupported) {\n throw new Error('pipeline is not supported in this browser');\n }\n const pipeline = new ProcessorPipeline([new DummyTransformer()], 'dummy');\n return pipeline;\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AAGA,IAAqB,oBAArB,MAA6E;AAAA,EAC3E,WAAW,cAAc;AACvB,WACE,OAAO,8BAA8B,eACrC,OAAO,8BAA8B;AAAA,EAEzC;AAAA,EAoBA,YAAY,cAA4C,MAAc;AACpE,SAAK,OAAO;AACZ,SAAK,eAAe;AAAA,EACtB;AAAA,EAEA,MAAM,KAAK,MAAoC;AAlCjD;AAmCI,SAAK,SAAS,KAAK;AACnB,UAAM,kBAAkB,KAAK,OAAO,eAAe;AACnD,UAAM,KAAK,OAAO,iBAAiB,iCAC9B,kBAD8B;AAAA;AAAA;AAAA;AAAA;AAAA,MAMjC,YAAY;AAAA,IACd,EAAC;AACD,SAAK,iBAAiB,KAAK,OAAO,YAAY;AAC9C,SAAK,cAAc,KAAK;AACxB,QAAI,KAAK,uBAAuB,kBAAkB;AAChD,WAAK,YAAY,SAAS,KAAK,eAAe;AAC9C,WAAK,YAAY,QAAQ,KAAK,eAAe;AAAA,IAC/C;AACA,QAAI,EAAE,KAAK,uBAAuB,mBAAmB;AACnD,YAAM,UAAU,iDAAiD;AAAA,IACnE;AAEA,SAAK,YAAY,IAAI,0BAA0B,EAAE,OAAO,KAAK,OAAO,CAAC;AAErE,SAAK,iBAAiB,IAAI,0BAA0B;AAAA,MAClD,MAAM;AAAA,MACN,cAAc,KAAK;AAAA,IACrB,CAAC;AAED,SAAK,SAAS,IAAI;AAAA,OAChB,UAAK,eAAe,UAApB,YAA6B;AAAA,OAC7B,UAAK,eAAe,WAApB,YAA8B;AAAA,IAChC;AAEA,QAAI,iBAAiB,KAAK,UAAU;AACpC,eAAW,eAAe,KAAK,cAAc;AAC3C,YAAM,YAAY,KAAK;AAAA,QACrB,cAAc,KAAK;AAAA,QACnB,cAAc,KAAK;AAAA,MACrB,CAAC;AACD,uBAAiB,eAAe,YAAY,YAAa,WAAY;AAAA,IACvE;AACA,mBACG,OAAO,KAAK,eAAe,QAAQ,EACnC,MAAM,CAAC,MAAM,QAAQ,MAAM,6BAA6B,CAAC,CAAC,EAC1D,QAAQ,MAAM,KAAK,QAAQ,CAAC;AAC/B,SAAK,iBAAiB,KAAK;AAAA,EAC7B;AAAA,EAEA,MAAM,UAAU;AAlFlB;AAmFI,eAAW,eAAe,KAAK,cAAc;AAC3C,YAAM,YAAY,QAAQ;AAAA,IAC5B;AACA,eAAK,mBAAL,mBAAqB;AAAA,EACvB;AACF;;;ACxFA,YAAY,YAAY;;;ACwBtB,mBAAgB;AAAA,EACd,uBAAuB;AAAA,EACvB,2BAA2B;AAC7B;;;ACzBF,IAA8B,mBAA9B,MAAgF;AAAA,EAAhF;AASE,SAAU,aAAuB;AAAA;AAAA,EAEjC,MAAM,KAAK;AAAA,IACT;AAAA,IACA,cAAc;AAAA,EAChB,GAA+C;AAhBjD;AAiBI,QAAI,EAAE,sBAAsB,mBAAmB;AAC7C,YAAM,UAAU,qDAAqD;AAAA,IACvE;AACA,SAAK,cAAc,IAAI,gBAAgB;AAAA,MACrC,WAAW,CAAC,OAAO,eAAe,KAAK,UAAU,OAAO,UAAU;AAAA,IACpE,CAAC;AACD,SAAK,SAAS,gBAAgB;AAC9B,QAAI,cAAc;AAChB,WAAK,QAAM,UAAK,WAAL,mBAAa,WAAW,UAAS;AAAA,IAC9C;AACA,SAAK,aAAa;AAClB,SAAK,aAAa;AAAA,EACpB;AAAA,EAEA,MAAM,UAAU;AACd,SAAK,aAAa;AAClB,SAAK,SAAS;AACd,SAAK,MAAM;AAAA,EACb;AAMF;;;AF5BA,IAAqB,sBAArB,cAAiD,iBAAiB;AAAA,EAehE,YAAY,MAAyB;AACnC,UAAM;AAPR,2BAAsC;AAQpC,SAAK,UAAU;AACf,QAAI,KAAK,YAAY;AACnB,WAAK,aAAa,KAAK;AAAA,IACzB,WAAW,KAAK,WAAW;AACzB,WAAK,eAAe,KAAK,SAAS;AAAA,IACpC;AAEA,SAAK,QAAQ;AAAA,EACf;AAAA,EAxBA,WAAW,cAAc;AACvB,WAAO,OAAO,oBAAoB;AAAA,EACpC;AAAA,EAwBA,MAAM,KAAK,EAAE,cAAc,cAAc,WAAW,GAAgC;AAClF,UAAM,MAAM,KAAK,EAAE,cAAc,cAAc,WAAW,CAAC;AAE3D,UAAM,UAAU,MAAa,uBAAgB;AAAA,MAC3C,wDAAwD,aAAa,yBAAyB,CAAC;AAAA,IACjG;AAEA,SAAK,iBAAiB,MAAa,sBAAe,kBAAkB,SAAS;AAAA,MAC3E,aAAa;AAAA,QACX,gBACE;AAAA,QACF,UAAU;AAAA,SACP,KAAK,QAAQ;AAAA,MAElB,aAAa;AAAA,MACb,oBAAoB;AAAA,MACpB,uBAAuB;AAAA,IACzB,CAAC;AAAA,EAGH;AAAA,EAEA,MAAM,UAAU;AA9DlB;AA+DI,UAAM,MAAM,QAAQ;AACpB,YAAM,UAAK,mBAAL,mBAAqB;AAC3B,SAAK,kBAAkB;AAAA,EACzB;AAAA,EAEA,MAAM,eAAe,MAAc;AACjC,UAAM,MAAM,IAAI,MAAM;AAEtB,UAAM,IAAI,QAAQ,CAAC,SAAS,WAAW;AACrC,UAAI,cAAc;AAClB,UAAI,SAAS,MAAM,QAAQ,GAAG;AAC9B,UAAI,UAAU,CAAC,QAAQ,OAAO,GAAG;AACjC,UAAI,MAAM;AAAA,IACZ,CAAC;AACD,UAAM,YAAY,MAAM,kBAAkB,GAAG;AAC7C,SAAK,kBAAkB;AAAA,EACzB;AAAA,EAEA,MAAM,UAAU,OAAmB,YAA0D;AAjF/F;AAkFI,QAAI;AACF,UAAI,KAAK,YAAY;AACnB,mBAAW,QAAQ,KAAK;AACxB;AAAA,MACF;AACA,UAAI,CAAC,KAAK,QAAQ;AAChB,cAAM,UAAU,sCAAsC;AAAA,MACxD;AACA,UAAI,cAAc,YAAY,IAAI;AAClC,iBAAK,mBAAL,mBAAqB;AAAA,QACnB,KAAK;AAAA,QACL;AAAA,QACA,CAAC,WAAY,KAAK,sBAAsB;AAAA;AAG1C,UAAI,KAAK,YAAY;AACnB,cAAM,KAAK,eAAe,KAAK;AAAA,MACjC,OAAO;AACL,cAAM,KAAK,sBAAsB,KAAK;AAAA,MACxC;AACA,YAAM,WAAW,IAAI,WAAW,KAAK,QAAQ;AAAA,QAC3C,WAAW,MAAM,aAAa,KAAK,IAAI;AAAA,MACzC,CAAC;AACD,iBAAW,QAAQ,QAAQ;AAAA,IAC7B,UAAE;AACA,YAAM,MAAM;AAAA,IACd;AAAA,EACF;AAAA,EAEA,MAAM,sBAAsB,OAAmB;AA/GjD;AAgHI,QAAI,CAAC,KAAK,UAAU,CAAC,KAAK,OAAO,CAAC,KAAK,uBAAuB,CAAC,KAAK;AAAY;AAGhF,SAAI,UAAK,wBAAL,mBAA0B,cAAc;AAC1C,WAAK,IAAI,SAAS;AAClB,WAAK,IAAI,2BAA2B;AACpC,YAAM,SAAS,MAAM;AAAA,QACnB,KAAK,oBAAoB;AAAA,QACzB,KAAK,WAAW;AAAA,QAChB,KAAK,WAAW;AAAA,MAClB;AACA,WAAK,IAAI,UAAU,QAAQ,GAAG,CAAC;AAC/B,WAAK,IAAI,SAAS;AAClB,WAAK,IAAI,2BAA2B;AACpC,UAAI,KAAK,iBAAiB;AACxB,aAAK,IAAI;AAAA,UACP,KAAK;AAAA,UACL;AAAA,UACA;AAAA,UACA,KAAK,gBAAgB;AAAA,UACrB,KAAK,gBAAgB;AAAA,UACrB;AAAA,UACA;AAAA,UACA,KAAK,OAAO;AAAA,UACZ,KAAK,OAAO;AAAA,QACd;AAAA,MACF,OAAO;AACL,aAAK,IAAI,YAAY;AACrB,aAAK,IAAI,SAAS,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AAAA,MAC/D;AAEA,WAAK,IAAI,2BAA2B;AAAA,IACtC;AACA,SAAK,IAAI,UAAU,OAAO,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AAAA,EAEvE;AAAA,EAEA,MAAM,eAAe,OAAmB;AArJ1C;AAsJI,QACE,CAAC,KAAK,OACN,CAAC,KAAK,UACN,GAAC,gBAAK,wBAAL,mBAA0B,iBAA1B,mBAAwC,WACzC,CAAC,KAAK,YACN;AACA;AAAA,IACF;AAEA,SAAK,IAAI,KAAK;AACd,SAAK,IAAI,2BAA2B;AAEpC,UAAM,SAAS,MAAM;AAAA,MACnB,KAAK,oBAAoB;AAAA,MACzB,KAAK,WAAW;AAAA,MAChB,KAAK,WAAW;AAAA,IAClB;AAEA,SAAK,IAAI,SAAS;AAClB,SAAK,IAAI,2BAA2B;AACpC,SAAK,IAAI,UAAU,QAAQ,GAAG,CAAC;AAC/B,SAAK,IAAI,SAAS;AAClB,SAAK,IAAI,2BAA2B;AACpC,SAAK,IAAI,UAAU,OAAO,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AACrE,SAAK,IAAI,2BAA2B;AACpC,SAAK,IAAI,SAAS,QAAQ,KAAK,UAAU;AACzC,SAAK,IAAI,UAAU,OAAO,GAAG,CAAC;AAC9B,SAAK,IAAI,QAAQ;AAAA,EACnB;AACF;AAEA,SAAS,aACP,MACA,YACA,aACsB;AACtB,QAAM,YAA+B,IAAI,kBAAkB,aAAa,cAAc,CAAC;AACvF,QAAM,SAAS,KAAK,gBAAgB;AACpC,WAAS,IAAI,GAAG,IAAI,OAAO,QAAQ,KAAK,GAAG;AACzC,cAAU,IAAI,CAAC,IAAI,OAAO,CAAC;AAC3B,cAAU,IAAI,IAAI,CAAC,IAAI,OAAO,CAAC;AAC/B,cAAU,IAAI,IAAI,CAAC,IAAI,OAAO,CAAC;AAC/B,cAAU,IAAI,IAAI,CAAC,IAAI,OAAO,CAAC;AAAA,EACjC;AACA,QAAM,UAAU,IAAI,UAAU,WAAW,YAAY,WAAW;AAEhE,SAAO,kBAAkB,OAAO;AAClC;;;AGnMA,IAAqB,mBAArB,cAA8C,iBAAiB;AAAA,EAC7D,MAAM,UAAU,OAAmB,YAA0D;AAC3F,eAAW,QAAQ,KAAK;AAAA,EAC1B;AAAA,EAEA,MAAM,UAAU;AAAA,EAEhB;AACF;;;ACNO,IAAM,iBAAiB,CAC5B,aAAqB,IACrB,qBACG;AACH,QAAM,sBAAsB,kBAAkB,eAAe,oBAAsB;AACnF,MAAI,CAAC,qBAAqB;AACxB,UAAM,IAAI,MAAM,2CAA2C;AAAA,EAC7D;AACA,QAAM,WAAW,IAAI;AAAA,IACnB,CAAC,IAAI,oBAAsB,EAAE,YAAY,iBAAiB,CAAC,CAAC;AAAA,IAC5D;AAAA,EACF;AACA,SAAO;AACT;AAEO,IAAM,oBAAoB,CAAC,WAAmB,qBAA4C;AAC/F,QAAM,sBAAsB,kBAAkB,eAAe,oBAAsB;AACnF,MAAI,CAAC,qBAAqB;AACxB,UAAM,IAAI,MAAM,2CAA2C;AAAA,EAC7D;AACA,QAAM,WAAW,IAAI;AAAA,IACnB,CAAC,IAAI,oBAAsB,EAAE,WAAW,iBAAiB,CAAC,CAAC;AAAA,IAC3D;AAAA,EACF;AACA,SAAO;AACT;AAEO,IAAM,QAAQ,MAAM;AACzB,QAAM,sBAAsB,kBAAkB,eAAe,oBAAsB;AACnF,MAAI,CAAC,qBAAqB;AACxB,UAAM,IAAI,MAAM,2CAA2C;AAAA,EAC7D;AACA,QAAM,WAAW,IAAI,kBAAkB,CAAC,IAAI,iBAAiB,CAAC,GAAG,OAAO;AACxE,SAAO;AACT;","names":[]}
|
|
1
|
+
{"version":3,"sources":["../src/ProcessorPipeline.ts","../src/transformers/BackgroundTransformer.ts","../package.json","../src/transformers/VideoTransformer.ts","../src/transformers/DummyTransformer.ts","../src/index.ts"],"sourcesContent":["import type { ProcessorOptions, Track, TrackProcessor } from 'livekit-client';\nimport { VideoTrackTransformer } from './transformers';\n\nexport default class ProcessorPipeline implements TrackProcessor<Track.Kind> {\n static get isSupported() {\n return (\n typeof MediaStreamTrackGenerator !== 'undefined' &&\n typeof MediaStreamTrackProcessor !== 'undefined'\n );\n }\n\n name: string;\n\n source?: MediaStreamVideoTrack;\n\n sourceSettings?: MediaTrackSettings;\n\n processor?: MediaStreamTrackProcessor<VideoFrame>;\n\n trackGenerator?: MediaStreamTrackGenerator<VideoFrame>;\n\n canvas?: OffscreenCanvas;\n\n sourceDummy?: HTMLMediaElement;\n\n processedTrack?: MediaStreamTrack;\n\n transformers: Array<VideoTrackTransformer>;\n\n constructor(transformers: Array<VideoTrackTransformer>, name: string) {\n this.name = name;\n this.transformers = transformers;\n }\n\n async init(opts: ProcessorOptions<Track.Kind>) {\n this.source = opts.track as MediaStreamVideoTrack;\n const origConstraints = this.source.getConstraints();\n await this.source.applyConstraints({\n ...origConstraints,\n // @ts-expect-error when a mediastream track is resized and/or cropped, the `VideoFrame` will have a coded height/width of the original video size\n // this leads to a shift of the underlying video as the frame itself is being rendered with the coded size\n // but image segmentation is based on the display dimensions (-> the cropped version)\n // in order to prevent this, we force the resize mode to \"none\"\n resizeMode: 'none',\n });\n this.sourceSettings = this.source.getSettings();\n this.sourceDummy = opts.element;\n if (this.sourceDummy instanceof HTMLVideoElement) {\n this.sourceDummy.height = this.sourceSettings.height ?? 300;\n this.sourceDummy.width = this.sourceSettings.width ?? 300;\n }\n if (!(this.sourceDummy instanceof HTMLVideoElement)) {\n throw TypeError('Currently only video transformers are supported');\n }\n // TODO explore if we can do all the processing work in a webworker\n this.processor = new MediaStreamTrackProcessor({ track: this.source });\n\n this.trackGenerator = new MediaStreamTrackGenerator({\n kind: 'video',\n signalTarget: this.source,\n });\n\n this.canvas = new OffscreenCanvas(\n this.sourceSettings.width ?? 300,\n this.sourceSettings.height ?? 300,\n );\n\n let readableStream = this.processor.readable;\n for (const transformer of this.transformers) {\n await transformer.init({\n outputCanvas: this.canvas,\n inputElement: this.sourceDummy!,\n });\n readableStream = readableStream.pipeThrough(transformer!.transformer!);\n }\n readableStream\n .pipeTo(this.trackGenerator.writable)\n .catch((e) => console.error('error when trying to pipe', e))\n .finally(() => this.destroy());\n this.processedTrack = this.trackGenerator as MediaStreamVideoTrack;\n }\n\n async destroy() {\n for (const transformer of this.transformers) {\n await transformer.destroy();\n }\n this.trackGenerator?.stop();\n }\n}\n","import * as vision from '@mediapipe/tasks-vision';\nimport { dependencies } from '../../package.json';\nimport VideoTransformer from './VideoTransformer';\nimport { VideoTransformerInitOptions } from './types';\n\nexport type SegmenterBaseOptions = Partial<vision.ImageSegmenterOptions['baseOptions']>;\n\nexport type BackgroundOptions = {\n blurRadius?: number;\n imagePath?: string;\n segmenterOptions?: SegmenterBaseOptions;\n};\n\nexport default class BackgroundProcessor extends VideoTransformer {\n static get isSupported() {\n return typeof OffscreenCanvas !== 'undefined';\n }\n\n imageSegmenter?: vision.ImageSegmenter;\n\n segmentationResults: vision.ImageSegmenterResult | undefined;\n\n backgroundImage: ImageBitmap | null = null;\n\n blurRadius?: number;\n\n options: BackgroundOptions;\n\n constructor(opts: BackgroundOptions) {\n super();\n this.options = opts;\n if (opts.blurRadius) {\n this.blurRadius = opts.blurRadius;\n } else if (opts.imagePath) {\n this.loadBackground(opts.imagePath);\n }\n\n this.options.segmenterOptions;\n }\n\n async init({ outputCanvas, inputElement: inputVideo }: VideoTransformerInitOptions) {\n await super.init({ outputCanvas, inputElement: inputVideo });\n\n const fileSet = await vision.FilesetResolver.forVisionTasks(\n `https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@${dependencies['@mediapipe/tasks-vision']}/wasm`,\n );\n\n this.imageSegmenter = await vision.ImageSegmenter.createFromOptions(fileSet, {\n baseOptions: {\n modelAssetPath:\n 'https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_segmenter/float16/latest/selfie_segmenter.tflite',\n delegate: 'GPU',\n ...this.options.segmenterOptions,\n },\n runningMode: 'VIDEO',\n outputCategoryMask: true,\n outputConfidenceMasks: false,\n });\n\n // this.loadBackground(opts.backgroundUrl).catch((e) => console.error(e));\n }\n\n async destroy() {\n await super.destroy();\n await this.imageSegmenter?.close();\n this.backgroundImage = null;\n }\n\n async loadBackground(path: string) {\n const img = new Image();\n\n await new Promise((resolve, reject) => {\n img.crossOrigin = 'Anonymous';\n img.onload = () => resolve(img);\n img.onerror = (err) => reject(err);\n img.src = path;\n });\n const imageData = await createImageBitmap(img);\n this.backgroundImage = imageData;\n }\n\n async transform(frame: VideoFrame, controller: TransformStreamDefaultController<VideoFrame>) {\n try {\n if (this.isDisabled) {\n controller.enqueue(frame);\n return;\n }\n if (!this.canvas) {\n throw TypeError('Canvas needs to be initialized first');\n }\n let startTimeMs = performance.now();\n this.imageSegmenter?.segmentForVideo(\n this.inputVideo!,\n startTimeMs,\n (result) => (this.segmentationResults = result),\n );\n\n if (this.blurRadius) {\n await this.blurBackground(frame);\n } else {\n await this.drawVirtualBackground(frame);\n }\n const newFrame = new VideoFrame(this.canvas, {\n timestamp: frame.timestamp || Date.now(),\n });\n controller.enqueue(newFrame);\n } finally {\n frame.close();\n }\n }\n\n async drawVirtualBackground(frame: VideoFrame) {\n if (!this.canvas || !this.ctx || !this.segmentationResults || !this.inputVideo) return;\n // this.ctx.save();\n // this.ctx.clearRect(0, 0, this.canvas.width, this.canvas.height);\n if (this.segmentationResults?.categoryMask) {\n this.ctx.filter = 'blur(3px)';\n this.ctx.globalCompositeOperation = 'copy';\n const bitmap = await maskToBitmap(\n this.segmentationResults.categoryMask,\n this.segmentationResults.categoryMask.width,\n this.segmentationResults.categoryMask.height,\n );\n this.ctx.drawImage(bitmap, 0, 0, this.canvas.width, this.canvas.height);\n this.ctx.filter = 'none';\n this.ctx.globalCompositeOperation = 'source-in';\n if (this.backgroundImage) {\n this.ctx.drawImage(\n this.backgroundImage,\n 0,\n 0,\n this.backgroundImage.width,\n this.backgroundImage.height,\n 0,\n 0,\n this.canvas.width,\n this.canvas.height,\n );\n } else {\n this.ctx.fillStyle = '#00FF00';\n this.ctx.fillRect(0, 0, this.canvas.width, this.canvas.height);\n }\n\n this.ctx.globalCompositeOperation = 'destination-over';\n }\n this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);\n // this.ctx.restore();\n }\n\n async blurBackground(frame: VideoFrame) {\n if (\n !this.ctx ||\n !this.canvas ||\n !this.segmentationResults?.categoryMask?.canvas ||\n !this.inputVideo\n ) {\n return;\n }\n\n this.ctx.save();\n this.ctx.globalCompositeOperation = 'copy';\n\n const bitmap = await maskToBitmap(\n this.segmentationResults.categoryMask,\n this.segmentationResults.categoryMask.width,\n this.segmentationResults.categoryMask.height,\n );\n\n this.ctx.filter = 'blur(3px)';\n this.ctx.globalCompositeOperation = 'copy';\n this.ctx.drawImage(bitmap, 0, 0, this.canvas.width, this.canvas.height);\n this.ctx.filter = 'none';\n this.ctx.globalCompositeOperation = 'source-out';\n this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);\n this.ctx.globalCompositeOperation = 'destination-over';\n this.ctx.filter = `blur(${this.blurRadius}px)`;\n this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);\n this.ctx.restore();\n }\n}\n\nfunction maskToBitmap(\n mask: vision.MPMask,\n videoWidth: number,\n videoHeight: number,\n): Promise<ImageBitmap> {\n const dataArray: Uint8ClampedArray = new Uint8ClampedArray(videoWidth * videoHeight * 4);\n const result = mask.getAsUint8Array();\n for (let i = 0; i < result.length; i += 1) {\n dataArray[i * 4] = result[i];\n dataArray[i * 4 + 1] = result[i];\n dataArray[i * 4 + 2] = result[i];\n dataArray[i * 4 + 3] = result[i];\n }\n const dataNew = new ImageData(dataArray, videoWidth, videoHeight);\n\n return createImageBitmap(dataNew);\n}\n","{\n \"name\": \"@livekit/track-processors\",\n \"version\": \"0.2.5\",\n \"description\": \"LiveKit track processors\",\n \"main\": \"dist/index.js\",\n \"module\": \"dist/index.mjs\",\n \"source\": \"src/index.ts\",\n \"types\": \"dist/src/index.d.ts\",\n \"repository\": \"git@github.com:livekit/livekit-track-processors.git\",\n \"author\": \"Lukas Seiler\",\n \"license\": \"Apache-2.0\",\n \"scripts\": {\n \"build\": \"tsup --onSuccess \\\"tsc --declaration --emitDeclarationOnly\\\"\",\n \"build-docs\": \"yarn exec typedoc\",\n \"build-sample\": \"cd example && vite build\",\n \"lint\": \"eslint src\",\n \"test\": \"jest\",\n \"sample\": \"vite serve example --port 8080 --open\",\n \"prepublish\": \"yarn build\"\n },\n \"files\": [\n \"dist\",\n \"src\"\n ],\n \"dependencies\": {\n \"@mediapipe/holistic\": \"0.5.1675471629\",\n \"@mediapipe/tasks-vision\": \"0.10.1\"\n },\n \"peerDependencies\": {\n \"livekit-client\": \"^1.12.0\"\n },\n \"devDependencies\": {\n \"@trivago/prettier-plugin-sort-imports\": \"^4.1.1\",\n \"@types/dom-mediacapture-transform\": \"^0.1.5\",\n \"@types/jest\": \"^27.0.3\",\n \"@types/offscreencanvas\": \"^2019.7.0\",\n \"@typescript-eslint/eslint-plugin\": \"^4.31.2\",\n \"@webpack-cli/serve\": \"^1.5.2\",\n \"eslint\": \"8.39.0\",\n \"eslint-config-airbnb-typescript\": \"17.0.0\",\n \"eslint-config-prettier\": \"8.8.0\",\n \"eslint-plugin-ecmascript-compat\": \"^3.0.0\",\n \"eslint-plugin-import\": \"2.27.5\",\n \"jest\": \"^27.4.3\",\n \"livekit-client\": \"^1.12.0\",\n \"prettier\": \"^2.8.8\",\n \"ts-jest\": \"^27.0.7\",\n \"ts-loader\": \"^8.1.0\",\n \"ts-proto\": \"^1.85.0\",\n \"tsup\": \"^7.1.0\",\n \"typedoc\": \"^0.20.35\",\n \"typedoc-plugin-no-inherit\": \"1.3.0\",\n \"typescript\": \"^5.0.4\",\n \"vite\": \"^4.3.8\",\n \"webpack\": \"^5.53.0\",\n \"webpack-cli\": \"^4.8.0\",\n \"webpack-dev-server\": \"^4.2.1\"\n }\n}\n","import { VideoTrackTransformer, VideoTransformerInitOptions } from './types';\n\nexport default abstract class VideoTransformer implements VideoTrackTransformer {\n transformer?: TransformStream;\n\n canvas?: OffscreenCanvas;\n\n ctx?: OffscreenCanvasRenderingContext2D;\n\n inputVideo?: HTMLVideoElement;\n\n protected isDisabled?: Boolean = false;\n\n async init({\n outputCanvas,\n inputElement: inputVideo,\n }: VideoTransformerInitOptions): Promise<void> {\n if (!(inputVideo instanceof HTMLVideoElement)) {\n throw TypeError('Video transformer needs a HTMLVideoElement as input');\n }\n this.transformer = new TransformStream({\n transform: (frame, controller) => this.transform(frame, controller),\n });\n this.canvas = outputCanvas || null;\n if (outputCanvas) {\n this.ctx = this.canvas?.getContext('2d') || undefined;\n }\n this.inputVideo = inputVideo;\n this.isDisabled = false;\n }\n\n async destroy() {\n this.isDisabled = true;\n this.canvas = undefined;\n this.ctx = undefined;\n }\n\n abstract transform(\n frame: VideoFrame,\n controller: TransformStreamDefaultController<VideoFrame>,\n ): void;\n}\n","import VideoTransformer from './VideoTransformer';\n\nexport default class DummyTransformer extends VideoTransformer {\n async transform(frame: VideoFrame, controller: TransformStreamDefaultController<VideoFrame>) {\n controller.enqueue(frame);\n }\n\n async destroy() {\n // nothing to do\n }\n}\n","import ProcessorPipeline from './ProcessorPipeline';\nimport BackgroundTransformer, { SegmenterBaseOptions } from './transformers/BackgroundTransformer';\nimport DummyTransformer from './transformers/DummyTransformer';\n\nexport * from './transformers/types';\n\nexport const BackgroundBlur = (\n blurRadius: number = 10,\n segmenterOptions?: SegmenterBaseOptions,\n) => {\n const isPipelineSupported = ProcessorPipeline.isSupported && BackgroundTransformer.isSupported;\n if (!isPipelineSupported) {\n throw new Error('pipeline is not supported in this browser');\n }\n const pipeline = new ProcessorPipeline(\n [new BackgroundTransformer({ blurRadius, segmenterOptions })],\n 'background-blur',\n );\n return pipeline;\n};\n\nexport const VirtualBackground = (imagePath: string, segmenterOptions?: SegmenterBaseOptions) => {\n const isPipelineSupported = ProcessorPipeline.isSupported && BackgroundTransformer.isSupported;\n if (!isPipelineSupported) {\n throw new Error('pipeline is not supported in this browser');\n }\n const pipeline = new ProcessorPipeline(\n [new BackgroundTransformer({ imagePath, segmenterOptions })],\n 'virtual-background',\n );\n return pipeline;\n};\n\nexport const Dummy = () => {\n const isPipelineSupported = ProcessorPipeline.isSupported && BackgroundTransformer.isSupported;\n if (!isPipelineSupported) {\n throw new Error('pipeline is not supported in this browser');\n }\n const pipeline = new ProcessorPipeline([new DummyTransformer()], 'dummy');\n return pipeline;\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AAGA,IAAqB,oBAArB,MAA6E;AAAA,EAC3E,WAAW,cAAc;AACvB,WACE,OAAO,8BAA8B,eACrC,OAAO,8BAA8B;AAAA,EAEzC;AAAA,EAoBA,YAAY,cAA4C,MAAc;AACpE,SAAK,OAAO;AACZ,SAAK,eAAe;AAAA,EACtB;AAAA,EAEA,MAAM,KAAK,MAAoC;AAlCjD;AAmCI,SAAK,SAAS,KAAK;AACnB,UAAM,kBAAkB,KAAK,OAAO,eAAe;AACnD,UAAM,KAAK,OAAO,iBAAiB,iCAC9B,kBAD8B;AAAA;AAAA;AAAA;AAAA;AAAA,MAMjC,YAAY;AAAA,IACd,EAAC;AACD,SAAK,iBAAiB,KAAK,OAAO,YAAY;AAC9C,SAAK,cAAc,KAAK;AACxB,QAAI,KAAK,uBAAuB,kBAAkB;AAChD,WAAK,YAAY,UAAS,UAAK,eAAe,WAApB,YAA8B;AACxD,WAAK,YAAY,SAAQ,UAAK,eAAe,UAApB,YAA6B;AAAA,IACxD;AACA,QAAI,EAAE,KAAK,uBAAuB,mBAAmB;AACnD,YAAM,UAAU,iDAAiD;AAAA,IACnE;AAEA,SAAK,YAAY,IAAI,0BAA0B,EAAE,OAAO,KAAK,OAAO,CAAC;AAErE,SAAK,iBAAiB,IAAI,0BAA0B;AAAA,MAClD,MAAM;AAAA,MACN,cAAc,KAAK;AAAA,IACrB,CAAC;AAED,SAAK,SAAS,IAAI;AAAA,OAChB,UAAK,eAAe,UAApB,YAA6B;AAAA,OAC7B,UAAK,eAAe,WAApB,YAA8B;AAAA,IAChC;AAEA,QAAI,iBAAiB,KAAK,UAAU;AACpC,eAAW,eAAe,KAAK,cAAc;AAC3C,YAAM,YAAY,KAAK;AAAA,QACrB,cAAc,KAAK;AAAA,QACnB,cAAc,KAAK;AAAA,MACrB,CAAC;AACD,uBAAiB,eAAe,YAAY,YAAa,WAAY;AAAA,IACvE;AACA,mBACG,OAAO,KAAK,eAAe,QAAQ,EACnC,MAAM,CAAC,MAAM,QAAQ,MAAM,6BAA6B,CAAC,CAAC,EAC1D,QAAQ,MAAM,KAAK,QAAQ,CAAC;AAC/B,SAAK,iBAAiB,KAAK;AAAA,EAC7B;AAAA,EAEA,MAAM,UAAU;AAlFlB;AAmFI,eAAW,eAAe,KAAK,cAAc;AAC3C,YAAM,YAAY,QAAQ;AAAA,IAC5B;AACA,eAAK,mBAAL,mBAAqB;AAAA,EACvB;AACF;;;ACxFA,YAAY,YAAY;;;ACwBtB,mBAAgB;AAAA,EACd,uBAAuB;AAAA,EACvB,2BAA2B;AAC7B;;;ACzBF,IAA8B,mBAA9B,MAAgF;AAAA,EAAhF;AASE,SAAU,aAAuB;AAAA;AAAA,EAEjC,MAAM,KAAK;AAAA,IACT;AAAA,IACA,cAAc;AAAA,EAChB,GAA+C;AAhBjD;AAiBI,QAAI,EAAE,sBAAsB,mBAAmB;AAC7C,YAAM,UAAU,qDAAqD;AAAA,IACvE;AACA,SAAK,cAAc,IAAI,gBAAgB;AAAA,MACrC,WAAW,CAAC,OAAO,eAAe,KAAK,UAAU,OAAO,UAAU;AAAA,IACpE,CAAC;AACD,SAAK,SAAS,gBAAgB;AAC9B,QAAI,cAAc;AAChB,WAAK,QAAM,UAAK,WAAL,mBAAa,WAAW,UAAS;AAAA,IAC9C;AACA,SAAK,aAAa;AAClB,SAAK,aAAa;AAAA,EACpB;AAAA,EAEA,MAAM,UAAU;AACd,SAAK,aAAa;AAClB,SAAK,SAAS;AACd,SAAK,MAAM;AAAA,EACb;AAMF;;;AF5BA,IAAqB,sBAArB,cAAiD,iBAAiB;AAAA,EAehE,YAAY,MAAyB;AACnC,UAAM;AAPR,2BAAsC;AAQpC,SAAK,UAAU;AACf,QAAI,KAAK,YAAY;AACnB,WAAK,aAAa,KAAK;AAAA,IACzB,WAAW,KAAK,WAAW;AACzB,WAAK,eAAe,KAAK,SAAS;AAAA,IACpC;AAEA,SAAK,QAAQ;AAAA,EACf;AAAA,EAxBA,WAAW,cAAc;AACvB,WAAO,OAAO,oBAAoB;AAAA,EACpC;AAAA,EAwBA,MAAM,KAAK,EAAE,cAAc,cAAc,WAAW,GAAgC;AAClF,UAAM,MAAM,KAAK,EAAE,cAAc,cAAc,WAAW,CAAC;AAE3D,UAAM,UAAU,MAAa,uBAAgB;AAAA,MAC3C,wDAAwD,aAAa,yBAAyB,CAAC;AAAA,IACjG;AAEA,SAAK,iBAAiB,MAAa,sBAAe,kBAAkB,SAAS;AAAA,MAC3E,aAAa;AAAA,QACX,gBACE;AAAA,QACF,UAAU;AAAA,SACP,KAAK,QAAQ;AAAA,MAElB,aAAa;AAAA,MACb,oBAAoB;AAAA,MACpB,uBAAuB;AAAA,IACzB,CAAC;AAAA,EAGH;AAAA,EAEA,MAAM,UAAU;AA9DlB;AA+DI,UAAM,MAAM,QAAQ;AACpB,YAAM,UAAK,mBAAL,mBAAqB;AAC3B,SAAK,kBAAkB;AAAA,EACzB;AAAA,EAEA,MAAM,eAAe,MAAc;AACjC,UAAM,MAAM,IAAI,MAAM;AAEtB,UAAM,IAAI,QAAQ,CAAC,SAAS,WAAW;AACrC,UAAI,cAAc;AAClB,UAAI,SAAS,MAAM,QAAQ,GAAG;AAC9B,UAAI,UAAU,CAAC,QAAQ,OAAO,GAAG;AACjC,UAAI,MAAM;AAAA,IACZ,CAAC;AACD,UAAM,YAAY,MAAM,kBAAkB,GAAG;AAC7C,SAAK,kBAAkB;AAAA,EACzB;AAAA,EAEA,MAAM,UAAU,OAAmB,YAA0D;AAjF/F;AAkFI,QAAI;AACF,UAAI,KAAK,YAAY;AACnB,mBAAW,QAAQ,KAAK;AACxB;AAAA,MACF;AACA,UAAI,CAAC,KAAK,QAAQ;AAChB,cAAM,UAAU,sCAAsC;AAAA,MACxD;AACA,UAAI,cAAc,YAAY,IAAI;AAClC,iBAAK,mBAAL,mBAAqB;AAAA,QACnB,KAAK;AAAA,QACL;AAAA,QACA,CAAC,WAAY,KAAK,sBAAsB;AAAA;AAG1C,UAAI,KAAK,YAAY;AACnB,cAAM,KAAK,eAAe,KAAK;AAAA,MACjC,OAAO;AACL,cAAM,KAAK,sBAAsB,KAAK;AAAA,MACxC;AACA,YAAM,WAAW,IAAI,WAAW,KAAK,QAAQ;AAAA,QAC3C,WAAW,MAAM,aAAa,KAAK,IAAI;AAAA,MACzC,CAAC;AACD,iBAAW,QAAQ,QAAQ;AAAA,IAC7B,UAAE;AACA,YAAM,MAAM;AAAA,IACd;AAAA,EACF;AAAA,EAEA,MAAM,sBAAsB,OAAmB;AA/GjD;AAgHI,QAAI,CAAC,KAAK,UAAU,CAAC,KAAK,OAAO,CAAC,KAAK,uBAAuB,CAAC,KAAK;AAAY;AAGhF,SAAI,UAAK,wBAAL,mBAA0B,cAAc;AAC1C,WAAK,IAAI,SAAS;AAClB,WAAK,IAAI,2BAA2B;AACpC,YAAM,SAAS,MAAM;AAAA,QACnB,KAAK,oBAAoB;AAAA,QACzB,KAAK,oBAAoB,aAAa;AAAA,QACtC,KAAK,oBAAoB,aAAa;AAAA,MACxC;AACA,WAAK,IAAI,UAAU,QAAQ,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AACtE,WAAK,IAAI,SAAS;AAClB,WAAK,IAAI,2BAA2B;AACpC,UAAI,KAAK,iBAAiB;AACxB,aAAK,IAAI;AAAA,UACP,KAAK;AAAA,UACL;AAAA,UACA;AAAA,UACA,KAAK,gBAAgB;AAAA,UACrB,KAAK,gBAAgB;AAAA,UACrB;AAAA,UACA;AAAA,UACA,KAAK,OAAO;AAAA,UACZ,KAAK,OAAO;AAAA,QACd;AAAA,MACF,OAAO;AACL,aAAK,IAAI,YAAY;AACrB,aAAK,IAAI,SAAS,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AAAA,MAC/D;AAEA,WAAK,IAAI,2BAA2B;AAAA,IACtC;AACA,SAAK,IAAI,UAAU,OAAO,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AAAA,EAEvE;AAAA,EAEA,MAAM,eAAe,OAAmB;AArJ1C;AAsJI,QACE,CAAC,KAAK,OACN,CAAC,KAAK,UACN,GAAC,gBAAK,wBAAL,mBAA0B,iBAA1B,mBAAwC,WACzC,CAAC,KAAK,YACN;AACA;AAAA,IACF;AAEA,SAAK,IAAI,KAAK;AACd,SAAK,IAAI,2BAA2B;AAEpC,UAAM,SAAS,MAAM;AAAA,MACnB,KAAK,oBAAoB;AAAA,MACzB,KAAK,oBAAoB,aAAa;AAAA,MACtC,KAAK,oBAAoB,aAAa;AAAA,IACxC;AAEA,SAAK,IAAI,SAAS;AAClB,SAAK,IAAI,2BAA2B;AACpC,SAAK,IAAI,UAAU,QAAQ,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AACtE,SAAK,IAAI,SAAS;AAClB,SAAK,IAAI,2BAA2B;AACpC,SAAK,IAAI,UAAU,OAAO,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AACrE,SAAK,IAAI,2BAA2B;AACpC,SAAK,IAAI,SAAS,QAAQ,KAAK,UAAU;AACzC,SAAK,IAAI,UAAU,OAAO,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AACrE,SAAK,IAAI,QAAQ;AAAA,EACnB;AACF;AAEA,SAAS,aACP,MACA,YACA,aACsB;AACtB,QAAM,YAA+B,IAAI,kBAAkB,aAAa,cAAc,CAAC;AACvF,QAAM,SAAS,KAAK,gBAAgB;AACpC,WAAS,IAAI,GAAG,IAAI,OAAO,QAAQ,KAAK,GAAG;AACzC,cAAU,IAAI,CAAC,IAAI,OAAO,CAAC;AAC3B,cAAU,IAAI,IAAI,CAAC,IAAI,OAAO,CAAC;AAC/B,cAAU,IAAI,IAAI,CAAC,IAAI,OAAO,CAAC;AAC/B,cAAU,IAAI,IAAI,CAAC,IAAI,OAAO,CAAC;AAAA,EACjC;AACA,QAAM,UAAU,IAAI,UAAU,WAAW,YAAY,WAAW;AAEhE,SAAO,kBAAkB,OAAO;AAClC;;;AGnMA,IAAqB,mBAArB,cAA8C,iBAAiB;AAAA,EAC7D,MAAM,UAAU,OAAmB,YAA0D;AAC3F,eAAW,QAAQ,KAAK;AAAA,EAC1B;AAAA,EAEA,MAAM,UAAU;AAAA,EAEhB;AACF;;;ACJO,IAAM,iBAAiB,CAC5B,aAAqB,IACrB,qBACG;AACH,QAAM,sBAAsB,kBAAkB,eAAe,oBAAsB;AACnF,MAAI,CAAC,qBAAqB;AACxB,UAAM,IAAI,MAAM,2CAA2C;AAAA,EAC7D;AACA,QAAM,WAAW,IAAI;AAAA,IACnB,CAAC,IAAI,oBAAsB,EAAE,YAAY,iBAAiB,CAAC,CAAC;AAAA,IAC5D;AAAA,EACF;AACA,SAAO;AACT;AAEO,IAAM,oBAAoB,CAAC,WAAmB,qBAA4C;AAC/F,QAAM,sBAAsB,kBAAkB,eAAe,oBAAsB;AACnF,MAAI,CAAC,qBAAqB;AACxB,UAAM,IAAI,MAAM,2CAA2C;AAAA,EAC7D;AACA,QAAM,WAAW,IAAI;AAAA,IACnB,CAAC,IAAI,oBAAsB,EAAE,WAAW,iBAAiB,CAAC,CAAC;AAAA,IAC3D;AAAA,EACF;AACA,SAAO;AACT;AAEO,IAAM,QAAQ,MAAM;AACzB,QAAM,sBAAsB,kBAAkB,eAAe,oBAAsB;AACnF,MAAI,CAAC,qBAAqB;AACxB,UAAM,IAAI,MAAM,2CAA2C;AAAA,EAC7D;AACA,QAAM,WAAW,IAAI,kBAAkB,CAAC,IAAI,iBAAiB,CAAC,GAAG,OAAO;AACxE,SAAO;AACT;","names":[]}
|
package/dist/src/index.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import ProcessorPipeline from './ProcessorPipeline';
|
|
2
2
|
import { SegmenterBaseOptions } from './transformers/BackgroundTransformer';
|
|
3
|
+
export * from './transformers/types';
|
|
3
4
|
export declare const BackgroundBlur: (blurRadius?: number, segmenterOptions?: SegmenterBaseOptions) => ProcessorPipeline;
|
|
4
5
|
export declare const VirtualBackground: (imagePath: string, segmenterOptions?: SegmenterBaseOptions) => ProcessorPipeline;
|
|
5
6
|
export declare const Dummy: () => ProcessorPipeline;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@livekit/track-processors",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.5",
|
|
4
4
|
"description": "LiveKit track processors",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"module": "dist/index.mjs",
|
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
"scripts": {
|
|
13
13
|
"build": "tsup --onSuccess \"tsc --declaration --emitDeclarationOnly\"",
|
|
14
14
|
"build-docs": "yarn exec typedoc",
|
|
15
|
-
"build-sample": "cd example &&
|
|
15
|
+
"build-sample": "cd example && vite build",
|
|
16
16
|
"lint": "eslint src",
|
|
17
17
|
"test": "jest",
|
|
18
18
|
"sample": "vite serve example --port 8080 --open",
|
|
@@ -42,7 +42,7 @@
|
|
|
42
42
|
"eslint-plugin-ecmascript-compat": "^3.0.0",
|
|
43
43
|
"eslint-plugin-import": "2.27.5",
|
|
44
44
|
"jest": "^27.4.3",
|
|
45
|
-
"livekit-client": "^
|
|
45
|
+
"livekit-client": "^1.12.0",
|
|
46
46
|
"prettier": "^2.8.8",
|
|
47
47
|
"ts-jest": "^27.0.7",
|
|
48
48
|
"ts-loader": "^8.1.0",
|
package/src/ProcessorPipeline.ts
CHANGED
|
@@ -46,8 +46,8 @@ export default class ProcessorPipeline implements TrackProcessor<Track.Kind> {
|
|
|
46
46
|
this.sourceSettings = this.source.getSettings();
|
|
47
47
|
this.sourceDummy = opts.element;
|
|
48
48
|
if (this.sourceDummy instanceof HTMLVideoElement) {
|
|
49
|
-
this.sourceDummy.height = this.sourceSettings.height
|
|
50
|
-
this.sourceDummy.width = this.sourceSettings.width
|
|
49
|
+
this.sourceDummy.height = this.sourceSettings.height ?? 300;
|
|
50
|
+
this.sourceDummy.width = this.sourceSettings.width ?? 300;
|
|
51
51
|
}
|
|
52
52
|
if (!(this.sourceDummy instanceof HTMLVideoElement)) {
|
|
53
53
|
throw TypeError('Currently only video transformers are supported');
|
package/src/index.ts
CHANGED
|
@@ -2,6 +2,8 @@ import ProcessorPipeline from './ProcessorPipeline';
|
|
|
2
2
|
import BackgroundTransformer, { SegmenterBaseOptions } from './transformers/BackgroundTransformer';
|
|
3
3
|
import DummyTransformer from './transformers/DummyTransformer';
|
|
4
4
|
|
|
5
|
+
export * from './transformers/types';
|
|
6
|
+
|
|
5
7
|
export const BackgroundBlur = (
|
|
6
8
|
blurRadius: number = 10,
|
|
7
9
|
segmenterOptions?: SegmenterBaseOptions,
|
|
@@ -118,10 +118,10 @@ export default class BackgroundProcessor extends VideoTransformer {
|
|
|
118
118
|
this.ctx.globalCompositeOperation = 'copy';
|
|
119
119
|
const bitmap = await maskToBitmap(
|
|
120
120
|
this.segmentationResults.categoryMask,
|
|
121
|
-
this.
|
|
122
|
-
this.
|
|
121
|
+
this.segmentationResults.categoryMask.width,
|
|
122
|
+
this.segmentationResults.categoryMask.height,
|
|
123
123
|
);
|
|
124
|
-
this.ctx.drawImage(bitmap, 0, 0);
|
|
124
|
+
this.ctx.drawImage(bitmap, 0, 0, this.canvas.width, this.canvas.height);
|
|
125
125
|
this.ctx.filter = 'none';
|
|
126
126
|
this.ctx.globalCompositeOperation = 'source-in';
|
|
127
127
|
if (this.backgroundImage) {
|
|
@@ -162,19 +162,19 @@ export default class BackgroundProcessor extends VideoTransformer {
|
|
|
162
162
|
|
|
163
163
|
const bitmap = await maskToBitmap(
|
|
164
164
|
this.segmentationResults.categoryMask,
|
|
165
|
-
this.
|
|
166
|
-
this.
|
|
165
|
+
this.segmentationResults.categoryMask.width,
|
|
166
|
+
this.segmentationResults.categoryMask.height,
|
|
167
167
|
);
|
|
168
168
|
|
|
169
169
|
this.ctx.filter = 'blur(3px)';
|
|
170
170
|
this.ctx.globalCompositeOperation = 'copy';
|
|
171
|
-
this.ctx.drawImage(bitmap, 0, 0);
|
|
171
|
+
this.ctx.drawImage(bitmap, 0, 0, this.canvas.width, this.canvas.height);
|
|
172
172
|
this.ctx.filter = 'none';
|
|
173
173
|
this.ctx.globalCompositeOperation = 'source-out';
|
|
174
174
|
this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);
|
|
175
175
|
this.ctx.globalCompositeOperation = 'destination-over';
|
|
176
176
|
this.ctx.filter = `blur(${this.blurRadius}px)`;
|
|
177
|
-
this.ctx.drawImage(frame, 0, 0);
|
|
177
|
+
this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);
|
|
178
178
|
this.ctx.restore();
|
|
179
179
|
}
|
|
180
180
|
}
|