@livekit/track-processors 0.2.1 → 0.2.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (10) hide show
  1. package/dist/index.js +21 -19
  2. package/dist/index.js.map +1 -1
  3. package/dist/index.mjs +21 -19
  4. package/dist/index.mjs.map +1 -1
  5. package/dist/src/index.d.ts +4 -2
  6. package/dist/src/transformers/BackgroundTransformer.d.ts +3 -0
  7. package/package.json +5 -4
  8. package/src/ProcessorPipeline.ts +2 -2
  9. package/src/index.ts +10 -5
  10. package/src/transformers/BackgroundTransformer.ts +17 -8
package/dist/index.js CHANGED
@@ -28,7 +28,7 @@ var ProcessorPipeline = class {
28
28
  this.transformers = transformers;
29
29
  }
30
30
  async init(opts) {
31
- var _a, _b;
31
+ var _a, _b, _c, _d;
32
32
  this.source = opts.track;
33
33
  const origConstraints = this.source.getConstraints();
34
34
  await this.source.applyConstraints(__spreadProps(__spreadValues({}, origConstraints), {
@@ -41,8 +41,8 @@ var ProcessorPipeline = class {
41
41
  this.sourceSettings = this.source.getSettings();
42
42
  this.sourceDummy = opts.element;
43
43
  if (this.sourceDummy instanceof HTMLVideoElement) {
44
- this.sourceDummy.height = this.sourceSettings.height;
45
- this.sourceDummy.width = this.sourceSettings.width;
44
+ this.sourceDummy.height = (_a = this.sourceSettings.height) != null ? _a : 300;
45
+ this.sourceDummy.width = (_b = this.sourceSettings.width) != null ? _b : 300;
46
46
  }
47
47
  if (!(this.sourceDummy instanceof HTMLVideoElement)) {
48
48
  throw TypeError("Currently only video transformers are supported");
@@ -53,8 +53,8 @@ var ProcessorPipeline = class {
53
53
  signalTarget: this.source
54
54
  });
55
55
  this.canvas = new OffscreenCanvas(
56
- (_a = this.sourceSettings.width) != null ? _a : 300,
57
- (_b = this.sourceSettings.height) != null ? _b : 300
56
+ (_c = this.sourceSettings.width) != null ? _c : 300,
57
+ (_d = this.sourceSettings.height) != null ? _d : 300
58
58
  );
59
59
  let readableStream = this.processor.readable;
60
60
  for (const transformer of this.transformers) {
@@ -120,11 +120,13 @@ var BackgroundProcessor = class extends VideoTransformer {
120
120
  constructor(opts) {
121
121
  super();
122
122
  this.backgroundImage = null;
123
+ this.options = opts;
123
124
  if (opts.blurRadius) {
124
125
  this.blurRadius = opts.blurRadius;
125
126
  } else if (opts.imagePath) {
126
127
  this.loadBackground(opts.imagePath);
127
128
  }
129
+ this.options.segmenterOptions;
128
130
  }
129
131
  static get isSupported() {
130
132
  return typeof OffscreenCanvas !== "undefined";
@@ -135,10 +137,10 @@ var BackgroundProcessor = class extends VideoTransformer {
135
137
  `https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@${dependencies["@mediapipe/tasks-vision"]}/wasm`
136
138
  );
137
139
  this.imageSegmenter = await vision.ImageSegmenter.createFromOptions(fileSet, {
138
- baseOptions: {
140
+ baseOptions: __spreadValues({
139
141
  modelAssetPath: "https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_segmenter/float16/latest/selfie_segmenter.tflite",
140
- delegate: "CPU"
141
- },
142
+ delegate: "GPU"
143
+ }, this.options.segmenterOptions),
142
144
  runningMode: "VIDEO",
143
145
  outputCategoryMask: true,
144
146
  outputConfidenceMasks: false
@@ -199,10 +201,10 @@ var BackgroundProcessor = class extends VideoTransformer {
199
201
  this.ctx.globalCompositeOperation = "copy";
200
202
  const bitmap = await maskToBitmap(
201
203
  this.segmentationResults.categoryMask,
202
- this.inputVideo.videoWidth,
203
- this.inputVideo.videoHeight
204
+ this.segmentationResults.categoryMask.width,
205
+ this.segmentationResults.categoryMask.height
204
206
  );
205
- this.ctx.drawImage(bitmap, 0, 0);
207
+ this.ctx.drawImage(bitmap, 0, 0, this.canvas.width, this.canvas.height);
206
208
  this.ctx.filter = "none";
207
209
  this.ctx.globalCompositeOperation = "source-in";
208
210
  if (this.backgroundImage) {
@@ -234,18 +236,18 @@ var BackgroundProcessor = class extends VideoTransformer {
234
236
  this.ctx.globalCompositeOperation = "copy";
235
237
  const bitmap = await maskToBitmap(
236
238
  this.segmentationResults.categoryMask,
237
- this.inputVideo.videoWidth,
238
- this.inputVideo.videoHeight
239
+ this.segmentationResults.categoryMask.width,
240
+ this.segmentationResults.categoryMask.height
239
241
  );
240
242
  this.ctx.filter = "blur(3px)";
241
243
  this.ctx.globalCompositeOperation = "copy";
242
- this.ctx.drawImage(bitmap, 0, 0);
244
+ this.ctx.drawImage(bitmap, 0, 0, this.canvas.width, this.canvas.height);
243
245
  this.ctx.filter = "none";
244
246
  this.ctx.globalCompositeOperation = "source-out";
245
247
  this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);
246
248
  this.ctx.globalCompositeOperation = "destination-over";
247
249
  this.ctx.filter = `blur(${this.blurRadius}px)`;
248
- this.ctx.drawImage(frame, 0, 0);
250
+ this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);
249
251
  this.ctx.restore();
250
252
  }
251
253
  };
@@ -272,24 +274,24 @@ var DummyTransformer = class extends VideoTransformer {
272
274
  };
273
275
 
274
276
  // src/index.ts
275
- var BackgroundBlur = (blurRadius = 10) => {
277
+ var BackgroundBlur = (blurRadius = 10, segmenterOptions) => {
276
278
  const isPipelineSupported = ProcessorPipeline.isSupported && BackgroundProcessor.isSupported;
277
279
  if (!isPipelineSupported) {
278
280
  throw new Error("pipeline is not supported in this browser");
279
281
  }
280
282
  const pipeline = new ProcessorPipeline(
281
- [new BackgroundProcessor({ blurRadius })],
283
+ [new BackgroundProcessor({ blurRadius, segmenterOptions })],
282
284
  "background-blur"
283
285
  );
284
286
  return pipeline;
285
287
  };
286
- var VirtualBackground = (imagePath) => {
288
+ var VirtualBackground = (imagePath, segmenterOptions) => {
287
289
  const isPipelineSupported = ProcessorPipeline.isSupported && BackgroundProcessor.isSupported;
288
290
  if (!isPipelineSupported) {
289
291
  throw new Error("pipeline is not supported in this browser");
290
292
  }
291
293
  const pipeline = new ProcessorPipeline(
292
- [new BackgroundProcessor({ imagePath })],
294
+ [new BackgroundProcessor({ imagePath, segmenterOptions })],
293
295
  "virtual-background"
294
296
  );
295
297
  return pipeline;
package/dist/index.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"sources":["../src/ProcessorPipeline.ts","../src/transformers/BackgroundTransformer.ts","../package.json","../src/transformers/VideoTransformer.ts","../src/transformers/DummyTransformer.ts","../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;AAGA,IAAqB,oBAArB,MAA6E;AAAA,EAC3E,WAAW,cAAc;AACvB,WACE,OAAO,8BAA8B,eACrC,OAAO,8BAA8B;AAAA,EAEzC;AAAA,EAoBA,YAAY,cAA4C,MAAc;AACpE,SAAK,OAAO;AACZ,SAAK,eAAe;AAAA,EACtB;AAAA,EAEA,MAAM,KAAK,MAAoC;AAlCjD;AAmCI,SAAK,SAAS,KAAK;AACnB,UAAM,kBAAkB,KAAK,OAAO,eAAe;AACnD,UAAM,KAAK,OAAO,iBAAiB,iCAC9B,kBAD8B;AAAA;AAAA;AAAA;AAAA;AAAA,MAMjC,YAAY;AAAA,IACd,EAAC;AACD,SAAK,iBAAiB,KAAK,OAAO,YAAY;AAC9C,SAAK,cAAc,KAAK;AACxB,QAAI,KAAK,uBAAuB,kBAAkB;AAChD,WAAK,YAAY,SAAS,KAAK,eAAe;AAC9C,WAAK,YAAY,QAAQ,KAAK,eAAe;AAAA,IAC/C;AACA,QAAI,EAAE,KAAK,uBAAuB,mBAAmB;AACnD,YAAM,UAAU,iDAAiD;AAAA,IACnE;AAEA,SAAK,YAAY,IAAI,0BAA0B,EAAE,OAAO,KAAK,OAAO,CAAC;AAErE,SAAK,iBAAiB,IAAI,0BAA0B;AAAA,MAClD,MAAM;AAAA,MACN,cAAc,KAAK;AAAA,IACrB,CAAC;AAED,SAAK,SAAS,IAAI;AAAA,OAChB,UAAK,eAAe,UAApB,YAA6B;AAAA,OAC7B,UAAK,eAAe,WAApB,YAA8B;AAAA,IAChC;AAEA,QAAI,iBAAiB,KAAK,UAAU;AACpC,eAAW,eAAe,KAAK,cAAc;AAC3C,YAAM,YAAY,KAAK;AAAA,QACrB,cAAc,KAAK;AAAA,QACnB,cAAc,KAAK;AAAA,MACrB,CAAC;AACD,uBAAiB,eAAe,YAAY,YAAa,WAAY;AAAA,IACvE;AACA,mBACG,OAAO,KAAK,eAAe,QAAQ,EACnC,MAAM,CAAC,MAAM,QAAQ,MAAM,6BAA6B,CAAC,CAAC,EAC1D,QAAQ,MAAM,KAAK,QAAQ,CAAC;AAC/B,SAAK,iBAAiB,KAAK;AAAA,EAC7B;AAAA,EAEA,MAAM,UAAU;AAlFlB;AAmFI,eAAW,eAAe,KAAK,cAAc;AAC3C,YAAM,YAAY,QAAQ;AAAA,IAC5B;AACA,eAAK,mBAAL,mBAAqB;AAAA,EACvB;AACF;;;ACxFA,YAAY,YAAY;;;ACuBtB,mBAAgB;AAAA,EACd,uBAAuB;AAAA,EACvB,2BAA2B;AAC7B;;;ACxBF,IAA8B,mBAA9B,MAAgF;AAAA,EAAhF;AASE,SAAU,aAAuB;AAAA;AAAA,EAEjC,MAAM,KAAK;AAAA,IACT;AAAA,IACA,cAAc;AAAA,EAChB,GAA+C;AAhBjD;AAiBI,QAAI,EAAE,sBAAsB,mBAAmB;AAC7C,YAAM,UAAU,qDAAqD;AAAA,IACvE;AACA,SAAK,cAAc,IAAI,gBAAgB;AAAA,MACrC,WAAW,CAAC,OAAO,eAAe,KAAK,UAAU,OAAO,UAAU;AAAA,IACpE,CAAC;AACD,SAAK,SAAS,gBAAgB;AAC9B,QAAI,cAAc;AAChB,WAAK,QAAM,UAAK,WAAL,mBAAa,WAAW,UAAS;AAAA,IAC9C;AACA,SAAK,aAAa;AAClB,SAAK,aAAa;AAAA,EACpB;AAAA,EAEA,MAAM,UAAU;AACd,SAAK,aAAa;AAClB,SAAK,SAAS;AACd,SAAK,MAAM;AAAA,EACb;AAMF;;;AF/BA,IAAqB,sBAArB,cAAiD,iBAAiB;AAAA,EAahE,YAAY,MAAyB;AACnC,UAAM;AALR,2BAAsC;AAMpC,QAAI,KAAK,YAAY;AACnB,WAAK,aAAa,KAAK;AAAA,IACzB,WAAW,KAAK,WAAW;AACzB,WAAK,eAAe,KAAK,SAAS;AAAA,IACpC;AAAA,EACF;AAAA,EAnBA,WAAW,cAAc;AACvB,WAAO,OAAO,oBAAoB;AAAA,EACpC;AAAA,EAmBA,MAAM,KAAK,EAAE,cAAc,cAAc,WAAW,GAAgC;AAClF,UAAM,MAAM,KAAK,EAAE,cAAc,cAAc,WAAW,CAAC;AAE3D,UAAM,UAAU,MAAa,uBAAgB;AAAA,MAC3C,wDAAwD,aAAa,yBAAyB,CAAC;AAAA,IACjG;AAEA,SAAK,iBAAiB,MAAa,sBAAe,kBAAkB,SAAS;AAAA,MAC3E,aAAa;AAAA,QACX,gBACE;AAAA,QACF,UAAU;AAAA,MACZ;AAAA,MACA,aAAa;AAAA,MACb,oBAAoB;AAAA,MACpB,uBAAuB;AAAA,IACzB,CAAC;AAAA,EAGH;AAAA,EAEA,MAAM,UAAU;AArDlB;AAsDI,UAAM,MAAM,QAAQ;AACpB,YAAM,UAAK,mBAAL,mBAAqB;AAC3B,SAAK,kBAAkB;AAAA,EACzB;AAAA,EAEA,MAAM,eAAe,MAAc;AACjC,UAAM,MAAM,IAAI,MAAM;AAEtB,UAAM,IAAI,QAAQ,CAAC,SAAS,WAAW;AACrC,UAAI,cAAc;AAClB,UAAI,SAAS,MAAM,QAAQ,GAAG;AAC9B,UAAI,UAAU,CAAC,QAAQ,OAAO,GAAG;AACjC,UAAI,MAAM;AAAA,IACZ,CAAC;AACD,UAAM,YAAY,MAAM,kBAAkB,GAAG;AAC7C,SAAK,kBAAkB;AAAA,EACzB;AAAA,EAEA,MAAM,UAAU,OAAmB,YAA0D;AAxE/F;AAyEI,QAAI;AACF,UAAI,KAAK,YAAY;AACnB,mBAAW,QAAQ,KAAK;AACxB;AAAA,MACF;AACA,UAAI,CAAC,KAAK,QAAQ;AAChB,cAAM,UAAU,sCAAsC;AAAA,MACxD;AACA,UAAI,cAAc,YAAY,IAAI;AAClC,iBAAK,mBAAL,mBAAqB;AAAA,QACnB,KAAK;AAAA,QACL;AAAA,QACA,CAAC,WAAY,KAAK,sBAAsB;AAAA;AAG1C,UAAI,KAAK,YAAY;AACnB,cAAM,KAAK,eAAe,KAAK;AAAA,MACjC,OAAO;AACL,cAAM,KAAK,sBAAsB,KAAK;AAAA,MACxC;AACA,YAAM,WAAW,IAAI,WAAW,KAAK,QAAQ;AAAA,QAC3C,WAAW,MAAM,aAAa,KAAK,IAAI;AAAA,MACzC,CAAC;AACD,iBAAW,QAAQ,QAAQ;AAAA,IAC7B,UAAE;AACA,YAAM,MAAM;AAAA,IACd;AAAA,EACF;AAAA,EAEA,MAAM,sBAAsB,OAAmB;AAtGjD;AAuGI,QAAI,CAAC,KAAK,UAAU,CAAC,KAAK,OAAO,CAAC,KAAK,uBAAuB,CAAC,KAAK;AAAY;AAGhF,SAAI,UAAK,wBAAL,mBAA0B,cAAc;AAC1C,WAAK,IAAI,SAAS;AAClB,WAAK,IAAI,2BAA2B;AACpC,YAAM,SAAS,MAAM;AAAA,QACnB,KAAK,oBAAoB;AAAA,QACzB,KAAK,WAAW;AAAA,QAChB,KAAK,WAAW;AAAA,MAClB;AACA,WAAK,IAAI,UAAU,QAAQ,GAAG,CAAC;AAC/B,WAAK,IAAI,SAAS;AAClB,WAAK,IAAI,2BAA2B;AACpC,UAAI,KAAK,iBAAiB;AACxB,aAAK,IAAI;AAAA,UACP,KAAK;AAAA,UACL;AAAA,UACA;AAAA,UACA,KAAK,gBAAgB;AAAA,UACrB,KAAK,gBAAgB;AAAA,UACrB;AAAA,UACA;AAAA,UACA,KAAK,OAAO;AAAA,UACZ,KAAK,OAAO;AAAA,QACd;AAAA,MACF,OAAO;AACL,aAAK,IAAI,YAAY;AACrB,aAAK,IAAI,SAAS,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AAAA,MAC/D;AAEA,WAAK,IAAI,2BAA2B;AAAA,IACtC;AACA,SAAK,IAAI,UAAU,OAAO,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AAAA,EAEvE;AAAA,EAEA,MAAM,eAAe,OAAmB;AA5I1C;AA6II,QACE,CAAC,KAAK,OACN,CAAC,KAAK,UACN,GAAC,gBAAK,wBAAL,mBAA0B,iBAA1B,mBAAwC,WACzC,CAAC,KAAK,YACN;AACA;AAAA,IACF;AAEA,SAAK,IAAI,KAAK;AACd,SAAK,IAAI,2BAA2B;AAEpC,UAAM,SAAS,MAAM;AAAA,MACnB,KAAK,oBAAoB;AAAA,MACzB,KAAK,WAAW;AAAA,MAChB,KAAK,WAAW;AAAA,IAClB;AAEA,SAAK,IAAI,SAAS;AAClB,SAAK,IAAI,2BAA2B;AACpC,SAAK,IAAI,UAAU,QAAQ,GAAG,CAAC;AAC/B,SAAK,IAAI,SAAS;AAClB,SAAK,IAAI,2BAA2B;AACpC,SAAK,IAAI,UAAU,OAAO,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AACrE,SAAK,IAAI,2BAA2B;AACpC,SAAK,IAAI,SAAS,QAAQ,KAAK,UAAU;AACzC,SAAK,IAAI,UAAU,OAAO,GAAG,CAAC;AAC9B,SAAK,IAAI,QAAQ;AAAA,EACnB;AACF;AAEA,SAAS,aACP,MACA,YACA,aACsB;AACtB,QAAM,YAA+B,IAAI,kBAAkB,aAAa,cAAc,CAAC;AACvF,QAAM,SAAS,KAAK,gBAAgB;AACpC,WAAS,IAAI,GAAG,IAAI,OAAO,QAAQ,KAAK,GAAG;AACzC,cAAU,IAAI,CAAC,IAAI,OAAO,CAAC;AAC3B,cAAU,IAAI,IAAI,CAAC,IAAI,OAAO,CAAC;AAC/B,cAAU,IAAI,IAAI,CAAC,IAAI,OAAO,CAAC;AAC/B,cAAU,IAAI,IAAI,CAAC,IAAI,OAAO,CAAC;AAAA,EACjC;AACA,QAAM,UAAU,IAAI,UAAU,WAAW,YAAY,WAAW;AAEhE,SAAO,kBAAkB,OAAO;AAClC;;;AG1LA,IAAqB,mBAArB,cAA8C,iBAAiB;AAAA,EAC7D,MAAM,UAAU,OAAmB,YAA0D;AAC3F,eAAW,QAAQ,KAAK;AAAA,EAC1B;AAAA,EAEA,MAAM,UAAU;AAAA,EAEhB;AACF;;;ACNO,IAAM,iBAAiB,CAAC,aAAqB,OAAO;AACzD,QAAM,sBAAsB,kBAAkB,eAAe,oBAAsB;AACnF,MAAI,CAAC,qBAAqB;AACxB,UAAM,IAAI,MAAM,2CAA2C;AAAA,EAC7D;AACA,QAAM,WAAW,IAAI;AAAA,IACnB,CAAC,IAAI,oBAAsB,EAAE,WAAW,CAAC,CAAC;AAAA,IAC1C;AAAA,EACF;AACA,SAAO;AACT;AAEO,IAAM,oBAAoB,CAAC,cAAsB;AACtD,QAAM,sBAAsB,kBAAkB,eAAe,oBAAsB;AACnF,MAAI,CAAC,qBAAqB;AACxB,UAAM,IAAI,MAAM,2CAA2C;AAAA,EAC7D;AACA,QAAM,WAAW,IAAI;AAAA,IACnB,CAAC,IAAI,oBAAsB,EAAE,UAAU,CAAC,CAAC;AAAA,IACzC;AAAA,EACF;AACA,SAAO;AACT;AAEO,IAAM,QAAQ,MAAM;AACzB,QAAM,sBAAsB,kBAAkB,eAAe,oBAAsB;AACnF,MAAI,CAAC,qBAAqB;AACxB,UAAM,IAAI,MAAM,2CAA2C;AAAA,EAC7D;AACA,QAAM,WAAW,IAAI,kBAAkB,CAAC,IAAI,iBAAiB,CAAC,GAAG,OAAO;AACxE,SAAO;AACT","sourcesContent":["import type { ProcessorOptions, Track, TrackProcessor } from 'livekit-client';\nimport { VideoTrackTransformer } from './transformers';\n\nexport default class ProcessorPipeline implements TrackProcessor<Track.Kind> {\n static get isSupported() {\n return (\n typeof MediaStreamTrackGenerator !== 'undefined' &&\n typeof MediaStreamTrackProcessor !== 'undefined'\n );\n }\n\n name: string;\n\n source?: MediaStreamVideoTrack;\n\n sourceSettings?: MediaTrackSettings;\n\n processor?: MediaStreamTrackProcessor<VideoFrame>;\n\n trackGenerator?: MediaStreamTrackGenerator<VideoFrame>;\n\n canvas?: OffscreenCanvas;\n\n sourceDummy?: HTMLMediaElement;\n\n processedTrack?: MediaStreamTrack;\n\n transformers: Array<VideoTrackTransformer>;\n\n constructor(transformers: Array<VideoTrackTransformer>, name: string) {\n this.name = name;\n this.transformers = transformers;\n }\n\n async init(opts: ProcessorOptions<Track.Kind>) {\n this.source = opts.track as MediaStreamVideoTrack;\n const origConstraints = this.source.getConstraints();\n await this.source.applyConstraints({\n ...origConstraints,\n // @ts-expect-error when a mediastream track is resized and/or cropped, the `VideoFrame` will have a coded height/width of the original video size\n // this leads to a shift of the underlying video as the frame itself is being rendered with the coded size\n // but image segmentation is based on the display dimensions (-> the cropped version)\n // in order to prevent this, we force the resize mode to \"none\"\n resizeMode: 'none',\n });\n this.sourceSettings = this.source.getSettings();\n this.sourceDummy = opts.element;\n if (this.sourceDummy instanceof HTMLVideoElement) {\n this.sourceDummy.height = this.sourceSettings.height!;\n this.sourceDummy.width = this.sourceSettings.width!;\n }\n if (!(this.sourceDummy instanceof HTMLVideoElement)) {\n throw TypeError('Currently only video transformers are supported');\n }\n // TODO explore if we can do all the processing work in a webworker\n this.processor = new MediaStreamTrackProcessor({ track: this.source });\n\n this.trackGenerator = new MediaStreamTrackGenerator({\n kind: 'video',\n signalTarget: this.source,\n });\n\n this.canvas = new OffscreenCanvas(\n this.sourceSettings.width ?? 300,\n this.sourceSettings.height ?? 300,\n );\n\n let readableStream = this.processor.readable;\n for (const transformer of this.transformers) {\n await transformer.init({\n outputCanvas: this.canvas,\n inputElement: this.sourceDummy!,\n });\n readableStream = readableStream.pipeThrough(transformer!.transformer!);\n }\n readableStream\n .pipeTo(this.trackGenerator.writable)\n .catch((e) => console.error('error when trying to pipe', e))\n .finally(() => this.destroy());\n this.processedTrack = this.trackGenerator as MediaStreamVideoTrack;\n }\n\n async destroy() {\n for (const transformer of this.transformers) {\n await transformer.destroy();\n }\n this.trackGenerator?.stop();\n }\n}\n","import * as vision from '@mediapipe/tasks-vision';\nimport { dependencies } from '../../package.json';\nimport VideoTransformer from './VideoTransformer';\nimport { VideoTransformerInitOptions } from './types';\n\nexport type BackgroundOptions = {\n blurRadius?: number;\n imagePath?: string;\n};\n\nexport default class BackgroundProcessor extends VideoTransformer {\n static get isSupported() {\n return typeof OffscreenCanvas !== 'undefined';\n }\n\n imageSegmenter?: vision.ImageSegmenter;\n\n segmentationResults: vision.ImageSegmenterResult | undefined;\n\n backgroundImage: ImageBitmap | null = null;\n\n blurRadius?: number;\n\n constructor(opts: BackgroundOptions) {\n super();\n if (opts.blurRadius) {\n this.blurRadius = opts.blurRadius;\n } else if (opts.imagePath) {\n this.loadBackground(opts.imagePath);\n }\n }\n\n async init({ outputCanvas, inputElement: inputVideo }: VideoTransformerInitOptions) {\n await super.init({ outputCanvas, inputElement: inputVideo });\n\n const fileSet = await vision.FilesetResolver.forVisionTasks(\n `https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@${dependencies['@mediapipe/tasks-vision']}/wasm`,\n );\n\n this.imageSegmenter = await vision.ImageSegmenter.createFromOptions(fileSet, {\n baseOptions: {\n modelAssetPath:\n 'https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_segmenter/float16/latest/selfie_segmenter.tflite',\n delegate: 'CPU',\n },\n runningMode: 'VIDEO',\n outputCategoryMask: true,\n outputConfidenceMasks: false,\n });\n\n // this.loadBackground(opts.backgroundUrl).catch((e) => console.error(e));\n }\n\n async destroy() {\n await super.destroy();\n await this.imageSegmenter?.close();\n this.backgroundImage = null;\n }\n\n async loadBackground(path: string) {\n const img = new Image();\n\n await new Promise((resolve, reject) => {\n img.crossOrigin = 'Anonymous';\n img.onload = () => resolve(img);\n img.onerror = (err) => reject(err);\n img.src = path;\n });\n const imageData = await createImageBitmap(img);\n this.backgroundImage = imageData;\n }\n\n async transform(frame: VideoFrame, controller: TransformStreamDefaultController<VideoFrame>) {\n try {\n if (this.isDisabled) {\n controller.enqueue(frame);\n return;\n }\n if (!this.canvas) {\n throw TypeError('Canvas needs to be initialized first');\n }\n let startTimeMs = performance.now();\n this.imageSegmenter?.segmentForVideo(\n this.inputVideo!,\n startTimeMs,\n (result) => (this.segmentationResults = result),\n );\n\n if (this.blurRadius) {\n await this.blurBackground(frame);\n } else {\n await this.drawVirtualBackground(frame);\n }\n const newFrame = new VideoFrame(this.canvas, {\n timestamp: frame.timestamp || Date.now(),\n });\n controller.enqueue(newFrame);\n } finally {\n frame.close();\n }\n }\n\n async drawVirtualBackground(frame: VideoFrame) {\n if (!this.canvas || !this.ctx || !this.segmentationResults || !this.inputVideo) return;\n // this.ctx.save();\n // this.ctx.clearRect(0, 0, this.canvas.width, this.canvas.height);\n if (this.segmentationResults?.categoryMask) {\n this.ctx.filter = 'blur(3px)';\n this.ctx.globalCompositeOperation = 'copy';\n const bitmap = await maskToBitmap(\n this.segmentationResults.categoryMask,\n this.inputVideo.videoWidth,\n this.inputVideo.videoHeight,\n );\n this.ctx.drawImage(bitmap, 0, 0);\n this.ctx.filter = 'none';\n this.ctx.globalCompositeOperation = 'source-in';\n if (this.backgroundImage) {\n this.ctx.drawImage(\n this.backgroundImage,\n 0,\n 0,\n this.backgroundImage.width,\n this.backgroundImage.height,\n 0,\n 0,\n this.canvas.width,\n this.canvas.height,\n );\n } else {\n this.ctx.fillStyle = '#00FF00';\n this.ctx.fillRect(0, 0, this.canvas.width, this.canvas.height);\n }\n\n this.ctx.globalCompositeOperation = 'destination-over';\n }\n this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);\n // this.ctx.restore();\n }\n\n async blurBackground(frame: VideoFrame) {\n if (\n !this.ctx ||\n !this.canvas ||\n !this.segmentationResults?.categoryMask?.canvas ||\n !this.inputVideo\n ) {\n return;\n }\n\n this.ctx.save();\n this.ctx.globalCompositeOperation = 'copy';\n\n const bitmap = await maskToBitmap(\n this.segmentationResults.categoryMask,\n this.inputVideo.videoWidth,\n this.inputVideo.videoHeight,\n );\n\n this.ctx.filter = 'blur(3px)';\n this.ctx.globalCompositeOperation = 'copy';\n this.ctx.drawImage(bitmap, 0, 0);\n this.ctx.filter = 'none';\n this.ctx.globalCompositeOperation = 'source-out';\n this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);\n this.ctx.globalCompositeOperation = 'destination-over';\n this.ctx.filter = `blur(${this.blurRadius}px)`;\n this.ctx.drawImage(frame, 0, 0);\n this.ctx.restore();\n }\n}\n\nfunction maskToBitmap(\n mask: vision.MPMask,\n videoWidth: number,\n videoHeight: number,\n): Promise<ImageBitmap> {\n const dataArray: Uint8ClampedArray = new Uint8ClampedArray(videoWidth * videoHeight * 4);\n const result = mask.getAsUint8Array();\n for (let i = 0; i < result.length; i += 1) {\n dataArray[i * 4] = result[i];\n dataArray[i * 4 + 1] = result[i];\n dataArray[i * 4 + 2] = result[i];\n dataArray[i * 4 + 3] = result[i];\n }\n const dataNew = new ImageData(dataArray, videoWidth, videoHeight);\n\n return createImageBitmap(dataNew);\n}\n","{\n \"name\": \"@livekit/track-processors\",\n \"version\": \"0.2.0\",\n \"description\": \"LiveKit track processors\",\n \"main\": \"dist/index.js\",\n \"module\": \"dist/index.mjs\",\n \"source\": \"src/index.ts\",\n \"types\": \"dist/src/index.d.ts\",\n \"repository\": \"git@github.com:livekit/livekit-track-processors.git\",\n \"author\": \"Lukas Seiler\",\n \"license\": \"Apache-2.0\",\n \"scripts\": {\n \"build\": \"tsup --onSuccess \\\"tsc --declaration --emitDeclarationOnly\\\"\",\n \"build-docs\": \"yarn exec typedoc\",\n \"build-sample\": \"cd example && webpack && cp styles.css index.html dist/\",\n \"lint\": \"eslint src\",\n \"test\": \"jest\",\n \"sample\": \"vite serve example --port 8080 --open\"\n },\n \"files\": [\n \"dist\",\n \"src\"\n ],\n \"dependencies\": {\n \"@mediapipe/holistic\": \"0.5.1675471629\",\n \"@mediapipe/tasks-vision\": \"0.10.1\"\n },\n \"peerDependencies\": {\n \"livekit-client\": \"^1.12.0\"\n },\n \"devDependencies\": {\n \"@trivago/prettier-plugin-sort-imports\": \"^4.1.1\",\n \"@types/dom-mediacapture-transform\": \"^0.1.5\",\n \"@types/jest\": \"^27.0.3\",\n \"@types/offscreencanvas\": \"^2019.7.0\",\n \"@typescript-eslint/eslint-plugin\": \"^4.31.2\",\n \"@webpack-cli/serve\": \"^1.5.2\",\n \"eslint\": \"8.39.0\",\n \"eslint-config-airbnb-typescript\": \"17.0.0\",\n \"eslint-config-prettier\": \"8.8.0\",\n \"eslint-plugin-ecmascript-compat\": \"^3.0.0\",\n \"eslint-plugin-import\": \"2.27.5\",\n \"jest\": \"^27.4.3\",\n \"livekit-client\": \"^0.16.6\",\n \"prettier\": \"^2.8.8\",\n \"ts-jest\": \"^27.0.7\",\n \"ts-loader\": \"^8.1.0\",\n \"ts-proto\": \"^1.85.0\",\n \"tsup\": \"^7.1.0\",\n \"typedoc\": \"^0.20.35\",\n \"typedoc-plugin-no-inherit\": \"1.3.0\",\n \"typescript\": \"^5.0.4\",\n \"vite\": \"^4.3.8\",\n \"webpack\": \"^5.53.0\",\n \"webpack-cli\": \"^4.8.0\",\n \"webpack-dev-server\": \"^4.2.1\"\n }\n}\n","import { VideoTrackTransformer, VideoTransformerInitOptions } from './types';\n\nexport default abstract class VideoTransformer implements VideoTrackTransformer {\n transformer?: TransformStream;\n\n canvas?: OffscreenCanvas;\n\n ctx?: OffscreenCanvasRenderingContext2D;\n\n inputVideo?: HTMLVideoElement;\n\n protected isDisabled?: Boolean = false;\n\n async init({\n outputCanvas,\n inputElement: inputVideo,\n }: VideoTransformerInitOptions): Promise<void> {\n if (!(inputVideo instanceof HTMLVideoElement)) {\n throw TypeError('Video transformer needs a HTMLVideoElement as input');\n }\n this.transformer = new TransformStream({\n transform: (frame, controller) => this.transform(frame, controller),\n });\n this.canvas = outputCanvas || null;\n if (outputCanvas) {\n this.ctx = this.canvas?.getContext('2d') || undefined;\n }\n this.inputVideo = inputVideo;\n this.isDisabled = false;\n }\n\n async destroy() {\n this.isDisabled = true;\n this.canvas = undefined;\n this.ctx = undefined;\n }\n\n abstract transform(\n frame: VideoFrame,\n controller: TransformStreamDefaultController<VideoFrame>,\n ): void;\n}\n","import VideoTransformer from './VideoTransformer';\n\nexport default class DummyTransformer extends VideoTransformer {\n async transform(frame: VideoFrame, controller: TransformStreamDefaultController<VideoFrame>) {\n controller.enqueue(frame);\n }\n\n async destroy() {\n // nothing to do\n }\n}\n","import ProcessorPipeline from './ProcessorPipeline';\nimport BackgroundTransformer from './transformers/BackgroundTransformer';\nimport DummyTransformer from './transformers/DummyTransformer';\n\nexport const BackgroundBlur = (blurRadius: number = 10) => {\n const isPipelineSupported = ProcessorPipeline.isSupported && BackgroundTransformer.isSupported;\n if (!isPipelineSupported) {\n throw new Error('pipeline is not supported in this browser');\n }\n const pipeline = new ProcessorPipeline(\n [new BackgroundTransformer({ blurRadius })],\n 'background-blur',\n );\n return pipeline;\n};\n\nexport const VirtualBackground = (imagePath: string) => {\n const isPipelineSupported = ProcessorPipeline.isSupported && BackgroundTransformer.isSupported;\n if (!isPipelineSupported) {\n throw new Error('pipeline is not supported in this browser');\n }\n const pipeline = new ProcessorPipeline(\n [new BackgroundTransformer({ imagePath })],\n 'virtual-background',\n );\n return pipeline;\n};\n\nexport const Dummy = () => {\n const isPipelineSupported = ProcessorPipeline.isSupported && BackgroundTransformer.isSupported;\n if (!isPipelineSupported) {\n throw new Error('pipeline is not supported in this browser');\n }\n const pipeline = new ProcessorPipeline([new DummyTransformer()], 'dummy');\n return pipeline;\n};\n"]}
1
+ {"version":3,"sources":["../src/ProcessorPipeline.ts","../src/transformers/BackgroundTransformer.ts","../package.json","../src/transformers/VideoTransformer.ts","../src/transformers/DummyTransformer.ts","../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;AAGA,IAAqB,oBAArB,MAA6E;AAAA,EAC3E,WAAW,cAAc;AACvB,WACE,OAAO,8BAA8B,eACrC,OAAO,8BAA8B;AAAA,EAEzC;AAAA,EAoBA,YAAY,cAA4C,MAAc;AACpE,SAAK,OAAO;AACZ,SAAK,eAAe;AAAA,EACtB;AAAA,EAEA,MAAM,KAAK,MAAoC;AAlCjD;AAmCI,SAAK,SAAS,KAAK;AACnB,UAAM,kBAAkB,KAAK,OAAO,eAAe;AACnD,UAAM,KAAK,OAAO,iBAAiB,iCAC9B,kBAD8B;AAAA;AAAA;AAAA;AAAA;AAAA,MAMjC,YAAY;AAAA,IACd,EAAC;AACD,SAAK,iBAAiB,KAAK,OAAO,YAAY;AAC9C,SAAK,cAAc,KAAK;AACxB,QAAI,KAAK,uBAAuB,kBAAkB;AAChD,WAAK,YAAY,UAAS,UAAK,eAAe,WAApB,YAA8B;AACxD,WAAK,YAAY,SAAQ,UAAK,eAAe,UAApB,YAA6B;AAAA,IACxD;AACA,QAAI,EAAE,KAAK,uBAAuB,mBAAmB;AACnD,YAAM,UAAU,iDAAiD;AAAA,IACnE;AAEA,SAAK,YAAY,IAAI,0BAA0B,EAAE,OAAO,KAAK,OAAO,CAAC;AAErE,SAAK,iBAAiB,IAAI,0BAA0B;AAAA,MAClD,MAAM;AAAA,MACN,cAAc,KAAK;AAAA,IACrB,CAAC;AAED,SAAK,SAAS,IAAI;AAAA,OAChB,UAAK,eAAe,UAApB,YAA6B;AAAA,OAC7B,UAAK,eAAe,WAApB,YAA8B;AAAA,IAChC;AAEA,QAAI,iBAAiB,KAAK,UAAU;AACpC,eAAW,eAAe,KAAK,cAAc;AAC3C,YAAM,YAAY,KAAK;AAAA,QACrB,cAAc,KAAK;AAAA,QACnB,cAAc,KAAK;AAAA,MACrB,CAAC;AACD,uBAAiB,eAAe,YAAY,YAAa,WAAY;AAAA,IACvE;AACA,mBACG,OAAO,KAAK,eAAe,QAAQ,EACnC,MAAM,CAAC,MAAM,QAAQ,MAAM,6BAA6B,CAAC,CAAC,EAC1D,QAAQ,MAAM,KAAK,QAAQ,CAAC;AAC/B,SAAK,iBAAiB,KAAK;AAAA,EAC7B;AAAA,EAEA,MAAM,UAAU;AAlFlB;AAmFI,eAAW,eAAe,KAAK,cAAc;AAC3C,YAAM,YAAY,QAAQ;AAAA,IAC5B;AACA,eAAK,mBAAL,mBAAqB;AAAA,EACvB;AACF;;;ACxFA,YAAY,YAAY;;;ACwBtB,mBAAgB;AAAA,EACd,uBAAuB;AAAA,EACvB,2BAA2B;AAC7B;;;ACzBF,IAA8B,mBAA9B,MAAgF;AAAA,EAAhF;AASE,SAAU,aAAuB;AAAA;AAAA,EAEjC,MAAM,KAAK;AAAA,IACT;AAAA,IACA,cAAc;AAAA,EAChB,GAA+C;AAhBjD;AAiBI,QAAI,EAAE,sBAAsB,mBAAmB;AAC7C,YAAM,UAAU,qDAAqD;AAAA,IACvE;AACA,SAAK,cAAc,IAAI,gBAAgB;AAAA,MACrC,WAAW,CAAC,OAAO,eAAe,KAAK,UAAU,OAAO,UAAU;AAAA,IACpE,CAAC;AACD,SAAK,SAAS,gBAAgB;AAC9B,QAAI,cAAc;AAChB,WAAK,QAAM,UAAK,WAAL,mBAAa,WAAW,UAAS;AAAA,IAC9C;AACA,SAAK,aAAa;AAClB,SAAK,aAAa;AAAA,EACpB;AAAA,EAEA,MAAM,UAAU;AACd,SAAK,aAAa;AAClB,SAAK,SAAS;AACd,SAAK,MAAM;AAAA,EACb;AAMF;;;AF5BA,IAAqB,sBAArB,cAAiD,iBAAiB;AAAA,EAehE,YAAY,MAAyB;AACnC,UAAM;AAPR,2BAAsC;AAQpC,SAAK,UAAU;AACf,QAAI,KAAK,YAAY;AACnB,WAAK,aAAa,KAAK;AAAA,IACzB,WAAW,KAAK,WAAW;AACzB,WAAK,eAAe,KAAK,SAAS;AAAA,IACpC;AAEA,SAAK,QAAQ;AAAA,EACf;AAAA,EAxBA,WAAW,cAAc;AACvB,WAAO,OAAO,oBAAoB;AAAA,EACpC;AAAA,EAwBA,MAAM,KAAK,EAAE,cAAc,cAAc,WAAW,GAAgC;AAClF,UAAM,MAAM,KAAK,EAAE,cAAc,cAAc,WAAW,CAAC;AAE3D,UAAM,UAAU,MAAa,uBAAgB;AAAA,MAC3C,wDAAwD,aAAa,yBAAyB,CAAC;AAAA,IACjG;AAEA,SAAK,iBAAiB,MAAa,sBAAe,kBAAkB,SAAS;AAAA,MAC3E,aAAa;AAAA,QACX,gBACE;AAAA,QACF,UAAU;AAAA,SACP,KAAK,QAAQ;AAAA,MAElB,aAAa;AAAA,MACb,oBAAoB;AAAA,MACpB,uBAAuB;AAAA,IACzB,CAAC;AAAA,EAGH;AAAA,EAEA,MAAM,UAAU;AA9DlB;AA+DI,UAAM,MAAM,QAAQ;AACpB,YAAM,UAAK,mBAAL,mBAAqB;AAC3B,SAAK,kBAAkB;AAAA,EACzB;AAAA,EAEA,MAAM,eAAe,MAAc;AACjC,UAAM,MAAM,IAAI,MAAM;AAEtB,UAAM,IAAI,QAAQ,CAAC,SAAS,WAAW;AACrC,UAAI,cAAc;AAClB,UAAI,SAAS,MAAM,QAAQ,GAAG;AAC9B,UAAI,UAAU,CAAC,QAAQ,OAAO,GAAG;AACjC,UAAI,MAAM;AAAA,IACZ,CAAC;AACD,UAAM,YAAY,MAAM,kBAAkB,GAAG;AAC7C,SAAK,kBAAkB;AAAA,EACzB;AAAA,EAEA,MAAM,UAAU,OAAmB,YAA0D;AAjF/F;AAkFI,QAAI;AACF,UAAI,KAAK,YAAY;AACnB,mBAAW,QAAQ,KAAK;AACxB;AAAA,MACF;AACA,UAAI,CAAC,KAAK,QAAQ;AAChB,cAAM,UAAU,sCAAsC;AAAA,MACxD;AACA,UAAI,cAAc,YAAY,IAAI;AAClC,iBAAK,mBAAL,mBAAqB;AAAA,QACnB,KAAK;AAAA,QACL;AAAA,QACA,CAAC,WAAY,KAAK,sBAAsB;AAAA;AAG1C,UAAI,KAAK,YAAY;AACnB,cAAM,KAAK,eAAe,KAAK;AAAA,MACjC,OAAO;AACL,cAAM,KAAK,sBAAsB,KAAK;AAAA,MACxC;AACA,YAAM,WAAW,IAAI,WAAW,KAAK,QAAQ;AAAA,QAC3C,WAAW,MAAM,aAAa,KAAK,IAAI;AAAA,MACzC,CAAC;AACD,iBAAW,QAAQ,QAAQ;AAAA,IAC7B,UAAE;AACA,YAAM,MAAM;AAAA,IACd;AAAA,EACF;AAAA,EAEA,MAAM,sBAAsB,OAAmB;AA/GjD;AAgHI,QAAI,CAAC,KAAK,UAAU,CAAC,KAAK,OAAO,CAAC,KAAK,uBAAuB,CAAC,KAAK;AAAY;AAGhF,SAAI,UAAK,wBAAL,mBAA0B,cAAc;AAC1C,WAAK,IAAI,SAAS;AAClB,WAAK,IAAI,2BAA2B;AACpC,YAAM,SAAS,MAAM;AAAA,QACnB,KAAK,oBAAoB;AAAA,QACzB,KAAK,oBAAoB,aAAa;AAAA,QACtC,KAAK,oBAAoB,aAAa;AAAA,MACxC;AACA,WAAK,IAAI,UAAU,QAAQ,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AACtE,WAAK,IAAI,SAAS;AAClB,WAAK,IAAI,2BAA2B;AACpC,UAAI,KAAK,iBAAiB;AACxB,aAAK,IAAI;AAAA,UACP,KAAK;AAAA,UACL;AAAA,UACA;AAAA,UACA,KAAK,gBAAgB;AAAA,UACrB,KAAK,gBAAgB;AAAA,UACrB;AAAA,UACA;AAAA,UACA,KAAK,OAAO;AAAA,UACZ,KAAK,OAAO;AAAA,QACd;AAAA,MACF,OAAO;AACL,aAAK,IAAI,YAAY;AACrB,aAAK,IAAI,SAAS,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AAAA,MAC/D;AAEA,WAAK,IAAI,2BAA2B;AAAA,IACtC;AACA,SAAK,IAAI,UAAU,OAAO,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AAAA,EAEvE;AAAA,EAEA,MAAM,eAAe,OAAmB;AArJ1C;AAsJI,QACE,CAAC,KAAK,OACN,CAAC,KAAK,UACN,GAAC,gBAAK,wBAAL,mBAA0B,iBAA1B,mBAAwC,WACzC,CAAC,KAAK,YACN;AACA;AAAA,IACF;AAEA,SAAK,IAAI,KAAK;AACd,SAAK,IAAI,2BAA2B;AAEpC,UAAM,SAAS,MAAM;AAAA,MACnB,KAAK,oBAAoB;AAAA,MACzB,KAAK,oBAAoB,aAAa;AAAA,MACtC,KAAK,oBAAoB,aAAa;AAAA,IACxC;AAEA,SAAK,IAAI,SAAS;AAClB,SAAK,IAAI,2BAA2B;AACpC,SAAK,IAAI,UAAU,QAAQ,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AACtE,SAAK,IAAI,SAAS;AAClB,SAAK,IAAI,2BAA2B;AACpC,SAAK,IAAI,UAAU,OAAO,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AACrE,SAAK,IAAI,2BAA2B;AACpC,SAAK,IAAI,SAAS,QAAQ,KAAK,UAAU;AACzC,SAAK,IAAI,UAAU,OAAO,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AACrE,SAAK,IAAI,QAAQ;AAAA,EACnB;AACF;AAEA,SAAS,aACP,MACA,YACA,aACsB;AACtB,QAAM,YAA+B,IAAI,kBAAkB,aAAa,cAAc,CAAC;AACvF,QAAM,SAAS,KAAK,gBAAgB;AACpC,WAAS,IAAI,GAAG,IAAI,OAAO,QAAQ,KAAK,GAAG;AACzC,cAAU,IAAI,CAAC,IAAI,OAAO,CAAC;AAC3B,cAAU,IAAI,IAAI,CAAC,IAAI,OAAO,CAAC;AAC/B,cAAU,IAAI,IAAI,CAAC,IAAI,OAAO,CAAC;AAC/B,cAAU,IAAI,IAAI,CAAC,IAAI,OAAO,CAAC;AAAA,EACjC;AACA,QAAM,UAAU,IAAI,UAAU,WAAW,YAAY,WAAW;AAEhE,SAAO,kBAAkB,OAAO;AAClC;;;AGnMA,IAAqB,mBAArB,cAA8C,iBAAiB;AAAA,EAC7D,MAAM,UAAU,OAAmB,YAA0D;AAC3F,eAAW,QAAQ,KAAK;AAAA,EAC1B;AAAA,EAEA,MAAM,UAAU;AAAA,EAEhB;AACF;;;ACJO,IAAM,iBAAiB,CAC5B,aAAqB,IACrB,qBACG;AACH,QAAM,sBAAsB,kBAAkB,eAAe,oBAAsB;AACnF,MAAI,CAAC,qBAAqB;AACxB,UAAM,IAAI,MAAM,2CAA2C;AAAA,EAC7D;AACA,QAAM,WAAW,IAAI;AAAA,IACnB,CAAC,IAAI,oBAAsB,EAAE,YAAY,iBAAiB,CAAC,CAAC;AAAA,IAC5D;AAAA,EACF;AACA,SAAO;AACT;AAEO,IAAM,oBAAoB,CAAC,WAAmB,qBAA4C;AAC/F,QAAM,sBAAsB,kBAAkB,eAAe,oBAAsB;AACnF,MAAI,CAAC,qBAAqB;AACxB,UAAM,IAAI,MAAM,2CAA2C;AAAA,EAC7D;AACA,QAAM,WAAW,IAAI;AAAA,IACnB,CAAC,IAAI,oBAAsB,EAAE,WAAW,iBAAiB,CAAC,CAAC;AAAA,IAC3D;AAAA,EACF;AACA,SAAO;AACT;AAEO,IAAM,QAAQ,MAAM;AACzB,QAAM,sBAAsB,kBAAkB,eAAe,oBAAsB;AACnF,MAAI,CAAC,qBAAqB;AACxB,UAAM,IAAI,MAAM,2CAA2C;AAAA,EAC7D;AACA,QAAM,WAAW,IAAI,kBAAkB,CAAC,IAAI,iBAAiB,CAAC,GAAG,OAAO;AACxE,SAAO;AACT","sourcesContent":["import type { ProcessorOptions, Track, TrackProcessor } from 'livekit-client';\nimport { VideoTrackTransformer } from './transformers';\n\nexport default class ProcessorPipeline implements TrackProcessor<Track.Kind> {\n static get isSupported() {\n return (\n typeof MediaStreamTrackGenerator !== 'undefined' &&\n typeof MediaStreamTrackProcessor !== 'undefined'\n );\n }\n\n name: string;\n\n source?: MediaStreamVideoTrack;\n\n sourceSettings?: MediaTrackSettings;\n\n processor?: MediaStreamTrackProcessor<VideoFrame>;\n\n trackGenerator?: MediaStreamTrackGenerator<VideoFrame>;\n\n canvas?: OffscreenCanvas;\n\n sourceDummy?: HTMLMediaElement;\n\n processedTrack?: MediaStreamTrack;\n\n transformers: Array<VideoTrackTransformer>;\n\n constructor(transformers: Array<VideoTrackTransformer>, name: string) {\n this.name = name;\n this.transformers = transformers;\n }\n\n async init(opts: ProcessorOptions<Track.Kind>) {\n this.source = opts.track as MediaStreamVideoTrack;\n const origConstraints = this.source.getConstraints();\n await this.source.applyConstraints({\n ...origConstraints,\n // @ts-expect-error when a mediastream track is resized and/or cropped, the `VideoFrame` will have a coded height/width of the original video size\n // this leads to a shift of the underlying video as the frame itself is being rendered with the coded size\n // but image segmentation is based on the display dimensions (-> the cropped version)\n // in order to prevent this, we force the resize mode to \"none\"\n resizeMode: 'none',\n });\n this.sourceSettings = this.source.getSettings();\n this.sourceDummy = opts.element;\n if (this.sourceDummy instanceof HTMLVideoElement) {\n this.sourceDummy.height = this.sourceSettings.height ?? 300;\n this.sourceDummy.width = this.sourceSettings.width ?? 300;\n }\n if (!(this.sourceDummy instanceof HTMLVideoElement)) {\n throw TypeError('Currently only video transformers are supported');\n }\n // TODO explore if we can do all the processing work in a webworker\n this.processor = new MediaStreamTrackProcessor({ track: this.source });\n\n this.trackGenerator = new MediaStreamTrackGenerator({\n kind: 'video',\n signalTarget: this.source,\n });\n\n this.canvas = new OffscreenCanvas(\n this.sourceSettings.width ?? 300,\n this.sourceSettings.height ?? 300,\n );\n\n let readableStream = this.processor.readable;\n for (const transformer of this.transformers) {\n await transformer.init({\n outputCanvas: this.canvas,\n inputElement: this.sourceDummy!,\n });\n readableStream = readableStream.pipeThrough(transformer!.transformer!);\n }\n readableStream\n .pipeTo(this.trackGenerator.writable)\n .catch((e) => console.error('error when trying to pipe', e))\n .finally(() => this.destroy());\n this.processedTrack = this.trackGenerator as MediaStreamVideoTrack;\n }\n\n async destroy() {\n for (const transformer of this.transformers) {\n await transformer.destroy();\n }\n this.trackGenerator?.stop();\n }\n}\n","import * as vision from '@mediapipe/tasks-vision';\nimport { dependencies } from '../../package.json';\nimport VideoTransformer from './VideoTransformer';\nimport { VideoTransformerInitOptions } from './types';\n\nexport type SegmenterBaseOptions = Partial<vision.ImageSegmenterOptions['baseOptions']>;\n\nexport type BackgroundOptions = {\n blurRadius?: number;\n imagePath?: string;\n segmenterOptions?: SegmenterBaseOptions;\n};\n\nexport default class BackgroundProcessor extends VideoTransformer {\n static get isSupported() {\n return typeof OffscreenCanvas !== 'undefined';\n }\n\n imageSegmenter?: vision.ImageSegmenter;\n\n segmentationResults: vision.ImageSegmenterResult | undefined;\n\n backgroundImage: ImageBitmap | null = null;\n\n blurRadius?: number;\n\n options: BackgroundOptions;\n\n constructor(opts: BackgroundOptions) {\n super();\n this.options = opts;\n if (opts.blurRadius) {\n this.blurRadius = opts.blurRadius;\n } else if (opts.imagePath) {\n this.loadBackground(opts.imagePath);\n }\n\n this.options.segmenterOptions;\n }\n\n async init({ outputCanvas, inputElement: inputVideo }: VideoTransformerInitOptions) {\n await super.init({ outputCanvas, inputElement: inputVideo });\n\n const fileSet = await vision.FilesetResolver.forVisionTasks(\n `https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@${dependencies['@mediapipe/tasks-vision']}/wasm`,\n );\n\n this.imageSegmenter = await vision.ImageSegmenter.createFromOptions(fileSet, {\n baseOptions: {\n modelAssetPath:\n 'https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_segmenter/float16/latest/selfie_segmenter.tflite',\n delegate: 'GPU',\n ...this.options.segmenterOptions,\n },\n runningMode: 'VIDEO',\n outputCategoryMask: true,\n outputConfidenceMasks: false,\n });\n\n // this.loadBackground(opts.backgroundUrl).catch((e) => console.error(e));\n }\n\n async destroy() {\n await super.destroy();\n await this.imageSegmenter?.close();\n this.backgroundImage = null;\n }\n\n async loadBackground(path: string) {\n const img = new Image();\n\n await new Promise((resolve, reject) => {\n img.crossOrigin = 'Anonymous';\n img.onload = () => resolve(img);\n img.onerror = (err) => reject(err);\n img.src = path;\n });\n const imageData = await createImageBitmap(img);\n this.backgroundImage = imageData;\n }\n\n async transform(frame: VideoFrame, controller: TransformStreamDefaultController<VideoFrame>) {\n try {\n if (this.isDisabled) {\n controller.enqueue(frame);\n return;\n }\n if (!this.canvas) {\n throw TypeError('Canvas needs to be initialized first');\n }\n let startTimeMs = performance.now();\n this.imageSegmenter?.segmentForVideo(\n this.inputVideo!,\n startTimeMs,\n (result) => (this.segmentationResults = result),\n );\n\n if (this.blurRadius) {\n await this.blurBackground(frame);\n } else {\n await this.drawVirtualBackground(frame);\n }\n const newFrame = new VideoFrame(this.canvas, {\n timestamp: frame.timestamp || Date.now(),\n });\n controller.enqueue(newFrame);\n } finally {\n frame.close();\n }\n }\n\n async drawVirtualBackground(frame: VideoFrame) {\n if (!this.canvas || !this.ctx || !this.segmentationResults || !this.inputVideo) return;\n // this.ctx.save();\n // this.ctx.clearRect(0, 0, this.canvas.width, this.canvas.height);\n if (this.segmentationResults?.categoryMask) {\n this.ctx.filter = 'blur(3px)';\n this.ctx.globalCompositeOperation = 'copy';\n const bitmap = await maskToBitmap(\n this.segmentationResults.categoryMask,\n this.segmentationResults.categoryMask.width,\n this.segmentationResults.categoryMask.height,\n );\n this.ctx.drawImage(bitmap, 0, 0, this.canvas.width, this.canvas.height);\n this.ctx.filter = 'none';\n this.ctx.globalCompositeOperation = 'source-in';\n if (this.backgroundImage) {\n this.ctx.drawImage(\n this.backgroundImage,\n 0,\n 0,\n this.backgroundImage.width,\n this.backgroundImage.height,\n 0,\n 0,\n this.canvas.width,\n this.canvas.height,\n );\n } else {\n this.ctx.fillStyle = '#00FF00';\n this.ctx.fillRect(0, 0, this.canvas.width, this.canvas.height);\n }\n\n this.ctx.globalCompositeOperation = 'destination-over';\n }\n this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);\n // this.ctx.restore();\n }\n\n async blurBackground(frame: VideoFrame) {\n if (\n !this.ctx ||\n !this.canvas ||\n !this.segmentationResults?.categoryMask?.canvas ||\n !this.inputVideo\n ) {\n return;\n }\n\n this.ctx.save();\n this.ctx.globalCompositeOperation = 'copy';\n\n const bitmap = await maskToBitmap(\n this.segmentationResults.categoryMask,\n this.segmentationResults.categoryMask.width,\n this.segmentationResults.categoryMask.height,\n );\n\n this.ctx.filter = 'blur(3px)';\n this.ctx.globalCompositeOperation = 'copy';\n this.ctx.drawImage(bitmap, 0, 0, this.canvas.width, this.canvas.height);\n this.ctx.filter = 'none';\n this.ctx.globalCompositeOperation = 'source-out';\n this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);\n this.ctx.globalCompositeOperation = 'destination-over';\n this.ctx.filter = `blur(${this.blurRadius}px)`;\n this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);\n this.ctx.restore();\n }\n}\n\nfunction maskToBitmap(\n mask: vision.MPMask,\n videoWidth: number,\n videoHeight: number,\n): Promise<ImageBitmap> {\n const dataArray: Uint8ClampedArray = new Uint8ClampedArray(videoWidth * videoHeight * 4);\n const result = mask.getAsUint8Array();\n for (let i = 0; i < result.length; i += 1) {\n dataArray[i * 4] = result[i];\n dataArray[i * 4 + 1] = result[i];\n dataArray[i * 4 + 2] = result[i];\n dataArray[i * 4 + 3] = result[i];\n }\n const dataNew = new ImageData(dataArray, videoWidth, videoHeight);\n\n return createImageBitmap(dataNew);\n}\n","{\n \"name\": \"@livekit/track-processors\",\n \"version\": \"0.2.5\",\n \"description\": \"LiveKit track processors\",\n \"main\": \"dist/index.js\",\n \"module\": \"dist/index.mjs\",\n \"source\": \"src/index.ts\",\n \"types\": \"dist/src/index.d.ts\",\n \"repository\": \"git@github.com:livekit/livekit-track-processors.git\",\n \"author\": \"Lukas Seiler\",\n \"license\": \"Apache-2.0\",\n \"scripts\": {\n \"build\": \"tsup --onSuccess \\\"tsc --declaration --emitDeclarationOnly\\\"\",\n \"build-docs\": \"yarn exec typedoc\",\n \"build-sample\": \"cd example && vite build\",\n \"lint\": \"eslint src\",\n \"test\": \"jest\",\n \"sample\": \"vite serve example --port 8080 --open\",\n \"prepublish\": \"yarn build\"\n },\n \"files\": [\n \"dist\",\n \"src\"\n ],\n \"dependencies\": {\n \"@mediapipe/holistic\": \"0.5.1675471629\",\n \"@mediapipe/tasks-vision\": \"0.10.1\"\n },\n \"peerDependencies\": {\n \"livekit-client\": \"^1.12.0\"\n },\n \"devDependencies\": {\n \"@trivago/prettier-plugin-sort-imports\": \"^4.1.1\",\n \"@types/dom-mediacapture-transform\": \"^0.1.5\",\n \"@types/jest\": \"^27.0.3\",\n \"@types/offscreencanvas\": \"^2019.7.0\",\n \"@typescript-eslint/eslint-plugin\": \"^4.31.2\",\n \"@webpack-cli/serve\": \"^1.5.2\",\n \"eslint\": \"8.39.0\",\n \"eslint-config-airbnb-typescript\": \"17.0.0\",\n \"eslint-config-prettier\": \"8.8.0\",\n \"eslint-plugin-ecmascript-compat\": \"^3.0.0\",\n \"eslint-plugin-import\": \"2.27.5\",\n \"jest\": \"^27.4.3\",\n \"livekit-client\": \"^1.12.0\",\n \"prettier\": \"^2.8.8\",\n \"ts-jest\": \"^27.0.7\",\n \"ts-loader\": \"^8.1.0\",\n \"ts-proto\": \"^1.85.0\",\n \"tsup\": \"^7.1.0\",\n \"typedoc\": \"^0.20.35\",\n \"typedoc-plugin-no-inherit\": \"1.3.0\",\n \"typescript\": \"^5.0.4\",\n \"vite\": \"^4.3.8\",\n \"webpack\": \"^5.53.0\",\n \"webpack-cli\": \"^4.8.0\",\n \"webpack-dev-server\": \"^4.2.1\"\n }\n}\n","import { VideoTrackTransformer, VideoTransformerInitOptions } from './types';\n\nexport default abstract class VideoTransformer implements VideoTrackTransformer {\n transformer?: TransformStream;\n\n canvas?: OffscreenCanvas;\n\n ctx?: OffscreenCanvasRenderingContext2D;\n\n inputVideo?: HTMLVideoElement;\n\n protected isDisabled?: Boolean = false;\n\n async init({\n outputCanvas,\n inputElement: inputVideo,\n }: VideoTransformerInitOptions): Promise<void> {\n if (!(inputVideo instanceof HTMLVideoElement)) {\n throw TypeError('Video transformer needs a HTMLVideoElement as input');\n }\n this.transformer = new TransformStream({\n transform: (frame, controller) => this.transform(frame, controller),\n });\n this.canvas = outputCanvas || null;\n if (outputCanvas) {\n this.ctx = this.canvas?.getContext('2d') || undefined;\n }\n this.inputVideo = inputVideo;\n this.isDisabled = false;\n }\n\n async destroy() {\n this.isDisabled = true;\n this.canvas = undefined;\n this.ctx = undefined;\n }\n\n abstract transform(\n frame: VideoFrame,\n controller: TransformStreamDefaultController<VideoFrame>,\n ): void;\n}\n","import VideoTransformer from './VideoTransformer';\n\nexport default class DummyTransformer extends VideoTransformer {\n async transform(frame: VideoFrame, controller: TransformStreamDefaultController<VideoFrame>) {\n controller.enqueue(frame);\n }\n\n async destroy() {\n // nothing to do\n }\n}\n","import ProcessorPipeline from './ProcessorPipeline';\nimport BackgroundTransformer, { SegmenterBaseOptions } from './transformers/BackgroundTransformer';\nimport DummyTransformer from './transformers/DummyTransformer';\n\nexport * from './transformers/types';\n\nexport const BackgroundBlur = (\n blurRadius: number = 10,\n segmenterOptions?: SegmenterBaseOptions,\n) => {\n const isPipelineSupported = ProcessorPipeline.isSupported && BackgroundTransformer.isSupported;\n if (!isPipelineSupported) {\n throw new Error('pipeline is not supported in this browser');\n }\n const pipeline = new ProcessorPipeline(\n [new BackgroundTransformer({ blurRadius, segmenterOptions })],\n 'background-blur',\n );\n return pipeline;\n};\n\nexport const VirtualBackground = (imagePath: string, segmenterOptions?: SegmenterBaseOptions) => {\n const isPipelineSupported = ProcessorPipeline.isSupported && BackgroundTransformer.isSupported;\n if (!isPipelineSupported) {\n throw new Error('pipeline is not supported in this browser');\n }\n const pipeline = new ProcessorPipeline(\n [new BackgroundTransformer({ imagePath, segmenterOptions })],\n 'virtual-background',\n );\n return pipeline;\n};\n\nexport const Dummy = () => {\n const isPipelineSupported = ProcessorPipeline.isSupported && BackgroundTransformer.isSupported;\n if (!isPipelineSupported) {\n throw new Error('pipeline is not supported in this browser');\n }\n const pipeline = new ProcessorPipeline([new DummyTransformer()], 'dummy');\n return pipeline;\n};\n"]}
package/dist/index.mjs CHANGED
@@ -28,7 +28,7 @@ var ProcessorPipeline = class {
28
28
  this.transformers = transformers;
29
29
  }
30
30
  async init(opts) {
31
- var _a, _b;
31
+ var _a, _b, _c, _d;
32
32
  this.source = opts.track;
33
33
  const origConstraints = this.source.getConstraints();
34
34
  await this.source.applyConstraints(__spreadProps(__spreadValues({}, origConstraints), {
@@ -41,8 +41,8 @@ var ProcessorPipeline = class {
41
41
  this.sourceSettings = this.source.getSettings();
42
42
  this.sourceDummy = opts.element;
43
43
  if (this.sourceDummy instanceof HTMLVideoElement) {
44
- this.sourceDummy.height = this.sourceSettings.height;
45
- this.sourceDummy.width = this.sourceSettings.width;
44
+ this.sourceDummy.height = (_a = this.sourceSettings.height) != null ? _a : 300;
45
+ this.sourceDummy.width = (_b = this.sourceSettings.width) != null ? _b : 300;
46
46
  }
47
47
  if (!(this.sourceDummy instanceof HTMLVideoElement)) {
48
48
  throw TypeError("Currently only video transformers are supported");
@@ -53,8 +53,8 @@ var ProcessorPipeline = class {
53
53
  signalTarget: this.source
54
54
  });
55
55
  this.canvas = new OffscreenCanvas(
56
- (_a = this.sourceSettings.width) != null ? _a : 300,
57
- (_b = this.sourceSettings.height) != null ? _b : 300
56
+ (_c = this.sourceSettings.width) != null ? _c : 300,
57
+ (_d = this.sourceSettings.height) != null ? _d : 300
58
58
  );
59
59
  let readableStream = this.processor.readable;
60
60
  for (const transformer of this.transformers) {
@@ -120,11 +120,13 @@ var BackgroundProcessor = class extends VideoTransformer {
120
120
  constructor(opts) {
121
121
  super();
122
122
  this.backgroundImage = null;
123
+ this.options = opts;
123
124
  if (opts.blurRadius) {
124
125
  this.blurRadius = opts.blurRadius;
125
126
  } else if (opts.imagePath) {
126
127
  this.loadBackground(opts.imagePath);
127
128
  }
129
+ this.options.segmenterOptions;
128
130
  }
129
131
  static get isSupported() {
130
132
  return typeof OffscreenCanvas !== "undefined";
@@ -135,10 +137,10 @@ var BackgroundProcessor = class extends VideoTransformer {
135
137
  `https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@${dependencies["@mediapipe/tasks-vision"]}/wasm`
136
138
  );
137
139
  this.imageSegmenter = await vision.ImageSegmenter.createFromOptions(fileSet, {
138
- baseOptions: {
140
+ baseOptions: __spreadValues({
139
141
  modelAssetPath: "https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_segmenter/float16/latest/selfie_segmenter.tflite",
140
- delegate: "CPU"
141
- },
142
+ delegate: "GPU"
143
+ }, this.options.segmenterOptions),
142
144
  runningMode: "VIDEO",
143
145
  outputCategoryMask: true,
144
146
  outputConfidenceMasks: false
@@ -199,10 +201,10 @@ var BackgroundProcessor = class extends VideoTransformer {
199
201
  this.ctx.globalCompositeOperation = "copy";
200
202
  const bitmap = await maskToBitmap(
201
203
  this.segmentationResults.categoryMask,
202
- this.inputVideo.videoWidth,
203
- this.inputVideo.videoHeight
204
+ this.segmentationResults.categoryMask.width,
205
+ this.segmentationResults.categoryMask.height
204
206
  );
205
- this.ctx.drawImage(bitmap, 0, 0);
207
+ this.ctx.drawImage(bitmap, 0, 0, this.canvas.width, this.canvas.height);
206
208
  this.ctx.filter = "none";
207
209
  this.ctx.globalCompositeOperation = "source-in";
208
210
  if (this.backgroundImage) {
@@ -234,18 +236,18 @@ var BackgroundProcessor = class extends VideoTransformer {
234
236
  this.ctx.globalCompositeOperation = "copy";
235
237
  const bitmap = await maskToBitmap(
236
238
  this.segmentationResults.categoryMask,
237
- this.inputVideo.videoWidth,
238
- this.inputVideo.videoHeight
239
+ this.segmentationResults.categoryMask.width,
240
+ this.segmentationResults.categoryMask.height
239
241
  );
240
242
  this.ctx.filter = "blur(3px)";
241
243
  this.ctx.globalCompositeOperation = "copy";
242
- this.ctx.drawImage(bitmap, 0, 0);
244
+ this.ctx.drawImage(bitmap, 0, 0, this.canvas.width, this.canvas.height);
243
245
  this.ctx.filter = "none";
244
246
  this.ctx.globalCompositeOperation = "source-out";
245
247
  this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);
246
248
  this.ctx.globalCompositeOperation = "destination-over";
247
249
  this.ctx.filter = `blur(${this.blurRadius}px)`;
248
- this.ctx.drawImage(frame, 0, 0);
250
+ this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);
249
251
  this.ctx.restore();
250
252
  }
251
253
  };
@@ -272,24 +274,24 @@ var DummyTransformer = class extends VideoTransformer {
272
274
  };
273
275
 
274
276
  // src/index.ts
275
- var BackgroundBlur = (blurRadius = 10) => {
277
+ var BackgroundBlur = (blurRadius = 10, segmenterOptions) => {
276
278
  const isPipelineSupported = ProcessorPipeline.isSupported && BackgroundProcessor.isSupported;
277
279
  if (!isPipelineSupported) {
278
280
  throw new Error("pipeline is not supported in this browser");
279
281
  }
280
282
  const pipeline = new ProcessorPipeline(
281
- [new BackgroundProcessor({ blurRadius })],
283
+ [new BackgroundProcessor({ blurRadius, segmenterOptions })],
282
284
  "background-blur"
283
285
  );
284
286
  return pipeline;
285
287
  };
286
- var VirtualBackground = (imagePath) => {
288
+ var VirtualBackground = (imagePath, segmenterOptions) => {
287
289
  const isPipelineSupported = ProcessorPipeline.isSupported && BackgroundProcessor.isSupported;
288
290
  if (!isPipelineSupported) {
289
291
  throw new Error("pipeline is not supported in this browser");
290
292
  }
291
293
  const pipeline = new ProcessorPipeline(
292
- [new BackgroundProcessor({ imagePath })],
294
+ [new BackgroundProcessor({ imagePath, segmenterOptions })],
293
295
  "virtual-background"
294
296
  );
295
297
  return pipeline;
package/dist/index.mjs.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"sources":["../src/ProcessorPipeline.ts","../src/transformers/BackgroundTransformer.ts","../package.json","../src/transformers/VideoTransformer.ts","../src/transformers/DummyTransformer.ts","../src/index.ts"],"sourcesContent":["import type { ProcessorOptions, Track, TrackProcessor } from 'livekit-client';\nimport { VideoTrackTransformer } from './transformers';\n\nexport default class ProcessorPipeline implements TrackProcessor<Track.Kind> {\n static get isSupported() {\n return (\n typeof MediaStreamTrackGenerator !== 'undefined' &&\n typeof MediaStreamTrackProcessor !== 'undefined'\n );\n }\n\n name: string;\n\n source?: MediaStreamVideoTrack;\n\n sourceSettings?: MediaTrackSettings;\n\n processor?: MediaStreamTrackProcessor<VideoFrame>;\n\n trackGenerator?: MediaStreamTrackGenerator<VideoFrame>;\n\n canvas?: OffscreenCanvas;\n\n sourceDummy?: HTMLMediaElement;\n\n processedTrack?: MediaStreamTrack;\n\n transformers: Array<VideoTrackTransformer>;\n\n constructor(transformers: Array<VideoTrackTransformer>, name: string) {\n this.name = name;\n this.transformers = transformers;\n }\n\n async init(opts: ProcessorOptions<Track.Kind>) {\n this.source = opts.track as MediaStreamVideoTrack;\n const origConstraints = this.source.getConstraints();\n await this.source.applyConstraints({\n ...origConstraints,\n // @ts-expect-error when a mediastream track is resized and/or cropped, the `VideoFrame` will have a coded height/width of the original video size\n // this leads to a shift of the underlying video as the frame itself is being rendered with the coded size\n // but image segmentation is based on the display dimensions (-> the cropped version)\n // in order to prevent this, we force the resize mode to \"none\"\n resizeMode: 'none',\n });\n this.sourceSettings = this.source.getSettings();\n this.sourceDummy = opts.element;\n if (this.sourceDummy instanceof HTMLVideoElement) {\n this.sourceDummy.height = this.sourceSettings.height!;\n this.sourceDummy.width = this.sourceSettings.width!;\n }\n if (!(this.sourceDummy instanceof HTMLVideoElement)) {\n throw TypeError('Currently only video transformers are supported');\n }\n // TODO explore if we can do all the processing work in a webworker\n this.processor = new MediaStreamTrackProcessor({ track: this.source });\n\n this.trackGenerator = new MediaStreamTrackGenerator({\n kind: 'video',\n signalTarget: this.source,\n });\n\n this.canvas = new OffscreenCanvas(\n this.sourceSettings.width ?? 300,\n this.sourceSettings.height ?? 300,\n );\n\n let readableStream = this.processor.readable;\n for (const transformer of this.transformers) {\n await transformer.init({\n outputCanvas: this.canvas,\n inputElement: this.sourceDummy!,\n });\n readableStream = readableStream.pipeThrough(transformer!.transformer!);\n }\n readableStream\n .pipeTo(this.trackGenerator.writable)\n .catch((e) => console.error('error when trying to pipe', e))\n .finally(() => this.destroy());\n this.processedTrack = this.trackGenerator as MediaStreamVideoTrack;\n }\n\n async destroy() {\n for (const transformer of this.transformers) {\n await transformer.destroy();\n }\n this.trackGenerator?.stop();\n }\n}\n","import * as vision from '@mediapipe/tasks-vision';\nimport { dependencies } from '../../package.json';\nimport VideoTransformer from './VideoTransformer';\nimport { VideoTransformerInitOptions } from './types';\n\nexport type BackgroundOptions = {\n blurRadius?: number;\n imagePath?: string;\n};\n\nexport default class BackgroundProcessor extends VideoTransformer {\n static get isSupported() {\n return typeof OffscreenCanvas !== 'undefined';\n }\n\n imageSegmenter?: vision.ImageSegmenter;\n\n segmentationResults: vision.ImageSegmenterResult | undefined;\n\n backgroundImage: ImageBitmap | null = null;\n\n blurRadius?: number;\n\n constructor(opts: BackgroundOptions) {\n super();\n if (opts.blurRadius) {\n this.blurRadius = opts.blurRadius;\n } else if (opts.imagePath) {\n this.loadBackground(opts.imagePath);\n }\n }\n\n async init({ outputCanvas, inputElement: inputVideo }: VideoTransformerInitOptions) {\n await super.init({ outputCanvas, inputElement: inputVideo });\n\n const fileSet = await vision.FilesetResolver.forVisionTasks(\n `https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@${dependencies['@mediapipe/tasks-vision']}/wasm`,\n );\n\n this.imageSegmenter = await vision.ImageSegmenter.createFromOptions(fileSet, {\n baseOptions: {\n modelAssetPath:\n 'https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_segmenter/float16/latest/selfie_segmenter.tflite',\n delegate: 'CPU',\n },\n runningMode: 'VIDEO',\n outputCategoryMask: true,\n outputConfidenceMasks: false,\n });\n\n // this.loadBackground(opts.backgroundUrl).catch((e) => console.error(e));\n }\n\n async destroy() {\n await super.destroy();\n await this.imageSegmenter?.close();\n this.backgroundImage = null;\n }\n\n async loadBackground(path: string) {\n const img = new Image();\n\n await new Promise((resolve, reject) => {\n img.crossOrigin = 'Anonymous';\n img.onload = () => resolve(img);\n img.onerror = (err) => reject(err);\n img.src = path;\n });\n const imageData = await createImageBitmap(img);\n this.backgroundImage = imageData;\n }\n\n async transform(frame: VideoFrame, controller: TransformStreamDefaultController<VideoFrame>) {\n try {\n if (this.isDisabled) {\n controller.enqueue(frame);\n return;\n }\n if (!this.canvas) {\n throw TypeError('Canvas needs to be initialized first');\n }\n let startTimeMs = performance.now();\n this.imageSegmenter?.segmentForVideo(\n this.inputVideo!,\n startTimeMs,\n (result) => (this.segmentationResults = result),\n );\n\n if (this.blurRadius) {\n await this.blurBackground(frame);\n } else {\n await this.drawVirtualBackground(frame);\n }\n const newFrame = new VideoFrame(this.canvas, {\n timestamp: frame.timestamp || Date.now(),\n });\n controller.enqueue(newFrame);\n } finally {\n frame.close();\n }\n }\n\n async drawVirtualBackground(frame: VideoFrame) {\n if (!this.canvas || !this.ctx || !this.segmentationResults || !this.inputVideo) return;\n // this.ctx.save();\n // this.ctx.clearRect(0, 0, this.canvas.width, this.canvas.height);\n if (this.segmentationResults?.categoryMask) {\n this.ctx.filter = 'blur(3px)';\n this.ctx.globalCompositeOperation = 'copy';\n const bitmap = await maskToBitmap(\n this.segmentationResults.categoryMask,\n this.inputVideo.videoWidth,\n this.inputVideo.videoHeight,\n );\n this.ctx.drawImage(bitmap, 0, 0);\n this.ctx.filter = 'none';\n this.ctx.globalCompositeOperation = 'source-in';\n if (this.backgroundImage) {\n this.ctx.drawImage(\n this.backgroundImage,\n 0,\n 0,\n this.backgroundImage.width,\n this.backgroundImage.height,\n 0,\n 0,\n this.canvas.width,\n this.canvas.height,\n );\n } else {\n this.ctx.fillStyle = '#00FF00';\n this.ctx.fillRect(0, 0, this.canvas.width, this.canvas.height);\n }\n\n this.ctx.globalCompositeOperation = 'destination-over';\n }\n this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);\n // this.ctx.restore();\n }\n\n async blurBackground(frame: VideoFrame) {\n if (\n !this.ctx ||\n !this.canvas ||\n !this.segmentationResults?.categoryMask?.canvas ||\n !this.inputVideo\n ) {\n return;\n }\n\n this.ctx.save();\n this.ctx.globalCompositeOperation = 'copy';\n\n const bitmap = await maskToBitmap(\n this.segmentationResults.categoryMask,\n this.inputVideo.videoWidth,\n this.inputVideo.videoHeight,\n );\n\n this.ctx.filter = 'blur(3px)';\n this.ctx.globalCompositeOperation = 'copy';\n this.ctx.drawImage(bitmap, 0, 0);\n this.ctx.filter = 'none';\n this.ctx.globalCompositeOperation = 'source-out';\n this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);\n this.ctx.globalCompositeOperation = 'destination-over';\n this.ctx.filter = `blur(${this.blurRadius}px)`;\n this.ctx.drawImage(frame, 0, 0);\n this.ctx.restore();\n }\n}\n\nfunction maskToBitmap(\n mask: vision.MPMask,\n videoWidth: number,\n videoHeight: number,\n): Promise<ImageBitmap> {\n const dataArray: Uint8ClampedArray = new Uint8ClampedArray(videoWidth * videoHeight * 4);\n const result = mask.getAsUint8Array();\n for (let i = 0; i < result.length; i += 1) {\n dataArray[i * 4] = result[i];\n dataArray[i * 4 + 1] = result[i];\n dataArray[i * 4 + 2] = result[i];\n dataArray[i * 4 + 3] = result[i];\n }\n const dataNew = new ImageData(dataArray, videoWidth, videoHeight);\n\n return createImageBitmap(dataNew);\n}\n","{\n \"name\": \"@livekit/track-processors\",\n \"version\": \"0.2.0\",\n \"description\": \"LiveKit track processors\",\n \"main\": \"dist/index.js\",\n \"module\": \"dist/index.mjs\",\n \"source\": \"src/index.ts\",\n \"types\": \"dist/src/index.d.ts\",\n \"repository\": \"git@github.com:livekit/livekit-track-processors.git\",\n \"author\": \"Lukas Seiler\",\n \"license\": \"Apache-2.0\",\n \"scripts\": {\n \"build\": \"tsup --onSuccess \\\"tsc --declaration --emitDeclarationOnly\\\"\",\n \"build-docs\": \"yarn exec typedoc\",\n \"build-sample\": \"cd example && webpack && cp styles.css index.html dist/\",\n \"lint\": \"eslint src\",\n \"test\": \"jest\",\n \"sample\": \"vite serve example --port 8080 --open\"\n },\n \"files\": [\n \"dist\",\n \"src\"\n ],\n \"dependencies\": {\n \"@mediapipe/holistic\": \"0.5.1675471629\",\n \"@mediapipe/tasks-vision\": \"0.10.1\"\n },\n \"peerDependencies\": {\n \"livekit-client\": \"^1.12.0\"\n },\n \"devDependencies\": {\n \"@trivago/prettier-plugin-sort-imports\": \"^4.1.1\",\n \"@types/dom-mediacapture-transform\": \"^0.1.5\",\n \"@types/jest\": \"^27.0.3\",\n \"@types/offscreencanvas\": \"^2019.7.0\",\n \"@typescript-eslint/eslint-plugin\": \"^4.31.2\",\n \"@webpack-cli/serve\": \"^1.5.2\",\n \"eslint\": \"8.39.0\",\n \"eslint-config-airbnb-typescript\": \"17.0.0\",\n \"eslint-config-prettier\": \"8.8.0\",\n \"eslint-plugin-ecmascript-compat\": \"^3.0.0\",\n \"eslint-plugin-import\": \"2.27.5\",\n \"jest\": \"^27.4.3\",\n \"livekit-client\": \"^0.16.6\",\n \"prettier\": \"^2.8.8\",\n \"ts-jest\": \"^27.0.7\",\n \"ts-loader\": \"^8.1.0\",\n \"ts-proto\": \"^1.85.0\",\n \"tsup\": \"^7.1.0\",\n \"typedoc\": \"^0.20.35\",\n \"typedoc-plugin-no-inherit\": \"1.3.0\",\n \"typescript\": \"^5.0.4\",\n \"vite\": \"^4.3.8\",\n \"webpack\": \"^5.53.0\",\n \"webpack-cli\": \"^4.8.0\",\n \"webpack-dev-server\": \"^4.2.1\"\n }\n}\n","import { VideoTrackTransformer, VideoTransformerInitOptions } from './types';\n\nexport default abstract class VideoTransformer implements VideoTrackTransformer {\n transformer?: TransformStream;\n\n canvas?: OffscreenCanvas;\n\n ctx?: OffscreenCanvasRenderingContext2D;\n\n inputVideo?: HTMLVideoElement;\n\n protected isDisabled?: Boolean = false;\n\n async init({\n outputCanvas,\n inputElement: inputVideo,\n }: VideoTransformerInitOptions): Promise<void> {\n if (!(inputVideo instanceof HTMLVideoElement)) {\n throw TypeError('Video transformer needs a HTMLVideoElement as input');\n }\n this.transformer = new TransformStream({\n transform: (frame, controller) => this.transform(frame, controller),\n });\n this.canvas = outputCanvas || null;\n if (outputCanvas) {\n this.ctx = this.canvas?.getContext('2d') || undefined;\n }\n this.inputVideo = inputVideo;\n this.isDisabled = false;\n }\n\n async destroy() {\n this.isDisabled = true;\n this.canvas = undefined;\n this.ctx = undefined;\n }\n\n abstract transform(\n frame: VideoFrame,\n controller: TransformStreamDefaultController<VideoFrame>,\n ): void;\n}\n","import VideoTransformer from './VideoTransformer';\n\nexport default class DummyTransformer extends VideoTransformer {\n async transform(frame: VideoFrame, controller: TransformStreamDefaultController<VideoFrame>) {\n controller.enqueue(frame);\n }\n\n async destroy() {\n // nothing to do\n }\n}\n","import ProcessorPipeline from './ProcessorPipeline';\nimport BackgroundTransformer from './transformers/BackgroundTransformer';\nimport DummyTransformer from './transformers/DummyTransformer';\n\nexport const BackgroundBlur = (blurRadius: number = 10) => {\n const isPipelineSupported = ProcessorPipeline.isSupported && BackgroundTransformer.isSupported;\n if (!isPipelineSupported) {\n throw new Error('pipeline is not supported in this browser');\n }\n const pipeline = new ProcessorPipeline(\n [new BackgroundTransformer({ blurRadius })],\n 'background-blur',\n );\n return pipeline;\n};\n\nexport const VirtualBackground = (imagePath: string) => {\n const isPipelineSupported = ProcessorPipeline.isSupported && BackgroundTransformer.isSupported;\n if (!isPipelineSupported) {\n throw new Error('pipeline is not supported in this browser');\n }\n const pipeline = new ProcessorPipeline(\n [new BackgroundTransformer({ imagePath })],\n 'virtual-background',\n );\n return pipeline;\n};\n\nexport const Dummy = () => {\n const isPipelineSupported = ProcessorPipeline.isSupported && BackgroundTransformer.isSupported;\n if (!isPipelineSupported) {\n throw new Error('pipeline is not supported in this browser');\n }\n const pipeline = new ProcessorPipeline([new DummyTransformer()], 'dummy');\n return pipeline;\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AAGA,IAAqB,oBAArB,MAA6E;AAAA,EAC3E,WAAW,cAAc;AACvB,WACE,OAAO,8BAA8B,eACrC,OAAO,8BAA8B;AAAA,EAEzC;AAAA,EAoBA,YAAY,cAA4C,MAAc;AACpE,SAAK,OAAO;AACZ,SAAK,eAAe;AAAA,EACtB;AAAA,EAEA,MAAM,KAAK,MAAoC;AAlCjD;AAmCI,SAAK,SAAS,KAAK;AACnB,UAAM,kBAAkB,KAAK,OAAO,eAAe;AACnD,UAAM,KAAK,OAAO,iBAAiB,iCAC9B,kBAD8B;AAAA;AAAA;AAAA;AAAA;AAAA,MAMjC,YAAY;AAAA,IACd,EAAC;AACD,SAAK,iBAAiB,KAAK,OAAO,YAAY;AAC9C,SAAK,cAAc,KAAK;AACxB,QAAI,KAAK,uBAAuB,kBAAkB;AAChD,WAAK,YAAY,SAAS,KAAK,eAAe;AAC9C,WAAK,YAAY,QAAQ,KAAK,eAAe;AAAA,IAC/C;AACA,QAAI,EAAE,KAAK,uBAAuB,mBAAmB;AACnD,YAAM,UAAU,iDAAiD;AAAA,IACnE;AAEA,SAAK,YAAY,IAAI,0BAA0B,EAAE,OAAO,KAAK,OAAO,CAAC;AAErE,SAAK,iBAAiB,IAAI,0BAA0B;AAAA,MAClD,MAAM;AAAA,MACN,cAAc,KAAK;AAAA,IACrB,CAAC;AAED,SAAK,SAAS,IAAI;AAAA,OAChB,UAAK,eAAe,UAApB,YAA6B;AAAA,OAC7B,UAAK,eAAe,WAApB,YAA8B;AAAA,IAChC;AAEA,QAAI,iBAAiB,KAAK,UAAU;AACpC,eAAW,eAAe,KAAK,cAAc;AAC3C,YAAM,YAAY,KAAK;AAAA,QACrB,cAAc,KAAK;AAAA,QACnB,cAAc,KAAK;AAAA,MACrB,CAAC;AACD,uBAAiB,eAAe,YAAY,YAAa,WAAY;AAAA,IACvE;AACA,mBACG,OAAO,KAAK,eAAe,QAAQ,EACnC,MAAM,CAAC,MAAM,QAAQ,MAAM,6BAA6B,CAAC,CAAC,EAC1D,QAAQ,MAAM,KAAK,QAAQ,CAAC;AAC/B,SAAK,iBAAiB,KAAK;AAAA,EAC7B;AAAA,EAEA,MAAM,UAAU;AAlFlB;AAmFI,eAAW,eAAe,KAAK,cAAc;AAC3C,YAAM,YAAY,QAAQ;AAAA,IAC5B;AACA,eAAK,mBAAL,mBAAqB;AAAA,EACvB;AACF;;;ACxFA,YAAY,YAAY;;;ACuBtB,mBAAgB;AAAA,EACd,uBAAuB;AAAA,EACvB,2BAA2B;AAC7B;;;ACxBF,IAA8B,mBAA9B,MAAgF;AAAA,EAAhF;AASE,SAAU,aAAuB;AAAA;AAAA,EAEjC,MAAM,KAAK;AAAA,IACT;AAAA,IACA,cAAc;AAAA,EAChB,GAA+C;AAhBjD;AAiBI,QAAI,EAAE,sBAAsB,mBAAmB;AAC7C,YAAM,UAAU,qDAAqD;AAAA,IACvE;AACA,SAAK,cAAc,IAAI,gBAAgB;AAAA,MACrC,WAAW,CAAC,OAAO,eAAe,KAAK,UAAU,OAAO,UAAU;AAAA,IACpE,CAAC;AACD,SAAK,SAAS,gBAAgB;AAC9B,QAAI,cAAc;AAChB,WAAK,QAAM,UAAK,WAAL,mBAAa,WAAW,UAAS;AAAA,IAC9C;AACA,SAAK,aAAa;AAClB,SAAK,aAAa;AAAA,EACpB;AAAA,EAEA,MAAM,UAAU;AACd,SAAK,aAAa;AAClB,SAAK,SAAS;AACd,SAAK,MAAM;AAAA,EACb;AAMF;;;AF/BA,IAAqB,sBAArB,cAAiD,iBAAiB;AAAA,EAahE,YAAY,MAAyB;AACnC,UAAM;AALR,2BAAsC;AAMpC,QAAI,KAAK,YAAY;AACnB,WAAK,aAAa,KAAK;AAAA,IACzB,WAAW,KAAK,WAAW;AACzB,WAAK,eAAe,KAAK,SAAS;AAAA,IACpC;AAAA,EACF;AAAA,EAnBA,WAAW,cAAc;AACvB,WAAO,OAAO,oBAAoB;AAAA,EACpC;AAAA,EAmBA,MAAM,KAAK,EAAE,cAAc,cAAc,WAAW,GAAgC;AAClF,UAAM,MAAM,KAAK,EAAE,cAAc,cAAc,WAAW,CAAC;AAE3D,UAAM,UAAU,MAAa,uBAAgB;AAAA,MAC3C,wDAAwD,aAAa,yBAAyB,CAAC;AAAA,IACjG;AAEA,SAAK,iBAAiB,MAAa,sBAAe,kBAAkB,SAAS;AAAA,MAC3E,aAAa;AAAA,QACX,gBACE;AAAA,QACF,UAAU;AAAA,MACZ;AAAA,MACA,aAAa;AAAA,MACb,oBAAoB;AAAA,MACpB,uBAAuB;AAAA,IACzB,CAAC;AAAA,EAGH;AAAA,EAEA,MAAM,UAAU;AArDlB;AAsDI,UAAM,MAAM,QAAQ;AACpB,YAAM,UAAK,mBAAL,mBAAqB;AAC3B,SAAK,kBAAkB;AAAA,EACzB;AAAA,EAEA,MAAM,eAAe,MAAc;AACjC,UAAM,MAAM,IAAI,MAAM;AAEtB,UAAM,IAAI,QAAQ,CAAC,SAAS,WAAW;AACrC,UAAI,cAAc;AAClB,UAAI,SAAS,MAAM,QAAQ,GAAG;AAC9B,UAAI,UAAU,CAAC,QAAQ,OAAO,GAAG;AACjC,UAAI,MAAM;AAAA,IACZ,CAAC;AACD,UAAM,YAAY,MAAM,kBAAkB,GAAG;AAC7C,SAAK,kBAAkB;AAAA,EACzB;AAAA,EAEA,MAAM,UAAU,OAAmB,YAA0D;AAxE/F;AAyEI,QAAI;AACF,UAAI,KAAK,YAAY;AACnB,mBAAW,QAAQ,KAAK;AACxB;AAAA,MACF;AACA,UAAI,CAAC,KAAK,QAAQ;AAChB,cAAM,UAAU,sCAAsC;AAAA,MACxD;AACA,UAAI,cAAc,YAAY,IAAI;AAClC,iBAAK,mBAAL,mBAAqB;AAAA,QACnB,KAAK;AAAA,QACL;AAAA,QACA,CAAC,WAAY,KAAK,sBAAsB;AAAA;AAG1C,UAAI,KAAK,YAAY;AACnB,cAAM,KAAK,eAAe,KAAK;AAAA,MACjC,OAAO;AACL,cAAM,KAAK,sBAAsB,KAAK;AAAA,MACxC;AACA,YAAM,WAAW,IAAI,WAAW,KAAK,QAAQ;AAAA,QAC3C,WAAW,MAAM,aAAa,KAAK,IAAI;AAAA,MACzC,CAAC;AACD,iBAAW,QAAQ,QAAQ;AAAA,IAC7B,UAAE;AACA,YAAM,MAAM;AAAA,IACd;AAAA,EACF;AAAA,EAEA,MAAM,sBAAsB,OAAmB;AAtGjD;AAuGI,QAAI,CAAC,KAAK,UAAU,CAAC,KAAK,OAAO,CAAC,KAAK,uBAAuB,CAAC,KAAK;AAAY;AAGhF,SAAI,UAAK,wBAAL,mBAA0B,cAAc;AAC1C,WAAK,IAAI,SAAS;AAClB,WAAK,IAAI,2BAA2B;AACpC,YAAM,SAAS,MAAM;AAAA,QACnB,KAAK,oBAAoB;AAAA,QACzB,KAAK,WAAW;AAAA,QAChB,KAAK,WAAW;AAAA,MAClB;AACA,WAAK,IAAI,UAAU,QAAQ,GAAG,CAAC;AAC/B,WAAK,IAAI,SAAS;AAClB,WAAK,IAAI,2BAA2B;AACpC,UAAI,KAAK,iBAAiB;AACxB,aAAK,IAAI;AAAA,UACP,KAAK;AAAA,UACL;AAAA,UACA;AAAA,UACA,KAAK,gBAAgB;AAAA,UACrB,KAAK,gBAAgB;AAAA,UACrB;AAAA,UACA;AAAA,UACA,KAAK,OAAO;AAAA,UACZ,KAAK,OAAO;AAAA,QACd;AAAA,MACF,OAAO;AACL,aAAK,IAAI,YAAY;AACrB,aAAK,IAAI,SAAS,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AAAA,MAC/D;AAEA,WAAK,IAAI,2BAA2B;AAAA,IACtC;AACA,SAAK,IAAI,UAAU,OAAO,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AAAA,EAEvE;AAAA,EAEA,MAAM,eAAe,OAAmB;AA5I1C;AA6II,QACE,CAAC,KAAK,OACN,CAAC,KAAK,UACN,GAAC,gBAAK,wBAAL,mBAA0B,iBAA1B,mBAAwC,WACzC,CAAC,KAAK,YACN;AACA;AAAA,IACF;AAEA,SAAK,IAAI,KAAK;AACd,SAAK,IAAI,2BAA2B;AAEpC,UAAM,SAAS,MAAM;AAAA,MACnB,KAAK,oBAAoB;AAAA,MACzB,KAAK,WAAW;AAAA,MAChB,KAAK,WAAW;AAAA,IAClB;AAEA,SAAK,IAAI,SAAS;AAClB,SAAK,IAAI,2BAA2B;AACpC,SAAK,IAAI,UAAU,QAAQ,GAAG,CAAC;AAC/B,SAAK,IAAI,SAAS;AAClB,SAAK,IAAI,2BAA2B;AACpC,SAAK,IAAI,UAAU,OAAO,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AACrE,SAAK,IAAI,2BAA2B;AACpC,SAAK,IAAI,SAAS,QAAQ,KAAK,UAAU;AACzC,SAAK,IAAI,UAAU,OAAO,GAAG,CAAC;AAC9B,SAAK,IAAI,QAAQ;AAAA,EACnB;AACF;AAEA,SAAS,aACP,MACA,YACA,aACsB;AACtB,QAAM,YAA+B,IAAI,kBAAkB,aAAa,cAAc,CAAC;AACvF,QAAM,SAAS,KAAK,gBAAgB;AACpC,WAAS,IAAI,GAAG,IAAI,OAAO,QAAQ,KAAK,GAAG;AACzC,cAAU,IAAI,CAAC,IAAI,OAAO,CAAC;AAC3B,cAAU,IAAI,IAAI,CAAC,IAAI,OAAO,CAAC;AAC/B,cAAU,IAAI,IAAI,CAAC,IAAI,OAAO,CAAC;AAC/B,cAAU,IAAI,IAAI,CAAC,IAAI,OAAO,CAAC;AAAA,EACjC;AACA,QAAM,UAAU,IAAI,UAAU,WAAW,YAAY,WAAW;AAEhE,SAAO,kBAAkB,OAAO;AAClC;;;AG1LA,IAAqB,mBAArB,cAA8C,iBAAiB;AAAA,EAC7D,MAAM,UAAU,OAAmB,YAA0D;AAC3F,eAAW,QAAQ,KAAK;AAAA,EAC1B;AAAA,EAEA,MAAM,UAAU;AAAA,EAEhB;AACF;;;ACNO,IAAM,iBAAiB,CAAC,aAAqB,OAAO;AACzD,QAAM,sBAAsB,kBAAkB,eAAe,oBAAsB;AACnF,MAAI,CAAC,qBAAqB;AACxB,UAAM,IAAI,MAAM,2CAA2C;AAAA,EAC7D;AACA,QAAM,WAAW,IAAI;AAAA,IACnB,CAAC,IAAI,oBAAsB,EAAE,WAAW,CAAC,CAAC;AAAA,IAC1C;AAAA,EACF;AACA,SAAO;AACT;AAEO,IAAM,oBAAoB,CAAC,cAAsB;AACtD,QAAM,sBAAsB,kBAAkB,eAAe,oBAAsB;AACnF,MAAI,CAAC,qBAAqB;AACxB,UAAM,IAAI,MAAM,2CAA2C;AAAA,EAC7D;AACA,QAAM,WAAW,IAAI;AAAA,IACnB,CAAC,IAAI,oBAAsB,EAAE,UAAU,CAAC,CAAC;AAAA,IACzC;AAAA,EACF;AACA,SAAO;AACT;AAEO,IAAM,QAAQ,MAAM;AACzB,QAAM,sBAAsB,kBAAkB,eAAe,oBAAsB;AACnF,MAAI,CAAC,qBAAqB;AACxB,UAAM,IAAI,MAAM,2CAA2C;AAAA,EAC7D;AACA,QAAM,WAAW,IAAI,kBAAkB,CAAC,IAAI,iBAAiB,CAAC,GAAG,OAAO;AACxE,SAAO;AACT;","names":[]}
1
+ {"version":3,"sources":["../src/ProcessorPipeline.ts","../src/transformers/BackgroundTransformer.ts","../package.json","../src/transformers/VideoTransformer.ts","../src/transformers/DummyTransformer.ts","../src/index.ts"],"sourcesContent":["import type { ProcessorOptions, Track, TrackProcessor } from 'livekit-client';\nimport { VideoTrackTransformer } from './transformers';\n\nexport default class ProcessorPipeline implements TrackProcessor<Track.Kind> {\n static get isSupported() {\n return (\n typeof MediaStreamTrackGenerator !== 'undefined' &&\n typeof MediaStreamTrackProcessor !== 'undefined'\n );\n }\n\n name: string;\n\n source?: MediaStreamVideoTrack;\n\n sourceSettings?: MediaTrackSettings;\n\n processor?: MediaStreamTrackProcessor<VideoFrame>;\n\n trackGenerator?: MediaStreamTrackGenerator<VideoFrame>;\n\n canvas?: OffscreenCanvas;\n\n sourceDummy?: HTMLMediaElement;\n\n processedTrack?: MediaStreamTrack;\n\n transformers: Array<VideoTrackTransformer>;\n\n constructor(transformers: Array<VideoTrackTransformer>, name: string) {\n this.name = name;\n this.transformers = transformers;\n }\n\n async init(opts: ProcessorOptions<Track.Kind>) {\n this.source = opts.track as MediaStreamVideoTrack;\n const origConstraints = this.source.getConstraints();\n await this.source.applyConstraints({\n ...origConstraints,\n // @ts-expect-error when a mediastream track is resized and/or cropped, the `VideoFrame` will have a coded height/width of the original video size\n // this leads to a shift of the underlying video as the frame itself is being rendered with the coded size\n // but image segmentation is based on the display dimensions (-> the cropped version)\n // in order to prevent this, we force the resize mode to \"none\"\n resizeMode: 'none',\n });\n this.sourceSettings = this.source.getSettings();\n this.sourceDummy = opts.element;\n if (this.sourceDummy instanceof HTMLVideoElement) {\n this.sourceDummy.height = this.sourceSettings.height ?? 300;\n this.sourceDummy.width = this.sourceSettings.width ?? 300;\n }\n if (!(this.sourceDummy instanceof HTMLVideoElement)) {\n throw TypeError('Currently only video transformers are supported');\n }\n // TODO explore if we can do all the processing work in a webworker\n this.processor = new MediaStreamTrackProcessor({ track: this.source });\n\n this.trackGenerator = new MediaStreamTrackGenerator({\n kind: 'video',\n signalTarget: this.source,\n });\n\n this.canvas = new OffscreenCanvas(\n this.sourceSettings.width ?? 300,\n this.sourceSettings.height ?? 300,\n );\n\n let readableStream = this.processor.readable;\n for (const transformer of this.transformers) {\n await transformer.init({\n outputCanvas: this.canvas,\n inputElement: this.sourceDummy!,\n });\n readableStream = readableStream.pipeThrough(transformer!.transformer!);\n }\n readableStream\n .pipeTo(this.trackGenerator.writable)\n .catch((e) => console.error('error when trying to pipe', e))\n .finally(() => this.destroy());\n this.processedTrack = this.trackGenerator as MediaStreamVideoTrack;\n }\n\n async destroy() {\n for (const transformer of this.transformers) {\n await transformer.destroy();\n }\n this.trackGenerator?.stop();\n }\n}\n","import * as vision from '@mediapipe/tasks-vision';\nimport { dependencies } from '../../package.json';\nimport VideoTransformer from './VideoTransformer';\nimport { VideoTransformerInitOptions } from './types';\n\nexport type SegmenterBaseOptions = Partial<vision.ImageSegmenterOptions['baseOptions']>;\n\nexport type BackgroundOptions = {\n blurRadius?: number;\n imagePath?: string;\n segmenterOptions?: SegmenterBaseOptions;\n};\n\nexport default class BackgroundProcessor extends VideoTransformer {\n static get isSupported() {\n return typeof OffscreenCanvas !== 'undefined';\n }\n\n imageSegmenter?: vision.ImageSegmenter;\n\n segmentationResults: vision.ImageSegmenterResult | undefined;\n\n backgroundImage: ImageBitmap | null = null;\n\n blurRadius?: number;\n\n options: BackgroundOptions;\n\n constructor(opts: BackgroundOptions) {\n super();\n this.options = opts;\n if (opts.blurRadius) {\n this.blurRadius = opts.blurRadius;\n } else if (opts.imagePath) {\n this.loadBackground(opts.imagePath);\n }\n\n this.options.segmenterOptions;\n }\n\n async init({ outputCanvas, inputElement: inputVideo }: VideoTransformerInitOptions) {\n await super.init({ outputCanvas, inputElement: inputVideo });\n\n const fileSet = await vision.FilesetResolver.forVisionTasks(\n `https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@${dependencies['@mediapipe/tasks-vision']}/wasm`,\n );\n\n this.imageSegmenter = await vision.ImageSegmenter.createFromOptions(fileSet, {\n baseOptions: {\n modelAssetPath:\n 'https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_segmenter/float16/latest/selfie_segmenter.tflite',\n delegate: 'GPU',\n ...this.options.segmenterOptions,\n },\n runningMode: 'VIDEO',\n outputCategoryMask: true,\n outputConfidenceMasks: false,\n });\n\n // this.loadBackground(opts.backgroundUrl).catch((e) => console.error(e));\n }\n\n async destroy() {\n await super.destroy();\n await this.imageSegmenter?.close();\n this.backgroundImage = null;\n }\n\n async loadBackground(path: string) {\n const img = new Image();\n\n await new Promise((resolve, reject) => {\n img.crossOrigin = 'Anonymous';\n img.onload = () => resolve(img);\n img.onerror = (err) => reject(err);\n img.src = path;\n });\n const imageData = await createImageBitmap(img);\n this.backgroundImage = imageData;\n }\n\n async transform(frame: VideoFrame, controller: TransformStreamDefaultController<VideoFrame>) {\n try {\n if (this.isDisabled) {\n controller.enqueue(frame);\n return;\n }\n if (!this.canvas) {\n throw TypeError('Canvas needs to be initialized first');\n }\n let startTimeMs = performance.now();\n this.imageSegmenter?.segmentForVideo(\n this.inputVideo!,\n startTimeMs,\n (result) => (this.segmentationResults = result),\n );\n\n if (this.blurRadius) {\n await this.blurBackground(frame);\n } else {\n await this.drawVirtualBackground(frame);\n }\n const newFrame = new VideoFrame(this.canvas, {\n timestamp: frame.timestamp || Date.now(),\n });\n controller.enqueue(newFrame);\n } finally {\n frame.close();\n }\n }\n\n async drawVirtualBackground(frame: VideoFrame) {\n if (!this.canvas || !this.ctx || !this.segmentationResults || !this.inputVideo) return;\n // this.ctx.save();\n // this.ctx.clearRect(0, 0, this.canvas.width, this.canvas.height);\n if (this.segmentationResults?.categoryMask) {\n this.ctx.filter = 'blur(3px)';\n this.ctx.globalCompositeOperation = 'copy';\n const bitmap = await maskToBitmap(\n this.segmentationResults.categoryMask,\n this.segmentationResults.categoryMask.width,\n this.segmentationResults.categoryMask.height,\n );\n this.ctx.drawImage(bitmap, 0, 0, this.canvas.width, this.canvas.height);\n this.ctx.filter = 'none';\n this.ctx.globalCompositeOperation = 'source-in';\n if (this.backgroundImage) {\n this.ctx.drawImage(\n this.backgroundImage,\n 0,\n 0,\n this.backgroundImage.width,\n this.backgroundImage.height,\n 0,\n 0,\n this.canvas.width,\n this.canvas.height,\n );\n } else {\n this.ctx.fillStyle = '#00FF00';\n this.ctx.fillRect(0, 0, this.canvas.width, this.canvas.height);\n }\n\n this.ctx.globalCompositeOperation = 'destination-over';\n }\n this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);\n // this.ctx.restore();\n }\n\n async blurBackground(frame: VideoFrame) {\n if (\n !this.ctx ||\n !this.canvas ||\n !this.segmentationResults?.categoryMask?.canvas ||\n !this.inputVideo\n ) {\n return;\n }\n\n this.ctx.save();\n this.ctx.globalCompositeOperation = 'copy';\n\n const bitmap = await maskToBitmap(\n this.segmentationResults.categoryMask,\n this.segmentationResults.categoryMask.width,\n this.segmentationResults.categoryMask.height,\n );\n\n this.ctx.filter = 'blur(3px)';\n this.ctx.globalCompositeOperation = 'copy';\n this.ctx.drawImage(bitmap, 0, 0, this.canvas.width, this.canvas.height);\n this.ctx.filter = 'none';\n this.ctx.globalCompositeOperation = 'source-out';\n this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);\n this.ctx.globalCompositeOperation = 'destination-over';\n this.ctx.filter = `blur(${this.blurRadius}px)`;\n this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);\n this.ctx.restore();\n }\n}\n\nfunction maskToBitmap(\n mask: vision.MPMask,\n videoWidth: number,\n videoHeight: number,\n): Promise<ImageBitmap> {\n const dataArray: Uint8ClampedArray = new Uint8ClampedArray(videoWidth * videoHeight * 4);\n const result = mask.getAsUint8Array();\n for (let i = 0; i < result.length; i += 1) {\n dataArray[i * 4] = result[i];\n dataArray[i * 4 + 1] = result[i];\n dataArray[i * 4 + 2] = result[i];\n dataArray[i * 4 + 3] = result[i];\n }\n const dataNew = new ImageData(dataArray, videoWidth, videoHeight);\n\n return createImageBitmap(dataNew);\n}\n","{\n \"name\": \"@livekit/track-processors\",\n \"version\": \"0.2.5\",\n \"description\": \"LiveKit track processors\",\n \"main\": \"dist/index.js\",\n \"module\": \"dist/index.mjs\",\n \"source\": \"src/index.ts\",\n \"types\": \"dist/src/index.d.ts\",\n \"repository\": \"git@github.com:livekit/livekit-track-processors.git\",\n \"author\": \"Lukas Seiler\",\n \"license\": \"Apache-2.0\",\n \"scripts\": {\n \"build\": \"tsup --onSuccess \\\"tsc --declaration --emitDeclarationOnly\\\"\",\n \"build-docs\": \"yarn exec typedoc\",\n \"build-sample\": \"cd example && vite build\",\n \"lint\": \"eslint src\",\n \"test\": \"jest\",\n \"sample\": \"vite serve example --port 8080 --open\",\n \"prepublish\": \"yarn build\"\n },\n \"files\": [\n \"dist\",\n \"src\"\n ],\n \"dependencies\": {\n \"@mediapipe/holistic\": \"0.5.1675471629\",\n \"@mediapipe/tasks-vision\": \"0.10.1\"\n },\n \"peerDependencies\": {\n \"livekit-client\": \"^1.12.0\"\n },\n \"devDependencies\": {\n \"@trivago/prettier-plugin-sort-imports\": \"^4.1.1\",\n \"@types/dom-mediacapture-transform\": \"^0.1.5\",\n \"@types/jest\": \"^27.0.3\",\n \"@types/offscreencanvas\": \"^2019.7.0\",\n \"@typescript-eslint/eslint-plugin\": \"^4.31.2\",\n \"@webpack-cli/serve\": \"^1.5.2\",\n \"eslint\": \"8.39.0\",\n \"eslint-config-airbnb-typescript\": \"17.0.0\",\n \"eslint-config-prettier\": \"8.8.0\",\n \"eslint-plugin-ecmascript-compat\": \"^3.0.0\",\n \"eslint-plugin-import\": \"2.27.5\",\n \"jest\": \"^27.4.3\",\n \"livekit-client\": \"^1.12.0\",\n \"prettier\": \"^2.8.8\",\n \"ts-jest\": \"^27.0.7\",\n \"ts-loader\": \"^8.1.0\",\n \"ts-proto\": \"^1.85.0\",\n \"tsup\": \"^7.1.0\",\n \"typedoc\": \"^0.20.35\",\n \"typedoc-plugin-no-inherit\": \"1.3.0\",\n \"typescript\": \"^5.0.4\",\n \"vite\": \"^4.3.8\",\n \"webpack\": \"^5.53.0\",\n \"webpack-cli\": \"^4.8.0\",\n \"webpack-dev-server\": \"^4.2.1\"\n }\n}\n","import { VideoTrackTransformer, VideoTransformerInitOptions } from './types';\n\nexport default abstract class VideoTransformer implements VideoTrackTransformer {\n transformer?: TransformStream;\n\n canvas?: OffscreenCanvas;\n\n ctx?: OffscreenCanvasRenderingContext2D;\n\n inputVideo?: HTMLVideoElement;\n\n protected isDisabled?: Boolean = false;\n\n async init({\n outputCanvas,\n inputElement: inputVideo,\n }: VideoTransformerInitOptions): Promise<void> {\n if (!(inputVideo instanceof HTMLVideoElement)) {\n throw TypeError('Video transformer needs a HTMLVideoElement as input');\n }\n this.transformer = new TransformStream({\n transform: (frame, controller) => this.transform(frame, controller),\n });\n this.canvas = outputCanvas || null;\n if (outputCanvas) {\n this.ctx = this.canvas?.getContext('2d') || undefined;\n }\n this.inputVideo = inputVideo;\n this.isDisabled = false;\n }\n\n async destroy() {\n this.isDisabled = true;\n this.canvas = undefined;\n this.ctx = undefined;\n }\n\n abstract transform(\n frame: VideoFrame,\n controller: TransformStreamDefaultController<VideoFrame>,\n ): void;\n}\n","import VideoTransformer from './VideoTransformer';\n\nexport default class DummyTransformer extends VideoTransformer {\n async transform(frame: VideoFrame, controller: TransformStreamDefaultController<VideoFrame>) {\n controller.enqueue(frame);\n }\n\n async destroy() {\n // nothing to do\n }\n}\n","import ProcessorPipeline from './ProcessorPipeline';\nimport BackgroundTransformer, { SegmenterBaseOptions } from './transformers/BackgroundTransformer';\nimport DummyTransformer from './transformers/DummyTransformer';\n\nexport * from './transformers/types';\n\nexport const BackgroundBlur = (\n blurRadius: number = 10,\n segmenterOptions?: SegmenterBaseOptions,\n) => {\n const isPipelineSupported = ProcessorPipeline.isSupported && BackgroundTransformer.isSupported;\n if (!isPipelineSupported) {\n throw new Error('pipeline is not supported in this browser');\n }\n const pipeline = new ProcessorPipeline(\n [new BackgroundTransformer({ blurRadius, segmenterOptions })],\n 'background-blur',\n );\n return pipeline;\n};\n\nexport const VirtualBackground = (imagePath: string, segmenterOptions?: SegmenterBaseOptions) => {\n const isPipelineSupported = ProcessorPipeline.isSupported && BackgroundTransformer.isSupported;\n if (!isPipelineSupported) {\n throw new Error('pipeline is not supported in this browser');\n }\n const pipeline = new ProcessorPipeline(\n [new BackgroundTransformer({ imagePath, segmenterOptions })],\n 'virtual-background',\n );\n return pipeline;\n};\n\nexport const Dummy = () => {\n const isPipelineSupported = ProcessorPipeline.isSupported && BackgroundTransformer.isSupported;\n if (!isPipelineSupported) {\n throw new Error('pipeline is not supported in this browser');\n }\n const pipeline = new ProcessorPipeline([new DummyTransformer()], 'dummy');\n return pipeline;\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AAGA,IAAqB,oBAArB,MAA6E;AAAA,EAC3E,WAAW,cAAc;AACvB,WACE,OAAO,8BAA8B,eACrC,OAAO,8BAA8B;AAAA,EAEzC;AAAA,EAoBA,YAAY,cAA4C,MAAc;AACpE,SAAK,OAAO;AACZ,SAAK,eAAe;AAAA,EACtB;AAAA,EAEA,MAAM,KAAK,MAAoC;AAlCjD;AAmCI,SAAK,SAAS,KAAK;AACnB,UAAM,kBAAkB,KAAK,OAAO,eAAe;AACnD,UAAM,KAAK,OAAO,iBAAiB,iCAC9B,kBAD8B;AAAA;AAAA;AAAA;AAAA;AAAA,MAMjC,YAAY;AAAA,IACd,EAAC;AACD,SAAK,iBAAiB,KAAK,OAAO,YAAY;AAC9C,SAAK,cAAc,KAAK;AACxB,QAAI,KAAK,uBAAuB,kBAAkB;AAChD,WAAK,YAAY,UAAS,UAAK,eAAe,WAApB,YAA8B;AACxD,WAAK,YAAY,SAAQ,UAAK,eAAe,UAApB,YAA6B;AAAA,IACxD;AACA,QAAI,EAAE,KAAK,uBAAuB,mBAAmB;AACnD,YAAM,UAAU,iDAAiD;AAAA,IACnE;AAEA,SAAK,YAAY,IAAI,0BAA0B,EAAE,OAAO,KAAK,OAAO,CAAC;AAErE,SAAK,iBAAiB,IAAI,0BAA0B;AAAA,MAClD,MAAM;AAAA,MACN,cAAc,KAAK;AAAA,IACrB,CAAC;AAED,SAAK,SAAS,IAAI;AAAA,OAChB,UAAK,eAAe,UAApB,YAA6B;AAAA,OAC7B,UAAK,eAAe,WAApB,YAA8B;AAAA,IAChC;AAEA,QAAI,iBAAiB,KAAK,UAAU;AACpC,eAAW,eAAe,KAAK,cAAc;AAC3C,YAAM,YAAY,KAAK;AAAA,QACrB,cAAc,KAAK;AAAA,QACnB,cAAc,KAAK;AAAA,MACrB,CAAC;AACD,uBAAiB,eAAe,YAAY,YAAa,WAAY;AAAA,IACvE;AACA,mBACG,OAAO,KAAK,eAAe,QAAQ,EACnC,MAAM,CAAC,MAAM,QAAQ,MAAM,6BAA6B,CAAC,CAAC,EAC1D,QAAQ,MAAM,KAAK,QAAQ,CAAC;AAC/B,SAAK,iBAAiB,KAAK;AAAA,EAC7B;AAAA,EAEA,MAAM,UAAU;AAlFlB;AAmFI,eAAW,eAAe,KAAK,cAAc;AAC3C,YAAM,YAAY,QAAQ;AAAA,IAC5B;AACA,eAAK,mBAAL,mBAAqB;AAAA,EACvB;AACF;;;ACxFA,YAAY,YAAY;;;ACwBtB,mBAAgB;AAAA,EACd,uBAAuB;AAAA,EACvB,2BAA2B;AAC7B;;;ACzBF,IAA8B,mBAA9B,MAAgF;AAAA,EAAhF;AASE,SAAU,aAAuB;AAAA;AAAA,EAEjC,MAAM,KAAK;AAAA,IACT;AAAA,IACA,cAAc;AAAA,EAChB,GAA+C;AAhBjD;AAiBI,QAAI,EAAE,sBAAsB,mBAAmB;AAC7C,YAAM,UAAU,qDAAqD;AAAA,IACvE;AACA,SAAK,cAAc,IAAI,gBAAgB;AAAA,MACrC,WAAW,CAAC,OAAO,eAAe,KAAK,UAAU,OAAO,UAAU;AAAA,IACpE,CAAC;AACD,SAAK,SAAS,gBAAgB;AAC9B,QAAI,cAAc;AAChB,WAAK,QAAM,UAAK,WAAL,mBAAa,WAAW,UAAS;AAAA,IAC9C;AACA,SAAK,aAAa;AAClB,SAAK,aAAa;AAAA,EACpB;AAAA,EAEA,MAAM,UAAU;AACd,SAAK,aAAa;AAClB,SAAK,SAAS;AACd,SAAK,MAAM;AAAA,EACb;AAMF;;;AF5BA,IAAqB,sBAArB,cAAiD,iBAAiB;AAAA,EAehE,YAAY,MAAyB;AACnC,UAAM;AAPR,2BAAsC;AAQpC,SAAK,UAAU;AACf,QAAI,KAAK,YAAY;AACnB,WAAK,aAAa,KAAK;AAAA,IACzB,WAAW,KAAK,WAAW;AACzB,WAAK,eAAe,KAAK,SAAS;AAAA,IACpC;AAEA,SAAK,QAAQ;AAAA,EACf;AAAA,EAxBA,WAAW,cAAc;AACvB,WAAO,OAAO,oBAAoB;AAAA,EACpC;AAAA,EAwBA,MAAM,KAAK,EAAE,cAAc,cAAc,WAAW,GAAgC;AAClF,UAAM,MAAM,KAAK,EAAE,cAAc,cAAc,WAAW,CAAC;AAE3D,UAAM,UAAU,MAAa,uBAAgB;AAAA,MAC3C,wDAAwD,aAAa,yBAAyB,CAAC;AAAA,IACjG;AAEA,SAAK,iBAAiB,MAAa,sBAAe,kBAAkB,SAAS;AAAA,MAC3E,aAAa;AAAA,QACX,gBACE;AAAA,QACF,UAAU;AAAA,SACP,KAAK,QAAQ;AAAA,MAElB,aAAa;AAAA,MACb,oBAAoB;AAAA,MACpB,uBAAuB;AAAA,IACzB,CAAC;AAAA,EAGH;AAAA,EAEA,MAAM,UAAU;AA9DlB;AA+DI,UAAM,MAAM,QAAQ;AACpB,YAAM,UAAK,mBAAL,mBAAqB;AAC3B,SAAK,kBAAkB;AAAA,EACzB;AAAA,EAEA,MAAM,eAAe,MAAc;AACjC,UAAM,MAAM,IAAI,MAAM;AAEtB,UAAM,IAAI,QAAQ,CAAC,SAAS,WAAW;AACrC,UAAI,cAAc;AAClB,UAAI,SAAS,MAAM,QAAQ,GAAG;AAC9B,UAAI,UAAU,CAAC,QAAQ,OAAO,GAAG;AACjC,UAAI,MAAM;AAAA,IACZ,CAAC;AACD,UAAM,YAAY,MAAM,kBAAkB,GAAG;AAC7C,SAAK,kBAAkB;AAAA,EACzB;AAAA,EAEA,MAAM,UAAU,OAAmB,YAA0D;AAjF/F;AAkFI,QAAI;AACF,UAAI,KAAK,YAAY;AACnB,mBAAW,QAAQ,KAAK;AACxB;AAAA,MACF;AACA,UAAI,CAAC,KAAK,QAAQ;AAChB,cAAM,UAAU,sCAAsC;AAAA,MACxD;AACA,UAAI,cAAc,YAAY,IAAI;AAClC,iBAAK,mBAAL,mBAAqB;AAAA,QACnB,KAAK;AAAA,QACL;AAAA,QACA,CAAC,WAAY,KAAK,sBAAsB;AAAA;AAG1C,UAAI,KAAK,YAAY;AACnB,cAAM,KAAK,eAAe,KAAK;AAAA,MACjC,OAAO;AACL,cAAM,KAAK,sBAAsB,KAAK;AAAA,MACxC;AACA,YAAM,WAAW,IAAI,WAAW,KAAK,QAAQ;AAAA,QAC3C,WAAW,MAAM,aAAa,KAAK,IAAI;AAAA,MACzC,CAAC;AACD,iBAAW,QAAQ,QAAQ;AAAA,IAC7B,UAAE;AACA,YAAM,MAAM;AAAA,IACd;AAAA,EACF;AAAA,EAEA,MAAM,sBAAsB,OAAmB;AA/GjD;AAgHI,QAAI,CAAC,KAAK,UAAU,CAAC,KAAK,OAAO,CAAC,KAAK,uBAAuB,CAAC,KAAK;AAAY;AAGhF,SAAI,UAAK,wBAAL,mBAA0B,cAAc;AAC1C,WAAK,IAAI,SAAS;AAClB,WAAK,IAAI,2BAA2B;AACpC,YAAM,SAAS,MAAM;AAAA,QACnB,KAAK,oBAAoB;AAAA,QACzB,KAAK,oBAAoB,aAAa;AAAA,QACtC,KAAK,oBAAoB,aAAa;AAAA,MACxC;AACA,WAAK,IAAI,UAAU,QAAQ,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AACtE,WAAK,IAAI,SAAS;AAClB,WAAK,IAAI,2BAA2B;AACpC,UAAI,KAAK,iBAAiB;AACxB,aAAK,IAAI;AAAA,UACP,KAAK;AAAA,UACL;AAAA,UACA;AAAA,UACA,KAAK,gBAAgB;AAAA,UACrB,KAAK,gBAAgB;AAAA,UACrB;AAAA,UACA;AAAA,UACA,KAAK,OAAO;AAAA,UACZ,KAAK,OAAO;AAAA,QACd;AAAA,MACF,OAAO;AACL,aAAK,IAAI,YAAY;AACrB,aAAK,IAAI,SAAS,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AAAA,MAC/D;AAEA,WAAK,IAAI,2BAA2B;AAAA,IACtC;AACA,SAAK,IAAI,UAAU,OAAO,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AAAA,EAEvE;AAAA,EAEA,MAAM,eAAe,OAAmB;AArJ1C;AAsJI,QACE,CAAC,KAAK,OACN,CAAC,KAAK,UACN,GAAC,gBAAK,wBAAL,mBAA0B,iBAA1B,mBAAwC,WACzC,CAAC,KAAK,YACN;AACA;AAAA,IACF;AAEA,SAAK,IAAI,KAAK;AACd,SAAK,IAAI,2BAA2B;AAEpC,UAAM,SAAS,MAAM;AAAA,MACnB,KAAK,oBAAoB;AAAA,MACzB,KAAK,oBAAoB,aAAa;AAAA,MACtC,KAAK,oBAAoB,aAAa;AAAA,IACxC;AAEA,SAAK,IAAI,SAAS;AAClB,SAAK,IAAI,2BAA2B;AACpC,SAAK,IAAI,UAAU,QAAQ,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AACtE,SAAK,IAAI,SAAS;AAClB,SAAK,IAAI,2BAA2B;AACpC,SAAK,IAAI,UAAU,OAAO,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AACrE,SAAK,IAAI,2BAA2B;AACpC,SAAK,IAAI,SAAS,QAAQ,KAAK,UAAU;AACzC,SAAK,IAAI,UAAU,OAAO,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AACrE,SAAK,IAAI,QAAQ;AAAA,EACnB;AACF;AAEA,SAAS,aACP,MACA,YACA,aACsB;AACtB,QAAM,YAA+B,IAAI,kBAAkB,aAAa,cAAc,CAAC;AACvF,QAAM,SAAS,KAAK,gBAAgB;AACpC,WAAS,IAAI,GAAG,IAAI,OAAO,QAAQ,KAAK,GAAG;AACzC,cAAU,IAAI,CAAC,IAAI,OAAO,CAAC;AAC3B,cAAU,IAAI,IAAI,CAAC,IAAI,OAAO,CAAC;AAC/B,cAAU,IAAI,IAAI,CAAC,IAAI,OAAO,CAAC;AAC/B,cAAU,IAAI,IAAI,CAAC,IAAI,OAAO,CAAC;AAAA,EACjC;AACA,QAAM,UAAU,IAAI,UAAU,WAAW,YAAY,WAAW;AAEhE,SAAO,kBAAkB,OAAO;AAClC;;;AGnMA,IAAqB,mBAArB,cAA8C,iBAAiB;AAAA,EAC7D,MAAM,UAAU,OAAmB,YAA0D;AAC3F,eAAW,QAAQ,KAAK;AAAA,EAC1B;AAAA,EAEA,MAAM,UAAU;AAAA,EAEhB;AACF;;;ACJO,IAAM,iBAAiB,CAC5B,aAAqB,IACrB,qBACG;AACH,QAAM,sBAAsB,kBAAkB,eAAe,oBAAsB;AACnF,MAAI,CAAC,qBAAqB;AACxB,UAAM,IAAI,MAAM,2CAA2C;AAAA,EAC7D;AACA,QAAM,WAAW,IAAI;AAAA,IACnB,CAAC,IAAI,oBAAsB,EAAE,YAAY,iBAAiB,CAAC,CAAC;AAAA,IAC5D;AAAA,EACF;AACA,SAAO;AACT;AAEO,IAAM,oBAAoB,CAAC,WAAmB,qBAA4C;AAC/F,QAAM,sBAAsB,kBAAkB,eAAe,oBAAsB;AACnF,MAAI,CAAC,qBAAqB;AACxB,UAAM,IAAI,MAAM,2CAA2C;AAAA,EAC7D;AACA,QAAM,WAAW,IAAI;AAAA,IACnB,CAAC,IAAI,oBAAsB,EAAE,WAAW,iBAAiB,CAAC,CAAC;AAAA,IAC3D;AAAA,EACF;AACA,SAAO;AACT;AAEO,IAAM,QAAQ,MAAM;AACzB,QAAM,sBAAsB,kBAAkB,eAAe,oBAAsB;AACnF,MAAI,CAAC,qBAAqB;AACxB,UAAM,IAAI,MAAM,2CAA2C;AAAA,EAC7D;AACA,QAAM,WAAW,IAAI,kBAAkB,CAAC,IAAI,iBAAiB,CAAC,GAAG,OAAO;AACxE,SAAO;AACT;","names":[]}
package/dist/src/index.d.ts CHANGED
@@ -1,4 +1,6 @@
1
1
  import ProcessorPipeline from './ProcessorPipeline';
2
- export declare const BackgroundBlur: (blurRadius?: number) => ProcessorPipeline;
3
- export declare const VirtualBackground: (imagePath: string) => ProcessorPipeline;
2
+ import { SegmenterBaseOptions } from './transformers/BackgroundTransformer';
3
+ export * from './transformers/types';
4
+ export declare const BackgroundBlur: (blurRadius?: number, segmenterOptions?: SegmenterBaseOptions) => ProcessorPipeline;
5
+ export declare const VirtualBackground: (imagePath: string, segmenterOptions?: SegmenterBaseOptions) => ProcessorPipeline;
4
6
  export declare const Dummy: () => ProcessorPipeline;
package/dist/src/transformers/BackgroundTransformer.d.ts CHANGED
@@ -2,9 +2,11 @@
2
2
  import * as vision from '@mediapipe/tasks-vision';
3
3
  import VideoTransformer from './VideoTransformer';
4
4
  import { VideoTransformerInitOptions } from './types';
5
+ export type SegmenterBaseOptions = Partial<vision.ImageSegmenterOptions['baseOptions']>;
5
6
  export type BackgroundOptions = {
6
7
  blurRadius?: number;
7
8
  imagePath?: string;
9
+ segmenterOptions?: SegmenterBaseOptions;
8
10
  };
9
11
  export default class BackgroundProcessor extends VideoTransformer {
10
12
  static get isSupported(): boolean;
@@ -12,6 +14,7 @@ export default class BackgroundProcessor extends VideoTransformer {
12
14
  segmentationResults: vision.ImageSegmenterResult | undefined;
13
15
  backgroundImage: ImageBitmap | null;
14
16
  blurRadius?: number;
17
+ options: BackgroundOptions;
15
18
  constructor(opts: BackgroundOptions);
16
19
  init({ outputCanvas, inputElement: inputVideo }: VideoTransformerInitOptions): Promise<void>;
17
20
  destroy(): Promise<void>;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@livekit/track-processors",
3
- "version": "0.2.1",
3
+ "version": "0.2.5",
4
4
  "description": "LiveKit track processors",
5
5
  "main": "dist/index.js",
6
6
  "module": "dist/index.mjs",
@@ -12,10 +12,11 @@
12
12
  "scripts": {
13
13
  "build": "tsup --onSuccess \"tsc --declaration --emitDeclarationOnly\"",
14
14
  "build-docs": "yarn exec typedoc",
15
- "build-sample": "cd example && webpack && cp styles.css index.html dist/",
15
+ "build-sample": "cd example && vite build",
16
16
  "lint": "eslint src",
17
17
  "test": "jest",
18
- "sample": "vite serve example --port 8080 --open"
18
+ "sample": "vite serve example --port 8080 --open",
19
+ "prepublish": "yarn build"
19
20
  },
20
21
  "files": [
21
22
  "dist",
@@ -41,7 +42,7 @@
41
42
  "eslint-plugin-ecmascript-compat": "^3.0.0",
42
43
  "eslint-plugin-import": "2.27.5",
43
44
  "jest": "^27.4.3",
44
- "livekit-client": "^0.16.6",
45
+ "livekit-client": "^1.12.0",
45
46
  "prettier": "^2.8.8",
46
47
  "ts-jest": "^27.0.7",
47
48
  "ts-loader": "^8.1.0",
package/src/ProcessorPipeline.ts CHANGED
@@ -46,8 +46,8 @@ export default class ProcessorPipeline implements TrackProcessor<Track.Kind> {
46
46
  this.sourceSettings = this.source.getSettings();
47
47
  this.sourceDummy = opts.element;
48
48
  if (this.sourceDummy instanceof HTMLVideoElement) {
49
- this.sourceDummy.height = this.sourceSettings.height!;
50
- this.sourceDummy.width = this.sourceSettings.width!;
49
+ this.sourceDummy.height = this.sourceSettings.height ?? 300;
50
+ this.sourceDummy.width = this.sourceSettings.width ?? 300;
51
51
  }
52
52
  if (!(this.sourceDummy instanceof HTMLVideoElement)) {
53
53
  throw TypeError('Currently only video transformers are supported');
package/src/index.ts CHANGED
@@ -1,26 +1,31 @@
1
1
  import ProcessorPipeline from './ProcessorPipeline';
2
- import BackgroundTransformer from './transformers/BackgroundTransformer';
2
+ import BackgroundTransformer, { SegmenterBaseOptions } from './transformers/BackgroundTransformer';
3
3
  import DummyTransformer from './transformers/DummyTransformer';
4
4
 
5
- export const BackgroundBlur = (blurRadius: number = 10) => {
5
+ export * from './transformers/types';
6
+
7
+ export const BackgroundBlur = (
8
+ blurRadius: number = 10,
9
+ segmenterOptions?: SegmenterBaseOptions,
10
+ ) => {
6
11
  const isPipelineSupported = ProcessorPipeline.isSupported && BackgroundTransformer.isSupported;
7
12
  if (!isPipelineSupported) {
8
13
  throw new Error('pipeline is not supported in this browser');
9
14
  }
10
15
  const pipeline = new ProcessorPipeline(
11
- [new BackgroundTransformer({ blurRadius })],
16
+ [new BackgroundTransformer({ blurRadius, segmenterOptions })],
12
17
  'background-blur',
13
18
  );
14
19
  return pipeline;
15
20
  };
16
21
 
17
- export const VirtualBackground = (imagePath: string) => {
22
+ export const VirtualBackground = (imagePath: string, segmenterOptions?: SegmenterBaseOptions) => {
18
23
  const isPipelineSupported = ProcessorPipeline.isSupported && BackgroundTransformer.isSupported;
19
24
  if (!isPipelineSupported) {
20
25
  throw new Error('pipeline is not supported in this browser');
21
26
  }
22
27
  const pipeline = new ProcessorPipeline(
23
- [new BackgroundTransformer({ imagePath })],
28
+ [new BackgroundTransformer({ imagePath, segmenterOptions })],
24
29
  'virtual-background',
25
30
  );
26
31
  return pipeline;
package/src/transformers/BackgroundTransformer.ts CHANGED
@@ -3,9 +3,12 @@ import { dependencies } from '../../package.json';
3
3
  import VideoTransformer from './VideoTransformer';
4
4
  import { VideoTransformerInitOptions } from './types';
5
5
 
6
+ export type SegmenterBaseOptions = Partial<vision.ImageSegmenterOptions['baseOptions']>;
7
+
6
8
  export type BackgroundOptions = {
7
9
  blurRadius?: number;
8
10
  imagePath?: string;
11
+ segmenterOptions?: SegmenterBaseOptions;
9
12
  };
10
13
 
11
14
  export default class BackgroundProcessor extends VideoTransformer {
@@ -21,13 +24,18 @@ export default class BackgroundProcessor extends VideoTransformer {
21
24
 
22
25
  blurRadius?: number;
23
26
 
27
+ options: BackgroundOptions;
28
+
24
29
  constructor(opts: BackgroundOptions) {
25
30
  super();
31
+ this.options = opts;
26
32
  if (opts.blurRadius) {
27
33
  this.blurRadius = opts.blurRadius;
28
34
  } else if (opts.imagePath) {
29
35
  this.loadBackground(opts.imagePath);
30
36
  }
37
+
38
+ this.options.segmenterOptions;
31
39
  }
32
40
 
33
41
  async init({ outputCanvas, inputElement: inputVideo }: VideoTransformerInitOptions) {
@@ -41,7 +49,8 @@ export default class BackgroundProcessor extends VideoTransformer {
41
49
  baseOptions: {
42
50
  modelAssetPath:
43
51
  'https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_segmenter/float16/latest/selfie_segmenter.tflite',
44
- delegate: 'CPU',
52
+ delegate: 'GPU',
53
+ ...this.options.segmenterOptions,
45
54
  },
46
55
  runningMode: 'VIDEO',
47
56
  outputCategoryMask: true,
@@ -109,10 +118,10 @@ export default class BackgroundProcessor extends VideoTransformer {
109
118
  this.ctx.globalCompositeOperation = 'copy';
110
119
  const bitmap = await maskToBitmap(
111
120
  this.segmentationResults.categoryMask,
112
- this.inputVideo.videoWidth,
113
- this.inputVideo.videoHeight,
121
+ this.segmentationResults.categoryMask.width,
122
+ this.segmentationResults.categoryMask.height,
114
123
  );
115
- this.ctx.drawImage(bitmap, 0, 0);
124
+ this.ctx.drawImage(bitmap, 0, 0, this.canvas.width, this.canvas.height);
116
125
  this.ctx.filter = 'none';
117
126
  this.ctx.globalCompositeOperation = 'source-in';
118
127
  if (this.backgroundImage) {
@@ -153,19 +162,19 @@ export default class BackgroundProcessor extends VideoTransformer {
153
162
 
154
163
  const bitmap = await maskToBitmap(
155
164
  this.segmentationResults.categoryMask,
156
- this.inputVideo.videoWidth,
157
- this.inputVideo.videoHeight,
165
+ this.segmentationResults.categoryMask.width,
166
+ this.segmentationResults.categoryMask.height,
158
167
  );
159
168
 
160
169
  this.ctx.filter = 'blur(3px)';
161
170
  this.ctx.globalCompositeOperation = 'copy';
162
- this.ctx.drawImage(bitmap, 0, 0);
171
+ this.ctx.drawImage(bitmap, 0, 0, this.canvas.width, this.canvas.height);
163
172
  this.ctx.filter = 'none';
164
173
  this.ctx.globalCompositeOperation = 'source-out';
165
174
  this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);
166
175
  this.ctx.globalCompositeOperation = 'destination-over';
167
176
  this.ctx.filter = `blur(${this.blurRadius}px)`;
168
- this.ctx.drawImage(frame, 0, 0);
177
+ this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);
169
178
  this.ctx.restore();
170
179
  }
171
180
  }