@livekit/track-processors 0.2.4 → 0.2.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (14) hide show
  1. package/dist/index.js +30 -15
  2. package/dist/index.js.map +1 -1
  3. package/dist/index.mjs +30 -15
  4. package/dist/index.mjs.map +1 -1
  5. package/dist/src/ProcessorPipeline.d.ts +2 -0
  6. package/dist/src/index.d.ts +1 -0
  7. package/dist/src/transformers/VideoTransformer.d.ts +1 -0
  8. package/dist/src/transformers/types.d.ts +1 -0
  9. package/package.json +8 -18
  10. package/src/ProcessorPipeline.ts +16 -4
  11. package/src/index.ts +2 -0
  12. package/src/transformers/BackgroundTransformer.ts +8 -10
  13. package/src/transformers/VideoTransformer.ts +8 -0
  14. package/src/transformers/types.ts +1 -0
package/dist/index.js CHANGED
@@ -27,8 +27,8 @@ var ProcessorPipeline = class {
27
27
  this.name = name;
28
28
  this.transformers = transformers;
29
29
  }
30
- async init(opts) {
31
- var _a, _b;
30
+ async setup(opts) {
31
+ var _a, _b, _c, _d;
32
32
  this.source = opts.track;
33
33
  const origConstraints = this.source.getConstraints();
34
34
  await this.source.applyConstraints(__spreadProps(__spreadValues({}, origConstraints), {
@@ -41,8 +41,8 @@ var ProcessorPipeline = class {
41
41
  this.sourceSettings = this.source.getSettings();
42
42
  this.sourceDummy = opts.element;
43
43
  if (this.sourceDummy instanceof HTMLVideoElement) {
44
- this.sourceDummy.height = this.sourceSettings.height;
45
- this.sourceDummy.width = this.sourceSettings.width;
44
+ this.sourceDummy.height = (_a = this.sourceSettings.height) != null ? _a : 300;
45
+ this.sourceDummy.width = (_b = this.sourceSettings.width) != null ? _b : 300;
46
46
  }
47
47
  if (!(this.sourceDummy instanceof HTMLVideoElement)) {
48
48
  throw TypeError("Currently only video transformers are supported");
@@ -53,9 +53,15 @@ var ProcessorPipeline = class {
53
53
  signalTarget: this.source
54
54
  });
55
55
  this.canvas = new OffscreenCanvas(
56
- (_a = this.sourceSettings.width) != null ? _a : 300,
57
- (_b = this.sourceSettings.height) != null ? _b : 300
56
+ (_c = this.sourceSettings.width) != null ? _c : 300,
57
+ (_d = this.sourceSettings.height) != null ? _d : 300
58
58
  );
59
+ }
60
+ async init(opts) {
61
+ await this.setup(opts);
62
+ if (!this.canvas || !this.processor || !this.trackGenerator) {
63
+ throw new TypeError("Expected both canvas and processor to be defined after setup");
64
+ }
59
65
  let readableStream = this.processor.readable;
60
66
  for (const transformer of this.transformers) {
61
67
  await transformer.init({
@@ -67,6 +73,10 @@ var ProcessorPipeline = class {
67
73
  readableStream.pipeTo(this.trackGenerator.writable).catch((e) => console.error("error when trying to pipe", e)).finally(() => this.destroy());
68
74
  this.processedTrack = this.trackGenerator;
69
75
  }
76
+ async restart(opts) {
77
+ await this.destroy();
78
+ return this.init(opts);
79
+ }
70
80
  async destroy() {
71
81
  var _a;
72
82
  for (const transformer of this.transformers) {
@@ -108,6 +118,12 @@ var VideoTransformer = class {
108
118
  this.inputVideo = inputVideo;
109
119
  this.isDisabled = false;
110
120
  }
121
+ async restart({ outputCanvas, inputElement: inputVideo }) {
122
+ this.canvas = outputCanvas || null;
123
+ this.ctx = this.canvas.getContext("2d") || void 0;
124
+ this.inputVideo = inputVideo;
125
+ this.isDisabled = false;
126
+ }
111
127
  async destroy() {
112
128
  this.isDisabled = true;
113
129
  this.canvas = void 0;
@@ -126,7 +142,6 @@ var BackgroundProcessor = class extends VideoTransformer {
126
142
  } else if (opts.imagePath) {
127
143
  this.loadBackground(opts.imagePath);
128
144
  }
129
- this.options.segmenterOptions;
130
145
  }
131
146
  static get isSupported() {
132
147
  return typeof OffscreenCanvas !== "undefined";
@@ -197,14 +212,14 @@ var BackgroundProcessor = class extends VideoTransformer {
197
212
  if (!this.canvas || !this.ctx || !this.segmentationResults || !this.inputVideo)
198
213
  return;
199
214
  if ((_a = this.segmentationResults) == null ? void 0 : _a.categoryMask) {
200
- this.ctx.filter = "blur(3px)";
215
+ this.ctx.filter = "blur(10px)";
201
216
  this.ctx.globalCompositeOperation = "copy";
202
217
  const bitmap = await maskToBitmap(
203
218
  this.segmentationResults.categoryMask,
204
- this.inputVideo.videoWidth,
205
- this.inputVideo.videoHeight
219
+ this.segmentationResults.categoryMask.width,
220
+ this.segmentationResults.categoryMask.height
206
221
  );
207
- this.ctx.drawImage(bitmap, 0, 0);
222
+ this.ctx.drawImage(bitmap, 0, 0, this.canvas.width, this.canvas.height);
208
223
  this.ctx.filter = "none";
209
224
  this.ctx.globalCompositeOperation = "source-in";
210
225
  if (this.backgroundImage) {
@@ -236,18 +251,18 @@ var BackgroundProcessor = class extends VideoTransformer {
236
251
  this.ctx.globalCompositeOperation = "copy";
237
252
  const bitmap = await maskToBitmap(
238
253
  this.segmentationResults.categoryMask,
239
- this.inputVideo.videoWidth,
240
- this.inputVideo.videoHeight
254
+ this.segmentationResults.categoryMask.width,
255
+ this.segmentationResults.categoryMask.height
241
256
  );
242
257
  this.ctx.filter = "blur(3px)";
243
258
  this.ctx.globalCompositeOperation = "copy";
244
- this.ctx.drawImage(bitmap, 0, 0);
259
+ this.ctx.drawImage(bitmap, 0, 0, this.canvas.width, this.canvas.height);
245
260
  this.ctx.filter = "none";
246
261
  this.ctx.globalCompositeOperation = "source-out";
247
262
  this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);
248
263
  this.ctx.globalCompositeOperation = "destination-over";
249
264
  this.ctx.filter = `blur(${this.blurRadius}px)`;
250
- this.ctx.drawImage(frame, 0, 0);
265
+ this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);
251
266
  this.ctx.restore();
252
267
  }
253
268
  };
package/dist/index.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"sources":["../src/ProcessorPipeline.ts","../src/transformers/BackgroundTransformer.ts","../package.json","../src/transformers/VideoTransformer.ts","../src/transformers/DummyTransformer.ts","../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;AAGA,IAAqB,oBAArB,MAA6E;AAAA,EAC3E,WAAW,cAAc;AACvB,WACE,OAAO,8BAA8B,eACrC,OAAO,8BAA8B;AAAA,EAEzC;AAAA,EAoBA,YAAY,cAA4C,MAAc;AACpE,SAAK,OAAO;AACZ,SAAK,eAAe;AAAA,EACtB;AAAA,EAEA,MAAM,KAAK,MAAoC;AAlCjD;AAmCI,SAAK,SAAS,KAAK;AACnB,UAAM,kBAAkB,KAAK,OAAO,eAAe;AACnD,UAAM,KAAK,OAAO,iBAAiB,iCAC9B,kBAD8B;AAAA;AAAA;AAAA;AAAA;AAAA,MAMjC,YAAY;AAAA,IACd,EAAC;AACD,SAAK,iBAAiB,KAAK,OAAO,YAAY;AAC9C,SAAK,cAAc,KAAK;AACxB,QAAI,KAAK,uBAAuB,kBAAkB;AAChD,WAAK,YAAY,SAAS,KAAK,eAAe;AAC9C,WAAK,YAAY,QAAQ,KAAK,eAAe;AAAA,IAC/C;AACA,QAAI,EAAE,KAAK,uBAAuB,mBAAmB;AACnD,YAAM,UAAU,iDAAiD;AAAA,IACnE;AAEA,SAAK,YAAY,IAAI,0BAA0B,EAAE,OAAO,KAAK,OAAO,CAAC;AAErE,SAAK,iBAAiB,IAAI,0BAA0B;AAAA,MAClD,MAAM;AAAA,MACN,cAAc,KAAK;AAAA,IACrB,CAAC;AAED,SAAK,SAAS,IAAI;AAAA,OAChB,UAAK,eAAe,UAApB,YAA6B;AAAA,OAC7B,UAAK,eAAe,WAApB,YAA8B;AAAA,IAChC;AAEA,QAAI,iBAAiB,KAAK,UAAU;AACpC,eAAW,eAAe,KAAK,cAAc;AAC3C,YAAM,YAAY,KAAK;AAAA,QACrB,cAAc,KAAK;AAAA,QACnB,cAAc,KAAK;AAAA,MACrB,CAAC;AACD,uBAAiB,eAAe,YAAY,YAAa,WAAY;AAAA,IACvE;AACA,mBACG,OAAO,KAAK,eAAe,QAAQ,EACnC,MAAM,CAAC,MAAM,QAAQ,MAAM,6BAA6B,CAAC,CAAC,EAC1D,QAAQ,MAAM,KAAK,QAAQ,CAAC;AAC/B,SAAK,iBAAiB,KAAK;AAAA,EAC7B;AAAA,EAEA,MAAM,UAAU;AAlFlB;AAmFI,eAAW,eAAe,KAAK,cAAc;AAC3C,YAAM,YAAY,QAAQ;AAAA,IAC5B;AACA,eAAK,mBAAL,mBAAqB;AAAA,EACvB;AACF;;;ACxFA,YAAY,YAAY;;;ACwBtB,mBAAgB;AAAA,EACd,uBAAuB;AAAA,EACvB,2BAA2B;AAC7B;;;ACzBF,IAA8B,mBAA9B,MAAgF;AAAA,EAAhF;AASE,SAAU,aAAuB;AAAA;AAAA,EAEjC,MAAM,KAAK;AAAA,IACT;AAAA,IACA,cAAc;AAAA,EAChB,GAA+C;AAhBjD;AAiBI,QAAI,EAAE,sBAAsB,mBAAmB;AAC7C,YAAM,UAAU,qDAAqD;AAAA,IACvE;AACA,SAAK,cAAc,IAAI,gBAAgB;AAAA,MACrC,WAAW,CAAC,OAAO,eAAe,KAAK,UAAU,OAAO,UAAU;AAAA,IACpE,CAAC;AACD,SAAK,SAAS,gBAAgB;AAC9B,QAAI,cAAc;AAChB,WAAK,QAAM,UAAK,WAAL,mBAAa,WAAW,UAAS;AAAA,IAC9C;AACA,SAAK,aAAa;AAClB,SAAK,aAAa;AAAA,EACpB;AAAA,EAEA,MAAM,UAAU;AACd,SAAK,aAAa;AAClB,SAAK,SAAS;AACd,SAAK,MAAM;AAAA,EACb;AAMF;;;AF5BA,IAAqB,sBAArB,cAAiD,iBAAiB;AAAA,EAehE,YAAY,MAAyB;AACnC,UAAM;AAPR,2BAAsC;AAQpC,SAAK,UAAU;AACf,QAAI,KAAK,YAAY;AACnB,WAAK,aAAa,KAAK;AAAA,IACzB,WAAW,KAAK,WAAW;AACzB,WAAK,eAAe,KAAK,SAAS;AAAA,IACpC;AAEA,SAAK,QAAQ;AAAA,EACf;AAAA,EAxBA,WAAW,cAAc;AACvB,WAAO,OAAO,oBAAoB;AAAA,EACpC;AAAA,EAwBA,MAAM,KAAK,EAAE,cAAc,cAAc,WAAW,GAAgC;AAClF,UAAM,MAAM,KAAK,EAAE,cAAc,cAAc,WAAW,CAAC;AAE3D,UAAM,UAAU,MAAa,uBAAgB;AAAA,MAC3C,wDAAwD,aAAa,yBAAyB,CAAC;AAAA,IACjG;AAEA,SAAK,iBAAiB,MAAa,sBAAe,kBAAkB,SAAS;AAAA,MAC3E,aAAa;AAAA,QACX,gBACE;AAAA,QACF,UAAU;AAAA,SACP,KAAK,QAAQ;AAAA,MAElB,aAAa;AAAA,MACb,oBAAoB;AAAA,MACpB,uBAAuB;AAAA,IACzB,CAAC;AAAA,EAGH;AAAA,EAEA,MAAM,UAAU;AA9DlB;AA+DI,UAAM,MAAM,QAAQ;AACpB,YAAM,UAAK,mBAAL,mBAAqB;AAC3B,SAAK,kBAAkB;AAAA,EACzB;AAAA,EAEA,MAAM,eAAe,MAAc;AACjC,UAAM,MAAM,IAAI,MAAM;AAEtB,UAAM,IAAI,QAAQ,CAAC,SAAS,WAAW;AACrC,UAAI,cAAc;AAClB,UAAI,SAAS,MAAM,QAAQ,GAAG;AAC9B,UAAI,UAAU,CAAC,QAAQ,OAAO,GAAG;AACjC,UAAI,MAAM;AAAA,IACZ,CAAC;AACD,UAAM,YAAY,MAAM,kBAAkB,GAAG;AAC7C,SAAK,kBAAkB;AAAA,EACzB;AAAA,EAEA,MAAM,UAAU,OAAmB,YAA0D;AAjF/F;AAkFI,QAAI;AACF,UAAI,KAAK,YAAY;AACnB,mBAAW,QAAQ,KAAK;AACxB;AAAA,MACF;AACA,UAAI,CAAC,KAAK,QAAQ;AAChB,cAAM,UAAU,sCAAsC;AAAA,MACxD;AACA,UAAI,cAAc,YAAY,IAAI;AAClC,iBAAK,mBAAL,mBAAqB;AAAA,QACnB,KAAK;AAAA,QACL;AAAA,QACA,CAAC,WAAY,KAAK,sBAAsB;AAAA;AAG1C,UAAI,KAAK,YAAY;AACnB,cAAM,KAAK,eAAe,KAAK;AAAA,MACjC,OAAO;AACL,cAAM,KAAK,sBAAsB,KAAK;AAAA,MACxC;AACA,YAAM,WAAW,IAAI,WAAW,KAAK,QAAQ;AAAA,QAC3C,WAAW,MAAM,aAAa,KAAK,IAAI;AAAA,MACzC,CAAC;AACD,iBAAW,QAAQ,QAAQ;AAAA,IAC7B,UAAE;AACA,YAAM,MAAM;AAAA,IACd;AAAA,EACF;AAAA,EAEA,MAAM,sBAAsB,OAAmB;AA/GjD;AAgHI,QAAI,CAAC,KAAK,UAAU,CAAC,KAAK,OAAO,CAAC,KAAK,uBAAuB,CAAC,KAAK;AAAY;AAGhF,SAAI,UAAK,wBAAL,mBAA0B,cAAc;AAC1C,WAAK,IAAI,SAAS;AAClB,WAAK,IAAI,2BAA2B;AACpC,YAAM,SAAS,MAAM;AAAA,QACnB,KAAK,oBAAoB;AAAA,QACzB,KAAK,WAAW;AAAA,QAChB,KAAK,WAAW;AAAA,MAClB;AACA,WAAK,IAAI,UAAU,QAAQ,GAAG,CAAC;AAC/B,WAAK,IAAI,SAAS;AAClB,WAAK,IAAI,2BAA2B;AACpC,UAAI,KAAK,iBAAiB;AACxB,aAAK,IAAI;AAAA,UACP,KAAK;AAAA,UACL;AAAA,UACA;AAAA,UACA,KAAK,gBAAgB;AAAA,UACrB,KAAK,gBAAgB;AAAA,UACrB;AAAA,UACA;AAAA,UACA,KAAK,OAAO;AAAA,UACZ,KAAK,OAAO;AAAA,QACd;AAAA,MACF,OAAO;AACL,aAAK,IAAI,YAAY;AACrB,aAAK,IAAI,SAAS,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AAAA,MAC/D;AAEA,WAAK,IAAI,2BAA2B;AAAA,IACtC;AACA,SAAK,IAAI,UAAU,OAAO,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AAAA,EAEvE;AAAA,EAEA,MAAM,eAAe,OAAmB;AArJ1C;AAsJI,QACE,CAAC,KAAK,OACN,CAAC,KAAK,UACN,GAAC,gBAAK,wBAAL,mBAA0B,iBAA1B,mBAAwC,WACzC,CAAC,KAAK,YACN;AACA;AAAA,IACF;AAEA,SAAK,IAAI,KAAK;AACd,SAAK,IAAI,2BAA2B;AAEpC,UAAM,SAAS,MAAM;AAAA,MACnB,KAAK,oBAAoB;AAAA,MACzB,KAAK,WAAW;AAAA,MAChB,KAAK,WAAW;AAAA,IAClB;AAEA,SAAK,IAAI,SAAS;AAClB,SAAK,IAAI,2BAA2B;AACpC,SAAK,IAAI,UAAU,QAAQ,GAAG,CAAC;AAC/B,SAAK,IAAI,SAAS;AAClB,SAAK,IAAI,2BAA2B;AACpC,SAAK,IAAI,UAAU,OAAO,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AACrE,SAAK,IAAI,2BAA2B;AACpC,SAAK,IAAI,SAAS,QAAQ,KAAK,UAAU;AACzC,SAAK,IAAI,UAAU,OAAO,GAAG,CAAC;AAC9B,SAAK,IAAI,QAAQ;AAAA,EACnB;AACF;AAEA,SAAS,aACP,MACA,YACA,aACsB;AACtB,QAAM,YAA+B,IAAI,kBAAkB,aAAa,cAAc,CAAC;AACvF,QAAM,SAAS,KAAK,gBAAgB;AACpC,WAAS,IAAI,GAAG,IAAI,OAAO,QAAQ,KAAK,GAAG;AACzC,cAAU,IAAI,CAAC,IAAI,OAAO,CAAC;AAC3B,cAAU,IAAI,IAAI,CAAC,IAAI,OAAO,CAAC;AAC/B,cAAU,IAAI,IAAI,CAAC,IAAI,OAAO,CAAC;AAC/B,cAAU,IAAI,IAAI,CAAC,IAAI,OAAO,CAAC;AAAA,EACjC;AACA,QAAM,UAAU,IAAI,UAAU,WAAW,YAAY,WAAW;AAEhE,SAAO,kBAAkB,OAAO;AAClC;;;AGnMA,IAAqB,mBAArB,cAA8C,iBAAiB;AAAA,EAC7D,MAAM,UAAU,OAAmB,YAA0D;AAC3F,eAAW,QAAQ,KAAK;AAAA,EAC1B;AAAA,EAEA,MAAM,UAAU;AAAA,EAEhB;AACF;;;ACNO,IAAM,iBAAiB,CAC5B,aAAqB,IACrB,qBACG;AACH,QAAM,sBAAsB,kBAAkB,eAAe,oBAAsB;AACnF,MAAI,CAAC,qBAAqB;AACxB,UAAM,IAAI,MAAM,2CAA2C;AAAA,EAC7D;AACA,QAAM,WAAW,IAAI;AAAA,IACnB,CAAC,IAAI,oBAAsB,EAAE,YAAY,iBAAiB,CAAC,CAAC;AAAA,IAC5D;AAAA,EACF;AACA,SAAO;AACT;AAEO,IAAM,oBAAoB,CAAC,WAAmB,qBAA4C;AAC/F,QAAM,sBAAsB,kBAAkB,eAAe,oBAAsB;AACnF,MAAI,CAAC,qBAAqB;AACxB,UAAM,IAAI,MAAM,2CAA2C;AAAA,EAC7D;AACA,QAAM,WAAW,IAAI;AAAA,IACnB,CAAC,IAAI,oBAAsB,EAAE,WAAW,iBAAiB,CAAC,CAAC;AAAA,IAC3D;AAAA,EACF;AACA,SAAO;AACT;AAEO,IAAM,QAAQ,MAAM;AACzB,QAAM,sBAAsB,kBAAkB,eAAe,oBAAsB;AACnF,MAAI,CAAC,qBAAqB;AACxB,UAAM,IAAI,MAAM,2CAA2C;AAAA,EAC7D;AACA,QAAM,WAAW,IAAI,kBAAkB,CAAC,IAAI,iBAAiB,CAAC,GAAG,OAAO;AACxE,SAAO;AACT","sourcesContent":["import type { ProcessorOptions, Track, TrackProcessor } from 'livekit-client';\nimport { VideoTrackTransformer } from './transformers';\n\nexport default class ProcessorPipeline implements TrackProcessor<Track.Kind> {\n static get isSupported() {\n return (\n typeof MediaStreamTrackGenerator !== 'undefined' &&\n typeof MediaStreamTrackProcessor !== 'undefined'\n );\n }\n\n name: string;\n\n source?: MediaStreamVideoTrack;\n\n sourceSettings?: MediaTrackSettings;\n\n processor?: MediaStreamTrackProcessor<VideoFrame>;\n\n trackGenerator?: MediaStreamTrackGenerator<VideoFrame>;\n\n canvas?: OffscreenCanvas;\n\n sourceDummy?: HTMLMediaElement;\n\n processedTrack?: MediaStreamTrack;\n\n transformers: Array<VideoTrackTransformer>;\n\n constructor(transformers: Array<VideoTrackTransformer>, name: string) {\n this.name = name;\n this.transformers = transformers;\n }\n\n async init(opts: ProcessorOptions<Track.Kind>) {\n this.source = opts.track as MediaStreamVideoTrack;\n const origConstraints = this.source.getConstraints();\n await this.source.applyConstraints({\n ...origConstraints,\n // @ts-expect-error when a mediastream track is resized and/or cropped, the `VideoFrame` will have a coded height/width of the original video size\n // this leads to a shift of the underlying video as the frame itself is being rendered with the coded size\n // but image segmentation is based on the display dimensions (-> the cropped version)\n // in order to prevent this, we force the resize mode to \"none\"\n resizeMode: 'none',\n });\n this.sourceSettings = this.source.getSettings();\n this.sourceDummy = opts.element;\n if (this.sourceDummy instanceof HTMLVideoElement) {\n this.sourceDummy.height = this.sourceSettings.height!;\n this.sourceDummy.width = this.sourceSettings.width!;\n }\n if (!(this.sourceDummy instanceof HTMLVideoElement)) {\n throw TypeError('Currently only video transformers are supported');\n }\n // TODO explore if we can do all the processing work in a webworker\n this.processor = new MediaStreamTrackProcessor({ track: this.source });\n\n this.trackGenerator = new MediaStreamTrackGenerator({\n kind: 'video',\n signalTarget: this.source,\n });\n\n this.canvas = new OffscreenCanvas(\n this.sourceSettings.width ?? 300,\n this.sourceSettings.height ?? 300,\n );\n\n let readableStream = this.processor.readable;\n for (const transformer of this.transformers) {\n await transformer.init({\n outputCanvas: this.canvas,\n inputElement: this.sourceDummy!,\n });\n readableStream = readableStream.pipeThrough(transformer!.transformer!);\n }\n readableStream\n .pipeTo(this.trackGenerator.writable)\n .catch((e) => console.error('error when trying to pipe', e))\n .finally(() => this.destroy());\n this.processedTrack = this.trackGenerator as MediaStreamVideoTrack;\n }\n\n async destroy() {\n for (const transformer of this.transformers) {\n await transformer.destroy();\n }\n this.trackGenerator?.stop();\n }\n}\n","import * as vision from '@mediapipe/tasks-vision';\nimport { dependencies } from '../../package.json';\nimport VideoTransformer from './VideoTransformer';\nimport { VideoTransformerInitOptions } from './types';\n\nexport type SegmenterBaseOptions = Partial<vision.ImageSegmenterOptions['baseOptions']>;\n\nexport type BackgroundOptions = {\n blurRadius?: number;\n imagePath?: string;\n segmenterOptions?: SegmenterBaseOptions;\n};\n\nexport default class BackgroundProcessor extends VideoTransformer {\n static get isSupported() {\n return typeof OffscreenCanvas !== 'undefined';\n }\n\n imageSegmenter?: vision.ImageSegmenter;\n\n segmentationResults: vision.ImageSegmenterResult | undefined;\n\n backgroundImage: ImageBitmap | null = null;\n\n blurRadius?: number;\n\n options: BackgroundOptions;\n\n constructor(opts: BackgroundOptions) {\n super();\n this.options = opts;\n if (opts.blurRadius) {\n this.blurRadius = opts.blurRadius;\n } else if (opts.imagePath) {\n this.loadBackground(opts.imagePath);\n }\n\n this.options.segmenterOptions;\n }\n\n async init({ outputCanvas, inputElement: inputVideo }: VideoTransformerInitOptions) {\n await super.init({ outputCanvas, inputElement: inputVideo });\n\n const fileSet = await vision.FilesetResolver.forVisionTasks(\n `https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@${dependencies['@mediapipe/tasks-vision']}/wasm`,\n );\n\n this.imageSegmenter = await vision.ImageSegmenter.createFromOptions(fileSet, {\n baseOptions: {\n modelAssetPath:\n 'https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_segmenter/float16/latest/selfie_segmenter.tflite',\n delegate: 'GPU',\n ...this.options.segmenterOptions,\n },\n runningMode: 'VIDEO',\n outputCategoryMask: true,\n outputConfidenceMasks: false,\n });\n\n // this.loadBackground(opts.backgroundUrl).catch((e) => console.error(e));\n }\n\n async destroy() {\n await super.destroy();\n await this.imageSegmenter?.close();\n this.backgroundImage = null;\n }\n\n async loadBackground(path: string) {\n const img = new Image();\n\n await new Promise((resolve, reject) => {\n img.crossOrigin = 'Anonymous';\n img.onload = () => resolve(img);\n img.onerror = (err) => reject(err);\n img.src = path;\n });\n const imageData = await createImageBitmap(img);\n this.backgroundImage = imageData;\n }\n\n async transform(frame: VideoFrame, controller: TransformStreamDefaultController<VideoFrame>) {\n try {\n if (this.isDisabled) {\n controller.enqueue(frame);\n return;\n }\n if (!this.canvas) {\n throw TypeError('Canvas needs to be initialized first');\n }\n let startTimeMs = performance.now();\n this.imageSegmenter?.segmentForVideo(\n this.inputVideo!,\n startTimeMs,\n (result) => (this.segmentationResults = result),\n );\n\n if (this.blurRadius) {\n await this.blurBackground(frame);\n } else {\n await this.drawVirtualBackground(frame);\n }\n const newFrame = new VideoFrame(this.canvas, {\n timestamp: frame.timestamp || Date.now(),\n });\n controller.enqueue(newFrame);\n } finally {\n frame.close();\n }\n }\n\n async drawVirtualBackground(frame: VideoFrame) {\n if (!this.canvas || !this.ctx || !this.segmentationResults || !this.inputVideo) return;\n // this.ctx.save();\n // this.ctx.clearRect(0, 0, this.canvas.width, this.canvas.height);\n if (this.segmentationResults?.categoryMask) {\n this.ctx.filter = 'blur(3px)';\n this.ctx.globalCompositeOperation = 'copy';\n const bitmap = await maskToBitmap(\n this.segmentationResults.categoryMask,\n this.inputVideo.videoWidth,\n this.inputVideo.videoHeight,\n );\n this.ctx.drawImage(bitmap, 0, 0);\n this.ctx.filter = 'none';\n this.ctx.globalCompositeOperation = 'source-in';\n if (this.backgroundImage) {\n this.ctx.drawImage(\n this.backgroundImage,\n 0,\n 0,\n this.backgroundImage.width,\n this.backgroundImage.height,\n 0,\n 0,\n this.canvas.width,\n this.canvas.height,\n );\n } else {\n this.ctx.fillStyle = '#00FF00';\n this.ctx.fillRect(0, 0, this.canvas.width, this.canvas.height);\n }\n\n this.ctx.globalCompositeOperation = 'destination-over';\n }\n this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);\n // this.ctx.restore();\n }\n\n async blurBackground(frame: VideoFrame) {\n if (\n !this.ctx ||\n !this.canvas ||\n !this.segmentationResults?.categoryMask?.canvas ||\n !this.inputVideo\n ) {\n return;\n }\n\n this.ctx.save();\n this.ctx.globalCompositeOperation = 'copy';\n\n const bitmap = await maskToBitmap(\n this.segmentationResults.categoryMask,\n this.inputVideo.videoWidth,\n this.inputVideo.videoHeight,\n );\n\n this.ctx.filter = 'blur(3px)';\n this.ctx.globalCompositeOperation = 'copy';\n this.ctx.drawImage(bitmap, 0, 0);\n this.ctx.filter = 'none';\n this.ctx.globalCompositeOperation = 'source-out';\n this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);\n this.ctx.globalCompositeOperation = 'destination-over';\n this.ctx.filter = `blur(${this.blurRadius}px)`;\n this.ctx.drawImage(frame, 0, 0);\n this.ctx.restore();\n }\n}\n\nfunction maskToBitmap(\n mask: vision.MPMask,\n videoWidth: number,\n videoHeight: number,\n): Promise<ImageBitmap> {\n const dataArray: Uint8ClampedArray = new Uint8ClampedArray(videoWidth * videoHeight * 4);\n const result = mask.getAsUint8Array();\n for (let i = 0; i < result.length; i += 1) {\n dataArray[i * 4] = result[i];\n dataArray[i * 4 + 1] = result[i];\n dataArray[i * 4 + 2] = result[i];\n dataArray[i * 4 + 3] = result[i];\n }\n const dataNew = new ImageData(dataArray, videoWidth, videoHeight);\n\n return createImageBitmap(dataNew);\n}\n","{\n \"name\": \"@livekit/track-processors\",\n \"version\": \"0.2.4\",\n \"description\": \"LiveKit track processors\",\n \"main\": \"dist/index.js\",\n \"module\": \"dist/index.mjs\",\n \"source\": \"src/index.ts\",\n \"types\": \"dist/src/index.d.ts\",\n \"repository\": \"git@github.com:livekit/livekit-track-processors.git\",\n \"author\": \"Lukas Seiler\",\n \"license\": \"Apache-2.0\",\n \"scripts\": {\n \"build\": \"tsup --onSuccess \\\"tsc --declaration --emitDeclarationOnly\\\"\",\n \"build-docs\": \"yarn exec typedoc\",\n \"build-sample\": \"cd example && webpack && cp styles.css index.html dist/\",\n \"lint\": \"eslint src\",\n \"test\": \"jest\",\n \"sample\": \"vite serve example --port 8080 --open\",\n \"prepublish\": \"yarn build\"\n },\n \"files\": [\n \"dist\",\n \"src\"\n ],\n \"dependencies\": {\n \"@mediapipe/holistic\": \"0.5.1675471629\",\n \"@mediapipe/tasks-vision\": \"0.10.1\"\n },\n \"peerDependencies\": {\n \"livekit-client\": \"^1.12.0\"\n },\n \"devDependencies\": {\n \"@trivago/prettier-plugin-sort-imports\": \"^4.1.1\",\n \"@types/dom-mediacapture-transform\": \"^0.1.5\",\n \"@types/jest\": \"^27.0.3\",\n \"@types/offscreencanvas\": \"^2019.7.0\",\n \"@typescript-eslint/eslint-plugin\": \"^4.31.2\",\n \"@webpack-cli/serve\": \"^1.5.2\",\n \"eslint\": \"8.39.0\",\n \"eslint-config-airbnb-typescript\": \"17.0.0\",\n \"eslint-config-prettier\": \"8.8.0\",\n \"eslint-plugin-ecmascript-compat\": \"^3.0.0\",\n \"eslint-plugin-import\": \"2.27.5\",\n \"jest\": \"^27.4.3\",\n \"livekit-client\": \"^0.16.6\",\n \"prettier\": \"^2.8.8\",\n \"ts-jest\": \"^27.0.7\",\n \"ts-loader\": \"^8.1.0\",\n \"ts-proto\": \"^1.85.0\",\n \"tsup\": \"^7.1.0\",\n \"typedoc\": \"^0.20.35\",\n \"typedoc-plugin-no-inherit\": \"1.3.0\",\n \"typescript\": \"^5.0.4\",\n \"vite\": \"^4.3.8\",\n \"webpack\": \"^5.53.0\",\n \"webpack-cli\": \"^4.8.0\",\n \"webpack-dev-server\": \"^4.2.1\"\n }\n}\n","import { VideoTrackTransformer, VideoTransformerInitOptions } from './types';\n\nexport default abstract class VideoTransformer implements VideoTrackTransformer {\n transformer?: TransformStream;\n\n canvas?: OffscreenCanvas;\n\n ctx?: OffscreenCanvasRenderingContext2D;\n\n inputVideo?: HTMLVideoElement;\n\n protected isDisabled?: Boolean = false;\n\n async init({\n outputCanvas,\n inputElement: inputVideo,\n }: VideoTransformerInitOptions): Promise<void> {\n if (!(inputVideo instanceof HTMLVideoElement)) {\n throw TypeError('Video transformer needs a HTMLVideoElement as input');\n }\n this.transformer = new TransformStream({\n transform: (frame, controller) => this.transform(frame, controller),\n });\n this.canvas = outputCanvas || null;\n if (outputCanvas) {\n this.ctx = this.canvas?.getContext('2d') || undefined;\n }\n this.inputVideo = inputVideo;\n this.isDisabled = false;\n }\n\n async destroy() {\n this.isDisabled = true;\n this.canvas = undefined;\n this.ctx = undefined;\n }\n\n abstract transform(\n frame: VideoFrame,\n controller: TransformStreamDefaultController<VideoFrame>,\n ): void;\n}\n","import VideoTransformer from './VideoTransformer';\n\nexport default class DummyTransformer extends VideoTransformer {\n async transform(frame: VideoFrame, controller: TransformStreamDefaultController<VideoFrame>) {\n controller.enqueue(frame);\n }\n\n async destroy() {\n // nothing to do\n }\n}\n","import ProcessorPipeline from './ProcessorPipeline';\nimport BackgroundTransformer, { SegmenterBaseOptions } from './transformers/BackgroundTransformer';\nimport DummyTransformer from './transformers/DummyTransformer';\n\nexport const BackgroundBlur = (\n blurRadius: number = 10,\n segmenterOptions?: SegmenterBaseOptions,\n) => {\n const isPipelineSupported = ProcessorPipeline.isSupported && BackgroundTransformer.isSupported;\n if (!isPipelineSupported) {\n throw new Error('pipeline is not supported in this browser');\n }\n const pipeline = new ProcessorPipeline(\n [new BackgroundTransformer({ blurRadius, segmenterOptions })],\n 'background-blur',\n );\n return pipeline;\n};\n\nexport const VirtualBackground = (imagePath: string, segmenterOptions?: SegmenterBaseOptions) => {\n const isPipelineSupported = ProcessorPipeline.isSupported && BackgroundTransformer.isSupported;\n if (!isPipelineSupported) {\n throw new Error('pipeline is not supported in this browser');\n }\n const pipeline = new ProcessorPipeline(\n [new BackgroundTransformer({ imagePath, segmenterOptions })],\n 'virtual-background',\n );\n return pipeline;\n};\n\nexport const Dummy = () => {\n const isPipelineSupported = ProcessorPipeline.isSupported && BackgroundTransformer.isSupported;\n if (!isPipelineSupported) {\n throw new Error('pipeline is not supported in this browser');\n }\n const pipeline = new ProcessorPipeline([new DummyTransformer()], 'dummy');\n return pipeline;\n};\n"]}
1
+ {"version":3,"sources":["../src/ProcessorPipeline.ts","../src/transformers/BackgroundTransformer.ts","../package.json","../src/transformers/VideoTransformer.ts","../src/transformers/DummyTransformer.ts","../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;AAGA,IAAqB,oBAArB,MAA6E;AAAA,EAC3E,WAAW,cAAc;AACvB,WACE,OAAO,8BAA8B,eACrC,OAAO,8BAA8B;AAAA,EAEzC;AAAA,EAoBA,YAAY,cAA4C,MAAc;AACpE,SAAK,OAAO;AACZ,SAAK,eAAe;AAAA,EACtB;AAAA,EAEA,MAAc,MAAM,MAAoC;AAlC1D;AAmCI,SAAK,SAAS,KAAK;AACnB,UAAM,kBAAkB,KAAK,OAAO,eAAe;AACnD,UAAM,KAAK,OAAO,iBAAiB,iCAC9B,kBAD8B;AAAA;AAAA;AAAA;AAAA;AAAA,MAMjC,YAAY;AAAA,IACd,EAAC;AACD,SAAK,iBAAiB,KAAK,OAAO,YAAY;AAC9C,SAAK,cAAc,KAAK;AACxB,QAAI,KAAK,uBAAuB,kBAAkB;AAChD,WAAK,YAAY,UAAS,UAAK,eAAe,WAApB,YAA8B;AACxD,WAAK,YAAY,SAAQ,UAAK,eAAe,UAApB,YAA6B;AAAA,IACxD;AACA,QAAI,EAAE,KAAK,uBAAuB,mBAAmB;AACnD,YAAM,UAAU,iDAAiD;AAAA,IACnE;AAEA,SAAK,YAAY,IAAI,0BAA0B,EAAE,OAAO,KAAK,OAAO,CAAC;AAErE,SAAK,iBAAiB,IAAI,0BAA0B;AAAA,MAClD,MAAM;AAAA,MACN,cAAc,KAAK;AAAA,IACrB,CAAC;AAED,SAAK,SAAS,IAAI;AAAA,OAChB,UAAK,eAAe,UAApB,YAA6B;AAAA,OAC7B,UAAK,eAAe,WAApB,YAA8B;AAAA,IAChC;AAAA,EACF;AAAA,EAEA,MAAM,KAAK,MAAoC;AAC7C,UAAM,KAAK,MAAM,IAAI;AACrB,QAAI,CAAC,KAAK,UAAU,CAAC,KAAK,aAAa,CAAC,KAAK,gBAAgB;AAC3D,YAAM,IAAI,UAAU,8DAA8D;AAAA,IACpF;AAEA,QAAI,iBAAiB,KAAK,UAAU;AACpC,eAAW,eAAe,KAAK,cAAc;AAC3C,YAAM,YAAY,KAAK;AAAA,QACrB,cAAc,KAAK;AAAA,QACnB,cAAc,KAAK;AAAA,MACrB,CAAC;AACD,uBAAiB,eAAe,YAAY,YAAa,WAAY;AAAA,IACvE;AACA,mBACG,OAAO,KAAK,eAAe,QAAQ,EACnC,MAAM,CAAC,MAAM,QAAQ,MAAM,6BAA6B,CAAC,CAAC,EAC1D,QAAQ,MAAM,KAAK,QAAQ,CAAC;AAC/B,SAAK,iBAAiB,KAAK;AAAA,EAC7B;AAAA,EAEA,MAAM,QAAQ,MAAoC;AAChD,UAAM,KAAK,QAAQ;AACnB,WAAO,KAAK,KAAK,IAAI;AAAA,EACvB;AAAA,EAEA,MAAM,UAAU;AA9FlB;AA+FI,eAAW,eAAe,KAAK,cAAc;AAC3C,YAAM,YAAY,QAAQ;AAAA,IAC5B;AACA,eAAK,mBAAL,mBAAqB;AAAA,EACvB;AACF;;;ACpGA,YAAY,YAAY;;;ACuBtB,mBAAgB;AAAA,EACd,uBAAuB;AAAA,EACvB,2BAA2B;AAC7B;;;ACxBF,IAA8B,mBAA9B,MAAgF;AAAA,EAAhF;AASE,SAAU,aAAuB;AAAA;AAAA,EAEjC,MAAM,KAAK;AAAA,IACT;AAAA,IACA,cAAc;AAAA,EAChB,GAA+C;AAhBjD;AAiBI,QAAI,EAAE,sBAAsB,mBAAmB;AAC7C,YAAM,UAAU,qDAAqD;AAAA,IACvE;AACA,SAAK,cAAc,IAAI,gBAAgB;AAAA,MACrC,WAAW,CAAC,OAAO,eAAe,KAAK,UAAU,OAAO,UAAU;AAAA,IACpE,CAAC;AACD,SAAK,SAAS,gBAAgB;AAC9B,QAAI,cAAc;AAChB,WAAK,QAAM,UAAK,WAAL,mBAAa,WAAW,UAAS;AAAA,IAC9C;AACA,SAAK,aAAa;AAClB,SAAK,aAAa;AAAA,EACpB;AAAA,EAEA,MAAM,QAAQ,EAAE,cAAc,cAAc,WAAW,GAAgC;AACrF,SAAK,SAAS,gBAAgB;AAC9B,SAAK,MAAM,KAAK,OAAO,WAAW,IAAI,KAAK;AAE3C,SAAK,aAAa;AAClB,SAAK,aAAa;AAAA,EACpB;AAAA,EAEA,MAAM,UAAU;AACd,SAAK,aAAa;AAClB,SAAK,SAAS;AACd,SAAK,MAAM;AAAA,EACb;AAMF;;;AFpCA,IAAqB,sBAArB,cAAiD,iBAAiB;AAAA,EAehE,YAAY,MAAyB;AACnC,UAAM;AAPR,2BAAsC;AAQpC,SAAK,UAAU;AACf,QAAI,KAAK,YAAY;AACnB,WAAK,aAAa,KAAK;AAAA,IACzB,WAAW,KAAK,WAAW;AACzB,WAAK,eAAe,KAAK,SAAS;AAAA,IACpC;AAAA,EACF;AAAA,EAtBA,WAAW,cAAc;AACvB,WAAO,OAAO,oBAAoB;AAAA,EACpC;AAAA,EAsBA,MAAM,KAAK,EAAE,cAAc,cAAc,WAAW,GAAgC;AAClF,UAAM,MAAM,KAAK,EAAE,cAAc,cAAc,WAAW,CAAC;AAE3D,UAAM,UAAU,MAAa,uBAAgB;AAAA,MAC3C,wDAAwD,aAAa,yBAAyB,CAAC;AAAA,IACjG;AAEA,SAAK,iBAAiB,MAAa,sBAAe,kBAAkB,SAAS;AAAA,MAC3E,aAAa;AAAA,QACX,gBACE;AAAA,QACF,UAAU;AAAA,SACP,KAAK,QAAQ;AAAA,MAElB,aAAa;AAAA,MACb,oBAAoB;AAAA,MACpB,uBAAuB;AAAA,IACzB,CAAC;AAAA,EAGH;AAAA,EAEA,MAAM,UAAU;AA5DlB;AA6DI,UAAM,MAAM,QAAQ;AACpB,YAAM,UAAK,mBAAL,mBAAqB;AAC3B,SAAK,kBAAkB;AAAA,EACzB;AAAA,EAEA,MAAM,eAAe,MAAc;AACjC,UAAM,MAAM,IAAI,MAAM;AAEtB,UAAM,IAAI,QAAQ,CAAC,SAAS,WAAW;AACrC,UAAI,cAAc;AAClB,UAAI,SAAS,MAAM,QAAQ,GAAG;AAC9B,UAAI,UAAU,CAAC,QAAQ,OAAO,GAAG;AACjC,UAAI,MAAM;AAAA,IACZ,CAAC;AACD,UAAM,YAAY,MAAM,kBAAkB,GAAG;AAC7C,SAAK,kBAAkB;AAAA,EACzB;AAAA,EAEA,MAAM,UAAU,OAAmB,YAA0D;AA/E/F;AAgFI,QAAI;AACF,UAAI,KAAK,YAAY;AACnB,mBAAW,QAAQ,KAAK;AACxB;AAAA,MACF;AACA,UAAI,CAAC,KAAK,QAAQ;AAChB,cAAM,UAAU,sCAAsC;AAAA,MACxD;AACA,UAAI,cAAc,YAAY,IAAI;AAClC,iBAAK,mBAAL,mBAAqB;AAAA,QACnB,KAAK;AAAA,QACL;AAAA,QACA,CAAC,WAAY,KAAK,sBAAsB;AAAA;AAG1C,UAAI,KAAK,YAAY;AACnB,cAAM,KAAK,eAAe,KAAK;AAAA,MACjC,OAAO;AACL,cAAM,KAAK,sBAAsB,KAAK;AAAA,MACxC;AACA,YAAM,WAAW,IAAI,WAAW,KAAK,QAAQ;AAAA,QAC3C,WAAW,MAAM,aAAa,KAAK,IAAI;AAAA,MACzC,CAAC;AACD,iBAAW,QAAQ,QAAQ;AAAA,IAC7B,UAAE;AACA,YAAM,MAAM;AAAA,IACd;AAAA,EACF;AAAA,EAEA,MAAM,sBAAsB,OAAmB;AA7GjD;AA8GI,QAAI,CAAC,KAAK,UAAU,CAAC,KAAK,OAAO,CAAC,KAAK,uBAAuB,CAAC,KAAK;AAAY;AAGhF,SAAI,UAAK,wBAAL,mBAA0B,cAAc;AAC1C,WAAK,IAAI,SAAS;AAClB,WAAK,IAAI,2BAA2B;AACpC,YAAM,SAAS,MAAM;AAAA,QACnB,KAAK,oBAAoB;AAAA,QACzB,KAAK,oBAAoB,aAAa;AAAA,QACtC,KAAK,oBAAoB,aAAa;AAAA,MACxC;AACA,WAAK,IAAI,UAAU,QAAQ,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AACtE,WAAK,IAAI,SAAS;AAClB,WAAK,IAAI,2BAA2B;AACpC,UAAI,KAAK,iBAAiB;AACxB,aAAK,IAAI;AAAA,UACP,KAAK;AAAA,UACL;AAAA,UACA;AAAA,UACA,KAAK,gBAAgB;AAAA,UACrB,KAAK,gBAAgB;AAAA,UACrB;AAAA,UACA;AAAA,UACA,KAAK,OAAO;AAAA,UACZ,KAAK,OAAO;AAAA,QACd;AAAA,MACF,OAAO;AACL,aAAK,IAAI,YAAY;AACrB,aAAK,IAAI,SAAS,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AAAA,MAC/D;AAEA,WAAK,IAAI,2BAA2B;AAAA,IACtC;AACA,SAAK,IAAI,UAAU,OAAO,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AAAA,EAEvE;AAAA,EAEA,MAAM,eAAe,OAAmB;AAnJ1C;AAoJI,QACE,CAAC,KAAK,OACN,CAAC,KAAK,UACN,GAAC,gBAAK,wBAAL,mBAA0B,iBAA1B,mBAAwC,WACzC,CAAC,KAAK,YACN;AACA;AAAA,IACF;AAEA,SAAK,IAAI,KAAK;AACd,SAAK,IAAI,2BAA2B;AAEpC,UAAM,SAAS,MAAM;AAAA,MACnB,KAAK,oBAAoB;AAAA,MACzB,KAAK,oBAAoB,aAAa;AAAA,MACtC,KAAK,oBAAoB,aAAa;AAAA,IACxC;AAEA,SAAK,IAAI,SAAS;AAClB,SAAK,IAAI,2BAA2B;AACpC,SAAK,IAAI,UAAU,QAAQ,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AACtE,SAAK,IAAI,SAAS;AAClB,SAAK,IAAI,2BAA2B;AACpC,SAAK,IAAI,UAAU,OAAO,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AACrE,SAAK,IAAI,2BAA2B;AACpC,SAAK,IAAI,SAAS,QAAQ,KAAK,UAAU;AACzC,SAAK,IAAI,UAAU,OAAO,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AACrE,SAAK,IAAI,QAAQ;AAAA,EACnB;AACF;AAEA,SAAS,aACP,MACA,YACA,aACsB;AACtB,QAAM,YAA+B,IAAI,kBAAkB,aAAa,cAAc,CAAC;AACvF,QAAM,SAAS,KAAK,gBAAgB;AACpC,WAAS,IAAI,GAAG,IAAI,OAAO,QAAQ,KAAK,GAAG;AACzC,cAAU,IAAI,CAAC,IAAI,OAAO,CAAC;AAC3B,cAAU,IAAI,IAAI,CAAC,IAAI,OAAO,CAAC;AAC/B,cAAU,IAAI,IAAI,CAAC,IAAI,OAAO,CAAC;AAC/B,cAAU,IAAI,IAAI,CAAC,IAAI,OAAO,CAAC;AAAA,EACjC;AACA,QAAM,UAAU,IAAI,UAAU,WAAW,YAAY,WAAW;AAEhE,SAAO,kBAAkB,OAAO;AAClC;;;AGjMA,IAAqB,mBAArB,cAA8C,iBAAiB;AAAA,EAC7D,MAAM,UAAU,OAAmB,YAA0D;AAC3F,eAAW,QAAQ,KAAK;AAAA,EAC1B;AAAA,EAEA,MAAM,UAAU;AAAA,EAEhB;AACF;;;ACJO,IAAM,iBAAiB,CAC5B,aAAqB,IACrB,qBACG;AACH,QAAM,sBAAsB,kBAAkB,eAAe,oBAAsB;AACnF,MAAI,CAAC,qBAAqB;AACxB,UAAM,IAAI,MAAM,2CAA2C;AAAA,EAC7D;AACA,QAAM,WAAW,IAAI;AAAA,IACnB,CAAC,IAAI,oBAAsB,EAAE,YAAY,iBAAiB,CAAC,CAAC;AAAA,IAC5D;AAAA,EACF;AACA,SAAO;AACT;AAEO,IAAM,oBAAoB,CAAC,WAAmB,qBAA4C;AAC/F,QAAM,sBAAsB,kBAAkB,eAAe,oBAAsB;AACnF,MAAI,CAAC,qBAAqB;AACxB,UAAM,IAAI,MAAM,2CAA2C;AAAA,EAC7D;AACA,QAAM,WAAW,IAAI;AAAA,IACnB,CAAC,IAAI,oBAAsB,EAAE,WAAW,iBAAiB,CAAC,CAAC;AAAA,IAC3D;AAAA,EACF;AACA,SAAO;AACT;AAEO,IAAM,QAAQ,MAAM;AACzB,QAAM,sBAAsB,kBAAkB,eAAe,oBAAsB;AACnF,MAAI,CAAC,qBAAqB;AACxB,UAAM,IAAI,MAAM,2CAA2C;AAAA,EAC7D;AACA,QAAM,WAAW,IAAI,kBAAkB,CAAC,IAAI,iBAAiB,CAAC,GAAG,OAAO;AACxE,SAAO;AACT","sourcesContent":["import type { ProcessorOptions, Track, TrackProcessor } from 'livekit-client';\nimport { VideoTrackTransformer } from './transformers';\n\nexport default class ProcessorPipeline implements TrackProcessor<Track.Kind> {\n static get isSupported() {\n return (\n typeof MediaStreamTrackGenerator !== 'undefined' &&\n typeof MediaStreamTrackProcessor !== 'undefined'\n );\n }\n\n name: string;\n\n source?: MediaStreamVideoTrack;\n\n sourceSettings?: MediaTrackSettings;\n\n processor?: MediaStreamTrackProcessor<VideoFrame>;\n\n trackGenerator?: MediaStreamTrackGenerator<VideoFrame>;\n\n canvas?: OffscreenCanvas;\n\n sourceDummy?: HTMLMediaElement;\n\n processedTrack?: MediaStreamTrack;\n\n transformers: Array<VideoTrackTransformer>;\n\n constructor(transformers: Array<VideoTrackTransformer>, name: string) {\n this.name = name;\n this.transformers = transformers;\n }\n\n private async setup(opts: ProcessorOptions<Track.Kind>) {\n this.source = opts.track as MediaStreamVideoTrack;\n const origConstraints = this.source.getConstraints();\n await this.source.applyConstraints({\n ...origConstraints,\n // @ts-expect-error when a mediastream track is resized and/or cropped, the `VideoFrame` will have a coded height/width of the original video size\n // this leads to a shift of the underlying video as the frame itself is being rendered with the coded size\n // but image segmentation is based on the display dimensions (-> the cropped version)\n // in order to prevent this, we force the resize mode to \"none\"\n resizeMode: 'none',\n });\n this.sourceSettings = this.source.getSettings();\n this.sourceDummy = opts.element;\n if (this.sourceDummy instanceof HTMLVideoElement) {\n this.sourceDummy.height = this.sourceSettings.height ?? 300;\n this.sourceDummy.width = this.sourceSettings.width ?? 300;\n }\n if (!(this.sourceDummy instanceof HTMLVideoElement)) {\n throw TypeError('Currently only video transformers are supported');\n }\n // TODO explore if we can do all the processing work in a webworker\n this.processor = new MediaStreamTrackProcessor({ track: this.source });\n\n this.trackGenerator = new MediaStreamTrackGenerator({\n kind: 'video',\n signalTarget: this.source,\n });\n\n this.canvas = new OffscreenCanvas(\n this.sourceSettings.width ?? 300,\n this.sourceSettings.height ?? 300,\n );\n }\n\n async init(opts: ProcessorOptions<Track.Kind>) {\n await this.setup(opts);\n if (!this.canvas || !this.processor || !this.trackGenerator) {\n throw new TypeError('Expected both canvas and processor to be defined after setup');\n }\n\n let readableStream = this.processor.readable;\n for (const transformer of this.transformers) {\n await transformer.init({\n outputCanvas: this.canvas,\n inputElement: this.sourceDummy as HTMLVideoElement,\n });\n readableStream = readableStream.pipeThrough(transformer!.transformer!);\n }\n readableStream\n .pipeTo(this.trackGenerator.writable)\n .catch((e) => console.error('error when trying to pipe', e))\n .finally(() => this.destroy());\n this.processedTrack = this.trackGenerator as MediaStreamVideoTrack;\n }\n\n async restart(opts: ProcessorOptions<Track.Kind>) {\n await this.destroy();\n return this.init(opts);\n }\n\n async destroy() {\n for (const transformer of this.transformers) {\n await transformer.destroy();\n }\n this.trackGenerator?.stop();\n }\n}\n","import * as vision from '@mediapipe/tasks-vision';\nimport { dependencies } from '../../package.json';\nimport VideoTransformer from './VideoTransformer';\nimport { VideoTransformerInitOptions } from './types';\n\nexport type SegmenterBaseOptions = Partial<vision.ImageSegmenterOptions['baseOptions']>;\n\nexport type BackgroundOptions = {\n blurRadius?: number;\n imagePath?: string;\n segmenterOptions?: SegmenterBaseOptions;\n};\n\nexport default class BackgroundProcessor extends VideoTransformer {\n static get isSupported() {\n return typeof OffscreenCanvas !== 'undefined';\n }\n\n imageSegmenter?: vision.ImageSegmenter;\n\n segmentationResults: vision.ImageSegmenterResult | undefined;\n\n backgroundImage: ImageBitmap | null = null;\n\n blurRadius?: number;\n\n options: BackgroundOptions;\n\n constructor(opts: BackgroundOptions) {\n super();\n this.options = opts;\n if (opts.blurRadius) {\n this.blurRadius = opts.blurRadius;\n } else if (opts.imagePath) {\n this.loadBackground(opts.imagePath);\n }\n }\n\n async init({ outputCanvas, inputElement: inputVideo }: VideoTransformerInitOptions) {\n await super.init({ outputCanvas, inputElement: inputVideo });\n\n const fileSet = await vision.FilesetResolver.forVisionTasks(\n `https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@${dependencies['@mediapipe/tasks-vision']}/wasm`,\n );\n\n this.imageSegmenter = await vision.ImageSegmenter.createFromOptions(fileSet, {\n baseOptions: {\n modelAssetPath:\n 'https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_segmenter/float16/latest/selfie_segmenter.tflite',\n delegate: 'GPU',\n ...this.options.segmenterOptions,\n },\n runningMode: 'VIDEO',\n outputCategoryMask: true,\n outputConfidenceMasks: false,\n });\n\n // this.loadBackground(opts.backgroundUrl).catch((e) => console.error(e));\n }\n\n async destroy() {\n await super.destroy();\n await this.imageSegmenter?.close();\n this.backgroundImage = null;\n }\n\n async loadBackground(path: string) {\n const img = new Image();\n\n await new Promise((resolve, reject) => {\n img.crossOrigin = 'Anonymous';\n img.onload = () => resolve(img);\n img.onerror = (err) => reject(err);\n img.src = path;\n });\n const imageData = await createImageBitmap(img);\n this.backgroundImage = imageData;\n }\n\n async transform(frame: VideoFrame, controller: TransformStreamDefaultController<VideoFrame>) {\n try {\n if (this.isDisabled) {\n controller.enqueue(frame);\n return;\n }\n if (!this.canvas) {\n throw TypeError('Canvas needs to be initialized first');\n }\n let startTimeMs = performance.now();\n this.imageSegmenter?.segmentForVideo(\n this.inputVideo!,\n startTimeMs,\n (result) => (this.segmentationResults = result),\n );\n\n if (this.blurRadius) {\n await this.blurBackground(frame);\n } else {\n await this.drawVirtualBackground(frame);\n }\n const newFrame = new VideoFrame(this.canvas, {\n timestamp: frame.timestamp || Date.now(),\n });\n controller.enqueue(newFrame);\n } finally {\n frame.close();\n }\n }\n\n async drawVirtualBackground(frame: VideoFrame) {\n if (!this.canvas || !this.ctx || !this.segmentationResults || !this.inputVideo) return;\n // this.ctx.save();\n // this.ctx.clearRect(0, 0, this.canvas.width, this.canvas.height);\n if (this.segmentationResults?.categoryMask) {\n this.ctx.filter = 'blur(10px)';\n this.ctx.globalCompositeOperation = 'copy';\n const bitmap = await maskToBitmap(\n this.segmentationResults.categoryMask,\n this.segmentationResults.categoryMask.width,\n this.segmentationResults.categoryMask.height,\n );\n this.ctx.drawImage(bitmap, 0, 0, this.canvas.width, this.canvas.height);\n this.ctx.filter = 'none';\n this.ctx.globalCompositeOperation = 'source-in';\n if (this.backgroundImage) {\n this.ctx.drawImage(\n this.backgroundImage,\n 0,\n 0,\n this.backgroundImage.width,\n this.backgroundImage.height,\n 0,\n 0,\n this.canvas.width,\n this.canvas.height,\n );\n } else {\n this.ctx.fillStyle = '#00FF00';\n this.ctx.fillRect(0, 0, this.canvas.width, this.canvas.height);\n }\n\n this.ctx.globalCompositeOperation = 'destination-over';\n }\n this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);\n // this.ctx.restore();\n }\n\n async blurBackground(frame: VideoFrame) {\n if (\n !this.ctx ||\n !this.canvas ||\n !this.segmentationResults?.categoryMask?.canvas ||\n !this.inputVideo\n ) {\n return;\n }\n\n this.ctx.save();\n this.ctx.globalCompositeOperation = 'copy';\n\n const bitmap = await maskToBitmap(\n this.segmentationResults.categoryMask,\n this.segmentationResults.categoryMask.width,\n this.segmentationResults.categoryMask.height,\n );\n\n this.ctx.filter = 'blur(3px)';\n this.ctx.globalCompositeOperation = 'copy';\n this.ctx.drawImage(bitmap, 0, 0, this.canvas.width, this.canvas.height);\n this.ctx.filter = 'none';\n this.ctx.globalCompositeOperation = 'source-out';\n this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);\n this.ctx.globalCompositeOperation = 'destination-over';\n this.ctx.filter = `blur(${this.blurRadius}px)`;\n this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);\n this.ctx.restore();\n }\n}\n\nfunction maskToBitmap(\n mask: vision.MPMask,\n videoWidth: number,\n videoHeight: number,\n): Promise<ImageBitmap> {\n const dataArray: Uint8ClampedArray = new Uint8ClampedArray(videoWidth * videoHeight * 4);\n const result = mask.getAsUint8Array();\n for (let i = 0; i < result.length; i += 1) {\n dataArray[i * 4] = result[i];\n dataArray[i * 4 + 1] = result[i];\n dataArray[i * 4 + 2] = result[i];\n dataArray[i * 4 + 3] = result[i];\n }\n const dataNew = new ImageData(dataArray, videoWidth, videoHeight);\n\n return createImageBitmap(dataNew);\n}\n","{\n \"name\": \"@livekit/track-processors\",\n \"version\": \"0.2.6\",\n \"description\": \"LiveKit track processors\",\n \"main\": \"dist/index.js\",\n \"module\": \"dist/index.mjs\",\n \"source\": \"src/index.ts\",\n \"types\": \"dist/src/index.d.ts\",\n \"repository\": \"git@github.com:livekit/livekit-track-processors.git\",\n \"author\": \"Lukas Seiler\",\n \"license\": \"Apache-2.0\",\n \"scripts\": {\n \"build\": \"tsup --onSuccess \\\"tsc --declaration --emitDeclarationOnly\\\"\",\n \"build-sample\": \"cd example && vite build\",\n \"lint\": \"eslint src\",\n \"release\": \"yarn build && changeset publish\",\n \"test\": \"jest\",\n \"sample\": \"vite serve example --port 8080 --open\"\n },\n \"files\": [\n \"dist\",\n \"src\"\n ],\n \"dependencies\": {\n \"@mediapipe/holistic\": \"0.5.1675471629\",\n \"@mediapipe/tasks-vision\": \"0.10.1\"\n },\n \"peerDependencies\": {\n \"livekit-client\": \"^1.12.0\"\n },\n \"devDependencies\": {\n \"@changesets/cli\": \"^2.26.2\",\n \"@livekit/changesets-changelog-github\": \"^0.0.4\",\n \"@trivago/prettier-plugin-sort-imports\": \"^4.1.1\",\n \"@types/dom-mediacapture-transform\": \"^0.1.5\",\n \"@types/offscreencanvas\": \"^2019.7.0\",\n \"@typescript-eslint/eslint-plugin\": \"^4.31.2\",\n \"eslint\": \"8.39.0\",\n \"eslint-config-airbnb-typescript\": \"17.0.0\",\n \"eslint-config-prettier\": \"8.8.0\",\n \"eslint-plugin-ecmascript-compat\": \"^3.0.0\",\n \"eslint-plugin-import\": \"2.27.5\",\n \"livekit-client\": \"^1.12.0\",\n \"prettier\": \"^2.8.8\",\n \"tsup\": \"^7.1.0\",\n \"typescript\": \"^5.0.4\",\n \"vite\": \"^4.3.8\"\n }\n}\n","import { VideoTrackTransformer, VideoTransformerInitOptions } from './types';\n\nexport default abstract class VideoTransformer implements VideoTrackTransformer {\n transformer?: TransformStream;\n\n canvas?: OffscreenCanvas;\n\n ctx?: OffscreenCanvasRenderingContext2D;\n\n inputVideo?: HTMLVideoElement;\n\n protected isDisabled?: Boolean = false;\n\n async init({\n outputCanvas,\n inputElement: inputVideo,\n }: VideoTransformerInitOptions): Promise<void> {\n if (!(inputVideo instanceof HTMLVideoElement)) {\n throw TypeError('Video transformer needs a HTMLVideoElement as input');\n }\n this.transformer = new TransformStream({\n transform: (frame, controller) => this.transform(frame, controller),\n });\n this.canvas = outputCanvas || null;\n if (outputCanvas) {\n this.ctx = this.canvas?.getContext('2d') || undefined;\n }\n this.inputVideo = inputVideo;\n this.isDisabled = false;\n }\n\n async restart({ outputCanvas, inputElement: inputVideo }: VideoTransformerInitOptions) {\n this.canvas = outputCanvas || null;\n this.ctx = this.canvas.getContext('2d') || undefined;\n\n this.inputVideo = inputVideo;\n this.isDisabled = false;\n }\n\n async destroy() {\n this.isDisabled = true;\n this.canvas = undefined;\n this.ctx = undefined;\n }\n\n abstract transform(\n frame: VideoFrame,\n controller: TransformStreamDefaultController<VideoFrame>,\n ): void;\n}\n","import VideoTransformer from './VideoTransformer';\n\nexport default class DummyTransformer extends VideoTransformer {\n async transform(frame: VideoFrame, controller: TransformStreamDefaultController<VideoFrame>) {\n controller.enqueue(frame);\n }\n\n async destroy() {\n // nothing to do\n }\n}\n","import ProcessorPipeline from './ProcessorPipeline';\nimport BackgroundTransformer, { SegmenterBaseOptions } from './transformers/BackgroundTransformer';\nimport DummyTransformer from './transformers/DummyTransformer';\n\nexport * from './transformers/types';\n\nexport const BackgroundBlur = (\n blurRadius: number = 10,\n segmenterOptions?: SegmenterBaseOptions,\n) => {\n const isPipelineSupported = ProcessorPipeline.isSupported && BackgroundTransformer.isSupported;\n if (!isPipelineSupported) {\n throw new Error('pipeline is not supported in this browser');\n }\n const pipeline = new ProcessorPipeline(\n [new BackgroundTransformer({ blurRadius, segmenterOptions })],\n 'background-blur',\n );\n return pipeline;\n};\n\nexport const VirtualBackground = (imagePath: string, segmenterOptions?: SegmenterBaseOptions) => {\n const isPipelineSupported = ProcessorPipeline.isSupported && BackgroundTransformer.isSupported;\n if (!isPipelineSupported) {\n throw new Error('pipeline is not supported in this browser');\n }\n const pipeline = new ProcessorPipeline(\n [new BackgroundTransformer({ imagePath, segmenterOptions })],\n 'virtual-background',\n );\n return pipeline;\n};\n\nexport const Dummy = () => {\n const isPipelineSupported = ProcessorPipeline.isSupported && BackgroundTransformer.isSupported;\n if (!isPipelineSupported) {\n throw new Error('pipeline is not supported in this browser');\n }\n const pipeline = new ProcessorPipeline([new DummyTransformer()], 'dummy');\n return pipeline;\n};\n"]}
package/dist/index.mjs CHANGED
@@ -27,8 +27,8 @@ var ProcessorPipeline = class {
27
27
  this.name = name;
28
28
  this.transformers = transformers;
29
29
  }
30
- async init(opts) {
31
- var _a, _b;
30
+ async setup(opts) {
31
+ var _a, _b, _c, _d;
32
32
  this.source = opts.track;
33
33
  const origConstraints = this.source.getConstraints();
34
34
  await this.source.applyConstraints(__spreadProps(__spreadValues({}, origConstraints), {
@@ -41,8 +41,8 @@ var ProcessorPipeline = class {
41
41
  this.sourceSettings = this.source.getSettings();
42
42
  this.sourceDummy = opts.element;
43
43
  if (this.sourceDummy instanceof HTMLVideoElement) {
44
- this.sourceDummy.height = this.sourceSettings.height;
45
- this.sourceDummy.width = this.sourceSettings.width;
44
+ this.sourceDummy.height = (_a = this.sourceSettings.height) != null ? _a : 300;
45
+ this.sourceDummy.width = (_b = this.sourceSettings.width) != null ? _b : 300;
46
46
  }
47
47
  if (!(this.sourceDummy instanceof HTMLVideoElement)) {
48
48
  throw TypeError("Currently only video transformers are supported");
@@ -53,9 +53,15 @@ var ProcessorPipeline = class {
53
53
  signalTarget: this.source
54
54
  });
55
55
  this.canvas = new OffscreenCanvas(
56
- (_a = this.sourceSettings.width) != null ? _a : 300,
57
- (_b = this.sourceSettings.height) != null ? _b : 300
56
+ (_c = this.sourceSettings.width) != null ? _c : 300,
57
+ (_d = this.sourceSettings.height) != null ? _d : 300
58
58
  );
59
+ }
60
+ async init(opts) {
61
+ await this.setup(opts);
62
+ if (!this.canvas || !this.processor || !this.trackGenerator) {
63
+ throw new TypeError("Expected both canvas and processor to be defined after setup");
64
+ }
59
65
  let readableStream = this.processor.readable;
60
66
  for (const transformer of this.transformers) {
61
67
  await transformer.init({
@@ -67,6 +73,10 @@ var ProcessorPipeline = class {
67
73
  readableStream.pipeTo(this.trackGenerator.writable).catch((e) => console.error("error when trying to pipe", e)).finally(() => this.destroy());
68
74
  this.processedTrack = this.trackGenerator;
69
75
  }
76
+ async restart(opts) {
77
+ await this.destroy();
78
+ return this.init(opts);
79
+ }
70
80
  async destroy() {
71
81
  var _a;
72
82
  for (const transformer of this.transformers) {
@@ -108,6 +118,12 @@ var VideoTransformer = class {
108
118
  this.inputVideo = inputVideo;
109
119
  this.isDisabled = false;
110
120
  }
121
+ async restart({ outputCanvas, inputElement: inputVideo }) {
122
+ this.canvas = outputCanvas || null;
123
+ this.ctx = this.canvas.getContext("2d") || void 0;
124
+ this.inputVideo = inputVideo;
125
+ this.isDisabled = false;
126
+ }
111
127
  async destroy() {
112
128
  this.isDisabled = true;
113
129
  this.canvas = void 0;
@@ -126,7 +142,6 @@ var BackgroundProcessor = class extends VideoTransformer {
126
142
  } else if (opts.imagePath) {
127
143
  this.loadBackground(opts.imagePath);
128
144
  }
129
- this.options.segmenterOptions;
130
145
  }
131
146
  static get isSupported() {
132
147
  return typeof OffscreenCanvas !== "undefined";
@@ -197,14 +212,14 @@ var BackgroundProcessor = class extends VideoTransformer {
197
212
  if (!this.canvas || !this.ctx || !this.segmentationResults || !this.inputVideo)
198
213
  return;
199
214
  if ((_a = this.segmentationResults) == null ? void 0 : _a.categoryMask) {
200
- this.ctx.filter = "blur(3px)";
215
+ this.ctx.filter = "blur(10px)";
201
216
  this.ctx.globalCompositeOperation = "copy";
202
217
  const bitmap = await maskToBitmap(
203
218
  this.segmentationResults.categoryMask,
204
- this.inputVideo.videoWidth,
205
- this.inputVideo.videoHeight
219
+ this.segmentationResults.categoryMask.width,
220
+ this.segmentationResults.categoryMask.height
206
221
  );
207
- this.ctx.drawImage(bitmap, 0, 0);
222
+ this.ctx.drawImage(bitmap, 0, 0, this.canvas.width, this.canvas.height);
208
223
  this.ctx.filter = "none";
209
224
  this.ctx.globalCompositeOperation = "source-in";
210
225
  if (this.backgroundImage) {
@@ -236,18 +251,18 @@ var BackgroundProcessor = class extends VideoTransformer {
236
251
  this.ctx.globalCompositeOperation = "copy";
237
252
  const bitmap = await maskToBitmap(
238
253
  this.segmentationResults.categoryMask,
239
- this.inputVideo.videoWidth,
240
- this.inputVideo.videoHeight
254
+ this.segmentationResults.categoryMask.width,
255
+ this.segmentationResults.categoryMask.height
241
256
  );
242
257
  this.ctx.filter = "blur(3px)";
243
258
  this.ctx.globalCompositeOperation = "copy";
244
- this.ctx.drawImage(bitmap, 0, 0);
259
+ this.ctx.drawImage(bitmap, 0, 0, this.canvas.width, this.canvas.height);
245
260
  this.ctx.filter = "none";
246
261
  this.ctx.globalCompositeOperation = "source-out";
247
262
  this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);
248
263
  this.ctx.globalCompositeOperation = "destination-over";
249
264
  this.ctx.filter = `blur(${this.blurRadius}px)`;
250
- this.ctx.drawImage(frame, 0, 0);
265
+ this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);
251
266
  this.ctx.restore();
252
267
  }
253
268
  };
package/dist/index.mjs.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"sources":["../src/ProcessorPipeline.ts","../src/transformers/BackgroundTransformer.ts","../package.json","../src/transformers/VideoTransformer.ts","../src/transformers/DummyTransformer.ts","../src/index.ts"],"sourcesContent":["import type { ProcessorOptions, Track, TrackProcessor } from 'livekit-client';\nimport { VideoTrackTransformer } from './transformers';\n\nexport default class ProcessorPipeline implements TrackProcessor<Track.Kind> {\n static get isSupported() {\n return (\n typeof MediaStreamTrackGenerator !== 'undefined' &&\n typeof MediaStreamTrackProcessor !== 'undefined'\n );\n }\n\n name: string;\n\n source?: MediaStreamVideoTrack;\n\n sourceSettings?: MediaTrackSettings;\n\n processor?: MediaStreamTrackProcessor<VideoFrame>;\n\n trackGenerator?: MediaStreamTrackGenerator<VideoFrame>;\n\n canvas?: OffscreenCanvas;\n\n sourceDummy?: HTMLMediaElement;\n\n processedTrack?: MediaStreamTrack;\n\n transformers: Array<VideoTrackTransformer>;\n\n constructor(transformers: Array<VideoTrackTransformer>, name: string) {\n this.name = name;\n this.transformers = transformers;\n }\n\n async init(opts: ProcessorOptions<Track.Kind>) {\n this.source = opts.track as MediaStreamVideoTrack;\n const origConstraints = this.source.getConstraints();\n await this.source.applyConstraints({\n ...origConstraints,\n // @ts-expect-error when a mediastream track is resized and/or cropped, the `VideoFrame` will have a coded height/width of the original video size\n // this leads to a shift of the underlying video as the frame itself is being rendered with the coded size\n // but image segmentation is based on the display dimensions (-> the cropped version)\n // in order to prevent this, we force the resize mode to \"none\"\n resizeMode: 'none',\n });\n this.sourceSettings = this.source.getSettings();\n this.sourceDummy = opts.element;\n if (this.sourceDummy instanceof HTMLVideoElement) {\n this.sourceDummy.height = this.sourceSettings.height!;\n this.sourceDummy.width = this.sourceSettings.width!;\n }\n if (!(this.sourceDummy instanceof HTMLVideoElement)) {\n throw TypeError('Currently only video transformers are supported');\n }\n // TODO explore if we can do all the processing work in a webworker\n this.processor = new MediaStreamTrackProcessor({ track: this.source });\n\n this.trackGenerator = new MediaStreamTrackGenerator({\n kind: 'video',\n signalTarget: this.source,\n });\n\n this.canvas = new OffscreenCanvas(\n this.sourceSettings.width ?? 300,\n this.sourceSettings.height ?? 300,\n );\n\n let readableStream = this.processor.readable;\n for (const transformer of this.transformers) {\n await transformer.init({\n outputCanvas: this.canvas,\n inputElement: this.sourceDummy!,\n });\n readableStream = readableStream.pipeThrough(transformer!.transformer!);\n }\n readableStream\n .pipeTo(this.trackGenerator.writable)\n .catch((e) => console.error('error when trying to pipe', e))\n .finally(() => this.destroy());\n this.processedTrack = this.trackGenerator as MediaStreamVideoTrack;\n }\n\n async destroy() {\n for (const transformer of this.transformers) {\n await transformer.destroy();\n }\n this.trackGenerator?.stop();\n }\n}\n","import * as vision from '@mediapipe/tasks-vision';\nimport { dependencies } from '../../package.json';\nimport VideoTransformer from './VideoTransformer';\nimport { VideoTransformerInitOptions } from './types';\n\nexport type SegmenterBaseOptions = Partial<vision.ImageSegmenterOptions['baseOptions']>;\n\nexport type BackgroundOptions = {\n blurRadius?: number;\n imagePath?: string;\n segmenterOptions?: SegmenterBaseOptions;\n};\n\nexport default class BackgroundProcessor extends VideoTransformer {\n static get isSupported() {\n return typeof OffscreenCanvas !== 'undefined';\n }\n\n imageSegmenter?: vision.ImageSegmenter;\n\n segmentationResults: vision.ImageSegmenterResult | undefined;\n\n backgroundImage: ImageBitmap | null = null;\n\n blurRadius?: number;\n\n options: BackgroundOptions;\n\n constructor(opts: BackgroundOptions) {\n super();\n this.options = opts;\n if (opts.blurRadius) {\n this.blurRadius = opts.blurRadius;\n } else if (opts.imagePath) {\n this.loadBackground(opts.imagePath);\n }\n\n this.options.segmenterOptions;\n }\n\n async init({ outputCanvas, inputElement: inputVideo }: VideoTransformerInitOptions) {\n await super.init({ outputCanvas, inputElement: inputVideo });\n\n const fileSet = await vision.FilesetResolver.forVisionTasks(\n `https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@${dependencies['@mediapipe/tasks-vision']}/wasm`,\n );\n\n this.imageSegmenter = await vision.ImageSegmenter.createFromOptions(fileSet, {\n baseOptions: {\n modelAssetPath:\n 'https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_segmenter/float16/latest/selfie_segmenter.tflite',\n delegate: 'GPU',\n ...this.options.segmenterOptions,\n },\n runningMode: 'VIDEO',\n outputCategoryMask: true,\n outputConfidenceMasks: false,\n });\n\n // this.loadBackground(opts.backgroundUrl).catch((e) => console.error(e));\n }\n\n async destroy() {\n await super.destroy();\n await this.imageSegmenter?.close();\n this.backgroundImage = null;\n }\n\n async loadBackground(path: string) {\n const img = new Image();\n\n await new Promise((resolve, reject) => {\n img.crossOrigin = 'Anonymous';\n img.onload = () => resolve(img);\n img.onerror = (err) => reject(err);\n img.src = path;\n });\n const imageData = await createImageBitmap(img);\n this.backgroundImage = imageData;\n }\n\n async transform(frame: VideoFrame, controller: TransformStreamDefaultController<VideoFrame>) {\n try {\n if (this.isDisabled) {\n controller.enqueue(frame);\n return;\n }\n if (!this.canvas) {\n throw TypeError('Canvas needs to be initialized first');\n }\n let startTimeMs = performance.now();\n this.imageSegmenter?.segmentForVideo(\n this.inputVideo!,\n startTimeMs,\n (result) => (this.segmentationResults = result),\n );\n\n if (this.blurRadius) {\n await this.blurBackground(frame);\n } else {\n await this.drawVirtualBackground(frame);\n }\n const newFrame = new VideoFrame(this.canvas, {\n timestamp: frame.timestamp || Date.now(),\n });\n controller.enqueue(newFrame);\n } finally {\n frame.close();\n }\n }\n\n async drawVirtualBackground(frame: VideoFrame) {\n if (!this.canvas || !this.ctx || !this.segmentationResults || !this.inputVideo) return;\n // this.ctx.save();\n // this.ctx.clearRect(0, 0, this.canvas.width, this.canvas.height);\n if (this.segmentationResults?.categoryMask) {\n this.ctx.filter = 'blur(3px)';\n this.ctx.globalCompositeOperation = 'copy';\n const bitmap = await maskToBitmap(\n this.segmentationResults.categoryMask,\n this.inputVideo.videoWidth,\n this.inputVideo.videoHeight,\n );\n this.ctx.drawImage(bitmap, 0, 0);\n this.ctx.filter = 'none';\n this.ctx.globalCompositeOperation = 'source-in';\n if (this.backgroundImage) {\n this.ctx.drawImage(\n this.backgroundImage,\n 0,\n 0,\n this.backgroundImage.width,\n this.backgroundImage.height,\n 0,\n 0,\n this.canvas.width,\n this.canvas.height,\n );\n } else {\n this.ctx.fillStyle = '#00FF00';\n this.ctx.fillRect(0, 0, this.canvas.width, this.canvas.height);\n }\n\n this.ctx.globalCompositeOperation = 'destination-over';\n }\n this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);\n // this.ctx.restore();\n }\n\n async blurBackground(frame: VideoFrame) {\n if (\n !this.ctx ||\n !this.canvas ||\n !this.segmentationResults?.categoryMask?.canvas ||\n !this.inputVideo\n ) {\n return;\n }\n\n this.ctx.save();\n this.ctx.globalCompositeOperation = 'copy';\n\n const bitmap = await maskToBitmap(\n this.segmentationResults.categoryMask,\n this.inputVideo.videoWidth,\n this.inputVideo.videoHeight,\n );\n\n this.ctx.filter = 'blur(3px)';\n this.ctx.globalCompositeOperation = 'copy';\n this.ctx.drawImage(bitmap, 0, 0);\n this.ctx.filter = 'none';\n this.ctx.globalCompositeOperation = 'source-out';\n this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);\n this.ctx.globalCompositeOperation = 'destination-over';\n this.ctx.filter = `blur(${this.blurRadius}px)`;\n this.ctx.drawImage(frame, 0, 0);\n this.ctx.restore();\n }\n}\n\nfunction maskToBitmap(\n mask: vision.MPMask,\n videoWidth: number,\n videoHeight: number,\n): Promise<ImageBitmap> {\n const dataArray: Uint8ClampedArray = new Uint8ClampedArray(videoWidth * videoHeight * 4);\n const result = mask.getAsUint8Array();\n for (let i = 0; i < result.length; i += 1) {\n dataArray[i * 4] = result[i];\n dataArray[i * 4 + 1] = result[i];\n dataArray[i * 4 + 2] = result[i];\n dataArray[i * 4 + 3] = result[i];\n }\n const dataNew = new ImageData(dataArray, videoWidth, videoHeight);\n\n return createImageBitmap(dataNew);\n}\n","{\n \"name\": \"@livekit/track-processors\",\n \"version\": \"0.2.4\",\n \"description\": \"LiveKit track processors\",\n \"main\": \"dist/index.js\",\n \"module\": \"dist/index.mjs\",\n \"source\": \"src/index.ts\",\n \"types\": \"dist/src/index.d.ts\",\n \"repository\": \"git@github.com:livekit/livekit-track-processors.git\",\n \"author\": \"Lukas Seiler\",\n \"license\": \"Apache-2.0\",\n \"scripts\": {\n \"build\": \"tsup --onSuccess \\\"tsc --declaration --emitDeclarationOnly\\\"\",\n \"build-docs\": \"yarn exec typedoc\",\n \"build-sample\": \"cd example && webpack && cp styles.css index.html dist/\",\n \"lint\": \"eslint src\",\n \"test\": \"jest\",\n \"sample\": \"vite serve example --port 8080 --open\",\n \"prepublish\": \"yarn build\"\n },\n \"files\": [\n \"dist\",\n \"src\"\n ],\n \"dependencies\": {\n \"@mediapipe/holistic\": \"0.5.1675471629\",\n \"@mediapipe/tasks-vision\": \"0.10.1\"\n },\n \"peerDependencies\": {\n \"livekit-client\": \"^1.12.0\"\n },\n \"devDependencies\": {\n \"@trivago/prettier-plugin-sort-imports\": \"^4.1.1\",\n \"@types/dom-mediacapture-transform\": \"^0.1.5\",\n \"@types/jest\": \"^27.0.3\",\n \"@types/offscreencanvas\": \"^2019.7.0\",\n \"@typescript-eslint/eslint-plugin\": \"^4.31.2\",\n \"@webpack-cli/serve\": \"^1.5.2\",\n \"eslint\": \"8.39.0\",\n \"eslint-config-airbnb-typescript\": \"17.0.0\",\n \"eslint-config-prettier\": \"8.8.0\",\n \"eslint-plugin-ecmascript-compat\": \"^3.0.0\",\n \"eslint-plugin-import\": \"2.27.5\",\n \"jest\": \"^27.4.3\",\n \"livekit-client\": \"^0.16.6\",\n \"prettier\": \"^2.8.8\",\n \"ts-jest\": \"^27.0.7\",\n \"ts-loader\": \"^8.1.0\",\n \"ts-proto\": \"^1.85.0\",\n \"tsup\": \"^7.1.0\",\n \"typedoc\": \"^0.20.35\",\n \"typedoc-plugin-no-inherit\": \"1.3.0\",\n \"typescript\": \"^5.0.4\",\n \"vite\": \"^4.3.8\",\n \"webpack\": \"^5.53.0\",\n \"webpack-cli\": \"^4.8.0\",\n \"webpack-dev-server\": \"^4.2.1\"\n }\n}\n","import { VideoTrackTransformer, VideoTransformerInitOptions } from './types';\n\nexport default abstract class VideoTransformer implements VideoTrackTransformer {\n transformer?: TransformStream;\n\n canvas?: OffscreenCanvas;\n\n ctx?: OffscreenCanvasRenderingContext2D;\n\n inputVideo?: HTMLVideoElement;\n\n protected isDisabled?: Boolean = false;\n\n async init({\n outputCanvas,\n inputElement: inputVideo,\n }: VideoTransformerInitOptions): Promise<void> {\n if (!(inputVideo instanceof HTMLVideoElement)) {\n throw TypeError('Video transformer needs a HTMLVideoElement as input');\n }\n this.transformer = new TransformStream({\n transform: (frame, controller) => this.transform(frame, controller),\n });\n this.canvas = outputCanvas || null;\n if (outputCanvas) {\n this.ctx = this.canvas?.getContext('2d') || undefined;\n }\n this.inputVideo = inputVideo;\n this.isDisabled = false;\n }\n\n async destroy() {\n this.isDisabled = true;\n this.canvas = undefined;\n this.ctx = undefined;\n }\n\n abstract transform(\n frame: VideoFrame,\n controller: TransformStreamDefaultController<VideoFrame>,\n ): void;\n}\n","import VideoTransformer from './VideoTransformer';\n\nexport default class DummyTransformer extends VideoTransformer {\n async transform(frame: VideoFrame, controller: TransformStreamDefaultController<VideoFrame>) {\n controller.enqueue(frame);\n }\n\n async destroy() {\n // nothing to do\n }\n}\n","import ProcessorPipeline from './ProcessorPipeline';\nimport BackgroundTransformer, { SegmenterBaseOptions } from './transformers/BackgroundTransformer';\nimport DummyTransformer from './transformers/DummyTransformer';\n\nexport const BackgroundBlur = (\n blurRadius: number = 10,\n segmenterOptions?: SegmenterBaseOptions,\n) => {\n const isPipelineSupported = ProcessorPipeline.isSupported && BackgroundTransformer.isSupported;\n if (!isPipelineSupported) {\n throw new Error('pipeline is not supported in this browser');\n }\n const pipeline = new ProcessorPipeline(\n [new BackgroundTransformer({ blurRadius, segmenterOptions })],\n 'background-blur',\n );\n return pipeline;\n};\n\nexport const VirtualBackground = (imagePath: string, segmenterOptions?: SegmenterBaseOptions) => {\n const isPipelineSupported = ProcessorPipeline.isSupported && BackgroundTransformer.isSupported;\n if (!isPipelineSupported) {\n throw new Error('pipeline is not supported in this browser');\n }\n const pipeline = new ProcessorPipeline(\n [new BackgroundTransformer({ imagePath, segmenterOptions })],\n 'virtual-background',\n );\n return pipeline;\n};\n\nexport const Dummy = () => {\n const isPipelineSupported = ProcessorPipeline.isSupported && BackgroundTransformer.isSupported;\n if (!isPipelineSupported) {\n throw new Error('pipeline is not supported in this browser');\n }\n const pipeline = new ProcessorPipeline([new DummyTransformer()], 'dummy');\n return pipeline;\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AAGA,IAAqB,oBAArB,MAA6E;AAAA,EAC3E,WAAW,cAAc;AACvB,WACE,OAAO,8BAA8B,eACrC,OAAO,8BAA8B;AAAA,EAEzC;AAAA,EAoBA,YAAY,cAA4C,MAAc;AACpE,SAAK,OAAO;AACZ,SAAK,eAAe;AAAA,EACtB;AAAA,EAEA,MAAM,KAAK,MAAoC;AAlCjD;AAmCI,SAAK,SAAS,KAAK;AACnB,UAAM,kBAAkB,KAAK,OAAO,eAAe;AACnD,UAAM,KAAK,OAAO,iBAAiB,iCAC9B,kBAD8B;AAAA;AAAA;AAAA;AAAA;AAAA,MAMjC,YAAY;AAAA,IACd,EAAC;AACD,SAAK,iBAAiB,KAAK,OAAO,YAAY;AAC9C,SAAK,cAAc,KAAK;AACxB,QAAI,KAAK,uBAAuB,kBAAkB;AAChD,WAAK,YAAY,SAAS,KAAK,eAAe;AAC9C,WAAK,YAAY,QAAQ,KAAK,eAAe;AAAA,IAC/C;AACA,QAAI,EAAE,KAAK,uBAAuB,mBAAmB;AACnD,YAAM,UAAU,iDAAiD;AAAA,IACnE;AAEA,SAAK,YAAY,IAAI,0BAA0B,EAAE,OAAO,KAAK,OAAO,CAAC;AAErE,SAAK,iBAAiB,IAAI,0BAA0B;AAAA,MAClD,MAAM;AAAA,MACN,cAAc,KAAK;AAAA,IACrB,CAAC;AAED,SAAK,SAAS,IAAI;AAAA,OAChB,UAAK,eAAe,UAApB,YAA6B;AAAA,OAC7B,UAAK,eAAe,WAApB,YAA8B;AAAA,IAChC;AAEA,QAAI,iBAAiB,KAAK,UAAU;AACpC,eAAW,eAAe,KAAK,cAAc;AAC3C,YAAM,YAAY,KAAK;AAAA,QACrB,cAAc,KAAK;AAAA,QACnB,cAAc,KAAK;AAAA,MACrB,CAAC;AACD,uBAAiB,eAAe,YAAY,YAAa,WAAY;AAAA,IACvE;AACA,mBACG,OAAO,KAAK,eAAe,QAAQ,EACnC,MAAM,CAAC,MAAM,QAAQ,MAAM,6BAA6B,CAAC,CAAC,EAC1D,QAAQ,MAAM,KAAK,QAAQ,CAAC;AAC/B,SAAK,iBAAiB,KAAK;AAAA,EAC7B;AAAA,EAEA,MAAM,UAAU;AAlFlB;AAmFI,eAAW,eAAe,KAAK,cAAc;AAC3C,YAAM,YAAY,QAAQ;AAAA,IAC5B;AACA,eAAK,mBAAL,mBAAqB;AAAA,EACvB;AACF;;;ACxFA,YAAY,YAAY;;;ACwBtB,mBAAgB;AAAA,EACd,uBAAuB;AAAA,EACvB,2BAA2B;AAC7B;;;ACzBF,IAA8B,mBAA9B,MAAgF;AAAA,EAAhF;AASE,SAAU,aAAuB;AAAA;AAAA,EAEjC,MAAM,KAAK;AAAA,IACT;AAAA,IACA,cAAc;AAAA,EAChB,GAA+C;AAhBjD;AAiBI,QAAI,EAAE,sBAAsB,mBAAmB;AAC7C,YAAM,UAAU,qDAAqD;AAAA,IACvE;AACA,SAAK,cAAc,IAAI,gBAAgB;AAAA,MACrC,WAAW,CAAC,OAAO,eAAe,KAAK,UAAU,OAAO,UAAU;AAAA,IACpE,CAAC;AACD,SAAK,SAAS,gBAAgB;AAC9B,QAAI,cAAc;AAChB,WAAK,QAAM,UAAK,WAAL,mBAAa,WAAW,UAAS;AAAA,IAC9C;AACA,SAAK,aAAa;AAClB,SAAK,aAAa;AAAA,EACpB;AAAA,EAEA,MAAM,UAAU;AACd,SAAK,aAAa;AAClB,SAAK,SAAS;AACd,SAAK,MAAM;AAAA,EACb;AAMF;;;AF5BA,IAAqB,sBAArB,cAAiD,iBAAiB;AAAA,EAehE,YAAY,MAAyB;AACnC,UAAM;AAPR,2BAAsC;AAQpC,SAAK,UAAU;AACf,QAAI,KAAK,YAAY;AACnB,WAAK,aAAa,KAAK;AAAA,IACzB,WAAW,KAAK,WAAW;AACzB,WAAK,eAAe,KAAK,SAAS;AAAA,IACpC;AAEA,SAAK,QAAQ;AAAA,EACf;AAAA,EAxBA,WAAW,cAAc;AACvB,WAAO,OAAO,oBAAoB;AAAA,EACpC;AAAA,EAwBA,MAAM,KAAK,EAAE,cAAc,cAAc,WAAW,GAAgC;AAClF,UAAM,MAAM,KAAK,EAAE,cAAc,cAAc,WAAW,CAAC;AAE3D,UAAM,UAAU,MAAa,uBAAgB;AAAA,MAC3C,wDAAwD,aAAa,yBAAyB,CAAC;AAAA,IACjG;AAEA,SAAK,iBAAiB,MAAa,sBAAe,kBAAkB,SAAS;AAAA,MAC3E,aAAa;AAAA,QACX,gBACE;AAAA,QACF,UAAU;AAAA,SACP,KAAK,QAAQ;AAAA,MAElB,aAAa;AAAA,MACb,oBAAoB;AAAA,MACpB,uBAAuB;AAAA,IACzB,CAAC;AAAA,EAGH;AAAA,EAEA,MAAM,UAAU;AA9DlB;AA+DI,UAAM,MAAM,QAAQ;AACpB,YAAM,UAAK,mBAAL,mBAAqB;AAC3B,SAAK,kBAAkB;AAAA,EACzB;AAAA,EAEA,MAAM,eAAe,MAAc;AACjC,UAAM,MAAM,IAAI,MAAM;AAEtB,UAAM,IAAI,QAAQ,CAAC,SAAS,WAAW;AACrC,UAAI,cAAc;AAClB,UAAI,SAAS,MAAM,QAAQ,GAAG;AAC9B,UAAI,UAAU,CAAC,QAAQ,OAAO,GAAG;AACjC,UAAI,MAAM;AAAA,IACZ,CAAC;AACD,UAAM,YAAY,MAAM,kBAAkB,GAAG;AAC7C,SAAK,kBAAkB;AAAA,EACzB;AAAA,EAEA,MAAM,UAAU,OAAmB,YAA0D;AAjF/F;AAkFI,QAAI;AACF,UAAI,KAAK,YAAY;AACnB,mBAAW,QAAQ,KAAK;AACxB;AAAA,MACF;AACA,UAAI,CAAC,KAAK,QAAQ;AAChB,cAAM,UAAU,sCAAsC;AAAA,MACxD;AACA,UAAI,cAAc,YAAY,IAAI;AAClC,iBAAK,mBAAL,mBAAqB;AAAA,QACnB,KAAK;AAAA,QACL;AAAA,QACA,CAAC,WAAY,KAAK,sBAAsB;AAAA;AAG1C,UAAI,KAAK,YAAY;AACnB,cAAM,KAAK,eAAe,KAAK;AAAA,MACjC,OAAO;AACL,cAAM,KAAK,sBAAsB,KAAK;AAAA,MACxC;AACA,YAAM,WAAW,IAAI,WAAW,KAAK,QAAQ;AAAA,QAC3C,WAAW,MAAM,aAAa,KAAK,IAAI;AAAA,MACzC,CAAC;AACD,iBAAW,QAAQ,QAAQ;AAAA,IAC7B,UAAE;AACA,YAAM,MAAM;AAAA,IACd;AAAA,EACF;AAAA,EAEA,MAAM,sBAAsB,OAAmB;AA/GjD;AAgHI,QAAI,CAAC,KAAK,UAAU,CAAC,KAAK,OAAO,CAAC,KAAK,uBAAuB,CAAC,KAAK;AAAY;AAGhF,SAAI,UAAK,wBAAL,mBAA0B,cAAc;AAC1C,WAAK,IAAI,SAAS;AAClB,WAAK,IAAI,2BAA2B;AACpC,YAAM,SAAS,MAAM;AAAA,QACnB,KAAK,oBAAoB;AAAA,QACzB,KAAK,WAAW;AAAA,QAChB,KAAK,WAAW;AAAA,MAClB;AACA,WAAK,IAAI,UAAU,QAAQ,GAAG,CAAC;AAC/B,WAAK,IAAI,SAAS;AAClB,WAAK,IAAI,2BAA2B;AACpC,UAAI,KAAK,iBAAiB;AACxB,aAAK,IAAI;AAAA,UACP,KAAK;AAAA,UACL;AAAA,UACA;AAAA,UACA,KAAK,gBAAgB;AAAA,UACrB,KAAK,gBAAgB;AAAA,UACrB;AAAA,UACA;AAAA,UACA,KAAK,OAAO;AAAA,UACZ,KAAK,OAAO;AAAA,QACd;AAAA,MACF,OAAO;AACL,aAAK,IAAI,YAAY;AACrB,aAAK,IAAI,SAAS,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AAAA,MAC/D;AAEA,WAAK,IAAI,2BAA2B;AAAA,IACtC;AACA,SAAK,IAAI,UAAU,OAAO,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AAAA,EAEvE;AAAA,EAEA,MAAM,eAAe,OAAmB;AArJ1C;AAsJI,QACE,CAAC,KAAK,OACN,CAAC,KAAK,UACN,GAAC,gBAAK,wBAAL,mBAA0B,iBAA1B,mBAAwC,WACzC,CAAC,KAAK,YACN;AACA;AAAA,IACF;AAEA,SAAK,IAAI,KAAK;AACd,SAAK,IAAI,2BAA2B;AAEpC,UAAM,SAAS,MAAM;AAAA,MACnB,KAAK,oBAAoB;AAAA,MACzB,KAAK,WAAW;AAAA,MAChB,KAAK,WAAW;AAAA,IAClB;AAEA,SAAK,IAAI,SAAS;AAClB,SAAK,IAAI,2BAA2B;AACpC,SAAK,IAAI,UAAU,QAAQ,GAAG,CAAC;AAC/B,SAAK,IAAI,SAAS;AAClB,SAAK,IAAI,2BAA2B;AACpC,SAAK,IAAI,UAAU,OAAO,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AACrE,SAAK,IAAI,2BAA2B;AACpC,SAAK,IAAI,SAAS,QAAQ,KAAK,UAAU;AACzC,SAAK,IAAI,UAAU,OAAO,GAAG,CAAC;AAC9B,SAAK,IAAI,QAAQ;AAAA,EACnB;AACF;AAEA,SAAS,aACP,MACA,YACA,aACsB;AACtB,QAAM,YAA+B,IAAI,kBAAkB,aAAa,cAAc,CAAC;AACvF,QAAM,SAAS,KAAK,gBAAgB;AACpC,WAAS,IAAI,GAAG,IAAI,OAAO,QAAQ,KAAK,GAAG;AACzC,cAAU,IAAI,CAAC,IAAI,OAAO,CAAC;AAC3B,cAAU,IAAI,IAAI,CAAC,IAAI,OAAO,CAAC;AAC/B,cAAU,IAAI,IAAI,CAAC,IAAI,OAAO,CAAC;AAC/B,cAAU,IAAI,IAAI,CAAC,IAAI,OAAO,CAAC;AAAA,EACjC;AACA,QAAM,UAAU,IAAI,UAAU,WAAW,YAAY,WAAW;AAEhE,SAAO,kBAAkB,OAAO;AAClC;;;AGnMA,IAAqB,mBAArB,cAA8C,iBAAiB;AAAA,EAC7D,MAAM,UAAU,OAAmB,YAA0D;AAC3F,eAAW,QAAQ,KAAK;AAAA,EAC1B;AAAA,EAEA,MAAM,UAAU;AAAA,EAEhB;AACF;;;ACNO,IAAM,iBAAiB,CAC5B,aAAqB,IACrB,qBACG;AACH,QAAM,sBAAsB,kBAAkB,eAAe,oBAAsB;AACnF,MAAI,CAAC,qBAAqB;AACxB,UAAM,IAAI,MAAM,2CAA2C;AAAA,EAC7D;AACA,QAAM,WAAW,IAAI;AAAA,IACnB,CAAC,IAAI,oBAAsB,EAAE,YAAY,iBAAiB,CAAC,CAAC;AAAA,IAC5D;AAAA,EACF;AACA,SAAO;AACT;AAEO,IAAM,oBAAoB,CAAC,WAAmB,qBAA4C;AAC/F,QAAM,sBAAsB,kBAAkB,eAAe,oBAAsB;AACnF,MAAI,CAAC,qBAAqB;AACxB,UAAM,IAAI,MAAM,2CAA2C;AAAA,EAC7D;AACA,QAAM,WAAW,IAAI;AAAA,IACnB,CAAC,IAAI,oBAAsB,EAAE,WAAW,iBAAiB,CAAC,CAAC;AAAA,IAC3D;AAAA,EACF;AACA,SAAO;AACT;AAEO,IAAM,QAAQ,MAAM;AACzB,QAAM,sBAAsB,kBAAkB,eAAe,oBAAsB;AACnF,MAAI,CAAC,qBAAqB;AACxB,UAAM,IAAI,MAAM,2CAA2C;AAAA,EAC7D;AACA,QAAM,WAAW,IAAI,kBAAkB,CAAC,IAAI,iBAAiB,CAAC,GAAG,OAAO;AACxE,SAAO;AACT;","names":[]}
1
+ {"version":3,"sources":["../src/ProcessorPipeline.ts","../src/transformers/BackgroundTransformer.ts","../package.json","../src/transformers/VideoTransformer.ts","../src/transformers/DummyTransformer.ts","../src/index.ts"],"sourcesContent":["import type { ProcessorOptions, Track, TrackProcessor } from 'livekit-client';\nimport { VideoTrackTransformer } from './transformers';\n\nexport default class ProcessorPipeline implements TrackProcessor<Track.Kind> {\n static get isSupported() {\n return (\n typeof MediaStreamTrackGenerator !== 'undefined' &&\n typeof MediaStreamTrackProcessor !== 'undefined'\n );\n }\n\n name: string;\n\n source?: MediaStreamVideoTrack;\n\n sourceSettings?: MediaTrackSettings;\n\n processor?: MediaStreamTrackProcessor<VideoFrame>;\n\n trackGenerator?: MediaStreamTrackGenerator<VideoFrame>;\n\n canvas?: OffscreenCanvas;\n\n sourceDummy?: HTMLMediaElement;\n\n processedTrack?: MediaStreamTrack;\n\n transformers: Array<VideoTrackTransformer>;\n\n constructor(transformers: Array<VideoTrackTransformer>, name: string) {\n this.name = name;\n this.transformers = transformers;\n }\n\n private async setup(opts: ProcessorOptions<Track.Kind>) {\n this.source = opts.track as MediaStreamVideoTrack;\n const origConstraints = this.source.getConstraints();\n await this.source.applyConstraints({\n ...origConstraints,\n // @ts-expect-error when a mediastream track is resized and/or cropped, the `VideoFrame` will have a coded height/width of the original video size\n // this leads to a shift of the underlying video as the frame itself is being rendered with the coded size\n // but image segmentation is based on the display dimensions (-> the cropped version)\n // in order to prevent this, we force the resize mode to \"none\"\n resizeMode: 'none',\n });\n this.sourceSettings = this.source.getSettings();\n this.sourceDummy = opts.element;\n if (this.sourceDummy instanceof HTMLVideoElement) {\n this.sourceDummy.height = this.sourceSettings.height ?? 300;\n this.sourceDummy.width = this.sourceSettings.width ?? 300;\n }\n if (!(this.sourceDummy instanceof HTMLVideoElement)) {\n throw TypeError('Currently only video transformers are supported');\n }\n // TODO explore if we can do all the processing work in a webworker\n this.processor = new MediaStreamTrackProcessor({ track: this.source });\n\n this.trackGenerator = new MediaStreamTrackGenerator({\n kind: 'video',\n signalTarget: this.source,\n });\n\n this.canvas = new OffscreenCanvas(\n this.sourceSettings.width ?? 300,\n this.sourceSettings.height ?? 300,\n );\n }\n\n async init(opts: ProcessorOptions<Track.Kind>) {\n await this.setup(opts);\n if (!this.canvas || !this.processor || !this.trackGenerator) {\n throw new TypeError('Expected both canvas and processor to be defined after setup');\n }\n\n let readableStream = this.processor.readable;\n for (const transformer of this.transformers) {\n await transformer.init({\n outputCanvas: this.canvas,\n inputElement: this.sourceDummy as HTMLVideoElement,\n });\n readableStream = readableStream.pipeThrough(transformer!.transformer!);\n }\n readableStream\n .pipeTo(this.trackGenerator.writable)\n .catch((e) => console.error('error when trying to pipe', e))\n .finally(() => this.destroy());\n this.processedTrack = this.trackGenerator as MediaStreamVideoTrack;\n }\n\n async restart(opts: ProcessorOptions<Track.Kind>) {\n await this.destroy();\n return this.init(opts);\n }\n\n async destroy() {\n for (const transformer of this.transformers) {\n await transformer.destroy();\n }\n this.trackGenerator?.stop();\n }\n}\n","import * as vision from '@mediapipe/tasks-vision';\nimport { dependencies } from '../../package.json';\nimport VideoTransformer from './VideoTransformer';\nimport { VideoTransformerInitOptions } from './types';\n\nexport type SegmenterBaseOptions = Partial<vision.ImageSegmenterOptions['baseOptions']>;\n\nexport type BackgroundOptions = {\n blurRadius?: number;\n imagePath?: string;\n segmenterOptions?: SegmenterBaseOptions;\n};\n\nexport default class BackgroundProcessor extends VideoTransformer {\n static get isSupported() {\n return typeof OffscreenCanvas !== 'undefined';\n }\n\n imageSegmenter?: vision.ImageSegmenter;\n\n segmentationResults: vision.ImageSegmenterResult | undefined;\n\n backgroundImage: ImageBitmap | null = null;\n\n blurRadius?: number;\n\n options: BackgroundOptions;\n\n constructor(opts: BackgroundOptions) {\n super();\n this.options = opts;\n if (opts.blurRadius) {\n this.blurRadius = opts.blurRadius;\n } else if (opts.imagePath) {\n this.loadBackground(opts.imagePath);\n }\n }\n\n async init({ outputCanvas, inputElement: inputVideo }: VideoTransformerInitOptions) {\n await super.init({ outputCanvas, inputElement: inputVideo });\n\n const fileSet = await vision.FilesetResolver.forVisionTasks(\n `https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@${dependencies['@mediapipe/tasks-vision']}/wasm`,\n );\n\n this.imageSegmenter = await vision.ImageSegmenter.createFromOptions(fileSet, {\n baseOptions: {\n modelAssetPath:\n 'https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_segmenter/float16/latest/selfie_segmenter.tflite',\n delegate: 'GPU',\n ...this.options.segmenterOptions,\n },\n runningMode: 'VIDEO',\n outputCategoryMask: true,\n outputConfidenceMasks: false,\n });\n\n // this.loadBackground(opts.backgroundUrl).catch((e) => console.error(e));\n }\n\n async destroy() {\n await super.destroy();\n await this.imageSegmenter?.close();\n this.backgroundImage = null;\n }\n\n async loadBackground(path: string) {\n const img = new Image();\n\n await new Promise((resolve, reject) => {\n img.crossOrigin = 'Anonymous';\n img.onload = () => resolve(img);\n img.onerror = (err) => reject(err);\n img.src = path;\n });\n const imageData = await createImageBitmap(img);\n this.backgroundImage = imageData;\n }\n\n async transform(frame: VideoFrame, controller: TransformStreamDefaultController<VideoFrame>) {\n try {\n if (this.isDisabled) {\n controller.enqueue(frame);\n return;\n }\n if (!this.canvas) {\n throw TypeError('Canvas needs to be initialized first');\n }\n let startTimeMs = performance.now();\n this.imageSegmenter?.segmentForVideo(\n this.inputVideo!,\n startTimeMs,\n (result) => (this.segmentationResults = result),\n );\n\n if (this.blurRadius) {\n await this.blurBackground(frame);\n } else {\n await this.drawVirtualBackground(frame);\n }\n const newFrame = new VideoFrame(this.canvas, {\n timestamp: frame.timestamp || Date.now(),\n });\n controller.enqueue(newFrame);\n } finally {\n frame.close();\n }\n }\n\n async drawVirtualBackground(frame: VideoFrame) {\n if (!this.canvas || !this.ctx || !this.segmentationResults || !this.inputVideo) return;\n // this.ctx.save();\n // this.ctx.clearRect(0, 0, this.canvas.width, this.canvas.height);\n if (this.segmentationResults?.categoryMask) {\n this.ctx.filter = 'blur(10px)';\n this.ctx.globalCompositeOperation = 'copy';\n const bitmap = await maskToBitmap(\n this.segmentationResults.categoryMask,\n this.segmentationResults.categoryMask.width,\n this.segmentationResults.categoryMask.height,\n );\n this.ctx.drawImage(bitmap, 0, 0, this.canvas.width, this.canvas.height);\n this.ctx.filter = 'none';\n this.ctx.globalCompositeOperation = 'source-in';\n if (this.backgroundImage) {\n this.ctx.drawImage(\n this.backgroundImage,\n 0,\n 0,\n this.backgroundImage.width,\n this.backgroundImage.height,\n 0,\n 0,\n this.canvas.width,\n this.canvas.height,\n );\n } else {\n this.ctx.fillStyle = '#00FF00';\n this.ctx.fillRect(0, 0, this.canvas.width, this.canvas.height);\n }\n\n this.ctx.globalCompositeOperation = 'destination-over';\n }\n this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);\n // this.ctx.restore();\n }\n\n async blurBackground(frame: VideoFrame) {\n if (\n !this.ctx ||\n !this.canvas ||\n !this.segmentationResults?.categoryMask?.canvas ||\n !this.inputVideo\n ) {\n return;\n }\n\n this.ctx.save();\n this.ctx.globalCompositeOperation = 'copy';\n\n const bitmap = await maskToBitmap(\n this.segmentationResults.categoryMask,\n this.segmentationResults.categoryMask.width,\n this.segmentationResults.categoryMask.height,\n );\n\n this.ctx.filter = 'blur(3px)';\n this.ctx.globalCompositeOperation = 'copy';\n this.ctx.drawImage(bitmap, 0, 0, this.canvas.width, this.canvas.height);\n this.ctx.filter = 'none';\n this.ctx.globalCompositeOperation = 'source-out';\n this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);\n this.ctx.globalCompositeOperation = 'destination-over';\n this.ctx.filter = `blur(${this.blurRadius}px)`;\n this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);\n this.ctx.restore();\n }\n}\n\nfunction maskToBitmap(\n mask: vision.MPMask,\n videoWidth: number,\n videoHeight: number,\n): Promise<ImageBitmap> {\n const dataArray: Uint8ClampedArray = new Uint8ClampedArray(videoWidth * videoHeight * 4);\n const result = mask.getAsUint8Array();\n for (let i = 0; i < result.length; i += 1) {\n dataArray[i * 4] = result[i];\n dataArray[i * 4 + 1] = result[i];\n dataArray[i * 4 + 2] = result[i];\n dataArray[i * 4 + 3] = result[i];\n }\n const dataNew = new ImageData(dataArray, videoWidth, videoHeight);\n\n return createImageBitmap(dataNew);\n}\n","{\n \"name\": \"@livekit/track-processors\",\n \"version\": \"0.2.6\",\n \"description\": \"LiveKit track processors\",\n \"main\": \"dist/index.js\",\n \"module\": \"dist/index.mjs\",\n \"source\": \"src/index.ts\",\n \"types\": \"dist/src/index.d.ts\",\n \"repository\": \"git@github.com:livekit/livekit-track-processors.git\",\n \"author\": \"Lukas Seiler\",\n \"license\": \"Apache-2.0\",\n \"scripts\": {\n \"build\": \"tsup --onSuccess \\\"tsc --declaration --emitDeclarationOnly\\\"\",\n \"build-sample\": \"cd example && vite build\",\n \"lint\": \"eslint src\",\n \"release\": \"yarn build && changeset publish\",\n \"test\": \"jest\",\n \"sample\": \"vite serve example --port 8080 --open\"\n },\n \"files\": [\n \"dist\",\n \"src\"\n ],\n \"dependencies\": {\n \"@mediapipe/holistic\": \"0.5.1675471629\",\n \"@mediapipe/tasks-vision\": \"0.10.1\"\n },\n \"peerDependencies\": {\n \"livekit-client\": \"^1.12.0\"\n },\n \"devDependencies\": {\n \"@changesets/cli\": \"^2.26.2\",\n \"@livekit/changesets-changelog-github\": \"^0.0.4\",\n \"@trivago/prettier-plugin-sort-imports\": \"^4.1.1\",\n \"@types/dom-mediacapture-transform\": \"^0.1.5\",\n \"@types/offscreencanvas\": \"^2019.7.0\",\n \"@typescript-eslint/eslint-plugin\": \"^4.31.2\",\n \"eslint\": \"8.39.0\",\n \"eslint-config-airbnb-typescript\": \"17.0.0\",\n \"eslint-config-prettier\": \"8.8.0\",\n \"eslint-plugin-ecmascript-compat\": \"^3.0.0\",\n \"eslint-plugin-import\": \"2.27.5\",\n \"livekit-client\": \"^1.12.0\",\n \"prettier\": \"^2.8.8\",\n \"tsup\": \"^7.1.0\",\n \"typescript\": \"^5.0.4\",\n \"vite\": \"^4.3.8\"\n }\n}\n","import { VideoTrackTransformer, VideoTransformerInitOptions } from './types';\n\nexport default abstract class VideoTransformer implements VideoTrackTransformer {\n transformer?: TransformStream;\n\n canvas?: OffscreenCanvas;\n\n ctx?: OffscreenCanvasRenderingContext2D;\n\n inputVideo?: HTMLVideoElement;\n\n protected isDisabled?: Boolean = false;\n\n async init({\n outputCanvas,\n inputElement: inputVideo,\n }: VideoTransformerInitOptions): Promise<void> {\n if (!(inputVideo instanceof HTMLVideoElement)) {\n throw TypeError('Video transformer needs a HTMLVideoElement as input');\n }\n this.transformer = new TransformStream({\n transform: (frame, controller) => this.transform(frame, controller),\n });\n this.canvas = outputCanvas || null;\n if (outputCanvas) {\n this.ctx = this.canvas?.getContext('2d') || undefined;\n }\n this.inputVideo = inputVideo;\n this.isDisabled = false;\n }\n\n async restart({ outputCanvas, inputElement: inputVideo }: VideoTransformerInitOptions) {\n this.canvas = outputCanvas || null;\n this.ctx = this.canvas.getContext('2d') || undefined;\n\n this.inputVideo = inputVideo;\n this.isDisabled = false;\n }\n\n async destroy() {\n this.isDisabled = true;\n this.canvas = undefined;\n this.ctx = undefined;\n }\n\n abstract transform(\n frame: VideoFrame,\n controller: TransformStreamDefaultController<VideoFrame>,\n ): void;\n}\n","import VideoTransformer from './VideoTransformer';\n\nexport default class DummyTransformer extends VideoTransformer {\n async transform(frame: VideoFrame, controller: TransformStreamDefaultController<VideoFrame>) {\n controller.enqueue(frame);\n }\n\n async destroy() {\n // nothing to do\n }\n}\n","import ProcessorPipeline from './ProcessorPipeline';\nimport BackgroundTransformer, { SegmenterBaseOptions } from './transformers/BackgroundTransformer';\nimport DummyTransformer from './transformers/DummyTransformer';\n\nexport * from './transformers/types';\n\nexport const BackgroundBlur = (\n blurRadius: number = 10,\n segmenterOptions?: SegmenterBaseOptions,\n) => {\n const isPipelineSupported = ProcessorPipeline.isSupported && BackgroundTransformer.isSupported;\n if (!isPipelineSupported) {\n throw new Error('pipeline is not supported in this browser');\n }\n const pipeline = new ProcessorPipeline(\n [new BackgroundTransformer({ blurRadius, segmenterOptions })],\n 'background-blur',\n );\n return pipeline;\n};\n\nexport const VirtualBackground = (imagePath: string, segmenterOptions?: SegmenterBaseOptions) => {\n const isPipelineSupported = ProcessorPipeline.isSupported && BackgroundTransformer.isSupported;\n if (!isPipelineSupported) {\n throw new Error('pipeline is not supported in this browser');\n }\n const pipeline = new ProcessorPipeline(\n [new BackgroundTransformer({ imagePath, segmenterOptions })],\n 'virtual-background',\n );\n return pipeline;\n};\n\nexport const Dummy = () => {\n const isPipelineSupported = ProcessorPipeline.isSupported && BackgroundTransformer.isSupported;\n if (!isPipelineSupported) {\n throw new Error('pipeline is not supported in this browser');\n }\n const pipeline = new ProcessorPipeline([new DummyTransformer()], 'dummy');\n return pipeline;\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AAGA,IAAqB,oBAArB,MAA6E;AAAA,EAC3E,WAAW,cAAc;AACvB,WACE,OAAO,8BAA8B,eACrC,OAAO,8BAA8B;AAAA,EAEzC;AAAA,EAoBA,YAAY,cAA4C,MAAc;AACpE,SAAK,OAAO;AACZ,SAAK,eAAe;AAAA,EACtB;AAAA,EAEA,MAAc,MAAM,MAAoC;AAlC1D;AAmCI,SAAK,SAAS,KAAK;AACnB,UAAM,kBAAkB,KAAK,OAAO,eAAe;AACnD,UAAM,KAAK,OAAO,iBAAiB,iCAC9B,kBAD8B;AAAA;AAAA;AAAA;AAAA;AAAA,MAMjC,YAAY;AAAA,IACd,EAAC;AACD,SAAK,iBAAiB,KAAK,OAAO,YAAY;AAC9C,SAAK,cAAc,KAAK;AACxB,QAAI,KAAK,uBAAuB,kBAAkB;AAChD,WAAK,YAAY,UAAS,UAAK,eAAe,WAApB,YAA8B;AACxD,WAAK,YAAY,SAAQ,UAAK,eAAe,UAApB,YAA6B;AAAA,IACxD;AACA,QAAI,EAAE,KAAK,uBAAuB,mBAAmB;AACnD,YAAM,UAAU,iDAAiD;AAAA,IACnE;AAEA,SAAK,YAAY,IAAI,0BAA0B,EAAE,OAAO,KAAK,OAAO,CAAC;AAErE,SAAK,iBAAiB,IAAI,0BAA0B;AAAA,MAClD,MAAM;AAAA,MACN,cAAc,KAAK;AAAA,IACrB,CAAC;AAED,SAAK,SAAS,IAAI;AAAA,OAChB,UAAK,eAAe,UAApB,YAA6B;AAAA,OAC7B,UAAK,eAAe,WAApB,YAA8B;AAAA,IAChC;AAAA,EACF;AAAA,EAEA,MAAM,KAAK,MAAoC;AAC7C,UAAM,KAAK,MAAM,IAAI;AACrB,QAAI,CAAC,KAAK,UAAU,CAAC,KAAK,aAAa,CAAC,KAAK,gBAAgB;AAC3D,YAAM,IAAI,UAAU,8DAA8D;AAAA,IACpF;AAEA,QAAI,iBAAiB,KAAK,UAAU;AACpC,eAAW,eAAe,KAAK,cAAc;AAC3C,YAAM,YAAY,KAAK;AAAA,QACrB,cAAc,KAAK;AAAA,QACnB,cAAc,KAAK;AAAA,MACrB,CAAC;AACD,uBAAiB,eAAe,YAAY,YAAa,WAAY;AAAA,IACvE;AACA,mBACG,OAAO,KAAK,eAAe,QAAQ,EACnC,MAAM,CAAC,MAAM,QAAQ,MAAM,6BAA6B,CAAC,CAAC,EAC1D,QAAQ,MAAM,KAAK,QAAQ,CAAC;AAC/B,SAAK,iBAAiB,KAAK;AAAA,EAC7B;AAAA,EAEA,MAAM,QAAQ,MAAoC;AAChD,UAAM,KAAK,QAAQ;AACnB,WAAO,KAAK,KAAK,IAAI;AAAA,EACvB;AAAA,EAEA,MAAM,UAAU;AA9FlB;AA+FI,eAAW,eAAe,KAAK,cAAc;AAC3C,YAAM,YAAY,QAAQ;AAAA,IAC5B;AACA,eAAK,mBAAL,mBAAqB;AAAA,EACvB;AACF;;;ACpGA,YAAY,YAAY;;;ACuBtB,mBAAgB;AAAA,EACd,uBAAuB;AAAA,EACvB,2BAA2B;AAC7B;;;ACxBF,IAA8B,mBAA9B,MAAgF;AAAA,EAAhF;AASE,SAAU,aAAuB;AAAA;AAAA,EAEjC,MAAM,KAAK;AAAA,IACT;AAAA,IACA,cAAc;AAAA,EAChB,GAA+C;AAhBjD;AAiBI,QAAI,EAAE,sBAAsB,mBAAmB;AAC7C,YAAM,UAAU,qDAAqD;AAAA,IACvE;AACA,SAAK,cAAc,IAAI,gBAAgB;AAAA,MACrC,WAAW,CAAC,OAAO,eAAe,KAAK,UAAU,OAAO,UAAU;AAAA,IACpE,CAAC;AACD,SAAK,SAAS,gBAAgB;AAC9B,QAAI,cAAc;AAChB,WAAK,QAAM,UAAK,WAAL,mBAAa,WAAW,UAAS;AAAA,IAC9C;AACA,SAAK,aAAa;AAClB,SAAK,aAAa;AAAA,EACpB;AAAA,EAEA,MAAM,QAAQ,EAAE,cAAc,cAAc,WAAW,GAAgC;AACrF,SAAK,SAAS,gBAAgB;AAC9B,SAAK,MAAM,KAAK,OAAO,WAAW,IAAI,KAAK;AAE3C,SAAK,aAAa;AAClB,SAAK,aAAa;AAAA,EACpB;AAAA,EAEA,MAAM,UAAU;AACd,SAAK,aAAa;AAClB,SAAK,SAAS;AACd,SAAK,MAAM;AAAA,EACb;AAMF;;;AFpCA,IAAqB,sBAArB,cAAiD,iBAAiB;AAAA,EAehE,YAAY,MAAyB;AACnC,UAAM;AAPR,2BAAsC;AAQpC,SAAK,UAAU;AACf,QAAI,KAAK,YAAY;AACnB,WAAK,aAAa,KAAK;AAAA,IACzB,WAAW,KAAK,WAAW;AACzB,WAAK,eAAe,KAAK,SAAS;AAAA,IACpC;AAAA,EACF;AAAA,EAtBA,WAAW,cAAc;AACvB,WAAO,OAAO,oBAAoB;AAAA,EACpC;AAAA,EAsBA,MAAM,KAAK,EAAE,cAAc,cAAc,WAAW,GAAgC;AAClF,UAAM,MAAM,KAAK,EAAE,cAAc,cAAc,WAAW,CAAC;AAE3D,UAAM,UAAU,MAAa,uBAAgB;AAAA,MAC3C,wDAAwD,aAAa,yBAAyB,CAAC;AAAA,IACjG;AAEA,SAAK,iBAAiB,MAAa,sBAAe,kBAAkB,SAAS;AAAA,MAC3E,aAAa;AAAA,QACX,gBACE;AAAA,QACF,UAAU;AAAA,SACP,KAAK,QAAQ;AAAA,MAElB,aAAa;AAAA,MACb,oBAAoB;AAAA,MACpB,uBAAuB;AAAA,IACzB,CAAC;AAAA,EAGH;AAAA,EAEA,MAAM,UAAU;AA5DlB;AA6DI,UAAM,MAAM,QAAQ;AACpB,YAAM,UAAK,mBAAL,mBAAqB;AAC3B,SAAK,kBAAkB;AAAA,EACzB;AAAA,EAEA,MAAM,eAAe,MAAc;AACjC,UAAM,MAAM,IAAI,MAAM;AAEtB,UAAM,IAAI,QAAQ,CAAC,SAAS,WAAW;AACrC,UAAI,cAAc;AAClB,UAAI,SAAS,MAAM,QAAQ,GAAG;AAC9B,UAAI,UAAU,CAAC,QAAQ,OAAO,GAAG;AACjC,UAAI,MAAM;AAAA,IACZ,CAAC;AACD,UAAM,YAAY,MAAM,kBAAkB,GAAG;AAC7C,SAAK,kBAAkB;AAAA,EACzB;AAAA,EAEA,MAAM,UAAU,OAAmB,YAA0D;AA/E/F;AAgFI,QAAI;AACF,UAAI,KAAK,YAAY;AACnB,mBAAW,QAAQ,KAAK;AACxB;AAAA,MACF;AACA,UAAI,CAAC,KAAK,QAAQ;AAChB,cAAM,UAAU,sCAAsC;AAAA,MACxD;AACA,UAAI,cAAc,YAAY,IAAI;AAClC,iBAAK,mBAAL,mBAAqB;AAAA,QACnB,KAAK;AAAA,QACL;AAAA,QACA,CAAC,WAAY,KAAK,sBAAsB;AAAA;AAG1C,UAAI,KAAK,YAAY;AACnB,cAAM,KAAK,eAAe,KAAK;AAAA,MACjC,OAAO;AACL,cAAM,KAAK,sBAAsB,KAAK;AAAA,MACxC;AACA,YAAM,WAAW,IAAI,WAAW,KAAK,QAAQ;AAAA,QAC3C,WAAW,MAAM,aAAa,KAAK,IAAI;AAAA,MACzC,CAAC;AACD,iBAAW,QAAQ,QAAQ;AAAA,IAC7B,UAAE;AACA,YAAM,MAAM;AAAA,IACd;AAAA,EACF;AAAA,EAEA,MAAM,sBAAsB,OAAmB;AA7GjD;AA8GI,QAAI,CAAC,KAAK,UAAU,CAAC,KAAK,OAAO,CAAC,KAAK,uBAAuB,CAAC,KAAK;AAAY;AAGhF,SAAI,UAAK,wBAAL,mBAA0B,cAAc;AAC1C,WAAK,IAAI,SAAS;AAClB,WAAK,IAAI,2BAA2B;AACpC,YAAM,SAAS,MAAM;AAAA,QACnB,KAAK,oBAAoB;AAAA,QACzB,KAAK,oBAAoB,aAAa;AAAA,QACtC,KAAK,oBAAoB,aAAa;AAAA,MACxC;AACA,WAAK,IAAI,UAAU,QAAQ,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AACtE,WAAK,IAAI,SAAS;AAClB,WAAK,IAAI,2BAA2B;AACpC,UAAI,KAAK,iBAAiB;AACxB,aAAK,IAAI;AAAA,UACP,KAAK;AAAA,UACL;AAAA,UACA;AAAA,UACA,KAAK,gBAAgB;AAAA,UACrB,KAAK,gBAAgB;AAAA,UACrB;AAAA,UACA;AAAA,UACA,KAAK,OAAO;AAAA,UACZ,KAAK,OAAO;AAAA,QACd;AAAA,MACF,OAAO;AACL,aAAK,IAAI,YAAY;AACrB,aAAK,IAAI,SAAS,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AAAA,MAC/D;AAEA,WAAK,IAAI,2BAA2B;AAAA,IACtC;AACA,SAAK,IAAI,UAAU,OAAO,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AAAA,EAEvE;AAAA,EAEA,MAAM,eAAe,OAAmB;AAnJ1C;AAoJI,QACE,CAAC,KAAK,OACN,CAAC,KAAK,UACN,GAAC,gBAAK,wBAAL,mBAA0B,iBAA1B,mBAAwC,WACzC,CAAC,KAAK,YACN;AACA;AAAA,IACF;AAEA,SAAK,IAAI,KAAK;AACd,SAAK,IAAI,2BAA2B;AAEpC,UAAM,SAAS,MAAM;AAAA,MACnB,KAAK,oBAAoB;AAAA,MACzB,KAAK,oBAAoB,aAAa;AAAA,MACtC,KAAK,oBAAoB,aAAa;AAAA,IACxC;AAEA,SAAK,IAAI,SAAS;AAClB,SAAK,IAAI,2BAA2B;AACpC,SAAK,IAAI,UAAU,QAAQ,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AACtE,SAAK,IAAI,SAAS;AAClB,SAAK,IAAI,2BAA2B;AACpC,SAAK,IAAI,UAAU,OAAO,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AACrE,SAAK,IAAI,2BAA2B;AACpC,SAAK,IAAI,SAAS,QAAQ,KAAK,UAAU;AACzC,SAAK,IAAI,UAAU,OAAO,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AACrE,SAAK,IAAI,QAAQ;AAAA,EACnB;AACF;AAEA,SAAS,aACP,MACA,YACA,aACsB;AACtB,QAAM,YAA+B,IAAI,kBAAkB,aAAa,cAAc,CAAC;AACvF,QAAM,SAAS,KAAK,gBAAgB;AACpC,WAAS,IAAI,GAAG,IAAI,OAAO,QAAQ,KAAK,GAAG;AACzC,cAAU,IAAI,CAAC,IAAI,OAAO,CAAC;AAC3B,cAAU,IAAI,IAAI,CAAC,IAAI,OAAO,CAAC;AAC/B,cAAU,IAAI,IAAI,CAAC,IAAI,OAAO,CAAC;AAC/B,cAAU,IAAI,IAAI,CAAC,IAAI,OAAO,CAAC;AAAA,EACjC;AACA,QAAM,UAAU,IAAI,UAAU,WAAW,YAAY,WAAW;AAEhE,SAAO,kBAAkB,OAAO;AAClC;;;AGjMA,IAAqB,mBAArB,cAA8C,iBAAiB;AAAA,EAC7D,MAAM,UAAU,OAAmB,YAA0D;AAC3F,eAAW,QAAQ,KAAK;AAAA,EAC1B;AAAA,EAEA,MAAM,UAAU;AAAA,EAEhB;AACF;;;ACJO,IAAM,iBAAiB,CAC5B,aAAqB,IACrB,qBACG;AACH,QAAM,sBAAsB,kBAAkB,eAAe,oBAAsB;AACnF,MAAI,CAAC,qBAAqB;AACxB,UAAM,IAAI,MAAM,2CAA2C;AAAA,EAC7D;AACA,QAAM,WAAW,IAAI;AAAA,IACnB,CAAC,IAAI,oBAAsB,EAAE,YAAY,iBAAiB,CAAC,CAAC;AAAA,IAC5D;AAAA,EACF;AACA,SAAO;AACT;AAEO,IAAM,oBAAoB,CAAC,WAAmB,qBAA4C;AAC/F,QAAM,sBAAsB,kBAAkB,eAAe,oBAAsB;AACnF,MAAI,CAAC,qBAAqB;AACxB,UAAM,IAAI,MAAM,2CAA2C;AAAA,EAC7D;AACA,QAAM,WAAW,IAAI;AAAA,IACnB,CAAC,IAAI,oBAAsB,EAAE,WAAW,iBAAiB,CAAC,CAAC;AAAA,IAC3D;AAAA,EACF;AACA,SAAO;AACT;AAEO,IAAM,QAAQ,MAAM;AACzB,QAAM,sBAAsB,kBAAkB,eAAe,oBAAsB;AACnF,MAAI,CAAC,qBAAqB;AACxB,UAAM,IAAI,MAAM,2CAA2C;AAAA,EAC7D;AACA,QAAM,WAAW,IAAI,kBAAkB,CAAC,IAAI,iBAAiB,CAAC,GAAG,OAAO;AACxE,SAAO;AACT;","names":[]}
package/dist/src/ProcessorPipeline.d.ts CHANGED
@@ -14,6 +14,8 @@ export default class ProcessorPipeline implements TrackProcessor<Track.Kind> {
14
14
  processedTrack?: MediaStreamTrack;
15
15
  transformers: Array<VideoTrackTransformer>;
16
16
  constructor(transformers: Array<VideoTrackTransformer>, name: string);
17
+ private setup;
17
18
  init(opts: ProcessorOptions<Track.Kind>): Promise<void>;
19
+ restart(opts: ProcessorOptions<Track.Kind>): Promise<void>;
18
20
  destroy(): Promise<void>;
19
21
  }
package/dist/src/index.d.ts CHANGED
@@ -1,5 +1,6 @@
1
1
  import ProcessorPipeline from './ProcessorPipeline';
2
2
  import { SegmenterBaseOptions } from './transformers/BackgroundTransformer';
3
+ export * from './transformers/types';
3
4
  export declare const BackgroundBlur: (blurRadius?: number, segmenterOptions?: SegmenterBaseOptions) => ProcessorPipeline;
4
5
  export declare const VirtualBackground: (imagePath: string, segmenterOptions?: SegmenterBaseOptions) => ProcessorPipeline;
5
6
  export declare const Dummy: () => ProcessorPipeline;
package/dist/src/transformers/VideoTransformer.d.ts CHANGED
@@ -7,6 +7,7 @@ export default abstract class VideoTransformer implements VideoTrackTransformer
7
7
  inputVideo?: HTMLVideoElement;
8
8
  protected isDisabled?: Boolean;
9
9
  init({ outputCanvas, inputElement: inputVideo, }: VideoTransformerInitOptions): Promise<void>;
10
+ restart({ outputCanvas, inputElement: inputVideo }: VideoTransformerInitOptions): Promise<void>;
10
11
  destroy(): Promise<void>;
11
12
  abstract transform(frame: VideoFrame, controller: TransformStreamDefaultController<VideoFrame>): void;
12
13
  }
package/dist/src/transformers/types.d.ts CHANGED
@@ -9,6 +9,7 @@ export interface VideoTransformerInitOptions extends TrackTransformerInitOptions
9
9
  export interface VideoTrackTransformer<T extends TrackTransformerInitOptions = VideoTransformerInitOptions> {
10
10
  init: (options: T) => void;
11
11
  destroy: () => void;
12
+ restart: (options: T) => void;
12
13
  transform: (frame: VideoFrame, controller: TransformStreamDefaultController) => void;
13
14
  transformer?: TransformStream;
14
15
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@livekit/track-processors",
3
- "version": "0.2.4",
3
+ "version": "0.2.6",
4
4
  "description": "LiveKit track processors",
5
5
  "main": "dist/index.js",
6
6
  "module": "dist/index.mjs",
@@ -11,12 +11,11 @@
11
11
  "license": "Apache-2.0",
12
12
  "scripts": {
13
13
  "build": "tsup --onSuccess \"tsc --declaration --emitDeclarationOnly\"",
14
- "build-docs": "yarn exec typedoc",
15
- "build-sample": "cd example && webpack && cp styles.css index.html dist/",
14
+ "build-sample": "cd example && vite build",
16
15
  "lint": "eslint src",
16
+ "release": "yarn build && changeset publish",
17
17
  "test": "jest",
18
- "sample": "vite serve example --port 8080 --open",
19
- "prepublish": "yarn build"
18
+ "sample": "vite serve example --port 8080 --open"
20
19
  },
21
20
  "files": [
22
21
  "dist",
@@ -30,30 +29,21 @@
30
29
  "livekit-client": "^1.12.0"
31
30
  },
32
31
  "devDependencies": {
32
+ "@changesets/cli": "^2.26.2",
33
+ "@livekit/changesets-changelog-github": "^0.0.4",
33
34
  "@trivago/prettier-plugin-sort-imports": "^4.1.1",
34
35
  "@types/dom-mediacapture-transform": "^0.1.5",
35
- "@types/jest": "^27.0.3",
36
36
  "@types/offscreencanvas": "^2019.7.0",
37
37
  "@typescript-eslint/eslint-plugin": "^4.31.2",
38
- "@webpack-cli/serve": "^1.5.2",
39
38
  "eslint": "8.39.0",
40
39
  "eslint-config-airbnb-typescript": "17.0.0",
41
40
  "eslint-config-prettier": "8.8.0",
42
41
  "eslint-plugin-ecmascript-compat": "^3.0.0",
43
42
  "eslint-plugin-import": "2.27.5",
44
- "jest": "^27.4.3",
45
- "livekit-client": "^0.16.6",
43
+ "livekit-client": "^1.12.0",
46
44
  "prettier": "^2.8.8",
47
- "ts-jest": "^27.0.7",
48
- "ts-loader": "^8.1.0",
49
- "ts-proto": "^1.85.0",
50
45
  "tsup": "^7.1.0",
51
- "typedoc": "^0.20.35",
52
- "typedoc-plugin-no-inherit": "1.3.0",
53
46
  "typescript": "^5.0.4",
54
- "vite": "^4.3.8",
55
- "webpack": "^5.53.0",
56
- "webpack-cli": "^4.8.0",
57
- "webpack-dev-server": "^4.2.1"
47
+ "vite": "^4.3.8"
58
48
  }
59
49
  }
package/src/ProcessorPipeline.ts CHANGED
@@ -32,7 +32,7 @@ export default class ProcessorPipeline implements TrackProcessor<Track.Kind> {
32
32
  this.transformers = transformers;
33
33
  }
34
34
 
35
- async init(opts: ProcessorOptions<Track.Kind>) {
35
+ private async setup(opts: ProcessorOptions<Track.Kind>) {
36
36
  this.source = opts.track as MediaStreamVideoTrack;
37
37
  const origConstraints = this.source.getConstraints();
38
38
  await this.source.applyConstraints({
@@ -46,8 +46,8 @@ export default class ProcessorPipeline implements TrackProcessor<Track.Kind> {
46
46
  this.sourceSettings = this.source.getSettings();
47
47
  this.sourceDummy = opts.element;
48
48
  if (this.sourceDummy instanceof HTMLVideoElement) {
49
- this.sourceDummy.height = this.sourceSettings.height!;
50
- this.sourceDummy.width = this.sourceSettings.width!;
49
+ this.sourceDummy.height = this.sourceSettings.height ?? 300;
50
+ this.sourceDummy.width = this.sourceSettings.width ?? 300;
51
51
  }
52
52
  if (!(this.sourceDummy instanceof HTMLVideoElement)) {
53
53
  throw TypeError('Currently only video transformers are supported');
@@ -64,12 +64,19 @@ export default class ProcessorPipeline implements TrackProcessor<Track.Kind> {
64
64
  this.sourceSettings.width ?? 300,
65
65
  this.sourceSettings.height ?? 300,
66
66
  );
67
+ }
68
+
69
+ async init(opts: ProcessorOptions<Track.Kind>) {
70
+ await this.setup(opts);
71
+ if (!this.canvas || !this.processor || !this.trackGenerator) {
72
+ throw new TypeError('Expected both canvas and processor to be defined after setup');
73
+ }
67
74
 
68
75
  let readableStream = this.processor.readable;
69
76
  for (const transformer of this.transformers) {
70
77
  await transformer.init({
71
78
  outputCanvas: this.canvas,
72
- inputElement: this.sourceDummy!,
79
+ inputElement: this.sourceDummy as HTMLVideoElement,
73
80
  });
74
81
  readableStream = readableStream.pipeThrough(transformer!.transformer!);
75
82
  }
@@ -80,6 +87,11 @@ export default class ProcessorPipeline implements TrackProcessor<Track.Kind> {
80
87
  this.processedTrack = this.trackGenerator as MediaStreamVideoTrack;
81
88
  }
82
89
 
90
+ async restart(opts: ProcessorOptions<Track.Kind>) {
91
+ await this.destroy();
92
+ return this.init(opts);
93
+ }
94
+
83
95
  async destroy() {
84
96
  for (const transformer of this.transformers) {
85
97
  await transformer.destroy();
package/src/index.ts CHANGED
@@ -2,6 +2,8 @@ import ProcessorPipeline from './ProcessorPipeline';
2
2
  import BackgroundTransformer, { SegmenterBaseOptions } from './transformers/BackgroundTransformer';
3
3
  import DummyTransformer from './transformers/DummyTransformer';
4
4
 
5
+ export * from './transformers/types';
6
+
5
7
  export const BackgroundBlur = (
6
8
  blurRadius: number = 10,
7
9
  segmenterOptions?: SegmenterBaseOptions,
package/src/transformers/BackgroundTransformer.ts CHANGED
@@ -34,8 +34,6 @@ export default class BackgroundProcessor extends VideoTransformer {
34
34
  } else if (opts.imagePath) {
35
35
  this.loadBackground(opts.imagePath);
36
36
  }
37
-
38
- this.options.segmenterOptions;
39
37
  }
40
38
 
41
39
  async init({ outputCanvas, inputElement: inputVideo }: VideoTransformerInitOptions) {
@@ -114,14 +112,14 @@ export default class BackgroundProcessor extends VideoTransformer {
114
112
  // this.ctx.save();
115
113
  // this.ctx.clearRect(0, 0, this.canvas.width, this.canvas.height);
116
114
  if (this.segmentationResults?.categoryMask) {
117
- this.ctx.filter = 'blur(3px)';
115
+ this.ctx.filter = 'blur(10px)';
118
116
  this.ctx.globalCompositeOperation = 'copy';
119
117
  const bitmap = await maskToBitmap(
120
118
  this.segmentationResults.categoryMask,
121
- this.inputVideo.videoWidth,
122
- this.inputVideo.videoHeight,
119
+ this.segmentationResults.categoryMask.width,
120
+ this.segmentationResults.categoryMask.height,
123
121
  );
124
- this.ctx.drawImage(bitmap, 0, 0);
122
+ this.ctx.drawImage(bitmap, 0, 0, this.canvas.width, this.canvas.height);
125
123
  this.ctx.filter = 'none';
126
124
  this.ctx.globalCompositeOperation = 'source-in';
127
125
  if (this.backgroundImage) {
@@ -162,19 +160,19 @@ export default class BackgroundProcessor extends VideoTransformer {
162
160
 
163
161
  const bitmap = await maskToBitmap(
164
162
  this.segmentationResults.categoryMask,
165
- this.inputVideo.videoWidth,
166
- this.inputVideo.videoHeight,
163
+ this.segmentationResults.categoryMask.width,
164
+ this.segmentationResults.categoryMask.height,
167
165
  );
168
166
 
169
167
  this.ctx.filter = 'blur(3px)';
170
168
  this.ctx.globalCompositeOperation = 'copy';
171
- this.ctx.drawImage(bitmap, 0, 0);
169
+ this.ctx.drawImage(bitmap, 0, 0, this.canvas.width, this.canvas.height);
172
170
  this.ctx.filter = 'none';
173
171
  this.ctx.globalCompositeOperation = 'source-out';
174
172
  this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);
175
173
  this.ctx.globalCompositeOperation = 'destination-over';
176
174
  this.ctx.filter = `blur(${this.blurRadius}px)`;
177
- this.ctx.drawImage(frame, 0, 0);
175
+ this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);
178
176
  this.ctx.restore();
179
177
  }
180
178
  }
package/src/transformers/VideoTransformer.ts CHANGED
@@ -29,6 +29,14 @@ export default abstract class VideoTransformer implements VideoTrackTransformer
29
29
  this.isDisabled = false;
30
30
  }
31
31
 
32
+ async restart({ outputCanvas, inputElement: inputVideo }: VideoTransformerInitOptions) {
33
+ this.canvas = outputCanvas || null;
34
+ this.ctx = this.canvas.getContext('2d') || undefined;
35
+
36
+ this.inputVideo = inputVideo;
37
+ this.isDisabled = false;
38
+ }
39
+
32
40
  async destroy() {
33
41
  this.isDisabled = true;
34
42
  this.canvas = undefined;
package/src/transformers/types.ts CHANGED
@@ -12,6 +12,7 @@ export interface VideoTrackTransformer<
12
12
  > {
13
13
  init: (options: T) => void;
14
14
  destroy: () => void;
15
+ restart: (options: T) => void;
15
16
  transform: (frame: VideoFrame, controller: TransformStreamDefaultController) => void;
16
17
  transformer?: TransformStream;
17
18
  }