@livekit/track-processors 0.4.0 → 0.4.1

package/dist/index.js

@@ -215,12 +215,15 @@ var BackgroundProcessor = class extends VideoTransformer {
       return;
     if (((_a = this.segmentationResults) == null ? void 0 : _a.categoryMask) && this.segmentationResults.categoryMask.width > 0) {
       this.ctx.globalCompositeOperation = "copy";
-      const bitmap = await maskToBitmap(
-        this.segmentationResults.categoryMask,
-        this.segmentationResults.categoryMask.width,
-        this.segmentationResults.categoryMask.height
+      this.ctx.putImageData(
+        maskToImageData(
+          this.segmentationResults.categoryMask,
+          this.segmentationResults.categoryMask.width,
+          this.segmentationResults.categoryMask.height
+        ),
+        0,
+        0
       );
-      this.ctx.drawImage(bitmap, 0, 0, this.canvas.width, this.canvas.height);
       this.ctx.filter = "none";
       this.ctx.globalCompositeOperation = "source-in";
       if (this.backgroundImage) {
@@ -251,13 +254,15 @@ var BackgroundProcessor = class extends VideoTransformer {
     this.ctx.save();
     this.ctx.globalCompositeOperation = "copy";
     if (((_c = this.segmentationResults) == null ? void 0 : _c.categoryMask) && this.segmentationResults.categoryMask.width > 0) {
-      const bitmap = await maskToBitmap(
-        this.segmentationResults.categoryMask,
-        this.segmentationResults.categoryMask.width,
-        this.segmentationResults.categoryMask.height
+      this.ctx.putImageData(
+        maskToImageData(
+          this.segmentationResults.categoryMask,
+          this.segmentationResults.categoryMask.width,
+          this.segmentationResults.categoryMask.height
+        ),
+        0,
+        0
       );
-      this.ctx.globalCompositeOperation = "copy";
-      this.ctx.drawImage(bitmap, 0, 0, this.canvas.width, this.canvas.height);
       this.ctx.filter = "none";
       this.ctx.globalCompositeOperation = "source-out";
       this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);
@@ -268,17 +273,17 @@ var BackgroundProcessor = class extends VideoTransformer {
     }
   }
 };
-function maskToBitmap(mask, videoWidth, videoHeight) {
+function maskToImageData(mask, videoWidth, videoHeight) {
   const dataArray = new Uint8ClampedArray(videoWidth * videoHeight * 4);
   const result = mask.getAsUint8Array();
   for (let i = 0; i < result.length; i += 1) {
-    dataArray[i * 4] = result[i];
-    dataArray[i * 4 + 1] = result[i];
-    dataArray[i * 4 + 2] = result[i];
-    dataArray[i * 4 + 3] = result[i];
+    const offset = i * 4;
+    dataArray[offset] = result[i];
+    dataArray[offset + 1] = result[i];
+    dataArray[offset + 2] = result[i];
+    dataArray[offset + 3] = result[i];
   }
-  const dataNew = new ImageData(dataArray, videoWidth, videoHeight);
-  return createImageBitmap(dataNew);
+  return new ImageData(dataArray, videoWidth, videoHeight);
 }
 
 // src/index.ts

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/utils.ts","../src/ProcessorWrapper.ts","../src/transformers/BackgroundTransformer.ts","../package.json","../src/transformers/VideoTransformer.ts","../src/index.ts"],"names":["BackgroundProcessor"],"mappings":";AAGA,eAAe,MAAM,MAAc;AACjC,SAAO,IAAI,QAAQ,CAAC,YAAY,WAAW,SAAS,IAAI,CAAC;AAC3D;AAEA,eAAsB,uBAAuB,OAAyB;AACpE,QAAM,UAAU;AAIhB,QAAM,MAAM,EAAE;AAEd,QAAM,UAAU,KAAK,IAAI;AACzB,SAAO,KAAK,IAAI,IAAI,UAAU,SAAS;AACrC,UAAM,EAAE,OAAO,OAAO,IAAI,MAAM,YAAY;AAC5C,QAAI,SAAS,QAAQ;AACnB,aAAO,EAAE,OAAO,OAAO;AAAA,IACzB;AACA,UAAM,MAAM,EAAE;AAAA,EAChB;AACA,SAAO,EAAE,OAAO,QAAW,QAAQ,OAAU;AAC/C;;;ACnBA,IAAqB,mBAArB,MAEA;AAAA,EACE,WAAW,cAAc;AACvB,WACE,OAAO,8BAA8B,eACrC,OAAO,8BAA8B;AAAA,EAEzC;AAAA,EAkBA,YAAY,aAAmD,MAAc;AAC3E,SAAK,OAAO;AACZ,SAAK,cAAc;AAAA,EACrB;AAAA,EAEA,MAAc,MAAM,MAAoC;AACtD,SAAK,SAAS,KAAK;AAEnB,UAAM,EAAE,OAAO,OAAO,IAAI,MAAM,uBAAuB,KAAK,MAAM;AAClE,SAAK,cAAc,KAAK;AAExB,QAAI,EAAE,KAAK,uBAAuB,mBAAmB;AACnD,YAAM,UAAU,iDAAiD;AAAA,IACnE;AAEA,QAAI,KAAK,uBAAuB,kBAAkB;AAChD,WAAK,YAAY,SAAS,0BAAU;AACpC,WAAK,YAAY,QAAQ,wBAAS;AAAA,IACpC;AAGA,SAAK,YAAY,IAAI,0BAA0B,EAAE,OAAO,KAAK,OAAO,CAAC;AAErE,SAAK,iBAAiB,IAAI,0BAA0B;AAAA,MAClD,MAAM;AAAA,MACN,cAAc,KAAK;AAAA,IACrB,CAAC;AAED,SAAK,SAAS,IAAI,gBAAgB,wBAAS,KAAK,0BAAU,GAAG;AAAA,EAC/D;AAAA,EAEA,MAAM,KAAK,MAAoC;AAC7C,UAAM,KAAK,MAAM,IAAI;AACrB,QAAI,CAAC,KAAK,UAAU,CAAC,KAAK,aAAa,CAAC,KAAK,gBAAgB;AAC3D,YAAM,IAAI,UAAU,8DAA8D;AAAA,IACpF;AAEA,UAAM,iBAAiB,KAAK,UAAU;AAEtC,UAAM,KAAK,YAAY,KAAK;AAAA,MAC1B,cAAc,KAAK;AAAA,MACnB,cAAc,KAAK;AAAA,IACrB,CAAC;AAED,UAAM,cAAc,eAAe,YAAY,KAAK,YAAa,WAAY;AAE7E,gBACG,OAAO,KAAK,eAAe,QAAQ,EACnC,MAAM,CAAC,MAAM,QAAQ,MAAM,6BAA6B,CAAC,CAAC,EAC1D,QAAQ,MAAM,KAAK,QAAQ,CAAC;AAC/B,SAAK,iBAAiB,KAAK;AAAA,EAC7B;AAAA,EAEA,MAAM,QAAQ,MAAoC;AAChD,UAAM,KAAK,QAAQ;AACnB,WAAO,KAAK,KAAK,IAAI;AAAA,EACvB;AAAA,EAEA,MAAM,sBAAsB,SAA2D;AAErF,SAAK,YAAY,QAAQ,QAAQ,CAAC,CAAC;AAAA,EACrC;AAAA,EAEA,MAAM,4BAA4B,SAA0D;AAC1F,SAAK,YAAY,OAAO,QAAQ,CAAC,CAAC;AAAA,EACpC;AAAA,EAEA,MAAM,UAAU;AAjGlB;AAkGI,UAAM,KAAK,YAAY,QAAQ;AAC/B,eAAK,mBAAL,mBAAqB;AAAA,EACvB;AACF;;;ACrGA,YAAY,YAAY;;;ACuBtB,mBAAgB;AAAA,EACd,2BAA2B;AAC7B;;;ACvBF,IAA8B,mBAA9B,MAEA;AAAA,EAFA;AAWE,SAAU,aAAuB;AAAA;AAAA,EAEjC,MAAM,KAAK;AAAA,IACT;AAAA,IACA,cAAc;AAAA,EAChB,GAA+C;AAlBjD;AAmBI,QAAI,EAAE,sBAAsB,mBAAmB;AAC7C,YAAM,UAAU,qDAAqD;AAAA,IACvE;AAEA,SAAK,cAAc,IAAI,gBAAgB;AAAA,MACrC,WAAW,CAAC,OAAO,eAAe,KAAK,UAAU,OAAO,UAAU;AAAA,IACpE,CAAC;AACD,SAAK,SAAS,gBAAgB;AAC9B,QAAI,cAAc;AAChB,WAAK,QAAM,UAAK,WAAL,mBAAa,WAAW,UAAS;AAAA,IAC9C;AACA,SAAK,aAAa;AAClB,SAAK,aAAa;AAAA,EACpB;AAAA,EAEA,MAAM,QAAQ,EAAE,cAAc,cAAc,WAAW,GAAgC;AACrF,SAAK,SAAS,gBAAgB;AAC9B,SAAK,MAAM,KAAK,OAAO,WAAW,IAAI,KAAK;AAE3C,SAAK,aAAa;AAClB,SAAK,aAAa;AAAA,EACpB;AAAA,EAEA,MAAM,UAAU;AACd,SAAK,aAAa;AAClB,SAAK,SAAS;AACd,SAAK,MAAM;AAAA,EACb;AAQF;;;AFtCA,IAAqB,sBAArB,cAAiD,iBAAoC;AAAA,EAenF,YAAY,MAAyB;AACnC,UAAM;AAPR,2BAAsC;AAQpC,SAAK,UAAU;AACf,SAAK,OAAO,IAAI;AAAA,EAClB;AAAA,EAlBA,WAAW,cAAc;AACvB,WAAO,OAAO,oBAAoB;AAAA,EACpC;AAAA,EAkBA,MAAM,KAAK,EAAE,cAAc,cAAc,WAAW,GAAgC;AArCtF;AAsCI,UAAM,MAAM,KAAK,EAAE,cAAc,cAAc,WAAW,CAAC;AAE3D,UAAM,UAAU,MAAa,uBAAgB;AAAA,OAC3C,gBAAK,QAAQ,eAAb,mBAAyB,uBAAzB,YACE,wDAAwD,aAAa,yBAAyB,CAAC;AAAA,IACnG;AAEA,SAAK,iBAAiB,MAAa,sBAAe,kBAAkB,SAAS;AAAA,MAC3E,aAAa;AAAA,QACX,iBACE,gBAAK,QAAQ,eAAb,mBAAyB,mBAAzB,YACA;AAAA,QACF,UAAU;AAAA,QACV,GAAG,KAAK,QAAQ;AAAA,MAClB;AAAA,MACA,aAAa;AAAA,MACb,oBAAoB;AAAA,MACpB,uBAAuB;AAAA,IACzB,CAAC;AAGD,UAAI,UAAK,YAAL,mBAAc,cAAa,CAAC,KAAK,iBAAiB;AACpD,YAAM,KAAK,eAAe,KAAK,QAAQ,SAAS,EAAE;AAAA,QAAM,CAAC,QACvD,QAAQ,MAAM,oDAAoD,GAAG;AAAA,MACvE;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,UAAU;AAlElB;AAmEI,UAAM,MAAM,QAAQ;AACpB,YAAM,UAAK,mBAAL,mBAAqB;AAC3B,SAAK,kBAAkB;AAAA,EACzB;AAAA,EAEA,MAAM,eAAe,MAAc;AACjC,UAAM,MAAM,IAAI,MAAM;AAEtB,UAAM,IAAI,QAAQ,CAAC,SAAS,WAAW;AACrC,UAAI,cAAc;AAClB,UAAI,SAAS,MAAM,QAAQ,GAAG;AAC9B,UAAI,UAAU,CAAC,QAAQ,OAAO,GAAG;AACjC,UAAI,MAAM;AAAA,IACZ,CAAC;AACD,UAAM,YAAY,MAAM,kBAAkB,GAAG;AAC7C,SAAK,kBAAkB;AAAA,EACzB;AAAA,EAEA,MAAM,UAAU,OAAmB,YAA0D;AArF/F;AAsFI,QAAI;AACF,UAAI,EAAE,iBAAiB,aAAa;AAClC,gBAAQ,MAAM,gCAAgC;AAC9C;AAAA,MACF;AACA,UAAI,KAAK,YAAY;AACnB,mBAAW,QAAQ,KAAK;AACxB;AAAA,MACF;AACA,UAAI,CAAC,KAAK,QAAQ;AAChB,cAAM,UAAU,sCAAsC;AAAA,MACxD;AACA,UAAI,cAAc,YAAY,IAAI;AAClC,iBAAK,mBAAL,mBAAqB;AAAA,QACnB,KAAK;AAAA,QACL;AAAA,QACA,CAAC,WAAY,KAAK,sBAAsB;AAAA;AAG1C,UAAI,KAAK,YAAY;AACnB,cAAM,KAAK,eAAe,KAAK;AAAA,MACjC,OAAO;AACL,cAAM,KAAK,sBAAsB,KAAK;AAAA,MACxC;AACA,YAAM,WAAW,IAAI,WAAW,KAAK,QAAQ;AAAA,QAC3C,WAAW,MAAM,aAAa,KAAK,IAAI;AAAA,MACzC,CAAC;AACD,iBAAW,QAAQ,QAAQ;AAAA,IAC7B,UAAE;AACA,qCAAO;AAAA,IACT;AAAA,EACF;AAAA,EAEA,MAAM,OAAO,MAAyB;AACpC,SAAK,UAAU;AACf,QAAI,KAAK,YAAY;AACnB,WAAK,aAAa,KAAK;AAAA,IACzB,WAAW,KAAK,WAAW;AACzB,YAAM,KAAK,eAAe,KAAK,SAAS;AAAA,IAC1C;AAAA,EACF;AAAA,EAEA,MAAM,sBAAsB,OAAmB;AAhIjD;AAiII,QAAI,CAAC,KAAK,UAAU,CAAC,KAAK,OAAO,CAAC,KAAK,uBAAuB,CAAC,KAAK;AAAY;AAGhF,UAAI,UAAK,wBAAL,mBAA0B,iBAAgB,KAAK,oBAAoB,aAAa,QAAQ,GAAG;AAC7F,WAAK,IAAI,2BAA2B;AACpC,YAAM,SAAS,MAAM;AAAA,QACnB,KAAK,oBAAoB;AAAA,QACzB,KAAK,oBAAoB,aAAa;AAAA,QACtC,KAAK,oBAAoB,aAAa;AAAA,MACxC;AACA,WAAK,IAAI,UAAU,QAAQ,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AACtE,WAAK,IAAI,SAAS;AAClB,WAAK,IAAI,2BAA2B;AACpC,UAAI,KAAK,iBAAiB;AACxB,aAAK,IAAI;AAAA,UACP,KAAK;AAAA,UACL;AAAA,UACA;AAAA,UACA,KAAK,gBAAgB;AAAA,UACrB,KAAK,gBAAgB;AAAA,UACrB;AAAA,UACA;AAAA,UACA,KAAK,OAAO;AAAA,UACZ,KAAK,OAAO;AAAA,QACd;AAAA,MACF,OAAO;AACL,aAAK,IAAI,YAAY;AACrB,aAAK,IAAI,SAAS,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AAAA,MAC/D;AAEA,WAAK,IAAI,2BAA2B;AAAA,IACtC;AACA,SAAK,IAAI,UAAU,OAAO,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AAAA,EAEvE;AAAA,EAEA,MAAM,eAAe,OAAmB;AArK1C;AAsKI,QACE,CAAC,KAAK,OACN,CAAC,KAAK,UACN,GAAC,gBAAK,wBAAL,mBAA0B,iBAA1B,mBAAwC,WACzC,CAAC,KAAK,YACN;AACA;AAAA,IACF;AAEA,SAAK,IAAI,KAAK;AACd,SAAK,IAAI,2BAA2B;AAEpC,UAAI,UAAK,wBAAL,mBAA0B,iBAAgB,KAAK,oBAAoB,aAAa,QAAQ,GAAG;AAC7F,YAAM,SAAS,MAAM;AAAA,QACnB,KAAK,oBAAoB;AAAA,QACzB,KAAK,oBAAoB,aAAa;AAAA,QACtC,KAAK,oBAAoB,aAAa;AAAA,MACxC;AAEA,WAAK,IAAI,2BAA2B;AACpC,WAAK,IAAI,UAAU,QAAQ,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AACtE,WAAK,IAAI,SAAS;AAClB,WAAK,IAAI,2BAA2B;AACpC,WAAK,IAAI,UAAU,OAAO,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AACrE,WAAK,IAAI,2BAA2B;AACpC,WAAK,IAAI,SAAS,QAAQ,KAAK,UAAU;AACzC,WAAK,IAAI,UAAU,OAAO,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AACrE,WAAK,IAAI,QAAQ;AAAA,IACnB;AAAA,EACF;AACF;AAEA,SAAS,aACP,MACA,YACA,aACsB;AACtB,QAAM,YAA+B,IAAI,kBAAkB,aAAa,cAAc,CAAC;AACvF,QAAM,SAAS,KAAK,gBAAgB;AACpC,WAAS,IAAI,GAAG,IAAI,OAAO,QAAQ,KAAK,GAAG;AACzC,cAAU,IAAI,CAAC,IAAI,OAAO,CAAC;AAC3B,cAAU,IAAI,IAAI,CAAC,IAAI,OAAO,CAAC;AAC/B,cAAU,IAAI,IAAI,CAAC,IAAI,OAAO,CAAC;AAC/B,cAAU,IAAI,IAAI,CAAC,IAAI,OAAO,CAAC;AAAA,EACjC;AACA,QAAM,UAAU,IAAI,UAAU,WAAW,YAAY,WAAW;AAEhE,SAAO,kBAAkB,OAAO;AAClC;;;AG5MO,IAAM,iBAAiB,CAAC,aAAqB,IAAI,qBAAwC;AAC9F,SAAOA,qBAAoB,EAAE,YAAY,iBAAiB,GAAG,iBAAiB;AAChF;AAEO,IAAM,oBAAoB,CAAC,WAAmB,qBAAwC;AAC3F,SAAOA,qBAAoB,EAAE,WAAW,iBAAiB,GAAG,oBAAoB;AAClF;AAEO,IAAMA,uBAAsB,CAAC,SAA4B,OAAO,2BAA2B;AAChG,QAAM,uBAAuB,iBAAiB,eAAe,oBAAsB;AACnF,MAAI,CAAC,sBAAsB;AACzB,UAAM,IAAI,MAAM,4CAA4C;AAAA,EAC9D;AACA,QAAM,YAAY,IAAI,iBAAiB,IAAI,oBAAsB,OAAO,GAAG,IAAI;AAC/E,SAAO;AACT","sourcesContent":["export const supportsProcessor = typeof MediaStreamTrackGenerator !== 'undefined';\nexport const supportsOffscreenCanvas = typeof OffscreenCanvas !== 'undefined';\n\nasync function sleep(time: number) {\n  return new Promise((resolve) => setTimeout(resolve, time));\n}\n\nexport async function waitForTrackResolution(track: MediaStreamTrack) {\n  const timeout = 500;\n\n  // browsers report wrong initial resolution on iOS.\n  // when slightly delaying the call to .getSettings(), the correct resolution is being reported\n  await sleep(10);\n\n  const started = Date.now();\n  while (Date.now() - started < timeout) {\n    const { width, height } = track.getSettings();\n    if (width && height) {\n      return { width, height };\n    }\n    await sleep(50);\n  }\n  return { width: undefined, height: undefined };\n}\n","import type { ProcessorOptions, Track, TrackProcessor } from 'livekit-client';\nimport { TrackTransformer } from './transformers';\nimport { waitForTrackResolution } from './utils';\n\nexport default class ProcessorWrapper<TransformerOptions extends Record<string, unknown>>\n  implements TrackProcessor<Track.Kind>\n{\n  static get isSupported() {\n    return (\n      typeof MediaStreamTrackGenerator !== 'undefined' &&\n      typeof MediaStreamTrackProcessor !== 'undefined'\n    );\n  }\n\n  name: string;\n\n  source?: MediaStreamVideoTrack;\n\n  processor?: MediaStreamTrackProcessor<VideoFrame>;\n\n  trackGenerator?: MediaStreamTrackGenerator<VideoFrame>;\n\n  canvas?: OffscreenCanvas;\n\n  sourceDummy?: HTMLMediaElement;\n\n  processedTrack?: MediaStreamTrack;\n\n  transformer: TrackTransformer<TransformerOptions>;\n\n  constructor(transformer: TrackTransformer<TransformerOptions>, name: string) {\n    this.name = name;\n    this.transformer = transformer;\n  }\n\n  private async setup(opts: ProcessorOptions<Track.Kind>) {\n    this.source = opts.track as MediaStreamVideoTrack;\n\n    const { width, height } = await waitForTrackResolution(this.source);\n    this.sourceDummy = opts.element;\n\n    if (!(this.sourceDummy instanceof HTMLVideoElement)) {\n      throw TypeError('Currently only video transformers are supported');\n    }\n\n    if (this.sourceDummy instanceof HTMLVideoElement) {\n      this.sourceDummy.height = height ?? 300;\n      this.sourceDummy.width = width ?? 300;\n    }\n\n    // TODO explore if we can do all the processing work in a webworker\n    this.processor = new MediaStreamTrackProcessor({ track: this.source });\n\n    this.trackGenerator = new MediaStreamTrackGenerator({\n      kind: 'video',\n      signalTarget: this.source,\n    });\n\n    this.canvas = new OffscreenCanvas(width ?? 300, height ?? 300);\n  }\n\n  async init(opts: ProcessorOptions<Track.Kind>) {\n    await this.setup(opts);\n    if (!this.canvas || !this.processor || !this.trackGenerator) {\n      throw new TypeError('Expected both canvas and processor to be defined after setup');\n    }\n\n    const readableStream = this.processor.readable;\n\n    await this.transformer.init({\n      outputCanvas: this.canvas,\n      inputElement: this.sourceDummy as HTMLVideoElement,\n    });\n\n    const pipedStream = readableStream.pipeThrough(this.transformer!.transformer!);\n\n    pipedStream\n      .pipeTo(this.trackGenerator.writable)\n      .catch((e) => console.error('error when trying to pipe', e))\n      .finally(() => this.destroy());\n    this.processedTrack = this.trackGenerator as MediaStreamVideoTrack;\n  }\n\n  async restart(opts: ProcessorOptions<Track.Kind>) {\n    await this.destroy();\n    return this.init(opts);\n  }\n\n  async restartTransformer(...options: Parameters<(typeof this.transformer)['restart']>) {\n    // @ts-ignore unclear why the restart method only accepts VideoTransformerInitOptions instead of either those or AudioTransformerInitOptions\n    this.transformer.restart(options[0]);\n  }\n\n  async updateTransformerOptions(...options: Parameters<(typeof this.transformer)['update']>) {\n    this.transformer.update(options[0]);\n  }\n\n  async destroy() {\n    await this.transformer.destroy();\n    this.trackGenerator?.stop();\n  }\n}\n","import * as vision from '@mediapipe/tasks-vision';\nimport { dependencies } from '../../package.json';\nimport VideoTransformer from './VideoTransformer';\nimport { VideoTransformerInitOptions } from './types';\n\nexport type SegmenterOptions = Partial<vision.ImageSegmenterOptions['baseOptions']>;\n\nexport type BackgroundOptions = {\n  blurRadius?: number;\n  imagePath?: string;\n  /** cannot be updated through the `update` method, needs a restart */\n  segmenterOptions?: SegmenterOptions;\n  /** cannot be updated through the `update` method, needs a restart */\n  assetPaths?: { tasksVisionFileSet?: string; modelAssetPath?: string };\n};\n\nexport default class BackgroundProcessor extends VideoTransformer<BackgroundOptions> {\n  static get isSupported() {\n    return typeof OffscreenCanvas !== 'undefined';\n  }\n\n  imageSegmenter?: vision.ImageSegmenter;\n\n  segmentationResults: vision.ImageSegmenterResult | undefined;\n\n  backgroundImage: ImageBitmap | null = null;\n\n  blurRadius?: number;\n\n  options: BackgroundOptions;\n\n  constructor(opts: BackgroundOptions) {\n    super();\n    this.options = opts;\n    this.update(opts);\n  }\n\n  async init({ outputCanvas, inputElement: inputVideo }: VideoTransformerInitOptions) {\n    await super.init({ outputCanvas, inputElement: inputVideo });\n\n    const fileSet = await vision.FilesetResolver.forVisionTasks(\n      this.options.assetPaths?.tasksVisionFileSet ??\n        `https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@${dependencies['@mediapipe/tasks-vision']}/wasm`,\n    );\n\n    this.imageSegmenter = await vision.ImageSegmenter.createFromOptions(fileSet, {\n      baseOptions: {\n        modelAssetPath:\n          this.options.assetPaths?.modelAssetPath ??\n          'https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_segmenter/float16/latest/selfie_segmenter.tflite',\n        delegate: 'GPU',\n        ...this.options.segmenterOptions,\n      },\n      runningMode: 'VIDEO',\n      outputCategoryMask: true,\n      outputConfidenceMasks: false,\n    });\n\n    // Skip loading the image here if update already loaded the image below\n    if (this.options?.imagePath && !this.backgroundImage) {\n      await this.loadBackground(this.options.imagePath).catch((err) =>\n        console.error('Error while loading processor background image: ', err),\n      );\n    }\n  }\n\n  async destroy() {\n    await super.destroy();\n    await this.imageSegmenter?.close();\n    this.backgroundImage = null;\n  }\n\n  async loadBackground(path: string) {\n    const img = new Image();\n\n    await new Promise((resolve, reject) => {\n      img.crossOrigin = 'Anonymous';\n      img.onload = () => resolve(img);\n      img.onerror = (err) => reject(err);\n      img.src = path;\n    });\n    const imageData = await createImageBitmap(img);\n    this.backgroundImage = imageData;\n  }\n\n  async transform(frame: VideoFrame, controller: TransformStreamDefaultController<VideoFrame>) {\n    try {\n      if (!(frame instanceof VideoFrame)) {\n        console.debug('empty frame detected, ignoring');\n        return;\n      }\n      if (this.isDisabled) {\n        controller.enqueue(frame);\n        return;\n      }\n      if (!this.canvas) {\n        throw TypeError('Canvas needs to be initialized first');\n      }\n      let startTimeMs = performance.now();\n      this.imageSegmenter?.segmentForVideo(\n        this.inputVideo!,\n        startTimeMs,\n        (result) => (this.segmentationResults = result),\n      );\n\n      if (this.blurRadius) {\n        await this.blurBackground(frame);\n      } else {\n        await this.drawVirtualBackground(frame);\n      }\n      const newFrame = new VideoFrame(this.canvas, {\n        timestamp: frame.timestamp || Date.now(),\n      });\n      controller.enqueue(newFrame);\n    } finally {\n      frame?.close();\n    }\n  }\n\n  async update(opts: BackgroundOptions) {\n    this.options = opts;\n    if (opts.blurRadius) {\n      this.blurRadius = opts.blurRadius;\n    } else if (opts.imagePath) {\n      await this.loadBackground(opts.imagePath);\n    }\n  }\n\n  async drawVirtualBackground(frame: VideoFrame) {\n    if (!this.canvas || !this.ctx || !this.segmentationResults || !this.inputVideo) return;\n    // this.ctx.save();\n    // this.ctx.clearRect(0, 0, this.canvas.width, this.canvas.height);\n    if (this.segmentationResults?.categoryMask && this.segmentationResults.categoryMask.width > 0) {\n      this.ctx.globalCompositeOperation = 'copy';\n      const bitmap = await maskToBitmap(\n        this.segmentationResults.categoryMask,\n        this.segmentationResults.categoryMask.width,\n        this.segmentationResults.categoryMask.height,\n      );\n      this.ctx.drawImage(bitmap, 0, 0, this.canvas.width, this.canvas.height);\n      this.ctx.filter = 'none';\n      this.ctx.globalCompositeOperation = 'source-in';\n      if (this.backgroundImage) {\n        this.ctx.drawImage(\n          this.backgroundImage,\n          0,\n          0,\n          this.backgroundImage.width,\n          this.backgroundImage.height,\n          0,\n          0,\n          this.canvas.width,\n          this.canvas.height,\n        );\n      } else {\n        this.ctx.fillStyle = '#00FF00';\n        this.ctx.fillRect(0, 0, this.canvas.width, this.canvas.height);\n      }\n\n      this.ctx.globalCompositeOperation = 'destination-over';\n    }\n    this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);\n    // this.ctx.restore();\n  }\n\n  async blurBackground(frame: VideoFrame) {\n    if (\n      !this.ctx ||\n      !this.canvas ||\n      !this.segmentationResults?.categoryMask?.canvas ||\n      !this.inputVideo\n    ) {\n      return;\n    }\n\n    this.ctx.save();\n    this.ctx.globalCompositeOperation = 'copy';\n\n    if (this.segmentationResults?.categoryMask && this.segmentationResults.categoryMask.width > 0) {\n      const bitmap = await maskToBitmap(\n        this.segmentationResults.categoryMask,\n        this.segmentationResults.categoryMask.width,\n        this.segmentationResults.categoryMask.height,\n      );\n\n      this.ctx.globalCompositeOperation = 'copy';\n      this.ctx.drawImage(bitmap, 0, 0, this.canvas.width, this.canvas.height);\n      this.ctx.filter = 'none';\n      this.ctx.globalCompositeOperation = 'source-out';\n      this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);\n      this.ctx.globalCompositeOperation = 'destination-over';\n      this.ctx.filter = `blur(${this.blurRadius}px)`;\n      this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);\n      this.ctx.restore();\n    }\n  }\n}\n\nfunction maskToBitmap(\n  mask: vision.MPMask,\n  videoWidth: number,\n  videoHeight: number,\n): Promise<ImageBitmap> {\n  const dataArray: Uint8ClampedArray = new Uint8ClampedArray(videoWidth * videoHeight * 4);\n  const result = mask.getAsUint8Array();\n  for (let i = 0; i < result.length; i += 1) {\n    dataArray[i * 4] = result[i];\n    dataArray[i * 4 + 1] = result[i];\n    dataArray[i * 4 + 2] = result[i];\n    dataArray[i * 4 + 3] = result[i];\n  }\n  const dataNew = new ImageData(dataArray, videoWidth, videoHeight);\n\n  return createImageBitmap(dataNew);\n}\n","{\n  \"name\": \"@livekit/track-processors\",\n  \"version\": \"0.4.0\",\n  \"description\": \"LiveKit track processors\",\n  \"main\": \"dist/index.js\",\n  \"module\": \"dist/index.mjs\",\n  \"source\": \"src/index.ts\",\n  \"types\": \"dist/src/index.d.ts\",\n  \"repository\": \"git@github.com:livekit/livekit-track-processors.git\",\n  \"author\": \"Lukas Seiler\",\n  \"license\": \"Apache-2.0\",\n  \"scripts\": {\n    \"build\": \"tsup --onSuccess \\\"tsc --declaration --emitDeclarationOnly\\\"\",\n    \"build-sample\": \"cd example && vite build\",\n    \"lint\": \"eslint src\",\n    \"release\": \"pnpm build && changeset publish\",\n    \"test\": \"jest\",\n    \"sample\": \"vite serve example --port 8080 --open\"\n  },\n  \"files\": [\n    \"dist\",\n    \"src\"\n  ],\n  \"dependencies\": {\n    \"@mediapipe/tasks-vision\": \"0.10.21\"\n  },\n  \"peerDependencies\": {\n    \"livekit-client\": \"^1.12.0 || ^2.1.0\"\n  },\n  \"devDependencies\": {\n    \"@changesets/cli\": \"^2.26.2\",\n    \"@livekit/changesets-changelog-github\": \"^0.0.4\",\n    \"@trivago/prettier-plugin-sort-imports\": \"^4.1.1\",\n    \"@types/dom-mediacapture-transform\": \"^0.1.6\",\n    \"@types/offscreencanvas\": \"^2019.7.0\",\n    \"@typescript-eslint/eslint-plugin\": \"^5.62.0\",\n    \"eslint\": \"8.39.0\",\n    \"eslint-config-airbnb-typescript\": \"17.0.0\",\n    \"eslint-config-prettier\": \"8.8.0\",\n    \"eslint-plugin-ecmascript-compat\": \"^3.0.0\",\n    \"eslint-plugin-import\": \"2.27.5\",\n    \"prettier\": \"^2.8.8\",\n    \"tsup\": \"^7.1.0\",\n    \"typescript\": \"^5.0.4\",\n    \"vite\": \"^4.3.8\"\n  },\n  \"packageManager\": \"pnpm@9.15.9+sha512.68046141893c66fad01c079231128e9afb89ef87e2691d69e4d40eee228988295fd4682181bae55b58418c3a253bde65a505ec7c5f9403ece5cc3cd37dcf2531\"\n}\n","import { VideoTrackTransformer, VideoTransformerInitOptions } from './types';\n\nexport default abstract class VideoTransformer<Options extends Record<string, unknown>>\n  implements VideoTrackTransformer<Options>\n{\n  transformer?: TransformStream;\n\n  canvas?: OffscreenCanvas;\n\n  ctx?: OffscreenCanvasRenderingContext2D;\n\n  inputVideo?: HTMLVideoElement;\n\n  protected isDisabled?: Boolean = false;\n\n  async init({\n    outputCanvas,\n    inputElement: inputVideo,\n  }: VideoTransformerInitOptions): Promise<void> {\n    if (!(inputVideo instanceof HTMLVideoElement)) {\n      throw TypeError('Video transformer needs a HTMLVideoElement as input');\n    }\n\n    this.transformer = new TransformStream({\n      transform: (frame, controller) => this.transform(frame, controller),\n    });\n    this.canvas = outputCanvas || null;\n    if (outputCanvas) {\n      this.ctx = this.canvas?.getContext('2d') || undefined;\n    }\n    this.inputVideo = inputVideo;\n    this.isDisabled = false;\n  }\n\n  async restart({ outputCanvas, inputElement: inputVideo }: VideoTransformerInitOptions) {\n    this.canvas = outputCanvas || null;\n    this.ctx = this.canvas.getContext('2d') || undefined;\n\n    this.inputVideo = inputVideo;\n    this.isDisabled = false;\n  }\n\n  async destroy() {\n    this.isDisabled = true;\n    this.canvas = undefined;\n    this.ctx = undefined;\n  }\n\n  abstract transform(\n    frame: VideoFrame,\n    controller: TransformStreamDefaultController<VideoFrame>,\n  ): void;\n\n  abstract update(options: Options): void;\n}\n","import ProcessorWrapper from './ProcessorWrapper';\nimport BackgroundTransformer, {\n  BackgroundOptions,\n  SegmenterOptions,\n} from './transformers/BackgroundTransformer';\n\nexport * from './transformers/types';\nexport { default as VideoTransformer } from './transformers/VideoTransformer';\nexport { ProcessorWrapper, type BackgroundOptions, type SegmenterOptions, BackgroundTransformer };\n\nexport const BackgroundBlur = (blurRadius: number = 10, segmenterOptions?: SegmenterOptions) => {\n  return BackgroundProcessor({ blurRadius, segmenterOptions }, 'background-blur');\n};\n\nexport const VirtualBackground = (imagePath: string, segmenterOptions?: SegmenterOptions) => {\n  return BackgroundProcessor({ imagePath, segmenterOptions }, 'virtual-background');\n};\n\nexport const BackgroundProcessor = (options: BackgroundOptions, name = 'background-processor') => {\n  const isProcessorSupported = ProcessorWrapper.isSupported && BackgroundTransformer.isSupported;\n  if (!isProcessorSupported) {\n    throw new Error('processor is not supported in this browser');\n  }\n  const processor = new ProcessorWrapper(new BackgroundTransformer(options), name);\n  return processor;\n};\n"]}
+{"version":3,"sources":["../src/utils.ts","../src/ProcessorWrapper.ts","../src/transformers/BackgroundTransformer.ts","../package.json","../src/transformers/VideoTransformer.ts","../src/index.ts"],"names":["BackgroundProcessor"],"mappings":";AAGA,eAAe,MAAM,MAAc;AACjC,SAAO,IAAI,QAAQ,CAAC,YAAY,WAAW,SAAS,IAAI,CAAC;AAC3D;AAEA,eAAsB,uBAAuB,OAAyB;AACpE,QAAM,UAAU;AAIhB,QAAM,MAAM,EAAE;AAEd,QAAM,UAAU,KAAK,IAAI;AACzB,SAAO,KAAK,IAAI,IAAI,UAAU,SAAS;AACrC,UAAM,EAAE,OAAO,OAAO,IAAI,MAAM,YAAY;AAC5C,QAAI,SAAS,QAAQ;AACnB,aAAO,EAAE,OAAO,OAAO;AAAA,IACzB;AACA,UAAM,MAAM,EAAE;AAAA,EAChB;AACA,SAAO,EAAE,OAAO,QAAW,QAAQ,OAAU;AAC/C;;;ACnBA,IAAqB,mBAArB,MAEA;AAAA,EACE,WAAW,cAAc;AACvB,WACE,OAAO,8BAA8B,eACrC,OAAO,8BAA8B;AAAA,EAEzC;AAAA,EAkBA,YAAY,aAAmD,MAAc;AAC3E,SAAK,OAAO;AACZ,SAAK,cAAc;AAAA,EACrB;AAAA,EAEA,MAAc,MAAM,MAAoC;AACtD,SAAK,SAAS,KAAK;AAEnB,UAAM,EAAE,OAAO,OAAO,IAAI,MAAM,uBAAuB,KAAK,MAAM;AAClE,SAAK,cAAc,KAAK;AAExB,QAAI,EAAE,KAAK,uBAAuB,mBAAmB;AACnD,YAAM,UAAU,iDAAiD;AAAA,IACnE;AAEA,QAAI,KAAK,uBAAuB,kBAAkB;AAChD,WAAK,YAAY,SAAS,0BAAU;AACpC,WAAK,YAAY,QAAQ,wBAAS;AAAA,IACpC;AAGA,SAAK,YAAY,IAAI,0BAA0B,EAAE,OAAO,KAAK,OAAO,CAAC;AAErE,SAAK,iBAAiB,IAAI,0BAA0B;AAAA,MAClD,MAAM;AAAA,MACN,cAAc,KAAK;AAAA,IACrB,CAAC;AAED,SAAK,SAAS,IAAI,gBAAgB,wBAAS,KAAK,0BAAU,GAAG;AAAA,EAC/D;AAAA,EAEA,MAAM,KAAK,MAAoC;AAC7C,UAAM,KAAK,MAAM,IAAI;AACrB,QAAI,CAAC,KAAK,UAAU,CAAC,KAAK,aAAa,CAAC,KAAK,gBAAgB;AAC3D,YAAM,IAAI,UAAU,8DAA8D;AAAA,IACpF;AAEA,UAAM,iBAAiB,KAAK,UAAU;AAEtC,UAAM,KAAK,YAAY,KAAK;AAAA,MAC1B,cAAc,KAAK;AAAA,MACnB,cAAc,KAAK;AAAA,IACrB,CAAC;AAED,UAAM,cAAc,eAAe,YAAY,KAAK,YAAa,WAAY;AAE7E,gBACG,OAAO,KAAK,eAAe,QAAQ,EACnC,MAAM,CAAC,MAAM,QAAQ,MAAM,6BAA6B,CAAC,CAAC,EAC1D,QAAQ,MAAM,KAAK,QAAQ,CAAC;AAC/B,SAAK,iBAAiB,KAAK;AAAA,EAC7B;AAAA,EAEA,MAAM,QAAQ,MAAoC;AAChD,UAAM,KAAK,QAAQ;AACnB,WAAO,KAAK,KAAK,IAAI;AAAA,EACvB;AAAA,EAEA,MAAM,sBAAsB,SAA2D;AAErF,SAAK,YAAY,QAAQ,QAAQ,CAAC,CAAC;AAAA,EACrC;AAAA,EAEA,MAAM,4BAA4B,SAA0D;AAC1F,SAAK,YAAY,OAAO,QAAQ,CAAC,CAAC;AAAA,EACpC;AAAA,EAEA,MAAM,UAAU;AAjGlB;AAkGI,UAAM,KAAK,YAAY,QAAQ;AAC/B,eAAK,mBAAL,mBAAqB;AAAA,EACvB;AACF;;;ACrGA,YAAY,YAAY;;;ACuBtB,mBAAgB;AAAA,EACd,2BAA2B;AAC7B;;;ACvBF,IAA8B,mBAA9B,MAEA;AAAA,EAFA;AAWE,SAAU,aAAuB;AAAA;AAAA,EAEjC,MAAM,KAAK;AAAA,IACT;AAAA,IACA,cAAc;AAAA,EAChB,GAA+C;AAlBjD;AAmBI,QAAI,EAAE,sBAAsB,mBAAmB;AAC7C,YAAM,UAAU,qDAAqD;AAAA,IACvE;AAEA,SAAK,cAAc,IAAI,gBAAgB;AAAA,MACrC,WAAW,CAAC,OAAO,eAAe,KAAK,UAAU,OAAO,UAAU;AAAA,IACpE,CAAC;AACD,SAAK,SAAS,gBAAgB;AAC9B,QAAI,cAAc;AAChB,WAAK,QAAM,UAAK,WAAL,mBAAa,WAAW,UAAS;AAAA,IAC9C;AACA,SAAK,aAAa;AAClB,SAAK,aAAa;AAAA,EACpB;AAAA,EAEA,MAAM,QAAQ,EAAE,cAAc,cAAc,WAAW,GAAgC;AACrF,SAAK,SAAS,gBAAgB;AAC9B,SAAK,MAAM,KAAK,OAAO,WAAW,IAAI,KAAK;AAE3C,SAAK,aAAa;AAClB,SAAK,aAAa;AAAA,EACpB;AAAA,EAEA,MAAM,UAAU;AACd,SAAK,aAAa;AAClB,SAAK,SAAS;AACd,SAAK,MAAM;AAAA,EACb;AAQF;;;AFtCA,IAAqB,sBAArB,cAAiD,iBAAoC;AAAA,EAenF,YAAY,MAAyB;AACnC,UAAM;AAPR,2BAAsC;AAQpC,SAAK,UAAU;AACf,SAAK,OAAO,IAAI;AAAA,EAClB;AAAA,EAlBA,WAAW,cAAc;AACvB,WAAO,OAAO,oBAAoB;AAAA,EACpC;AAAA,EAkBA,MAAM,KAAK,EAAE,cAAc,cAAc,WAAW,GAAgC;AArCtF;AAsCI,UAAM,MAAM,KAAK,EAAE,cAAc,cAAc,WAAW,CAAC;AAE3D,UAAM,UAAU,MAAa,uBAAgB;AAAA,OAC3C,gBAAK,QAAQ,eAAb,mBAAyB,uBAAzB,YACE,wDAAwD,aAAa,yBAAyB,CAAC;AAAA,IACnG;AAEA,SAAK,iBAAiB,MAAa,sBAAe,kBAAkB,SAAS;AAAA,MAC3E,aAAa;AAAA,QACX,iBACE,gBAAK,QAAQ,eAAb,mBAAyB,mBAAzB,YACA;AAAA,QACF,UAAU;AAAA,QACV,GAAG,KAAK,QAAQ;AAAA,MAClB;AAAA,MACA,aAAa;AAAA,MACb,oBAAoB;AAAA,MACpB,uBAAuB;AAAA,IACzB,CAAC;AAGD,UAAI,UAAK,YAAL,mBAAc,cAAa,CAAC,KAAK,iBAAiB;AACpD,YAAM,KAAK,eAAe,KAAK,QAAQ,SAAS,EAAE;AAAA,QAAM,CAAC,QACvD,QAAQ,MAAM,oDAAoD,GAAG;AAAA,MACvE;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,UAAU;AAlElB;AAmEI,UAAM,MAAM,QAAQ;AACpB,YAAM,UAAK,mBAAL,mBAAqB;AAC3B,SAAK,kBAAkB;AAAA,EACzB;AAAA,EAEA,MAAM,eAAe,MAAc;AACjC,UAAM,MAAM,IAAI,MAAM;AAEtB,UAAM,IAAI,QAAQ,CAAC,SAAS,WAAW;AACrC,UAAI,cAAc;AAClB,UAAI,SAAS,MAAM,QAAQ,GAAG;AAC9B,UAAI,UAAU,CAAC,QAAQ,OAAO,GAAG;AACjC,UAAI,MAAM;AAAA,IACZ,CAAC;AACD,UAAM,YAAY,MAAM,kBAAkB,GAAG;AAC7C,SAAK,kBAAkB;AAAA,EACzB;AAAA,EAEA,MAAM,UAAU,OAAmB,YAA0D;AArF/F;AAsFI,QAAI;AACF,UAAI,EAAE,iBAAiB,aAAa;AAClC,gBAAQ,MAAM,gCAAgC;AAC9C;AAAA,MACF;AACA,UAAI,KAAK,YAAY;AACnB,mBAAW,QAAQ,KAAK;AACxB;AAAA,MACF;AACA,UAAI,CAAC,KAAK,QAAQ;AAChB,cAAM,UAAU,sCAAsC;AAAA,MACxD;AACA,UAAI,cAAc,YAAY,IAAI;AAClC,iBAAK,mBAAL,mBAAqB;AAAA,QACnB,KAAK;AAAA,QACL;AAAA,QACA,CAAC,WAAY,KAAK,sBAAsB;AAAA;AAG1C,UAAI,KAAK,YAAY;AACnB,cAAM,KAAK,eAAe,KAAK;AAAA,MACjC,OAAO;AACL,cAAM,KAAK,sBAAsB,KAAK;AAAA,MACxC;AACA,YAAM,WAAW,IAAI,WAAW,KAAK,QAAQ;AAAA,QAC3C,WAAW,MAAM,aAAa,KAAK,IAAI;AAAA,MACzC,CAAC;AACD,iBAAW,QAAQ,QAAQ;AAAA,IAC7B,UAAE;AACA,qCAAO;AAAA,IACT;AAAA,EACF;AAAA,EAEA,MAAM,OAAO,MAAyB;AACpC,SAAK,UAAU;AACf,QAAI,KAAK,YAAY;AACnB,WAAK,aAAa,KAAK;AAAA,IACzB,WAAW,KAAK,WAAW;AACzB,YAAM,KAAK,eAAe,KAAK,SAAS;AAAA,IAC1C;AAAA,EACF;AAAA,EAEA,MAAM,sBAAsB,OAAmB;AAhIjD;AAiII,QAAI,CAAC,KAAK,UAAU,CAAC,KAAK,OAAO,CAAC,KAAK,uBAAuB,CAAC,KAAK;AAAY;AAGhF,UAAI,UAAK,wBAAL,mBAA0B,iBAAgB,KAAK,oBAAoB,aAAa,QAAQ,GAAG;AAC7F,WAAK,IAAI,2BAA2B;AAEpC,WAAK,IAAI;AAAA,QACP;AAAA,UACE,KAAK,oBAAoB;AAAA,UACzB,KAAK,oBAAoB,aAAa;AAAA,UACtC,KAAK,oBAAoB,aAAa;AAAA,QACxC;AAAA,QACA;AAAA,QACA;AAAA,MACF;AACA,WAAK,IAAI,SAAS;AAClB,WAAK,IAAI,2BAA2B;AACpC,UAAI,KAAK,iBAAiB;AACxB,aAAK,IAAI;AAAA,UACP,KAAK;AAAA,UACL;AAAA,UACA;AAAA,UACA,KAAK,gBAAgB;AAAA,UACrB,KAAK,gBAAgB;AAAA,UACrB;AAAA,UACA;AAAA,UACA,KAAK,OAAO;AAAA,UACZ,KAAK,OAAO;AAAA,QACd;AAAA,MACF,OAAO;AACL,aAAK,IAAI,YAAY;AACrB,aAAK,IAAI,SAAS,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AAAA,MAC/D;AAEA,WAAK,IAAI,2BAA2B;AAAA,IACtC;AACA,SAAK,IAAI,UAAU,OAAO,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AAAA,EACvE;AAAA,EAEA,MAAM,eAAe,OAAmB;AAxK1C;AAyKI,QACE,CAAC,KAAK,OACN,CAAC,KAAK,UACN,GAAC,gBAAK,wBAAL,mBAA0B,iBAA1B,mBAAwC,WACzC,CAAC,KAAK,YACN;AACA;AAAA,IACF;AAEA,SAAK,IAAI,KAAK;AACd,SAAK,IAAI,2BAA2B;AAEpC,UAAI,UAAK,wBAAL,mBAA0B,iBAAgB,KAAK,oBAAoB,aAAa,QAAQ,GAAG;AAC7F,WAAK,IAAI;AAAA,QACP;AAAA,UACE,KAAK,oBAAoB;AAAA,UACzB,KAAK,oBAAoB,aAAa;AAAA,UACtC,KAAK,oBAAoB,aAAa;AAAA,QACxC;AAAA,QACA;AAAA,QACA;AAAA,MACF;AACA,WAAK,IAAI,SAAS;AAClB,WAAK,IAAI,2BAA2B;AACpC,WAAK,IAAI,UAAU,OAAO,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AACrE,WAAK,IAAI,2BAA2B;AACpC,WAAK,IAAI,SAAS,QAAQ,KAAK,UAAU;AACzC,WAAK,IAAI,UAAU,OAAO,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AACrE,WAAK,IAAI,QAAQ;AAAA,IACnB;AAAA,EACF;AACF;AAEA,SAAS,gBAAgB,MAAqB,YAAoB,aAAgC;AAChG,QAAM,YAA+B,IAAI,kBAAkB,aAAa,cAAc,CAAC;AACvF,QAAM,SAAS,KAAK,gBAAgB;AACpC,WAAS,IAAI,GAAG,IAAI,OAAO,QAAQ,KAAK,GAAG;AACzC,UAAM,SAAS,IAAI;AACnB,cAAU,MAAM,IAAI,OAAO,CAAC;AAC5B,cAAU,SAAS,CAAC,IAAI,OAAO,CAAC;AAChC,cAAU,SAAS,CAAC,IAAI,OAAO,CAAC;AAChC,cAAU,SAAS,CAAC,IAAI,OAAO,CAAC;AAAA,EAClC;AACA,SAAO,IAAI,UAAU,WAAW,YAAY,WAAW;AACzD;;;AG3MO,IAAM,iBAAiB,CAAC,aAAqB,IAAI,qBAAwC;AAC9F,SAAOA,qBAAoB,EAAE,YAAY,iBAAiB,GAAG,iBAAiB;AAChF;AAEO,IAAM,oBAAoB,CAAC,WAAmB,qBAAwC;AAC3F,SAAOA,qBAAoB,EAAE,WAAW,iBAAiB,GAAG,oBAAoB;AAClF;AAEO,IAAMA,uBAAsB,CAAC,SAA4B,OAAO,2BAA2B;AAChG,QAAM,uBAAuB,iBAAiB,eAAe,oBAAsB;AACnF,MAAI,CAAC,sBAAsB;AACzB,UAAM,IAAI,MAAM,4CAA4C;AAAA,EAC9D;AACA,QAAM,YAAY,IAAI,iBAAiB,IAAI,oBAAsB,OAAO,GAAG,IAAI;AAC/E,SAAO;AACT","sourcesContent":["export const supportsProcessor = typeof MediaStreamTrackGenerator !== 'undefined';\nexport const supportsOffscreenCanvas = typeof OffscreenCanvas !== 'undefined';\n\nasync function sleep(time: number) {\n  return new Promise((resolve) => setTimeout(resolve, time));\n}\n\nexport async function waitForTrackResolution(track: MediaStreamTrack) {\n  const timeout = 500;\n\n  // browsers report wrong initial resolution on iOS.\n  // when slightly delaying the call to .getSettings(), the correct resolution is being reported\n  await sleep(10);\n\n  const started = Date.now();\n  while (Date.now() - started < timeout) {\n    const { width, height } = track.getSettings();\n    if (width && height) {\n      return { width, height };\n    }\n    await sleep(50);\n  }\n  return { width: undefined, height: undefined };\n}\n","import type { ProcessorOptions, Track, TrackProcessor } from 'livekit-client';\nimport { TrackTransformer } from './transformers';\nimport { waitForTrackResolution } from './utils';\n\nexport default class ProcessorWrapper<TransformerOptions extends Record<string, unknown>>\n  implements TrackProcessor<Track.Kind>\n{\n  static get isSupported() {\n    return (\n      typeof MediaStreamTrackGenerator !== 'undefined' &&\n      typeof MediaStreamTrackProcessor !== 'undefined'\n    );\n  }\n\n  name: string;\n\n  source?: MediaStreamVideoTrack;\n\n  processor?: MediaStreamTrackProcessor<VideoFrame>;\n\n  trackGenerator?: MediaStreamTrackGenerator<VideoFrame>;\n\n  canvas?: OffscreenCanvas;\n\n  sourceDummy?: HTMLMediaElement;\n\n  processedTrack?: MediaStreamTrack;\n\n  transformer: TrackTransformer<TransformerOptions>;\n\n  constructor(transformer: TrackTransformer<TransformerOptions>, name: string) {\n    this.name = name;\n    this.transformer = transformer;\n  }\n\n  private async setup(opts: ProcessorOptions<Track.Kind>) {\n    this.source = opts.track as MediaStreamVideoTrack;\n\n    const { width, height } = await waitForTrackResolution(this.source);\n    this.sourceDummy = opts.element;\n\n    if (!(this.sourceDummy instanceof HTMLVideoElement)) {\n      throw TypeError('Currently only video transformers are supported');\n    }\n\n    if (this.sourceDummy instanceof HTMLVideoElement) {\n      this.sourceDummy.height = height ?? 300;\n      this.sourceDummy.width = width ?? 300;\n    }\n\n    // TODO explore if we can do all the processing work in a webworker\n    this.processor = new MediaStreamTrackProcessor({ track: this.source });\n\n    this.trackGenerator = new MediaStreamTrackGenerator({\n      kind: 'video',\n      signalTarget: this.source,\n    });\n\n    this.canvas = new OffscreenCanvas(width ?? 300, height ?? 300);\n  }\n\n  async init(opts: ProcessorOptions<Track.Kind>) {\n    await this.setup(opts);\n    if (!this.canvas || !this.processor || !this.trackGenerator) {\n      throw new TypeError('Expected both canvas and processor to be defined after setup');\n    }\n\n    const readableStream = this.processor.readable;\n\n    await this.transformer.init({\n      outputCanvas: this.canvas,\n      inputElement: this.sourceDummy as HTMLVideoElement,\n    });\n\n    const pipedStream = readableStream.pipeThrough(this.transformer!.transformer!);\n\n    pipedStream\n      .pipeTo(this.trackGenerator.writable)\n      .catch((e) => console.error('error when trying to pipe', e))\n      .finally(() => this.destroy());\n    this.processedTrack = this.trackGenerator as MediaStreamVideoTrack;\n  }\n\n  async restart(opts: ProcessorOptions<Track.Kind>) {\n    await this.destroy();\n    return this.init(opts);\n  }\n\n  async restartTransformer(...options: Parameters<(typeof this.transformer)['restart']>) {\n    // @ts-ignore unclear why the restart method only accepts VideoTransformerInitOptions instead of either those or AudioTransformerInitOptions\n    this.transformer.restart(options[0]);\n  }\n\n  async updateTransformerOptions(...options: Parameters<(typeof this.transformer)['update']>) {\n    this.transformer.update(options[0]);\n  }\n\n  async destroy() {\n    await this.transformer.destroy();\n    this.trackGenerator?.stop();\n  }\n}\n","import * as vision from '@mediapipe/tasks-vision';\nimport { dependencies } from '../../package.json';\nimport VideoTransformer from './VideoTransformer';\nimport { VideoTransformerInitOptions } from './types';\n\nexport type SegmenterOptions = Partial<vision.ImageSegmenterOptions['baseOptions']>;\n\nexport type BackgroundOptions = {\n  blurRadius?: number;\n  imagePath?: string;\n  /** cannot be updated through the `update` method, needs a restart */\n  segmenterOptions?: SegmenterOptions;\n  /** cannot be updated through the `update` method, needs a restart */\n  assetPaths?: { tasksVisionFileSet?: string; modelAssetPath?: string };\n};\n\nexport default class BackgroundProcessor extends VideoTransformer<BackgroundOptions> {\n  static get isSupported() {\n    return typeof OffscreenCanvas !== 'undefined';\n  }\n\n  imageSegmenter?: vision.ImageSegmenter;\n\n  segmentationResults: vision.ImageSegmenterResult | undefined;\n\n  backgroundImage: ImageBitmap | null = null;\n\n  blurRadius?: number;\n\n  options: BackgroundOptions;\n\n  constructor(opts: BackgroundOptions) {\n    super();\n    this.options = opts;\n    this.update(opts);\n  }\n\n  async init({ outputCanvas, inputElement: inputVideo }: VideoTransformerInitOptions) {\n    await super.init({ outputCanvas, inputElement: inputVideo });\n\n    const fileSet = await vision.FilesetResolver.forVisionTasks(\n      this.options.assetPaths?.tasksVisionFileSet ??\n        `https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@${dependencies['@mediapipe/tasks-vision']}/wasm`,\n    );\n\n    this.imageSegmenter = await vision.ImageSegmenter.createFromOptions(fileSet, {\n      baseOptions: {\n        modelAssetPath:\n          this.options.assetPaths?.modelAssetPath ??\n          'https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_segmenter/float16/latest/selfie_segmenter.tflite',\n        delegate: 'GPU',\n        ...this.options.segmenterOptions,\n      },\n      runningMode: 'VIDEO',\n      outputCategoryMask: true,\n      outputConfidenceMasks: false,\n    });\n\n    // Skip loading the image here if update already loaded the image below\n    if (this.options?.imagePath && !this.backgroundImage) {\n      await this.loadBackground(this.options.imagePath).catch((err) =>\n        console.error('Error while loading processor background image: ', err),\n      );\n    }\n  }\n\n  async destroy() {\n    await super.destroy();\n    await this.imageSegmenter?.close();\n    this.backgroundImage = null;\n  }\n\n  async loadBackground(path: string) {\n    const img = new Image();\n\n    await new Promise((resolve, reject) => {\n      img.crossOrigin = 'Anonymous';\n      img.onload = () => resolve(img);\n      img.onerror = (err) => reject(err);\n      img.src = path;\n    });\n    const imageData = await createImageBitmap(img);\n    this.backgroundImage = imageData;\n  }\n\n  async transform(frame: VideoFrame, controller: TransformStreamDefaultController<VideoFrame>) {\n    try {\n      if (!(frame instanceof VideoFrame)) {\n        console.debug('empty frame detected, ignoring');\n        return;\n      }\n      if (this.isDisabled) {\n        controller.enqueue(frame);\n        return;\n      }\n      if (!this.canvas) {\n        throw TypeError('Canvas needs to be initialized first');\n      }\n      let startTimeMs = performance.now();\n      this.imageSegmenter?.segmentForVideo(\n        this.inputVideo!,\n        startTimeMs,\n        (result) => (this.segmentationResults = result),\n      );\n\n      if (this.blurRadius) {\n        await this.blurBackground(frame);\n      } else {\n        await this.drawVirtualBackground(frame);\n      }\n      const newFrame = new VideoFrame(this.canvas, {\n        timestamp: frame.timestamp || Date.now(),\n      });\n      controller.enqueue(newFrame);\n    } finally {\n      frame?.close();\n    }\n  }\n\n  async update(opts: BackgroundOptions) {\n    this.options = opts;\n    if (opts.blurRadius) {\n      this.blurRadius = opts.blurRadius;\n    } else if (opts.imagePath) {\n      await this.loadBackground(opts.imagePath);\n    }\n  }\n\n  async drawVirtualBackground(frame: VideoFrame) {\n    if (!this.canvas || !this.ctx || !this.segmentationResults || !this.inputVideo) return;\n    // this.ctx.save();\n    // this.ctx.clearRect(0, 0, this.canvas.width, this.canvas.height);\n    if (this.segmentationResults?.categoryMask && this.segmentationResults.categoryMask.width > 0) {\n      this.ctx.globalCompositeOperation = 'copy';\n\n      this.ctx.putImageData(\n        maskToImageData(\n          this.segmentationResults.categoryMask,\n          this.segmentationResults.categoryMask.width,\n          this.segmentationResults.categoryMask.height,\n        ),\n        0,\n        0,\n      );\n      this.ctx.filter = 'none';\n      this.ctx.globalCompositeOperation = 'source-in';\n      if (this.backgroundImage) {\n        this.ctx.drawImage(\n          this.backgroundImage,\n          0,\n          0,\n          this.backgroundImage.width,\n          this.backgroundImage.height,\n          0,\n          0,\n          this.canvas.width,\n          this.canvas.height,\n        );\n      } else {\n        this.ctx.fillStyle = '#00FF00';\n        this.ctx.fillRect(0, 0, this.canvas.width, this.canvas.height);\n      }\n\n      this.ctx.globalCompositeOperation = 'destination-over';\n    }\n    this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);\n  }\n\n  async blurBackground(frame: VideoFrame) {\n    if (\n      !this.ctx ||\n      !this.canvas ||\n      !this.segmentationResults?.categoryMask?.canvas ||\n      !this.inputVideo\n    ) {\n      return;\n    }\n\n    this.ctx.save();\n    this.ctx.globalCompositeOperation = 'copy';\n\n    if (this.segmentationResults?.categoryMask && this.segmentationResults.categoryMask.width > 0) {\n      this.ctx.putImageData(\n        maskToImageData(\n          this.segmentationResults.categoryMask,\n          this.segmentationResults.categoryMask.width,\n          this.segmentationResults.categoryMask.height,\n        ),\n        0,\n        0,\n      );\n      this.ctx.filter = 'none';\n      this.ctx.globalCompositeOperation = 'source-out';\n      this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);\n      this.ctx.globalCompositeOperation = 'destination-over';\n      this.ctx.filter = `blur(${this.blurRadius}px)`;\n      this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);\n      this.ctx.restore();\n    }\n  }\n}\n\nfunction maskToImageData(mask: vision.MPMask, videoWidth: number, videoHeight: number): ImageData {\n  const dataArray: Uint8ClampedArray = new Uint8ClampedArray(videoWidth * videoHeight * 4);\n  const result = mask.getAsUint8Array();\n  for (let i = 0; i < result.length; i += 1) {\n    const offset = i * 4;\n    dataArray[offset] = result[i];\n    dataArray[offset + 1] = result[i];\n    dataArray[offset + 2] = result[i];\n    dataArray[offset + 3] = result[i];\n  }\n  return new ImageData(dataArray, videoWidth, videoHeight);\n}\n","{\n  \"name\": \"@livekit/track-processors\",\n  \"version\": \"0.4.1\",\n  \"description\": \"LiveKit track processors\",\n  \"main\": \"dist/index.js\",\n  \"module\": \"dist/index.mjs\",\n  \"source\": \"src/index.ts\",\n  \"types\": \"dist/src/index.d.ts\",\n  \"repository\": \"git@github.com:livekit/livekit-track-processors.git\",\n  \"author\": \"Lukas Seiler\",\n  \"license\": \"Apache-2.0\",\n  \"scripts\": {\n    \"build\": \"tsup --onSuccess \\\"tsc --declaration --emitDeclarationOnly\\\"\",\n    \"build-sample\": \"cd example && vite build\",\n    \"lint\": \"eslint src\",\n    \"release\": \"pnpm build && changeset publish\",\n    \"test\": \"jest\",\n    \"sample\": \"vite serve example --port 8080 --open\"\n  },\n  \"files\": [\n    \"dist\",\n    \"src\"\n  ],\n  \"dependencies\": {\n    \"@mediapipe/tasks-vision\": \"0.10.21\"\n  },\n  \"peerDependencies\": {\n    \"livekit-client\": \"^1.12.0 || ^2.1.0\"\n  },\n  \"devDependencies\": {\n    \"@changesets/cli\": \"^2.26.2\",\n    \"@livekit/changesets-changelog-github\": \"^0.0.4\",\n    \"@trivago/prettier-plugin-sort-imports\": \"^4.1.1\",\n    \"@types/dom-mediacapture-transform\": \"^0.1.6\",\n    \"@types/offscreencanvas\": \"^2019.7.0\",\n    \"@typescript-eslint/eslint-plugin\": \"^5.62.0\",\n    \"eslint\": \"8.39.0\",\n    \"eslint-config-airbnb-typescript\": \"17.0.0\",\n    \"eslint-config-prettier\": \"8.8.0\",\n    \"eslint-plugin-ecmascript-compat\": \"^3.0.0\",\n    \"eslint-plugin-import\": \"2.27.5\",\n    \"prettier\": \"^2.8.8\",\n    \"tsup\": \"^7.1.0\",\n    \"typescript\": \"^5.0.4\",\n    \"vite\": \"^4.3.8\"\n  },\n  \"packageManager\": \"pnpm@9.15.9+sha512.68046141893c66fad01c079231128e9afb89ef87e2691d69e4d40eee228988295fd4682181bae55b58418c3a253bde65a505ec7c5f9403ece5cc3cd37dcf2531\"\n}\n","import { VideoTrackTransformer, VideoTransformerInitOptions } from './types';\n\nexport default abstract class VideoTransformer<Options extends Record<string, unknown>>\n  implements VideoTrackTransformer<Options>\n{\n  transformer?: TransformStream;\n\n  canvas?: OffscreenCanvas;\n\n  ctx?: OffscreenCanvasRenderingContext2D;\n\n  inputVideo?: HTMLVideoElement;\n\n  protected isDisabled?: Boolean = false;\n\n  async init({\n    outputCanvas,\n    inputElement: inputVideo,\n  }: VideoTransformerInitOptions): Promise<void> {\n    if (!(inputVideo instanceof HTMLVideoElement)) {\n      throw TypeError('Video transformer needs a HTMLVideoElement as input');\n    }\n\n    this.transformer = new TransformStream({\n      transform: (frame, controller) => this.transform(frame, controller),\n    });\n    this.canvas = outputCanvas || null;\n    if (outputCanvas) {\n      this.ctx = this.canvas?.getContext('2d') || undefined;\n    }\n    this.inputVideo = inputVideo;\n    this.isDisabled = false;\n  }\n\n  async restart({ outputCanvas, inputElement: inputVideo }: VideoTransformerInitOptions) {\n    this.canvas = outputCanvas || null;\n    this.ctx = this.canvas.getContext('2d') || undefined;\n\n    this.inputVideo = inputVideo;\n    this.isDisabled = false;\n  }\n\n  async destroy() {\n    this.isDisabled = true;\n    this.canvas = undefined;\n    this.ctx = undefined;\n  }\n\n  abstract transform(\n    frame: VideoFrame,\n    controller: TransformStreamDefaultController<VideoFrame>,\n  ): void;\n\n  abstract update(options: Options): void;\n}\n","import ProcessorWrapper from './ProcessorWrapper';\nimport BackgroundTransformer, {\n  BackgroundOptions,\n  SegmenterOptions,\n} from './transformers/BackgroundTransformer';\n\nexport * from './transformers/types';\nexport { default as VideoTransformer } from './transformers/VideoTransformer';\nexport { ProcessorWrapper, type BackgroundOptions, type SegmenterOptions, BackgroundTransformer };\n\nexport const BackgroundBlur = (blurRadius: number = 10, segmenterOptions?: SegmenterOptions) => {\n  return BackgroundProcessor({ blurRadius, segmenterOptions }, 'background-blur');\n};\n\nexport const VirtualBackground = (imagePath: string, segmenterOptions?: SegmenterOptions) => {\n  return BackgroundProcessor({ imagePath, segmenterOptions }, 'virtual-background');\n};\n\nexport const BackgroundProcessor = (options: BackgroundOptions, name = 'background-processor') => {\n  const isProcessorSupported = ProcessorWrapper.isSupported && BackgroundTransformer.isSupported;\n  if (!isProcessorSupported) {\n    throw new Error('processor is not supported in this browser');\n  }\n  const processor = new ProcessorWrapper(new BackgroundTransformer(options), name);\n  return processor;\n};\n"]}

package/dist/index.mjs

@@ -215,12 +215,15 @@ var BackgroundProcessor = class extends VideoTransformer {
       return;
     if (((_a = this.segmentationResults) == null ? void 0 : _a.categoryMask) && this.segmentationResults.categoryMask.width > 0) {
       this.ctx.globalCompositeOperation = "copy";
-      const bitmap = await maskToBitmap(
-        this.segmentationResults.categoryMask,
-        this.segmentationResults.categoryMask.width,
-        this.segmentationResults.categoryMask.height
+      this.ctx.putImageData(
+        maskToImageData(
+          this.segmentationResults.categoryMask,
+          this.segmentationResults.categoryMask.width,
+          this.segmentationResults.categoryMask.height
+        ),
+        0,
+        0
       );
-      this.ctx.drawImage(bitmap, 0, 0, this.canvas.width, this.canvas.height);
       this.ctx.filter = "none";
       this.ctx.globalCompositeOperation = "source-in";
       if (this.backgroundImage) {
@@ -251,13 +254,15 @@ var BackgroundProcessor = class extends VideoTransformer {
     this.ctx.save();
     this.ctx.globalCompositeOperation = "copy";
     if (((_c = this.segmentationResults) == null ? void 0 : _c.categoryMask) && this.segmentationResults.categoryMask.width > 0) {
-      const bitmap = await maskToBitmap(
-        this.segmentationResults.categoryMask,
-        this.segmentationResults.categoryMask.width,
-        this.segmentationResults.categoryMask.height
+      this.ctx.putImageData(
+        maskToImageData(
+          this.segmentationResults.categoryMask,
+          this.segmentationResults.categoryMask.width,
+          this.segmentationResults.categoryMask.height
+        ),
+        0,
+        0
       );
-      this.ctx.globalCompositeOperation = "copy";
-      this.ctx.drawImage(bitmap, 0, 0, this.canvas.width, this.canvas.height);
       this.ctx.filter = "none";
       this.ctx.globalCompositeOperation = "source-out";
       this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);
@@ -268,17 +273,17 @@ var BackgroundProcessor = class extends VideoTransformer {
     }
   }
 };
-function maskToBitmap(mask, videoWidth, videoHeight) {
+function maskToImageData(mask, videoWidth, videoHeight) {
   const dataArray = new Uint8ClampedArray(videoWidth * videoHeight * 4);
   const result = mask.getAsUint8Array();
   for (let i = 0; i < result.length; i += 1) {
-    dataArray[i * 4] = result[i];
-    dataArray[i * 4 + 1] = result[i];
-    dataArray[i * 4 + 2] = result[i];
-    dataArray[i * 4 + 3] = result[i];
+    const offset = i * 4;
+    dataArray[offset] = result[i];
+    dataArray[offset + 1] = result[i];
+    dataArray[offset + 2] = result[i];
+    dataArray[offset + 3] = result[i];
   }
-  const dataNew = new ImageData(dataArray, videoWidth, videoHeight);
-  return createImageBitmap(dataNew);
+  return new ImageData(dataArray, videoWidth, videoHeight);
 }
 
 // src/index.ts

package/dist/index.mjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/utils.ts","../src/ProcessorWrapper.ts","../src/transformers/BackgroundTransformer.ts","../package.json","../src/transformers/VideoTransformer.ts","../src/index.ts"],"sourcesContent":["export const supportsProcessor = typeof MediaStreamTrackGenerator !== 'undefined';\nexport const supportsOffscreenCanvas = typeof OffscreenCanvas !== 'undefined';\n\nasync function sleep(time: number) {\n  return new Promise((resolve) => setTimeout(resolve, time));\n}\n\nexport async function waitForTrackResolution(track: MediaStreamTrack) {\n  const timeout = 500;\n\n  // browsers report wrong initial resolution on iOS.\n  // when slightly delaying the call to .getSettings(), the correct resolution is being reported\n  await sleep(10);\n\n  const started = Date.now();\n  while (Date.now() - started < timeout) {\n    const { width, height } = track.getSettings();\n    if (width && height) {\n      return { width, height };\n    }\n    await sleep(50);\n  }\n  return { width: undefined, height: undefined };\n}\n","import type { ProcessorOptions, Track, TrackProcessor } from 'livekit-client';\nimport { TrackTransformer } from './transformers';\nimport { waitForTrackResolution } from './utils';\n\nexport default class ProcessorWrapper<TransformerOptions extends Record<string, unknown>>\n  implements TrackProcessor<Track.Kind>\n{\n  static get isSupported() {\n    return (\n      typeof MediaStreamTrackGenerator !== 'undefined' &&\n      typeof MediaStreamTrackProcessor !== 'undefined'\n    );\n  }\n\n  name: string;\n\n  source?: MediaStreamVideoTrack;\n\n  processor?: MediaStreamTrackProcessor<VideoFrame>;\n\n  trackGenerator?: MediaStreamTrackGenerator<VideoFrame>;\n\n  canvas?: OffscreenCanvas;\n\n  sourceDummy?: HTMLMediaElement;\n\n  processedTrack?: MediaStreamTrack;\n\n  transformer: TrackTransformer<TransformerOptions>;\n\n  constructor(transformer: TrackTransformer<TransformerOptions>, name: string) {\n    this.name = name;\n    this.transformer = transformer;\n  }\n\n  private async setup(opts: ProcessorOptions<Track.Kind>) {\n    this.source = opts.track as MediaStreamVideoTrack;\n\n    const { width, height } = await waitForTrackResolution(this.source);\n    this.sourceDummy = opts.element;\n\n    if (!(this.sourceDummy instanceof HTMLVideoElement)) {\n      throw TypeError('Currently only video transformers are supported');\n    }\n\n    if (this.sourceDummy instanceof HTMLVideoElement) {\n      this.sourceDummy.height = height ?? 300;\n      this.sourceDummy.width = width ?? 300;\n    }\n\n    // TODO explore if we can do all the processing work in a webworker\n    this.processor = new MediaStreamTrackProcessor({ track: this.source });\n\n    this.trackGenerator = new MediaStreamTrackGenerator({\n      kind: 'video',\n      signalTarget: this.source,\n    });\n\n    this.canvas = new OffscreenCanvas(width ?? 300, height ?? 300);\n  }\n\n  async init(opts: ProcessorOptions<Track.Kind>) {\n    await this.setup(opts);\n    if (!this.canvas || !this.processor || !this.trackGenerator) {\n      throw new TypeError('Expected both canvas and processor to be defined after setup');\n    }\n\n    const readableStream = this.processor.readable;\n\n    await this.transformer.init({\n      outputCanvas: this.canvas,\n      inputElement: this.sourceDummy as HTMLVideoElement,\n    });\n\n    const pipedStream = readableStream.pipeThrough(this.transformer!.transformer!);\n\n    pipedStream\n      .pipeTo(this.trackGenerator.writable)\n      .catch((e) => console.error('error when trying to pipe', e))\n      .finally(() => this.destroy());\n    this.processedTrack = this.trackGenerator as MediaStreamVideoTrack;\n  }\n\n  async restart(opts: ProcessorOptions<Track.Kind>) {\n    await this.destroy();\n    return this.init(opts);\n  }\n\n  async restartTransformer(...options: Parameters<(typeof this.transformer)['restart']>) {\n    // @ts-ignore unclear why the restart method only accepts VideoTransformerInitOptions instead of either those or AudioTransformerInitOptions\n    this.transformer.restart(options[0]);\n  }\n\n  async updateTransformerOptions(...options: Parameters<(typeof this.transformer)['update']>) {\n    this.transformer.update(options[0]);\n  }\n\n  async destroy() {\n    await this.transformer.destroy();\n    this.trackGenerator?.stop();\n  }\n}\n","import * as vision from '@mediapipe/tasks-vision';\nimport { dependencies } from '../../package.json';\nimport VideoTransformer from './VideoTransformer';\nimport { VideoTransformerInitOptions } from './types';\n\nexport type SegmenterOptions = Partial<vision.ImageSegmenterOptions['baseOptions']>;\n\nexport type BackgroundOptions = {\n  blurRadius?: number;\n  imagePath?: string;\n  /** cannot be updated through the `update` method, needs a restart */\n  segmenterOptions?: SegmenterOptions;\n  /** cannot be updated through the `update` method, needs a restart */\n  assetPaths?: { tasksVisionFileSet?: string; modelAssetPath?: string };\n};\n\nexport default class BackgroundProcessor extends VideoTransformer<BackgroundOptions> {\n  static get isSupported() {\n    return typeof OffscreenCanvas !== 'undefined';\n  }\n\n  imageSegmenter?: vision.ImageSegmenter;\n\n  segmentationResults: vision.ImageSegmenterResult | undefined;\n\n  backgroundImage: ImageBitmap | null = null;\n\n  blurRadius?: number;\n\n  options: BackgroundOptions;\n\n  constructor(opts: BackgroundOptions) {\n    super();\n    this.options = opts;\n    this.update(opts);\n  }\n\n  async init({ outputCanvas, inputElement: inputVideo }: VideoTransformerInitOptions) {\n    await super.init({ outputCanvas, inputElement: inputVideo });\n\n    const fileSet = await vision.FilesetResolver.forVisionTasks(\n      this.options.assetPaths?.tasksVisionFileSet ??\n        `https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@${dependencies['@mediapipe/tasks-vision']}/wasm`,\n    );\n\n    this.imageSegmenter = await vision.ImageSegmenter.createFromOptions(fileSet, {\n      baseOptions: {\n        modelAssetPath:\n          this.options.assetPaths?.modelAssetPath ??\n          'https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_segmenter/float16/latest/selfie_segmenter.tflite',\n        delegate: 'GPU',\n        ...this.options.segmenterOptions,\n      },\n      runningMode: 'VIDEO',\n      outputCategoryMask: true,\n      outputConfidenceMasks: false,\n    });\n\n    // Skip loading the image here if update already loaded the image below\n    if (this.options?.imagePath && !this.backgroundImage) {\n      await this.loadBackground(this.options.imagePath).catch((err) =>\n        console.error('Error while loading processor background image: ', err),\n      );\n    }\n  }\n\n  async destroy() {\n    await super.destroy();\n    await this.imageSegmenter?.close();\n    this.backgroundImage = null;\n  }\n\n  async loadBackground(path: string) {\n    const img = new Image();\n\n    await new Promise((resolve, reject) => {\n      img.crossOrigin = 'Anonymous';\n      img.onload = () => resolve(img);\n      img.onerror = (err) => reject(err);\n      img.src = path;\n    });\n    const imageData = await createImageBitmap(img);\n    this.backgroundImage = imageData;\n  }\n\n  async transform(frame: VideoFrame, controller: TransformStreamDefaultController<VideoFrame>) {\n    try {\n      if (!(frame instanceof VideoFrame)) {\n        console.debug('empty frame detected, ignoring');\n        return;\n      }\n      if (this.isDisabled) {\n        controller.enqueue(frame);\n        return;\n      }\n      if (!this.canvas) {\n        throw TypeError('Canvas needs to be initialized first');\n      }\n      let startTimeMs = performance.now();\n      this.imageSegmenter?.segmentForVideo(\n        this.inputVideo!,\n        startTimeMs,\n        (result) => (this.segmentationResults = result),\n      );\n\n      if (this.blurRadius) {\n        await this.blurBackground(frame);\n      } else {\n        await this.drawVirtualBackground(frame);\n      }\n      const newFrame = new VideoFrame(this.canvas, {\n        timestamp: frame.timestamp || Date.now(),\n      });\n      controller.enqueue(newFrame);\n    } finally {\n      frame?.close();\n    }\n  }\n\n  async update(opts: BackgroundOptions) {\n    this.options = opts;\n    if (opts.blurRadius) {\n      this.blurRadius = opts.blurRadius;\n    } else if (opts.imagePath) {\n      await this.loadBackground(opts.imagePath);\n    }\n  }\n\n  async drawVirtualBackground(frame: VideoFrame) {\n    if (!this.canvas || !this.ctx || !this.segmentationResults || !this.inputVideo) return;\n    // this.ctx.save();\n    // this.ctx.clearRect(0, 0, this.canvas.width, this.canvas.height);\n    if (this.segmentationResults?.categoryMask && this.segmentationResults.categoryMask.width > 0) {\n      this.ctx.globalCompositeOperation = 'copy';\n      const bitmap = await maskToBitmap(\n        this.segmentationResults.categoryMask,\n        this.segmentationResults.categoryMask.width,\n        this.segmentationResults.categoryMask.height,\n      );\n      this.ctx.drawImage(bitmap, 0, 0, this.canvas.width, this.canvas.height);\n      this.ctx.filter = 'none';\n      this.ctx.globalCompositeOperation = 'source-in';\n      if (this.backgroundImage) {\n        this.ctx.drawImage(\n          this.backgroundImage,\n          0,\n          0,\n          this.backgroundImage.width,\n          this.backgroundImage.height,\n          0,\n          0,\n          this.canvas.width,\n          this.canvas.height,\n        );\n      } else {\n        this.ctx.fillStyle = '#00FF00';\n        this.ctx.fillRect(0, 0, this.canvas.width, this.canvas.height);\n      }\n\n      this.ctx.globalCompositeOperation = 'destination-over';\n    }\n    this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);\n    // this.ctx.restore();\n  }\n\n  async blurBackground(frame: VideoFrame) {\n    if (\n      !this.ctx ||\n      !this.canvas ||\n      !this.segmentationResults?.categoryMask?.canvas ||\n      !this.inputVideo\n    ) {\n      return;\n    }\n\n    this.ctx.save();\n    this.ctx.globalCompositeOperation = 'copy';\n\n    if (this.segmentationResults?.categoryMask && this.segmentationResults.categoryMask.width > 0) {\n      const bitmap = await maskToBitmap(\n        this.segmentationResults.categoryMask,\n        this.segmentationResults.categoryMask.width,\n        this.segmentationResults.categoryMask.height,\n      );\n\n      this.ctx.globalCompositeOperation = 'copy';\n      this.ctx.drawImage(bitmap, 0, 0, this.canvas.width, this.canvas.height);\n      this.ctx.filter = 'none';\n      this.ctx.globalCompositeOperation = 'source-out';\n      this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);\n      this.ctx.globalCompositeOperation = 'destination-over';\n      this.ctx.filter = `blur(${this.blurRadius}px)`;\n      this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);\n      this.ctx.restore();\n    }\n  }\n}\n\nfunction maskToBitmap(\n  mask: vision.MPMask,\n  videoWidth: number,\n  videoHeight: number,\n): Promise<ImageBitmap> {\n  const dataArray: Uint8ClampedArray = new Uint8ClampedArray(videoWidth * videoHeight * 4);\n  const result = mask.getAsUint8Array();\n  for (let i = 0; i < result.length; i += 1) {\n    dataArray[i * 4] = result[i];\n    dataArray[i * 4 + 1] = result[i];\n    dataArray[i * 4 + 2] = result[i];\n    dataArray[i * 4 + 3] = result[i];\n  }\n  const dataNew = new ImageData(dataArray, videoWidth, videoHeight);\n\n  return createImageBitmap(dataNew);\n}\n","{\n  \"name\": \"@livekit/track-processors\",\n  \"version\": \"0.4.0\",\n  \"description\": \"LiveKit track processors\",\n  \"main\": \"dist/index.js\",\n  \"module\": \"dist/index.mjs\",\n  \"source\": \"src/index.ts\",\n  \"types\": \"dist/src/index.d.ts\",\n  \"repository\": \"git@github.com:livekit/livekit-track-processors.git\",\n  \"author\": \"Lukas Seiler\",\n  \"license\": \"Apache-2.0\",\n  \"scripts\": {\n    \"build\": \"tsup --onSuccess \\\"tsc --declaration --emitDeclarationOnly\\\"\",\n    \"build-sample\": \"cd example && vite build\",\n    \"lint\": \"eslint src\",\n    \"release\": \"pnpm build && changeset publish\",\n    \"test\": \"jest\",\n    \"sample\": \"vite serve example --port 8080 --open\"\n  },\n  \"files\": [\n    \"dist\",\n    \"src\"\n  ],\n  \"dependencies\": {\n    \"@mediapipe/tasks-vision\": \"0.10.21\"\n  },\n  \"peerDependencies\": {\n    \"livekit-client\": \"^1.12.0 || ^2.1.0\"\n  },\n  \"devDependencies\": {\n    \"@changesets/cli\": \"^2.26.2\",\n    \"@livekit/changesets-changelog-github\": \"^0.0.4\",\n    \"@trivago/prettier-plugin-sort-imports\": \"^4.1.1\",\n    \"@types/dom-mediacapture-transform\": \"^0.1.6\",\n    \"@types/offscreencanvas\": \"^2019.7.0\",\n    \"@typescript-eslint/eslint-plugin\": \"^5.62.0\",\n    \"eslint\": \"8.39.0\",\n    \"eslint-config-airbnb-typescript\": \"17.0.0\",\n    \"eslint-config-prettier\": \"8.8.0\",\n    \"eslint-plugin-ecmascript-compat\": \"^3.0.0\",\n    \"eslint-plugin-import\": \"2.27.5\",\n    \"prettier\": \"^2.8.8\",\n    \"tsup\": \"^7.1.0\",\n    \"typescript\": \"^5.0.4\",\n    \"vite\": \"^4.3.8\"\n  },\n  \"packageManager\": \"pnpm@9.15.9+sha512.68046141893c66fad01c079231128e9afb89ef87e2691d69e4d40eee228988295fd4682181bae55b58418c3a253bde65a505ec7c5f9403ece5cc3cd37dcf2531\"\n}\n","import { VideoTrackTransformer, VideoTransformerInitOptions } from './types';\n\nexport default abstract class VideoTransformer<Options extends Record<string, unknown>>\n  implements VideoTrackTransformer<Options>\n{\n  transformer?: TransformStream;\n\n  canvas?: OffscreenCanvas;\n\n  ctx?: OffscreenCanvasRenderingContext2D;\n\n  inputVideo?: HTMLVideoElement;\n\n  protected isDisabled?: Boolean = false;\n\n  async init({\n    outputCanvas,\n    inputElement: inputVideo,\n  }: VideoTransformerInitOptions): Promise<void> {\n    if (!(inputVideo instanceof HTMLVideoElement)) {\n      throw TypeError('Video transformer needs a HTMLVideoElement as input');\n    }\n\n    this.transformer = new TransformStream({\n      transform: (frame, controller) => this.transform(frame, controller),\n    });\n    this.canvas = outputCanvas || null;\n    if (outputCanvas) {\n      this.ctx = this.canvas?.getContext('2d') || undefined;\n    }\n    this.inputVideo = inputVideo;\n    this.isDisabled = false;\n  }\n\n  async restart({ outputCanvas, inputElement: inputVideo }: VideoTransformerInitOptions) {\n    this.canvas = outputCanvas || null;\n    this.ctx = this.canvas.getContext('2d') || undefined;\n\n    this.inputVideo = inputVideo;\n    this.isDisabled = false;\n  }\n\n  async destroy() {\n    this.isDisabled = true;\n    this.canvas = undefined;\n    this.ctx = undefined;\n  }\n\n  abstract transform(\n    frame: VideoFrame,\n    controller: TransformStreamDefaultController<VideoFrame>,\n  ): void;\n\n  abstract update(options: Options): void;\n}\n","import ProcessorWrapper from './ProcessorWrapper';\nimport BackgroundTransformer, {\n  BackgroundOptions,\n  SegmenterOptions,\n} from './transformers/BackgroundTransformer';\n\nexport * from './transformers/types';\nexport { default as VideoTransformer } from './transformers/VideoTransformer';\nexport { ProcessorWrapper, type BackgroundOptions, type SegmenterOptions, BackgroundTransformer };\n\nexport const BackgroundBlur = (blurRadius: number = 10, segmenterOptions?: SegmenterOptions) => {\n  return BackgroundProcessor({ blurRadius, segmenterOptions }, 'background-blur');\n};\n\nexport const VirtualBackground = (imagePath: string, segmenterOptions?: SegmenterOptions) => {\n  return BackgroundProcessor({ imagePath, segmenterOptions }, 'virtual-background');\n};\n\nexport const BackgroundProcessor = (options: BackgroundOptions, name = 'background-processor') => {\n  const isProcessorSupported = ProcessorWrapper.isSupported && BackgroundTransformer.isSupported;\n  if (!isProcessorSupported) {\n    throw new Error('processor is not supported in this browser');\n  }\n  const processor = new ProcessorWrapper(new BackgroundTransformer(options), name);\n  return processor;\n};\n"],"mappings":";AAGA,eAAe,MAAM,MAAc;AACjC,SAAO,IAAI,QAAQ,CAAC,YAAY,WAAW,SAAS,IAAI,CAAC;AAC3D;AAEA,eAAsB,uBAAuB,OAAyB;AACpE,QAAM,UAAU;AAIhB,QAAM,MAAM,EAAE;AAEd,QAAM,UAAU,KAAK,IAAI;AACzB,SAAO,KAAK,IAAI,IAAI,UAAU,SAAS;AACrC,UAAM,EAAE,OAAO,OAAO,IAAI,MAAM,YAAY;AAC5C,QAAI,SAAS,QAAQ;AACnB,aAAO,EAAE,OAAO,OAAO;AAAA,IACzB;AACA,UAAM,MAAM,EAAE;AAAA,EAChB;AACA,SAAO,EAAE,OAAO,QAAW,QAAQ,OAAU;AAC/C;;;ACnBA,IAAqB,mBAArB,MAEA;AAAA,EACE,WAAW,cAAc;AACvB,WACE,OAAO,8BAA8B,eACrC,OAAO,8BAA8B;AAAA,EAEzC;AAAA,EAkBA,YAAY,aAAmD,MAAc;AAC3E,SAAK,OAAO;AACZ,SAAK,cAAc;AAAA,EACrB;AAAA,EAEA,MAAc,MAAM,MAAoC;AACtD,SAAK,SAAS,KAAK;AAEnB,UAAM,EAAE,OAAO,OAAO,IAAI,MAAM,uBAAuB,KAAK,MAAM;AAClE,SAAK,cAAc,KAAK;AAExB,QAAI,EAAE,KAAK,uBAAuB,mBAAmB;AACnD,YAAM,UAAU,iDAAiD;AAAA,IACnE;AAEA,QAAI,KAAK,uBAAuB,kBAAkB;AAChD,WAAK,YAAY,SAAS,0BAAU;AACpC,WAAK,YAAY,QAAQ,wBAAS;AAAA,IACpC;AAGA,SAAK,YAAY,IAAI,0BAA0B,EAAE,OAAO,KAAK,OAAO,CAAC;AAErE,SAAK,iBAAiB,IAAI,0BAA0B;AAAA,MAClD,MAAM;AAAA,MACN,cAAc,KAAK;AAAA,IACrB,CAAC;AAED,SAAK,SAAS,IAAI,gBAAgB,wBAAS,KAAK,0BAAU,GAAG;AAAA,EAC/D;AAAA,EAEA,MAAM,KAAK,MAAoC;AAC7C,UAAM,KAAK,MAAM,IAAI;AACrB,QAAI,CAAC,KAAK,UAAU,CAAC,KAAK,aAAa,CAAC,KAAK,gBAAgB;AAC3D,YAAM,IAAI,UAAU,8DAA8D;AAAA,IACpF;AAEA,UAAM,iBAAiB,KAAK,UAAU;AAEtC,UAAM,KAAK,YAAY,KAAK;AAAA,MAC1B,cAAc,KAAK;AAAA,MACnB,cAAc,KAAK;AAAA,IACrB,CAAC;AAED,UAAM,cAAc,eAAe,YAAY,KAAK,YAAa,WAAY;AAE7E,gBACG,OAAO,KAAK,eAAe,QAAQ,EACnC,MAAM,CAAC,MAAM,QAAQ,MAAM,6BAA6B,CAAC,CAAC,EAC1D,QAAQ,MAAM,KAAK,QAAQ,CAAC;AAC/B,SAAK,iBAAiB,KAAK;AAAA,EAC7B;AAAA,EAEA,MAAM,QAAQ,MAAoC;AAChD,UAAM,KAAK,QAAQ;AACnB,WAAO,KAAK,KAAK,IAAI;AAAA,EACvB;AAAA,EAEA,MAAM,sBAAsB,SAA2D;AAErF,SAAK,YAAY,QAAQ,QAAQ,CAAC,CAAC;AAAA,EACrC;AAAA,EAEA,MAAM,4BAA4B,SAA0D;AAC1F,SAAK,YAAY,OAAO,QAAQ,CAAC,CAAC;AAAA,EACpC;AAAA,EAEA,MAAM,UAAU;AAjGlB;AAkGI,UAAM,KAAK,YAAY,QAAQ;AAC/B,eAAK,mBAAL,mBAAqB;AAAA,EACvB;AACF;;;ACrGA,YAAY,YAAY;;;ACuBtB,mBAAgB;AAAA,EACd,2BAA2B;AAC7B;;;ACvBF,IAA8B,mBAA9B,MAEA;AAAA,EAFA;AAWE,SAAU,aAAuB;AAAA;AAAA,EAEjC,MAAM,KAAK;AAAA,IACT;AAAA,IACA,cAAc;AAAA,EAChB,GAA+C;AAlBjD;AAmBI,QAAI,EAAE,sBAAsB,mBAAmB;AAC7C,YAAM,UAAU,qDAAqD;AAAA,IACvE;AAEA,SAAK,cAAc,IAAI,gBAAgB;AAAA,MACrC,WAAW,CAAC,OAAO,eAAe,KAAK,UAAU,OAAO,UAAU;AAAA,IACpE,CAAC;AACD,SAAK,SAAS,gBAAgB;AAC9B,QAAI,cAAc;AAChB,WAAK,QAAM,UAAK,WAAL,mBAAa,WAAW,UAAS;AAAA,IAC9C;AACA,SAAK,aAAa;AAClB,SAAK,aAAa;AAAA,EACpB;AAAA,EAEA,MAAM,QAAQ,EAAE,cAAc,cAAc,WAAW,GAAgC;AACrF,SAAK,SAAS,gBAAgB;AAC9B,SAAK,MAAM,KAAK,OAAO,WAAW,IAAI,KAAK;AAE3C,SAAK,aAAa;AAClB,SAAK,aAAa;AAAA,EACpB;AAAA,EAEA,MAAM,UAAU;AACd,SAAK,aAAa;AAClB,SAAK,SAAS;AACd,SAAK,MAAM;AAAA,EACb;AAQF;;;AFtCA,IAAqB,sBAArB,cAAiD,iBAAoC;AAAA,EAenF,YAAY,MAAyB;AACnC,UAAM;AAPR,2BAAsC;AAQpC,SAAK,UAAU;AACf,SAAK,OAAO,IAAI;AAAA,EAClB;AAAA,EAlBA,WAAW,cAAc;AACvB,WAAO,OAAO,oBAAoB;AAAA,EACpC;AAAA,EAkBA,MAAM,KAAK,EAAE,cAAc,cAAc,WAAW,GAAgC;AArCtF;AAsCI,UAAM,MAAM,KAAK,EAAE,cAAc,cAAc,WAAW,CAAC;AAE3D,UAAM,UAAU,MAAa,uBAAgB;AAAA,OAC3C,gBAAK,QAAQ,eAAb,mBAAyB,uBAAzB,YACE,wDAAwD,aAAa,yBAAyB,CAAC;AAAA,IACnG;AAEA,SAAK,iBAAiB,MAAa,sBAAe,kBAAkB,SAAS;AAAA,MAC3E,aAAa;AAAA,QACX,iBACE,gBAAK,QAAQ,eAAb,mBAAyB,mBAAzB,YACA;AAAA,QACF,UAAU;AAAA,QACV,GAAG,KAAK,QAAQ;AAAA,MAClB;AAAA,MACA,aAAa;AAAA,MACb,oBAAoB;AAAA,MACpB,uBAAuB;AAAA,IACzB,CAAC;AAGD,UAAI,UAAK,YAAL,mBAAc,cAAa,CAAC,KAAK,iBAAiB;AACpD,YAAM,KAAK,eAAe,KAAK,QAAQ,SAAS,EAAE;AAAA,QAAM,CAAC,QACvD,QAAQ,MAAM,oDAAoD,GAAG;AAAA,MACvE;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,UAAU;AAlElB;AAmEI,UAAM,MAAM,QAAQ;AACpB,YAAM,UAAK,mBAAL,mBAAqB;AAC3B,SAAK,kBAAkB;AAAA,EACzB;AAAA,EAEA,MAAM,eAAe,MAAc;AACjC,UAAM,MAAM,IAAI,MAAM;AAEtB,UAAM,IAAI,QAAQ,CAAC,SAAS,WAAW;AACrC,UAAI,cAAc;AAClB,UAAI,SAAS,MAAM,QAAQ,GAAG;AAC9B,UAAI,UAAU,CAAC,QAAQ,OAAO,GAAG;AACjC,UAAI,MAAM;AAAA,IACZ,CAAC;AACD,UAAM,YAAY,MAAM,kBAAkB,GAAG;AAC7C,SAAK,kBAAkB;AAAA,EACzB;AAAA,EAEA,MAAM,UAAU,OAAmB,YAA0D;AArF/F;AAsFI,QAAI;AACF,UAAI,EAAE,iBAAiB,aAAa;AAClC,gBAAQ,MAAM,gCAAgC;AAC9C;AAAA,MACF;AACA,UAAI,KAAK,YAAY;AACnB,mBAAW,QAAQ,KAAK;AACxB;AAAA,MACF;AACA,UAAI,CAAC,KAAK,QAAQ;AAChB,cAAM,UAAU,sCAAsC;AAAA,MACxD;AACA,UAAI,cAAc,YAAY,IAAI;AAClC,iBAAK,mBAAL,mBAAqB;AAAA,QACnB,KAAK;AAAA,QACL;AAAA,QACA,CAAC,WAAY,KAAK,sBAAsB;AAAA;AAG1C,UAAI,KAAK,YAAY;AACnB,cAAM,KAAK,eAAe,KAAK;AAAA,MACjC,OAAO;AACL,cAAM,KAAK,sBAAsB,KAAK;AAAA,MACxC;AACA,YAAM,WAAW,IAAI,WAAW,KAAK,QAAQ;AAAA,QAC3C,WAAW,MAAM,aAAa,KAAK,IAAI;AAAA,MACzC,CAAC;AACD,iBAAW,QAAQ,QAAQ;AAAA,IAC7B,UAAE;AACA,qCAAO;AAAA,IACT;AAAA,EACF;AAAA,EAEA,MAAM,OAAO,MAAyB;AACpC,SAAK,UAAU;AACf,QAAI,KAAK,YAAY;AACnB,WAAK,aAAa,KAAK;AAAA,IACzB,WAAW,KAAK,WAAW;AACzB,YAAM,KAAK,eAAe,KAAK,SAAS;AAAA,IAC1C;AAAA,EACF;AAAA,EAEA,MAAM,sBAAsB,OAAmB;AAhIjD;AAiII,QAAI,CAAC,KAAK,UAAU,CAAC,KAAK,OAAO,CAAC,KAAK,uBAAuB,CAAC,KAAK;AAAY;AAGhF,UAAI,UAAK,wBAAL,mBAA0B,iBAAgB,KAAK,oBAAoB,aAAa,QAAQ,GAAG;AAC7F,WAAK,IAAI,2BAA2B;AACpC,YAAM,SAAS,MAAM;AAAA,QACnB,KAAK,oBAAoB;AAAA,QACzB,KAAK,oBAAoB,aAAa;AAAA,QACtC,KAAK,oBAAoB,aAAa;AAAA,MACxC;AACA,WAAK,IAAI,UAAU,QAAQ,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AACtE,WAAK,IAAI,SAAS;AAClB,WAAK,IAAI,2BAA2B;AACpC,UAAI,KAAK,iBAAiB;AACxB,aAAK,IAAI;AAAA,UACP,KAAK;AAAA,UACL;AAAA,UACA;AAAA,UACA,KAAK,gBAAgB;AAAA,UACrB,KAAK,gBAAgB;AAAA,UACrB;AAAA,UACA;AAAA,UACA,KAAK,OAAO;AAAA,UACZ,KAAK,OAAO;AAAA,QACd;AAAA,MACF,OAAO;AACL,aAAK,IAAI,YAAY;AACrB,aAAK,IAAI,SAAS,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AAAA,MAC/D;AAEA,WAAK,IAAI,2BAA2B;AAAA,IACtC;AACA,SAAK,IAAI,UAAU,OAAO,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AAAA,EAEvE;AAAA,EAEA,MAAM,eAAe,OAAmB;AArK1C;AAsKI,QACE,CAAC,KAAK,OACN,CAAC,KAAK,UACN,GAAC,gBAAK,wBAAL,mBAA0B,iBAA1B,mBAAwC,WACzC,CAAC,KAAK,YACN;AACA;AAAA,IACF;AAEA,SAAK,IAAI,KAAK;AACd,SAAK,IAAI,2BAA2B;AAEpC,UAAI,UAAK,wBAAL,mBAA0B,iBAAgB,KAAK,oBAAoB,aAAa,QAAQ,GAAG;AAC7F,YAAM,SAAS,MAAM;AAAA,QACnB,KAAK,oBAAoB;AAAA,QACzB,KAAK,oBAAoB,aAAa;AAAA,QACtC,KAAK,oBAAoB,aAAa;AAAA,MACxC;AAEA,WAAK,IAAI,2BAA2B;AACpC,WAAK,IAAI,UAAU,QAAQ,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AACtE,WAAK,IAAI,SAAS;AAClB,WAAK,IAAI,2BAA2B;AACpC,WAAK,IAAI,UAAU,OAAO,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AACrE,WAAK,IAAI,2BAA2B;AACpC,WAAK,IAAI,SAAS,QAAQ,KAAK,UAAU;AACzC,WAAK,IAAI,UAAU,OAAO,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AACrE,WAAK,IAAI,QAAQ;AAAA,IACnB;AAAA,EACF;AACF;AAEA,SAAS,aACP,MACA,YACA,aACsB;AACtB,QAAM,YAA+B,IAAI,kBAAkB,aAAa,cAAc,CAAC;AACvF,QAAM,SAAS,KAAK,gBAAgB;AACpC,WAAS,IAAI,GAAG,IAAI,OAAO,QAAQ,KAAK,GAAG;AACzC,cAAU,IAAI,CAAC,IAAI,OAAO,CAAC;AAC3B,cAAU,IAAI,IAAI,CAAC,IAAI,OAAO,CAAC;AAC/B,cAAU,IAAI,IAAI,CAAC,IAAI,OAAO,CAAC;AAC/B,cAAU,IAAI,IAAI,CAAC,IAAI,OAAO,CAAC;AAAA,EACjC;AACA,QAAM,UAAU,IAAI,UAAU,WAAW,YAAY,WAAW;AAEhE,SAAO,kBAAkB,OAAO;AAClC;;;AG5MO,IAAM,iBAAiB,CAAC,aAAqB,IAAI,qBAAwC;AAC9F,SAAOA,qBAAoB,EAAE,YAAY,iBAAiB,GAAG,iBAAiB;AAChF;AAEO,IAAM,oBAAoB,CAAC,WAAmB,qBAAwC;AAC3F,SAAOA,qBAAoB,EAAE,WAAW,iBAAiB,GAAG,oBAAoB;AAClF;AAEO,IAAMA,uBAAsB,CAAC,SAA4B,OAAO,2BAA2B;AAChG,QAAM,uBAAuB,iBAAiB,eAAe,oBAAsB;AACnF,MAAI,CAAC,sBAAsB;AACzB,UAAM,IAAI,MAAM,4CAA4C;AAAA,EAC9D;AACA,QAAM,YAAY,IAAI,iBAAiB,IAAI,oBAAsB,OAAO,GAAG,IAAI;AAC/E,SAAO;AACT;","names":["BackgroundProcessor"]}
+{"version":3,"sources":["../src/utils.ts","../src/ProcessorWrapper.ts","../src/transformers/BackgroundTransformer.ts","../package.json","../src/transformers/VideoTransformer.ts","../src/index.ts"],"sourcesContent":["export const supportsProcessor = typeof MediaStreamTrackGenerator !== 'undefined';\nexport const supportsOffscreenCanvas = typeof OffscreenCanvas !== 'undefined';\n\nasync function sleep(time: number) {\n  return new Promise((resolve) => setTimeout(resolve, time));\n}\n\nexport async function waitForTrackResolution(track: MediaStreamTrack) {\n  const timeout = 500;\n\n  // browsers report wrong initial resolution on iOS.\n  // when slightly delaying the call to .getSettings(), the correct resolution is being reported\n  await sleep(10);\n\n  const started = Date.now();\n  while (Date.now() - started < timeout) {\n    const { width, height } = track.getSettings();\n    if (width && height) {\n      return { width, height };\n    }\n    await sleep(50);\n  }\n  return { width: undefined, height: undefined };\n}\n","import type { ProcessorOptions, Track, TrackProcessor } from 'livekit-client';\nimport { TrackTransformer } from './transformers';\nimport { waitForTrackResolution } from './utils';\n\nexport default class ProcessorWrapper<TransformerOptions extends Record<string, unknown>>\n  implements TrackProcessor<Track.Kind>\n{\n  static get isSupported() {\n    return (\n      typeof MediaStreamTrackGenerator !== 'undefined' &&\n      typeof MediaStreamTrackProcessor !== 'undefined'\n    );\n  }\n\n  name: string;\n\n  source?: MediaStreamVideoTrack;\n\n  processor?: MediaStreamTrackProcessor<VideoFrame>;\n\n  trackGenerator?: MediaStreamTrackGenerator<VideoFrame>;\n\n  canvas?: OffscreenCanvas;\n\n  sourceDummy?: HTMLMediaElement;\n\n  processedTrack?: MediaStreamTrack;\n\n  transformer: TrackTransformer<TransformerOptions>;\n\n  constructor(transformer: TrackTransformer<TransformerOptions>, name: string) {\n    this.name = name;\n    this.transformer = transformer;\n  }\n\n  private async setup(opts: ProcessorOptions<Track.Kind>) {\n    this.source = opts.track as MediaStreamVideoTrack;\n\n    const { width, height } = await waitForTrackResolution(this.source);\n    this.sourceDummy = opts.element;\n\n    if (!(this.sourceDummy instanceof HTMLVideoElement)) {\n      throw TypeError('Currently only video transformers are supported');\n    }\n\n    if (this.sourceDummy instanceof HTMLVideoElement) {\n      this.sourceDummy.height = height ?? 300;\n      this.sourceDummy.width = width ?? 300;\n    }\n\n    // TODO explore if we can do all the processing work in a webworker\n    this.processor = new MediaStreamTrackProcessor({ track: this.source });\n\n    this.trackGenerator = new MediaStreamTrackGenerator({\n      kind: 'video',\n      signalTarget: this.source,\n    });\n\n    this.canvas = new OffscreenCanvas(width ?? 300, height ?? 300);\n  }\n\n  async init(opts: ProcessorOptions<Track.Kind>) {\n    await this.setup(opts);\n    if (!this.canvas || !this.processor || !this.trackGenerator) {\n      throw new TypeError('Expected both canvas and processor to be defined after setup');\n    }\n\n    const readableStream = this.processor.readable;\n\n    await this.transformer.init({\n      outputCanvas: this.canvas,\n      inputElement: this.sourceDummy as HTMLVideoElement,\n    });\n\n    const pipedStream = readableStream.pipeThrough(this.transformer!.transformer!);\n\n    pipedStream\n      .pipeTo(this.trackGenerator.writable)\n      .catch((e) => console.error('error when trying to pipe', e))\n      .finally(() => this.destroy());\n    this.processedTrack = this.trackGenerator as MediaStreamVideoTrack;\n  }\n\n  async restart(opts: ProcessorOptions<Track.Kind>) {\n    await this.destroy();\n    return this.init(opts);\n  }\n\n  async restartTransformer(...options: Parameters<(typeof this.transformer)['restart']>) {\n    // @ts-ignore unclear why the restart method only accepts VideoTransformerInitOptions instead of either those or AudioTransformerInitOptions\n    this.transformer.restart(options[0]);\n  }\n\n  async updateTransformerOptions(...options: Parameters<(typeof this.transformer)['update']>) {\n    this.transformer.update(options[0]);\n  }\n\n  async destroy() {\n    await this.transformer.destroy();\n    this.trackGenerator?.stop();\n  }\n}\n","import * as vision from '@mediapipe/tasks-vision';\nimport { dependencies } from '../../package.json';\nimport VideoTransformer from './VideoTransformer';\nimport { VideoTransformerInitOptions } from './types';\n\nexport type SegmenterOptions = Partial<vision.ImageSegmenterOptions['baseOptions']>;\n\nexport type BackgroundOptions = {\n  blurRadius?: number;\n  imagePath?: string;\n  /** cannot be updated through the `update` method, needs a restart */\n  segmenterOptions?: SegmenterOptions;\n  /** cannot be updated through the `update` method, needs a restart */\n  assetPaths?: { tasksVisionFileSet?: string; modelAssetPath?: string };\n};\n\nexport default class BackgroundProcessor extends VideoTransformer<BackgroundOptions> {\n  static get isSupported() {\n    return typeof OffscreenCanvas !== 'undefined';\n  }\n\n  imageSegmenter?: vision.ImageSegmenter;\n\n  segmentationResults: vision.ImageSegmenterResult | undefined;\n\n  backgroundImage: ImageBitmap | null = null;\n\n  blurRadius?: number;\n\n  options: BackgroundOptions;\n\n  constructor(opts: BackgroundOptions) {\n    super();\n    this.options = opts;\n    this.update(opts);\n  }\n\n  async init({ outputCanvas, inputElement: inputVideo }: VideoTransformerInitOptions) {\n    await super.init({ outputCanvas, inputElement: inputVideo });\n\n    const fileSet = await vision.FilesetResolver.forVisionTasks(\n      this.options.assetPaths?.tasksVisionFileSet ??\n        `https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@${dependencies['@mediapipe/tasks-vision']}/wasm`,\n    );\n\n    this.imageSegmenter = await vision.ImageSegmenter.createFromOptions(fileSet, {\n      baseOptions: {\n        modelAssetPath:\n          this.options.assetPaths?.modelAssetPath ??\n          'https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_segmenter/float16/latest/selfie_segmenter.tflite',\n        delegate: 'GPU',\n        ...this.options.segmenterOptions,\n      },\n      runningMode: 'VIDEO',\n      outputCategoryMask: true,\n      outputConfidenceMasks: false,\n    });\n\n    // Skip loading the image here if update already loaded the image below\n    if (this.options?.imagePath && !this.backgroundImage) {\n      await this.loadBackground(this.options.imagePath).catch((err) =>\n        console.error('Error while loading processor background image: ', err),\n      );\n    }\n  }\n\n  async destroy() {\n    await super.destroy();\n    await this.imageSegmenter?.close();\n    this.backgroundImage = null;\n  }\n\n  async loadBackground(path: string) {\n    const img = new Image();\n\n    await new Promise((resolve, reject) => {\n      img.crossOrigin = 'Anonymous';\n      img.onload = () => resolve(img);\n      img.onerror = (err) => reject(err);\n      img.src = path;\n    });\n    const imageData = await createImageBitmap(img);\n    this.backgroundImage = imageData;\n  }\n\n  async transform(frame: VideoFrame, controller: TransformStreamDefaultController<VideoFrame>) {\n    try {\n      if (!(frame instanceof VideoFrame)) {\n        console.debug('empty frame detected, ignoring');\n        return;\n      }\n      if (this.isDisabled) {\n        controller.enqueue(frame);\n        return;\n      }\n      if (!this.canvas) {\n        throw TypeError('Canvas needs to be initialized first');\n      }\n      let startTimeMs = performance.now();\n      this.imageSegmenter?.segmentForVideo(\n        this.inputVideo!,\n        startTimeMs,\n        (result) => (this.segmentationResults = result),\n      );\n\n      if (this.blurRadius) {\n        await this.blurBackground(frame);\n      } else {\n        await this.drawVirtualBackground(frame);\n      }\n      const newFrame = new VideoFrame(this.canvas, {\n        timestamp: frame.timestamp || Date.now(),\n      });\n      controller.enqueue(newFrame);\n    } finally {\n      frame?.close();\n    }\n  }\n\n  async update(opts: BackgroundOptions) {\n    this.options = opts;\n    if (opts.blurRadius) {\n      this.blurRadius = opts.blurRadius;\n    } else if (opts.imagePath) {\n      await this.loadBackground(opts.imagePath);\n    }\n  }\n\n  async drawVirtualBackground(frame: VideoFrame) {\n    if (!this.canvas || !this.ctx || !this.segmentationResults || !this.inputVideo) return;\n    // this.ctx.save();\n    // this.ctx.clearRect(0, 0, this.canvas.width, this.canvas.height);\n    if (this.segmentationResults?.categoryMask && this.segmentationResults.categoryMask.width > 0) {\n      this.ctx.globalCompositeOperation = 'copy';\n\n      this.ctx.putImageData(\n        maskToImageData(\n          this.segmentationResults.categoryMask,\n          this.segmentationResults.categoryMask.width,\n          this.segmentationResults.categoryMask.height,\n        ),\n        0,\n        0,\n      );\n      this.ctx.filter = 'none';\n      this.ctx.globalCompositeOperation = 'source-in';\n      if (this.backgroundImage) {\n        this.ctx.drawImage(\n          this.backgroundImage,\n          0,\n          0,\n          this.backgroundImage.width,\n          this.backgroundImage.height,\n          0,\n          0,\n          this.canvas.width,\n          this.canvas.height,\n        );\n      } else {\n        this.ctx.fillStyle = '#00FF00';\n        this.ctx.fillRect(0, 0, this.canvas.width, this.canvas.height);\n      }\n\n      this.ctx.globalCompositeOperation = 'destination-over';\n    }\n    this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);\n  }\n\n  async blurBackground(frame: VideoFrame) {\n    if (\n      !this.ctx ||\n      !this.canvas ||\n      !this.segmentationResults?.categoryMask?.canvas ||\n      !this.inputVideo\n    ) {\n      return;\n    }\n\n    this.ctx.save();\n    this.ctx.globalCompositeOperation = 'copy';\n\n    if (this.segmentationResults?.categoryMask && this.segmentationResults.categoryMask.width > 0) {\n      this.ctx.putImageData(\n        maskToImageData(\n          this.segmentationResults.categoryMask,\n          this.segmentationResults.categoryMask.width,\n          this.segmentationResults.categoryMask.height,\n        ),\n        0,\n        0,\n      );\n      this.ctx.filter = 'none';\n      this.ctx.globalCompositeOperation = 'source-out';\n      this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);\n      this.ctx.globalCompositeOperation = 'destination-over';\n      this.ctx.filter = `blur(${this.blurRadius}px)`;\n      this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);\n      this.ctx.restore();\n    }\n  }\n}\n\nfunction maskToImageData(mask: vision.MPMask, videoWidth: number, videoHeight: number): ImageData {\n  const dataArray: Uint8ClampedArray = new Uint8ClampedArray(videoWidth * videoHeight * 4);\n  const result = mask.getAsUint8Array();\n  for (let i = 0; i < result.length; i += 1) {\n    const offset = i * 4;\n    dataArray[offset] = result[i];\n    dataArray[offset + 1] = result[i];\n    dataArray[offset + 2] = result[i];\n    dataArray[offset + 3] = result[i];\n  }\n  return new ImageData(dataArray, videoWidth, videoHeight);\n}\n","{\n  \"name\": \"@livekit/track-processors\",\n  \"version\": \"0.4.1\",\n  \"description\": \"LiveKit track processors\",\n  \"main\": \"dist/index.js\",\n  \"module\": \"dist/index.mjs\",\n  \"source\": \"src/index.ts\",\n  \"types\": \"dist/src/index.d.ts\",\n  \"repository\": \"git@github.com:livekit/livekit-track-processors.git\",\n  \"author\": \"Lukas Seiler\",\n  \"license\": \"Apache-2.0\",\n  \"scripts\": {\n    \"build\": \"tsup --onSuccess \\\"tsc --declaration --emitDeclarationOnly\\\"\",\n    \"build-sample\": \"cd example && vite build\",\n    \"lint\": \"eslint src\",\n    \"release\": \"pnpm build && changeset publish\",\n    \"test\": \"jest\",\n    \"sample\": \"vite serve example --port 8080 --open\"\n  },\n  \"files\": [\n    \"dist\",\n    \"src\"\n  ],\n  \"dependencies\": {\n    \"@mediapipe/tasks-vision\": \"0.10.21\"\n  },\n  \"peerDependencies\": {\n    \"livekit-client\": \"^1.12.0 || ^2.1.0\"\n  },\n  \"devDependencies\": {\n    \"@changesets/cli\": \"^2.26.2\",\n    \"@livekit/changesets-changelog-github\": \"^0.0.4\",\n    \"@trivago/prettier-plugin-sort-imports\": \"^4.1.1\",\n    \"@types/dom-mediacapture-transform\": \"^0.1.6\",\n    \"@types/offscreencanvas\": \"^2019.7.0\",\n    \"@typescript-eslint/eslint-plugin\": \"^5.62.0\",\n    \"eslint\": \"8.39.0\",\n    \"eslint-config-airbnb-typescript\": \"17.0.0\",\n    \"eslint-config-prettier\": \"8.8.0\",\n    \"eslint-plugin-ecmascript-compat\": \"^3.0.0\",\n    \"eslint-plugin-import\": \"2.27.5\",\n    \"prettier\": \"^2.8.8\",\n    \"tsup\": \"^7.1.0\",\n    \"typescript\": \"^5.0.4\",\n    \"vite\": \"^4.3.8\"\n  },\n  \"packageManager\": \"pnpm@9.15.9+sha512.68046141893c66fad01c079231128e9afb89ef87e2691d69e4d40eee228988295fd4682181bae55b58418c3a253bde65a505ec7c5f9403ece5cc3cd37dcf2531\"\n}\n","import { VideoTrackTransformer, VideoTransformerInitOptions } from './types';\n\nexport default abstract class VideoTransformer<Options extends Record<string, unknown>>\n  implements VideoTrackTransformer<Options>\n{\n  transformer?: TransformStream;\n\n  canvas?: OffscreenCanvas;\n\n  ctx?: OffscreenCanvasRenderingContext2D;\n\n  inputVideo?: HTMLVideoElement;\n\n  protected isDisabled?: Boolean = false;\n\n  async init({\n    outputCanvas,\n    inputElement: inputVideo,\n  }: VideoTransformerInitOptions): Promise<void> {\n    if (!(inputVideo instanceof HTMLVideoElement)) {\n      throw TypeError('Video transformer needs a HTMLVideoElement as input');\n    }\n\n    this.transformer = new TransformStream({\n      transform: (frame, controller) => this.transform(frame, controller),\n    });\n    this.canvas = outputCanvas || null;\n    if (outputCanvas) {\n      this.ctx = this.canvas?.getContext('2d') || undefined;\n    }\n    this.inputVideo = inputVideo;\n    this.isDisabled = false;\n  }\n\n  async restart({ outputCanvas, inputElement: inputVideo }: VideoTransformerInitOptions) {\n    this.canvas = outputCanvas || null;\n    this.ctx = this.canvas.getContext('2d') || undefined;\n\n    this.inputVideo = inputVideo;\n    this.isDisabled = false;\n  }\n\n  async destroy() {\n    this.isDisabled = true;\n    this.canvas = undefined;\n    this.ctx = undefined;\n  }\n\n  abstract transform(\n    frame: VideoFrame,\n    controller: TransformStreamDefaultController<VideoFrame>,\n  ): void;\n\n  abstract update(options: Options): void;\n}\n","import ProcessorWrapper from './ProcessorWrapper';\nimport BackgroundTransformer, {\n  BackgroundOptions,\n  SegmenterOptions,\n} from './transformers/BackgroundTransformer';\n\nexport * from './transformers/types';\nexport { default as VideoTransformer } from './transformers/VideoTransformer';\nexport { ProcessorWrapper, type BackgroundOptions, type SegmenterOptions, BackgroundTransformer };\n\nexport const BackgroundBlur = (blurRadius: number = 10, segmenterOptions?: SegmenterOptions) => {\n  return BackgroundProcessor({ blurRadius, segmenterOptions }, 'background-blur');\n};\n\nexport const VirtualBackground = (imagePath: string, segmenterOptions?: SegmenterOptions) => {\n  return BackgroundProcessor({ imagePath, segmenterOptions }, 'virtual-background');\n};\n\nexport const BackgroundProcessor = (options: BackgroundOptions, name = 'background-processor') => {\n  const isProcessorSupported = ProcessorWrapper.isSupported && BackgroundTransformer.isSupported;\n  if (!isProcessorSupported) {\n    throw new Error('processor is not supported in this browser');\n  }\n  const processor = new ProcessorWrapper(new BackgroundTransformer(options), name);\n  return processor;\n};\n"],"mappings":";AAGA,eAAe,MAAM,MAAc;AACjC,SAAO,IAAI,QAAQ,CAAC,YAAY,WAAW,SAAS,IAAI,CAAC;AAC3D;AAEA,eAAsB,uBAAuB,OAAyB;AACpE,QAAM,UAAU;AAIhB,QAAM,MAAM,EAAE;AAEd,QAAM,UAAU,KAAK,IAAI;AACzB,SAAO,KAAK,IAAI,IAAI,UAAU,SAAS;AACrC,UAAM,EAAE,OAAO,OAAO,IAAI,MAAM,YAAY;AAC5C,QAAI,SAAS,QAAQ;AACnB,aAAO,EAAE,OAAO,OAAO;AAAA,IACzB;AACA,UAAM,MAAM,EAAE;AAAA,EAChB;AACA,SAAO,EAAE,OAAO,QAAW,QAAQ,OAAU;AAC/C;;;ACnBA,IAAqB,mBAArB,MAEA;AAAA,EACE,WAAW,cAAc;AACvB,WACE,OAAO,8BAA8B,eACrC,OAAO,8BAA8B;AAAA,EAEzC;AAAA,EAkBA,YAAY,aAAmD,MAAc;AAC3E,SAAK,OAAO;AACZ,SAAK,cAAc;AAAA,EACrB;AAAA,EAEA,MAAc,MAAM,MAAoC;AACtD,SAAK,SAAS,KAAK;AAEnB,UAAM,EAAE,OAAO,OAAO,IAAI,MAAM,uBAAuB,KAAK,MAAM;AAClE,SAAK,cAAc,KAAK;AAExB,QAAI,EAAE,KAAK,uBAAuB,mBAAmB;AACnD,YAAM,UAAU,iDAAiD;AAAA,IACnE;AAEA,QAAI,KAAK,uBAAuB,kBAAkB;AAChD,WAAK,YAAY,SAAS,0BAAU;AACpC,WAAK,YAAY,QAAQ,wBAAS;AAAA,IACpC;AAGA,SAAK,YAAY,IAAI,0BAA0B,EAAE,OAAO,KAAK,OAAO,CAAC;AAErE,SAAK,iBAAiB,IAAI,0BAA0B;AAAA,MAClD,MAAM;AAAA,MACN,cAAc,KAAK;AAAA,IACrB,CAAC;AAED,SAAK,SAAS,IAAI,gBAAgB,wBAAS,KAAK,0BAAU,GAAG;AAAA,EAC/D;AAAA,EAEA,MAAM,KAAK,MAAoC;AAC7C,UAAM,KAAK,MAAM,IAAI;AACrB,QAAI,CAAC,KAAK,UAAU,CAAC,KAAK,aAAa,CAAC,KAAK,gBAAgB;AAC3D,YAAM,IAAI,UAAU,8DAA8D;AAAA,IACpF;AAEA,UAAM,iBAAiB,KAAK,UAAU;AAEtC,UAAM,KAAK,YAAY,KAAK;AAAA,MAC1B,cAAc,KAAK;AAAA,MACnB,cAAc,KAAK;AAAA,IACrB,CAAC;AAED,UAAM,cAAc,eAAe,YAAY,KAAK,YAAa,WAAY;AAE7E,gBACG,OAAO,KAAK,eAAe,QAAQ,EACnC,MAAM,CAAC,MAAM,QAAQ,MAAM,6BAA6B,CAAC,CAAC,EAC1D,QAAQ,MAAM,KAAK,QAAQ,CAAC;AAC/B,SAAK,iBAAiB,KAAK;AAAA,EAC7B;AAAA,EAEA,MAAM,QAAQ,MAAoC;AAChD,UAAM,KAAK,QAAQ;AACnB,WAAO,KAAK,KAAK,IAAI;AAAA,EACvB;AAAA,EAEA,MAAM,sBAAsB,SAA2D;AAErF,SAAK,YAAY,QAAQ,QAAQ,CAAC,CAAC;AAAA,EACrC;AAAA,EAEA,MAAM,4BAA4B,SAA0D;AAC1F,SAAK,YAAY,OAAO,QAAQ,CAAC,CAAC;AAAA,EACpC;AAAA,EAEA,MAAM,UAAU;AAjGlB;AAkGI,UAAM,KAAK,YAAY,QAAQ;AAC/B,eAAK,mBAAL,mBAAqB;AAAA,EACvB;AACF;;;ACrGA,YAAY,YAAY;;;ACuBtB,mBAAgB;AAAA,EACd,2BAA2B;AAC7B;;;ACvBF,IAA8B,mBAA9B,MAEA;AAAA,EAFA;AAWE,SAAU,aAAuB;AAAA;AAAA,EAEjC,MAAM,KAAK;AAAA,IACT;AAAA,IACA,cAAc;AAAA,EAChB,GAA+C;AAlBjD;AAmBI,QAAI,EAAE,sBAAsB,mBAAmB;AAC7C,YAAM,UAAU,qDAAqD;AAAA,IACvE;AAEA,SAAK,cAAc,IAAI,gBAAgB;AAAA,MACrC,WAAW,CAAC,OAAO,eAAe,KAAK,UAAU,OAAO,UAAU;AAAA,IACpE,CAAC;AACD,SAAK,SAAS,gBAAgB;AAC9B,QAAI,cAAc;AAChB,WAAK,QAAM,UAAK,WAAL,mBAAa,WAAW,UAAS;AAAA,IAC9C;AACA,SAAK,aAAa;AAClB,SAAK,aAAa;AAAA,EACpB;AAAA,EAEA,MAAM,QAAQ,EAAE,cAAc,cAAc,WAAW,GAAgC;AACrF,SAAK,SAAS,gBAAgB;AAC9B,SAAK,MAAM,KAAK,OAAO,WAAW,IAAI,KAAK;AAE3C,SAAK,aAAa;AAClB,SAAK,aAAa;AAAA,EACpB;AAAA,EAEA,MAAM,UAAU;AACd,SAAK,aAAa;AAClB,SAAK,SAAS;AACd,SAAK,MAAM;AAAA,EACb;AAQF;;;AFtCA,IAAqB,sBAArB,cAAiD,iBAAoC;AAAA,EAenF,YAAY,MAAyB;AACnC,UAAM;AAPR,2BAAsC;AAQpC,SAAK,UAAU;AACf,SAAK,OAAO,IAAI;AAAA,EAClB;AAAA,EAlBA,WAAW,cAAc;AACvB,WAAO,OAAO,oBAAoB;AAAA,EACpC;AAAA,EAkBA,MAAM,KAAK,EAAE,cAAc,cAAc,WAAW,GAAgC;AArCtF;AAsCI,UAAM,MAAM,KAAK,EAAE,cAAc,cAAc,WAAW,CAAC;AAE3D,UAAM,UAAU,MAAa,uBAAgB;AAAA,OAC3C,gBAAK,QAAQ,eAAb,mBAAyB,uBAAzB,YACE,wDAAwD,aAAa,yBAAyB,CAAC;AAAA,IACnG;AAEA,SAAK,iBAAiB,MAAa,sBAAe,kBAAkB,SAAS;AAAA,MAC3E,aAAa;AAAA,QACX,iBACE,gBAAK,QAAQ,eAAb,mBAAyB,mBAAzB,YACA;AAAA,QACF,UAAU;AAAA,QACV,GAAG,KAAK,QAAQ;AAAA,MAClB;AAAA,MACA,aAAa;AAAA,MACb,oBAAoB;AAAA,MACpB,uBAAuB;AAAA,IACzB,CAAC;AAGD,UAAI,UAAK,YAAL,mBAAc,cAAa,CAAC,KAAK,iBAAiB;AACpD,YAAM,KAAK,eAAe,KAAK,QAAQ,SAAS,EAAE;AAAA,QAAM,CAAC,QACvD,QAAQ,MAAM,oDAAoD,GAAG;AAAA,MACvE;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,UAAU;AAlElB;AAmEI,UAAM,MAAM,QAAQ;AACpB,YAAM,UAAK,mBAAL,mBAAqB;AAC3B,SAAK,kBAAkB;AAAA,EACzB;AAAA,EAEA,MAAM,eAAe,MAAc;AACjC,UAAM,MAAM,IAAI,MAAM;AAEtB,UAAM,IAAI,QAAQ,CAAC,SAAS,WAAW;AACrC,UAAI,cAAc;AAClB,UAAI,SAAS,MAAM,QAAQ,GAAG;AAC9B,UAAI,UAAU,CAAC,QAAQ,OAAO,GAAG;AACjC,UAAI,MAAM;AAAA,IACZ,CAAC;AACD,UAAM,YAAY,MAAM,kBAAkB,GAAG;AAC7C,SAAK,kBAAkB;AAAA,EACzB;AAAA,EAEA,MAAM,UAAU,OAAmB,YAA0D;AArF/F;AAsFI,QAAI;AACF,UAAI,EAAE,iBAAiB,aAAa;AAClC,gBAAQ,MAAM,gCAAgC;AAC9C;AAAA,MACF;AACA,UAAI,KAAK,YAAY;AACnB,mBAAW,QAAQ,KAAK;AACxB;AAAA,MACF;AACA,UAAI,CAAC,KAAK,QAAQ;AAChB,cAAM,UAAU,sCAAsC;AAAA,MACxD;AACA,UAAI,cAAc,YAAY,IAAI;AAClC,iBAAK,mBAAL,mBAAqB;AAAA,QACnB,KAAK;AAAA,QACL;AAAA,QACA,CAAC,WAAY,KAAK,sBAAsB;AAAA;AAG1C,UAAI,KAAK,YAAY;AACnB,cAAM,KAAK,eAAe,KAAK;AAAA,MACjC,OAAO;AACL,cAAM,KAAK,sBAAsB,KAAK;AAAA,MACxC;AACA,YAAM,WAAW,IAAI,WAAW,KAAK,QAAQ;AAAA,QAC3C,WAAW,MAAM,aAAa,KAAK,IAAI;AAAA,MACzC,CAAC;AACD,iBAAW,QAAQ,QAAQ;AAAA,IAC7B,UAAE;AACA,qCAAO;AAAA,IACT;AAAA,EACF;AAAA,EAEA,MAAM,OAAO,MAAyB;AACpC,SAAK,UAAU;AACf,QAAI,KAAK,YAAY;AACnB,WAAK,aAAa,KAAK;AAAA,IACzB,WAAW,KAAK,WAAW;AACzB,YAAM,KAAK,eAAe,KAAK,SAAS;AAAA,IAC1C;AAAA,EACF;AAAA,EAEA,MAAM,sBAAsB,OAAmB;AAhIjD;AAiII,QAAI,CAAC,KAAK,UAAU,CAAC,KAAK,OAAO,CAAC,KAAK,uBAAuB,CAAC,KAAK;AAAY;AAGhF,UAAI,UAAK,wBAAL,mBAA0B,iBAAgB,KAAK,oBAAoB,aAAa,QAAQ,GAAG;AAC7F,WAAK,IAAI,2BAA2B;AAEpC,WAAK,IAAI;AAAA,QACP;AAAA,UACE,KAAK,oBAAoB;AAAA,UACzB,KAAK,oBAAoB,aAAa;AAAA,UACtC,KAAK,oBAAoB,aAAa;AAAA,QACxC;AAAA,QACA;AAAA,QACA;AAAA,MACF;AACA,WAAK,IAAI,SAAS;AAClB,WAAK,IAAI,2BAA2B;AACpC,UAAI,KAAK,iBAAiB;AACxB,aAAK,IAAI;AAAA,UACP,KAAK;AAAA,UACL;AAAA,UACA;AAAA,UACA,KAAK,gBAAgB;AAAA,UACrB,KAAK,gBAAgB;AAAA,UACrB;AAAA,UACA;AAAA,UACA,KAAK,OAAO;AAAA,UACZ,KAAK,OAAO;AAAA,QACd;AAAA,MACF,OAAO;AACL,aAAK,IAAI,YAAY;AACrB,aAAK,IAAI,SAAS,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AAAA,MAC/D;AAEA,WAAK,IAAI,2BAA2B;AAAA,IACtC;AACA,SAAK,IAAI,UAAU,OAAO,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AAAA,EACvE;AAAA,EAEA,MAAM,eAAe,OAAmB;AAxK1C;AAyKI,QACE,CAAC,KAAK,OACN,CAAC,KAAK,UACN,GAAC,gBAAK,wBAAL,mBAA0B,iBAA1B,mBAAwC,WACzC,CAAC,KAAK,YACN;AACA;AAAA,IACF;AAEA,SAAK,IAAI,KAAK;AACd,SAAK,IAAI,2BAA2B;AAEpC,UAAI,UAAK,wBAAL,mBAA0B,iBAAgB,KAAK,oBAAoB,aAAa,QAAQ,GAAG;AAC7F,WAAK,IAAI;AAAA,QACP;AAAA,UACE,KAAK,oBAAoB;AAAA,UACzB,KAAK,oBAAoB,aAAa;AAAA,UACtC,KAAK,oBAAoB,aAAa;AAAA,QACxC;AAAA,QACA;AAAA,QACA;AAAA,MACF;AACA,WAAK,IAAI,SAAS;AAClB,WAAK,IAAI,2BAA2B;AACpC,WAAK,IAAI,UAAU,OAAO,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AACrE,WAAK,IAAI,2BAA2B;AACpC,WAAK,IAAI,SAAS,QAAQ,KAAK,UAAU;AACzC,WAAK,IAAI,UAAU,OAAO,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AACrE,WAAK,IAAI,QAAQ;AAAA,IACnB;AAAA,EACF;AACF;AAEA,SAAS,gBAAgB,MAAqB,YAAoB,aAAgC;AAChG,QAAM,YAA+B,IAAI,kBAAkB,aAAa,cAAc,CAAC;AACvF,QAAM,SAAS,KAAK,gBAAgB;AACpC,WAAS,IAAI,GAAG,IAAI,OAAO,QAAQ,KAAK,GAAG;AACzC,UAAM,SAAS,IAAI;AACnB,cAAU,MAAM,IAAI,OAAO,CAAC;AAC5B,cAAU,SAAS,CAAC,IAAI,OAAO,CAAC;AAChC,cAAU,SAAS,CAAC,IAAI,OAAO,CAAC;AAChC,cAAU,SAAS,CAAC,IAAI,OAAO,CAAC;AAAA,EAClC;AACA,SAAO,IAAI,UAAU,WAAW,YAAY,WAAW;AACzD;;;AG3MO,IAAM,iBAAiB,CAAC,aAAqB,IAAI,qBAAwC;AAC9F,SAAOA,qBAAoB,EAAE,YAAY,iBAAiB,GAAG,iBAAiB;AAChF;AAEO,IAAM,oBAAoB,CAAC,WAAmB,qBAAwC;AAC3F,SAAOA,qBAAoB,EAAE,WAAW,iBAAiB,GAAG,oBAAoB;AAClF;AAEO,IAAMA,uBAAsB,CAAC,SAA4B,OAAO,2BAA2B;AAChG,QAAM,uBAAuB,iBAAiB,eAAe,oBAAsB;AACnF,MAAI,CAAC,sBAAsB;AACzB,UAAM,IAAI,MAAM,4CAA4C;AAAA,EAC9D;AACA,QAAM,YAAY,IAAI,iBAAiB,IAAI,oBAAsB,OAAO,GAAG,IAAI;AAC/E,SAAO;AACT;","names":["BackgroundProcessor"]}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@livekit/track-processors",
-  "version": "0.4.0",
+  "version": "0.4.1",
   "description": "LiveKit track processors",
   "main": "dist/index.js",
   "module": "dist/index.mjs",

package/src/transformers/BackgroundTransformer.ts

@@ -132,12 +132,16 @@ export default class BackgroundProcessor extends VideoTransformer<BackgroundOpti
     // this.ctx.clearRect(0, 0, this.canvas.width, this.canvas.height);
     if (this.segmentationResults?.categoryMask && this.segmentationResults.categoryMask.width > 0) {
       this.ctx.globalCompositeOperation = 'copy';
-      const bitmap = await maskToBitmap(
-        this.segmentationResults.categoryMask,
-        this.segmentationResults.categoryMask.width,
-        this.segmentationResults.categoryMask.height,
+
+      this.ctx.putImageData(
+        maskToImageData(
+          this.segmentationResults.categoryMask,
+          this.segmentationResults.categoryMask.width,
+          this.segmentationResults.categoryMask.height,
+        ),
+        0,
+        0,
       );
-      this.ctx.drawImage(bitmap, 0, 0, this.canvas.width, this.canvas.height);
       this.ctx.filter = 'none';
       this.ctx.globalCompositeOperation = 'source-in';
       if (this.backgroundImage) {
@@ -160,7 +164,6 @@ export default class BackgroundProcessor extends VideoTransformer<BackgroundOpti
       this.ctx.globalCompositeOperation = 'destination-over';
     }
     this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);
-    // this.ctx.restore();
   }
 
   async blurBackground(frame: VideoFrame) {
@@ -177,14 +180,15 @@ export default class BackgroundProcessor extends VideoTransformer<BackgroundOpti
     this.ctx.globalCompositeOperation = 'copy';
 
     if (this.segmentationResults?.categoryMask && this.segmentationResults.categoryMask.width > 0) {
-      const bitmap = await maskToBitmap(
-        this.segmentationResults.categoryMask,
-        this.segmentationResults.categoryMask.width,
-        this.segmentationResults.categoryMask.height,
+      this.ctx.putImageData(
+        maskToImageData(
+          this.segmentationResults.categoryMask,
+          this.segmentationResults.categoryMask.width,
+          this.segmentationResults.categoryMask.height,
+        ),
+        0,
+        0,
       );
-
-      this.ctx.globalCompositeOperation = 'copy';
-      this.ctx.drawImage(bitmap, 0, 0, this.canvas.width, this.canvas.height);
       this.ctx.filter = 'none';
       this.ctx.globalCompositeOperation = 'source-out';
       this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);
@@ -196,20 +200,15 @@ export default class BackgroundProcessor extends VideoTransformer<BackgroundOpti
   }
 }
 
-function maskToBitmap(
-  mask: vision.MPMask,
-  videoWidth: number,
-  videoHeight: number,
-): Promise<ImageBitmap> {
+function maskToImageData(mask: vision.MPMask, videoWidth: number, videoHeight: number): ImageData {
   const dataArray: Uint8ClampedArray = new Uint8ClampedArray(videoWidth * videoHeight * 4);
   const result = mask.getAsUint8Array();
   for (let i = 0; i < result.length; i += 1) {
-    dataArray[i * 4] = result[i];
-    dataArray[i * 4 + 1] = result[i];
-    dataArray[i * 4 + 2] = result[i];
-    dataArray[i * 4 + 3] = result[i];
+    const offset = i * 4;
+    dataArray[offset] = result[i];
+    dataArray[offset + 1] = result[i];
+    dataArray[offset + 2] = result[i];
+    dataArray[offset + 3] = result[i];
   }
-  const dataNew = new ImageData(dataArray, videoWidth, videoHeight);
-
-  return createImageBitmap(dataNew);
+  return new ImageData(dataArray, videoWidth, videoHeight);
 }