@livekit/track-processors 0.1.7 → 0.2.0

package/dist/index.js

@@ -1,4 +1,24 @@
-"use strict";Object.defineProperty(exports, "__esModule", {value: true}); function _interopRequireWildcard(obj) { if (obj && obj.__esModule) { return obj; } else { var newObj = {}; if (obj != null) { for (var key in obj) { if (Object.prototype.hasOwnProperty.call(obj, key)) { newObj[key] = obj[key]; } } } newObj.default = obj; return newObj; } }// src/ProcessorPipeline.ts
+"use strict";Object.defineProperty(exports, "__esModule", {value: true}); function _interopRequireWildcard(obj) { if (obj && obj.__esModule) { return obj; } else { var newObj = {}; if (obj != null) { for (var key in obj) { if (Object.prototype.hasOwnProperty.call(obj, key)) { newObj[key] = obj[key]; } } } newObj.default = obj; return newObj; } }var __defProp = Object.defineProperty;
+var __defProps = Object.defineProperties;
+var __getOwnPropDescs = Object.getOwnPropertyDescriptors;
+var __getOwnPropSymbols = Object.getOwnPropertySymbols;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __propIsEnum = Object.prototype.propertyIsEnumerable;
+var __defNormalProp = (obj, key, value) => key in obj ? __defProp(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
+var __spreadValues = (a, b) => {
+  for (var prop in b || (b = {}))
+    if (__hasOwnProp.call(b, prop))
+      __defNormalProp(a, prop, b[prop]);
+  if (__getOwnPropSymbols)
+    for (var prop of __getOwnPropSymbols(b)) {
+      if (__propIsEnum.call(b, prop))
+        __defNormalProp(a, prop, b[prop]);
+    }
+  return a;
+};
+var __spreadProps = (a, b) => __defProps(a, __getOwnPropDescs(b));
+
+// src/ProcessorPipeline.ts
 var ProcessorPipeline = class {
   static get isSupported() {
     return typeof MediaStreamTrackGenerator !== "undefined" && typeof MediaStreamTrackProcessor !== "undefined";
@@ -10,26 +30,41 @@ var ProcessorPipeline = class {
   async init(opts) {
     var _a, _b;
     this.source = opts.track;
+    const origConstraints = this.source.getConstraints();
+    await this.source.applyConstraints(__spreadProps(__spreadValues({}, origConstraints), {
+      // @ts-expect-error when a mediastream track is resized and/or cropped, the `VideoFrame` will have a coded height/width of the original video size
+      // this leads to a shift of the underlying video as the frame itself is being rendered with the coded size
+      // but image segmentation is based on the display dimensions (-> the cropped version)
+      // in order to prevent this, we force the resize mode to "none"
+      resizeMode: "none"
+    }));
     this.sourceSettings = this.source.getSettings();
     this.sourceDummy = opts.element;
+    if (this.sourceDummy instanceof HTMLVideoElement) {
+      this.sourceDummy.height = this.sourceSettings.height;
+      this.sourceDummy.width = this.sourceSettings.width;
+    }
     if (!(this.sourceDummy instanceof HTMLVideoElement)) {
       throw TypeError("Currently only video transformers are supported");
     }
     this.processor = new MediaStreamTrackProcessor({ track: this.source });
-    this.trackGenerator = new MediaStreamTrackGenerator({ kind: "video" });
+    this.trackGenerator = new MediaStreamTrackGenerator({
+      kind: "video",
+      signalTarget: this.source
+    });
     this.canvas = new OffscreenCanvas(
       (_a = this.sourceSettings.width) != null ? _a : 300,
       (_b = this.sourceSettings.height) != null ? _b : 300
     );
     let readableStream = this.processor.readable;
     for (const transformer of this.transformers) {
-      transformer.init({
+      await transformer.init({
         outputCanvas: this.canvas,
         inputElement: this.sourceDummy
       });
       readableStream = readableStream.pipeThrough(transformer.transformer);
     }
-    readableStream.pipeTo(this.trackGenerator.writable);
+    readableStream.pipeTo(this.trackGenerator.writable).catch((e) => console.error("error when trying to pipe", e)).finally(() => this.destroy());
     this.processedTrack = this.trackGenerator;
   }
   async destroy() {
@@ -55,7 +90,10 @@ var VideoTransformer = class {
   constructor() {
     this.isDisabled = false;
   }
-  init({ outputCanvas, inputElement: inputVideo }) {
+  async init({
+    outputCanvas,
+    inputElement: inputVideo
+  }) {
     var _a;
     if (!(inputVideo instanceof HTMLVideoElement)) {
       throw TypeError("Video transformer needs a HTMLVideoElement as input");
@@ -92,7 +130,7 @@ var BackgroundProcessor = class extends VideoTransformer {
     return typeof OffscreenCanvas !== "undefined";
   }
   async init({ outputCanvas, inputElement: inputVideo }) {
-    super.init({ outputCanvas, inputElement: inputVideo });
+    await super.init({ outputCanvas, inputElement: inputVideo });
     const fileSet = await vision.FilesetResolver.forVisionTasks(
       `https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@${dependencies["@mediapipe/tasks-vision"]}/wasm`
     );
@@ -112,22 +150,6 @@ var BackgroundProcessor = class extends VideoTransformer {
     await ((_a = this.imageSegmenter) == null ? void 0 : _a.close());
     this.backgroundImage = null;
   }
-  async sendFramesContinuouslyForSegmentation(videoEl) {
-    var _a;
-    if (!this.isDisabled) {
-      if (videoEl.videoWidth > 0 && videoEl.videoHeight > 0) {
-        let startTimeMs = performance.now();
-        (_a = this.imageSegmenter) == null ? void 0 : _a.segmentForVideo(
-          videoEl,
-          startTimeMs,
-          (result) => this.segmentationResults = result
-        );
-      }
-      videoEl.requestVideoFrameCallback(() => {
-        this.sendFramesContinuouslyForSegmentation(videoEl);
-      });
-    }
-  }
   async loadBackground(path) {
     const img = new Image();
     await new Promise((resolve, reject) => {
@@ -141,29 +163,32 @@ var BackgroundProcessor = class extends VideoTransformer {
   }
   async transform(frame, controller) {
     var _a;
-    if (this.isDisabled) {
-      controller.enqueue(frame);
-      return;
-    }
-    if (!this.canvas) {
-      throw TypeError("Canvas needs to be initialized first");
-    }
-    let startTimeMs = performance.now();
-    (_a = this.imageSegmenter) == null ? void 0 : _a.segmentForVideo(
-      this.inputVideo,
-      startTimeMs,
-      (result) => this.segmentationResults = result
-    );
-    if (this.blurRadius) {
-      await this.blurBackground(frame);
-    } else {
-      await this.drawVirtualBackground(frame);
+    try {
+      if (this.isDisabled) {
+        controller.enqueue(frame);
+        return;
+      }
+      if (!this.canvas) {
+        throw TypeError("Canvas needs to be initialized first");
+      }
+      let startTimeMs = performance.now();
+      (_a = this.imageSegmenter) == null ? void 0 : _a.segmentForVideo(
+        this.inputVideo,
+        startTimeMs,
+        (result) => this.segmentationResults = result
+      );
+      if (this.blurRadius) {
+        await this.blurBackground(frame);
+      } else {
+        await this.drawVirtualBackground(frame);
+      }
+      const newFrame = new VideoFrame(this.canvas, {
+        timestamp: frame.timestamp || Date.now()
+      });
+      controller.enqueue(newFrame);
+    } finally {
+      frame.close();
     }
-    const newFrame = new VideoFrame(this.canvas, {
-      timestamp: frame.timestamp || Date.now()
-    });
-    frame.close();
-    controller.enqueue(newFrame);
   }
   async drawVirtualBackground(frame) {
     var _a;
@@ -205,6 +230,8 @@ var BackgroundProcessor = class extends VideoTransformer {
     if (!this.ctx || !this.canvas || !((_b = (_a = this.segmentationResults) == null ? void 0 : _a.categoryMask) == null ? void 0 : _b.canvas) || !this.inputVideo) {
       return;
     }
+    this.canvas.width = frame.codedWidth;
+    this.canvas.height = frame.codedHeight;
     this.ctx.save();
     this.ctx.globalCompositeOperation = "copy";
     const bitmap = await maskToBitmap(

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/ProcessorPipeline.ts","../src/transformers/BackgroundTransformer.ts","../package.json","../src/transformers/VideoTransformer.ts","../src/transformers/DummyTransformer.ts","../src/index.ts"],"names":[],"mappings":";AAGA,IAAqB,oBAArB,MAA6E;AAAA,EAC3E,WAAW,cAAc;AACvB,WACE,OAAO,8BAA8B,eACrC,OAAO,8BAA8B;AAAA,EAEzC;AAAA,EAoBA,YAAY,cAA4C,MAAc;AACpE,SAAK,OAAO;AACZ,SAAK,eAAe;AAAA,EACtB;AAAA,EAEA,MAAM,KAAK,MAAoC;AAlCjD;AAmCI,SAAK,SAAS,KAAK;AACnB,SAAK,iBAAiB,KAAK,OAAO,YAAY;AAC9C,SAAK,cAAc,KAAK;AACxB,QAAI,EAAE,KAAK,uBAAuB,mBAAmB;AACnD,YAAM,UAAU,iDAAiD;AAAA,IACnE;AAEA,SAAK,YAAY,IAAI,0BAA0B,EAAE,OAAO,KAAK,OAAO,CAAC;AACrE,SAAK,iBAAiB,IAAI,0BAA0B,EAAE,MAAM,QAAQ,CAAC;AAErE,SAAK,SAAS,IAAI;AAAA,OAChB,UAAK,eAAe,UAApB,YAA6B;AAAA,OAC7B,UAAK,eAAe,WAApB,YAA8B;AAAA,IAChC;AAEA,QAAI,iBAAiB,KAAK,UAAU;AACpC,eAAW,eAAe,KAAK,cAAc;AAC3C,kBAAY,KAAK;AAAA,QACf,cAAc,KAAK;AAAA,QACnB,cAAc,KAAK;AAAA,MACrB,CAAC;AACD,uBAAiB,eAAe,YAAY,YAAa,WAAY;AAAA,IACvE;AACA,mBAAe,OAAO,KAAK,eAAe,QAAQ;AAClD,SAAK,iBAAiB,KAAK;AAAA,EAC7B;AAAA,EAEA,MAAM,UAAU;AA9DlB;AA+DI,eAAW,eAAe,KAAK,cAAc;AAC3C,YAAM,YAAY,QAAQ;AAAA,IAC5B;AACA,eAAK,mBAAL,mBAAqB;AAAA,EACvB;AACF;;;ACpEA,YAAY,YAAY;;;ACuBtB,mBAAgB;AAAA,EACd,uBAAuB;AAAA,EACvB,2BAA2B;AAC7B;;;ACxBF,IAA8B,mBAA9B,MAAgF;AAAA,EAAhF;AASE,SAAU,aAAuB;AAAA;AAAA,EAEjC,KAAK,EAAE,cAAc,cAAc,WAAW,GAAsC;AAbtF;AAcI,QAAI,EAAE,sBAAsB,mBAAmB;AAC7C,YAAM,UAAU,qDAAqD;AAAA,IACvE;AACA,SAAK,cAAc,IAAI,gBAAgB;AAAA,MACrC,WAAW,CAAC,OAAO,eAAe,KAAK,UAAU,OAAO,UAAU;AAAA,IACpE,CAAC;AACD,SAAK,SAAS,gBAAgB;AAC9B,QAAI,cAAc;AAChB,WAAK,QAAM,UAAK,WAAL,mBAAa,WAAW,UAAS;AAAA,IAC9C;AACA,SAAK,aAAa;AAClB,SAAK,aAAa;AAAA,EACpB;AAAA,EAEA,MAAM,UAAU;AACd,SAAK,aAAa;AAClB,SAAK,SAAS;AACd,SAAK,MAAM;AAAA,EACb;AAMF;;;AF5BA,IAAqB,sBAArB,cAAiD,iBAAiB;AAAA,EAahE,YAAY,MAAyB;AACnC,UAAM;AALR,2BAAsC;AAMpC,QAAI,KAAK,YAAY;AACnB,WAAK,aAAa,KAAK;AAAA,IACzB,WAAW,KAAK,WAAW;AACzB,WAAK,eAAe,KAAK,SAAS;AAAA,IACpC;AAAA,EACF;AAAA,EAnBA,WAAW,cAAc;AACvB,WAAO,OAAO,oBAAoB;AAAA,EACpC;AAAA,EAmBA,MAAM,KAAK,EAAE,cAAc,cAAc,WAAW,GAAgC;AAClF,UAAM,KAAK,EAAE,cAAc,cAAc,WAAW,CAAC;AAErD,UAAM,UAAU,MAAa,uBAAgB;AAAA,MAC3C,wDAAwD,aAAa,yBAAyB,CAAC;AAAA,IACjG;AAEA,SAAK,iBAAiB,MAAa,sBAAe,kBAAkB,SAAS;AAAA,MAC3E,aAAa;AAAA,QACX,gBACE;AAAA,QACF,UAAU;AAAA,MACZ;AAAA,MACA,aAAa;AAAA,MACb,oBAAoB;AAAA,MACpB,uBAAuB;AAAA,IACzB,CAAC;AAAA,EAIH;AAAA,EAEA,MAAM,UAAU;AAtDlB;AAuDI,UAAM,MAAM,QAAQ;AACpB,YAAM,UAAK,mBAAL,mBAAqB;AAC3B,SAAK,kBAAkB;AAAA,EACzB;AAAA,EAEA,MAAM,sCAAsC,SAA2B;AA5DzE;AA6DI,QAAI,CAAC,KAAK,YAAY;AACpB,UAAI,QAAQ,aAAa,KAAK,QAAQ,cAAc,GAAG;AACrD,YAAI,cAAc,YAAY,IAAI;AAClC,mBAAK,mBAAL,mBAAqB;AAAA,UACnB;AAAA,UACA;AAAA,UACA,CAAC,WAAY,KAAK,sBAAsB;AAAA;AAAA,MAE5C;AACA,cAAQ,0BAA0B,MAAM;AACtC,aAAK,sCAAsC,OAAO;AAAA,MACpD,CAAC;AAAA,IACH;AAAA,EACF;AAAA,EAEA,MAAM,eAAe,MAAc;AACjC,UAAM,MAAM,IAAI,MAAM;AAEtB,UAAM,IAAI,QAAQ,CAAC,SAAS,WAAW;AACrC,UAAI,cAAc;AAClB,UAAI,SAAS,MAAM,QAAQ,GAAG;AAC9B,UAAI,UAAU,CAAC,QAAQ,OAAO,GAAG;AACjC,UAAI,MAAM;AAAA,IACZ,CAAC;AACD,UAAM,YAAY,MAAM,kBAAkB,GAAG;AAC7C,SAAK,kBAAkB;AAAA,EACzB;AAAA,EAEA,MAAM,UAAU,OAAmB,YAA0D;AAzF/F;AA0FI,QAAI,KAAK,YAAY;AACnB,iBAAW,QAAQ,KAAK;AACxB;AAAA,IACF;AACA,QAAI,CAAC,KAAK,QAAQ;AAChB,YAAM,UAAU,sCAAsC;AAAA,IACxD;AACA,QAAI,cAAc,YAAY,IAAI;AAClC,eAAK,mBAAL,mBAAqB;AAAA,MACnB,KAAK;AAAA,MACL;AAAA,MACA,CAAC,WAAY,KAAK,sBAAsB;AAAA;AAG1C,QAAI,KAAK,YAAY;AACnB,YAAM,KAAK,eAAe,KAAK;AAAA,IACjC,OAAO;AACL,YAAM,KAAK,sBAAsB,KAAK;AAAA,IACxC;AACA,UAAM,WAAW,IAAI,WAAW,KAAK,QAAQ;AAAA,MAC3C,WAAW,MAAM,aAAa,KAAK,IAAI;AAAA,IACzC,CAAC;AACD,UAAM,MAAM;AACZ,eAAW,QAAQ,QAAQ;AAAA,EAC7B;AAAA,EAEA,MAAM,sBAAsB,OAAmB;AApHjD;AAqHI,QAAI,CAAC,KAAK,UAAU,CAAC,KAAK,OAAO,CAAC,KAAK,uBAAuB,CAAC,KAAK;AAAY;AAGhF,SAAI,UAAK,wBAAL,mBAA0B,cAAc;AAC1C,WAAK,IAAI,SAAS;AAClB,WAAK,IAAI,2BAA2B;AACpC,YAAM,SAAS,MAAM;AAAA,QACnB,KAAK,oBAAoB;AAAA,QACzB,KAAK,WAAW;AAAA,QAChB,KAAK,WAAW;AAAA,MAClB;AACA,WAAK,IAAI,UAAU,QAAQ,GAAG,CAAC;AAC/B,WAAK,IAAI,SAAS;AAClB,WAAK,IAAI,2BAA2B;AACpC,UAAI,KAAK,iBAAiB;AACxB,aAAK,IAAI;AAAA,UACP,KAAK;AAAA,UACL;AAAA,UACA;AAAA,UACA,KAAK,gBAAgB;AAAA,UACrB,KAAK,gBAAgB;AAAA,UACrB;AAAA,UACA;AAAA,UACA,KAAK,OAAO;AAAA,UACZ,KAAK,OAAO;AAAA,QACd;AAAA,MACF,OAAO;AACL,aAAK,IAAI,YAAY;AACrB,aAAK,IAAI,SAAS,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AAAA,MAC/D;AAEA,WAAK,IAAI,2BAA2B;AAAA,IACtC;AACA,SAAK,IAAI,UAAU,OAAO,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AAAA,EAEvE;AAAA,EAEA,MAAM,eAAe,OAAmB;AA1J1C;AA2JI,QACE,CAAC,KAAK,OACN,CAAC,KAAK,UACN,GAAC,gBAAK,wBAAL,mBAA0B,iBAA1B,mBAAwC,WACzC,CAAC,KAAK,YACN;AACA;AAAA,IACF;AACA,SAAK,IAAI,KAAK;AACd,SAAK,IAAI,2BAA2B;AAEpC,UAAM,SAAS,MAAM;AAAA,MACnB,KAAK,oBAAoB;AAAA,MACzB,KAAK,WAAW;AAAA,MAChB,KAAK,WAAW;AAAA,IAClB;AACA,SAAK,IAAI,SAAS;AAClB,SAAK,IAAI,2BAA2B;AACpC,SAAK,IAAI,UAAU,QAAQ,GAAG,CAAC;AAC/B,SAAK,IAAI,SAAS;AAClB,SAAK,IAAI,2BAA2B;AACpC,SAAK,IAAI,UAAU,OAAO,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AACrE,SAAK,IAAI,2BAA2B;AACpC,SAAK,IAAI,SAAS,QAAQ,KAAK,UAAU;AACzC,SAAK,IAAI,UAAU,OAAO,GAAG,CAAC;AAC9B,SAAK,IAAI,QAAQ;AAAA,EACnB;AACF;AAEA,SAAS,aACP,MACA,YACA,aACsB;AACtB,QAAM,YAA+B,IAAI,kBAAkB,aAAa,cAAc,CAAC;AACvF,QAAM,SAAS,KAAK,gBAAgB;AACpC,WAAS,IAAI,GAAG,IAAI,OAAO,QAAQ,KAAK,GAAG;AACzC,cAAU,IAAI,CAAC,IAAI,OAAO,CAAC;AAC3B,cAAU,IAAI,IAAI,CAAC,IAAI,OAAO,CAAC;AAC/B,cAAU,IAAI,IAAI,CAAC,IAAI,OAAO,CAAC;AAC/B,cAAU,IAAI,IAAI,CAAC,IAAI,OAAO,CAAC;AAAA,EACjC;AACA,QAAM,UAAU,IAAI,UAAU,WAAW,YAAY,WAAW;AAEhE,SAAO,kBAAkB,OAAO;AAClC;;;AGtMA,IAAqB,mBAArB,cAA8C,iBAAiB;AAAA,EAC7D,MAAM,UAAU,OAAmB,YAA0D;AAC3F,eAAW,QAAQ,KAAK;AAAA,EAC1B;AAAA,EAEA,MAAM,UAAU;AAAA,EAEhB;AACF;;;ACNO,IAAM,iBAAiB,CAAC,aAAqB,OAAO;AACzD,QAAM,sBAAsB,kBAAkB,eAAe,oBAAsB;AACnF,MAAI,CAAC,qBAAqB;AACxB,UAAM,IAAI,MAAM,2CAA2C;AAAA,EAC7D;AACA,QAAM,WAAW,IAAI;AAAA,IACnB,CAAC,IAAI,oBAAsB,EAAE,WAAW,CAAC,CAAC;AAAA,IAC1C;AAAA,EACF;AACA,SAAO;AACT;AAEO,IAAM,oBAAoB,CAAC,cAAsB;AACtD,QAAM,sBAAsB,kBAAkB,eAAe,oBAAsB;AACnF,MAAI,CAAC,qBAAqB;AACxB,UAAM,IAAI,MAAM,2CAA2C;AAAA,EAC7D;AACA,QAAM,WAAW,IAAI;AAAA,IACnB,CAAC,IAAI,oBAAsB,EAAE,UAAU,CAAC,CAAC;AAAA,IACzC;AAAA,EACF;AACA,SAAO;AACT;AAEO,IAAM,QAAQ,MAAM;AACzB,QAAM,sBAAsB,kBAAkB,eAAe,oBAAsB;AACnF,MAAI,CAAC,qBAAqB;AACxB,UAAM,IAAI,MAAM,2CAA2C;AAAA,EAC7D;AACA,QAAM,WAAW,IAAI,kBAAkB,CAAC,IAAI,iBAAiB,CAAC,GAAG,OAAO;AACxE,SAAO;AACT","sourcesContent":["import type { ProcessorOptions, Track, TrackProcessor } from 'livekit-client';\nimport { VideoTrackTransformer } from './transformers';\n\nexport default class ProcessorPipeline implements TrackProcessor<Track.Kind> {\n  static get isSupported() {\n    return (\n      typeof MediaStreamTrackGenerator !== 'undefined' &&\n      typeof MediaStreamTrackProcessor !== 'undefined'\n    );\n  }\n\n  name: string;\n\n  source?: MediaStreamVideoTrack;\n\n  sourceSettings?: MediaTrackSettings;\n\n  processor?: MediaStreamTrackProcessor<VideoFrame>;\n\n  trackGenerator?: MediaStreamTrackGenerator<VideoFrame>;\n\n  canvas?: OffscreenCanvas;\n\n  sourceDummy?: HTMLMediaElement;\n\n  processedTrack?: MediaStreamTrack;\n\n  transformers: Array<VideoTrackTransformer>;\n\n  constructor(transformers: Array<VideoTrackTransformer>, name: string) {\n    this.name = name;\n    this.transformers = transformers;\n  }\n\n  async init(opts: ProcessorOptions<Track.Kind>) {\n    this.source = opts.track as MediaStreamVideoTrack;\n    this.sourceSettings = this.source.getSettings();\n    this.sourceDummy = opts.element;\n    if (!(this.sourceDummy instanceof HTMLVideoElement)) {\n      throw TypeError('Currently only video transformers are supported');\n    }\n    // TODO explore if we can do all the processing work in a webworker\n    this.processor = new MediaStreamTrackProcessor({ track: this.source });\n    this.trackGenerator = new MediaStreamTrackGenerator({ kind: 'video' });\n\n    this.canvas = new OffscreenCanvas(\n      this.sourceSettings.width ?? 300,\n      this.sourceSettings.height ?? 300,\n    );\n\n    let readableStream = this.processor.readable;\n    for (const transformer of this.transformers) {\n      transformer.init({\n        outputCanvas: this.canvas,\n        inputElement: this.sourceDummy!,\n      });\n      readableStream = readableStream.pipeThrough(transformer!.transformer!);\n    }\n    readableStream.pipeTo(this.trackGenerator.writable);\n    this.processedTrack = this.trackGenerator as MediaStreamVideoTrack;\n  }\n\n  async destroy() {\n    for (const transformer of this.transformers) {\n      await transformer.destroy();\n    }\n    this.trackGenerator?.stop();\n  }\n}\n","import * as vision from '@mediapipe/tasks-vision';\nimport { dependencies } from '../../package.json';\nimport VideoTransformer from './VideoTransformer';\nimport { VideoTransformerInitOptions } from './types';\n\nexport type BackgroundOptions = {\n  blurRadius?: number;\n  imagePath?: string;\n};\n\nexport default class BackgroundProcessor extends VideoTransformer {\n  static get isSupported() {\n    return typeof OffscreenCanvas !== 'undefined';\n  }\n\n  imageSegmenter?: vision.ImageSegmenter;\n\n  segmentationResults: vision.ImageSegmenterResult | undefined;\n\n  backgroundImage: ImageBitmap | null = null;\n\n  blurRadius?: number;\n\n  constructor(opts: BackgroundOptions) {\n    super();\n    if (opts.blurRadius) {\n      this.blurRadius = opts.blurRadius;\n    } else if (opts.imagePath) {\n      this.loadBackground(opts.imagePath);\n    }\n  }\n\n  async init({ outputCanvas, inputElement: inputVideo }: VideoTransformerInitOptions) {\n    super.init({ outputCanvas, inputElement: inputVideo });\n\n    const fileSet = await vision.FilesetResolver.forVisionTasks(\n      `https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@${dependencies['@mediapipe/tasks-vision']}/wasm`,\n    );\n\n    this.imageSegmenter = await vision.ImageSegmenter.createFromOptions(fileSet, {\n      baseOptions: {\n        modelAssetPath:\n          'https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_segmenter/float16/latest/selfie_segmenter.tflite',\n        delegate: 'CPU',\n      },\n      runningMode: 'VIDEO',\n      outputCategoryMask: true,\n      outputConfidenceMasks: false,\n    });\n\n    // this.loadBackground(opts.backgroundUrl).catch((e) => console.error(e));\n    // this.sendFramesContinuouslyForSegmentation(this.inputVideo!);\n  }\n\n  async destroy() {\n    await super.destroy();\n    await this.imageSegmenter?.close();\n    this.backgroundImage = null;\n  }\n\n  async sendFramesContinuouslyForSegmentation(videoEl: HTMLVideoElement) {\n    if (!this.isDisabled) {\n      if (videoEl.videoWidth > 0 && videoEl.videoHeight > 0) {\n        let startTimeMs = performance.now();\n        this.imageSegmenter?.segmentForVideo(\n          videoEl,\n          startTimeMs,\n          (result) => (this.segmentationResults = result),\n        );\n      }\n      videoEl.requestVideoFrameCallback(() => {\n        this.sendFramesContinuouslyForSegmentation(videoEl);\n      });\n    }\n  }\n\n  async loadBackground(path: string) {\n    const img = new Image();\n\n    await new Promise((resolve, reject) => {\n      img.crossOrigin = 'Anonymous';\n      img.onload = () => resolve(img);\n      img.onerror = (err) => reject(err);\n      img.src = path;\n    });\n    const imageData = await createImageBitmap(img);\n    this.backgroundImage = imageData;\n  }\n\n  async transform(frame: VideoFrame, controller: TransformStreamDefaultController<VideoFrame>) {\n    if (this.isDisabled) {\n      controller.enqueue(frame);\n      return;\n    }\n    if (!this.canvas) {\n      throw TypeError('Canvas needs to be initialized first');\n    }\n    let startTimeMs = performance.now();\n    this.imageSegmenter?.segmentForVideo(\n      this.inputVideo!,\n      startTimeMs,\n      (result) => (this.segmentationResults = result),\n    );\n\n    if (this.blurRadius) {\n      await this.blurBackground(frame);\n    } else {\n      await this.drawVirtualBackground(frame);\n    }\n    const newFrame = new VideoFrame(this.canvas, {\n      timestamp: frame.timestamp || Date.now(),\n    });\n    frame.close();\n    controller.enqueue(newFrame);\n  }\n\n  async drawVirtualBackground(frame: VideoFrame) {\n    if (!this.canvas || !this.ctx || !this.segmentationResults || !this.inputVideo) return;\n    // this.ctx.save();\n    // this.ctx.clearRect(0, 0, this.canvas.width, this.canvas.height);\n    if (this.segmentationResults?.categoryMask) {\n      this.ctx.filter = 'blur(3px)';\n      this.ctx.globalCompositeOperation = 'copy';\n      const bitmap = await maskToBitmap(\n        this.segmentationResults.categoryMask,\n        this.inputVideo.videoWidth,\n        this.inputVideo.videoHeight,\n      );\n      this.ctx.drawImage(bitmap, 0, 0);\n      this.ctx.filter = 'none';\n      this.ctx.globalCompositeOperation = 'source-in';\n      if (this.backgroundImage) {\n        this.ctx.drawImage(\n          this.backgroundImage,\n          0,\n          0,\n          this.backgroundImage.width,\n          this.backgroundImage.height,\n          0,\n          0,\n          this.canvas.width,\n          this.canvas.height,\n        );\n      } else {\n        this.ctx.fillStyle = '#00FF00';\n        this.ctx.fillRect(0, 0, this.canvas.width, this.canvas.height);\n      }\n\n      this.ctx.globalCompositeOperation = 'destination-over';\n    }\n    this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);\n    // this.ctx.restore();\n  }\n\n  async blurBackground(frame: VideoFrame) {\n    if (\n      !this.ctx ||\n      !this.canvas ||\n      !this.segmentationResults?.categoryMask?.canvas ||\n      !this.inputVideo\n    ) {\n      return;\n    }\n    this.ctx.save();\n    this.ctx.globalCompositeOperation = 'copy';\n\n    const bitmap = await maskToBitmap(\n      this.segmentationResults.categoryMask,\n      this.inputVideo.videoWidth,\n      this.inputVideo.videoHeight,\n    );\n    this.ctx.filter = 'blur(3px)';\n    this.ctx.globalCompositeOperation = 'copy';\n    this.ctx.drawImage(bitmap, 0, 0);\n    this.ctx.filter = 'none';\n    this.ctx.globalCompositeOperation = 'source-out';\n    this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);\n    this.ctx.globalCompositeOperation = 'destination-over';\n    this.ctx.filter = `blur(${this.blurRadius}px)`;\n    this.ctx.drawImage(frame, 0, 0);\n    this.ctx.restore();\n  }\n}\n\nfunction maskToBitmap(\n  mask: vision.MPMask,\n  videoWidth: number,\n  videoHeight: number,\n): Promise<ImageBitmap> {\n  const dataArray: Uint8ClampedArray = new Uint8ClampedArray(videoWidth * videoHeight * 4);\n  const result = mask.getAsUint8Array();\n  for (let i = 0; i < result.length; i += 1) {\n    dataArray[i * 4] = result[i];\n    dataArray[i * 4 + 1] = result[i];\n    dataArray[i * 4 + 2] = result[i];\n    dataArray[i * 4 + 3] = result[i];\n  }\n  const dataNew = new ImageData(dataArray, videoWidth, videoHeight);\n\n  return createImageBitmap(dataNew);\n}\n","{\n  \"name\": \"@livekit/track-processors\",\n  \"version\": \"0.1.6\",\n  \"description\": \"LiveKit track processors\",\n  \"main\": \"dist/index.js\",\n  \"module\": \"dist/index.mjs\",\n  \"source\": \"src/index.ts\",\n  \"types\": \"dist/src/index.d.ts\",\n  \"repository\": \"git@github.com:livekit/livekit-track-processors.git\",\n  \"author\": \"Lukas Seiler\",\n  \"license\": \"Apache-2.0\",\n  \"scripts\": {\n    \"build\": \"tsup --onSuccess \\\"tsc --declaration --emitDeclarationOnly\\\"\",\n    \"build-docs\": \"yarn exec typedoc\",\n    \"build-sample\": \"cd example && webpack && cp styles.css index.html dist/\",\n    \"lint\": \"eslint src\",\n    \"test\": \"jest\",\n    \"sample\": \"vite serve example --port 8080 --open\"\n  },\n  \"files\": [\n    \"dist\",\n    \"src\"\n  ],\n  \"dependencies\": {\n    \"@mediapipe/holistic\": \"0.5.1675471629\",\n    \"@mediapipe/tasks-vision\": \"0.10.1\"\n  },\n  \"peerDependencies\": {\n    \"livekit-client\": \"^1.10.0\"\n  },\n  \"devDependencies\": {\n    \"@trivago/prettier-plugin-sort-imports\": \"^4.1.1\",\n    \"@types/dom-mediacapture-transform\": \"^0.1.5\",\n    \"@types/jest\": \"^27.0.3\",\n    \"@types/offscreencanvas\": \"^2019.7.0\",\n    \"@typescript-eslint/eslint-plugin\": \"^4.31.2\",\n    \"@webpack-cli/serve\": \"^1.5.2\",\n    \"eslint\": \"8.39.0\",\n    \"eslint-config-airbnb-typescript\": \"17.0.0\",\n    \"eslint-config-prettier\": \"8.8.0\",\n    \"eslint-plugin-ecmascript-compat\": \"^3.0.0\",\n    \"eslint-plugin-import\": \"2.27.5\",\n    \"jest\": \"^27.4.3\",\n    \"livekit-client\": \"^0.16.6\",\n    \"prettier\": \"^2.8.8\",\n    \"ts-jest\": \"^27.0.7\",\n    \"ts-loader\": \"^8.1.0\",\n    \"ts-proto\": \"^1.85.0\",\n    \"tsup\": \"^7.1.0\",\n    \"typedoc\": \"^0.20.35\",\n    \"typedoc-plugin-no-inherit\": \"1.3.0\",\n    \"typescript\": \"^5.0.4\",\n    \"vite\": \"^4.3.8\",\n    \"webpack\": \"^5.53.0\",\n    \"webpack-cli\": \"^4.8.0\",\n    \"webpack-dev-server\": \"^4.2.1\"\n  }\n}\n","import { VideoTrackTransformer, VideoTransformerInitOptions } from './types';\n\nexport default abstract class VideoTransformer implements VideoTrackTransformer {\n  transformer?: TransformStream;\n\n  canvas?: OffscreenCanvas;\n\n  ctx?: OffscreenCanvasRenderingContext2D;\n\n  inputVideo?: HTMLVideoElement;\n\n  protected isDisabled?: Boolean = false;\n\n  init({ outputCanvas, inputElement: inputVideo }: VideoTransformerInitOptions): void {\n    if (!(inputVideo instanceof HTMLVideoElement)) {\n      throw TypeError('Video transformer needs a HTMLVideoElement as input');\n    }\n    this.transformer = new TransformStream({\n      transform: (frame, controller) => this.transform(frame, controller),\n    });\n    this.canvas = outputCanvas || null;\n    if (outputCanvas) {\n      this.ctx = this.canvas?.getContext('2d') || undefined;\n    }\n    this.inputVideo = inputVideo;\n    this.isDisabled = false;\n  }\n\n  async destroy() {\n    this.isDisabled = true;\n    this.canvas = undefined;\n    this.ctx = undefined;\n  }\n\n  abstract transform(\n    frame: VideoFrame,\n    controller: TransformStreamDefaultController<VideoFrame>,\n  ): void;\n}\n","import VideoTransformer from './VideoTransformer';\n\nexport default class DummyTransformer extends VideoTransformer {\n  async transform(frame: VideoFrame, controller: TransformStreamDefaultController<VideoFrame>) {\n    controller.enqueue(frame);\n  }\n\n  async destroy() {\n    // nothing to do\n  }\n}\n","import ProcessorPipeline from './ProcessorPipeline';\nimport BackgroundTransformer from './transformers/BackgroundTransformer';\nimport DummyTransformer from './transformers/DummyTransformer';\n\nexport const BackgroundBlur = (blurRadius: number = 10) => {\n  const isPipelineSupported = ProcessorPipeline.isSupported && BackgroundTransformer.isSupported;\n  if (!isPipelineSupported) {\n    throw new Error('pipeline is not supported in this browser');\n  }\n  const pipeline = new ProcessorPipeline(\n    [new BackgroundTransformer({ blurRadius })],\n    'background-blur',\n  );\n  return pipeline;\n};\n\nexport const VirtualBackground = (imagePath: string) => {\n  const isPipelineSupported = ProcessorPipeline.isSupported && BackgroundTransformer.isSupported;\n  if (!isPipelineSupported) {\n    throw new Error('pipeline is not supported in this browser');\n  }\n  const pipeline = new ProcessorPipeline(\n    [new BackgroundTransformer({ imagePath })],\n    'virtual-background',\n  );\n  return pipeline;\n};\n\nexport const Dummy = () => {\n  const isPipelineSupported = ProcessorPipeline.isSupported && BackgroundTransformer.isSupported;\n  if (!isPipelineSupported) {\n    throw new Error('pipeline is not supported in this browser');\n  }\n  const pipeline = new ProcessorPipeline([new DummyTransformer()], 'dummy');\n  return pipeline;\n};\n"]}
+{"version":3,"sources":["../src/ProcessorPipeline.ts","../src/transformers/BackgroundTransformer.ts","../package.json","../src/transformers/VideoTransformer.ts","../src/transformers/DummyTransformer.ts","../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;AAGA,IAAqB,oBAArB,MAA6E;AAAA,EAC3E,WAAW,cAAc;AACvB,WACE,OAAO,8BAA8B,eACrC,OAAO,8BAA8B;AAAA,EAEzC;AAAA,EAoBA,YAAY,cAA4C,MAAc;AACpE,SAAK,OAAO;AACZ,SAAK,eAAe;AAAA,EACtB;AAAA,EAEA,MAAM,KAAK,MAAoC;AAlCjD;AAmCI,SAAK,SAAS,KAAK;AACnB,UAAM,kBAAkB,KAAK,OAAO,eAAe;AACnD,UAAM,KAAK,OAAO,iBAAiB,iCAC9B,kBAD8B;AAAA;AAAA;AAAA;AAAA;AAAA,MAMjC,YAAY;AAAA,IACd,EAAC;AACD,SAAK,iBAAiB,KAAK,OAAO,YAAY;AAC9C,SAAK,cAAc,KAAK;AACxB,QAAI,KAAK,uBAAuB,kBAAkB;AAChD,WAAK,YAAY,SAAS,KAAK,eAAe;AAC9C,WAAK,YAAY,QAAQ,KAAK,eAAe;AAAA,IAC/C;AACA,QAAI,EAAE,KAAK,uBAAuB,mBAAmB;AACnD,YAAM,UAAU,iDAAiD;AAAA,IACnE;AAEA,SAAK,YAAY,IAAI,0BAA0B,EAAE,OAAO,KAAK,OAAO,CAAC;AAErE,SAAK,iBAAiB,IAAI,0BAA0B;AAAA,MAClD,MAAM;AAAA,MACN,cAAc,KAAK;AAAA,IACrB,CAAC;AAED,SAAK,SAAS,IAAI;AAAA,OAChB,UAAK,eAAe,UAApB,YAA6B;AAAA,OAC7B,UAAK,eAAe,WAApB,YAA8B;AAAA,IAChC;AAEA,QAAI,iBAAiB,KAAK,UAAU;AACpC,eAAW,eAAe,KAAK,cAAc;AAC3C,YAAM,YAAY,KAAK;AAAA,QACrB,cAAc,KAAK;AAAA,QACnB,cAAc,KAAK;AAAA,MACrB,CAAC;AACD,uBAAiB,eAAe,YAAY,YAAa,WAAY;AAAA,IACvE;AACA,mBACG,OAAO,KAAK,eAAe,QAAQ,EACnC,MAAM,CAAC,MAAM,QAAQ,MAAM,6BAA6B,CAAC,CAAC,EAC1D,QAAQ,MAAM,KAAK,QAAQ,CAAC;AAC/B,SAAK,iBAAiB,KAAK;AAAA,EAC7B;AAAA,EAEA,MAAM,UAAU;AAlFlB;AAmFI,eAAW,eAAe,KAAK,cAAc;AAC3C,YAAM,YAAY,QAAQ;AAAA,IAC5B;AACA,eAAK,mBAAL,mBAAqB;AAAA,EACvB;AACF;;;ACxFA,YAAY,YAAY;;;ACuBtB,mBAAgB;AAAA,EACd,uBAAuB;AAAA,EACvB,2BAA2B;AAC7B;;;ACxBF,IAA8B,mBAA9B,MAAgF;AAAA,EAAhF;AASE,SAAU,aAAuB;AAAA;AAAA,EAEjC,MAAM,KAAK;AAAA,IACT;AAAA,IACA,cAAc;AAAA,EAChB,GAA+C;AAhBjD;AAiBI,QAAI,EAAE,sBAAsB,mBAAmB;AAC7C,YAAM,UAAU,qDAAqD;AAAA,IACvE;AACA,SAAK,cAAc,IAAI,gBAAgB;AAAA,MACrC,WAAW,CAAC,OAAO,eAAe,KAAK,UAAU,OAAO,UAAU;AAAA,IACpE,CAAC;AACD,SAAK,SAAS,gBAAgB;AAC9B,QAAI,cAAc;AAChB,WAAK,QAAM,UAAK,WAAL,mBAAa,WAAW,UAAS;AAAA,IAC9C;AACA,SAAK,aAAa;AAClB,SAAK,aAAa;AAAA,EACpB;AAAA,EAEA,MAAM,UAAU;AACd,SAAK,aAAa;AAClB,SAAK,SAAS;AACd,SAAK,MAAM;AAAA,EACb;AAMF;;;AF/BA,IAAqB,sBAArB,cAAiD,iBAAiB;AAAA,EAahE,YAAY,MAAyB;AACnC,UAAM;AALR,2BAAsC;AAMpC,QAAI,KAAK,YAAY;AACnB,WAAK,aAAa,KAAK;AAAA,IACzB,WAAW,KAAK,WAAW;AACzB,WAAK,eAAe,KAAK,SAAS;AAAA,IACpC;AAAA,EACF;AAAA,EAnBA,WAAW,cAAc;AACvB,WAAO,OAAO,oBAAoB;AAAA,EACpC;AAAA,EAmBA,MAAM,KAAK,EAAE,cAAc,cAAc,WAAW,GAAgC;AAClF,UAAM,MAAM,KAAK,EAAE,cAAc,cAAc,WAAW,CAAC;AAE3D,UAAM,UAAU,MAAa,uBAAgB;AAAA,MAC3C,wDAAwD,aAAa,yBAAyB,CAAC;AAAA,IACjG;AAEA,SAAK,iBAAiB,MAAa,sBAAe,kBAAkB,SAAS;AAAA,MAC3E,aAAa;AAAA,QACX,gBACE;AAAA,QACF,UAAU;AAAA,MACZ;AAAA,MACA,aAAa;AAAA,MACb,oBAAoB;AAAA,MACpB,uBAAuB;AAAA,IACzB,CAAC;AAAA,EAGH;AAAA,EAEA,MAAM,UAAU;AArDlB;AAsDI,UAAM,MAAM,QAAQ;AACpB,YAAM,UAAK,mBAAL,mBAAqB;AAC3B,SAAK,kBAAkB;AAAA,EACzB;AAAA,EAEA,MAAM,eAAe,MAAc;AACjC,UAAM,MAAM,IAAI,MAAM;AAEtB,UAAM,IAAI,QAAQ,CAAC,SAAS,WAAW;AACrC,UAAI,cAAc;AAClB,UAAI,SAAS,MAAM,QAAQ,GAAG;AAC9B,UAAI,UAAU,CAAC,QAAQ,OAAO,GAAG;AACjC,UAAI,MAAM;AAAA,IACZ,CAAC;AACD,UAAM,YAAY,MAAM,kBAAkB,GAAG;AAC7C,SAAK,kBAAkB;AAAA,EACzB;AAAA,EAEA,MAAM,UAAU,OAAmB,YAA0D;AAxE/F;AAyEI,QAAI;AACF,UAAI,KAAK,YAAY;AACnB,mBAAW,QAAQ,KAAK;AACxB;AAAA,MACF;AACA,UAAI,CAAC,KAAK,QAAQ;AAChB,cAAM,UAAU,sCAAsC;AAAA,MACxD;AACA,UAAI,cAAc,YAAY,IAAI;AAClC,iBAAK,mBAAL,mBAAqB;AAAA,QACnB,KAAK;AAAA,QACL;AAAA,QACA,CAAC,WAAY,KAAK,sBAAsB;AAAA;AAG1C,UAAI,KAAK,YAAY;AACnB,cAAM,KAAK,eAAe,KAAK;AAAA,MACjC,OAAO;AACL,cAAM,KAAK,sBAAsB,KAAK;AAAA,MACxC;AACA,YAAM,WAAW,IAAI,WAAW,KAAK,QAAQ;AAAA,QAC3C,WAAW,MAAM,aAAa,KAAK,IAAI;AAAA,MACzC,CAAC;AACD,iBAAW,QAAQ,QAAQ;AAAA,IAC7B,UAAE;AACA,YAAM,MAAM;AAAA,IACd;AAAA,EACF;AAAA,EAEA,MAAM,sBAAsB,OAAmB;AAtGjD;AAuGI,QAAI,CAAC,KAAK,UAAU,CAAC,KAAK,OAAO,CAAC,KAAK,uBAAuB,CAAC,KAAK;AAAY;AAGhF,SAAI,UAAK,wBAAL,mBAA0B,cAAc;AAC1C,WAAK,IAAI,SAAS;AAClB,WAAK,IAAI,2BAA2B;AACpC,YAAM,SAAS,MAAM;AAAA,QACnB,KAAK,oBAAoB;AAAA,QACzB,KAAK,WAAW;AAAA,QAChB,KAAK,WAAW;AAAA,MAClB;AACA,WAAK,IAAI,UAAU,QAAQ,GAAG,CAAC;AAC/B,WAAK,IAAI,SAAS;AAClB,WAAK,IAAI,2BAA2B;AACpC,UAAI,KAAK,iBAAiB;AACxB,aAAK,IAAI;AAAA,UACP,KAAK;AAAA,UACL;AAAA,UACA;AAAA,UACA,KAAK,gBAAgB;AAAA,UACrB,KAAK,gBAAgB;AAAA,UACrB;AAAA,UACA;AAAA,UACA,KAAK,OAAO;AAAA,UACZ,KAAK,OAAO;AAAA,QACd;AAAA,MACF,OAAO;AACL,aAAK,IAAI,YAAY;AACrB,aAAK,IAAI,SAAS,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AAAA,MAC/D;AAEA,WAAK,IAAI,2BAA2B;AAAA,IACtC;AACA,SAAK,IAAI,UAAU,OAAO,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AAAA,EAEvE;AAAA,EAEA,MAAM,eAAe,OAAmB;AA5I1C;AA6II,QACE,CAAC,KAAK,OACN,CAAC,KAAK,UACN,GAAC,gBAAK,wBAAL,mBAA0B,iBAA1B,mBAAwC,WACzC,CAAC,KAAK,YACN;AACA;AAAA,IACF;AACA,SAAK,OAAO,QAAQ,MAAM;AAC1B,SAAK,OAAO,SAAS,MAAM;AAC3B,SAAK,IAAI,KAAK;AACd,SAAK,IAAI,2BAA2B;AAEpC,UAAM,SAAS,MAAM;AAAA,MACnB,KAAK,oBAAoB;AAAA,MACzB,KAAK,WAAW;AAAA,MAChB,KAAK,WAAW;AAAA,IAClB;AAEA,SAAK,IAAI,SAAS;AAClB,SAAK,IAAI,2BAA2B;AACpC,SAAK,IAAI,UAAU,QAAQ,GAAG,CAAC;AAC/B,SAAK,IAAI,SAAS;AAClB,SAAK,IAAI,2BAA2B;AACpC,SAAK,IAAI,UAAU,OAAO,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AACrE,SAAK,IAAI,2BAA2B;AACpC,SAAK,IAAI,SAAS,QAAQ,KAAK,UAAU;AACzC,SAAK,IAAI,UAAU,OAAO,GAAG,CAAC;AAC9B,SAAK,IAAI,QAAQ;AAAA,EACnB;AACF;AAEA,SAAS,aACP,MACA,YACA,aACsB;AACtB,QAAM,YAA+B,IAAI,kBAAkB,aAAa,cAAc,CAAC;AACvF,QAAM,SAAS,KAAK,gBAAgB;AACpC,WAAS,IAAI,GAAG,IAAI,OAAO,QAAQ,KAAK,GAAG;AACzC,cAAU,IAAI,CAAC,IAAI,OAAO,CAAC;AAC3B,cAAU,IAAI,IAAI,CAAC,IAAI,OAAO,CAAC;AAC/B,cAAU,IAAI,IAAI,CAAC,IAAI,OAAO,CAAC;AAC/B,cAAU,IAAI,IAAI,CAAC,IAAI,OAAO,CAAC;AAAA,EACjC;AACA,QAAM,UAAU,IAAI,UAAU,WAAW,YAAY,WAAW;AAEhE,SAAO,kBAAkB,OAAO;AAClC;;;AG3LA,IAAqB,mBAArB,cAA8C,iBAAiB;AAAA,EAC7D,MAAM,UAAU,OAAmB,YAA0D;AAC3F,eAAW,QAAQ,KAAK;AAAA,EAC1B;AAAA,EAEA,MAAM,UAAU;AAAA,EAEhB;AACF;;;ACNO,IAAM,iBAAiB,CAAC,aAAqB,OAAO;AACzD,QAAM,sBAAsB,kBAAkB,eAAe,oBAAsB;AACnF,MAAI,CAAC,qBAAqB;AACxB,UAAM,IAAI,MAAM,2CAA2C;AAAA,EAC7D;AACA,QAAM,WAAW,IAAI;AAAA,IACnB,CAAC,IAAI,oBAAsB,EAAE,WAAW,CAAC,CAAC;AAAA,IAC1C;AAAA,EACF;AACA,SAAO;AACT;AAEO,IAAM,oBAAoB,CAAC,cAAsB;AACtD,QAAM,sBAAsB,kBAAkB,eAAe,oBAAsB;AACnF,MAAI,CAAC,qBAAqB;AACxB,UAAM,IAAI,MAAM,2CAA2C;AAAA,EAC7D;AACA,QAAM,WAAW,IAAI;AAAA,IACnB,CAAC,IAAI,oBAAsB,EAAE,UAAU,CAAC,CAAC;AAAA,IACzC;AAAA,EACF;AACA,SAAO;AACT;AAEO,IAAM,QAAQ,MAAM;AACzB,QAAM,sBAAsB,kBAAkB,eAAe,oBAAsB;AACnF,MAAI,CAAC,qBAAqB;AACxB,UAAM,IAAI,MAAM,2CAA2C;AAAA,EAC7D;AACA,QAAM,WAAW,IAAI,kBAAkB,CAAC,IAAI,iBAAiB,CAAC,GAAG,OAAO;AACxE,SAAO;AACT","sourcesContent":["import type { ProcessorOptions, Track, TrackProcessor } from 'livekit-client';\nimport { VideoTrackTransformer } from './transformers';\n\nexport default class ProcessorPipeline implements TrackProcessor<Track.Kind> {\n  static get isSupported() {\n    return (\n      typeof MediaStreamTrackGenerator !== 'undefined' &&\n      typeof MediaStreamTrackProcessor !== 'undefined'\n    );\n  }\n\n  name: string;\n\n  source?: MediaStreamVideoTrack;\n\n  sourceSettings?: MediaTrackSettings;\n\n  processor?: MediaStreamTrackProcessor<VideoFrame>;\n\n  trackGenerator?: MediaStreamTrackGenerator<VideoFrame>;\n\n  canvas?: OffscreenCanvas;\n\n  sourceDummy?: HTMLMediaElement;\n\n  processedTrack?: MediaStreamTrack;\n\n  transformers: Array<VideoTrackTransformer>;\n\n  constructor(transformers: Array<VideoTrackTransformer>, name: string) {\n    this.name = name;\n    this.transformers = transformers;\n  }\n\n  async init(opts: ProcessorOptions<Track.Kind>) {\n    this.source = opts.track as MediaStreamVideoTrack;\n    const origConstraints = this.source.getConstraints();\n    await this.source.applyConstraints({\n      ...origConstraints,\n      // @ts-expect-error when a mediastream track is resized and/or cropped, the `VideoFrame` will have a coded height/width of the original video size\n      // this leads to a shift of the underlying video as the frame itself is being rendered with the coded size\n      // but image segmentation is based on the display dimensions (-> the cropped version)\n      // in order to prevent this, we force the resize mode to \"none\"\n      resizeMode: 'none',\n    });\n    this.sourceSettings = this.source.getSettings();\n    this.sourceDummy = opts.element;\n    if (this.sourceDummy instanceof HTMLVideoElement) {\n      this.sourceDummy.height = this.sourceSettings.height!;\n      this.sourceDummy.width = this.sourceSettings.width!;\n    }\n    if (!(this.sourceDummy instanceof HTMLVideoElement)) {\n      throw TypeError('Currently only video transformers are supported');\n    }\n    // TODO explore if we can do all the processing work in a webworker\n    this.processor = new MediaStreamTrackProcessor({ track: this.source });\n\n    this.trackGenerator = new MediaStreamTrackGenerator({\n      kind: 'video',\n      signalTarget: this.source,\n    });\n\n    this.canvas = new OffscreenCanvas(\n      this.sourceSettings.width ?? 300,\n      this.sourceSettings.height ?? 300,\n    );\n\n    let readableStream = this.processor.readable;\n    for (const transformer of this.transformers) {\n      await transformer.init({\n        outputCanvas: this.canvas,\n        inputElement: this.sourceDummy!,\n      });\n      readableStream = readableStream.pipeThrough(transformer!.transformer!);\n    }\n    readableStream\n      .pipeTo(this.trackGenerator.writable)\n      .catch((e) => console.error('error when trying to pipe', e))\n      .finally(() => this.destroy());\n    this.processedTrack = this.trackGenerator as MediaStreamVideoTrack;\n  }\n\n  async destroy() {\n    for (const transformer of this.transformers) {\n      await transformer.destroy();\n    }\n    this.trackGenerator?.stop();\n  }\n}\n","import * as vision from '@mediapipe/tasks-vision';\nimport { dependencies } from '../../package.json';\nimport VideoTransformer from './VideoTransformer';\nimport { VideoTransformerInitOptions } from './types';\n\nexport type BackgroundOptions = {\n  blurRadius?: number;\n  imagePath?: string;\n};\n\nexport default class BackgroundProcessor extends VideoTransformer {\n  static get isSupported() {\n    return typeof OffscreenCanvas !== 'undefined';\n  }\n\n  imageSegmenter?: vision.ImageSegmenter;\n\n  segmentationResults: vision.ImageSegmenterResult | undefined;\n\n  backgroundImage: ImageBitmap | null = null;\n\n  blurRadius?: number;\n\n  constructor(opts: BackgroundOptions) {\n    super();\n    if (opts.blurRadius) {\n      this.blurRadius = opts.blurRadius;\n    } else if (opts.imagePath) {\n      this.loadBackground(opts.imagePath);\n    }\n  }\n\n  async init({ outputCanvas, inputElement: inputVideo }: VideoTransformerInitOptions) {\n    await super.init({ outputCanvas, inputElement: inputVideo });\n\n    const fileSet = await vision.FilesetResolver.forVisionTasks(\n      `https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@${dependencies['@mediapipe/tasks-vision']}/wasm`,\n    );\n\n    this.imageSegmenter = await vision.ImageSegmenter.createFromOptions(fileSet, {\n      baseOptions: {\n        modelAssetPath:\n          'https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_segmenter/float16/latest/selfie_segmenter.tflite',\n        delegate: 'CPU',\n      },\n      runningMode: 'VIDEO',\n      outputCategoryMask: true,\n      outputConfidenceMasks: false,\n    });\n\n    // this.loadBackground(opts.backgroundUrl).catch((e) => console.error(e));\n  }\n\n  async destroy() {\n    await super.destroy();\n    await this.imageSegmenter?.close();\n    this.backgroundImage = null;\n  }\n\n  async loadBackground(path: string) {\n    const img = new Image();\n\n    await new Promise((resolve, reject) => {\n      img.crossOrigin = 'Anonymous';\n      img.onload = () => resolve(img);\n      img.onerror = (err) => reject(err);\n      img.src = path;\n    });\n    const imageData = await createImageBitmap(img);\n    this.backgroundImage = imageData;\n  }\n\n  async transform(frame: VideoFrame, controller: TransformStreamDefaultController<VideoFrame>) {\n    try {\n      if (this.isDisabled) {\n        controller.enqueue(frame);\n        return;\n      }\n      if (!this.canvas) {\n        throw TypeError('Canvas needs to be initialized first');\n      }\n      let startTimeMs = performance.now();\n      this.imageSegmenter?.segmentForVideo(\n        this.inputVideo!,\n        startTimeMs,\n        (result) => (this.segmentationResults = result),\n      );\n\n      if (this.blurRadius) {\n        await this.blurBackground(frame);\n      } else {\n        await this.drawVirtualBackground(frame);\n      }\n      const newFrame = new VideoFrame(this.canvas, {\n        timestamp: frame.timestamp || Date.now(),\n      });\n      controller.enqueue(newFrame);\n    } finally {\n      frame.close();\n    }\n  }\n\n  async drawVirtualBackground(frame: VideoFrame) {\n    if (!this.canvas || !this.ctx || !this.segmentationResults || !this.inputVideo) return;\n    // this.ctx.save();\n    // this.ctx.clearRect(0, 0, this.canvas.width, this.canvas.height);\n    if (this.segmentationResults?.categoryMask) {\n      this.ctx.filter = 'blur(3px)';\n      this.ctx.globalCompositeOperation = 'copy';\n      const bitmap = await maskToBitmap(\n        this.segmentationResults.categoryMask,\n        this.inputVideo.videoWidth,\n        this.inputVideo.videoHeight,\n      );\n      this.ctx.drawImage(bitmap, 0, 0);\n      this.ctx.filter = 'none';\n      this.ctx.globalCompositeOperation = 'source-in';\n      if (this.backgroundImage) {\n        this.ctx.drawImage(\n          this.backgroundImage,\n          0,\n          0,\n          this.backgroundImage.width,\n          this.backgroundImage.height,\n          0,\n          0,\n          this.canvas.width,\n          this.canvas.height,\n        );\n      } else {\n        this.ctx.fillStyle = '#00FF00';\n        this.ctx.fillRect(0, 0, this.canvas.width, this.canvas.height);\n      }\n\n      this.ctx.globalCompositeOperation = 'destination-over';\n    }\n    this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);\n    // this.ctx.restore();\n  }\n\n  async blurBackground(frame: VideoFrame) {\n    if (\n      !this.ctx ||\n      !this.canvas ||\n      !this.segmentationResults?.categoryMask?.canvas ||\n      !this.inputVideo\n    ) {\n      return;\n    }\n    this.canvas.width = frame.codedWidth;\n    this.canvas.height = frame.codedHeight;\n    this.ctx.save();\n    this.ctx.globalCompositeOperation = 'copy';\n\n    const bitmap = await maskToBitmap(\n      this.segmentationResults.categoryMask,\n      this.inputVideo.videoWidth,\n      this.inputVideo.videoHeight,\n    );\n\n    this.ctx.filter = 'blur(3px)';\n    this.ctx.globalCompositeOperation = 'copy';\n    this.ctx.drawImage(bitmap, 0, 0);\n    this.ctx.filter = 'none';\n    this.ctx.globalCompositeOperation = 'source-out';\n    this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);\n    this.ctx.globalCompositeOperation = 'destination-over';\n    this.ctx.filter = `blur(${this.blurRadius}px)`;\n    this.ctx.drawImage(frame, 0, 0);\n    this.ctx.restore();\n  }\n}\n\nfunction maskToBitmap(\n  mask: vision.MPMask,\n  videoWidth: number,\n  videoHeight: number,\n): Promise<ImageBitmap> {\n  const dataArray: Uint8ClampedArray = new Uint8ClampedArray(videoWidth * videoHeight * 4);\n  const result = mask.getAsUint8Array();\n  for (let i = 0; i < result.length; i += 1) {\n    dataArray[i * 4] = result[i];\n    dataArray[i * 4 + 1] = result[i];\n    dataArray[i * 4 + 2] = result[i];\n    dataArray[i * 4 + 3] = result[i];\n  }\n  const dataNew = new ImageData(dataArray, videoWidth, videoHeight);\n\n  return createImageBitmap(dataNew);\n}\n","{\n  \"name\": \"@livekit/track-processors\",\n  \"version\": \"0.1.8\",\n  \"description\": \"LiveKit track processors\",\n  \"main\": \"dist/index.js\",\n  \"module\": \"dist/index.mjs\",\n  \"source\": \"src/index.ts\",\n  \"types\": \"dist/src/index.d.ts\",\n  \"repository\": \"git@github.com:livekit/livekit-track-processors.git\",\n  \"author\": \"Lukas Seiler\",\n  \"license\": \"Apache-2.0\",\n  \"scripts\": {\n    \"build\": \"tsup --onSuccess \\\"tsc --declaration --emitDeclarationOnly\\\"\",\n    \"build-docs\": \"yarn exec typedoc\",\n    \"build-sample\": \"cd example && webpack && cp styles.css index.html dist/\",\n    \"lint\": \"eslint src\",\n    \"test\": \"jest\",\n    \"sample\": \"vite serve example --port 8080 --open\"\n  },\n  \"files\": [\n    \"dist\",\n    \"src\"\n  ],\n  \"dependencies\": {\n    \"@mediapipe/holistic\": \"0.5.1675471629\",\n    \"@mediapipe/tasks-vision\": \"0.10.1\"\n  },\n  \"peerDependencies\": {\n    \"livekit-client\": \"^1.12.0\"\n  },\n  \"devDependencies\": {\n    \"@trivago/prettier-plugin-sort-imports\": \"^4.1.1\",\n    \"@types/dom-mediacapture-transform\": \"^0.1.5\",\n    \"@types/jest\": \"^27.0.3\",\n    \"@types/offscreencanvas\": \"^2019.7.0\",\n    \"@typescript-eslint/eslint-plugin\": \"^4.31.2\",\n    \"@webpack-cli/serve\": \"^1.5.2\",\n    \"eslint\": \"8.39.0\",\n    \"eslint-config-airbnb-typescript\": \"17.0.0\",\n    \"eslint-config-prettier\": \"8.8.0\",\n    \"eslint-plugin-ecmascript-compat\": \"^3.0.0\",\n    \"eslint-plugin-import\": \"2.27.5\",\n    \"jest\": \"^27.4.3\",\n    \"livekit-client\": \"^0.16.6\",\n    \"prettier\": \"^2.8.8\",\n    \"ts-jest\": \"^27.0.7\",\n    \"ts-loader\": \"^8.1.0\",\n    \"ts-proto\": \"^1.85.0\",\n    \"tsup\": \"^7.1.0\",\n    \"typedoc\": \"^0.20.35\",\n    \"typedoc-plugin-no-inherit\": \"1.3.0\",\n    \"typescript\": \"^5.0.4\",\n    \"vite\": \"^4.3.8\",\n    \"webpack\": \"^5.53.0\",\n    \"webpack-cli\": \"^4.8.0\",\n    \"webpack-dev-server\": \"^4.2.1\"\n  }\n}\n","import { VideoTrackTransformer, VideoTransformerInitOptions } from './types';\n\nexport default abstract class VideoTransformer implements VideoTrackTransformer {\n  transformer?: TransformStream;\n\n  canvas?: OffscreenCanvas;\n\n  ctx?: OffscreenCanvasRenderingContext2D;\n\n  inputVideo?: HTMLVideoElement;\n\n  protected isDisabled?: Boolean = false;\n\n  async init({\n    outputCanvas,\n    inputElement: inputVideo,\n  }: VideoTransformerInitOptions): Promise<void> {\n    if (!(inputVideo instanceof HTMLVideoElement)) {\n      throw TypeError('Video transformer needs a HTMLVideoElement as input');\n    }\n    this.transformer = new TransformStream({\n      transform: (frame, controller) => this.transform(frame, controller),\n    });\n    this.canvas = outputCanvas || null;\n    if (outputCanvas) {\n      this.ctx = this.canvas?.getContext('2d') || undefined;\n    }\n    this.inputVideo = inputVideo;\n    this.isDisabled = false;\n  }\n\n  async destroy() {\n    this.isDisabled = true;\n    this.canvas = undefined;\n    this.ctx = undefined;\n  }\n\n  abstract transform(\n    frame: VideoFrame,\n    controller: TransformStreamDefaultController<VideoFrame>,\n  ): void;\n}\n","import VideoTransformer from './VideoTransformer';\n\nexport default class DummyTransformer extends VideoTransformer {\n  async transform(frame: VideoFrame, controller: TransformStreamDefaultController<VideoFrame>) {\n    controller.enqueue(frame);\n  }\n\n  async destroy() {\n    // nothing to do\n  }\n}\n","import ProcessorPipeline from './ProcessorPipeline';\nimport BackgroundTransformer from './transformers/BackgroundTransformer';\nimport DummyTransformer from './transformers/DummyTransformer';\n\nexport const BackgroundBlur = (blurRadius: number = 10) => {\n  const isPipelineSupported = ProcessorPipeline.isSupported && BackgroundTransformer.isSupported;\n  if (!isPipelineSupported) {\n    throw new Error('pipeline is not supported in this browser');\n  }\n  const pipeline = new ProcessorPipeline(\n    [new BackgroundTransformer({ blurRadius })],\n    'background-blur',\n  );\n  return pipeline;\n};\n\nexport const VirtualBackground = (imagePath: string) => {\n  const isPipelineSupported = ProcessorPipeline.isSupported && BackgroundTransformer.isSupported;\n  if (!isPipelineSupported) {\n    throw new Error('pipeline is not supported in this browser');\n  }\n  const pipeline = new ProcessorPipeline(\n    [new BackgroundTransformer({ imagePath })],\n    'virtual-background',\n  );\n  return pipeline;\n};\n\nexport const Dummy = () => {\n  const isPipelineSupported = ProcessorPipeline.isSupported && BackgroundTransformer.isSupported;\n  if (!isPipelineSupported) {\n    throw new Error('pipeline is not supported in this browser');\n  }\n  const pipeline = new ProcessorPipeline([new DummyTransformer()], 'dummy');\n  return pipeline;\n};\n"]}

package/dist/index.mjs

@@ -1,3 +1,23 @@
+var __defProp = Object.defineProperty;
+var __defProps = Object.defineProperties;
+var __getOwnPropDescs = Object.getOwnPropertyDescriptors;
+var __getOwnPropSymbols = Object.getOwnPropertySymbols;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __propIsEnum = Object.prototype.propertyIsEnumerable;
+var __defNormalProp = (obj, key, value) => key in obj ? __defProp(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
+var __spreadValues = (a, b) => {
+  for (var prop in b || (b = {}))
+    if (__hasOwnProp.call(b, prop))
+      __defNormalProp(a, prop, b[prop]);
+  if (__getOwnPropSymbols)
+    for (var prop of __getOwnPropSymbols(b)) {
+      if (__propIsEnum.call(b, prop))
+        __defNormalProp(a, prop, b[prop]);
+    }
+  return a;
+};
+var __spreadProps = (a, b) => __defProps(a, __getOwnPropDescs(b));
+
 // src/ProcessorPipeline.ts
 var ProcessorPipeline = class {
   static get isSupported() {
@@ -10,26 +30,41 @@ var ProcessorPipeline = class {
   async init(opts) {
     var _a, _b;
     this.source = opts.track;
+    const origConstraints = this.source.getConstraints();
+    await this.source.applyConstraints(__spreadProps(__spreadValues({}, origConstraints), {
+      // @ts-expect-error when a mediastream track is resized and/or cropped, the `VideoFrame` will have a coded height/width of the original video size
+      // this leads to a shift of the underlying video as the frame itself is being rendered with the coded size
+      // but image segmentation is based on the display dimensions (-> the cropped version)
+      // in order to prevent this, we force the resize mode to "none"
+      resizeMode: "none"
+    }));
     this.sourceSettings = this.source.getSettings();
     this.sourceDummy = opts.element;
+    if (this.sourceDummy instanceof HTMLVideoElement) {
+      this.sourceDummy.height = this.sourceSettings.height;
+      this.sourceDummy.width = this.sourceSettings.width;
+    }
     if (!(this.sourceDummy instanceof HTMLVideoElement)) {
       throw TypeError("Currently only video transformers are supported");
     }
     this.processor = new MediaStreamTrackProcessor({ track: this.source });
-    this.trackGenerator = new MediaStreamTrackGenerator({ kind: "video" });
+    this.trackGenerator = new MediaStreamTrackGenerator({
+      kind: "video",
+      signalTarget: this.source
+    });
     this.canvas = new OffscreenCanvas(
       (_a = this.sourceSettings.width) != null ? _a : 300,
       (_b = this.sourceSettings.height) != null ? _b : 300
     );
     let readableStream = this.processor.readable;
     for (const transformer of this.transformers) {
-      transformer.init({
+      await transformer.init({
         outputCanvas: this.canvas,
         inputElement: this.sourceDummy
       });
       readableStream = readableStream.pipeThrough(transformer.transformer);
     }
-    readableStream.pipeTo(this.trackGenerator.writable);
+    readableStream.pipeTo(this.trackGenerator.writable).catch((e) => console.error("error when trying to pipe", e)).finally(() => this.destroy());
     this.processedTrack = this.trackGenerator;
   }
   async destroy() {
@@ -55,7 +90,10 @@ var VideoTransformer = class {
   constructor() {
     this.isDisabled = false;
   }
-  init({ outputCanvas, inputElement: inputVideo }) {
+  async init({
+    outputCanvas,
+    inputElement: inputVideo
+  }) {
     var _a;
     if (!(inputVideo instanceof HTMLVideoElement)) {
       throw TypeError("Video transformer needs a HTMLVideoElement as input");
@@ -92,7 +130,7 @@ var BackgroundProcessor = class extends VideoTransformer {
     return typeof OffscreenCanvas !== "undefined";
   }
   async init({ outputCanvas, inputElement: inputVideo }) {
-    super.init({ outputCanvas, inputElement: inputVideo });
+    await super.init({ outputCanvas, inputElement: inputVideo });
     const fileSet = await vision.FilesetResolver.forVisionTasks(
       `https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@${dependencies["@mediapipe/tasks-vision"]}/wasm`
     );
@@ -112,22 +150,6 @@ var BackgroundProcessor = class extends VideoTransformer {
     await ((_a = this.imageSegmenter) == null ? void 0 : _a.close());
     this.backgroundImage = null;
   }
-  async sendFramesContinuouslyForSegmentation(videoEl) {
-    var _a;
-    if (!this.isDisabled) {
-      if (videoEl.videoWidth > 0 && videoEl.videoHeight > 0) {
-        let startTimeMs = performance.now();
-        (_a = this.imageSegmenter) == null ? void 0 : _a.segmentForVideo(
-          videoEl,
-          startTimeMs,
-          (result) => this.segmentationResults = result
-        );
-      }
-      videoEl.requestVideoFrameCallback(() => {
-        this.sendFramesContinuouslyForSegmentation(videoEl);
-      });
-    }
-  }
   async loadBackground(path) {
     const img = new Image();
     await new Promise((resolve, reject) => {
@@ -141,29 +163,32 @@ var BackgroundProcessor = class extends VideoTransformer {
   }
   async transform(frame, controller) {
     var _a;
-    if (this.isDisabled) {
-      controller.enqueue(frame);
-      return;
-    }
-    if (!this.canvas) {
-      throw TypeError("Canvas needs to be initialized first");
-    }
-    let startTimeMs = performance.now();
-    (_a = this.imageSegmenter) == null ? void 0 : _a.segmentForVideo(
-      this.inputVideo,
-      startTimeMs,
-      (result) => this.segmentationResults = result
-    );
-    if (this.blurRadius) {
-      await this.blurBackground(frame);
-    } else {
-      await this.drawVirtualBackground(frame);
+    try {
+      if (this.isDisabled) {
+        controller.enqueue(frame);
+        return;
+      }
+      if (!this.canvas) {
+        throw TypeError("Canvas needs to be initialized first");
+      }
+      let startTimeMs = performance.now();
+      (_a = this.imageSegmenter) == null ? void 0 : _a.segmentForVideo(
+        this.inputVideo,
+        startTimeMs,
+        (result) => this.segmentationResults = result
+      );
+      if (this.blurRadius) {
+        await this.blurBackground(frame);
+      } else {
+        await this.drawVirtualBackground(frame);
+      }
+      const newFrame = new VideoFrame(this.canvas, {
+        timestamp: frame.timestamp || Date.now()
+      });
+      controller.enqueue(newFrame);
+    } finally {
+      frame.close();
     }
-    const newFrame = new VideoFrame(this.canvas, {
-      timestamp: frame.timestamp || Date.now()
-    });
-    frame.close();
-    controller.enqueue(newFrame);
   }
   async drawVirtualBackground(frame) {
     var _a;
@@ -205,6 +230,8 @@ var BackgroundProcessor = class extends VideoTransformer {
     if (!this.ctx || !this.canvas || !((_b = (_a = this.segmentationResults) == null ? void 0 : _a.categoryMask) == null ? void 0 : _b.canvas) || !this.inputVideo) {
       return;
     }
+    this.canvas.width = frame.codedWidth;
+    this.canvas.height = frame.codedHeight;
     this.ctx.save();
     this.ctx.globalCompositeOperation = "copy";
     const bitmap = await maskToBitmap(

package/dist/index.mjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/ProcessorPipeline.ts","../src/transformers/BackgroundTransformer.ts","../package.json","../src/transformers/VideoTransformer.ts","../src/transformers/DummyTransformer.ts","../src/index.ts"],"sourcesContent":["import type { ProcessorOptions, Track, TrackProcessor } from 'livekit-client';\nimport { VideoTrackTransformer } from './transformers';\n\nexport default class ProcessorPipeline implements TrackProcessor<Track.Kind> {\n  static get isSupported() {\n    return (\n      typeof MediaStreamTrackGenerator !== 'undefined' &&\n      typeof MediaStreamTrackProcessor !== 'undefined'\n    );\n  }\n\n  name: string;\n\n  source?: MediaStreamVideoTrack;\n\n  sourceSettings?: MediaTrackSettings;\n\n  processor?: MediaStreamTrackProcessor<VideoFrame>;\n\n  trackGenerator?: MediaStreamTrackGenerator<VideoFrame>;\n\n  canvas?: OffscreenCanvas;\n\n  sourceDummy?: HTMLMediaElement;\n\n  processedTrack?: MediaStreamTrack;\n\n  transformers: Array<VideoTrackTransformer>;\n\n  constructor(transformers: Array<VideoTrackTransformer>, name: string) {\n    this.name = name;\n    this.transformers = transformers;\n  }\n\n  async init(opts: ProcessorOptions<Track.Kind>) {\n    this.source = opts.track as MediaStreamVideoTrack;\n    this.sourceSettings = this.source.getSettings();\n    this.sourceDummy = opts.element;\n    if (!(this.sourceDummy instanceof HTMLVideoElement)) {\n      throw TypeError('Currently only video transformers are supported');\n    }\n    // TODO explore if we can do all the processing work in a webworker\n    this.processor = new MediaStreamTrackProcessor({ track: this.source });\n    this.trackGenerator = new MediaStreamTrackGenerator({ kind: 'video' });\n\n    this.canvas = new OffscreenCanvas(\n      this.sourceSettings.width ?? 300,\n      this.sourceSettings.height ?? 300,\n    );\n\n    let readableStream = this.processor.readable;\n    for (const transformer of this.transformers) {\n      transformer.init({\n        outputCanvas: this.canvas,\n        inputElement: this.sourceDummy!,\n      });\n      readableStream = readableStream.pipeThrough(transformer!.transformer!);\n    }\n    readableStream.pipeTo(this.trackGenerator.writable);\n    this.processedTrack = this.trackGenerator as MediaStreamVideoTrack;\n  }\n\n  async destroy() {\n    for (const transformer of this.transformers) {\n      await transformer.destroy();\n    }\n    this.trackGenerator?.stop();\n  }\n}\n","import * as vision from '@mediapipe/tasks-vision';\nimport { dependencies } from '../../package.json';\nimport VideoTransformer from './VideoTransformer';\nimport { VideoTransformerInitOptions } from './types';\n\nexport type BackgroundOptions = {\n  blurRadius?: number;\n  imagePath?: string;\n};\n\nexport default class BackgroundProcessor extends VideoTransformer {\n  static get isSupported() {\n    return typeof OffscreenCanvas !== 'undefined';\n  }\n\n  imageSegmenter?: vision.ImageSegmenter;\n\n  segmentationResults: vision.ImageSegmenterResult | undefined;\n\n  backgroundImage: ImageBitmap | null = null;\n\n  blurRadius?: number;\n\n  constructor(opts: BackgroundOptions) {\n    super();\n    if (opts.blurRadius) {\n      this.blurRadius = opts.blurRadius;\n    } else if (opts.imagePath) {\n      this.loadBackground(opts.imagePath);\n    }\n  }\n\n  async init({ outputCanvas, inputElement: inputVideo }: VideoTransformerInitOptions) {\n    super.init({ outputCanvas, inputElement: inputVideo });\n\n    const fileSet = await vision.FilesetResolver.forVisionTasks(\n      `https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@${dependencies['@mediapipe/tasks-vision']}/wasm`,\n    );\n\n    this.imageSegmenter = await vision.ImageSegmenter.createFromOptions(fileSet, {\n      baseOptions: {\n        modelAssetPath:\n          'https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_segmenter/float16/latest/selfie_segmenter.tflite',\n        delegate: 'CPU',\n      },\n      runningMode: 'VIDEO',\n      outputCategoryMask: true,\n      outputConfidenceMasks: false,\n    });\n\n    // this.loadBackground(opts.backgroundUrl).catch((e) => console.error(e));\n    // this.sendFramesContinuouslyForSegmentation(this.inputVideo!);\n  }\n\n  async destroy() {\n    await super.destroy();\n    await this.imageSegmenter?.close();\n    this.backgroundImage = null;\n  }\n\n  async sendFramesContinuouslyForSegmentation(videoEl: HTMLVideoElement) {\n    if (!this.isDisabled) {\n      if (videoEl.videoWidth > 0 && videoEl.videoHeight > 0) {\n        let startTimeMs = performance.now();\n        this.imageSegmenter?.segmentForVideo(\n          videoEl,\n          startTimeMs,\n          (result) => (this.segmentationResults = result),\n        );\n      }\n      videoEl.requestVideoFrameCallback(() => {\n        this.sendFramesContinuouslyForSegmentation(videoEl);\n      });\n    }\n  }\n\n  async loadBackground(path: string) {\n    const img = new Image();\n\n    await new Promise((resolve, reject) => {\n      img.crossOrigin = 'Anonymous';\n      img.onload = () => resolve(img);\n      img.onerror = (err) => reject(err);\n      img.src = path;\n    });\n    const imageData = await createImageBitmap(img);\n    this.backgroundImage = imageData;\n  }\n\n  async transform(frame: VideoFrame, controller: TransformStreamDefaultController<VideoFrame>) {\n    if (this.isDisabled) {\n      controller.enqueue(frame);\n      return;\n    }\n    if (!this.canvas) {\n      throw TypeError('Canvas needs to be initialized first');\n    }\n    let startTimeMs = performance.now();\n    this.imageSegmenter?.segmentForVideo(\n      this.inputVideo!,\n      startTimeMs,\n      (result) => (this.segmentationResults = result),\n    );\n\n    if (this.blurRadius) {\n      await this.blurBackground(frame);\n    } else {\n      await this.drawVirtualBackground(frame);\n    }\n    const newFrame = new VideoFrame(this.canvas, {\n      timestamp: frame.timestamp || Date.now(),\n    });\n    frame.close();\n    controller.enqueue(newFrame);\n  }\n\n  async drawVirtualBackground(frame: VideoFrame) {\n    if (!this.canvas || !this.ctx || !this.segmentationResults || !this.inputVideo) return;\n    // this.ctx.save();\n    // this.ctx.clearRect(0, 0, this.canvas.width, this.canvas.height);\n    if (this.segmentationResults?.categoryMask) {\n      this.ctx.filter = 'blur(3px)';\n      this.ctx.globalCompositeOperation = 'copy';\n      const bitmap = await maskToBitmap(\n        this.segmentationResults.categoryMask,\n        this.inputVideo.videoWidth,\n        this.inputVideo.videoHeight,\n      );\n      this.ctx.drawImage(bitmap, 0, 0);\n      this.ctx.filter = 'none';\n      this.ctx.globalCompositeOperation = 'source-in';\n      if (this.backgroundImage) {\n        this.ctx.drawImage(\n          this.backgroundImage,\n          0,\n          0,\n          this.backgroundImage.width,\n          this.backgroundImage.height,\n          0,\n          0,\n          this.canvas.width,\n          this.canvas.height,\n        );\n      } else {\n        this.ctx.fillStyle = '#00FF00';\n        this.ctx.fillRect(0, 0, this.canvas.width, this.canvas.height);\n      }\n\n      this.ctx.globalCompositeOperation = 'destination-over';\n    }\n    this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);\n    // this.ctx.restore();\n  }\n\n  async blurBackground(frame: VideoFrame) {\n    if (\n      !this.ctx ||\n      !this.canvas ||\n      !this.segmentationResults?.categoryMask?.canvas ||\n      !this.inputVideo\n    ) {\n      return;\n    }\n    this.ctx.save();\n    this.ctx.globalCompositeOperation = 'copy';\n\n    const bitmap = await maskToBitmap(\n      this.segmentationResults.categoryMask,\n      this.inputVideo.videoWidth,\n      this.inputVideo.videoHeight,\n    );\n    this.ctx.filter = 'blur(3px)';\n    this.ctx.globalCompositeOperation = 'copy';\n    this.ctx.drawImage(bitmap, 0, 0);\n    this.ctx.filter = 'none';\n    this.ctx.globalCompositeOperation = 'source-out';\n    this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);\n    this.ctx.globalCompositeOperation = 'destination-over';\n    this.ctx.filter = `blur(${this.blurRadius}px)`;\n    this.ctx.drawImage(frame, 0, 0);\n    this.ctx.restore();\n  }\n}\n\nfunction maskToBitmap(\n  mask: vision.MPMask,\n  videoWidth: number,\n  videoHeight: number,\n): Promise<ImageBitmap> {\n  const dataArray: Uint8ClampedArray = new Uint8ClampedArray(videoWidth * videoHeight * 4);\n  const result = mask.getAsUint8Array();\n  for (let i = 0; i < result.length; i += 1) {\n    dataArray[i * 4] = result[i];\n    dataArray[i * 4 + 1] = result[i];\n    dataArray[i * 4 + 2] = result[i];\n    dataArray[i * 4 + 3] = result[i];\n  }\n  const dataNew = new ImageData(dataArray, videoWidth, videoHeight);\n\n  return createImageBitmap(dataNew);\n}\n","{\n  \"name\": \"@livekit/track-processors\",\n  \"version\": \"0.1.6\",\n  \"description\": \"LiveKit track processors\",\n  \"main\": \"dist/index.js\",\n  \"module\": \"dist/index.mjs\",\n  \"source\": \"src/index.ts\",\n  \"types\": \"dist/src/index.d.ts\",\n  \"repository\": \"git@github.com:livekit/livekit-track-processors.git\",\n  \"author\": \"Lukas Seiler\",\n  \"license\": \"Apache-2.0\",\n  \"scripts\": {\n    \"build\": \"tsup --onSuccess \\\"tsc --declaration --emitDeclarationOnly\\\"\",\n    \"build-docs\": \"yarn exec typedoc\",\n    \"build-sample\": \"cd example && webpack && cp styles.css index.html dist/\",\n    \"lint\": \"eslint src\",\n    \"test\": \"jest\",\n    \"sample\": \"vite serve example --port 8080 --open\"\n  },\n  \"files\": [\n    \"dist\",\n    \"src\"\n  ],\n  \"dependencies\": {\n    \"@mediapipe/holistic\": \"0.5.1675471629\",\n    \"@mediapipe/tasks-vision\": \"0.10.1\"\n  },\n  \"peerDependencies\": {\n    \"livekit-client\": \"^1.10.0\"\n  },\n  \"devDependencies\": {\n    \"@trivago/prettier-plugin-sort-imports\": \"^4.1.1\",\n    \"@types/dom-mediacapture-transform\": \"^0.1.5\",\n    \"@types/jest\": \"^27.0.3\",\n    \"@types/offscreencanvas\": \"^2019.7.0\",\n    \"@typescript-eslint/eslint-plugin\": \"^4.31.2\",\n    \"@webpack-cli/serve\": \"^1.5.2\",\n    \"eslint\": \"8.39.0\",\n    \"eslint-config-airbnb-typescript\": \"17.0.0\",\n    \"eslint-config-prettier\": \"8.8.0\",\n    \"eslint-plugin-ecmascript-compat\": \"^3.0.0\",\n    \"eslint-plugin-import\": \"2.27.5\",\n    \"jest\": \"^27.4.3\",\n    \"livekit-client\": \"^0.16.6\",\n    \"prettier\": \"^2.8.8\",\n    \"ts-jest\": \"^27.0.7\",\n    \"ts-loader\": \"^8.1.0\",\n    \"ts-proto\": \"^1.85.0\",\n    \"tsup\": \"^7.1.0\",\n    \"typedoc\": \"^0.20.35\",\n    \"typedoc-plugin-no-inherit\": \"1.3.0\",\n    \"typescript\": \"^5.0.4\",\n    \"vite\": \"^4.3.8\",\n    \"webpack\": \"^5.53.0\",\n    \"webpack-cli\": \"^4.8.0\",\n    \"webpack-dev-server\": \"^4.2.1\"\n  }\n}\n","import { VideoTrackTransformer, VideoTransformerInitOptions } from './types';\n\nexport default abstract class VideoTransformer implements VideoTrackTransformer {\n  transformer?: TransformStream;\n\n  canvas?: OffscreenCanvas;\n\n  ctx?: OffscreenCanvasRenderingContext2D;\n\n  inputVideo?: HTMLVideoElement;\n\n  protected isDisabled?: Boolean = false;\n\n  init({ outputCanvas, inputElement: inputVideo }: VideoTransformerInitOptions): void {\n    if (!(inputVideo instanceof HTMLVideoElement)) {\n      throw TypeError('Video transformer needs a HTMLVideoElement as input');\n    }\n    this.transformer = new TransformStream({\n      transform: (frame, controller) => this.transform(frame, controller),\n    });\n    this.canvas = outputCanvas || null;\n    if (outputCanvas) {\n      this.ctx = this.canvas?.getContext('2d') || undefined;\n    }\n    this.inputVideo = inputVideo;\n    this.isDisabled = false;\n  }\n\n  async destroy() {\n    this.isDisabled = true;\n    this.canvas = undefined;\n    this.ctx = undefined;\n  }\n\n  abstract transform(\n    frame: VideoFrame,\n    controller: TransformStreamDefaultController<VideoFrame>,\n  ): void;\n}\n","import VideoTransformer from './VideoTransformer';\n\nexport default class DummyTransformer extends VideoTransformer {\n  async transform(frame: VideoFrame, controller: TransformStreamDefaultController<VideoFrame>) {\n    controller.enqueue(frame);\n  }\n\n  async destroy() {\n    // nothing to do\n  }\n}\n","import ProcessorPipeline from './ProcessorPipeline';\nimport BackgroundTransformer from './transformers/BackgroundTransformer';\nimport DummyTransformer from './transformers/DummyTransformer';\n\nexport const BackgroundBlur = (blurRadius: number = 10) => {\n  const isPipelineSupported = ProcessorPipeline.isSupported && BackgroundTransformer.isSupported;\n  if (!isPipelineSupported) {\n    throw new Error('pipeline is not supported in this browser');\n  }\n  const pipeline = new ProcessorPipeline(\n    [new BackgroundTransformer({ blurRadius })],\n    'background-blur',\n  );\n  return pipeline;\n};\n\nexport const VirtualBackground = (imagePath: string) => {\n  const isPipelineSupported = ProcessorPipeline.isSupported && BackgroundTransformer.isSupported;\n  if (!isPipelineSupported) {\n    throw new Error('pipeline is not supported in this browser');\n  }\n  const pipeline = new ProcessorPipeline(\n    [new BackgroundTransformer({ imagePath })],\n    'virtual-background',\n  );\n  return pipeline;\n};\n\nexport const Dummy = () => {\n  const isPipelineSupported = ProcessorPipeline.isSupported && BackgroundTransformer.isSupported;\n  if (!isPipelineSupported) {\n    throw new Error('pipeline is not supported in this browser');\n  }\n  const pipeline = new ProcessorPipeline([new DummyTransformer()], 'dummy');\n  return pipeline;\n};\n"],"mappings":";AAGA,IAAqB,oBAArB,MAA6E;AAAA,EAC3E,WAAW,cAAc;AACvB,WACE,OAAO,8BAA8B,eACrC,OAAO,8BAA8B;AAAA,EAEzC;AAAA,EAoBA,YAAY,cAA4C,MAAc;AACpE,SAAK,OAAO;AACZ,SAAK,eAAe;AAAA,EACtB;AAAA,EAEA,MAAM,KAAK,MAAoC;AAlCjD;AAmCI,SAAK,SAAS,KAAK;AACnB,SAAK,iBAAiB,KAAK,OAAO,YAAY;AAC9C,SAAK,cAAc,KAAK;AACxB,QAAI,EAAE,KAAK,uBAAuB,mBAAmB;AACnD,YAAM,UAAU,iDAAiD;AAAA,IACnE;AAEA,SAAK,YAAY,IAAI,0BAA0B,EAAE,OAAO,KAAK,OAAO,CAAC;AACrE,SAAK,iBAAiB,IAAI,0BAA0B,EAAE,MAAM,QAAQ,CAAC;AAErE,SAAK,SAAS,IAAI;AAAA,OAChB,UAAK,eAAe,UAApB,YAA6B;AAAA,OAC7B,UAAK,eAAe,WAApB,YAA8B;AAAA,IAChC;AAEA,QAAI,iBAAiB,KAAK,UAAU;AACpC,eAAW,eAAe,KAAK,cAAc;AAC3C,kBAAY,KAAK;AAAA,QACf,cAAc,KAAK;AAAA,QACnB,cAAc,KAAK;AAAA,MACrB,CAAC;AACD,uBAAiB,eAAe,YAAY,YAAa,WAAY;AAAA,IACvE;AACA,mBAAe,OAAO,KAAK,eAAe,QAAQ;AAClD,SAAK,iBAAiB,KAAK;AAAA,EAC7B;AAAA,EAEA,MAAM,UAAU;AA9DlB;AA+DI,eAAW,eAAe,KAAK,cAAc;AAC3C,YAAM,YAAY,QAAQ;AAAA,IAC5B;AACA,eAAK,mBAAL,mBAAqB;AAAA,EACvB;AACF;;;ACpEA,YAAY,YAAY;;;ACuBtB,mBAAgB;AAAA,EACd,uBAAuB;AAAA,EACvB,2BAA2B;AAC7B;;;ACxBF,IAA8B,mBAA9B,MAAgF;AAAA,EAAhF;AASE,SAAU,aAAuB;AAAA;AAAA,EAEjC,KAAK,EAAE,cAAc,cAAc,WAAW,GAAsC;AAbtF;AAcI,QAAI,EAAE,sBAAsB,mBAAmB;AAC7C,YAAM,UAAU,qDAAqD;AAAA,IACvE;AACA,SAAK,cAAc,IAAI,gBAAgB;AAAA,MACrC,WAAW,CAAC,OAAO,eAAe,KAAK,UAAU,OAAO,UAAU;AAAA,IACpE,CAAC;AACD,SAAK,SAAS,gBAAgB;AAC9B,QAAI,cAAc;AAChB,WAAK,QAAM,UAAK,WAAL,mBAAa,WAAW,UAAS;AAAA,IAC9C;AACA,SAAK,aAAa;AAClB,SAAK,aAAa;AAAA,EACpB;AAAA,EAEA,MAAM,UAAU;AACd,SAAK,aAAa;AAClB,SAAK,SAAS;AACd,SAAK,MAAM;AAAA,EACb;AAMF;;;AF5BA,IAAqB,sBAArB,cAAiD,iBAAiB;AAAA,EAahE,YAAY,MAAyB;AACnC,UAAM;AALR,2BAAsC;AAMpC,QAAI,KAAK,YAAY;AACnB,WAAK,aAAa,KAAK;AAAA,IACzB,WAAW,KAAK,WAAW;AACzB,WAAK,eAAe,KAAK,SAAS;AAAA,IACpC;AAAA,EACF;AAAA,EAnBA,WAAW,cAAc;AACvB,WAAO,OAAO,oBAAoB;AAAA,EACpC;AAAA,EAmBA,MAAM,KAAK,EAAE,cAAc,cAAc,WAAW,GAAgC;AAClF,UAAM,KAAK,EAAE,cAAc,cAAc,WAAW,CAAC;AAErD,UAAM,UAAU,MAAa,uBAAgB;AAAA,MAC3C,wDAAwD,aAAa,yBAAyB,CAAC;AAAA,IACjG;AAEA,SAAK,iBAAiB,MAAa,sBAAe,kBAAkB,SAAS;AAAA,MAC3E,aAAa;AAAA,QACX,gBACE;AAAA,QACF,UAAU;AAAA,MACZ;AAAA,MACA,aAAa;AAAA,MACb,oBAAoB;AAAA,MACpB,uBAAuB;AAAA,IACzB,CAAC;AAAA,EAIH;AAAA,EAEA,MAAM,UAAU;AAtDlB;AAuDI,UAAM,MAAM,QAAQ;AACpB,YAAM,UAAK,mBAAL,mBAAqB;AAC3B,SAAK,kBAAkB;AAAA,EACzB;AAAA,EAEA,MAAM,sCAAsC,SAA2B;AA5DzE;AA6DI,QAAI,CAAC,KAAK,YAAY;AACpB,UAAI,QAAQ,aAAa,KAAK,QAAQ,cAAc,GAAG;AACrD,YAAI,cAAc,YAAY,IAAI;AAClC,mBAAK,mBAAL,mBAAqB;AAAA,UACnB;AAAA,UACA;AAAA,UACA,CAAC,WAAY,KAAK,sBAAsB;AAAA;AAAA,MAE5C;AACA,cAAQ,0BAA0B,MAAM;AACtC,aAAK,sCAAsC,OAAO;AAAA,MACpD,CAAC;AAAA,IACH;AAAA,EACF;AAAA,EAEA,MAAM,eAAe,MAAc;AACjC,UAAM,MAAM,IAAI,MAAM;AAEtB,UAAM,IAAI,QAAQ,CAAC,SAAS,WAAW;AACrC,UAAI,cAAc;AAClB,UAAI,SAAS,MAAM,QAAQ,GAAG;AAC9B,UAAI,UAAU,CAAC,QAAQ,OAAO,GAAG;AACjC,UAAI,MAAM;AAAA,IACZ,CAAC;AACD,UAAM,YAAY,MAAM,kBAAkB,GAAG;AAC7C,SAAK,kBAAkB;AAAA,EACzB;AAAA,EAEA,MAAM,UAAU,OAAmB,YAA0D;AAzF/F;AA0FI,QAAI,KAAK,YAAY;AACnB,iBAAW,QAAQ,KAAK;AACxB;AAAA,IACF;AACA,QAAI,CAAC,KAAK,QAAQ;AAChB,YAAM,UAAU,sCAAsC;AAAA,IACxD;AACA,QAAI,cAAc,YAAY,IAAI;AAClC,eAAK,mBAAL,mBAAqB;AAAA,MACnB,KAAK;AAAA,MACL;AAAA,MACA,CAAC,WAAY,KAAK,sBAAsB;AAAA;AAG1C,QAAI,KAAK,YAAY;AACnB,YAAM,KAAK,eAAe,KAAK;AAAA,IACjC,OAAO;AACL,YAAM,KAAK,sBAAsB,KAAK;AAAA,IACxC;AACA,UAAM,WAAW,IAAI,WAAW,KAAK,QAAQ;AAAA,MAC3C,WAAW,MAAM,aAAa,KAAK,IAAI;AAAA,IACzC,CAAC;AACD,UAAM,MAAM;AACZ,eAAW,QAAQ,QAAQ;AAAA,EAC7B;AAAA,EAEA,MAAM,sBAAsB,OAAmB;AApHjD;AAqHI,QAAI,CAAC,KAAK,UAAU,CAAC,KAAK,OAAO,CAAC,KAAK,uBAAuB,CAAC,KAAK;AAAY;AAGhF,SAAI,UAAK,wBAAL,mBAA0B,cAAc;AAC1C,WAAK,IAAI,SAAS;AAClB,WAAK,IAAI,2BAA2B;AACpC,YAAM,SAAS,MAAM;AAAA,QACnB,KAAK,oBAAoB;AAAA,QACzB,KAAK,WAAW;AAAA,QAChB,KAAK,WAAW;AAAA,MAClB;AACA,WAAK,IAAI,UAAU,QAAQ,GAAG,CAAC;AAC/B,WAAK,IAAI,SAAS;AAClB,WAAK,IAAI,2BAA2B;AACpC,UAAI,KAAK,iBAAiB;AACxB,aAAK,IAAI;AAAA,UACP,KAAK;AAAA,UACL;AAAA,UACA;AAAA,UACA,KAAK,gBAAgB;AAAA,UACrB,KAAK,gBAAgB;AAAA,UACrB;AAAA,UACA;AAAA,UACA,KAAK,OAAO;AAAA,UACZ,KAAK,OAAO;AAAA,QACd;AAAA,MACF,OAAO;AACL,aAAK,IAAI,YAAY;AACrB,aAAK,IAAI,SAAS,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AAAA,MAC/D;AAEA,WAAK,IAAI,2BAA2B;AAAA,IACtC;AACA,SAAK,IAAI,UAAU,OAAO,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AAAA,EAEvE;AAAA,EAEA,MAAM,eAAe,OAAmB;AA1J1C;AA2JI,QACE,CAAC,KAAK,OACN,CAAC,KAAK,UACN,GAAC,gBAAK,wBAAL,mBAA0B,iBAA1B,mBAAwC,WACzC,CAAC,KAAK,YACN;AACA;AAAA,IACF;AACA,SAAK,IAAI,KAAK;AACd,SAAK,IAAI,2BAA2B;AAEpC,UAAM,SAAS,MAAM;AAAA,MACnB,KAAK,oBAAoB;AAAA,MACzB,KAAK,WAAW;AAAA,MAChB,KAAK,WAAW;AAAA,IAClB;AACA,SAAK,IAAI,SAAS;AAClB,SAAK,IAAI,2BAA2B;AACpC,SAAK,IAAI,UAAU,QAAQ,GAAG,CAAC;AAC/B,SAAK,IAAI,SAAS;AAClB,SAAK,IAAI,2BAA2B;AACpC,SAAK,IAAI,UAAU,OAAO,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AACrE,SAAK,IAAI,2BAA2B;AACpC,SAAK,IAAI,SAAS,QAAQ,KAAK,UAAU;AACzC,SAAK,IAAI,UAAU,OAAO,GAAG,CAAC;AAC9B,SAAK,IAAI,QAAQ;AAAA,EACnB;AACF;AAEA,SAAS,aACP,MACA,YACA,aACsB;AACtB,QAAM,YAA+B,IAAI,kBAAkB,aAAa,cAAc,CAAC;AACvF,QAAM,SAAS,KAAK,gBAAgB;AACpC,WAAS,IAAI,GAAG,IAAI,OAAO,QAAQ,KAAK,GAAG;AACzC,cAAU,IAAI,CAAC,IAAI,OAAO,CAAC;AAC3B,cAAU,IAAI,IAAI,CAAC,IAAI,OAAO,CAAC;AAC/B,cAAU,IAAI,IAAI,CAAC,IAAI,OAAO,CAAC;AAC/B,cAAU,IAAI,IAAI,CAAC,IAAI,OAAO,CAAC;AAAA,EACjC;AACA,QAAM,UAAU,IAAI,UAAU,WAAW,YAAY,WAAW;AAEhE,SAAO,kBAAkB,OAAO;AAClC;;;AGtMA,IAAqB,mBAArB,cAA8C,iBAAiB;AAAA,EAC7D,MAAM,UAAU,OAAmB,YAA0D;AAC3F,eAAW,QAAQ,KAAK;AAAA,EAC1B;AAAA,EAEA,MAAM,UAAU;AAAA,EAEhB;AACF;;;ACNO,IAAM,iBAAiB,CAAC,aAAqB,OAAO;AACzD,QAAM,sBAAsB,kBAAkB,eAAe,oBAAsB;AACnF,MAAI,CAAC,qBAAqB;AACxB,UAAM,IAAI,MAAM,2CAA2C;AAAA,EAC7D;AACA,QAAM,WAAW,IAAI;AAAA,IACnB,CAAC,IAAI,oBAAsB,EAAE,WAAW,CAAC,CAAC;AAAA,IAC1C;AAAA,EACF;AACA,SAAO;AACT;AAEO,IAAM,oBAAoB,CAAC,cAAsB;AACtD,QAAM,sBAAsB,kBAAkB,eAAe,oBAAsB;AACnF,MAAI,CAAC,qBAAqB;AACxB,UAAM,IAAI,MAAM,2CAA2C;AAAA,EAC7D;AACA,QAAM,WAAW,IAAI;AAAA,IACnB,CAAC,IAAI,oBAAsB,EAAE,UAAU,CAAC,CAAC;AAAA,IACzC;AAAA,EACF;AACA,SAAO;AACT;AAEO,IAAM,QAAQ,MAAM;AACzB,QAAM,sBAAsB,kBAAkB,eAAe,oBAAsB;AACnF,MAAI,CAAC,qBAAqB;AACxB,UAAM,IAAI,MAAM,2CAA2C;AAAA,EAC7D;AACA,QAAM,WAAW,IAAI,kBAAkB,CAAC,IAAI,iBAAiB,CAAC,GAAG,OAAO;AACxE,SAAO;AACT;","names":[]}
+{"version":3,"sources":["../src/ProcessorPipeline.ts","../src/transformers/BackgroundTransformer.ts","../package.json","../src/transformers/VideoTransformer.ts","../src/transformers/DummyTransformer.ts","../src/index.ts"],"sourcesContent":["import type { ProcessorOptions, Track, TrackProcessor } from 'livekit-client';\nimport { VideoTrackTransformer } from './transformers';\n\nexport default class ProcessorPipeline implements TrackProcessor<Track.Kind> {\n  static get isSupported() {\n    return (\n      typeof MediaStreamTrackGenerator !== 'undefined' &&\n      typeof MediaStreamTrackProcessor !== 'undefined'\n    );\n  }\n\n  name: string;\n\n  source?: MediaStreamVideoTrack;\n\n  sourceSettings?: MediaTrackSettings;\n\n  processor?: MediaStreamTrackProcessor<VideoFrame>;\n\n  trackGenerator?: MediaStreamTrackGenerator<VideoFrame>;\n\n  canvas?: OffscreenCanvas;\n\n  sourceDummy?: HTMLMediaElement;\n\n  processedTrack?: MediaStreamTrack;\n\n  transformers: Array<VideoTrackTransformer>;\n\n  constructor(transformers: Array<VideoTrackTransformer>, name: string) {\n    this.name = name;\n    this.transformers = transformers;\n  }\n\n  async init(opts: ProcessorOptions<Track.Kind>) {\n    this.source = opts.track as MediaStreamVideoTrack;\n    const origConstraints = this.source.getConstraints();\n    await this.source.applyConstraints({\n      ...origConstraints,\n      // @ts-expect-error when a mediastream track is resized and/or cropped, the `VideoFrame` will have a coded height/width of the original video size\n      // this leads to a shift of the underlying video as the frame itself is being rendered with the coded size\n      // but image segmentation is based on the display dimensions (-> the cropped version)\n      // in order to prevent this, we force the resize mode to \"none\"\n      resizeMode: 'none',\n    });\n    this.sourceSettings = this.source.getSettings();\n    this.sourceDummy = opts.element;\n    if (this.sourceDummy instanceof HTMLVideoElement) {\n      this.sourceDummy.height = this.sourceSettings.height!;\n      this.sourceDummy.width = this.sourceSettings.width!;\n    }\n    if (!(this.sourceDummy instanceof HTMLVideoElement)) {\n      throw TypeError('Currently only video transformers are supported');\n    }\n    // TODO explore if we can do all the processing work in a webworker\n    this.processor = new MediaStreamTrackProcessor({ track: this.source });\n\n    this.trackGenerator = new MediaStreamTrackGenerator({\n      kind: 'video',\n      signalTarget: this.source,\n    });\n\n    this.canvas = new OffscreenCanvas(\n      this.sourceSettings.width ?? 300,\n      this.sourceSettings.height ?? 300,\n    );\n\n    let readableStream = this.processor.readable;\n    for (const transformer of this.transformers) {\n      await transformer.init({\n        outputCanvas: this.canvas,\n        inputElement: this.sourceDummy!,\n      });\n      readableStream = readableStream.pipeThrough(transformer!.transformer!);\n    }\n    readableStream\n      .pipeTo(this.trackGenerator.writable)\n      .catch((e) => console.error('error when trying to pipe', e))\n      .finally(() => this.destroy());\n    this.processedTrack = this.trackGenerator as MediaStreamVideoTrack;\n  }\n\n  async destroy() {\n    for (const transformer of this.transformers) {\n      await transformer.destroy();\n    }\n    this.trackGenerator?.stop();\n  }\n}\n","import * as vision from '@mediapipe/tasks-vision';\nimport { dependencies } from '../../package.json';\nimport VideoTransformer from './VideoTransformer';\nimport { VideoTransformerInitOptions } from './types';\n\nexport type BackgroundOptions = {\n  blurRadius?: number;\n  imagePath?: string;\n};\n\nexport default class BackgroundProcessor extends VideoTransformer {\n  static get isSupported() {\n    return typeof OffscreenCanvas !== 'undefined';\n  }\n\n  imageSegmenter?: vision.ImageSegmenter;\n\n  segmentationResults: vision.ImageSegmenterResult | undefined;\n\n  backgroundImage: ImageBitmap | null = null;\n\n  blurRadius?: number;\n\n  constructor(opts: BackgroundOptions) {\n    super();\n    if (opts.blurRadius) {\n      this.blurRadius = opts.blurRadius;\n    } else if (opts.imagePath) {\n      this.loadBackground(opts.imagePath);\n    }\n  }\n\n  async init({ outputCanvas, inputElement: inputVideo }: VideoTransformerInitOptions) {\n    await super.init({ outputCanvas, inputElement: inputVideo });\n\n    const fileSet = await vision.FilesetResolver.forVisionTasks(\n      `https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@${dependencies['@mediapipe/tasks-vision']}/wasm`,\n    );\n\n    this.imageSegmenter = await vision.ImageSegmenter.createFromOptions(fileSet, {\n      baseOptions: {\n        modelAssetPath:\n          'https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_segmenter/float16/latest/selfie_segmenter.tflite',\n        delegate: 'CPU',\n      },\n      runningMode: 'VIDEO',\n      outputCategoryMask: true,\n      outputConfidenceMasks: false,\n    });\n\n    // this.loadBackground(opts.backgroundUrl).catch((e) => console.error(e));\n  }\n\n  async destroy() {\n    await super.destroy();\n    await this.imageSegmenter?.close();\n    this.backgroundImage = null;\n  }\n\n  async loadBackground(path: string) {\n    const img = new Image();\n\n    await new Promise((resolve, reject) => {\n      img.crossOrigin = 'Anonymous';\n      img.onload = () => resolve(img);\n      img.onerror = (err) => reject(err);\n      img.src = path;\n    });\n    const imageData = await createImageBitmap(img);\n    this.backgroundImage = imageData;\n  }\n\n  async transform(frame: VideoFrame, controller: TransformStreamDefaultController<VideoFrame>) {\n    try {\n      if (this.isDisabled) {\n        controller.enqueue(frame);\n        return;\n      }\n      if (!this.canvas) {\n        throw TypeError('Canvas needs to be initialized first');\n      }\n      let startTimeMs = performance.now();\n      this.imageSegmenter?.segmentForVideo(\n        this.inputVideo!,\n        startTimeMs,\n        (result) => (this.segmentationResults = result),\n      );\n\n      if (this.blurRadius) {\n        await this.blurBackground(frame);\n      } else {\n        await this.drawVirtualBackground(frame);\n      }\n      const newFrame = new VideoFrame(this.canvas, {\n        timestamp: frame.timestamp || Date.now(),\n      });\n      controller.enqueue(newFrame);\n    } finally {\n      frame.close();\n    }\n  }\n\n  async drawVirtualBackground(frame: VideoFrame) {\n    if (!this.canvas || !this.ctx || !this.segmentationResults || !this.inputVideo) return;\n    // this.ctx.save();\n    // this.ctx.clearRect(0, 0, this.canvas.width, this.canvas.height);\n    if (this.segmentationResults?.categoryMask) {\n      this.ctx.filter = 'blur(3px)';\n      this.ctx.globalCompositeOperation = 'copy';\n      const bitmap = await maskToBitmap(\n        this.segmentationResults.categoryMask,\n        this.inputVideo.videoWidth,\n        this.inputVideo.videoHeight,\n      );\n      this.ctx.drawImage(bitmap, 0, 0);\n      this.ctx.filter = 'none';\n      this.ctx.globalCompositeOperation = 'source-in';\n      if (this.backgroundImage) {\n        this.ctx.drawImage(\n          this.backgroundImage,\n          0,\n          0,\n          this.backgroundImage.width,\n          this.backgroundImage.height,\n          0,\n          0,\n          this.canvas.width,\n          this.canvas.height,\n        );\n      } else {\n        this.ctx.fillStyle = '#00FF00';\n        this.ctx.fillRect(0, 0, this.canvas.width, this.canvas.height);\n      }\n\n      this.ctx.globalCompositeOperation = 'destination-over';\n    }\n    this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);\n    // this.ctx.restore();\n  }\n\n  async blurBackground(frame: VideoFrame) {\n    if (\n      !this.ctx ||\n      !this.canvas ||\n      !this.segmentationResults?.categoryMask?.canvas ||\n      !this.inputVideo\n    ) {\n      return;\n    }\n    this.canvas.width = frame.codedWidth;\n    this.canvas.height = frame.codedHeight;\n    this.ctx.save();\n    this.ctx.globalCompositeOperation = 'copy';\n\n    const bitmap = await maskToBitmap(\n      this.segmentationResults.categoryMask,\n      this.inputVideo.videoWidth,\n      this.inputVideo.videoHeight,\n    );\n\n    this.ctx.filter = 'blur(3px)';\n    this.ctx.globalCompositeOperation = 'copy';\n    this.ctx.drawImage(bitmap, 0, 0);\n    this.ctx.filter = 'none';\n    this.ctx.globalCompositeOperation = 'source-out';\n    this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);\n    this.ctx.globalCompositeOperation = 'destination-over';\n    this.ctx.filter = `blur(${this.blurRadius}px)`;\n    this.ctx.drawImage(frame, 0, 0);\n    this.ctx.restore();\n  }\n}\n\nfunction maskToBitmap(\n  mask: vision.MPMask,\n  videoWidth: number,\n  videoHeight: number,\n): Promise<ImageBitmap> {\n  const dataArray: Uint8ClampedArray = new Uint8ClampedArray(videoWidth * videoHeight * 4);\n  const result = mask.getAsUint8Array();\n  for (let i = 0; i < result.length; i += 1) {\n    dataArray[i * 4] = result[i];\n    dataArray[i * 4 + 1] = result[i];\n    dataArray[i * 4 + 2] = result[i];\n    dataArray[i * 4 + 3] = result[i];\n  }\n  const dataNew = new ImageData(dataArray, videoWidth, videoHeight);\n\n  return createImageBitmap(dataNew);\n}\n","{\n  \"name\": \"@livekit/track-processors\",\n  \"version\": \"0.1.8\",\n  \"description\": \"LiveKit track processors\",\n  \"main\": \"dist/index.js\",\n  \"module\": \"dist/index.mjs\",\n  \"source\": \"src/index.ts\",\n  \"types\": \"dist/src/index.d.ts\",\n  \"repository\": \"git@github.com:livekit/livekit-track-processors.git\",\n  \"author\": \"Lukas Seiler\",\n  \"license\": \"Apache-2.0\",\n  \"scripts\": {\n    \"build\": \"tsup --onSuccess \\\"tsc --declaration --emitDeclarationOnly\\\"\",\n    \"build-docs\": \"yarn exec typedoc\",\n    \"build-sample\": \"cd example && webpack && cp styles.css index.html dist/\",\n    \"lint\": \"eslint src\",\n    \"test\": \"jest\",\n    \"sample\": \"vite serve example --port 8080 --open\"\n  },\n  \"files\": [\n    \"dist\",\n    \"src\"\n  ],\n  \"dependencies\": {\n    \"@mediapipe/holistic\": \"0.5.1675471629\",\n    \"@mediapipe/tasks-vision\": \"0.10.1\"\n  },\n  \"peerDependencies\": {\n    \"livekit-client\": \"^1.12.0\"\n  },\n  \"devDependencies\": {\n    \"@trivago/prettier-plugin-sort-imports\": \"^4.1.1\",\n    \"@types/dom-mediacapture-transform\": \"^0.1.5\",\n    \"@types/jest\": \"^27.0.3\",\n    \"@types/offscreencanvas\": \"^2019.7.0\",\n    \"@typescript-eslint/eslint-plugin\": \"^4.31.2\",\n    \"@webpack-cli/serve\": \"^1.5.2\",\n    \"eslint\": \"8.39.0\",\n    \"eslint-config-airbnb-typescript\": \"17.0.0\",\n    \"eslint-config-prettier\": \"8.8.0\",\n    \"eslint-plugin-ecmascript-compat\": \"^3.0.0\",\n    \"eslint-plugin-import\": \"2.27.5\",\n    \"jest\": \"^27.4.3\",\n    \"livekit-client\": \"^0.16.6\",\n    \"prettier\": \"^2.8.8\",\n    \"ts-jest\": \"^27.0.7\",\n    \"ts-loader\": \"^8.1.0\",\n    \"ts-proto\": \"^1.85.0\",\n    \"tsup\": \"^7.1.0\",\n    \"typedoc\": \"^0.20.35\",\n    \"typedoc-plugin-no-inherit\": \"1.3.0\",\n    \"typescript\": \"^5.0.4\",\n    \"vite\": \"^4.3.8\",\n    \"webpack\": \"^5.53.0\",\n    \"webpack-cli\": \"^4.8.0\",\n    \"webpack-dev-server\": \"^4.2.1\"\n  }\n}\n","import { VideoTrackTransformer, VideoTransformerInitOptions } from './types';\n\nexport default abstract class VideoTransformer implements VideoTrackTransformer {\n  transformer?: TransformStream;\n\n  canvas?: OffscreenCanvas;\n\n  ctx?: OffscreenCanvasRenderingContext2D;\n\n  inputVideo?: HTMLVideoElement;\n\n  protected isDisabled?: Boolean = false;\n\n  async init({\n    outputCanvas,\n    inputElement: inputVideo,\n  }: VideoTransformerInitOptions): Promise<void> {\n    if (!(inputVideo instanceof HTMLVideoElement)) {\n      throw TypeError('Video transformer needs a HTMLVideoElement as input');\n    }\n    this.transformer = new TransformStream({\n      transform: (frame, controller) => this.transform(frame, controller),\n    });\n    this.canvas = outputCanvas || null;\n    if (outputCanvas) {\n      this.ctx = this.canvas?.getContext('2d') || undefined;\n    }\n    this.inputVideo = inputVideo;\n    this.isDisabled = false;\n  }\n\n  async destroy() {\n    this.isDisabled = true;\n    this.canvas = undefined;\n    this.ctx = undefined;\n  }\n\n  abstract transform(\n    frame: VideoFrame,\n    controller: TransformStreamDefaultController<VideoFrame>,\n  ): void;\n}\n","import VideoTransformer from './VideoTransformer';\n\nexport default class DummyTransformer extends VideoTransformer {\n  async transform(frame: VideoFrame, controller: TransformStreamDefaultController<VideoFrame>) {\n    controller.enqueue(frame);\n  }\n\n  async destroy() {\n    // nothing to do\n  }\n}\n","import ProcessorPipeline from './ProcessorPipeline';\nimport BackgroundTransformer from './transformers/BackgroundTransformer';\nimport DummyTransformer from './transformers/DummyTransformer';\n\nexport const BackgroundBlur = (blurRadius: number = 10) => {\n  const isPipelineSupported = ProcessorPipeline.isSupported && BackgroundTransformer.isSupported;\n  if (!isPipelineSupported) {\n    throw new Error('pipeline is not supported in this browser');\n  }\n  const pipeline = new ProcessorPipeline(\n    [new BackgroundTransformer({ blurRadius })],\n    'background-blur',\n  );\n  return pipeline;\n};\n\nexport const VirtualBackground = (imagePath: string) => {\n  const isPipelineSupported = ProcessorPipeline.isSupported && BackgroundTransformer.isSupported;\n  if (!isPipelineSupported) {\n    throw new Error('pipeline is not supported in this browser');\n  }\n  const pipeline = new ProcessorPipeline(\n    [new BackgroundTransformer({ imagePath })],\n    'virtual-background',\n  );\n  return pipeline;\n};\n\nexport const Dummy = () => {\n  const isPipelineSupported = ProcessorPipeline.isSupported && BackgroundTransformer.isSupported;\n  if (!isPipelineSupported) {\n    throw new Error('pipeline is not supported in this browser');\n  }\n  const pipeline = new ProcessorPipeline([new DummyTransformer()], 'dummy');\n  return pipeline;\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AAGA,IAAqB,oBAArB,MAA6E;AAAA,EAC3E,WAAW,cAAc;AACvB,WACE,OAAO,8BAA8B,eACrC,OAAO,8BAA8B;AAAA,EAEzC;AAAA,EAoBA,YAAY,cAA4C,MAAc;AACpE,SAAK,OAAO;AACZ,SAAK,eAAe;AAAA,EACtB;AAAA,EAEA,MAAM,KAAK,MAAoC;AAlCjD;AAmCI,SAAK,SAAS,KAAK;AACnB,UAAM,kBAAkB,KAAK,OAAO,eAAe;AACnD,UAAM,KAAK,OAAO,iBAAiB,iCAC9B,kBAD8B;AAAA;AAAA;AAAA;AAAA;AAAA,MAMjC,YAAY;AAAA,IACd,EAAC;AACD,SAAK,iBAAiB,KAAK,OAAO,YAAY;AAC9C,SAAK,cAAc,KAAK;AACxB,QAAI,KAAK,uBAAuB,kBAAkB;AAChD,WAAK,YAAY,SAAS,KAAK,eAAe;AAC9C,WAAK,YAAY,QAAQ,KAAK,eAAe;AAAA,IAC/C;AACA,QAAI,EAAE,KAAK,uBAAuB,mBAAmB;AACnD,YAAM,UAAU,iDAAiD;AAAA,IACnE;AAEA,SAAK,YAAY,IAAI,0BAA0B,EAAE,OAAO,KAAK,OAAO,CAAC;AAErE,SAAK,iBAAiB,IAAI,0BAA0B;AAAA,MAClD,MAAM;AAAA,MACN,cAAc,KAAK;AAAA,IACrB,CAAC;AAED,SAAK,SAAS,IAAI;AAAA,OAChB,UAAK,eAAe,UAApB,YAA6B;AAAA,OAC7B,UAAK,eAAe,WAApB,YAA8B;AAAA,IAChC;AAEA,QAAI,iBAAiB,KAAK,UAAU;AACpC,eAAW,eAAe,KAAK,cAAc;AAC3C,YAAM,YAAY,KAAK;AAAA,QACrB,cAAc,KAAK;AAAA,QACnB,cAAc,KAAK;AAAA,MACrB,CAAC;AACD,uBAAiB,eAAe,YAAY,YAAa,WAAY;AAAA,IACvE;AACA,mBACG,OAAO,KAAK,eAAe,QAAQ,EACnC,MAAM,CAAC,MAAM,QAAQ,MAAM,6BAA6B,CAAC,CAAC,EAC1D,QAAQ,MAAM,KAAK,QAAQ,CAAC;AAC/B,SAAK,iBAAiB,KAAK;AAAA,EAC7B;AAAA,EAEA,MAAM,UAAU;AAlFlB;AAmFI,eAAW,eAAe,KAAK,cAAc;AAC3C,YAAM,YAAY,QAAQ;AAAA,IAC5B;AACA,eAAK,mBAAL,mBAAqB;AAAA,EACvB;AACF;;;ACxFA,YAAY,YAAY;;;ACuBtB,mBAAgB;AAAA,EACd,uBAAuB;AAAA,EACvB,2BAA2B;AAC7B;;;ACxBF,IAA8B,mBAA9B,MAAgF;AAAA,EAAhF;AASE,SAAU,aAAuB;AAAA;AAAA,EAEjC,MAAM,KAAK;AAAA,IACT;AAAA,IACA,cAAc;AAAA,EAChB,GAA+C;AAhBjD;AAiBI,QAAI,EAAE,sBAAsB,mBAAmB;AAC7C,YAAM,UAAU,qDAAqD;AAAA,IACvE;AACA,SAAK,cAAc,IAAI,gBAAgB;AAAA,MACrC,WAAW,CAAC,OAAO,eAAe,KAAK,UAAU,OAAO,UAAU;AAAA,IACpE,CAAC;AACD,SAAK,SAAS,gBAAgB;AAC9B,QAAI,cAAc;AAChB,WAAK,QAAM,UAAK,WAAL,mBAAa,WAAW,UAAS;AAAA,IAC9C;AACA,SAAK,aAAa;AAClB,SAAK,aAAa;AAAA,EACpB;AAAA,EAEA,MAAM,UAAU;AACd,SAAK,aAAa;AAClB,SAAK,SAAS;AACd,SAAK,MAAM;AAAA,EACb;AAMF;;;AF/BA,IAAqB,sBAArB,cAAiD,iBAAiB;AAAA,EAahE,YAAY,MAAyB;AACnC,UAAM;AALR,2BAAsC;AAMpC,QAAI,KAAK,YAAY;AACnB,WAAK,aAAa,KAAK;AAAA,IACzB,WAAW,KAAK,WAAW;AACzB,WAAK,eAAe,KAAK,SAAS;AAAA,IACpC;AAAA,EACF;AAAA,EAnBA,WAAW,cAAc;AACvB,WAAO,OAAO,oBAAoB;AAAA,EACpC;AAAA,EAmBA,MAAM,KAAK,EAAE,cAAc,cAAc,WAAW,GAAgC;AAClF,UAAM,MAAM,KAAK,EAAE,cAAc,cAAc,WAAW,CAAC;AAE3D,UAAM,UAAU,MAAa,uBAAgB;AAAA,MAC3C,wDAAwD,aAAa,yBAAyB,CAAC;AAAA,IACjG;AAEA,SAAK,iBAAiB,MAAa,sBAAe,kBAAkB,SAAS;AAAA,MAC3E,aAAa;AAAA,QACX,gBACE;AAAA,QACF,UAAU;AAAA,MACZ;AAAA,MACA,aAAa;AAAA,MACb,oBAAoB;AAAA,MACpB,uBAAuB;AAAA,IACzB,CAAC;AAAA,EAGH;AAAA,EAEA,MAAM,UAAU;AArDlB;AAsDI,UAAM,MAAM,QAAQ;AACpB,YAAM,UAAK,mBAAL,mBAAqB;AAC3B,SAAK,kBAAkB;AAAA,EACzB;AAAA,EAEA,MAAM,eAAe,MAAc;AACjC,UAAM,MAAM,IAAI,MAAM;AAEtB,UAAM,IAAI,QAAQ,CAAC,SAAS,WAAW;AACrC,UAAI,cAAc;AAClB,UAAI,SAAS,MAAM,QAAQ,GAAG;AAC9B,UAAI,UAAU,CAAC,QAAQ,OAAO,GAAG;AACjC,UAAI,MAAM;AAAA,IACZ,CAAC;AACD,UAAM,YAAY,MAAM,kBAAkB,GAAG;AAC7C,SAAK,kBAAkB;AAAA,EACzB;AAAA,EAEA,MAAM,UAAU,OAAmB,YAA0D;AAxE/F;AAyEI,QAAI;AACF,UAAI,KAAK,YAAY;AACnB,mBAAW,QAAQ,KAAK;AACxB;AAAA,MACF;AACA,UAAI,CAAC,KAAK,QAAQ;AAChB,cAAM,UAAU,sCAAsC;AAAA,MACxD;AACA,UAAI,cAAc,YAAY,IAAI;AAClC,iBAAK,mBAAL,mBAAqB;AAAA,QACnB,KAAK;AAAA,QACL;AAAA,QACA,CAAC,WAAY,KAAK,sBAAsB;AAAA;AAG1C,UAAI,KAAK,YAAY;AACnB,cAAM,KAAK,eAAe,KAAK;AAAA,MACjC,OAAO;AACL,cAAM,KAAK,sBAAsB,KAAK;AAAA,MACxC;AACA,YAAM,WAAW,IAAI,WAAW,KAAK,QAAQ;AAAA,QAC3C,WAAW,MAAM,aAAa,KAAK,IAAI;AAAA,MACzC,CAAC;AACD,iBAAW,QAAQ,QAAQ;AAAA,IAC7B,UAAE;AACA,YAAM,MAAM;AAAA,IACd;AAAA,EACF;AAAA,EAEA,MAAM,sBAAsB,OAAmB;AAtGjD;AAuGI,QAAI,CAAC,KAAK,UAAU,CAAC,KAAK,OAAO,CAAC,KAAK,uBAAuB,CAAC,KAAK;AAAY;AAGhF,SAAI,UAAK,wBAAL,mBAA0B,cAAc;AAC1C,WAAK,IAAI,SAAS;AAClB,WAAK,IAAI,2BAA2B;AACpC,YAAM,SAAS,MAAM;AAAA,QACnB,KAAK,oBAAoB;AAAA,QACzB,KAAK,WAAW;AAAA,QAChB,KAAK,WAAW;AAAA,MAClB;AACA,WAAK,IAAI,UAAU,QAAQ,GAAG,CAAC;AAC/B,WAAK,IAAI,SAAS;AAClB,WAAK,IAAI,2BAA2B;AACpC,UAAI,KAAK,iBAAiB;AACxB,aAAK,IAAI;AAAA,UACP,KAAK;AAAA,UACL;AAAA,UACA;AAAA,UACA,KAAK,gBAAgB;AAAA,UACrB,KAAK,gBAAgB;AAAA,UACrB;AAAA,UACA;AAAA,UACA,KAAK,OAAO;AAAA,UACZ,KAAK,OAAO;AAAA,QACd;AAAA,MACF,OAAO;AACL,aAAK,IAAI,YAAY;AACrB,aAAK,IAAI,SAAS,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AAAA,MAC/D;AAEA,WAAK,IAAI,2BAA2B;AAAA,IACtC;AACA,SAAK,IAAI,UAAU,OAAO,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AAAA,EAEvE;AAAA,EAEA,MAAM,eAAe,OAAmB;AA5I1C;AA6II,QACE,CAAC,KAAK,OACN,CAAC,KAAK,UACN,GAAC,gBAAK,wBAAL,mBAA0B,iBAA1B,mBAAwC,WACzC,CAAC,KAAK,YACN;AACA;AAAA,IACF;AACA,SAAK,OAAO,QAAQ,MAAM;AAC1B,SAAK,OAAO,SAAS,MAAM;AAC3B,SAAK,IAAI,KAAK;AACd,SAAK,IAAI,2BAA2B;AAEpC,UAAM,SAAS,MAAM;AAAA,MACnB,KAAK,oBAAoB;AAAA,MACzB,KAAK,WAAW;AAAA,MAChB,KAAK,WAAW;AAAA,IAClB;AAEA,SAAK,IAAI,SAAS;AAClB,SAAK,IAAI,2BAA2B;AACpC,SAAK,IAAI,UAAU,QAAQ,GAAG,CAAC;AAC/B,SAAK,IAAI,SAAS;AAClB,SAAK,IAAI,2BAA2B;AACpC,SAAK,IAAI,UAAU,OAAO,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AACrE,SAAK,IAAI,2BAA2B;AACpC,SAAK,IAAI,SAAS,QAAQ,KAAK,UAAU;AACzC,SAAK,IAAI,UAAU,OAAO,GAAG,CAAC;AAC9B,SAAK,IAAI,QAAQ;AAAA,EACnB;AACF;AAEA,SAAS,aACP,MACA,YACA,aACsB;AACtB,QAAM,YAA+B,IAAI,kBAAkB,aAAa,cAAc,CAAC;AACvF,QAAM,SAAS,KAAK,gBAAgB;AACpC,WAAS,IAAI,GAAG,IAAI,OAAO,QAAQ,KAAK,GAAG;AACzC,cAAU,IAAI,CAAC,IAAI,OAAO,CAAC;AAC3B,cAAU,IAAI,IAAI,CAAC,IAAI,OAAO,CAAC;AAC/B,cAAU,IAAI,IAAI,CAAC,IAAI,OAAO,CAAC;AAC/B,cAAU,IAAI,IAAI,CAAC,IAAI,OAAO,CAAC;AAAA,EACjC;AACA,QAAM,UAAU,IAAI,UAAU,WAAW,YAAY,WAAW;AAEhE,SAAO,kBAAkB,OAAO;AAClC;;;AG3LA,IAAqB,mBAArB,cAA8C,iBAAiB;AAAA,EAC7D,MAAM,UAAU,OAAmB,YAA0D;AAC3F,eAAW,QAAQ,KAAK;AAAA,EAC1B;AAAA,EAEA,MAAM,UAAU;AAAA,EAEhB;AACF;;;ACNO,IAAM,iBAAiB,CAAC,aAAqB,OAAO;AACzD,QAAM,sBAAsB,kBAAkB,eAAe,oBAAsB;AACnF,MAAI,CAAC,qBAAqB;AACxB,UAAM,IAAI,MAAM,2CAA2C;AAAA,EAC7D;AACA,QAAM,WAAW,IAAI;AAAA,IACnB,CAAC,IAAI,oBAAsB,EAAE,WAAW,CAAC,CAAC;AAAA,IAC1C;AAAA,EACF;AACA,SAAO;AACT;AAEO,IAAM,oBAAoB,CAAC,cAAsB;AACtD,QAAM,sBAAsB,kBAAkB,eAAe,oBAAsB;AACnF,MAAI,CAAC,qBAAqB;AACxB,UAAM,IAAI,MAAM,2CAA2C;AAAA,EAC7D;AACA,QAAM,WAAW,IAAI;AAAA,IACnB,CAAC,IAAI,oBAAsB,EAAE,UAAU,CAAC,CAAC;AAAA,IACzC;AAAA,EACF;AACA,SAAO;AACT;AAEO,IAAM,QAAQ,MAAM;AACzB,QAAM,sBAAsB,kBAAkB,eAAe,oBAAsB;AACnF,MAAI,CAAC,qBAAqB;AACxB,UAAM,IAAI,MAAM,2CAA2C;AAAA,EAC7D;AACA,QAAM,WAAW,IAAI,kBAAkB,CAAC,IAAI,iBAAiB,CAAC,GAAG,OAAO;AACxE,SAAO;AACT;","names":[]}

package/dist/src/transformers/BackgroundTransformer.d.ts

@@ -15,7 +15,6 @@ export default class BackgroundProcessor extends VideoTransformer {
     constructor(opts: BackgroundOptions);
     init({ outputCanvas, inputElement: inputVideo }: VideoTransformerInitOptions): Promise<void>;
     destroy(): Promise<void>;
-    sendFramesContinuouslyForSegmentation(videoEl: HTMLVideoElement): Promise<void>;
     loadBackground(path: string): Promise<void>;
     transform(frame: VideoFrame, controller: TransformStreamDefaultController<VideoFrame>): Promise<void>;
     drawVirtualBackground(frame: VideoFrame): Promise<void>;

package/dist/src/transformers/MediaPipeHolisticTrackerTransformer.d.ts

@@ -11,7 +11,7 @@ export default class MediaPipeHolisticTrackerTransformer extends VideoTransforme
     callback: (results: Results) => void;
     static get isSupported(): boolean;
     constructor({ holisticOptions, callback }: MediaPipeHolisticTrackerTransformerOptions);
-    init({ inputElement: inputVideo, outputCanvas }: VideoTransformerInitOptions): void;
+    init({ inputElement: inputVideo, outputCanvas, }: VideoTransformerInitOptions): Promise<void>;
     destroy(): Promise<void>;
     transform(): Promise<void>;
     sendFramesContinuouslyForTracking(videoEl: HTMLVideoElement): Promise<void>;

package/dist/src/transformers/VideoTransformer.d.ts

@@ -6,7 +6,7 @@ export default abstract class VideoTransformer implements VideoTrackTransformer
     ctx?: OffscreenCanvasRenderingContext2D;
     inputVideo?: HTMLVideoElement;
     protected isDisabled?: Boolean;
-    init({ outputCanvas, inputElement: inputVideo }: VideoTransformerInitOptions): void;
+    init({ outputCanvas, inputElement: inputVideo, }: VideoTransformerInitOptions): Promise<void>;
     destroy(): Promise<void>;
     abstract transform(frame: VideoFrame, controller: TransformStreamDefaultController<VideoFrame>): void;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@livekit/track-processors",
-  "version": "0.1.7",
+  "version": "0.2.0",
   "description": "LiveKit track processors",
   "main": "dist/index.js",
   "module": "dist/index.mjs",
@@ -26,7 +26,7 @@
     "@mediapipe/tasks-vision": "0.10.1"
   },
   "peerDependencies": {
-    "livekit-client": "^1.10.0"
+    "livekit-client": "^1.12.0"
   },
   "devDependencies": {
     "@trivago/prettier-plugin-sort-imports": "^4.1.1",

package/src/ProcessorPipeline.ts

@@ -34,14 +34,31 @@ export default class ProcessorPipeline implements TrackProcessor<Track.Kind> {
 
   async init(opts: ProcessorOptions<Track.Kind>) {
     this.source = opts.track as MediaStreamVideoTrack;
+    const origConstraints = this.source.getConstraints();
+    await this.source.applyConstraints({
+      ...origConstraints,
+      // @ts-expect-error when a mediastream track is resized and/or cropped, the `VideoFrame` will have a coded height/width of the original video size
+      // this leads to a shift of the underlying video as the frame itself is being rendered with the coded size
+      // but image segmentation is based on the display dimensions (-> the cropped version)
+      // in order to prevent this, we force the resize mode to "none"
+      resizeMode: 'none',
+    });
     this.sourceSettings = this.source.getSettings();
     this.sourceDummy = opts.element;
+    if (this.sourceDummy instanceof HTMLVideoElement) {
+      this.sourceDummy.height = this.sourceSettings.height!;
+      this.sourceDummy.width = this.sourceSettings.width!;
+    }
     if (!(this.sourceDummy instanceof HTMLVideoElement)) {
       throw TypeError('Currently only video transformers are supported');
     }
     // TODO explore if we can do all the processing work in a webworker
     this.processor = new MediaStreamTrackProcessor({ track: this.source });
-    this.trackGenerator = new MediaStreamTrackGenerator({ kind: 'video' });
+
+    this.trackGenerator = new MediaStreamTrackGenerator({
+      kind: 'video',
+      signalTarget: this.source,
+    });
 
     this.canvas = new OffscreenCanvas(
       this.sourceSettings.width ?? 300,
@@ -50,13 +67,16 @@ export default class ProcessorPipeline implements TrackProcessor<Track.Kind> {
 
     let readableStream = this.processor.readable;
     for (const transformer of this.transformers) {
-      transformer.init({
+      await transformer.init({
         outputCanvas: this.canvas,
         inputElement: this.sourceDummy!,
       });
       readableStream = readableStream.pipeThrough(transformer!.transformer!);
     }
-    readableStream.pipeTo(this.trackGenerator.writable);
+    readableStream
+      .pipeTo(this.trackGenerator.writable)
+      .catch((e) => console.error('error when trying to pipe', e))
+      .finally(() => this.destroy());
     this.processedTrack = this.trackGenerator as MediaStreamVideoTrack;
   }
 

package/src/transformers/BackgroundTransformer.ts

@@ -31,7 +31,7 @@ export default class BackgroundProcessor extends VideoTransformer {
   }
 
   async init({ outputCanvas, inputElement: inputVideo }: VideoTransformerInitOptions) {
-    super.init({ outputCanvas, inputElement: inputVideo });
+    await super.init({ outputCanvas, inputElement: inputVideo });
 
     const fileSet = await vision.FilesetResolver.forVisionTasks(
       `https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@${dependencies['@mediapipe/tasks-vision']}/wasm`,
@@ -49,7 +49,6 @@ export default class BackgroundProcessor extends VideoTransformer {
     });
 
     // this.loadBackground(opts.backgroundUrl).catch((e) => console.error(e));
-    // this.sendFramesContinuouslyForSegmentation(this.inputVideo!);
   }
 
   async destroy() {
@@ -58,22 +57,6 @@ export default class BackgroundProcessor extends VideoTransformer {
     this.backgroundImage = null;
   }
 
-  async sendFramesContinuouslyForSegmentation(videoEl: HTMLVideoElement) {
-    if (!this.isDisabled) {
-      if (videoEl.videoWidth > 0 && videoEl.videoHeight > 0) {
-        let startTimeMs = performance.now();
-        this.imageSegmenter?.segmentForVideo(
-          videoEl,
-          startTimeMs,
-          (result) => (this.segmentationResults = result),
-        );
-      }
-      videoEl.requestVideoFrameCallback(() => {
-        this.sendFramesContinuouslyForSegmentation(videoEl);
-      });
-    }
-  }
-
   async loadBackground(path: string) {
     const img = new Image();
 
@@ -88,30 +71,33 @@ export default class BackgroundProcessor extends VideoTransformer {
   }
 
   async transform(frame: VideoFrame, controller: TransformStreamDefaultController<VideoFrame>) {
-    if (this.isDisabled) {
-      controller.enqueue(frame);
-      return;
-    }
-    if (!this.canvas) {
-      throw TypeError('Canvas needs to be initialized first');
-    }
-    let startTimeMs = performance.now();
-    this.imageSegmenter?.segmentForVideo(
-      this.inputVideo!,
-      startTimeMs,
-      (result) => (this.segmentationResults = result),
-    );
+    try {
+      if (this.isDisabled) {
+        controller.enqueue(frame);
+        return;
+      }
+      if (!this.canvas) {
+        throw TypeError('Canvas needs to be initialized first');
+      }
+      let startTimeMs = performance.now();
+      this.imageSegmenter?.segmentForVideo(
+        this.inputVideo!,
+        startTimeMs,
+        (result) => (this.segmentationResults = result),
+      );
 
-    if (this.blurRadius) {
-      await this.blurBackground(frame);
-    } else {
-      await this.drawVirtualBackground(frame);
+      if (this.blurRadius) {
+        await this.blurBackground(frame);
+      } else {
+        await this.drawVirtualBackground(frame);
+      }
+      const newFrame = new VideoFrame(this.canvas, {
+        timestamp: frame.timestamp || Date.now(),
+      });
+      controller.enqueue(newFrame);
+    } finally {
+      frame.close();
     }
-    const newFrame = new VideoFrame(this.canvas, {
-      timestamp: frame.timestamp || Date.now(),
-    });
-    frame.close();
-    controller.enqueue(newFrame);
   }
 
   async drawVirtualBackground(frame: VideoFrame) {
@@ -161,6 +147,8 @@ export default class BackgroundProcessor extends VideoTransformer {
     ) {
       return;
     }
+    this.canvas.width = frame.codedWidth;
+    this.canvas.height = frame.codedHeight;
     this.ctx.save();
     this.ctx.globalCompositeOperation = 'copy';
 
@@ -169,6 +157,7 @@ export default class BackgroundProcessor extends VideoTransformer {
       this.inputVideo.videoWidth,
       this.inputVideo.videoHeight,
     );
+
     this.ctx.filter = 'blur(3px)';
     this.ctx.globalCompositeOperation = 'copy';
     this.ctx.drawImage(bitmap, 0, 0);

package/src/transformers/MediaPipeHolisticTrackerTransformer.ts

@@ -22,7 +22,10 @@ export default class MediaPipeHolisticTrackerTransformer extends VideoTransforme
     this.holisticOptions = holisticOptions || {};
   }
 
-  init({ inputElement: inputVideo, outputCanvas }: VideoTransformerInitOptions): void {
+  async init({
+    inputElement: inputVideo,
+    outputCanvas,
+  }: VideoTransformerInitOptions): Promise<void> {
     super.init({ outputCanvas, inputElement: inputVideo });
 
     this.holistic = new Holistic({

package/src/transformers/VideoTransformer.ts

@@ -11,7 +11,10 @@ export default abstract class VideoTransformer implements VideoTrackTransformer
 
   protected isDisabled?: Boolean = false;
 
-  init({ outputCanvas, inputElement: inputVideo }: VideoTransformerInitOptions): void {
+  async init({
+    outputCanvas,
+    inputElement: inputVideo,
+  }: VideoTransformerInitOptions): Promise<void> {
     if (!(inputVideo instanceof HTMLVideoElement)) {
       throw TypeError('Video transformer needs a HTMLVideoElement as input');
     }