@100mslive/hms-virtual-background 1.3.2 → 1.3.3-alpha

package/dist/HMSVirtualBackgroundPlugin.d.ts

@@ -5,7 +5,9 @@ export declare class HMSVirtualBackgroundPlugin implements HMSVideoPlugin {
     personMaskWidth: number;
     personMaskHeight: number;
     isVirtualBackground: boolean;
-    backgroundImage: HTMLImageElement | undefined;
+    backgroundImage: HTMLImageElement | null;
+    backgroundVideo: HTMLVideoElement | null;
+    backgroundType: string;
     loadModelCalled: boolean;
     blurValue: any;
     tfLite: any;
@@ -20,12 +22,14 @@ export declare class HMSVirtualBackgroundPlugin implements HMSVideoPlugin {
     personMask: ImageData;
     personMaskCanvas: HTMLCanvasElement;
     personMaskCtx: any;
-    constructor(background: string);
+    filters: any;
+    enableSharpening?: boolean | false;
+    constructor(background: string, enableSharpening?: boolean);
     init(): Promise<void>;
     isSupported(): boolean;
     getName(): string;
     getPluginType(): HMSVideoPluginType;
-    setBackground(path?: string | HTMLImageElement): Promise<void>;
+    setBackground(path?: string | HTMLImageElement | HTMLVideoElement): Promise<void>;
     stop(): void;
     processVideoFrame(input: HTMLCanvasElement, output: HTMLCanvasElement, skipProcessing?: boolean): Promise<void> | void;
     private setImage;
@@ -33,9 +37,12 @@ export declare class HMSVirtualBackgroundPlugin implements HMSVideoPlugin {
     private resizeInputData;
     private infer;
     private postProcessing;
+    private sharpenFilter;
     private drawPersonMask;
     private drawSegmentedBackground;
     private runSegmentation;
+    private fitVideoToBackground;
     private fitImageToBackground;
     private addBlurToBackground;
+    private initSharpenFilter;
 }

package/dist/hms-virtual-background.cjs.development.js

@@ -921,8 +921,16 @@ var TAG$1 = 'VBProcessor';
 var pkg$1 = /*#__PURE__*/require("../package.json");
 
 var HMSVirtualBackgroundPlugin = /*#__PURE__*/function () {
-  function HMSVirtualBackgroundPlugin(background) {
+  function HMSVirtualBackgroundPlugin(background, enableSharpening) {
+    if (enableSharpening === void 0) {
+      enableSharpening = false;
+    }
+
+    this.backgroundType = "none";
     this.background = background;
+    this.enableSharpening = enableSharpening;
+    this.backgroundImage = null;
+    this.backgroundVideo = null;
     this.personMaskWidth = 256;
     this.personMaskHeight = 144;
     this.isVirtualBackground = false;
@@ -941,6 +949,7 @@ var HMSVirtualBackgroundPlugin = /*#__PURE__*/function () {
     this.personMaskCanvas.width = this.personMaskWidth;
     this.personMaskCanvas.height = this.personMaskHeight;
     this.personMaskCtx = this.personMaskCanvas.getContext('2d');
+    this.filters = {};
     this.log(TAG$1, "Virtual Background plugin created");
     this.setBackground(this.background);
   }
@@ -974,6 +983,11 @@ var HMSVirtualBackgroundPlugin = /*#__PURE__*/function () {
               return this.tfLitePromise;
 
             case 11:
+              if (this.enableSharpening) {
+                this.initSharpenFilter();
+              }
+
+            case 12:
             case "end":
               return _context.stop();
           }
@@ -1002,71 +1016,121 @@ var HMSVirtualBackgroundPlugin = /*#__PURE__*/function () {
   };
 
   _proto.setBackground = /*#__PURE__*/function () {
-    var _setBackground = /*#__PURE__*/_asyncToGenerator( /*#__PURE__*/runtime_1.mark(function _callee2(path) {
+    var _setBackground = /*#__PURE__*/_asyncToGenerator( /*#__PURE__*/runtime_1.mark(function _callee3(path) {
+      var _this = this;
+
       var img;
-      return runtime_1.wrap(function _callee2$(_context2) {
+      return runtime_1.wrap(function _callee3$(_context3) {
         while (1) {
-          switch (_context2.prev = _context2.next) {
+          switch (_context3.prev = _context3.next) {
             case 0:
               if (!(path !== "")) {
-                _context2.next = 24;
+                _context3.next = 43;
                 break;
               }
 
               if (!(path === "none")) {
-                _context2.next = 7;
+                _context3.next = 8;
                 break;
               }
 
               this.log(TAG$1, 'setting background to :', path);
               this.background = "none";
+              this.backgroundType = "none";
               this.isVirtualBackground = false;
-              _context2.next = 22;
+              _context3.next = 41;
               break;
 
-            case 7:
+            case 8:
               if (!(path === "blur")) {
-                _context2.next = 13;
+                _context3.next = 15;
                 break;
               }
 
               this.log(TAG$1, 'setting background to :', path);
               this.background = "blur";
+              this.backgroundType = "blur"; //TODO: check this
+
               this.isVirtualBackground = false;
-              _context2.next = 22;
+              _context3.next = 41;
               break;
 
-            case 13:
-              _context2.next = 15;
+            case 15:
+              if (!(path instanceof HTMLImageElement)) {
+                _context3.next = 29;
+                break;
+              }
+
+              this.log("setting background to image", path);
+              _context3.next = 19;
               return this.setImage(path);
 
-            case 15:
-              img = _context2.sent;
+            case 19:
+              img = _context3.sent;
 
               if (!(!img || !img.complete || !img.naturalHeight)) {
-                _context2.next = 20;
+                _context3.next = 24;
                 break;
               }
 
               throw new Error('Invalid image. Provide a valid and successfully loaded HTMLImageElement');
 
-            case 20:
+            case 24:
               this.isVirtualBackground = true;
               this.backgroundImage = img;
+              this.backgroundType = "image";
 
-            case 22:
-              _context2.next = 25;
+            case 27:
+              _context3.next = 41;
               break;
 
-            case 24:
-              this.log(TAG$1, "Not updating anything using the previous background Settings");
+            case 29:
+              if (!(path instanceof HTMLVideoElement)) {
+                _context3.next = 40;
+                break;
+              }
+
+              this.log("setting background to video", path);
+              this.isVirtualBackground = true;
+              this.backgroundType = "video";
+              this.backgroundVideo = path;
+              this.backgroundVideo.crossOrigin = 'anonymous';
+              this.backgroundVideo.muted = true;
+              this.backgroundVideo.loop = true;
+              this.backgroundVideo.oncanplaythrough = /*#__PURE__*/_asyncToGenerator( /*#__PURE__*/runtime_1.mark(function _callee2() {
+                return runtime_1.wrap(function _callee2$(_context2) {
+                  while (1) {
+                    switch (_context2.prev = _context2.next) {
+                      case 0:
+                        _context2.next = 2;
+                        return _this.backgroundVideo.play();
+
+                      case 2:
+                      case "end":
+                        return _context2.stop();
+                    }
+                  }
+                }, _callee2);
+              }));
+              _context3.next = 41;
+              break;
 
-            case 25:
+            case 40:
+              throw new Error("Invalid background supplied, see the docs to check supported background type");
+
+            case 41:
+              _context3.next = 44;
+              break;
+
+            case 43:
+              throw new Error("Invalid background supplied, see the docs to check supported background type");
+
+            case 44:
             case "end":
-              return _context2.stop();
+              return _context3.stop();
           }
         }
-      }, _callee2, this);
+      }, _callee3, this);
     }));
 
     function setBackground(_x) {
@@ -1078,9 +1142,15 @@ var HMSVirtualBackgroundPlugin = /*#__PURE__*/function () {
 
   _proto.stop = function stop() {
     if (this.isVirtualBackground) {
-      var _this$backgroundImage;
+      var _this$backgroundImage, _this$backgroundVideo;
 
       (_this$backgroundImage = this.backgroundImage) == null ? void 0 : _this$backgroundImage.removeAttribute('src');
+      (_this$backgroundVideo = this.backgroundVideo) == null ? void 0 : _this$backgroundVideo.removeAttribute('src');
+
+      if (this.backgroundType == "video") {
+        this.backgroundVideo.loop = false;
+        this.backgroundVideo = null;
+      }
     }
 
     if (this.outputCtx) {
@@ -1090,18 +1160,32 @@ var HMSVirtualBackgroundPlugin = /*#__PURE__*/function () {
   };
 
   _proto.processVideoFrame = function processVideoFrame(input, output, skipProcessing) {
-    var _this = this;
+    var _this2 = this;
 
     if (!input || !output) {
       throw new Error('Plugin invalid input/output');
     }
 
     this.input = input;
-    this.output = output;
+    this.output = output; // if(!this.t1){
+    //   this.output.style.position = "fixed"
+    //   this.output.style.top = String(0)
+    //   document.body.append(this.output)
+    //   this.t1 = true
+    // }
+
     var ctx = output.getContext('2d');
 
     if (ctx.canvas.width !== input.width) {
       ctx.canvas.width = input.width;
+
+      if (this.backgroundType == "video") {
+        this.backgroundVideo.width = input.width;
+        this.backgroundVideo.height = input.height;
+      } // this.backgroundVideo!.style.top = String(0)
+      // this.backgroundVideo!.style.position = "fixed"
+      // document.body.append(this.backgroundVideo!)
+
     }
 
     if (ctx.canvas.height !== input.height) {
@@ -1116,24 +1200,24 @@ var HMSVirtualBackgroundPlugin = /*#__PURE__*/function () {
     }
 
     var process = /*#__PURE__*/function () {
-      var _ref = _asyncToGenerator( /*#__PURE__*/runtime_1.mark(function _callee3() {
-        return runtime_1.wrap(function _callee3$(_context3) {
+      var _ref2 = _asyncToGenerator( /*#__PURE__*/runtime_1.mark(function _callee4() {
+        return runtime_1.wrap(function _callee4$(_context4) {
           while (1) {
-            switch (_context3.prev = _context3.next) {
+            switch (_context4.prev = _context4.next) {
               case 0:
-                _context3.next = 2;
-                return _this.runSegmentation(skipProcessing);
+                _context4.next = 2;
+                return _this2.runSegmentation(skipProcessing);
 
               case 2:
               case "end":
-                return _context3.stop();
+                return _context4.stop();
             }
           }
-        }, _callee3);
+        }, _callee4);
       }));
 
       return function process() {
-        return _ref.apply(this, arguments);
+        return _ref2.apply(this, arguments);
       };
     }();
 
@@ -1147,13 +1231,13 @@ var HMSVirtualBackgroundPlugin = /*#__PURE__*/function () {
   };
 
   _proto.setImage = /*#__PURE__*/function () {
-    var _setImage = /*#__PURE__*/_asyncToGenerator( /*#__PURE__*/runtime_1.mark(function _callee4(image) {
-      return runtime_1.wrap(function _callee4$(_context4) {
+    var _setImage = /*#__PURE__*/_asyncToGenerator( /*#__PURE__*/runtime_1.mark(function _callee5(image) {
+      return runtime_1.wrap(function _callee5$(_context5) {
         while (1) {
-          switch (_context4.prev = _context4.next) {
+          switch (_context5.prev = _context5.next) {
             case 0:
               image.crossOrigin = 'anonymous';
-              return _context4.abrupt("return", new Promise(function (resolve, reject) {
+              return _context5.abrupt("return", new Promise(function (resolve, reject) {
                 image.onload = function () {
                   return resolve(image);
                 };
@@ -1163,10 +1247,10 @@ var HMSVirtualBackgroundPlugin = /*#__PURE__*/function () {
 
             case 2:
             case "end":
-              return _context4.stop();
+              return _context5.stop();
           }
         }
-      }, _callee4);
+      }, _callee5);
     }));
 
     function setImage(_x2) {
@@ -1188,7 +1272,8 @@ var HMSVirtualBackgroundPlugin = /*#__PURE__*/function () {
 
   _proto.resizeInputData = function resizeInputData() {
     this.personMaskCtx.drawImage(this.input, 0, 0, this.input.width, this.input.height, 0, 0, this.personMaskWidth, this.personMaskHeight);
-    var imageData = this.personMaskCtx.getImageData(0, 0, this.personMaskWidth, this.personMaskHeight);
+    var imageData = this.personMaskCtx.getImageData(0, 0, this.personMaskWidth, this.personMaskHeight); //
+
     var inputMemoryOffset = this.tfLite._getInputMemoryOffset() / 4;
 
     for (var i = 0; i < this.personMaskPixelCount; i++) {
@@ -1237,11 +1322,32 @@ var HMSVirtualBackgroundPlugin = /*#__PURE__*/function () {
     this.outputCtx.globalCompositeOperation = 'source-in';
     this.outputCtx.filter = 'none'; // //Draw the foreground
 
-    this.outputCtx.drawImage(this.input, 0, 0); // //Draw the background
+    this.outputCtx.drawImage(this.input, 0, 0);
+
+    if (this.enableSharpening) {
+      this.sharpenFilter();
+    } // //Draw the background
+
 
     this.drawSegmentedBackground();
   };
 
+  _proto.sharpenFilter = function sharpenFilter() {
+    // adding sharpening filter to each frame to improve edges and brightness
+    // The basic idea is that you take the weighed sum of a rectangle of pixels from the source image and use that as the output value using convolution filter
+    // It is applied intermediate output with black background and only mask data in frame
+    // Filter currently used is 3 x 3 sharpening filter with values as shown:
+    // [  0, -1,  0,
+    //   -1,  5, -1,
+    //   0, -1,  0 ]
+    var outputImageData = this.outputCtx.getImageData(0, 0, this.output.width, this.output.height); // filters you may try
+    // [-1, -1, -1, -1, 9, -1, -1, -1, -1]
+    //[0, -1, 0, -1, 5, -1, 0, -1, 0]
+
+    var output = this.filters.convolute(outputImageData);
+    this.outputCtx.putImageData(output, 0, 0);
+  };
+
   _proto.drawPersonMask = function drawPersonMask() {
     this.outputCtx.drawImage(this.personMaskCanvas, 0, 0, this.personMaskWidth, this.personMaskHeight, 0, 0, this.output.width, this.output.height);
   };
@@ -1252,26 +1358,30 @@ var HMSVirtualBackgroundPlugin = /*#__PURE__*/function () {
     this.outputCtx.imageSmoothingQuality = 'high';
 
     if (this.isVirtualBackground) {
-      this.fitImageToBackground();
+      if (this.backgroundType === "video") {
+        this.fitVideoToBackground();
+      } else if (this.backgroundType === "image") {
+        this.fitImageToBackground();
+      }
     } else {
       this.addBlurToBackground();
     }
   };
 
   _proto.runSegmentation = /*#__PURE__*/function () {
-    var _runSegmentation = /*#__PURE__*/_asyncToGenerator( /*#__PURE__*/runtime_1.mark(function _callee5(skipProcessing) {
-      return runtime_1.wrap(function _callee5$(_context5) {
+    var _runSegmentation = /*#__PURE__*/_asyncToGenerator( /*#__PURE__*/runtime_1.mark(function _callee6(skipProcessing) {
+      return runtime_1.wrap(function _callee6$(_context6) {
         while (1) {
-          switch (_context5.prev = _context5.next) {
+          switch (_context6.prev = _context6.next) {
             case 0:
               if (!this.tfLite) {
-                _context5.next = 5;
+                _context6.next = 5;
                 break;
               }
 
               // const start = performance.now();
               this.resizeInputData();
-              _context5.next = 4;
+              _context6.next = 4;
               return this.infer(skipProcessing);
 
             case 4:
@@ -1280,10 +1390,10 @@ var HMSVirtualBackgroundPlugin = /*#__PURE__*/function () {
 
             case 5:
             case "end":
-              return _context5.stop();
+              return _context6.stop();
           }
         }
-      }, _callee5, this);
+      }, _callee6, this);
     }));
 
     function runSegmentation(_x3) {
@@ -1293,6 +1403,23 @@ var HMSVirtualBackgroundPlugin = /*#__PURE__*/function () {
     return runSegmentation;
   }();
 
+  _proto.fitVideoToBackground = function fitVideoToBackground() {
+    // let input_width: any, input_height: any, x_off: any, y_off: any
+    // if ((this.backgroundVideo!.width / this.backgroundVideo!.height) < this.imageAspectRatio) {
+    //   input_width = this.backgroundVideo!.width;
+    //   input_height = this.backgroundVideo!.width/ this.imageAspectRatio;
+    //   x_off = 0;
+    //   y_off = (this.backgroundVideo!.height - input_height) / 2;
+    // }
+    // else {
+    //   input_height = this.backgroundVideo!.height;
+    //   input_width = this.backgroundVideo!.height * this.imageAspectRatio;
+    //   y_off = 0;
+    //   x_off = (this.backgroundVideo!.width - input_width) / 2;
+    // }
+    this.outputCtx.drawImage(this.backgroundVideo, 0, 0, this.output.width, this.output.height);
+  };
+
   _proto.fitImageToBackground = function fitImageToBackground() {
     var input_width, input_height, x_offset, y_offset;
 
@@ -1312,11 +1439,11 @@ var HMSVirtualBackgroundPlugin = /*#__PURE__*/function () {
   };
 
   _proto.addBlurToBackground = /*#__PURE__*/function () {
-    var _addBlurToBackground = /*#__PURE__*/_asyncToGenerator( /*#__PURE__*/runtime_1.mark(function _callee6() {
+    var _addBlurToBackground = /*#__PURE__*/_asyncToGenerator( /*#__PURE__*/runtime_1.mark(function _callee7() {
       var blurValue;
-      return runtime_1.wrap(function _callee6$(_context6) {
+      return runtime_1.wrap(function _callee7$(_context7) {
         while (1) {
-          switch (_context6.prev = _context6.next) {
+          switch (_context7.prev = _context7.next) {
             case 0:
               blurValue = '15px';
               if (this.input.width <= 160) blurValue = '5px';else if (this.input.width <= 320) blurValue = '10px';else if (this.input.width <= 640) blurValue = '15px';else if (this.input.width <= 960) blurValue = '20px';else if (this.input.width <= 1280) blurValue = '25px';else if (this.input.width <= 1920) blurValue = '30px';
@@ -1325,10 +1452,10 @@ var HMSVirtualBackgroundPlugin = /*#__PURE__*/function () {
 
             case 4:
             case "end":
-              return _context6.stop();
+              return _context7.stop();
           }
         }
-      }, _callee6, this);
+      }, _callee7, this);
     }));
 
     function addBlurToBackground() {
@@ -1338,6 +1465,80 @@ var HMSVirtualBackgroundPlugin = /*#__PURE__*/function () {
     return addBlurToBackground;
   }();
 
+  _proto.initSharpenFilter = function initSharpenFilter() {
+    var _this3 = this;
+
+    this.filters.tmpCanvas = document.createElement('canvas');
+    this.filters.tmpCtx = this.filters.tmpCanvas.getContext('2d');
+
+    this.filters.createImageData = function (w, h) {
+      return _this3.filters.tmpCtx.createImageData(w, h);
+    };
+
+    this.filters.convolute = function (pixels, weights, opaque) {
+      if (weights === void 0) {
+        weights = [0, -1, 0, -1, 5, -1, 0, -1, 0];
+      }
+
+      var side = Math.round(Math.sqrt(weights.length));
+      var halfSide = Math.floor(side / 2);
+      var src = pixels.data;
+      var sw = pixels.width;
+      var sh = pixels.height; // pad output by the convolution matrix
+
+      var w = sw;
+      var h = sh;
+
+      var output = _this3.filters.createImageData(w, h);
+
+      var dst = output.data; // go through the destination image pixels
+
+      var alphaFac = opaque ? 1 : 0;
+
+      for (var y = 0; y < h; y = y + 1) {
+        for (var x = 0; x < w; x = x + 1) {
+          var dstOff = (y * w + x) * 4;
+
+          if (src[dstOff + 3] == 0) {
+            continue;
+          } else if (x < w && y < h) {
+            var sy = y;
+            var sx = x; // calculate the weighed sum of the source image pixels that
+            // fall under the convolution matrix
+
+            var r = 0,
+                g = 0,
+                b = 0,
+                a = 0;
+
+            for (var cy = 0; cy < side; cy++) {
+              for (var cx = 0; cx < side; cx++) {
+                var scy = sy + cy - halfSide;
+                var scx = sx + cx - halfSide;
+
+                if (scy >= 0 && scy < sh && scx >= 0 && scx < sw) {
+                  var srcOff = (scy * sw + scx) * 4;
+                  var wt = weights[cy * side + cx];
+                  r += src[srcOff] * wt;
+                  g += src[srcOff + 1] * wt;
+                  b += src[srcOff + 2] * wt;
+                  a += src[srcOff + 3] * wt;
+                }
+              }
+            }
+
+            dst[dstOff] = r;
+            dst[dstOff + 1] = g;
+            dst[dstOff + 2] = b;
+            dst[dstOff + 3] = a + alphaFac * (255 - a);
+          }
+        }
+      }
+
+      return output;
+    };
+  };
+
   return HMSVirtualBackgroundPlugin;
 }();