@100mslive/hms-virtual-background 1.3.1-7e → 1.3.4

package/dist/HMSVirtualBackgroundPlugin.d.ts

@@ -1,5 +1,5 @@
 import '@tensorflow/tfjs-backend-webgl';
-import { HMSVideoPlugin, HMSVideoPluginType } from '@100mslive/hms-video';
+import { HMSVideoPlugin, HMSVideoPluginType } from "@100mslive/hms-video";
 export declare class HMSVirtualBackgroundPlugin implements HMSVideoPlugin {
     background: string | HTMLImageElement;
     personMaskWidth: number;
@@ -20,13 +20,13 @@ export declare class HMSVirtualBackgroundPlugin implements HMSVideoPlugin {
     personMask: ImageData;
     personMaskCanvas: HTMLCanvasElement;
     personMaskCtx: any;
-    Filters: any;
-    constructor(background: string);
+    filters: any;
+    enableSharpening?: boolean | false;
+    constructor(background: string, enableSharpening?: boolean);
     init(): Promise<void>;
     isSupported(): boolean;
     getName(): string;
     getPluginType(): HMSVideoPluginType;
-    initSharpenFilter(): any;
     setBackground(path?: string | HTMLImageElement): Promise<void>;
     stop(): void;
     processVideoFrame(input: HTMLCanvasElement, output: HTMLCanvasElement, skipProcessing?: boolean): Promise<void> | void;
@@ -35,9 +35,11 @@ export declare class HMSVirtualBackgroundPlugin implements HMSVideoPlugin {
     private resizeInputData;
     private infer;
     private postProcessing;
+    sharpenFilter(): void;
     private drawPersonMask;
     private drawSegmentedBackground;
     private runSegmentation;
     private fitImageToBackground;
     private addBlurToBackground;
+    initSharpenFilter(): any;
 }

package/dist/hms-virtual-background.cjs.development.js

@@ -1,5 +1,7 @@
 'use strict';
 
+Object.defineProperty(exports, '__esModule', { value: true });
+
 require('@tensorflow/tfjs-backend-webgl');
 var hmsVideo = require('@100mslive/hms-video');
 
@@ -916,10 +918,18 @@ var loadTFLite = /*#__PURE__*/function () {
 
 var TAG$1 = 'VBProcessor';
 
-var pkg$1 = /*#__PURE__*/require('../package.json');
+var pkg$1 = /*#__PURE__*/require("../package.json");
 
+var minVideoWidthForSharpening = 214;
+var maxVideoWidthForSharpening = 855;
+var minVideoHeightForSharpening = 120;
+var maxVideoHeightForSharpening = 720;
 var HMSVirtualBackgroundPlugin = /*#__PURE__*/function () {
-  function HMSVirtualBackgroundPlugin(background) {
+  function HMSVirtualBackgroundPlugin(background, enableSharpening) {
+    if (enableSharpening === void 0) {
+      enableSharpening = false;
+    }
+
     this.background = background;
     this.personMaskWidth = 256;
     this.personMaskHeight = 144;
@@ -939,8 +949,9 @@ var HMSVirtualBackgroundPlugin = /*#__PURE__*/function () {
     this.personMaskCanvas.width = this.personMaskWidth;
     this.personMaskCanvas.height = this.personMaskHeight;
     this.personMaskCtx = this.personMaskCanvas.getContext('2d');
-    this.Filters = {};
-    this.log(TAG$1, 'Virtual Background plugin created');
+    this.filters = {};
+    this.enableSharpening = enableSharpening;
+    this.log(TAG$1, "Virtual Background plugin created");
     this.setBackground(this.background);
   }
 
@@ -957,7 +968,7 @@ var HMSVirtualBackgroundPlugin = /*#__PURE__*/function () {
                 break;
               }
 
-              this.log(TAG$1, 'PREVIOUS LOADED MODEL IS ', this.tfLite);
+              this.log(TAG$1, "PREVIOUS LOADED MODEL IS ", this.tfLite);
               this.loadModelCalled = true;
               this.tfLitePromise = loadTFLite();
               _context.next = 6;
@@ -973,7 +984,9 @@ var HMSVirtualBackgroundPlugin = /*#__PURE__*/function () {
               return this.tfLitePromise;
 
             case 11:
-              this.initSharpenFilter();
+              if (this.enableSharpening) {
+                this.initSharpenFilter();
+              }
 
             case 12:
             case "end":
@@ -992,7 +1005,7 @@ var HMSVirtualBackgroundPlugin = /*#__PURE__*/function () {
 
   _proto.isSupported = function isSupported() {
     //support chrome, firefox, edge TODO: check this
-    return navigator.userAgent.indexOf('Chrome') != -1 || navigator.userAgent.indexOf('Firefox') != -1 || navigator.userAgent.indexOf('Edg') != -1;
+    return navigator.userAgent.indexOf("Chrome") != -1 || navigator.userAgent.indexOf("Firefox") != -1 || navigator.userAgent.indexOf("Edg") != -1;
   };
 
   _proto.getName = function getName() {
@@ -1003,91 +1016,6 @@ var HMSVirtualBackgroundPlugin = /*#__PURE__*/function () {
     return hmsVideo.HMSVideoPluginType.TRANSFORM;
   };
 
-  _proto.initSharpenFilter = function initSharpenFilter() {
-    var _arguments = arguments,
-        _this = this;
-
-    this.Filters.filterImage = function (filter, image, var_args) {
-      var args = [image];
-
-      for (var i = 2; i < _arguments.length; i++) {
-        args.push(_arguments[i]);
-      }
-
-      return filter.apply(null, args);
-    };
-
-    this.Filters.tmpCanvas = document.createElement('canvas');
-    this.Filters.tmpCtx = this.Filters.tmpCanvas.getContext('2d');
-
-    this.Filters.createImageData = function (w, h) {
-      return _this.Filters.tmpCtx.createImageData(w, h);
-    };
-
-    this.Filters.convolute = function (pixels, weights, opaque) {
-      if (weights === void 0) {
-        weights = [0, -1, 0, -1, 5, -1, 0, -1, 0];
-      }
-
-      var side = Math.round(Math.sqrt(weights.length));
-      var halfSide = Math.floor(side / 2);
-      var src = pixels.data;
-      var sw = pixels.width;
-      var sh = pixels.height; // pad output by the convolution matrix
-
-      var w = sw;
-      var h = sh;
-
-      var output = _this.Filters.createImageData(w, h);
-
-      var dst = output.data; // go through the destination image pixels
-
-      var alphaFac = opaque ? 1 : 0;
-
-      for (var y = 0; y < h; y = y + 1) {
-        for (var x = 0; x < w; x = x + 1) {
-          var dstOff = (y * w + x) * 4;
-
-          if (src[dstOff + 3] == 0) {
-            continue;
-          } else if (x < w && y < h) {
-            var sy = y;
-            var sx = x; // calculate the weighed sum of the source image pixels that
-            // fall under the convolution matrix
-
-            var r = 0,
-                g = 0,
-                b = 0,
-                a = 0;
-
-            for (var cy = 0; cy < side; cy++) {
-              for (var cx = 0; cx < side; cx++) {
-                var scy = sy + cy - halfSide;
-                var scx = sx + cx - halfSide;
-
-                if (scy >= 0 && scy < sh && scx >= 0 && scx < sw) {
-                  var srcOff = (scy * sw + scx) * 4;
-                  var wt = weights[cy * side + cx];
-                  r += src[srcOff] * wt;
-                  g += src[srcOff + 1] * wt;
-                  b += src[srcOff + 2] * wt;
-                  a += src[srcOff + 3] * wt;
-                }
-              }
-            }
-
-            dst[dstOff] = r;
-            dst[dstOff + 1] = g;
-            dst[dstOff + 2] = b;
-            dst[dstOff + 3] = a + alphaFac * (255 - a);
-          }
-        }
-      }
-
-      return output;
-    };
-  };
-
   _proto.setBackground = /*#__PURE__*/function () {
     var _setBackground = /*#__PURE__*/_asyncToGenerator( /*#__PURE__*/runtime_1.mark(function _callee2(path) {
       var img;
@@ -1095,30 +1023,30 @@ var HMSVirtualBackgroundPlugin = /*#__PURE__*/function () {
         while (1) {
           switch (_context2.prev = _context2.next) {
             case 0:
-              if (!(path !== '')) {
+              if (!(path !== "")) {
                 _context2.next = 24;
                 break;
               }
 
-              if (!(path === 'none')) {
+              if (!(path === "none")) {
                 _context2.next = 7;
                 break;
               }
 
               this.log(TAG$1, 'setting background to :', path);
-              this.background = 'none';
+              this.background = "none";
               this.isVirtualBackground = false;
               _context2.next = 22;
               break;
 
             case 7:
-              if (!(path === 'blur')) {
+              if (!(path === "blur")) {
                 _context2.next = 13;
                 break;
               }
 
               this.log(TAG$1, 'setting background to :', path);
-              this.background = 'blur';
+              this.background = "blur";
               this.isVirtualBackground = false;
               _context2.next = 22;
               break;
@@ -1146,7 +1074,7 @@ var HMSVirtualBackgroundPlugin = /*#__PURE__*/function () {
               break;
 
             case 24:
-              this.log(TAG$1, 'Not updating anything using the previous background Settings');
+              this.log(TAG$1, "Not updating anything using the previous background Settings");
 
             case 25:
             case "end":
@@ -1177,7 +1105,7 @@ var HMSVirtualBackgroundPlugin = /*#__PURE__*/function () {
   };
 
   _proto.processVideoFrame = function processVideoFrame(input, output, skipProcessing) {
-    var _this2 = this;
+    var _this = this;
 
     if (!input || !output) {
       throw new Error('Plugin invalid input/output');
@@ -1199,7 +1127,7 @@ var HMSVirtualBackgroundPlugin = /*#__PURE__*/function () {
     this.imageAspectRatio = input.width / input.height;
 
     if (this.imageAspectRatio <= 0) {
-      throw new Error('Invalid input width/height');
+      throw new Error("Invalid input width/height");
     }
 
     var process = /*#__PURE__*/function () {
@@ -1209,7 +1137,7 @@ var HMSVirtualBackgroundPlugin = /*#__PURE__*/function () {
             switch (_context3.prev = _context3.next) {
               case 0:
                 _context3.next = 2;
-                return _this2.runSegmentation(skipProcessing);
+                return _this.runSegmentation(skipProcessing);
 
               case 2:
               case "end":
@@ -1224,7 +1152,7 @@ var HMSVirtualBackgroundPlugin = /*#__PURE__*/function () {
       };
     }();
 
-    if (this.background === 'none' && !this.isVirtualBackground) {
+    if (this.background === "none" && !this.isVirtualBackground) {
       this.outputCtx.globalCompositeOperation = 'copy';
       this.outputCtx.filter = 'none';
       this.outputCtx.drawImage(input, 0, 0, input.width, input.height);
@@ -1322,31 +1250,33 @@ var HMSVirtualBackgroundPlugin = /*#__PURE__*/function () {
 
     this.drawPersonMask();
     this.outputCtx.globalCompositeOperation = 'source-in';
-    this.outputCtx.filter = 'none'; // // //Draw the foreground
+    this.outputCtx.filter = 'none'; // //Draw the foreground
 
     this.outputCtx.drawImage(this.input, 0, 0);
-    var outputImageData = this.outputCtx.getImageData(0, 0, this.output.width, this.output.height); // [-1, -1, -1, -1, 9, -1, -1, -1, -1]
+
+    if (this.enableSharpening && this.output.width > minVideoWidthForSharpening && // minimum and maximum resolution to enable sharpening filter
+    this.output.height > minVideoHeightForSharpening && this.output.width < maxVideoWidthForSharpening && this.output.height < maxVideoHeightForSharpening) {
+      this.sharpenFilter();
+    } // //Draw the background
+
+
+    this.drawSegmentedBackground();
+  };
+
+  _proto.sharpenFilter = function sharpenFilter() {
+    // adding sharpening filter to each frame to improve edges and brightness
+    // The basic idea is that you take the weighed sum of a rectangle of pixels from the source image and use that as the output value using convolution filter
+    // It is applied intermediate output with black background and only mask data in frame
+    // Filter currently used is 3 x 3 sharpening filter with values as shown:
+    // [  0, -1,  0,
+    //   -1,  5, -1,
+    //   0, -1,  0 ]
+    var outputImageData = this.outputCtx.getImageData(0, 0, this.output.width, this.output.height); // filters you may try
+    // [-1, -1, -1, -1, 9, -1, -1, -1, -1]
     //[0, -1, 0, -1, 5, -1, 0, -1, 0]
 
-    var output = this.Filters.filterImage(this.Filters.convolute, outputImageData);
-    this.outputCtx.putImageData(output, 0, 0); // //Draw the background
-
-    this.drawSegmentedBackground(); // //const t0 = performance.now();
-    // let outputImageData = this.outputCtx!.getImageData(
-    //   0,
-    //   0,
-    //   this.output!.width,
-    //   this.output!.height
-    // );
-    // // [-1, -1, -1, -1, 9, -1, -1, -1, -1]
-    // //[0, -1, 0, -1, 5, -1, 0, -1, 0]
-    // let output = this.Filters.filterImage(
-    //   this.Filters.convolute,
-    //   outputImageData
-    // );
-    // this.outputCtx!.putImageData(output, 0, 0);
-    // const t1 = performance.now();
-    //this.log(TAG, t1 - t0);
+    var output = this.filters.convolute(outputImageData);
+    this.outputCtx.putImageData(output, 0, 0);
   };
 
   _proto.drawPersonMask = function drawPersonMask() {
@@ -1367,27 +1297,25 @@ var HMSVirtualBackgroundPlugin = /*#__PURE__*/function () {
 
   _proto.runSegmentation = /*#__PURE__*/function () {
     var _runSegmentation = /*#__PURE__*/_asyncToGenerator( /*#__PURE__*/runtime_1.mark(function _callee5(skipProcessing) {
-      var start, end;
       return runtime_1.wrap(function _callee5$(_context5) {
         while (1) {
           switch (_context5.prev = _context5.next) {
             case 0:
               if (!this.tfLite) {
-                _context5.next = 8;
+                _context5.next = 5;
                 break;
               }
 
-              start = performance.now();
+              // const start = performance.now();
               this.resizeInputData();
-              _context5.next = 5;
+              _context5.next = 4;
               return this.infer(skipProcessing);
 
-            case 5:
-              this.postProcessing();
-              end = performance.now();
-              this.log(TAG$1, 'time taken', end - start);
+            case 4:
+              this.postProcessing(); // const end = performance.now();
+              // this.log(TAG,"time taken",end -start);
 
-            case 8:
+            case 5:
             case "end":
               return _context5.stop();
           }
@@ -1447,6 +1375,80 @@ var HMSVirtualBackgroundPlugin = /*#__PURE__*/function () {
     return addBlurToBackground;
   }();
 
+  _proto.initSharpenFilter = function initSharpenFilter() {
+    var _this2 = this;
+
+    this.filters.tmpCanvas = document.createElement('canvas');
+    this.filters.tmpCtx = this.filters.tmpCanvas.getContext('2d');
+
+    this.filters.createImageData = function (w, h) {
+      return _this2.filters.tmpCtx.createImageData(w, h);
+    };
+
+    this.filters.convolute = function (pixels, weights, opaque) {
+      if (weights === void 0) {
+        weights = [0, -1, 0, -1, 5, -1, 0, -1, 0];
+      }
+
+      var side = Math.round(Math.sqrt(weights.length));
+      var halfSide = Math.floor(side / 2);
+      var src = pixels.data;
+      var sw = pixels.width;
+      var sh = pixels.height; // pad output by the convolution matrix
+
+      var w = sw;
+      var h = sh;
+
+      var output = _this2.filters.createImageData(w, h);
+
+      var dst = output.data; // go through the destination image pixels
+
+      var alphaFac = opaque ? 1 : 0;
+
+      for (var y = 0; y < h; y = y + 1) {
+        for (var x = 0; x < w; x = x + 1) {
+          var dstOff = (y * w + x) * 4;
+
+          if (src[dstOff + 3] == 0) {
+            continue;
+          } else if (x < w && y < h) {
+            var sy = y;
+            var sx = x; // calculate the weighed sum of the source image pixels that
+            // fall under the convolution matrix
+
+            var r = 0,
+                g = 0,
+                b = 0,
+                a = 0;
+
+            for (var cy = 0; cy < side; cy++) {
+              for (var cx = 0; cx < side; cx++) {
+                var scy = sy + cy - halfSide;
+                var scx = sx + cx - halfSide;
+
+                if (scy >= 0 && scy < sh && scx >= 0 && scx < sw) {
+                  var srcOff = (scy * sw + scx) * 4;
+                  var wt = weights[cy * side + cx];
+                  r += src[srcOff] * wt;
+                  g += src[srcOff + 1] * wt;
+                  b += src[srcOff + 2] * wt;
+                  a += src[srcOff + 3] * wt;
+                }
+              }
+            }
+
+            dst[dstOff] = r;
+            dst[dstOff + 1] = g;
+            dst[dstOff + 2] = b;
+            dst[dstOff + 3] = a + alphaFac * (255 - a);
+          }
+        }
+      }
+
+      return output;
+    };
+  };
+
   return HMSVirtualBackgroundPlugin;
 }();