idmission-web-sdk 2.3.115 → 2.3.116

package/dist/sdk2.umd.development.js

@@ -211,7 +211,7 @@
     return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
   };
 
-  var webSdkVersion = '2.3.115';
+  var webSdkVersion = '2.3.116';
 
   function getPlatform() {
     // eslint-disable-next-line @typescript-eslint/ban-ts-comment
@@ -8252,8 +8252,8 @@
 
   var DEFAULT_CDN_URL = 'https://websdk-cdn-dev.idmission.com/assets';
 
-  var defaultDocumentDetectorModelPath = "".concat(DEFAULT_CDN_URL, "/models/DocumentDetector/DocumentDetector-20251018_121234.tflite");
-  var defaultDocumentDetectorModelHash = 'cTLjb4g6H+a+RHnLaYrvVF8050qv/Ox04rMtxFXBDR6Xv8Pv+X//hsaO3K3jNPog';
+  var defaultDocumentDetectorModelPath = "".concat(DEFAULT_CDN_URL, "/models/DocumentDetector/DocumentDetector-20250815_115859.tflite");
+  var defaultDocumentDetectorModelHash = '64LSaO2ctQsTkGJoZ6Et4v2KH28Fz1ettgkxD+qFg6frxY7ufL9l/lkjw2ljX2cR';
 
   var defaultFocusModelPath = "".concat(DEFAULT_CDN_URL, "/models/Focus/Focus-20241008_102708.tflite");
   var defaultFocusModelHash = 'HTJNLB7QybtPXIIyUI9oNqW40vE5bgSV5V24R1wXLQHknNEVOSyDQDo/QzRgJ8jb';
@@ -8570,8 +8570,8 @@
     singlePage: 'Single page',
     none: 'None'
   };
-  var detector = null;
-  var detectorSettings = null;
+  var detector$1 = null;
+  var detectorSettings$1 = null;
   function loadDocumentDetector() {
     return __awaiter(this, arguments, void 0, function (modelAssetPath, scoreThreshold) {
       var _a, _b;
@@ -8584,7 +8584,7 @@
       return __generator(this, function (_c) {
         switch (_c.label) {
           case 0:
-            if (detector && (detectorSettings === null || detectorSettings === void 0 ? void 0 : detectorSettings.modelAssetPath) === modelAssetPath && (detectorSettings === null || detectorSettings === void 0 ? void 0 : detectorSettings.scoreThreshold) === scoreThreshold) return [2 /*return*/, detector];
+            if (detector$1 && (detectorSettings$1 === null || detectorSettings$1 === void 0 ? void 0 : detectorSettings$1.modelAssetPath) === modelAssetPath && (detectorSettings$1 === null || detectorSettings$1 === void 0 ? void 0 : detectorSettings$1.scoreThreshold) === scoreThreshold) return [2 /*return*/, detector$1];
             closeDocumentDetector();
             return [4 /*yield*/, preloadDocumentDetectorDependencies()];
           case 1:
@@ -8605,20 +8605,20 @@
               runningMode: 'VIDEO'
             }])];
           case 3:
-            detector = _c.sent();
-            detectorSettings = {
+            detector$1 = _c.sent();
+            detectorSettings$1 = {
               modelAssetPath: modelAssetPath,
               scoreThreshold: scoreThreshold
             };
-            return [2 /*return*/, detector];
+            return [2 /*return*/, detector$1];
         }
       });
     });
   }
   function closeDocumentDetector() {
-    detector === null || detector === void 0 ? void 0 : detector.close();
-    detector = null;
-    detectorSettings = null;
+    detector$1 === null || detector$1 === void 0 ? void 0 : detector$1.close();
+    detector$1 = null;
+    detectorSettings$1 = null;
   }
   function useLoadDocumentDetector(_a) {
     var _b = _a.shouldLoadModels,
@@ -8719,12 +8719,12 @@
     return __awaiter(this, void 0, void 0, function () {
       var startedAt, prediction, time, frameWidth, frameHeight;
       return __generator(this, function (_a) {
-        if (!detector) return [2 /*return*/, null];
+        if (!detector$1) return [2 /*return*/, null];
         startedAt = performance.now();
         // Detectors can throw errors, for example when using custom URLs that
         // contain a model that doesn't provide the expected output.
         try {
-          prediction = detector.detectForVideo(frame, performance.now());
+          prediction = detector$1.detectForVideo(frame, performance.now());
           time = performance.now() - startedAt;
           frameWidth = frame.width;
           frameHeight = frame.height;
@@ -15922,7 +15922,6 @@
       scaledHeight = _c.scaledHeight,
       xOffset = _c.xOffset,
       yOffset = _c.yOffset;
-    if (!point) return /*#__PURE__*/React.createElement(React.Fragment, null);
     var left, top;
     if (horizontal) {
       left = point.x / videoWidth * scaledWidth - xOffset;
@@ -21012,6 +21011,136 @@
   var templateObject_1$n, templateObject_2$l;
 
   var defaultSelfieCaptureModelLoadTimeoutMs = 45000;
+  var detector = null;
+  var detectorSettings = null;
+  function loadFaceDetector() {
+    return __awaiter(this, arguments, void 0, function (modelAssetPath) {
+      var _a, _b;
+      if (modelAssetPath === void 0) {
+        modelAssetPath = defaultFaceDetectorModelPath;
+      }
+      return __generator(this, function (_c) {
+        switch (_c.label) {
+          case 0:
+            if (detector && (detectorSettings === null || detectorSettings === void 0 ? void 0 : detectorSettings.modelAssetPath) === modelAssetPath) return [2 /*return*/, detector];
+            closeFaceDetector();
+            return [4 /*yield*/, preloadFaceDetectorDependencies()];
+          case 1:
+            _c.sent();
+            if (modelCapabilities.delegate === 'NONE') {
+              throw new Error('No available delegate for face detector.');
+            }
+            _b = (_a = ya).createFromOptions;
+            return [4 /*yield*/, ao.forVisionTasks(visionTasksBasePath)];
+          case 2:
+            return [4 /*yield*/, _b.apply(_a, [_c.sent(), {
+              // canvas: document.createElement('canvas'),
+              baseOptions: {
+                modelAssetPath: modelAssetPath,
+                delegate: modelCapabilities.delegate
+              },
+              runningMode: 'VIDEO'
+            }])];
+          case 3:
+            detector = _c.sent();
+            detectorSettings = {
+              modelAssetPath: modelAssetPath
+            };
+            return [2 /*return*/, detector];
+        }
+      });
+    });
+  }
+  function closeFaceDetector() {
+    detector === null || detector === void 0 ? void 0 : detector.close();
+    detector = null;
+    detectorSettings = null;
+  }
+  function useLoadFaceDetector(_a) {
+    var onModelError = _a.onModelError,
+      _b = _a.modelLoadTimeoutMs,
+      modelLoadTimeoutMs = _b === void 0 ? defaultSelfieCaptureModelLoadTimeoutMs : _b,
+      videoRef = _a.videoRef;
+    var _c = React.useState('not-started'),
+      modelLoadState = _c[0],
+      setModelLoadState = _c[1];
+    var _d = React.useState(0),
+      modelDownloadProgress = _d[0],
+      setModelDownloadProgress = _d[1];
+    var _e = React.useState(null),
+      modelWarmingStartedAt = _e[0],
+      setModelWarmingStartedAt = _e[1];
+    var _f = React.useState(null),
+      modelError = _f[0],
+      setModelError = _f[1];
+    React.useEffect(function loadModel() {
+      var _this = this;
+      setModelLoadState('downloading');
+      setModelWarmingStartedAt(null);
+      var modelLoadTimeout = setTimeout(function () {
+        setModelError(new Error('Model loading time limit exceeded.'));
+      }, modelLoadTimeoutMs);
+      function handleDownloadProgress(event) {
+        setModelDownloadProgress(progressToPercentage(event.detail));
+      }
+      document.addEventListener('idmission.preloadProgress.faceDetection', handleDownloadProgress);
+      var cancelVideoReady = function cancelVideoReady() {};
+      loadFaceDetector().then(function (model) {
+        return __awaiter(_this, void 0, void 0, function () {
+          var _a, videoReady, cancel, cancelled;
+          return __generator(this, function (_b) {
+            switch (_b.label) {
+              case 0:
+                setModelDownloadProgress(100);
+                clearTimeout(modelLoadTimeout);
+                setModelLoadState('warming');
+                setModelWarmingStartedAt(performance.now());
+                return [4 /*yield*/, testFaceDetectionAgainstKnownImage(model)];
+              case 1:
+                _b.sent();
+                _a = waitForVideoReady(videoRef), videoReady = _a[0], cancel = _a[1];
+                cancelled = false;
+                cancelVideoReady = function cancelVideoReady() {
+                  cancelled = true;
+                  cancel();
+                };
+                return [4 /*yield*/, videoReady];
+              case 2:
+                _b.sent();
+                if (cancelled) return [2 /*return*/];
+                model.detectForVideo(videoRef.current, performance.now());
+                setModelLoadState('ready');
+                return [2 /*return*/];
+            }
+          });
+        });
+      })["catch"](function (e) {
+        setModelError(e);
+        setModelLoadState('error');
+      })["finally"](function () {
+        clearTimeout(modelLoadTimeout);
+      });
+      return function () {
+        log('unloading face detection model');
+        cancelVideoReady();
+        closeFaceDetector();
+        clearTimeout(modelLoadTimeout);
+        document.removeEventListener('idmission.preloadProgress.faceDetection', handleDownloadProgress);
+      };
+    }, [modelLoadTimeoutMs, videoRef]);
+    React.useEffect(function handleModelError() {
+      if (modelError) onModelError === null || onModelError === void 0 ? void 0 : onModelError(modelError);
+    }, [modelError, onModelError]);
+    return React.useMemo(function () {
+      return {
+        ready: modelLoadState === 'ready',
+        modelLoadState: modelLoadState,
+        modelDownloadProgress: modelDownloadProgress,
+        modelWarmingStartedAt: modelWarmingStartedAt,
+        modelError: modelError
+      };
+    }, [modelLoadState, modelDownloadProgress, modelWarmingStartedAt, modelError]);
+  }
   var lastFaceDetectionAt = 0;
   var lastFaceDetectionTime = 0;
   function setLastFaceDetectionAt(time) {
@@ -21061,68 +21190,24 @@
       if (lastNNosePairs.length > framesNeeded - 1) lastNNosePairs.length = framesNeeded - 1;
     }
   }
-  function isFaceDetection(detection) {
-    return detection.boundingBox && detection.categories.some(function (c) {
-      return c.categoryName === 'Primary face' || c.categoryName === 'Secondary face';
-    });
-  }
-  function isNoseDetection(detection) {
-    return detection.categories.some(function (c) {
-      return c.categoryName === 'Nose';
+  function makeFaceDetectorPrediction(imageData) {
+    if (!detector) return null;
+    var prediction = detector.detectForVideo(imageData, performance.now());
+    var faces = prediction.detections.map(function (d) {
+      return {
+        box: convertBoundingBox(d.boundingBox),
+        keypoints: d.keypoints.map(function (k) {
+          var _a;
+          return _assign(_assign({}, k), {
+            x: k.x * imageData.width,
+            y: k.y * imageData.height,
+            name: (_a = k.label) !== null && _a !== void 0 ? _a : ''
+          });
+        })
+      };
     });
-  }
-  function makeFacePredictionWithDocumentDetector(frame) {
-    return __awaiter(this, void 0, void 0, function () {
-      var prediction, faceDetections, noseDetections, _i, _a, detection, faces, _b, faceDetections_1, d, faceBox, nose, _c, noseDetections_1, n, noseBox, cX, cY;
-      return __generator(this, function (_d) {
-        switch (_d.label) {
-          case 0:
-            return [4 /*yield*/, makeDocumentDetectorPrediction(frame)];
-          case 1:
-            prediction = _d.sent();
-            if (!prediction) return [2 /*return*/, null];
-            faceDetections = [];
-            noseDetections = [];
-            for (_i = 0, _a = prediction.detections; _i < _a.length; _i++) {
-              detection = _a[_i];
-              if (isNoseDetection(detection)) {
-                noseDetections.push(detection);
-              } else if (isFaceDetection(detection)) {
-                faceDetections.push(detection);
-              }
-            }
-            faces = [];
-            for (_b = 0, faceDetections_1 = faceDetections; _b < faceDetections_1.length; _b++) {
-              d = faceDetections_1[_b];
-              faceBox = d.boundingBox;
-              if (!faceBox) continue;
-              nose = null;
-              for (_c = 0, noseDetections_1 = noseDetections; _c < noseDetections_1.length; _c++) {
-                n = noseDetections_1[_c];
-                noseBox = n.boundingBox;
-                if (!noseBox) continue;
-                cX = noseBox.originX + noseBox.width / 2;
-                cY = noseBox.originY + noseBox.height / 2;
-                if (cX >= faceBox.originX && cX <= faceBox.originX + faceBox.width && cY >= faceBox.originY && cY <= faceBox.originY + faceBox.height) {
-                  nose = {
-                    x: cX,
-                    y: cY,
-                    name: 'nose'
-                  };
-                  break;
-                }
-              }
-              faces.push({
-                box: convertBoundingBox(d.boundingBox),
-                keypoints: [null, null, nose, null, null]
-              });
-            }
-            debug('faces', faces);
-            return [2 /*return*/, _assign(_assign({}, prediction), {
-              faces: faces
-            })];
-        }
-      });
+    return _assign(_assign({}, prediction), {
+      faces: faces
     });
   }
   function processFaceDetectorPrediction(_a) {
@@ -21150,9 +21235,13 @@
       // this represents the edge that the centroid of the face should not cross -- 12.5% of video width
       yCentroidBoundary = _h === void 0 ? 0.125 : _h,
       // this represents the edge that the centroid of the face should not cross -- 12.5% of video height
-      _j = _a.noseTrackingThreshold,
+      _j = _a.foreheadRatio,
       // this represents the edge that the centroid of the face should not cross -- 12.5% of video height
-      noseTrackingThreshold = _j === void 0 ? 0.2 : _j,
+      foreheadRatio = _j === void 0 ? 0.275 : _j,
+      // we found that the bounding box ends at the brow and misses the forehead. this ratio represents how much we should extend the box to include the forehead.
+      _k = _a.noseTrackingThreshold,
+      // we found that the bounding box ends at the brow and misses the forehead. this ratio represents how much we should extend the box to include the forehead.
+      noseTrackingThreshold = _k === void 0 ? 0.2 : _k,
       // this represents the maximum distance that the nose can be from the center of the face box -- 20% of the face box width or height
       minCaptureBrightnessThreshold = _a.minCaptureBrightnessThreshold,
       minCaptureRangeThreshold = _a.minCaptureRangeThreshold,
@@ -21179,10 +21268,10 @@
       var frameCX = videoWidth / 2;
       var frameCY = videoHeight / 2;
       // calculate head bounding box, with forehead extension
-      // const foreheadSize = face.box.height * foreheadRatio
+      var foreheadSize = face.box.height * foreheadRatio;
       var headXMin = face.box.xMin;
       var headXMax = face.box.xMax;
-      var headYMin = face.box.yMin; // - foreheadSize
+      var headYMin = face.box.yMin - foreheadSize;
       var headYMax = face.box.yMax;
       // calculate head centroids
       var headCX = (headXMin + headXMax) / 2;
@@ -21238,6 +21327,26 @@
       faceVisibilityTooLow: faceVisibilityTooLow
     };
   }
+  function testFaceDetectionAgainstKnownImage(detector) {
+    return new Promise(function (resolve, reject) {
+      var img = new Image();
+      img.crossOrigin = 'anonymous';
+      img.onload = function () {
+        var prediction = detector.detectForVideo(img, performance.now());
+        if (prediction.detections.length > 0) {
+          debug('face detection test result', prediction.detections);
+          resolve(void 0);
+        } else {
+          warn('face detection test failed');
+          reject(new Error('testFaceDetectionAgainstKnownImage failed to predict'));
+        }
+      };
+      img.onerror = function () {
+        return reject(new Error('testFaceDetectionAgainstKnownImage failed to load image'));
+      };
+      img.src = "".concat(DEFAULT_CDN_URL, "/head-test.jpg");
+    });
+  }
 
   function detectBrightnessAndContrast(frame, brightnessAverager) {
     var ctx = frame.getContext('2d');
@@ -21389,27 +21498,16 @@
     var canvasRef = React.useRef(null);
     var onPredictionHandler = React.useRef();
     var addToAverage = useRunningAvg(5).addToAverage;
-    var _f = useLoadDocumentDetector({
-        videoRef: videoRef,
+    var _f = useLoadFaceDetector({
         onModelError: onModelError,
-        modelLoadTimeoutMs: modelLoadTimeoutMs
+        modelLoadTimeoutMs: modelLoadTimeoutMs,
+        videoRef: videoRef
       }),
       ready = _f.ready,
       modelLoadState = _f.modelLoadState,
       modelDownloadProgress = _f.modelDownloadProgress,
       modelWarmingStartedAt = _f.modelWarmingStartedAt,
       modelError = _f.modelError;
-    // const {
-    //   ready,
-    //   modelLoadState,
-    //   modelDownloadProgress,
-    //   modelWarmingStartedAt,
-    //   modelError,
-    // } = useLoadFaceDetector({
-    //   onModelError,
-    //   modelLoadTimeoutMs,
-    //   videoRef,
-    // })
     var _g = useFrameLoop(React.useCallback(function () {
         return __awaiter(_this, void 0, void 0, function () {
           var vw, vh, ctx, thresholdsProvided, brightnessResults, brightness, range, variance, prediction, processed, e_1;
@@ -21423,21 +21521,19 @@
                 ctx = canvasRef.current.getContext('2d');
                 canvasRef.current.width = vw;
                 canvasRef.current.height = vh;
-                if (!(ctx && videoRef.current.readyState === 4)) return [3 /*break*/, 5];
+                if (!(ctx && videoRef.current.readyState === 4)) return [3 /*break*/, 4];
                 ctx.translate(vw, 0);
                 ctx.scale(-1, 1);
                 ctx.drawImage(videoRef.current, 0, 0, vw, vh);
                 _c.label = 1;
               case 1:
-                _c.trys.push([1, 4,, 5]);
+                _c.trys.push([1, 3,, 4]);
                 thresholdsProvided = minCaptureBrightnessThreshold !== undefined || minCaptureRangeThreshold !== undefined || minCaptureVarianceThreshold !== undefined;
                 brightnessResults = thresholdsProvided ? detectBrightnessAndContrast(canvasRef.current, addToAverage) : undefined;
                 brightness = brightnessResults === null || brightnessResults === void 0 ? void 0 : brightnessResults.brightness;
                 range = brightnessResults === null || brightnessResults === void 0 ? void 0 : brightnessResults.range;
                 variance = brightnessResults === null || brightnessResults === void 0 ? void 0 : brightnessResults.variance;
-                return [4 /*yield*/, makeFacePredictionWithDocumentDetector(canvasRef.current)];
-              case 2:
-                prediction = _c.sent();
+                prediction = makeFaceDetectorPrediction(canvasRef.current);
                 processed = processFaceDetectorPrediction({
                   faces: (_a = prediction === null || prediction === void 0 ? void 0 : prediction.faces) !== null && _a !== void 0 ? _a : [],
                   videoWidth: vw,
@@ -21453,15 +21549,15 @@
                 setLastFaceDetectionAt(new Date().getTime());
                 // setLastPrediction(processed)
                 return [4 /*yield*/, (_b = onPredictionHandler.current) === null || _b === void 0 ? void 0 : _b.call(onPredictionHandler, processed)];
-              case 3:
+              case 2:
                 // setLastPrediction(processed)
                 _c.sent();
-                return [3 /*break*/, 5];
-              case 4:
+                return [3 /*break*/, 4];
+              case 3:
                 e_1 = _c.sent();
                 error('caught face detection error', e_1);
-                return [3 /*break*/, 5];
-              case 5:
+                return [3 /*break*/, 4];
+              case 4:
                 return [2 /*return*/];
             }
           });
@@ -24766,8 +24862,7 @@
       var rightEdge = videoWidth * (1 - headTrackingBoundaryPercentage);
       var topEdge = videoHeight * headTrackingBoundaryPercentage;
       var bottomEdge = videoHeight * (1 - headTrackingBoundaryPercentage);
-      var nose = face === null || face === void 0 ? void 0 : face.keypoints[2];
-      var nearBoundary = !!face && !!nose && (headTrackingBoundaryType === 'nose' ? nose.x < leftEdge || nose.x > rightEdge || nose.y < topEdge || nose.y > bottomEdge : (face === null || face === void 0 ? void 0 : face.box.xMin) < leftEdge || (face === null || face === void 0 ? void 0 : face.box.xMax) > rightEdge || (face === null || face === void 0 ? void 0 : face.box.yMin) < topEdge || (face === null || face === void 0 ? void 0 : face.box.yMax) > bottomEdge);
+      var nearBoundary = !!face && (headTrackingBoundaryType === 'nose' ? face.keypoints[2].x < leftEdge || face.keypoints[2].x > rightEdge || face.keypoints[2].y < topEdge || face.keypoints[2].y > bottomEdge : (face === null || face === void 0 ? void 0 : face.box.xMin) < leftEdge || (face === null || face === void 0 ? void 0 : face.box.xMax) > rightEdge || (face === null || face === void 0 ? void 0 : face.box.yMin) < topEdge || (face === null || face === void 0 ? void 0 : face.box.yMax) > bottomEdge);
       setLastFace(face);
       setHeadTrackingSatisfied(!!face && !nearBoundary);
       setNumFramesWithoutFaces(face ? 0 : function (n) {