@smileid/web-components 2.0.1 → 10.0.0

package/lib/components/selfie/src/smartselfie-capture/hooks/useFaceCapture.ts

@@ -0,0 +1,558 @@
+import { useRef } from 'preact/hooks';
+import { useSignal, useComputed } from '@preact/signals';
+import { FaceLandmarker } from '@mediapipe/tasks-vision';
+import {
+  calculateFaceSize,
+  isFaceInBounds,
+  calculateMouthOpening,
+} from '../utils/faceDetection';
+import {
+  createCroppedVideoFrame,
+  drawFaceMesh,
+  clearCanvas,
+} from '../utils/canvas';
+import { captureImageFromVideo } from '../utils/imageCapture';
+import { ImageType } from '../constants';
+import { MESSAGES, type MessageKey } from '../utils/alertMessages';
+import { getMediapipeInstance } from '../utils/mediapipeManager';
+import packageJson from '../../../../../../package.json';
+
+const COMPONENTS_VERSION = packageJson.version;
+
+interface UseFaceCaptureProps {
+  videoRef: React.RefObject<HTMLVideoElement>;
+  canvasRef: React.RefObject<HTMLCanvasElement>;
+  interval: number;
+  duration: number;
+  smileThreshold: number;
+  mouthOpenThreshold: number;
+  minFaceSize: number;
+  maxFaceSize: number;
+  smileCooldown: number;
+}
+
+export const useFaceCapture = ({
+  videoRef,
+  canvasRef,
+  interval,
+  duration,
+  smileThreshold,
+  mouthOpenThreshold,
+  minFaceSize,
+  maxFaceSize,
+  smileCooldown,
+}: UseFaceCaptureProps) => {
+  const faceLandmarkerRef = useRef<FaceLandmarker | null>(null);
+  const animationFrameRef = useRef<number | null>(null);
+  const captureTimerRef = useRef<NodeJS.Timeout | null>(null);
+  const resumeCaptureRef = useRef<(() => void) | null>(null);
+
+  const faceDetected = useSignal(false);
+  const faceInBounds = useSignal(false);
+  const faceProximity = useSignal<'too-close' | 'too-far' | 'good'>('good');
+  const multipleFaces = useSignal(false);
+  const videoAspectRatio = useSignal(16 / 9);
+  const faceLandmarks = useSignal<any[]>([]);
+  const currentSmileScore = useSignal(0);
+  const currentFaceSize = useSignal(0);
+  const currentMouthOpen = useSignal(0);
+  const lastSmileTime = useSignal(0);
+  const alertTitle = useSignal('');
+
+  const isCapturing = useSignal(false);
+  const isPaused = useSignal(false);
+  const countdown = useSignal(0);
+  const capturedImages = useSignal<string[]>([]);
+  const referencePhoto = useSignal<string | null>(null);
+  const totalCaptures = useSignal(1);
+  const capturesTaken = useSignal(0);
+  const hasFinishedCapture = useSignal(false);
+
+  const smileCheckpoint = useComputed(() =>
+    Math.floor(totalCaptures.value * 0.4),
+  );
+  const neutralZone = useComputed(() => Math.floor(totalCaptures.value * 0.2));
+
+  const isReadyToCapture = useComputed(
+    () =>
+      faceDetected.value &&
+      faceInBounds.value &&
+      faceProximity.value === 'good' &&
+      !multipleFaces.value,
+  );
+
+  const initializeFaceLandmarker = async () => {
+    try {
+      faceLandmarkerRef.current = await getMediapipeInstance();
+    } catch (error) {
+      console.error('Failed to initialize MediaPipe:', error);
+    }
+  };
+
+  const updateAlert = (messageKey: MessageKey | null) => {
+    if (messageKey && MESSAGES[messageKey]) {
+      alertTitle.value = MESSAGES[messageKey];
+    } else {
+      alertTitle.value = '';
+    }
+  };
+
+  const setupCanvas = () => {
+    if (videoRef.current && canvasRef.current) {
+      const { videoWidth, videoHeight } = videoRef.current;
+
+      videoAspectRatio.value = videoWidth / videoHeight;
+
+      canvasRef.current.width = videoWidth;
+      canvasRef.current.height = videoHeight;
+
+      const container = videoRef.current.parentElement;
+      if (container) {
+        canvasRef.current.style.left = '50%';
+        canvasRef.current.style.top = '50%';
+        canvasRef.current.style.transform = 'translate(-50%, -50%) scaleX(-1)';
+      }
+    }
+  };
+
+  const updateCaptureAlerts = () => {
+    const isInNeutralZone = capturesTaken.value < neutralZone.value;
+    const isInSmileZone = capturesTaken.value >= smileCheckpoint.value;
+
+    if (isInNeutralZone && currentSmileScore.value >= smileThreshold) {
+      updateAlert('neutral-expression');
+    } else if (isInNeutralZone) {
+      alertTitle.value = 'Capturing...';
+    } else if (isInSmileZone) {
+      const timeSinceSmile = Date.now() - lastSmileTime.value;
+      if (timeSinceSmile > smileCooldown) {
+        if (
+          currentSmileScore.value >= smileThreshold &&
+          currentMouthOpen.value < mouthOpenThreshold
+        ) {
+          updateAlert('open-mouth-smile');
+        } else {
+          updateAlert('smile-required');
+        }
+      } else {
+        alertTitle.value = 'Keep smiling!';
+      }
+    } else {
+      updateAlert(null);
+    }
+  };
+
+  const updateAlerts = () => {
+    if (multipleFaces.value) {
+      updateAlert('multiple-faces');
+    } else if (!faceDetected.value) {
+      updateAlert('no-face');
+    } else if (faceProximity.value === 'too-close') {
+      updateAlert('too-close');
+    } else if (faceProximity.value === 'too-far') {
+      updateAlert('too-far');
+    } else if (!faceInBounds.value) {
+      updateAlert('out-of-bounds');
+    } else if (isCapturing.value) {
+      updateCaptureAlerts();
+    } else {
+      alertTitle.value = 'Ready to capture';
+    }
+  };
+
+  const stopDetectionLoop = () => {
+    if (animationFrameRef.current) {
+      cancelAnimationFrame(animationFrameRef.current);
+      animationFrameRef.current = null;
+    }
+  };
+
+  const detectFace = async () => {
+    if (!faceLandmarkerRef.current || !videoRef.current) {
+      stopDetectionLoop();
+      return;
+    }
+
+    try {
+      const croppedCanvas = createCroppedVideoFrame(videoRef.current);
+      const detectionSource = croppedCanvas || videoRef.current;
+
+      const results = faceLandmarkerRef.current.detectForVideo(
+        detectionSource,
+        performance.now(),
+      );
+
+      faceLandmarks.value = results.faceLandmarks || [];
+
+      if (results.faceLandmarks && canvasRef.current && videoRef.current) {
+        // we run detection on a cropped video frame
+        // adjust landmark coordinates back to full video space
+        if (croppedCanvas) {
+          const sourceWidth = videoRef.current.videoWidth;
+          const sourceHeight = videoRef.current.videoHeight;
+          const squareSize = Math.min(sourceWidth, sourceHeight);
+          const offsetX = (sourceWidth - squareSize) / (2 * sourceWidth);
+          const offsetY = (sourceHeight - squareSize) / (2 * sourceHeight);
+          const scaleFactor = squareSize / sourceWidth;
+          const scaleFactorY = squareSize / sourceHeight;
+
+          const adjustedLandmarks = results.faceLandmarks.map((face) =>
+            face.map((landmark: any) => ({
+              x: landmark.x * scaleFactor + offsetX,
+              y: landmark.y * scaleFactorY + offsetY,
+              z: landmark.z,
+            })),
+          );
+
+          drawFaceMesh(
+            canvasRef.current,
+            adjustedLandmarks,
+            capturesTaken.value,
+            smileCheckpoint.value,
+          );
+        } else {
+          drawFaceMesh(
+            canvasRef.current,
+            results.faceLandmarks,
+            capturesTaken.value,
+            smileCheckpoint.value,
+          );
+        }
+      } else if (canvasRef.current) {
+        clearCanvas(canvasRef.current);
+      }
+
+      // Check number of faces
+      const numFaces = results.faceLandmarks ? results.faceLandmarks.length : 0;
+      multipleFaces.value = numFaces > 1;
+
+      // Check if face is detected
+      const hasFace =
+        results.faceBlendshapes &&
+        results.faceBlendshapes.length > 0 &&
+        numFaces === 1;
+      faceDetected.value = hasFace;
+
+      if (hasFace && results.faceLandmarks) {
+        // Calculate face size and position
+        const faceSize = calculateFaceSize(results.faceLandmarks);
+        currentFaceSize.value = faceSize;
+
+        // Check face proximity
+        if (faceSize > maxFaceSize) {
+          faceProximity.value = 'too-close';
+        } else if (faceSize < minFaceSize) {
+          faceProximity.value = 'too-far';
+        } else {
+          faceProximity.value = 'good';
+        }
+
+        // Check face position
+        faceInBounds.value = isFaceInBounds(
+          results.faceLandmarks,
+          videoAspectRatio.value,
+        );
+
+        // Get smile and mouth open data
+        const blendshapes = results.faceBlendshapes[0].categories;
+        const smileLeft =
+          blendshapes.find((b) => b.categoryName === 'mouthSmileLeft')?.score ||
+          0;
+        const smileRight =
+          blendshapes.find((b) => b.categoryName === 'mouthSmileRight')
+            ?.score || 0;
+        const mouthOpen = calculateMouthOpening(results.faceLandmarks);
+        const smileScore = (smileLeft + smileRight) / 2;
+
+        currentSmileScore.value = smileScore;
+        currentMouthOpen.value = mouthOpen;
+
+        if (smileScore >= smileThreshold && mouthOpen >= mouthOpenThreshold) {
+          lastSmileTime.value = Date.now();
+
+          if (isPaused.value && isCapturing.value && resumeCaptureRef.current) {
+            // defer execution
+            setTimeout(() => {
+              const stillSmiling = Date.now() - lastSmileTime.value <= 100;
+              if (
+                stillSmiling &&
+                isPaused.value &&
+                isCapturing.value &&
+                resumeCaptureRef.current
+              ) {
+                resumeCaptureRef.current();
+              }
+            }, 0);
+          }
+        }
+      } else {
+        // No face detected or multiple faces - reset values
+        currentSmileScore.value = 0;
+        currentFaceSize.value = 0;
+        currentMouthOpen.value = 0;
+        faceInBounds.value = false;
+        faceProximity.value = 'good';
+      }
+
+      updateAlerts();
+    } catch (error) {
+      faceDetected.value = false;
+      faceInBounds.value = false;
+      multipleFaces.value = false;
+      faceProximity.value = 'good';
+      currentMouthOpen.value = 0;
+
+      if (isCapturing.value) {
+        updateAlert('no-face');
+      }
+    }
+
+    animationFrameRef.current = requestAnimationFrame(detectFace);
+  };
+
+  const startDetectionLoop = () => {
+    if (animationFrameRef.current) {
+      cancelAnimationFrame(animationFrameRef.current);
+    }
+    animationFrameRef.current = requestAnimationFrame(detectFace);
+  };
+
+  const captureImage = () => {
+    if (!videoRef.current) return;
+
+    const isReference = capturesTaken.value === totalCaptures.value - 1;
+    const imageData = captureImageFromVideo(videoRef.current, isReference);
+
+    if (!imageData) return;
+
+    if (isReference) {
+      referencePhoto.value = imageData;
+    } else {
+      capturedImages.value = [...capturedImages.value, imageData];
+    }
+
+    capturesTaken.value++;
+    countdown.value = totalCaptures.value - capturesTaken.value;
+  };
+
+  const stopCapture = () => {
+    if (captureTimerRef.current) {
+      clearInterval(captureTimerRef.current);
+      captureTimerRef.current = null;
+    }
+
+    isCapturing.value = false;
+    isPaused.value = false;
+
+    if (capturesTaken.value >= totalCaptures.value && referencePhoto.value) {
+      const livenessImages = capturedImages.value.map((img) => ({
+        image: img.split(',')[1],
+        image_type_id: ImageType.LIVENESS_IMAGE_BASE64,
+      }));
+
+      const referenceImage = {
+        image: referencePhoto.value.split(',')[1],
+        image_type_id: ImageType.SELFIE_IMAGE_BASE64,
+      };
+
+      const eventDetail = {
+        images: [...livenessImages, referenceImage],
+        referenceImage: referencePhoto.value,
+        previewImage: referencePhoto.value,
+        meta: { libraryVersion: COMPONENTS_VERSION },
+      };
+
+      window.dispatchEvent(
+        new CustomEvent('selfie-capture.publish', {
+          detail: eventDetail,
+        }),
+      );
+
+      const smartCameraWeb = document.querySelector('smart-camera-web');
+      smartCameraWeb?.dispatchEvent(
+        new CustomEvent('metadata.selfie-capture-end'),
+      );
+
+      hasFinishedCapture.value = true;
+    }
+  };
+
+  const pauseCapture = () => {
+    if (captureTimerRef.current) {
+      clearInterval(captureTimerRef.current);
+      captureTimerRef.current = null;
+    }
+    isPaused.value = true;
+
+    if (
+      !multipleFaces.value &&
+      faceDetected.value &&
+      faceInBounds.value &&
+      faceProximity.value === 'good'
+    ) {
+      updateAlert('smile-required');
+    }
+  };
+
+  const startCaptureInterval = () => {
+    if (captureTimerRef.current) {
+      clearInterval(captureTimerRef.current);
+    }
+
+    captureTimerRef.current = setInterval(() => {
+      if (capturesTaken.value >= totalCaptures.value) {
+        stopCapture();
+        return;
+      }
+
+      if (multipleFaces.value) {
+        pauseCapture();
+        return;
+      }
+
+      if (!faceDetected.value) {
+        return;
+      }
+
+      if (!faceInBounds.value) {
+        pauseCapture();
+        return;
+      }
+
+      if (faceProximity.value !== 'good') {
+        pauseCapture();
+        return;
+      }
+
+      const isInNeutralZone = capturesTaken.value < neutralZone.value;
+      const isInSmileZone = capturesTaken.value >= smileCheckpoint.value;
+
+      if (isInNeutralZone && currentSmileScore.value >= smileThreshold) {
+        return;
+      }
+
+      if (isInSmileZone) {
+        const timeSinceSmile = Date.now() - lastSmileTime.value;
+        if (timeSinceSmile > smileCooldown) {
+          pauseCapture();
+          return;
+        }
+      }
+
+      captureImage();
+    }, interval);
+  };
+
+  const resumeCapture = () => {
+    if (
+      faceDetected.value &&
+      faceProximity.value === 'good' &&
+      faceInBounds.value &&
+      !multipleFaces.value
+    ) {
+      const isInSmileZone = capturesTaken.value >= smileCheckpoint.value;
+      if (isInSmileZone) {
+        const timeSinceSmile = Date.now() - lastSmileTime.value;
+        if (timeSinceSmile > smileCooldown) {
+          return;
+        }
+      }
+
+      isPaused.value = false;
+      updateAlert(null);
+      startCaptureInterval();
+    }
+  };
+
+  resumeCaptureRef.current = resumeCapture;
+
+  const startCapture = async () => {
+    capturedImages.value = [];
+    isCapturing.value = true;
+    isPaused.value = false;
+    totalCaptures.value = Math.ceil(duration / interval);
+    capturesTaken.value = 0;
+    countdown.value = totalCaptures.value;
+
+    startCaptureInterval();
+  };
+
+  const handleCancel = () => {
+    stopCapture();
+    window.dispatchEvent(
+      new CustomEvent('selfie-capture.cancelled', {
+        detail: { meta: { libraryVersion: COMPONENTS_VERSION } },
+      }),
+    );
+
+    // TODO: remove - for backwards compatibility
+    window.dispatchEvent(
+      new CustomEvent('selfie-capture-screens.cancelled', {
+        detail: { meta: { libraryVersion: COMPONENTS_VERSION } },
+      }),
+    );
+  };
+
+  const handleClose = () => {
+    stopCapture();
+
+    window.dispatchEvent(
+      new CustomEvent('selfie-capture.close', {
+        detail: { meta: { libraryVersion: COMPONENTS_VERSION } },
+      }),
+    );
+
+    // TODO: remove - backwards compatibility
+    window.dispatchEvent(
+      new CustomEvent('selfie-capture-screens.close', {
+        detail: { meta: { libraryVersion: COMPONENTS_VERSION } },
+      }),
+    );
+  };
+
+  const cleanup = () => {
+    if (captureTimerRef.current) {
+      clearInterval(captureTimerRef.current);
+    }
+    stopDetectionLoop();
+  };
+
+  return {
+    faceDetected,
+    faceInBounds,
+    faceProximity,
+    multipleFaces,
+    videoAspectRatio,
+    faceLandmarks,
+    currentSmileScore,
+    currentFaceSize,
+    currentMouthOpen,
+    lastSmileTime,
+    alertTitle,
+    isReadyToCapture,
+
+    isCapturing,
+    isPaused,
+    countdown,
+    capturedImages,
+    referencePhoto,
+    totalCaptures,
+    capturesTaken,
+    hasFinishedCapture,
+    smileCheckpoint,
+    neutralZone,
+
+    initializeFaceLandmarker,
+    setupCanvas,
+    startDetectionLoop,
+    stopDetectionLoop,
+    updateAlert,
+    startCapture,
+    stopCapture,
+    pauseCapture,
+    resumeCapture,
+    handleCancel,
+    handleClose,
+    cleanup,
+  };
+};

package/lib/components/selfie/src/smartselfie-capture/index.ts

@@ -0,0 +1 @@
+export { default as SmartSelfieCapture } from './SmartSelfieCapture';

package/lib/components/selfie/src/smartselfie-capture/utils/alertMessages.ts

@@ -0,0 +1,12 @@
+export const MESSAGES = {
+  'multiple-faces': 'Ensure only one face is visible',
+  'no-face': 'Position your face in the oval',
+  'out-of-bounds': 'Position your face in the oval',
+  'too-close': 'Move farther away',
+  'too-far': 'Move closer',
+  'neutral-expression': 'Neutral expression',
+  'smile-required': 'Smile!',
+  'open-mouth-smile': 'Bigger smile!',
+};
+
+export type MessageKey = keyof typeof MESSAGES;

package/lib/components/selfie/src/smartselfie-capture/utils/canvas.ts

@@ -0,0 +1,105 @@
+import { DrawingUtils, FaceLandmarker } from '@mediapipe/tasks-vision';
+
+/**
+ * Create a cropped square canvas from video for face detection
+ */
+export const createCroppedVideoFrame = (
+  videoElement: HTMLVideoElement,
+): HTMLCanvasElement | null => {
+  const canvas = document.createElement('canvas');
+  const ctx = canvas.getContext('2d');
+  if (!ctx) return null;
+
+  const sourceWidth = videoElement.videoWidth;
+  const sourceHeight = videoElement.videoHeight;
+
+  const squareSize = Math.min(sourceWidth, sourceHeight);
+  const cropX = (sourceWidth - squareSize) / 2;
+  const cropY = (sourceHeight - squareSize) / 2;
+
+  canvas.width = squareSize;
+  canvas.height = squareSize;
+
+  ctx.drawImage(
+    videoElement,
+    cropX,
+    cropY,
+    squareSize,
+    squareSize,
+    0,
+    0,
+    squareSize,
+    squareSize,
+  );
+
+  return canvas;
+};
+
+/**
+ * Draw face mesh overlay on canvas
+ */
+export const drawFaceMesh = (
+  canvas: HTMLCanvasElement,
+  landmarks: any[],
+  capturesTaken: number,
+  smileCheckpoint: number,
+): void => {
+  const ctx = canvas.getContext('2d');
+  if (!ctx) return;
+
+  const canvasWidth = canvas.width;
+  const canvasHeight = canvas.height;
+
+  ctx.clearRect(0, 0, canvasWidth, canvasHeight);
+  const drawingUtils = new DrawingUtils(ctx);
+
+  // use this if scaling is needed
+  // const scaleFactor = Math.sqrt(canvasWidth * canvasHeight) / 500;
+
+  landmarks.forEach((landmark) => {
+    if (!landmark || landmark.length === 0) return;
+
+    const outlineColor = 'rgba(162, 155, 254,0.4)';
+    const lineWidth = 2; // Math.max(1, scaleFactor * 2);
+    ctx.strokeStyle = 'rgba(255, 255, 255, 0.6)';
+    ctx.lineWidth = lineWidth;
+    ctx.lineCap = 'round';
+    ctx.lineJoin = 'round';
+
+    drawingUtils.drawLandmarks(landmark, {
+      color: 'rgba(9, 132, 227,0.7)',
+      lineWidth: 0.5,
+      radius: 0.5,
+    });
+    drawingUtils.drawConnectors(
+      landmark,
+      FaceLandmarker.FACE_LANDMARKS_FACE_OVAL,
+      {
+        color: outlineColor,
+        lineWidth,
+      },
+    );
+
+    const isInSmileZone = capturesTaken > 0 && capturesTaken >= smileCheckpoint;
+    if (isInSmileZone) {
+      drawingUtils.drawConnectors(
+        landmark,
+        FaceLandmarker.FACE_LANDMARKS_LIPS,
+        {
+          color: outlineColor,
+          lineWidth,
+        },
+      );
+    }
+  });
+};
+
+/**
+ * Clear canvas completely
+ */
+export const clearCanvas = (canvas: HTMLCanvasElement): void => {
+  const ctx = canvas.getContext('2d');
+  if (ctx) {
+    ctx.clearRect(0, 0, canvas.width, canvas.height);
+  }
+};