@trustchex/react-native-sdk 1.355.1 → 1.357.0

package/src/Shared/Components/FaceCamera.tsx

@@ -1,46 +1,60 @@
 import { useIsFocused } from '@react-navigation/native';
 import { useKeepAwake } from '../Libs/native-keep-awake.utils';
-import React, { useEffect, useState, useMemo } from 'react';
+import { debugLog, debugWarn, debugError, logError } from '../Libs/debug.utils';
+import React, { useEffect, useState, useRef, useCallback } from 'react';
 import {
   StyleSheet,
   Text,
   View,
   Platform,
   Linking,
-  Dimensions,
   ActivityIndicator,
   NativeModules,
+  PermissionsAndroid,
+  type NativeSyntheticEvent,
 } from 'react-native';
 import {
-  useCameraDevice,
-  useCameraPermission,
-  Camera,
-  useFrameProcessor,
-  useCameraFormat,
-  runAtTargetFps,
-  useMicrophonePermission,
-} from 'react-native-vision-camera';
-import type { Frame } from 'react-native-vision-camera';
-import { runAsync } from '../Libs/worklet.utils';
-import {
-  type Face,
-  useFaceDetector,
-} from '../VisionCameraPlugins/FaceDetector';
-import { Worklets, useSharedValue } from 'react-native-worklets-core';
-import { crop } from '../VisionCameraPlugins/Cropper';
-import { isCircularRegionBright, isBlurry } from '../Libs/camera.utils';
+  TrustchexCamera,
+  type TrustchexCameraHandle,
+  type Frame,
+} from './TrustchexCamera';
 import { useTranslation } from 'react-i18next';
 import StyledButton from './StyledButton';
 import { useTheme } from '../Contexts/ThemeContext';
 import { SafeAreaView } from 'react-native-safe-area-context';
+import { Dimensions } from 'react-native';
+
+const { width: windowWidth, height: windowHeight } = Dimensions.get('window');
+const { OpenCVModule } = NativeModules;
+const MIN_BRIGHTNESS_THRESHOLD = 80;
+const BLUR_VARIANCE_THRESHOLD = 60;
+const REGION_BRIGHTNESS_THRESHOLD = 70;
+
+export type Face = {
+  bounds: {
+    x: number;
+    y: number;
+    width: number;
+    height: number;
+  };
+  yawAngle: number;
+  pitchAngle: number;
+  rollAngle: number;
+  trackingId?: number;
+  smilingProbability: number;
+  leftEyeOpenProbability: number;
+  rightEyeOpenProbability: number;
+};
 
 export type FaceCameraProps = {
   onFacesDetected: (
     faces: Face[],
     image: string,
-    isImageBright: boolean
+    isImageBright: boolean,
+    frameWidth: number,
+    frameHeight: number
   ) => void;
-  onCameraInitialized: (camera: Camera) => void;
+  onCameraInitialized: (camera: TrustchexCameraHandle) => void;
   previewRect?: {
     minX: number;
     minY: number;
@@ -49,12 +63,6 @@ export type FaceCameraProps = {
   };
 };
 
-const VIDEO_WIDTH = 1280;
-const VIDEO_HEIGHT = 720;
-
-const windowWidth = Dimensions.get('window').width;
-const windowHeight = Dimensions.get('window').height;
-
 const FaceCamera = ({
   onFacesDetected,
   onCameraInitialized,
@@ -62,65 +70,38 @@ const FaceCamera = ({
 }: FaceCameraProps) => {
   useKeepAwake();
   const theme = useTheme();
-  const cameraPermission = useCameraPermission();
-  const microphonePermission = useMicrophonePermission();
+  const [hasPermission, setHasPermission] = useState(false);
   const [permissionsRequested, setPermissionsRequested] = useState(false);
-  const cameraRef = React.useRef<Camera | null>(null);
+  const cameraRef = useRef<TrustchexCameraHandle>(null);
   const isFocused = useIsFocused();
   const [isActive, setIsActive] = useState(false);
-  const device = useCameraDevice('front');
-  const format = useCameraFormat(device, [
-    {
-      videoResolution: { width: VIDEO_WIDTH, height: VIDEO_HEIGHT },
-      iso: 'max',
-      photoHdr: false,
-      videoHdr: false,
-      fps: 30,
-    },
-  ]);
-  const isCameraInitialized = useSharedValue(false);
-
-  const faceDetectorOptions = useMemo(
-    () => ({
-      contourMode: 'none' as const,
-      landmarkMode: 'none' as const,
-      classificationMode: 'all' as const,
-      performanceMode: 'accurate' as const,
-      trackingEnabled: false,
-      autoScale: true,
-      windowWidth: windowWidth,
-      windowHeight: windowHeight,
-    }),
-    [windowWidth, windowHeight]
-  );
-
-  const { detectFaces } = useFaceDetector(faceDetectorOptions);
+  const isCameraInitializedRef = useRef(false);
+  const lastFrameTimeRef = useRef(0);
   const { t } = useTranslation();
 
   useEffect(() => {
     const requestPermissions = async () => {
-      if (!cameraPermission.hasPermission) {
-        await cameraPermission.requestPermission();
-      }
-      if (!microphonePermission.hasPermission) {
-        await microphonePermission.requestPermission();
+      if (Platform.OS === 'android') {
+        const granted = await PermissionsAndroid.request(
+          PermissionsAndroid.PERMISSIONS.CAMERA
+        );
+        setHasPermission(granted === PermissionsAndroid.RESULTS.GRANTED);
+      } else {
+        // iOS permissions handled by native camera
+        setHasPermission(true);
       }
       setPermissionsRequested(true);
     };
     requestPermissions();
-  }, [cameraPermission, microphonePermission]);
-
-  const handleFaces = Worklets.createRunOnJS(
-    (faces: Face[], image: string, isBright: boolean) => {
-      if (faces.length > 0) {
-        console.log('[FaceCamera] handleFaces called - faces:', faces.length, 'bright:', isBright);
-        onFacesDetected(faces, image, isBright);
-      }
-    }
-  );
+  }, []);
 
   useEffect(() => {
-    if (!!device && !!format && isFocused) {
+    debugLog('FaceCamera', 'isActive state changed:', {
+      isFocused,
+      hasPermission,
+      isActive: isFocused && hasPermission,
+    });
+    if (isFocused && hasPermission) {
       setIsActive(true);
     } else {
       setIsActive(false);
@@ -129,7 +110,7 @@ const FaceCamera = ({
     return () => {
       setIsActive(false);
     };
-  }, [device, format, isFocused]);
+  }, [isFocused, hasPermission]);
 
   // Set screen brightness to maximum when camera is active
   useEffect(() => {
@@ -142,9 +123,13 @@ const FaceCamera = ({
       if (isActive) {
         try {
           originalBrightness = await DeviceBrightness.getBrightness();
-          await DeviceBrightness.setBrightness(1.0);
+          await DeviceBrightness.setBrightness(1);
         } catch (error) {
-          console.log('Failed to set brightness:', error);
+          debugLog(
+            'FaceCamera',
+            '[FaceCamera] Failed to set brightness:',
+            error
+          );
         }
       }
     };
@@ -153,103 +138,178 @@ const FaceCamera = ({
 
     return () => {
       if (originalBrightness >= 0) {
-        DeviceBrightness.setBrightness(originalBrightness).catch(() => { });
+        DeviceBrightness.setBrightness(originalBrightness).catch(() => {});
       }
     };
   }, [isActive]);
 
-  const handleWorklet = (frame: Frame) => {
-    'worklet';
-    try {
-      // Calculate brightness based on the circular preview area if provided
-      // Otherwise fall back to entire frame brightness
-      let isBright = false;
-      if (previewRect) {
-        // Convert preview rect from screen coordinates to frame coordinates
-        const scaleX = frame.width / windowWidth;
-        const scaleY = frame.height / windowHeight;
-        const frameCircleRect = {
-          minX: Math.floor(previewRect.minX * scaleX),
-          minY: Math.floor(previewRect.minY * scaleY),
-          width: Math.floor(previewRect.width * scaleX),
-          height: Math.floor(previewRect.height * scaleY),
-        };
-        isBright = isCircularRegionBright(frame, frameCircleRect, 60);
-      } else {
-        // Fallback: check entire frame brightness (use legacy method)
-        const buffer = frame.toArrayBuffer();
-        const data = new Uint8Array(buffer);
-        const width = frame.width;
-        const height = frame.height;
-        let luminanceSum = 0;
-        let pixelCount = 0;
-        const centerX = Math.floor(width / 2);
-        const centerY = Math.floor(height / 2);
-        const halfSizeX = Math.floor(width / 2);
-        const halfSizeY = Math.floor(height / 2);
-
-        for (let y = centerY - halfSizeY; y < centerY + halfSizeY; y++) {
-          for (let x = centerX - halfSizeX; x < centerX + halfSizeX; x++) {
-            const index = y * width + x;
-            if (data[index] !== undefined) {
-              luminanceSum += data[index];
-              pixelCount++;
-            }
-          }
-        }
-        isBright = (luminanceSum / pixelCount) > 60;
-      }
+  const isProcessingFrame = useRef(false);
 
-      // Skip blurry frames to ensure sharp face captures
-      // Lower threshold (10) for better frame acceptance
-      if (isBlurry(frame, 10)) {
+  const handleFrame = useCallback(
+    async (event: NativeSyntheticEvent<{ frame: Frame }>) => {
+      debugLog('FaceCamera', '*** FRAME CALLBACK INVOKED ***', {
+        isProcessing: isProcessingFrame.current,
+        timestamp: Date.now(),
+      });
+
+      if (isProcessingFrame.current) {
+        debugLog('FaceCamera', 'Frame dropped - already processing');
         return;
       }
 
-      const image = crop(frame, {
-        cropRegion: {
-          top: 0,
-          left: 0,
-          width: 100,
-          height: 100,
-        },
-        includeImageBase64: true,
-        saveAsFile: false,
-      });
+      const now = Date.now();
+      const timeSinceLastFrame = now - lastFrameTimeRef.current;
+      // Throttle to ~6.7 FPS for face scanning (150ms interval matches Flutter SDK for responsive detection)
+      if (timeSinceLastFrame < 150) {
+        debugLog(
+          'FaceCamera',
+          `Frame dropped - throttling (${timeSinceLastFrame}ms < 150ms)`
+        );
+        return;
+      }
+      lastFrameTimeRef.current = now;
 
-      if (image && image.base64) {
-        const faces = detectFaces(frame);
-        if (faces.length > 0) {
-          console.log('[FaceCamera] Faces detected:', faces.length);
-        }
-        handleFaces(faces, image.base64, isBright);
+      if (!isCameraInitializedRef.current) {
+        debugLog('FaceCamera', 'Camera not initialized yet');
+        return;
       }
-    } catch (error) {
-      console.error('[FaceCamera] Face detection error:', error);
-    }
-  };
 
-  const frameProcessor = useFrameProcessor(
-    (frame) => {
-      'worklet';
-      if (!isCameraInitialized.value) {
+      if (!isActive) {
+        debugLog('FaceCamera', 'Camera not active');
         return;
       }
-      // iOS: Run at target 6 FPS using runAtTargetFps
-      if (Platform.OS === 'ios') {
-        runAtTargetFps(6, () => {
-          'worklet';
-          handleWorklet(frame);
-        });
-      } else {
-        // Android: Run async without throttling
-        runAsync(frame, () => {
-          'worklet';
-          handleWorklet(frame);
-        });
+
+      isProcessingFrame.current = true;
+      const { frame } = event.nativeEvent;
+
+      try {
+        if (
+          !frame.width ||
+          !frame.height ||
+          frame.width <= 0 ||
+          frame.height <= 0
+        ) {
+          debugLog('FaceCamera', 'Invalid frame dimensions');
+          return;
+        }
+
+        const base64Image = frame.base64Image ?? '';
+
+        // Read faces directly from native ML Kit results
+        const nativeFaces = frame.faces ?? [];
+        debugLog('FaceCamera', 'Native faces:', nativeFaces.length);
+
+        // Map NativeFace to FaceCamera's Face type first
+        // ML Kit classification probabilities may be undefined if detection fails
+        // Default to 0 so comparisons don't fail with undefined
+        const faces: Face[] = nativeFaces.map((f) => ({
+          bounds: {
+            x: f.bounds.x,
+            y: f.bounds.y,
+            width: f.bounds.width,
+            height: f.bounds.height,
+          },
+          yawAngle: f.yawAngle ?? 0,
+          pitchAngle: f.pitchAngle ?? 0,
+          rollAngle: f.rollAngle ?? 0,
+          smilingProbability: f.smilingProbability ?? 0,
+          leftEyeOpenProbability: f.leftEyeOpenProbability ?? 0,
+          rightEyeOpenProbability: f.rightEyeOpenProbability ?? 0,
+        }));
+
+        // Check overall brightness - computed natively on the Y plane
+        const avgBrightness = frame.brightness ?? 128;
+        const isBright = avgBrightness >= MIN_BRIGHTNESS_THRESHOLD;
+
+        // Check brightness in the region of interest (circular preview area)
+        let isRegionBright = isBright; // Default to overall brightness
+        if (previewRect && base64Image && OpenCVModule) {
+          try {
+            // Scale preview rect from screen coordinates to frame coordinates
+            const scaleX = frame.width / windowWidth;
+            const scaleY = frame.height / windowHeight;
+            const regionMinX = Math.round(previewRect.minX * scaleX);
+            const regionMinY = Math.round(previewRect.minY * scaleY);
+            const regionWidth = Math.round(previewRect.width * scaleX);
+            const regionHeight = Math.round(previewRect.height * scaleY);
+
+            isRegionBright = await OpenCVModule.isCircularRegionBright(
+              base64Image,
+              regionMinX,
+              regionMinY,
+              regionWidth,
+              regionHeight,
+              REGION_BRIGHTNESS_THRESHOLD
+            );
+          } catch (error) {
+            debugWarn('FaceCamera', 'Region brightness check failed:', error);
+          }
+        }
+
+        // Check for blur using OpenCV Laplacian variance
+        let isNotBlurry = true;
+        try {
+          if (base64Image && OpenCVModule) {
+            const isBlurryResult = await OpenCVModule.checkBlurry(
+              base64Image,
+              BLUR_VARIANCE_THRESHOLD
+            );
+            isNotBlurry = !isBlurryResult;
+          }
+        } catch (error) {
+          debugWarn('FaceCamera', 'Blur check failed:', error);
+          // If check fails, assume not blurry to continue
+        }
+
+        // Check quality but still pass detected faces
+        // Let liveness detection logic decide whether to accept poor quality frames
+        const hasGoodQuality = isRegionBright && isNotBlurry;
+        if (!hasGoodQuality) {
+          debugLog('FaceCamera', 'Frame quality insufficient:', {
+            avgBrightness,
+            isBright,
+            isRegionBright,
+            isNotBlurry,
+            faceCount: faces.length,
+          });
+        }
+
+        debugLog(
+          'FaceCamera',
+          `Calling onFacesDetected with ${faces.length} faces`
+        );
+        // Always call onFacesDetected with detected faces
+        // isRegionBright indicates whether the region of interest has acceptable brightness
+        // This allows the UI to show warnings while still detecting faces
+        onFacesDetected(
+          faces,
+          base64Image,
+          isRegionBright,
+          frame.width,
+          frame.height
+        );
+      } catch (error: any) {
+        debugError('FaceCamera', 'Face detection error:', error);
+      } finally {
+        isProcessingFrame.current = false;
       }
     },
-    [handleFaces, isCameraInitialized]
+    [isActive, previewRect, onFacesDetected]
+  );
+
+  const handleCameraReady = useCallback(() => {
+    debugLog('FaceCamera', 'Camera ready, initializing...');
+    isCameraInitializedRef.current = true;
+    if (cameraRef.current) {
+      onCameraInitialized(cameraRef.current);
+    }
+  }, [onCameraInitialized]);
+
+  const handleCameraError = useCallback(
+    (event: NativeSyntheticEvent<{ error: string }>) => {
+      logError('[FaceCamera] Camera error:', event.nativeEvent.error);
+    },
+    []
   );
 
   if (!permissionsRequested) {
@@ -260,8 +320,7 @@ const FaceCamera = ({
     );
   }
 
-  if (!cameraPermission.hasPermission) {
-    // Camera permission denied by user - their choice, not actionable
+  if (!hasPermission) {
     return (
       <SafeAreaView style={styles.permissionContainer}>
         <Text style={styles.permissionText}>
@@ -279,54 +338,28 @@ const FaceCamera = ({
     );
   }
 
-  if (!microphonePermission.hasPermission) {
-    // Microphone permission denied by user - their choice, not actionable
-    return (
-      <SafeAreaView style={styles.permissionContainer}>
-        <Text style={styles.permissionText}>
-          {t('general.noMicrophonePermissionGiven')}
-        </Text>
-        <StyledButton
-          mode="contained"
-          onPress={() => {
-            Linking.openSettings();
-          }}
-        >
-          {t('general.openSettings')}
-        </StyledButton>
-      </SafeAreaView>
-    );
-  }
-
-  if (device == null) {
-    // No camera device - device limitation, not actionable
-    return (
-      <SafeAreaView style={styles.permissionContainer}>
-        <Text style={styles.permissionText}>
-          {t('general.noCameraDetected')}
-        </Text>
-      </SafeAreaView>
-    );
-  }
+  debugLog('FaceCamera', 'Rendering TrustchexCamera with props:', {
+    isActive,
+    enableFrameProcessing: isActive,
+    enableFaceDetection: isActive,
+    includeBase64: true,
+    targetFps: 10,
+  });
 
   return (
     <View style={StyleSheet.absoluteFill}>
-      <Camera
+      <TrustchexCamera
         ref={cameraRef}
-        frameProcessor={frameProcessor}
         style={StyleSheet.absoluteFill}
-        device={device}
-        format={format}
-        isActive={isActive}
-        video={true}
-        photo={false}
-        audio={true}
-        videoBitRate={'extra-low'}
-        fps={Math.max(format?.minFps ?? 0, 30)}
-        onInitialized={() => {
-          isCameraInitialized.value = true;
-          onCameraInitialized(cameraRef.current!);
-        }}
+        cameraType="front"
+        torchEnabled={false}
+        enableFrameProcessing={isActive}
+        enableFaceDetection={isActive}
+        includeBase64={true} // Enabled to capture photos for liveness steps
+        targetFps={5}
+        onFrameAvailable={handleFrame}
+        onCameraReady={handleCameraReady}
+        onCameraError={handleCameraError}
       />
     </View>
   );