@arfuhad/react-native-smart-camera 0.1.2 → 0.1.5

package/src/hooks/useWebRTC.ts

@@ -0,0 +1,453 @@
+/**
+ * useWebRTC Hook
+ * 
+ * React hook for managing WebRTC video calling with peer connections,
+ * local/remote streams, and signaling integration.
+ * 
+ * This hook provides all the functionality needed for 1-to-1 video calls.
+ * Users are responsible for implementing their own signaling mechanism
+ * (WebSocket, Socket.io, etc.) to exchange SDP offers/answers and ICE candidates.
+ */
+
+import { useState, useCallback, useRef, useEffect } from 'react';
+import {
+  WebRTCManager,
+  isWebRTCAvailable,
+} from '../webrtc/WebRTCManager';
+import type {
+  UseWebRTCOptions,
+  UseWebRTCResult,
+  PeerConnectionConfig,
+  MediaConstraints,
+  SessionDescription,
+  ICECandidateEvent,
+  CallState,
+  WebRTCConnectionState,
+  ICEConnectionState,
+} from '../webrtc/types';
+
+/**
+ * Hook for WebRTC video calling
+ * 
+ * @param options - Configuration options
+ * @returns WebRTC controls and state
+ * 
+ * @example
+ * ```tsx
+ * import { useWebRTC } from '@arfuhad/react-native-smart-camera';
+ * import { RTCView } from 'react-native-webrtc';
+ * 
+ * function VideoCall() {
+ *   const {
+ *     localStream,
+ *     remoteStream,
+ *     callState,
+ *     startLocalStream,
+ *     createPeerConnection,
+ *     createOffer,
+ *     setRemoteDescription,
+ *     addIceCandidate,
+ *     onIceCandidate,
+ *     switchCamera,
+ *     toggleAudio,
+ *     toggleVideo,
+ *     cleanup,
+ *   } = useWebRTC({
+ *     config: {
+ *       iceServers: [{ urls: 'stun:stun.l.google.com:19302' }],
+ *     },
+ *     onRemoteStream: (stream) => console.log('Remote stream received'),
+ *   });
+ * 
+ *   // Start a call
+ *   const startCall = async () => {
+ *     await startLocalStream();
+ *     createPeerConnection();
+ *     const offer = await createOffer();
+ *     // Send offer via your signaling server
+ *     socket.emit('offer', offer);
+ *   };
+ * 
+ *   // Handle incoming call
+ *   useEffect(() => {
+ *     socket.on('offer', async (offer) => {
+ *       await startLocalStream();
+ *       createPeerConnection();
+ *       await setRemoteDescription(offer);
+ *       const answer = await createAnswer();
+ *       socket.emit('answer', answer);
+ *     });
+ * 
+ *     socket.on('ice-candidate', (candidate) => {
+ *       addIceCandidate(candidate);
+ *     });
+ * 
+ *     onIceCandidate((event) => {
+ *       if (event.candidate) {
+ *         socket.emit('ice-candidate', event.candidate);
+ *       }
+ *     });
+ *   }, []);
+ * 
+ *   return (
+ *     <View>
+ *       {remoteStream && <RTCView streamURL={remoteStream.toURL()} />}
+ *       {localStream && <RTCView streamURL={localStream.toURL()} />}
+ *       <Button onPress={switchCamera} title="Switch Camera" />
+ *       <Button onPress={toggleAudio} title="Mute/Unmute" />
+ *     </View>
+ *   );
+ * }
+ * ```
+ */
+export function useWebRTC(options: UseWebRTCOptions = {}): UseWebRTCResult {
+  const {
+    config,
+    mediaConstraints,
+    initialCamera = 'front',
+    autoStartLocalStream = false,
+    onCallStateChange,
+    onConnectionStateChange,
+    onIceConnectionStateChange,
+    onRemoteStream,
+    onRemoteStreamRemoved,
+    onError,
+  } = options;
+
+  // State
+  const [localStream, setLocalStream] = useState<MediaStream | null>(null);
+  const [remoteStream, setRemoteStream] = useState<MediaStream | null>(null);
+  const [callState, setCallState] = useState<CallState>('idle');
+  const [isAudioEnabled, setIsAudioEnabled] = useState(true);
+  const [isVideoEnabled, setIsVideoEnabled] = useState(true);
+  const [currentCamera, setCurrentCamera] = useState<'front' | 'back'>(initialCamera);
+  const [peerConnection, setPeerConnection] = useState<RTCPeerConnection | null>(null);
+
+  // Refs
+  const managerRef = useRef<WebRTCManager | null>(null);
+  const isMountedRef = useRef(true);
+  const iceCandidateCallbackRef = useRef<((event: ICECandidateEvent) => void) | null>(null);
+  const callbacksRef = useRef({
+    onCallStateChange,
+    onConnectionStateChange,
+    onIceConnectionStateChange,
+    onRemoteStream,
+    onRemoteStreamRemoved,
+    onError,
+  });
+
+  // Keep callbacks ref updated
+  useEffect(() => {
+    callbacksRef.current = {
+      onCallStateChange,
+      onConnectionStateChange,
+      onIceConnectionStateChange,
+      onRemoteStream,
+      onRemoteStreamRemoved,
+      onError,
+    };
+  }, [
+    onCallStateChange,
+    onConnectionStateChange,
+    onIceConnectionStateChange,
+    onRemoteStream,
+    onRemoteStreamRemoved,
+    onError,
+  ]);
+
+  // Initialize manager
+  useEffect(() => {
+    if (!isWebRTCAvailable()) {
+      console.warn('[useWebRTC] react-native-webrtc is not available');
+      return;
+    }
+
+    managerRef.current = new WebRTCManager(config);
+
+    return () => {
+      isMountedRef.current = false;
+      managerRef.current?.cleanup();
+    };
+  }, []);
+
+  // Auto-start local stream if configured
+  useEffect(() => {
+    if (autoStartLocalStream && isWebRTCAvailable()) {
+      startLocalStream();
+    }
+  }, [autoStartLocalStream]);
+
+  // Update call state and notify
+  const updateCallState = useCallback((newState: CallState) => {
+    if (isMountedRef.current) {
+      setCallState(newState);
+      callbacksRef.current.onCallStateChange?.(newState);
+    }
+  }, []);
+
+  // ==========================================================================
+  // Local Stream Controls
+  // ==========================================================================
+
+  /**
+   * Start local camera/microphone stream
+   */
+  const startLocalStream = useCallback(async (): Promise<void> => {
+    if (!managerRef.current) {
+      throw new Error('WebRTC not initialized');
+    }
+
+    try {
+      const constraints: MediaConstraints = {
+        video: {
+          width: { ideal: 1280 },
+          height: { ideal: 720 },
+          frameRate: { ideal: 30 },
+          facingMode: currentCamera === 'front' ? 'user' : 'environment',
+        },
+        audio: {
+          echoCancellation: true,
+          noiseSuppression: true,
+          autoGainControl: true,
+        },
+        ...mediaConstraints,
+      };
+
+      const stream = await managerRef.current.getLocalStream(constraints);
+      
+      if (isMountedRef.current) {
+        setLocalStream(stream);
+        setIsAudioEnabled(true);
+        setIsVideoEnabled(true);
+      }
+    } catch (error) {
+      callbacksRef.current.onError?.(error as Error);
+      throw error;
+    }
+  }, [currentCamera, mediaConstraints]);
+
+  /**
+   * Stop local stream
+   */
+  const stopLocalStream = useCallback((): void => {
+    managerRef.current?.stopLocalStream();
+    if (isMountedRef.current) {
+      setLocalStream(null);
+    }
+  }, []);
+
+  /**
+   * Switch between front and back camera
+   */
+  const switchCamera = useCallback(async (): Promise<void> => {
+    try {
+      await managerRef.current?.switchCamera();
+      if (isMountedRef.current) {
+        setCurrentCamera((prev) => (prev === 'front' ? 'back' : 'front'));
+      }
+    } catch (error) {
+      callbacksRef.current.onError?.(error as Error);
+      throw error;
+    }
+  }, []);
+
+  /**
+   * Toggle local audio on/off
+   */
+  const toggleAudio = useCallback((): void => {
+    const newState = !isAudioEnabled;
+    managerRef.current?.toggleAudio(newState);
+    if (isMountedRef.current) {
+      setIsAudioEnabled(newState);
+    }
+  }, [isAudioEnabled]);
+
+  /**
+   * Toggle local video on/off
+   */
+  const toggleVideo = useCallback((): void => {
+    const newState = !isVideoEnabled;
+    managerRef.current?.toggleVideo(newState);
+    if (isMountedRef.current) {
+      setIsVideoEnabled(newState);
+    }
+  }, [isVideoEnabled]);
+
+  // ==========================================================================
+  // Peer Connection Management
+  // ==========================================================================
+
+  /**
+   * Create a new peer connection
+   */
+  const createPeerConnection = useCallback((pcConfig?: PeerConnectionConfig): RTCPeerConnection => {
+    if (!managerRef.current) {
+      throw new Error('WebRTC not initialized');
+    }
+
+    const pc = managerRef.current.createPeerConnection(pcConfig || config);
+
+    // Setup callbacks
+    managerRef.current.onRemoteStream((stream) => {
+      if (isMountedRef.current) {
+        setRemoteStream(stream);
+        updateCallState('connected');
+        callbacksRef.current.onRemoteStream?.(stream);
+      }
+    });
+
+    managerRef.current.onConnectionStateChange((state) => {
+      callbacksRef.current.onConnectionStateChange?.(state);
+      
+      // Update call state based on connection state
+      if (state === 'connected') {
+        updateCallState('connected');
+      } else if (state === 'disconnected' || state === 'failed') {
+        updateCallState('disconnected');
+      } else if (state === 'connecting') {
+        updateCallState('connecting');
+      }
+    });
+
+    managerRef.current.onIceConnectionStateChange((state) => {
+      callbacksRef.current.onIceConnectionStateChange?.(state);
+    });
+
+    managerRef.current.onIceCandidate((event) => {
+      iceCandidateCallbackRef.current?.(event);
+    });
+
+    // Add local stream to peer connection if available
+    const stream = managerRef.current.getLocalStreamInstance();
+    if (stream) {
+      managerRef.current.addLocalStream(stream);
+    }
+
+    if (isMountedRef.current) {
+      setPeerConnection(pc);
+      updateCallState('connecting');
+    }
+
+    return pc;
+  }, [config, updateCallState]);
+
+  /**
+   * Close the peer connection
+   */
+  const closePeerConnection = useCallback((): void => {
+    managerRef.current?.closePeerConnection();
+    if (isMountedRef.current) {
+      setPeerConnection(null);
+      setRemoteStream(null);
+      updateCallState('idle');
+      callbacksRef.current.onRemoteStreamRemoved?.();
+    }
+  }, [updateCallState]);
+
+  // ==========================================================================
+  // Signaling Helpers
+  // ==========================================================================
+
+  /**
+   * Create an SDP offer
+   */
+  const createOffer = useCallback(async (): Promise<SessionDescription> => {
+    if (!managerRef.current) {
+      throw new Error('WebRTC not initialized');
+    }
+    return managerRef.current.createOffer();
+  }, []);
+
+  /**
+   * Create an SDP answer
+   */
+  const createAnswer = useCallback(async (): Promise<SessionDescription> => {
+    if (!managerRef.current) {
+      throw new Error('WebRTC not initialized');
+    }
+    return managerRef.current.createAnswer();
+  }, []);
+
+  /**
+   * Set the remote SDP description
+   */
+  const setRemoteDescription = useCallback(async (description: SessionDescription): Promise<void> => {
+    if (!managerRef.current) {
+      throw new Error('WebRTC not initialized');
+    }
+    await managerRef.current.setRemoteDescription(description);
+  }, []);
+
+  /**
+   * Add an ICE candidate from remote peer
+   */
+  const addIceCandidate = useCallback(async (candidate: RTCIceCandidate): Promise<void> => {
+    if (!managerRef.current) {
+      throw new Error('WebRTC not initialized');
+    }
+    await managerRef.current.addIceCandidate(candidate);
+  }, []);
+
+  /**
+   * Register callback for ICE candidates
+   */
+  const onIceCandidate = useCallback((callback: (event: ICECandidateEvent) => void): void => {
+    iceCandidateCallbackRef.current = callback;
+  }, []);
+
+  // ==========================================================================
+  // Cleanup
+  // ==========================================================================
+
+  /**
+   * Clean up all resources
+   */
+  const cleanup = useCallback((): void => {
+    managerRef.current?.cleanup();
+    if (isMountedRef.current) {
+      setLocalStream(null);
+      setRemoteStream(null);
+      setPeerConnection(null);
+      setCallState('idle');
+      setIsAudioEnabled(true);
+      setIsVideoEnabled(true);
+    }
+  }, []);
+
+  return {
+    // Streams
+    localStream,
+    remoteStream,
+    
+    // State
+    callState,
+    isAudioEnabled,
+    isVideoEnabled,
+    currentCamera,
+    peerConnection,
+    
+    // Local stream controls
+    startLocalStream,
+    stopLocalStream,
+    switchCamera,
+    toggleAudio,
+    toggleVideo,
+    
+    // Peer connection management
+    createPeerConnection,
+    closePeerConnection,
+    
+    // Signaling helpers
+    createOffer,
+    createAnswer,
+    setRemoteDescription,
+    addIceCandidate,
+    onIceCandidate,
+    
+    // Cleanup
+    cleanup,
+  };
+}
+
+export type { UseWebRTCOptions, UseWebRTCResult };
+

package/src/hooks/useWebRTCWithDetection.ts

@@ -0,0 +1,181 @@
+/**
+ * useWebRTCWithDetection Hook
+ * 
+ * Combined hook that provides WebRTC video calling functionality
+ * alongside face detection and eye tracking capabilities.
+ * 
+ * This hook allows you to:
+ * - Make video calls using WebRTC
+ * - Detect faces in the local camera feed using VisionCamera
+ * - Track eye status for blink detection
+ */
+
+import { useState, useCallback, useRef, useEffect } from 'react';
+import { useRunOnJS } from 'react-native-worklets-core';
+import type { Frame } from 'react-native-vision-camera';
+import { useWebRTC } from './useWebRTC';
+import { useFaceDetector } from './useFaceDetector';
+import { useBlinkDetection } from './useBlinkDetection';
+import type { Face, EyeStatusResult } from '../types';
+import type {
+  UseWebRTCOptions,
+  UseWebRTCResult,
+  UseWebRTCWithDetectionOptions,
+} from '../webrtc/types';
+
+// Re-export the options type
+export type { UseWebRTCWithDetectionOptions } from '../webrtc/types';
+
+/**
+ * Result type for useWebRTCWithDetection hook
+ */
+export interface UseWebRTCWithDetectionResult extends UseWebRTCResult {
+  // Face detection
+  /** Currently detected faces */
+  faces: Face[];
+  /** Detect faces in a frame (for use in VisionCamera frame processor) */
+  detectFaces: (frame: Frame) => Face[];
+  
+  // Eye tracking
+  /** Current eye status */
+  eyeStatus: EyeStatusResult | null;
+  /** Process faces for eye tracking */
+  processEyeStatus: (faces: Face[]) => void;
+  /** Reset eye status */
+  resetEyeStatus: () => void;
+  
+  // Combined handler for frame processor
+  /** Process a frame for both face detection and eye tracking */
+  processFrame: (frame: Frame) => void;
+}
+
+/**
+ * Hook combining WebRTC video calling with face detection and eye tracking
+ * 
+ * @param options - Configuration options
+ * @returns Combined WebRTC, face detection, and eye tracking controls
+ * 
+ * @example
+ * ```tsx
+ * import { useWebRTCWithDetection } from '@arfuhad/react-native-smart-camera';
+ * import { Camera, useFrameProcessor } from 'react-native-vision-camera';
+ * import { RTCView } from 'react-native-webrtc';
+ * 
+ * function VideoCallWithDetection() {
+ *   const {
+ *     // WebRTC
+ *     localStream,
+ *     remoteStream,
+ *     callState,
+ *     startLocalStream,
+ *     createPeerConnection,
+ *     // Face detection
+ *     faces,
+ *     detectFaces,
+ *     // Eye tracking
+ *     eyeStatus,
+ *     // Combined handler
+ *     processFrame,
+ *   } = useWebRTCWithDetection({
+ *     config: { iceServers: [{ urls: 'stun:stun.l.google.com:19302' }] },
+ *     faceDetection: { enabled: true, classificationMode: 'all' },
+ *     eyeTracking: { enabled: true, eyeClosedThreshold: 0.3 },
+ *   });
+ * 
+ *   // Use VisionCamera for local preview + detection
+ *   // Use RTCView for remote stream
+ *   const frameProcessor = useFrameProcessor((frame) => {
+ *     'worklet';
+ *     processFrame(frame);
+ *   }, [processFrame]);
+ * 
+ *   return (
+ *     <View>
+ *       <Camera
+ *         device={device}
+ *         isActive={true}
+ *         frameProcessor={frameProcessor}
+ *       />
+ *       {remoteStream && <RTCView streamURL={remoteStream.toURL()} />}
+ *       <Text>Faces: {faces.length}</Text>
+ *       <Text>Left Eye: {eyeStatus?.leftEye.openProbability.toFixed(2)}</Text>
+ *     </View>
+ *   );
+ * }
+ * ```
+ */
+export function useWebRTCWithDetection(
+  options: UseWebRTCWithDetectionOptions = {}
+): UseWebRTCWithDetectionResult {
+  const {
+    faceDetection = {},
+    eyeTracking = {},
+    ...webrtcOptions
+  } = options;
+
+  // WebRTC hook
+  const webrtc = useWebRTC(webrtcOptions);
+
+  // Face detection state
+  const [faces, setFaces] = useState<Face[]>([]);
+
+  // Face detector hook
+  const { detectFaces } = useFaceDetector({
+    performanceMode: faceDetection.performanceMode || 'fast',
+    landmarkMode: faceDetection.landmarkMode || 'none',
+    classificationMode: faceDetection.classificationMode || 'all', // Needed for eye tracking
+    trackingEnabled: true,
+  });
+
+  // Blink detection hook
+  const {
+    eyeStatus,
+    processEyeStatus,
+    reset: resetEyeStatus,
+  } = useBlinkDetection({
+    enabled: eyeTracking.enabled !== false,
+    eyeClosedThreshold: eyeTracking.eyeClosedThreshold ?? 0.5,
+  });
+
+  // Handle detected faces on JS thread
+  const handleFaces = useRunOnJS((detectedFaces: Face[]) => {
+    setFaces(detectedFaces);
+    
+    // Process for eye tracking if enabled
+    if (eyeTracking.enabled !== false) {
+      processEyeStatus(detectedFaces);
+    }
+  }, [processEyeStatus, eyeTracking.enabled]);
+
+  // Combined frame processor function
+  const processFrame = useCallback((frame: Frame) => {
+    'worklet';
+    
+    if (faceDetection.enabled === false) {
+      return;
+    }
+    
+    const detectedFaces = detectFaces(frame);
+    handleFaces(detectedFaces);
+  }, [detectFaces, handleFaces, faceDetection.enabled]);
+
+  return {
+    // WebRTC
+    ...webrtc,
+    
+    // Face detection
+    faces,
+    detectFaces,
+    
+    // Eye tracking
+    eyeStatus,
+    processEyeStatus,
+    resetEyeStatus,
+    
+    // Combined handler
+    processFrame,
+  };
+}
+
+// Types are exported from the interface definition above
+