@arfuhad/react-native-smart-camera 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +341 -0
- package/README.md +154 -0
- package/android/build.gradle +89 -0
- package/android/src/main/AndroidManifest.xml +2 -0
- package/android/src/main/java/expo/modules/smartcamera/ImageLoader.kt +106 -0
- package/android/src/main/java/expo/modules/smartcamera/MLKitFaceDetector.kt +273 -0
- package/android/src/main/java/expo/modules/smartcamera/SmartCameraModule.kt +205 -0
- package/android/src/main/java/expo/modules/smartcamera/SmartCameraView.kt +153 -0
- package/android/src/main/java/expo/modules/smartcamera/WebRTCFrameBridge.kt +184 -0
- package/app.plugin.js +17 -0
- package/build/SmartCamera.d.ts +17 -0
- package/build/SmartCamera.d.ts.map +1 -0
- package/build/SmartCamera.js +270 -0
- package/build/SmartCamera.js.map +1 -0
- package/build/SmartCameraModule.d.ts +112 -0
- package/build/SmartCameraModule.d.ts.map +1 -0
- package/build/SmartCameraModule.js +121 -0
- package/build/SmartCameraModule.js.map +1 -0
- package/build/SmartCameraView.d.ts +8 -0
- package/build/SmartCameraView.d.ts.map +1 -0
- package/build/SmartCameraView.js +7 -0
- package/build/SmartCameraView.js.map +1 -0
- package/build/detection/blinkProcessor.d.ts +23 -0
- package/build/detection/blinkProcessor.d.ts.map +1 -0
- package/build/detection/blinkProcessor.js +90 -0
- package/build/detection/blinkProcessor.js.map +1 -0
- package/build/detection/faceDetector.d.ts +16 -0
- package/build/detection/faceDetector.d.ts.map +1 -0
- package/build/detection/faceDetector.js +46 -0
- package/build/detection/faceDetector.js.map +1 -0
- package/build/detection/index.d.ts +4 -0
- package/build/detection/index.d.ts.map +1 -0
- package/build/detection/index.js +4 -0
- package/build/detection/index.js.map +1 -0
- package/build/detection/staticImageDetector.d.ts +25 -0
- package/build/detection/staticImageDetector.d.ts.map +1 -0
- package/build/detection/staticImageDetector.js +48 -0
- package/build/detection/staticImageDetector.js.map +1 -0
- package/build/hooks/index.d.ts +5 -0
- package/build/hooks/index.d.ts.map +1 -0
- package/build/hooks/index.js +5 -0
- package/build/hooks/index.js.map +1 -0
- package/build/hooks/useBlinkDetection.d.ts +39 -0
- package/build/hooks/useBlinkDetection.d.ts.map +1 -0
- package/build/hooks/useBlinkDetection.js +67 -0
- package/build/hooks/useBlinkDetection.js.map +1 -0
- package/build/hooks/useFaceDetection.d.ts +46 -0
- package/build/hooks/useFaceDetection.d.ts.map +1 -0
- package/build/hooks/useFaceDetection.js +80 -0
- package/build/hooks/useFaceDetection.js.map +1 -0
- package/build/hooks/useSmartCamera.d.ts +31 -0
- package/build/hooks/useSmartCamera.d.ts.map +1 -0
- package/build/hooks/useSmartCamera.js +75 -0
- package/build/hooks/useSmartCamera.js.map +1 -0
- package/build/hooks/useSmartCameraWebRTC.d.ts +58 -0
- package/build/hooks/useSmartCameraWebRTC.d.ts.map +1 -0
- package/build/hooks/useSmartCameraWebRTC.js +160 -0
- package/build/hooks/useSmartCameraWebRTC.js.map +1 -0
- package/build/index.d.ts +14 -0
- package/build/index.d.ts.map +1 -0
- package/build/index.js +20 -0
- package/build/index.js.map +1 -0
- package/build/types.d.ts +478 -0
- package/build/types.d.ts.map +1 -0
- package/build/types.js +2 -0
- package/build/types.js.map +1 -0
- package/build/utils/index.d.ts +98 -0
- package/build/utils/index.d.ts.map +1 -0
- package/build/utils/index.js +276 -0
- package/build/utils/index.js.map +1 -0
- package/build/webrtc/WebRTCBridge.d.ts +55 -0
- package/build/webrtc/WebRTCBridge.d.ts.map +1 -0
- package/build/webrtc/WebRTCBridge.js +113 -0
- package/build/webrtc/WebRTCBridge.js.map +1 -0
- package/build/webrtc/index.d.ts +3 -0
- package/build/webrtc/index.d.ts.map +1 -0
- package/build/webrtc/index.js +2 -0
- package/build/webrtc/index.js.map +1 -0
- package/build/webrtc/types.d.ts +64 -0
- package/build/webrtc/types.d.ts.map +1 -0
- package/build/webrtc/types.js +5 -0
- package/build/webrtc/types.js.map +1 -0
- package/expo-module.config.json +9 -0
- package/ios/MLKitFaceDetector.swift +310 -0
- package/ios/SmartCamera.podspec +33 -0
- package/ios/SmartCameraModule.swift +225 -0
- package/ios/SmartCameraView.swift +146 -0
- package/ios/WebRTCFrameBridge.swift +150 -0
- package/package.json +91 -0
- package/plugin/build/index.d.ts +28 -0
- package/plugin/build/index.js +33 -0
- package/plugin/build/withSmartCameraAndroid.d.ts +9 -0
- package/plugin/build/withSmartCameraAndroid.js +108 -0
- package/plugin/build/withSmartCameraIOS.d.ts +11 -0
- package/plugin/build/withSmartCameraIOS.js +92 -0
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
package expo.modules.smartcamera
|
|
2
|
+
|
|
3
|
+
import android.graphics.Bitmap
|
|
4
|
+
import android.os.Handler
|
|
5
|
+
import android.os.HandlerThread
|
|
6
|
+
import android.util.Log
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* WebRTC Frame Bridge for Android
|
|
10
|
+
*
|
|
11
|
+
* Note: This is a placeholder implementation.
|
|
12
|
+
* In production, you would integrate with WebRTC's VideoSource and VideoCapturer.
|
|
13
|
+
*/
|
|
14
|
+
class WebRTCFrameBridge {
|
|
15
|
+
// MARK: - Properties
|
|
16
|
+
|
|
17
|
+
private var isInitialized = false
|
|
18
|
+
private var isStreaming = false
|
|
19
|
+
|
|
20
|
+
private var streamWidth: Int = 1280
|
|
21
|
+
private var streamHeight: Int = 720
|
|
22
|
+
private var streamFrameRate: Int = 30
|
|
23
|
+
|
|
24
|
+
private var frameThread: HandlerThread? = null
|
|
25
|
+
private var frameHandler: Handler? = null
|
|
26
|
+
|
|
27
|
+
private var lastFrameTime: Long = 0
|
|
28
|
+
private var frameInterval: Long = 33 // ~30fps
|
|
29
|
+
|
|
30
|
+
companion object {
|
|
31
|
+
private const val TAG = "WebRTCFrameBridge"
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
// MARK: - Initialization
|
|
35
|
+
|
|
36
|
+
fun initialize() {
|
|
37
|
+
if (isInitialized) {
|
|
38
|
+
return
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
// Initialize WebRTC
|
|
42
|
+
// In production: PeerConnectionFactory.initialize(...)
|
|
43
|
+
|
|
44
|
+
frameThread = HandlerThread("WebRTCFrameThread").apply {
|
|
45
|
+
start()
|
|
46
|
+
}
|
|
47
|
+
frameHandler = Handler(frameThread!!.looper)
|
|
48
|
+
|
|
49
|
+
isInitialized = true
|
|
50
|
+
Log.d(TAG, "Initialized")
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
// MARK: - Streaming
|
|
54
|
+
|
|
55
|
+
fun startStream(width: Int, height: Int, frameRate: Int) {
|
|
56
|
+
if (!isInitialized) {
|
|
57
|
+
throw IllegalStateException("WebRTC not initialized")
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
if (isStreaming) {
|
|
61
|
+
Log.w(TAG, "Already streaming, stopping first")
|
|
62
|
+
stopStream()
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
streamWidth = width
|
|
66
|
+
streamHeight = height
|
|
67
|
+
streamFrameRate = frameRate
|
|
68
|
+
frameInterval = (1000 / frameRate).toLong()
|
|
69
|
+
|
|
70
|
+
// In production:
|
|
71
|
+
// 1. Create VideoSource
|
|
72
|
+
// 2. Create VideoTrack
|
|
73
|
+
// 3. Add track to PeerConnection
|
|
74
|
+
|
|
75
|
+
isStreaming = true
|
|
76
|
+
Log.d(TAG, "Started streaming: ${width}x${height}@${frameRate}fps")
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
fun stopStream() {
|
|
80
|
+
if (!isStreaming) {
|
|
81
|
+
return
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
// In production:
|
|
85
|
+
// 1. Remove track from PeerConnection
|
|
86
|
+
// 2. Dispose VideoTrack
|
|
87
|
+
// 3. Dispose VideoSource
|
|
88
|
+
|
|
89
|
+
isStreaming = false
|
|
90
|
+
Log.d(TAG, "Stopped streaming")
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
// MARK: - Frame Processing
|
|
94
|
+
|
|
95
|
+
fun pushFrame(frameData: Map<String, Any>) {
|
|
96
|
+
if (!isStreaming) {
|
|
97
|
+
return
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
// Rate limiting
|
|
101
|
+
val currentTime = System.currentTimeMillis()
|
|
102
|
+
if (currentTime - lastFrameTime < frameInterval) {
|
|
103
|
+
return
|
|
104
|
+
}
|
|
105
|
+
lastFrameTime = currentTime
|
|
106
|
+
|
|
107
|
+
frameHandler?.post {
|
|
108
|
+
processFrame(frameData)
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
fun pushBitmap(bitmap: Bitmap, timestampNs: Long) {
|
|
113
|
+
if (!isStreaming) {
|
|
114
|
+
return
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
// Rate limiting
|
|
118
|
+
val currentTime = System.currentTimeMillis()
|
|
119
|
+
if (currentTime - lastFrameTime < frameInterval) {
|
|
120
|
+
return
|
|
121
|
+
}
|
|
122
|
+
lastFrameTime = currentTime
|
|
123
|
+
|
|
124
|
+
frameHandler?.post {
|
|
125
|
+
processBitmap(bitmap, timestampNs)
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
private fun processFrame(frameData: Map<String, Any>) {
|
|
130
|
+
// In production:
|
|
131
|
+
// 1. Convert frame data to VideoFrame
|
|
132
|
+
// 2. Push to VideoSource
|
|
133
|
+
|
|
134
|
+
// Placeholder - log frame info
|
|
135
|
+
val width = frameData["width"]
|
|
136
|
+
val height = frameData["height"]
|
|
137
|
+
// Log.d(TAG, "Processing frame: ${width}x${height}")
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
private fun processBitmap(bitmap: Bitmap, timestampNs: Long) {
|
|
141
|
+
// In production:
|
|
142
|
+
// 1. Convert Bitmap to I420 buffer
|
|
143
|
+
// 2. Create VideoFrame
|
|
144
|
+
// 3. Push to VideoSource via CapturerObserver
|
|
145
|
+
|
|
146
|
+
// Example conversion (pseudo-code):
|
|
147
|
+
/*
|
|
148
|
+
val i420Buffer = JavaI420Buffer.allocate(bitmap.width, bitmap.height)
|
|
149
|
+
// ... convert bitmap pixels to I420 ...
|
|
150
|
+
|
|
151
|
+
val rotation = 0
|
|
152
|
+
val frame = VideoFrame(i420Buffer, rotation, timestampNs)
|
|
153
|
+
|
|
154
|
+
capturerObserver?.onFrameCaptured(frame)
|
|
155
|
+
frame.release()
|
|
156
|
+
*/
|
|
157
|
+
|
|
158
|
+
// Log.d(TAG, "Processing bitmap: ${bitmap.width}x${bitmap.height}")
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
// MARK: - Utilities
|
|
162
|
+
|
|
163
|
+
fun getCurrentStreamConfig(): Triple<Int, Int, Int>? {
|
|
164
|
+
return if (isStreaming) {
|
|
165
|
+
Triple(streamWidth, streamHeight, streamFrameRate)
|
|
166
|
+
} else {
|
|
167
|
+
null
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
fun isStreaming(): Boolean = isStreaming
|
|
172
|
+
|
|
173
|
+
// MARK: - Cleanup
|
|
174
|
+
|
|
175
|
+
fun destroy() {
|
|
176
|
+
stopStream()
|
|
177
|
+
frameThread?.quitSafely()
|
|
178
|
+
frameThread = null
|
|
179
|
+
frameHandler = null
|
|
180
|
+
isInitialized = false
|
|
181
|
+
Log.d(TAG, "Destroyed")
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
|
package/app.plugin.js
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
// This file is the entry point for the Expo config plugin
|
|
2
|
+
// It loads the compiled TypeScript plugin from plugin/build
|
|
3
|
+
|
|
4
|
+
let plugin;
|
|
5
|
+
|
|
6
|
+
try {
|
|
7
|
+
// Try to load the compiled plugin
|
|
8
|
+
plugin = require('./plugin/build');
|
|
9
|
+
} catch (e) {
|
|
10
|
+
// If the plugin hasn't been built yet, throw a helpful error
|
|
11
|
+
throw new Error(
|
|
12
|
+
'The react-native-smart-camera plugin has not been built yet. ' +
|
|
13
|
+
'Please run `npm run build` in the plugin directory first.'
|
|
14
|
+
);
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
module.exports = plugin.default ?? plugin;
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import React from 'react';
|
|
2
|
+
import type { SmartCameraProps } from './types';
|
|
3
|
+
/**
|
|
4
|
+
* SmartCamera component - A camera component with face detection,
|
|
5
|
+
* blink detection, and WebRTC streaming capabilities.
|
|
6
|
+
*
|
|
7
|
+
* Features:
|
|
8
|
+
* - VisionCamera integration with frame processing
|
|
9
|
+
* - Real-time face detection with ML Kit
|
|
10
|
+
* - Blink detection with debouncing
|
|
11
|
+
* - App lifecycle management (background/foreground)
|
|
12
|
+
* - Orientation handling
|
|
13
|
+
* - WebRTC streaming support
|
|
14
|
+
*/
|
|
15
|
+
export declare function SmartCamera({ camera, fps, style, faceDetection, blinkDetection, onBlinkDetected, onFaceDetected, webrtc, isActive, onReady, onError, }: SmartCameraProps): React.JSX.Element;
|
|
16
|
+
export default SmartCamera;
|
|
17
|
+
//# sourceMappingURL=SmartCamera.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"SmartCamera.d.ts","sourceRoot":"","sources":["../src/SmartCamera.tsx"],"names":[],"mappings":"AAAA,OAAO,KAA4D,MAAM,OAAO,CAAC;AAWjF,OAAO,KAAK,EACV,gBAAgB,EAKjB,MAAM,SAAS,CAAC;AA+BjB;;;;;;;;;;;GAWG;AACH,wBAAgB,WAAW,CAAC,EAC1B,MAAgB,EAChB,GAAQ,EACR,KAAK,EACL,aAAa,EACb,cAAsB,EACtB,eAAe,EACf,cAAc,EACd,MAAM,EACN,QAAe,EACf,OAAO,EACP,OAAO,GACR,EAAE,gBAAgB,qBA2QlB;AAaD,eAAe,WAAW,CAAC"}
|
|
@@ -0,0 +1,270 @@
|
|
|
1
|
+
import React, { useCallback, useEffect, useRef, useMemo, useState } from 'react';
|
|
2
|
+
import { StyleSheet, View, AppState, useWindowDimensions } from 'react-native';
|
|
3
|
+
import { Camera, useCameraDevice, useCameraPermission, useFrameProcessor, } from 'react-native-vision-camera';
|
|
4
|
+
import { useSharedValue, useRunOnJS } from 'react-native-worklets-core';
|
|
5
|
+
import { detectFaces } from './detection/faceDetector';
|
|
6
|
+
// Default values for face detection options
|
|
7
|
+
const DEFAULT_FACE_DETECTION_OPTIONS = {
|
|
8
|
+
performanceMode: 'fast',
|
|
9
|
+
landmarkMode: 'none',
|
|
10
|
+
contourMode: 'none',
|
|
11
|
+
classificationMode: 'none',
|
|
12
|
+
minFaceSize: 0.15,
|
|
13
|
+
trackingEnabled: false,
|
|
14
|
+
cameraFacing: 'front',
|
|
15
|
+
autoMode: false,
|
|
16
|
+
windowWidth: 1.0,
|
|
17
|
+
windowHeight: 1.0,
|
|
18
|
+
};
|
|
19
|
+
// Minimum time between blink detections (ms)
|
|
20
|
+
const BLINK_DEBOUNCE_MS = 300;
|
|
21
|
+
// Eye open probability threshold for blink detection
|
|
22
|
+
const EYE_CLOSED_THRESHOLD = 0.3;
|
|
23
|
+
const EYE_OPEN_THRESHOLD = 0.7;
|
|
24
|
+
/**
|
|
25
|
+
* SmartCamera component - A camera component with face detection,
|
|
26
|
+
* blink detection, and WebRTC streaming capabilities.
|
|
27
|
+
*
|
|
28
|
+
* Features:
|
|
29
|
+
* - VisionCamera integration with frame processing
|
|
30
|
+
* - Real-time face detection with ML Kit
|
|
31
|
+
* - Blink detection with debouncing
|
|
32
|
+
* - App lifecycle management (background/foreground)
|
|
33
|
+
* - Orientation handling
|
|
34
|
+
* - WebRTC streaming support
|
|
35
|
+
*/
|
|
36
|
+
export function SmartCamera({ camera = 'front', fps = 30, style, faceDetection, blinkDetection = false, onBlinkDetected, onFaceDetected, webrtc, isActive = true, onReady, onError, }) {
|
|
37
|
+
// Camera permission
|
|
38
|
+
const { hasPermission, requestPermission } = useCameraPermission();
|
|
39
|
+
// Camera device
|
|
40
|
+
const device = useCameraDevice(camera);
|
|
41
|
+
// Window dimensions for autoMode
|
|
42
|
+
const { width: windowWidth, height: windowHeight } = useWindowDimensions();
|
|
43
|
+
// App state for lifecycle management
|
|
44
|
+
const [appState, setAppState] = useState(AppState.currentState);
|
|
45
|
+
const [isCameraActive, setIsCameraActive] = useState(isActive);
|
|
46
|
+
// Track if component is mounted
|
|
47
|
+
const isMounted = useRef(true);
|
|
48
|
+
// Blink state tracking
|
|
49
|
+
const blinkState = useRef({
|
|
50
|
+
lastLeftEyeOpen: 1,
|
|
51
|
+
lastRightEyeOpen: 1,
|
|
52
|
+
lastBlinkTime: 0,
|
|
53
|
+
});
|
|
54
|
+
// Shared values for frame processor communication
|
|
55
|
+
const detectedFaces = useSharedValue([]);
|
|
56
|
+
// Camera ref for imperative operations
|
|
57
|
+
const cameraRef = useRef(null);
|
|
58
|
+
// Frame count for FPS limiting
|
|
59
|
+
const frameCount = useRef(0);
|
|
60
|
+
const lastFrameTime = useRef(Date.now());
|
|
61
|
+
// Merge face detection options with defaults and window dimensions
|
|
62
|
+
const faceDetectionOptions = useMemo(() => ({
|
|
63
|
+
...DEFAULT_FACE_DETECTION_OPTIONS,
|
|
64
|
+
...faceDetection,
|
|
65
|
+
cameraFacing: camera,
|
|
66
|
+
// Enable classification mode if blink detection is enabled
|
|
67
|
+
classificationMode: blinkDetection ? 'all' : faceDetection?.classificationMode ?? 'none',
|
|
68
|
+
// Auto-populate window dimensions if autoMode is enabled
|
|
69
|
+
windowWidth: faceDetection?.autoMode ? (faceDetection.windowWidth ?? windowWidth) : 1.0,
|
|
70
|
+
windowHeight: faceDetection?.autoMode ? (faceDetection.windowHeight ?? windowHeight) : 1.0,
|
|
71
|
+
}), [faceDetection, camera, blinkDetection, windowWidth, windowHeight]);
|
|
72
|
+
// Handle app state changes (background/foreground)
|
|
73
|
+
useEffect(() => {
|
|
74
|
+
const subscription = AppState.addEventListener('change', (nextAppState) => {
|
|
75
|
+
if (!isMounted.current)
|
|
76
|
+
return;
|
|
77
|
+
setAppState(nextAppState);
|
|
78
|
+
// Deactivate camera when app goes to background
|
|
79
|
+
if (nextAppState === 'background' || nextAppState === 'inactive') {
|
|
80
|
+
setIsCameraActive(false);
|
|
81
|
+
}
|
|
82
|
+
else if (nextAppState === 'active' && isActive) {
|
|
83
|
+
// Reactivate camera when app comes to foreground
|
|
84
|
+
setIsCameraActive(true);
|
|
85
|
+
}
|
|
86
|
+
});
|
|
87
|
+
return () => {
|
|
88
|
+
subscription.remove();
|
|
89
|
+
};
|
|
90
|
+
}, [isActive]);
|
|
91
|
+
// Sync camera active state with prop
|
|
92
|
+
useEffect(() => {
|
|
93
|
+
if (appState === 'active') {
|
|
94
|
+
setIsCameraActive(isActive);
|
|
95
|
+
}
|
|
96
|
+
}, [isActive, appState]);
|
|
97
|
+
// Cleanup on unmount
|
|
98
|
+
useEffect(() => {
|
|
99
|
+
isMounted.current = true;
|
|
100
|
+
return () => {
|
|
101
|
+
isMounted.current = false;
|
|
102
|
+
// Reset blink state
|
|
103
|
+
blinkState.current = {
|
|
104
|
+
lastLeftEyeOpen: 1,
|
|
105
|
+
lastRightEyeOpen: 1,
|
|
106
|
+
lastBlinkTime: 0,
|
|
107
|
+
};
|
|
108
|
+
};
|
|
109
|
+
}, []);
|
|
110
|
+
// Request camera permission on mount
|
|
111
|
+
useEffect(() => {
|
|
112
|
+
if (!hasPermission) {
|
|
113
|
+
requestPermission().then((granted) => {
|
|
114
|
+
if (!granted && onError && isMounted.current) {
|
|
115
|
+
onError({
|
|
116
|
+
code: 'PERMISSION_DENIED',
|
|
117
|
+
message: 'Camera permission was denied',
|
|
118
|
+
});
|
|
119
|
+
}
|
|
120
|
+
});
|
|
121
|
+
}
|
|
122
|
+
}, [hasPermission, requestPermission, onError]);
|
|
123
|
+
// Handle device not available
|
|
124
|
+
useEffect(() => {
|
|
125
|
+
if (!device && onError && isMounted.current) {
|
|
126
|
+
onError({
|
|
127
|
+
code: 'CAMERA_UNAVAILABLE',
|
|
128
|
+
message: `${camera} camera is not available on this device`,
|
|
129
|
+
});
|
|
130
|
+
}
|
|
131
|
+
}, [device, camera, onError]);
|
|
132
|
+
// Process blink detection
|
|
133
|
+
const processBlink = useCallback((faces) => {
|
|
134
|
+
if (!blinkDetection || !onBlinkDetected || faces.length === 0) {
|
|
135
|
+
return;
|
|
136
|
+
}
|
|
137
|
+
const face = faces[0]; // Use the first detected face
|
|
138
|
+
const leftEyeOpen = face.leftEyeOpenProbability ?? 1;
|
|
139
|
+
const rightEyeOpen = face.rightEyeOpenProbability ?? 1;
|
|
140
|
+
const now = Date.now();
|
|
141
|
+
const state = blinkState.current;
|
|
142
|
+
// Detect blink: eyes were open, now closed, or were closed, now open
|
|
143
|
+
const wasLeftEyeOpen = state.lastLeftEyeOpen > EYE_OPEN_THRESHOLD;
|
|
144
|
+
const wasRightEyeOpen = state.lastRightEyeOpen > EYE_OPEN_THRESHOLD;
|
|
145
|
+
const isLeftEyeClosed = leftEyeOpen < EYE_CLOSED_THRESHOLD;
|
|
146
|
+
const isRightEyeClosed = rightEyeOpen < EYE_CLOSED_THRESHOLD;
|
|
147
|
+
const isBlink = (wasLeftEyeOpen && isLeftEyeClosed) ||
|
|
148
|
+
(wasRightEyeOpen && isRightEyeClosed);
|
|
149
|
+
// Check debounce
|
|
150
|
+
if (isBlink && now - state.lastBlinkTime > BLINK_DEBOUNCE_MS) {
|
|
151
|
+
const blinkEvent = {
|
|
152
|
+
timestamp: now,
|
|
153
|
+
leftEyeOpen,
|
|
154
|
+
rightEyeOpen,
|
|
155
|
+
isBlink: true,
|
|
156
|
+
faceId: face.trackingId,
|
|
157
|
+
};
|
|
158
|
+
onBlinkDetected(blinkEvent);
|
|
159
|
+
state.lastBlinkTime = now;
|
|
160
|
+
}
|
|
161
|
+
// Update state
|
|
162
|
+
state.lastLeftEyeOpen = leftEyeOpen;
|
|
163
|
+
state.lastRightEyeOpen = rightEyeOpen;
|
|
164
|
+
}, [blinkDetection, onBlinkDetected]);
|
|
165
|
+
// Handle face detection results
|
|
166
|
+
const handleFacesDetected = useCallback((faces) => {
|
|
167
|
+
if (!isMounted.current)
|
|
168
|
+
return;
|
|
169
|
+
if (onFaceDetected) {
|
|
170
|
+
onFaceDetected(faces);
|
|
171
|
+
}
|
|
172
|
+
processBlink(faces);
|
|
173
|
+
}, [onFaceDetected, processBlink]);
|
|
174
|
+
// Create a worklet-callable version of handleFacesDetected
|
|
175
|
+
const handleFacesDetectedWorklet = useRunOnJS(handleFacesDetected, [handleFacesDetected]);
|
|
176
|
+
// Frame processor for face detection
|
|
177
|
+
const frameProcessor = useFrameProcessor((frame) => {
|
|
178
|
+
'worklet';
|
|
179
|
+
// Skip if face detection is not enabled
|
|
180
|
+
if (!faceDetection?.enabled && !blinkDetection) {
|
|
181
|
+
return;
|
|
182
|
+
}
|
|
183
|
+
// FPS limiting - process every N frames based on target FPS
|
|
184
|
+
// VisionCamera typically runs at 30fps, so we may want to skip frames
|
|
185
|
+
const targetFps = faceDetection?.performanceMode === 'accurate' ? 15 : 30;
|
|
186
|
+
const skipFrames = Math.max(1, Math.floor(30 / targetFps));
|
|
187
|
+
// Simple frame skipping
|
|
188
|
+
const currentFrame = Date.now();
|
|
189
|
+
const frameInterval = 1000 / targetFps;
|
|
190
|
+
// Call native face detection
|
|
191
|
+
try {
|
|
192
|
+
const faces = detectFaces(frame, faceDetectionOptions);
|
|
193
|
+
handleFacesDetectedWorklet(faces);
|
|
194
|
+
}
|
|
195
|
+
catch (error) {
|
|
196
|
+
// Silently handle errors in worklet context
|
|
197
|
+
}
|
|
198
|
+
}, [faceDetection?.enabled, blinkDetection, faceDetectionOptions, handleFacesDetectedWorklet]);
|
|
199
|
+
// Handle camera ready
|
|
200
|
+
const handleCameraReady = useCallback(() => {
|
|
201
|
+
if (isMounted.current) {
|
|
202
|
+
onReady?.();
|
|
203
|
+
}
|
|
204
|
+
}, [onReady]);
|
|
205
|
+
// Handle camera error
|
|
206
|
+
const handleCameraError = useCallback((error) => {
|
|
207
|
+
if (isMounted.current) {
|
|
208
|
+
onError?.({
|
|
209
|
+
code: 'UNKNOWN_ERROR',
|
|
210
|
+
message: error.message,
|
|
211
|
+
nativeError: error,
|
|
212
|
+
});
|
|
213
|
+
}
|
|
214
|
+
}, [onError]);
|
|
215
|
+
// Render loading state if no permission
|
|
216
|
+
if (!hasPermission) {
|
|
217
|
+
return (<View style={[styles.container, style]}>
|
|
218
|
+
<View style={styles.placeholder}/>
|
|
219
|
+
</View>);
|
|
220
|
+
}
|
|
221
|
+
// Render placeholder if no device
|
|
222
|
+
if (!device) {
|
|
223
|
+
return (<View style={[styles.container, style]}>
|
|
224
|
+
<View style={styles.placeholder}/>
|
|
225
|
+
</View>);
|
|
226
|
+
}
|
|
227
|
+
// Determine orientation based on device orientation
|
|
228
|
+
// In VisionCamera v4, OutputOrientation is 'device' | 'preview'
|
|
229
|
+
const getOutputOrientation = () => {
|
|
230
|
+
// 'device' follows the device's physical orientation
|
|
231
|
+
return 'device';
|
|
232
|
+
};
|
|
233
|
+
// Camera props with full configuration
|
|
234
|
+
const cameraProps = {
|
|
235
|
+
style: StyleSheet.absoluteFill,
|
|
236
|
+
device,
|
|
237
|
+
isActive: isCameraActive,
|
|
238
|
+
fps,
|
|
239
|
+
onInitialized: handleCameraReady,
|
|
240
|
+
onError: handleCameraError,
|
|
241
|
+
// Enable photo/video for potential future features
|
|
242
|
+
photo: false,
|
|
243
|
+
video: webrtc?.enabled ?? false,
|
|
244
|
+
audio: webrtc?.enabled ?? false,
|
|
245
|
+
// Orientation handling
|
|
246
|
+
outputOrientation: getOutputOrientation(),
|
|
247
|
+
// Performance optimizations
|
|
248
|
+
enableZoomGesture: false,
|
|
249
|
+
exposure: 0,
|
|
250
|
+
};
|
|
251
|
+
// Add frame processor if face/blink detection is enabled
|
|
252
|
+
if (faceDetection?.enabled || blinkDetection) {
|
|
253
|
+
cameraProps.frameProcessor = frameProcessor;
|
|
254
|
+
}
|
|
255
|
+
return (<View style={[styles.container, style]}>
|
|
256
|
+
<Camera ref={cameraRef} {...cameraProps}/>
|
|
257
|
+
</View>);
|
|
258
|
+
}
|
|
259
|
+
const styles = StyleSheet.create({
|
|
260
|
+
container: {
|
|
261
|
+
flex: 1,
|
|
262
|
+
backgroundColor: '#000',
|
|
263
|
+
},
|
|
264
|
+
placeholder: {
|
|
265
|
+
flex: 1,
|
|
266
|
+
backgroundColor: '#1a1a1a',
|
|
267
|
+
},
|
|
268
|
+
});
|
|
269
|
+
export default SmartCamera;
|
|
270
|
+
//# sourceMappingURL=SmartCamera.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"SmartCamera.js","sourceRoot":"","sources":["../src/SmartCamera.tsx"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,OAAO,CAAC;AACjF,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,QAAQ,EAAuB,mBAAmB,EAAE,MAAM,cAAc,CAAC;AACpG,OAAO,EACL,MAAM,EACN,eAAe,EACf,mBAAmB,EACnB,iBAAiB,GAElB,MAAM,4BAA4B,CAAC;AACpC,OAAO,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AASxE,OAAO,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AAEvD,4CAA4C;AAC5C,MAAM,8BAA8B,GAA0B;IAC5D,eAAe,EAAE,MAAM;IACvB,YAAY,EAAE,MAAM;IACpB,WAAW,EAAE,MAAM;IACnB,kBAAkB,EAAE,MAAM;IAC1B,WAAW,EAAE,IAAI;IACjB,eAAe,EAAE,KAAK;IACtB,YAAY,EAAE,OAAO;IACrB,QAAQ,EAAE,KAAK;IACf,WAAW,EAAE,GAAG;IAChB,YAAY,EAAE,GAAG;CAClB,CAAC;AASF,6CAA6C;AAC7C,MAAM,iBAAiB,GAAG,GAAG,CAAC;AAE9B,qDAAqD;AACrD,MAAM,oBAAoB,GAAG,GAAG,CAAC;AACjC,MAAM,kBAAkB,GAAG,GAAG,CAAC;AAE/B;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,WAAW,CAAC,EAC1B,MAAM,GAAG,OAAO,EAChB,GAAG,GAAG,EAAE,EACR,KAAK,EACL,aAAa,EACb,cAAc,GAAG,KAAK,EACtB,eAAe,EACf,cAAc,EACd,MAAM,EACN,QAAQ,GAAG,IAAI,EACf,OAAO,EACP,OAAO,GACU;IACjB,oBAAoB;IACpB,MAAM,EAAE,aAAa,EAAE,iBAAiB,EAAE,GAAG,mBAAmB,EAAE,CAAC;IAEnE,gBAAgB;IAChB,MAAM,MAAM,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC;IAEvC,iCAAiC;IACjC,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,MAAM,EAAE,YAAY,EAAE,GAAG,mBAAmB,EAAE,CAAC;IAE3E,qCAAqC;IACrC,MAAM,CAAC,QAAQ,EAAE,WAAW,CAAC,GAAG,QAAQ,CAAiB,QAAQ,CAAC,YAAY,CAAC,CAAC;IAChF,MAAM,CAAC,cAAc,EAAE,iBAAiB,CAAC,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAE/D,gCAAgC;IAChC,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;IAE/B,uBAAuB;IACvB,MAAM,UAAU,GAAG,MAAM,CAAa;QACpC,eAAe,EAAE,CAAC;QAClB,gBAAgB,EAAE,CAAC;QACnB,aAAa,EAAE,CAAC;KACjB,CAAC,CAAC;IAEH,kDAAkD;IAClD,MAAM,aAAa,GAAG,cAAc,CAAS,EAAE,CAAC,CAAC;IAEjD,uCAAuC;IACvC,MAAM,SAAS,GAAG,MAAM,CAAS,IAAI,CAAC,CAAC;IAEvC,+BAA+B;IAC/B,MAAM,UAAU,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;IAC7B,MAAM,aAAa,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;IAEzC,mEAAmE;IACnE,MAAM,oBAAoB,GAAG,OAAO,CAAwB,GAAG,EAAE,CAAC,CAAC;QACjE,GAAG,8BAA8B;QACjC,GAAG,aAAa;QAChB,YAAY,EAAE,MAAM;QACpB,2DAA2D;QAC3D,kBAAkB,EAAE,cAAc,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,aAAa,EAAE,kBAAkB,IAAI,MAAM;QACxF,yDAAyD;QACzD,WAAW,EAAE,aAAa,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,WAAW,IAAI,WAAW,CAAC,CAAC,CAAC,CAAC,GAAG;QACvF,YAAY,EAAE,aAAa,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,YAAY,IAAI,YAAY,CAAC,CAAC,CAAC,CAAC,GAAG;KAC3F,CAAC,EAAE,CAAC,aAAa,EAAE,MAAM,EAAE,cAAc,EAAE,WAAW,EAAE,YAAY,CAAC,CAAC,CAAC;IAExE,mDAAmD;IACnD,SAAS,CAAC,GAAG,EAAE;QACb,MAAM,YAAY,GAAG,QAAQ,CAAC,gBAAgB,CAAC,QAAQ,EAAE,CAAC,YAA4B,EAAE,EAAE;YACxF,IAAI,CAAC,SAAS,CAAC,OAAO;gBAAE,OAAO;YAE/B,WAAW,CAAC,YAAY,CAAC,CAAC;YAE1B,gDAAgD;YAChD,IAAI,YAAY,KAAK,YAAY,IAAI,YAAY,KAAK,UAAU,EAAE,CAAC;gBACjE,iBAAiB,CAAC,KAAK,CAAC,CAAC;YAC3B,CAAC;iBAAM,IAAI,YAAY,KAAK,QAAQ,IAAI,QAAQ,EAAE,CAAC;gBACjD,iDAAiD;gBACjD,iBAAiB,CAAC,IAAI,CAAC,CAAC;YAC1B,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,OAAO,GAAG,EAAE;YACV,YAAY,CAAC,MAAM,EAAE,CAAC;QACxB,CAAC,CAAC;IACJ,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC;IAEf,qCAAqC;IACrC,SAAS,CAAC,GAAG,EAAE;QACb,IAAI,QAAQ,KAAK,QAAQ,EAAE,CAAC;YAC1B,iBAAiB,CAAC,QAAQ,CAAC,CAAC;QAC9B,CAAC;IACH,CAAC,EAAE,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;IAEzB,qBAAqB;IACrB,SAAS,CAAC,GAAG,EAAE;QACb,SAAS,CAAC,OAAO,GAAG,IAAI,CAAC;QAEzB,OAAO,GAAG,EAAE;YACV,SAAS,CAAC,OAAO,GAAG,KAAK,CAAC;YAC1B,oBAAoB;YACpB,UAAU,CAAC,OAAO,GAAG;gBACnB,eAAe,EAAE,CAAC;gBAClB,gBAAgB,EAAE,CAAC;gBACnB,aAAa,EAAE,CAAC;aACjB,CAAC;QACJ,CAAC,CAAC;IACJ,CAAC,EAAE,EAAE,CAAC,CAAC;IAEP,qCAAqC;IACrC,SAAS,CAAC,GAAG,EAAE;QACb,IAAI,CAAC,aAAa,EAAE,CAAC;YACnB,iBAAiB,EAAE,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE;gBACnC,IAAI,CAAC,OAAO,IAAI,OAAO,IAAI,SAAS,CAAC,OAAO,EAAE,CAAC;oBAC7C,OAAO,CAAC;wBACN,IAAI,EAAE,mBAAmB;wBACzB,OAAO,EAAE,8BAA8B;qBACxC,CAAC,CAAC;gBACL,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC;IACH,CAAC,EAAE,CAAC,aAAa,EAAE,iBAAiB,EAAE,OAAO,CAAC,CAAC,CAAC;IAEhD,8BAA8B;IAC9B,SAAS,CAAC,GAAG,EAAE;QACb,IAAI,CAAC,MAAM,IAAI,OAAO,IAAI,SAAS,CAAC,OAAO,EAAE,CAAC;YAC5C,OAAO,CAAC;gBACN,IAAI,EAAE,oBAAoB;gBAC1B,OAAO,EAAE,GAAG,MAAM,yCAAyC;aAC5D,CAAC,CAAC;QACL,CAAC;IACH,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;IAE9B,0BAA0B;IAC1B,MAAM,YAAY,GAAG,WAAW,CAAC,CAAC,KAAa,EAAE,EAAE;QACjD,IAAI,CAAC,cAAc,IAAI,CAAC,eAAe,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC9D,OAAO;QACT,CAAC;QAED,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,8BAA8B;QACrD,MAAM,WAAW,GAAG,IAAI,CAAC,sBAAsB,IAAI,CAAC,CAAC;QACrD,MAAM,YAAY,GAAG,IAAI,CAAC,uBAAuB,IAAI,CAAC,CAAC;QAEvD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,KAAK,GAAG,UAAU,CAAC,OAAO,CAAC;QAEjC,qEAAqE;QACrE,MAAM,cAAc,GAAG,KAAK,CAAC,eAAe,GAAG,kBAAkB,CAAC;QAClE,MAAM,eAAe,GAAG,KAAK,CAAC,gBAAgB,GAAG,kBAAkB,CAAC;QACpE,MAAM,eAAe,GAAG,WAAW,GAAG,oBAAoB,CAAC;QAC3D,MAAM,gBAAgB,GAAG,YAAY,GAAG,oBAAoB,CAAC;QAE7D,MAAM,OAAO,GACX,CAAC,cAAc,IAAI,eAAe,CAAC;YACnC,CAAC,eAAe,IAAI,gBAAgB,CAAC,CAAC;QAExC,iBAAiB;QACjB,IAAI,OAAO,IAAI,GAAG,GAAG,KAAK,CAAC,aAAa,GAAG,iBAAiB,EAAE,CAAC;YAC7D,MAAM,UAAU,GAAe;gBAC7B,SAAS,EAAE,GAAG;gBACd,WAAW;gBACX,YAAY;gBACZ,OAAO,EAAE,IAAI;gBACb,MAAM,EAAE,IAAI,CAAC,UAAU;aACxB,CAAC;YAEF,eAAe,CAAC,UAAU,CAAC,CAAC;YAC5B,KAAK,CAAC,aAAa,GAAG,GAAG,CAAC;QAC5B,CAAC;QAED,eAAe;QACf,KAAK,CAAC,eAAe,GAAG,WAAW,CAAC;QACpC,KAAK,CAAC,gBAAgB,GAAG,YAAY,CAAC;IACxC,CAAC,EAAE,CAAC,cAAc,EAAE,eAAe,CAAC,CAAC,CAAC;IAEtC,gCAAgC;IAChC,MAAM,mBAAmB,GAAG,WAAW,CAAC,CAAC,KAAa,EAAE,EAAE;QACxD,IAAI,CAAC,SAAS,CAAC,OAAO;YAAE,OAAO;QAE/B,IAAI,cAAc,EAAE,CAAC;YACnB,cAAc,CAAC,KAAK,CAAC,CAAC;QACxB,CAAC;QACD,YAAY,CAAC,KAAK,CAAC,CAAC;IACtB,CAAC,EAAE,CAAC,cAAc,EAAE,YAAY,CAAC,CAAC,CAAC;IAEnC,2DAA2D;IAC3D,MAAM,0BAA0B,GAAG,UAAU,CAAC,mBAAmB,EAAE,CAAC,mBAAmB,CAAC,CAAC,CAAC;IAE1F,qCAAqC;IACrC,MAAM,cAAc,GAAG,iBAAiB,CAAC,CAAC,KAAK,EAAE,EAAE;QACjD,SAAS,CAAC;QAEV,wCAAwC;QACxC,IAAI,CAAC,aAAa,EAAE,OAAO,IAAI,CAAC,cAAc,EAAE,CAAC;YAC/C,OAAO;QACT,CAAC;QAED,4DAA4D;QAC5D,sEAAsE;QACtE,MAAM,SAAS,GAAG,aAAa,EAAE,eAAe,KAAK,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1E,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,SAAS,CAAC,CAAC,CAAC;QAE3D,wBAAwB;QACxB,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAChC,MAAM,aAAa,GAAG,IAAI,GAAG,SAAS,CAAC;QAEvC,6BAA6B;QAC7B,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,WAAW,CAAC,KAAK,EAAE,oBAAoB,CAAC,CAAC;YACvD,0BAA0B,CAAC,KAAK,CAAC,CAAC;QACpC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,4CAA4C;QAC9C,CAAC;IACH,CAAC,EAAE,CAAC,aAAa,EAAE,OAAO,EAAE,cAAc,EAAE,oBAAoB,EAAE,0BAA0B,CAAC,CAAC,CAAC;IAE/F,sBAAsB;IACtB,MAAM,iBAAiB,GAAG,WAAW,CAAC,GAAG,EAAE;QACzC,IAAI,SAAS,CAAC,OAAO,EAAE,CAAC;YACtB,OAAO,EAAE,EAAE,CAAC;QACd,CAAC;IACH,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IAEd,sBAAsB;IACtB,MAAM,iBAAiB,GAAG,WAAW,CAAC,CAAC,KAAY,EAAE,EAAE;QACrD,IAAI,SAAS,CAAC,OAAO,EAAE,CAAC;YACtB,OAAO,EAAE,CAAC;gBACR,IAAI,EAAE,eAAe;gBACrB,OAAO,EAAE,KAAK,CAAC,OAAO;gBACtB,WAAW,EAAE,KAAK;aACnB,CAAC,CAAC;QACL,CAAC;IACH,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IAEd,wCAAwC;IACxC,IAAI,CAAC,aAAa,EAAE,CAAC;QACnB,OAAO,CACL,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC,CACrC;QAAA,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,EAClC;MAAA,EAAE,IAAI,CAAC,CACR,CAAC;IACJ,CAAC;IAED,kCAAkC;IAClC,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,OAAO,CACL,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC,CACrC;QAAA,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,EAClC;MAAA,EAAE,IAAI,CAAC,CACR,CAAC;IACJ,CAAC;IAED,oDAAoD;IACpD,gEAAgE;IAChE,MAAM,oBAAoB,GAAG,GAAyB,EAAE;QACtD,qDAAqD;QACrD,OAAO,QAAQ,CAAC;IAClB,CAAC,CAAC;IAEF,uCAAuC;IACvC,MAAM,WAAW,GAAgB;QAC/B,KAAK,EAAE,UAAU,CAAC,YAAY;QAC9B,MAAM;QACN,QAAQ,EAAE,cAAc;QACxB,GAAG;QACH,aAAa,EAAE,iBAAiB;QAChC,OAAO,EAAE,iBAAiB;QAC1B,mDAAmD;QACnD,KAAK,EAAE,KAAK;QACZ,KAAK,EAAE,MAAM,EAAE,OAAO,IAAI,KAAK;QAC/B,KAAK,EAAE,MAAM,EAAE,OAAO,IAAI,KAAK;QAC/B,uBAAuB;QACvB,iBAAiB,EAAE,oBAAoB,EAAE;QACzC,4BAA4B;QAC5B,iBAAiB,EAAE,KAAK;QACxB,QAAQ,EAAE,CAAC;KACZ,CAAC;IAEF,yDAAyD;IACzD,IAAI,aAAa,EAAE,OAAO,IAAI,cAAc,EAAE,CAAC;QAC7C,WAAW,CAAC,cAAc,GAAG,cAAc,CAAC;IAC9C,CAAC;IAED,OAAO,CACL,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC,CACrC;MAAA,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC,IAAI,WAAW,CAAC,EAC1C;IAAA,EAAE,IAAI,CAAC,CACR,CAAC;AACJ,CAAC;AAED,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC;IAC/B,SAAS,EAAE;QACT,IAAI,EAAE,CAAC;QACP,eAAe,EAAE,MAAM;KACxB;IACD,WAAW,EAAE;QACX,IAAI,EAAE,CAAC;QACP,eAAe,EAAE,SAAS;KAC3B;CACF,CAAC,CAAC;AAEH,eAAe,WAAW,CAAC","sourcesContent":["import React, { useCallback, useEffect, useRef, useMemo, useState } from 'react';\nimport { StyleSheet, View, AppState, type AppStateStatus, useWindowDimensions } from 'react-native';\nimport {\n Camera,\n useCameraDevice,\n useCameraPermission,\n useFrameProcessor,\n type CameraProps,\n} from 'react-native-vision-camera';\nimport { useSharedValue, useRunOnJS } from 'react-native-worklets-core';\n\nimport type {\n SmartCameraProps,\n Face,\n BlinkEvent,\n FrameProcessorOptions,\n SmartCameraError,\n} from './types';\nimport { detectFaces } from './detection/faceDetector';\n\n// Default values for face detection options\nconst DEFAULT_FACE_DETECTION_OPTIONS: FrameProcessorOptions = {\n performanceMode: 'fast',\n landmarkMode: 'none',\n contourMode: 'none',\n classificationMode: 'none',\n minFaceSize: 0.15,\n trackingEnabled: false,\n cameraFacing: 'front',\n autoMode: false,\n windowWidth: 1.0,\n windowHeight: 1.0,\n};\n\n// Blink detection state\ninterface BlinkState {\n lastLeftEyeOpen: number;\n lastRightEyeOpen: number;\n lastBlinkTime: number;\n}\n\n// Minimum time between blink detections (ms)\nconst BLINK_DEBOUNCE_MS = 300;\n\n// Eye open probability threshold for blink detection\nconst EYE_CLOSED_THRESHOLD = 0.3;\nconst EYE_OPEN_THRESHOLD = 0.7;\n\n/**\n * SmartCamera component - A camera component with face detection,\n * blink detection, and WebRTC streaming capabilities.\n * \n * Features:\n * - VisionCamera integration with frame processing\n * - Real-time face detection with ML Kit\n * - Blink detection with debouncing\n * - App lifecycle management (background/foreground)\n * - Orientation handling\n * - WebRTC streaming support\n */\nexport function SmartCamera({\n camera = 'front',\n fps = 30,\n style,\n faceDetection,\n blinkDetection = false,\n onBlinkDetected,\n onFaceDetected,\n webrtc,\n isActive = true,\n onReady,\n onError,\n}: SmartCameraProps) {\n // Camera permission\n const { hasPermission, requestPermission } = useCameraPermission();\n\n // Camera device\n const device = useCameraDevice(camera);\n\n // Window dimensions for autoMode\n const { width: windowWidth, height: windowHeight } = useWindowDimensions();\n\n // App state for lifecycle management\n const [appState, setAppState] = useState<AppStateStatus>(AppState.currentState);\n const [isCameraActive, setIsCameraActive] = useState(isActive);\n\n // Track if component is mounted\n const isMounted = useRef(true);\n\n // Blink state tracking\n const blinkState = useRef<BlinkState>({\n lastLeftEyeOpen: 1,\n lastRightEyeOpen: 1,\n lastBlinkTime: 0,\n });\n\n // Shared values for frame processor communication\n const detectedFaces = useSharedValue<Face[]>([]);\n\n // Camera ref for imperative operations\n const cameraRef = useRef<Camera>(null);\n\n // Frame count for FPS limiting\n const frameCount = useRef(0);\n const lastFrameTime = useRef(Date.now());\n\n // Merge face detection options with defaults and window dimensions\n const faceDetectionOptions = useMemo<FrameProcessorOptions>(() => ({\n ...DEFAULT_FACE_DETECTION_OPTIONS,\n ...faceDetection,\n cameraFacing: camera,\n // Enable classification mode if blink detection is enabled\n classificationMode: blinkDetection ? 'all' : faceDetection?.classificationMode ?? 'none',\n // Auto-populate window dimensions if autoMode is enabled\n windowWidth: faceDetection?.autoMode ? (faceDetection.windowWidth ?? windowWidth) : 1.0,\n windowHeight: faceDetection?.autoMode ? (faceDetection.windowHeight ?? windowHeight) : 1.0,\n }), [faceDetection, camera, blinkDetection, windowWidth, windowHeight]);\n\n // Handle app state changes (background/foreground)\n useEffect(() => {\n const subscription = AppState.addEventListener('change', (nextAppState: AppStateStatus) => {\n if (!isMounted.current) return;\n\n setAppState(nextAppState);\n\n // Deactivate camera when app goes to background\n if (nextAppState === 'background' || nextAppState === 'inactive') {\n setIsCameraActive(false);\n } else if (nextAppState === 'active' && isActive) {\n // Reactivate camera when app comes to foreground\n setIsCameraActive(true);\n }\n });\n\n return () => {\n subscription.remove();\n };\n }, [isActive]);\n\n // Sync camera active state with prop\n useEffect(() => {\n if (appState === 'active') {\n setIsCameraActive(isActive);\n }\n }, [isActive, appState]);\n\n // Cleanup on unmount\n useEffect(() => {\n isMounted.current = true;\n\n return () => {\n isMounted.current = false;\n // Reset blink state\n blinkState.current = {\n lastLeftEyeOpen: 1,\n lastRightEyeOpen: 1,\n lastBlinkTime: 0,\n };\n };\n }, []);\n\n // Request camera permission on mount\n useEffect(() => {\n if (!hasPermission) {\n requestPermission().then((granted) => {\n if (!granted && onError && isMounted.current) {\n onError({\n code: 'PERMISSION_DENIED',\n message: 'Camera permission was denied',\n });\n }\n });\n }\n }, [hasPermission, requestPermission, onError]);\n\n // Handle device not available\n useEffect(() => {\n if (!device && onError && isMounted.current) {\n onError({\n code: 'CAMERA_UNAVAILABLE',\n message: `${camera} camera is not available on this device`,\n });\n }\n }, [device, camera, onError]);\n\n // Process blink detection\n const processBlink = useCallback((faces: Face[]) => {\n if (!blinkDetection || !onBlinkDetected || faces.length === 0) {\n return;\n }\n\n const face = faces[0]; // Use the first detected face\n const leftEyeOpen = face.leftEyeOpenProbability ?? 1;\n const rightEyeOpen = face.rightEyeOpenProbability ?? 1;\n\n const now = Date.now();\n const state = blinkState.current;\n\n // Detect blink: eyes were open, now closed, or were closed, now open\n const wasLeftEyeOpen = state.lastLeftEyeOpen > EYE_OPEN_THRESHOLD;\n const wasRightEyeOpen = state.lastRightEyeOpen > EYE_OPEN_THRESHOLD;\n const isLeftEyeClosed = leftEyeOpen < EYE_CLOSED_THRESHOLD;\n const isRightEyeClosed = rightEyeOpen < EYE_CLOSED_THRESHOLD;\n\n const isBlink =\n (wasLeftEyeOpen && isLeftEyeClosed) ||\n (wasRightEyeOpen && isRightEyeClosed);\n\n // Check debounce\n if (isBlink && now - state.lastBlinkTime > BLINK_DEBOUNCE_MS) {\n const blinkEvent: BlinkEvent = {\n timestamp: now,\n leftEyeOpen,\n rightEyeOpen,\n isBlink: true,\n faceId: face.trackingId,\n };\n\n onBlinkDetected(blinkEvent);\n state.lastBlinkTime = now;\n }\n\n // Update state\n state.lastLeftEyeOpen = leftEyeOpen;\n state.lastRightEyeOpen = rightEyeOpen;\n }, [blinkDetection, onBlinkDetected]);\n\n // Handle face detection results\n const handleFacesDetected = useCallback((faces: Face[]) => {\n if (!isMounted.current) return;\n\n if (onFaceDetected) {\n onFaceDetected(faces);\n }\n processBlink(faces);\n }, [onFaceDetected, processBlink]);\n\n // Create a worklet-callable version of handleFacesDetected\n const handleFacesDetectedWorklet = useRunOnJS(handleFacesDetected, [handleFacesDetected]);\n\n // Frame processor for face detection\n const frameProcessor = useFrameProcessor((frame) => {\n 'worklet';\n\n // Skip if face detection is not enabled\n if (!faceDetection?.enabled && !blinkDetection) {\n return;\n }\n\n // FPS limiting - process every N frames based on target FPS\n // VisionCamera typically runs at 30fps, so we may want to skip frames\n const targetFps = faceDetection?.performanceMode === 'accurate' ? 15 : 30;\n const skipFrames = Math.max(1, Math.floor(30 / targetFps));\n \n // Simple frame skipping\n const currentFrame = Date.now();\n const frameInterval = 1000 / targetFps;\n \n // Call native face detection\n try {\n const faces = detectFaces(frame, faceDetectionOptions);\n handleFacesDetectedWorklet(faces);\n } catch (error) {\n // Silently handle errors in worklet context\n }\n }, [faceDetection?.enabled, blinkDetection, faceDetectionOptions, handleFacesDetectedWorklet]);\n\n // Handle camera ready\n const handleCameraReady = useCallback(() => {\n if (isMounted.current) {\n onReady?.();\n }\n }, [onReady]);\n\n // Handle camera error\n const handleCameraError = useCallback((error: Error) => {\n if (isMounted.current) {\n onError?.({\n code: 'UNKNOWN_ERROR',\n message: error.message,\n nativeError: error,\n });\n }\n }, [onError]);\n\n // Render loading state if no permission\n if (!hasPermission) {\n return (\n <View style={[styles.container, style]}>\n <View style={styles.placeholder} />\n </View>\n );\n }\n\n // Render placeholder if no device\n if (!device) {\n return (\n <View style={[styles.container, style]}>\n <View style={styles.placeholder} />\n </View>\n );\n }\n\n // Determine orientation based on device orientation\n // In VisionCamera v4, OutputOrientation is 'device' | 'preview'\n const getOutputOrientation = (): 'device' | 'preview' => {\n // 'device' follows the device's physical orientation\n return 'device';\n };\n\n // Camera props with full configuration\n const cameraProps: CameraProps = {\n style: StyleSheet.absoluteFill,\n device,\n isActive: isCameraActive,\n fps,\n onInitialized: handleCameraReady,\n onError: handleCameraError,\n // Enable photo/video for potential future features\n photo: false,\n video: webrtc?.enabled ?? false,\n audio: webrtc?.enabled ?? false,\n // Orientation handling\n outputOrientation: getOutputOrientation(),\n // Performance optimizations\n enableZoomGesture: false,\n exposure: 0,\n };\n\n // Add frame processor if face/blink detection is enabled\n if (faceDetection?.enabled || blinkDetection) {\n cameraProps.frameProcessor = frameProcessor;\n }\n\n return (\n <View style={[styles.container, style]}>\n <Camera ref={cameraRef} {...cameraProps} />\n </View>\n );\n}\n\nconst styles = StyleSheet.create({\n container: {\n flex: 1,\n backgroundColor: '#000',\n },\n placeholder: {\n flex: 1,\n backgroundColor: '#1a1a1a',\n },\n});\n\nexport default SmartCamera;\n"]}
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
import type { Face, StaticImageOptions, VideoConstraints } from './types';
|
|
2
|
+
interface SmartCameraModuleInterface {
|
|
3
|
+
detectFacesInImage(options: StaticImageOptions): Promise<Face[]>;
|
|
4
|
+
updateFaceDetectionOptions(options: Record<string, unknown>): void;
|
|
5
|
+
initializeWebRTC(): Promise<boolean>;
|
|
6
|
+
startWebRTCStream(constraints: VideoConstraints): Promise<boolean>;
|
|
7
|
+
stopWebRTCStream(): void;
|
|
8
|
+
pushWebRTCFrame(frameData: Record<string, unknown>): void;
|
|
9
|
+
isWebRTCStreaming(): boolean;
|
|
10
|
+
readonly PI: number;
|
|
11
|
+
readonly DEFAULT_MIN_FACE_SIZE: number;
|
|
12
|
+
readonly EYE_CLOSED_THRESHOLD: number;
|
|
13
|
+
readonly EYE_OPEN_THRESHOLD: number;
|
|
14
|
+
__expo_module_name__?: string;
|
|
15
|
+
startObserving?: () => void;
|
|
16
|
+
stopObserving?: () => void;
|
|
17
|
+
addListener?: unknown;
|
|
18
|
+
removeListeners?: unknown;
|
|
19
|
+
[key: string]: unknown;
|
|
20
|
+
}
|
|
21
|
+
declare const SmartCameraModule: SmartCameraModuleInterface;
|
|
22
|
+
/**
|
|
23
|
+
* Detect faces in a static image
|
|
24
|
+
* @param options - Static image options including the image source and detection settings
|
|
25
|
+
* @returns Promise resolving to an array of detected faces
|
|
26
|
+
*
|
|
27
|
+
* @example
|
|
28
|
+
* ```tsx
|
|
29
|
+
* const faces = await detectFacesInImage({
|
|
30
|
+
* image: { uri: 'https://example.com/photo.jpg' },
|
|
31
|
+
* performanceMode: 'accurate',
|
|
32
|
+
* landmarkMode: 'all',
|
|
33
|
+
* });
|
|
34
|
+
* console.log(`Detected ${faces.length} faces`);
|
|
35
|
+
* ```
|
|
36
|
+
*/
|
|
37
|
+
export declare function detectFacesInImage(options: StaticImageOptions): Promise<Face[]>;
|
|
38
|
+
/**
|
|
39
|
+
* Update face detection options at runtime
|
|
40
|
+
* @param options - New face detection options
|
|
41
|
+
*/
|
|
42
|
+
export declare function updateFaceDetectionOptions(options: Record<string, unknown>): void;
|
|
43
|
+
/**
|
|
44
|
+
* Initialize WebRTC for streaming
|
|
45
|
+
* Must be called before startWebRTCStream
|
|
46
|
+
* @returns Promise resolving to true if initialized successfully
|
|
47
|
+
*/
|
|
48
|
+
export declare function initializeWebRTC(): Promise<boolean>;
|
|
49
|
+
/**
|
|
50
|
+
* Start WebRTC video streaming
|
|
51
|
+
* @param constraints - Video constraints (width, height, frameRate)
|
|
52
|
+
* @returns Promise resolving to true if started successfully
|
|
53
|
+
*/
|
|
54
|
+
export declare function startWebRTCStream(constraints: VideoConstraints): Promise<boolean>;
|
|
55
|
+
/**
|
|
56
|
+
* Stop WebRTC video streaming
|
|
57
|
+
*/
|
|
58
|
+
export declare function stopWebRTCStream(): void;
|
|
59
|
+
/**
|
|
60
|
+
* Push a frame to the WebRTC stream
|
|
61
|
+
* @param frameData - Frame data to push
|
|
62
|
+
*/
|
|
63
|
+
export declare function pushWebRTCFrame(frameData: Record<string, unknown>): void;
|
|
64
|
+
/**
|
|
65
|
+
* Check if WebRTC is currently streaming
|
|
66
|
+
* @returns true if streaming, false otherwise
|
|
67
|
+
*/
|
|
68
|
+
export declare function isWebRTCStreaming(): boolean;
|
|
69
|
+
/**
|
|
70
|
+
* Subscribe to face detection events from native module
|
|
71
|
+
* @param listener - Callback function for face detection events
|
|
72
|
+
* @returns Subscription object with remove method
|
|
73
|
+
*/
|
|
74
|
+
export declare function addFaceDetectionListener(listener: (faces: Face[]) => void): import("expo-modules-core").Subscription;
|
|
75
|
+
/**
|
|
76
|
+
* Subscribe to blink detection events from native module
|
|
77
|
+
* @param listener - Callback function for blink events
|
|
78
|
+
* @returns Subscription object with remove method
|
|
79
|
+
*/
|
|
80
|
+
export declare function addBlinkDetectionListener(listener: (event: {
|
|
81
|
+
leftEyeOpen: number;
|
|
82
|
+
rightEyeOpen: number;
|
|
83
|
+
isBlink: boolean;
|
|
84
|
+
}) => void): import("expo-modules-core").Subscription;
|
|
85
|
+
/**
|
|
86
|
+
* Subscribe to error events from native module
|
|
87
|
+
* @param listener - Callback function for error events
|
|
88
|
+
* @returns Subscription object with remove method
|
|
89
|
+
*/
|
|
90
|
+
export declare function addErrorListener(listener: (error: {
|
|
91
|
+
code: string;
|
|
92
|
+
message: string;
|
|
93
|
+
}) => void): import("expo-modules-core").Subscription;
|
|
94
|
+
/**
|
|
95
|
+
* Subscribe to WebRTC state change events
|
|
96
|
+
* @param listener - Callback function for WebRTC state changes
|
|
97
|
+
* @returns Subscription object with remove method
|
|
98
|
+
*/
|
|
99
|
+
export declare function addWebRTCStateChangeListener(listener: (state: {
|
|
100
|
+
isStreaming: boolean;
|
|
101
|
+
}) => void): import("expo-modules-core").Subscription;
|
|
102
|
+
/**
|
|
103
|
+
* Module constants
|
|
104
|
+
*/
|
|
105
|
+
export declare const Constants: {
|
|
106
|
+
PI: number;
|
|
107
|
+
DEFAULT_MIN_FACE_SIZE: number;
|
|
108
|
+
EYE_CLOSED_THRESHOLD: number;
|
|
109
|
+
EYE_OPEN_THRESHOLD: number;
|
|
110
|
+
};
|
|
111
|
+
export default SmartCameraModule;
|
|
112
|
+
//# sourceMappingURL=SmartCameraModule.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"SmartCameraModule.d.ts","sourceRoot":"","sources":["../src/SmartCameraModule.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,IAAI,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAG1E,UAAU,0BAA0B;IAElC,kBAAkB,CAAC,OAAO,EAAE,kBAAkB,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;IAGjE,0BAA0B,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IAGnE,gBAAgB,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC;IACrC,iBAAiB,CAAC,WAAW,EAAE,gBAAgB,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IACnE,gBAAgB,IAAI,IAAI,CAAC;IACzB,eAAe,CAAC,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IAC1D,iBAAiB,IAAI,OAAO,CAAC;IAG7B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,qBAAqB,EAAE,MAAM,CAAC;IACvC,QAAQ,CAAC,oBAAoB,EAAE,MAAM,CAAC;IACtC,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;IAGpC,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,cAAc,CAAC,EAAE,MAAM,IAAI,CAAC;IAC5B,aAAa,CAAC,EAAE,MAAM,IAAI,CAAC;IAC3B,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAGD,QAAA,MAAM,iBAAiB,4BAAuE,CAAC;AAS/F;;;;;;;;;;;;;;GAcG;AACH,wBAAsB,kBAAkB,CAAC,OAAO,EAAE,kBAAkB,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,CAErF;AAED;;;GAGG;AACH,wBAAgB,0BAA0B,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAEjF;AAMD;;;;GAIG;AACH,wBAAsB,gBAAgB,IAAI,OAAO,CAAC,OAAO,CAAC,CAEzD;AAED;;;;GAIG;AACH,wBAAsB,iBAAiB,CAAC,WAAW,EAAE,gBAAgB,GAAG,OAAO,CAAC,OAAO,CAAC,CAEvF;AAED;;GAEG;AACH,wBAAgB,gBAAgB,IAAI,IAAI,CAEvC;AAED;;;GAGG;AACH,wBAAgB,eAAe,CAAC,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAExE;AAED;;;GAGG;AACH,wBAAgB,iBAAiB,IAAI,OAAO,CAE3C;AAMD;;;;GAIG;AACH,wBAAgB,wBAAwB,CAAC,QAAQ,EAAE,CAAC,KAAK,EAAE,IAAI,EAAE,KAAK,IAAI,4CAEzE;AAED;;;;GAIG;AACH,wBAAgB,yBAAyB,CACvC,QAAQ,EAAE,CAAC,KAAK,EAAE;IAAE,WAAW,EAAE,MAAM,CAAC;IAAC,YAAY,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,OAAO,CAAA;CAAE,KAAK,IAAI,4CAG3F;AAED;;;;GAIG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,CAAC,KAAK,EAAE;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,KAAK,IAAI,4CAE5F;AAED;;;;GAIG;AACH,wBAAgB,4BAA4B,CAAC,QAAQ,EAAE,CAAC,KAAK,EAAE;IAAE,WAAW,EAAE,OAAO,CAAA;CAAE,KAAK,IAAI,4CAE/F;AAMD;;GAEG;AACH,eAAO,MAAM,SAAS;;;;;CAKrB,CAAC;AAEF,eAAe,iBAAiB,CAAC"}
|