@souscheflabs/ml-vision 0.1.1 → 0.1.3

package/dist/esm/hooks/useProductDetector.js

@@ -0,0 +1,714 @@
+/**
+ * useProductDetector Hook
+ *
+ * React hook for visual product detection using TFLite models.
+ * Detects products in photos or camera frames using YOLOv8.
+ *
+ * @example
+ * ```typescript
+ * import { useProductDetector } from '@souschef/ml-vision';
+ *
+ * function FridgeScanner() {
+ *   const {
+ *     isModelLoaded,
+ *     isDetecting,
+ *     detections,
+ *     detectProducts,
+ *   } = useProductDetector({
+ *     minConfidence: 0.5,
+ *     onDetected: (products) => console.log('Found:', products),
+ *   });
+ *
+ *   const handleCapture = async (photoUri: string) => {
+ *     const results = await detectProducts(photoUri);
+ *     // results contain detected products with bounding boxes
+ *   };
+ *
+ *   return (
+ *     // Your camera UI
+ *   );
+ * }
+ * ```
+ */
+import { useState, useCallback, useEffect, useRef, useMemo } from 'react';
+import { Platform, Image } from 'react-native';
+import { useMLVisionContext } from '../core/MLVisionProvider';
+import { CLASS_LABELS, getClassInfo, NUM_CLASSES } from './classLabels';
+import { NUM_CLASSES_COCO, getCocoClassInfo } from './classLabelsCoco';
+import { preprocessImage, isSkiaAvailable, initSkia } from '../utils/imagePreprocessor';
+import { processYoloOutput as processYoloOutputUtil, scaleDetections } from '../utils/yoloProcessor';
+import { logger } from '../utils/logger';
+/**
+ * Singleton class for managing TFLite model lifecycle
+ * Ensures only one model instance exists and handles cleanup properly
+ */
+class TFLiteModelManager {
+    constructor() {
+        this.tfliteModule = null;
+        this.model = null;
+        this.loadPromise = null;
+        this.refCount = 0;
+    }
+    static getInstance() {
+        if (!TFLiteModelManager.instance) {
+            TFLiteModelManager.instance = new TFLiteModelManager();
+        }
+        return TFLiteModelManager.instance;
+    }
+    /**
+     * Load the TFLite module
+     */
+    async loadModule() {
+        if (this.tfliteModule)
+            return this.tfliteModule;
+        try {
+            this.tfliteModule = await import('react-native-fast-tflite');
+            return this.tfliteModule;
+        }
+        catch (error) {
+            logger.error('Failed to load react-native-fast-tflite:', error);
+            return null;
+        }
+    }
+    /**
+     * Get or load the TFLite model
+     */
+    async getModel(modelAsset, delegate) {
+        // If already loaded, increment ref count and return
+        if (this.model) {
+            this.refCount++;
+            return this.model;
+        }
+        // If loading in progress, wait for it
+        if (this.loadPromise) {
+            const result = await this.loadPromise;
+            if (result)
+                this.refCount++;
+            return result;
+        }
+        // Start loading
+        this.loadPromise = this.loadModelInternal(modelAsset, delegate);
+        const result = await this.loadPromise;
+        this.loadPromise = null;
+        if (result) {
+            this.model = result;
+            this.refCount = 1;
+        }
+        return result;
+    }
+    async loadModelInternal(modelAsset, delegate) {
+        const tflite = await this.loadModule();
+        if (!tflite)
+            return null;
+        try {
+            logger.debug('Loading TFLite model...');
+            let model;
+            try {
+                model = await tflite.loadTensorflowModel(modelAsset, delegate);
+            }
+            catch (loadErr) {
+                logger.warn('Primary model load failed:', loadErr.message);
+                // Fallback for Android assets
+                if (Platform.OS === 'android') {
+                    logger.debug('Trying Android asset fallback...');
+                    model = await tflite.loadTensorflowModel({ url: 'file:///android_asset/souschef_products_v2_int8.tflite' }, delegate);
+                    logger.info('Android asset fallback succeeded');
+                }
+                else {
+                    throw loadErr;
+                }
+            }
+            logger.info('TFLite model loaded successfully');
+            logger.debug('Model inputs:', JSON.stringify(model.inputs));
+            logger.debug('Model outputs:', JSON.stringify(model.outputs));
+            return model;
+        }
+        catch (error) {
+            logger.error('Failed to load TFLite model:', error);
+            return null;
+        }
+    }
+    /**
+     * Release a reference to the model
+     * Model is unloaded when ref count reaches 0
+     */
+    releaseModel() {
+        if (this.refCount > 0) {
+            this.refCount--;
+            logger.debug(`Model ref count: ${this.refCount}`);
+        }
+        // Note: react-native-fast-tflite doesn't expose a dispose method
+        // The model will be garbage collected when no longer referenced
+        // We keep the model cached for performance (hot reload scenarios)
+    }
+    /**
+     * Force unload the model (for testing or memory pressure)
+     */
+    forceUnload() {
+        this.model = null;
+        this.refCount = 0;
+        logger.debug('Model force unloaded');
+    }
+    /**
+     * Check if model is loaded
+     */
+    isLoaded() {
+        return this.model !== null;
+    }
+    /**
+     * Get current ref count (for debugging)
+     */
+    getRefCount() {
+        return this.refCount;
+    }
+}
+// Export singleton instance for use in hook
+const modelManager = TFLiteModelManager.getInstance();
+// ============================================================================
+// Constants
+// ============================================================================
+/** Model input size (must match training config) */
+const MODEL_INPUT_SIZE = 640;
+/**
+ * Model asset - loaded via Metro bundler
+ * The require() returns an asset ID that react-native-fast-tflite can resolve
+ *
+ * Using SousChef products v2 model (43 food classes)
+ */
+const MODEL_ASSET = require('../../models/souschef_products_v2_int8.tflite');
+/** Whether we're using the COCO model (80 classes) or custom food model (43 classes) */
+const USE_COCO_MODEL = false;
+/** Get the appropriate number of classes based on model */
+const getNumClasses = () => USE_COCO_MODEL ? NUM_CLASSES_COCO : NUM_CLASSES;
+/** Model version identifier */
+const MODEL_VERSION = '2.0.0';
+/**
+ * Simple hash function for image URIs
+ * For a real implementation, use a perceptual hash or file content hash
+ */
+function simpleHash(str) {
+    let hash = 5381;
+    for (let i = 0; i < str.length; i++) {
+        hash = (hash * 33 + str.charCodeAt(i)) % 2147483647;
+    }
+    return `img_${Math.abs(hash).toString(36)}`;
+}
+/**
+ * Process YOLOv8 output tensor to get detections
+ *
+ * YOLOv8 output format: [1, 34, 8400] where:
+ * - 34 = 4 (bbox) + 30 (class scores)
+ * - 8400 = number of anchor boxes (80x80 + 40x40 + 20x20)
+ *
+ * @deprecated Use processYoloOutput from utils/yoloProcessor instead
+ */
+function _processYoloOutput(output, numClasses = NUM_CLASSES, confThreshold = 0.25, iouThreshold = 0.45) {
+    const numAnchors = 8400;
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    const numOutputs = 4 + numClasses; // x, y, w, h + class scores
+    const detections = [];
+    // Process each anchor
+    for (let i = 0; i < numAnchors; i++) {
+        // Extract bbox: [x_center, y_center, width, height] (normalized 0-1)
+        const x = output[0 * numAnchors + i];
+        const y = output[1 * numAnchors + i];
+        const w = output[2 * numAnchors + i];
+        const h = output[3 * numAnchors + i];
+        // Find max class score and index
+        let maxScore = 0;
+        let maxClassIndex = 0;
+        for (let c = 0; c < numClasses; c++) {
+            const score = output[(4 + c) * numAnchors + i];
+            if (score > maxScore) {
+                maxScore = score;
+                maxClassIndex = c;
+            }
+        }
+        // Filter by confidence
+        if (maxScore >= confThreshold) {
+            detections.push({
+                x: x - w / 2, // Convert from center to top-left
+                y: y - h / 2,
+                width: w,
+                height: h,
+                classIndex: maxClassIndex,
+                confidence: maxScore,
+            });
+        }
+    }
+    // Apply Non-Maximum Suppression (NMS)
+    return applyNMS(detections, iouThreshold);
+}
+/**
+ * Non-Maximum Suppression to remove overlapping detections
+ */
+function applyNMS(detections, iouThreshold) {
+    if (detections.length === 0)
+        return [];
+    // Sort by confidence (descending)
+    const sorted = [...detections].sort((a, b) => b.confidence - a.confidence);
+    const kept = [];
+    while (sorted.length > 0) {
+        const best = sorted.shift();
+        kept.push(best);
+        // Remove overlapping detections of same class
+        for (let i = sorted.length - 1; i >= 0; i--) {
+            if (sorted[i].classIndex === best.classIndex) {
+                const iou = calculateIoU(best, sorted[i]);
+                if (iou > iouThreshold) {
+                    sorted.splice(i, 1);
+                }
+            }
+        }
+    }
+    return kept;
+}
+/**
+ * Calculate Intersection over Union (IoU) between two boxes
+ */
+function calculateIoU(box1, box2) {
+    const x1 = Math.max(box1.x, box2.x);
+    const y1 = Math.max(box1.y, box2.y);
+    const x2 = Math.min(box1.x + box1.width, box2.x + box2.width);
+    const y2 = Math.min(box1.y + box1.height, box2.y + box2.height);
+    const intersection = Math.max(0, x2 - x1) * Math.max(0, y2 - y1);
+    const area1 = box1.width * box1.height;
+    const area2 = box2.width * box2.height;
+    const union = area1 + area2 - intersection;
+    return union > 0 ? intersection / union : 0;
+}
+// ============================================================================
+// Hook Implementation
+// ============================================================================
+/**
+ * Hook for visual product detection using TFLite
+ */
+export function useProductDetector(options = {}) {
+    const { model: modelType = 'fast', maxDetections = 20, minConfidence = 0.5, categories, serverFallback = true, serverFallbackThreshold = 0.4, serverUrl, onDetected, onError, hapticFeedback = true, } = options;
+    // Context
+    const context = useMLVisionContext();
+    const { serverClient, serverAvailable, config, cacheManager } = context;
+    // State
+    const [isModelLoaded, setIsModelLoaded] = useState(false);
+    const [isDetecting, setIsDetecting] = useState(false);
+    const [detections, setDetections] = useState([]);
+    const [error, setError] = useState(null);
+    // Refs
+    const modelRef = useRef(null);
+    const mountedRef = useRef(true);
+    // Computed server URL
+    const effectiveServerUrl = serverUrl || config.serverUrl;
+    // ============================================================================
+    // Model Loading
+    // ============================================================================
+    useEffect(() => {
+        mountedRef.current = true;
+        async function loadModel() {
+            if (modelType !== 'fast') {
+                // Server-only mode
+                setIsModelLoaded(true);
+                return;
+            }
+            try {
+                logger.debug('Loading TFLite model via singleton manager...');
+                logger.debug('MODEL_ASSET:', MODEL_ASSET, typeof MODEL_ASSET);
+                // CoreML is iOS-only, use default CPU delegate on Android for stability
+                const delegate = config.enableGPUDelegate && Platform.OS === 'ios'
+                    ? 'core-ml'
+                    : 'default';
+                logger.debug('Using delegate:', delegate);
+                // Use singleton model manager
+                const model = await modelManager.getModel(MODEL_ASSET, delegate);
+                if (!model) {
+                    throw new Error('Failed to load TFLite model');
+                }
+                modelRef.current = model;
+                if (mountedRef.current) {
+                    setIsModelLoaded(true);
+                    logger.info('Model ready for inference');
+                }
+            }
+            catch (err) {
+                const loadError = err instanceof Error ? err : new Error('Failed to load model');
+                logger.error('Model loading error:', loadError);
+                if (mountedRef.current) {
+                    setError(loadError);
+                    onError?.(loadError);
+                }
+            }
+        }
+        loadModel();
+        return () => {
+            mountedRef.current = false;
+            // Release model reference on unmount
+            modelManager.releaseModel();
+        };
+    }, [modelType, config.enableGPUDelegate, onError]);
+    // ============================================================================
+    // Detection Functions
+    // ============================================================================
+    /**
+     * Detect products in a photo
+     */
+    const detectProducts = useCallback(async (uri) => {
+        if (isDetecting) {
+            logger.warn('Detection already in progress');
+            return [];
+        }
+        setIsDetecting(true);
+        setError(null);
+        try {
+            let results = [];
+            const imageHash = simpleHash(uri);
+            // Check cache first
+            if (cacheManager) {
+                const cached = cacheManager.getProductRecognition(imageHash);
+                if (cached && cached.modelVersion === MODEL_VERSION) {
+                    logger.debug('[Detection] Using cached results');
+                    // Convert cached data to ProductDetectionResult format
+                    results = cached.detections.map((det, idx) => ({
+                        id: `cached_${imageHash}_${idx}`,
+                        type: 'product',
+                        confidence: det.confidence,
+                        boundingBox: det.boundingBox,
+                        data: {
+                            category: getClassInfo(parseInt(Object.entries(CLASS_LABELS).find(([, v]) => v.label === det.classLabel)?.[0] ?? '0', 10)).category,
+                            classLabel: det.classLabel,
+                            classIndex: parseInt(Object.entries(CLASS_LABELS).find(([, v]) => v.label === det.classLabel)?.[0] ?? '0', 10),
+                            name: det.classLabel.replace(/_/g, ' '),
+                        },
+                        source: 'on_device',
+                        processingTimeMs: 0, // Cached, no processing time
+                        timestamp: Date.now(),
+                    }));
+                    if (mountedRef.current) {
+                        setDetections(results);
+                        setIsDetecting(false);
+                    }
+                    onDetected?.(results);
+                    return results;
+                }
+            }
+            // Get image dimensions
+            const imageSize = await new Promise((resolve, reject) => {
+                Image.getSize(uri, (width, height) => resolve({ width, height }), reject);
+            });
+            // Try on-device detection first
+            if (modelType === 'fast' && modelRef.current) {
+                try {
+                    results = await runOnDeviceDetection(uri, imageSize.width, imageSize.height);
+                }
+                catch {
+                    // On-device detection failed (e.g., preprocessing not implemented)
+                    // This is expected for photo-based detection - silently fall back to server
+                    logger.debug('[Detection] On-device detection unavailable, will use server');
+                }
+            }
+            // Server fallback for:
+            // - Server-only mode (modelType === 'accurate')
+            // - On-device detection failed or returned no results
+            // - Low confidence results
+            const shouldFallback = modelType === 'accurate' ||
+                (serverFallback &&
+                    effectiveServerUrl &&
+                    serverAvailable &&
+                    (results.length === 0 ||
+                        results.every((r) => r.confidence < serverFallbackThreshold)));
+            if (shouldFallback && serverClient) {
+                logger.debug('[Detection] Using server detection');
+                try {
+                    const serverResults = await runServerDetection(uri);
+                    // Merge results, preferring server for duplicates
+                    results = mergeDetections(results, serverResults);
+                }
+                catch (serverErr) {
+                    logger.error('[Detection] Server detection failed:', serverErr);
+                    // If server fails too, we have no results - throw the error
+                    if (results.length === 0) {
+                        throw new Error('Detection failed: Server unavailable and on-device preprocessing not implemented');
+                    }
+                }
+            }
+            // Filter by confidence and categories
+            results = results
+                .filter((r) => r.confidence >= minConfidence)
+                .filter((r) => !categories || categories.includes(r.data.category))
+                .slice(0, maxDetections);
+            // Update state
+            if (mountedRef.current) {
+                setDetections(results);
+            }
+            // Cache results
+            if (cacheManager && results.length > 0) {
+                const cacheData = {
+                    imageHash,
+                    detections: results.map((r) => ({
+                        classLabel: r.data.classLabel,
+                        confidence: r.confidence,
+                        boundingBox: r.boundingBox,
+                    })),
+                    modelVersion: MODEL_VERSION,
+                };
+                cacheManager.setProductRecognition(imageHash, cacheData);
+            }
+            // Callbacks
+            if (results.length > 0) {
+                onDetected?.(results);
+                if (hapticFeedback) {
+                    // Trigger haptic feedback (implement based on your haptic service)
+                    // HapticService.success();
+                }
+            }
+            return results;
+        }
+        catch (err) {
+            const detectError = err instanceof Error ? err : new Error('Detection failed');
+            logger.error('[Detection] Detection error:', detectError);
+            if (mountedRef.current) {
+                setError(detectError);
+            }
+            onError?.(detectError);
+            return [];
+        }
+        finally {
+            if (mountedRef.current) {
+                setIsDetecting(false);
+            }
+        }
+    }, 
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+    [
+        isDetecting,
+        modelType,
+        serverFallback,
+        serverFallbackThreshold,
+        effectiveServerUrl,
+        serverAvailable,
+        serverClient,
+        cacheManager,
+        minConfidence,
+        categories,
+        maxDetections,
+        onDetected,
+        onError,
+        hapticFeedback,
+        // Note: runOnDeviceDetection and runServerDetection are intentionally excluded
+        // They are stable helper functions defined within this hook
+    ]);
+    /**
+     * Run detection on device using TFLite
+     *
+     * Requires @shopify/react-native-skia for image preprocessing.
+     * Install with: npm install @shopify/react-native-skia
+     *
+     * For real-time detection (VisionCamera frame processor), use the
+     * vision-camera-resize-plugin which provides direct pixel access.
+     */
+    async function runOnDeviceDetection(uri, imageWidth, imageHeight) {
+        const model = modelRef.current;
+        if (!model) {
+            throw new Error('Model not loaded');
+        }
+        const startTime = Date.now();
+        // Try to initialize Skia for preprocessing
+        await initSkia();
+        if (!isSkiaAvailable()) {
+            logger.debug('[Detection] On-device detection requires @shopify/react-native-skia');
+            logger.debug('[Detection] Install with: npm install @shopify/react-native-skia');
+            logger.debug('[Detection] Falling back to server detection...');
+            throw new Error('On-device photo detection requires @shopify/react-native-skia. ' +
+                'Install it or use server fallback.');
+        }
+        logger.debug('[Detection] Running on-device detection...');
+        logger.debug('[Detection] Image:', imageWidth, 'x', imageHeight);
+        try {
+            // Preprocess image: resize to 640x640 and normalize to [0, 1]
+            const preprocessed = await preprocessImage(uri, {
+                width: MODEL_INPUT_SIZE,
+                height: MODEL_INPUT_SIZE,
+                normalize: true,
+                letterbox: true,
+            });
+            logger.debug('[Detection] Preprocessed image, running inference...');
+            // Run TFLite inference
+            const output = await model.run([preprocessed.data]);
+            const rawOutput = output[0];
+            const inferenceTime = Date.now() - startTime;
+            logger.debug('[Detection] Inference complete:', inferenceTime, 'ms');
+            // Process YOLO output
+            const numClasses = getNumClasses();
+            const rawDetections = processYoloOutputUtil(rawOutput, numClasses, 0.25, 0.45);
+            // Scale detections back to original image coordinates
+            const scaledDetections = scaleDetections(rawDetections, MODEL_INPUT_SIZE, imageWidth, imageHeight);
+            logger.debug('[Detection] Found', scaledDetections.length, 'detections');
+            // Convert to ProductDetectionResult format
+            const processingTimeMs = Date.now() - startTime;
+            return scaledDetections.map((det) => toProductDetectionResult(det, imageWidth, imageHeight, processingTimeMs));
+        }
+        catch (preprocessError) {
+            logger.error('[Detection] On-device detection failed:', preprocessError);
+            throw preprocessError;
+        }
+    }
+    /**
+     * Convert raw detection to ProductDetectionResult
+     */
+    function toProductDetectionResult(det, imageWidth, imageHeight, processingTimeMs) {
+        // Get class info based on model type
+        const classInfo = USE_COCO_MODEL
+            ? getCocoClassInfo(det.classIndex)
+            : getClassInfo(det.classIndex);
+        return {
+            id: `det_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`,
+            type: 'product',
+            confidence: det.confidence,
+            boundingBox: {
+                x: det.x,
+                y: det.y,
+                width: det.width,
+                height: det.height,
+            },
+            data: {
+                category: classInfo.category,
+                classLabel: classInfo.label,
+                classIndex: det.classIndex,
+                name: classInfo.label.replace(/_/g, ' '),
+            },
+            source: 'on_device',
+            processingTimeMs,
+            timestamp: Date.now(),
+        };
+    }
+    /**
+     * Run detection on server
+     */
+    async function runServerDetection(uri) {
+        logger.debug('[Detection] runServerDetection called');
+        logger.debug('[Detection] Server client available:', !!serverClient);
+        logger.debug('[Detection] Server URL:', effectiveServerUrl);
+        logger.debug('[Detection] Image URI:', uri);
+        if (!serverClient) {
+            logger.error('[Detection] Server client not available!');
+            throw new Error('Server client not available');
+        }
+        // Convert URI to ImageUploadData format
+        const imageData = {
+            image: uri,
+            isBase64: false,
+        };
+        logger.debug('[Detection] Calling serverClient.detectProducts...');
+        try {
+            const results = await serverClient.detectProducts(imageData, {
+                maxDetections,
+                minConfidence: minConfidence * 0.8, // Slightly lower threshold for server
+            });
+            logger.debug('[Detection] Server returned', results?.length || 0, 'detections');
+            logger.debug('[Detection] Server results:', JSON.stringify(results));
+            // Server already returns ProductDetectionResult[] with all required fields
+            return results;
+        }
+        catch (serverErr) {
+            logger.error('[Detection] Server request failed:', serverErr);
+            throw serverErr;
+        }
+    }
+    /**
+     * Merge on-device and server detections
+     */
+    function mergeDetections(onDevice, server) {
+        // Simple merge: include all server detections, add on-device only if no overlap
+        const merged = [...server];
+        for (const det of onDevice) {
+            const overlaps = server.some((s) => s.data.classLabel === det.data.classLabel &&
+                calculateIoU({
+                    x: det.boundingBox.x,
+                    y: det.boundingBox.y,
+                    width: det.boundingBox.width,
+                    height: det.boundingBox.height,
+                    classIndex: det.data.classIndex,
+                    confidence: det.confidence,
+                }, {
+                    x: s.boundingBox.x,
+                    y: s.boundingBox.y,
+                    width: s.boundingBox.width,
+                    height: s.boundingBox.height,
+                    classIndex: s.data.classIndex,
+                    confidence: s.confidence,
+                }) > 0.5);
+            if (!overlaps) {
+                merged.push(det);
+            }
+        }
+        return merged;
+    }
+    /**
+     * Detect products in current camera frame (placeholder)
+     */
+    const detectInFrame = useCallback(async () => {
+        // This would be called from a frame processor
+        // For now, return current detections
+        return detections;
+    }, [detections]);
+    /**
+     * Update model (check for new version)
+     */
+    const updateModel = useCallback(async () => {
+        // TODO: Implement model update checking
+        logger.debug('[Detection] Model update check not implemented');
+    }, []);
+    /**
+     * Clear all detections
+     */
+    const clearDetections = useCallback(() => {
+        setDetections([]);
+        setError(null);
+    }, []);
+    // ============================================================================
+    // Frame Processor (placeholder)
+    // ============================================================================
+    // Frame processor for real-time detection would be implemented here
+    // This requires worklet functions from react-native-vision-camera
+    const frameProcessor = useMemo(() => {
+        // Placeholder - actual implementation needs VisionCamera worklets
+        return null;
+    }, []);
+    // ============================================================================
+    // Model Info
+    // ============================================================================
+    const modelInfo = useMemo(() => {
+        if (!isModelLoaded)
+            return null;
+        return {
+            name: 'souschef_products_v2_int8',
+            version: MODEL_VERSION,
+            sizeBytes: 26392350, // ~25 MB (43 food classes, YOLOv8m)
+            lastUpdated: new Date('2026-01-19'),
+            type: 'detection',
+            isLoaded: isModelLoaded,
+        };
+    }, [isModelLoaded]);
+    // ============================================================================
+    // Return Value
+    // ============================================================================
+    return {
+        // State
+        isModelLoaded,
+        isDetecting,
+        detections,
+        error,
+        modelVersion: MODEL_VERSION,
+        // Model (for frame processor use)
+        model: modelRef.current,
+        // Actions
+        detectProducts,
+        detectInFrame,
+        updateModel,
+        clearDetections,
+        // Frame processor
+        frameProcessor,
+        // Model info
+        modelInfo,
+    };
+}
+export default useProductDetector;
+//# sourceMappingURL=useProductDetector.js.map