@souscheflabs/ml-vision 0.1.0

package/dist/processors/detectionProcessor.js

@@ -0,0 +1,124 @@
+"use strict";
+/**
+ * Detection Frame Processor
+ *
+ * VisionCamera frame processor for real-time product detection.
+ * Uses react-native-fast-tflite for on-device inference.
+ *
+ * @example
+ * ```typescript
+ * import { useFrameProcessor } from 'react-native-vision-camera';
+ * import { detectProducts } from '@souschef/ml-vision/processors';
+ *
+ * function CameraScreen() {
+ *   const frameProcessor = useFrameProcessor((frame) => {
+ *     'worklet';
+ *     const detections = detectProducts(frame);
+ *     // Handle detections...
+ *   }, []);
+ *
+ *   return <Camera frameProcessor={frameProcessor} />;
+ * }
+ * ```
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CLASS_LABELS = exports.NUM_CLASSES = void 0;
+exports.createDetectionProcessor = createDetectionProcessor;
+exports.frameDetectionToResult = frameDetectionToResult;
+exports.getClassLabel = getClassLabel;
+const classLabels_1 = require("../hooks/classLabels");
+Object.defineProperty(exports, "CLASS_LABELS", { enumerable: true, get: function () { return classLabels_1.CLASS_LABELS; } });
+Object.defineProperty(exports, "NUM_CLASSES", { enumerable: true, get: function () { return classLabels_1.NUM_CLASSES; } });
+// ============================================================================
+// Frame Counter for Skipping
+// ============================================================================
+let frameCounter = 0;
+// ============================================================================
+// Frame Processor Plugin
+// ============================================================================
+/**
+ * Create a detection frame processor
+ *
+ * This returns a function that can be used with VisionCamera's useFrameProcessor.
+ *
+ * @example
+ * ```typescript
+ * const detectFrame = createDetectionProcessor({
+ *   minConfidence: 0.5,
+ *   maxDetections: 10,
+ * });
+ *
+ * const frameProcessor = useFrameProcessor((frame) => {
+ *   'worklet';
+ *   const detections = detectFrame(frame);
+ *   runOnJS(handleDetections)(detections);
+ * }, []);
+ * ```
+ */
+function createDetectionProcessor(options = {}) {
+    const { frameSkip = 3 } = options;
+    /**
+     * The actual frame processor function
+     *
+     * Note: This is a placeholder that shows the expected interface.
+     * The actual TFLite inference requires native module integration
+     * via react-native-fast-tflite's useRunOnce or similar.
+     */
+    return function detectProductsInFrame(_frame) {
+        'worklet';
+        // Frame skipping for performance
+        frameCounter++;
+        if (frameCounter % frameSkip !== 0) {
+            return [];
+        }
+        // In a real implementation, this would:
+        // 1. Get frame data from VisionCamera
+        // 2. Preprocess for TFLite input
+        // 3. Run inference via react-native-fast-tflite
+        // 4. Post-process output
+        // Placeholder - actual implementation requires native TFLite integration
+        // with VisionCamera frame processors using Reanimated worklets
+        return [];
+    };
+}
+// ============================================================================
+// Utility Functions
+// ============================================================================
+/**
+ * Convert FrameDetection to ProductDetectionResult
+ *
+ * Call this on the JS thread after receiving worklet results.
+ */
+function frameDetectionToResult(detection, imageWidth, imageHeight) {
+    const classInfo = classLabels_1.CLASS_LABELS[detection.classIndex] ?? {
+        label: `class_${detection.classIndex}`,
+        category: 'unknown',
+    };
+    const boundingBox = {
+        x: detection.box.x * imageWidth,
+        y: detection.box.y * imageHeight,
+        width: detection.box.width * imageWidth,
+        height: detection.box.height * imageHeight,
+    };
+    return {
+        id: `frame_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`,
+        type: 'product',
+        confidence: detection.confidence,
+        boundingBox,
+        data: {
+            category: classInfo.category,
+            classLabel: classInfo.label,
+            classIndex: detection.classIndex,
+            name: classInfo.label.replace(/_/g, ' '),
+        },
+        source: 'on_device',
+        processingTimeMs: 0, // Frame processor timing handled externally
+        timestamp: Date.now(),
+    };
+}
+/**
+ * Get label for class index (can be called from worklet via shared value)
+ */
+function getClassLabel(classIndex) {
+    return classLabels_1.CLASS_LABELS[classIndex]?.label ?? `class_${classIndex}`;
+}

package/dist/processors/index.d.ts

@@ -0,0 +1,5 @@
+/**
+ * Processors exports
+ */
+export { createDetectionProcessor, frameDetectionToResult, getClassLabel, NUM_CLASSES, CLASS_LABELS, type FrameDetection, type DetectionProcessorOptions, } from './detectionProcessor';
+export { useTFLiteFrameProcessor, createTFLiteFrameProcessor, type FrameDetection as TFLiteFrameDetection, type FrameProcessorResult, type UseTFLiteFrameProcessorOptions, } from './tfliteFrameProcessor';

package/dist/processors/index.js

@@ -0,0 +1,16 @@
+"use strict";
+/**
+ * Processors exports
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createTFLiteFrameProcessor = exports.useTFLiteFrameProcessor = exports.CLASS_LABELS = exports.NUM_CLASSES = exports.getClassLabel = exports.frameDetectionToResult = exports.createDetectionProcessor = void 0;
+var detectionProcessor_1 = require("./detectionProcessor");
+Object.defineProperty(exports, "createDetectionProcessor", { enumerable: true, get: function () { return detectionProcessor_1.createDetectionProcessor; } });
+Object.defineProperty(exports, "frameDetectionToResult", { enumerable: true, get: function () { return detectionProcessor_1.frameDetectionToResult; } });
+Object.defineProperty(exports, "getClassLabel", { enumerable: true, get: function () { return detectionProcessor_1.getClassLabel; } });
+Object.defineProperty(exports, "NUM_CLASSES", { enumerable: true, get: function () { return detectionProcessor_1.NUM_CLASSES; } });
+Object.defineProperty(exports, "CLASS_LABELS", { enumerable: true, get: function () { return detectionProcessor_1.CLASS_LABELS; } });
+// TFLite Frame Processor for real-time video scanning
+var tfliteFrameProcessor_1 = require("./tfliteFrameProcessor");
+Object.defineProperty(exports, "useTFLiteFrameProcessor", { enumerable: true, get: function () { return tfliteFrameProcessor_1.useTFLiteFrameProcessor; } });
+Object.defineProperty(exports, "createTFLiteFrameProcessor", { enumerable: true, get: function () { return tfliteFrameProcessor_1.createTFLiteFrameProcessor; } });

package/dist/processors/tfliteFrameProcessor.d.ts

@@ -0,0 +1,90 @@
+/**
+ * TFLite Frame Processor for Real-time Video Detection
+ *
+ * Uses vision-camera-resize-plugin to resize frames and run
+ * TFLite inference directly in the frame processor worklet.
+ *
+ * Requirements:
+ * - npm install vision-camera-resize-plugin
+ * - npm install react-native-fast-tflite
+ * - npm install react-native-worklets-core
+ */
+import type { TensorflowModel } from 'react-native-fast-tflite';
+import type { Frame } from 'react-native-vision-camera';
+import { useResizePlugin } from 'vision-camera-resize-plugin';
+export interface FrameDetection {
+    x: number;
+    y: number;
+    width: number;
+    height: number;
+    classIndex: number;
+    confidence: number;
+    label: string;
+}
+export interface FrameProcessorResult {
+    detections: FrameDetection[];
+    inferenceTimeMs: number;
+    frameWidth: number;
+    frameHeight: number;
+}
+export interface UseTFLiteFrameProcessorOptions {
+    /** Loaded TFLite model */
+    model: TensorflowModel | null;
+    /** Model input size (default: 640) */
+    inputSize?: number;
+    /** Number of classes in model (default: 80 for COCO) */
+    numClasses?: number;
+    /** Confidence threshold (default: 0.25) */
+    confThreshold?: number;
+    /** IoU threshold for NMS (default: 0.45) */
+    iouThreshold?: number;
+    /** Class labels array */
+    classLabels?: string[];
+    /** Only return food classes (for COCO) */
+    foodOnly?: boolean;
+    /** Callback when detections are found */
+    onDetections?: (detections: FrameDetection[]) => void;
+}
+/**
+ * Hook to create a TFLite-powered frame processor
+ *
+ * @example
+ * ```typescript
+ * const { frameProcessor, detections, isProcessing } = useTFLiteFrameProcessor({
+ *   model: loadedModel,
+ *   numClasses: 80,
+ *   foodOnly: true,
+ * });
+ *
+ * // Use with VisionCamera
+ * <Camera frameProcessor={frameProcessor} />
+ * ```
+ */
+export declare function useTFLiteFrameProcessor(options: UseTFLiteFrameProcessorOptions): {
+    frameProcessor: ((frame: Frame) => void) | undefined;
+    detections: FrameDetection[];
+    isProcessing: boolean;
+    isReady: boolean;
+};
+/** Type for the resize function from useResizePlugin */
+type ResizeFunction = ReturnType<typeof useResizePlugin>['resize'];
+/**
+ * Create a simple frame processor function for use with useFrameProcessor
+ *
+ * This is for manual integration when you need more control.
+ * The resize function must be obtained from useResizePlugin hook.
+ *
+ * @example
+ * ```typescript
+ * const { resize } = useResizePlugin();
+ * const processor = createTFLiteFrameProcessor(model, resize, { foodOnly: true });
+ * ```
+ */
+export declare function createTFLiteFrameProcessor(model: TensorflowModel, resizeFn: ResizeFunction, options?: {
+    inputSize?: number;
+    numClasses?: number;
+    confThreshold?: number;
+    classLabels?: string[];
+    foodOnly?: boolean;
+}): (frame: Frame) => FrameDetection[];
+export {};

package/dist/processors/tfliteFrameProcessor.js

@@ -0,0 +1,213 @@
+"use strict";
+/**
+ * TFLite Frame Processor for Real-time Video Detection
+ *
+ * Uses vision-camera-resize-plugin to resize frames and run
+ * TFLite inference directly in the frame processor worklet.
+ *
+ * Requirements:
+ * - npm install vision-camera-resize-plugin
+ * - npm install react-native-fast-tflite
+ * - npm install react-native-worklets-core
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.useTFLiteFrameProcessor = useTFLiteFrameProcessor;
+exports.createTFLiteFrameProcessor = createTFLiteFrameProcessor;
+const react_1 = require("react");
+const react_native_reanimated_1 = require("react-native-reanimated");
+const react_native_worklets_core_1 = require("react-native-worklets-core");
+const vision_camera_resize_plugin_1 = require("vision-camera-resize-plugin");
+// COCO food class indices
+const COCO_FOOD_INDICES = new Set([46, 47, 48, 49, 50, 51, 52, 53, 54, 55]);
+// COCO class names
+const COCO_LABELS = [
+    'person', 'bicycle', 'car', 'motorcycle', 'airplane', 'bus', 'train', 'truck',
+    'boat', 'traffic light', 'fire hydrant', 'stop sign', 'parking meter', 'bench',
+    'bird', 'cat', 'dog', 'horse', 'sheep', 'cow', 'elephant', 'bear', 'zebra',
+    'giraffe', 'backpack', 'umbrella', 'handbag', 'tie', 'suitcase', 'frisbee',
+    'skis', 'snowboard', 'sports ball', 'kite', 'baseball bat', 'baseball glove',
+    'skateboard', 'surfboard', 'tennis racket', 'bottle', 'wine glass', 'cup',
+    'fork', 'knife', 'spoon', 'bowl', 'banana', 'apple', 'sandwich', 'orange',
+    'broccoli', 'carrot', 'hot dog', 'pizza', 'donut', 'cake', 'chair', 'couch',
+    'potted plant', 'bed', 'dining table', 'toilet', 'tv', 'laptop', 'mouse',
+    'remote', 'keyboard', 'cell phone', 'microwave', 'oven', 'toaster', 'sink',
+    'refrigerator', 'book', 'clock', 'vase', 'scissors', 'teddy bear', 'hair drier',
+    'toothbrush'
+];
+/**
+ * Process YOLOv8 output in worklet (simplified for worklet compatibility)
+ * This runs synchronously in the frame processor thread
+ */
+function processYoloOutputWorklet(output, numClasses, confThreshold, _iouThreshold, labels, foodOnly) {
+    'worklet';
+    const data = new Float32Array(output);
+    const numPredictions = 8400;
+    const detections = [];
+    // Extract detections above threshold
+    for (let i = 0; i < numPredictions; i++) {
+        let maxScore = 0;
+        let maxClassIdx = 0;
+        // Find best class
+        for (let c = 0; c < numClasses; c++) {
+            const score = data[(4 + c) * numPredictions + i];
+            if (score > maxScore) {
+                maxScore = score;
+                maxClassIdx = c;
+            }
+        }
+        if (maxScore >= confThreshold) {
+            // Filter for food classes if requested
+            if (foodOnly && !COCO_FOOD_INDICES.has(maxClassIdx)) {
+                continue;
+            }
+            // Extract box (center x, center y, width, height)
+            const cx = data[0 * numPredictions + i];
+            const cy = data[1 * numPredictions + i];
+            const w = data[2 * numPredictions + i];
+            const h = data[3 * numPredictions + i];
+            detections.push({
+                x: Math.max(0, cx - w / 2),
+                y: Math.max(0, cy - h / 2),
+                width: w,
+                height: h,
+                classIndex: maxClassIdx,
+                confidence: maxScore,
+                label: labels[maxClassIdx] || `class_${maxClassIdx}`,
+            });
+        }
+    }
+    // Simple NMS (keep top detection per class)
+    const kept = [];
+    const seenClasses = new Set();
+    // Sort by confidence
+    detections.sort((a, b) => b.confidence - a.confidence);
+    for (const det of detections) {
+        if (!seenClasses.has(det.classIndex)) {
+            kept.push(det);
+            seenClasses.add(det.classIndex);
+        }
+    }
+    return kept.slice(0, 10); // Max 10 detections
+}
+/**
+ * Hook to create a TFLite-powered frame processor
+ *
+ * @example
+ * ```typescript
+ * const { frameProcessor, detections, isProcessing } = useTFLiteFrameProcessor({
+ *   model: loadedModel,
+ *   numClasses: 80,
+ *   foodOnly: true,
+ * });
+ *
+ * // Use with VisionCamera
+ * <Camera frameProcessor={frameProcessor} />
+ * ```
+ */
+function useTFLiteFrameProcessor(options) {
+    const { model, inputSize = 640, numClasses = 80, confThreshold = 0.25, iouThreshold = 0.45, classLabels = COCO_LABELS, foodOnly = false, onDetections, } = options;
+    const [detections, setDetections] = (0, react_1.useState)([]);
+    const [isProcessing, _setIsProcessing] = (0, react_1.useState)(false);
+    const [frameCount, setFrameCount] = (0, react_1.useState)(0);
+    // Use shared value for throttle timing (accessible from worklet)
+    const lastProcessTime = (0, react_native_reanimated_1.useSharedValue)(0);
+    // Use the resize plugin hook
+    const { resize } = (0, vision_camera_resize_plugin_1.useResizePlugin)();
+    // Log state on mount and when model changes
+    (0, react_1.useEffect)(() => {
+        console.log('[TFLiteFrameProcessor] Hook state:', {
+            model: !!model,
+            resize: !!resize,
+        });
+    }, [model, resize]);
+    // Debug: log frame count changes
+    (0, react_1.useEffect)(() => {
+        if (frameCount > 0) {
+            console.log(`[TFLiteFrameProcessor] Processed ${frameCount} frames`);
+        }
+    }, [frameCount]);
+    // Helper to log from worklet
+    const logFromWorklet = (0, react_1.useCallback)((msg) => {
+        console.log(msg);
+    }, []);
+    const incrementFrameCount = (0, react_1.useCallback)(() => {
+        setFrameCount(prev => prev + 1);
+    }, []);
+    // Create frame processor
+    const frameProcessor = (0, react_1.useCallback)((frame) => {
+        'worklet';
+        if (!model) {
+            react_native_worklets_core_1.Worklets.runOnJS(() => logFromWorklet('[TFLiteFrameProcessor] No model'));
+            return;
+        }
+        // Throttle to ~10fps for processing
+        const now = Date.now();
+        if (now - lastProcessTime.value < 100) {
+            return;
+        }
+        lastProcessTime.value = now;
+        react_native_worklets_core_1.Worklets.runOnJS(() => incrementFrameCount());
+        try {
+            // Resize frame to model input size and get RGB float32 tensor
+            const resized = resize(frame, {
+                scale: {
+                    width: inputSize,
+                    height: inputSize,
+                },
+                pixelFormat: 'rgb',
+                dataType: 'float32',
+            });
+            // Run TFLite inference synchronously in worklet
+            const outputs = model.runSync([resized]);
+            // Process YOLO output
+            // outputs[0] is a TypedArray, we need its underlying buffer
+            const outputData = outputs[0];
+            const frameDetections = processYoloOutputWorklet(outputData.buffer ?? outputs[0], numClasses, confThreshold, iouThreshold, classLabels, foodOnly);
+            // Update state on JS thread
+            if (frameDetections.length > 0) {
+                react_native_worklets_core_1.Worklets.runOnJS(() => setDetections(frameDetections));
+                if (onDetections) {
+                    react_native_worklets_core_1.Worklets.runOnJS(() => onDetections(frameDetections));
+                }
+            }
+        }
+        catch (error) {
+            react_native_worklets_core_1.Worklets.runOnJS(() => logFromWorklet(`[TFLiteFrameProcessor] Error: ${error}`));
+        }
+    }, [model, resize, inputSize, numClasses, confThreshold, iouThreshold, classLabels, foodOnly, onDetections, lastProcessTime, logFromWorklet, incrementFrameCount]);
+    return {
+        frameProcessor: model ? frameProcessor : undefined,
+        detections,
+        isProcessing,
+        isReady: !!model,
+    };
+}
+/**
+ * Create a simple frame processor function for use with useFrameProcessor
+ *
+ * This is for manual integration when you need more control.
+ * The resize function must be obtained from useResizePlugin hook.
+ *
+ * @example
+ * ```typescript
+ * const { resize } = useResizePlugin();
+ * const processor = createTFLiteFrameProcessor(model, resize, { foodOnly: true });
+ * ```
+ */
+function createTFLiteFrameProcessor(model, resizeFn, options = {}) {
+    const { inputSize = 640, numClasses = 80, confThreshold = 0.25, classLabels = COCO_LABELS, foodOnly = false, } = options;
+    return (frame) => {
+        'worklet';
+        // Resize to model input
+        const resized = resizeFn(frame, {
+            scale: { width: inputSize, height: inputSize },
+            pixelFormat: 'rgb',
+            dataType: 'float32',
+        });
+        // Run inference
+        const outputs = model.runSync([resized]);
+        // Process output - access the underlying buffer from TypedArray
+        const outputData = outputs[0];
+        return processYoloOutputWorklet(outputData.buffer ?? outputs[0], numClasses, confThreshold, 0.45, classLabels, foodOnly);
+    };
+}

package/dist/types/barcode.d.ts

@@ -0,0 +1,91 @@
+/**
+ * Barcode Detection Types
+ *
+ * Types for multi-barcode scanning functionality.
+ * Designed to be compatible with react-native-vision-camera's CodeScanner.
+ */
+import type { BaseDetectionResult } from './index';
+/**
+ * Supported barcode formats
+ * Matches VisionCamera's CodeType for compatibility
+ */
+export type BarcodeFormat = 'ean-13' | 'ean-8' | 'upc-a' | 'upc-e' | 'code-128' | 'code-39' | 'code-93' | 'itf' | 'codabar' | 'qr' | 'data-matrix' | 'pdf-417' | 'aztec';
+/**
+ * Common grocery barcode formats (optimized subset)
+ */
+export declare const GROCERY_BARCODE_FORMATS: BarcodeFormat[];
+/**
+ * Data extracted from a scanned barcode
+ */
+export interface BarcodeData {
+    /** The barcode format */
+    format: BarcodeFormat;
+    /** The decoded barcode value */
+    value: string;
+    /** Raw bytes if available */
+    rawValue?: string;
+    /** Display value (formatted for UI) */
+    displayValue?: string;
+}
+/**
+ * Complete barcode detection result
+ */
+export interface BarcodeDetectionResult extends BaseDetectionResult {
+    type: 'barcode';
+    data: BarcodeData;
+}
+/**
+ * Options for useMultiBarcodeScanner hook
+ */
+export interface UseMultiBarcodeScannerOptions {
+    /** Barcode formats to scan for (default: GROCERY_BARCODE_FORMATS) */
+    formats?: BarcodeFormat[];
+    /** Maximum number of barcodes to detect per frame (default: 20) */
+    maxBarcodes?: number;
+    /** Minimum interval between scans in ms (default: 100) */
+    scanInterval?: number;
+    /** Callback when barcodes are detected */
+    onDetected?: (barcodes: BarcodeDetectionResult[]) => void;
+    /** Callback on error */
+    onError?: (error: Error) => void;
+    /** Enable haptic feedback on detection (default: true) */
+    hapticFeedback?: boolean;
+    /** Deduplicate barcodes within session (default: true) */
+    deduplicate?: boolean;
+}
+/**
+ * Return type for useMultiBarcodeScanner hook
+ */
+export interface UseMultiBarcodeScannerReturn {
+    /** Whether the scanner is initialized and ready */
+    isReady: boolean;
+    /** Whether actively scanning */
+    isScanning: boolean;
+    /** Array of detected barcodes */
+    results: BarcodeDetectionResult[];
+    /** Current error if any */
+    error: Error | null;
+    /** Start continuous scanning */
+    startScanning: () => void;
+    /** Stop scanning */
+    stopScanning: () => void;
+    /** Scan a single photo for barcodes */
+    scanPhoto: (uri: string) => Promise<BarcodeDetectionResult[]>;
+    /** Clear all results */
+    clearResults: () => void;
+    /** Reset scanner state */
+    reset: () => void;
+    frameProcessor: unknown;
+}
+/**
+ * Cached barcode lookup result
+ * Maps barcode value to product data (from your backend)
+ */
+export interface BarcodeCacheEntry {
+    barcode: string;
+    format: BarcodeFormat;
+    productId?: string;
+    productName?: string;
+    cachedAt: number;
+    ttl: number;
+}

package/dist/types/barcode.js

@@ -0,0 +1,19 @@
+"use strict";
+/**
+ * Barcode Detection Types
+ *
+ * Types for multi-barcode scanning functionality.
+ * Designed to be compatible with react-native-vision-camera's CodeScanner.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.GROCERY_BARCODE_FORMATS = void 0;
+/**
+ * Common grocery barcode formats (optimized subset)
+ */
+exports.GROCERY_BARCODE_FORMATS = [
+    'ean-13',
+    'ean-8',
+    'upc-a',
+    'upc-e',
+    'qr',
+];