@souscheflabs/ml-vision 0.1.0

package/dist/hooks/useReceiptScanner.d.ts

@@ -0,0 +1,37 @@
+/**
+ * useReceiptScanner Hook
+ *
+ * React hook for scanning and parsing grocery receipts.
+ * Uses MLKit text recognition with intelligent parsing.
+ *
+ * @example
+ * ```typescript
+ * import { useReceiptScanner } from '@souschef/ml-vision';
+ *
+ * function ReceiptScreen() {
+ *   const {
+ *     items,
+ *     metadata,
+ *     isProcessing,
+ *     scanReceipt,
+ *   } = useReceiptScanner({
+ *     onParseComplete: (result) => console.log('Parsed:', result),
+ *   });
+ *
+ *   const handleCapture = async (photoUri: string) => {
+ *     const result = await scanReceipt(photoUri);
+ *     // result.items contains parsed line items
+ *   };
+ *
+ *   return (
+ *     // Your UI
+ *   );
+ * }
+ * ```
+ */
+import type { UseReceiptScannerOptions, UseReceiptScannerReturn } from '../types';
+/**
+ * Hook for receipt scanning and parsing
+ */
+export declare function useReceiptScanner(options?: UseReceiptScannerOptions): UseReceiptScannerReturn;
+export default useReceiptScanner;

package/dist/hooks/useReceiptScanner.js

@@ -0,0 +1,405 @@
+"use strict";
+/**
+ * useReceiptScanner Hook
+ *
+ * React hook for scanning and parsing grocery receipts.
+ * Uses MLKit text recognition with intelligent parsing.
+ *
+ * @example
+ * ```typescript
+ * import { useReceiptScanner } from '@souschef/ml-vision';
+ *
+ * function ReceiptScreen() {
+ *   const {
+ *     items,
+ *     metadata,
+ *     isProcessing,
+ *     scanReceipt,
+ *   } = useReceiptScanner({
+ *     onParseComplete: (result) => console.log('Parsed:', result),
+ *   });
+ *
+ *   const handleCapture = async (photoUri: string) => {
+ *     const result = await scanReceipt(photoUri);
+ *     // result.items contains parsed line items
+ *   };
+ *
+ *   return (
+ *     // Your UI
+ *   );
+ * }
+ * ```
+ */
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.useReceiptScanner = useReceiptScanner;
+const react_1 = require("react");
+const text_recognition_1 = __importDefault(require("@react-native-ml-kit/text-recognition"));
+const MLVisionProvider_1 = require("../core/MLVisionProvider");
+// ============================================================================
+// Receipt Parsing Patterns
+// ============================================================================
+/** Price pattern: matches $X.XX, X.XX, etc. */
+const PRICE_PATTERN = /\$?\d{1,4}\.\d{2}/g;
+/** Quantity pattern: matches Xkg, X lb, X ct, etc. */
+const QUANTITY_PATTERN = /(\d+(?:\.\d+)?)\s*(kg|lb|oz|ct|ea|pc|g|ml|l|gal)?/i;
+/** Date patterns */
+const DATE_PATTERNS = [
+    /(\d{1,2})[/-](\d{1,2})[/-](\d{2,4})/, // MM/DD/YYYY or DD/MM/YYYY
+    /(\w{3})\s+(\d{1,2}),?\s*(\d{4})/i, // Jan 15, 2024
+];
+/** Time pattern */
+const TIME_PATTERN = /\d{1,2}:\d{2}(?::\d{2})?\s*(?:AM|PM)?/i;
+// Total/Subtotal patterns (used for line classification)
+// TOTAL|GRAND TOTAL|AMOUNT DUE, SUBTOTAL, TAX
+/** Known store patterns for detection */
+const STORE_PATTERNS = [
+    { pattern: /WALMART/i, store: 'walmart' },
+    { pattern: /TARGET/i, store: 'target' },
+    { pattern: /KROGER/i, store: 'kroger' },
+    { pattern: /SAFEWAY/i, store: 'safeway' },
+    { pattern: /COSTCO/i, store: 'costco' },
+    { pattern: /TRADER\s*JOE/i, store: 'trader_joes' },
+    { pattern: /WHOLE\s*FOODS/i, store: 'whole_foods' },
+    { pattern: /PUBLIX/i, store: 'publix' },
+    { pattern: /ALDI/i, store: 'aldi' },
+];
+// ============================================================================
+// Hook Implementation
+// ============================================================================
+/**
+ * Hook for receipt scanning and parsing
+ */
+function useReceiptScanner(options = {}) {
+    const { serverFallback = true, serverUrl, minConfidence = 0.7, onItemDetected, onParseComplete, onError, } = options;
+    // Context
+    const { serverClient, serverAvailable, config } = (0, MLVisionProvider_1.useMLVisionContext)();
+    // State
+    const [isProcessing, setIsProcessing] = (0, react_1.useState)(false);
+    const [items, setItems] = (0, react_1.useState)([]);
+    const [metadata, setMetadata] = (0, react_1.useState)(null);
+    const [rawText, setRawText] = (0, react_1.useState)('');
+    const [confidence, setConfidence] = (0, react_1.useState)(0);
+    const [source, setSource] = (0, react_1.useState)(null);
+    const [error, setError] = (0, react_1.useState)(null);
+    // Refs
+    const mountedRef = (0, react_1.useRef)(true);
+    // Computed server URL
+    const effectiveServerUrl = serverUrl || config.serverUrl;
+    // ============================================================================
+    // Receipt Parsing Logic
+    // ============================================================================
+    /**
+     * Parse raw OCR text into receipt items
+     */
+    const parseReceiptText = (0, react_1.useCallback)((text) => {
+        const lines = text.split('\n').filter(line => line.trim());
+        const parsedLines = lines.map((line, index) => classifyLine(line, index));
+        // Extract metadata
+        const extractedMetadata = {};
+        // Find store name
+        for (const { pattern, store } of STORE_PATTERNS) {
+            if (pattern.test(text)) {
+                extractedMetadata.storeName = store.toUpperCase().replace('_', ' ');
+                break;
+            }
+        }
+        // Find date and time
+        for (const line of lines.slice(0, 10)) {
+            for (const datePattern of DATE_PATTERNS) {
+                const match = line.match(datePattern);
+                if (match) {
+                    try {
+                        extractedMetadata.date = new Date(match[0]);
+                    }
+                    catch {
+                        // Invalid date
+                    }
+                    break;
+                }
+            }
+            const timeMatch = line.match(TIME_PATTERN);
+            if (timeMatch) {
+                extractedMetadata.time = timeMatch[0];
+            }
+        }
+        // Find totals
+        for (const parsedLine of parsedLines) {
+            if (parsedLine.type === 'total' && parsedLine.values.price) {
+                extractedMetadata.total = parsedLine.values.price;
+            }
+            else if (parsedLine.type === 'subtotal' && parsedLine.values.price) {
+                extractedMetadata.subtotal = parsedLine.values.price;
+            }
+            else if (parsedLine.type === 'tax' && parsedLine.values.price) {
+                extractedMetadata.tax = parsedLine.values.price;
+            }
+        }
+        // Extract items
+        const extractedItems = parsedLines
+            .filter(line => line.type === 'item')
+            .map(line => {
+            const item = {
+                name: line.values.name || line.text,
+                normalizedName: normalizeItemName(line.values.name || line.text),
+                quantity: line.values.quantity,
+                totalPrice: line.values.price,
+                confidence: line.confidence,
+                rawText: line.text,
+                lineNumber: line.lineNumber,
+                needsReview: line.confidence < minConfidence,
+            };
+            // Notify per item
+            onItemDetected?.(item);
+            return item;
+        });
+        return { items: extractedItems, metadata: extractedMetadata };
+    }, [minConfidence, onItemDetected]);
+    /**
+     * Classify a single receipt line
+     */
+    function classifyLine(text, lineNumber) {
+        const trimmed = text.trim();
+        let type = 'unknown';
+        let lineConfidence = 0.5;
+        const values = {};
+        // Check for totals first
+        if (/^TOTAL/i.test(trimmed) || /GRAND\s*TOTAL/i.test(trimmed)) {
+            type = 'total';
+            lineConfidence = 0.95;
+        }
+        else if (/^SUBTOTAL/i.test(trimmed)) {
+            type = 'subtotal';
+            lineConfidence = 0.95;
+        }
+        else if (/^TAX/i.test(trimmed) || /SALES\s*TAX/i.test(trimmed)) {
+            type = 'tax';
+            lineConfidence = 0.95;
+        }
+        else if (/^(CASH|CREDIT|DEBIT|VISA|MASTERCARD|AMEX)/i.test(trimmed)) {
+            type = 'payment';
+            lineConfidence = 0.9;
+        }
+        else if (DATE_PATTERNS.some(p => p.test(trimmed))) {
+            type = 'date_time';
+            lineConfidence = 0.85;
+        }
+        else if (STORE_PATTERNS.some(({ pattern }) => pattern.test(trimmed))) {
+            type = 'store_info';
+            lineConfidence = 0.9;
+        }
+        else if (/^\d{10,20}$/.test(trimmed.replace(/[\s-]/g, ''))) {
+            type = 'transaction';
+            lineConfidence = 0.7;
+        }
+        else if (trimmed.length > 3) {
+            // Likely an item line
+            type = 'item';
+            // Extract price
+            const prices = trimmed.match(PRICE_PATTERN);
+            if (prices && prices.length > 0) {
+                const lastPrice = prices[prices.length - 1];
+                values.price = parseFloat(lastPrice.replace('$', ''));
+                lineConfidence = 0.8;
+            }
+            // Extract quantity
+            const qtyMatch = trimmed.match(QUANTITY_PATTERN);
+            if (qtyMatch) {
+                values.quantity = parseFloat(qtyMatch[1]);
+            }
+            // Extract name (text before price)
+            if (prices && prices.length > 0) {
+                const priceIndex = trimmed.indexOf(prices[0]);
+                if (priceIndex > 0) {
+                    values.name = trimmed.substring(0, priceIndex).trim();
+                }
+            }
+            else {
+                values.name = trimmed;
+                lineConfidence = 0.6; // Lower confidence without price
+            }
+        }
+        // Extract price for non-item lines too
+        if (type !== 'item' && type !== 'unknown') {
+            const prices = trimmed.match(PRICE_PATTERN);
+            if (prices && prices.length > 0) {
+                const lastPrice = prices[prices.length - 1];
+                values.price = parseFloat(lastPrice.replace('$', ''));
+            }
+        }
+        return { text: trimmed, type, values, lineNumber, confidence: lineConfidence };
+    }
+    /**
+     * Normalize item name for matching
+     */
+    function normalizeItemName(name) {
+        return name
+            .toLowerCase()
+            .replace(/[^a-z0-9\s]/g, '')
+            .replace(/\s+/g, ' ')
+            .trim();
+    }
+    // ============================================================================
+    // Actions
+    // ============================================================================
+    /**
+     * Scan a receipt photo
+     */
+    const scanReceipt = (0, react_1.useCallback)(async (uri) => {
+        setIsProcessing(true);
+        setError(null);
+        const startTime = Date.now();
+        try {
+            let ocrText = '';
+            let processSource = 'on_device';
+            // Try server OCR first if available (better accuracy for receipts)
+            if (serverFallback &&
+                effectiveServerUrl &&
+                serverClient &&
+                serverAvailable) {
+                try {
+                    // Note: Server OCR endpoint would return text
+                    // For now, simulate with placeholder
+                    console.log('[useReceiptScanner] Server OCR not yet implemented');
+                }
+                catch {
+                    console.log('[useReceiptScanner] Server OCR failed, using on-device');
+                }
+            }
+            // Use on-device OCR with MLKit
+            if (!ocrText) {
+                console.log('[useReceiptScanner] Running MLKit text recognition...');
+                try {
+                    const result = await text_recognition_1.default.recognize(uri);
+                    ocrText = result.text;
+                    console.log(`[useReceiptScanner] MLKit recognized ${result.blocks.length} blocks, ${ocrText.length} chars`);
+                    processSource = 'on_device';
+                }
+                catch (ocrErr) {
+                    console.error('[useReceiptScanner] MLKit OCR error:', ocrErr);
+                    throw new Error(`OCR failed: ${ocrErr instanceof Error ? ocrErr.message : 'Unknown error'}`);
+                }
+            }
+            // Parse the OCR text
+            const { items: parsedItems, metadata: parsedMetadata } = parseReceiptText(ocrText);
+            const processingTimeMs = Date.now() - startTime;
+            const overallConfidence = parsedItems.length > 0
+                ? parsedItems.reduce((sum, item) => sum + item.confidence, 0) /
+                    parsedItems.length
+                : 0;
+            // Build result
+            const result = {
+                id: `receipt_${Date.now()}_${Math.random()
+                    .toString(36)
+                    .substr(2, 9)}`,
+                type: 'receipt',
+                items: parsedItems,
+                metadata: parsedMetadata,
+                rawText: ocrText,
+                textLines: [], // Would be populated by MLKit
+                confidence: overallConfidence,
+                itemsNeedingReview: parsedItems.filter(i => i.needsReview).length,
+                source: processSource,
+                processingTimeMs,
+                timestamp: Date.now(),
+            };
+            // Update state
+            if (mountedRef.current) {
+                setItems(parsedItems);
+                setMetadata(parsedMetadata);
+                setRawText(ocrText);
+                setConfidence(overallConfidence);
+                setSource(processSource);
+            }
+            // Callback
+            onParseComplete?.(result);
+            return result;
+        }
+        catch (err) {
+            const scanError = err instanceof Error ? err : new Error('Receipt scan failed');
+            console.error('[useReceiptScanner] Error:', scanError);
+            if (mountedRef.current) {
+                setError(scanError);
+            }
+            onError?.(scanError);
+            throw scanError;
+        }
+        finally {
+            if (mountedRef.current) {
+                setIsProcessing(false);
+            }
+        }
+    }, [
+        serverFallback,
+        effectiveServerUrl,
+        serverClient,
+        serverAvailable,
+        parseReceiptText,
+        onParseComplete,
+        onError,
+    ]);
+    /**
+     * Scan current camera frame
+     */
+    const scanFrame = (0, react_1.useCallback)(async () => {
+        // This would capture current frame and scan
+        // Requires VisionCamera integration
+        throw new Error('Frame scanning not implemented - use scanReceipt with photo URI');
+    }, []);
+    /**
+     * Confirm an item is correct
+     */
+    const confirmItem = (0, react_1.useCallback)((item) => {
+        setItems(prev => prev.map(i => i.lineNumber === item.lineNumber ? { ...i, needsReview: false } : i));
+    }, []);
+    /**
+     * Reject/remove an item
+     */
+    const rejectItem = (0, react_1.useCallback)((item) => {
+        setItems(prev => prev.filter(i => i.lineNumber !== item.lineNumber));
+    }, []);
+    /**
+     * Edit an item
+     */
+    const editItem = (0, react_1.useCallback)((item, updates) => {
+        setItems(prev => prev.map(i => i.lineNumber === item.lineNumber
+            ? { ...i, ...updates, needsReview: false }
+            : i));
+    }, []);
+    /**
+     * Clear all results
+     */
+    const clearResults = (0, react_1.useCallback)(() => {
+        setItems([]);
+        setMetadata(null);
+        setRawText('');
+        setConfidence(0);
+        setSource(null);
+        setError(null);
+    }, []);
+    // ============================================================================
+    // Return
+    // ============================================================================
+    return {
+        // State
+        isProcessing,
+        items,
+        metadata,
+        rawText,
+        confidence,
+        source,
+        error,
+        // Actions
+        scanReceipt,
+        scanFrame,
+        confirmItem,
+        rejectItem,
+        editItem,
+        clearResults,
+        // Frame processor (placeholder)
+        frameProcessor: null,
+    };
+}
+exports.default = useReceiptScanner;

package/dist/hooks/useVideoScanner.d.ts

@@ -0,0 +1,118 @@
+/**
+ * useVideoScanner Hook
+ *
+ * React hook for real-time video scanning combining product detection,
+ * barcode scanning, and text recognition.
+ *
+ * @example
+ * ```typescript
+ * import { useVideoScanner } from '@souschef/ml-vision';
+ *
+ * function LiveScanner() {
+ *   const {
+ *     isScanning,
+ *     products,
+ *     barcodes,
+ *     startScanning,
+ *     stopScanning,
+ *     cameraRef,
+ *   } = useVideoScanner({
+ *     modes: ['products', 'barcodes'],
+ *     onDetected: (results) => console.log('Found:', results),
+ *   });
+ *
+ *   return (
+ *     <Camera ref={cameraRef} />
+ *   );
+ * }
+ * ```
+ */
+import type { TensorflowModel } from 'react-native-fast-tflite';
+import type { Camera, Frame } from 'react-native-vision-camera';
+import type { ProductDetectionResult, BarcodeDetectionResult, ProductCategory } from '../types';
+/**
+ * Scanning mode
+ */
+export type ScanMode = 'products' | 'barcodes' | 'text';
+/**
+ * Combined detection result
+ */
+export interface VideoScanResult {
+    products: ProductDetectionResult[];
+    barcodes: BarcodeDetectionResult[];
+    timestamp: number;
+    frameNumber: number;
+}
+/**
+ * Options for useVideoScanner hook
+ */
+export interface UseVideoScannerOptions {
+    /** Scanning modes to enable (default: ['products', 'barcodes']) */
+    modes?: ScanMode[];
+    /** Frame rate for processing (default: 10 fps) */
+    targetFps?: number;
+    /** Minimum confidence for product detection (default: 0.5) */
+    minProductConfidence?: number;
+    /** Product categories to detect */
+    productCategories?: ProductCategory[];
+    /** Maximum products per frame (default: 10) */
+    maxProducts?: number;
+    /** Maximum barcodes per frame (default: 5) */
+    maxBarcodes?: number;
+    /** Enable temporal smoothing (default: true) */
+    temporalSmoothing?: boolean;
+    /** Number of frames for smoothing (default: 3) */
+    smoothingFrames?: number;
+    /** Callback when items are detected */
+    onDetected?: (results: VideoScanResult) => void;
+    /** Callback on error */
+    onError?: (error: Error) => void;
+    /** Enable haptic feedback (default: true) */
+    hapticFeedback?: boolean;
+    /** Loaded TFLite model for on-device frame processing */
+    tfliteModel?: TensorflowModel | null;
+    /** Use on-device TFLite frame processor instead of server (default: true if model provided) */
+    useOnDeviceFrameProcessor?: boolean;
+    /** Only detect food classes when using COCO model (default: true) */
+    foodOnly?: boolean;
+}
+/**
+ * Return type for useVideoScanner hook
+ */
+export interface UseVideoScannerReturn {
+    /** Whether scanning is active */
+    isScanning: boolean;
+    /** Whether all models are loaded */
+    isReady: boolean;
+    /** Detected products (smoothed) */
+    products: ProductDetectionResult[];
+    /** Detected barcodes */
+    barcodes: BarcodeDetectionResult[];
+    /** Current frame number */
+    frameNumber: number;
+    /** Average FPS */
+    fps: number;
+    /** Current error if any */
+    error: Error | null;
+    /** Last inference time in milliseconds */
+    inferenceTimeMs: number;
+    /** Start video scanning */
+    startScanning: () => void;
+    /** Stop video scanning */
+    stopScanning: () => void;
+    /** Toggle scanning */
+    toggleScanning: () => void;
+    /** Clear all detections */
+    clearDetections: () => void;
+    /** Capture current frame as photo */
+    captureFrame: () => Promise<string | null>;
+    cameraRef: React.RefObject<Camera | null>;
+    frameProcessorConfig: unknown;
+    /** Frame processor function for use with VisionCamera (if using TFLite) */
+    frameProcessor: ((frame: Frame) => void) | undefined;
+}
+/**
+ * Hook for real-time video scanning
+ */
+export declare function useVideoScanner(options?: UseVideoScannerOptions): UseVideoScannerReturn;
+export default useVideoScanner;