@souscheflabs/ml-vision 0.1.0

package/README.md

@@ -0,0 +1,274 @@
+# @souschef/ml-vision
+
+ML-powered product detection for React Native. Speed up kitchen inventory onboarding with multi-barcode scanning, receipt OCR, and visual product recognition.
+
+## Features
+
+- **Multi-Barcode Scanning** - Scan multiple barcodes from a single photo
+- **Receipt OCR** - Photograph grocery receipts to auto-extract items
+- **Visual Product Recognition** - Recognize fridge/pantry contents by appearance
+- **Video Scanning** - Real-time product detection from camera feed
+- **On-Device ML** - Fast inference using TFLite with GPU acceleration
+- **Server Fallback** - Optional server-side processing for better accuracy
+
+## Installation
+
+```bash
+npm install @souschef/ml-vision
+```
+
+### Peer Dependencies
+
+This package requires the following peer dependencies:
+
+```bash
+npm install react-native-vision-camera react-native-mmkv
+```
+
+For on-device ML inference (Phase 3+):
+```bash
+npm install react-native-fast-tflite
+```
+
+### iOS Setup
+
+```bash
+cd ios && pod install
+```
+
+Add camera permissions to `Info.plist`:
+```xml
+<key>NSCameraUsageDescription</key>
+<string>Camera is used to scan products and receipts</string>
+```
+
+### Android Setup
+
+Add camera permission to `AndroidManifest.xml`:
+```xml
+<uses-permission android:name="android.permission.CAMERA" />
+```
+
+## Quick Start
+
+### 1. Wrap your app with the provider
+
+```typescript
+import { MLVisionProvider } from '@souschef/ml-vision';
+import { MMKV } from 'react-native-mmkv';
+
+// Create MMKV instance for caching
+const storage = new MMKV({ id: 'ml-vision-cache' });
+
+function App() {
+  return (
+    <MLVisionProvider
+      config={{
+        serverUrl: 'http://192.168.1.100:8000', // Optional: your ML server
+        cacheEnabled: true,
+      }}
+      storage={storage}
+    >
+      <YourApp />
+    </MLVisionProvider>
+  );
+}
+```
+
+### 2. Use the hooks in your components
+
+```typescript
+import { useMultiBarcodeScanner } from '@souschef/ml-vision';
+import { Camera } from 'react-native-vision-camera';
+
+function BarcodeScanScreen() {
+  const {
+    results,
+    isScanning,
+    startScanning,
+    stopScanning,
+    frameProcessor,
+  } = useMultiBarcodeScanner({
+    formats: ['ean-13', 'upc-a', 'qr'],
+    maxBarcodes: 20,
+    onDetected: (barcodes) => {
+      console.log('Found barcodes:', barcodes);
+    },
+  });
+
+  return (
+    <Camera
+      device={device}
+      isActive={isScanning}
+      frameProcessor={frameProcessor}
+    />
+  );
+}
+```
+
+## API Reference
+
+### MLVisionProvider
+
+The context provider that must wrap your app.
+
+```typescript
+<MLVisionProvider
+  config={{
+    serverUrl?: string;        // ML server URL for fallback
+    serverTimeout?: number;    // Request timeout (default: 5000ms)
+    cacheEnabled?: boolean;    // Enable result caching (default: true)
+    cacheTTL?: number;         // Cache TTL in ms (default: 24 hours)
+    enableGPUDelegate?: boolean; // Use GPU for inference (default: true)
+  }}
+  storage={mmkvInstance}       // MMKV instance for caching
+  onInitialized={() => {}}     // Called when ready
+  onError={(error) => {}}      // Called on init error
+>
+```
+
+### useMultiBarcodeScanner
+
+Scan multiple barcodes from camera frames or photos.
+
+```typescript
+const {
+  // State
+  isReady: boolean;
+  isScanning: boolean;
+  results: BarcodeDetectionResult[];
+  error: Error | null;
+
+  // Actions
+  startScanning: () => void;
+  stopScanning: () => void;
+  scanPhoto: (uri: string) => Promise<BarcodeDetectionResult[]>;
+  clearResults: () => void;
+
+  // For VisionCamera
+  frameProcessor: FrameProcessor;
+} = useMultiBarcodeScanner(options);
+```
+
+### useProductDetector
+
+Detect products in images using trained ML models.
+
+```typescript
+const {
+  // State
+  isModelLoaded: boolean;
+  isDetecting: boolean;
+  detections: ProductDetectionResult[];
+  modelVersion: string;
+
+  // Actions
+  detectProducts: (uri: string) => Promise<ProductDetectionResult[]>;
+  updateModel: () => Promise<void>;
+
+  // For VisionCamera
+  frameProcessor: FrameProcessor;
+} = useProductDetector({
+  model: 'fast' | 'accurate';
+  minConfidence?: number;
+  serverFallback?: boolean;
+});
+```
+
+### useReceiptScanner
+
+Extract items from receipt photos.
+
+```typescript
+const {
+  // State
+  isProcessing: boolean;
+  items: ReceiptItem[];
+  confidence: number;
+
+  // Actions
+  scanReceipt: (uri: string) => Promise<ReceiptScanResult>;
+  confirmItem: (item: ReceiptItem) => void;
+  rejectItem: (item: ReceiptItem) => void;
+} = useReceiptScanner({
+  serverFallback?: boolean;
+  minConfidence?: number;
+});
+```
+
+## Architecture
+
+```
+┌─────────────────────────────────────────────────────────────────┐
+│                    Your React Native App                        │
+├─────────────────────────────────────────────────────────────────┤
+│  @souschef/ml-vision                                            │
+│  ├── Hooks (useMultiBarcodeScanner, useProductDetector, etc.)  │
+│  ├── Frame Processors (barcode, OCR, TFLite detection)         │
+│  └── CacheManager (MMKV-based caching)                         │
+├─────────────────────────────────────────────────────────────────┤
+│  On-Device Processing (fast, offline)                           │
+│  ├── MLKit (barcodes, text recognition)                        │
+│  ├── react-native-fast-tflite (custom models)                  │
+│  └── CoreML/NNAPI delegates (GPU acceleration)                 │
+├─────────────────────────────────────────────────────────────────┤
+│  Server Fallback (accurate, requires network)                   │
+│  ├── FastAPI inference server                                   │
+│  ├── YOLOv8 full models                                        │
+│  └── PaddleOCR for complex receipts                            │
+└─────────────────────────────────────────────────────────────────┘
+```
+
+## Training Your Own Models
+
+See the [Training Guide](../../docker/README.md) for instructions on:
+- Setting up the Docker training environment
+- Preparing your dataset
+- Training YOLOv8 models
+- Exporting to TFLite for mobile
+
+## Project Structure
+
+```
+packages/ml-vision/
+├── src/
+│   ├── index.ts              # Main exports
+│   ├── types/                # TypeScript definitions
+│   ├── hooks/                # React hooks
+│   ├── core/                 # Provider, cache, server client
+│   ├── processors/           # Frame processors
+│   └── utils/                # Utilities
+├── models/                   # Bundled TFLite models
+├── docs/                     # Documentation
+└── dist/                     # Compiled output
+```
+
+## Development
+
+```bash
+# Build the package
+npm run build
+
+# Watch for changes
+npm run watch
+
+# Type check
+npm run typecheck
+```
+
+## License
+
+ISC
+
+## Contributing
+
+1. Fork the repository
+2. Create your feature branch
+3. Make your changes
+4. Run tests and type checks
+5. Submit a pull request
+
+## Support
+
+- [GitHub Issues](https://github.com/SousChefLabs/ml-vision/issues)
+- [Documentation](./docs/)

package/dist/components/DetectionOverlay.d.ts

@@ -0,0 +1,57 @@
+/**
+ * DetectionOverlay Component
+ *
+ * Renders bounding boxes over detected products.
+ * Overlay this on top of your camera view or image.
+ *
+ * @example
+ * ```typescript
+ * import { DetectionOverlay } from '@souschef/ml-vision';
+ *
+ * function CameraScreen() {
+ *   const { detections } = useProductDetector();
+ *
+ *   return (
+ *     <View style={styles.container}>
+ *       <Camera style={StyleSheet.absoluteFill} />
+ *       <DetectionOverlay
+ *         detections={detections}
+ *         imageWidth={screenWidth}
+ *         imageHeight={screenHeight}
+ *       />
+ *     </View>
+ *   );
+ * }
+ * ```
+ */
+import React from 'react';
+import { type ViewStyle, type TextStyle } from 'react-native';
+import type { ProductDetectionResult, ProductCategory } from '../types';
+export interface DetectionOverlayProps {
+    /** Array of detection results to display */
+    detections: ProductDetectionResult[];
+    /** Width of the image/camera view */
+    imageWidth: number;
+    /** Height of the image/camera view */
+    imageHeight: number;
+    /** Show confidence score (default: true) */
+    showConfidence?: boolean;
+    /** Show class label (default: true) */
+    showLabel?: boolean;
+    /** Minimum confidence to display (default: 0) */
+    minConfidence?: number;
+    /** Custom colors per category */
+    categoryColors?: Partial<Record<ProductCategory, string>>;
+    /** Custom box style override */
+    boxStyle?: ViewStyle;
+    /** Custom label style override */
+    labelStyle?: TextStyle;
+    /** Callback when a detection is tapped */
+    onDetectionPress?: (detection: ProductDetectionResult) => void;
+}
+export declare function DetectionOverlay({ detections, imageWidth: _imageWidth, imageHeight: _imageHeight, showConfidence, showLabel, minConfidence, categoryColors, boxStyle, labelStyle, onDetectionPress, }: DetectionOverlayProps): React.ReactElement;
+/**
+ * Simple loading indicator for when model is loading
+ */
+export declare function DetectionLoadingOverlay(): React.ReactElement;
+export default DetectionOverlay;

package/dist/components/DetectionOverlay.js

@@ -0,0 +1,133 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DetectionOverlay = DetectionOverlay;
+exports.DetectionLoadingOverlay = DetectionLoadingOverlay;
+const jsx_runtime_1 = require("react/jsx-runtime");
+/**
+ * DetectionOverlay Component
+ *
+ * Renders bounding boxes over detected products.
+ * Overlay this on top of your camera view or image.
+ *
+ * @example
+ * ```typescript
+ * import { DetectionOverlay } from '@souschef/ml-vision';
+ *
+ * function CameraScreen() {
+ *   const { detections } = useProductDetector();
+ *
+ *   return (
+ *     <View style={styles.container}>
+ *       <Camera style={StyleSheet.absoluteFill} />
+ *       <DetectionOverlay
+ *         detections={detections}
+ *         imageWidth={screenWidth}
+ *         imageHeight={screenHeight}
+ *       />
+ *     </View>
+ *   );
+ * }
+ * ```
+ */
+const react_1 = require("react");
+const react_native_1 = require("react-native");
+// ============================================================================
+// Default Colors
+// ============================================================================
+const DEFAULT_CATEGORY_COLORS = {
+    produce: '#4CAF50', // Green
+    meat: '#F44336', // Red
+    dairy: '#2196F3', // Blue
+    packaged: '#FF9800', // Orange
+    beverage: '#9C27B0', // Purple
+    condiment: '#FFEB3B', // Yellow
+    spice: '#795548', // Brown
+    bakery: '#FFC107', // Amber
+    frozen: '#00BCD4', // Cyan
+    snack: '#E91E63', // Pink
+    unknown: '#9E9E9E', // Gray
+};
+// ============================================================================
+// Component
+// ============================================================================
+function DetectionOverlay({ detections, imageWidth: _imageWidth, imageHeight: _imageHeight, showConfidence = true, showLabel = true, minConfidence = 0, categoryColors = {}, boxStyle, labelStyle, onDetectionPress, }) {
+    // Merge custom colors with defaults
+    const colors = (0, react_1.useMemo)(() => ({ ...DEFAULT_CATEGORY_COLORS, ...categoryColors }), [categoryColors]);
+    // Filter detections by confidence
+    const visibleDetections = (0, react_1.useMemo)(() => detections.filter((d) => d.confidence >= minConfidence), [detections, minConfidence]);
+    return ((0, jsx_runtime_1.jsx)(react_native_1.View, { style: styles.container, pointerEvents: "box-none", children: visibleDetections.map((detection) => {
+            if (!detection.boundingBox)
+                return null;
+            const { x, y, width, height } = detection.boundingBox;
+            const color = colors[detection.data.category];
+            const label = detection.data.name || detection.data.classLabel;
+            const confidence = Math.round(detection.confidence * 100);
+            return ((0, jsx_runtime_1.jsx)(react_native_1.View, { style: [
+                    styles.box,
+                    {
+                        left: x,
+                        top: y,
+                        width: width,
+                        height: height,
+                        borderColor: color,
+                    },
+                    boxStyle,
+                ], onTouchEnd: () => onDetectionPress?.(detection), children: (showLabel || showConfidence) && ((0, jsx_runtime_1.jsx)(react_native_1.View, { style: [styles.labelContainer, { backgroundColor: color }], children: (0, jsx_runtime_1.jsxs)(react_native_1.Text, { style: [styles.labelText, labelStyle], numberOfLines: 1, ellipsizeMode: "tail", children: [showLabel && label, showLabel && showConfidence && ' ', showConfidence && `${confidence}%`] }) })) }, detection.id));
+        }) }));
+}
+// ============================================================================
+// Styles
+// ============================================================================
+const styles = react_native_1.StyleSheet.create({
+    container: {
+        ...react_native_1.StyleSheet.absoluteFillObject,
+    },
+    box: {
+        position: 'absolute',
+        borderWidth: 2,
+        borderRadius: 4,
+    },
+    labelContainer: {
+        position: 'absolute',
+        top: -24,
+        left: -2,
+        paddingHorizontal: 6,
+        paddingVertical: 2,
+        borderRadius: 4,
+        minWidth: 40,
+    },
+    labelText: {
+        color: '#FFFFFF',
+        fontSize: 12,
+        fontWeight: '600',
+    },
+});
+// ============================================================================
+// Utility Components
+// ============================================================================
+/**
+ * Simple loading indicator for when model is loading
+ */
+function DetectionLoadingOverlay() {
+    return ((0, jsx_runtime_1.jsx)(react_native_1.View, { style: loadingStyles.container, children: (0, jsx_runtime_1.jsx)(react_native_1.View, { style: loadingStyles.box, children: (0, jsx_runtime_1.jsx)(react_native_1.Text, { style: loadingStyles.text, children: "Loading model..." }) }) }));
+}
+const loadingStyles = react_native_1.StyleSheet.create({
+    container: {
+        ...react_native_1.StyleSheet.absoluteFillObject,
+        justifyContent: 'center',
+        alignItems: 'center',
+        backgroundColor: 'rgba(0, 0, 0, 0.3)',
+    },
+    box: {
+        backgroundColor: 'rgba(0, 0, 0, 0.7)',
+        paddingHorizontal: 20,
+        paddingVertical: 12,
+        borderRadius: 8,
+    },
+    text: {
+        color: '#FFFFFF',
+        fontSize: 16,
+        fontWeight: '500',
+    },
+});
+exports.default = DetectionOverlay;

package/dist/components/index.d.ts

@@ -0,0 +1,4 @@
+/**
+ * Components exports
+ */
+export { DetectionOverlay, DetectionLoadingOverlay, type DetectionOverlayProps, } from './DetectionOverlay';

package/dist/components/index.js

@@ -0,0 +1,9 @@
+"use strict";
+/**
+ * Components exports
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DetectionLoadingOverlay = exports.DetectionOverlay = void 0;
+var DetectionOverlay_1 = require("./DetectionOverlay");
+Object.defineProperty(exports, "DetectionOverlay", { enumerable: true, get: function () { return DetectionOverlay_1.DetectionOverlay; } });
+Object.defineProperty(exports, "DetectionLoadingOverlay", { enumerable: true, get: function () { return DetectionOverlay_1.DetectionLoadingOverlay; } });

package/dist/core/CacheManager.d.ts

@@ -0,0 +1,168 @@
+/**
+ * CacheManager - MMKV-based caching for ML Vision results
+ *
+ * Provides fast, synchronous caching for:
+ * - Barcode to product lookups
+ * - Product recognition results
+ * - Receipt parsing results
+ * - Model metadata
+ *
+ * Uses react-native-mmkv for ~30x faster access than AsyncStorage.
+ */
+import type { MMKV } from 'react-native-mmkv';
+import type { CacheStats, BarcodeFormat } from '../types';
+/**
+ * Default TTL values (in milliseconds)
+ */
+export declare const CACHE_TTL: {
+    /** Barcode lookups - 7 days */
+    readonly BARCODE: number;
+    /** Product recognition - 24 hours */
+    readonly PRODUCT: number;
+    /** Receipt parsing - 1 hour */
+    readonly RECEIPT: number;
+    /** Model metadata - until manually cleared */
+    readonly MODEL: number;
+    /** Server responses - 5 minutes */
+    readonly SERVER: number;
+};
+/**
+ * Cache key prefixes for namespacing
+ */
+declare const CACHE_PREFIX: {
+    readonly BARCODE: "mlv_barcode_";
+    readonly PRODUCT: "mlv_product_";
+    readonly RECEIPT: "mlv_receipt_";
+    readonly MODEL: "mlv_model_";
+    readonly CONFIG: "mlv_config_";
+};
+/**
+ * Cached barcode lookup
+ */
+export interface CachedBarcodeLookup {
+    barcode: string;
+    format: BarcodeFormat;
+    productId?: string;
+    productName?: string;
+    imageUrl?: string;
+    found: boolean;
+}
+/**
+ * Cached product recognition
+ */
+export interface CachedProductRecognition {
+    imageHash: string;
+    detections: Array<{
+        classLabel: string;
+        confidence: number;
+        boundingBox?: {
+            x: number;
+            y: number;
+            width: number;
+            height: number;
+        };
+    }>;
+    modelVersion: string;
+}
+/**
+ * CacheManager provides fast key-value caching using MMKV
+ *
+ * @example
+ * ```typescript
+ * const cache = new CacheManager(mmkvInstance);
+ *
+ * // Cache a barcode lookup
+ * cache.setBarcode('0123456789012', {
+ *   barcode: '0123456789012',
+ *   format: 'ean-13',
+ *   productId: 'prod_123',
+ *   productName: 'Example Product',
+ *   found: true,
+ * });
+ *
+ * // Retrieve later
+ * const product = cache.getBarcode('0123456789012');
+ * ```
+ */
+export declare class CacheManager {
+    private storage;
+    private defaultTTL;
+    constructor(storage: MMKV, customTTL?: Partial<typeof CACHE_TTL>);
+    /**
+     * Cache a barcode lookup result
+     */
+    setBarcode(barcode: string, data: CachedBarcodeLookup, ttl?: number): void;
+    /**
+     * Get a cached barcode lookup
+     * @returns The cached data or null if not found/expired
+     */
+    getBarcode(barcode: string): CachedBarcodeLookup | null;
+    /**
+     * Check if a barcode is cached
+     */
+    hasBarcode(barcode: string): boolean;
+    /**
+     * Delete a cached barcode
+     */
+    deleteBarcode(barcode: string): void;
+    /**
+     * Cache product recognition results for an image
+     * @param imageHash - Hash of the image (e.g., MD5 or perceptual hash)
+     */
+    setProductRecognition(imageHash: string, data: CachedProductRecognition, ttl?: number): void;
+    /**
+     * Get cached product recognition for an image
+     */
+    getProductRecognition(imageHash: string): CachedProductRecognition | null;
+    /**
+     * Store model metadata
+     */
+    setModelMetadata(modelId: string, metadata: Record<string, unknown>): void;
+    /**
+     * Get model metadata
+     */
+    getModelMetadata<T = Record<string, unknown>>(modelId: string): T | null;
+    /**
+     * Store a configuration value
+     */
+    setConfig<T>(key: string, value: T): void;
+    /**
+     * Get a configuration value
+     */
+    getConfig<T>(key: string, defaultValue: T): T;
+    /**
+     * Clear all expired entries from the cache
+     * Call this periodically (e.g., on app start)
+     */
+    cleanupExpired(): number;
+    /**
+     * Clear all ML Vision cache entries
+     */
+    clearAll(): void;
+    /**
+     * Clear cache by type
+     */
+    clearByType(type: keyof typeof CACHE_PREFIX): void;
+    /**
+     * Get cache statistics
+     */
+    getStats(): CacheStats;
+    /**
+     * Check if a cache entry has expired
+     */
+    private isExpired;
+}
+/**
+ * Create a CacheManager instance
+ * This is the recommended way to create a CacheManager
+ *
+ * @example
+ * ```typescript
+ * import { MMKV } from 'react-native-mmkv';
+ *
+ * const storage = new MMKV({ id: 'ml-vision-cache' });
+ * const cache = createCacheManager(storage);
+ * ```
+ */
+export declare function createCacheManager(storage: MMKV, customTTL?: Partial<typeof CACHE_TTL>): CacheManager;
+export {};