@souscheflabs/ml-vision 0.1.0

package/dist/types/detection.d.ts

@@ -0,0 +1,166 @@
+/**
+ * Object Detection Types
+ *
+ * Types for visual product detection (fridge, pantry, spice rack scanning).
+ * Used with TFLite models for on-device inference.
+ */
+import type { BaseDetectionResult } from './index';
+/**
+ * High-level product categories for detection
+ */
+export type ProductCategory = 'produce' | 'meat' | 'dairy' | 'packaged' | 'beverage' | 'condiment' | 'spice' | 'bakery' | 'frozen' | 'snack' | 'unknown';
+/**
+ * Detailed produce types
+ */
+export type ProduceType = 'apple' | 'banana' | 'orange' | 'lemon' | 'lime' | 'tomato' | 'potato' | 'onion' | 'garlic' | 'carrot' | 'broccoli' | 'lettuce' | 'cucumber' | 'pepper' | 'avocado' | 'mango' | 'strawberry' | 'blueberry' | 'grape' | 'watermelon' | 'pineapple' | 'other_fruit' | 'other_vegetable';
+/**
+ * Produce quality assessment
+ */
+export type ProduceQuality = 'fresh' | 'ripe' | 'overripe' | 'spoiled' | 'unknown';
+/**
+ * Data for detected products
+ */
+export interface ProductData {
+    /** Primary category */
+    category: ProductCategory;
+    /** Specific item type if classified */
+    itemType?: string;
+    /** Subcategory for more detail */
+    subcategory?: string;
+    /** Detected brand name if visible */
+    brand?: string;
+    /** Product name if recognized */
+    name?: string;
+    /** Class label from the model */
+    classLabel: string;
+    /** Class index from the model */
+    classIndex: number;
+    /** Suggested matches from your product database */
+    suggestedMatches?: ProductMatch[];
+}
+/**
+ * Suggested product match from database
+ */
+export interface ProductMatch {
+    /** Product ID in your database */
+    productId: string;
+    /** Product name */
+    name: string;
+    /** Match confidence (0-1) */
+    matchConfidence: number;
+    /** Image URL if available */
+    imageUrl?: string;
+}
+/**
+ * Extended data for produce items
+ */
+export interface ProduceData extends ProductData {
+    category: 'produce';
+    /** Specific produce type */
+    produceType: ProduceType;
+    /** Variety if identifiable (e.g., "Granny Smith" for apple) */
+    variety?: string;
+    /** Quality assessment */
+    quality: ProduceQuality;
+    /** Ripeness level (0-1) */
+    ripenessScore?: number;
+}
+/**
+ * Complete product detection result
+ */
+export interface ProductDetectionResult extends BaseDetectionResult {
+    type: 'product';
+    data: ProductData | ProduceData;
+}
+/**
+ * Model selection for detection
+ */
+export type DetectionModel = 'fast' | 'accurate';
+/**
+ * Options for useProductDetector hook
+ */
+export interface UseProductDetectorOptions {
+    /** Model to use: 'fast' (on-device TFLite) or 'accurate' (server) */
+    model?: DetectionModel;
+    /** Maximum detections per image (default: 20) */
+    maxDetections?: number;
+    /** Minimum confidence threshold (default: 0.5) */
+    minConfidence?: number;
+    /** Categories to detect (default: all) */
+    categories?: ProductCategory[];
+    /** Enable server fallback for low-confidence results (default: true) */
+    serverFallback?: boolean;
+    /** Confidence threshold below which to use server (default: 0.4) */
+    serverFallbackThreshold?: number;
+    /** Server URL (uses MLVisionProvider config if not specified) */
+    serverUrl?: string;
+    /** Callback when products are detected */
+    onDetected?: (products: ProductDetectionResult[]) => void;
+    /** Callback on error */
+    onError?: (error: Error) => void;
+    /** Enable haptic feedback (default: true) */
+    hapticFeedback?: boolean;
+}
+/**
+ * TensorflowModel type placeholder
+ * The actual type comes from react-native-fast-tflite
+ * This is intentionally loose to avoid type conflicts
+ */
+export type TensorflowModel = any;
+/**
+ * Return type for useProductDetector hook
+ */
+export interface UseProductDetectorReturn {
+    /** Whether the model is loaded and ready */
+    isModelLoaded: boolean;
+    /** Whether currently detecting */
+    isDetecting: boolean;
+    /** Array of detected products */
+    detections: ProductDetectionResult[];
+    /** Current error if any */
+    error: Error | null;
+    /** Model version string */
+    modelVersion: string;
+    /** The loaded TFLite model (for frame processor use) */
+    model: TensorflowModel | null;
+    /** Detect products in a photo */
+    detectProducts: (uri: string) => Promise<ProductDetectionResult[]>;
+    /** Detect products in current camera frame */
+    detectInFrame: () => Promise<ProductDetectionResult[]>;
+    /** Check for and download model updates */
+    updateModel: () => Promise<void>;
+    /** Clear all detections */
+    clearDetections: () => void;
+    frameProcessor: unknown;
+    modelInfo: {
+        name: string;
+        version: string;
+        sizeBytes: number;
+        lastUpdated: Date;
+        type: 'detection';
+        isLoaded: boolean;
+    } | null;
+}
+/**
+ * TFLite model output format (YOLOv8 style)
+ */
+export interface TFLiteDetectionOutput {
+    /** Bounding boxes [x, y, w, h] normalized 0-1 */
+    boxes: number[][];
+    /** Class indices */
+    classes: number[];
+    /** Confidence scores */
+    scores: number[];
+    /** Number of detections */
+    numDetections: number;
+}
+/**
+ * Class label mapping for the model
+ */
+export interface ClassLabelMap {
+    [classIndex: number]: {
+        label: string;
+        category: ProductCategory;
+        subcategory?: string;
+    };
+}

package/dist/types/detection.js

@@ -0,0 +1,8 @@
+"use strict";
+/**
+ * Object Detection Types
+ *
+ * Types for visual product detection (fridge, pantry, spice rack scanning).
+ * Used with TFLite models for on-device inference.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/types/index.d.ts

@@ -0,0 +1,126 @@
+/**
+ * @souschef/ml-vision Type Definitions
+ *
+ * Core types for the ML Vision package. These types are designed to be
+ * compatible with your existing Sous Chef app patterns.
+ */
+export * from './barcode';
+export * from './detection';
+export * from './ocr';
+/**
+ * Configuration for the MLVisionProvider
+ */
+export interface MLVisionConfig {
+    /** URL for the ML inference server (your Docker server) */
+    serverUrl?: string;
+    /** Timeout for server requests in milliseconds (default: 5000) */
+    serverTimeout?: number;
+    /** Enable MMKV caching for results (default: true) */
+    cacheEnabled?: boolean;
+    /** Cache TTL in milliseconds (default: 24 hours) */
+    cacheTTL?: number;
+    /** Check for model updates on app start (default: true) */
+    modelUpdateCheck?: boolean;
+    /** Enable GPU delegates for on-device inference (default: true) */
+    enableGPUDelegate?: boolean;
+}
+/**
+ * Bounding box for detected items
+ */
+export interface BoundingBox {
+    /** X coordinate (0-1 normalized) */
+    x: number;
+    /** Y coordinate (0-1 normalized) */
+    y: number;
+    /** Width (0-1 normalized) */
+    width: number;
+    /** Height (0-1 normalized) */
+    height: number;
+    /** Rotation in degrees (optional) */
+    rotation?: number;
+}
+/**
+ * Base detection result that all specific results extend
+ */
+export interface BaseDetectionResult {
+    /** Unique identifier for this detection */
+    id: string;
+    /** Confidence score (0-1) */
+    confidence: number;
+    /** Bounding box in the image */
+    boundingBox?: BoundingBox;
+    /** Whether this was processed on-device or via server */
+    source: 'on_device' | 'server';
+    /** Processing time in milliseconds */
+    processingTimeMs: number;
+    /** Timestamp of detection */
+    timestamp: number;
+}
+/**
+ * Information about a loaded ML model
+ */
+export interface ModelInfo {
+    /** Model identifier */
+    id: string;
+    /** Human-readable name */
+    name: string;
+    /** Semantic version */
+    version: string;
+    /** File size in bytes */
+    sizeBytes: number;
+    /** When the model was last updated */
+    lastUpdated: Date;
+    /** Model type */
+    type: 'detection' | 'classification' | 'ocr';
+    /** Whether the model is currently loaded */
+    isLoaded: boolean;
+}
+/**
+ * Model download progress
+ */
+export interface ModelDownloadProgress {
+    modelId: string;
+    bytesDownloaded: number;
+    totalBytes: number;
+    progress: number;
+}
+/**
+ * Cache entry wrapper
+ */
+export interface CacheEntry<T> {
+    data: T;
+    timestamp: number;
+    ttl: number;
+}
+/**
+ * Cache statistics
+ */
+export interface CacheStats {
+    barcodeEntries: number;
+    productEntries: number;
+    receiptEntries: number;
+    totalSizeBytes: number;
+}
+/**
+ * Server health response
+ */
+export interface ServerHealthResponse {
+    status: 'healthy' | 'degraded' | 'unhealthy';
+    version: string;
+    models: {
+        id: string;
+        loaded: boolean;
+        lastUsed?: string;
+    }[];
+    gpuAvailable: boolean;
+    gpuMemoryUsed?: number;
+    gpuMemoryTotal?: number;
+}
+/**
+ * Server error response
+ */
+export interface ServerErrorResponse {
+    error: string;
+    code: string;
+    details?: Record<string, unknown>;
+}

package/dist/types/index.js

@@ -0,0 +1,25 @@
+"use strict";
+/**
+ * @souschef/ml-vision Type Definitions
+ *
+ * Core types for the ML Vision package. These types are designed to be
+ * compatible with your existing Sous Chef app patterns.
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./barcode"), exports);
+__exportStar(require("./detection"), exports);
+__exportStar(require("./ocr"), exports);

package/dist/types/ocr.d.ts

@@ -0,0 +1,202 @@
+/**
+ * OCR and Receipt Scanning Types
+ *
+ * Types for text recognition and receipt parsing functionality.
+ * Uses MLKit Text Recognition on-device with server fallback.
+ */
+import type { BaseDetectionResult, BoundingBox } from './index';
+/**
+ * A line of recognized text
+ */
+export interface TextLine {
+    /** The recognized text content */
+    text: string;
+    /** Confidence score (0-1) */
+    confidence: number;
+    /** Bounding box of the text line */
+    boundingBox: BoundingBox;
+    /** Language code if detected */
+    language?: string;
+}
+/**
+ * A block of text (paragraph)
+ */
+export interface TextBlock {
+    /** All text in the block */
+    text: string;
+    /** Individual lines */
+    lines: TextLine[];
+    /** Bounding box of the entire block */
+    boundingBox: BoundingBox;
+    /** Confidence score (0-1) */
+    confidence: number;
+}
+/**
+ * Complete text recognition result
+ */
+export interface TextRecognitionResult {
+    /** All recognized text concatenated */
+    fullText: string;
+    /** Text blocks (paragraphs) */
+    blocks: TextBlock[];
+    /** All lines across all blocks */
+    lines: TextLine[];
+    /** Detected language */
+    language?: string;
+    /** Overall confidence */
+    confidence: number;
+    /** Processing source */
+    source: 'on_device' | 'server';
+    /** Processing time in ms */
+    processingTimeMs: number;
+}
+/**
+ * A parsed item from a receipt
+ */
+export interface ReceiptItem {
+    /** Item name as it appears on receipt */
+    name: string;
+    /** Cleaned/normalized item name */
+    normalizedName?: string;
+    /** Quantity purchased */
+    quantity?: number;
+    /** Unit of measurement */
+    unit?: string;
+    /** Price per unit */
+    unitPrice?: number;
+    /** Total price for this item */
+    totalPrice?: number;
+    /** Confidence in the parsing (0-1) */
+    confidence: number;
+    /** Original text from receipt */
+    rawText: string;
+    /** Line number in receipt */
+    lineNumber: number;
+    /** Whether this item needs manual review */
+    needsReview: boolean;
+    /** Suggested product matches from database */
+    suggestedMatches?: {
+        productId: string;
+        name: string;
+        confidence: number;
+    }[];
+}
+/**
+ * Receipt metadata
+ */
+export interface ReceiptMetadata {
+    /** Store name if detected */
+    storeName?: string;
+    /** Store location/address */
+    storeAddress?: string;
+    /** Receipt date if detected */
+    date?: Date;
+    /** Receipt time */
+    time?: string;
+    /** Transaction/receipt number */
+    transactionId?: string;
+    /** Subtotal before tax */
+    subtotal?: number;
+    /** Tax amount */
+    tax?: number;
+    /** Total amount */
+    total?: number;
+    /** Payment method */
+    paymentMethod?: string;
+}
+/**
+ * Complete receipt scan result
+ */
+export interface ReceiptScanResult extends BaseDetectionResult {
+    type: 'receipt';
+    /** Parsed items from the receipt */
+    items: ReceiptItem[];
+    /** Receipt metadata */
+    metadata: ReceiptMetadata;
+    /** Raw OCR result */
+    rawText: string;
+    /** All text lines */
+    textLines: TextLine[];
+    /** Overall parsing confidence */
+    confidence: number;
+    /** Number of items that need review */
+    itemsNeedingReview: number;
+}
+/**
+ * Known store formats for optimized parsing
+ */
+export type KnownStore = 'walmart' | 'target' | 'kroger' | 'safeway' | 'costco' | 'trader_joes' | 'whole_foods' | 'publix' | 'aldi' | 'unknown';
+/**
+ * Options for useReceiptScanner hook
+ */
+export interface UseReceiptScannerOptions {
+    /** Enable server fallback for complex receipts (default: true) */
+    serverFallback?: boolean;
+    /** Server URL (uses MLVisionProvider config if not specified) */
+    serverUrl?: string;
+    /** Minimum confidence to accept an item (default: 0.7) */
+    minConfidence?: number;
+    /** Known store formats for optimized parsing */
+    recognizedStores?: KnownStore[];
+    /** Callback when items are detected */
+    onItemDetected?: (item: ReceiptItem) => void;
+    /** Callback when parsing is complete */
+    onParseComplete?: (result: ReceiptScanResult) => void;
+    /** Callback on error */
+    onError?: (error: Error) => void;
+}
+/**
+ * Return type for useReceiptScanner hook
+ */
+export interface UseReceiptScannerReturn {
+    /** Whether currently processing */
+    isProcessing: boolean;
+    /** Parsed receipt items */
+    items: ReceiptItem[];
+    /** Receipt metadata */
+    metadata: ReceiptMetadata | null;
+    /** Raw OCR text */
+    rawText: string;
+    /** Overall confidence */
+    confidence: number;
+    /** Processing source */
+    source: 'on_device' | 'server' | null;
+    /** Current error if any */
+    error: Error | null;
+    /** Scan a receipt photo */
+    scanReceipt: (uri: string) => Promise<ReceiptScanResult>;
+    /** Scan current camera frame */
+    scanFrame: () => Promise<ReceiptScanResult>;
+    /** Confirm an item is correct */
+    confirmItem: (item: ReceiptItem) => void;
+    /** Reject/remove an item */
+    rejectItem: (item: ReceiptItem) => void;
+    /** Edit an item */
+    editItem: (item: ReceiptItem, updates: Partial<ReceiptItem>) => void;
+    /** Clear all results */
+    clearResults: () => void;
+    frameProcessor: unknown;
+}
+/**
+ * Receipt line classification
+ */
+export type ReceiptLineType = 'item' | 'subtotal' | 'tax' | 'total' | 'payment' | 'store_info' | 'date_time' | 'transaction' | 'discount' | 'unknown';
+/**
+ * Parsed receipt line before item extraction
+ */
+export interface ParsedReceiptLine {
+    /** Original text */
+    text: string;
+    /** Classified line type */
+    type: ReceiptLineType;
+    /** Extracted values */
+    values: {
+        name?: string;
+        quantity?: number;
+        price?: number;
+    };
+    /** Line number */
+    lineNumber: number;
+    /** Confidence in classification */
+    confidence: number;
+}

package/dist/types/ocr.js

@@ -0,0 +1,8 @@
+"use strict";
+/**
+ * OCR and Receipt Scanning Types
+ *
+ * Types for text recognition and receipt parsing functionality.
+ * Uses MLKit Text Recognition on-device with server fallback.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/utils/imagePreprocessor.d.ts

@@ -0,0 +1,85 @@
+/**
+ * Image Preprocessor for TFLite Models
+ *
+ * Converts images to the format expected by TFLite models:
+ * - Resized to model input size (e.g., 640x640)
+ * - Normalized to [0, 1] range
+ * - As Float32Array in NHWC format (batch, height, width, channels)
+ *
+ * This implementation uses react-native-skia for image manipulation.
+ * Install with: npm install @shopify/react-native-skia
+ */
+export interface PreprocessOptions {
+    /** Target width (default: 640) */
+    width?: number;
+    /** Target height (default: 640) */
+    height?: number;
+    /** Normalize to [0, 1] range (default: true) */
+    normalize?: boolean;
+    /** Whether to apply letterboxing to maintain aspect ratio (default: true) */
+    letterbox?: boolean;
+}
+export interface PreprocessResult {
+    /** Preprocessed pixel data as Float32Array */
+    data: Float32Array;
+    /** Original image width */
+    originalWidth: number;
+    /** Original image height */
+    originalHeight: number;
+    /** Scale factors for mapping back to original */
+    scale: {
+        x: number;
+        y: number;
+    };
+    /** Padding applied (if letterboxing) */
+    padding: {
+        x: number;
+        y: number;
+    };
+}
+/**
+ * Preprocess an image for TFLite model inference
+ *
+ * @param uri Image URI (file:// or http://)
+ * @param options Preprocessing options
+ * @returns Preprocessed image data
+ */
+export declare function preprocessImage(uri: string, options?: PreprocessOptions): Promise<PreprocessResult>;
+/**
+ * Preprocess a VisionCamera frame for TFLite inference
+ *
+ * This requires vision-camera-resize-plugin to be installed:
+ * npm install vision-camera-resize-plugin
+ *
+ * Usage in a frame processor:
+ * ```
+ * const frameProcessor = useFrameProcessor((frame) => {
+ *   'worklet';
+ *   const resized = resize(frame, {
+ *     size: { width: 640, height: 640 },
+ *     pixelFormat: 'rgb',
+ *     dataType: 'float32',
+ *   });
+ *   const result = model.runSync([resized]);
+ *   // Process result...
+ * }, [model]);
+ * ```
+ */
+export declare function getFrameProcessorInstructions(): string;
+/**
+ * Convert Float32Array from HWC to NHWC format (add batch dimension)
+ */
+export declare function addBatchDimension(data: Float32Array): Float32Array;
+/**
+ * Check if Skia is available for image preprocessing
+ */
+export declare function isSkiaAvailable(): boolean;
+/**
+ * Initialize Skia module (call before using Skia-dependent functions)
+ * @returns true if Skia was loaded successfully
+ */
+export declare function initSkia(): Promise<boolean>;
+/**
+ * Check if resize plugin is available for frame processing
+ */
+export declare function isResizePluginAvailable(): Promise<boolean>;