@souscheflabs/ml-vision 0.1.0

package/dist/utils/imagePreprocessor.js

@@ -0,0 +1,304 @@
+"use strict";
+/**
+ * Image Preprocessor for TFLite Models
+ *
+ * Converts images to the format expected by TFLite models:
+ * - Resized to model input size (e.g., 640x640)
+ * - Normalized to [0, 1] range
+ * - As Float32Array in NHWC format (batch, height, width, channels)
+ *
+ * This implementation uses react-native-skia for image manipulation.
+ * Install with: npm install @shopify/react-native-skia
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.preprocessImage = preprocessImage;
+exports.getFrameProcessorInstructions = getFrameProcessorInstructions;
+exports.addBatchDimension = addBatchDimension;
+exports.isSkiaAvailable = isSkiaAvailable;
+exports.initSkia = initSkia;
+exports.isResizePluginAvailable = isResizePluginAvailable;
+const react_native_1 = require("react-native");
+// Lazy-loaded Skia references (to avoid crashes when Skia isn't needed)
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+let SkiaModule = null;
+let skiaLoadAttempted = false;
+/**
+ * Load Skia module lazily (only when needed for photo preprocessing)
+ */
+async function loadSkiaModule() {
+    if (SkiaModule)
+        return true;
+    if (skiaLoadAttempted)
+        return SkiaModule !== null;
+    skiaLoadAttempted = true;
+    try {
+        console.log('[imagePreprocessor] Loading Skia module...');
+        // Lazy require to avoid loading Skia when not needed (e.g., video scanner)
+        SkiaModule = require('@shopify/react-native-skia');
+        console.log('[imagePreprocessor] Skia loaded successfully');
+        return true;
+    }
+    catch (error) {
+        console.error('[imagePreprocessor] Failed to load Skia:', error);
+        SkiaModule = null;
+        return false;
+    }
+}
+/**
+ * Check if Skia is available (synchronous check after loading)
+ */
+function checkSkiaAvailable() {
+    if (!SkiaModule)
+        return false;
+    try {
+        const { Skia } = SkiaModule;
+        return !!(Skia?.Data && Skia?.Image && Skia?.Surface);
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Preprocess an image for TFLite model inference
+ *
+ * @param uri Image URI (file:// or http://)
+ * @param options Preprocessing options
+ * @returns Preprocessed image data
+ */
+async function preprocessImage(uri, options = {}) {
+    const { width: targetWidth = 640, height: targetHeight = 640, normalize = true, letterbox = true, } = options;
+    // Load Skia module first (lazy loading)
+    const loaded = await loadSkiaModule();
+    if (!loaded || !checkSkiaAvailable()) {
+        throw new Error('Image preprocessing requires @shopify/react-native-skia. ' +
+            'Install with: npm install @shopify/react-native-skia\n\n' +
+            'Alternatively, use server-side inference or VisionCamera frame processor.');
+    }
+    // Get original image dimensions
+    const { width: originalWidth, height: originalHeight } = await getImageSize(uri);
+    // Use Skia for preprocessing
+    return preprocessWithSkia(uri, originalWidth, originalHeight, targetWidth, targetHeight, normalize, letterbox);
+}
+/**
+ * Get image dimensions from URI
+ */
+async function getImageSize(uri) {
+    return new Promise((resolve, reject) => {
+        react_native_1.Image.getSize(uri, (width, height) => resolve({ width, height }), (error) => reject(new Error(`Failed to get image size: ${error}`)));
+    });
+}
+/**
+ * Preprocess using react-native-skia
+ * Uses Skia.Data.fromURI() for proper image loading
+ */
+async function preprocessWithSkia(uri, originalWidth, originalHeight, targetWidth, targetHeight, normalize, letterbox) {
+    // Get Skia APIs from the lazy-loaded module
+    const { Skia, ColorType, AlphaType } = SkiaModule;
+    console.log('[imagePreprocessor] preprocessWithSkia called');
+    console.log('[imagePreprocessor] URI:', uri);
+    console.log('[imagePreprocessor] Original size:', originalWidth, 'x', originalHeight);
+    console.log('[imagePreprocessor] Target size:', targetWidth, 'x', targetHeight);
+    // Load image data using Skia's fromURI (handles file:// URIs properly)
+    console.log('[imagePreprocessor] Loading image data with Skia.Data.fromURI...');
+    const data = await Skia.Data.fromURI(uri);
+    console.log('[imagePreprocessor] Data loaded:', !!data);
+    // Create SkImage from encoded data
+    const image = Skia.Image.MakeImageFromEncoded(data);
+    console.log('[imagePreprocessor] Image created:', !!image);
+    if (!image) {
+        throw new Error('Failed to decode image with Skia. The image format may not be supported.');
+    }
+    console.log('[imagePreprocessor] Image dimensions:', image.width(), 'x', image.height());
+    // Calculate scaling and padding for letterboxing
+    let scale = { x: 1, y: 1 };
+    let padding = { x: 0, y: 0 };
+    if (letterbox) {
+        const ratio = Math.min(targetWidth / originalWidth, targetHeight / originalHeight);
+        scale = { x: ratio, y: ratio };
+        padding = {
+            x: (targetWidth - originalWidth * ratio) / 2,
+            y: (targetHeight - originalHeight * ratio) / 2,
+        };
+    }
+    else {
+        scale = { x: targetWidth / originalWidth, y: targetHeight / originalHeight };
+    }
+    // Create a surface to draw the resized image
+    console.log('[imagePreprocessor] Creating surface...');
+    const surface = Skia.Surface.Make(targetWidth, targetHeight);
+    if (!surface) {
+        throw new Error('Failed to create Skia surface. This may happen on devices without GPU support.');
+    }
+    const canvas = surface.getCanvas();
+    // Fill with gray (letterbox padding color) - Skia.Color takes a color string
+    if (letterbox) {
+        canvas.clear(Skia.Color('gray'));
+    }
+    // Draw the resized image
+    const srcRect = Skia.XYWHRect(0, 0, originalWidth, originalHeight);
+    const dstRect = Skia.XYWHRect(padding.x, padding.y, originalWidth * scale.x, originalHeight * scale.y);
+    canvas.drawImageRect(image, srcRect, dstRect, Skia.Paint());
+    // Get pixel data from the snapshot
+    console.log('[imagePreprocessor] Reading pixels...');
+    const snapshot = surface.makeImageSnapshot();
+    // readPixels returns Uint8Array directly in newer Skia versions
+    // ColorType and AlphaType are separate exports from @shopify/react-native-skia
+    const pixels = snapshot.readPixels(0, 0, {
+        width: targetWidth,
+        height: targetHeight,
+        colorType: ColorType.RGBA_8888,
+        alphaType: AlphaType.Unpremul,
+    });
+    if (!pixels) {
+        throw new Error('Failed to read pixels from Skia image');
+    }
+    console.log('[imagePreprocessor] Pixels read, length:', pixels.length);
+    // Convert RGBA Uint8Array to RGB Float32Array
+    const rgbData = new Float32Array(targetWidth * targetHeight * 3);
+    for (let i = 0, j = 0; i < pixels.length; i += 4, j += 3) {
+        const r = pixels[i];
+        const g = pixels[i + 1];
+        const b = pixels[i + 2];
+        // Skip alpha (i + 3)
+        if (normalize) {
+            rgbData[j] = r / 255.0;
+            rgbData[j + 1] = g / 255.0;
+            rgbData[j + 2] = b / 255.0;
+        }
+        else {
+            rgbData[j] = r;
+            rgbData[j + 1] = g;
+            rgbData[j + 2] = b;
+        }
+    }
+    // Clean up if dispose methods exist
+    if (surface.dispose)
+        surface.dispose();
+    if (data.dispose)
+        data.dispose();
+    console.log('[imagePreprocessor] Preprocessing complete, output size:', rgbData.length);
+    return {
+        data: rgbData,
+        originalWidth,
+        originalHeight,
+        scale,
+        padding,
+    };
+}
+/**
+ * Preprocess a VisionCamera frame for TFLite inference
+ *
+ * This requires vision-camera-resize-plugin to be installed:
+ * npm install vision-camera-resize-plugin
+ *
+ * Usage in a frame processor:
+ * ```
+ * const frameProcessor = useFrameProcessor((frame) => {
+ *   'worklet';
+ *   const resized = resize(frame, {
+ *     size: { width: 640, height: 640 },
+ *     pixelFormat: 'rgb',
+ *     dataType: 'float32',
+ *   });
+ *   const result = model.runSync([resized]);
+ *   // Process result...
+ * }, [model]);
+ * ```
+ */
+function getFrameProcessorInstructions() {
+    return `
+To use on-device inference with VisionCamera frame processor:
+
+1. Install the resize plugin:
+   npm install vision-camera-resize-plugin
+   cd ios && pod install
+
+2. Use in your frame processor:
+   import { resize } from 'vision-camera-resize-plugin';
+
+   const frameProcessor = useFrameProcessor((frame) => {
+     'worklet';
+     const resized = resize(frame, {
+       size: { width: 640, height: 640 },
+       pixelFormat: 'rgb',
+       dataType: 'float32',
+     });
+     const result = model.runSync([resized]);
+     // Process result...
+   }, [model]);
+`;
+}
+/**
+ * Convert Float32Array from HWC to NHWC format (add batch dimension)
+ */
+function addBatchDimension(data) {
+    // Data is already in HWC format, just need to ensure it's treated as batch of 1
+    // The TFLite model expects [1, H, W, C] but since we pass a flat array,
+    // and the model knows the shape, we don't need to actually reshape
+    return data;
+}
+/**
+ * Check if Skia is available for image preprocessing
+ */
+function isSkiaAvailable() {
+    return checkSkiaAvailable();
+}
+/**
+ * Initialize Skia module (call before using Skia-dependent functions)
+ * @returns true if Skia was loaded successfully
+ */
+async function initSkia() {
+    return checkSkiaAvailable();
+}
+// Cached resize plugin availability
+let resizePluginAvailable = null;
+/**
+ * Check if resize plugin is available for frame processing
+ */
+async function isResizePluginAvailable() {
+    if (resizePluginAvailable !== null) {
+        return resizePluginAvailable;
+    }
+    try {
+        await Promise.resolve().then(() => __importStar(require('vision-camera-resize-plugin')));
+        resizePluginAvailable = true;
+        return true;
+    }
+    catch {
+        resizePluginAvailable = false;
+        return false;
+    }
+}

package/dist/utils/yoloProcessor.d.ts

@@ -0,0 +1,40 @@
+/**
+ * YOLO Output Processor
+ *
+ * Processes raw YOLOv8 TFLite model output into detection results.
+ * YOLOv8 output format: [1, 84, 8400] where:
+ *   - 84 = 4 (bbox) + 80 (class scores for COCO)
+ *   - 8400 = number of predictions (from 3 detection heads)
+ *
+ * For custom models with N classes: [1, 4+N, 8400]
+ */
+export interface RawDetection {
+    x: number;
+    y: number;
+    width: number;
+    height: number;
+    classIndex: number;
+    confidence: number;
+}
+export interface ProcessedDetection {
+    x: number;
+    y: number;
+    width: number;
+    height: number;
+    classIndex: number;
+    confidence: number;
+}
+/**
+ * Process YOLOv8 model output into detections
+ *
+ * @param output Raw model output as Float32Array
+ * @param numClasses Number of classes in the model
+ * @param confThreshold Minimum confidence threshold (0-1)
+ * @param iouThreshold IoU threshold for NMS (0-1)
+ * @returns Array of processed detections
+ */
+export declare function processYoloOutput(output: Float32Array, numClasses: number, confThreshold?: number, iouThreshold?: number): ProcessedDetection[];
+/**
+ * Scale detection coordinates to actual image dimensions
+ */
+export declare function scaleDetections(detections: ProcessedDetection[], modelSize: number, imageWidth: number, imageHeight: number): ProcessedDetection[];

package/dist/utils/yoloProcessor.js

@@ -0,0 +1,154 @@
+"use strict";
+/**
+ * YOLO Output Processor
+ *
+ * Processes raw YOLOv8 TFLite model output into detection results.
+ * YOLOv8 output format: [1, 84, 8400] where:
+ *   - 84 = 4 (bbox) + 80 (class scores for COCO)
+ *   - 8400 = number of predictions (from 3 detection heads)
+ *
+ * For custom models with N classes: [1, 4+N, 8400]
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.processYoloOutput = processYoloOutput;
+exports.scaleDetections = scaleDetections;
+/**
+ * Process YOLOv8 model output into detections
+ *
+ * @param output Raw model output as Float32Array
+ * @param numClasses Number of classes in the model
+ * @param confThreshold Minimum confidence threshold (0-1)
+ * @param iouThreshold IoU threshold for NMS (0-1)
+ * @returns Array of processed detections
+ */
+function processYoloOutput(output, numClasses, confThreshold = 0.25, iouThreshold = 0.45) {
+    // YOLOv8 output shape: [1, 4+numClasses, 8400]
+    // We need to transpose to [8400, 4+numClasses] for easier processing
+    const numPredictions = 8400;
+    // Note: numOutputs = 4 + numClasses (4 bbox coords + class scores)
+    // Extract raw detections above confidence threshold
+    const rawDetections = [];
+    for (let i = 0; i < numPredictions; i++) {
+        // Find the class with highest score
+        let maxScore = 0;
+        let maxClassIdx = 0;
+        for (let c = 0; c < numClasses; c++) {
+            // Output is in format [1, 84, 8400], so index is: (4 + c) * 8400 + i
+            const score = output[(4 + c) * numPredictions + i];
+            if (score > maxScore) {
+                maxScore = score;
+                maxClassIdx = c;
+            }
+        }
+        if (maxScore >= confThreshold) {
+            // Extract bounding box (center x, center y, width, height)
+            const cx = output[0 * numPredictions + i];
+            const cy = output[1 * numPredictions + i];
+            const w = output[2 * numPredictions + i];
+            const h = output[3 * numPredictions + i];
+            rawDetections.push({
+                x: cx,
+                y: cy,
+                width: w,
+                height: h,
+                classIndex: maxClassIdx,
+                confidence: maxScore,
+            });
+        }
+    }
+    // Apply Non-Maximum Suppression
+    const nmsDetections = nonMaxSuppression(rawDetections, iouThreshold);
+    // Convert from center format to corner format
+    return nmsDetections.map((det) => ({
+        x: Math.max(0, det.x - det.width / 2),
+        y: Math.max(0, det.y - det.height / 2),
+        width: det.width,
+        height: det.height,
+        classIndex: det.classIndex,
+        confidence: det.confidence,
+    }));
+}
+/**
+ * Non-Maximum Suppression to remove overlapping detections
+ */
+function nonMaxSuppression(detections, iouThreshold) {
+    if (detections.length === 0)
+        return [];
+    // Sort by confidence (highest first)
+    const sorted = [...detections].sort((a, b) => b.confidence - a.confidence);
+    const kept = [];
+    const suppressed = new Set();
+    for (let i = 0; i < sorted.length; i++) {
+        if (suppressed.has(i))
+            continue;
+        const current = sorted[i];
+        kept.push(current);
+        // Suppress overlapping detections of the same class
+        for (let j = i + 1; j < sorted.length; j++) {
+            if (suppressed.has(j))
+                continue;
+            const other = sorted[j];
+            // Only suppress same class
+            if (current.classIndex !== other.classIndex)
+                continue;
+            const iou = calculateIoU(current, other);
+            if (iou > iouThreshold) {
+                suppressed.add(j);
+            }
+        }
+    }
+    return kept;
+}
+/**
+ * Calculate Intersection over Union between two boxes
+ * Boxes are in center format (cx, cy, w, h)
+ */
+function calculateIoU(box1, box2) {
+    // Convert to corner format
+    const x1_min = box1.x - box1.width / 2;
+    const y1_min = box1.y - box1.height / 2;
+    const x1_max = box1.x + box1.width / 2;
+    const y1_max = box1.y + box1.height / 2;
+    const x2_min = box2.x - box2.width / 2;
+    const y2_min = box2.y - box2.height / 2;
+    const x2_max = box2.x + box2.width / 2;
+    const y2_max = box2.y + box2.height / 2;
+    // Calculate intersection
+    const intersect_x_min = Math.max(x1_min, x2_min);
+    const intersect_y_min = Math.max(y1_min, y2_min);
+    const intersect_x_max = Math.min(x1_max, x2_max);
+    const intersect_y_max = Math.min(y1_max, y2_max);
+    const intersect_width = Math.max(0, intersect_x_max - intersect_x_min);
+    const intersect_height = Math.max(0, intersect_y_max - intersect_y_min);
+    const intersect_area = intersect_width * intersect_height;
+    // Calculate union
+    const area1 = box1.width * box1.height;
+    const area2 = box2.width * box2.height;
+    const union_area = area1 + area2 - intersect_area;
+    if (union_area === 0)
+        return 0;
+    return intersect_area / union_area;
+}
+/**
+ * Scale detection coordinates to actual image dimensions
+ */
+function scaleDetections(detections, modelSize, imageWidth, imageHeight) {
+    // The model was trained on square images, so we need to account for letterboxing
+    const scale = Math.min(modelSize / imageWidth, modelSize / imageHeight);
+    const padX = (modelSize - imageWidth * scale) / 2;
+    const padY = (modelSize - imageHeight * scale) / 2;
+    return detections.map((det) => {
+        // Remove padding and scale back to original image dimensions
+        const x = (det.x * modelSize - padX) / scale / imageWidth;
+        const y = (det.y * modelSize - padY) / scale / imageHeight;
+        const w = (det.width * modelSize) / scale / imageWidth;
+        const h = (det.height * modelSize) / scale / imageHeight;
+        return {
+            ...det,
+            x: Math.max(0, Math.min(1, x)),
+            y: Math.max(0, Math.min(1, y)),
+            width: Math.max(0, Math.min(1 - x, w)),
+            height: Math.max(0, Math.min(1 - y, h)),
+        };
+    });
+}

package/package.json

@@ -0,0 +1,78 @@
+{
+  "name": "@souscheflabs/ml-vision",
+  "version": "0.1.0",
+  "description": "ML-powered product detection for React Native - multi-barcode scanning, receipt OCR, and visual product recognition",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "scripts": {
+    "build": "tsc",
+    "watch": "tsc --watch",
+    "clean": "rm -rf dist",
+    "prepublishOnly": "npm run lint && npm run clean && npm run build",
+    "typecheck": "tsc --noEmit",
+    "lint": "eslint src --ext .ts,.tsx",
+    "lint:fix": "eslint src --ext .ts,.tsx --fix",
+    "test": "jest",
+    "test:watch": "jest --watch",
+    "test:coverage": "jest --coverage"
+  },
+  "keywords": [
+    "react-native",
+    "ml",
+    "machine-learning",
+    "barcode",
+    "ocr",
+    "product-detection",
+    "vision-camera",
+    "tflite"
+  ],
+  "author": "Artan Muzhaqi",
+  "license": "ISC",
+  "type": "commonjs",
+  "devDependencies": {
+    "@testing-library/react-native": "^12.4.3",
+    "@types/jest": "^29.5.12",
+    "@types/react": "^19.2.8",
+    "@types/react-native": "^0.73.0",
+    "jest": "^29.7.0",
+    "react-native-reanimated": "^4.2.1",
+    "ts-jest": "^29.1.2",
+    "typescript": "^5.9.3"
+  },
+  "peerDependencies": {
+    "react": ">=18.0.0",
+    "react-native": ">=0.73.0",
+    "react-native-mmkv": ">=3.0.0",
+    "react-native-reanimated": ">=3.0.0",
+    "react-native-vision-camera": ">=4.0.0"
+  },
+  "peerDependenciesMeta": {
+    "react-native-mmkv": {
+      "optional": false
+    },
+    "react-native-vision-camera": {
+      "optional": false
+    },
+    "react-native-reanimated": {
+      "optional": true
+    }
+  },
+  "dependencies": {
+    "react-native-fast-tflite": "^1.5.0"
+  },
+  "engines": {
+    "node": ">=18.0.0"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git@github.com:SousChefLabs/ml-vision.git"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "files": [
+    "dist",
+    "README.md",
+    "LICENSE"
+  ]
+}