@souscheflabs/ml-vision 0.1.0 → 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (149) hide show
  1. package/CHANGELOG.md +54 -0
  2. package/LICENSE +15 -0
  3. package/README.md +50 -5
  4. package/dist/{components → cjs/components}/DetectionOverlay.js +1 -0
  5. package/dist/cjs/components/DetectionOverlay.js.map +1 -0
  6. package/dist/{components → cjs/components}/index.js +1 -0
  7. package/dist/cjs/components/index.js.map +1 -0
  8. package/dist/{core → cjs/core}/CacheManager.js +1 -0
  9. package/dist/cjs/core/CacheManager.js.map +1 -0
  10. package/dist/{core → cjs/core}/MLVisionProvider.js +4 -2
  11. package/dist/cjs/core/MLVisionProvider.js.map +1 -0
  12. package/dist/{core → cjs/core}/ServerClient.js +33 -13
  13. package/dist/cjs/core/ServerClient.js.map +1 -0
  14. package/dist/{core → cjs/core}/index.js +1 -0
  15. package/dist/cjs/core/index.js.map +1 -0
  16. package/dist/{hooks → cjs/hooks}/classLabels.js +1 -0
  17. package/dist/cjs/hooks/classLabels.js.map +1 -0
  18. package/dist/{hooks → cjs/hooks}/classLabelsCoco.js +1 -0
  19. package/dist/cjs/hooks/classLabelsCoco.js.map +1 -0
  20. package/dist/{hooks → cjs/hooks}/index.js +1 -0
  21. package/dist/cjs/hooks/index.js.map +1 -0
  22. package/dist/{hooks → cjs/hooks}/useMultiBarcodeScanner.js +6 -3
  23. package/dist/cjs/hooks/useMultiBarcodeScanner.js.map +1 -0
  24. package/dist/{hooks → cjs/hooks}/useProductDetector.js +161 -90
  25. package/dist/cjs/hooks/useProductDetector.js.map +1 -0
  26. package/dist/{hooks → cjs/hooks}/useReceiptScanner.js +60 -52
  27. package/dist/cjs/hooks/useReceiptScanner.js.map +1 -0
  28. package/dist/{hooks → cjs/hooks}/useVideoScanner.js +8 -5
  29. package/dist/cjs/hooks/useVideoScanner.js.map +1 -0
  30. package/dist/{index.js → cjs/index.js} +8 -4
  31. package/dist/cjs/index.js.map +1 -0
  32. package/dist/{processors → cjs/processors}/detectionProcessor.js +1 -0
  33. package/dist/cjs/processors/detectionProcessor.js.map +1 -0
  34. package/dist/{processors → cjs/processors}/index.js +1 -0
  35. package/dist/cjs/processors/index.js.map +1 -0
  36. package/dist/{processors → cjs/processors}/tfliteFrameProcessor.js +108 -25
  37. package/dist/cjs/processors/tfliteFrameProcessor.js.map +1 -0
  38. package/dist/{types → cjs/types}/barcode.js +1 -0
  39. package/dist/cjs/types/barcode.js.map +1 -0
  40. package/dist/{types → cjs/types}/detection.js +1 -0
  41. package/dist/cjs/types/detection.js.map +1 -0
  42. package/dist/{types → cjs/types}/index.js +1 -0
  43. package/dist/cjs/types/index.js.map +1 -0
  44. package/dist/{types → cjs/types}/ocr.js +1 -0
  45. package/dist/cjs/types/ocr.js.map +1 -0
  46. package/dist/{utils → cjs/utils}/imagePreprocessor.js +30 -25
  47. package/dist/cjs/utils/imagePreprocessor.js.map +1 -0
  48. package/dist/cjs/utils/logger.js +99 -0
  49. package/dist/cjs/utils/logger.js.map +1 -0
  50. package/dist/{utils → cjs/utils}/yoloProcessor.js +1 -0
  51. package/dist/cjs/utils/yoloProcessor.js.map +1 -0
  52. package/dist/esm/components/DetectionOverlay.js +130 -0
  53. package/dist/esm/components/DetectionOverlay.js.map +1 -0
  54. package/dist/esm/components/index.js +5 -0
  55. package/dist/esm/components/index.js.map +1 -0
  56. package/dist/esm/core/CacheManager.js +327 -0
  57. package/dist/esm/core/CacheManager.js.map +1 -0
  58. package/dist/esm/core/MLVisionProvider.js +185 -0
  59. package/dist/esm/core/MLVisionProvider.js.map +1 -0
  60. package/dist/esm/core/ServerClient.js +305 -0
  61. package/dist/esm/core/ServerClient.js.map +1 -0
  62. package/dist/esm/core/index.js +7 -0
  63. package/dist/esm/core/index.js.map +1 -0
  64. package/dist/esm/hooks/classLabels.js +436 -0
  65. package/dist/esm/hooks/classLabels.js.map +1 -0
  66. package/dist/esm/hooks/classLabelsCoco.js +98 -0
  67. package/dist/esm/hooks/classLabelsCoco.js.map +1 -0
  68. package/dist/esm/hooks/index.js +9 -0
  69. package/dist/esm/hooks/index.js.map +1 -0
  70. package/dist/esm/hooks/useMultiBarcodeScanner.js +290 -0
  71. package/dist/esm/hooks/useMultiBarcodeScanner.js.map +1 -0
  72. package/dist/esm/hooks/useProductDetector.js +714 -0
  73. package/dist/esm/hooks/useProductDetector.js.map +1 -0
  74. package/dist/esm/hooks/useReceiptScanner.js +407 -0
  75. package/dist/esm/hooks/useReceiptScanner.js.map +1 -0
  76. package/dist/esm/hooks/useVideoScanner.js +383 -0
  77. package/dist/esm/hooks/useVideoScanner.js.map +1 -0
  78. package/dist/esm/index.js +93 -0
  79. package/dist/esm/index.js.map +1 -0
  80. package/dist/esm/processors/detectionProcessor.js +121 -0
  81. package/dist/esm/processors/detectionProcessor.js.map +1 -0
  82. package/dist/esm/processors/index.js +7 -0
  83. package/dist/esm/processors/index.js.map +1 -0
  84. package/dist/esm/processors/tfliteFrameProcessor.js +292 -0
  85. package/dist/esm/processors/tfliteFrameProcessor.js.map +1 -0
  86. package/dist/esm/types/barcode.js +17 -0
  87. package/dist/esm/types/barcode.js.map +1 -0
  88. package/dist/esm/types/detection.js +8 -0
  89. package/dist/esm/types/detection.js.map +1 -0
  90. package/dist/esm/types/index.js +10 -0
  91. package/dist/esm/types/index.js.map +1 -0
  92. package/dist/esm/types/ocr.js +8 -0
  93. package/dist/esm/types/ocr.js.map +1 -0
  94. package/dist/esm/utils/imagePreprocessor.js +268 -0
  95. package/dist/esm/utils/imagePreprocessor.js.map +1 -0
  96. package/dist/esm/utils/logger.js +94 -0
  97. package/dist/esm/utils/logger.js.map +1 -0
  98. package/dist/esm/utils/yoloProcessor.js +151 -0
  99. package/dist/esm/utils/yoloProcessor.js.map +1 -0
  100. package/dist/{components → types/components}/DetectionOverlay.d.ts +1 -0
  101. package/dist/types/components/DetectionOverlay.d.ts.map +1 -0
  102. package/dist/{components → types/components}/index.d.ts +1 -0
  103. package/dist/types/components/index.d.ts.map +1 -0
  104. package/dist/{core → types/core}/CacheManager.d.ts +1 -0
  105. package/dist/types/core/CacheManager.d.ts.map +1 -0
  106. package/dist/{core → types/core}/MLVisionProvider.d.ts +1 -0
  107. package/dist/types/core/MLVisionProvider.d.ts.map +1 -0
  108. package/dist/{core → types/core}/ServerClient.d.ts +1 -0
  109. package/dist/types/core/ServerClient.d.ts.map +1 -0
  110. package/dist/{core → types/core}/index.d.ts +1 -0
  111. package/dist/types/core/index.d.ts.map +1 -0
  112. package/dist/{hooks → types/hooks}/classLabels.d.ts +1 -0
  113. package/dist/types/hooks/classLabels.d.ts.map +1 -0
  114. package/dist/{hooks → types/hooks}/classLabelsCoco.d.ts +1 -0
  115. package/dist/types/hooks/classLabelsCoco.d.ts.map +1 -0
  116. package/dist/{hooks → types/hooks}/index.d.ts +1 -0
  117. package/dist/types/hooks/index.d.ts.map +1 -0
  118. package/dist/{hooks → types/hooks}/useMultiBarcodeScanner.d.ts +1 -0
  119. package/dist/types/hooks/useMultiBarcodeScanner.d.ts.map +1 -0
  120. package/dist/{hooks → types/hooks}/useProductDetector.d.ts +1 -0
  121. package/dist/types/hooks/useProductDetector.d.ts.map +1 -0
  122. package/dist/{hooks → types/hooks}/useReceiptScanner.d.ts +2 -1
  123. package/dist/types/hooks/useReceiptScanner.d.ts.map +1 -0
  124. package/dist/{hooks → types/hooks}/useVideoScanner.d.ts +1 -0
  125. package/dist/types/hooks/useVideoScanner.d.ts.map +1 -0
  126. package/dist/types/index.d.ts +58 -124
  127. package/dist/types/index.d.ts.map +1 -0
  128. package/dist/{processors → types/processors}/detectionProcessor.d.ts +1 -0
  129. package/dist/types/processors/detectionProcessor.d.ts.map +1 -0
  130. package/dist/{processors → types/processors}/index.d.ts +1 -0
  131. package/dist/types/processors/index.d.ts.map +1 -0
  132. package/dist/{processors → types/processors}/tfliteFrameProcessor.d.ts +5 -4
  133. package/dist/types/processors/tfliteFrameProcessor.d.ts.map +1 -0
  134. package/dist/types/{barcode.d.ts → types/barcode.d.ts} +1 -0
  135. package/dist/types/types/barcode.d.ts.map +1 -0
  136. package/dist/types/{detection.d.ts → types/detection.d.ts} +1 -0
  137. package/dist/types/types/detection.d.ts.map +1 -0
  138. package/dist/types/types/index.d.ts +127 -0
  139. package/dist/types/types/index.d.ts.map +1 -0
  140. package/dist/types/{ocr.d.ts → types/ocr.d.ts} +1 -0
  141. package/dist/types/types/ocr.d.ts.map +1 -0
  142. package/dist/{utils → types/utils}/imagePreprocessor.d.ts +1 -0
  143. package/dist/types/utils/imagePreprocessor.d.ts.map +1 -0
  144. package/dist/types/utils/logger.d.ts +52 -0
  145. package/dist/types/utils/logger.d.ts.map +1 -0
  146. package/dist/{utils → types/utils}/yoloProcessor.d.ts +1 -0
  147. package/dist/types/utils/yoloProcessor.d.ts.map +1 -0
  148. package/package.json +62 -21
  149. package/dist/index.d.ts +0 -58
@@ -0,0 +1,292 @@
1
+ /**
2
+ * TFLite Frame Processor for Real-time Video Detection
3
+ *
4
+ * Uses vision-camera-resize-plugin to resize frames and run
5
+ * TFLite inference directly in the frame processor worklet.
6
+ *
7
+ * Requirements:
8
+ * - npm install vision-camera-resize-plugin
9
+ * - npm install react-native-fast-tflite
10
+ * - npm install react-native-worklets-core
11
+ */
12
+ import { useCallback, useState, useEffect } from "react";
13
+ import { useSharedValue } from "react-native-reanimated";
14
+ import { Worklets } from "react-native-worklets-core";
15
+ import { useResizePlugin } from "vision-camera-resize-plugin";
16
+ import { logger } from "../utils/logger";
17
+ // COCO food class indices
18
+ const COCO_FOOD_INDICES = new Set([46, 47, 48, 49, 50, 51, 52, 53, 54, 55]);
19
+ // COCO class names
20
+ const COCO_LABELS = [
21
+ "person",
22
+ "bicycle",
23
+ "car",
24
+ "motorcycle",
25
+ "airplane",
26
+ "bus",
27
+ "train",
28
+ "truck",
29
+ "boat",
30
+ "traffic light",
31
+ "fire hydrant",
32
+ "stop sign",
33
+ "parking meter",
34
+ "bench",
35
+ "bird",
36
+ "cat",
37
+ "dog",
38
+ "horse",
39
+ "sheep",
40
+ "cow",
41
+ "elephant",
42
+ "bear",
43
+ "zebra",
44
+ "giraffe",
45
+ "backpack",
46
+ "umbrella",
47
+ "handbag",
48
+ "tie",
49
+ "suitcase",
50
+ "frisbee",
51
+ "skis",
52
+ "snowboard",
53
+ "sports ball",
54
+ "kite",
55
+ "baseball bat",
56
+ "baseball glove",
57
+ "skateboard",
58
+ "surfboard",
59
+ "tennis racket",
60
+ "bottle",
61
+ "wine glass",
62
+ "cup",
63
+ "fork",
64
+ "knife",
65
+ "spoon",
66
+ "bowl",
67
+ "banana",
68
+ "apple",
69
+ "sandwich",
70
+ "orange",
71
+ "broccoli",
72
+ "carrot",
73
+ "hot dog",
74
+ "pizza",
75
+ "donut",
76
+ "cake",
77
+ "chair",
78
+ "couch",
79
+ "potted plant",
80
+ "bed",
81
+ "dining table",
82
+ "toilet",
83
+ "tv",
84
+ "laptop",
85
+ "mouse",
86
+ "remote",
87
+ "keyboard",
88
+ "cell phone",
89
+ "microwave",
90
+ "oven",
91
+ "toaster",
92
+ "sink",
93
+ "refrigerator",
94
+ "book",
95
+ "clock",
96
+ "vase",
97
+ "scissors",
98
+ "teddy bear",
99
+ "hair drier",
100
+ "toothbrush",
101
+ ];
102
+ /**
103
+ * Process YOLOv8 output in worklet (simplified for worklet compatibility)
104
+ * This runs synchronously in the frame processor thread
105
+ */
106
+ function processYoloOutputWorklet(output, numClasses, confThreshold, _iouThreshold, labels, foodOnly) {
107
+ "worklet";
108
+ const data = new Float32Array(output);
109
+ const numPredictions = 8400;
110
+ const detections = [];
111
+ // Extract detections above threshold
112
+ for (let i = 0; i < numPredictions; i++) {
113
+ let maxScore = 0;
114
+ let maxClassIdx = 0;
115
+ // Find best class
116
+ for (let c = 0; c < numClasses; c++) {
117
+ const score = data[(4 + c) * numPredictions + i];
118
+ if (score > maxScore) {
119
+ maxScore = score;
120
+ maxClassIdx = c;
121
+ }
122
+ }
123
+ if (maxScore >= confThreshold) {
124
+ // Filter for food classes if requested
125
+ if (foodOnly && !COCO_FOOD_INDICES.has(maxClassIdx)) {
126
+ continue;
127
+ }
128
+ // Extract box (center x, center y, width, height)
129
+ const cx = data[0 * numPredictions + i];
130
+ const cy = data[1 * numPredictions + i];
131
+ const w = data[2 * numPredictions + i];
132
+ const h = data[3 * numPredictions + i];
133
+ detections.push({
134
+ x: Math.max(0, cx - w / 2),
135
+ y: Math.max(0, cy - h / 2),
136
+ width: w,
137
+ height: h,
138
+ classIndex: maxClassIdx,
139
+ confidence: maxScore,
140
+ label: labels[maxClassIdx] || `class_${maxClassIdx}`,
141
+ });
142
+ }
143
+ }
144
+ // Simple NMS (keep top detection per class)
145
+ const kept = [];
146
+ const seenClasses = new Set();
147
+ // Sort by confidence
148
+ detections.sort((a, b) => b.confidence - a.confidence);
149
+ for (const det of detections) {
150
+ if (!seenClasses.has(det.classIndex)) {
151
+ kept.push(det);
152
+ seenClasses.add(det.classIndex);
153
+ }
154
+ }
155
+ return kept.slice(0, 10); // Max 10 detections
156
+ }
157
+ /**
158
+ * Hook to create a TFLite-powered frame processor
159
+ *
160
+ * @example
161
+ * ```typescript
162
+ * const { frameProcessor, detections, isProcessing } = useTFLiteFrameProcessor({
163
+ * model: loadedModel,
164
+ * numClasses: 80,
165
+ * foodOnly: true,
166
+ * });
167
+ *
168
+ * // Use with VisionCamera
169
+ * <Camera frameProcessor={frameProcessor} />
170
+ * ```
171
+ */
172
+ export function useTFLiteFrameProcessor(options) {
173
+ const { model, inputSize = 640, numClasses = 80, confThreshold = 0.25, iouThreshold = 0.45, classLabels = COCO_LABELS, foodOnly = false, onDetections, } = options;
174
+ const [detections, setDetections] = useState([]);
175
+ const [isProcessing, _setIsProcessing] = useState(false);
176
+ const [frameCount, setFrameCount] = useState(0);
177
+ // Use shared value for throttle timing (accessible from worklet)
178
+ const lastProcessTime = useSharedValue(0);
179
+ // Use the resize plugin hook
180
+ const { resize } = useResizePlugin();
181
+ // Log state on mount and when model changes
182
+ useEffect(() => {
183
+ logger.debug("[TFLiteFrameProcessor] Hook state:", {
184
+ model: !!model,
185
+ resize: !!resize,
186
+ });
187
+ }, [model, resize]);
188
+ // Debug: log frame count changes
189
+ useEffect(() => {
190
+ if (frameCount > 0) {
191
+ logger.debug(`[TFLiteFrameProcessor] Processed ${frameCount} frames`);
192
+ }
193
+ }, [frameCount]);
194
+ // Helper to log from worklet
195
+ const logFromWorklet = useCallback((msg) => {
196
+ logger.debug(msg);
197
+ }, []);
198
+ const incrementFrameCount = useCallback(() => {
199
+ setFrameCount((prev) => prev + 1);
200
+ }, []);
201
+ // Create frame processor
202
+ const frameProcessor = useCallback((frame) => {
203
+ "worklet";
204
+ if (!model) {
205
+ Worklets.runOnJS(() => logFromWorklet("[TFLiteFrameProcessor] No model"));
206
+ return;
207
+ }
208
+ // Throttle to ~10fps for processing
209
+ const now = Date.now();
210
+ if (now - lastProcessTime.value < 100) {
211
+ return;
212
+ }
213
+ lastProcessTime.value = now;
214
+ Worklets.runOnJS(() => incrementFrameCount());
215
+ try {
216
+ // Resize frame to model input size and get RGB float32 tensor
217
+ const resized = resize(frame, {
218
+ scale: {
219
+ width: inputSize,
220
+ height: inputSize,
221
+ },
222
+ pixelFormat: "rgb",
223
+ dataType: "float32",
224
+ });
225
+ // Run TFLite inference synchronously in worklet
226
+ const outputs = model.runSync([resized]);
227
+ // Process YOLO output
228
+ // outputs[0] is a TypedArray, we need its underlying buffer
229
+ const outputData = outputs[0];
230
+ const frameDetections = processYoloOutputWorklet(outputData.buffer ?? outputs[0], numClasses, confThreshold, iouThreshold, classLabels, foodOnly);
231
+ // Update state on JS thread
232
+ if (frameDetections.length > 0) {
233
+ Worklets.runOnJS(() => setDetections(frameDetections));
234
+ if (onDetections) {
235
+ Worklets.runOnJS(() => onDetections(frameDetections));
236
+ }
237
+ }
238
+ }
239
+ catch (error) {
240
+ Worklets.runOnJS(() => logFromWorklet(`[TFLiteFrameProcessor] Error: ${error}`));
241
+ }
242
+ }, [
243
+ model,
244
+ resize,
245
+ inputSize,
246
+ numClasses,
247
+ confThreshold,
248
+ iouThreshold,
249
+ classLabels,
250
+ foodOnly,
251
+ onDetections,
252
+ lastProcessTime,
253
+ logFromWorklet,
254
+ incrementFrameCount,
255
+ ]);
256
+ return {
257
+ frameProcessor: model ? frameProcessor : undefined,
258
+ detections,
259
+ isProcessing,
260
+ isReady: !!model,
261
+ };
262
+ }
263
+ /**
264
+ * Create a simple frame processor function for use with useFrameProcessor
265
+ *
266
+ * This is for manual integration when you need more control.
267
+ * The resize function must be obtained from useResizePlugin hook.
268
+ *
269
+ * @example
270
+ * ```typescript
271
+ * const { resize } = useResizePlugin();
272
+ * const processor = createTFLiteFrameProcessor(model, resize, { foodOnly: true });
273
+ * ```
274
+ */
275
+ export function createTFLiteFrameProcessor(model, resizeFn, options = {}) {
276
+ const { inputSize = 640, numClasses = 80, confThreshold = 0.25, classLabels = COCO_LABELS, foodOnly = false, } = options;
277
+ return (frame) => {
278
+ "worklet";
279
+ // Resize to model input
280
+ const resized = resizeFn(frame, {
281
+ scale: { width: inputSize, height: inputSize },
282
+ pixelFormat: "rgb",
283
+ dataType: "float32",
284
+ });
285
+ // Run inference
286
+ const outputs = model.runSync([resized]);
287
+ // Process output - access the underlying buffer from TypedArray
288
+ const outputData = outputs[0];
289
+ return processYoloOutputWorklet(outputData.buffer ?? outputs[0], numClasses, confThreshold, 0.45, classLabels, foodOnly);
290
+ };
291
+ }
292
+ //# sourceMappingURL=tfliteFrameProcessor.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"tfliteFrameProcessor.js","sourceRoot":"","sources":["../../../src/processors/tfliteFrameProcessor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,WAAW,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,OAAO,CAAC;AAGzD,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AACzD,OAAO,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAC;AACtD,OAAO,EAAE,eAAe,EAAE,MAAM,6BAA6B,CAAC;AAC9D,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAuCzC,0BAA0B;AAC1B,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC;AAE5E,mBAAmB;AACnB,MAAM,WAAW,GAAG;IAClB,QAAQ;IACR,SAAS;IACT,KAAK;IACL,YAAY;IACZ,UAAU;IACV,KAAK;IACL,OAAO;IACP,OAAO;IACP,MAAM;IACN,eAAe;IACf,cAAc;IACd,WAAW;IACX,eAAe;IACf,OAAO;IACP,MAAM;IACN,KAAK;IACL,KAAK;IACL,OAAO;IACP,OAAO;IACP,KAAK;IACL,UAAU;IACV,MAAM;IACN,OAAO;IACP,SAAS;IACT,UAAU;IACV,UAAU;IACV,SAAS;IACT,KAAK;IACL,UAAU;IACV,SAAS;IACT,MAAM;IACN,WAAW;IACX,aAAa;IACb,MAAM;IACN,cAAc;IACd,gBAAgB;IAChB,YAAY;IACZ,WAAW;IACX,eAAe;IACf,QAAQ;IACR,YAAY;IACZ,KAAK;IACL,MAAM;IACN,OAAO;IACP,OAAO;IACP,MAAM;IACN,QAAQ;IACR,OAAO;IACP,UAAU;IACV,QAAQ;IACR,UAAU;IACV,QAAQ;IACR,SAAS;IACT,OAAO;IACP,OAAO;IACP,MAAM;IACN,OAAO;IACP,OAAO;IACP,cAAc;IACd,KAAK;IACL,cAAc;IACd,QAAQ;IACR,IAAI;IACJ,QAAQ;IACR,OAAO;IACP,QAAQ;IACR,UAAU;IACV,YAAY;IACZ,WAAW;IACX,MAAM;IACN,SAAS;IACT,MAAM;IACN,cAAc;IACd,MAAM;IACN,OAAO;IACP,MAAM;IACN,UAAU;IACV,YAAY;IACZ,YAAY;IACZ,YAAY;CACb,CAAC;AAEF;;;GAGG;AACH,SAAS,wBAAwB,CAC/B,MAAmB,EACnB,UAAkB,EAClB,aAAqB,EACrB,aAAqB,EACrB,MAAgB,EAChB,QAAiB;IAEjB,SAAS,CAAC;IAEV,MAAM,IAAI,GAAG,IAAI,YAAY,CAAC,MAAM,CAAC,CAAC;IACtC,MAAM,cAAc,GAAG,IAAI,CAAC;IAC5B,MAAM,UAAU,GAAqB,EAAE,CAAC;IAExC,qCAAqC;IACrC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,cAAc,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,IAAI,WAAW,GAAG,CAAC,CAAC;QAEpB,kBAAkB;QAClB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;YACpC,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,cAAc,GAAG,CAAC,CAAC,CAAC;YACjD,IAAI,KAAK,GAAG,QAAQ,EAAE,CAAC;gBACrB,QAAQ,GAAG,KAAK,CAAC;gBACjB,WAAW,GAAG,CAAC,CAAC;YAClB,CAAC;QACH,CAAC;QAED,IAAI,QAAQ,IAAI,aAAa,EAAE,CAAC;YAC9B,uCAAuC;YACvC,IAAI,QAAQ,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,CAAC;gBACpD,SAAS;YACX,CAAC;YAED,kDAAkD;YAClD,MAAM,EAAE,GAAG,IAAI,CAAC,CAAC,GAAG,cAAc,GAAG,CAAC,CAAC,CAAC;YACxC,MAAM,EAAE,GAAG,IAAI,CAAC,CAAC,GAAG,cAAc,GAAG,CAAC,CAAC,CAAC;YACxC,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC,GAAG,cAAc,GAAG,CAAC,CAAC,CAAC;YACvC,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC,GAAG,cAAc,GAAG,CAAC,CAAC,CAAC;YAEvC,UAAU,CAAC,IAAI,CAAC;gBACd,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;gBAC1B,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;gBAC1B,KAAK,EAAE,CAAC;gBACR,MAAM,EAAE,CAAC;gBACT,UAAU,EAAE,WAAW;gBACvB,UAAU,EAAE,QAAQ;gBACpB,KAAK,EAAE,MAAM,CAAC,WAAW,CAAC,IAAI,SAAS,WAAW,EAAE;aACrD,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,4CAA4C;IAC5C,MAAM,IAAI,GAAqB,EAAE,CAAC;IAClC,MAAM,WAAW,GAAG,IAAI,GAAG,EAAU,CAAC;IAEtC,qBAAqB;IACrB,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC;IAEvD,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;QAC7B,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;YACrC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACf,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAClC,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,oBAAoB;AAChD,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,UAAU,uBAAuB,CACrC,OAAuC;IAEvC,MAAM,EACJ,KAAK,EACL,SAAS,GAAG,GAAG,EACf,UAAU,GAAG,EAAE,EACf,aAAa,GAAG,IAAI,EACpB,YAAY,GAAG,IAAI,EACnB,WAAW,GAAG,WAAW,EACzB,QAAQ,GAAG,KAAK,EAChB,YAAY,GACb,GAAG,OAAO,CAAC;IAEZ,MAAM,CAAC,UAAU,EAAE,aAAa,CAAC,GAAG,QAAQ,CAAmB,EAAE,CAAC,CAAC;IACnE,MAAM,CAAC,YAAY,EAAE,gBAAgB,CAAC,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;IACzD,MAAM,CAAC,UAAU,EAAE,aAAa,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;IAEhD,iEAAiE;IACjE,MAAM,eAAe,GAAG,cAAc,CAAC,CAAC,CAAC,CAAC;IAE1C,6BAA6B;IAC7B,MAAM,EAAE,MAAM,EAAE,GAAG,eAAe,EAAE,CAAC;IAErC,4CAA4C;IAC5C,SAAS,CAAC,GAAG,EAAE;QACb,MAAM,CAAC,KAAK,CAAC,oCAAoC,EAAE;YACjD,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,MAAM,EAAE,CAAC,CAAC,MAAM;SACjB,CAAC,CAAC;IACL,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC;IAEpB,iCAAiC;IACjC,SAAS,CAAC,GAAG,EAAE;QACb,IAAI,UAAU,GAAG,CAAC,EAAE,CAAC;YACnB,MAAM,CAAC,KAAK,CAAC,oCAAoC,UAAU,SAAS,CAAC,CAAC;QACxE,CAAC;IACH,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC;IAEjB,6BAA6B;IAC7B,MAAM,cAAc,GAAG,WAAW,CAAC,CAAC,GAAW,EAAE,EAAE;QACjD,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACpB,CAAC,EAAE,EAAE,CAAC,CAAC;IAEP,MAAM,mBAAmB,GAAG,WAAW,CAAC,GAAG,EAAE;QAC3C,aAAa,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC;IACpC,CAAC,EAAE,EAAE,CAAC,CAAC;IAEP,yBAAyB;IACzB,MAAM,cAAc,GAAG,WAAW,CAChC,CAAC,KAAY,EAAE,EAAE;QACf,SAAS,CAAC;QAEV,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,QAAQ,CAAC,OAAO,CAAC,GAAG,EAAE,CACpB,cAAc,CAAC,iCAAiC,CAAC,CAClD,CAAC;YACF,OAAO;QACT,CAAC;QAED,oCAAoC;QACpC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,IAAI,GAAG,GAAG,eAAe,CAAC,KAAK,GAAG,GAAG,EAAE,CAAC;YACtC,OAAO;QACT,CAAC;QACD,eAAe,CAAC,KAAK,GAAG,GAAG,CAAC;QAE5B,QAAQ,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC,CAAC;QAE9C,IAAI,CAAC;YACH,8DAA8D;YAC9D,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,EAAE;gBAC5B,KAAK,EAAE;oBACL,KAAK,EAAE,SAAS;oBAChB,MAAM,EAAE,SAAS;iBAClB;gBACD,WAAW,EAAE,KAAK;gBAClB,QAAQ,EAAE,SAAS;aACpB,CAAC,CAAC;YAEH,gDAAgD;YAChD,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;YAEzC,sBAAsB;YACtB,4DAA4D;YAC5D,MAAM,UAAU,GAAG,OAAO,CAAC,CAAC,CAAuC,CAAC;YACpE,MAAM,eAAe,GAAG,wBAAwB,CAC9C,UAAU,CAAC,MAAM,IAAK,OAAO,CAAC,CAAC,CAA4B,EAC3D,UAAU,EACV,aAAa,EACb,YAAY,EACZ,WAAW,EACX,QAAQ,CACT,CAAC;YAEF,4BAA4B;YAC5B,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC/B,QAAQ,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,aAAa,CAAC,eAAe,CAAC,CAAC,CAAC;gBACvD,IAAI,YAAY,EAAE,CAAC;oBACjB,QAAQ,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,YAAY,CAAC,eAAe,CAAC,CAAC,CAAC;gBACxD,CAAC;YACH,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,QAAQ,CAAC,OAAO,CAAC,GAAG,EAAE,CACpB,cAAc,CAAC,iCAAiC,KAAK,EAAE,CAAC,CACzD,CAAC;QACJ,CAAC;IACH,CAAC,EACD;QACE,KAAK;QACL,MAAM;QACN,SAAS;QACT,UAAU;QACV,aAAa;QACb,YAAY;QACZ,WAAW;QACX,QAAQ;QACR,YAAY;QACZ,eAAe;QACf,cAAc;QACd,mBAAmB;KACpB,CACF,CAAC;IAEF,OAAO;QACL,cAAc,EAAE,KAAK,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,SAAS;QAClD,UAAU;QACV,YAAY;QACZ,OAAO,EAAE,CAAC,CAAC,KAAK;KACjB,CAAC;AACJ,CAAC;AAKD;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,0BAA0B,CACxC,KAAsB,EACtB,QAAwB,EACxB,UAMI,EAAE;IAEN,MAAM,EACJ,SAAS,GAAG,GAAG,EACf,UAAU,GAAG,EAAE,EACf,aAAa,GAAG,IAAI,EACpB,WAAW,GAAG,WAAW,EACzB,QAAQ,GAAG,KAAK,GACjB,GAAG,OAAO,CAAC;IAEZ,OAAO,CAAC,KAAY,EAAoB,EAAE;QACxC,SAAS,CAAC;QAEV,wBAAwB;QACxB,MAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,EAAE;YAC9B,KAAK,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE;YAC9C,WAAW,EAAE,KAAK;YAClB,QAAQ,EAAE,SAAS;SACpB,CAAC,CAAC;QAEH,gBAAgB;QAChB,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;QAEzC,gEAAgE;QAChE,MAAM,UAAU,GAAG,OAAO,CAAC,CAAC,CAAuC,CAAC;QACpE,OAAO,wBAAwB,CAC7B,UAAU,CAAC,MAAM,IAAK,OAAO,CAAC,CAAC,CAA4B,EAC3D,UAAU,EACV,aAAa,EACb,IAAI,EACJ,WAAW,EACX,QAAQ,CACT,CAAC;IACJ,CAAC,CAAC;AACJ,CAAC"}
@@ -0,0 +1,17 @@
1
+ /**
2
+ * Barcode Detection Types
3
+ *
4
+ * Types for multi-barcode scanning functionality.
5
+ * Designed to be compatible with react-native-vision-camera's CodeScanner.
6
+ */
7
+ /**
8
+ * Common grocery barcode formats (optimized subset)
9
+ */
10
+ export const GROCERY_BARCODE_FORMATS = [
11
+ 'ean-13',
12
+ 'ean-8',
13
+ 'upc-a',
14
+ 'upc-e',
15
+ 'qr',
16
+ ];
17
+ //# sourceMappingURL=barcode.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"barcode.js","sourceRoot":"","sources":["../../../src/types/barcode.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AA2BH;;GAEG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAoB;IACtD,QAAQ;IACR,OAAO;IACP,OAAO;IACP,OAAO;IACP,IAAI;CACL,CAAC"}
@@ -0,0 +1,8 @@
1
+ /**
2
+ * Object Detection Types
3
+ *
4
+ * Types for visual product detection (fridge, pantry, spice rack scanning).
5
+ * Used with TFLite models for on-device inference.
6
+ */
7
+ export {};
8
+ //# sourceMappingURL=detection.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"detection.js","sourceRoot":"","sources":["../../../src/types/detection.ts"],"names":[],"mappings":"AAAA;;;;;GAKG"}
@@ -0,0 +1,10 @@
1
+ /**
2
+ * @souschef/ml-vision Type Definitions
3
+ *
4
+ * Core types for the ML Vision package. These types are designed to be
5
+ * compatible with your existing Sous Chef app patterns.
6
+ */
7
+ export * from './barcode';
8
+ export * from './detection';
9
+ export * from './ocr';
10
+ //# sourceMappingURL=index.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/types/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,cAAc,WAAW,CAAC;AAC1B,cAAc,aAAa,CAAC;AAC5B,cAAc,OAAO,CAAC"}
@@ -0,0 +1,8 @@
1
+ /**
2
+ * OCR and Receipt Scanning Types
3
+ *
4
+ * Types for text recognition and receipt parsing functionality.
5
+ * Uses MLKit Text Recognition on-device with server fallback.
6
+ */
7
+ export {};
8
+ //# sourceMappingURL=ocr.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"ocr.js","sourceRoot":"","sources":["../../../src/types/ocr.ts"],"names":[],"mappings":"AAAA;;;;;GAKG"}
@@ -0,0 +1,268 @@
1
+ /**
2
+ * Image Preprocessor for TFLite Models
3
+ *
4
+ * Converts images to the format expected by TFLite models:
5
+ * - Resized to model input size (e.g., 640x640)
6
+ * - Normalized to [0, 1] range
7
+ * - As Float32Array in NHWC format (batch, height, width, channels)
8
+ *
9
+ * This implementation uses react-native-skia for image manipulation.
10
+ * Install with: npm install @shopify/react-native-skia
11
+ */
12
+ import { Image } from "react-native";
13
+ import { logger } from "../utils/logger";
14
+ // Lazy-loaded Skia references (to avoid crashes when Skia isn't needed)
15
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
16
+ let SkiaModule = null;
17
+ let skiaLoadAttempted = false;
18
+ /**
19
+ * Load Skia module lazily (only when needed for photo preprocessing)
20
+ */
21
+ async function loadSkiaModule() {
22
+ if (SkiaModule)
23
+ return true;
24
+ if (skiaLoadAttempted)
25
+ return SkiaModule !== null;
26
+ skiaLoadAttempted = true;
27
+ try {
28
+ logger.debug("[imagePreprocessor] Loading Skia module...");
29
+ // Lazy require to avoid loading Skia when not needed (e.g., video scanner)
30
+ SkiaModule = require("@shopify/react-native-skia");
31
+ logger.debug("[imagePreprocessor] Skia loaded successfully");
32
+ return true;
33
+ }
34
+ catch (error) {
35
+ logger.error("[imagePreprocessor] Failed to load Skia:", error);
36
+ SkiaModule = null;
37
+ return false;
38
+ }
39
+ }
40
+ /**
41
+ * Check if Skia is available (synchronous check after loading)
42
+ */
43
+ function checkSkiaAvailable() {
44
+ if (!SkiaModule)
45
+ return false;
46
+ try {
47
+ const { Skia } = SkiaModule;
48
+ return !!(Skia?.Data && Skia?.Image && Skia?.Surface);
49
+ }
50
+ catch {
51
+ return false;
52
+ }
53
+ }
54
+ /**
55
+ * Preprocess an image for TFLite model inference
56
+ *
57
+ * @param uri Image URI (file:// or http://)
58
+ * @param options Preprocessing options
59
+ * @returns Preprocessed image data
60
+ */
61
+ export async function preprocessImage(uri, options = {}) {
62
+ const { width: targetWidth = 640, height: targetHeight = 640, normalize = true, letterbox = true, } = options;
63
+ // Load Skia module first (lazy loading)
64
+ const loaded = await loadSkiaModule();
65
+ if (!loaded || !checkSkiaAvailable()) {
66
+ throw new Error("Image preprocessing requires @shopify/react-native-skia. " +
67
+ "Install with: npm install @shopify/react-native-skia\n\n" +
68
+ "Alternatively, use server-side inference or VisionCamera frame processor.");
69
+ }
70
+ // Get original image dimensions
71
+ const { width: originalWidth, height: originalHeight } = await getImageSize(uri);
72
+ // Use Skia for preprocessing
73
+ return preprocessWithSkia(uri, originalWidth, originalHeight, targetWidth, targetHeight, normalize, letterbox);
74
+ }
75
+ /**
76
+ * Get image dimensions from URI
77
+ */
78
+ async function getImageSize(uri) {
79
+ return new Promise((resolve, reject) => {
80
+ Image.getSize(uri, (width, height) => resolve({ width, height }), (error) => reject(new Error(`Failed to get image size: ${error}`)));
81
+ });
82
+ }
83
+ /**
84
+ * Preprocess using react-native-skia
85
+ * Uses Skia.Data.fromURI() for proper image loading
86
+ */
87
+ async function preprocessWithSkia(uri, originalWidth, originalHeight, targetWidth, targetHeight, normalize, letterbox) {
88
+ // Get Skia APIs from the lazy-loaded module
89
+ const { Skia, ColorType, AlphaType } = SkiaModule;
90
+ logger.debug("[imagePreprocessor] preprocessWithSkia called");
91
+ logger.debug("[imagePreprocessor] URI:", uri);
92
+ logger.debug("[imagePreprocessor] Original size:", originalWidth, "x", originalHeight);
93
+ logger.debug("[imagePreprocessor] Target size:", targetWidth, "x", targetHeight);
94
+ // Load image data using Skia's fromURI (handles file:// URIs properly)
95
+ logger.debug("[imagePreprocessor] Loading image data with Skia.Data.fromURI...");
96
+ const data = await Skia.Data.fromURI(uri);
97
+ logger.debug("[imagePreprocessor] Data loaded:", !!data);
98
+ // Create SkImage from encoded data
99
+ const image = Skia.Image.MakeImageFromEncoded(data);
100
+ logger.debug("[imagePreprocessor] Image created:", !!image);
101
+ if (!image) {
102
+ throw new Error("Failed to decode image with Skia. The image format may not be supported.");
103
+ }
104
+ logger.debug("[imagePreprocessor] Image dimensions:", image.width(), "x", image.height());
105
+ // Calculate scaling and padding for letterboxing
106
+ let scale = { x: 1, y: 1 };
107
+ let padding = { x: 0, y: 0 };
108
+ if (letterbox) {
109
+ const ratio = Math.min(targetWidth / originalWidth, targetHeight / originalHeight);
110
+ scale = { x: ratio, y: ratio };
111
+ padding = {
112
+ x: (targetWidth - originalWidth * ratio) / 2,
113
+ y: (targetHeight - originalHeight * ratio) / 2,
114
+ };
115
+ }
116
+ else {
117
+ scale = {
118
+ x: targetWidth / originalWidth,
119
+ y: targetHeight / originalHeight,
120
+ };
121
+ }
122
+ // Create a surface to draw the resized image
123
+ logger.debug("[imagePreprocessor] Creating surface...");
124
+ const surface = Skia.Surface.Make(targetWidth, targetHeight);
125
+ if (!surface) {
126
+ throw new Error("Failed to create Skia surface. This may happen on devices without GPU support.");
127
+ }
128
+ const canvas = surface.getCanvas();
129
+ // Fill with gray (letterbox padding color) - Skia.Color takes a color string
130
+ if (letterbox) {
131
+ canvas.clear(Skia.Color("gray"));
132
+ }
133
+ // Draw the resized image
134
+ const srcRect = Skia.XYWHRect(0, 0, originalWidth, originalHeight);
135
+ const dstRect = Skia.XYWHRect(padding.x, padding.y, originalWidth * scale.x, originalHeight * scale.y);
136
+ canvas.drawImageRect(image, srcRect, dstRect, Skia.Paint());
137
+ // Get pixel data from the snapshot
138
+ logger.debug("[imagePreprocessor] Reading pixels...");
139
+ const snapshot = surface.makeImageSnapshot();
140
+ // readPixels returns Uint8Array directly in newer Skia versions
141
+ // ColorType and AlphaType are separate exports from @shopify/react-native-skia
142
+ const pixels = snapshot.readPixels(0, 0, {
143
+ width: targetWidth,
144
+ height: targetHeight,
145
+ colorType: ColorType.RGBA_8888,
146
+ alphaType: AlphaType.Unpremul,
147
+ });
148
+ if (!pixels) {
149
+ throw new Error("Failed to read pixels from Skia image");
150
+ }
151
+ logger.debug("[imagePreprocessor] Pixels read, length:", pixels.length);
152
+ // Convert RGBA Uint8Array to RGB Float32Array
153
+ const rgbData = new Float32Array(targetWidth * targetHeight * 3);
154
+ for (let i = 0, j = 0; i < pixels.length; i += 4, j += 3) {
155
+ const r = pixels[i];
156
+ const g = pixels[i + 1];
157
+ const b = pixels[i + 2];
158
+ // Skip alpha (i + 3)
159
+ if (normalize) {
160
+ rgbData[j] = r / 255.0;
161
+ rgbData[j + 1] = g / 255.0;
162
+ rgbData[j + 2] = b / 255.0;
163
+ }
164
+ else {
165
+ rgbData[j] = r;
166
+ rgbData[j + 1] = g;
167
+ rgbData[j + 2] = b;
168
+ }
169
+ }
170
+ // Clean up if dispose methods exist
171
+ if (surface.dispose)
172
+ surface.dispose();
173
+ if (data.dispose)
174
+ data.dispose();
175
+ logger.debug("[imagePreprocessor] Preprocessing complete, output size:", rgbData.length);
176
+ return {
177
+ data: rgbData,
178
+ originalWidth,
179
+ originalHeight,
180
+ scale,
181
+ padding,
182
+ };
183
+ }
184
+ /**
185
+ * Preprocess a VisionCamera frame for TFLite inference
186
+ *
187
+ * This requires vision-camera-resize-plugin to be installed:
188
+ * npm install vision-camera-resize-plugin
189
+ *
190
+ * Usage in a frame processor:
191
+ * ```
192
+ * const frameProcessor = useFrameProcessor((frame) => {
193
+ * 'worklet';
194
+ * const resized = resize(frame, {
195
+ * size: { width: 640, height: 640 },
196
+ * pixelFormat: 'rgb',
197
+ * dataType: 'float32',
198
+ * });
199
+ * const result = model.runSync([resized]);
200
+ * // Process result...
201
+ * }, [model]);
202
+ * ```
203
+ */
204
+ export function getFrameProcessorInstructions() {
205
+ return `
206
+ To use on-device inference with VisionCamera frame processor:
207
+
208
+ 1. Install the resize plugin:
209
+ npm install vision-camera-resize-plugin
210
+ cd ios && pod install
211
+
212
+ 2. Use in your frame processor:
213
+ import { resize } from 'vision-camera-resize-plugin';
214
+
215
+ const frameProcessor = useFrameProcessor((frame) => {
216
+ 'worklet';
217
+ const resized = resize(frame, {
218
+ size: { width: 640, height: 640 },
219
+ pixelFormat: 'rgb',
220
+ dataType: 'float32',
221
+ });
222
+ const result = model.runSync([resized]);
223
+ // Process result...
224
+ }, [model]);
225
+ `;
226
+ }
227
+ /**
228
+ * Convert Float32Array from HWC to NHWC format (add batch dimension)
229
+ */
230
+ export function addBatchDimension(data) {
231
+ // Data is already in HWC format, just need to ensure it's treated as batch of 1
232
+ // The TFLite model expects [1, H, W, C] but since we pass a flat array,
233
+ // and the model knows the shape, we don't need to actually reshape
234
+ return data;
235
+ }
236
+ /**
237
+ * Check if Skia is available for image preprocessing
238
+ */
239
+ export function isSkiaAvailable() {
240
+ return checkSkiaAvailable();
241
+ }
242
+ /**
243
+ * Initialize Skia module (call before using Skia-dependent functions)
244
+ * @returns true if Skia was loaded successfully
245
+ */
246
+ export async function initSkia() {
247
+ return checkSkiaAvailable();
248
+ }
249
+ // Cached resize plugin availability
250
+ let resizePluginAvailable = null;
251
+ /**
252
+ * Check if resize plugin is available for frame processing
253
+ */
254
+ export async function isResizePluginAvailable() {
255
+ if (resizePluginAvailable !== null) {
256
+ return resizePluginAvailable;
257
+ }
258
+ try {
259
+ await import("vision-camera-resize-plugin");
260
+ resizePluginAvailable = true;
261
+ return true;
262
+ }
263
+ catch {
264
+ resizePluginAvailable = false;
265
+ return false;
266
+ }
267
+ }
268
+ //# sourceMappingURL=imagePreprocessor.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"imagePreprocessor.js","sourceRoot":"","sources":["../../../src/utils/imagePreprocessor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,KAAK,EAAE,MAAM,cAAc,CAAC;AACrC,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAEzC,wEAAwE;AACxE,8DAA8D;AAC9D,IAAI,UAAU,GAAQ,IAAI,CAAC;AAC3B,IAAI,iBAAiB,GAAG,KAAK,CAAC;AAE9B;;GAEG;AACH,KAAK,UAAU,cAAc;IAC3B,IAAI,UAAU;QAAE,OAAO,IAAI,CAAC;IAC5B,IAAI,iBAAiB;QAAE,OAAO,UAAU,KAAK,IAAI,CAAC;IAElD,iBAAiB,GAAG,IAAI,CAAC;IACzB,IAAI,CAAC;QACH,MAAM,CAAC,KAAK,CAAC,4CAA4C,CAAC,CAAC;QAC3D,2EAA2E;QAC3E,UAAU,GAAG,OAAO,CAAC,4BAA4B,CAAC,CAAC;QACnD,MAAM,CAAC,KAAK,CAAC,8CAA8C,CAAC,CAAC;QAC7D,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,CAAC,KAAK,CAAC,0CAA0C,EAAE,KAAK,CAAC,CAAC;QAChE,UAAU,GAAG,IAAI,CAAC;QAClB,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,kBAAkB;IACzB,IAAI,CAAC,UAAU;QAAE,OAAO,KAAK,CAAC;IAC9B,IAAI,CAAC;QACH,MAAM,EAAE,IAAI,EAAE,GAAG,UAAU,CAAC;QAC5B,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,IAAI,IAAI,EAAE,KAAK,IAAI,IAAI,EAAE,OAAO,CAAC,CAAC;IACxD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AA0BD;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,GAAW,EACX,UAA6B,EAAE;IAE/B,MAAM,EACJ,KAAK,EAAE,WAAW,GAAG,GAAG,EACxB,MAAM,EAAE,YAAY,GAAG,GAAG,EAC1B,SAAS,GAAG,IAAI,EAChB,SAAS,GAAG,IAAI,GACjB,GAAG,OAAO,CAAC;IAEZ,wCAAwC;IACxC,MAAM,MAAM,GAAG,MAAM,cAAc,EAAE,CAAC;IACtC,IAAI,CAAC,MAAM,IAAI,CAAC,kBAAkB,EAAE,EAAE,CAAC;QACrC,MAAM,IAAI,KAAK,CACb,2DAA2D;YACzD,0DAA0D;YAC1D,2EAA2E,CAC9E,CAAC;IACJ,CAAC;IAED,gCAAgC;IAChC,MAAM,EAAE,KAAK,EAAE,aAAa,EAAE,MAAM,EAAE,cAAc,EAAE,GACpD,MAAM,YAAY,CAAC,GAAG,CAAC,CAAC;IAE1B,6BAA6B;IAC7B,OAAO,kBAAkB,CACvB,GAAG,EACH,aAAa,EACb,cAAc,EACd,WAAW,EACX,YAAY,EACZ,SAAS,EACT,SAAS,CACV,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,YAAY,CACzB,GAAW;IAEX,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrC,KAAK,CAAC,OAAO,CACX,GAAG,EACH,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC,OAAO,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,EAC7C,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,6BAA6B,KAAK,EAAE,CAAC,CAAC,CACnE,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,kBAAkB,CAC/B,GAAW,EACX,aAAqB,EACrB,cAAsB,EACtB,WAAmB,EACnB,YAAoB,EACpB,SAAkB,EAClB,SAAkB;IAElB,4CAA4C;IAC5C,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,UAAU,CAAC;IAElD,MAAM,CAAC,KAAK,CAAC,+CAA+C,CAAC,CAAC;IAC9D,MAAM,CAAC,KAAK,CAAC,0BAA0B,EAAE,GAAG,CAAC,CAAC;IAC9C,MAAM,CAAC,KAAK,CACV,oCAAoC,EACpC,aAAa,EACb,GAAG,EACH,cAAc,CACf,CAAC;IACF,MAAM,CAAC,KAAK,CACV,kCAAkC,EAClC,WAAW,EACX,GAAG,EACH,YAAY,CACb,CAAC;IAEF,uEAAuE;IACvE,MAAM,CAAC,KAAK,CACV,kEAAkE,CACnE,CAAC;IACF,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAC1C,MAAM,CAAC,KAAK,CAAC,kCAAkC,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;IAEzD,mCAAmC;IACnC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,oBAAoB,CAAC,IAAI,CAAC,CAAC;IACpD,MAAM,CAAC,KAAK,CAAC,oCAAoC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC;IAE5D,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CACb,0EAA0E,CAC3E,CAAC;IACJ,CAAC;IAED,MAAM,CAAC,KAAK,CACV,uCAAuC,EACvC,KAAK,CAAC,KAAK,EAAE,EACb,GAAG,EACH,KAAK,CAAC,MAAM,EAAE,CACf,CAAC;IAEF,iDAAiD;IACjD,IAAI,KAAK,GAAG,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC;IAC3B,IAAI,OAAO,GAAG,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC;IAE7B,IAAI,SAAS,EAAE,CAAC;QACd,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CACpB,WAAW,GAAG,aAAa,EAC3B,YAAY,GAAG,cAAc,CAC9B,CAAC;QACF,KAAK,GAAG,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC;QAC/B,OAAO,GAAG;YACR,CAAC,EAAE,CAAC,WAAW,GAAG,aAAa,GAAG,KAAK,CAAC,GAAG,CAAC;YAC5C,CAAC,EAAE,CAAC,YAAY,GAAG,cAAc,GAAG,KAAK,CAAC,GAAG,CAAC;SAC/C,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,KAAK,GAAG;YACN,CAAC,EAAE,WAAW,GAAG,aAAa;YAC9B,CAAC,EAAE,YAAY,GAAG,cAAc;SACjC,CAAC;IACJ,CAAC;IAED,6CAA6C;IAC7C,MAAM,CAAC,KAAK,CAAC,yCAAyC,CAAC,CAAC;IACxD,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,WAAW,EAAE,YAAY,CAAC,CAAC;IAC7D,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,MAAM,IAAI,KAAK,CACb,gFAAgF,CACjF,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC;IAEnC,6EAA6E;IAC7E,IAAI,SAAS,EAAE,CAAC;QACd,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;IACnC,CAAC;IAED,yBAAyB;IACzB,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,EAAE,aAAa,EAAE,cAAc,CAAC,CAAC;IACnE,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAC3B,OAAO,CAAC,CAAC,EACT,OAAO,CAAC,CAAC,EACT,aAAa,GAAG,KAAK,CAAC,CAAC,EACvB,cAAc,GAAG,KAAK,CAAC,CAAC,CACzB,CAAC;IACF,MAAM,CAAC,aAAa,CAAC,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC;IAE5D,mCAAmC;IACnC,MAAM,CAAC,KAAK,CAAC,uCAAuC,CAAC,CAAC;IACtD,MAAM,QAAQ,GAAG,OAAO,CAAC,iBAAiB,EAAE,CAAC;IAE7C,gEAAgE;IAChE,+EAA+E;IAC/E,MAAM,MAAM,GAAG,QAAQ,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC,EAAE;QACvC,KAAK,EAAE,WAAW;QAClB,MAAM,EAAE,YAAY;QACpB,SAAS,EAAE,SAAS,CAAC,SAAS;QAC9B,SAAS,EAAE,SAAS,CAAC,QAAQ;KAC9B,CAAC,CAAC;IAEH,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;IAC3D,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,0CAA0C,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;IAExE,8CAA8C;IAC9C,MAAM,OAAO,GAAG,IAAI,YAAY,CAAC,WAAW,GAAG,YAAY,GAAG,CAAC,CAAC,CAAC;IAEjE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;QACzD,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QACpB,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACxB,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACxB,qBAAqB;QAErB,IAAI,SAAS,EAAE,CAAC;YACd,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;YACvB,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;YAC3B,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;QAC7B,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;YACf,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;YACnB,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;QACrB,CAAC;IACH,CAAC;IAED,oCAAoC;IACpC,IAAI,OAAO,CAAC,OAAO;QAAE,OAAO,CAAC,OAAO,EAAE,CAAC;IACvC,IAAI,IAAI,CAAC,OAAO;QAAE,IAAI,CAAC,OAAO,EAAE,CAAC;IAEjC,MAAM,CAAC,KAAK,CACV,0DAA0D,EAC1D,OAAO,CAAC,MAAM,CACf,CAAC;IAEF,OAAO;QACL,IAAI,EAAE,OAAO;QACb,aAAa;QACb,cAAc;QACd,KAAK;QACL,OAAO;KACR,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,MAAM,UAAU,6BAA6B;IAC3C,OAAO;;;;;;;;;;;;;;;;;;;;CAoBR,CAAC;AACF,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,iBAAiB,CAAC,IAAkB;IAClD,gFAAgF;IAChF,wEAAwE;IACxE,mEAAmE;IACnE,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,eAAe;IAC7B,OAAO,kBAAkB,EAAE,CAAC;AAC9B,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ;IAC5B,OAAO,kBAAkB,EAAE,CAAC;AAC9B,CAAC;AAED,oCAAoC;AACpC,IAAI,qBAAqB,GAAmB,IAAI,CAAC;AAEjD;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,uBAAuB;IAC3C,IAAI,qBAAqB,KAAK,IAAI,EAAE,CAAC;QACnC,OAAO,qBAAqB,CAAC;IAC/B,CAAC;IACD,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,6BAA6B,CAAC,CAAC;QAC5C,qBAAqB,GAAG,IAAI,CAAC;QAC7B,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,qBAAqB,GAAG,KAAK,CAAC;QAC9B,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC"}