@switchlabs/verify-ai-react-native 1.1.0 → 1.1.1

package/src/hooks/useVerifyAI.ts

@@ -1,13 +1,17 @@
-import { useMemo, useCallback, useState, useEffect } from 'react';
-import { AppState, type AppStateStatus } from 'react-native';
+import React, { useMemo, useCallback, useState, useEffect, useRef } from 'react';
+import { AppState, Platform, type AppStateStatus } from 'react-native';
 import { VerifyAIClient, VerifyAIRequestError } from '../client';
 import { OfflineQueue } from '../storage/offlineQueue';
+import { TelemetryContext } from '../telemetry/TelemetryContext';
+import type { TelemetryReporter } from '../telemetry/TelemetryReporter';
 import type {
   VerifyAIConfig,
   VerificationRequest,
+  MultipartVerificationRequest,
   VerificationResult,
   VerificationListParams,
   VerificationListResponse,
+  VerifyOptions,
 } from '../types';
 
 function isQueueableError(error: Error): boolean {
@@ -28,9 +32,30 @@ function isQueueableError(error: Error): boolean {
   );
 }
 
+function isOfflineError(error: Error): boolean {
+  const message = error.message.toLowerCase();
+  return (
+    message.includes('network') ||
+    message.includes('failed to fetch') ||
+    message.includes('no internet')
+  );
+}
+
+export interface UseVerifyAIConfig extends VerifyAIConfig {
+  /** Enable on-device ML inference for faster, offline-capable verification. */
+  enableOnDeviceML?: boolean;
+  /** Minimum confidence threshold for on-device results. Below this, falls back to cloud. Default: 0.7 */
+  onDeviceConfidenceThreshold?: number;
+}
+
 export interface UseVerifyAIReturn {
-  /** Submit a verification. Uses offline queue if offlineMode is enabled and request fails. */
-  verify: (request: VerificationRequest) => Promise<VerificationResult | null>;
+  /** Submit a verification. Uses on-device ML if enabled and model is cached. */
+  verify: (request: VerificationRequest, options?: VerifyOptions) => Promise<VerificationResult | null>;
+  /** Submit a multipart verification. Uses on-device ML if enabled and model is cached. */
+  verifyMultipart: (
+    request: MultipartVerificationRequest,
+    options?: VerifyOptions,
+  ) => Promise<VerificationResult | null>;
   /** List past verifications. */
   listVerifications: (params?: VerificationListParams) => Promise<VerificationListResponse>;
   /** Get a single verification by ID. */
@@ -49,41 +74,102 @@ export interface UseVerifyAIReturn {
   client: VerifyAIClient;
   /** The offline queue instance (null if offlineMode is disabled). */
   offlineQueue: OfflineQueue | null;
+  /** Whether an on-device model is loaded and ready. */
+  mlModelReady: boolean;
+  /** Initialize or update the on-device ML model for a policy. */
+  initializeMLModel: (policyId: string) => Promise<void>;
+  /** Telemetry reporter instance (null if telemetry is disabled). */
+  telemetry: TelemetryReporter | null;
+  /** TelemetryProvider component — wrap your scanner tree with this. */
+  TelemetryProvider: React.FC<{ children: React.ReactNode }>;
 }
 
 /**
- * React hook for Verify AI. Provides verification methods,
- * loading/error state, and optional offline queue management.
+ * React hook for Verify AI with optional on-device ML inference.
  *
  * @example
  * ```tsx
- * const { verify, loading, lastResult, error } = useVerifyAI({
+ * const { verify, loading, lastResult, mlModelReady, initializeMLModel } = useVerifyAI({
  *   apiKey: 'vai_your_api_key',
  *   offlineMode: true,
+ *   enableOnDeviceML: true,
  * });
  *
+ * // Initialize ML model on mount
+ * useEffect(() => {
+ *   initializeMLModel('scooter_parking');
+ * }, []);
+ *
  * const handleCapture = async (base64Image: string) => {
  *   const result = await verify({
  *     image: base64Image,
  *     policy: 'scooter_parking',
  *   });
- *   if (result?.is_compliant) {
- *     // Success
- *   }
  * };
  * ```
  */
-export function useVerifyAI(config: VerifyAIConfig): UseVerifyAIReturn {
-  const client = useMemo(() => new VerifyAIClient(config), [config.apiKey, config.baseUrl, config.timeout]);
+export function useVerifyAI(config: UseVerifyAIConfig): UseVerifyAIReturn {
+  const {
+    enableOnDeviceML = false,
+    onDeviceConfidenceThreshold = 0.7,
+    ...baseConfig
+  } = config;
+
+  const client = useMemo(
+    () => new VerifyAIClient({
+      apiKey: baseConfig.apiKey,
+      baseUrl: baseConfig.baseUrl,
+      timeout: baseConfig.timeout,
+      offlineMode: baseConfig.offlineMode,
+      telemetry: baseConfig.telemetry,
+    }),
+    [baseConfig.apiKey, baseConfig.baseUrl, baseConfig.timeout, baseConfig.offlineMode, baseConfig.telemetry],
+  );
   const offlineQueue = useMemo(
-    () => (config.offlineMode ? new OfflineQueue(client) : null),
-    [client, config.offlineMode]
+    () => (baseConfig.offlineMode ? new OfflineQueue(client) : null),
+    [client, baseConfig.offlineMode]
   );
 
+  // ML components — lazily loaded to avoid requiring ML deps when not enabled
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  const modelManagerRef = useRef<any>(null);
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  const inferenceEngineRef = useRef<any>(null);
+
   const [loading, setLoading] = useState(false);
   const [error, setError] = useState<Error | null>(null);
   const [lastResult, setLastResult] = useState<VerificationResult | null>(null);
   const [queueSize, setQueueSize] = useState(0);
+  const [mlModelReady, setMlModelReady] = useState(false);
+
+  // Initialize ML components lazily via dynamic import
+  useEffect(() => {
+    if (!enableOnDeviceML || !baseConfig.apiKey) return;
+
+    let disposed = false;
+
+    (async () => {
+      try {
+        const [{ ModelManager }, { InferenceEngine }] = await Promise.all([
+          import('../ml/modelManager'),
+          import('../ml/inferenceEngine'),
+        ]);
+        if (disposed) return;
+        modelManagerRef.current = new ModelManager({
+          apiKey: baseConfig.apiKey,
+          baseUrl: baseConfig.baseUrl,
+        });
+        inferenceEngineRef.current = new InferenceEngine();
+      } catch (err) {
+        console.warn('[VerifyAI] Failed to load ML modules:', err);
+      }
+    })();
+
+    return () => {
+      disposed = true;
+      inferenceEngineRef.current?.dispose();
+    };
+  }, [enableOnDeviceML, baseConfig.apiKey, baseConfig.baseUrl]);
 
   // Refresh queue size
   const refreshQueueSize = useCallback(async () => {
@@ -109,20 +195,176 @@ export function useVerifyAI(config: VerifyAIConfig): UseVerifyAIReturn {
     return () => subscription.remove();
   }, [offlineQueue, refreshQueueSize]);
 
+  /** Initialize or update the on-device ML model for a policy. */
+  const initializeMLModel = useCallback(async (policyId: string) => {
+    if (!enableOnDeviceML || !modelManagerRef.current || !inferenceEngineRef.current) return;
+
+    try {
+      // Load cached bundle first
+      const bundle = await modelManagerRef.current.loadCachedBundle(policyId);
+
+      if (bundle) {
+        await inferenceEngineRef.current.loadModel(bundle);
+        setMlModelReady(true);
+      }
+
+      // Check for updates in background
+      const platform = Platform.OS === 'ios' ? 'ios' : 'android';
+      const updatedBundle = await modelManagerRef.current.checkForUpdates(policyId, platform);
+
+      if (updatedBundle && updatedBundle.bundleVersion !== bundle?.bundleVersion) {
+        await inferenceEngineRef.current.loadModel(updatedBundle);
+        setMlModelReady(true);
+      }
+    } catch (err) {
+      setMlModelReady(false);
+      client.telemetry?.track('ml_model_load_failure', {
+        component: 'useVerifyAI',
+        error: err,
+      });
+    }
+  }, [enableOnDeviceML, client.telemetry]);
+
+  /** Try on-device verification. Returns null if inference fails or low confidence. */
+  const tryOnDeviceVerification = useCallback(
+    async (
+      params: {
+        policy: string;
+        metadata?: Record<string, unknown>;
+        preprocessedInput: Float32Array;
+      },
+      ignoreThreshold = false,
+    ): Promise<VerificationResult | null> => {
+      try {
+        const modelManager = modelManagerRef.current;
+        const inferenceEngine = inferenceEngineRef.current;
+        const bundle = modelManager?.currentBundle;
+
+        if (!inferenceEngine?.isLoaded || !bundle) return null;
+
+        // Lazily import ML modules
+        const [{ extractFeatures }, { evaluatePolicy }] = await Promise.all([
+          import('../ml/featureExtractor'),
+          import('../ml/policyEngine'),
+        ]);
+
+        // Run inference
+        const rawDetections = inferenceEngine.runInference(params.preprocessedInput);
+
+        // Extract features
+        const features = extractFeatures(rawDetections);
+
+        // Evaluate policy
+        const policyResult = evaluatePolicy(bundle.policyAst, features);
+
+        // Check confidence threshold
+        if (!ignoreThreshold && policyResult.confidence < onDeviceConfidenceThreshold) {
+          return null;
+        }
+
+        return {
+          id: `local_${Date.now()}`,
+          created_at: new Date().toISOString(),
+          status: 'success',
+          is_compliant: policyResult.is_compliant,
+          confidence: policyResult.confidence,
+          policy: params.policy,
+          violation_reasons: policyResult.violation_reasons,
+          feedback: policyResult.feedback,
+          metadata: {
+            ...params.metadata,
+            evaluation_source: 'on_device',
+            bundle_version: bundle.bundleVersion,
+            category: policyResult.category_id,
+          },
+          image_url: null,
+        };
+      } catch (err) {
+        client.telemetry?.track('ml_inference_failure', {
+          component: 'useVerifyAI',
+          error: err,
+        });
+        return null;
+      }
+    },
+    [onDeviceConfidenceThreshold, client.telemetry],
+  );
+
+  const tryOnDeviceBase64Verification = useCallback(
+    async (
+      request: VerificationRequest,
+      ignoreThreshold = false,
+    ): Promise<VerificationResult | null> => {
+      try {
+        const { preprocessImageBase64 } = await import('../ml/imagePreprocessor');
+        const preprocessedInput = preprocessImageBase64(request.image);
+        return tryOnDeviceVerification({
+          policy: request.policy,
+          metadata: request.metadata,
+          preprocessedInput,
+        }, ignoreThreshold);
+      } catch {
+        return null;
+      }
+    },
+    [tryOnDeviceVerification],
+  );
+
+  const tryOnDeviceMultipartVerification = useCallback(
+    async (
+      request: MultipartVerificationRequest,
+      ignoreThreshold = false,
+    ): Promise<VerificationResult | null> => {
+      try {
+        const { preprocessImageUri } = await import('../ml/imagePreprocessor');
+        const preprocessedInput = await preprocessImageUri(request.imageUri);
+        return tryOnDeviceVerification({
+          policy: request.policy,
+          metadata: request.metadata,
+          preprocessedInput,
+        }, ignoreThreshold);
+      } catch {
+        return null;
+      }
+    },
+    [tryOnDeviceVerification],
+  );
+
   const verify = useCallback(
-    async (request: VerificationRequest): Promise<VerificationResult | null> => {
+    async (request: VerificationRequest, options?: VerifyOptions): Promise<VerificationResult | null> => {
       setLoading(true);
       setError(null);
 
       try {
-        const result = await client.verify(request);
+        // Try on-device inference first
+        if (enableOnDeviceML && mlModelReady) {
+          const localResult = await tryOnDeviceBase64Verification(request);
+          if (localResult) {
+            setLastResult(localResult);
+            setLoading(false);
+            return localResult;
+          }
+          // Fall through to cloud
+        }
+
+        const result = await client.verify(request, options);
         setLastResult(result);
         return result;
       } catch (err) {
         const error = err instanceof Error ? err : new Error(String(err));
         setError(error);
 
-        // Queue only transient failures so invalid requests are surfaced immediately.
+        // If offline and we have a model, try on-device regardless of confidence
+        if (enableOnDeviceML && mlModelReady && isOfflineError(error)) {
+          const localResult = await tryOnDeviceBase64Verification(request, true);
+          if (localResult) {
+            setLastResult(localResult);
+            setLoading(false);
+            return localResult;
+          }
+        }
+
+        // Queue only transient failures
         if (offlineQueue && isQueueableError(error)) {
           await offlineQueue.enqueue(request);
           await refreshQueueSize();
@@ -134,7 +376,47 @@ export function useVerifyAI(config: VerifyAIConfig): UseVerifyAIReturn {
         setLoading(false);
       }
     },
-    [client, offlineQueue, refreshQueueSize]
+    [client, offlineQueue, refreshQueueSize, enableOnDeviceML, mlModelReady, tryOnDeviceBase64Verification]
+  );
+
+  const verifyMultipart = useCallback(
+    async (
+      request: MultipartVerificationRequest,
+      options?: VerifyOptions,
+    ): Promise<VerificationResult | null> => {
+      setLoading(true);
+      setError(null);
+
+      try {
+        if (enableOnDeviceML && mlModelReady) {
+          const localResult = await tryOnDeviceMultipartVerification(request);
+          if (localResult) {
+            setLastResult(localResult);
+            return localResult;
+          }
+        }
+
+        const result = await client.verifyMultipart(request, options);
+        setLastResult(result);
+        return result;
+      } catch (err) {
+        const error = err instanceof Error ? err : new Error(String(err));
+        setError(error);
+
+        if (enableOnDeviceML && mlModelReady && isOfflineError(error)) {
+          const localResult = await tryOnDeviceMultipartVerification(request, true);
+          if (localResult) {
+            setLastResult(localResult);
+            return localResult;
+          }
+        }
+
+        throw error;
+      } finally {
+        setLoading(false);
+      }
+    },
+    [client, enableOnDeviceML, mlModelReady, tryOnDeviceMultipartVerification],
   );
 
   const listVerifications = useCallback(
@@ -153,8 +435,36 @@ export function useVerifyAI(config: VerifyAIConfig): UseVerifyAIReturn {
     await refreshQueueSize();
   }, [offlineQueue, refreshQueueSize]);
 
+  // Flush telemetry on app background, dispose on unmount
+  useEffect(() => {
+    const reporter = client.telemetry;
+    if (!reporter) return;
+
+    const handleAppState = (state: AppStateStatus) => {
+      if (state === 'background' || state === 'inactive') {
+        reporter.flush();
+      }
+    };
+
+    const subscription = AppState.addEventListener('change', handleAppState);
+    return () => {
+      subscription.remove();
+      reporter.dispose();
+    };
+  }, [client.telemetry]);
+
+  // Memoized TelemetryProvider component
+  const TelemetryProvider = useMemo(() => {
+    const reporter = client.telemetry;
+    const Provider: React.FC<{ children: React.ReactNode }> = ({ children }) =>
+      React.createElement(TelemetryContext.Provider, { value: reporter }, children);
+    Provider.displayName = 'TelemetryProvider';
+    return Provider;
+  }, [client.telemetry]);
+
   return {
     verify,
+    verifyMultipart,
     listVerifications,
     getVerification,
     loading,
@@ -164,5 +474,9 @@ export function useVerifyAI(config: VerifyAIConfig): UseVerifyAIReturn {
     processQueue,
     client,
     offlineQueue,
+    mlModelReady,
+    initializeMLModel,
+    telemetry: client.telemetry,
+    TelemetryProvider,
   };
 }

package/src/index.ts

@@ -3,19 +3,38 @@ export { VerifyAIClient, VerifyAIRequestError } from './client';
 
 // Hooks
 export { useVerifyAI } from './hooks/useVerifyAI';
-export type { UseVerifyAIReturn } from './hooks/useVerifyAI';
+export type { UseVerifyAIReturn, UseVerifyAIConfig } from './hooks/useVerifyAI';
 
 // Components
 export { VerifyAIScanner } from './components/VerifyAIScanner';
 export type { VerifyAIScannerProps } from './components/VerifyAIScanner';
 
+// Telemetry
+export { TelemetryReporter } from './telemetry/TelemetryReporter';
+export { TelemetryContext } from './telemetry/TelemetryContext';
+
 // Offline Queue
 export { OfflineQueue } from './storage/offlineQueue';
 
+// ML (on-device inference) — types are safe to re-export statically.
+// Value exports use a lazy namespace so consuming apps don't need ML deps
+// (buffer, jpeg-js, @noble/hashes, react-native-fast-tflite) unless they
+// explicitly opt in via `enableOnDeviceML: true`.
+export type {
+  BundleManifest,
+  ModelArtifact,
+  FeatureVector,
+  Detection as MLDetection,
+  PolicyAST,
+  PolicyResult,
+  RuleResult,
+} from './ml/types';
+
 // Types
 export type {
   VerifyAIConfig,
   VerificationRequest,
+  MultipartVerificationRequest,
   VerificationResult,
   VerificationListResponse,
   VerificationListParams,

package/src/ml/featureExtractor.ts

@@ -0,0 +1,160 @@
+/**
+ * Feature extractor — converts raw detections into a structured FeatureVector.
+ * Must produce identical output to the Dart and server TypeScript implementations.
+ */
+
+import type { FeatureVector, Detection, ImageQuality } from './types';
+
+const SCHEMA_VERSION = '1.0.0';
+
+/** Vehicle class IDs (must match ontology.ts) */
+const VEHICLE_CLASS_IDS = new Set([1, 2, 3, 4, 5]);
+/** Surface class IDs */
+const SURFACE_CLASS_IDS = new Set([60, 61, 62, 63, 64, 65, 66]);
+
+/** Ontology class name mapping (must match ontology.ts) */
+export const ONTOLOGY_CLASS_NAMES: Record<number, string> = {
+  1: 'scooter', 2: 'bicycle', 3: 'ebike', 4: 'moped', 5: 'motorcycle',
+  20: 'kickstand', 21: 'handlebar', 22: 'wheel', 23: 'saddle', 24: 'basket',
+  40: 'bike_rack', 41: 'curb', 42: 'parking_bay', 43: 'green_bay',
+  44: 'tactile_paving', 45: 'bollard', 46: 'fence', 47: 'pole', 48: 'yellow_lines',
+  60: 'sidewalk', 61: 'road', 62: 'grass', 63: 'crosswalk',
+  64: 'dirt', 65: 'gravel', 66: 'parking_lot',
+  80: 'person', 81: 'entrance', 82: 'obstruction', 83: 'bench',
+  84: 'garbage_bin', 85: 'fire_escape', 86: 'ramp', 87: 'driveway',
+};
+
+/** Detection model class index -> ontology class ID mapping. */
+export const MODEL_OUTPUT_CLASS_IDS = [
+  1, 2, 3, 4, 5,
+  20, 21, 22, 23, 24,
+  40, 41, 42, 43, 44, 45, 46, 47, 48,
+  60, 61, 62, 63, 64, 65, 66,
+  80, 81, 82, 83, 84, 85, 86, 87,
+] as const;
+
+interface RawDetection {
+  classId: number;
+  confidence: number;
+  bbox: [number, number, number, number];
+}
+
+export function extractFeatures(rawDetections: RawDetection[]): FeatureVector {
+  // Convert raw detections to typed detections
+  const detections: Detection[] = rawDetections.map((raw) => ({
+    class_id: raw.classId,
+    class_name: ONTOLOGY_CLASS_NAMES[raw.classId] || `unknown_${raw.classId}`,
+    confidence: raw.confidence,
+    bbox: raw.bbox,
+  }));
+
+  // Find primary vehicle (highest confidence)
+  let primaryVehicle: Detection | null = null;
+  for (const det of detections) {
+    if (VEHICLE_CLASS_IDS.has(det.class_id)) {
+      if (!primaryVehicle || det.confidence > primaryVehicle.confidence) {
+        primaryVehicle = det;
+      }
+    }
+  }
+
+  // Determine surface under vehicle
+  let vehicleOnSurface: string | null = null;
+  if (primaryVehicle) {
+    vehicleOnSurface = findSurfaceUnderVehicle(primaryVehicle, detections);
+  }
+
+  // Find objects near the vehicle
+  const vehicleNear: string[] = [];
+  if (primaryVehicle) {
+    vehicleNear.push(...findObjectsNearVehicle(primaryVehicle, detections));
+  }
+
+  // Assess image quality
+  const imageQuality: ImageQuality = {
+    is_blurry: false,
+    is_dark: false,
+    has_vehicle: primaryVehicle !== null,
+  };
+
+  return {
+    schema_version: SCHEMA_VERSION,
+    detections,
+    primary_vehicle: primaryVehicle,
+    image_quality: imageQuality,
+    vehicle_on_surface: vehicleOnSurface,
+    vehicle_near: vehicleNear,
+  };
+}
+
+function findSurfaceUnderVehicle(vehicle: Detection, allDetections: Detection[]): string | null {
+  const surfaces = allDetections.filter((d) => SURFACE_CLASS_IDS.has(d.class_id));
+
+  let bestSurface: string | null = null;
+  let bestOverlap = 0;
+
+  for (const surface of surfaces) {
+    // Check overlap with vehicle bottom half
+    const vehicleBottom: [number, number, number, number] = [
+      vehicle.bbox[0],
+      (vehicle.bbox[1] + vehicle.bbox[3]) / 2,
+      vehicle.bbox[2],
+      vehicle.bbox[3],
+    ];
+    const overlap = computeIoU(vehicleBottom, surface.bbox);
+
+    if (overlap > bestOverlap) {
+      bestOverlap = overlap;
+      bestSurface = surface.class_name;
+    }
+  }
+
+  return bestOverlap > 0.1 ? bestSurface : null;
+}
+
+function findObjectsNearVehicle(vehicle: Detection, allDetections: Detection[]): string[] {
+  const nearObjects = new Set<string>();
+
+  for (const det of allDetections) {
+    if (VEHICLE_CLASS_IDS.has(det.class_id)) continue;
+    if (SURFACE_CLASS_IDS.has(det.class_id)) continue;
+
+    if (isNearby(vehicle.bbox, det.bbox, 0.2)) {
+      nearObjects.add(det.class_name);
+    }
+  }
+
+  return Array.from(nearObjects);
+}
+
+function isNearby(
+  bboxA: [number, number, number, number],
+  bboxB: [number, number, number, number],
+  threshold: number,
+): boolean {
+  const centerAx = (bboxA[0] + bboxA[2]) / 2;
+  const centerAy = (bboxA[1] + bboxA[3]) / 2;
+  const centerBx = (bboxB[0] + bboxB[2]) / 2;
+  const centerBy = (bboxB[1] + bboxB[3]) / 2;
+
+  return Math.abs(centerAx - centerBx) < threshold || Math.abs(centerAy - centerBy) < threshold;
+}
+
+function computeIoU(
+  a: [number, number, number, number],
+  b: [number, number, number, number],
+): number {
+  const x1 = Math.max(a[0], b[0]);
+  const y1 = Math.max(a[1], b[1]);
+  const x2 = Math.min(a[2], b[2]);
+  const y2 = Math.min(a[3], b[3]);
+
+  if (x2 <= x1 || y2 <= y1) return 0;
+
+  const intersection = (x2 - x1) * (y2 - y1);
+  const areaA = (a[2] - a[0]) * (a[3] - a[1]);
+  const areaB = (b[2] - b[0]) * (b[3] - b[1]);
+  const union = areaA + areaB - intersection;
+
+  return union > 0 ? intersection / union : 0;
+}