@switchlabs/verify-ai-react-native 1.1.0 → 1.1.2

package/src/ml/imagePreprocessor.ts

@@ -0,0 +1,72 @@
+import { Buffer } from 'buffer';
+import { decode as decodeJpeg } from 'jpeg-js';
+
+const DEFAULT_INPUT_SIZE = 640;
+
+function stripDataUriPrefix(base64: string): string {
+  const marker = 'base64,';
+  const markerIndex = base64.indexOf(marker);
+  return markerIndex >= 0 ? base64.slice(markerIndex + marker.length) : base64;
+}
+
+function isJpeg(bytes: Uint8Array): boolean {
+  return bytes.length >= 2 && bytes[0] === 0xff && bytes[1] === 0xd8;
+}
+
+function decodeBase64(base64: string): Uint8Array {
+  return Uint8Array.from(Buffer.from(stripDataUriPrefix(base64), 'base64'));
+}
+
+function resizeRgbaToChw(
+  rgba: Uint8Array,
+  sourceWidth: number,
+  sourceHeight: number,
+  inputSize: number,
+): Float32Array {
+  const totalPixels = inputSize * inputSize;
+  const tensor = new Float32Array(3 * totalPixels);
+
+  for (let y = 0; y < inputSize; y++) {
+    const sourceY = Math.min(sourceHeight - 1, Math.floor((y * sourceHeight) / inputSize));
+    for (let x = 0; x < inputSize; x++) {
+      const sourceX = Math.min(sourceWidth - 1, Math.floor((x * sourceWidth) / inputSize));
+      const sourceOffset = (sourceY * sourceWidth + sourceX) * 4;
+      const outputIndex = y * inputSize + x;
+
+      tensor[outputIndex] = rgba[sourceOffset] / 255;
+      tensor[totalPixels + outputIndex] = rgba[sourceOffset + 1] / 255;
+      tensor[2 * totalPixels + outputIndex] = rgba[sourceOffset + 2] / 255;
+    }
+  }
+
+  return tensor;
+}
+
+export function preprocessImageBase64(
+  base64: string,
+  inputSize = DEFAULT_INPUT_SIZE,
+): Float32Array {
+  const bytes = decodeBase64(base64);
+  if (!isJpeg(bytes)) {
+    throw new Error('On-device inference currently supports JPEG inputs only');
+  }
+
+  const decoded = decodeJpeg(bytes, { useTArray: true, tolerantDecoding: true });
+  if (!decoded.width || !decoded.height) {
+    throw new Error('Failed to decode JPEG for on-device inference');
+  }
+
+  return resizeRgbaToChw(decoded.data, decoded.width, decoded.height, inputSize);
+}
+
+export async function preprocessImageUri(
+  imageUri: string,
+  inputSize = DEFAULT_INPUT_SIZE,
+): Promise<Float32Array> {
+  const fileSystem = await import('expo-file-system');
+  const base64 = await fileSystem.readAsStringAsync(imageUri, {
+    encoding: fileSystem.EncodingType.Base64,
+  });
+
+  return preprocessImageBase64(base64, inputSize);
+}

package/src/ml/index.ts

@@ -0,0 +1,14 @@
+export { ModelManager } from './modelManager';
+export { InferenceEngine } from './inferenceEngine';
+export { extractFeatures, ONTOLOGY_CLASS_NAMES } from './featureExtractor';
+export { evaluatePolicy } from './policyEngine';
+export type {
+  BundleManifest,
+  ModelArtifact,
+  FeatureVector,
+  Detection,
+  ImageQuality,
+  PolicyAST,
+  PolicyResult,
+  RuleResult,
+} from './types';

package/src/ml/inferenceEngine.ts

@@ -0,0 +1,200 @@
+/**
+ * On-device inference engine for React Native.
+ * Uses react-native-fast-tflite for TFLite inference (uses CoreML delegate on iOS).
+ */
+
+import { Platform } from 'react-native';
+import type { BundleManifest } from './types';
+import { MODEL_OUTPUT_CLASS_IDS } from './featureExtractor';
+
+interface RawDetection {
+  classId: number;
+  confidence: number;
+  bbox: [number, number, number, number];
+}
+
+interface TfliteModel {
+  runSync(inputs: [Float32Array]): unknown[];
+}
+
+type TfliteDelegate = 'core-ml' | 'android-gpu' | 'nnapi';
+
+interface TfliteModule {
+  loadTensorflowModel(source: { url: string }, delegate?: TfliteDelegate): Promise<TfliteModel>;
+}
+
+export class InferenceEngine {
+  private model: TfliteModel | null = null;
+  private _isLoaded = false;
+  private classIds: readonly number[] = MODEL_OUTPUT_CLASS_IDS;
+
+  get isLoaded(): boolean {
+    return this._isLoaded;
+  }
+
+  /** Load the model from a cached bundle artifact. */
+  async loadModel(bundle: BundleManifest): Promise<void> {
+    const artifact = bundle.artifacts.find((a) => a.format === 'tflite');
+
+    if (!artifact?.localPath) {
+      throw new Error('No TFLite artifact with local path in bundle');
+    }
+
+    try {
+      const tfliteModule = await import('react-native-fast-tflite') as TfliteModule;
+      const { loadTensorflowModel } = tfliteModule;
+      const preferredDelegate: TfliteDelegate | undefined =
+        Platform.OS === 'ios' ? 'core-ml' : undefined;
+
+      try {
+        this.model = await loadTensorflowModel({ url: artifact.localPath }, preferredDelegate);
+      } catch {
+        this.model = await loadTensorflowModel({ url: artifact.localPath });
+      }
+      this.classIds = bundle.classIds.length > 0 ? bundle.classIds : MODEL_OUTPUT_CLASS_IDS;
+
+      this._isLoaded = true;
+    } catch (err) {
+      this._isLoaded = false;
+      throw new Error(`Failed to load TFLite model: ${err}`);
+    }
+  }
+
+  /** Run inference on preprocessed image data. */
+  runInference(input: Float32Array, threshold = 0.3): RawDetection[] {
+    if (!this._isLoaded || !this.model) {
+      throw new Error('Model not loaded');
+    }
+
+    // Run inference
+    const outputs = this.model.runSync([input]);
+
+    return this.parseDetections(outputs, threshold);
+  }
+
+  private parseDetections(outputs: unknown[], threshold: number): RawDetection[] {
+    const detections: RawDetection[] = [];
+
+    if (!outputs || outputs.length === 0) return detections;
+
+    if (outputs.length === 1) {
+      // Single combined output: [1, N, 4+num_classes]
+      const output = extractMatrix(outputs[0]);
+      const numDetections = output.length;
+
+      for (let i = 0; i < numDetections; i++) {
+        const det = output[i];
+        if (det.length < 5) continue;
+
+        const bbox = det.slice(0, 4) as [number, number, number, number];
+        const scores = det.slice(4);
+        let maxScore = 0;
+        let classIndex = 0;
+
+        for (let j = 0; j < scores.length; j++) {
+          if (scores[j] > maxScore) {
+            maxScore = scores[j];
+            classIndex = j;
+          }
+        }
+
+        if (maxScore >= threshold) {
+          detections.push({
+            classId: this.resolveClassId(classIndex),
+            confidence: maxScore,
+            bbox,
+          });
+        }
+      }
+    } else if (outputs.length >= 2) {
+      // Separate boxes and scores
+      const boxes = extractMatrix(outputs[0]);
+      const scoreMatrix = extractMatrix(outputs[1]);
+      const scoreVector = extractVector(outputs[1]);
+
+      for (let i = 0; i < boxes.length; i++) {
+        const box = boxes[i] as [number, number, number, number];
+
+        let maxScore = 0;
+        let classIndex = 0;
+
+        const scoreRow = scoreMatrix[i];
+        if (scoreRow) {
+          for (let j = 0; j < scoreRow.length; j++) {
+            if (scoreRow[j] > maxScore) {
+              maxScore = scoreRow[j];
+              classIndex = j;
+            }
+          }
+        } else {
+          const scalarScore = scoreVector[i] ?? null;
+          if (scalarScore === null) continue;
+          maxScore = scalarScore;
+        }
+
+        if (maxScore >= threshold) {
+          detections.push({
+            classId: this.resolveClassId(classIndex),
+            confidence: maxScore,
+            bbox: box,
+          });
+        }
+      }
+    }
+
+    return detections;
+  }
+
+  private resolveClassId(classIndex: number): number {
+    return this.classIds[classIndex] ?? classIndex;
+  }
+
+  dispose(): void {
+    this.model = null;
+    this._isLoaded = false;
+    this.classIds = MODEL_OUTPUT_CLASS_IDS;
+  }
+}
+
+function extractMatrix(value: unknown): number[][] {
+  if (!Array.isArray(value)) {
+    return [];
+  }
+
+  if (value.length === 0) {
+    return [];
+  }
+
+  const first = value[0];
+  if (Array.isArray(first) && (first.length === 0 || typeof first[0] === 'number')) {
+    return value
+      .filter(Array.isArray)
+      .map((row) => (row as unknown[]).map((entry) => Number(entry)));
+  }
+
+  if (Array.isArray(first)) {
+    return extractMatrix(first);
+  }
+
+  return [];
+}
+
+function extractVector(value: unknown): number[] {
+  if (!Array.isArray(value)) {
+    return [];
+  }
+
+  if (value.length === 0) {
+    return [];
+  }
+
+  if (typeof value[0] === 'number') {
+    return value.map((entry) => Number(entry));
+  }
+
+  if (Array.isArray(value[0])) {
+    return extractVector(value[0]);
+  }
+
+  return [];
+}

package/src/ml/modelManager.ts

@@ -0,0 +1,265 @@
+/**
+ * Model Manager for React Native SDK.
+ * Downloads, caches, and version-checks model bundles.
+ */
+
+import { Buffer } from 'buffer';
+import { sha256 } from '@noble/hashes/sha2.js';
+import type { BundleManifest, ModelArtifact } from './types';
+
+export type { BundleManifest, ModelArtifact };
+
+type ExpoFileSystemModule = typeof import('expo-file-system');
+type BundleManifestResponse = {
+  bundle_version: number;
+  ontology_version: string;
+  schema_version: string;
+  policy_id: string;
+  class_ids?: number[];
+  artifacts?: Array<{
+    role: ModelArtifact['role'];
+    architecture: string;
+    version: number;
+    format: ModelArtifact['format'];
+    storage_path: string;
+    size_bytes: number;
+    sha256: string;
+    download_url?: string;
+  }>;
+  policy_ast: BundleManifest['policyAst'];
+};
+
+async function loadFileSystem(): Promise<ExpoFileSystemModule | null> {
+  try {
+    return await import('expo-file-system');
+  } catch {
+    return null;
+  }
+}
+
+function bytesToSha256Hex(bytes: Uint8Array): string {
+  return Buffer.from(sha256(bytes)).toString('hex');
+}
+
+interface ModelManagerConfig {
+  apiKey: string;
+  baseUrl?: string;
+}
+
+export class ModelManager {
+  private apiKey: string;
+  private baseUrl: string;
+  private cachedETag: string | null = null;
+  private _currentBundle: BundleManifest | null = null;
+
+  constructor(config: ModelManagerConfig) {
+    this.apiKey = config.apiKey;
+    this.baseUrl = config.baseUrl || 'https://verify.switchlabs.dev/api/v1';
+  }
+
+  get currentBundle(): BundleManifest | null {
+    return this._currentBundle;
+  }
+
+  /** Check for bundle updates from the server. */
+  async checkForUpdates(policyId: string, platform: 'ios' | 'android'): Promise<BundleManifest | null> {
+    try {
+      const headers: Record<string, string> = {
+        'X-API-Key': this.apiKey,
+        'Accept': 'application/json',
+      };
+
+      if (this.cachedETag) {
+        headers['If-None-Match'] = this.cachedETag;
+      }
+
+      const url = `${this.baseUrl}/models/latest?policy=${policyId}&platform=${platform}`;
+      const response = await fetch(url, { headers });
+
+      if (response.status === 304) {
+        return this._currentBundle;
+      }
+
+      if (!response.ok) {
+        return this._currentBundle;
+      }
+
+      this.cachedETag = response.headers.get('etag');
+      const body = (await response.json()) as { bundles?: BundleManifestResponse[] };
+      const bundles = body.bundles;
+
+      if (!bundles || bundles.length === 0) {
+        return null;
+      }
+
+      const bundleJson = bundles.find((bundle) => bundle.policy_id === policyId) || bundles[0];
+      const manifest = parseBundleManifest(bundleJson);
+
+      if (this._currentBundle?.bundleVersion === manifest.bundleVersion) {
+        return this._currentBundle;
+      }
+
+      // Download artifacts
+      await this.downloadArtifacts(manifest, policyId);
+      await this.saveBundleManifest(policyId, manifest);
+      this._currentBundle = manifest;
+      return manifest;
+    } catch {
+      return this._currentBundle;
+    }
+  }
+
+  /** Download model artifacts and verify integrity. */
+  private async downloadArtifacts(manifest: BundleManifest, policyId: string): Promise<void> {
+    const fileSystem = await loadFileSystem();
+    if (!fileSystem || !fileSystem.documentDirectory) {
+      console.warn('[VerifyAI] expo-file-system not available, skipping download');
+      return;
+    }
+
+    const cacheDir = `${fileSystem.documentDirectory}verify-ai/bundles/${policyId}/`;
+    await fileSystem.makeDirectoryAsync(cacheDir, { intermediates: true }).catch(() => {});
+
+    for (const artifact of manifest.artifacts) {
+      if (!artifact.downloadUrl) continue;
+
+      const ext = artifact.format === 'coreml' ? 'mlpackage.zip' : artifact.format;
+      const localPath = `${cacheDir}${artifact.format}_v${artifact.version}.${ext}`;
+
+      // Check if already downloaded and intact
+      const info = await fileSystem.getInfoAsync(localPath).catch(() => ({ exists: false }));
+      if (info.exists) {
+        const existingHash = await this.readFileSha256(fileSystem, localPath);
+        if (existingHash === artifact.sha256) {
+          artifact.localPath = localPath;
+          continue;
+        }
+        await fileSystem.deleteAsync(localPath, { idempotent: true }).catch(() => {});
+      }
+
+      // Download
+      try {
+        const download = await fileSystem.downloadAsync(artifact.downloadUrl, localPath);
+        if (download.status === 200) {
+          const downloadedHash = await this.readFileSha256(fileSystem, localPath);
+          if (downloadedHash === artifact.sha256) {
+            artifact.localPath = localPath;
+          } else {
+            await fileSystem.deleteAsync(localPath, { idempotent: true }).catch(() => {});
+          }
+        } else {
+          await fileSystem.deleteAsync(localPath, { idempotent: true }).catch(() => {});
+        }
+      } catch {
+        await fileSystem.deleteAsync(localPath, { idempotent: true }).catch(() => {});
+      }
+    }
+  }
+
+  private async readFileSha256(
+    fileSystem: ExpoFileSystemModule,
+    localPath: string,
+  ): Promise<string | null> {
+    try {
+      const base64 = await fileSystem.readAsStringAsync(localPath, {
+        encoding: fileSystem.EncodingType.Base64,
+      });
+      return bytesToSha256Hex(Uint8Array.from(Buffer.from(base64, 'base64')));
+    } catch {
+      return null;
+    }
+  }
+
+  private async saveBundleManifest(policyId: string, manifest: BundleManifest): Promise<void> {
+    const fileSystem = await loadFileSystem();
+    if (!fileSystem || !fileSystem.documentDirectory) {
+      return;
+    }
+
+    const cacheDir = `${fileSystem.documentDirectory}verify-ai/bundles/${policyId}/`;
+    const manifestPath = `${cacheDir}manifest.json`;
+    await fileSystem.makeDirectoryAsync(cacheDir, { intermediates: true }).catch(() => {});
+    await fileSystem.writeAsStringAsync(manifestPath, JSON.stringify(serializeBundleManifest(manifest)));
+  }
+
+  /** Load a cached bundle from disk on startup. */
+  async loadCachedBundle(policyId: string): Promise<BundleManifest | null> {
+    const fileSystem = await loadFileSystem();
+    if (!fileSystem || !fileSystem.documentDirectory) {
+      return null;
+    }
+
+    try {
+      const cacheDir = `${fileSystem.documentDirectory}verify-ai/bundles/${policyId}/`;
+      const manifestPath = `${cacheDir}manifest.json`;
+
+      const info = await fileSystem.getInfoAsync(manifestPath);
+      if (!info.exists) return null;
+
+      const content = await fileSystem.readAsStringAsync(manifestPath);
+      const manifest = parseBundleManifest(JSON.parse(content) as BundleManifestResponse);
+
+      // Verify local paths exist
+      for (const artifact of manifest.artifacts) {
+        const ext = artifact.format === 'coreml' ? 'mlpackage.zip' : artifact.format;
+        const localPath = `${cacheDir}${artifact.format}_v${artifact.version}.${ext}`;
+        const fileInfo = await fileSystem.getInfoAsync(localPath).catch(() => ({ exists: false }));
+        if (fileInfo.exists) {
+          artifact.localPath = localPath;
+        }
+      }
+
+      if (manifest.artifacts.some((a) => a.localPath)) {
+        this._currentBundle = manifest;
+        return manifest;
+      }
+
+      return null;
+    } catch {
+      return null;
+    }
+  }
+}
+
+function parseBundleManifest(json: BundleManifestResponse): BundleManifest {
+  return {
+    bundleVersion: json.bundle_version,
+    ontologyVersion: json.ontology_version,
+    schemaVersion: json.schema_version,
+    policyId: json.policy_id,
+    classIds: json.class_ids || [],
+    artifacts: (json.artifacts || []).map((artifact) => ({
+      role: artifact.role,
+      architecture: artifact.architecture,
+      version: artifact.version,
+      format: artifact.format,
+      storagePath: artifact.storage_path,
+      sizeBytes: artifact.size_bytes,
+      sha256: artifact.sha256,
+      downloadUrl: artifact.download_url,
+      localPath: undefined,
+    })),
+    policyAst: json.policy_ast,
+  };
+}
+
+function serializeBundleManifest(manifest: BundleManifest): BundleManifestResponse {
+  return {
+    bundle_version: manifest.bundleVersion,
+    ontology_version: manifest.ontologyVersion,
+    schema_version: manifest.schemaVersion,
+    policy_id: manifest.policyId,
+    class_ids: manifest.classIds,
+    artifacts: manifest.artifacts.map((artifact) => ({
+      role: artifact.role,
+      architecture: artifact.architecture,
+      version: artifact.version,
+      format: artifact.format,
+      storage_path: artifact.storagePath,
+      size_bytes: artifact.sizeBytes,
+      sha256: artifact.sha256,
+      download_url: artifact.downloadUrl,
+    })),
+    policy_ast: manifest.policyAst,
+  };
+}