@ateam-ai-mobile/expo 0.1.4 → 0.1.7

package/src/vision/index.ts

@@ -0,0 +1,144 @@
+// ada-vision — on-device image understanding, our own native code (no camera lib).
+//
+// On-device, frames never leave the phone:
+//   assessQuality(uri)          — one-shot quality of a LOCAL frame file.
+//   startVision()/onVisionFrame — the headless camera session that publishes a
+//                                 timeline frame (id + quality, never pixels) at a
+//                                 cadence; readText/describe/detectObjects resolve
+//                                 a published id ON-DEVICE.
+//
+// The luma math mirrors ./analysis.js (which has the exhaustive unit tests).
+//
+// requireOptionalNativeModule: on a binary WITHOUT the native module, the import
+// is null, isSupported() is false, and the session calls refuse cleanly (no crash).
+
+import type { ComponentType } from 'react';
+import { requireOptionalNativeModule } from 'expo-modules-core';
+import { requireNativeView } from 'expo';
+
+/** TEST-ONLY live preview bound to the same headless session (so a tester can
+ *  aim). The real Guide runs headless and never mounts this. Null on a binary
+ *  without the native view, so importing this module never crashes. */
+export const CameraPreview: ComponentType<any> | null = (() => {
+  try { return requireNativeView('AdaVision'); } catch { return null; }
+})();
+
+export interface RawFrameQuality {
+  blur: 'sharp' | 'slight' | 'severe' | 'unknown';
+  exposure: 'good' | 'dark' | 'bright' | 'unknown';
+  /** true = lens covered; false = clearly not; null = not assessed. */
+  obstructed: boolean | null;
+}
+
+interface Subscription { remove(): void }
+
+/** A published timeline frame from the vision session (metadata only — never
+ *  pixels; the buffer stays in the native ring, resolved by id). */
+export interface VisionFrame {
+  id: string;
+  ts: number;
+  expiresAt: number;
+  blur: RawFrameQuality['blur'];
+  exposure: RawFrameQuality['exposure'];
+  obstructed: RawFrameQuality['obstructed'];
+}
+
+/** One OCR result on a frame — matches the SDK's VisionText. */
+export interface VisionText {
+  text: string;
+  /** 0..1 recogniser confidence (never a 1.0 filler). */
+  confidence: number | null;
+  /** Normalised 0..1 box, origin TOP-left. */
+  box?: { x: number; y: number; width: number; height: number };
+}
+
+/** One classifier label — a vocabulary term, NOT prose. Matches VisionLabel. */
+export interface VisionLabel {
+  label: string;
+  confidence: number | null;
+}
+
+/** One detected object — a located item with a box, from the bundled detector. */
+export interface VisionObject {
+  /** Open-Images class name (e.g. "Car", "Traffic light", "Building", "Person"). */
+  label: string;
+  /** 0..1 detector confidence. */
+  confidence: number | null;
+  /** Normalised 0..1 box, origin TOP-left. */
+  box?: { x: number; y: number; width: number; height: number };
+}
+
+interface AdaVisionNative {
+  isSupported(): boolean;
+  assessQuality(uri: string): Promise<RawFrameQuality>;
+  startVision(intervalMs: number): Promise<void>;
+  stopVision(): void;
+  visionRetentionMs(): number;
+  qualityById(id: string): RawFrameQuality | null;
+  readText(id: string, languages: string[] | null): Promise<VisionText[]>;
+  describe(id: string, maxLabels: number): Promise<VisionLabel[]>;
+  detectObjects(id: string, maxItems: number, minConfidence: number): Promise<VisionObject[]>;
+  addListener(event: 'visionFrame', cb: (f: VisionFrame) => void): Subscription;
+}
+
+const native = requireOptionalNativeModule<AdaVisionNative>('AdaVision');
+
+/** Is the native vision module present in this binary? */
+export function isSupported(): boolean {
+  try { return native?.isSupported() ?? false; } catch { return false; }
+}
+
+/** One-shot: measure a frame's quality from its LOCAL file. Throws if it cannot look. */
+export async function assessQuality(uri: string): Promise<RawFrameQuality> {
+  if (!native) throw new Error('ada-vision native module not present in this build');
+  return native.assessQuality(uri);
+}
+
+// ── Vision session (feeds the Visual Engine) ──
+
+/** Start the vision session — publishes a timeline frame every `intervalMs`. */
+export async function startVision(intervalMs = 1000): Promise<void> {
+  if (!native) throw new Error('ada-vision native module not present in this build');
+  return native.startVision(intervalMs);
+}
+
+/** Stop the vision session and release the camera. */
+export function stopVision(): void {
+  native?.stopVision?.();
+}
+
+/** Subscribe to published timeline frames (id + quality + expiry; no pixels). */
+export function onVisionFrame(cb: (f: VisionFrame) => void): Subscription {
+  if (!native) return { remove() {} };
+  return native.addListener('visionFrame', cb);
+}
+
+/** How far back the native ring can still resolve a frame id (ms). */
+export function visionRetentionMs(): number {
+  return native?.visionRetentionMs?.() ?? 0;
+}
+
+/** Resolve a published frame id → its measured quality (null if evicted). */
+export function qualityById(id: string): RawFrameQuality | null {
+  return native?.qualityById?.(id) ?? null;
+}
+
+/** On-device OCR on a published frame. [] = looked, no text; throws = couldn't look. */
+export async function readText(id: string, languages?: string[]): Promise<VisionText[]> {
+  if (!native) throw new Error('ada-vision native module not present in this build');
+  return native.readText(id, languages ?? null);
+}
+
+/** On-device image classification → labels (not prose). [] = nothing recognised. */
+export async function describe(id: string, maxLabels = 5): Promise<VisionLabel[]> {
+  if (!native) throw new Error('ada-vision native module not present in this build');
+  return native.describe(id, maxLabels);
+}
+
+/** On-device object detection (bundled YOLOv8n, Open-Images vocabulary) → located
+ *  items with boxes. [] = looked, found nothing above `minConfidence`; throws =
+ *  couldn't look (id aged out, or the model isn't in this build). */
+export async function detectObjects(id: string, maxItems = 10, minConfidence = 0.25): Promise<VisionObject[]> {
+  if (!native) throw new Error('ada-vision native module not present in this build');
+  return native.detectObjects(id, maxItems, minConfidence);
+}

package/src/vision/ring.js

@@ -0,0 +1,35 @@
+// Reference for the native VisionSession frame ring (ios/AdaVisionModule.swift).
+// The native side mirrors this; the tests pin the invariant GPT flagged:
+//   a frame id from an OLD camera session, or one evicted from the ring, must
+//   NEVER resolve. "Cannot look at that frame" (null) must never be confusable
+//   with a stale hit from a previous session.
+//
+// Two independent guarantees, both modelled here: ids are generation-scoped
+// (vf_<gen>_<ts>), AND the ring is cleared on start(). Either alone prevents a
+// stale resolve; together it can't happen.
+
+export function createRing({ cap = 12, intervalMs = 1000 } = {}) {
+  let generation = 0;
+  let entries = []; // { id, quality }
+
+  return {
+    /** A new camera session: bump generation, clear the ring. */
+    start() { generation += 1; entries = []; },
+    /** Session ended: nothing is resolvable. */
+    stop() { entries = []; },
+    /** The ring's actual retention (ms) — feeds the engine's window clamp. */
+    retentionMs() { return cap * intervalMs; },
+    /** Publish a frame → mint a generation-scoped id, retain (bounded). */
+    publish(quality, tsMs) {
+      const id = `vf_${generation}_${tsMs}`;
+      entries.push({ id, quality });
+      if (entries.length > cap) entries.splice(0, entries.length - cap);
+      return id;
+    },
+    /** Resolve by id → quality, or null when evicted / old-generation / unknown. */
+    qualityById(id) {
+      const e = entries.find(x => x.id === id);
+      return e ? e.quality : null;
+    },
+  };
+}