@smileid/usesmileid 12.0.0-beta01

package/src/camera/document_image_pipeline.ts

@@ -0,0 +1,101 @@
+import { Directory, File, Paths } from "expo-file-system";
+import { ImageManipulator, SaveFormat } from "expo-image-manipulator";
+import {
+  DocumentSide,
+  UseSmileIDCrashReporting,
+} from "@smileid/usesmileid_bridge";
+import { CameraDefaults } from "./camera_defaults";
+import { computeDocumentResizeTarget } from "./document_resize_target";
+
+export { computeDocumentResizeTarget } from "./document_resize_target";
+
+/**
+ * Imported document image pipeline — brings a manual-shutter still or gallery
+ * image onto the SAME byte policy as the native auto-capture artifact: longest
+ * edge downscaled to {@link CameraDefaults.documentMaxDimension} at
+ * {@link CameraDefaults.documentJpegQuality}, written under the canonical
+ * per-side filename. The Smile EXIF `UserComment` tag is stamped downstream by
+ * `SmileExifTagger.finalize`, so every upload is byte-policy-identical
+ * regardless of capture path.
+ */
+
+/** Subdirectory under the cache directory holding imported document artifacts. */
+const IMPORT_SUBDIRECTORY = "usesmileid_documents";
+
+/** Canonical per-side artifact basename, matching the auto-capture naming. */
+export function documentArtifactFileName(side: DocumentSide): string {
+  return side === DocumentSide.Back
+    ? "document_back.jpg"
+    : "document_front.jpg";
+}
+
+/** Source image for an imported document capture (manual shutter or gallery). */
+export type ImportedDocumentSource = {
+  /** `file://` URI (or bare path) of the still / picked image. */
+  readonly uri: string;
+  /** Pixel dimensions when known. Omitted ⇒ re-encode at document quality without a resize (never upscaled). */
+  readonly width?: number;
+  readonly height?: number;
+};
+
+/**
+ * Resample an imported document image and write it under the canonical per-side
+ * filename. Returns the `file://` URI of the processed artifact.
+ *
+ * Canonical naming is best-effort: the native auto-capture session directory is
+ * not exposed to JS, so the artifact lands in the app cache under the same
+ * basename. On rename failure the resampled file's own URI is returned so the
+ * byte-correct artifact is never lost — the backend validates the EXIF
+ * image-type tag and JPEG bytes, not the containing directory.
+ */
+export async function resampleImportedDocument(
+  source: ImportedDocumentSource,
+  side: DocumentSide,
+): Promise<string> {
+  const target =
+    source.width !== undefined && source.height !== undefined
+      ? computeDocumentResizeTarget(
+          source.width,
+          source.height,
+          CameraDefaults.documentMaxDimension,
+        )
+      : null;
+
+  const context = ImageManipulator.manipulate(source.uri);
+  if (target) {
+    context.resize(target);
+  }
+  const rendered = await context.renderAsync();
+  const result = await rendered.saveAsync({
+    compress: CameraDefaults.documentJpegQuality,
+    format: SaveFormat.JPEG,
+  });
+
+  return persistAsCanonicalArtifact(result.uri, side);
+}
+
+/** Move the resampled file to `<cache>/usesmileid_documents/document_<side>.jpg` (synchronous `expo-file-system` API). */
+function persistAsCanonicalArtifact(
+  resampledUri: string,
+  side: DocumentSide,
+): string {
+  try {
+    const directory = new Directory(Paths.cache, IMPORT_SUBDIRECTORY);
+    if (!directory.exists) {
+      directory.create({ intermediates: true });
+    }
+    const destination = new File(directory, documentArtifactFileName(side));
+    if (destination.exists) {
+      destination.delete();
+    }
+    new File(resampledUri).move(destination);
+    return destination.uri;
+  } catch (error) {
+    // Best-effort rename: a file-system error must not drop the byte-correct
+    // artifact. Surface non-fatal and fall back to the resampled file's URI.
+    UseSmileIDCrashReporting.captureException(
+      error instanceof Error ? error : new Error(String(error)),
+    );
+    return resampledUri;
+  }
+}

package/src/camera/document_resize_target.ts

@@ -0,0 +1,22 @@
+/**
+ * Resize target so the LONGEST edge equals `maxDimension`, preserving aspect
+ * ratio (`expo-image-manipulator` derives the other edge from one). Returns
+ * `null` when the image already fits — downscale-only, never upscale (upscaling
+ * adds bytes and blur, diverging from the native auto-capture artifact).
+ * Dependency-free so the math is unit-testable without the native image bridge.
+ */
+export function computeDocumentResizeTarget(
+  width: number,
+  height: number,
+  maxDimension: number,
+): { readonly width: number } | { readonly height: number } | null {
+  const longestEdge = Math.max(width, height);
+  if (
+    !Number.isFinite(longestEdge) ||
+    longestEdge <= 0 ||
+    longestEdge <= maxDimension
+  ) {
+    return null;
+  }
+  return width >= height ? { width: maxDimension } : { height: maxDimension };
+}

package/src/camera/focus_state.ts

@@ -0,0 +1,26 @@
+/**
+ * Tap-to-focus state machine — `unspecified` / `running` / `success`
+ * / `failure`. The `point` payload (where on the preview the partner
+ * tapped) is `undefined` only for the `unspecified` state.
+ */
+
+export type FocusPoint = {
+  /** Surface-coordinate X (0..1, normalized to preview width). */
+  readonly x: number;
+  /** Surface-coordinate Y (0..1, normalized to preview height). */
+  readonly y: number;
+};
+
+export type FocusState =
+  | { readonly kind: "unspecified" }
+  | { readonly kind: "running"; readonly point: FocusPoint }
+  | { readonly kind: "success"; readonly point: FocusPoint }
+  | { readonly kind: "failure"; readonly point: FocusPoint };
+
+/** Convenience constructors for the `FocusState` discriminated union. */
+export const FocusStates = {
+  unspecified: (): FocusState => ({ kind: "unspecified" }),
+  running: (point: FocusPoint): FocusState => ({ kind: "running", point }),
+  success: (point: FocusPoint): FocusState => ({ kind: "success", point }),
+  failure: (point: FocusPoint): FocusState => ({ kind: "failure", point }),
+} as const;

package/src/camera/orientation_degrees.ts

@@ -0,0 +1,22 @@
+import type { Orientation } from "react-native-vision-camera";
+
+/**
+ * Map VisionCamera's gravity-derived device {@link Orientation} to the
+ * clockwise degrees the native scan sessions expect (`0` = upright
+ * portrait). Must reflect the physical device tilt — feed it from
+ * `onOutputOrientationChanged` with `outputOrientation="device"`. Never
+ * derive it from `frame.orientation`: that reports the sensor mounting,
+ * not how the user is holding the phone.
+ */
+export function orientationToDegrees(orientation: Orientation): number {
+  switch (orientation) {
+    case "portrait":
+      return 0;
+    case "landscape-left":
+      return 90;
+    case "portrait-upside-down":
+      return 180;
+    case "landscape-right":
+      return 270;
+  }
+}

package/src/camera/use_smile_id_camera_preview.tsx

@@ -0,0 +1,256 @@
+import { useCallback, useEffect, useMemo, useRef, useState } from "react";
+import type { ReactNode } from "react";
+import {
+  Pressable,
+  StyleSheet,
+  View,
+  type GestureResponderEvent,
+  type LayoutChangeEvent,
+  type StyleProp,
+  type ViewStyle,
+} from "react-native";
+import {
+  Camera,
+  useCameraDevice,
+  useCameraFormat,
+  VisionCameraProxy,
+} from "react-native-vision-camera";
+import { useFrameProcessor } from "react-native-vision-camera";
+import { Worklets } from "react-native-worklets-core";
+import { FrameProcessorPluginNotRegisteredException } from "@smileid/usesmileid_bridge";
+import { CameraDefaults } from "./camera_defaults";
+import { orientationToDegrees } from "./orientation_degrees";
+import type { UseSmileIDCameraSystem } from "./use_smile_id_camera_system";
+import type { CameraSystemState } from "./camera_configuration";
+import type { FocusState } from "./focus_state";
+
+/**
+ * Per-frame analysis wiring for the preview. Pixels never cross into
+ * JS: the named native frame processor plugin runs the provider's scan
+ * session and the worklet forwards only the result map to `onResult`
+ * on the JS thread.
+ */
+export type UseSmileIDFrameAnalysis = {
+  /** VisionCamera frame-processor plugin name — the analyzer's `frameProcessorPluginName`. */
+  readonly pluginName: string;
+  /**
+   * Receives each frame's raw scan-result map (JS thread); feed it
+   * `useSmileIDFrameStream(...).onFrameResult`. Memoise the
+   * `frameAnalysis` object — a new identity per render rebuilds the
+   * worklet bridge and the camera frame processor.
+   */
+  readonly onResult: (rawResult: unknown) => void;
+  /**
+   * Called once if the named plugin isn't registered natively. Without
+   * a handler the failure is silent — frames never produce results.
+   */
+  readonly onError?: (error: unknown) => void;
+};
+
+export type UseSmileIDCameraPreviewProps = {
+  /** The orchestrator instance — partner creates with `new UseSmileIDCameraSystem()`. */
+  readonly cameraSystem: UseSmileIDCameraSystem;
+  /**
+   * Optional per-frame analysis wiring — the preview initialises the
+   * named plugin and streams each frame's scan-result map to `onResult`.
+   */
+  readonly frameAnalysis?: UseSmileIDFrameAnalysis;
+  /**
+   * Overlay children (face oval, focus reticle, partner branding).
+   * Receives the live `state` and `focusState` via the function-as-
+   * children form so overlays react to focus transitions without
+   * subscribing to the system manually.
+   */
+  readonly children?:
+    | ReactNode
+    | ((args: {
+        readonly state: CameraSystemState;
+        readonly focusState: FocusState;
+      }) => ReactNode);
+  readonly style?: StyleProp<ViewStyle>;
+};
+
+/**
+ * Camera preview component.
+ *
+ * Wraps `react-native-vision-camera`'s `<Camera>`, attaches its ref
+ * to the supplied `cameraSystem` so the orchestrator can drive still
+ * captures + tap-to-focus, and — when `frameAnalysis` is supplied —
+ * wires a frame processor that runs the provider's native plugin per
+ * frame and streams its result maps to the JS thread.
+ *
+ * Lifecycle: ref attaches on mount, detaches on unmount. Partners
+ * call `cameraSystem.start()` once they're ready for the camera to
+ * begin streaming (typically inside a `useEffect` watching the active
+ * `FlowStep`); the preview reads `state.isStreaming` to drive the
+ * `<Camera isActive>` prop.
+ */
+export function UseSmileIDCameraPreview({
+  cameraSystem,
+  frameAnalysis,
+  children,
+  style,
+}: UseSmileIDCameraPreviewProps) {
+  const cameraRef = useRef<Camera | null>(null);
+  const [state, setState] = useState<CameraSystemState>(() =>
+    cameraSystem.getState(),
+  );
+  const [focusState, setFocusState] = useState<FocusState>(() =>
+    cameraSystem.getFocusState(),
+  );
+  const [viewSize, setViewSize] = useState<{ width: number; height: number }>({
+    width: 0,
+    height: 0,
+  });
+
+  const device = useCameraDevice(state.currentLens);
+  // Pin a 30 fps capture format at ~720p — explicit selection avoids
+  // vision-camera's default "highest supported" heuristic which on
+  // some devices lands on 60 fps with shorter per-frame exposure
+  // (more visible ISO noise) or a much higher resolution than the
+  // analysis pipeline needs.
+  const format = useCameraFormat(device, [
+    // 4:3 first: the scan sessions' geometry checks and smile heuristic
+    // are calibrated on 4:3 frames — 16:9 skews normalized landmark
+    // ratios and silently shifts every threshold.
+    { videoAspectRatio: 4 / 3 },
+    {
+      videoResolution: {
+        width: CameraDefaults.analysisTargetResolution.width,
+        height: CameraDefaults.analysisTargetResolution.height,
+      },
+    },
+    { fps: CameraDefaults.analysisTargetFps },
+  ]);
+  const fps =
+    cameraSystem.getConfiguration().fps ?? CameraDefaults.analysisTargetFps;
+
+  // Subscribe once. cameraSystem reference is stable across renders
+  // when the partner holds it in a `useMemo` — re-running the effect
+  // would leak listeners.
+  useEffect(() => {
+    return cameraSystem.subscribe((nextState, nextFocus) => {
+      setState(nextState);
+      setFocusState(nextFocus);
+    });
+  }, [cameraSystem]);
+
+  // Attach the Camera ref once mounted. Detach on unmount so the
+  // system doesn't try to drive a torn-down view (e.g. `focus()`).
+  useEffect(() => {
+    cameraSystem.attachRef(cameraRef.current);
+    return () => {
+      cameraSystem.attachRef(null);
+    };
+  }, [cameraSystem]);
+
+  // `null` when analysis is off or the plugin isn't registered natively.
+  const pluginName = frameAnalysis?.pluginName ?? null;
+  const plugin = useMemo(() => {
+    if (pluginName === null) return null;
+    return VisionCameraProxy.initFrameProcessorPlugin(pluginName, {}) ?? null;
+  }, [pluginName]);
+
+  // Surface a missing plugin once per name — otherwise the worklet
+  // silently no-ops on every frame. The handler ref keeps an unstable
+  // `onError` identity from re-firing the effect.
+  const onAnalysisErrorRef = useRef(frameAnalysis?.onError);
+  useEffect(() => {
+    onAnalysisErrorRef.current = frameAnalysis?.onError;
+  }, [frameAnalysis?.onError]);
+  useEffect(() => {
+    if (pluginName === null || plugin !== null) return;
+    onAnalysisErrorRef.current?.(
+      new FrameProcessorPluginNotRegisteredException({ pluginName }),
+    );
+  }, [pluginName, plugin]);
+
+  // `Worklets.createRunOnJS` posts results to the JS runtime without
+  // blocking the worklet. Backpressure lives natively: the plugin
+  // blocks the frame-processor thread per scan, so mid-scan frames are
+  // dropped before reaching JS.
+  const onFrameResult = frameAnalysis?.onResult ?? null;
+  const onResultRunOnJS = useMemo(() => {
+    if (!onFrameResult) return null;
+    return Worklets.createRunOnJS((rawResult: unknown) => {
+      onFrameResult(rawResult);
+    });
+  }, [onFrameResult]);
+
+  // Physical device tilt (0 = upright portrait) for the native
+  // session's upright validator and head-pose math; gravity-sourced,
+  // never derived from `frame.orientation` (fixed sensor mounting).
+  const [deviceOrientationDegrees, setDeviceOrientationDegrees] = useState(0);
+
+  const frameProcessor = useFrameProcessor(
+    (frame) => {
+      "worklet";
+      if (!plugin || !onResultRunOnJS) return;
+      const rawResult = plugin.call(frame, {
+        rotation: deviceOrientationDegrees,
+      });
+      onResultRunOnJS(rawResult);
+    },
+    [plugin, onResultRunOnJS, deviceOrientationDegrees],
+  );
+
+  const handleLayout = useCallback((event: LayoutChangeEvent) => {
+    const { width, height } = event.nativeEvent.layout;
+    setViewSize({ width, height });
+  }, []);
+
+  const handleTap = useCallback(
+    (event: GestureResponderEvent) => {
+      if (viewSize.width === 0 || viewSize.height === 0) return;
+      const { locationX, locationY } = event.nativeEvent;
+      void cameraSystem.tapToFocus({
+        x: locationX / viewSize.width,
+        y: locationY / viewSize.height,
+      });
+    },
+    [cameraSystem, viewSize.height, viewSize.width],
+  );
+
+  const overlay =
+    typeof children === "function" ? children({ state, focusState }) : children;
+
+  return (
+    <View style={[styles.container, style]} onLayout={handleLayout}>
+      {device ? (
+        <Camera
+          ref={cameraRef}
+          style={StyleSheet.absoluteFill}
+          device={device}
+          fps={fps}
+          isActive={state.isStreaming}
+          photo={true}
+          // Gravity-based device orientation — keeps reporting tilt even
+          // when the app's UI orientation is locked to portrait.
+          outputOrientation="device"
+          onOutputOrientationChanged={(orientation) => {
+            setDeviceOrientationDegrees(orientationToDegrees(orientation));
+          }}
+          // Conditional spreads instead of `prop={undefined}` —
+          // `exactOptionalPropertyTypes` in tsconfig rejects passing
+          // `undefined` to an optional prop, so we omit the key when
+          // the value is missing.
+          {...(format ? { format } : {})}
+          {...(plugin && onResultRunOnJS ? { frameProcessor } : {})}
+        />
+      ) : null}
+      <Pressable
+        style={StyleSheet.absoluteFill}
+        onPress={handleTap}
+        // Pass-through hit testing for partner overlay UIs that need
+        // to capture taps themselves (buttons, links, etc.).
+        pointerEvents="box-none"
+      >
+        {overlay}
+      </Pressable>
+    </View>
+  );
+}
+
+const styles = StyleSheet.create({
+  container: { flex: 1, backgroundColor: "black", overflow: "hidden" },
+});

package/src/camera/use_smile_id_camera_system.ts

@@ -0,0 +1,236 @@
+import type { Camera } from "react-native-vision-camera";
+import { CameraDefaults } from "./camera_defaults";
+import { CameraLens } from "./camera_lens";
+import type {
+  CameraConfiguration,
+  CameraSystemState,
+} from "./camera_configuration";
+import type { FocusPoint, FocusState } from "./focus_state";
+import { FocusStates } from "./focus_state";
+import {
+  getCameraPermissionStatus,
+  requestCameraPermission,
+} from "./camera_permissions";
+import { UseSmileIDCrashReporting } from "@smileid/usesmileid_bridge";
+
+/**
+ * Camera orchestrator.
+ *
+ * Key responsibilities:
+ *   1. Drive permission prompt + state
+ *   2. Toggle the camera on/off — frame streaming itself lives in
+ *      `<UseSmileIDCameraPreview>`'s `useFrameProcessor` worklet, so
+ *      this class doesn't own the frame loop
+ *   3. Surface tap-to-focus — drives the visual reticle and calls
+ *      `Camera.focus(point)` so the lens actually re-focuses
+ *
+ * The SDK does not ship a still-capture API. Per-frame analysis via
+ * `analyzer.analyze(bytes, metadata, rotation)` + `flush()` is the
+ * canonical capture path; the `UseSmileIDCaptureScreen` owns that
+ * wiring end-to-end.
+ *
+ * Lifecycle: `configure(config)` then `start()` to flip the running
+ * state on (camera renders in preview), `stop()` to pause without
+ * tearing down the React tree, `release()` to detach all refs and
+ * mark the system unusable. After `release()`, `start()` throws.
+ */
+
+type CameraInstance = Camera;
+type Listener = (state: CameraSystemState, focus: FocusState) => void;
+
+export class UseSmileIDCameraSystem {
+  private _cameraRef: CameraInstance | null = null;
+  private _state: CameraSystemState;
+  private _focusState: FocusState = FocusStates.unspecified();
+  private readonly _listeners = new Set<Listener>();
+  private _config: CameraConfiguration = {};
+  private _released = false;
+  private _focusRunningTimeout: ReturnType<typeof setTimeout> | null = null;
+  private _focusSettledTimeout: ReturnType<typeof setTimeout> | null = null;
+
+  constructor(initialLens: CameraLens = CameraLens.front) {
+    this._state = {
+      isCameraRunning: false,
+      isStreaming: false,
+      currentLens: initialLens,
+      hasPermission: false,
+    };
+  }
+
+  /**
+   * @internal Called by `<UseSmileIDCameraPreview>` when the underlying
+   * `<Camera>` mounts / unmounts. Partners do not call this directly.
+   */
+  attachRef(ref: CameraInstance | null): void {
+    this._cameraRef = ref;
+  }
+
+  /** Snapshot of the runtime state. Stable across calls between updates. */
+  getState(): CameraSystemState {
+    return this._state;
+  }
+
+  /** Snapshot of the focus state. */
+  getFocusState(): FocusState {
+    return this._focusState;
+  }
+
+  /** Snapshot of the configuration last passed to `configure()`. */
+  getConfiguration(): CameraConfiguration {
+    return this._config;
+  }
+
+  /**
+   * Subscribe to state + focus-state changes. Returns an unsubscribe
+   * function; safe to call after `release()`.
+   */
+  subscribe(listener: Listener): () => void {
+    this._listeners.add(listener);
+    return () => {
+      this._listeners.delete(listener);
+    };
+  }
+
+  /**
+   * Replace the active configuration. Safe to call before `start()`;
+   * if the camera is already running, the preview reads the new lens
+   * / fps / format on its next render.
+   */
+  configure(config: CameraConfiguration): void {
+    this._config = config;
+    const lens = config.lens ?? this._state.currentLens;
+    if (lens !== this._state.currentLens) {
+      this._setState({ ...this._state, currentLens: lens });
+    }
+  }
+
+  /**
+   * Ensure permission and flip the running state on. The preview
+   * component watches `state.isCameraRunning` and `state.isStreaming`
+   * to decide when to mount the native `<Camera>` and start its frame
+   * processor.
+   *
+   * Returns once permissions have settled. The first frame can take
+   * 100–300 ms to arrive while the underlying capture session warms.
+   */
+  async start(): Promise<void> {
+    if (this._released) {
+      throw new Error(
+        "UseSmileIDCameraSystem.start called after release() — instantiate a new system",
+      );
+    }
+
+    const status = await this._ensurePermission();
+    if (status !== "granted") {
+      this._setState({ ...this._state, hasPermission: false });
+      return;
+    }
+
+    this._setState({
+      ...this._state,
+      hasPermission: true,
+      isCameraRunning: true,
+      isStreaming: true,
+    });
+  }
+
+  /**
+   * Pause the camera without tearing down the React tree. Frame
+   * processor stops firing; the preview shows the last frame.
+   * Re-callable via `start()`.
+   */
+  stop(): void {
+    if (this._released) return;
+    this._setState({
+      ...this._state,
+      isCameraRunning: false,
+      isStreaming: false,
+    });
+  }
+
+  /**
+   * Drive the focus-indicator state machine AND call vision-camera's
+   * `Camera.focus(point)` to actually re-focus the lens. The point is
+   * in normalized [0..1] preview coordinates; vision-camera converts
+   * to device coordinates internally.
+   */
+  async tapToFocus(point: FocusPoint): Promise<void> {
+    if (this._released) return;
+
+    this._setFocusState(FocusStates.running(point));
+
+    if (this._focusRunningTimeout) clearTimeout(this._focusRunningTimeout);
+    if (this._focusSettledTimeout) clearTimeout(this._focusSettledTimeout);
+
+    const ref = this._cameraRef;
+    if (ref) {
+      try {
+        await ref.focus({ x: point.x, y: point.y });
+      } catch (error) {
+        // Focus can fail on devices whose camera doesn't support
+        // programmatic focus (e.g. fixed-focus front cameras). Surface
+        // via crash reporting and keep the visual state machine
+        // moving — the user still gets reticle feedback.
+        UseSmileIDCrashReporting.captureException(error);
+      }
+    }
+
+    this._focusRunningTimeout = setTimeout(() => {
+      this._setFocusState(FocusStates.success(point));
+      this._focusSettledTimeout = setTimeout(() => {
+        this._setFocusState(FocusStates.unspecified());
+      }, CameraDefaults.focusSettledDurationMs);
+    }, CameraDefaults.focusRunningDurationMs);
+  }
+
+  /**
+   * Stop the camera, clear any focus timers, drop the ref.
+   * Idempotent — safe to call from `useEffect` cleanup.
+   *
+   * After release, calling `start()` throws — the partner must
+   * instantiate a new system.
+   */
+  release(): void {
+    this._released = true;
+    if (this._focusRunningTimeout) clearTimeout(this._focusRunningTimeout);
+    if (this._focusSettledTimeout) clearTimeout(this._focusSettledTimeout);
+    this._focusRunningTimeout = null;
+    this._focusSettledTimeout = null;
+    this._cameraRef = null;
+    this._setState({
+      ...this._state,
+      isCameraRunning: false,
+      isStreaming: false,
+    });
+  }
+
+  // --- internals -----------------------------------------------------------
+
+  private async _ensurePermission(): Promise<
+    "granted" | "denied" | "undetermined"
+  > {
+    const current = await getCameraPermissionStatus();
+    if (current === "granted") return "granted";
+    return await requestCameraPermission();
+  }
+
+  private _setState(next: CameraSystemState): void {
+    this._state = next;
+    this._notify();
+  }
+
+  private _setFocusState(next: FocusState): void {
+    this._focusState = next;
+    this._notify();
+  }
+
+  private _notify(): void {
+    for (const listener of this._listeners) {
+      try {
+        listener(this._state, this._focusState);
+      } catch {
+        /* never let a buggy listener break the dispatch chain */
+      }
+    }
+  }
+}