@pose-tracker/react-native-pose-estimation-light 0.1.0

package/src/backends/PoseBackend.ts

@@ -0,0 +1,59 @@
+/**
+ * Inference backend abstraction.
+ *
+ * v1 ships a single implementation (`TfjsMoveNetBackend`, TF.js +
+ * bundleResourceIO). The interface is deliberately minimal so a native
+ * TFLite backend (react-native-fast-tflite + custom models downloaded from
+ * the manifest) can be plugged in later without touching the engine or the
+ * provider: anything that turns an input frame into a `Pose` qualifies.
+ */
+
+import type { Pose } from '../types/pose';
+import type { ModelDescriptor } from '../types/manifest';
+import type { AccelerationDiagnostics } from '../types/acceleration';
+
+/**
+ * Input frame handed to the backend. For the TF.js backend this is a
+ * `tf.Tensor3D` ([height, width, 3], int32/float32). Kept opaque here so the
+ * interface does not force a TF.js dependency on future native backends
+ * (which would receive a native frame/buffer reference instead).
+ */
+export type PoseInputFrame = unknown;
+
+export interface PoseBackendInitOptions {
+  /** Model resolved from the manifest for the active profile. */
+  model: ModelDescriptor;
+  /**
+   * Local filesystem path of a downloaded model archive, when the manifest
+   * provided one and it passed integrity validation. Undefined = use the
+   * model bundled in the SDK package.
+   */
+  localModelPath?: string;
+}
+
+export interface PoseBackend {
+  readonly name: string;
+
+  /** Prepare the runtime (TF.js ready + backend selection for tfjs). */
+  init(options: PoseBackendInitOptions): Promise<void>;
+
+  /**
+   * Run 1–2 dummy inferences so shaders/kernels are compiled before the
+   * first real camera frame. No-op if already warm.
+   */
+  warmup(): Promise<void>;
+
+  /** Estimate a single pose from an input frame. */
+  estimatePose(frame: PoseInputFrame): Promise<Pose | null>;
+
+  /** Release model and GPU resources. */
+  dispose(): Promise<void>;
+
+  /**
+   * Optional: GPU-acceleration verdict + diagnostics collected during
+   * `warmup()` (see docs/ANDROID_GL_ACCELERATION.md). Backends that cannot
+   * introspect their accelerator (or a future native TFLite backend using
+   * GPU/NNAPI delegates) may omit it or return null before warm-up.
+   */
+  getAcceleration?(): AccelerationDiagnostics | null;
+}

package/src/backends/vision/VisionPoseBackend.ts

@@ -0,0 +1,238 @@
+/**
+ * Apple Vision implementation of `PoseBackend` (iOS only).
+ *
+ * Uses `VNDetectHumanBodyPoseRequest` via a VisionCamera frame-processor
+ * plugin shipped in this package's `ios/` folder. Up to 19 anatomical
+ * joints (COCO-17 + `neck` + `root`). Primary fast path is
+ * `PoseCameraView` → worklet → native plugin (CVPixelBuffer, no RGB copy).
+ *
+ * Never loaded in Expo Go (`isExpoGo()` guard). On Android,
+ * `init()` throws {@link VisionUnavailableError} so the client can fall
+ * back to TFLite / TF.js when selection is `'auto'`.
+ */
+
+import { Platform } from 'react-native';
+
+import type { PoseBackend, PoseBackendInitOptions, PoseInputFrame } from '../PoseBackend';
+import type { Pose } from '../../types/pose';
+import type {
+  AccelerationDiagnostics,
+  AccelerationState,
+  DiagnosticListener,
+} from '../../types/acceleration';
+import { isExpoGo } from '../../support/optionalModules';
+import {
+  getVisionBodyPosePlugin,
+  isVisionBackendAvailable,
+  type VisionFrameProcessorPlugin,
+} from './optionalVision';
+import {
+  poseFromVisionPluginResult,
+  type VisionPluginResult,
+} from './mapVisionJoints';
+
+const DEFAULT_MAX_ACCEPTABLE_INFERENCE_MS = 50;
+
+/** Thrown when Apple Vision cannot run (wrong platform, Expo Go, missing pod). */
+export class VisionUnavailableError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = 'VisionUnavailableError';
+  }
+}
+
+export interface VisionPoseBackendOptions {
+  maxAcceptableInferenceMs?: number;
+  onDiagnostic?: DiagnosticListener;
+  onAccelerationChange?: (state: AccelerationState) => void;
+}
+
+/**
+ * Opaque frame accepted on the JS path: either a VisionCamera `Frame`
+ * (preferred — passed through to the plugin) or a pre-decoded
+ * {@link VisionPluginResult} (for tests / hosts that already ran Vision).
+ */
+export type VisionPoseInput = VisionPluginResult | { __visionCameraFrame: true };
+
+export class VisionPoseBackend implements PoseBackend {
+  readonly name = 'apple-vision';
+
+  private plugin: VisionFrameProcessorPlugin | null = null;
+  private warm = false;
+  private acceleration: AccelerationState = 'unknown';
+  private inferenceTimesMs: number[] = [];
+  private reasons: string[] = [];
+
+  private readonly maxAcceptableInferenceMs: number;
+  private readonly onDiagnostic: DiagnosticListener | undefined;
+  private readonly onAccelerationChange: ((state: AccelerationState) => void) | undefined;
+
+  constructor(private readonly options: VisionPoseBackendOptions = {}) {
+    this.maxAcceptableInferenceMs =
+      options.maxAcceptableInferenceMs ?? DEFAULT_MAX_ACCEPTABLE_INFERENCE_MS;
+    this.onDiagnostic = options.onDiagnostic;
+    this.onAccelerationChange = options.onAccelerationChange;
+  }
+
+  async init(_options: PoseBackendInitOptions): Promise<void> {
+    if (this.plugin) {
+      return;
+    }
+    if (Platform.OS !== 'ios') {
+      throw new VisionUnavailableError(
+        'Apple Vision body pose is iOS-only. Use preferredBackend "tflite" or "tfjs" on Android.',
+      );
+    }
+    if (isExpoGo()) {
+      throw new VisionUnavailableError(
+        'Apple Vision requires a native iOS build — not available in Expo Go.',
+      );
+    }
+    if (!isVisionBackendAvailable()) {
+      throw new VisionUnavailableError(
+        'Apple Vision frame-processor plugin is not linked. Rebuild with the SDK ios/ pod ' +
+          '(VisionCamera + PoseTrackerVision) and ensure react-native-vision-camera + worklets are installed.',
+      );
+    }
+    this.plugin = getVisionBodyPosePlugin();
+    if (!this.plugin) {
+      throw new VisionUnavailableError('Failed to initialize detectBodyPose frame-processor plugin.');
+    }
+    this.onDiagnostic?.(
+      '[posetracker] apple-vision backend ready (VNDetectHumanBodyPoseRequest, up to 19 joints)',
+    );
+  }
+
+  /**
+   * Vision has no separate model load — warm-up records the accelerator
+   * verdict. Real timings come from the first camera frames via
+   * {@link recordInferenceTime}.
+   */
+  async warmup(): Promise<void> {
+    this.requirePlugin();
+    if (this.warm) {
+      return;
+    }
+    // Neural Engine / GPU path — Vision does not expose a CPU-only mode.
+    this.setAcceleration('gpu');
+    this.warm = true;
+    this.onDiagnostic?.(
+      `[posetracker] apple-vision warm-up done: acceleration=${this.acceleration} ` +
+        '(per-frame ms reported from PoseCameraView)',
+    );
+  }
+
+  /**
+   * JS-path estimation. Accepts:
+   * - a VisionCamera `Frame` (has `.width`/`.height` and is callable via plugin);
+   * - a precomputed {@link VisionPluginResult} `{ joints }`.
+   *
+   * Prefer `PoseCameraView` (worklet) for production — this path is for
+   * TensorCamera-style hosts and unit tests.
+   */
+  async estimatePose(frame: PoseInputFrame): Promise<Pose | null> {
+    const plugin = this.requirePlugin();
+    const asResult = frame as VisionPluginResult;
+    if (asResult && Array.isArray(asResult.joints)) {
+      return poseFromVisionPluginResult(asResult);
+    }
+
+    // VisionCamera Frame — structural duck-type (worklet Frame has width/height).
+    const anyFrame = frame as { width?: number; height?: number };
+    if (
+      anyFrame &&
+      typeof anyFrame.width === 'number' &&
+      typeof anyFrame.height === 'number' &&
+      typeof (plugin as VisionFrameProcessorPlugin).call === 'function'
+    ) {
+      const start = Date.now();
+      const raw = plugin.call(frame) as VisionPluginResult | null;
+      const elapsed = Date.now() - start;
+      this.recordInferenceTime(elapsed);
+      return poseFromVisionPluginResult(raw ?? undefined);
+    }
+
+    throw new Error(
+      'VisionPoseBackend: pass a VisionCamera Frame, a { joints } plugin result, ' +
+        'or use PoseCameraView (worklet path).',
+    );
+  }
+
+  getAcceleration(): AccelerationDiagnostics | null {
+    if (!this.plugin && this.acceleration === 'unknown') {
+      return null;
+    }
+    return {
+      state: this.acceleration,
+      tfjsBackend: null,
+      runtime: 'vision',
+      delegate: 'apple-vision',
+      medianInferenceMs: median(this.inferenceTimesMs),
+      inferenceTimesMs: [...this.inferenceTimesMs],
+      maxAcceptableInferenceMs: this.maxAcceptableInferenceMs,
+      capabilities: null,
+      flags: {},
+      contextLossCount: 0,
+      reasons: [...this.reasons],
+    };
+  }
+
+  async dispose(): Promise<void> {
+    this.plugin = null;
+    this.warm = false;
+    this.acceleration = 'unknown';
+    this.inferenceTimesMs = [];
+    this.reasons = [];
+  }
+
+  /** Plugin handle for PoseCameraView worklets. */
+  getFrameProcessorPlugin(): VisionFrameProcessorPlugin | null {
+    return this.plugin;
+  }
+
+  /** Feed live inference timings from the camera worklet into diagnostics. */
+  recordInferenceTime(ms: number): void {
+    this.inferenceTimesMs.push(ms);
+    if (this.inferenceTimesMs.length > 30) {
+      this.inferenceTimesMs.shift();
+    }
+    const med = median(this.inferenceTimesMs);
+    if (med !== null && med > this.maxAcceptableInferenceMs && this.acceleration === 'gpu') {
+      this.noteReason(
+        `apple-vision median ${med.toFixed(0)} ms exceeds ${this.maxAcceptableInferenceMs} ms budget`,
+      );
+      this.setAcceleration('cpu-fallback');
+    }
+  }
+
+  private setAcceleration(state: AccelerationState): void {
+    if (this.acceleration !== state) {
+      this.acceleration = state;
+      this.onAccelerationChange?.(state);
+    }
+  }
+
+  private noteReason(reason: string): void {
+    this.reasons.push(reason);
+    this.onDiagnostic?.(`[posetracker] ${reason}`);
+  }
+
+  private requirePlugin(): VisionFrameProcessorPlugin {
+    if (!this.plugin) {
+      throw new Error('VisionPoseBackend not initialized — call init() first.');
+    }
+    return this.plugin;
+  }
+}
+
+function median(values: number[]): number | null {
+  if (values.length === 0) {
+    return null;
+  }
+  const sorted = [...values].sort((a, b) => a - b);
+  const mid = Math.floor(sorted.length / 2);
+  if (sorted.length % 2 === 0) {
+    return ((sorted[mid - 1] as number) + (sorted[mid] as number)) / 2;
+  }
+  return sorted[mid] as number;
+}

package/src/backends/vision/mapVisionJoints.ts

@@ -0,0 +1,185 @@
+/**
+ * Apple Vision body-pose joint → SDK keypoint mapping.
+ *
+ * `VNDetectHumanBodyPoseRequest` exposes 19 joints. Seventeen align 1:1 with
+ * the COCO / MoveNet topology; `neck` and `root` (mid-hip) are Vision extras
+ * kept under their Vision names so A/B overlays can show the full set.
+ *
+ * Vision normalized points use a **bottom-left** origin — the native plugin
+ * already flips Y (`y = 1 - visionY`) before returning joints here.
+ *
+ * `poseFromVisionPluginResult` carries a `'worklet'` directive so
+ * PoseCameraView can call it on the frame-processor thread.
+ */
+
+import type { Keypoint, KeypointName, Pose } from '../../types/pose';
+
+/** Vision `VNHumanBodyPoseObservation.JointName` camelCase → SDK name. */
+export const VISION_JOINT_TO_SDK: Readonly<Record<string, KeypointName>> = {
+  nose: 'nose',
+  leftEye: 'left_eye',
+  rightEye: 'right_eye',
+  leftEar: 'left_ear',
+  rightEar: 'right_ear',
+  leftShoulder: 'left_shoulder',
+  rightShoulder: 'right_shoulder',
+  leftElbow: 'left_elbow',
+  rightElbow: 'right_elbow',
+  leftWrist: 'left_wrist',
+  rightWrist: 'right_wrist',
+  leftHip: 'left_hip',
+  rightHip: 'right_hip',
+  leftKnee: 'left_knee',
+  rightKnee: 'right_knee',
+  leftAnkle: 'left_ankle',
+  rightAnkle: 'right_ankle',
+  neck: 'neck',
+  root: 'root',
+};
+
+/** One joint as returned by the native frame-processor plugin. */
+export interface VisionPluginJoint {
+  name: string;
+  x: number;
+  y: number;
+  score: number;
+}
+
+/** Payload of `detectBodyPose` frame-processor plugin (already Y-flipped). */
+export interface VisionPluginResult {
+  joints: VisionPluginJoint[];
+  /** Wall time of the Vision request on the frame-processor thread, ms. */
+  inferenceTimeMs?: number;
+}
+
+/**
+ * Build a SDK `Pose` from the native plugin result. Unknown joint names are
+ * dropped; missing joints are filled with score 0 so overlays stay
+ * topology-stable when a joint is absent.
+ */
+export function poseFromVisionPluginResult(
+  result: VisionPluginResult | null | undefined,
+  timestampMs: number = Date.now(),
+): Pose | null {
+  'worklet';
+  // Inlined — worklets cannot capture module-level imports reliably.
+  const ORDER: KeypointName[] = [
+    'nose',
+    'left_eye',
+    'right_eye',
+    'left_ear',
+    'right_ear',
+    'left_shoulder',
+    'right_shoulder',
+    'left_elbow',
+    'right_elbow',
+    'left_wrist',
+    'right_wrist',
+    'left_hip',
+    'right_hip',
+    'left_knee',
+    'right_knee',
+    'left_ankle',
+    'right_ankle',
+    'neck',
+    'root',
+  ];
+  const ALIAS: Record<string, KeypointName> = {
+    nose: 'nose',
+    leftEye: 'left_eye',
+    rightEye: 'right_eye',
+    leftEar: 'left_ear',
+    rightEar: 'right_ear',
+    leftShoulder: 'left_shoulder',
+    rightShoulder: 'right_shoulder',
+    leftElbow: 'left_elbow',
+    rightElbow: 'right_elbow',
+    leftWrist: 'left_wrist',
+    rightWrist: 'right_wrist',
+    leftHip: 'left_hip',
+    rightHip: 'right_hip',
+    leftKnee: 'left_knee',
+    rightKnee: 'right_knee',
+    leftAnkle: 'left_ankle',
+    rightAnkle: 'right_ankle',
+    neck: 'neck',
+    root: 'root',
+    left_eye: 'left_eye',
+    right_eye: 'right_eye',
+    left_ear: 'left_ear',
+    right_ear: 'right_ear',
+    left_shoulder: 'left_shoulder',
+    right_shoulder: 'right_shoulder',
+    left_elbow: 'left_elbow',
+    right_elbow: 'right_elbow',
+    left_wrist: 'left_wrist',
+    right_wrist: 'right_wrist',
+    left_hip: 'left_hip',
+    right_hip: 'right_hip',
+    left_knee: 'left_knee',
+    right_knee: 'right_knee',
+    left_ankle: 'left_ankle',
+    right_ankle: 'right_ankle',
+  };
+
+  if (!result || !result.joints || result.joints.length === 0) {
+    return null;
+  }
+
+  const byName: Partial<Record<KeypointName, Keypoint>> = {};
+  for (let i = 0; i < result.joints.length; i++) {
+    const joint = result.joints[i];
+    if (!joint) {
+      continue;
+    }
+    const mapped = ALIAS[joint.name];
+    if (!mapped) {
+      continue;
+    }
+    const x = clamp01(joint.x);
+    const y = clamp01(joint.y);
+    const score = clamp01(joint.score);
+    byName[mapped] = { name: mapped, x, y, score };
+  }
+
+  const keypoints: Keypoint[] = [];
+  let scoreSum = 0;
+  let scoreCount = 0;
+  for (let i = 0; i < ORDER.length; i++) {
+    const name = ORDER[i] as KeypointName;
+    const existing = byName[name];
+    if (existing) {
+      keypoints.push(existing);
+      if (existing.score > 0) {
+        scoreSum += existing.score;
+        scoreCount += 1;
+      }
+    } else {
+      keypoints.push({ name, x: 0, y: 0, score: 0 });
+    }
+  }
+
+  if (scoreCount === 0) {
+    return null;
+  }
+
+  return {
+    keypoints,
+    score: scoreSum / scoreCount,
+    timestampMs,
+  };
+}
+
+function clamp01(n: number): number {
+  'worklet';
+  if (!Number.isFinite(n)) {
+    return 0;
+  }
+  if (n < 0) {
+    return 0;
+  }
+  if (n > 1) {
+    return 1;
+  }
+  return n;
+}

package/src/backends/vision/optionalVision.ts

@@ -0,0 +1,94 @@
+/**
+ * Guarded access to the Apple Vision body-pose frame-processor plugin.
+ *
+ * The plugin lives in this package's `ios/` folder (VisionCamera Swift
+ * FrameProcessorPlugin + `VNDetectHumanBodyPoseRequest`). It is only linked
+ * in native iOS builds — never in Expo Go. All requires are gated by
+ * `Platform.OS === 'ios' && !isExpoGo()` so Metro/Expo Go never evaluate
+ * Nitro/VisionCamera code paths unintentionally.
+ */
+
+import { Platform } from 'react-native';
+
+import { getVisionCamera, getWorkletsCore, isExpoGo } from '../../support/optionalModules';
+
+/* eslint-disable @typescript-eslint/no-explicit-any */
+declare function require(name: string): unknown;
+
+/** JS name registered via `VISION_EXPORT_SWIFT_FRAME_PROCESSOR(..., detectBodyPose)`. */
+export const VISION_BODY_POSE_PLUGIN_NAME = 'detectBodyPose';
+
+export interface VisionFrameProcessorPlugin {
+  call(frame: unknown, args?: Record<string, unknown>): unknown;
+}
+
+interface VisionCameraProxyModule {
+  VisionCameraProxy: {
+    initFrameProcessorPlugin(
+      name: string,
+      options?: Record<string, unknown>,
+    ): VisionFrameProcessorPlugin | null | undefined;
+  };
+}
+
+let cachedPlugin: VisionFrameProcessorPlugin | null | undefined;
+
+/**
+ * True when we are on a native iOS build (not Expo Go) where the Vision
+ * frame-processor plugin can be loaded. Does not require Nitro / TFLite.
+ */
+export function isVisionBackendAvailable(): boolean {
+  if (Platform.OS !== 'ios' || isExpoGo()) {
+    return false;
+  }
+  if (getVisionCamera() === null || getWorkletsCore() === null) {
+    return false;
+  }
+  return getVisionBodyPosePlugin() !== null;
+}
+
+/**
+ * vision-camera + worklets present on iOS native build — enough to mount the
+ * Vision path of `PoseCameraView` once the backend has initialized.
+ */
+export function isVisionPoseCameraAvailable(): boolean {
+  return (
+    Platform.OS === 'ios' &&
+    !isExpoGo() &&
+    getVisionCamera() !== null &&
+    getWorkletsCore() !== null
+  );
+}
+
+/**
+ * Lazily init the VisionCamera frame-processor plugin. Cached; `null` when
+ * the native binary does not include our Swift plugin (Expo Go, Android,
+ * or a host that excluded the pod).
+ */
+export function getVisionBodyPosePlugin(): VisionFrameProcessorPlugin | null {
+  if (cachedPlugin !== undefined) {
+    return cachedPlugin;
+  }
+  if (Platform.OS !== 'ios' || isExpoGo()) {
+    cachedPlugin = null;
+    return cachedPlugin;
+  }
+  try {
+    const mod = require('react-native-vision-camera') as VisionCameraProxyModule;
+    const proxy = mod.VisionCameraProxy;
+    if (!proxy || typeof proxy.initFrameProcessorPlugin !== 'function') {
+      cachedPlugin = null;
+      return cachedPlugin;
+    }
+    const plugin = proxy.initFrameProcessorPlugin(VISION_BODY_POSE_PLUGIN_NAME, {});
+    cachedPlugin = plugin ?? null;
+  } catch {
+    cachedPlugin = null;
+  }
+  return cachedPlugin;
+}
+
+/** Test helper — clears the plugin cache between availability probes. */
+export function resetVisionPluginCacheForTests(): void {
+  cachedPlugin = undefined;
+}