@pose-tracker/react-native-pose-estimation-light 0.1.0

package/src/PoseTrackerProvider.tsx

@@ -0,0 +1,332 @@
+/**
+ * React layer: `PoseTrackerProvider` owns a `PoseTrackerClient`, and
+ * `usePoseTracker` gives screens access to the pipeline plus typed event
+ * callbacks.
+ *
+ * Recommended usage:
+ * - Mount the provider near the app root — with or without an API token
+ *   (without token, the SDK runs in keypoints-only mode).
+ * - **Mounting the provider does not load MoveNet** unless `autoPreload` is
+ *   true. On the screen *before* the camera (e.g. home), call `preload()` /
+ *   `warmup()` (default **basic** = model only, no camera permission). For
+ *   WebView, mount a 1×1 `<WebViewPoseView coldStart="basic" />` warmer so
+ *   MoveNet can load — see docs/PRELOAD.md.
+ * - Call `configure(apiToken)` at any time to upgrade keypoints-only →
+ *   full-engine without restarting the camera pipeline.
+ */
+
+import React, {
+  createContext,
+  useCallback,
+  useContext,
+  useEffect,
+  useMemo,
+  useRef,
+  useState,
+} from 'react';
+
+import { PoseTrackerClient, type PoseTrackerClientOptions, type StartExerciseOptions } from './client';
+import type { PoseInputFrame } from './backends/PoseBackend';
+import type { ExerciseConfig, SdkManifest } from './types/manifest';
+import type { PreloadOptions } from './types/preload';
+import type {
+  ErrorEvent,
+  PoseTrackerCallbacks,
+  PoseTrackerEvent,
+  PoseTrackerEventListener,
+  PoseTrackerMode,
+  PoseTrackerStatus,
+} from './types/events';
+import type { Pose } from './types/pose';
+import type { AccelerationDiagnostics, AccelerationState } from './types/acceleration';
+import type { QualityState } from './quality/AdaptiveQualityController';
+
+export interface PoseTrackerContextValue {
+  /**
+   * The underlying orchestrator. Escape hatch for advanced integrations
+   * (e.g. `PoseCameraView` reads the active backend and ingests poses
+   * computed in the vision-camera worklet). Stable across renders.
+   */
+  client: PoseTrackerClient;
+  status: PoseTrackerStatus;
+  /** 'keypoints-only' (no validated key/session) or 'full-engine'. */
+  mode: PoseTrackerMode;
+  /**
+   * GPU-acceleration verdict from the warm-up health check: 'unknown' until
+   * warm-up completes, then 'gpu' | 'cpu-fallback' | 'unavailable'. Can
+   * downgrade mid-session after an Android GL context loss. See
+   * docs/ANDROID_GL_ACCELERATION.md.
+   */
+  acceleration: AccelerationState;
+  /** Full GPU diagnostics (backend, timings, GL renderer, flags, downgrade trail). */
+  accelerationDiagnostics: AccelerationDiagnostics | null;
+  /**
+   * Adaptive camera-quality profile (AdaptiveChoice + crash-guard + FPS
+   * auto-downgrade). `capturePriority` reflects the host option (`performance`
+   * default vs `quality`). See docs/ADAPTIVE_QUALITY.md.
+   */
+  quality: QualityState;
+  /** Last (possibly non-fatal) error; `ready` + keypoints-only can coexist with it. */
+  error: ErrorEvent | null;
+  manifest: SdkManifest | null;
+  /** Empty in keypoints-only mode. */
+  exercises: ExerciseConfig[];
+  /**
+   * Warm-up (handshake + engine + model). Default `coldStart: 'basic'` —
+   * no getUserMedia. Pass `{ coldStart: 'full' }` to also open the camera.
+   */
+  preload: (options?: PreloadOptions) => Promise<void>;
+  /** Alias of preload(). */
+  warmup: (options?: PreloadOptions) => Promise<void>;
+  /**
+   * Hot (re)configuration: validates the token and loads the engine without
+   * touching the camera pipeline. Resolves to true on full-engine mode.
+   */
+  configure: (apiToken?: string) => Promise<boolean>;
+  /**
+   * Start an FSM exercise (manifest) or a custom one (`jump_analysis`,
+   * `air_time_jump`). Options: `difficulty`, `userHeightCm`, `devicePitchDeg`.
+   */
+  startExercise: (exerciseId: string, options?: StartExerciseOptions) => void;
+  stopExercise: () => void;
+  estimatePose: (frame: PoseInputFrame) => Promise<Pose | null>;
+  processFrame: (frame: PoseInputFrame) => Promise<Pose | null>;
+  addEventListener: (listener: PoseTrackerEventListener) => () => void;
+  /**
+   * Classic PoseTracker WebView JSON stream (`sendDataToNative` shape).
+   * Same as `usePoseTracker({ onMessage })`.
+   */
+  addMessageListener: (
+    listener: (message: import('./events/classicMessage').ClassicNativeMessage) => void,
+  ) => () => void;
+}
+
+const PoseTrackerContext = createContext<PoseTrackerContextValue | null>(null);
+
+export interface PoseTrackerProviderProps {
+  /** Optional: without a token the SDK runs in keypoints-only mode. */
+  apiToken?: string;
+  options?: PoseTrackerClientOptions;
+  /** Start preloading as soon as the provider mounts. Default: false. */
+  autoPreload?: boolean;
+  children: React.ReactNode;
+}
+
+export function PoseTrackerProvider({
+  apiToken,
+  options,
+  autoPreload = false,
+  children,
+}: PoseTrackerProviderProps): React.JSX.Element {
+  const clientRef = useRef<PoseTrackerClient | null>(null);
+  if (!clientRef.current) {
+    clientRef.current = new PoseTrackerClient(apiToken, options);
+  }
+  const client = clientRef.current;
+
+  const [status, setStatus] = useState<PoseTrackerStatus>(client.getStatus());
+  const [mode, setMode] = useState<PoseTrackerMode>(client.getMode());
+  const [error, setError] = useState<ErrorEvent | null>(null);
+  const [manifest, setManifest] = useState<SdkManifest | null>(null);
+  const [exercises, setExercises] = useState<ExerciseConfig[]>([]);
+  const [acceleration, setAcceleration] = useState<AccelerationState>(client.getAcceleration());
+  const [accelerationDiagnostics, setAccelerationDiagnostics] =
+    useState<AccelerationDiagnostics | null>(client.getAccelerationDiagnostics());
+  const [quality, setQuality] = useState<QualityState>(client.getQualityState());
+
+  useEffect(() => {
+    const offState = client.onStateChange(() => {
+      setStatus(client.getStatus());
+      setMode(client.getMode());
+      setError(client.getError());
+      setManifest(client.getManifest());
+      setExercises(client.getAvailableExercises());
+      setAcceleration(client.getAcceleration());
+      setAccelerationDiagnostics(client.getAccelerationDiagnostics());
+      setQuality(client.getQualityState());
+    });
+    if (autoPreload) {
+      client.preload().catch(() => {
+        // Error is surfaced through status/error state and the error event.
+      });
+    }
+    return () => {
+      offState();
+    };
+  }, [client, autoPreload]);
+
+  useEffect(() => {
+    return () => {
+      client.dispose().catch(() => {});
+    };
+  }, [client]);
+
+  const preload = useCallback(
+    (options?: PreloadOptions) => client.preload(options),
+    [client],
+  );
+  const configureFn = useCallback((token?: string) => client.configure(token), [client]);
+  const startExercise = useCallback(
+    (id: string, exerciseOptions?: StartExerciseOptions) => client.startExercise(id, exerciseOptions),
+    [client],
+  );
+  const stopExercise = useCallback(() => client.stopExercise(), [client]);
+  const estimatePose = useCallback(
+    (frame: PoseInputFrame) => client.estimatePose(frame),
+    [client],
+  );
+  const processFrame = useCallback(
+    (frame: PoseInputFrame) => client.processFrame(frame),
+    [client],
+  );
+  const addEventListener = useCallback(
+    (listener: PoseTrackerEventListener) => client.addEventListener(listener),
+    [client],
+  );
+  const addMessageListener = useCallback(
+    (listener: (message: import('./events/classicMessage').ClassicNativeMessage) => void) =>
+      client.addMessageListener(listener),
+    [client],
+  );
+
+  const value = useMemo<PoseTrackerContextValue>(
+    () => ({
+      client,
+      status,
+      mode,
+      acceleration,
+      accelerationDiagnostics,
+      quality,
+      error,
+      manifest,
+      exercises,
+      preload,
+      warmup: preload,
+      configure: configureFn,
+      startExercise,
+      stopExercise,
+      estimatePose,
+      processFrame,
+      addEventListener,
+      addMessageListener,
+    }),
+    [
+      client,
+      status,
+      mode,
+      acceleration,
+      accelerationDiagnostics,
+      quality,
+      error,
+      manifest,
+      exercises,
+      preload,
+      configureFn,
+      startExercise,
+      stopExercise,
+      estimatePose,
+      processFrame,
+      addEventListener,
+      addMessageListener,
+    ],
+  );
+
+  return <PoseTrackerContext.Provider value={value}>{children}</PoseTrackerContext.Provider>;
+}
+
+/**
+ * Access the PoseTracker pipeline and subscribe to typed events.
+ *
+ * ```tsx
+ * const { status, mode, processFrame, configure } = usePoseTracker({
+ *   onKeypoints: (e) => drawSkeleton(e.keypoints),   // both modes
+ *   onCounter: (e) => setReps(e.count),              // full-engine only
+ * });
+ * ```
+ */
+export function usePoseTracker(callbacks: PoseTrackerCallbacks = {}): PoseTrackerContextValue {
+  const context = useContext(PoseTrackerContext);
+  if (!context) {
+    throw new Error('usePoseTracker must be used within a <PoseTrackerProvider>.');
+  }
+
+  const callbacksRef = useRef(callbacks);
+  callbacksRef.current = callbacks;
+
+  useEffect(() => {
+    const offEvents = context.addEventListener((event: PoseTrackerEvent) => {
+      const cb = callbacksRef.current;
+      switch (event.type) {
+        case 'initialization':
+          cb.onInitialization?.(event);
+          break;
+        case 'error':
+          cb.onError?.(event);
+          break;
+        case 'warning':
+          cb.onWarning?.(event);
+          break;
+        case 'keypoints':
+          cb.onKeypoints?.(event);
+          break;
+        case 'angles':
+          cb.onAngles?.(event);
+          break;
+        case 'counter':
+          cb.onCounter?.(event);
+          break;
+        case 'posture':
+          cb.onPosture?.(event);
+          break;
+        case 'progression':
+          cb.onProgression?.(event);
+          break;
+        case 'recommendations':
+          cb.onRecommendations?.(event);
+          break;
+        case 'form_score':
+          cb.onFormScore?.(event);
+          break;
+        case 'exercise_summary':
+          cb.onExerciseSummary?.(event);
+          break;
+        case 'jump_calibration':
+          cb.onJumpCalibration?.(event);
+          break;
+        case 'jump_started':
+          cb.onJumpStarted?.(event);
+          break;
+        case 'jump_height':
+          cb.onJumpHeight?.(event);
+          break;
+        case 'jump_discarded':
+          cb.onJumpDiscarded?.(event);
+          break;
+        case 'jump_result':
+          cb.onJumpResult?.(event);
+          break;
+        case 'jump_summary':
+          cb.onJumpSummary?.(event);
+          break;
+        case 'quality_changed':
+          cb.onQualityChanged?.(event);
+          break;
+        case 'performance_warning':
+          cb.onPerformanceWarning?.(event);
+          break;
+        case 'runtime_download_progress':
+          cb.onRuntimeDownloadProgress?.(event);
+          break;
+      }
+    });
+    const offMessages = context.addMessageListener((message) => {
+      callbacksRef.current.onMessage?.(message);
+    });
+    return () => {
+      offEvents();
+      offMessages();
+    };
+  }, [context]);
+
+  return context;
+}

package/src/api/configure.ts

@@ -0,0 +1,82 @@
+/**
+ * Handshake client: `configure(apiToken)` calls the Strapi backend, which
+ * validates the API token (plan/quota) and returns the `SdkManifest` —
+ * exercise configs, engine bundle descriptor, pose-runtime payload
+ * descriptor, model profiles, reference movements. All server-driven
+ * behavior flows from this single call.
+ *
+ * Keyless mode (`apiToken` null): the backend returns the PUBLIC manifest —
+ * pose-runtime descriptor only, no session/engine/exercises. This is how
+ * keypoints-only installs bootstrap the pose estimation payload.
+ */
+
+import { Platform } from 'react-native';
+
+import { SDK_VERSION } from '../sdkVersion';
+import type { ConfigureRequest, PoseModelProfile, SdkManifest } from '../types/manifest';
+
+export const SDK_NAME = 'posetracker-rn-light' as const;
+export { SDK_VERSION };
+
+/** Overridable for staging/self-hosted backends. */
+export const DEFAULT_BASE_URL =
+  'https://movelytics-strapi-c78a339b7070.herokuapp.com';
+
+export interface ConfigureOptions {
+  baseUrl?: string;
+  poseModelProfile?: PoseModelProfile;
+  locale?: string;
+  /** Local cache versions, so the backend can answer `upToDate` per module. */
+  localVersions?: ConfigureRequest['localVersions'];
+}
+
+export class ConfigureError extends Error {
+  constructor(
+    readonly code: 'invalid_token' | 'quota_exceeded' | 'network' | 'internal',
+    message: string,
+  ) {
+    super(message);
+    this.name = 'ConfigureError';
+  }
+}
+
+export async function configure(
+  apiToken: string | null,
+  options: ConfigureOptions = {},
+): Promise<SdkManifest> {
+  const baseUrl = (options.baseUrl ?? DEFAULT_BASE_URL).replace(/\/$/, '');
+  const body: ConfigureRequest = {
+    ...(apiToken ? { apiToken } : {}),
+    sdkName: SDK_NAME,
+    sdkVersion: SDK_VERSION,
+    targetPlatform: Platform.OS === 'ios' ? 'ios' : 'android',
+    poseModelProfile: options.poseModelProfile ?? 'AdaptiveChoice',
+    locale: options.locale,
+    localVersions: options.localVersions,
+  };
+
+  let response: Response;
+  try {
+    response = await fetch(`${baseUrl}/api/sdk/configure`, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify(body),
+    });
+  } catch (err) {
+    throw new ConfigureError('network', `Handshake request failed: ${String(err)}`);
+  }
+
+  if (response.status === 401 || response.status === 403) {
+    // The 401 body carries `revoked: true` — the caller purges the sealed
+    // engine/session caches (a revoked key must not keep the engine alive).
+    throw new ConfigureError('invalid_token', 'Invalid, revoked or unauthorized API token.');
+  }
+  if (response.status === 429) {
+    throw new ConfigureError('quota_exceeded', 'API call quota exceeded for the current plan.');
+  }
+  if (!response.ok) {
+    throw new ConfigureError('internal', `Handshake failed with HTTP ${response.status}.`);
+  }
+
+  return (await response.json()) as SdkManifest;
+}

package/src/api/skeleton.ts

@@ -0,0 +1,60 @@
+/**
+ * Fetch a custom skeleton overlay by Strapi `api_uuid`
+ * (WebView `?skeleton=<uuid>` parity).
+ */
+
+import type { SkeletonDefinition } from '../types/skeleton';
+import { DEFAULT_BASE_URL } from './configure';
+
+export interface FetchSkeletonOptions {
+  baseUrl?: string;
+}
+
+export class SkeletonFetchError extends Error {
+  constructor(
+    readonly code: 'not_found' | 'network' | 'invalid',
+    message: string,
+  ) {
+    super(message);
+    this.name = 'SkeletonFetchError';
+  }
+}
+
+/**
+ * GET /api/sdk/skeleton?uuid=…
+ * Returns the overlay definition or throws {@link SkeletonFetchError}.
+ */
+export async function fetchSkeletonDefinition(
+  uuid: string,
+  options: FetchSkeletonOptions = {},
+): Promise<SkeletonDefinition> {
+  const id = uuid.trim();
+  if (!id || id === 'true' || id === 'false') {
+    throw new SkeletonFetchError('invalid', 'skeleton uuid is required');
+  }
+  const baseUrl = (options.baseUrl ?? DEFAULT_BASE_URL).replace(/\/$/, '');
+  let response: Response;
+  try {
+    response = await fetch(`${baseUrl}/api/sdk/skeleton?uuid=${encodeURIComponent(id)}`);
+  } catch (err) {
+    throw new SkeletonFetchError('network', `Skeleton fetch failed: ${String(err)}`);
+  }
+  if (response.status === 404) {
+    throw new SkeletonFetchError('not_found', `Skeleton '${id}' not found`);
+  }
+  if (!response.ok) {
+    throw new SkeletonFetchError('network', `Skeleton fetch HTTP ${response.status}`);
+  }
+  const body = (await response.json()) as SkeletonDefinition & { api_uuid?: string };
+  if (!body?.keypoints || !body?.keypoint_lines || !body?.circles || !body?.lines) {
+    throw new SkeletonFetchError('invalid', 'Skeleton payload incomplete');
+  }
+  return {
+    keypoints: body.keypoints,
+    keypoint_lines: body.keypoint_lines,
+    keypoint_angles: body.keypoint_angles ?? [],
+    circles: body.circles,
+    lines: body.lines,
+    angles: body.angles,
+  };
+}

package/src/api/track.ts

@@ -0,0 +1,182 @@
+/**
+ * Usage tracking client — `POST /api/sdk/track`.
+ *
+ * One call per real usage: when the camera starts with the model ready
+ * (`camera_start`), NOT on handshake and NOT on preload. Server-side, a
+ * valid API key increments the same usage counters as the WebView API
+ * (quota-checked) and inserts a `sdk-usage-event` row with the params;
+ * keyless calls insert an anonymous row.
+ *
+ * Delivery rules (see ARCHITECTURE.md §Offline):
+ * - metered (with API key): the track call MUST succeed online — offline
+ *   metered sessions are refused because usage cannot be counted;
+ * - anonymous (no key): fire-and-forget with a local retry queue, flushed
+ *   on the next connection.
+ */
+
+import { Platform } from 'react-native';
+
+import { DEFAULT_BASE_URL, SDK_VERSION } from './configure';
+import { createAsyncKeyValueStore, type KeyValueStore } from '../engine/EngineLoader';
+
+const QUEUE_KEY = 'posetracker.track.queue';
+const QUEUE_LIMIT = 100;
+
+export class TrackError extends Error {
+  constructor(
+    readonly code: 'invalid_token' | 'quota_exceeded' | 'network' | 'internal',
+    message: string,
+  ) {
+    super(message);
+    this.name = 'TrackError';
+  }
+}
+
+export interface TrackRequest {
+  event?: string;
+  apiToken?: string | null;
+  params?: Record<string, unknown>;
+}
+
+export interface TrackResponse {
+  tracked: boolean;
+  anonymous?: boolean;
+  monthly_usage_counter?: number;
+  remainingCalls?: number;
+  type?: string;
+}
+
+interface QueuedEvent {
+  event: string;
+  platform: string;
+  sdkVersion: string;
+  params: Record<string, unknown> | undefined;
+  queuedAtMs: number;
+}
+
+export interface UsageTrackerOptions {
+  baseUrl?: string;
+  keyValueStore?: KeyValueStore | null;
+  fetchFn?: typeof fetch;
+}
+
+export class UsageTracker {
+  private readonly baseUrl: string;
+  private readonly kv: KeyValueStore | null;
+  private readonly fetchFn: typeof fetch;
+
+  constructor(options: UsageTrackerOptions = {}) {
+    this.baseUrl = (options.baseUrl ?? DEFAULT_BASE_URL).replace(/\/$/, '');
+    this.kv =
+      options.keyValueStore !== undefined ? options.keyValueStore : createAsyncKeyValueStore();
+    this.fetchFn = options.fetchFn ?? fetch;
+  }
+
+  /**
+   * Metered track (API key): throws on failure — the caller refuses to start
+   * a metered session it cannot count.
+   */
+  async trackMetered(request: TrackRequest & { apiToken: string }): Promise<TrackResponse> {
+    return this.post({
+      event: request.event ?? 'camera_start',
+      apiToken: request.apiToken,
+      platform: Platform.OS,
+      sdkVersion: SDK_VERSION,
+      params: request.params,
+    });
+  }
+
+  /**
+   * Anonymous track (no key): never throws. Failed sends are queued locally
+   * and flushed on the next connection.
+   */
+  async trackAnonymous(request: TrackRequest = {}): Promise<void> {
+    const payload: QueuedEvent = {
+      event: request.event ?? 'camera_start',
+      platform: Platform.OS,
+      sdkVersion: SDK_VERSION,
+      params: request.params,
+      queuedAtMs: Date.now(),
+    };
+    try {
+      await this.post({ ...payload });
+      // A send just worked: opportunistically drain older queued events.
+      void this.flushQueue();
+    } catch {
+      await this.enqueue(payload);
+    }
+  }
+
+  /** Send queued anonymous events (called on preload and after online sends). */
+  async flushQueue(): Promise<void> {
+    const queue = await this.readQueue();
+    if (queue.length === 0) {
+      return;
+    }
+    const remaining: QueuedEvent[] = [];
+    for (const item of queue) {
+      try {
+        await this.post({ ...item, params: { ...(item.params ?? {}), queuedAtMs: item.queuedAtMs } });
+      } catch {
+        remaining.push(item);
+      }
+    }
+    await this.writeQueue(remaining);
+  }
+
+  // -------------------------------------------------------------------------
+
+  private async post(body: Record<string, unknown>): Promise<TrackResponse> {
+    let response: Response;
+    try {
+      response = await this.fetchFn(`${this.baseUrl}/api/sdk/track`, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify(body),
+      });
+    } catch (err) {
+      throw new TrackError('network', `Usage tracking request failed: ${String(err)}`);
+    }
+    if (response.status === 401 || response.status === 403) {
+      throw new TrackError('invalid_token', 'Invalid, revoked or unauthorized API token.');
+    }
+    if (response.status === 429) {
+      throw new TrackError('quota_exceeded', 'API call quota exceeded for the current plan.');
+    }
+    if (!response.ok) {
+      throw new TrackError('internal', `Usage tracking failed with HTTP ${response.status}.`);
+    }
+    return (await response.json()) as TrackResponse;
+  }
+
+  private async readQueue(): Promise<QueuedEvent[]> {
+    try {
+      const raw = await this.kv?.getItem(QUEUE_KEY);
+      if (!raw) {
+        return [];
+      }
+      const parsed = JSON.parse(raw) as QueuedEvent[];
+      return Array.isArray(parsed) ? parsed : [];
+    } catch {
+      return [];
+    }
+  }
+
+  private async writeQueue(queue: QueuedEvent[]): Promise<void> {
+    try {
+      if (queue.length === 0) {
+        await this.kv?.removeItem(QUEUE_KEY);
+      } else {
+        await this.kv?.setItem(QUEUE_KEY, JSON.stringify(queue.slice(-QUEUE_LIMIT)));
+      }
+    } catch {
+      // Queue persistence is best-effort.
+    }
+  }
+
+  private async enqueue(item: QueuedEvent): Promise<void> {
+    const queue = await this.readQueue();
+    queue.push(item);
+    await this.writeQueue(queue);
+  }
+}