@pose-tracker/react-native-pose-estimation-light 0.1.0

package/src/events/classicMessage.ts

@@ -0,0 +1,271 @@
+/**
+ * Classic PoseTracker WebView → native message parity.
+ *
+ * Host apps that already parse `sendDataToNative` JSON
+ * (`PoseTrackerFront/lib/v3/sendDataToNativeContract.js`) can subscribe via
+ * `addMessageListener` / `usePoseTracker({ onMessage })` and keep the same
+ * `type` + payload field names (`data`, `current_count`, `message`, …).
+ *
+ * Typed {@link PoseTrackerEvent} remains the preferred API for new apps.
+ */
+
+import type { AngleValue, PoseTrackerEvent } from '../types/events';
+
+/** Loose JSON envelope matching the frozen WebView → native contract. */
+export type ClassicNativeMessage = {
+  type: string;
+  [key: string]: unknown;
+};
+
+export type ClassicMessageListener = (message: ClassicNativeMessage) => void;
+
+/** Relation keys matching engine `PUBLIC_ANGLES` / Front `computeAngles`. */
+const ANGLE_RELATIONS: Record<string, string> = {
+  left_knee: 'from_hip_to_ankle',
+  right_knee: 'from_hip_to_ankle',
+  left_hip: 'from_knee_to_shoulder',
+  right_hip: 'from_knee_to_shoulder',
+  left_shoulder: 'from_hip_to_elbow',
+  right_shoulder: 'from_hip_to_elbow',
+  left_elbow: 'from_shoulder_to_wrist',
+  right_elbow: 'from_shoulder_to_wrist',
+};
+
+/**
+ * Rebuild the WebView `angles.data` tree (`left_side` / `right_side`) from
+ * the typed flat `AngleValue[]` list.
+ */
+export function anglesToClassicTree(angles: AngleValue[]): {
+  left_side: Record<string, Record<string, number>>;
+  right_side: Record<string, Record<string, number>>;
+} {
+  const data = {
+    left_side: {} as Record<string, Record<string, number>>,
+    right_side: {} as Record<string, Record<string, number>>,
+  };
+  for (const a of angles) {
+    if (a.side !== 'left' && a.side !== 'right') continue;
+    const side = a.side === 'left' ? data.left_side : data.right_side;
+    const joint = a.id.replace(/^(left_|right_)/, '');
+    const angleType = `${joint}_angle`;
+    const relation = ANGLE_RELATIONS[a.id] ?? 'value';
+    if (!side[angleType]) side[angleType] = {};
+    side[angleType]![relation] = a.degrees;
+    side[angleType]![`score_${relation}`] = a.score;
+  }
+  return data;
+}
+
+/**
+ * Map a typed SDK event to a classic PoseTracker `sendDataToNative` payload.
+ * Returns null for SDK-only events that have no classic equivalent and should
+ * not be mirrored (none today — quality/performance map to error/warning).
+ */
+export function toClassicNativeMessage(event: PoseTrackerEvent): ClassicNativeMessage {
+  switch (event.type) {
+    case 'initialization':
+      return {
+        type: 'initialization',
+        // WebView-literal parity: TrackingAppV3 emits exactly
+        // "checking you plan and access" (sic) while the plan loads — hosts
+        // migrating from the WebView may string-match it.
+        message:
+          event.step === 'configuring'
+            ? 'checking you plan and access'
+            : event.message ?? humanInitStep(event.step),
+        ready: event.ready,
+        step: event.step,
+        ...(event.mode != null ? { mode: event.mode } : {}),
+        ...(event.acceleration != null ? { acceleration: event.acceleration } : {}),
+      };
+    case 'error':
+      return {
+        type: 'error',
+        error: event.code,
+        message: event.message,
+      };
+    case 'warning':
+      return {
+        type: 'warning',
+        error: event.code,
+        message: event.message,
+      };
+    case 'keypoints':
+      return {
+        type: 'keypoints',
+        data: event.keypoints.map((k) => ({
+          name: k.name,
+          x: k.x,
+          y: k.y,
+          ...(k.z != null ? { z: k.z } : {}),
+          score: k.score,
+        })),
+        score: event.score,
+        timestampMs: event.timestampMs,
+      };
+    case 'angles':
+      return {
+        type: 'angles',
+        // WebView V3 shape: nested left_side / right_side tree (not the typed flat array).
+        data: anglesToClassicTree(event.angles),
+        timestampMs: event.timestampMs,
+      };
+    case 'counter':
+      return {
+        type: 'counter',
+        current_count: event.count,
+        // GitBook: authoritative grade for a counted rep lives on counter.form_score.
+        ...(event.formScore != null
+          ? {
+              form_score: {
+                score: event.formScore.score,
+                avg_score: event.formScore.average,
+                grade: event.formScore.grade,
+              },
+            }
+          : {}),
+        ...(event.referenceScore != null
+          ? { reference_score: { overallScore: event.referenceScore } }
+          : {}),
+        timestampMs: event.timestampMs,
+      };
+    case 'posture':
+      return {
+        type: 'posture',
+        message: event.hint,
+        ready: event.ready,
+        direction: event.direction ?? '',
+        requirements: { missingKeypoints: event.missingKeypoints },
+        timestampMs: event.timestampMs,
+      };
+    case 'progression':
+      return {
+        type: 'progression',
+        value: event.value,
+        timestampMs: event.timestampMs,
+      };
+    case 'recommendations':
+      return {
+        type: 'recommendations',
+        data: event.recommendations,
+        timestampMs: event.timestampMs,
+      };
+    case 'form_score':
+      // Optional live / convenience stream — do not treat as authoritative
+      // for the counted rep (prefer counter.form_score). Kept for hosts that
+      // already subscribe to a standalone form_score type.
+      return {
+        type: 'form_score',
+        score: event.score,
+        avg_score: event.average,
+        grade: event.grade,
+        timestampMs: event.timestampMs,
+      };
+    case 'exercise_summary':
+      return {
+        type: 'exercise_summary',
+        exercise: event.exercise,
+        counter: event.counter,
+        avg_rep_score: event.averageFormScore,
+        avg_similarity: event.averageSimilarity,
+        grade: event.grade,
+        history: event.history,
+        durationMs: event.durationMs,
+        timestampMs: event.timestampMs,
+      };
+    // Custom jump exercises: the classic contract already uses these exact
+    // type strings and camelCase field names — pass through minus timestampMs
+    // restructuring (front payloads have no timestampMs, but extra fields are
+    // harmless for string-matching hosts).
+    case 'jump_calibration':
+      return {
+        type: 'jump_calibration',
+        calibrated: event.calibrated,
+        cmPerPixel: event.cmPerPixel,
+        baselineY: event.baselineY,
+        visibleHips: event.visibleHips,
+        message: event.message,
+        timestampMs: event.timestampMs,
+      };
+    case 'jump_started':
+      return {
+        type: 'jump_started',
+        message: event.message,
+        timestampMs: event.timestampMs,
+      };
+    case 'jump_height': {
+      const { type, timestampMs, ...rest } = event;
+      return { type, ...rest, timestampMs };
+    }
+    case 'jump_discarded':
+      return {
+        type: 'jump_discarded',
+        reason: event.reason,
+        userMessage: event.userMessage,
+        timestampMs: event.timestampMs,
+      };
+    case 'jump_result': {
+      const { type, timestampMs, ...rest } = event;
+      return { type, ...rest, timestampMs };
+    }
+    case 'jump_summary': {
+      const { type, timestampMs, ...rest } = event;
+      return { type, ...rest, timestampMs };
+    }
+    case 'performance_warning':
+      // Classic V3 uses type "error" with degradation codes for device capability.
+      return {
+        type: 'error',
+        error: event.code,
+        message: event.message,
+        meanFps: event.meanFps,
+        thresholdFps: event.thresholdFps,
+        activeProfile: event.activeProfile,
+        medianInferenceMs: event.medianInferenceMs,
+        videoSize: event.videoSize,
+        timestampMs: event.timestampMs,
+      };
+    case 'quality_changed':
+      return {
+        type: 'warning',
+        error: 'quality_downgraded',
+        message: event.detail,
+        previousProfile: event.previousProfile,
+        activeProfile: event.activeProfile,
+        reason: event.reason,
+        timestampMs: event.timestampMs,
+      };
+    case 'runtime_download_progress':
+      // Classic contract has no download event — mirror as initialization.
+      return {
+        type: 'initialization',
+        message: `downloading pose runtime ${event.completedParts + 1}/${event.totalParts}`,
+        ready: false,
+        step: 'downloading',
+        part: event.part,
+        completedParts: event.completedParts,
+        totalParts: event.totalParts,
+        timestampMs: event.timestampMs,
+      };
+    default: {
+      const _exhaustive: never = event;
+      return { type: 'unknown', event: _exhaustive };
+    }
+  }
+}
+
+function humanInitStep(step: string): string {
+  switch (step) {
+    case 'configuring':
+      // Front literal (sic) — TrackingAppV3.js.
+      return 'checking you plan and access';
+    case 'downloading':
+      return 'downloading engine';
+    case 'warming':
+      return 'loading pose model';
+    case 'ready':
+      return 'running';
+    default:
+      return step;
+  }
+}

package/src/exercises/aliases.ts

@@ -0,0 +1,50 @@
+/**
+ * V2 Strapi movement names → canonical V3 exercise keys.
+ * Mirrors PoseTrackerFront/lib/v3/exercises/index.js `EXERCISE_ALIASES`.
+ *
+ * The RN SDK manifest still often ships legacy `face_*` ids from Strapi while
+ * host apps / demos call V3 keys (`squat`, `push_up`, …).
+ */
+
+export const EXERCISE_ALIASES: Readonly<Record<string, string>> = {
+  face_squat: 'squat',
+  face_pushup: 'push_up',
+  face_plank: 'plank',
+  face_lunge: 'lunge',
+  face_jumping_jack: 'jumping_jack',
+  face_low_impact_jack: 'low_impact_jack',
+  face_balance_leg: 'balance_leg',
+  face_balance_leg_left: 'balance_leg_left',
+  face_balance_leg_right: 'balance_leg_right',
+  jump: 'air_time_jump',
+};
+
+/** Resolve a legacy or V3 id to the canonical V3 key. */
+export function resolveExerciseName(name: string): string {
+  return EXERCISE_ALIASES[name] ?? name;
+}
+
+/**
+ * Pick the best matching exercise config for a requested id.
+ * Tries exact id, then any legacy alias that maps to the same V3 key,
+ * then any config whose id resolves to the same V3 key.
+ */
+export function findExerciseByIdOrAlias<T extends { id: string }>(
+  exerciseId: string,
+  exercises: readonly T[],
+): T | undefined {
+  const exact = exercises.find((e) => e.id === exerciseId);
+  if (exact) return exact;
+
+  const canonical = resolveExerciseName(exerciseId);
+  const legacyIds = Object.entries(EXERCISE_ALIASES)
+    .filter(([, v3]) => v3 === canonical)
+    .map(([legacy]) => legacy);
+
+  for (const legacy of legacyIds) {
+    const hit = exercises.find((e) => e.id === legacy);
+    if (hit) return hit;
+  }
+
+  return exercises.find((e) => resolveExerciseName(e.id) === canonical);
+}

package/src/index.ts

@@ -0,0 +1,243 @@
+/**
+ * @pose-tracker/react-native-pose-estimation-light — public API surface.
+ *
+ * Light / fully online variant: TF.js loads from CDN and MoveNet (default)
+ * from a remote URL each WebView boot. The movement engine is still
+ * downloaded from the PoseTracker API after an authenticated handshake.
+ *
+ * Runtime model:
+ * - Default (both platforms, incl. Expo Go): MoveNet SinglePose Lightning
+ *   (17 keypoints, 192×192) inside a Chromium/WKWebView — TF.js WebGL.
+ *   Mount {@link WebViewPoseView}. Requires network for TF.js + model.
+ * - Opt-in (`preferredBackend: 'vision'`, iOS native builds): Apple Vision
+ *   `VNDetectHumanBodyPoseRequest` via {@link PoseCameraView}.
+ */
+
+// Handshake + usage tracking
+export { configure, ConfigureError, DEFAULT_BASE_URL, SDK_NAME, SDK_VERSION } from './api/configure';
+export type { ConfigureOptions } from './api/configure';
+export {
+  EXERCISE_ALIASES,
+  findExerciseByIdOrAlias,
+  resolveExerciseName,
+} from './exercises/aliases';
+export { UsageTracker, TrackError } from './api/track';
+export type { TrackRequest, TrackResponse, UsageTrackerOptions } from './api/track';
+
+// Online pose-runtime
+export { RuntimeCache } from './runtime/RuntimeCache';
+export type {
+  PoseRuntimeParts,
+  RuntimeCacheOptions,
+  RuntimeCacheProgress,
+} from './runtime/RuntimeCache';
+export {
+  getOnlineRuntimeParts,
+  getOnlineRuntimeVersion,
+  defaultOnlineRuntimeUrls,
+  ONLINE_TFJS_VERSION,
+} from './backends/webview/onlineRuntime';
+export type {
+  GetOnlineRuntimeOptions,
+  OnlineRuntimeParts,
+  OnlineRuntimeUrls,
+} from './backends/webview/onlineRuntime';
+export {
+  DEFAULT_MOVENET_LIGHTNING_URL,
+  originFromModelUrl,
+  resolvePoseModel,
+} from './models/poseModels';
+export type {
+  PoseModelAlias,
+  ResolvePoseModelOptions,
+  ResolvedPoseModel,
+} from './models/poseModels';
+
+// Orchestrator + React layer
+export { PoseTrackerClient } from './client';
+export type { PoseTrackerClientOptions, PreferredBackend, StartExerciseOptions } from './client';
+export type { ColdStartMode, PreloadOptions } from './types/preload';
+export type {
+  SkeletonAnglesStyle,
+  SkeletonCirclesStyle,
+  SkeletonDefinition,
+  SkeletonLinesStyle,
+} from './types/skeleton';
+export { DEFAULT_SKELETON_DEFINITION } from './types/skeleton';
+export { fetchSkeletonDefinition, SkeletonFetchError } from './api/skeleton';
+export type { FetchSkeletonOptions } from './api/skeleton';
+export { PoseTrackerProvider, usePoseTracker } from './PoseTrackerProvider';
+export type { PoseTrackerContextValue, PoseTrackerProviderProps } from './PoseTrackerProvider';
+
+// Inference backends
+export type { PoseBackend, PoseBackendInitOptions, PoseInputFrame } from './backends/PoseBackend';
+
+// WebView MoveNet — light online runtime (both platforms)
+export { WebViewPoseBackend, isWebViewPoseBackend } from './backends/webview/WebViewPoseBackend';
+export type { WebViewPoseBackendOptions, WebViewPoseMessage } from './backends/webview/WebViewPoseBackend';
+export { WebViewPoseView } from './camera/WebViewPoseView';
+export type { WebViewPoseViewProps } from './camera/WebViewPoseView';
+export {
+  buildPoseHtml,
+  DEFAULT_LOADING_TEXT,
+  POSE_HTML_BUILD,
+} from './backends/webview/poseHtml';
+export type { PoseHtmlOptions } from './backends/webview/poseHtml';
+export { POSETRACKER_LOGO_DATA_URL } from './backends/webview/brandAssets';
+
+// Apple Vision (iOS native builds, opt-in via preferredBackend: 'vision')
+export { VisionPoseBackend, VisionUnavailableError } from './backends/vision/VisionPoseBackend';
+export type { VisionPoseBackendOptions } from './backends/vision/VisionPoseBackend';
+export {
+  isVisionBackendAvailable,
+  isVisionPoseCameraAvailable,
+  VISION_BODY_POSE_PLUGIN_NAME,
+} from './backends/vision/optionalVision';
+export {
+  poseFromVisionPluginResult,
+  VISION_JOINT_TO_SDK,
+} from './backends/vision/mapVisionJoints';
+export type { VisionPluginJoint, VisionPluginResult } from './backends/vision/mapVisionJoints';
+export { PoseCameraView } from './camera/PoseCameraView';
+export type {
+  PoseCameraStats,
+  PoseCameraUnavailableReason,
+  PoseCameraViewProps,
+} from './camera/PoseCameraView';
+
+// Environment probes
+export { isExpoGo } from './support/optionalModules';
+
+// Acceleration / diagnostics types
+export type {
+  AccelerationDiagnostics,
+  AccelerationState,
+  DiagnosticListener,
+  GlCapabilities,
+  InferenceRuntime,
+  NativeDelegateLabel,
+  TfjsFlagSnapshot,
+  VisionDelegateLabel,
+} from './types/acceleration';
+
+// Engine
+export {
+  EngineLoader,
+  createExpoFileStore,
+  createRnfsFileStore,
+  createBlobUtilFileStore,
+  createMemoryFileStore,
+  createNativeFileStore,
+} from './engine/EngineLoader';
+export type { EngineLoadResult, EngineLoaderOptions, FileStore, KeyValueStore } from './engine/EngineLoader';
+export type {
+  CustomExerciseDescriptor,
+  CustomSessionOptions,
+  EngineModuleExports,
+  EngineSession,
+  EngineSessionFeatures,
+  EngineSessionOptions,
+  EventSink,
+  PoseTrackerEngine,
+} from './engine/types';
+
+// Metro / logcat diagnostic helpers
+export {
+  defaultDiagnosticLogger,
+  logAccelerationReport,
+  logFrameStats,
+  logPlatformBanner,
+} from './diagnostics/logReport';
+
+// Classic PoseTracker WebView message parity (sendDataToNative JSON)
+export {
+  toClassicNativeMessage,
+} from './events/classicMessage';
+export type {
+  ClassicMessageListener,
+  ClassicNativeMessage,
+} from './events/classicMessage';
+
+// Adaptive camera quality (AdaptiveChoice + crash-loop guard + FPS downgrade)
+export {
+  AdaptiveQualityController,
+} from './quality/AdaptiveQualityController';
+export type {
+  AdaptiveQualityControllerOptions,
+  QualityState,
+  QualityStatsSample,
+} from './quality/AdaptiveQualityController';
+export {
+  CRITICAL_FPS_THRESHOLD,
+  IDEAL_FPS_RANGE_ANDROID,
+  IDEAL_FPS_RANGE_IOS,
+  LOW_FPS_STREAK_BEFORE_DOWNGRADE,
+  QUALITY_SETTLE_MS,
+  MIN_TARGET_FPS_ANDROID,
+  MIN_TARGET_FPS_DEFAULT,
+  MIN_TARGET_FPS_IOS,
+  QUALITY_LADDER,
+  QUALITY_PROFILES,
+  TARGET_FPS,
+  TARGET_MEDIAN_MS,
+  currentQualityPlatform,
+  estimatedFpsFromMedianMs,
+  getCriticalFpsThreshold,
+  getIdealFpsRange,
+  getMinTargetFps,
+  getQualityProfile,
+  isQualityProfileId,
+  lowerQualityProfile,
+  minMedianMsForMinTarget,
+  nextLowerQualityProfile,
+  profileFromWarmupMedianMs,
+} from './quality/profiles';
+export {
+  ANDROID_INFER_FRAME_SKIP,
+  ANDROID_MIN_TARGET_FPS,
+  ANDROID_PERF_DEBUG,
+  ANDROID_PREPROCESS_PATH,
+  ANDROID_SOFT_CAP_PROFILE,
+  CAPTURE_CONSTRAINT_MODE,
+  ENABLE_MALI_HARD_CAP,
+} from './quality/captureMode';
+export type {
+  AndroidPreprocessPath,
+  AndroidSoftCapProfile,
+  CaptureConstraintMode,
+} from './quality/captureMode';
+export type {
+  FpsRange,
+  CapturePriority,
+  QualityChoice,
+  QualityPlatform,
+  QualityProfile,
+  QualityProfileId,
+} from './quality/profiles';
+export {
+  isMaliRenderer,
+  profileFromScore,
+  scoreDeviceCapability,
+} from './quality/deviceCapability';
+export type { DeviceCapabilitySnapshot } from './quality/deviceCapability';
+export { RuntimeGuard } from './quality/RuntimeGuard';
+export type { GuardState } from './quality/RuntimeGuard';
+
+// Tracking features (WebView query-param parity + plan gating)
+export {
+  DEFAULT_FEATURES,
+  FREE_PLAN_FEATURES_MESSAGE,
+  INVALID_TOKEN_MESSAGE,
+  COMBINED_REFERENCE_EXERCISE_MESSAGE,
+  featureNotSupportedMessage,
+  freeBlockedFeatures,
+  isPaidPlan,
+  shouldShowWatermark,
+  resolveFeatures,
+} from './types/features';
+export type { PoseTrackerFeatures, ResolvedFeatures } from './types/features';
+
+// Types
+export * from './types/events';
+export * from './types/manifest';
+export * from './types/pose';

package/src/models/poseModels.ts

@@ -0,0 +1,93 @@
+/**
+ * Online pose-model catalog for the light SDK.
+ *
+ * Defaults match the PoseTracker Front / Docs API product path:
+ * MoveNet SinglePose Lightning hosted at app.posetracker.com (same URL as
+ * `usePoseDetection` / tracking WebView `modelUrl`).
+ *
+ * BlazePose is accepted as a `model` alias for API parity but is not wired
+ * in the RN WebView yet (needs MediaPipe). Pass a TF.js `modelUrl` for any
+ * custom graph model.
+ */
+
+/** Production MoveNet SinglePose Lightning topology (Front default). */
+export const DEFAULT_MOVENET_LIGHTNING_URL =
+  'https://app.posetracker.com/scripts/tmp_model_to_remove.json';
+
+/** Docs API / Front `model` query aliases. */
+export type PoseModelAlias =
+  | 'movenet'
+  | 'movenet-singlepose-lightning'
+  | 'lightning'
+  | 'blazepose'
+  | (string & {});
+
+export interface ResolvePoseModelOptions {
+  /**
+   * Docs API `model` query parity (`movenet` default, `blazepose`, …).
+   * Ignored when {@link modelUrl} is set.
+   */
+  model?: PoseModelAlias;
+  /** Explicit TF.js graph-model topology URL (weights resolve as siblings). */
+  modelUrl?: string;
+}
+
+export interface ResolvedPoseModel {
+  /** Stable id reported in diagnostics / track params. */
+  modelId: string;
+  /** TF.js `loadGraphModel` URL, or null when unsupported without override. */
+  modelUrl: string | null;
+  /** Human-readable reason when {@link modelUrl} is null. */
+  unsupportedReason?: string;
+}
+
+/**
+ * Resolve which TF.js graph model the light WebView should fetch.
+ * Default: MoveNet SinglePose Lightning (product URL).
+ */
+export function resolvePoseModel(options: ResolvePoseModelOptions = {}): ResolvedPoseModel {
+  const explicit =
+    typeof options.modelUrl === 'string' && options.modelUrl.trim().length > 0
+      ? options.modelUrl.trim()
+      : null;
+  if (explicit) {
+    const alias =
+      typeof options.model === 'string' && options.model.trim().length > 0
+        ? options.model.trim()
+        : 'custom';
+    return { modelId: alias, modelUrl: explicit };
+  }
+
+  const key = (options.model ?? 'movenet').trim().toLowerCase();
+  if (key === 'movenet' || key === 'movenet-singlepose-lightning' || key === 'lightning') {
+    return {
+      modelId: 'movenet-singlepose-lightning',
+      modelUrl: DEFAULT_MOVENET_LIGHTNING_URL,
+    };
+  }
+  if (key === 'blazepose') {
+    return {
+      modelId: 'blazepose',
+      modelUrl: null,
+      unsupportedReason:
+        'BlazePose is not wired in the light RN WebView yet (needs MediaPipe). ' +
+        'Use model="movenet" or pass a TF.js graph modelUrl.',
+    };
+  }
+  return {
+    modelId: key,
+    modelUrl: null,
+    unsupportedReason:
+      `Unknown model "${options.model}". Use "movenet" (default) or pass modelUrl.`,
+  };
+}
+
+/** Origin used as WebView `baseUrl` so model fetches are same-origin when possible. */
+export function originFromModelUrl(modelUrl: string): string {
+  try {
+    const u = new URL(modelUrl);
+    return `${u.protocol}//${u.host}/`;
+  } catch {
+    return 'https://app.posetracker.com/';
+  }
+}