@pose-tracker/react-native-pose-estimation-light 0.1.0

package/src/runtime/RuntimeCache.ts

@@ -0,0 +1,300 @@
+/**
+ * Versioned filesystem cache of a remote pose-runtime payload (legacy
+ * stub-loader helper).
+ *
+ * The light SDK’s default path does **not** use this cache: TF.js loads from
+ * CDN and the model from a URL inside the WebView each boot. This class
+ * remains exported for advanced hosts that still want to download / verify
+ * Strapi `/api/sdk/pose-runtime` parts.
+ *
+ * Guarantees when used:
+ * - sha256 is verified BEFORE a part is committed to the cache.
+ * - `purge()` on revocation signals.
+ */
+
+import { sha256 } from 'js-sha256';
+
+import type { PoseRuntimeDescriptor } from '../types/manifest';
+import {
+  createMemoryFileStore,
+  createNativeFileStore,
+  type FileStore,
+} from '../engine/EngineLoader';
+
+/** Part names, as declared by the backend runtime manifest. */
+const PART_NAMES = ['tfjs', 'tfjs-wasm', 'model', 'weights', 'pipeline', 'runtime'] as const;
+type PartName = (typeof PART_NAMES)[number];
+
+/** Required parts — `pipeline` is optional (JS fallback lives in the runtime part). */
+const REQUIRED_PARTS: PartName[] = ['tfjs', 'tfjs-wasm', 'model', 'weights', 'runtime'];
+
+const CURRENT_KEY = 'current.json';
+
+interface CurrentPointer {
+  version: string;
+  parts: Partial<Record<PartName, { key: string; sha256: string }>>;
+}
+
+/** Decoded, ready-to-inject runtime parts. */
+export interface PoseRuntimeParts {
+  version: string;
+  /** TF.js bundle (core + converter + webgl + wasm backends). */
+  tfjsJs: string;
+  /** XNNPACK wasm binaries (base64). */
+  tfjsWasmB64: { plain: string; simd: string; threadedSimd: string };
+  /** MoveNet graph model topology (raw model.json). */
+  modelJson: string;
+  /** Weight shards (base64, manifest order). */
+  weightsB64: string[];
+  /** Proprietary pose pipeline wasm (base64), null when not deployed. */
+  pipelineWasmB64: string | null;
+  /** Page runtime (camera, presets, inference loop, events). */
+  runtimeJs: string;
+}
+
+export interface RuntimeCacheProgress {
+  part: string;
+  completedParts: number;
+  totalParts: number;
+  /** Bytes of the part being downloaded (from the manifest). */
+  partBytes: number;
+}
+
+export interface RuntimeCacheOptions {
+  fileStore?: FileStore | null;
+  /** Injectable for tests. */
+  fetchFn?: typeof fetch;
+}
+
+export class RuntimeCache {
+  private readonly files: FileStore | null;
+  private readonly fetchFn: typeof fetch;
+  private memoryParts: PoseRuntimeParts | null = null;
+  /** True when no persistent FS module is installed (memory-only fallback). */
+  readonly persistent: boolean;
+
+  constructor(options: RuntimeCacheOptions = {}) {
+    let files =
+      options.fileStore !== undefined
+        ? options.fileStore
+        : createNativeFileStore('posetracker-runtime');
+    this.persistent = files !== null;
+    if (files === null) {
+      // Bare RN without any filesystem lib: stay functional with an
+      // in-memory store — online works, but every cold start re-downloads.
+      files = createMemoryFileStore();
+    }
+    this.files = files;
+    this.fetchFn = options.fetchFn ?? fetch;
+  }
+
+  /** Version currently committed to the cache, or null when cold/empty. */
+  async getCachedVersion(): Promise<string | null> {
+    const current = await this.readCurrent();
+    return current?.version ?? null;
+  }
+
+  /**
+   * Ensure the cache holds the version described by the manifest. Downloads
+   * only what is missing/outdated; every part is checksum-verified BEFORE
+   * being committed. The previous version keeps serving until the new one is
+   * fully committed (atomic flip of `current.json`, then best-effort GC).
+   *
+   * Returns true when the cache is warm (either already up-to-date or
+   * successfully updated); false when it could not be warmed (offline...) —
+   * the caller decides whether a stale/absent cache is acceptable.
+   */
+  async warm(
+    descriptor: PoseRuntimeDescriptor | null,
+    onProgress?: (progress: RuntimeCacheProgress) => void,
+  ): Promise<boolean> {
+    if (!this.files) {
+      return false;
+    }
+    const current = await this.readCurrent();
+    if (!descriptor) {
+      // Offline handshake: the existing cache (if any) is served as-is.
+      return current !== null;
+    }
+    if (current && current.version === descriptor.version) {
+      return true;
+    }
+
+    const wanted = Object.entries(descriptor.parts) as Array<
+      [string, { url: string; sha256: string; bytes: number }]
+    >;
+    const next: CurrentPointer = { version: descriptor.version, parts: {} };
+    let completed = 0;
+
+    try {
+      for (const [name, part] of wanted) {
+        if (!this.isKnownPart(name)) {
+          continue;
+        }
+        onProgress?.({
+          part: name,
+          completedParts: completed,
+          totalParts: wanted.length,
+          partBytes: part.bytes,
+        });
+        const key = `${name}@${descriptor.version}`;
+
+        // Reuse without network when the exact content is already cached
+        // (same key from an interrupted warm, or same sha in the previous
+        // version under another key).
+        let content = await this.readVerified(key, part.sha256);
+        if (content === null) {
+          const previous = current?.parts[name];
+          if (previous && previous.sha256 === part.sha256) {
+            content = await this.readVerified(previous.key, part.sha256);
+          }
+        }
+        if (content === null) {
+          const response = await this.fetchFn(part.url);
+          if (!response.ok) {
+            throw new Error(`pose-runtime part '${name}' HTTP ${response.status}`);
+          }
+          content = await response.text();
+          // Integrity gate BEFORE commit: a truncated/corrupted download
+          // never reaches the cache.
+          if (sha256(content) !== part.sha256) {
+            throw new Error(`pose-runtime part '${name}' failed sha256 verification`);
+          }
+        }
+        await this.files.write(key, content);
+        next.parts[name as PartName] = { key, sha256: part.sha256 };
+        completed += 1;
+      }
+
+      for (const required of REQUIRED_PARTS) {
+        if (!next.parts[required]) {
+          throw new Error(`pose-runtime manifest is missing required part '${required}'`);
+        }
+      }
+
+      // Commit point: flip the pointer only once every part is verified.
+      await this.files.write(CURRENT_KEY, JSON.stringify(next));
+      this.memoryParts = null;
+
+      // Best-effort GC of the previous version's files.
+      if (current) {
+        for (const entry of Object.values(current.parts)) {
+          if (entry && !Object.values(next.parts).some((p) => p && p.key === entry.key)) {
+            await this.files.remove(entry.key).catch(() => {});
+          }
+        }
+      }
+      return true;
+    } catch {
+      // Warm failed: the previous committed version stays intact.
+      return current !== null;
+    }
+  }
+
+  /** Load (and decode) the committed runtime parts; null when cache is cold. */
+  async load(): Promise<PoseRuntimeParts | null> {
+    if (this.memoryParts) {
+      return this.memoryParts;
+    }
+    if (!this.files) {
+      return null;
+    }
+    const current = await this.readCurrent();
+    if (!current) {
+      return null;
+    }
+
+    const read = async (name: PartName): Promise<string | null> => {
+      const entry = current.parts[name];
+      if (!entry) {
+        return null;
+      }
+      return this.readVerified(entry.key, entry.sha256);
+    };
+
+    const [tfjsJs, tfjsWasmRaw, modelJson, weightsRaw, pipelineWasmB64, runtimeJs] =
+      await Promise.all([
+        read('tfjs'),
+        read('tfjs-wasm'),
+        read('model'),
+        read('weights'),
+        read('pipeline'),
+        read('runtime'),
+      ]);
+
+    if (!tfjsJs || !tfjsWasmRaw || !modelJson || !weightsRaw || !runtimeJs) {
+      // A part is missing or failed integrity: treat the cache as cold.
+      return null;
+    }
+
+    try {
+      this.memoryParts = {
+        version: current.version,
+        tfjsJs,
+        tfjsWasmB64: JSON.parse(tfjsWasmRaw) as PoseRuntimeParts['tfjsWasmB64'],
+        modelJson,
+        weightsB64: JSON.parse(weightsRaw) as string[],
+        pipelineWasmB64: pipelineWasmB64 ?? null,
+        runtimeJs,
+      };
+      return this.memoryParts;
+    } catch {
+      return null;
+    }
+  }
+
+  /** Wipe the whole runtime cache (revocation / recovery). */
+  async purge(): Promise<void> {
+    this.memoryParts = null;
+    if (!this.files) {
+      return;
+    }
+    const current = await this.readCurrent();
+    if (current) {
+      for (const entry of Object.values(current.parts)) {
+        if (entry) {
+          await this.files.remove(entry.key).catch(() => {});
+        }
+      }
+    }
+    await this.files.remove(CURRENT_KEY).catch(() => {});
+  }
+
+  // -------------------------------------------------------------------------
+
+  private isKnownPart(name: string): name is PartName {
+    return (PART_NAMES as readonly string[]).includes(name);
+  }
+
+  private async readCurrent(): Promise<CurrentPointer | null> {
+    try {
+      const raw = await this.files?.read(CURRENT_KEY);
+      if (!raw) {
+        return null;
+      }
+      const parsed = JSON.parse(raw) as CurrentPointer;
+      if (!parsed.version || !parsed.parts) {
+        return null;
+      }
+      return parsed;
+    } catch {
+      return null;
+    }
+  }
+
+  private async readVerified(key: string, expectedSha: string): Promise<string | null> {
+    try {
+      const content = await this.files?.read(key);
+      if (content === null || content === undefined) {
+        return null;
+      }
+      if (sha256(content) !== expectedSha) {
+        await this.files?.remove(key).catch(() => {});
+        return null;
+      }
+      return content;
+    } catch {
+      return null;
+    }
+  }
+}

package/src/sdkVersion.ts

@@ -0,0 +1,2 @@
+/** AUTO-GENERATED by scripts/sync-sdk-version.mjs — do not edit. */
+export const SDK_VERSION = '0.1.0';

package/src/support/optionalModules.ts

@@ -0,0 +1,114 @@
+/**
+ * Guarded access to the OPTIONAL native modules used by the Apple Vision
+ * backend: react-native-vision-camera + react-native-worklets-core.
+ *
+ * They are `peerDependenciesMeta.optional` — a host that only uses the
+ * WebView runtime (the default, Expo Go compatible) does not install them.
+ *
+ * 1. Every `require` is wrapped in try/catch. Metro marks dependencies
+ *    required inside a try block as *optional* when
+ *    `transformer.allowOptionalDependencies` is enabled (the Expo default),
+ *    so bundling never fails when a module is absent.
+ * 2. The SDK must not depend on these packages' TypeScript types, so the
+ *    interfaces below are minimal *structural* typings of the APIs we
+ *    consume (react-native-vision-camera 4.x, worklets-core 1.x).
+ */
+
+declare function require(name: string): unknown;
+
+/**
+ * Expo Go has no vision-camera native binaries. Requiring them there throws
+ * (and React Native LogBox still surfaces it even when wrapped in
+ * try/catch). Detect Expo Go *before* any of those requires.
+ */
+export function isExpoGo(): boolean {
+  try {
+    // Avoid a hard dependency on expo-constants: same signal Nitro uses.
+    // eslint-disable-next-line @typescript-eslint/no-require-imports
+    const { NativeModules } = require('react-native') as {
+      NativeModules: {
+        NativeUnimoduleProxy?: {
+          modulesConstants?: { ExponentConstants?: { appOwnership?: string } };
+        };
+      };
+    };
+    return (
+      NativeModules.NativeUnimoduleProxy?.modulesConstants?.ExponentConstants
+        ?.appOwnership === 'expo'
+    );
+  } catch {
+    return false;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Structural typings (subset actually consumed by the SDK)
+// ---------------------------------------------------------------------------
+
+/** react-native-vision-camera 4.x — subset used by PoseCameraView. */
+export interface VisionCameraFrame {
+  width: number;
+  height: number;
+  orientation: string; // 'portrait' | 'portrait-upside-down' | 'landscape-left' | 'landscape-right'
+  timestamp: number;
+}
+
+export interface VisionCameraModule {
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  Camera: any; // React component class
+  useCameraDevice(position: 'front' | 'back'): unknown | undefined;
+  useCameraPermission(): { hasPermission: boolean; requestPermission: () => Promise<boolean> };
+  useFrameProcessor(processor: (frame: VisionCameraFrame) => void, deps: unknown[]): unknown;
+}
+
+/** react-native-worklets-core 1.x — subset used by PoseCameraView. */
+export interface WorkletsCoreModule {
+  useSharedValue<T>(initial: T): { value: T };
+  useRunOnJS<A extends unknown[]>(fn: (...args: A) => void, deps: unknown[]): (...args: A) => void;
+}
+
+// ---------------------------------------------------------------------------
+// Guarded requires (cached; `undefined` = not probed yet, `null` = missing)
+// ---------------------------------------------------------------------------
+
+// NOTE: every `require` below uses a STRING LITERAL — Metro cannot resolve
+// dynamic `require(variable)` calls and would fail the build.
+
+let visionCamera: VisionCameraModule | null | undefined;
+let workletsCore: WorkletsCoreModule | null | undefined;
+
+export function getVisionCamera(): VisionCameraModule | null {
+  if (visionCamera === undefined) {
+    if (isExpoGo()) {
+      visionCamera = null;
+    } else {
+      try {
+        visionCamera = require('react-native-vision-camera') as VisionCameraModule;
+      } catch {
+        visionCamera = null;
+      }
+      if (visionCamera && typeof visionCamera.useFrameProcessor !== 'function') {
+        visionCamera = null;
+      }
+    }
+  }
+  return visionCamera;
+}
+
+export function getWorkletsCore(): WorkletsCoreModule | null {
+  if (workletsCore === undefined) {
+    if (isExpoGo()) {
+      workletsCore = null;
+    } else {
+      try {
+        workletsCore = require('react-native-worklets-core') as WorkletsCoreModule;
+      } catch {
+        workletsCore = null;
+      }
+      if (workletsCore && typeof workletsCore.useRunOnJS !== 'function') {
+        workletsCore = null;
+      }
+    }
+  }
+  return workletsCore;
+}

package/src/types/acceleration.ts

@@ -0,0 +1,94 @@
+/**
+ * GPU-acceleration status types.
+ *
+ * Historical context (see docs/ANDROID_GL_ACCELERATION.md): on Android the
+ * previous product generation had to run TF.js inside a WebView because it
+ * was the only reliable way to get a working WebGL backend. The native path
+ * (tfjs-react-native over expo-gl) can silently degrade — the platform
+ * adapter *shims* float-texture extension queries, so TF.js may believe the
+ * GPU supports float32 render targets when it does not, and expo-gl contexts
+ * die on Android when the surface is backgrounded. These types exist so the
+ * SDK reports an explicit acceleration verdict instead of failing silently
+ * into a 1-2 fps CPU fallback.
+ */
+
+/**
+ * - 'unknown':       warm-up has not completed yet.
+ * - 'gpu':           rn-webgl (expo-gl) active and the warm-up health check
+ *                    produced sane outputs within the latency budget.
+ * - 'cpu-fallback':  inference works but is NOT GPU-accelerated (rn-webgl
+ *                    failed to initialize, produced NaN/garbage, or was
+ *                    pathologically slow — i.e. software rendering).
+ * - 'unavailable':   no backend could run the model at all (fatal; the
+ *                    client also reports status 'error').
+ */
+export type AccelerationState = 'unknown' | 'gpu' | 'cpu-fallback' | 'unavailable';
+
+/** Raw GL context facts, gathered best-effort after backend init. */
+export interface GlCapabilities {
+  /** e.g. "OpenGL ES 3.0 ..." */
+  glVersion: string | null;
+  /** e.g. "Adreno (TM) 640", "Mali-G78", "PowerVR ..." — key for triage. */
+  renderer: string | null;
+  vendor: string | null;
+  maxTextureSize: number | null;
+  /**
+   * Extension presence *as reported to TF.js*. WARNING: on rn-webgl the
+   * tfjs-react-native platform adapter shims `getExtension` and always
+   * reports EXT_color_buffer_float (Android) / EXT_color_buffer_half_float
+   * as present, so `true` here does NOT prove hardware support. That lie is
+   * exactly why the SDK forces f16 textures on Android and validates real
+   * outputs in the health check.
+   */
+  colorBufferFloat: boolean;
+  colorBufferHalfFloat: boolean;
+  textureFloat: boolean;
+  textureHalfFloat: boolean;
+  /** True when the extension answers above come from the shimmed adapter. */
+  extensionQueriesShimmed: boolean;
+}
+
+/** Snapshot of the TF.js WebGL flags that matter for Android stability. */
+export type TfjsFlagSnapshot = Record<string, boolean | number | string | null>;
+
+/**
+ * Inference runtime that produced the acceleration verdict:
+ * - 'webview': TF.js MoveNet inside a Chromium/WKWebView (ANGLE WebGL) —
+ *              the SDK's base runtime, offline, same stack as the
+ *              PoseTracker WebView product.
+ * - 'vision':  Apple Vision (`VNDetectHumanBodyPoseRequest`) — iOS only.
+ */
+export type InferenceRuntime = 'webview' | 'vision';
+
+/** Native accelerator label (Apple Vision). */
+export type VisionDelegateLabel = 'apple-vision';
+
+export type NativeDelegateLabel = VisionDelegateLabel;
+
+export interface AccelerationDiagnostics {
+  state: AccelerationState;
+  /** Active TF.js backend after init/fallbacks: 'rn-webgl' | 'cpu' | null. */
+  tfjsBackend: string | null;
+  /** Which runtime produced this verdict (absent = 'tfjs', pre-TFLite SDKs). */
+  runtime?: InferenceRuntime;
+  /** Active native accelerator; null/absent for the TF.js backend. */
+  delegate?: NativeDelegateLabel | null;
+  /** Median duration of the timed warm-up inferences, in ms. */
+  medianInferenceMs: number | null;
+  /** Individual timed warm-up runs, in ms (first run includes shader compilation). */
+  inferenceTimesMs: number[];
+  /** Latency budget used by the health check to accept the GPU path. */
+  maxAcceptableInferenceMs: number;
+  capabilities: GlCapabilities | null;
+  flags: TfjsFlagSnapshot;
+  /** Number of GL context losses recovered since init (Android lifecycle). */
+  contextLossCount: number;
+  /**
+   * Human-readable trail of every downgrade / recovery decision, e.g.
+   * "rn-webgl produced NaN outputs — falling back to CPU".
+   */
+  reasons: string[];
+}
+
+/** Callback for host apps that want SDK diagnostics in their own logger. */
+export type DiagnosticListener = (message: string) => void;