@pose-tracker/react-native-pose-estimation-light 0.1.0

package/src/quality/captureMode.ts

@@ -0,0 +1,97 @@
+/**
+ * Camera capture + Android perf experiment knobs (WebView getUserMedia / MoveNet).
+ *
+ * iOS is never mutated by the Android-only flags below — {@link buildPoseHtml}
+ * forces frame-skip / soft-cap / preprocess overrides off on iOS.
+ *
+ * ## Experiment stack 2026-08-06
+ * 1. `device-native` cold open (Front-aligned) + optional Android soft-cap
+ * 2. Android frame-skip between inferences
+ * 3. Android lighter preprocess (`canvas-direct`)
+ * 4. Android min-target floor 10 FPS (downgrade only when truly needed)
+ * 5. Verbose perf debug in WebView stats / HUD
+ *
+ * ## REVERT to previous stable behaviour
+ * 1. `CAPTURE_CONSTRAINT_MODE = 'profile-constrained'`
+ * 2. `ENABLE_MALI_HARD_CAP = true`
+ * 3. `ANDROID_INFER_FRAME_SKIP = 0`
+ * 4. `ANDROID_PREPROCESS_PATH = 'imagebitmap'`
+ * 5. `ANDROID_SOFT_CAP_PROFILE = null` (experiment currently pins `'basic'`)
+ * 6. `ANDROID_MIN_TARGET_FPS = 15`
+ * 7. `ANDROID_PERF_DEBUG = false`
+ * 8. Rebuild: `npm run build:runtime-payload && npm run build`
+ * 9. Refresh testapp link + `npx expo start -c`
+ *
+ * You also have a zip archive of the previous SDK as a full rollback.
+ */
+
+/** How getUserMedia chooses resolution on cold open. */
+export type CaptureConstraintMode = 'device-native' | 'profile-constrained';
+
+/**
+ * Letterbox preprocess path inside the WebView (Android experiment only).
+ * - `imagebitmap` — draw 192² canvas → createImageBitmap → fromPixels (legacy)
+ * - `canvas-direct` — draw 192² canvas → fromPixels(canvas) (skips ImageBitmap)
+ */
+export type AndroidPreprocessPath = 'imagebitmap' | 'canvas-direct';
+
+/** Soft-cap ladder ids (mirrors QualityProfileId; kept local to avoid import cycles). */
+export type AndroidSoftCapProfile =
+  | 'prime'
+  | 'pro'
+  | 'lite'
+  | 'ultralite'
+  | 'basic';
+
+/**
+ * - `device-native` — facingMode only (like PoseTrackerFront). Preview = device default (often HD).
+ * - `profile-constrained` — ideal width/height/frameRate from the AdaptiveChoice ladder.
+ */
+export const CAPTURE_CONSTRAINT_MODE: CaptureConstraintMode = 'device-native';
+
+/**
+ * When true, Mali GL renderers are hard-capped to UltraLite at capability score
+ * time (previous SDK default). Disable for the Front-aligned experiment so RN
+ * does not immediately setQuality() back down to 480×360 after a native HD open.
+ */
+export const ENABLE_MALI_HARD_CAP = false;
+
+/**
+ * Android-only: ready rAF ticks to skip between MoveNet inferences.
+ * Preview stays at full capture quality.
+ *
+ * - `0` — off
+ * - `1` — every other ready tick
+ * - `2` — every 3rd, etc.
+ */
+export const ANDROID_INFER_FRAME_SKIP = 1;
+
+/**
+ * Android-only preprocess path. iOS always uses `imagebitmap` (unchanged).
+ */
+export const ANDROID_PREPROCESS_PATH: AndroidPreprocessPath = 'canvas-direct';
+
+/**
+ * Android-only cold-open soft cap. When set, getUserMedia uses this profile’s
+ * width/height as ideal+max even in `device-native` mode (so the device cannot
+ * open unrestricted 1080p). Set `null` for fully unconstrained facingMode-only.
+ *
+ * Mali-G52 evidence (2026-08-06): `basic` 240×320 ≈ 10 FPS median; `ultralite`
+ * 360×480 ≈ 8 FPS — pin soft-cap to `basic` to hold the 10 FPS floor.
+ * Live `setQuality` cannot go lower than basic.
+ */
+export const ANDROID_SOFT_CAP_PROFILE: AndroidSoftCapProfile | null = 'basic';
+
+/**
+ * Android AdaptiveChoice floor (min target FPS). Downgrade when median
+ * inference ms stays above ~1.2 × (1000/floor). iOS stays on
+ * {@link MIN_TARGET_FPS_IOS} (30).
+ */
+export const ANDROID_MIN_TARGET_FPS = 10;
+
+/**
+ * When true (Android), WebView posts rich `experiments` + `leverHints` on each
+ * 1 Hz stats tick and mirrors a compact line on the HUD. RN logs the same via
+ * the existing diagnostic listener. iOS ignores this flag.
+ */
+export const ANDROID_PERF_DEBUG = true;

package/src/quality/deviceCapability.ts

@@ -0,0 +1,187 @@
+/**
+ * Lightweight device capability score → initial quality profile.
+ *
+ * Sency's AdaptiveChoice uses RAM / CPU freq / first API level. In Expo Go we
+ * only have a subset of signals without extra native deps; we combine:
+ *   - Platform (iOS is typically 3–5× faster on WebGL MoveNet)
+ *   - Android API level
+ *   - Optional RAM via expo-device / react-native DeviceInfo when present
+ *   - GL renderer hint (Mali mid-range) once known from the WebView
+ *
+ * Score is 0–100; mapped to a QualityProfileId. Brand/year caps from Sency
+ * are approximated via API level + Mali detection.
+ */
+
+import { Platform } from 'react-native';
+
+import { ENABLE_MALI_HARD_CAP } from './captureMode';
+import type { QualityProfileId } from './profiles';
+
+export interface DeviceCapabilitySnapshot {
+  platform: 'ios' | 'android' | 'other';
+  /** Android API level or iOS major version when available. */
+  osVersion: number | null;
+  /** Total device RAM in GiB when a probe is available. */
+  totalMemoryGiB: number | null;
+  /** WebGL UNMASKED_RENDERER when known (after first WebView ready). */
+  glRenderer: string | null;
+  /** 0–100 composite score. */
+  score: number;
+  /** Profile suggested by the score (before crash-guard / caps). */
+  suggestedProfile: QualityProfileId;
+  reasons: string[];
+}
+
+function probeTotalMemoryGiB(): number | null {
+  // Optional peers — never required.
+  try {
+    // eslint-disable-next-line @typescript-eslint/no-var-requires
+    const Device = require('expo-device');
+    const bytes = Device.totalMemory as number | null | undefined;
+    if (typeof bytes === 'number' && bytes > 0) return bytes / (1024 * 1024 * 1024);
+  } catch {
+    /* ignore */
+  }
+  try {
+    // eslint-disable-next-line @typescript-eslint/no-var-requires
+    const DeviceInfo = require('react-native-device-info');
+    // sync API may exist
+    if (typeof DeviceInfo.getTotalMemorySync === 'function') {
+      const bytes = DeviceInfo.getTotalMemorySync() as number;
+      if (typeof bytes === 'number' && bytes > 0) return bytes / (1024 * 1024 * 1024);
+    }
+  } catch {
+    /* ignore */
+  }
+  return null;
+}
+
+function parseOsVersion(): number | null {
+  const v = Platform.Version;
+  if (typeof v === 'number') return v;
+  if (typeof v === 'string') {
+    const major = parseInt(v.split('.')[0] ?? '', 10);
+    return Number.isFinite(major) ? major : null;
+  }
+  return null;
+}
+
+export function isMaliRenderer(renderer: string | null | undefined): boolean {
+  return !!(renderer && /mali/i.test(renderer));
+}
+
+/**
+ * Map capability score → profile (before Mali / crash-guard caps).
+ * Thresholds calibrated so mid-range Android (Mali-G52 era) lands on UltraLite.
+ */
+export function profileFromScore(score: number): QualityProfileId {
+  if (score >= 78) return 'prime';
+  if (score >= 62) return 'pro';
+  if (score >= 48) return 'lite';
+  if (score >= 32) return 'ultralite';
+  return 'basic';
+}
+
+export function scoreDeviceCapability(options?: {
+  glRenderer?: string | null;
+}): DeviceCapabilitySnapshot {
+  const reasons: string[] = [];
+  const platform =
+    Platform.OS === 'ios' ? 'ios' : Platform.OS === 'android' ? 'android' : 'other';
+  const osVersion = parseOsVersion();
+  const totalMemoryGiB = probeTotalMemoryGiB();
+  const glRenderer = options?.glRenderer ?? null;
+
+  let score = 0;
+
+  if (platform === 'ios') {
+    // Apple GPU WebGL MoveNet is typically <20 ms — start high.
+    score += 55;
+    reasons.push('platform=ios (+55)');
+    if (osVersion != null) {
+      if (osVersion >= 17) {
+        score += 15;
+        reasons.push(`ios>=17 (+15)`);
+      } else if (osVersion >= 15) {
+        score += 10;
+        reasons.push(`ios>=15 (+10)`);
+      } else {
+        score += 5;
+        reasons.push(`ios legacy (+5)`);
+      }
+    }
+  } else if (platform === 'android') {
+    score += 18;
+    reasons.push('platform=android (+18)');
+    if (osVersion != null) {
+      if (osVersion >= 34) {
+        score += 14;
+        reasons.push(`api>=34 (+14)`);
+      } else if (osVersion >= 31) {
+        score += 10;
+        reasons.push(`api>=31 (+10)`);
+      } else if (osVersion >= 28) {
+        score += 6;
+        reasons.push(`api>=28 (+6)`);
+      } else {
+        reasons.push(`api=${osVersion} (+0)`);
+      }
+    }
+  } else {
+    score += 20;
+    reasons.push('platform=other (+20)');
+  }
+
+  if (totalMemoryGiB != null) {
+    if (totalMemoryGiB >= 8) {
+      score += 20;
+      reasons.push(`ram>=8GiB (+20)`);
+    } else if (totalMemoryGiB >= 6) {
+      score += 14;
+      reasons.push(`ram>=6GiB (+14)`);
+    } else if (totalMemoryGiB >= 4) {
+      score += 8;
+      reasons.push(`ram>=4GiB (+8)`);
+    } else {
+      reasons.push(`ram=${totalMemoryGiB.toFixed(1)}GiB (+0)`);
+    }
+  } else {
+    // Neutral default when RAM unknown: don't assume high-end.
+    score += 6;
+    reasons.push('ram=unknown (+6)');
+  }
+
+  if (isMaliRenderer(glRenderer)) {
+    // Mali mid-range: HD camera preprocess alone can cost ~100 ms/frame.
+    score = Math.min(score, 40);
+    reasons.push('gl=Mali cap score≤40');
+  }
+
+  score = Math.max(0, Math.min(100, score));
+  let suggested = profileFromScore(score);
+
+  // Hard caps (Sency-style).
+  if (platform === 'android' && suggested === 'prime') {
+    suggested = 'pro';
+    reasons.push('android cap: prime→pro');
+  }
+  // Mali → ultralite hard cap: gated by captureMode (see captureMode.ts REVERT).
+  if (
+    ENABLE_MALI_HARD_CAP &&
+    isMaliRenderer(glRenderer) &&
+    (suggested === 'prime' || suggested === 'pro')
+  ) {
+    suggested = 'ultralite';
+    reasons.push('mali cap: →ultralite');
+  }
+
+  return {
+    platform,
+    osVersion,
+    totalMemoryGiB,
+    glRenderer,
+    score,
+    suggestedProfile: suggested,
+    reasons,
+  };
+}

package/src/quality/profiles.ts

@@ -0,0 +1,253 @@
+/**
+ * Camera / preprocess quality tiers for the WebView MoveNet runtime.
+ *
+ * Inspired by Sency's PoseModelChoice ladder (Prime → … → Basic), adapted to
+ * our stack: MoveNet Lightning always runs at 192×192; what changes is the
+ * **camera capture resolution**.
+ *
+ * FPS policy uses a **minimum target** (floor), not a single setpoint:
+ *   - iOS: min 30 — acceptable/ideal band roughly 30–50+ FPS
+ *   - Android: min from {@link ANDROID_MIN_TARGET_FPS} (experiment default 10)
+ *
+ * Warm-up benches estimate execute cost; AdaptiveChoice picks a capture
+ * profile before getUserMedia when the estimate is below the platform floor.
+ */
+
+import { Platform } from 'react-native';
+
+import { ANDROID_MIN_TARGET_FPS } from './captureMode';
+
+export type QualityProfileId = 'prime' | 'pro' | 'lite' | 'ultralite' | 'basic';
+
+/** Host-facing choice, mirrors Sency's AdaptiveChoice + explicit override. */
+export type QualityChoice = 'AdaptiveChoice' | QualityProfileId;
+
+/**
+ * Host trade-off between pose FPS and camera preview sharpness.
+ *
+ * - `performance` (default) — AdaptiveChoice may lower capture (esp. Android
+ *   soft-cap / warm-up / live downgrade) so inference stays near the platform
+ *   FPS floor. Preview can look soft on mid-range Android.
+ * - `quality` — keep high capture / skip FPS-driven capture downgrades.
+ *   Preview stays sharp; pose FPS may drop well below the floor. The SDK still
+ *   emits `performance_warning` when slow.
+ *
+ * Explicit {@link QualityChoice} profile ids still pin the starting tier;
+ * `quality` only disables automatic FPS-driven drops.
+ */
+export type CapturePriority = 'performance' | 'quality';
+
+export type QualityPlatform = 'ios' | 'android' | 'other';
+
+export interface FpsRange {
+  /** Soft lower bound of the “good experience” band (same as min target on that OS). */
+  min: number;
+  /**
+   * Soft upper reference for docs / UI (“we’re happy around here”).
+   * Not a hard cap — exceeding it is fine.
+   */
+  idealMax: number;
+}
+
+export interface QualityProfile {
+  id: QualityProfileId;
+  /** Human label for diagnostics / docs. */
+  label: string;
+  /** getUserMedia ideal width (CSS pixels / sensor). */
+  idealWidth: number;
+  /** getUserMedia ideal height. */
+  idealHeight: number;
+  /** getUserMedia ideal frameRate. */
+  idealFrameRate: number;
+  /**
+   * Per-profile floor used when AdaptiveChoice has not overridden via platform.
+   * Live controller uses {@link getMinTargetFps} instead.
+   */
+  minStableFps: number;
+}
+
+/**
+ * Ordered highest → lowest. `nextLowerQualityProfile` walks this ladder.
+ */
+export const QUALITY_LADDER: readonly QualityProfileId[] = [
+  'prime',
+  'pro',
+  'lite',
+  'ultralite',
+  'basic',
+] as const;
+
+/**
+ * Android minimum acceptable inference FPS (floor).
+ * Sourced from {@link ANDROID_MIN_TARGET_FPS} so the experiment flag is the
+ * single knob (see captureMode.ts).
+ */
+export const MIN_TARGET_FPS_ANDROID = ANDROID_MIN_TARGET_FPS;
+
+/**
+ * iOS minimum acceptable inference FPS (floor).
+ * Acceptable/ideal experience is typically 30–50+ on Apple GPU.
+ */
+export const MIN_TARGET_FPS_IOS = 30;
+
+/** Fallback floor for unknown platforms. */
+export const MIN_TARGET_FPS_DEFAULT = MIN_TARGET_FPS_ANDROID;
+
+/**
+ * Documented “good experience” bands (not hard caps).
+ * Downgrades / warnings use the **min** only.
+ */
+export const IDEAL_FPS_RANGE_IOS: FpsRange = { min: MIN_TARGET_FPS_IOS, idealMax: 50 };
+export const IDEAL_FPS_RANGE_ANDROID: FpsRange = { min: MIN_TARGET_FPS_ANDROID, idealMax: 30 };
+
+/**
+ * @deprecated Use {@link getMinTargetFps} / {@link MIN_TARGET_FPS_ANDROID}.
+ * Kept as the Android floor so older imports keep a sensible number.
+ */
+export const TARGET_FPS = MIN_TARGET_FPS_ANDROID;
+
+/** @deprecated Use {@link minMedianMsForMinTarget}. */
+export const TARGET_MEDIAN_MS = 1000 / MIN_TARGET_FPS_ANDROID;
+
+/**
+ * Critical alert when FPS stays well below the platform minimum target
+ * (even on `basic`). Android: <10; iOS: <20.
+ */
+export function getCriticalFpsThreshold(platform: QualityPlatform = currentQualityPlatform()): number {
+  if (platform === 'ios') return 20;
+  return 10;
+}
+
+/** @deprecated Prefer {@link getCriticalFpsThreshold}. */
+export const CRITICAL_FPS_THRESHOLD = 10;
+
+/** Consecutive samples below min-target before auto-downgrade. */
+/** Sustained samples below the inference budget before a live downgrade. */
+export const LOW_FPS_STREAK_BEFORE_DOWNGRADE = 4;
+
+/**
+ * Ignore live downgrade decisions for this long after ready / a quality swap
+ * (camera restart tanks the first 1 Hz stats samples).
+ */
+export const QUALITY_SETTLE_MS = 5_000;
+
+export const QUALITY_PROFILES: Record<QualityProfileId, QualityProfile> = {
+  prime: {
+    id: 'prime',
+    label: 'Prime (1280×720 @30)',
+    idealWidth: 1280,
+    idealHeight: 720,
+    idealFrameRate: 30,
+    minStableFps: MIN_TARGET_FPS_ANDROID,
+  },
+  pro: {
+    id: 'pro',
+    label: 'Pro (960×540 @30)',
+    idealWidth: 960,
+    idealHeight: 540,
+    idealFrameRate: 30,
+    minStableFps: MIN_TARGET_FPS_ANDROID,
+  },
+  lite: {
+    id: 'lite',
+    label: 'Lite (640×480 @30)',
+    idealWidth: 640,
+    idealHeight: 480,
+    idealFrameRate: 30,
+    minStableFps: MIN_TARGET_FPS_ANDROID,
+  },
+  ultralite: {
+    id: 'ultralite',
+    label: 'UltraLite (480×360 @24)',
+    idealWidth: 480,
+    idealHeight: 360,
+    idealFrameRate: 24,
+    minStableFps: MIN_TARGET_FPS_ANDROID,
+  },
+  basic: {
+    id: 'basic',
+    label: 'Basic (320×240 @20)',
+    idealWidth: 320,
+    idealHeight: 240,
+    idealFrameRate: 20,
+    minStableFps: MIN_TARGET_FPS_ANDROID,
+  },
+};
+
+export function currentQualityPlatform(): QualityPlatform {
+  if (Platform.OS === 'ios') return 'ios';
+  if (Platform.OS === 'android') return 'android';
+  return 'other';
+}
+
+/** Platform minimum target FPS (floor — not a setpoint). */
+export function getMinTargetFps(platform: QualityPlatform = currentQualityPlatform()): number {
+  if (platform === 'ios') return MIN_TARGET_FPS_IOS;
+  if (platform === 'android') return ANDROID_MIN_TARGET_FPS;
+  return MIN_TARGET_FPS_DEFAULT;
+}
+
+export function getIdealFpsRange(platform: QualityPlatform = currentQualityPlatform()): FpsRange {
+  if (platform === 'ios') return IDEAL_FPS_RANGE_IOS;
+  if (platform === 'android') return IDEAL_FPS_RANGE_ANDROID;
+  return { min: MIN_TARGET_FPS_DEFAULT, idealMax: 30 };
+}
+
+/** Max warm-up median ms that still meets a given min-target FPS. */
+export function minMedianMsForMinTarget(minTargetFps: number): number {
+  return 1000 / minTargetFps;
+}
+
+export function getQualityProfile(id: QualityProfileId): QualityProfile {
+  return QUALITY_PROFILES[id];
+}
+
+export function nextLowerQualityProfile(id: QualityProfileId): QualityProfileId | null {
+  const idx = QUALITY_LADDER.indexOf(id);
+  if (idx < 0 || idx >= QUALITY_LADDER.length - 1) return null;
+  return QUALITY_LADDER[idx + 1]!;
+}
+
+export function isQualityProfileId(value: string): value is QualityProfileId {
+  return Object.prototype.hasOwnProperty.call(QUALITY_PROFILES, value);
+}
+
+export function qualityLadderIndex(id: QualityProfileId): number {
+  return QUALITY_LADDER.indexOf(id);
+}
+
+/** Lower or equal quality (higher index) wins. */
+export function lowerQualityProfile(
+  a: QualityProfileId,
+  b: QualityProfileId,
+): QualityProfileId {
+  return qualityLadderIndex(a) >= qualityLadderIndex(b) ? a : b;
+}
+
+/**
+ * Map warm-up execute median (zeros, no camera) → max safe capture profile.
+ *
+ * Thresholds scale with the platform **minimum target** FPS. Warm-up ignores
+ * bitmap cost, so anything over the floor budget must stay on `basic` —
+ * the old `≤ 1.3× budget → ultralite` band upgraded Mali devices that were
+ * already below the floor (e.g. 120 ms zeros → ultralite → live ~8 FPS).
+ */
+export function profileFromWarmupMedianMs(
+  medianMs: number | null,
+  minTargetFps: number = getMinTargetFps(),
+): QualityProfileId {
+  if (medianMs == null || !Number.isFinite(medianMs)) return 'basic';
+  const budget = minMedianMsForMinTarget(minTargetFps);
+  // Over the floor on zeros alone → basic only (camera adds more cost).
+  if (medianMs > budget) return 'basic';
+  // At / under the floor with headroom → higher capture ok
+  if (medianMs <= budget * 0.6) return 'prime';
+  if (medianMs <= budget * 0.75) return 'pro';
+  if (medianMs <= budget * 0.9) return 'lite';
+  return 'ultralite';
+}
+
+export function estimatedFpsFromMedianMs(medianMs: number | null): number | null {
+  if (medianMs == null || !Number.isFinite(medianMs) || medianMs <= 0) return null;
+  return 1000 / medianMs;
+}