@pose-tracker/react-native-pose-estimation-light 0.1.0 → 0.2.1

package/src/backends/webview/poseHtml.ts

@@ -2,9 +2,10 @@
  * WebView HTML assembler (light / online).
  *
  * Loads TF.js from CDN script tags and MoveNet (or a custom graph model)
- * from {@link OnlineRuntimeParts.modelUrl}. The thin page runtime is still
- * inlined from the package. Same camera / adaptive quality / postMessage
- * events as the offline SDK.
+ * from {@link OnlineRuntimeParts.modelUrl}. For BlazePose, also loads
+ * `@tensorflow-models/pose-detection` (no graph URL). The thin page runtime
+ * is still inlined from the package. Same camera / adaptive quality /
+ * postMessage events as the offline SDK.
  */
 
 import { Platform } from 'react-native';
@@ -21,7 +22,7 @@ import { POSETRACKER_LOGO_DATA_URL } from './brandAssets';
 import type { OnlineRuntimeParts } from './onlineRuntime';
 
 /** Bumped on every assembler-path change — appears in WebView diag logs. */
-export const POSE_HTML_BUILD = '20260811-onlineLight-localhostBase';
+export const POSE_HTML_BUILD = '20260812-onlineLight-mediaSources';
 
 /** Default boot overlay copy (WebView `loading_message` parity). */
 export const DEFAULT_LOADING_TEXT = 'AI Loading';
@@ -76,16 +77,23 @@ export interface PoseHtmlOptions {
   showWatermark?: boolean;
   /** Show the technical `#hud` overlay (FPS / backend). Default false. */
   debugHud?: boolean;
+  /**
+   * Input source mode. Default `camera`.
+   * For `video` / `image`, pass a URI the WebView can load (`sourceUrl`).
+   */
+  sourceType?: 'camera' | 'video' | 'image';
+  /** file://, content://, https://, or data: URL for video/image modes. */
+  sourceUrl?: string;
 }
 
 const PAGE_CSS = `
     html, body { margin:0; padding:0; width:100%; height:100%; background:#000; overflow:hidden; }
     #wrap { position:relative; width:100%; height:100%; background:#000; }
-    video, canvas {
+    video, img#still, canvas {
       position:absolute; inset:0; width:100%; height:100%;
       object-fit:cover; opacity:0; transition:opacity 180ms ease-out;
     }
-    video { z-index:1; background:#000; }
+    video, img#still { z-index:1; background:#000; }
     canvas { z-index:2; pointer-events:none; }
     #boot {
       position:absolute; inset:0; z-index:4;
@@ -204,11 +212,17 @@ export function buildPoseHtml(parts: OnlineRuntimeParts, options?: PoseHtmlOptio
     loadingText,
     showWatermark: watermarkOn,
     debugHud,
+    sourceType:
+      options?.sourceType === 'video' || options?.sourceType === 'image'
+        ? options.sourceType
+        : 'camera',
+    sourceUrl: typeof options?.sourceUrl === 'string' ? options.sourceUrl : null,
     captureConstraintMode: CAPTURE_CONSTRAINT_MODE,
     capturePriority,
     platform: Platform.OS,
     modelUrl: parts.modelUrl,
     modelId: parts.modelId,
+    modelKind: parts.modelKind,
     tfjsWasmPath: parts.tfjsWasmPath,
     inferFrameSkip:
       isAndroid && ANDROID_INFER_FRAME_SKIP > 0 ? ANDROID_INFER_FRAME_SKIP : 0,
@@ -228,6 +242,7 @@ export function buildPoseHtml(parts: OnlineRuntimeParts, options?: PoseHtmlOptio
     '<style>', PAGE_CSS, '</style></head><body>',
     '<div id="wrap">',
     '<video id="video" playsinline muted autoplay></video>',
+    '<img id="still" alt="" draggable="false" />',
     '<canvas id="overlay"></canvas>',
     '<div id="wm">',
     '<div class="wm-powered">powered by</div>',
@@ -247,6 +262,8 @@ export function buildPoseHtml(parts: OnlineRuntimeParts, options?: PoseHtmlOptio
     '<script>window.__PT_BUILD=', JSON.stringify(`${POSE_HTML_BUILD}/${parts.version}`),
     ';window.__PT_CONFIG=', config,
     ';window.__PT_MODEL_URL=', JSON.stringify(parts.modelUrl),
+    ';window.__PT_MODEL_ID=', JSON.stringify(parts.modelId),
+    ';window.__PT_MODEL_KIND=', JSON.stringify(parts.modelKind),
     ';window.__PT_WASM_PATH=', JSON.stringify(parts.tfjsWasmPath),
     ';window.__PT_MODEL_JSON=null',
     ';window.__PT_WEIGHTS_B64=null',

package/src/camera/WebViewPoseView.tsx

@@ -120,6 +120,18 @@ export interface WebViewPoseViewProps {
    * Fetched via `GET /api/sdk/skeleton`. Ignored if `skeletonDef` is set.
    */
   skeletonUuid?: string | null;
+  /**
+   * Input source. Default `camera` (getUserMedia).
+   * For `video` / `image`, the host app picks a file and passes `sourceUri`
+   * (file://, content://, https, data URL) or `sourceBase64`.
+   */
+  source?: 'camera' | 'video' | 'image';
+  /** URI injected into the WebView for video/image modes. */
+  sourceUri?: string;
+  /** Optional base64 payload (without data: prefix); paired with sourceMime. */
+  sourceBase64?: string;
+  /** MIME for sourceBase64 (default image/jpeg or video/mp4). */
+  sourceMime?: string;
   onPose?: (pose: Pose, stats: { inferenceTimeMs: number; timestampMs: number }) => void;
   children?: React.ReactNode;
 }
@@ -266,6 +278,8 @@ export function WebViewPoseView(props: WebViewPoseViewProps): React.ReactElement
       // injected via __PT_SET_SKELETON (avoids remounting on uuid fetch).
       skeletonDef: props.skeletonDef ?? null,
       capturePriority: client.getQualityState().capturePriority,
+      sourceType: props.source ?? 'camera',
+      sourceUrl: props.sourceUri,
     });
   }, [
     facingMode,
@@ -276,6 +290,8 @@ export function WebViewPoseView(props: WebViewPoseViewProps): React.ReactElement
     coldStart,
     loadingText,
     props.skeletonDef,
+    props.source,
+    props.sourceUri,
   ]);
 
   // Apply custom skeleton (uuid fetch or prop) without remounting the page.
@@ -307,6 +323,33 @@ export function WebViewPoseView(props: WebViewPoseViewProps): React.ReactElement
     );
   }, [isWebViewBackend, html, loadingText]);
 
+  // Live source switches after first HTML boot (CFG already has initial source).
+  const lastSourceHtmlRef = useRef<string | null>(null);
+  useEffect(() => {
+    if (!isWebViewBackend || !html) return;
+    if (lastSourceHtmlRef.current !== html) {
+      lastSourceHtmlRef.current = html;
+      return;
+    }
+    const type = props.source ?? 'camera';
+    const payload = JSON.stringify({
+      type,
+      url: props.sourceUri ?? null,
+      base64: props.sourceBase64 ?? null,
+      mime: props.sourceMime ?? null,
+    });
+    webRef.current?.injectJavaScript?.(
+      `window.__PT_SET_SOURCE && window.__PT_SET_SOURCE(${payload}); true;`,
+    );
+  }, [
+    isWebViewBackend,
+    html,
+    props.source,
+    props.sourceUri,
+    props.sourceBase64,
+    props.sourceMime,
+  ]);
+
   useEffect(() => {
     if (!(backend instanceof WebViewPoseBackend)) {
       return;

package/src/client.ts

@@ -69,7 +69,6 @@ import {
 import {
   toClassicNativeMessage,
   type ClassicMessageListener,
-  type ClassicNativeMessage,
 } from './events/classicMessage';
 import {
   FREE_PLAN_FEATURES_MESSAGE,
@@ -122,8 +121,8 @@ export interface PoseTrackerClientOptions extends ConfigureOptions {
   usageTracker?: UsageTracker;
   /**
    * Docs API `model` query parity (`movenet` default, `blazepose`, …).
-   * BlazePose is not wired in the light WebView yet — use `modelUrl` for a
-   * custom TF.js graph model. Ignored when {@link modelUrl} is set.
+   * BlazePose loads via CDN pose-detection in the WebView (heavier than
+   * MoveNet). Ignored when {@link modelUrl} is set.
    */
   model?: PoseModelAlias;
   /**
@@ -164,6 +163,12 @@ export interface PoseTrackerClientOptions extends ConfigureOptions {
    * triaging Android GPU issues in the field.
    */
   onDiagnostic?: (message: string) => void;
+  /**
+   * When true, exercise sessions request `engine_debug` snapshots from the
+   * remote engine (bundle ≥ 1.2.4). Overridable per call via
+   * `startExercise(id, { debug: true|false })`.
+   */
+  debugEngine?: boolean;
 }
 
 /**
@@ -181,6 +186,11 @@ export interface StartExerciseOptions {
   userHeightCm?: number;
   /** Device pitch in degrees — jump exercises compensate camera tilt. */
   devicePitchDeg?: number;
+  /**
+   * Request throttled `engine_debug` events for this session (movement FSM
+   * QA). Defaults to {@link PoseTrackerClientOptions.debugEngine}.
+   */
+  debug?: boolean;
 }
 
 export class PoseTrackerClient {
@@ -579,7 +589,7 @@ export class PoseTrackerClient {
     });
     this.options.onDiagnostic?.(
       `[posetracker-light] pose-runtime online version=${parts.version} ` +
-        `modelId=${parts.modelId} modelUrl=${parts.modelUrl}`,
+        `modelId=${parts.modelId} modelKind=${parts.modelKind} modelUrl=${parts.modelUrl}`,
     );
     return parts;
   }
@@ -1040,12 +1050,14 @@ export class PoseTrackerClient {
     this.stopExercise();
     this.currentExerciseId = exerciseId;
     this.keypointsSuppressionLogged = false;
+    const debug = options.debug ?? this.options.debugEngine === true;
     this.session = this.engine.createSession(
       {
         exercise,
         locale: this.options.locale ?? 'en',
         difficulty: options.difficulty,
         minGrade: this.features.minGrade ?? undefined,
+        debug,
         features: {
           angles: this.features.angles,
           recommendations: this.features.recommendations,
@@ -1054,6 +1066,10 @@ export class PoseTrackerClient {
       },
       (event) => this.emitFromSession(event),
     );
+    this.options.onDiagnostic?.(
+      `[posetracker] startExercise id=${exercise.id} difficulty=${options.difficulty ?? 'medium'} ` +
+        `debug=${debug} engine=${this.engine.version} minGrade=${this.features.minGrade ?? 'off'}`,
+    );
   }
 
   /** Custom engine session (jump_analysis / air_time_jump). */
@@ -1280,7 +1296,11 @@ export class PoseTrackerClient {
       // Skip the classic-message conversion entirely (per-frame allocation).
       return;
     }
-    const classic: ClassicNativeMessage = toClassicNativeMessage(event);
+    const classic = toClassicNativeMessage(event);
+    if (classic == null) {
+      // SDK-only events (e.g. engine_debug) have no classic equivalent.
+      return;
+    }
     this.messageListeners.forEach((l) => {
       try {
         l(classic);

package/src/engine/types.ts

@@ -39,6 +39,11 @@ export interface EngineSessionOptions {
   minGrade?: MinGrade;
   /** Emission flags — engines older than 1.1.0 ignore them (client re-filters). */
   features?: EngineSessionFeatures;
+  /**
+   * When true, the engine emits throttled `engine_debug` events (movement
+   * FSM snapshot). Requires engine bundle ≥ 1.2.4. Ignored by older engines.
+   */
+  debug?: boolean;
 }
 
 export interface EngineSession {

package/src/events/classicMessage.ts

@@ -59,9 +59,11 @@ export function anglesToClassicTree(angles: AngleValue[]): {
 /**
  * Map a typed SDK event to a classic PoseTracker `sendDataToNative` payload.
  * Returns null for SDK-only events that have no classic equivalent and should
- * not be mirrored (none today — quality/performance map to error/warning).
+ * not be mirrored (e.g. `engine_debug` QA snapshots).
  */
-export function toClassicNativeMessage(event: PoseTrackerEvent): ClassicNativeMessage {
+export function toClassicNativeMessage(
+  event: PoseTrackerEvent,
+): ClassicNativeMessage | null {
   switch (event.type) {
     case 'initialization':
       return {
@@ -247,6 +249,9 @@ export function toClassicNativeMessage(event: PoseTrackerEvent): ClassicNativeMe
         totalParts: event.totalParts,
         timestampMs: event.timestampMs,
       };
+    case 'engine_debug':
+      // SDK-only QA stream — no classic WebView equivalent; skip onMessage.
+      return null;
     default: {
       const _exhaustive: never = event;
       return { type: 'unknown', event: _exhaustive };

package/src/index.ts

@@ -35,7 +35,9 @@ export {
   getOnlineRuntimeParts,
   getOnlineRuntimeVersion,
   defaultOnlineRuntimeUrls,
+  defaultPoseDetectionCdnUrl,
   ONLINE_TFJS_VERSION,
+  ONLINE_POSE_DETECTION_VERSION,
 } from './backends/webview/onlineRuntime';
 export type {
   GetOnlineRuntimeOptions,
@@ -49,6 +51,7 @@ export {
 } from './models/poseModels';
 export type {
   PoseModelAlias,
+  PoseModelKind,
   ResolvePoseModelOptions,
   ResolvedPoseModel,
 } from './models/poseModels';

package/src/models/poseModels.ts

@@ -5,9 +5,9 @@
  * MoveNet SinglePose Lightning hosted at app.posetracker.com (same URL as
  * `usePoseDetection` / tracking WebView `modelUrl`).
  *
- * BlazePose is accepted as a `model` alias for API parity but is not wired
- * in the RN WebView yet (needs MediaPipe). Pass a TF.js `modelUrl` for any
- * custom graph model.
+ * BlazePose loads via CDN `@tensorflow-models/pose-detection` inside the
+ * WebView (no graph `modelUrl`). Pass a TF.js `modelUrl` for any custom
+ * graph model.
  */
 
 /** Production MoveNet SinglePose Lightning topology (Front default). */
@@ -32,17 +32,20 @@ export interface ResolvePoseModelOptions {
   modelUrl?: string;
 }
 
+export type PoseModelKind = 'movenet-graph' | 'blazepose' | 'custom-graph' | 'unsupported';
+
 export interface ResolvedPoseModel {
   /** Stable id reported in diagnostics / track params. */
   modelId: string;
-  /** TF.js `loadGraphModel` URL, or null when unsupported without override. */
+  kind: PoseModelKind;
+  /** TF.js `loadGraphModel` URL when kind is movenet-graph or custom-graph. */
   modelUrl: string | null;
-  /** Human-readable reason when {@link modelUrl} is null. */
+  /** Human-readable reason when kind is unsupported. */
   unsupportedReason?: string;
 }
 
 /**
- * Resolve which TF.js graph model the light WebView should fetch.
+ * Resolve which online pose model the light WebView should run.
  * Default: MoveNet SinglePose Lightning (product URL).
  */
 export function resolvePoseModel(options: ResolvePoseModelOptions = {}): ResolvedPoseModel {
@@ -55,30 +58,30 @@ export function resolvePoseModel(options: ResolvePoseModelOptions = {}): Resolve
       typeof options.model === 'string' && options.model.trim().length > 0
         ? options.model.trim()
         : 'custom';
-    return { modelId: alias, modelUrl: explicit };
+    return { modelId: alias, kind: 'custom-graph', modelUrl: explicit };
   }
 
   const key = (options.model ?? 'movenet').trim().toLowerCase();
   if (key === 'movenet' || key === 'movenet-singlepose-lightning' || key === 'lightning') {
     return {
       modelId: 'movenet-singlepose-lightning',
+      kind: 'movenet-graph',
       modelUrl: DEFAULT_MOVENET_LIGHTNING_URL,
     };
   }
   if (key === 'blazepose') {
     return {
       modelId: 'blazepose',
+      kind: 'blazepose',
       modelUrl: null,
-      unsupportedReason:
-        'BlazePose is not wired in the light RN WebView yet (needs MediaPipe). ' +
-        'Use model="movenet" or pass a TF.js graph modelUrl.',
     };
   }
   return {
     modelId: key,
+    kind: 'unsupported',
     modelUrl: null,
     unsupportedReason:
-      `Unknown model "${options.model}". Use "movenet" (default) or pass modelUrl.`,
+      `Unknown model "${options.model}". Use "movenet" (default), "blazepose", or pass modelUrl.`,
   };
 }
 

package/src/sdkVersion.ts

@@ -1,2 +1,2 @@
 /** AUTO-GENERATED by scripts/sync-sdk-version.mjs — do not edit. */
-export const SDK_VERSION = '0.1.0';
+export const SDK_VERSION = '0.2.1';

package/src/types/events.ts

@@ -38,6 +38,7 @@ export type InitializationStep =
   | 'downloading'
   | 'warming'
   | 'accessing_webcam'
+  | 'loading_media'
   | 'loading_pose_model'
   | 'ready';
 
@@ -197,6 +198,38 @@ export interface ProgressionEvent {
   timestampMs: number;
 }
 
+/**
+ * Throttled movement / rep-FSM snapshot for QA (engine ≥ 1.2.4).
+ * Enable via `PoseTrackerClientOptions.debugEngine` or `startExercise(..., { debug: true })`.
+ */
+export interface EngineDebugEvent {
+  type: 'engine_debug';
+  exerciseId: string;
+  engineVersion: string;
+  ready: boolean;
+  postureHint: string;
+  direction?: string;
+  missingKeypoints: string[];
+  stepIndex: number;
+  stepCount: number;
+  stepPosition: number | null;
+  stepProgression: number;
+  progression: number;
+  /** Raw FSM rep count (before minGrade gate). */
+  rawCount: number;
+  /** Public counter after minGrade gate. */
+  publicCount: number;
+  difficulty: string;
+  minGrade: MinGrade | null;
+  pointOfView: 'low' | 'up';
+  computeError?: string;
+  lastSkipReason: string | null;
+  posesProcessed: number;
+  visibleKeypoints: number;
+  knees: { left: number | null; right: number | null };
+  timestampMs: number;
+}
+
 export interface RecommendationsEvent {
   type: 'recommendations';
   /** Localized form advice strings. */
@@ -388,6 +421,7 @@ export type PoseTrackerEvent =
   | CounterEvent
   | PostureEvent
   | ProgressionEvent
+  | EngineDebugEvent
   | RecommendationsEvent
   | FormScoreEvent
   | ExerciseSummaryEvent
@@ -417,6 +451,7 @@ export interface PoseTrackerCallbacks {
   onCounter?: (event: CounterEvent) => void;
   onPosture?: (event: PostureEvent) => void;
   onProgression?: (event: ProgressionEvent) => void;
+  onEngineDebug?: (event: EngineDebugEvent) => void;
   onRecommendations?: (event: RecommendationsEvent) => void;
   onFormScore?: (event: FormScoreEvent) => void;
   onExerciseSummary?: (event: ExerciseSummaryEvent) => void;

package/src/types/features.ts

@@ -11,8 +11,8 @@
  * - `keypoints` DURING an exercise session requires a paid plan too
  *   (free may stream keypoints only in pose-only mode, i.e. no exercise).
  * - `blazepose` / `poseEngine` / `mediapipeModel` / `poseBackend` /
- *   `runInWorker` are NOT part of this SDK (MoveNet Lightning only) and are
- *   rejected with a clear error.
+ *   `runInWorker` are NOT feature flags here — select BlazePose with
+ *   `options.model = 'blazepose'` instead of `{ features: { blazepose: true } }`.
  */
 
 import type { MinGrade } from './events';
@@ -75,12 +75,12 @@ const KNOWN_FEATURE_KEYS = new Set<string>([
 ]);
 
 /**
- * WebView params that this SDK deliberately does NOT ship (one SDK, one
- * model: MoveNet Lightning). Passing them (possible from untyped JS hosts)
- * produces a clear `feature_not_supported` error.
+ * WebView params that are NOT feature flags in this SDK.
+ * Note: `options.model = 'blazepose'` is supported on light; passing
+ * `{ blazepose: true }` inside `features` is rejected (WebView query parity).
  */
 const UNSUPPORTED_FEATURE_HINTS: Record<string, string> = {
-  blazepose: 'BlazePose',
+  blazepose: 'BlazePose as a features flag (use options.model = "blazepose" instead)',
   poseEngine: 'pose engine selection (MediaPipe/PoseLandmarker)',
   mediapipeModel: 'MediaPipe model selection',
   poseBackend: 'pose backend selection',
@@ -108,9 +108,8 @@ export const COMBINED_REFERENCE_EXERCISE_MESSAGE =
 export function featureNotSupportedMessage(key: string): string {
   const hint = UNSUPPORTED_FEATURE_HINTS[key] ?? `'${key}'`;
   return (
-    `The '${key}' option (${hint}) is not available in this SDK. ` +
-    'This SDK ships MoveNet Lightning only — pose engine and model cannot be changed. ' +
-    'Remove the option, or use the PoseTracker WebView product if you need it. ' +
+    `The '${key}' option (${hint}) is not available as a features flag. ` +
+    'Remove the option, or use options.model / modelUrl for model selection. ' +
     '(visit: https://posetracker.gitbook.io/posetracker-api/tracking-endpoint)'
   );
 }