@pose-tracker/react-native-pose-estimation-light 0.3.5 → 0.3.6

package/README.md

@@ -133,6 +133,50 @@ Same contract as the offline SDK / web tracking URL:
 </PoseTrackerProvider>
 ```
 
+## External frames (your camera, our data)
+
+Full guide, including Vision Camera and expo-camera: https://docs.posetracker.com/external-frames
+
+Optional. Use it when your app already owns the camera (VisionCamera, a
+custom pipeline, a recorded file) and you only need the data. PoseTracker
+runs pose estimation and the exercise engine on every frame you push, and
+returns the same events as the camera flow: keypoints, posture/placement,
+counter, form score, progression. **Nothing is drawn**, so you render what you want.
+
+```tsx
+const { warmupExternal, startExercise, processFrame } = usePoseTracker({
+  onCounter: (e) => setReps(e.count), // listeners still fire
+});
+
+await warmupExternal(); // no camera permission, no getUserMedia
+startExercise('squat');
+
+// For each frame from your camera:
+const { dropped, pose, events } = await processFrame({
+  base64: jpegBase64, // or uri: 'file:///…' / 'data:image/jpeg;base64,…'
+  width: 256,
+  height: 192,
+  timestampMs: Date.now(),
+  mirrored: true, // front camera (default). false for the back camera.
+});
+if (!dropped) {
+  // pose.keypoints are normalized (0..1) to the frame you sent
+  // events: everything emitted for this frame (counter, posture, …)
+}
+```
+
+- The Provider mounts a hidden 1×1 warmer after `warmupExternal()`. It is not
+  needed if a `<WebViewPoseView />` is already on screen. The light package
+  still needs network for the model.
+- One frame in flight at a time: an overlapping call resolves at once with
+  `{ dropped: true, pose: lastPose, events: [] }`. Await before pushing the next one.
+- Keep the longest side of each frame at about 256 px or less. A 1080p JPEG per
+  frame is not real-time over the bridge.
+- The engine is temporal: push frames in order and keep the same session
+  across frames.
+- Don't mix the two flows. `processFrame` throws while the SDK camera is open.
+- `processFrame` before `warmupExternal()` throws.
+
 ## Cold-start
 
 | Mode | API | Camera permission |
@@ -169,6 +213,9 @@ offline MoveNet.
 **Who sees the watermark?**  
 Keyless and free plans. Hidden for paid plans.
 
+**How does Claude or Cursor integrate this SDK?**  
+Add the connector `https://mcp.posetracker.com/api/mcp`, sign in, and click Allow at `https://app.posetracker.com/oauth/authorize`. The agent writes the snippet on your account. Docs: https://docs.posetracker.com/ai/mcp-connector.
+
 ## License
 
 **Proprietary** — Movelytics SAS / PoseTracker. See [`LICENSE`](./LICENSE).

package/lib/PoseTrackerProvider.d.ts

@@ -17,6 +17,7 @@
 import React from 'react';
 import { PoseTrackerClient, type PoseTrackerClientOptions, type StartExerciseOptions } from './client';
 import type { PoseInputFrame } from './backends/PoseBackend';
+import type { ExternalFrame, ExternalFrameResult } from './types/externalFrame';
 import type { ExerciseConfig, SdkManifest } from './types/manifest';
 import type { PreloadOptions } from './types/preload';
 import type { ErrorEvent, PoseTrackerCallbacks, PoseTrackerEventListener, PoseTrackerMode, PoseTrackerStatus } from './types/events';
@@ -72,7 +73,18 @@ export interface PoseTrackerContextValue {
     startExercise: (exerciseId: string, options?: StartExerciseOptions) => void;
     stopExercise: () => void;
     estimatePose: (frame: PoseInputFrame) => Promise<Pose | null>;
-    processFrame: (frame: PoseInputFrame) => Promise<Pose | null>;
+    /**
+     * Pass `{ base64 | uri, width, height, timestampMs }` from your own camera
+     * after {@link warmupExternal}: resolves `{ dropped, pose, events }`.
+     * Any other frame keeps the original pose + engine behaviour.
+     */
+    processFrame: ProcessFrameFn;
+    /**
+     * Opt-in: load the model for frames from your own camera. Mounts a hidden
+     * 1×1 basic WebView (no camera, nothing drawn). Not needed with
+     * `WebViewPoseView`.
+     */
+    warmupExternal: () => Promise<void>;
     addEventListener: (listener: PoseTrackerEventListener) => () => void;
     /**
      * Classic PoseTracker WebView JSON stream (`sendDataToNative` shape).
@@ -80,6 +92,10 @@ export interface PoseTrackerContextValue {
      */
     addMessageListener: (listener: (message: import('./events/classicMessage').ClassicNativeMessage) => void) => () => void;
 }
+export interface ProcessFrameFn {
+    (frame: ExternalFrame): Promise<ExternalFrameResult>;
+    (frame: PoseInputFrame): Promise<Pose | null>;
+}
 export interface PoseTrackerProviderProps {
     /** Optional: without a token the SDK runs in keypoints-only mode. */
     apiToken?: string;

package/lib/PoseTrackerProvider.js

@@ -20,7 +20,9 @@ const jsx_runtime_1 = require("react/jsx-runtime");
  *   full-engine without restarting the camera pipeline.
  */
 const react_1 = require("react");
+const react_native_1 = require("react-native");
 const client_1 = require("./client");
+const WebViewPoseBackend_1 = require("./backends/webview/WebViewPoseBackend");
 const PoseTrackerContext = (0, react_1.createContext)(null);
 function PoseTrackerProvider({ apiToken, engine, options, autoPreload = false, children, }) {
     const clientRef = (0, react_1.useRef)(null);
@@ -39,8 +41,22 @@ function PoseTrackerProvider({ apiToken, engine, options, autoPreload = false, c
     const [acceleration, setAcceleration] = (0, react_1.useState)(client.getAcceleration());
     const [accelerationDiagnostics, setAccelerationDiagnostics] = (0, react_1.useState)(client.getAccelerationDiagnostics());
     const [quality, setQuality] = (0, react_1.useState)(client.getQualityState());
+    /** Hidden warmer for external frames; false unless warmupExternal() ran. */
+    const [mountExternalWarmer, setMountExternalWarmer] = (0, react_1.useState)(false);
     (0, react_1.useEffect)(() => {
         const offState = client.onStateChange(() => {
+            if (client.isExternalWarmerRequested()) {
+                setMountExternalWarmer((mounted) => {
+                    if (mounted)
+                        return true;
+                    const backend = client.getBackend();
+                    // Reuse a WebViewPoseView the host already mounted.
+                    return backend instanceof WebViewPoseBackend_1.WebViewPoseBackend && !backend.isAttached();
+                });
+            }
+            else {
+                setMountExternalWarmer(false);
+            }
             setStatus(client.getStatus());
             setMode(client.getMode());
             setError(client.getError());
@@ -69,7 +85,8 @@ function PoseTrackerProvider({ apiToken, engine, options, autoPreload = false, c
     const startExercise = (0, react_1.useCallback)((id, exerciseOptions) => client.startExercise(id, exerciseOptions), [client]);
     const stopExercise = (0, react_1.useCallback)(() => client.stopExercise(), [client]);
     const estimatePose = (0, react_1.useCallback)((frame) => client.estimatePose(frame), [client]);
-    const processFrame = (0, react_1.useCallback)((frame) => client.processFrame(frame), [client]);
+    const processFrame = (0, react_1.useCallback)(((frame) => client.processFrame(frame)), [client]);
+    const warmupExternal = (0, react_1.useCallback)(() => client.warmupExternal(), [client]);
     const addEventListener = (0, react_1.useCallback)((listener) => client.addEventListener(listener), [client]);
     const addMessageListener = (0, react_1.useCallback)((listener) => client.addMessageListener(listener), [client]);
     const value = (0, react_1.useMemo)(() => ({
@@ -89,6 +106,7 @@ function PoseTrackerProvider({ apiToken, engine, options, autoPreload = false, c
         stopExercise,
         estimatePose,
         processFrame,
+        warmupExternal,
         addEventListener,
         addMessageListener,
     }), [
@@ -107,11 +125,27 @@ function PoseTrackerProvider({ apiToken, engine, options, autoPreload = false, c
         stopExercise,
         estimatePose,
         processFrame,
+        warmupExternal,
         addEventListener,
         addMessageListener,
     ]);
-    return (0, jsx_runtime_1.jsx)(PoseTrackerContext.Provider, { value: value, children: children });
+    return ((0, jsx_runtime_1.jsxs)(PoseTrackerContext.Provider, { value: value, children: [children, mountExternalWarmer ? (0, jsx_runtime_1.jsx)(ExternalFramesWarmer, {}) : null] }));
+}
+/** Hidden basic page for external frames: no camera, nothing drawn. */
+function ExternalFramesWarmer() {
+    // Required lazily: WebViewPoseView itself imports this module.
+    const { WebViewPoseView } = require('./camera/WebViewPoseView');
+    return ((0, jsx_runtime_1.jsx)(react_native_1.View, { pointerEvents: "none", style: styles.externalWarmer, children: (0, jsx_runtime_1.jsx)(WebViewPoseView, { coldStart: "basic", drawSkeleton: false, drawPlacementBox: false, showWatermark: false }) }));
 }
+const styles = react_native_1.StyleSheet.create({
+    externalWarmer: {
+        position: 'absolute',
+        width: 1,
+        height: 1,
+        opacity: 0,
+        overflow: 'hidden',
+    },
+});
 /**
  * Access the PoseTracker pipeline and subscribe to typed events.
  *

package/lib/backends/webview/WebViewPoseBackend.d.ts

@@ -7,14 +7,16 @@
  * only one real-time across the whole Android park in Expo Go. Fully
  * offline: TF.js and the model ship inside the npm package (poseHtml.ts).
  *
- * The WebView owns the camera (`getUserMedia`). Frames are NOT pushed from
- * React Native — `estimatePose()` returns the latest pose posted by the page.
- * Mount {@link WebViewPoseView} (or a 1×1 warmer) to attach the runtime.
+ * By default the WebView owns the camera (`getUserMedia`) and
+ * `estimatePose()` returns the latest pose posted by the page. Mount
+ * {@link WebViewPoseView} (or a 1×1 warmer) to attach the runtime.
+ * Opt-in: {@link pushFrame} infers host-provided frames on a basic warmer.
  */
 import type { PoseBackend, PoseBackendInitOptions, PoseInputFrame } from '../PoseBackend';
 import type { Pose } from '../../types/pose';
 import type { AccelerationDiagnostics, AccelerationState, DiagnosticListener } from '../../types/acceleration';
 import type { PoseTrackerEvent } from '../../types/events';
+import type { ExternalFrame } from '../../types/externalFrame';
 export type WebViewPoseMessage = {
     type: 'ready';
     backend: string;
@@ -59,6 +61,13 @@ export type WebViewPoseMessage = {
     score: number;
     inferenceTimeMs: number;
     timestampMs: number;
+    /** Set when the pose answers a pushed external frame. */
+    frameId?: number;
+} | {
+    type: 'frame_result';
+    id: number;
+    dropped: boolean;
+    error?: string;
 } | {
     type: 'stats';
     fps: number;
@@ -172,6 +181,9 @@ export declare class WebViewPoseBackend implements PoseBackend {
     /** True after the page successfully opened getUserMedia. */
     private cameraOpened;
     private openCameraHandler;
+    private pushFrameHandler;
+    private frameSeq;
+    private readonly pendingFrames;
     private lastPose;
     private inferenceTimesMs;
     private medianInferenceMs;
@@ -192,11 +204,26 @@ export declare class WebViewPoseBackend implements PoseBackend {
     constructor(options?: WebViewPoseBackendOptions);
     /** Host view calls this when the WebView mounts / unmounts. */
     setAttached(attached: boolean): void;
+    isAttached(): boolean;
+    /** Model warm in the attached page (basic or full cold-start). */
+    isWarm(): boolean;
+    getLastPose(): Pose | null;
     setOnPose(handler: ((pose: Pose, inferenceTimeMs: number) => void) | undefined): void;
     setOnReady(handler: WebViewPoseBackendOptions['onReady']): void;
     setOnStats(handler: WebViewPoseBackendOptions['onStats']): void;
     /** Feed a message from the WebView `onMessage` handler. */
     handleMessage(raw: string): void;
+    /** Host view injects `__PT_PUSH_FRAME` with the JSON payload. */
+    setPushFrameHandler(handler: ((payload: string) => void) | undefined): void;
+    /**
+     * Infer one host-provided frame in the attached page. Resolves after the
+     * page posted the pose (already forwarded to `onPose`, hence the engine).
+     */
+    pushFrame(frame: ExternalFrame): Promise<{
+        dropped: boolean;
+        pose: Pose | null;
+    }>;
+    private rejectPendingFrames;
     init(_options: PoseBackendInitOptions): Promise<void>;
     warmup(): Promise<void>;
     /** Whether getUserMedia has already run in the attached page. */

package/lib/backends/webview/WebViewPoseBackend.js

@@ -8,13 +8,15 @@
  * only one real-time across the whole Android park in Expo Go. Fully
  * offline: TF.js and the model ship inside the npm package (poseHtml.ts).
  *
- * The WebView owns the camera (`getUserMedia`). Frames are NOT pushed from
- * React Native — `estimatePose()` returns the latest pose posted by the page.
- * Mount {@link WebViewPoseView} (or a 1×1 warmer) to attach the runtime.
+ * By default the WebView owns the camera (`getUserMedia`) and
+ * `estimatePose()` returns the latest pose posted by the page. Mount
+ * {@link WebViewPoseView} (or a 1×1 warmer) to attach the runtime.
+ * Opt-in: {@link pushFrame} infers host-provided frames on a basic warmer.
  */
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.WebViewPoseBackend = void 0;
 exports.isWebViewPoseBackend = isWebViewPoseBackend;
+const PUSH_FRAME_TIMEOUT_MS = 10000;
 class WebViewPoseBackend {
     constructor(options = {}) {
         this.name = 'webview-movenet';
@@ -24,6 +26,8 @@ class WebViewPoseBackend {
         this.warm = false;
         /** True after the page successfully opened getUserMedia. */
         this.cameraOpened = false;
+        this.frameSeq = 0;
+        this.pendingFrames = new Map();
         this.lastPose = null;
         this.inferenceTimesMs = [];
         this.medianInferenceMs = null;
@@ -54,8 +58,20 @@ class WebViewPoseBackend {
             // Drop stale GPU verdict from a previous warmer/page so the host does
             // not show acceleration=gpu while the next WebView is still booting.
             this.setAcceleration('unknown');
+            this.pushFrameHandler = undefined;
+            this.rejectPendingFrames('WebView detached while a frame was in flight.');
         }
     }
+    isAttached() {
+        return this.attached;
+    }
+    /** Model warm in the attached page (basic or full cold-start). */
+    isWarm() {
+        return this.warm;
+    }
+    getLastPose() {
+        return this.lastPose;
+    }
     setOnPose(handler) {
         this.onPose = handler;
     }
@@ -222,8 +238,66 @@ class WebViewPoseBackend {
                 timestampMs: msg.timestampMs,
             };
             this.lastPose = pose;
+            if (msg.frameId != null) {
+                const pending = this.pendingFrames.get(msg.frameId);
+                if (pending)
+                    pending.pose = pose;
+            }
             this.onPose?.(pose, msg.inferenceTimeMs);
+            return;
+        }
+        if (msg.type === 'frame_result') {
+            const pending = this.pendingFrames.get(msg.id);
+            if (!pending)
+                return;
+            this.pendingFrames.delete(msg.id);
+            clearTimeout(pending.timer);
+            if (msg.error) {
+                pending.reject(new Error(`processFrame: ${msg.error}`));
+            }
+            else {
+                pending.resolve({ dropped: msg.dropped === true, pose: pending.pose });
+            }
+        }
+    }
+    /** Host view injects `__PT_PUSH_FRAME` with the JSON payload. */
+    setPushFrameHandler(handler) {
+        this.pushFrameHandler = handler;
+    }
+    /**
+     * Infer one host-provided frame in the attached page. Resolves after the
+     * page posted the pose (already forwarded to `onPose`, hence the engine).
+     */
+    pushFrame(frame) {
+        const handler = this.pushFrameHandler;
+        if (!handler) {
+            return Promise.reject(new Error('processFrame: no WebViewPoseView is attached. Call warmupExternal() first.'));
+        }
+        const id = ++this.frameSeq;
+        return new Promise((resolve, reject) => {
+            const timer = setTimeout(() => {
+                this.pendingFrames.delete(id);
+                reject(new Error('processFrame: timed out waiting for the pose runtime.'));
+            }, PUSH_FRAME_TIMEOUT_MS);
+            this.pendingFrames.set(id, { pose: null, resolve, reject, timer });
+            handler(JSON.stringify({
+                id,
+                base64: frame.base64 ?? null,
+                uri: frame.uri ?? null,
+                mime: frame.mime ?? null,
+                width: frame.width,
+                height: frame.height,
+                timestampMs: frame.timestampMs,
+                mirrored: frame.mirrored !== false,
+            }));
+        });
+    }
+    rejectPendingFrames(message) {
+        for (const pending of this.pendingFrames.values()) {
+            clearTimeout(pending.timer);
+            pending.reject(new Error(`processFrame: ${message}`));
         }
+        this.pendingFrames.clear();
     }
     async init(_options) {
         this.onDiagnostic?.('[posetracker] WebViewPoseBackend.init — waiting for Chromium WebView to mount (WebViewPoseView)');
@@ -312,6 +386,8 @@ class WebViewPoseBackend {
         this.cameraOpened = false;
         this.attached = false;
         this.openCameraHandler = undefined;
+        this.pushFrameHandler = undefined;
+        this.rejectPendingFrames('client disposed.');
         this.lastPose = null;
         this.acceleration = 'unknown';
         this.readyWaiters = [];