@pose-tracker/react-native-pose-estimation-light 0.1.0

package/lib/backends/vision/VisionPoseBackend.js

@@ -0,0 +1,177 @@
+"use strict";
+/**
+ * Apple Vision implementation of `PoseBackend` (iOS only).
+ *
+ * Uses `VNDetectHumanBodyPoseRequest` via a VisionCamera frame-processor
+ * plugin shipped in this package's `ios/` folder. Up to 19 anatomical
+ * joints (COCO-17 + `neck` + `root`). Primary fast path is
+ * `PoseCameraView` → worklet → native plugin (CVPixelBuffer, no RGB copy).
+ *
+ * Never loaded in Expo Go (`isExpoGo()` guard). On Android,
+ * `init()` throws {@link VisionUnavailableError} so the client can fall
+ * back to TFLite / TF.js when selection is `'auto'`.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.VisionPoseBackend = exports.VisionUnavailableError = void 0;
+const react_native_1 = require("react-native");
+const optionalModules_1 = require("../../support/optionalModules");
+const optionalVision_1 = require("./optionalVision");
+const mapVisionJoints_1 = require("./mapVisionJoints");
+const DEFAULT_MAX_ACCEPTABLE_INFERENCE_MS = 50;
+/** Thrown when Apple Vision cannot run (wrong platform, Expo Go, missing pod). */
+class VisionUnavailableError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = 'VisionUnavailableError';
+    }
+}
+exports.VisionUnavailableError = VisionUnavailableError;
+class VisionPoseBackend {
+    constructor(options = {}) {
+        this.options = options;
+        this.name = 'apple-vision';
+        this.plugin = null;
+        this.warm = false;
+        this.acceleration = 'unknown';
+        this.inferenceTimesMs = [];
+        this.reasons = [];
+        this.maxAcceptableInferenceMs =
+            options.maxAcceptableInferenceMs ?? DEFAULT_MAX_ACCEPTABLE_INFERENCE_MS;
+        this.onDiagnostic = options.onDiagnostic;
+        this.onAccelerationChange = options.onAccelerationChange;
+    }
+    async init(_options) {
+        if (this.plugin) {
+            return;
+        }
+        if (react_native_1.Platform.OS !== 'ios') {
+            throw new VisionUnavailableError('Apple Vision body pose is iOS-only. Use preferredBackend "tflite" or "tfjs" on Android.');
+        }
+        if ((0, optionalModules_1.isExpoGo)()) {
+            throw new VisionUnavailableError('Apple Vision requires a native iOS build — not available in Expo Go.');
+        }
+        if (!(0, optionalVision_1.isVisionBackendAvailable)()) {
+            throw new VisionUnavailableError('Apple Vision frame-processor plugin is not linked. Rebuild with the SDK ios/ pod ' +
+                '(VisionCamera + PoseTrackerVision) and ensure react-native-vision-camera + worklets are installed.');
+        }
+        this.plugin = (0, optionalVision_1.getVisionBodyPosePlugin)();
+        if (!this.plugin) {
+            throw new VisionUnavailableError('Failed to initialize detectBodyPose frame-processor plugin.');
+        }
+        this.onDiagnostic?.('[posetracker] apple-vision backend ready (VNDetectHumanBodyPoseRequest, up to 19 joints)');
+    }
+    /**
+     * Vision has no separate model load — warm-up records the accelerator
+     * verdict. Real timings come from the first camera frames via
+     * {@link recordInferenceTime}.
+     */
+    async warmup() {
+        this.requirePlugin();
+        if (this.warm) {
+            return;
+        }
+        // Neural Engine / GPU path — Vision does not expose a CPU-only mode.
+        this.setAcceleration('gpu');
+        this.warm = true;
+        this.onDiagnostic?.(`[posetracker] apple-vision warm-up done: acceleration=${this.acceleration} ` +
+            '(per-frame ms reported from PoseCameraView)');
+    }
+    /**
+     * JS-path estimation. Accepts:
+     * - a VisionCamera `Frame` (has `.width`/`.height` and is callable via plugin);
+     * - a precomputed {@link VisionPluginResult} `{ joints }`.
+     *
+     * Prefer `PoseCameraView` (worklet) for production — this path is for
+     * TensorCamera-style hosts and unit tests.
+     */
+    async estimatePose(frame) {
+        const plugin = this.requirePlugin();
+        const asResult = frame;
+        if (asResult && Array.isArray(asResult.joints)) {
+            return (0, mapVisionJoints_1.poseFromVisionPluginResult)(asResult);
+        }
+        // VisionCamera Frame — structural duck-type (worklet Frame has width/height).
+        const anyFrame = frame;
+        if (anyFrame &&
+            typeof anyFrame.width === 'number' &&
+            typeof anyFrame.height === 'number' &&
+            typeof plugin.call === 'function') {
+            const start = Date.now();
+            const raw = plugin.call(frame);
+            const elapsed = Date.now() - start;
+            this.recordInferenceTime(elapsed);
+            return (0, mapVisionJoints_1.poseFromVisionPluginResult)(raw ?? undefined);
+        }
+        throw new Error('VisionPoseBackend: pass a VisionCamera Frame, a { joints } plugin result, ' +
+            'or use PoseCameraView (worklet path).');
+    }
+    getAcceleration() {
+        if (!this.plugin && this.acceleration === 'unknown') {
+            return null;
+        }
+        return {
+            state: this.acceleration,
+            tfjsBackend: null,
+            runtime: 'vision',
+            delegate: 'apple-vision',
+            medianInferenceMs: median(this.inferenceTimesMs),
+            inferenceTimesMs: [...this.inferenceTimesMs],
+            maxAcceptableInferenceMs: this.maxAcceptableInferenceMs,
+            capabilities: null,
+            flags: {},
+            contextLossCount: 0,
+            reasons: [...this.reasons],
+        };
+    }
+    async dispose() {
+        this.plugin = null;
+        this.warm = false;
+        this.acceleration = 'unknown';
+        this.inferenceTimesMs = [];
+        this.reasons = [];
+    }
+    /** Plugin handle for PoseCameraView worklets. */
+    getFrameProcessorPlugin() {
+        return this.plugin;
+    }
+    /** Feed live inference timings from the camera worklet into diagnostics. */
+    recordInferenceTime(ms) {
+        this.inferenceTimesMs.push(ms);
+        if (this.inferenceTimesMs.length > 30) {
+            this.inferenceTimesMs.shift();
+        }
+        const med = median(this.inferenceTimesMs);
+        if (med !== null && med > this.maxAcceptableInferenceMs && this.acceleration === 'gpu') {
+            this.noteReason(`apple-vision median ${med.toFixed(0)} ms exceeds ${this.maxAcceptableInferenceMs} ms budget`);
+            this.setAcceleration('cpu-fallback');
+        }
+    }
+    setAcceleration(state) {
+        if (this.acceleration !== state) {
+            this.acceleration = state;
+            this.onAccelerationChange?.(state);
+        }
+    }
+    noteReason(reason) {
+        this.reasons.push(reason);
+        this.onDiagnostic?.(`[posetracker] ${reason}`);
+    }
+    requirePlugin() {
+        if (!this.plugin) {
+            throw new Error('VisionPoseBackend not initialized — call init() first.');
+        }
+        return this.plugin;
+    }
+}
+exports.VisionPoseBackend = VisionPoseBackend;
+function median(values) {
+    if (values.length === 0) {
+        return null;
+    }
+    const sorted = [...values].sort((a, b) => a - b);
+    const mid = Math.floor(sorted.length / 2);
+    if (sorted.length % 2 === 0) {
+        return (sorted[mid - 1] + sorted[mid]) / 2;
+    }
+    return sorted[mid];
+}

package/lib/backends/vision/mapVisionJoints.d.ts

@@ -0,0 +1,35 @@
+/**
+ * Apple Vision body-pose joint → SDK keypoint mapping.
+ *
+ * `VNDetectHumanBodyPoseRequest` exposes 19 joints. Seventeen align 1:1 with
+ * the COCO / MoveNet topology; `neck` and `root` (mid-hip) are Vision extras
+ * kept under their Vision names so A/B overlays can show the full set.
+ *
+ * Vision normalized points use a **bottom-left** origin — the native plugin
+ * already flips Y (`y = 1 - visionY`) before returning joints here.
+ *
+ * `poseFromVisionPluginResult` carries a `'worklet'` directive so
+ * PoseCameraView can call it on the frame-processor thread.
+ */
+import type { KeypointName, Pose } from '../../types/pose';
+/** Vision `VNHumanBodyPoseObservation.JointName` camelCase → SDK name. */
+export declare const VISION_JOINT_TO_SDK: Readonly<Record<string, KeypointName>>;
+/** One joint as returned by the native frame-processor plugin. */
+export interface VisionPluginJoint {
+    name: string;
+    x: number;
+    y: number;
+    score: number;
+}
+/** Payload of `detectBodyPose` frame-processor plugin (already Y-flipped). */
+export interface VisionPluginResult {
+    joints: VisionPluginJoint[];
+    /** Wall time of the Vision request on the frame-processor thread, ms. */
+    inferenceTimeMs?: number;
+}
+/**
+ * Build a SDK `Pose` from the native plugin result. Unknown joint names are
+ * dropped; missing joints are filled with score 0 so overlays stay
+ * topology-stable when a joint is absent.
+ */
+export declare function poseFromVisionPluginResult(result: VisionPluginResult | null | undefined, timestampMs?: number): Pose | null;

package/lib/backends/vision/mapVisionJoints.js

@@ -0,0 +1,162 @@
+"use strict";
+/**
+ * Apple Vision body-pose joint → SDK keypoint mapping.
+ *
+ * `VNDetectHumanBodyPoseRequest` exposes 19 joints. Seventeen align 1:1 with
+ * the COCO / MoveNet topology; `neck` and `root` (mid-hip) are Vision extras
+ * kept under their Vision names so A/B overlays can show the full set.
+ *
+ * Vision normalized points use a **bottom-left** origin — the native plugin
+ * already flips Y (`y = 1 - visionY`) before returning joints here.
+ *
+ * `poseFromVisionPluginResult` carries a `'worklet'` directive so
+ * PoseCameraView can call it on the frame-processor thread.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.VISION_JOINT_TO_SDK = void 0;
+exports.poseFromVisionPluginResult = poseFromVisionPluginResult;
+/** Vision `VNHumanBodyPoseObservation.JointName` camelCase → SDK name. */
+exports.VISION_JOINT_TO_SDK = {
+    nose: 'nose',
+    leftEye: 'left_eye',
+    rightEye: 'right_eye',
+    leftEar: 'left_ear',
+    rightEar: 'right_ear',
+    leftShoulder: 'left_shoulder',
+    rightShoulder: 'right_shoulder',
+    leftElbow: 'left_elbow',
+    rightElbow: 'right_elbow',
+    leftWrist: 'left_wrist',
+    rightWrist: 'right_wrist',
+    leftHip: 'left_hip',
+    rightHip: 'right_hip',
+    leftKnee: 'left_knee',
+    rightKnee: 'right_knee',
+    leftAnkle: 'left_ankle',
+    rightAnkle: 'right_ankle',
+    neck: 'neck',
+    root: 'root',
+};
+/**
+ * Build a SDK `Pose` from the native plugin result. Unknown joint names are
+ * dropped; missing joints are filled with score 0 so overlays stay
+ * topology-stable when a joint is absent.
+ */
+function poseFromVisionPluginResult(result, timestampMs = Date.now()) {
+    'worklet';
+    // Inlined — worklets cannot capture module-level imports reliably.
+    const ORDER = [
+        'nose',
+        'left_eye',
+        'right_eye',
+        'left_ear',
+        'right_ear',
+        'left_shoulder',
+        'right_shoulder',
+        'left_elbow',
+        'right_elbow',
+        'left_wrist',
+        'right_wrist',
+        'left_hip',
+        'right_hip',
+        'left_knee',
+        'right_knee',
+        'left_ankle',
+        'right_ankle',
+        'neck',
+        'root',
+    ];
+    const ALIAS = {
+        nose: 'nose',
+        leftEye: 'left_eye',
+        rightEye: 'right_eye',
+        leftEar: 'left_ear',
+        rightEar: 'right_ear',
+        leftShoulder: 'left_shoulder',
+        rightShoulder: 'right_shoulder',
+        leftElbow: 'left_elbow',
+        rightElbow: 'right_elbow',
+        leftWrist: 'left_wrist',
+        rightWrist: 'right_wrist',
+        leftHip: 'left_hip',
+        rightHip: 'right_hip',
+        leftKnee: 'left_knee',
+        rightKnee: 'right_knee',
+        leftAnkle: 'left_ankle',
+        rightAnkle: 'right_ankle',
+        neck: 'neck',
+        root: 'root',
+        left_eye: 'left_eye',
+        right_eye: 'right_eye',
+        left_ear: 'left_ear',
+        right_ear: 'right_ear',
+        left_shoulder: 'left_shoulder',
+        right_shoulder: 'right_shoulder',
+        left_elbow: 'left_elbow',
+        right_elbow: 'right_elbow',
+        left_wrist: 'left_wrist',
+        right_wrist: 'right_wrist',
+        left_hip: 'left_hip',
+        right_hip: 'right_hip',
+        left_knee: 'left_knee',
+        right_knee: 'right_knee',
+        left_ankle: 'left_ankle',
+        right_ankle: 'right_ankle',
+    };
+    if (!result || !result.joints || result.joints.length === 0) {
+        return null;
+    }
+    const byName = {};
+    for (let i = 0; i < result.joints.length; i++) {
+        const joint = result.joints[i];
+        if (!joint) {
+            continue;
+        }
+        const mapped = ALIAS[joint.name];
+        if (!mapped) {
+            continue;
+        }
+        const x = clamp01(joint.x);
+        const y = clamp01(joint.y);
+        const score = clamp01(joint.score);
+        byName[mapped] = { name: mapped, x, y, score };
+    }
+    const keypoints = [];
+    let scoreSum = 0;
+    let scoreCount = 0;
+    for (let i = 0; i < ORDER.length; i++) {
+        const name = ORDER[i];
+        const existing = byName[name];
+        if (existing) {
+            keypoints.push(existing);
+            if (existing.score > 0) {
+                scoreSum += existing.score;
+                scoreCount += 1;
+            }
+        }
+        else {
+            keypoints.push({ name, x: 0, y: 0, score: 0 });
+        }
+    }
+    if (scoreCount === 0) {
+        return null;
+    }
+    return {
+        keypoints,
+        score: scoreSum / scoreCount,
+        timestampMs,
+    };
+}
+function clamp01(n) {
+    'worklet';
+    if (!Number.isFinite(n)) {
+        return 0;
+    }
+    if (n < 0) {
+        return 0;
+    }
+    if (n > 1) {
+        return 1;
+    }
+    return n;
+}

package/lib/backends/vision/optionalVision.d.ts

@@ -0,0 +1,32 @@
+/**
+ * Guarded access to the Apple Vision body-pose frame-processor plugin.
+ *
+ * The plugin lives in this package's `ios/` folder (VisionCamera Swift
+ * FrameProcessorPlugin + `VNDetectHumanBodyPoseRequest`). It is only linked
+ * in native iOS builds — never in Expo Go. All requires are gated by
+ * `Platform.OS === 'ios' && !isExpoGo()` so Metro/Expo Go never evaluate
+ * Nitro/VisionCamera code paths unintentionally.
+ */
+/** JS name registered via `VISION_EXPORT_SWIFT_FRAME_PROCESSOR(..., detectBodyPose)`. */
+export declare const VISION_BODY_POSE_PLUGIN_NAME = "detectBodyPose";
+export interface VisionFrameProcessorPlugin {
+    call(frame: unknown, args?: Record<string, unknown>): unknown;
+}
+/**
+ * True when we are on a native iOS build (not Expo Go) where the Vision
+ * frame-processor plugin can be loaded. Does not require Nitro / TFLite.
+ */
+export declare function isVisionBackendAvailable(): boolean;
+/**
+ * vision-camera + worklets present on iOS native build — enough to mount the
+ * Vision path of `PoseCameraView` once the backend has initialized.
+ */
+export declare function isVisionPoseCameraAvailable(): boolean;
+/**
+ * Lazily init the VisionCamera frame-processor plugin. Cached; `null` when
+ * the native binary does not include our Swift plugin (Expo Go, Android,
+ * or a host that excluded the pod).
+ */
+export declare function getVisionBodyPosePlugin(): VisionFrameProcessorPlugin | null;
+/** Test helper — clears the plugin cache between availability probes. */
+export declare function resetVisionPluginCacheForTests(): void;

package/lib/backends/vision/optionalVision.js

@@ -0,0 +1,76 @@
+"use strict";
+/**
+ * Guarded access to the Apple Vision body-pose frame-processor plugin.
+ *
+ * The plugin lives in this package's `ios/` folder (VisionCamera Swift
+ * FrameProcessorPlugin + `VNDetectHumanBodyPoseRequest`). It is only linked
+ * in native iOS builds — never in Expo Go. All requires are gated by
+ * `Platform.OS === 'ios' && !isExpoGo()` so Metro/Expo Go never evaluate
+ * Nitro/VisionCamera code paths unintentionally.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.VISION_BODY_POSE_PLUGIN_NAME = void 0;
+exports.isVisionBackendAvailable = isVisionBackendAvailable;
+exports.isVisionPoseCameraAvailable = isVisionPoseCameraAvailable;
+exports.getVisionBodyPosePlugin = getVisionBodyPosePlugin;
+exports.resetVisionPluginCacheForTests = resetVisionPluginCacheForTests;
+const react_native_1 = require("react-native");
+const optionalModules_1 = require("../../support/optionalModules");
+/** JS name registered via `VISION_EXPORT_SWIFT_FRAME_PROCESSOR(..., detectBodyPose)`. */
+exports.VISION_BODY_POSE_PLUGIN_NAME = 'detectBodyPose';
+let cachedPlugin;
+/**
+ * True when we are on a native iOS build (not Expo Go) where the Vision
+ * frame-processor plugin can be loaded. Does not require Nitro / TFLite.
+ */
+function isVisionBackendAvailable() {
+    if (react_native_1.Platform.OS !== 'ios' || (0, optionalModules_1.isExpoGo)()) {
+        return false;
+    }
+    if ((0, optionalModules_1.getVisionCamera)() === null || (0, optionalModules_1.getWorkletsCore)() === null) {
+        return false;
+    }
+    return getVisionBodyPosePlugin() !== null;
+}
+/**
+ * vision-camera + worklets present on iOS native build — enough to mount the
+ * Vision path of `PoseCameraView` once the backend has initialized.
+ */
+function isVisionPoseCameraAvailable() {
+    return (react_native_1.Platform.OS === 'ios' &&
+        !(0, optionalModules_1.isExpoGo)() &&
+        (0, optionalModules_1.getVisionCamera)() !== null &&
+        (0, optionalModules_1.getWorkletsCore)() !== null);
+}
+/**
+ * Lazily init the VisionCamera frame-processor plugin. Cached; `null` when
+ * the native binary does not include our Swift plugin (Expo Go, Android,
+ * or a host that excluded the pod).
+ */
+function getVisionBodyPosePlugin() {
+    if (cachedPlugin !== undefined) {
+        return cachedPlugin;
+    }
+    if (react_native_1.Platform.OS !== 'ios' || (0, optionalModules_1.isExpoGo)()) {
+        cachedPlugin = null;
+        return cachedPlugin;
+    }
+    try {
+        const mod = require('react-native-vision-camera');
+        const proxy = mod.VisionCameraProxy;
+        if (!proxy || typeof proxy.initFrameProcessorPlugin !== 'function') {
+            cachedPlugin = null;
+            return cachedPlugin;
+        }
+        const plugin = proxy.initFrameProcessorPlugin(exports.VISION_BODY_POSE_PLUGIN_NAME, {});
+        cachedPlugin = plugin ?? null;
+    }
+    catch {
+        cachedPlugin = null;
+    }
+    return cachedPlugin;
+}
+/** Test helper — clears the plugin cache between availability probes. */
+function resetVisionPluginCacheForTests() {
+    cachedPlugin = undefined;
+}