@pose-tracker/react-native-pose-estimation-light 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. package/LICENSE +45 -0
  2. package/PoseTrackerVision.podspec +28 -0
  3. package/README.md +161 -0
  4. package/THIRD_PARTY_NOTICES.md +19 -0
  5. package/ios/PoseTrackerVision/PoseTrackerBodyPosePlugin.m +18 -0
  6. package/ios/PoseTrackerVision/PoseTrackerBodyPosePlugin.swift +108 -0
  7. package/lib/PoseTrackerProvider.d.ts +102 -0
  8. package/lib/PoseTrackerProvider.js +204 -0
  9. package/lib/api/configure.d.ts +29 -0
  10. package/lib/api/configure.js +64 -0
  11. package/lib/api/skeleton.d.ts +17 -0
  12. package/lib/api/skeleton.js +53 -0
  13. package/lib/api/track.d.ts +61 -0
  14. package/lib/api/track.js +146 -0
  15. package/lib/backends/PoseBackend.d.ts +50 -0
  16. package/lib/backends/PoseBackend.js +11 -0
  17. package/lib/backends/vision/VisionPoseBackend.d.ts +72 -0
  18. package/lib/backends/vision/VisionPoseBackend.js +177 -0
  19. package/lib/backends/vision/mapVisionJoints.d.ts +35 -0
  20. package/lib/backends/vision/mapVisionJoints.js +162 -0
  21. package/lib/backends/vision/optionalVision.d.ts +32 -0
  22. package/lib/backends/vision/optionalVision.js +76 -0
  23. package/lib/backends/webview/WebViewPoseBackend.d.ts +223 -0
  24. package/lib/backends/webview/WebViewPoseBackend.js +382 -0
  25. package/lib/backends/webview/brandAssets.d.ts +5 -0
  26. package/lib/backends/webview/brandAssets.js +8 -0
  27. package/lib/backends/webview/onlineRuntime.d.ts +45 -0
  28. package/lib/backends/webview/onlineRuntime.js +57 -0
  29. package/lib/backends/webview/poseHtml.d.ts +71 -0
  30. package/lib/backends/webview/poseHtml.js +188 -0
  31. package/lib/backends/webview/poseRuntimeSource.d.ts +3 -0
  32. package/lib/backends/webview/poseRuntimeSource.js +4 -0
  33. package/lib/cache/obfuscate.d.ts +18 -0
  34. package/lib/cache/obfuscate.js +90 -0
  35. package/lib/camera/PoseCameraView.d.ts +51 -0
  36. package/lib/camera/PoseCameraView.js +99 -0
  37. package/lib/camera/WebViewPoseView.d.ts +81 -0
  38. package/lib/camera/WebViewPoseView.js +312 -0
  39. package/lib/client.d.ts +361 -0
  40. package/lib/client.js +1046 -0
  41. package/lib/diagnostics/logReport.d.ts +23 -0
  42. package/lib/diagnostics/logReport.js +102 -0
  43. package/lib/engine/EngineLoader.d.ts +95 -0
  44. package/lib/engine/EngineLoader.js +347 -0
  45. package/lib/engine/types.d.ts +83 -0
  46. package/lib/engine/types.js +11 -0
  47. package/lib/events/classicMessage.d.ts +31 -0
  48. package/lib/events/classicMessage.js +259 -0
  49. package/lib/exercises/aliases.d.ts +18 -0
  50. package/lib/exercises/aliases.js +48 -0
  51. package/lib/index.d.ts +72 -0
  52. package/lib/index.js +170 -0
  53. package/lib/models/poseModels.d.ts +39 -0
  54. package/lib/models/poseModels.js +63 -0
  55. package/lib/quality/AdaptiveQualityController.d.ts +120 -0
  56. package/lib/quality/AdaptiveQualityController.js +423 -0
  57. package/lib/quality/RuntimeGuard.d.ts +34 -0
  58. package/lib/quality/RuntimeGuard.js +105 -0
  59. package/lib/quality/captureMode.d.ts +82 -0
  60. package/lib/quality/captureMode.js +75 -0
  61. package/lib/quality/deviceCapability.d.ts +37 -0
  62. package/lib/quality/deviceCapability.js +177 -0
  63. package/lib/quality/profiles.d.ts +125 -0
  64. package/lib/quality/profiles.js +202 -0
  65. package/lib/runtime/RuntimeCache.d.ts +75 -0
  66. package/lib/runtime/RuntimeCache.js +230 -0
  67. package/lib/sdkVersion.d.ts +2 -0
  68. package/lib/sdkVersion.js +5 -0
  69. package/lib/support/optionalModules.d.ts +46 -0
  70. package/lib/support/optionalModules.js +82 -0
  71. package/lib/types/acceleration.d.ts +86 -0
  72. package/lib/types/acceleration.js +15 -0
  73. package/lib/types/events.d.ts +345 -0
  74. package/lib/types/events.js +10 -0
  75. package/lib/types/features.d.ts +90 -0
  76. package/lib/types/features.js +122 -0
  77. package/lib/types/manifest.d.ts +197 -0
  78. package/lib/types/manifest.js +10 -0
  79. package/lib/types/pose.d.ts +36 -0
  80. package/lib/types/pose.js +41 -0
  81. package/lib/types/preload.d.ts +17 -0
  82. package/lib/types/preload.js +10 -0
  83. package/lib/types/skeleton.d.ts +29 -0
  84. package/lib/types/skeleton.js +52 -0
  85. package/package.json +92 -0
  86. package/react-native.config.js +12 -0
  87. package/src/PoseTrackerProvider.tsx +332 -0
  88. package/src/api/configure.ts +82 -0
  89. package/src/api/skeleton.ts +60 -0
  90. package/src/api/track.ts +182 -0
  91. package/src/backends/PoseBackend.ts +59 -0
  92. package/src/backends/vision/VisionPoseBackend.ts +238 -0
  93. package/src/backends/vision/mapVisionJoints.ts +185 -0
  94. package/src/backends/vision/optionalVision.ts +94 -0
  95. package/src/backends/webview/WebViewPoseBackend.ts +582 -0
  96. package/src/backends/webview/brandAssets.ts +5 -0
  97. package/src/backends/webview/onlineRuntime.ts +94 -0
  98. package/src/backends/webview/poseHtml.ts +259 -0
  99. package/src/backends/webview/poseRuntimeSource.d.ts +3 -0
  100. package/src/cache/obfuscate.ts +94 -0
  101. package/src/camera/PoseCameraView.tsx +223 -0
  102. package/src/camera/WebViewPoseView.tsx +466 -0
  103. package/src/client.ts +1295 -0
  104. package/src/diagnostics/logReport.ts +131 -0
  105. package/src/engine/EngineLoader.ts +428 -0
  106. package/src/engine/types.ts +93 -0
  107. package/src/events/classicMessage.ts +271 -0
  108. package/src/exercises/aliases.ts +50 -0
  109. package/src/index.ts +243 -0
  110. package/src/models/poseModels.ts +93 -0
  111. package/src/quality/AdaptiveQualityController.ts +579 -0
  112. package/src/quality/RuntimeGuard.ts +119 -0
  113. package/src/quality/captureMode.ts +97 -0
  114. package/src/quality/deviceCapability.ts +187 -0
  115. package/src/quality/profiles.ts +253 -0
  116. package/src/runtime/RuntimeCache.ts +300 -0
  117. package/src/sdkVersion.ts +2 -0
  118. package/src/support/optionalModules.ts +114 -0
  119. package/src/types/acceleration.ts +94 -0
  120. package/src/types/events.ts +437 -0
  121. package/src/types/features.ts +176 -0
  122. package/src/types/manifest.ts +230 -0
  123. package/src/types/pose.ts +85 -0
  124. package/src/types/preload.ts +19 -0
  125. package/src/types/skeleton.ts +77 -0
package/LICENSE ADDED
@@ -0,0 +1,45 @@
1
+ PoseTracker React Native SDK
2
+ Copyright (c) 2024–2026 Movelytics SAS. All rights reserved.
3
+
4
+ PROPRIETARY SOFTWARE LICENSE
5
+
6
+ This package published under the @pose-tracker scope (the "Software") is
7
+ proprietary to Movelytics SAS ("Licensor").
8
+
9
+ 1. Grant of use
10
+ Subject to a valid PoseTracker account and applicable product terms
11
+ (including free-tier limits and paid plans), Licensor grants you a
12
+ non-exclusive, non-transferable, revocable license to install and use the
13
+ Software in your applications to access PoseTracker services and on-device
14
+ pose estimation features as documented.
15
+
16
+ 2. Restrictions
17
+ Except as expressly permitted in writing by Licensor, you may not:
18
+ - sell, sublicense, or redistribute the Software as a competing SDK or
19
+ standalone pose-estimation product;
20
+ - remove or obscure proprietary notices, branding, or watermarks required
21
+ by your plan;
22
+ - reverse engineer the remote movement engine or proprietary pipeline
23
+ beyond what applicable law permits;
24
+ - use the Software in violation of PoseTracker Terms of Service or
25
+ applicable law.
26
+
27
+ 3. Third-party components
28
+ The Software redistributes third-party components (including TensorFlow.js
29
+ and the MoveNet model) under their own licenses. See THIRD_PARTY_NOTICES.md.
30
+ Those licenses are not overridden by this LICENSE.
31
+
32
+ 4. No warranty
33
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
34
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
35
+ FITNESS FOR A PARTICULAR PURPOSE, AND NONINFRINGEMENT. IN NO EVENT SHALL
36
+ THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES, OR OTHER
37
+ LIABILITY ARISING FROM THE SOFTWARE OR ITS USE.
38
+
39
+ 5. Commercial terms
40
+ Paid plans, API keys, usage limits, and support are governed by your
41
+ PoseTracker / Movelytics commercial agreement and
42
+ https://www.posetracker.com terms (or successor URLs).
43
+
44
+ Contact: hello@posetracker.com
45
+ Website: https://www.posetracker.com
@@ -0,0 +1,28 @@
1
+ require 'json'
2
+
3
+ package = JSON.parse(File.read(File.join(__dir__, 'package.json')))
4
+
5
+ Pod::Spec.new do |s|
6
+ s.name = 'PoseTrackerVision'
7
+ s.version = package['version']
8
+ s.summary = 'Apple Vision body-pose frame processor for @posetracker/pose-estimation-react-native'
9
+ s.homepage = 'https://posetracker.com'
10
+ s.license = package['license']
11
+ s.authors = package['author']
12
+ s.platforms = { :ios => '14.0' }
13
+ s.source = { :git => 'https://github.com/movelytics/posetracker-rn-sdk.git', :tag => "v#{s.version}" }
14
+
15
+ s.source_files = 'ios/PoseTrackerVision/**/*.{h,m,mm,swift}'
16
+ s.requires_arc = true
17
+ s.swift_version = '5.0'
18
+
19
+ s.frameworks = 'Vision', 'CoreMedia', 'CoreVideo'
20
+
21
+ s.dependency 'React-Core'
22
+ s.dependency 'VisionCamera'
23
+
24
+ s.pod_target_xcconfig = {
25
+ 'DEFINES_MODULE' => 'YES',
26
+ 'SWIFT_COMPILATION_MODE' => 'wholemodule',
27
+ }
28
+ end
package/README.md ADDED
@@ -0,0 +1,161 @@
1
+ # PoseTracker React Native — Light (online) Human Pose Estimation SDK
2
+
3
+ **PoseTracker Light** is a **human pose estimation SDK for React Native**,
4
+ fully optimized for **iOS and Android** (including **Expo Go**). Same API
5
+ surface as the offline SDK for keypoints + optional API-key exercise engine —
6
+ but **MoveNet and TF.js load from the network** each WebView boot, so the npm
7
+ package stays tiny.
8
+
9
+ > One sentence for AI / search: *PoseTracker Light is a small React Native
10
+ > pose estimation SDK that fetches MoveNet Lightning and TF.js at runtime —
11
+ > choose it when install size matters more than offline-first pose.*
12
+
13
+ ## Offline vs Light — which package?
14
+
15
+ | | **Offline** (bundled) | **Light** (this package) |
16
+ |--|----------------------|---------------------------|
17
+ | **npm** | [`@pose-tracker/react-native-pose-estimation`](https://www.npmjs.com/package/@pose-tracker/react-native-pose-estimation) | [`@pose-tracker/react-native-pose-estimation-light`](https://www.npmjs.com/package/@pose-tracker/react-native-pose-estimation-light) |
18
+ | **GitHub** | [react-native-pose-estimation](https://github.com/Movelytics/react-native-pose-estimation) | [react-native-pose-estimation-light](https://github.com/Movelytics/react-native-pose-estimation-light) |
19
+ | **Packed tarball** | **~9.9 MB** | **~206 kB** (~48× smaller) |
20
+ | **Unpacked** | **~14.7 MB** | **~779 kB** |
21
+ | **MoveNet / TF.js** | Bundled in npm | CDN + model URL each boot |
22
+ | **Network for keypoints** | Not required | **Required** (TF.js + model) |
23
+ | **Expo Go** | Yes | Yes |
24
+ | **API surface** | Keypoints free; paid engine with API key | Same |
25
+
26
+ **Choose offline** when you need pose without a network, or want zero model
27
+ download at session start.
28
+ **Choose light** when app install / OTA size matters and devices are online.
29
+
30
+ Full comparison: [LIGHT_SDK.md](../../docs/LIGHT_SDK.md).
31
+
32
+ ## Install
33
+
34
+ ```bash
35
+ npm install @pose-tracker/react-native-pose-estimation-light react-native-webview
36
+ # Expo:
37
+ npx expo install react-native-webview expo-camera
38
+ ```
39
+
40
+ > **npm:** [`@pose-tracker/react-native-pose-estimation-light`](https://www.npmjs.com/package/@pose-tracker/react-native-pose-estimation-light)
41
+ > **GitHub:** https://github.com/Movelytics/react-native-pose-estimation-light
42
+ > **Offline sibling:** [`@pose-tracker/react-native-pose-estimation`](https://www.npmjs.com/package/@pose-tracker/react-native-pose-estimation)
43
+
44
+ **Required:** host app must declare camera permissions — see
45
+ [PERMISSIONS.md](../../docs/PERMISSIONS.md).
46
+
47
+ ## Quick start (keypoints — needs network, no API key)
48
+
49
+ ```tsx
50
+ import {
51
+ PoseTrackerProvider,
52
+ WebViewPoseView,
53
+ usePoseTracker,
54
+ } from '@pose-tracker/react-native-pose-estimation-light';
55
+
56
+ function App() {
57
+ return (
58
+ <PoseTrackerProvider
59
+ options={{
60
+ model: 'movenet', // Docs API parity (default)
61
+ // modelUrl: 'https://…/model.json', // optional override
62
+ }}
63
+ >
64
+ <CameraScreen />
65
+ </PoseTrackerProvider>
66
+ );
67
+ }
68
+
69
+ function CameraScreen() {
70
+ usePoseTracker({
71
+ onKeypoints: (e) => {
72
+ console.log(e.keypoints.length, e.score);
73
+ },
74
+ });
75
+
76
+ return (
77
+ <WebViewPoseView
78
+ style={{ flex: 1 }}
79
+ drawSkeleton
80
+ loadingText="AI Loading"
81
+ coldStart="full"
82
+ />
83
+ );
84
+ }
85
+ ```
86
+
87
+ ## Default model URL
88
+
89
+ ```
90
+ https://app.posetracker.com/scripts/tmp_model_to_remove.json
91
+ ```
92
+
93
+ Same MoveNet SinglePose Lightning topology as the PoseTracker Front tracking
94
+ product. Weight shards are relative neighbors (`group1-shard1of2.bin`, …).
95
+ TF.js loads from jsDelivr (`@tensorflow/*@4.22.0`) unless you override
96
+ `tfjsCdnBase` / `tfjsVersion`.
97
+
98
+ ## WebView `baseUrl`
99
+
100
+ The WebView uses `baseUrl: https://localhost/` (same as the offline SDK) so
101
+ `getUserMedia` works reliably. The model is fetched via **absolute** HTTPS URLs
102
+ with CORS — the document origin does **not** need to match `app.posetracker.com`.
103
+
104
+ ## Full tracking (API key)
105
+
106
+ Same contract as the offline SDK / web tracking URL:
107
+
108
+ ```tsx
109
+ <PoseTrackerProvider
110
+ apiToken="YOUR_API_KEY"
111
+ options={{ features: { angles: true, progression: true, minGrade: 'B' } }}
112
+ >
113
+ <WebViewPoseView drawSkeleton skeletonUuid="OPTIONAL_CUSTOM_SKELETON_UUID" />
114
+ </PoseTrackerProvider>
115
+ ```
116
+
117
+ ## Cold-start
118
+
119
+ | Mode | API | Camera permission |
120
+ |------|-----|-------------------|
121
+ | **basic** (default) | `preload()` | No — TF.js/model warm-up only |
122
+ | **full** | `preload({ coldStart: 'full' })` | Yes — when user expects camera |
123
+
124
+ ## Documentation
125
+
126
+ | Doc | Topic |
127
+ |-----|--------|
128
+ | [LIGHT_SDK.md](../../docs/LIGHT_SDK.md) | Offline vs light, sizes, model URL |
129
+ | [PERMISSIONS.md](../../docs/PERMISSIONS.md) | Camera permission setup (required) |
130
+ | [PRELOAD.md](../../docs/PRELOAD.md) | Preload / warm-up / lifecycle |
131
+ | [FEATURES.md](../../docs/FEATURES.md) | Plan gating, watermark, loading text |
132
+ | [EVENTS.md](../../docs/EVENTS.md) | Typed events + classic `onMessage` |
133
+
134
+ ## FAQ
135
+
136
+ **Does it work without an API key?**
137
+ Yes for keypoints — but a network is still required to load TF.js + MoveNet.
138
+
139
+ **Does it support Expo Go?**
140
+ Yes (`react-native-webview` peer).
141
+
142
+ **BlazePose?**
143
+ `model: 'blazepose'` is accepted for API parity but **not wired** yet. Pass a
144
+ TF.js `modelUrl` for custom graph models.
145
+
146
+ **Who sees the watermark?**
147
+ Keyless and free plans. Hidden for paid plans.
148
+
149
+ ## License
150
+
151
+ **Proprietary** — Movelytics SAS / PoseTracker. See [`LICENSE`](./LICENSE).
152
+
153
+ Third-party components (TensorFlow.js, MoveNet Lightning) are **Apache 2.0** —
154
+ see [`THIRD_PARTY_NOTICES.md`](./THIRD_PARTY_NOTICES.md).
155
+
156
+ ## Links
157
+
158
+ - Product: https://www.posetracker.com
159
+ - API docs: https://posetracker.gitbook.io/posetracker-api
160
+ - Offline SDK: https://github.com/Movelytics/react-native-pose-estimation
161
+ - Light demo: https://github.com/Movelytics/react-native-pose-estimation-light-demo
@@ -0,0 +1,19 @@
1
+ # Third-party notices
2
+
3
+ See the repository root `THIRD_PARTY_NOTICES.md` (same content shipped with
4
+ this npm package).
5
+
6
+ ## TensorFlow.js / TensorFlow.js WASM backend
7
+
8
+ - Copyright: Google LLC and TensorFlow authors
9
+ - License: Apache License 2.0 — https://www.apache.org/licenses/LICENSE-2.0
10
+
11
+ ## MoveNet SinglePose Lightning (model weights)
12
+
13
+ - Copyright: Google LLC
14
+ - License: Apache License 2.0 — https://www.apache.org/licenses/LICENSE-2.0
15
+ - Hub: https://tfhub.dev/google/movenet/singlepose/lightning/4
16
+
17
+ ## Other dependencies
18
+
19
+ Declared on npm for each package (e.g. `js-sha256`, `react-native-webview`).
@@ -0,0 +1,18 @@
1
+ /**
2
+ * Registers PoseTrackerBodyPosePlugin with VisionCamera's frame-processor
3
+ * registry under the JS name `detectBodyPose`.
4
+ */
5
+
6
+ #import <VisionCamera/FrameProcessorPlugin.h>
7
+ #import <VisionCamera/FrameProcessorPluginRegistry.h>
8
+
9
+ #if __has_include(<PoseTrackerVision/PoseTrackerVision-Swift.h>)
10
+ #import <PoseTrackerVision/PoseTrackerVision-Swift.h>
11
+ #elif __has_include("PoseTrackerVision-Swift.h")
12
+ #import "PoseTrackerVision-Swift.h"
13
+ #else
14
+ // Autolinking / use_frameworks! variants — generated Swift header name.
15
+ #import "PoseTrackerVision-Swift.h"
16
+ #endif
17
+
18
+ VISION_EXPORT_SWIFT_FRAME_PROCESSOR(PoseTrackerBodyPosePlugin, detectBodyPose)
@@ -0,0 +1,108 @@
1
+ /**
2
+ * VisionCamera frame-processor plugin: Apple Vision body pose.
3
+ *
4
+ * Runs `VNDetectHumanBodyPoseRequest` on the camera `CMSampleBuffer`
5
+ * (zero RGB copy). Returns up to 19 joints with normalized coords in
6
+ * top-left origin [0,1] (Vision's bottom-left Y is flipped here) and
7
+ * confidence scores in [0,1].
8
+ *
9
+ * Registered as JS plugin name `detectBodyPose`.
10
+ */
11
+
12
+ import Foundation
13
+ import Vision
14
+ import CoreMedia
15
+ import VisionCamera
16
+
17
+ @objc(PoseTrackerBodyPosePlugin)
18
+ public class PoseTrackerBodyPosePlugin: FrameProcessorPlugin {
19
+ private let request = VNDetectHumanBodyPoseRequest()
20
+
21
+ private static let jointSpecs: [(VNHumanBodyPoseObservation.JointName, String)] = [
22
+ (.nose, "nose"),
23
+ (.leftEye, "leftEye"),
24
+ (.rightEye, "rightEye"),
25
+ (.leftEar, "leftEar"),
26
+ (.rightEar, "rightEar"),
27
+ (.leftShoulder, "leftShoulder"),
28
+ (.rightShoulder, "rightShoulder"),
29
+ (.leftElbow, "leftElbow"),
30
+ (.rightElbow, "rightElbow"),
31
+ (.leftWrist, "leftWrist"),
32
+ (.rightWrist, "rightWrist"),
33
+ (.leftHip, "leftHip"),
34
+ (.rightHip, "rightHip"),
35
+ (.leftKnee, "leftKnee"),
36
+ (.rightKnee, "rightKnee"),
37
+ (.leftAnkle, "leftAnkle"),
38
+ (.rightAnkle, "rightAnkle"),
39
+ (.neck, "neck"),
40
+ (.root, "root"),
41
+ ]
42
+
43
+ public override init(proxy: VisionCameraProxyHolder, options: [AnyHashable: Any]! = [:]) {
44
+ super.init(proxy: proxy, options: options)
45
+ }
46
+
47
+ public override func callback(_ frame: Frame, withArguments arguments: [AnyHashable: Any]?) -> Any? {
48
+ let buffer = frame.buffer
49
+ let orientation = Self.cgOrientation(from: frame.orientation)
50
+
51
+ let start = CFAbsoluteTimeGetCurrent()
52
+ let handler = VNImageRequestHandler(
53
+ cmSampleBuffer: buffer,
54
+ orientation: orientation,
55
+ options: [:]
56
+ )
57
+
58
+ do {
59
+ try handler.perform([request])
60
+ } catch {
61
+ return nil
62
+ }
63
+
64
+ guard let observation = request.results?.first as? VNHumanBodyPoseObservation else {
65
+ return [
66
+ "joints": [] as [[String: Any]],
67
+ "inferenceTimeMs": (CFAbsoluteTimeGetCurrent() - start) * 1000.0,
68
+ ]
69
+ }
70
+
71
+ var joints: [[String: Any]] = []
72
+ joints.reserveCapacity(Self.jointSpecs.count)
73
+
74
+ for (jointName, jsName) in Self.jointSpecs {
75
+ guard let point = try? observation.recognizedPoint(jointName) else {
76
+ continue
77
+ }
78
+ // Vision uses bottom-left origin; SDK / MoveNet use top-left.
79
+ joints.append([
80
+ "name": jsName,
81
+ "x": Double(point.location.x),
82
+ "y": 1.0 - Double(point.location.y),
83
+ "score": Double(point.confidence),
84
+ ])
85
+ }
86
+
87
+ let elapsedMs = (CFAbsoluteTimeGetCurrent() - start) * 1000.0
88
+ return [
89
+ "joints": joints,
90
+ "inferenceTimeMs": elapsedMs,
91
+ ]
92
+ }
93
+
94
+ private static func cgOrientation(from orientation: Orientation) -> CGImagePropertyOrientation {
95
+ switch orientation {
96
+ case .portrait:
97
+ return .right
98
+ case .portraitUpsideDown:
99
+ return .left
100
+ case .landscapeLeft:
101
+ return .up
102
+ case .landscapeRight:
103
+ return .down
104
+ @unknown default:
105
+ return .right
106
+ }
107
+ }
108
+ }
@@ -0,0 +1,102 @@
1
+ /**
2
+ * React layer: `PoseTrackerProvider` owns a `PoseTrackerClient`, and
3
+ * `usePoseTracker` gives screens access to the pipeline plus typed event
4
+ * callbacks.
5
+ *
6
+ * Recommended usage:
7
+ * - Mount the provider near the app root — with or without an API token
8
+ * (without token, the SDK runs in keypoints-only mode).
9
+ * - **Mounting the provider does not load MoveNet** unless `autoPreload` is
10
+ * true. On the screen *before* the camera (e.g. home), call `preload()` /
11
+ * `warmup()` (default **basic** = model only, no camera permission). For
12
+ * WebView, mount a 1×1 `<WebViewPoseView coldStart="basic" />` warmer so
13
+ * MoveNet can load — see docs/PRELOAD.md.
14
+ * - Call `configure(apiToken)` at any time to upgrade keypoints-only →
15
+ * full-engine without restarting the camera pipeline.
16
+ */
17
+ import React from 'react';
18
+ import { PoseTrackerClient, type PoseTrackerClientOptions, type StartExerciseOptions } from './client';
19
+ import type { PoseInputFrame } from './backends/PoseBackend';
20
+ import type { ExerciseConfig, SdkManifest } from './types/manifest';
21
+ import type { PreloadOptions } from './types/preload';
22
+ import type { ErrorEvent, PoseTrackerCallbacks, PoseTrackerEventListener, PoseTrackerMode, PoseTrackerStatus } from './types/events';
23
+ import type { Pose } from './types/pose';
24
+ import type { AccelerationDiagnostics, AccelerationState } from './types/acceleration';
25
+ import type { QualityState } from './quality/AdaptiveQualityController';
26
+ export interface PoseTrackerContextValue {
27
+ /**
28
+ * The underlying orchestrator. Escape hatch for advanced integrations
29
+ * (e.g. `PoseCameraView` reads the active backend and ingests poses
30
+ * computed in the vision-camera worklet). Stable across renders.
31
+ */
32
+ client: PoseTrackerClient;
33
+ status: PoseTrackerStatus;
34
+ /** 'keypoints-only' (no validated key/session) or 'full-engine'. */
35
+ mode: PoseTrackerMode;
36
+ /**
37
+ * GPU-acceleration verdict from the warm-up health check: 'unknown' until
38
+ * warm-up completes, then 'gpu' | 'cpu-fallback' | 'unavailable'. Can
39
+ * downgrade mid-session after an Android GL context loss. See
40
+ * docs/ANDROID_GL_ACCELERATION.md.
41
+ */
42
+ acceleration: AccelerationState;
43
+ /** Full GPU diagnostics (backend, timings, GL renderer, flags, downgrade trail). */
44
+ accelerationDiagnostics: AccelerationDiagnostics | null;
45
+ /**
46
+ * Adaptive camera-quality profile (AdaptiveChoice + crash-guard + FPS
47
+ * auto-downgrade). `capturePriority` reflects the host option (`performance`
48
+ * default vs `quality`). See docs/ADAPTIVE_QUALITY.md.
49
+ */
50
+ quality: QualityState;
51
+ /** Last (possibly non-fatal) error; `ready` + keypoints-only can coexist with it. */
52
+ error: ErrorEvent | null;
53
+ manifest: SdkManifest | null;
54
+ /** Empty in keypoints-only mode. */
55
+ exercises: ExerciseConfig[];
56
+ /**
57
+ * Warm-up (handshake + engine + model). Default `coldStart: 'basic'` —
58
+ * no getUserMedia. Pass `{ coldStart: 'full' }` to also open the camera.
59
+ */
60
+ preload: (options?: PreloadOptions) => Promise<void>;
61
+ /** Alias of preload(). */
62
+ warmup: (options?: PreloadOptions) => Promise<void>;
63
+ /**
64
+ * Hot (re)configuration: validates the token and loads the engine without
65
+ * touching the camera pipeline. Resolves to true on full-engine mode.
66
+ */
67
+ configure: (apiToken?: string) => Promise<boolean>;
68
+ /**
69
+ * Start an FSM exercise (manifest) or a custom one (`jump_analysis`,
70
+ * `air_time_jump`). Options: `difficulty`, `userHeightCm`, `devicePitchDeg`.
71
+ */
72
+ startExercise: (exerciseId: string, options?: StartExerciseOptions) => void;
73
+ stopExercise: () => void;
74
+ estimatePose: (frame: PoseInputFrame) => Promise<Pose | null>;
75
+ processFrame: (frame: PoseInputFrame) => Promise<Pose | null>;
76
+ addEventListener: (listener: PoseTrackerEventListener) => () => void;
77
+ /**
78
+ * Classic PoseTracker WebView JSON stream (`sendDataToNative` shape).
79
+ * Same as `usePoseTracker({ onMessage })`.
80
+ */
81
+ addMessageListener: (listener: (message: import('./events/classicMessage').ClassicNativeMessage) => void) => () => void;
82
+ }
83
+ export interface PoseTrackerProviderProps {
84
+ /** Optional: without a token the SDK runs in keypoints-only mode. */
85
+ apiToken?: string;
86
+ options?: PoseTrackerClientOptions;
87
+ /** Start preloading as soon as the provider mounts. Default: false. */
88
+ autoPreload?: boolean;
89
+ children: React.ReactNode;
90
+ }
91
+ export declare function PoseTrackerProvider({ apiToken, options, autoPreload, children, }: PoseTrackerProviderProps): React.JSX.Element;
92
+ /**
93
+ * Access the PoseTracker pipeline and subscribe to typed events.
94
+ *
95
+ * ```tsx
96
+ * const { status, mode, processFrame, configure } = usePoseTracker({
97
+ * onKeypoints: (e) => drawSkeleton(e.keypoints), // both modes
98
+ * onCounter: (e) => setReps(e.count), // full-engine only
99
+ * });
100
+ * ```
101
+ */
102
+ export declare function usePoseTracker(callbacks?: PoseTrackerCallbacks): PoseTrackerContextValue;
@@ -0,0 +1,204 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.PoseTrackerProvider = PoseTrackerProvider;
4
+ exports.usePoseTracker = usePoseTracker;
5
+ const jsx_runtime_1 = require("react/jsx-runtime");
6
+ /**
7
+ * React layer: `PoseTrackerProvider` owns a `PoseTrackerClient`, and
8
+ * `usePoseTracker` gives screens access to the pipeline plus typed event
9
+ * callbacks.
10
+ *
11
+ * Recommended usage:
12
+ * - Mount the provider near the app root — with or without an API token
13
+ * (without token, the SDK runs in keypoints-only mode).
14
+ * - **Mounting the provider does not load MoveNet** unless `autoPreload` is
15
+ * true. On the screen *before* the camera (e.g. home), call `preload()` /
16
+ * `warmup()` (default **basic** = model only, no camera permission). For
17
+ * WebView, mount a 1×1 `<WebViewPoseView coldStart="basic" />` warmer so
18
+ * MoveNet can load — see docs/PRELOAD.md.
19
+ * - Call `configure(apiToken)` at any time to upgrade keypoints-only →
20
+ * full-engine without restarting the camera pipeline.
21
+ */
22
+ const react_1 = require("react");
23
+ const client_1 = require("./client");
24
+ const PoseTrackerContext = (0, react_1.createContext)(null);
25
+ function PoseTrackerProvider({ apiToken, options, autoPreload = false, children, }) {
26
+ const clientRef = (0, react_1.useRef)(null);
27
+ if (!clientRef.current) {
28
+ clientRef.current = new client_1.PoseTrackerClient(apiToken, options);
29
+ }
30
+ const client = clientRef.current;
31
+ const [status, setStatus] = (0, react_1.useState)(client.getStatus());
32
+ const [mode, setMode] = (0, react_1.useState)(client.getMode());
33
+ const [error, setError] = (0, react_1.useState)(null);
34
+ const [manifest, setManifest] = (0, react_1.useState)(null);
35
+ const [exercises, setExercises] = (0, react_1.useState)([]);
36
+ const [acceleration, setAcceleration] = (0, react_1.useState)(client.getAcceleration());
37
+ const [accelerationDiagnostics, setAccelerationDiagnostics] = (0, react_1.useState)(client.getAccelerationDiagnostics());
38
+ const [quality, setQuality] = (0, react_1.useState)(client.getQualityState());
39
+ (0, react_1.useEffect)(() => {
40
+ const offState = client.onStateChange(() => {
41
+ setStatus(client.getStatus());
42
+ setMode(client.getMode());
43
+ setError(client.getError());
44
+ setManifest(client.getManifest());
45
+ setExercises(client.getAvailableExercises());
46
+ setAcceleration(client.getAcceleration());
47
+ setAccelerationDiagnostics(client.getAccelerationDiagnostics());
48
+ setQuality(client.getQualityState());
49
+ });
50
+ if (autoPreload) {
51
+ client.preload().catch(() => {
52
+ // Error is surfaced through status/error state and the error event.
53
+ });
54
+ }
55
+ return () => {
56
+ offState();
57
+ };
58
+ }, [client, autoPreload]);
59
+ (0, react_1.useEffect)(() => {
60
+ return () => {
61
+ client.dispose().catch(() => { });
62
+ };
63
+ }, [client]);
64
+ const preload = (0, react_1.useCallback)((options) => client.preload(options), [client]);
65
+ const configureFn = (0, react_1.useCallback)((token) => client.configure(token), [client]);
66
+ const startExercise = (0, react_1.useCallback)((id, exerciseOptions) => client.startExercise(id, exerciseOptions), [client]);
67
+ const stopExercise = (0, react_1.useCallback)(() => client.stopExercise(), [client]);
68
+ const estimatePose = (0, react_1.useCallback)((frame) => client.estimatePose(frame), [client]);
69
+ const processFrame = (0, react_1.useCallback)((frame) => client.processFrame(frame), [client]);
70
+ const addEventListener = (0, react_1.useCallback)((listener) => client.addEventListener(listener), [client]);
71
+ const addMessageListener = (0, react_1.useCallback)((listener) => client.addMessageListener(listener), [client]);
72
+ const value = (0, react_1.useMemo)(() => ({
73
+ client,
74
+ status,
75
+ mode,
76
+ acceleration,
77
+ accelerationDiagnostics,
78
+ quality,
79
+ error,
80
+ manifest,
81
+ exercises,
82
+ preload,
83
+ warmup: preload,
84
+ configure: configureFn,
85
+ startExercise,
86
+ stopExercise,
87
+ estimatePose,
88
+ processFrame,
89
+ addEventListener,
90
+ addMessageListener,
91
+ }), [
92
+ client,
93
+ status,
94
+ mode,
95
+ acceleration,
96
+ accelerationDiagnostics,
97
+ quality,
98
+ error,
99
+ manifest,
100
+ exercises,
101
+ preload,
102
+ configureFn,
103
+ startExercise,
104
+ stopExercise,
105
+ estimatePose,
106
+ processFrame,
107
+ addEventListener,
108
+ addMessageListener,
109
+ ]);
110
+ return (0, jsx_runtime_1.jsx)(PoseTrackerContext.Provider, { value: value, children: children });
111
+ }
112
+ /**
113
+ * Access the PoseTracker pipeline and subscribe to typed events.
114
+ *
115
+ * ```tsx
116
+ * const { status, mode, processFrame, configure } = usePoseTracker({
117
+ * onKeypoints: (e) => drawSkeleton(e.keypoints), // both modes
118
+ * onCounter: (e) => setReps(e.count), // full-engine only
119
+ * });
120
+ * ```
121
+ */
122
+ function usePoseTracker(callbacks = {}) {
123
+ const context = (0, react_1.useContext)(PoseTrackerContext);
124
+ if (!context) {
125
+ throw new Error('usePoseTracker must be used within a <PoseTrackerProvider>.');
126
+ }
127
+ const callbacksRef = (0, react_1.useRef)(callbacks);
128
+ callbacksRef.current = callbacks;
129
+ (0, react_1.useEffect)(() => {
130
+ const offEvents = context.addEventListener((event) => {
131
+ const cb = callbacksRef.current;
132
+ switch (event.type) {
133
+ case 'initialization':
134
+ cb.onInitialization?.(event);
135
+ break;
136
+ case 'error':
137
+ cb.onError?.(event);
138
+ break;
139
+ case 'warning':
140
+ cb.onWarning?.(event);
141
+ break;
142
+ case 'keypoints':
143
+ cb.onKeypoints?.(event);
144
+ break;
145
+ case 'angles':
146
+ cb.onAngles?.(event);
147
+ break;
148
+ case 'counter':
149
+ cb.onCounter?.(event);
150
+ break;
151
+ case 'posture':
152
+ cb.onPosture?.(event);
153
+ break;
154
+ case 'progression':
155
+ cb.onProgression?.(event);
156
+ break;
157
+ case 'recommendations':
158
+ cb.onRecommendations?.(event);
159
+ break;
160
+ case 'form_score':
161
+ cb.onFormScore?.(event);
162
+ break;
163
+ case 'exercise_summary':
164
+ cb.onExerciseSummary?.(event);
165
+ break;
166
+ case 'jump_calibration':
167
+ cb.onJumpCalibration?.(event);
168
+ break;
169
+ case 'jump_started':
170
+ cb.onJumpStarted?.(event);
171
+ break;
172
+ case 'jump_height':
173
+ cb.onJumpHeight?.(event);
174
+ break;
175
+ case 'jump_discarded':
176
+ cb.onJumpDiscarded?.(event);
177
+ break;
178
+ case 'jump_result':
179
+ cb.onJumpResult?.(event);
180
+ break;
181
+ case 'jump_summary':
182
+ cb.onJumpSummary?.(event);
183
+ break;
184
+ case 'quality_changed':
185
+ cb.onQualityChanged?.(event);
186
+ break;
187
+ case 'performance_warning':
188
+ cb.onPerformanceWarning?.(event);
189
+ break;
190
+ case 'runtime_download_progress':
191
+ cb.onRuntimeDownloadProgress?.(event);
192
+ break;
193
+ }
194
+ });
195
+ const offMessages = context.addMessageListener((message) => {
196
+ callbacksRef.current.onMessage?.(message);
197
+ });
198
+ return () => {
199
+ offEvents();
200
+ offMessages();
201
+ };
202
+ }, [context]);
203
+ return context;
204
+ }