@pose-tracker/react-native-pose-estimation-light 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +45 -0
- package/PoseTrackerVision.podspec +28 -0
- package/README.md +161 -0
- package/THIRD_PARTY_NOTICES.md +19 -0
- package/ios/PoseTrackerVision/PoseTrackerBodyPosePlugin.m +18 -0
- package/ios/PoseTrackerVision/PoseTrackerBodyPosePlugin.swift +108 -0
- package/lib/PoseTrackerProvider.d.ts +102 -0
- package/lib/PoseTrackerProvider.js +204 -0
- package/lib/api/configure.d.ts +29 -0
- package/lib/api/configure.js +64 -0
- package/lib/api/skeleton.d.ts +17 -0
- package/lib/api/skeleton.js +53 -0
- package/lib/api/track.d.ts +61 -0
- package/lib/api/track.js +146 -0
- package/lib/backends/PoseBackend.d.ts +50 -0
- package/lib/backends/PoseBackend.js +11 -0
- package/lib/backends/vision/VisionPoseBackend.d.ts +72 -0
- package/lib/backends/vision/VisionPoseBackend.js +177 -0
- package/lib/backends/vision/mapVisionJoints.d.ts +35 -0
- package/lib/backends/vision/mapVisionJoints.js +162 -0
- package/lib/backends/vision/optionalVision.d.ts +32 -0
- package/lib/backends/vision/optionalVision.js +76 -0
- package/lib/backends/webview/WebViewPoseBackend.d.ts +223 -0
- package/lib/backends/webview/WebViewPoseBackend.js +382 -0
- package/lib/backends/webview/brandAssets.d.ts +5 -0
- package/lib/backends/webview/brandAssets.js +8 -0
- package/lib/backends/webview/onlineRuntime.d.ts +45 -0
- package/lib/backends/webview/onlineRuntime.js +57 -0
- package/lib/backends/webview/poseHtml.d.ts +71 -0
- package/lib/backends/webview/poseHtml.js +188 -0
- package/lib/backends/webview/poseRuntimeSource.d.ts +3 -0
- package/lib/backends/webview/poseRuntimeSource.js +4 -0
- package/lib/cache/obfuscate.d.ts +18 -0
- package/lib/cache/obfuscate.js +90 -0
- package/lib/camera/PoseCameraView.d.ts +51 -0
- package/lib/camera/PoseCameraView.js +99 -0
- package/lib/camera/WebViewPoseView.d.ts +81 -0
- package/lib/camera/WebViewPoseView.js +312 -0
- package/lib/client.d.ts +361 -0
- package/lib/client.js +1046 -0
- package/lib/diagnostics/logReport.d.ts +23 -0
- package/lib/diagnostics/logReport.js +102 -0
- package/lib/engine/EngineLoader.d.ts +95 -0
- package/lib/engine/EngineLoader.js +347 -0
- package/lib/engine/types.d.ts +83 -0
- package/lib/engine/types.js +11 -0
- package/lib/events/classicMessage.d.ts +31 -0
- package/lib/events/classicMessage.js +259 -0
- package/lib/exercises/aliases.d.ts +18 -0
- package/lib/exercises/aliases.js +48 -0
- package/lib/index.d.ts +72 -0
- package/lib/index.js +170 -0
- package/lib/models/poseModels.d.ts +39 -0
- package/lib/models/poseModels.js +63 -0
- package/lib/quality/AdaptiveQualityController.d.ts +120 -0
- package/lib/quality/AdaptiveQualityController.js +423 -0
- package/lib/quality/RuntimeGuard.d.ts +34 -0
- package/lib/quality/RuntimeGuard.js +105 -0
- package/lib/quality/captureMode.d.ts +82 -0
- package/lib/quality/captureMode.js +75 -0
- package/lib/quality/deviceCapability.d.ts +37 -0
- package/lib/quality/deviceCapability.js +177 -0
- package/lib/quality/profiles.d.ts +125 -0
- package/lib/quality/profiles.js +202 -0
- package/lib/runtime/RuntimeCache.d.ts +75 -0
- package/lib/runtime/RuntimeCache.js +230 -0
- package/lib/sdkVersion.d.ts +2 -0
- package/lib/sdkVersion.js +5 -0
- package/lib/support/optionalModules.d.ts +46 -0
- package/lib/support/optionalModules.js +82 -0
- package/lib/types/acceleration.d.ts +86 -0
- package/lib/types/acceleration.js +15 -0
- package/lib/types/events.d.ts +345 -0
- package/lib/types/events.js +10 -0
- package/lib/types/features.d.ts +90 -0
- package/lib/types/features.js +122 -0
- package/lib/types/manifest.d.ts +197 -0
- package/lib/types/manifest.js +10 -0
- package/lib/types/pose.d.ts +36 -0
- package/lib/types/pose.js +41 -0
- package/lib/types/preload.d.ts +17 -0
- package/lib/types/preload.js +10 -0
- package/lib/types/skeleton.d.ts +29 -0
- package/lib/types/skeleton.js +52 -0
- package/package.json +92 -0
- package/react-native.config.js +12 -0
- package/src/PoseTrackerProvider.tsx +332 -0
- package/src/api/configure.ts +82 -0
- package/src/api/skeleton.ts +60 -0
- package/src/api/track.ts +182 -0
- package/src/backends/PoseBackend.ts +59 -0
- package/src/backends/vision/VisionPoseBackend.ts +238 -0
- package/src/backends/vision/mapVisionJoints.ts +185 -0
- package/src/backends/vision/optionalVision.ts +94 -0
- package/src/backends/webview/WebViewPoseBackend.ts +582 -0
- package/src/backends/webview/brandAssets.ts +5 -0
- package/src/backends/webview/onlineRuntime.ts +94 -0
- package/src/backends/webview/poseHtml.ts +259 -0
- package/src/backends/webview/poseRuntimeSource.d.ts +3 -0
- package/src/cache/obfuscate.ts +94 -0
- package/src/camera/PoseCameraView.tsx +223 -0
- package/src/camera/WebViewPoseView.tsx +466 -0
- package/src/client.ts +1295 -0
- package/src/diagnostics/logReport.ts +131 -0
- package/src/engine/EngineLoader.ts +428 -0
- package/src/engine/types.ts +93 -0
- package/src/events/classicMessage.ts +271 -0
- package/src/exercises/aliases.ts +50 -0
- package/src/index.ts +243 -0
- package/src/models/poseModels.ts +93 -0
- package/src/quality/AdaptiveQualityController.ts +579 -0
- package/src/quality/RuntimeGuard.ts +119 -0
- package/src/quality/captureMode.ts +97 -0
- package/src/quality/deviceCapability.ts +187 -0
- package/src/quality/profiles.ts +253 -0
- package/src/runtime/RuntimeCache.ts +300 -0
- package/src/sdkVersion.ts +2 -0
- package/src/support/optionalModules.ts +114 -0
- package/src/types/acceleration.ts +94 -0
- package/src/types/events.ts +437 -0
- package/src/types/features.ts +176 -0
- package/src/types/manifest.ts +230 -0
- package/src/types/pose.ts +85 -0
- package/src/types/preload.ts +19 -0
- package/src/types/skeleton.ts +77 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
PoseTracker React Native SDK
|
|
2
|
+
Copyright (c) 2024–2026 Movelytics SAS. All rights reserved.
|
|
3
|
+
|
|
4
|
+
PROPRIETARY SOFTWARE LICENSE
|
|
5
|
+
|
|
6
|
+
This package published under the @pose-tracker scope (the "Software") is
|
|
7
|
+
proprietary to Movelytics SAS ("Licensor").
|
|
8
|
+
|
|
9
|
+
1. Grant of use
|
|
10
|
+
Subject to a valid PoseTracker account and applicable product terms
|
|
11
|
+
(including free-tier limits and paid plans), Licensor grants you a
|
|
12
|
+
non-exclusive, non-transferable, revocable license to install and use the
|
|
13
|
+
Software in your applications to access PoseTracker services and on-device
|
|
14
|
+
pose estimation features as documented.
|
|
15
|
+
|
|
16
|
+
2. Restrictions
|
|
17
|
+
Except as expressly permitted in writing by Licensor, you may not:
|
|
18
|
+
- sell, sublicense, or redistribute the Software as a competing SDK or
|
|
19
|
+
standalone pose-estimation product;
|
|
20
|
+
- remove or obscure proprietary notices, branding, or watermarks required
|
|
21
|
+
by your plan;
|
|
22
|
+
- reverse engineer the remote movement engine or proprietary pipeline
|
|
23
|
+
beyond what applicable law permits;
|
|
24
|
+
- use the Software in violation of PoseTracker Terms of Service or
|
|
25
|
+
applicable law.
|
|
26
|
+
|
|
27
|
+
3. Third-party components
|
|
28
|
+
The Software redistributes third-party components (including TensorFlow.js
|
|
29
|
+
and the MoveNet model) under their own licenses. See THIRD_PARTY_NOTICES.md.
|
|
30
|
+
Those licenses are not overridden by this LICENSE.
|
|
31
|
+
|
|
32
|
+
4. No warranty
|
|
33
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
34
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
35
|
+
FITNESS FOR A PARTICULAR PURPOSE, AND NONINFRINGEMENT. IN NO EVENT SHALL
|
|
36
|
+
THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES, OR OTHER
|
|
37
|
+
LIABILITY ARISING FROM THE SOFTWARE OR ITS USE.
|
|
38
|
+
|
|
39
|
+
5. Commercial terms
|
|
40
|
+
Paid plans, API keys, usage limits, and support are governed by your
|
|
41
|
+
PoseTracker / Movelytics commercial agreement and
|
|
42
|
+
https://www.posetracker.com terms (or successor URLs).
|
|
43
|
+
|
|
44
|
+
Contact: hello@posetracker.com
|
|
45
|
+
Website: https://www.posetracker.com
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
require 'json'
|
|
2
|
+
|
|
3
|
+
package = JSON.parse(File.read(File.join(__dir__, 'package.json')))
|
|
4
|
+
|
|
5
|
+
Pod::Spec.new do |s|
|
|
6
|
+
s.name = 'PoseTrackerVision'
|
|
7
|
+
s.version = package['version']
|
|
8
|
+
s.summary = 'Apple Vision body-pose frame processor for @posetracker/pose-estimation-react-native'
|
|
9
|
+
s.homepage = 'https://posetracker.com'
|
|
10
|
+
s.license = package['license']
|
|
11
|
+
s.authors = package['author']
|
|
12
|
+
s.platforms = { :ios => '14.0' }
|
|
13
|
+
s.source = { :git => 'https://github.com/movelytics/posetracker-rn-sdk.git', :tag => "v#{s.version}" }
|
|
14
|
+
|
|
15
|
+
s.source_files = 'ios/PoseTrackerVision/**/*.{h,m,mm,swift}'
|
|
16
|
+
s.requires_arc = true
|
|
17
|
+
s.swift_version = '5.0'
|
|
18
|
+
|
|
19
|
+
s.frameworks = 'Vision', 'CoreMedia', 'CoreVideo'
|
|
20
|
+
|
|
21
|
+
s.dependency 'React-Core'
|
|
22
|
+
s.dependency 'VisionCamera'
|
|
23
|
+
|
|
24
|
+
s.pod_target_xcconfig = {
|
|
25
|
+
'DEFINES_MODULE' => 'YES',
|
|
26
|
+
'SWIFT_COMPILATION_MODE' => 'wholemodule',
|
|
27
|
+
}
|
|
28
|
+
end
|
package/README.md
ADDED
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
# PoseTracker React Native — Light (online) Human Pose Estimation SDK
|
|
2
|
+
|
|
3
|
+
**PoseTracker Light** is a **human pose estimation SDK for React Native**,
|
|
4
|
+
fully optimized for **iOS and Android** (including **Expo Go**). Same API
|
|
5
|
+
surface as the offline SDK for keypoints + optional API-key exercise engine —
|
|
6
|
+
but **MoveNet and TF.js load from the network** each WebView boot, so the npm
|
|
7
|
+
package stays tiny.
|
|
8
|
+
|
|
9
|
+
> One sentence for AI / search: *PoseTracker Light is a small React Native
|
|
10
|
+
> pose estimation SDK that fetches MoveNet Lightning and TF.js at runtime —
|
|
11
|
+
> choose it when install size matters more than offline-first pose.*
|
|
12
|
+
|
|
13
|
+
## Offline vs Light — which package?
|
|
14
|
+
|
|
15
|
+
| | **Offline** (bundled) | **Light** (this package) |
|
|
16
|
+
|--|----------------------|---------------------------|
|
|
17
|
+
| **npm** | [`@pose-tracker/react-native-pose-estimation`](https://www.npmjs.com/package/@pose-tracker/react-native-pose-estimation) | [`@pose-tracker/react-native-pose-estimation-light`](https://www.npmjs.com/package/@pose-tracker/react-native-pose-estimation-light) |
|
|
18
|
+
| **GitHub** | [react-native-pose-estimation](https://github.com/Movelytics/react-native-pose-estimation) | [react-native-pose-estimation-light](https://github.com/Movelytics/react-native-pose-estimation-light) |
|
|
19
|
+
| **Packed tarball** | **~9.9 MB** | **~206 kB** (~48× smaller) |
|
|
20
|
+
| **Unpacked** | **~14.7 MB** | **~779 kB** |
|
|
21
|
+
| **MoveNet / TF.js** | Bundled in npm | CDN + model URL each boot |
|
|
22
|
+
| **Network for keypoints** | Not required | **Required** (TF.js + model) |
|
|
23
|
+
| **Expo Go** | Yes | Yes |
|
|
24
|
+
| **API surface** | Keypoints free; paid engine with API key | Same |
|
|
25
|
+
|
|
26
|
+
**Choose offline** when you need pose without a network, or want zero model
|
|
27
|
+
download at session start.
|
|
28
|
+
**Choose light** when app install / OTA size matters and devices are online.
|
|
29
|
+
|
|
30
|
+
Full comparison: [LIGHT_SDK.md](../../docs/LIGHT_SDK.md).
|
|
31
|
+
|
|
32
|
+
## Install
|
|
33
|
+
|
|
34
|
+
```bash
|
|
35
|
+
npm install @pose-tracker/react-native-pose-estimation-light react-native-webview
|
|
36
|
+
# Expo:
|
|
37
|
+
npx expo install react-native-webview expo-camera
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
> **npm:** [`@pose-tracker/react-native-pose-estimation-light`](https://www.npmjs.com/package/@pose-tracker/react-native-pose-estimation-light)
|
|
41
|
+
> **GitHub:** https://github.com/Movelytics/react-native-pose-estimation-light
|
|
42
|
+
> **Offline sibling:** [`@pose-tracker/react-native-pose-estimation`](https://www.npmjs.com/package/@pose-tracker/react-native-pose-estimation)
|
|
43
|
+
|
|
44
|
+
**Required:** host app must declare camera permissions — see
|
|
45
|
+
[PERMISSIONS.md](../../docs/PERMISSIONS.md).
|
|
46
|
+
|
|
47
|
+
## Quick start (keypoints — needs network, no API key)
|
|
48
|
+
|
|
49
|
+
```tsx
|
|
50
|
+
import {
|
|
51
|
+
PoseTrackerProvider,
|
|
52
|
+
WebViewPoseView,
|
|
53
|
+
usePoseTracker,
|
|
54
|
+
} from '@pose-tracker/react-native-pose-estimation-light';
|
|
55
|
+
|
|
56
|
+
function App() {
|
|
57
|
+
return (
|
|
58
|
+
<PoseTrackerProvider
|
|
59
|
+
options={{
|
|
60
|
+
model: 'movenet', // Docs API parity (default)
|
|
61
|
+
// modelUrl: 'https://…/model.json', // optional override
|
|
62
|
+
}}
|
|
63
|
+
>
|
|
64
|
+
<CameraScreen />
|
|
65
|
+
</PoseTrackerProvider>
|
|
66
|
+
);
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
function CameraScreen() {
|
|
70
|
+
usePoseTracker({
|
|
71
|
+
onKeypoints: (e) => {
|
|
72
|
+
console.log(e.keypoints.length, e.score);
|
|
73
|
+
},
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
return (
|
|
77
|
+
<WebViewPoseView
|
|
78
|
+
style={{ flex: 1 }}
|
|
79
|
+
drawSkeleton
|
|
80
|
+
loadingText="AI Loading"
|
|
81
|
+
coldStart="full"
|
|
82
|
+
/>
|
|
83
|
+
);
|
|
84
|
+
}
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
## Default model URL
|
|
88
|
+
|
|
89
|
+
```
|
|
90
|
+
https://app.posetracker.com/scripts/tmp_model_to_remove.json
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
Same MoveNet SinglePose Lightning topology as the PoseTracker Front tracking
|
|
94
|
+
product. Weight shards are relative neighbors (`group1-shard1of2.bin`, …).
|
|
95
|
+
TF.js loads from jsDelivr (`@tensorflow/*@4.22.0`) unless you override
|
|
96
|
+
`tfjsCdnBase` / `tfjsVersion`.
|
|
97
|
+
|
|
98
|
+
## WebView `baseUrl`
|
|
99
|
+
|
|
100
|
+
The WebView uses `baseUrl: https://localhost/` (same as the offline SDK) so
|
|
101
|
+
`getUserMedia` works reliably. The model is fetched via **absolute** HTTPS URLs
|
|
102
|
+
with CORS — the document origin does **not** need to match `app.posetracker.com`.
|
|
103
|
+
|
|
104
|
+
## Full tracking (API key)
|
|
105
|
+
|
|
106
|
+
Same contract as the offline SDK / web tracking URL:
|
|
107
|
+
|
|
108
|
+
```tsx
|
|
109
|
+
<PoseTrackerProvider
|
|
110
|
+
apiToken="YOUR_API_KEY"
|
|
111
|
+
options={{ features: { angles: true, progression: true, minGrade: 'B' } }}
|
|
112
|
+
>
|
|
113
|
+
<WebViewPoseView drawSkeleton skeletonUuid="OPTIONAL_CUSTOM_SKELETON_UUID" />
|
|
114
|
+
</PoseTrackerProvider>
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
## Cold-start
|
|
118
|
+
|
|
119
|
+
| Mode | API | Camera permission |
|
|
120
|
+
|------|-----|-------------------|
|
|
121
|
+
| **basic** (default) | `preload()` | No — TF.js/model warm-up only |
|
|
122
|
+
| **full** | `preload({ coldStart: 'full' })` | Yes — when user expects camera |
|
|
123
|
+
|
|
124
|
+
## Documentation
|
|
125
|
+
|
|
126
|
+
| Doc | Topic |
|
|
127
|
+
|-----|--------|
|
|
128
|
+
| [LIGHT_SDK.md](../../docs/LIGHT_SDK.md) | Offline vs light, sizes, model URL |
|
|
129
|
+
| [PERMISSIONS.md](../../docs/PERMISSIONS.md) | Camera permission setup (required) |
|
|
130
|
+
| [PRELOAD.md](../../docs/PRELOAD.md) | Preload / warm-up / lifecycle |
|
|
131
|
+
| [FEATURES.md](../../docs/FEATURES.md) | Plan gating, watermark, loading text |
|
|
132
|
+
| [EVENTS.md](../../docs/EVENTS.md) | Typed events + classic `onMessage` |
|
|
133
|
+
|
|
134
|
+
## FAQ
|
|
135
|
+
|
|
136
|
+
**Does it work without an API key?**
|
|
137
|
+
Yes for keypoints — but a network is still required to load TF.js + MoveNet.
|
|
138
|
+
|
|
139
|
+
**Does it support Expo Go?**
|
|
140
|
+
Yes (`react-native-webview` peer).
|
|
141
|
+
|
|
142
|
+
**BlazePose?**
|
|
143
|
+
`model: 'blazepose'` is accepted for API parity but **not wired** yet. Pass a
|
|
144
|
+
TF.js `modelUrl` for custom graph models.
|
|
145
|
+
|
|
146
|
+
**Who sees the watermark?**
|
|
147
|
+
Keyless and free plans. Hidden for paid plans.
|
|
148
|
+
|
|
149
|
+
## License
|
|
150
|
+
|
|
151
|
+
**Proprietary** — Movelytics SAS / PoseTracker. See [`LICENSE`](./LICENSE).
|
|
152
|
+
|
|
153
|
+
Third-party components (TensorFlow.js, MoveNet Lightning) are **Apache 2.0** —
|
|
154
|
+
see [`THIRD_PARTY_NOTICES.md`](./THIRD_PARTY_NOTICES.md).
|
|
155
|
+
|
|
156
|
+
## Links
|
|
157
|
+
|
|
158
|
+
- Product: https://www.posetracker.com
|
|
159
|
+
- API docs: https://posetracker.gitbook.io/posetracker-api
|
|
160
|
+
- Offline SDK: https://github.com/Movelytics/react-native-pose-estimation
|
|
161
|
+
- Light demo: https://github.com/Movelytics/react-native-pose-estimation-light-demo
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
# Third-party notices
|
|
2
|
+
|
|
3
|
+
See the repository root `THIRD_PARTY_NOTICES.md` (same content shipped with
|
|
4
|
+
this npm package).
|
|
5
|
+
|
|
6
|
+
## TensorFlow.js / TensorFlow.js WASM backend
|
|
7
|
+
|
|
8
|
+
- Copyright: Google LLC and TensorFlow authors
|
|
9
|
+
- License: Apache License 2.0 — https://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
+
|
|
11
|
+
## MoveNet SinglePose Lightning (model weights)
|
|
12
|
+
|
|
13
|
+
- Copyright: Google LLC
|
|
14
|
+
- License: Apache License 2.0 — https://www.apache.org/licenses/LICENSE-2.0
|
|
15
|
+
- Hub: https://tfhub.dev/google/movenet/singlepose/lightning/4
|
|
16
|
+
|
|
17
|
+
## Other dependencies
|
|
18
|
+
|
|
19
|
+
Declared on npm for each package (e.g. `js-sha256`, `react-native-webview`).
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Registers PoseTrackerBodyPosePlugin with VisionCamera's frame-processor
|
|
3
|
+
* registry under the JS name `detectBodyPose`.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
#import <VisionCamera/FrameProcessorPlugin.h>
|
|
7
|
+
#import <VisionCamera/FrameProcessorPluginRegistry.h>
|
|
8
|
+
|
|
9
|
+
#if __has_include(<PoseTrackerVision/PoseTrackerVision-Swift.h>)
|
|
10
|
+
#import <PoseTrackerVision/PoseTrackerVision-Swift.h>
|
|
11
|
+
#elif __has_include("PoseTrackerVision-Swift.h")
|
|
12
|
+
#import "PoseTrackerVision-Swift.h"
|
|
13
|
+
#else
|
|
14
|
+
// Autolinking / use_frameworks! variants — generated Swift header name.
|
|
15
|
+
#import "PoseTrackerVision-Swift.h"
|
|
16
|
+
#endif
|
|
17
|
+
|
|
18
|
+
VISION_EXPORT_SWIFT_FRAME_PROCESSOR(PoseTrackerBodyPosePlugin, detectBodyPose)
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* VisionCamera frame-processor plugin: Apple Vision body pose.
|
|
3
|
+
*
|
|
4
|
+
* Runs `VNDetectHumanBodyPoseRequest` on the camera `CMSampleBuffer`
|
|
5
|
+
* (zero RGB copy). Returns up to 19 joints with normalized coords in
|
|
6
|
+
* top-left origin [0,1] (Vision's bottom-left Y is flipped here) and
|
|
7
|
+
* confidence scores in [0,1].
|
|
8
|
+
*
|
|
9
|
+
* Registered as JS plugin name `detectBodyPose`.
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
import Foundation
|
|
13
|
+
import Vision
|
|
14
|
+
import CoreMedia
|
|
15
|
+
import VisionCamera
|
|
16
|
+
|
|
17
|
+
@objc(PoseTrackerBodyPosePlugin)
|
|
18
|
+
public class PoseTrackerBodyPosePlugin: FrameProcessorPlugin {
|
|
19
|
+
private let request = VNDetectHumanBodyPoseRequest()
|
|
20
|
+
|
|
21
|
+
private static let jointSpecs: [(VNHumanBodyPoseObservation.JointName, String)] = [
|
|
22
|
+
(.nose, "nose"),
|
|
23
|
+
(.leftEye, "leftEye"),
|
|
24
|
+
(.rightEye, "rightEye"),
|
|
25
|
+
(.leftEar, "leftEar"),
|
|
26
|
+
(.rightEar, "rightEar"),
|
|
27
|
+
(.leftShoulder, "leftShoulder"),
|
|
28
|
+
(.rightShoulder, "rightShoulder"),
|
|
29
|
+
(.leftElbow, "leftElbow"),
|
|
30
|
+
(.rightElbow, "rightElbow"),
|
|
31
|
+
(.leftWrist, "leftWrist"),
|
|
32
|
+
(.rightWrist, "rightWrist"),
|
|
33
|
+
(.leftHip, "leftHip"),
|
|
34
|
+
(.rightHip, "rightHip"),
|
|
35
|
+
(.leftKnee, "leftKnee"),
|
|
36
|
+
(.rightKnee, "rightKnee"),
|
|
37
|
+
(.leftAnkle, "leftAnkle"),
|
|
38
|
+
(.rightAnkle, "rightAnkle"),
|
|
39
|
+
(.neck, "neck"),
|
|
40
|
+
(.root, "root"),
|
|
41
|
+
]
|
|
42
|
+
|
|
43
|
+
public override init(proxy: VisionCameraProxyHolder, options: [AnyHashable: Any]! = [:]) {
|
|
44
|
+
super.init(proxy: proxy, options: options)
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
public override func callback(_ frame: Frame, withArguments arguments: [AnyHashable: Any]?) -> Any? {
|
|
48
|
+
let buffer = frame.buffer
|
|
49
|
+
let orientation = Self.cgOrientation(from: frame.orientation)
|
|
50
|
+
|
|
51
|
+
let start = CFAbsoluteTimeGetCurrent()
|
|
52
|
+
let handler = VNImageRequestHandler(
|
|
53
|
+
cmSampleBuffer: buffer,
|
|
54
|
+
orientation: orientation,
|
|
55
|
+
options: [:]
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
do {
|
|
59
|
+
try handler.perform([request])
|
|
60
|
+
} catch {
|
|
61
|
+
return nil
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
guard let observation = request.results?.first as? VNHumanBodyPoseObservation else {
|
|
65
|
+
return [
|
|
66
|
+
"joints": [] as [[String: Any]],
|
|
67
|
+
"inferenceTimeMs": (CFAbsoluteTimeGetCurrent() - start) * 1000.0,
|
|
68
|
+
]
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
var joints: [[String: Any]] = []
|
|
72
|
+
joints.reserveCapacity(Self.jointSpecs.count)
|
|
73
|
+
|
|
74
|
+
for (jointName, jsName) in Self.jointSpecs {
|
|
75
|
+
guard let point = try? observation.recognizedPoint(jointName) else {
|
|
76
|
+
continue
|
|
77
|
+
}
|
|
78
|
+
// Vision uses bottom-left origin; SDK / MoveNet use top-left.
|
|
79
|
+
joints.append([
|
|
80
|
+
"name": jsName,
|
|
81
|
+
"x": Double(point.location.x),
|
|
82
|
+
"y": 1.0 - Double(point.location.y),
|
|
83
|
+
"score": Double(point.confidence),
|
|
84
|
+
])
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
let elapsedMs = (CFAbsoluteTimeGetCurrent() - start) * 1000.0
|
|
88
|
+
return [
|
|
89
|
+
"joints": joints,
|
|
90
|
+
"inferenceTimeMs": elapsedMs,
|
|
91
|
+
]
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
private static func cgOrientation(from orientation: Orientation) -> CGImagePropertyOrientation {
|
|
95
|
+
switch orientation {
|
|
96
|
+
case .portrait:
|
|
97
|
+
return .right
|
|
98
|
+
case .portraitUpsideDown:
|
|
99
|
+
return .left
|
|
100
|
+
case .landscapeLeft:
|
|
101
|
+
return .up
|
|
102
|
+
case .landscapeRight:
|
|
103
|
+
return .down
|
|
104
|
+
@unknown default:
|
|
105
|
+
return .right
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
}
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* React layer: `PoseTrackerProvider` owns a `PoseTrackerClient`, and
|
|
3
|
+
* `usePoseTracker` gives screens access to the pipeline plus typed event
|
|
4
|
+
* callbacks.
|
|
5
|
+
*
|
|
6
|
+
* Recommended usage:
|
|
7
|
+
* - Mount the provider near the app root — with or without an API token
|
|
8
|
+
* (without token, the SDK runs in keypoints-only mode).
|
|
9
|
+
* - **Mounting the provider does not load MoveNet** unless `autoPreload` is
|
|
10
|
+
* true. On the screen *before* the camera (e.g. home), call `preload()` /
|
|
11
|
+
* `warmup()` (default **basic** = model only, no camera permission). For
|
|
12
|
+
* WebView, mount a 1×1 `<WebViewPoseView coldStart="basic" />` warmer so
|
|
13
|
+
* MoveNet can load — see docs/PRELOAD.md.
|
|
14
|
+
* - Call `configure(apiToken)` at any time to upgrade keypoints-only →
|
|
15
|
+
* full-engine without restarting the camera pipeline.
|
|
16
|
+
*/
|
|
17
|
+
import React from 'react';
|
|
18
|
+
import { PoseTrackerClient, type PoseTrackerClientOptions, type StartExerciseOptions } from './client';
|
|
19
|
+
import type { PoseInputFrame } from './backends/PoseBackend';
|
|
20
|
+
import type { ExerciseConfig, SdkManifest } from './types/manifest';
|
|
21
|
+
import type { PreloadOptions } from './types/preload';
|
|
22
|
+
import type { ErrorEvent, PoseTrackerCallbacks, PoseTrackerEventListener, PoseTrackerMode, PoseTrackerStatus } from './types/events';
|
|
23
|
+
import type { Pose } from './types/pose';
|
|
24
|
+
import type { AccelerationDiagnostics, AccelerationState } from './types/acceleration';
|
|
25
|
+
import type { QualityState } from './quality/AdaptiveQualityController';
|
|
26
|
+
export interface PoseTrackerContextValue {
|
|
27
|
+
/**
|
|
28
|
+
* The underlying orchestrator. Escape hatch for advanced integrations
|
|
29
|
+
* (e.g. `PoseCameraView` reads the active backend and ingests poses
|
|
30
|
+
* computed in the vision-camera worklet). Stable across renders.
|
|
31
|
+
*/
|
|
32
|
+
client: PoseTrackerClient;
|
|
33
|
+
status: PoseTrackerStatus;
|
|
34
|
+
/** 'keypoints-only' (no validated key/session) or 'full-engine'. */
|
|
35
|
+
mode: PoseTrackerMode;
|
|
36
|
+
/**
|
|
37
|
+
* GPU-acceleration verdict from the warm-up health check: 'unknown' until
|
|
38
|
+
* warm-up completes, then 'gpu' | 'cpu-fallback' | 'unavailable'. Can
|
|
39
|
+
* downgrade mid-session after an Android GL context loss. See
|
|
40
|
+
* docs/ANDROID_GL_ACCELERATION.md.
|
|
41
|
+
*/
|
|
42
|
+
acceleration: AccelerationState;
|
|
43
|
+
/** Full GPU diagnostics (backend, timings, GL renderer, flags, downgrade trail). */
|
|
44
|
+
accelerationDiagnostics: AccelerationDiagnostics | null;
|
|
45
|
+
/**
|
|
46
|
+
* Adaptive camera-quality profile (AdaptiveChoice + crash-guard + FPS
|
|
47
|
+
* auto-downgrade). `capturePriority` reflects the host option (`performance`
|
|
48
|
+
* default vs `quality`). See docs/ADAPTIVE_QUALITY.md.
|
|
49
|
+
*/
|
|
50
|
+
quality: QualityState;
|
|
51
|
+
/** Last (possibly non-fatal) error; `ready` + keypoints-only can coexist with it. */
|
|
52
|
+
error: ErrorEvent | null;
|
|
53
|
+
manifest: SdkManifest | null;
|
|
54
|
+
/** Empty in keypoints-only mode. */
|
|
55
|
+
exercises: ExerciseConfig[];
|
|
56
|
+
/**
|
|
57
|
+
* Warm-up (handshake + engine + model). Default `coldStart: 'basic'` —
|
|
58
|
+
* no getUserMedia. Pass `{ coldStart: 'full' }` to also open the camera.
|
|
59
|
+
*/
|
|
60
|
+
preload: (options?: PreloadOptions) => Promise<void>;
|
|
61
|
+
/** Alias of preload(). */
|
|
62
|
+
warmup: (options?: PreloadOptions) => Promise<void>;
|
|
63
|
+
/**
|
|
64
|
+
* Hot (re)configuration: validates the token and loads the engine without
|
|
65
|
+
* touching the camera pipeline. Resolves to true on full-engine mode.
|
|
66
|
+
*/
|
|
67
|
+
configure: (apiToken?: string) => Promise<boolean>;
|
|
68
|
+
/**
|
|
69
|
+
* Start an FSM exercise (manifest) or a custom one (`jump_analysis`,
|
|
70
|
+
* `air_time_jump`). Options: `difficulty`, `userHeightCm`, `devicePitchDeg`.
|
|
71
|
+
*/
|
|
72
|
+
startExercise: (exerciseId: string, options?: StartExerciseOptions) => void;
|
|
73
|
+
stopExercise: () => void;
|
|
74
|
+
estimatePose: (frame: PoseInputFrame) => Promise<Pose | null>;
|
|
75
|
+
processFrame: (frame: PoseInputFrame) => Promise<Pose | null>;
|
|
76
|
+
addEventListener: (listener: PoseTrackerEventListener) => () => void;
|
|
77
|
+
/**
|
|
78
|
+
* Classic PoseTracker WebView JSON stream (`sendDataToNative` shape).
|
|
79
|
+
* Same as `usePoseTracker({ onMessage })`.
|
|
80
|
+
*/
|
|
81
|
+
addMessageListener: (listener: (message: import('./events/classicMessage').ClassicNativeMessage) => void) => () => void;
|
|
82
|
+
}
|
|
83
|
+
export interface PoseTrackerProviderProps {
|
|
84
|
+
/** Optional: without a token the SDK runs in keypoints-only mode. */
|
|
85
|
+
apiToken?: string;
|
|
86
|
+
options?: PoseTrackerClientOptions;
|
|
87
|
+
/** Start preloading as soon as the provider mounts. Default: false. */
|
|
88
|
+
autoPreload?: boolean;
|
|
89
|
+
children: React.ReactNode;
|
|
90
|
+
}
|
|
91
|
+
export declare function PoseTrackerProvider({ apiToken, options, autoPreload, children, }: PoseTrackerProviderProps): React.JSX.Element;
|
|
92
|
+
/**
|
|
93
|
+
* Access the PoseTracker pipeline and subscribe to typed events.
|
|
94
|
+
*
|
|
95
|
+
* ```tsx
|
|
96
|
+
* const { status, mode, processFrame, configure } = usePoseTracker({
|
|
97
|
+
* onKeypoints: (e) => drawSkeleton(e.keypoints), // both modes
|
|
98
|
+
* onCounter: (e) => setReps(e.count), // full-engine only
|
|
99
|
+
* });
|
|
100
|
+
* ```
|
|
101
|
+
*/
|
|
102
|
+
export declare function usePoseTracker(callbacks?: PoseTrackerCallbacks): PoseTrackerContextValue;
|
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.PoseTrackerProvider = PoseTrackerProvider;
|
|
4
|
+
exports.usePoseTracker = usePoseTracker;
|
|
5
|
+
const jsx_runtime_1 = require("react/jsx-runtime");
|
|
6
|
+
/**
|
|
7
|
+
* React layer: `PoseTrackerProvider` owns a `PoseTrackerClient`, and
|
|
8
|
+
* `usePoseTracker` gives screens access to the pipeline plus typed event
|
|
9
|
+
* callbacks.
|
|
10
|
+
*
|
|
11
|
+
* Recommended usage:
|
|
12
|
+
* - Mount the provider near the app root — with or without an API token
|
|
13
|
+
* (without token, the SDK runs in keypoints-only mode).
|
|
14
|
+
* - **Mounting the provider does not load MoveNet** unless `autoPreload` is
|
|
15
|
+
* true. On the screen *before* the camera (e.g. home), call `preload()` /
|
|
16
|
+
* `warmup()` (default **basic** = model only, no camera permission). For
|
|
17
|
+
* WebView, mount a 1×1 `<WebViewPoseView coldStart="basic" />` warmer so
|
|
18
|
+
* MoveNet can load — see docs/PRELOAD.md.
|
|
19
|
+
* - Call `configure(apiToken)` at any time to upgrade keypoints-only →
|
|
20
|
+
* full-engine without restarting the camera pipeline.
|
|
21
|
+
*/
|
|
22
|
+
const react_1 = require("react");
|
|
23
|
+
const client_1 = require("./client");
|
|
24
|
+
const PoseTrackerContext = (0, react_1.createContext)(null);
|
|
25
|
+
function PoseTrackerProvider({ apiToken, options, autoPreload = false, children, }) {
|
|
26
|
+
const clientRef = (0, react_1.useRef)(null);
|
|
27
|
+
if (!clientRef.current) {
|
|
28
|
+
clientRef.current = new client_1.PoseTrackerClient(apiToken, options);
|
|
29
|
+
}
|
|
30
|
+
const client = clientRef.current;
|
|
31
|
+
const [status, setStatus] = (0, react_1.useState)(client.getStatus());
|
|
32
|
+
const [mode, setMode] = (0, react_1.useState)(client.getMode());
|
|
33
|
+
const [error, setError] = (0, react_1.useState)(null);
|
|
34
|
+
const [manifest, setManifest] = (0, react_1.useState)(null);
|
|
35
|
+
const [exercises, setExercises] = (0, react_1.useState)([]);
|
|
36
|
+
const [acceleration, setAcceleration] = (0, react_1.useState)(client.getAcceleration());
|
|
37
|
+
const [accelerationDiagnostics, setAccelerationDiagnostics] = (0, react_1.useState)(client.getAccelerationDiagnostics());
|
|
38
|
+
const [quality, setQuality] = (0, react_1.useState)(client.getQualityState());
|
|
39
|
+
(0, react_1.useEffect)(() => {
|
|
40
|
+
const offState = client.onStateChange(() => {
|
|
41
|
+
setStatus(client.getStatus());
|
|
42
|
+
setMode(client.getMode());
|
|
43
|
+
setError(client.getError());
|
|
44
|
+
setManifest(client.getManifest());
|
|
45
|
+
setExercises(client.getAvailableExercises());
|
|
46
|
+
setAcceleration(client.getAcceleration());
|
|
47
|
+
setAccelerationDiagnostics(client.getAccelerationDiagnostics());
|
|
48
|
+
setQuality(client.getQualityState());
|
|
49
|
+
});
|
|
50
|
+
if (autoPreload) {
|
|
51
|
+
client.preload().catch(() => {
|
|
52
|
+
// Error is surfaced through status/error state and the error event.
|
|
53
|
+
});
|
|
54
|
+
}
|
|
55
|
+
return () => {
|
|
56
|
+
offState();
|
|
57
|
+
};
|
|
58
|
+
}, [client, autoPreload]);
|
|
59
|
+
(0, react_1.useEffect)(() => {
|
|
60
|
+
return () => {
|
|
61
|
+
client.dispose().catch(() => { });
|
|
62
|
+
};
|
|
63
|
+
}, [client]);
|
|
64
|
+
const preload = (0, react_1.useCallback)((options) => client.preload(options), [client]);
|
|
65
|
+
const configureFn = (0, react_1.useCallback)((token) => client.configure(token), [client]);
|
|
66
|
+
const startExercise = (0, react_1.useCallback)((id, exerciseOptions) => client.startExercise(id, exerciseOptions), [client]);
|
|
67
|
+
const stopExercise = (0, react_1.useCallback)(() => client.stopExercise(), [client]);
|
|
68
|
+
const estimatePose = (0, react_1.useCallback)((frame) => client.estimatePose(frame), [client]);
|
|
69
|
+
const processFrame = (0, react_1.useCallback)((frame) => client.processFrame(frame), [client]);
|
|
70
|
+
const addEventListener = (0, react_1.useCallback)((listener) => client.addEventListener(listener), [client]);
|
|
71
|
+
const addMessageListener = (0, react_1.useCallback)((listener) => client.addMessageListener(listener), [client]);
|
|
72
|
+
const value = (0, react_1.useMemo)(() => ({
|
|
73
|
+
client,
|
|
74
|
+
status,
|
|
75
|
+
mode,
|
|
76
|
+
acceleration,
|
|
77
|
+
accelerationDiagnostics,
|
|
78
|
+
quality,
|
|
79
|
+
error,
|
|
80
|
+
manifest,
|
|
81
|
+
exercises,
|
|
82
|
+
preload,
|
|
83
|
+
warmup: preload,
|
|
84
|
+
configure: configureFn,
|
|
85
|
+
startExercise,
|
|
86
|
+
stopExercise,
|
|
87
|
+
estimatePose,
|
|
88
|
+
processFrame,
|
|
89
|
+
addEventListener,
|
|
90
|
+
addMessageListener,
|
|
91
|
+
}), [
|
|
92
|
+
client,
|
|
93
|
+
status,
|
|
94
|
+
mode,
|
|
95
|
+
acceleration,
|
|
96
|
+
accelerationDiagnostics,
|
|
97
|
+
quality,
|
|
98
|
+
error,
|
|
99
|
+
manifest,
|
|
100
|
+
exercises,
|
|
101
|
+
preload,
|
|
102
|
+
configureFn,
|
|
103
|
+
startExercise,
|
|
104
|
+
stopExercise,
|
|
105
|
+
estimatePose,
|
|
106
|
+
processFrame,
|
|
107
|
+
addEventListener,
|
|
108
|
+
addMessageListener,
|
|
109
|
+
]);
|
|
110
|
+
return (0, jsx_runtime_1.jsx)(PoseTrackerContext.Provider, { value: value, children: children });
|
|
111
|
+
}
|
|
112
|
+
/**
|
|
113
|
+
* Access the PoseTracker pipeline and subscribe to typed events.
|
|
114
|
+
*
|
|
115
|
+
* ```tsx
|
|
116
|
+
* const { status, mode, processFrame, configure } = usePoseTracker({
|
|
117
|
+
* onKeypoints: (e) => drawSkeleton(e.keypoints), // both modes
|
|
118
|
+
* onCounter: (e) => setReps(e.count), // full-engine only
|
|
119
|
+
* });
|
|
120
|
+
* ```
|
|
121
|
+
*/
|
|
122
|
+
function usePoseTracker(callbacks = {}) {
|
|
123
|
+
const context = (0, react_1.useContext)(PoseTrackerContext);
|
|
124
|
+
if (!context) {
|
|
125
|
+
throw new Error('usePoseTracker must be used within a <PoseTrackerProvider>.');
|
|
126
|
+
}
|
|
127
|
+
const callbacksRef = (0, react_1.useRef)(callbacks);
|
|
128
|
+
callbacksRef.current = callbacks;
|
|
129
|
+
(0, react_1.useEffect)(() => {
|
|
130
|
+
const offEvents = context.addEventListener((event) => {
|
|
131
|
+
const cb = callbacksRef.current;
|
|
132
|
+
switch (event.type) {
|
|
133
|
+
case 'initialization':
|
|
134
|
+
cb.onInitialization?.(event);
|
|
135
|
+
break;
|
|
136
|
+
case 'error':
|
|
137
|
+
cb.onError?.(event);
|
|
138
|
+
break;
|
|
139
|
+
case 'warning':
|
|
140
|
+
cb.onWarning?.(event);
|
|
141
|
+
break;
|
|
142
|
+
case 'keypoints':
|
|
143
|
+
cb.onKeypoints?.(event);
|
|
144
|
+
break;
|
|
145
|
+
case 'angles':
|
|
146
|
+
cb.onAngles?.(event);
|
|
147
|
+
break;
|
|
148
|
+
case 'counter':
|
|
149
|
+
cb.onCounter?.(event);
|
|
150
|
+
break;
|
|
151
|
+
case 'posture':
|
|
152
|
+
cb.onPosture?.(event);
|
|
153
|
+
break;
|
|
154
|
+
case 'progression':
|
|
155
|
+
cb.onProgression?.(event);
|
|
156
|
+
break;
|
|
157
|
+
case 'recommendations':
|
|
158
|
+
cb.onRecommendations?.(event);
|
|
159
|
+
break;
|
|
160
|
+
case 'form_score':
|
|
161
|
+
cb.onFormScore?.(event);
|
|
162
|
+
break;
|
|
163
|
+
case 'exercise_summary':
|
|
164
|
+
cb.onExerciseSummary?.(event);
|
|
165
|
+
break;
|
|
166
|
+
case 'jump_calibration':
|
|
167
|
+
cb.onJumpCalibration?.(event);
|
|
168
|
+
break;
|
|
169
|
+
case 'jump_started':
|
|
170
|
+
cb.onJumpStarted?.(event);
|
|
171
|
+
break;
|
|
172
|
+
case 'jump_height':
|
|
173
|
+
cb.onJumpHeight?.(event);
|
|
174
|
+
break;
|
|
175
|
+
case 'jump_discarded':
|
|
176
|
+
cb.onJumpDiscarded?.(event);
|
|
177
|
+
break;
|
|
178
|
+
case 'jump_result':
|
|
179
|
+
cb.onJumpResult?.(event);
|
|
180
|
+
break;
|
|
181
|
+
case 'jump_summary':
|
|
182
|
+
cb.onJumpSummary?.(event);
|
|
183
|
+
break;
|
|
184
|
+
case 'quality_changed':
|
|
185
|
+
cb.onQualityChanged?.(event);
|
|
186
|
+
break;
|
|
187
|
+
case 'performance_warning':
|
|
188
|
+
cb.onPerformanceWarning?.(event);
|
|
189
|
+
break;
|
|
190
|
+
case 'runtime_download_progress':
|
|
191
|
+
cb.onRuntimeDownloadProgress?.(event);
|
|
192
|
+
break;
|
|
193
|
+
}
|
|
194
|
+
});
|
|
195
|
+
const offMessages = context.addMessageListener((message) => {
|
|
196
|
+
callbacksRef.current.onMessage?.(message);
|
|
197
|
+
});
|
|
198
|
+
return () => {
|
|
199
|
+
offEvents();
|
|
200
|
+
offMessages();
|
|
201
|
+
};
|
|
202
|
+
}, [context]);
|
|
203
|
+
return context;
|
|
204
|
+
}
|