@pose-tracker/react-native-pose-estimation-light 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +45 -0
- package/PoseTrackerVision.podspec +28 -0
- package/README.md +161 -0
- package/THIRD_PARTY_NOTICES.md +19 -0
- package/ios/PoseTrackerVision/PoseTrackerBodyPosePlugin.m +18 -0
- package/ios/PoseTrackerVision/PoseTrackerBodyPosePlugin.swift +108 -0
- package/lib/PoseTrackerProvider.d.ts +102 -0
- package/lib/PoseTrackerProvider.js +204 -0
- package/lib/api/configure.d.ts +29 -0
- package/lib/api/configure.js +64 -0
- package/lib/api/skeleton.d.ts +17 -0
- package/lib/api/skeleton.js +53 -0
- package/lib/api/track.d.ts +61 -0
- package/lib/api/track.js +146 -0
- package/lib/backends/PoseBackend.d.ts +50 -0
- package/lib/backends/PoseBackend.js +11 -0
- package/lib/backends/vision/VisionPoseBackend.d.ts +72 -0
- package/lib/backends/vision/VisionPoseBackend.js +177 -0
- package/lib/backends/vision/mapVisionJoints.d.ts +35 -0
- package/lib/backends/vision/mapVisionJoints.js +162 -0
- package/lib/backends/vision/optionalVision.d.ts +32 -0
- package/lib/backends/vision/optionalVision.js +76 -0
- package/lib/backends/webview/WebViewPoseBackend.d.ts +223 -0
- package/lib/backends/webview/WebViewPoseBackend.js +382 -0
- package/lib/backends/webview/brandAssets.d.ts +5 -0
- package/lib/backends/webview/brandAssets.js +8 -0
- package/lib/backends/webview/onlineRuntime.d.ts +45 -0
- package/lib/backends/webview/onlineRuntime.js +57 -0
- package/lib/backends/webview/poseHtml.d.ts +71 -0
- package/lib/backends/webview/poseHtml.js +188 -0
- package/lib/backends/webview/poseRuntimeSource.d.ts +3 -0
- package/lib/backends/webview/poseRuntimeSource.js +4 -0
- package/lib/cache/obfuscate.d.ts +18 -0
- package/lib/cache/obfuscate.js +90 -0
- package/lib/camera/PoseCameraView.d.ts +51 -0
- package/lib/camera/PoseCameraView.js +99 -0
- package/lib/camera/WebViewPoseView.d.ts +81 -0
- package/lib/camera/WebViewPoseView.js +312 -0
- package/lib/client.d.ts +361 -0
- package/lib/client.js +1046 -0
- package/lib/diagnostics/logReport.d.ts +23 -0
- package/lib/diagnostics/logReport.js +102 -0
- package/lib/engine/EngineLoader.d.ts +95 -0
- package/lib/engine/EngineLoader.js +347 -0
- package/lib/engine/types.d.ts +83 -0
- package/lib/engine/types.js +11 -0
- package/lib/events/classicMessage.d.ts +31 -0
- package/lib/events/classicMessage.js +259 -0
- package/lib/exercises/aliases.d.ts +18 -0
- package/lib/exercises/aliases.js +48 -0
- package/lib/index.d.ts +72 -0
- package/lib/index.js +170 -0
- package/lib/models/poseModels.d.ts +39 -0
- package/lib/models/poseModels.js +63 -0
- package/lib/quality/AdaptiveQualityController.d.ts +120 -0
- package/lib/quality/AdaptiveQualityController.js +423 -0
- package/lib/quality/RuntimeGuard.d.ts +34 -0
- package/lib/quality/RuntimeGuard.js +105 -0
- package/lib/quality/captureMode.d.ts +82 -0
- package/lib/quality/captureMode.js +75 -0
- package/lib/quality/deviceCapability.d.ts +37 -0
- package/lib/quality/deviceCapability.js +177 -0
- package/lib/quality/profiles.d.ts +125 -0
- package/lib/quality/profiles.js +202 -0
- package/lib/runtime/RuntimeCache.d.ts +75 -0
- package/lib/runtime/RuntimeCache.js +230 -0
- package/lib/sdkVersion.d.ts +2 -0
- package/lib/sdkVersion.js +5 -0
- package/lib/support/optionalModules.d.ts +46 -0
- package/lib/support/optionalModules.js +82 -0
- package/lib/types/acceleration.d.ts +86 -0
- package/lib/types/acceleration.js +15 -0
- package/lib/types/events.d.ts +345 -0
- package/lib/types/events.js +10 -0
- package/lib/types/features.d.ts +90 -0
- package/lib/types/features.js +122 -0
- package/lib/types/manifest.d.ts +197 -0
- package/lib/types/manifest.js +10 -0
- package/lib/types/pose.d.ts +36 -0
- package/lib/types/pose.js +41 -0
- package/lib/types/preload.d.ts +17 -0
- package/lib/types/preload.js +10 -0
- package/lib/types/skeleton.d.ts +29 -0
- package/lib/types/skeleton.js +52 -0
- package/package.json +92 -0
- package/react-native.config.js +12 -0
- package/src/PoseTrackerProvider.tsx +332 -0
- package/src/api/configure.ts +82 -0
- package/src/api/skeleton.ts +60 -0
- package/src/api/track.ts +182 -0
- package/src/backends/PoseBackend.ts +59 -0
- package/src/backends/vision/VisionPoseBackend.ts +238 -0
- package/src/backends/vision/mapVisionJoints.ts +185 -0
- package/src/backends/vision/optionalVision.ts +94 -0
- package/src/backends/webview/WebViewPoseBackend.ts +582 -0
- package/src/backends/webview/brandAssets.ts +5 -0
- package/src/backends/webview/onlineRuntime.ts +94 -0
- package/src/backends/webview/poseHtml.ts +259 -0
- package/src/backends/webview/poseRuntimeSource.d.ts +3 -0
- package/src/cache/obfuscate.ts +94 -0
- package/src/camera/PoseCameraView.tsx +223 -0
- package/src/camera/WebViewPoseView.tsx +466 -0
- package/src/client.ts +1295 -0
- package/src/diagnostics/logReport.ts +131 -0
- package/src/engine/EngineLoader.ts +428 -0
- package/src/engine/types.ts +93 -0
- package/src/events/classicMessage.ts +271 -0
- package/src/exercises/aliases.ts +50 -0
- package/src/index.ts +243 -0
- package/src/models/poseModels.ts +93 -0
- package/src/quality/AdaptiveQualityController.ts +579 -0
- package/src/quality/RuntimeGuard.ts +119 -0
- package/src/quality/captureMode.ts +97 -0
- package/src/quality/deviceCapability.ts +187 -0
- package/src/quality/profiles.ts +253 -0
- package/src/runtime/RuntimeCache.ts +300 -0
- package/src/sdkVersion.ts +2 -0
- package/src/support/optionalModules.ts +114 -0
- package/src/types/acceleration.ts +94 -0
- package/src/types/events.ts +437 -0
- package/src/types/features.ts +176 -0
- package/src/types/manifest.ts +230 -0
- package/src/types/pose.ts +85 -0
- package/src/types/preload.ts +19 -0
- package/src/types/skeleton.ts +77 -0
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Online (light) pose-runtime loader.
|
|
3
|
+
*
|
|
4
|
+
* The npm package ships only the thin page runtime (`pose-runtime.js`).
|
|
5
|
+
* TF.js + WASM backends load from a CDN; MoveNet (or a custom graph model)
|
|
6
|
+
* loads from a URL on every WebView boot — same idea as the historical
|
|
7
|
+
* PoseTracker Front tracking page.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import { POSE_RUNTIME_JS, POSE_RUNTIME_VERSION } from './poseRuntimeSource';
|
|
11
|
+
import {
|
|
12
|
+
resolvePoseModel,
|
|
13
|
+
type PoseModelAlias,
|
|
14
|
+
type ResolvedPoseModel,
|
|
15
|
+
} from '../../models/poseModels';
|
|
16
|
+
|
|
17
|
+
/** TF.js version pinned for CDN scripts (matches offline SDK build). */
|
|
18
|
+
export const ONLINE_TFJS_VERSION = '4.22.0';
|
|
19
|
+
|
|
20
|
+
const JSDELIVR = 'https://cdn.jsdelivr.net/npm';
|
|
21
|
+
|
|
22
|
+
export interface OnlineRuntimeUrls {
|
|
23
|
+
/** Ordered <script src> list (core → converter → webgl → wasm). */
|
|
24
|
+
tfjsScriptUrls: string[];
|
|
25
|
+
/** Directory passed to `tf.wasm.setWasmPaths`. */
|
|
26
|
+
tfjsWasmPath: string;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export interface OnlineRuntimeParts {
|
|
30
|
+
version: string;
|
|
31
|
+
delivery: 'online';
|
|
32
|
+
modelId: string;
|
|
33
|
+
modelUrl: string;
|
|
34
|
+
tfjsScriptUrls: string[];
|
|
35
|
+
tfjsWasmPath: string;
|
|
36
|
+
/** Thin page runtime shipped in the package (~50 KB). */
|
|
37
|
+
runtimeJs: string;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export interface GetOnlineRuntimeOptions {
|
|
41
|
+
model?: PoseModelAlias;
|
|
42
|
+
modelUrl?: string;
|
|
43
|
+
/**
|
|
44
|
+
* Override jsDelivr package root, e.g.
|
|
45
|
+
* `https://cdn.jsdelivr.net/npm` (default) or a mirrored host.
|
|
46
|
+
*/
|
|
47
|
+
tfjsCdnBase?: string;
|
|
48
|
+
/** Override TF.js version segment in CDN URLs. */
|
|
49
|
+
tfjsVersion?: string;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
export function defaultOnlineRuntimeUrls(
|
|
53
|
+
cdnBase: string = JSDELIVR,
|
|
54
|
+
tfjsVersion: string = ONLINE_TFJS_VERSION,
|
|
55
|
+
): OnlineRuntimeUrls {
|
|
56
|
+
const base = cdnBase.replace(/\/$/, '');
|
|
57
|
+
return {
|
|
58
|
+
tfjsScriptUrls: [
|
|
59
|
+
`${base}/@tensorflow/tfjs-core@${tfjsVersion}/dist/tf-core.min.js`,
|
|
60
|
+
`${base}/@tensorflow/tfjs-converter@${tfjsVersion}/dist/tf-converter.min.js`,
|
|
61
|
+
`${base}/@tensorflow/tfjs-backend-webgl@${tfjsVersion}/dist/tf-backend-webgl.min.js`,
|
|
62
|
+
`${base}/@tensorflow/tfjs-backend-wasm@${tfjsVersion}/dist/tf-backend-wasm.min.js`,
|
|
63
|
+
],
|
|
64
|
+
tfjsWasmPath: `${base}/@tensorflow/tfjs-backend-wasm@${tfjsVersion}/dist/`,
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* Build the online runtime descriptor used by {@link buildPoseHtml}.
|
|
70
|
+
* Throws when the selected model cannot be loaded (e.g. blazepose without URL).
|
|
71
|
+
*/
|
|
72
|
+
export function getOnlineRuntimeParts(options: GetOnlineRuntimeOptions = {}): OnlineRuntimeParts {
|
|
73
|
+
const resolved: ResolvedPoseModel = resolvePoseModel({
|
|
74
|
+
model: options.model,
|
|
75
|
+
modelUrl: options.modelUrl,
|
|
76
|
+
});
|
|
77
|
+
if (!resolved.modelUrl) {
|
|
78
|
+
throw new Error(resolved.unsupportedReason ?? `Model "${resolved.modelId}" is unavailable.`);
|
|
79
|
+
}
|
|
80
|
+
const urls = defaultOnlineRuntimeUrls(options.tfjsCdnBase, options.tfjsVersion);
|
|
81
|
+
return {
|
|
82
|
+
version: POSE_RUNTIME_VERSION,
|
|
83
|
+
delivery: 'online',
|
|
84
|
+
modelId: resolved.modelId,
|
|
85
|
+
modelUrl: resolved.modelUrl,
|
|
86
|
+
tfjsScriptUrls: urls.tfjsScriptUrls,
|
|
87
|
+
tfjsWasmPath: urls.tfjsWasmPath,
|
|
88
|
+
runtimeJs: POSE_RUNTIME_JS,
|
|
89
|
+
};
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
export function getOnlineRuntimeVersion(): string {
|
|
93
|
+
return POSE_RUNTIME_VERSION;
|
|
94
|
+
}
|
|
@@ -0,0 +1,259 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* WebView HTML assembler (light / online).
|
|
3
|
+
*
|
|
4
|
+
* Loads TF.js from CDN script tags and MoveNet (or a custom graph model)
|
|
5
|
+
* from {@link OnlineRuntimeParts.modelUrl}. The thin page runtime is still
|
|
6
|
+
* inlined from the package. Same camera / adaptive quality / postMessage
|
|
7
|
+
* events as the offline SDK.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import { Platform } from 'react-native';
|
|
11
|
+
import {
|
|
12
|
+
ANDROID_INFER_FRAME_SKIP,
|
|
13
|
+
ANDROID_PERF_DEBUG,
|
|
14
|
+
ANDROID_PREPROCESS_PATH,
|
|
15
|
+
ANDROID_SOFT_CAP_PROFILE,
|
|
16
|
+
CAPTURE_CONSTRAINT_MODE,
|
|
17
|
+
} from '../../quality/captureMode';
|
|
18
|
+
import type { CapturePriority } from '../../quality/profiles';
|
|
19
|
+
import type { SkeletonDefinition } from '../../types/skeleton';
|
|
20
|
+
import { POSETRACKER_LOGO_DATA_URL } from './brandAssets';
|
|
21
|
+
import type { OnlineRuntimeParts } from './onlineRuntime';
|
|
22
|
+
|
|
23
|
+
/** Bumped on every assembler-path change — appears in WebView diag logs. */
|
|
24
|
+
export const POSE_HTML_BUILD = '20260811-onlineLight-localhostBase';
|
|
25
|
+
|
|
26
|
+
/** Default boot overlay copy (WebView `loading_message` parity). */
|
|
27
|
+
export const DEFAULT_LOADING_TEXT = 'AI Loading';
|
|
28
|
+
|
|
29
|
+
export interface PoseHtmlOptions {
|
|
30
|
+
facingMode?: 'user' | 'environment';
|
|
31
|
+
/** Min keypoint score for the in-page skeleton. Default 0.25. */
|
|
32
|
+
minScore?: number;
|
|
33
|
+
/**
|
|
34
|
+
* getUserMedia resolution hint. Defaults follow the adaptive quality
|
|
35
|
+
* profile (often UltraLite on Android, Pro/Prime on iOS). Inference always
|
|
36
|
+
* letterboxes down to 192×192 regardless.
|
|
37
|
+
*/
|
|
38
|
+
idealWidth?: number;
|
|
39
|
+
idealHeight?: number;
|
|
40
|
+
idealFrameRate?: number;
|
|
41
|
+
/** Active adaptive quality profile id (ceiling for warm-up downgrades). */
|
|
42
|
+
profileId?: string;
|
|
43
|
+
/**
|
|
44
|
+
* Platform minimum target FPS (floor). iOS ≈ 30, Android ≈ 10 (experiment).
|
|
45
|
+
* Passed from RN so the HTML page shares the same AdaptiveChoice policy.
|
|
46
|
+
*/
|
|
47
|
+
minTargetFps?: number;
|
|
48
|
+
/** Draw the skeleton overlay inside the page. Default true. */
|
|
49
|
+
drawSkeleton?: boolean;
|
|
50
|
+
/**
|
|
51
|
+
* `performance` (default) applies Android soft-cap / warm-up floor mapping.
|
|
52
|
+
* `quality` disables soft-cap and opens at prime / device-native HD.
|
|
53
|
+
*/
|
|
54
|
+
capturePriority?: CapturePriority;
|
|
55
|
+
/**
|
|
56
|
+
* `basic` (default for warmers): model + WebGL zeros only — no getUserMedia.
|
|
57
|
+
* `full`: open the camera after warm-up (camera screens / explicit full preload).
|
|
58
|
+
*/
|
|
59
|
+
coldStart?: 'basic' | 'full';
|
|
60
|
+
/**
|
|
61
|
+
* Custom skeleton overlay (Strapi skeleton document). When omitted, the
|
|
62
|
+
* page uses PoseTrackerFront `DEFAULT_SKELETON` (navy + gold).
|
|
63
|
+
*/
|
|
64
|
+
skeletonDef?: SkeletonDefinition | null;
|
|
65
|
+
/**
|
|
66
|
+
* Boot overlay label (WebView `loading_message` query param parity).
|
|
67
|
+
* Default {@link DEFAULT_LOADING_TEXT}. Technical progress stays on
|
|
68
|
+
* `initialization` / `diag` events — never on this UI.
|
|
69
|
+
*/
|
|
70
|
+
loadingText?: string;
|
|
71
|
+
/**
|
|
72
|
+
* Bottom-right “powered by” + join-logo watermark on the live camera
|
|
73
|
+
* surface. Hosts typically derive this from plan (`!isPaidPlan`).
|
|
74
|
+
* Ignored / forced off for `coldStart: 'basic'` warmers.
|
|
75
|
+
*/
|
|
76
|
+
showWatermark?: boolean;
|
|
77
|
+
/** Show the technical `#hud` overlay (FPS / backend). Default false. */
|
|
78
|
+
debugHud?: boolean;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
const PAGE_CSS = `
|
|
82
|
+
html, body { margin:0; padding:0; width:100%; height:100%; background:#000; overflow:hidden; }
|
|
83
|
+
#wrap { position:relative; width:100%; height:100%; background:#000; }
|
|
84
|
+
video, canvas {
|
|
85
|
+
position:absolute; inset:0; width:100%; height:100%;
|
|
86
|
+
object-fit:cover; opacity:0; transition:opacity 180ms ease-out;
|
|
87
|
+
}
|
|
88
|
+
video { z-index:1; background:#000; }
|
|
89
|
+
canvas { z-index:2; pointer-events:none; }
|
|
90
|
+
#boot {
|
|
91
|
+
position:absolute; inset:0; z-index:4;
|
|
92
|
+
display:flex; flex-direction:column; align-items:center; justify-content:center;
|
|
93
|
+
background:#0a0a0a; color:#fff;
|
|
94
|
+
font:15px/1.4 -apple-system,system-ui,sans-serif;
|
|
95
|
+
transition:opacity 180ms ease-out;
|
|
96
|
+
gap:0;
|
|
97
|
+
padding:24px;
|
|
98
|
+
box-sizing:border-box;
|
|
99
|
+
}
|
|
100
|
+
#boot.hide { opacity:0; pointer-events:none; }
|
|
101
|
+
#boot .boot-powered {
|
|
102
|
+
color:rgba(255,255,255,0.55);
|
|
103
|
+
font-size:11px;
|
|
104
|
+
font-weight:600;
|
|
105
|
+
letter-spacing:0.14em;
|
|
106
|
+
text-transform:uppercase;
|
|
107
|
+
margin-bottom:10px;
|
|
108
|
+
}
|
|
109
|
+
#boot .boot-logo {
|
|
110
|
+
width:min(220px, 62vw);
|
|
111
|
+
height:auto;
|
|
112
|
+
margin-bottom:22px;
|
|
113
|
+
display:block;
|
|
114
|
+
box-sizing:border-box;
|
|
115
|
+
background:#3a3a3a;
|
|
116
|
+
padding:14px 16px;
|
|
117
|
+
border-radius:10px;
|
|
118
|
+
}
|
|
119
|
+
#boot .boot-spinner {
|
|
120
|
+
width:28px; height:28px; margin-bottom:16px;
|
|
121
|
+
border:2px solid #333; border-top-color:#ffc300; border-radius:50%;
|
|
122
|
+
animation:ptspin 0.8s linear infinite;
|
|
123
|
+
}
|
|
124
|
+
#boot .boot-msg {
|
|
125
|
+
color:#e8e8e8; font-size:14px; font-weight:500;
|
|
126
|
+
text-align:center; padding:0 12px; min-height:1.2em;
|
|
127
|
+
}
|
|
128
|
+
#boot .boot-msg.is-error { color:#FE8370; font-size:13px; }
|
|
129
|
+
@keyframes ptspin { to { transform:rotate(360deg); } }
|
|
130
|
+
#wm {
|
|
131
|
+
position:absolute; right:10px; bottom:10px; z-index:3;
|
|
132
|
+
display:none; flex-direction:column; align-items:flex-end;
|
|
133
|
+
pointer-events:none;
|
|
134
|
+
text-shadow:0 1px 2px rgba(0,0,0,0.55);
|
|
135
|
+
}
|
|
136
|
+
#wm.show { display:flex; }
|
|
137
|
+
#wm .wm-powered {
|
|
138
|
+
color:rgba(255,255,255,0.7);
|
|
139
|
+
font:600 11px/1.2 -apple-system,system-ui,sans-serif;
|
|
140
|
+
letter-spacing:0.12em;
|
|
141
|
+
text-transform:uppercase;
|
|
142
|
+
margin-bottom:4px;
|
|
143
|
+
}
|
|
144
|
+
#wm .wm-logo {
|
|
145
|
+
width:130px; height:auto; display:block;
|
|
146
|
+
opacity:0.92;
|
|
147
|
+
filter:drop-shadow(0 1px 2px rgba(0,0,0,0.45));
|
|
148
|
+
}
|
|
149
|
+
#hud {
|
|
150
|
+
position:absolute; left:8px; right:8px; bottom:8px; z-index:3;
|
|
151
|
+
color:#ffc300; font:12px/1.3 -apple-system,system-ui,sans-serif;
|
|
152
|
+
text-shadow:0 1px 2px #000; pointer-events:none;
|
|
153
|
+
display:none;
|
|
154
|
+
}
|
|
155
|
+
#hud.debug { display:block; }
|
|
156
|
+
`;
|
|
157
|
+
|
|
158
|
+
/** Escape text for safe HTML text-node content. */
|
|
159
|
+
function escapeHtml(text: string): string {
|
|
160
|
+
return text
|
|
161
|
+
.replace(/&/g, '&')
|
|
162
|
+
.replace(/</g, '<')
|
|
163
|
+
.replace(/>/g, '>')
|
|
164
|
+
.replace(/"/g, '"')
|
|
165
|
+
.replace(/'/g, ''');
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
/** `</script` inside injected JS text would terminate the inline script tag. */
|
|
169
|
+
function escapeScript(js: string): string {
|
|
170
|
+
return js.replace(/<\/script/gi, '<\\/script');
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
function escapeAttr(value: string): string {
|
|
174
|
+
return value.replace(/&/g, '&').replace(/"/g, '"').replace(/</g, '<');
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
/**
|
|
178
|
+
* Assemble the pose page for online delivery: CDN TF.js + remote model URL
|
|
179
|
+
* + inlined thin page runtime.
|
|
180
|
+
*/
|
|
181
|
+
export function buildPoseHtml(parts: OnlineRuntimeParts, options?: PoseHtmlOptions): string {
|
|
182
|
+
const isAndroid = Platform.OS === 'android';
|
|
183
|
+
const capturePriority: CapturePriority =
|
|
184
|
+
options?.capturePriority === 'quality' ? 'quality' : 'performance';
|
|
185
|
+
const preferQuality = capturePriority === 'quality';
|
|
186
|
+
const coldStart = options?.coldStart === 'full' ? 'full' : 'basic';
|
|
187
|
+
const loadingText =
|
|
188
|
+
typeof options?.loadingText === 'string' && options.loadingText.trim().length > 0
|
|
189
|
+
? options.loadingText.trim()
|
|
190
|
+
: DEFAULT_LOADING_TEXT;
|
|
191
|
+
const watermarkOn = coldStart === 'full' && (options?.showWatermark ?? true);
|
|
192
|
+
const debugHud = !!options?.debugHud;
|
|
193
|
+
const config = JSON.stringify({
|
|
194
|
+
facingMode: options?.facingMode ?? 'user',
|
|
195
|
+
minScore: options?.minScore ?? 0.25,
|
|
196
|
+
idealWidth: options?.idealWidth ?? 480,
|
|
197
|
+
idealHeight: options?.idealHeight ?? 360,
|
|
198
|
+
idealFrameRate: options?.idealFrameRate ?? 24,
|
|
199
|
+
profileId: options?.profileId ?? 'ultralite',
|
|
200
|
+
minTargetFps: options?.minTargetFps ?? 15,
|
|
201
|
+
drawSkeleton: options?.drawSkeleton ?? true,
|
|
202
|
+
coldStart,
|
|
203
|
+
skeletonDef: options?.skeletonDef ?? null,
|
|
204
|
+
loadingText,
|
|
205
|
+
showWatermark: watermarkOn,
|
|
206
|
+
debugHud,
|
|
207
|
+
captureConstraintMode: CAPTURE_CONSTRAINT_MODE,
|
|
208
|
+
capturePriority,
|
|
209
|
+
platform: Platform.OS,
|
|
210
|
+
modelUrl: parts.modelUrl,
|
|
211
|
+
modelId: parts.modelId,
|
|
212
|
+
tfjsWasmPath: parts.tfjsWasmPath,
|
|
213
|
+
inferFrameSkip:
|
|
214
|
+
isAndroid && ANDROID_INFER_FRAME_SKIP > 0 ? ANDROID_INFER_FRAME_SKIP : 0,
|
|
215
|
+
preprocessPath: isAndroid ? ANDROID_PREPROCESS_PATH : 'imagebitmap',
|
|
216
|
+
softCapProfile:
|
|
217
|
+
isAndroid && !preferQuality ? ANDROID_SOFT_CAP_PROFILE : null,
|
|
218
|
+
perfDebug: isAndroid && ANDROID_PERF_DEBUG,
|
|
219
|
+
});
|
|
220
|
+
|
|
221
|
+
const cdnScripts = parts.tfjsScriptUrls
|
|
222
|
+
.map((url) => `<script src="${escapeAttr(url)}"></script>`)
|
|
223
|
+
.join('');
|
|
224
|
+
|
|
225
|
+
return [
|
|
226
|
+
'<!DOCTYPE html><html><head><meta charset="utf-8" />',
|
|
227
|
+
'<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1, user-scalable=no" />',
|
|
228
|
+
'<style>', PAGE_CSS, '</style></head><body>',
|
|
229
|
+
'<div id="wrap">',
|
|
230
|
+
'<video id="video" playsinline muted autoplay></video>',
|
|
231
|
+
'<canvas id="overlay"></canvas>',
|
|
232
|
+
'<div id="wm">',
|
|
233
|
+
'<div class="wm-powered">powered by</div>',
|
|
234
|
+
'<img class="wm-logo" alt="PoseTracker" src="', POSETRACKER_LOGO_DATA_URL, '" />',
|
|
235
|
+
'</div>',
|
|
236
|
+
'<div id="boot">',
|
|
237
|
+
'<div class="boot-powered">powered by</div>',
|
|
238
|
+
'<img class="boot-logo" alt="PoseTracker" src="', POSETRACKER_LOGO_DATA_URL, '" />',
|
|
239
|
+
'<div class="boot-spinner"></div>',
|
|
240
|
+
'<div class="boot-msg">', escapeHtml(loadingText), '</div>',
|
|
241
|
+
'</div>',
|
|
242
|
+
'<div id="hud"', debugHud || (isAndroid && ANDROID_PERF_DEBUG) ? ' class="debug"' : '', '>',
|
|
243
|
+
'booting…',
|
|
244
|
+
'</div>',
|
|
245
|
+
'</div>',
|
|
246
|
+
cdnScripts,
|
|
247
|
+
'<script>window.__PT_BUILD=', JSON.stringify(`${POSE_HTML_BUILD}/${parts.version}`),
|
|
248
|
+
';window.__PT_CONFIG=', config,
|
|
249
|
+
';window.__PT_MODEL_URL=', JSON.stringify(parts.modelUrl),
|
|
250
|
+
';window.__PT_WASM_PATH=', JSON.stringify(parts.tfjsWasmPath),
|
|
251
|
+
';window.__PT_MODEL_JSON=null',
|
|
252
|
+
';window.__PT_WEIGHTS_B64=null',
|
|
253
|
+
';window.__PT_WASM_B64=null',
|
|
254
|
+
';window.__PT_PIPELINE_WASM_B64=null',
|
|
255
|
+
';</script>',
|
|
256
|
+
'<script>', escapeScript(parts.runtimeJs), '</script>',
|
|
257
|
+
'</body></html>',
|
|
258
|
+
].join('');
|
|
259
|
+
}
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Obfuscation of the local session cache (manifest + engine bundle).
|
|
3
|
+
*
|
|
4
|
+
* Purpose: the cached business logic and exercise configs must not be
|
|
5
|
+
* readable without the API key that earned them (commercial boundary). The
|
|
6
|
+
* scheme is an SHA-256-based XOR keystream keyed by the API token — this is
|
|
7
|
+
* deliberate *obfuscation*, not cryptographic security against an attacker
|
|
8
|
+
* who owns both the device and a valid token (such an attacker can already
|
|
9
|
+
* read the downloaded engine at runtime). What it guarantees:
|
|
10
|
+
* - a device with no API token cannot exploit a cache left by another app
|
|
11
|
+
* or a previous integration;
|
|
12
|
+
* - the cache written for token A is unreadable after switching to token B.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
import { sha256 } from 'js-sha256';
|
|
16
|
+
|
|
17
|
+
const MAGIC = 'ptc1:';
|
|
18
|
+
|
|
19
|
+
/** Derives the cache secret from the API token (never stored on disk). */
|
|
20
|
+
export function deriveCacheSecret(apiToken: string): string {
|
|
21
|
+
return sha256(`posetracker-sdk-cache:${apiToken}`);
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
function utf8Encode(text: string): Uint8Array {
|
|
25
|
+
// encodeURIComponent-based UTF-8, avoids relying on TextEncoder (Hermes).
|
|
26
|
+
const escaped = unescape(encodeURIComponent(text));
|
|
27
|
+
const bytes = new Uint8Array(escaped.length);
|
|
28
|
+
for (let i = 0; i < escaped.length; i++) {
|
|
29
|
+
bytes[i] = escaped.charCodeAt(i);
|
|
30
|
+
}
|
|
31
|
+
return bytes;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function utf8Decode(bytes: Uint8Array): string {
|
|
35
|
+
let escaped = '';
|
|
36
|
+
for (let i = 0; i < bytes.length; i++) {
|
|
37
|
+
escaped += String.fromCharCode(bytes[i]!);
|
|
38
|
+
}
|
|
39
|
+
return decodeURIComponent(escape(escaped));
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
function toHex(bytes: Uint8Array): string {
|
|
43
|
+
let hex = '';
|
|
44
|
+
for (let i = 0; i < bytes.length; i++) {
|
|
45
|
+
hex += bytes[i]!.toString(16).padStart(2, '0');
|
|
46
|
+
}
|
|
47
|
+
return hex;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function fromHex(hex: string): Uint8Array | null {
|
|
51
|
+
if (hex.length % 2 !== 0 || /[^0-9a-f]/.test(hex)) {
|
|
52
|
+
return null;
|
|
53
|
+
}
|
|
54
|
+
const bytes = new Uint8Array(hex.length / 2);
|
|
55
|
+
for (let i = 0; i < bytes.length; i++) {
|
|
56
|
+
bytes[i] = parseInt(hex.slice(i * 2, i * 2 + 2), 16);
|
|
57
|
+
}
|
|
58
|
+
return bytes;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/** SHA-256 counter-mode keystream. */
|
|
62
|
+
function xorWithKeystream(bytes: Uint8Array, secret: string): Uint8Array {
|
|
63
|
+
const out = new Uint8Array(bytes.length);
|
|
64
|
+
let counter = 0;
|
|
65
|
+
let offset = 0;
|
|
66
|
+
while (offset < bytes.length) {
|
|
67
|
+
const block = sha256.array(`${secret}:${counter++}`);
|
|
68
|
+
for (let i = 0; i < block.length && offset < bytes.length; i++, offset++) {
|
|
69
|
+
out[offset] = bytes[offset]! ^ block[i]!;
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
return out;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
export function sealString(plain: string, secret: string): string {
|
|
76
|
+
return MAGIC + toHex(xorWithKeystream(utf8Encode(plain), secret));
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/** Returns null when the payload is not a sealed blob or cannot be decoded. */
|
|
80
|
+
export function openString(sealed: string, secret: string): string | null {
|
|
81
|
+
if (!sealed.startsWith(MAGIC)) {
|
|
82
|
+
return null;
|
|
83
|
+
}
|
|
84
|
+
const bytes = fromHex(sealed.slice(MAGIC.length));
|
|
85
|
+
if (!bytes) {
|
|
86
|
+
return null;
|
|
87
|
+
}
|
|
88
|
+
try {
|
|
89
|
+
return utf8Decode(xorWithKeystream(bytes, secret));
|
|
90
|
+
} catch {
|
|
91
|
+
// Wrong secret produces invalid UTF-8 → decodeURIComponent throws.
|
|
92
|
+
return null;
|
|
93
|
+
}
|
|
94
|
+
}
|
|
@@ -0,0 +1,223 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Camera component for the Apple Vision backend (iOS native builds only):
|
|
3
|
+
*
|
|
4
|
+
* vision-camera frame ──detectBodyPose plugin──▶ VNDetectHumanBodyPoseRequest
|
|
5
|
+
* ──19 joints (Y-flipped)──▶ Pose
|
|
6
|
+
*
|
|
7
|
+
* Everything up to the throttled `runOnJS` happens on the frame-processor
|
|
8
|
+
* thread. Requires a native build (NOT Expo Go) and the Vision backend
|
|
9
|
+
* active (`preferredBackend: 'vision'`). For every other case use
|
|
10
|
+
* {@link WebViewPoseView} — the SDK's default runtime.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import React, { useRef } from 'react';
|
|
14
|
+
import { StyleSheet, Text, View, type StyleProp, type ViewStyle } from 'react-native';
|
|
15
|
+
|
|
16
|
+
import { usePoseTracker } from '../PoseTrackerProvider';
|
|
17
|
+
import {
|
|
18
|
+
getVisionCamera,
|
|
19
|
+
getWorkletsCore,
|
|
20
|
+
type VisionCameraFrame,
|
|
21
|
+
type VisionCameraModule,
|
|
22
|
+
type WorkletsCoreModule,
|
|
23
|
+
} from '../support/optionalModules';
|
|
24
|
+
import { VisionPoseBackend } from '../backends/vision/VisionPoseBackend';
|
|
25
|
+
import {
|
|
26
|
+
getVisionBodyPosePlugin,
|
|
27
|
+
isVisionPoseCameraAvailable,
|
|
28
|
+
type VisionFrameProcessorPlugin,
|
|
29
|
+
} from '../backends/vision/optionalVision';
|
|
30
|
+
import {
|
|
31
|
+
poseFromVisionPluginResult,
|
|
32
|
+
type VisionPluginResult,
|
|
33
|
+
} from '../backends/vision/mapVisionJoints';
|
|
34
|
+
import type { Pose } from '../types/pose';
|
|
35
|
+
|
|
36
|
+
export type PoseCameraUnavailableReason =
|
|
37
|
+
/** Optional native modules not installed (Expo Go or missing deps). */
|
|
38
|
+
| 'modules-missing'
|
|
39
|
+
/** Active backend is not Apple Vision (use WebViewPoseView instead). */
|
|
40
|
+
| 'native-backend-inactive'
|
|
41
|
+
/** Backend not initialized yet — preload()/warmup() has not completed. */
|
|
42
|
+
| 'backend-not-ready'
|
|
43
|
+
/** vision-camera found no camera device for the requested position. */
|
|
44
|
+
| 'no-camera-device';
|
|
45
|
+
|
|
46
|
+
export interface PoseCameraStats {
|
|
47
|
+
/** Synchronous inference duration for this frame, in ms. */
|
|
48
|
+
inferenceTimeMs: number;
|
|
49
|
+
/** Wall-clock timestamp of the processed frame (host clock, ms). */
|
|
50
|
+
timestampMs: number;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export interface PoseCameraViewProps {
|
|
54
|
+
style?: StyleProp<ViewStyle>;
|
|
55
|
+
/** Camera to use. Default: 'front'. */
|
|
56
|
+
position?: 'front' | 'back';
|
|
57
|
+
/** Stream frames when true (and the SDK is ready). Default: true. */
|
|
58
|
+
isActive?: boolean;
|
|
59
|
+
/** Camera fps hint passed to vision-camera (inference runs per frame). */
|
|
60
|
+
targetFps?: number;
|
|
61
|
+
/**
|
|
62
|
+
* Throttle for the worklet → JS pose callbacks (`runOnJS`). Inference
|
|
63
|
+
* still runs on every camera frame; only the JS notifications (keypoints
|
|
64
|
+
* events, overlays) are capped. Default: 30.
|
|
65
|
+
*/
|
|
66
|
+
maxPoseCallbacksPerSecond?: number;
|
|
67
|
+
/** Per-pose callback (after the pose was ingested into the SDK pipeline). */
|
|
68
|
+
onPose?: (pose: Pose, stats: PoseCameraStats) => void;
|
|
69
|
+
/** Rendered when the native camera path cannot run. */
|
|
70
|
+
renderFallback?: (reason: PoseCameraUnavailableReason) => React.ReactElement | null;
|
|
71
|
+
/** Overlay content (skeleton, HUD…), rendered above the camera preview. */
|
|
72
|
+
children?: React.ReactNode;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
export function PoseCameraView(props: PoseCameraViewProps): React.ReactElement | null {
|
|
76
|
+
const { client } = usePoseTracker();
|
|
77
|
+
const backend = client.getBackend();
|
|
78
|
+
|
|
79
|
+
if (!(backend instanceof VisionPoseBackend)) {
|
|
80
|
+
return fallback(props, 'native-backend-inactive');
|
|
81
|
+
}
|
|
82
|
+
if (!isVisionPoseCameraAvailable()) {
|
|
83
|
+
return fallback(props, 'modules-missing');
|
|
84
|
+
}
|
|
85
|
+
const plugin = backend.getFrameProcessorPlugin() ?? getVisionBodyPosePlugin();
|
|
86
|
+
if (!plugin) {
|
|
87
|
+
return fallback(props, 'backend-not-ready');
|
|
88
|
+
}
|
|
89
|
+
return (
|
|
90
|
+
<VisionPoseCameraInner
|
|
91
|
+
{...props}
|
|
92
|
+
backend={backend}
|
|
93
|
+
plugin={plugin}
|
|
94
|
+
visionCamera={getVisionCamera() as VisionCameraModule}
|
|
95
|
+
workletsCore={getWorkletsCore() as WorkletsCoreModule}
|
|
96
|
+
/>
|
|
97
|
+
);
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
function fallback(
|
|
101
|
+
props: PoseCameraViewProps,
|
|
102
|
+
reason: PoseCameraUnavailableReason,
|
|
103
|
+
): React.ReactElement | null {
|
|
104
|
+
if (props.renderFallback) {
|
|
105
|
+
return props.renderFallback(reason);
|
|
106
|
+
}
|
|
107
|
+
return (
|
|
108
|
+
<View style={[styles.fallback, props.style]}>
|
|
109
|
+
<Text style={styles.fallbackText}>{FALLBACK_MESSAGES[reason]}</Text>
|
|
110
|
+
</View>
|
|
111
|
+
);
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
const FALLBACK_MESSAGES: Record<PoseCameraUnavailableReason, string> = {
|
|
115
|
+
'modules-missing':
|
|
116
|
+
'PoseCameraView requires an iOS native build with react-native-vision-camera + ' +
|
|
117
|
+
'react-native-worklets-core. Not available in Expo Go — use WebViewPoseView.',
|
|
118
|
+
'native-backend-inactive':
|
|
119
|
+
"PoseCameraView requires the Apple Vision backend (preferredBackend: 'vision') — " +
|
|
120
|
+
'the client is on the WebView runtime (use WebViewPoseView instead).',
|
|
121
|
+
'backend-not-ready': 'PoseTracker is not ready yet — call preload() and wait for status "ready".',
|
|
122
|
+
'no-camera-device': 'No camera device available for the requested position.',
|
|
123
|
+
};
|
|
124
|
+
|
|
125
|
+
// ---------------------------------------------------------------------------
|
|
126
|
+
// Apple Vision path (iOS)
|
|
127
|
+
// ---------------------------------------------------------------------------
|
|
128
|
+
|
|
129
|
+
interface VisionPoseCameraInnerProps extends PoseCameraViewProps {
|
|
130
|
+
backend: VisionPoseBackend;
|
|
131
|
+
plugin: VisionFrameProcessorPlugin;
|
|
132
|
+
visionCamera: VisionCameraModule;
|
|
133
|
+
workletsCore: WorkletsCoreModule;
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
function VisionPoseCameraInner(props: VisionPoseCameraInnerProps): React.ReactElement {
|
|
137
|
+
const {
|
|
138
|
+
backend,
|
|
139
|
+
plugin,
|
|
140
|
+
visionCamera,
|
|
141
|
+
workletsCore,
|
|
142
|
+
position = 'front',
|
|
143
|
+
isActive = true,
|
|
144
|
+
targetFps,
|
|
145
|
+
maxPoseCallbacksPerSecond = 30,
|
|
146
|
+
onPose,
|
|
147
|
+
style,
|
|
148
|
+
children,
|
|
149
|
+
} = props;
|
|
150
|
+
|
|
151
|
+
const { client, status } = usePoseTracker();
|
|
152
|
+
|
|
153
|
+
const onPoseRef = useRef(onPose);
|
|
154
|
+
onPoseRef.current = onPose;
|
|
155
|
+
|
|
156
|
+
const device = visionCamera.useCameraDevice(position);
|
|
157
|
+
const lastEmitMs = workletsCore.useSharedValue(0);
|
|
158
|
+
|
|
159
|
+
const emitPose = workletsCore.useRunOnJS(
|
|
160
|
+
(pose: Pose, inferenceTimeMs: number) => {
|
|
161
|
+
client.ingestPose(pose);
|
|
162
|
+
backend.recordInferenceTime(inferenceTimeMs);
|
|
163
|
+
onPoseRef.current?.(pose, { inferenceTimeMs, timestampMs: pose.timestampMs });
|
|
164
|
+
},
|
|
165
|
+
[client, backend],
|
|
166
|
+
);
|
|
167
|
+
|
|
168
|
+
const minEmitIntervalMs = Math.max(0, Math.floor(1000 / Math.max(1, maxPoseCallbacksPerSecond)));
|
|
169
|
+
|
|
170
|
+
const frameProcessor = visionCamera.useFrameProcessor(
|
|
171
|
+
(frame: VisionCameraFrame) => {
|
|
172
|
+
'worklet';
|
|
173
|
+
const raw = plugin.call(frame) as VisionPluginResult | null;
|
|
174
|
+
if (!raw || !raw.joints || raw.joints.length === 0) {
|
|
175
|
+
return;
|
|
176
|
+
}
|
|
177
|
+
const inferenceTimeMs =
|
|
178
|
+
typeof raw.inferenceTimeMs === 'number' ? raw.inferenceTimeMs : 0;
|
|
179
|
+
const pose = poseFromVisionPluginResult(raw, Date.now());
|
|
180
|
+
if (!pose) {
|
|
181
|
+
return;
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
const now = Date.now();
|
|
185
|
+
if (now - lastEmitMs.value >= minEmitIntervalMs) {
|
|
186
|
+
lastEmitMs.value = now;
|
|
187
|
+
emitPose(pose, inferenceTimeMs);
|
|
188
|
+
}
|
|
189
|
+
},
|
|
190
|
+
[plugin, minEmitIntervalMs, emitPose, lastEmitMs],
|
|
191
|
+
);
|
|
192
|
+
|
|
193
|
+
if (!device) {
|
|
194
|
+
return fallback(props, 'no-camera-device') ?? <View style={style} />;
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
const Camera = visionCamera.Camera;
|
|
198
|
+
return (
|
|
199
|
+
<View style={[styles.container, style]}>
|
|
200
|
+
<Camera
|
|
201
|
+
style={StyleSheet.absoluteFill}
|
|
202
|
+
device={device}
|
|
203
|
+
isActive={isActive && status === 'ready'}
|
|
204
|
+
frameProcessor={frameProcessor}
|
|
205
|
+
// Vision reads CMSampleBuffer / CVPixelBuffer directly — yuv is fine.
|
|
206
|
+
pixelFormat="yuv"
|
|
207
|
+
{...(targetFps !== undefined ? { fps: targetFps } : {})}
|
|
208
|
+
/>
|
|
209
|
+
{children}
|
|
210
|
+
</View>
|
|
211
|
+
);
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
const styles = StyleSheet.create({
|
|
215
|
+
container: { overflow: 'hidden' },
|
|
216
|
+
fallback: {
|
|
217
|
+
alignItems: 'center',
|
|
218
|
+
justifyContent: 'center',
|
|
219
|
+
padding: 24,
|
|
220
|
+
backgroundColor: '#111',
|
|
221
|
+
},
|
|
222
|
+
fallbackText: { color: '#ccc', fontSize: 13, textAlign: 'center', lineHeight: 18 },
|
|
223
|
+
});
|