@pose-tracker/react-native-pose-estimation-light 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. package/LICENSE +45 -0
  2. package/PoseTrackerVision.podspec +28 -0
  3. package/README.md +161 -0
  4. package/THIRD_PARTY_NOTICES.md +19 -0
  5. package/ios/PoseTrackerVision/PoseTrackerBodyPosePlugin.m +18 -0
  6. package/ios/PoseTrackerVision/PoseTrackerBodyPosePlugin.swift +108 -0
  7. package/lib/PoseTrackerProvider.d.ts +102 -0
  8. package/lib/PoseTrackerProvider.js +204 -0
  9. package/lib/api/configure.d.ts +29 -0
  10. package/lib/api/configure.js +64 -0
  11. package/lib/api/skeleton.d.ts +17 -0
  12. package/lib/api/skeleton.js +53 -0
  13. package/lib/api/track.d.ts +61 -0
  14. package/lib/api/track.js +146 -0
  15. package/lib/backends/PoseBackend.d.ts +50 -0
  16. package/lib/backends/PoseBackend.js +11 -0
  17. package/lib/backends/vision/VisionPoseBackend.d.ts +72 -0
  18. package/lib/backends/vision/VisionPoseBackend.js +177 -0
  19. package/lib/backends/vision/mapVisionJoints.d.ts +35 -0
  20. package/lib/backends/vision/mapVisionJoints.js +162 -0
  21. package/lib/backends/vision/optionalVision.d.ts +32 -0
  22. package/lib/backends/vision/optionalVision.js +76 -0
  23. package/lib/backends/webview/WebViewPoseBackend.d.ts +223 -0
  24. package/lib/backends/webview/WebViewPoseBackend.js +382 -0
  25. package/lib/backends/webview/brandAssets.d.ts +5 -0
  26. package/lib/backends/webview/brandAssets.js +8 -0
  27. package/lib/backends/webview/onlineRuntime.d.ts +45 -0
  28. package/lib/backends/webview/onlineRuntime.js +57 -0
  29. package/lib/backends/webview/poseHtml.d.ts +71 -0
  30. package/lib/backends/webview/poseHtml.js +188 -0
  31. package/lib/backends/webview/poseRuntimeSource.d.ts +3 -0
  32. package/lib/backends/webview/poseRuntimeSource.js +4 -0
  33. package/lib/cache/obfuscate.d.ts +18 -0
  34. package/lib/cache/obfuscate.js +90 -0
  35. package/lib/camera/PoseCameraView.d.ts +51 -0
  36. package/lib/camera/PoseCameraView.js +99 -0
  37. package/lib/camera/WebViewPoseView.d.ts +81 -0
  38. package/lib/camera/WebViewPoseView.js +312 -0
  39. package/lib/client.d.ts +361 -0
  40. package/lib/client.js +1046 -0
  41. package/lib/diagnostics/logReport.d.ts +23 -0
  42. package/lib/diagnostics/logReport.js +102 -0
  43. package/lib/engine/EngineLoader.d.ts +95 -0
  44. package/lib/engine/EngineLoader.js +347 -0
  45. package/lib/engine/types.d.ts +83 -0
  46. package/lib/engine/types.js +11 -0
  47. package/lib/events/classicMessage.d.ts +31 -0
  48. package/lib/events/classicMessage.js +259 -0
  49. package/lib/exercises/aliases.d.ts +18 -0
  50. package/lib/exercises/aliases.js +48 -0
  51. package/lib/index.d.ts +72 -0
  52. package/lib/index.js +170 -0
  53. package/lib/models/poseModels.d.ts +39 -0
  54. package/lib/models/poseModels.js +63 -0
  55. package/lib/quality/AdaptiveQualityController.d.ts +120 -0
  56. package/lib/quality/AdaptiveQualityController.js +423 -0
  57. package/lib/quality/RuntimeGuard.d.ts +34 -0
  58. package/lib/quality/RuntimeGuard.js +105 -0
  59. package/lib/quality/captureMode.d.ts +82 -0
  60. package/lib/quality/captureMode.js +75 -0
  61. package/lib/quality/deviceCapability.d.ts +37 -0
  62. package/lib/quality/deviceCapability.js +177 -0
  63. package/lib/quality/profiles.d.ts +125 -0
  64. package/lib/quality/profiles.js +202 -0
  65. package/lib/runtime/RuntimeCache.d.ts +75 -0
  66. package/lib/runtime/RuntimeCache.js +230 -0
  67. package/lib/sdkVersion.d.ts +2 -0
  68. package/lib/sdkVersion.js +5 -0
  69. package/lib/support/optionalModules.d.ts +46 -0
  70. package/lib/support/optionalModules.js +82 -0
  71. package/lib/types/acceleration.d.ts +86 -0
  72. package/lib/types/acceleration.js +15 -0
  73. package/lib/types/events.d.ts +345 -0
  74. package/lib/types/events.js +10 -0
  75. package/lib/types/features.d.ts +90 -0
  76. package/lib/types/features.js +122 -0
  77. package/lib/types/manifest.d.ts +197 -0
  78. package/lib/types/manifest.js +10 -0
  79. package/lib/types/pose.d.ts +36 -0
  80. package/lib/types/pose.js +41 -0
  81. package/lib/types/preload.d.ts +17 -0
  82. package/lib/types/preload.js +10 -0
  83. package/lib/types/skeleton.d.ts +29 -0
  84. package/lib/types/skeleton.js +52 -0
  85. package/package.json +92 -0
  86. package/react-native.config.js +12 -0
  87. package/src/PoseTrackerProvider.tsx +332 -0
  88. package/src/api/configure.ts +82 -0
  89. package/src/api/skeleton.ts +60 -0
  90. package/src/api/track.ts +182 -0
  91. package/src/backends/PoseBackend.ts +59 -0
  92. package/src/backends/vision/VisionPoseBackend.ts +238 -0
  93. package/src/backends/vision/mapVisionJoints.ts +185 -0
  94. package/src/backends/vision/optionalVision.ts +94 -0
  95. package/src/backends/webview/WebViewPoseBackend.ts +582 -0
  96. package/src/backends/webview/brandAssets.ts +5 -0
  97. package/src/backends/webview/onlineRuntime.ts +94 -0
  98. package/src/backends/webview/poseHtml.ts +259 -0
  99. package/src/backends/webview/poseRuntimeSource.d.ts +3 -0
  100. package/src/cache/obfuscate.ts +94 -0
  101. package/src/camera/PoseCameraView.tsx +223 -0
  102. package/src/camera/WebViewPoseView.tsx +466 -0
  103. package/src/client.ts +1295 -0
  104. package/src/diagnostics/logReport.ts +131 -0
  105. package/src/engine/EngineLoader.ts +428 -0
  106. package/src/engine/types.ts +93 -0
  107. package/src/events/classicMessage.ts +271 -0
  108. package/src/exercises/aliases.ts +50 -0
  109. package/src/index.ts +243 -0
  110. package/src/models/poseModels.ts +93 -0
  111. package/src/quality/AdaptiveQualityController.ts +579 -0
  112. package/src/quality/RuntimeGuard.ts +119 -0
  113. package/src/quality/captureMode.ts +97 -0
  114. package/src/quality/deviceCapability.ts +187 -0
  115. package/src/quality/profiles.ts +253 -0
  116. package/src/runtime/RuntimeCache.ts +300 -0
  117. package/src/sdkVersion.ts +2 -0
  118. package/src/support/optionalModules.ts +114 -0
  119. package/src/types/acceleration.ts +94 -0
  120. package/src/types/events.ts +437 -0
  121. package/src/types/features.ts +176 -0
  122. package/src/types/manifest.ts +230 -0
  123. package/src/types/pose.ts +85 -0
  124. package/src/types/preload.ts +19 -0
  125. package/src/types/skeleton.ts +77 -0
package/src/client.ts ADDED
@@ -0,0 +1,1295 @@
1
+ /**
2
+ * Framework-agnostic orchestrator. `PoseTrackerProvider` is a thin React
3
+ * wrapper around this class.
4
+ *
5
+ * Lifecycle (exposed as `PoseTrackerStatus`):
6
+ * idle → configuring (handshake attempt) → downloading (engine bundle)
7
+ * → warming (TF runtime + model + dummy inferences) → ready | error
8
+ *
9
+ * Operating modes (commercial boundary, see ARCHITECTURE.md §Modes):
10
+ * - 'keypoints-only': always reachable — online MoveNet (CDN TF.js + model
11
+ * URL) + warm-up + camera pipeline, raw `keypoints` events only. A failed
12
+ * handshake (offline API, missing/invalid token, quota) NEVER blocks
13
+ * `ready`: it degrades to this mode with a non-fatal `error` event.
14
+ * `error` status is reserved for unrecoverable local failures (model load).
15
+ * - 'full-engine': requires a validated handshake — live, or replayed from
16
+ * the encrypted session cache written after a previous successful
17
+ * handshake (Sency-style offline cold start). The npm package itself
18
+ * ships ZERO movement intelligence.
19
+ *
20
+ * The mode upgrades at runtime without restarting the camera pipeline:
21
+ * `configure()` can be called at any time (e.g. when the network comes back
22
+ * or the host obtains a token) — the inference backend is untouched, and
23
+ * business events simply start flowing once the engine is up.
24
+ */
25
+
26
+ import { configure as handshake, ConfigureError, type ConfigureOptions } from './api/configure';
27
+ import { TrackError, UsageTracker } from './api/track';
28
+ import { EngineLoader, type EngineLoadResult, type FileStore } from './engine/EngineLoader';
29
+ import { createNativeFileStore } from './engine/EngineLoader';
30
+ import {
31
+ getOnlineRuntimeParts,
32
+ getOnlineRuntimeVersion,
33
+ type OnlineRuntimeParts,
34
+ } from './backends/webview/onlineRuntime';
35
+ import { deriveCacheSecret, openString, sealString } from './cache/obfuscate';
36
+ import type { CustomExerciseDescriptor, EngineSession, PoseTrackerEngine } from './engine/types';
37
+ import type { PoseBackend, PoseInputFrame } from './backends/PoseBackend';
38
+ import { VisionPoseBackend } from './backends/vision/VisionPoseBackend';
39
+ import { WebViewPoseBackend } from './backends/webview/WebViewPoseBackend';
40
+ import { findExerciseByIdOrAlias } from './exercises/aliases';
41
+ import { fetchSkeletonDefinition } from './api/skeleton';
42
+ import type { ExerciseConfig, ModelDescriptor, SdkManifest } from './types/manifest';
43
+ import type { ColdStartMode, PreloadOptions } from './types/preload';
44
+ import type { SkeletonDefinition } from './types/skeleton';
45
+ import type {
46
+ ErrorEvent,
47
+ PoseTrackerEvent,
48
+ PoseTrackerEventListener,
49
+ PoseTrackerMode,
50
+ PoseTrackerStatus,
51
+ } from './types/events';
52
+ import type { Pose } from './types/pose';
53
+ import type { AccelerationDiagnostics, AccelerationState } from './types/acceleration';
54
+ import {
55
+ defaultDiagnosticLogger,
56
+ logAccelerationReport,
57
+ logPlatformBanner,
58
+ } from './diagnostics/logReport';
59
+ import {
60
+ AdaptiveQualityController,
61
+ type QualityState,
62
+ } from './quality/AdaptiveQualityController';
63
+ import {
64
+ isQualityProfileId,
65
+ type CapturePriority,
66
+ type QualityChoice,
67
+ type QualityProfile,
68
+ } from './quality/profiles';
69
+ import {
70
+ toClassicNativeMessage,
71
+ type ClassicMessageListener,
72
+ type ClassicNativeMessage,
73
+ } from './events/classicMessage';
74
+ import {
75
+ FREE_PLAN_FEATURES_MESSAGE,
76
+ INVALID_TOKEN_MESSAGE,
77
+ featureNotSupportedMessage,
78
+ freeBlockedFeatures,
79
+ resolveFeatures,
80
+ type PoseTrackerFeatures,
81
+ type ResolvedFeatures,
82
+ } from './types/features';
83
+ import type { PoseModelAlias } from './models/poseModels';
84
+
85
+ const MANIFEST_CACHE_KEY = 'session.sealed';
86
+ const ENGINE_VERSION_KEY = 'engine.version';
87
+
88
+ /** Consecutive engine `processPose` failures before the session is stopped. */
89
+ const SESSION_ERROR_STREAK_LIMIT = 30;
90
+
91
+ /**
92
+ * Fallback descriptor when no manifest is available. Light SDK loads the
93
+ * graph model from {@link PoseTrackerClientOptions.modelUrl} (or the product
94
+ * default URL); this descriptor only feeds backend init metadata.
95
+ */
96
+ const DEFAULT_MOVENET: ModelDescriptor = {
97
+ modelId: 'movenet-singlepose-lightning',
98
+ format: 'tfjs-graph-model',
99
+ inputSize: 192,
100
+ version: '4',
101
+ };
102
+
103
+ /**
104
+ * Inference backend selection:
105
+ * - 'auto' (default) / 'webview': MoveNet SinglePose Lightning (17 keypoints,
106
+ * 192×192) inside a Chromium/WKWebView — TF.js WebGL, **online** (CDN TF.js
107
+ * + model URL each boot). Base runtime on BOTH platforms; works in Expo Go.
108
+ * - 'vision': Apple Vision (`VNDetectHumanBodyPoseRequest`) — iOS native
109
+ * builds only, explicit opt-in. No automatic fallback: an init failure
110
+ * surfaces as status 'error'.
111
+ */
112
+ export type PreferredBackend = 'auto' | 'webview' | 'vision';
113
+
114
+ export interface PoseTrackerClientOptions extends ConfigureOptions {
115
+ /** Injectable for tests / custom hosts (takes precedence over `preferredBackend`). */
116
+ backend?: PoseBackend;
117
+ /** See {@link PreferredBackend}. */
118
+ preferredBackend?: PreferredBackend;
119
+ engineLoader?: EngineLoader;
120
+ fileStore?: FileStore | null;
121
+ /** Injectable for tests. */
122
+ usageTracker?: UsageTracker;
123
+ /**
124
+ * Docs API `model` query parity (`movenet` default, `blazepose`, …).
125
+ * BlazePose is not wired in the light WebView yet — use `modelUrl` for a
126
+ * custom TF.js graph model. Ignored when {@link modelUrl} is set.
127
+ */
128
+ model?: PoseModelAlias;
129
+ /**
130
+ * Explicit TF.js graph-model topology URL. Default: PoseTracker Front
131
+ * MoveNet SinglePose Lightning
132
+ * (`https://app.posetracker.com/scripts/tmp_model_to_remove.json`).
133
+ */
134
+ modelUrl?: string;
135
+ /** Override jsDelivr (or mirror) root for TF.js scripts. */
136
+ tfjsCdnBase?: string;
137
+ /** Override TF.js CDN version pin (default `4.22.0`). */
138
+ tfjsVersion?: string;
139
+ /**
140
+ * Camera / preprocess quality tier. Default `AdaptiveChoice` — picks a
141
+ * profile from device capability, crash-loop guard, and live FPS.
142
+ * See docs/ADAPTIVE_QUALITY.md.
143
+ */
144
+ qualityChoice?: QualityChoice;
145
+ /**
146
+ * Trade-off between pose FPS and camera preview sharpness.
147
+ * Default `performance` — SDK may lower capture (esp. Android) to hold the
148
+ * FPS floor. Pass `quality` to keep a sharp preview and accept slower pose
149
+ * updates (no FPS-driven capture downgrade). See docs/ADAPTIVE_QUALITY.md.
150
+ */
151
+ capturePriority?: CapturePriority;
152
+ /**
153
+ * Tracking feature flags — SDK port of the WebView query params
154
+ * (`angles`, `recommendations`, `progression`, `keypoints`, `minGrade`)
155
+ * with the SAME plan gating and error messages as the WebView product:
156
+ * paid plans only; `free` gets the exact `TrackingAppV3` error strings.
157
+ * Defaults: everything off (WebView parity). Pose-only keypoints (no
158
+ * exercise) always stream and are never gated. See docs/FEATURES.md.
159
+ */
160
+ features?: PoseTrackerFeatures;
161
+ /**
162
+ * Receives SDK diagnostic lines (GL flags, GPU health check, fallbacks,
163
+ * context-loss recoveries). Route to your logger/telemetry; useful for
164
+ * triaging Android GPU issues in the field.
165
+ */
166
+ onDiagnostic?: (message: string) => void;
167
+ }
168
+
169
+ /**
170
+ * Per-session options for {@link PoseTrackerClient.startExercise} — the SDK
171
+ * port of the WebView query params `difficulty`, `userHeightCm` and
172
+ * `devicePitchDeg`.
173
+ */
174
+ export interface StartExerciseOptions {
175
+ /**
176
+ * Difficulty key into the movement `scale_acceptance` maps (FSM exercises).
177
+ * WebView `difficulty` param parity. Default: 'medium'.
178
+ */
179
+ difficulty?: string;
180
+ /** Athlete height in cm — REQUIRED by `jump_analysis` (cm/pixel calibration). */
181
+ userHeightCm?: number;
182
+ /** Device pitch in degrees — jump exercises compensate camera tilt. */
183
+ devicePitchDeg?: number;
184
+ }
185
+
186
+ export class PoseTrackerClient {
187
+ private status: PoseTrackerStatus = 'idle';
188
+ private mode: PoseTrackerMode = 'keypoints-only';
189
+ private lastError: ErrorEvent | null = null;
190
+ private manifest: SdkManifest | null = null;
191
+ private engine: PoseTrackerEngine | null = null;
192
+ private engineSource: EngineLoadResult['source'] | null = null;
193
+ private session: EngineSession | null = null;
194
+ private currentExerciseId: string | null = null;
195
+ private apiToken: string | null;
196
+
197
+ private backend: PoseBackend;
198
+ private readonly engineLoader: EngineLoader;
199
+ private readonly files: FileStore | null;
200
+ private readonly tracker: UsageTracker;
201
+ private readonly quality: AdaptiveQualityController;
202
+ private readonly listeners = new Set<PoseTrackerEventListener>();
203
+ private readonly messageListeners = new Set<ClassicMessageListener>();
204
+ private readonly stateListeners = new Set<() => void>();
205
+ private preloadPromise: Promise<void> | null = null;
206
+ /** Last requested cold-start mode (default basic — no getUserMedia). */
207
+ private coldStartMode: ColdStartMode = 'basic';
208
+ private configurePromise: Promise<boolean> | null = null;
209
+ /** One handshake per configuration cycle, shared by runtime warm + engine load. */
210
+ private handshakePromise: Promise<SdkManifest | null> | null = null;
211
+ /** Online pose-runtime descriptor (CDN TF.js + model URL + thin page runtime). */
212
+ private runtimePromise: Promise<OnlineRuntimeParts> | null = null;
213
+ /**
214
+ * Metered-session gate: with an API key, the `camera_start` track call must
215
+ * succeed online before key-gated features run ('refused' = offline/quota).
216
+ */
217
+ private meteredSessionState: 'idle' | 'pending' | 'validated' | 'refused' = 'idle';
218
+ /** Last camera_start payload — lets configure() retry a refused metered gate. */
219
+ private lastCameraStartInfo: { backend: string; profileId: string | null } | null = null;
220
+ /** Consecutive engine processPose failures (see SESSION_ERROR_STREAK_LIMIT). */
221
+ private sessionErrorStreak = 0;
222
+
223
+ /** Requested tracking features with WebView-parity defaults applied. */
224
+ private readonly features: ResolvedFeatures;
225
+ /** WebView-only keys passed by untyped hosts (blazepose, poseEngine, …). */
226
+ private readonly unsupportedFeatureKeys: string[];
227
+ /** One-shot flags so plan-gating errors are not re-emitted on every retry. */
228
+ private featureGateReported = { unsupported: false, freeBlock: false, missingToken: false };
229
+ private keypointsSuppressionLogged = false;
230
+
231
+ private readonly options: PoseTrackerClientOptions;
232
+
233
+ constructor(apiToken?: string, options: PoseTrackerClientOptions = {}) {
234
+ // Default: dump every diagnostic line to Metro / logcat so Android FPS
235
+ // issues (CPU fallback) are visible without any host wiring.
236
+ this.options = {
237
+ ...options,
238
+ onDiagnostic: options.onDiagnostic ?? defaultDiagnosticLogger,
239
+ };
240
+ this.apiToken = apiToken ?? null;
241
+ const resolved = resolveFeatures(options.features);
242
+ this.features = resolved.features;
243
+ this.unsupportedFeatureKeys = resolved.unsupportedKeys;
244
+ logPlatformBanner();
245
+ this.options.onDiagnostic?.(
246
+ `[posetracker] client created preferredBackend=${this.options.preferredBackend ?? 'auto'} ` +
247
+ `qualityChoice=${this.options.qualityChoice ?? 'AdaptiveChoice'} ` +
248
+ `capturePriority=${this.options.capturePriority ?? 'performance'} ` +
249
+ `hasApiToken=${Boolean(this.apiToken)} ` +
250
+ `features=${JSON.stringify(this.features)}` +
251
+ (this.unsupportedFeatureKeys.length > 0
252
+ ? ` unsupportedFeatureKeys=${this.unsupportedFeatureKeys.join(',')}`
253
+ : ''),
254
+ );
255
+ this.quality = new AdaptiveQualityController({
256
+ choice: this.options.qualityChoice ?? 'AdaptiveChoice',
257
+ capturePriority: this.options.capturePriority ?? 'performance',
258
+ onDiagnostic: this.options.onDiagnostic,
259
+ onQualityChanged: (event) => {
260
+ // Typed: quality_changed. Classic onMessage maps it to type "warning".
261
+ this.emit(event);
262
+ this.notifyState();
263
+ },
264
+ onPerformanceWarning: (event) => {
265
+ this.emit(event);
266
+ },
267
+ });
268
+ this.backend = options.backend ?? this.createBackend();
269
+ this.engineLoader =
270
+ options.engineLoader ??
271
+ new EngineLoader({
272
+ fileStore: options.fileStore,
273
+ onDiagnostic: this.options.onDiagnostic,
274
+ });
275
+ this.files =
276
+ options.fileStore !== undefined ? options.fileStore : createNativeFileStore('posetracker-engine');
277
+ this.tracker = options.usageTracker ?? new UsageTracker({ baseUrl: options.baseUrl });
278
+ this.options.onDiagnostic?.(
279
+ `[posetracker-light] selected backend=${this.backend.name} ` +
280
+ `onlinePoseRuntime=${getOnlineRuntimeVersion()} ` +
281
+ `fileStore=${this.files ? 'native' : 'none'}`,
282
+ );
283
+ }
284
+
285
+ /** Backend selection — see {@link PreferredBackend}. */
286
+ private createBackend(): PoseBackend {
287
+ const shared = {
288
+ onDiagnostic: this.options.onDiagnostic,
289
+ // Surface mid-session downgrades to the provider/hook without polling.
290
+ onAccelerationChange: () => this.notifyState(),
291
+ };
292
+ const preferred = this.options.preferredBackend ?? 'auto';
293
+
294
+ if (preferred === 'vision') {
295
+ // Explicit opt-in only (iOS native build). On Android / Expo Go,
296
+ // init() throws a clear error — no silent swap, the host asked for
297
+ // Vision. The WebView runtime remains available as a manual retry.
298
+ return new VisionPoseBackend(shared);
299
+ }
300
+
301
+ // 'auto' | 'webview': the offline WebView MoveNet runtime, both platforms.
302
+ return new WebViewPoseBackend({
303
+ ...shared,
304
+ onWarmupEstimate: (info) => {
305
+ const pageProfile = isQualityProfileId(info.profileId) ? info.profileId : null;
306
+ void this.quality.onWarmupEstimate({
307
+ medianInferenceMs: info.medianInferenceMs,
308
+ glRenderer: info.glRenderer,
309
+ pageSelectedProfile: pageProfile,
310
+ });
311
+ },
312
+ onReady: (info) => {
313
+ const pageProfile =
314
+ info.profileId && isQualityProfileId(info.profileId) ? info.profileId : null;
315
+ void this.quality.onRuntimeReady({
316
+ glRenderer: info.glRenderer,
317
+ medianInferenceMs: info.medianInferenceMs,
318
+ pageSelectedProfile: pageProfile,
319
+ });
320
+ // Usage metering happens HERE: camera started with the model ready
321
+ // (not at handshake, not at preload).
322
+ void this.handleCameraStart({
323
+ backend: info.backend,
324
+ profileId: info.profileId ?? null,
325
+ });
326
+ },
327
+ onStats: (stats) => {
328
+ void this.quality.onStats({
329
+ fps: stats.fps,
330
+ medianInferenceMs: stats.medianInferenceMs,
331
+ videoSize: stats.videoSize,
332
+ backend: stats.backend,
333
+ });
334
+ },
335
+ onTrackerEvent: (event) => {
336
+ this.emit(event);
337
+ },
338
+ });
339
+ }
340
+
341
+ // -------------------------------------------------------------------------
342
+ // Introspection & events
343
+ // -------------------------------------------------------------------------
344
+
345
+ getStatus(): PoseTrackerStatus {
346
+ return this.status;
347
+ }
348
+
349
+ getMode(): PoseTrackerMode {
350
+ return this.mode;
351
+ }
352
+
353
+ /** Last non-fatal or fatal error (also emitted as an `error` event). */
354
+ getError(): ErrorEvent | null {
355
+ return this.lastError;
356
+ }
357
+
358
+ getManifest(): SdkManifest | null {
359
+ return this.manifest;
360
+ }
361
+
362
+ /** Plan type from the manifest ('free', 'developer', …) or null (keyless/offline). */
363
+ getPlanType(): string | null {
364
+ return this.manifest?.plan?.plan ?? null;
365
+ }
366
+
367
+ /** Requested tracking features with WebView-parity defaults applied. */
368
+ getFeatures(): ResolvedFeatures {
369
+ return { ...this.features };
370
+ }
371
+
372
+ /** 'remote-cache' | 'remote-download' | null (keypoints-only). */
373
+ getEngineSource(): EngineLoadResult['source'] | null {
374
+ return this.engineSource;
375
+ }
376
+
377
+ /**
378
+ * GPU-acceleration verdict from the warm-up health check ('unknown' until
379
+ * preload/warmup completes). See docs/ANDROID_GL_ACCELERATION.md.
380
+ */
381
+ getAcceleration(): AccelerationState {
382
+ return this.getAccelerationDiagnostics()?.state ?? 'unknown';
383
+ }
384
+
385
+ /** Full diagnostics (backend, timings, GL renderer, flags, downgrade trail). */
386
+ getAccelerationDiagnostics(): AccelerationDiagnostics | null {
387
+ return this.backend.getAcceleration?.() ?? null;
388
+ }
389
+
390
+ /**
391
+ * Adaptive camera-quality state (profile, capability score, mean FPS).
392
+ * See docs/ADAPTIVE_QUALITY.md.
393
+ */
394
+ getQualityState(): QualityState {
395
+ return this.quality.getState();
396
+ }
397
+
398
+ /** Resolve (and cache) the initial quality profile before mounting the WebView. */
399
+ resolveQualityProfile(): Promise<QualityProfile> {
400
+ return this.quality.resolveInitialProfile();
401
+ }
402
+
403
+ /** Mark the active profile as PROBING (crash-loop guard) before page boot. */
404
+ beginQualitySession(): Promise<void> {
405
+ return this.quality.beginSession();
406
+ }
407
+
408
+ /**
409
+ * Wire the live WebView injector so auto-downgrades can restart getUserMedia.
410
+ * Called by {@link WebViewPoseView}.
411
+ */
412
+ setQualityApplyHandler(fn: ((profile: QualityProfile) => void) | undefined): void {
413
+ this.quality.setApplyProfile(fn);
414
+ }
415
+
416
+ addEventListener(listener: PoseTrackerEventListener): () => void {
417
+ this.listeners.add(listener);
418
+ return () => this.listeners.delete(listener);
419
+ }
420
+
421
+ /**
422
+ * Classic PoseTracker WebView parity: receive `sendDataToNative`-shaped JSON
423
+ * (`{ type: 'keypoints', data: [...] }`, `{ type: 'initialization', message, ready }`, …).
424
+ * Prefer typed {@link addEventListener} for new apps; use this to migrate
425
+ * existing `onMessage` parsers with minimal changes.
426
+ */
427
+ addMessageListener(listener: ClassicMessageListener): () => void {
428
+ this.messageListeners.add(listener);
429
+ return () => this.messageListeners.delete(listener);
430
+ }
431
+
432
+ /** Fired on any status/mode/manifest change. */
433
+ onStateChange(listener: () => void): () => void {
434
+ this.stateListeners.add(listener);
435
+ return () => this.stateListeners.delete(listener);
436
+ }
437
+
438
+ // -------------------------------------------------------------------------
439
+ // Preload / warmup
440
+ // -------------------------------------------------------------------------
441
+
442
+ /**
443
+ * Idempotent warm-up: bundled pose runtime, optional engine handshake,
444
+ * then backend cold-start. Does **not** run when the provider mounts
445
+ * (unless `autoPreload`). For WebView, mount a `WebViewPoseView` so the
446
+ * page can load MoveNet — see docs/PRELOAD.md.
447
+ *
448
+ * Cold-start modes (`options.coldStart`):
449
+ * - `basic` (**default**): model + WebGL zeros only — **no getUserMedia**,
450
+ * so lobby / home preload never prompts for camera permission.
451
+ * - `full`: also open the camera (legacy). Pair with
452
+ * `<WebViewPoseView coldStart="full" />` or call after basic ready to
453
+ * upgrade via `__PT_OPEN_CAMERA`.
454
+ *
455
+ * Always reaches `ready` unless the local model path fails; a failed
456
+ * handshake degrades to keypoints-only mode.
457
+ */
458
+ preload(options?: PreloadOptions): Promise<void> {
459
+ const mode: ColdStartMode = options?.coldStart === 'full' ? 'full' : 'basic';
460
+ this.coldStartMode = mode;
461
+
462
+ // Already model-ready: only upgrade to full if requested.
463
+ if (this.status === 'ready' && this.preloadPromise) {
464
+ return this.preloadPromise.then(() => this.ensureColdStartMode(mode));
465
+ }
466
+
467
+ if (!this.preloadPromise) {
468
+ this.preloadPromise = this.doPreload()
469
+ .then(() => this.ensureColdStartMode(mode))
470
+ .catch((err) => {
471
+ // Allow a retry after a failed preload (model failure).
472
+ this.preloadPromise = null;
473
+ throw err;
474
+ });
475
+ } else if (mode === 'full') {
476
+ // Upgrade an in-flight basic preload once it finishes.
477
+ return this.preloadPromise.then(() => this.ensureColdStartMode('full'));
478
+ }
479
+ return this.preloadPromise;
480
+ }
481
+
482
+ /** Alias of preload(), matching the Sency-style naming. */
483
+ warmup(options?: PreloadOptions): Promise<void> {
484
+ return this.preload(options);
485
+ }
486
+
487
+ /** Current preferred cold-start mode (for hosts / WebView wiring). */
488
+ getColdStartMode(): ColdStartMode {
489
+ return this.coldStartMode;
490
+ }
491
+
492
+ private async doPreload(): Promise<void> {
493
+ // 1. Online runtime descriptor (thin page runtime + CDN/model URLs).
494
+ // Network fetch of TF.js/model happens inside the WebView at warm-up.
495
+ await this.getRuntimeParts();
496
+
497
+ // 2. Handshake — optional for keypoints-only; required path for engine.
498
+ // Never fatal by itself (offline API / no key → keypoints-only).
499
+ this.setStatus('configuring');
500
+ await this.resolveManifestOnce();
501
+
502
+ // 3. Engine (API key path, never fatal — degrades to keypoints-only) ---
503
+ this.setStatus('downloading');
504
+ await this.tryConfigure({ silentStatus: true });
505
+
506
+ // 4. Flush queued anonymous usage events (best-effort) -----------------
507
+ void this.tracker.flushQueue();
508
+
509
+ // 5. WebView backend warm-up (fatal on failure) ------------------------
510
+ // Waits for page `ready` after CDN TF.js + remote model load.
511
+ try {
512
+ this.setStatus('warming');
513
+ await this.initAndWarmupBackend();
514
+ } catch (err) {
515
+ this.reportError({
516
+ type: 'error',
517
+ code: 'model_load_failed',
518
+ message: err instanceof Error ? err.message : String(err),
519
+ });
520
+ this.setStatus('error');
521
+ throw err;
522
+ }
523
+
524
+ this.setStatus('ready');
525
+ this.emit({
526
+ type: 'initialization',
527
+ step: 'ready',
528
+ message: 'running',
529
+ ready: true,
530
+ mode: this.mode,
531
+ acceleration: this.getAcceleration(),
532
+ });
533
+ }
534
+
535
+ /**
536
+ * After model ready, optionally open the camera for `coldStart: 'full'`.
537
+ * No-op for basic, or when the WebView already booted with coldStart=full.
538
+ */
539
+ private async ensureColdStartMode(mode: ColdStartMode): Promise<void> {
540
+ if (mode !== 'full') return;
541
+ const backend = this.backend;
542
+ if (!(backend instanceof WebViewPoseBackend)) return;
543
+ if (backend.isCameraOpened()) return;
544
+ await backend.openCamera();
545
+ }
546
+
547
+ // -------------------------------------------------------------------------
548
+ // Pose-runtime (online — CDN TF.js + remote model URL)
549
+ // -------------------------------------------------------------------------
550
+
551
+ /**
552
+ * Online runtime descriptor used by {@link WebViewPoseView} / {@link buildPoseHtml}.
553
+ * Resolves model URL + CDN script list synchronously; the WebView performs
554
+ * the actual network fetches at boot.
555
+ */
556
+ getRuntimeParts(): Promise<OnlineRuntimeParts> {
557
+ if (!this.runtimePromise) {
558
+ this.runtimePromise = Promise.resolve(this.loadOnlineRuntime());
559
+ }
560
+ return this.runtimePromise;
561
+ }
562
+
563
+ /**
564
+ * Load a custom skeleton overlay by Strapi `api_uuid`
565
+ * (WebView `?skeleton=<uuid>`). Pass the result to
566
+ * `<WebViewPoseView skeletonDef={…} />` or let the view fetch via
567
+ * `skeletonUuid`.
568
+ */
569
+ fetchSkeleton(uuid: string): Promise<SkeletonDefinition> {
570
+ return fetchSkeletonDefinition(uuid, { baseUrl: this.options.baseUrl });
571
+ }
572
+
573
+ private loadOnlineRuntime(): OnlineRuntimeParts {
574
+ const parts = getOnlineRuntimeParts({
575
+ model: this.options.model,
576
+ modelUrl: this.options.modelUrl,
577
+ tfjsCdnBase: this.options.tfjsCdnBase,
578
+ tfjsVersion: this.options.tfjsVersion,
579
+ });
580
+ this.options.onDiagnostic?.(
581
+ `[posetracker-light] pose-runtime online version=${parts.version} ` +
582
+ `modelId=${parts.modelId} modelUrl=${parts.modelUrl}`,
583
+ );
584
+ return parts;
585
+ }
586
+
587
+ /**
588
+ * One handshake per configuration cycle — shared by the pose-runtime warm
589
+ * and the engine load, with or without API key. Never throws; resolves to
590
+ * null when unreachable AND no sealed session cache can be replayed.
591
+ */
592
+ private resolveManifestOnce(): Promise<SdkManifest | null> {
593
+ if (!this.handshakePromise) {
594
+ this.handshakePromise = this.doResolveManifestOnce().then((manifest) => {
595
+ if (!manifest) {
596
+ // Allow later retries (network may come back).
597
+ this.handshakePromise = null;
598
+ }
599
+ return manifest;
600
+ });
601
+ }
602
+ return this.handshakePromise;
603
+ }
604
+
605
+ private async doResolveManifestOnce(): Promise<SdkManifest | null> {
606
+ const localVersions = {
607
+ poseRuntime: getOnlineRuntimeVersion(),
608
+ engine: (await this.files?.read(ENGINE_VERSION_KEY).catch(() => null)) ?? null,
609
+ };
610
+
611
+ if (this.apiToken) {
612
+ const manifest = await this.resolveManifest(this.apiToken, localVersions);
613
+ if (manifest) {
614
+ this.manifest = manifest;
615
+ if (manifest.revoked === true) {
616
+ await this.handleRevocation('Access revoked by the backend.');
617
+ }
618
+ }
619
+ return manifest;
620
+ }
621
+
622
+ // Keyless handshake: public manifest (pose-runtime descriptor only).
623
+ try {
624
+ const manifest = await handshake(null, { ...this.options, localVersions });
625
+ this.manifest = manifest;
626
+ return manifest;
627
+ } catch {
628
+ // Offline keyless: the runtime cache decides what is possible.
629
+ return null;
630
+ }
631
+ }
632
+
633
+ /**
634
+ * Revocation signal: purge the sealed engine artifacts and downgrade to
635
+ * keypoints-only. The public pose-runtime cache is NOT purged (public
636
+ * payload, keypoints-only keeps working).
637
+ */
638
+ private async handleRevocation(message: string): Promise<void> {
639
+ const engineVersion = await this.files?.read(ENGINE_VERSION_KEY).catch(() => null);
640
+ if (engineVersion) {
641
+ await this.files?.remove(`engine-${engineVersion}.sealed`).catch(() => {});
642
+ await this.files?.remove(ENGINE_VERSION_KEY).catch(() => {});
643
+ }
644
+ await this.files?.remove(MANIFEST_CACHE_KEY).catch(() => {});
645
+ this.engine = null;
646
+ this.engineSource = null;
647
+ this.stopExercise();
648
+ this.setMode('keypoints-only');
649
+ this.reportError({ type: 'error', code: 'invalid_token', message });
650
+ }
651
+
652
+ /**
653
+ * Usage metering — fired when the WebView reports the camera started with
654
+ * the model ready. With an API key the call MUST succeed (quota check +
655
+ * counter increment + usage row): offline metered sessions are refused.
656
+ * Without a key the event is fire-and-forget with a local retry queue.
657
+ */
658
+ private async handleCameraStart(info: {
659
+ backend: string;
660
+ profileId: string | null;
661
+ }): Promise<void> {
662
+ this.lastCameraStartInfo = info;
663
+ const params = {
664
+ backend: info.backend,
665
+ profileId: info.profileId,
666
+ poseModelProfile: this.options.poseModelProfile ?? 'AdaptiveChoice',
667
+ qualityChoice: this.options.qualityChoice ?? 'AdaptiveChoice',
668
+ mode: this.mode,
669
+ exercise: this.currentExerciseId,
670
+ runtimeVersion: getOnlineRuntimeVersion(),
671
+ model: this.options.model ?? 'movenet',
672
+ modelUrl: this.options.modelUrl ?? null,
673
+ };
674
+
675
+ if (!this.apiToken) {
676
+ void this.tracker.trackAnonymous({ event: 'camera_start', params });
677
+ return;
678
+ }
679
+
680
+ this.meteredSessionState = 'pending';
681
+ try {
682
+ await this.tracker.trackMetered({ event: 'camera_start', apiToken: this.apiToken, params });
683
+ this.meteredSessionState = 'validated';
684
+ this.options.onDiagnostic?.('[posetracker] camera_start tracked (metered session validated)');
685
+ } catch (err) {
686
+ this.meteredSessionState = 'refused';
687
+ if (err instanceof TrackError && err.code === 'network') {
688
+ this.reportError({
689
+ type: 'error',
690
+ code: 'offline_metered',
691
+ message:
692
+ 'API-key features are not available offline: PoseTracker cannot ' +
693
+ 'count their usage. Keypoints-only keeps running from the cache; ' +
694
+ 'reconnect to start a metered session.',
695
+ });
696
+ } else if (err instanceof TrackError && err.code === 'invalid_token') {
697
+ await this.handleRevocation('API key invalid or revoked — engine cache purged.');
698
+ } else if (err instanceof TrackError && err.code === 'quota_exceeded') {
699
+ this.reportError({ type: 'error', code: 'quota_exceeded', message: err.message });
700
+ } else {
701
+ this.reportError({
702
+ type: 'error',
703
+ code: 'internal',
704
+ message: `Usage tracking failed: ${err instanceof Error ? err.message : String(err)}`,
705
+ });
706
+ }
707
+ }
708
+ }
709
+
710
+ /** Backend init + warm-up. An init failure surfaces as status 'error'. */
711
+ private async initAndWarmupBackend(): Promise<void> {
712
+ try {
713
+ await this.backend.init({ model: this.resolveModel() });
714
+ await this.backend.warmup();
715
+ } catch (err) {
716
+ logAccelerationReport(this.getAccelerationDiagnostics(), {
717
+ phase: 'init-failed',
718
+ error: err instanceof Error ? err.message : String(err),
719
+ });
720
+ throw err;
721
+ }
722
+ // Always dump the full report to Metro after warm-up.
723
+ logAccelerationReport(this.getAccelerationDiagnostics(), {
724
+ phase: 'warmup-complete',
725
+ backend: this.backend.name,
726
+ mode: this.mode,
727
+ });
728
+ }
729
+
730
+ /**
731
+ * Runtime (re)configuration — the keypoints-only → full-engine upgrade
732
+ * path. Can be called before or after `ready`; when the camera pipeline
733
+ * is already running it keeps running, business events simply start once
734
+ * the engine is loaded. Resolves to true when full-engine mode is active.
735
+ */
736
+ configure(apiToken?: string): Promise<boolean> {
737
+ if (apiToken !== undefined && apiToken !== this.apiToken) {
738
+ this.apiToken = apiToken;
739
+ this.configurePromise = null;
740
+ // New credentials: re-handshake (the previous one may be keyless).
741
+ this.handshakePromise = null;
742
+ // The plan may change with the token: re-arm the feature gating errors.
743
+ this.featureGateReported.freeBlock = false;
744
+ this.featureGateReported.missingToken = false;
745
+ }
746
+ // Explicit configure() (e.g. Test key) must be allowed to retry even when
747
+ // a previous attempt failed or the crash-guard blocked the bundle.
748
+ if (this.mode !== 'full-engine') {
749
+ this.configurePromise = null;
750
+ }
751
+ if (!this.configurePromise) {
752
+ this.configurePromise = this.tryConfigure({
753
+ silentStatus: this.status === 'ready',
754
+ forceEngineRetry: true,
755
+ })
756
+ .then((ok) => {
757
+ if (!ok) {
758
+ // Allow retries (network may come back).
759
+ this.configurePromise = null;
760
+ }
761
+ // Network is (possibly) back: a metered session refused at camera
762
+ // start (offline) can now be validated without restarting the camera.
763
+ this.retryMeteredSessionIfRefused();
764
+ return ok;
765
+ })
766
+ .catch(() => {
767
+ this.configurePromise = null;
768
+ return false;
769
+ });
770
+ }
771
+ return this.configurePromise;
772
+ }
773
+
774
+ /**
775
+ * Re-run the `camera_start` metered gate after a configure() retry. A
776
+ * session refused offline becomes usable as soon as the network is back —
777
+ * WebView parity: the front revalidates on reload, the SDK on configure().
778
+ */
779
+ private retryMeteredSessionIfRefused(): void {
780
+ if (this.meteredSessionState === 'refused' && this.apiToken && this.lastCameraStartInfo) {
781
+ this.options.onDiagnostic?.(
782
+ '[posetracker] retrying metered camera_start after configure() (was refused)',
783
+ );
784
+ void this.handleCameraStart(this.lastCameraStartInfo);
785
+ }
786
+ }
787
+
788
+ /**
789
+ * Handshake + engine load. Never throws; returns whether full-engine mode
790
+ * was reached. `silentStatus` avoids status regressions (e.g. hot upgrade
791
+ * while `ready` and the camera is streaming).
792
+ */
793
+ private async tryConfigure({
794
+ silentStatus,
795
+ forceEngineRetry = false,
796
+ }: {
797
+ silentStatus: boolean;
798
+ forceEngineRetry?: boolean;
799
+ }): Promise<boolean> {
800
+ if (this.mode === 'full-engine') {
801
+ return true;
802
+ }
803
+ if (!this.apiToken) {
804
+ // No API key: keypoints-only, by design. Not an error — unless the
805
+ // host requested key-gated features (WebView parity: token required).
806
+ this.validateRequestedFeatures();
807
+ this.setMode('keypoints-only');
808
+ return false;
809
+ }
810
+
811
+ const manifest = await this.resolveManifestOnce();
812
+ if (!manifest || manifest.revoked === true) {
813
+ this.setMode('keypoints-only');
814
+ return false;
815
+ }
816
+ this.manifest = manifest;
817
+ // Plan is now known: replicate the TrackingAppV3 load-time gating.
818
+ this.validateRequestedFeatures();
819
+
820
+ if (!silentStatus) {
821
+ this.setStatus('downloading');
822
+ }
823
+ const secret = deriveCacheSecret(this.apiToken);
824
+ if (forceEngineRetry && manifest.engine) {
825
+ await this.engineLoader.clearGuard(manifest.engine);
826
+ }
827
+ const result = await this.engineLoader.load(manifest.engine ?? null, secret, {
828
+ forceRetry: forceEngineRetry,
829
+ });
830
+ if (!result) {
831
+ const detail = this.engineLoader.lastError;
832
+ this.reportError({
833
+ type: 'error',
834
+ code: 'engine_load_failed',
835
+ message: detail
836
+ ? `Engine bundle unavailable — ${detail}`
837
+ : 'Engine bundle unavailable (offline without cache, or integrity/evaluation failure) — running keypoints-only.',
838
+ });
839
+ this.setMode('keypoints-only');
840
+ return false;
841
+ }
842
+
843
+ this.engine = result.engine;
844
+ this.engineSource = result.source;
845
+ // Remember the sealed engine version so a revocation can purge it.
846
+ if (manifest.engine?.version) {
847
+ await this.files?.write(ENGINE_VERSION_KEY, manifest.engine.version).catch(() => {});
848
+ }
849
+ this.setMode('full-engine');
850
+ return true;
851
+ }
852
+
853
+ /**
854
+ * Handshake with the session-cache fallback:
855
+ * - live call → cache the manifest, sealed with the token-derived secret;
856
+ * - network/server failure → replay the sealed cached manifest (only
857
+ * readable with the same token: the "already configured once" proof);
858
+ * - invalid token → purge the cache (a revoked key must not keep the
859
+ * engine alive) and report a non-fatal error;
860
+ * - quota exceeded → non-fatal error, cache kept (transient condition)
861
+ * but not replayed this session.
862
+ */
863
+ private async resolveManifest(
864
+ apiToken: string,
865
+ localVersions?: ConfigureOptions['localVersions'],
866
+ ): Promise<SdkManifest | null> {
867
+ const secret = deriveCacheSecret(apiToken);
868
+ try {
869
+ const manifest = await handshake(apiToken, { ...this.options, localVersions });
870
+ await this.files?.write(MANIFEST_CACHE_KEY, sealString(JSON.stringify(manifest), secret)).catch(() => {});
871
+ return manifest;
872
+ } catch (err) {
873
+ if (err instanceof ConfigureError && err.code === 'invalid_token') {
874
+ // Revoked/invalid key: purge the sealed session AND engine caches —
875
+ // a revoked key must not keep the business logic alive.
876
+ await this.handleRevocation(err.message);
877
+ return null;
878
+ }
879
+ if (err instanceof ConfigureError && err.code === 'quota_exceeded') {
880
+ this.reportError({ type: 'error', code: 'quota_exceeded', message: err.message });
881
+ return null;
882
+ }
883
+
884
+ // Network/server failure: replay the encrypted session cache.
885
+ const sealed = await this.files?.read(MANIFEST_CACHE_KEY);
886
+ if (sealed) {
887
+ const plain = openString(sealed, secret);
888
+ if (plain) {
889
+ try {
890
+ return JSON.parse(plain) as SdkManifest;
891
+ } catch {
892
+ await this.files?.remove(MANIFEST_CACHE_KEY).catch(() => {});
893
+ }
894
+ }
895
+ }
896
+ this.reportError({
897
+ type: 'error',
898
+ code: 'network',
899
+ message: 'Handshake unreachable and no cached session — running keypoints-only.',
900
+ });
901
+ return null;
902
+ }
903
+ }
904
+
905
+ /**
906
+ * WebView parity — the load-time gating of `TrackingAppV3`:
907
+ * - `blazepose` / `poseEngine` / other WebView-only keys → clear error
908
+ * (this SDK ships MoveNet Lightning only);
909
+ * - developer features requested WITHOUT an API key → the front's exact
910
+ * "Invalid params… token=YOUR API_KEY…" message;
911
+ * - plan `free` + angles/recommendations/progression → the front's exact
912
+ * "You cannot use developer features." message (keypoints alone stays
913
+ * allowed here: pose-only mode is free; the +exercise case is enforced
914
+ * in {@link startExercise}).
915
+ * All non-fatal: keypoints-only pose estimation keeps running. One-shot
916
+ * per condition so `configure()` retries don't spam the host.
917
+ */
918
+ private validateRequestedFeatures(): void {
919
+ if (this.unsupportedFeatureKeys.length > 0 && !this.featureGateReported.unsupported) {
920
+ this.featureGateReported.unsupported = true;
921
+ this.reportError({
922
+ type: 'error',
923
+ code: 'feature_not_supported',
924
+ message: featureNotSupportedMessage(this.unsupportedFeatureKeys[0]!),
925
+ });
926
+ }
927
+
928
+ const f = this.features;
929
+ const requestsDevFeatures = f.angles || f.recommendations || f.progression || f.keypoints;
930
+ if (!requestsDevFeatures) {
931
+ return;
932
+ }
933
+
934
+ if (!this.apiToken) {
935
+ if (!this.featureGateReported.missingToken) {
936
+ this.featureGateReported.missingToken = true;
937
+ this.reportError({ type: 'error', code: 'invalid_token', message: INVALID_TOKEN_MESSAGE });
938
+ }
939
+ return;
940
+ }
941
+
942
+ if (this.getPlanType() === 'free') {
943
+ const blocked = freeBlockedFeatures(f, { withExercise: false });
944
+ if (blocked.length > 0 && !this.featureGateReported.freeBlock) {
945
+ this.featureGateReported.freeBlock = true;
946
+ this.options.onDiagnostic?.(
947
+ `[posetracker] free plan blocked features: ${blocked.join(', ')}`,
948
+ );
949
+ this.reportError({
950
+ type: 'error',
951
+ code: 'free_plan_feature_blocked',
952
+ message: FREE_PLAN_FEATURES_MESSAGE,
953
+ });
954
+ }
955
+ }
956
+ }
957
+
958
+ private resolveModel(): ModelDescriptor {
959
+ if (this.manifest) {
960
+ const model = this.manifest.models[this.manifest.resolvedProfile];
961
+ if (model) {
962
+ return model;
963
+ }
964
+ }
965
+ return DEFAULT_MOVENET;
966
+ }
967
+
968
+ // -------------------------------------------------------------------------
969
+ // Exercise sessions (full-engine mode only)
970
+ // -------------------------------------------------------------------------
971
+
972
+ getAvailableExercises(): ExerciseConfig[] {
973
+ return this.mode === 'full-engine' ? this.manifest?.exercises ?? [] : [];
974
+ }
975
+
976
+ /**
977
+ * Custom exercises shipped inside the engine bundle (jump_analysis,
978
+ * air_time_jump — WebView `customHandlers.js` parity). Empty in
979
+ * keypoints-only mode or with an engine bundle older than 1.2.0.
980
+ */
981
+ getAvailableCustomExercises(): CustomExerciseDescriptor[] {
982
+ if (this.mode !== 'full-engine' || !this.engine?.listCustomExercises) {
983
+ return [];
984
+ }
985
+ return this.engine.listCustomExercises();
986
+ }
987
+
988
+ startExercise(exerciseId: string, options: StartExerciseOptions = {}): void {
989
+ if (this.status !== 'ready') {
990
+ throw new Error(`Cannot start exercise while status is '${this.status}' — call preload() first.`);
991
+ }
992
+ if (this.mode !== 'full-engine' || !this.engine) {
993
+ throw new Error(
994
+ "Exercises require full-engine mode (validated API key). The SDK is running keypoints-only — call configure(apiToken) first.",
995
+ );
996
+ }
997
+ if (this.meteredSessionState === 'refused') {
998
+ throw new Error(
999
+ 'API-key features are not available offline: PoseTracker could not ' +
1000
+ 'track this session (camera_start failed — no network or quota ' +
1001
+ 'exceeded). Reconnect and retry.',
1002
+ );
1003
+ }
1004
+ if (this.unsupportedFeatureKeys.length > 0) {
1005
+ const message = featureNotSupportedMessage(this.unsupportedFeatureKeys[0]!);
1006
+ this.reportError({ type: 'error', code: 'feature_not_supported', message });
1007
+ throw new Error(message);
1008
+ }
1009
+ // WebView parity: `free` cannot run developer features — and keypoints
1010
+ // combined with an exercise counts as one (pose-only keypoints stay free).
1011
+ if (this.getPlanType() === 'free') {
1012
+ const blocked = freeBlockedFeatures(this.features, { withExercise: true });
1013
+ if (blocked.length > 0) {
1014
+ this.reportError({
1015
+ type: 'error',
1016
+ code: 'free_plan_feature_blocked',
1017
+ message: FREE_PLAN_FEATURES_MESSAGE,
1018
+ });
1019
+ throw new Error(FREE_PLAN_FEATURES_MESSAGE);
1020
+ }
1021
+ }
1022
+ const available = this.getAvailableExercises();
1023
+ const exercise = findExerciseByIdOrAlias(exerciseId, available);
1024
+ if (!exercise) {
1025
+ // Custom exercises live in the engine bundle, not the movement manifest
1026
+ // (WebView customHandlers.js parity: jump_analysis, air_time_jump).
1027
+ const customs = this.getAvailableCustomExercises();
1028
+ const custom =
1029
+ customs.find((e) => e.id === exerciseId) ??
1030
+ findExerciseByIdOrAlias(exerciseId, customs);
1031
+ if (custom) {
1032
+ this.startCustomExercise(custom, options);
1033
+ return;
1034
+ }
1035
+ // Front literal (CameraFeedV3): error `invalid_exercise`.
1036
+ const message = `Exercise '${exerciseId}' is not available in V3 engine`;
1037
+ this.reportError({ type: 'error', code: 'invalid_exercise', message });
1038
+ throw new Error(message);
1039
+ }
1040
+ this.stopExercise();
1041
+ this.currentExerciseId = exerciseId;
1042
+ this.keypointsSuppressionLogged = false;
1043
+ this.session = this.engine.createSession(
1044
+ {
1045
+ exercise,
1046
+ locale: this.options.locale ?? 'en',
1047
+ difficulty: options.difficulty,
1048
+ minGrade: this.features.minGrade ?? undefined,
1049
+ features: {
1050
+ angles: this.features.angles,
1051
+ recommendations: this.features.recommendations,
1052
+ progression: this.features.progression,
1053
+ },
1054
+ },
1055
+ (event) => this.emitFromSession(event),
1056
+ );
1057
+ }
1058
+
1059
+ /** Custom engine session (jump_analysis / air_time_jump). */
1060
+ private startCustomExercise(custom: CustomExerciseDescriptor, options: StartExerciseOptions): void {
1061
+ if (!this.engine?.createCustomSession) {
1062
+ throw new Error(
1063
+ `The cached engine bundle is too old for custom exercise '${custom.id}' — reconnect so the SDK can update it.`,
1064
+ );
1065
+ }
1066
+ // Front literal (CameraFeedV3): error `jump_analysis_missing_height`.
1067
+ if (custom.id === 'jump_analysis' && (!options.userHeightCm || options.userHeightCm <= 0)) {
1068
+ const message = 'User height (userHeightCm) must be provided for jump_analysis exercise';
1069
+ this.reportError({ type: 'error', code: 'jump_analysis_missing_height', message });
1070
+ throw new Error(message);
1071
+ }
1072
+ this.stopExercise();
1073
+ this.currentExerciseId = custom.id;
1074
+ this.keypointsSuppressionLogged = false;
1075
+ this.session = this.engine.createCustomSession(
1076
+ {
1077
+ exerciseId: custom.id,
1078
+ locale: this.options.locale ?? 'en',
1079
+ userHeightCm: options.userHeightCm,
1080
+ devicePitchDeg: options.devicePitchDeg,
1081
+ },
1082
+ (event) => this.emitFromSession(event),
1083
+ );
1084
+ }
1085
+
1086
+ /**
1087
+ * Engine → host emission gate (WebView parity): `angles`,
1088
+ * `recommendations` and `progression` only stream when their flag is on.
1089
+ * Double safety on top of the engine-side flags — a cached engine bundle
1090
+ * predating features support still gets filtered here.
1091
+ */
1092
+ private emitFromSession(event: PoseTrackerEvent): void {
1093
+ if (event.type === 'angles' && !this.features.angles) return;
1094
+ if (event.type === 'recommendations' && !this.features.recommendations) return;
1095
+ if (event.type === 'progression' && !this.features.progression) return;
1096
+ this.emit(event);
1097
+ }
1098
+
1099
+ /** Ends the active session (emits a final `exercise_summary`). */
1100
+ stopExercise(): void {
1101
+ // end() runs remotely-delivered engine code — a throw must not leave the
1102
+ // client stuck with a dead session (startExercise calls stopExercise).
1103
+ try {
1104
+ this.session?.end();
1105
+ } catch (err) {
1106
+ this.options.onDiagnostic?.(
1107
+ '[posetracker] engine session end() threw: ' +
1108
+ (err instanceof Error ? err.message : String(err)),
1109
+ );
1110
+ }
1111
+ this.session = null;
1112
+ this.currentExerciseId = null;
1113
+ this.sessionErrorStreak = 0;
1114
+ }
1115
+
1116
+ getCurrentExerciseId(): string | null {
1117
+ return this.currentExerciseId;
1118
+ }
1119
+
1120
+ // -------------------------------------------------------------------------
1121
+ // Frame pipeline (both modes)
1122
+ // -------------------------------------------------------------------------
1123
+
1124
+ /** Active inference backend (auto-selected, forced, or injected). */
1125
+ getBackend(): PoseBackend {
1126
+ return this.backend;
1127
+ }
1128
+
1129
+ /** Raw pose estimation, no engine involvement. */
1130
+ estimatePose(frame: PoseInputFrame): Promise<Pose | null> {
1131
+ return this.backend.estimatePose(frame);
1132
+ }
1133
+
1134
+ /**
1135
+ * Full pipeline for one camera frame: pose estimation + `keypoints` event
1136
+ * (both modes), then engine processing when a session is active
1137
+ * (full-engine mode). Mode upgrades take effect transparently here.
1138
+ */
1139
+ async processFrame(frame: PoseInputFrame): Promise<Pose | null> {
1140
+ const pose = await this.estimatePose(frame);
1141
+ if (!pose) {
1142
+ return null;
1143
+ }
1144
+ this.ingestPose(pose);
1145
+ return pose;
1146
+ }
1147
+
1148
+ /**
1149
+ * Feed an externally-estimated pose into the pipeline: `keypoints` event
1150
+ * (both modes) + engine processing when a session is active. This is how
1151
+ * the vision-camera path (`PoseCameraView`) delivers poses computed
1152
+ * synchronously inside the frame-processor worklet.
1153
+ */
1154
+ ingestPose(pose: Pose): void {
1155
+ // WebView parity: DURING an exercise session, raw keypoints only stream
1156
+ // when the `keypoints` feature is on (paid plans — free + keypoints +
1157
+ // exercise is rejected in startExercise). Pose-only mode (no session)
1158
+ // always streams: that is the SDK's free offline base.
1159
+ if (!this.session || this.features.keypoints) {
1160
+ this.emit({
1161
+ type: 'keypoints',
1162
+ keypoints: pose.keypoints,
1163
+ score: pose.score,
1164
+ timestampMs: pose.timestampMs,
1165
+ });
1166
+ } else if (!this.keypointsSuppressionLogged) {
1167
+ this.keypointsSuppressionLogged = true;
1168
+ this.options.onDiagnostic?.(
1169
+ '[posetracker] keypoints events paused during the exercise session ' +
1170
+ '(features.keypoints=false, WebView parity) — they resume on stopExercise().',
1171
+ );
1172
+ }
1173
+ if (this.session) {
1174
+ // The engine is remotely-delivered code: one bad frame must not crash
1175
+ // the ingest path, and a session that fails on EVERY frame must not
1176
+ // keep throwing at camera rate (battery). After a streak of failures
1177
+ // the session is terminated with an error event.
1178
+ try {
1179
+ this.session.processPose(pose);
1180
+ this.sessionErrorStreak = 0;
1181
+ } catch (err) {
1182
+ this.sessionErrorStreak += 1;
1183
+ const message = err instanceof Error ? err.message : String(err);
1184
+ if (this.sessionErrorStreak === 1) {
1185
+ this.options.onDiagnostic?.(
1186
+ `[posetracker] engine session processPose threw: ${message}`,
1187
+ );
1188
+ }
1189
+ if (this.sessionErrorStreak >= SESSION_ERROR_STREAK_LIMIT) {
1190
+ const exerciseId = this.currentExerciseId;
1191
+ this.session = null; // skip end(): the session is already broken
1192
+ this.currentExerciseId = null;
1193
+ this.sessionErrorStreak = 0;
1194
+ this.reportError({
1195
+ type: 'error',
1196
+ code: 'internal',
1197
+ message:
1198
+ `Exercise session '${exerciseId ?? '?'}' failed repeatedly ` +
1199
+ `(${message}) — session stopped, keypoints keep streaming.`,
1200
+ });
1201
+ }
1202
+ }
1203
+ }
1204
+ }
1205
+
1206
+ async dispose(): Promise<void> {
1207
+ this.stopExercise();
1208
+ this.quality.setApplyProfile(undefined);
1209
+ await this.backend.dispose();
1210
+ this.listeners.clear();
1211
+ this.messageListeners.clear();
1212
+ this.stateListeners.clear();
1213
+ this.preloadPromise = null;
1214
+ this.configurePromise = null;
1215
+ this.handshakePromise = null;
1216
+ this.runtimePromise = null;
1217
+ this.meteredSessionState = 'idle';
1218
+ this.lastCameraStartInfo = null;
1219
+ this.featureGateReported = { unsupported: false, freeBlock: false, missingToken: false };
1220
+ this.keypointsSuppressionLogged = false;
1221
+ this.engine = null;
1222
+ this.engineSource = null;
1223
+ this.manifest = null;
1224
+ this.mode = 'keypoints-only';
1225
+ this.setStatus('idle');
1226
+ }
1227
+
1228
+ // -------------------------------------------------------------------------
1229
+
1230
+ private setStatus(status: PoseTrackerStatus): void {
1231
+ this.status = status;
1232
+ if (status === 'configuring' || status === 'downloading' || status === 'warming') {
1233
+ this.emit({
1234
+ type: 'initialization',
1235
+ step: status,
1236
+ message:
1237
+ status === 'configuring'
1238
+ // Front / GitBook literal (sic): hosts may string-match this.
1239
+ ? 'checking you plan and access'
1240
+ : status === 'downloading'
1241
+ ? 'downloading engine'
1242
+ : 'loading pose model',
1243
+ ready: false,
1244
+ });
1245
+ }
1246
+ this.notifyState();
1247
+ }
1248
+
1249
+ private setMode(mode: PoseTrackerMode): void {
1250
+ if (this.mode !== mode) {
1251
+ this.mode = mode;
1252
+ this.notifyState();
1253
+ }
1254
+ }
1255
+
1256
+ private reportError(error: ErrorEvent): void {
1257
+ this.lastError = error;
1258
+ this.emit(error);
1259
+ this.notifyState();
1260
+ }
1261
+
1262
+ private notifyState(): void {
1263
+ this.stateListeners.forEach((l) => l());
1264
+ }
1265
+
1266
+ private emit(event: PoseTrackerEvent): void {
1267
+ // A throwing host listener must never break other listeners or the frame
1268
+ // pipeline (emit is called from the per-frame ingest path).
1269
+ this.listeners.forEach((l) => {
1270
+ try {
1271
+ l(event);
1272
+ } catch (err) {
1273
+ this.options.onDiagnostic?.(
1274
+ `[posetracker] event listener threw on '${event.type}': ` +
1275
+ (err instanceof Error ? err.message : String(err)),
1276
+ );
1277
+ }
1278
+ });
1279
+ if (this.messageListeners.size === 0) {
1280
+ // Skip the classic-message conversion entirely (per-frame allocation).
1281
+ return;
1282
+ }
1283
+ const classic: ClassicNativeMessage = toClassicNativeMessage(event);
1284
+ this.messageListeners.forEach((l) => {
1285
+ try {
1286
+ l(classic);
1287
+ } catch (err) {
1288
+ this.options.onDiagnostic?.(
1289
+ `[posetracker] message listener threw on '${event.type}': ` +
1290
+ (err instanceof Error ? err.message : String(err)),
1291
+ );
1292
+ }
1293
+ });
1294
+ }
1295
+ }