@pose-tracker/react-native-pose-estimation-light 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. package/LICENSE +45 -0
  2. package/PoseTrackerVision.podspec +28 -0
  3. package/README.md +161 -0
  4. package/THIRD_PARTY_NOTICES.md +19 -0
  5. package/ios/PoseTrackerVision/PoseTrackerBodyPosePlugin.m +18 -0
  6. package/ios/PoseTrackerVision/PoseTrackerBodyPosePlugin.swift +108 -0
  7. package/lib/PoseTrackerProvider.d.ts +102 -0
  8. package/lib/PoseTrackerProvider.js +204 -0
  9. package/lib/api/configure.d.ts +29 -0
  10. package/lib/api/configure.js +64 -0
  11. package/lib/api/skeleton.d.ts +17 -0
  12. package/lib/api/skeleton.js +53 -0
  13. package/lib/api/track.d.ts +61 -0
  14. package/lib/api/track.js +146 -0
  15. package/lib/backends/PoseBackend.d.ts +50 -0
  16. package/lib/backends/PoseBackend.js +11 -0
  17. package/lib/backends/vision/VisionPoseBackend.d.ts +72 -0
  18. package/lib/backends/vision/VisionPoseBackend.js +177 -0
  19. package/lib/backends/vision/mapVisionJoints.d.ts +35 -0
  20. package/lib/backends/vision/mapVisionJoints.js +162 -0
  21. package/lib/backends/vision/optionalVision.d.ts +32 -0
  22. package/lib/backends/vision/optionalVision.js +76 -0
  23. package/lib/backends/webview/WebViewPoseBackend.d.ts +223 -0
  24. package/lib/backends/webview/WebViewPoseBackend.js +382 -0
  25. package/lib/backends/webview/brandAssets.d.ts +5 -0
  26. package/lib/backends/webview/brandAssets.js +8 -0
  27. package/lib/backends/webview/onlineRuntime.d.ts +45 -0
  28. package/lib/backends/webview/onlineRuntime.js +57 -0
  29. package/lib/backends/webview/poseHtml.d.ts +71 -0
  30. package/lib/backends/webview/poseHtml.js +188 -0
  31. package/lib/backends/webview/poseRuntimeSource.d.ts +3 -0
  32. package/lib/backends/webview/poseRuntimeSource.js +4 -0
  33. package/lib/cache/obfuscate.d.ts +18 -0
  34. package/lib/cache/obfuscate.js +90 -0
  35. package/lib/camera/PoseCameraView.d.ts +51 -0
  36. package/lib/camera/PoseCameraView.js +99 -0
  37. package/lib/camera/WebViewPoseView.d.ts +81 -0
  38. package/lib/camera/WebViewPoseView.js +312 -0
  39. package/lib/client.d.ts +361 -0
  40. package/lib/client.js +1046 -0
  41. package/lib/diagnostics/logReport.d.ts +23 -0
  42. package/lib/diagnostics/logReport.js +102 -0
  43. package/lib/engine/EngineLoader.d.ts +95 -0
  44. package/lib/engine/EngineLoader.js +347 -0
  45. package/lib/engine/types.d.ts +83 -0
  46. package/lib/engine/types.js +11 -0
  47. package/lib/events/classicMessage.d.ts +31 -0
  48. package/lib/events/classicMessage.js +259 -0
  49. package/lib/exercises/aliases.d.ts +18 -0
  50. package/lib/exercises/aliases.js +48 -0
  51. package/lib/index.d.ts +72 -0
  52. package/lib/index.js +170 -0
  53. package/lib/models/poseModels.d.ts +39 -0
  54. package/lib/models/poseModels.js +63 -0
  55. package/lib/quality/AdaptiveQualityController.d.ts +120 -0
  56. package/lib/quality/AdaptiveQualityController.js +423 -0
  57. package/lib/quality/RuntimeGuard.d.ts +34 -0
  58. package/lib/quality/RuntimeGuard.js +105 -0
  59. package/lib/quality/captureMode.d.ts +82 -0
  60. package/lib/quality/captureMode.js +75 -0
  61. package/lib/quality/deviceCapability.d.ts +37 -0
  62. package/lib/quality/deviceCapability.js +177 -0
  63. package/lib/quality/profiles.d.ts +125 -0
  64. package/lib/quality/profiles.js +202 -0
  65. package/lib/runtime/RuntimeCache.d.ts +75 -0
  66. package/lib/runtime/RuntimeCache.js +230 -0
  67. package/lib/sdkVersion.d.ts +2 -0
  68. package/lib/sdkVersion.js +5 -0
  69. package/lib/support/optionalModules.d.ts +46 -0
  70. package/lib/support/optionalModules.js +82 -0
  71. package/lib/types/acceleration.d.ts +86 -0
  72. package/lib/types/acceleration.js +15 -0
  73. package/lib/types/events.d.ts +345 -0
  74. package/lib/types/events.js +10 -0
  75. package/lib/types/features.d.ts +90 -0
  76. package/lib/types/features.js +122 -0
  77. package/lib/types/manifest.d.ts +197 -0
  78. package/lib/types/manifest.js +10 -0
  79. package/lib/types/pose.d.ts +36 -0
  80. package/lib/types/pose.js +41 -0
  81. package/lib/types/preload.d.ts +17 -0
  82. package/lib/types/preload.js +10 -0
  83. package/lib/types/skeleton.d.ts +29 -0
  84. package/lib/types/skeleton.js +52 -0
  85. package/package.json +92 -0
  86. package/react-native.config.js +12 -0
  87. package/src/PoseTrackerProvider.tsx +332 -0
  88. package/src/api/configure.ts +82 -0
  89. package/src/api/skeleton.ts +60 -0
  90. package/src/api/track.ts +182 -0
  91. package/src/backends/PoseBackend.ts +59 -0
  92. package/src/backends/vision/VisionPoseBackend.ts +238 -0
  93. package/src/backends/vision/mapVisionJoints.ts +185 -0
  94. package/src/backends/vision/optionalVision.ts +94 -0
  95. package/src/backends/webview/WebViewPoseBackend.ts +582 -0
  96. package/src/backends/webview/brandAssets.ts +5 -0
  97. package/src/backends/webview/onlineRuntime.ts +94 -0
  98. package/src/backends/webview/poseHtml.ts +259 -0
  99. package/src/backends/webview/poseRuntimeSource.d.ts +3 -0
  100. package/src/cache/obfuscate.ts +94 -0
  101. package/src/camera/PoseCameraView.tsx +223 -0
  102. package/src/camera/WebViewPoseView.tsx +466 -0
  103. package/src/client.ts +1295 -0
  104. package/src/diagnostics/logReport.ts +131 -0
  105. package/src/engine/EngineLoader.ts +428 -0
  106. package/src/engine/types.ts +93 -0
  107. package/src/events/classicMessage.ts +271 -0
  108. package/src/exercises/aliases.ts +50 -0
  109. package/src/index.ts +243 -0
  110. package/src/models/poseModels.ts +93 -0
  111. package/src/quality/AdaptiveQualityController.ts +579 -0
  112. package/src/quality/RuntimeGuard.ts +119 -0
  113. package/src/quality/captureMode.ts +97 -0
  114. package/src/quality/deviceCapability.ts +187 -0
  115. package/src/quality/profiles.ts +253 -0
  116. package/src/runtime/RuntimeCache.ts +300 -0
  117. package/src/sdkVersion.ts +2 -0
  118. package/src/support/optionalModules.ts +114 -0
  119. package/src/types/acceleration.ts +94 -0
  120. package/src/types/events.ts +437 -0
  121. package/src/types/features.ts +176 -0
  122. package/src/types/manifest.ts +230 -0
  123. package/src/types/pose.ts +85 -0
  124. package/src/types/preload.ts +19 -0
  125. package/src/types/skeleton.ts +77 -0
@@ -0,0 +1,579 @@
1
+ /**
2
+ * Adaptive camera-quality controller (Sency AdaptiveChoice + RuntimeGuard).
3
+ *
4
+ * Uses a platform **minimum target FPS** (floor), not a single setpoint:
5
+ * - iOS: min 30 (acceptable band ~30–50+)
6
+ * - Android: min 15 (mid-range GPUs rarely match iOS)
7
+ *
8
+ * Flow:
9
+ * 1. Resolve an initial profile from device capability + crash-loop guard.
10
+ * 2. WebView benches MoveNet on zeros *before* opening the camera → estimated
11
+ * FPS. If estimate < min-target, drop capture profile then open webcam.
12
+ * 3. Live 1 Hz stats: if mean FPS < min-target → downgrade again.
13
+ * 4. Emit `performance_warning` when even `basic` cannot hold the floor.
14
+ */
15
+
16
+ import { SDK_VERSION } from '../api/configure';
17
+ import { createAsyncKeyValueStore, type KeyValueStore } from '../engine/EngineLoader';
18
+ import type {
19
+ PerformanceWarningEvent,
20
+ QualityChangedEvent,
21
+ } from '../types/events';
22
+ import { scoreDeviceCapability, type DeviceCapabilitySnapshot } from './deviceCapability';
23
+ import {
24
+ LOW_FPS_STREAK_BEFORE_DOWNGRADE,
25
+ QUALITY_SETTLE_MS,
26
+ currentQualityPlatform,
27
+ estimatedFpsFromMedianMs,
28
+ getCriticalFpsThreshold,
29
+ getIdealFpsRange,
30
+ getMinTargetFps,
31
+ getQualityProfile,
32
+ isQualityProfileId,
33
+ lowerQualityProfile,
34
+ nextLowerQualityProfile,
35
+ profileFromWarmupMedianMs,
36
+ qualityLadderIndex,
37
+ type FpsRange,
38
+ type CapturePriority,
39
+ type QualityChoice,
40
+ type QualityProfile,
41
+ type QualityProfileId,
42
+ } from './profiles';
43
+ import { RuntimeGuard } from './RuntimeGuard';
44
+
45
+ const ACTIVE_KEY = 'posetracker.quality.active';
46
+
47
+ export interface QualityStatsSample {
48
+ fps: number;
49
+ medianInferenceMs: number | null;
50
+ videoSize?: string;
51
+ backend?: string;
52
+ }
53
+
54
+ export interface AdaptiveQualityControllerOptions {
55
+ /** Explicit override; default AdaptiveChoice. */
56
+ choice?: QualityChoice;
57
+ /**
58
+ * `performance` (default) vs `quality` — see {@link CapturePriority}.
59
+ * When `quality`, FPS-driven capture downgrades are skipped.
60
+ */
61
+ capturePriority?: CapturePriority;
62
+ keyValueStore?: KeyValueStore | null;
63
+ onDiagnostic?: (message: string) => void;
64
+ onQualityChanged?: (event: QualityChangedEvent) => void;
65
+ onPerformanceWarning?: (event: PerformanceWarningEvent) => void;
66
+ /** Apply constraints to the live WebView (restart getUserMedia). */
67
+ applyProfile?: (profile: QualityProfile) => void;
68
+ }
69
+
70
+ export interface QualityState {
71
+ choice: QualityChoice;
72
+ /** Host capture vs FPS trade-off (default `performance`). */
73
+ capturePriority: CapturePriority;
74
+ activeProfile: QualityProfileId;
75
+ profile: QualityProfile;
76
+ capability: DeviceCapabilitySnapshot;
77
+ meanFps: number | null;
78
+ /** Last warm-up estimate (1000/medianMs), before or at camera open. */
79
+ warmupEstimatedFps: number | null;
80
+ warmupMedianMs: number | null;
81
+ lowFpsStreak: number;
82
+ lastWarningAtMs: number | null;
83
+ /** Platform floor (iOS 30 / Android 15) — not a setpoint. */
84
+ minTargetFps: number;
85
+ /** Soft “good experience” band for docs / host UI. */
86
+ idealFpsRange: FpsRange;
87
+ /** @deprecated Use {@link minTargetFps}. */
88
+ targetFps: number;
89
+ }
90
+
91
+ export class AdaptiveQualityController {
92
+ private readonly choice: QualityChoice;
93
+ private readonly capturePriority: CapturePriority;
94
+ private readonly kv: KeyValueStore | null;
95
+ private readonly guard: RuntimeGuard;
96
+ private readonly onDiagnostic: ((message: string) => void) | undefined;
97
+ private readonly onQualityChanged: ((event: QualityChangedEvent) => void) | undefined;
98
+ private readonly onPerformanceWarning: ((event: PerformanceWarningEvent) => void) | undefined;
99
+ private applyProfile: ((profile: QualityProfile) => void) | undefined;
100
+
101
+ private capability: DeviceCapabilitySnapshot;
102
+ private activeProfile: QualityProfileId;
103
+ private resolved = false;
104
+ private fpsWindow: number[] = [];
105
+ private lowFpsStreak = 0;
106
+ private lastWarningAtMs: number | null = null;
107
+ private downgradeInFlight = false;
108
+ private warmupEstimatedFps: number | null = null;
109
+ private warmupMedianMs: number | null = null;
110
+ private warmupApplied = false;
111
+ /** Earliest time live low-FPS downgrades are allowed (settle after camera open/swap). */
112
+ private settleUntilMs = 0;
113
+
114
+ constructor(options: AdaptiveQualityControllerOptions = {}) {
115
+ this.choice = options.choice ?? 'AdaptiveChoice';
116
+ this.capturePriority = options.capturePriority === 'quality' ? 'quality' : 'performance';
117
+ this.kv =
118
+ options.keyValueStore !== undefined
119
+ ? options.keyValueStore
120
+ : createAsyncKeyValueStore();
121
+ this.guard = new RuntimeGuard(this.kv, SDK_VERSION);
122
+ this.onDiagnostic = options.onDiagnostic;
123
+ this.onQualityChanged = options.onQualityChanged;
124
+ this.onPerformanceWarning = options.onPerformanceWarning;
125
+ this.applyProfile = options.applyProfile;
126
+ this.capability = scoreDeviceCapability();
127
+ this.activeProfile =
128
+ this.choice === 'AdaptiveChoice'
129
+ ? this.capturePriority === 'quality'
130
+ ? 'prime'
131
+ : this.capability.suggestedProfile
132
+ : this.choice;
133
+ }
134
+
135
+ /** Host capture vs FPS trade-off. */
136
+ getCapturePriority(): CapturePriority {
137
+ return this.capturePriority;
138
+ }
139
+
140
+ prefersCaptureQuality(): boolean {
141
+ return this.capturePriority === 'quality';
142
+ }
143
+
144
+ setApplyProfile(fn: ((profile: QualityProfile) => void) | undefined): void {
145
+ this.applyProfile = fn;
146
+ }
147
+
148
+ getState(): QualityState {
149
+ const platform = currentQualityPlatform();
150
+ const minTargetFps = getMinTargetFps(platform);
151
+ return {
152
+ choice: this.choice,
153
+ capturePriority: this.capturePriority,
154
+ activeProfile: this.activeProfile,
155
+ profile: getQualityProfile(this.activeProfile),
156
+ capability: this.capability,
157
+ meanFps: this.meanFps(),
158
+ warmupEstimatedFps: this.warmupEstimatedFps,
159
+ warmupMedianMs: this.warmupMedianMs,
160
+ lowFpsStreak: this.lowFpsStreak,
161
+ lastWarningAtMs: this.lastWarningAtMs,
162
+ minTargetFps,
163
+ idealFpsRange: getIdealFpsRange(platform),
164
+ targetFps: minTargetFps,
165
+ };
166
+ }
167
+
168
+ /** Platform minimum target FPS (floor). */
169
+ getMinTargetFps(): number {
170
+ return getMinTargetFps(currentQualityPlatform());
171
+ }
172
+
173
+ getActiveProfile(): QualityProfile {
174
+ return getQualityProfile(this.activeProfile);
175
+ }
176
+
177
+ /**
178
+ * Resolve the profile to boot with. Call once before building WebView HTML.
179
+ * Walks down the ladder past any FAILED / crash-PROBING keys.
180
+ */
181
+ async resolveInitialProfile(): Promise<QualityProfile> {
182
+ if (this.resolved) return this.getActiveProfile();
183
+
184
+ this.capability = scoreDeviceCapability();
185
+ let candidate: QualityProfileId =
186
+ this.choice === 'AdaptiveChoice'
187
+ ? this.capturePriority === 'quality'
188
+ ? 'prime'
189
+ : this.capability.suggestedProfile
190
+ : this.choice;
191
+
192
+ // capturePriority=quality: do not restore a low last-good (that would
193
+ // defeat the host's request for sharp preview). Crash-guard still applies.
194
+ if (this.choice === 'AdaptiveChoice' && this.kv && this.capturePriority !== 'quality') {
195
+ try {
196
+ const saved = await this.kv.getItem(ACTIVE_KEY);
197
+ if (saved && isQualityProfileId(saved) && isAtMost(saved, candidate)) {
198
+ // Honor last-good only when it is at most one tier below the
199
+ // capability suggestion. A bigger gap is almost always a stale
200
+ // floor (e.g. iPhone score→pro stuck on ultralite after an old
201
+ // crash / bad session). Crash-guard FAILED marks still apply below.
202
+ const gap = qualityLadderIndex(saved) - qualityLadderIndex(candidate);
203
+ if (gap <= 1) {
204
+ candidate = saved;
205
+ this.onDiagnostic?.(
206
+ `[posetracker] quality: restored last-good profile=${candidate}`,
207
+ );
208
+ } else {
209
+ this.onDiagnostic?.(
210
+ `[posetracker] quality: ignoring stale last-good=${saved} ` +
211
+ `(suggested=${candidate}, gap=${gap} tiers) — re-probing from capability`,
212
+ );
213
+ }
214
+ }
215
+ } catch {
216
+ /* ignore */
217
+ }
218
+ }
219
+
220
+ const suggested = candidate;
221
+ let safety = 0;
222
+ while (safety++ < 8) {
223
+ const crashed = await this.guard.consumeCrashIfProbing(candidate);
224
+ const failed = crashed || (await this.guard.isFailed(candidate));
225
+ if (!failed) break;
226
+ const lower = nextLowerQualityProfile(candidate);
227
+ this.onDiagnostic?.(
228
+ `[posetracker] quality: profile=${candidate} marked failed (crash-guard) → ` +
229
+ (lower ?? 'none'),
230
+ );
231
+ if (!lower) break;
232
+ candidate = lower;
233
+ }
234
+
235
+ // Poisoned ladder: aborted getUserMedia races used to mark every tier
236
+ // FAILED and pin the device on basic. If we dropped ≥2 tiers below the
237
+ // capability suggestion, clear those marks and re-probe from suggested.
238
+ const drop = qualityLadderIndex(candidate) - qualityLadderIndex(suggested);
239
+ if (drop >= 2 && this.choice === 'AdaptiveChoice') {
240
+ this.onDiagnostic?.(
241
+ `[posetracker] quality: clearing poisoned crash-guard ` +
242
+ `(${suggested}→${candidate}, drop=${drop}) — re-probing from ${suggested}`,
243
+ );
244
+ let walk: QualityProfileId | null = suggested;
245
+ while (walk) {
246
+ await this.guard.clear(walk);
247
+ if (walk === candidate) break;
248
+ walk = nextLowerQualityProfile(walk);
249
+ }
250
+ candidate = suggested;
251
+ }
252
+
253
+ this.activeProfile = candidate;
254
+ this.resolved = true;
255
+ const minTarget = this.getMinTargetFps();
256
+ const band = getIdealFpsRange();
257
+ this.onDiagnostic?.(
258
+ `[posetracker] quality: initial profile=${candidate} ` +
259
+ `minTargetFps=${minTarget} idealBand=${band.min}-${band.idealMax}+ ` +
260
+ `choice=${this.choice} capturePriority=${this.capturePriority} ` +
261
+ `score=${this.capability.score} ` +
262
+ `reasons=[${this.capability.reasons.join('; ')}]`,
263
+ );
264
+ return this.getActiveProfile();
265
+ }
266
+
267
+ /** Call immediately before the WebView page boots this profile. */
268
+ async beginSession(): Promise<void> {
269
+ if (!this.resolved) await this.resolveInitialProfile();
270
+ await this.guard.markProbing(this.activeProfile);
271
+ this.onDiagnostic?.(
272
+ `[posetracker] quality: guard PROBING profile=${this.activeProfile}`,
273
+ );
274
+ }
275
+
276
+ /**
277
+ * Warm-up zeros bench result (before / around getUserMedia).
278
+ * Picks the safer of (warmup median → profile) and (device capability).
279
+ * May **upgrade** away from a stale last-good when the bench shows headroom
280
+ * (e.g. 15 ms median on iPhone stuck on ultralite).
281
+ */
282
+ async onWarmupEstimate(options: {
283
+ medianInferenceMs: number | null;
284
+ glRenderer?: string | null;
285
+ /** Profile the HTML page already selected from the same estimate (sync). */
286
+ pageSelectedProfile?: QualityProfileId | null;
287
+ }): Promise<void> {
288
+ const med = options.medianInferenceMs;
289
+ const est = estimatedFpsFromMedianMs(med);
290
+ const minTarget = this.getMinTargetFps();
291
+ this.warmupMedianMs = med;
292
+ this.warmupEstimatedFps = est;
293
+
294
+ if (options.glRenderer) {
295
+ this.capability = scoreDeviceCapability({ glRenderer: options.glRenderer });
296
+ }
297
+
298
+ const fromWarmup = profileFromWarmupMedianMs(med, minTarget);
299
+ const fromGl =
300
+ this.choice === 'AdaptiveChoice' ? this.capability.suggestedProfile : this.activeProfile;
301
+ // Safer of bench vs capability — do NOT clamp to activeProfile/page
302
+ // (that permanently froze devices on a stale ultralite last-good).
303
+ let next = lowerQualityProfile(fromWarmup, fromGl);
304
+ if (this.capturePriority === 'quality') {
305
+ // Host opted out of FPS-driven capture cuts — keep / raise to prime.
306
+ next = this.choice === 'AdaptiveChoice' ? 'prime' : this.activeProfile;
307
+ this.onDiagnostic?.(
308
+ `[posetracker] quality: capturePriority=quality — skipping FPS warm-up downgrade ` +
309
+ `(est=${est != null ? est.toFixed(1) : '?'} fps); holding profile=${next}`,
310
+ );
311
+ } else {
312
+ // Never upgrade when zeros warm-up is already below the floor — camera
313
+ // preprocess only adds cost (Mali: basic~10fps, ultralite upgrade~8fps).
314
+ const belowFloor = est != null && est < minTarget;
315
+ if (belowFloor && qualityLadderIndex(next) < qualityLadderIndex(this.activeProfile)) {
316
+ this.onDiagnostic?.(
317
+ `[posetracker] quality: warmup below floor (est=${est.toFixed(1)} < ${minTarget}) ` +
318
+ `— blocking upgrade ${this.activeProfile} → ${next}, staying ${this.activeProfile}`,
319
+ );
320
+ next = this.activeProfile;
321
+ }
322
+ }
323
+ const upgrading = qualityLadderIndex(next) < qualityLadderIndex(this.activeProfile);
324
+
325
+ this.onDiagnostic?.(
326
+ `[posetracker] quality: warmup medianMs=${med != null ? med.toFixed(1) : '?'} ` +
327
+ `estimatedFps=${est != null ? est.toFixed(1) : '?'} ` +
328
+ `minTarget=${minTarget} → profile=${next} (was ${this.activeProfile}` +
329
+ `${upgrading ? ', upgrade' : ''})`,
330
+ );
331
+
332
+ if (next !== this.activeProfile) {
333
+ // Page posts warmup_estimate *before* getUserMedia and now opens at
334
+ // fromWarmup. Never inject setQuality during that window — it aborts
335
+ // the in-flight getUserMedia ("The operation was aborted") and fails init.
336
+ // Sync RN state only; live setQuality remains for post-ready downgrades.
337
+ const pageAlreadyApplied =
338
+ options.pageSelectedProfile != null && options.pageSelectedProfile === next;
339
+ const skipApply = pageAlreadyApplied || upgrading;
340
+ const detail = upgrading
341
+ ? `Warm-up estimated inference FPS ${est != null ? est.toFixed(1) : '?'} ` +
342
+ `(median ${med != null ? med.toFixed(1) : '?'} ms) has headroom above the ` +
343
+ `minimum target of ${minTarget} FPS. Upgrading camera profile ` +
344
+ `"${this.activeProfile}" → "${next}"` +
345
+ (skipApply ? ' (page will open / already selected — no mid-boot setQuality).' : '.')
346
+ : `Warm-up estimated inference FPS ${est != null ? est.toFixed(1) : '?'} ` +
347
+ `(median ${med != null ? med.toFixed(1) : '?'} ms) is below the ` +
348
+ `platform minimum target of ${minTarget} FPS. Switching camera profile ` +
349
+ `to "${next}" before opening the webcam.`;
350
+ if (upgrading) {
351
+ // Drop stale FAILED marks for the tier we're recovering to.
352
+ await this.guard.clear(next);
353
+ }
354
+ await this.downgradeTo(next, 'warmup_estimate', detail, {
355
+ estimatedFps: est,
356
+ medianInferenceMs: med,
357
+ skipApply,
358
+ });
359
+ } else {
360
+ this.warmupApplied = true;
361
+ await this.guard.markPassed(this.activeProfile);
362
+ await this.persistActive(this.activeProfile);
363
+ }
364
+ }
365
+
366
+ /** Call when the WebView posts `ready`. */
367
+ async onRuntimeReady(options?: {
368
+ glRenderer?: string | null;
369
+ medianInferenceMs?: number | null;
370
+ pageSelectedProfile?: QualityProfileId | null;
371
+ }): Promise<void> {
372
+ // Prefer warm-up path; ready often carries the same medianMs.
373
+ if (!this.warmupApplied && options?.medianInferenceMs != null) {
374
+ await this.onWarmupEstimate({
375
+ medianInferenceMs: options.medianInferenceMs,
376
+ glRenderer: options.glRenderer,
377
+ pageSelectedProfile: options.pageSelectedProfile,
378
+ });
379
+ return;
380
+ }
381
+
382
+ if (options?.glRenderer) {
383
+ this.capability = scoreDeviceCapability({ glRenderer: options.glRenderer });
384
+ if (
385
+ this.choice === 'AdaptiveChoice' &&
386
+ isStrictlyLower(this.capability.suggestedProfile, this.activeProfile)
387
+ ) {
388
+ await this.downgradeTo(
389
+ this.capability.suggestedProfile,
390
+ 'device_capability',
391
+ 'GL renderer capability suggests a lower camera profile for stable FPS.',
392
+ );
393
+ return;
394
+ }
395
+ }
396
+
397
+ await this.guard.markPassed(this.activeProfile);
398
+ await this.persistActive(this.activeProfile);
399
+ this.settleUntilMs = Date.now() + QUALITY_SETTLE_MS;
400
+ this.lowFpsStreak = 0;
401
+ this.fpsWindow = [];
402
+ this.onDiagnostic?.(
403
+ `[posetracker] quality: guard PASSED profile=${this.activeProfile} ` +
404
+ `(live downgrade settle ${QUALITY_SETTLE_MS}ms)`,
405
+ );
406
+ }
407
+
408
+ /** Feed 1 Hz WebView stats. May auto-downgrade and/or warn. */
409
+ async onStats(sample: QualityStatsSample): Promise<void> {
410
+ if (!Number.isFinite(sample.fps) || sample.fps < 0) return;
411
+ this.fpsWindow.push(sample.fps);
412
+ if (this.fpsWindow.length > 10) this.fpsWindow.shift();
413
+
414
+ const profile = this.getActiveProfile();
415
+ const mean = this.meanFps();
416
+ if (mean == null) return;
417
+
418
+ const minTarget = this.getMinTargetFps();
419
+ const critical = getCriticalFpsThreshold();
420
+ const now = Date.now();
421
+
422
+ // Pipeline FPS (rAF loop) is often well below 1000/medianMs because it
423
+ // includes camera + bitmap. Downgrade on **inference median** only — e.g.
424
+ // iPhone pro at fps=22 / medianMs=28 is healthy vs a 30 FPS floor (33 ms).
425
+ const budgetMs = 1000 / minTarget;
426
+ const med = sample.medianInferenceMs;
427
+ const inferenceTooSlow =
428
+ med != null && Number.isFinite(med) && med > budgetMs * 1.2;
429
+
430
+ if (now < this.settleUntilMs) {
431
+ this.lowFpsStreak = 0;
432
+ return;
433
+ }
434
+
435
+ if (inferenceTooSlow) {
436
+ this.lowFpsStreak += 1;
437
+ } else {
438
+ this.lowFpsStreak = 0;
439
+ }
440
+
441
+ // Warnings may still look at pipeline FPS (UX feel), but never after settle only.
442
+ if (mean < critical || (mean < minTarget && !nextLowerQualityProfile(this.activeProfile))) {
443
+ if (this.lastWarningAtMs == null || now - this.lastWarningAtMs > 30_000) {
444
+ this.lastWarningAtMs = now;
445
+ const qualityNote =
446
+ this.capturePriority === 'quality'
447
+ ? ' capturePriority is "quality" (sharp preview preferred over FPS) — this is expected on mid-range Android.'
448
+ : '';
449
+ const event: PerformanceWarningEvent = {
450
+ type: 'performance_warning',
451
+ code: 'device_too_slow',
452
+ message:
453
+ `This device appears unsuitable for real-time pose estimation with the ` +
454
+ `current configuration (mean pipeline FPS=${mean.toFixed(1)} < ` +
455
+ `minimum target ${minTarget} FPS` +
456
+ (med != null ? `, median inference ${med.toFixed(1)} ms` : '') +
457
+ `). User experience may be significantly impacted.` +
458
+ qualityNote,
459
+ meanFps: mean,
460
+ thresholdFps: minTarget,
461
+ activeProfile: this.activeProfile,
462
+ medianInferenceMs: sample.medianInferenceMs,
463
+ videoSize: sample.videoSize,
464
+ timestampMs: now,
465
+ };
466
+ this.onDiagnostic?.(`[posetracker] WARNING: ${event.message}`);
467
+ this.onPerformanceWarning?.(event);
468
+ }
469
+ }
470
+
471
+ if (
472
+ this.capturePriority !== 'quality' &&
473
+ this.lowFpsStreak >= LOW_FPS_STREAK_BEFORE_DOWNGRADE &&
474
+ !this.downgradeInFlight
475
+ ) {
476
+ const lower = nextLowerQualityProfile(this.activeProfile);
477
+ if (lower) {
478
+ await this.downgradeTo(
479
+ lower,
480
+ 'low_fps',
481
+ `Median inference ${med != null ? med.toFixed(1) : '?'} ms stayed above the ` +
482
+ `budget of ${(budgetMs * 1.2).toFixed(0)} ms (min target ${minTarget} FPS) ` +
483
+ `for ${this.lowFpsStreak} samples on profile "${profile.id}". ` +
484
+ `Automatically switching to "${lower}".`,
485
+ { estimatedFps: mean, medianInferenceMs: sample.medianInferenceMs },
486
+ );
487
+ }
488
+ } else if (
489
+ this.capturePriority === 'quality' &&
490
+ this.lowFpsStreak >= LOW_FPS_STREAK_BEFORE_DOWNGRADE
491
+ ) {
492
+ if (this.lowFpsStreak === LOW_FPS_STREAK_BEFORE_DOWNGRADE) {
493
+ this.onDiagnostic?.(
494
+ `[posetracker] quality: capturePriority=quality — not downgrading from ` +
495
+ `"${profile.id}" despite slow median ${med != null ? med.toFixed(1) : '?'} ms`,
496
+ );
497
+ }
498
+ }
499
+ }
500
+
501
+ private async downgradeTo(
502
+ next: QualityProfileId,
503
+ reason: QualityChangedEvent['reason'],
504
+ detail: string,
505
+ extras?: {
506
+ estimatedFps?: number | null;
507
+ medianInferenceMs?: number | null;
508
+ /** When true, update state/events but do not restart getUserMedia. */
509
+ skipApply?: boolean;
510
+ },
511
+ ): Promise<void> {
512
+ if (next === this.activeProfile || this.downgradeInFlight) return;
513
+ this.downgradeInFlight = true;
514
+ const previous = this.activeProfile;
515
+ try {
516
+ // Only crash_guard / hard capability caps poison a tier. Live low_fps
517
+ // must NOT markFailed — that pinned iPhones on basic after a cascade.
518
+ if (reason === 'crash_guard' || reason === 'device_capability') {
519
+ await this.guard.markFailed(previous);
520
+ } else if (qualityLadderIndex(next) < qualityLadderIndex(previous)) {
521
+ await this.guard.clear(previous);
522
+ }
523
+ this.activeProfile = next;
524
+ this.lowFpsStreak = 0;
525
+ this.fpsWindow = [];
526
+ this.warmupApplied = true;
527
+ this.settleUntilMs = Date.now() + QUALITY_SETTLE_MS;
528
+ await this.guard.markProbing(next);
529
+ await this.persistActive(next);
530
+ const profile = getQualityProfile(next);
531
+ if (!extras?.skipApply) {
532
+ this.applyProfile?.(profile);
533
+ }
534
+ const event: QualityChangedEvent = {
535
+ type: 'quality_changed',
536
+ previousProfile: previous,
537
+ activeProfile: next,
538
+ reason,
539
+ detail,
540
+ profile,
541
+ estimatedFps: extras?.estimatedFps,
542
+ medianInferenceMs: extras?.medianInferenceMs,
543
+ timestampMs: Date.now(),
544
+ };
545
+ this.onDiagnostic?.(
546
+ `[posetracker] quality: ${previous} → ${next} reason=${reason}` +
547
+ (extras?.skipApply ? ' (page-applied)' : '') +
548
+ ` — ${detail}`,
549
+ );
550
+ this.onQualityChanged?.(event);
551
+ } finally {
552
+ this.downgradeInFlight = false;
553
+ }
554
+ }
555
+
556
+ private meanFps(): number | null {
557
+ if (!this.fpsWindow.length) return null;
558
+ const sum = this.fpsWindow.reduce((a, b) => a + b, 0);
559
+ return sum / this.fpsWindow.length;
560
+ }
561
+
562
+ private async persistActive(id: QualityProfileId): Promise<void> {
563
+ if (!this.kv) return;
564
+ try {
565
+ await this.kv.setItem(ACTIVE_KEY, id);
566
+ } catch {
567
+ /* ignore */
568
+ }
569
+ }
570
+ }
571
+
572
+ /** True if `a` is equal or lower quality than `b`. */
573
+ function isAtMost(a: QualityProfileId, b: QualityProfileId): boolean {
574
+ return qualityLadderIndex(a) >= qualityLadderIndex(b) && qualityLadderIndex(a) >= 0;
575
+ }
576
+
577
+ function isStrictlyLower(a: QualityProfileId, b: QualityProfileId): boolean {
578
+ return qualityLadderIndex(a) > qualityLadderIndex(b);
579
+ }
@@ -0,0 +1,119 @@
1
+ /**
2
+ * Crash-loop guard for quality / runtime profiles.
3
+ *
4
+ * Analog of Sency's `TfliteRuntimeGuard` (PROBING → PASSED | FAILED):
5
+ * 1. Before activating a profile, `markProbing(key)`.
6
+ * 2. After a healthy ready signal, `markPassed(key)`.
7
+ * 3. If the app is killed/crashes while still PROBING, the next launch
8
+ * sees PROBING → treats it as FAILED → caller must pick a lower profile.
9
+ *
10
+ * Persisted via {@link KeyValueStore} (AsyncStorage when available). Without
11
+ * a store the guard is in-memory only (still useful within one process for
12
+ * the 30s same-process timeout).
13
+ */
14
+
15
+ import type { KeyValueStore } from '../engine/EngineLoader';
16
+
17
+ export type GuardState = 'unknown' | 'probing' | 'passed' | 'failed';
18
+
19
+ const KEY_PREFIX = 'posetracker.quality.guard.';
20
+ /** Same-process: if still probing after this, treat as failed. */
21
+ const PROBING_TIMEOUT_MS = 30_000;
22
+
23
+ interface GuardRecord {
24
+ state: GuardState;
25
+ updatedAt: number;
26
+ sdkVersion: string;
27
+ }
28
+
29
+ export class RuntimeGuard {
30
+ private readonly memory = new Map<string, GuardRecord>();
31
+
32
+ constructor(
33
+ private readonly kv: KeyValueStore | null,
34
+ private readonly sdkVersion: string,
35
+ ) {}
36
+
37
+ private storageKey(profileKey: string): string {
38
+ return `${KEY_PREFIX}${this.sdkVersion}.${profileKey}`;
39
+ }
40
+
41
+ async getState(profileKey: string): Promise<GuardState> {
42
+ const key = this.storageKey(profileKey);
43
+ let record = this.memory.get(key) ?? null;
44
+ if (!record && this.kv) {
45
+ try {
46
+ const raw = await this.kv.getItem(key);
47
+ if (raw) record = JSON.parse(raw) as GuardRecord;
48
+ } catch {
49
+ record = null;
50
+ }
51
+ }
52
+ if (!record) return 'unknown';
53
+
54
+ if (
55
+ record.state === 'probing' &&
56
+ Date.now() - record.updatedAt > PROBING_TIMEOUT_MS
57
+ ) {
58
+ await this.write(profileKey, 'failed');
59
+ return 'failed';
60
+ }
61
+ return record.state;
62
+ }
63
+
64
+ /**
65
+ * If previous launch died while probing this key, mark failed and return true.
66
+ */
67
+ async consumeCrashIfProbing(profileKey: string): Promise<boolean> {
68
+ const state = await this.getState(profileKey);
69
+ if (state === 'probing') {
70
+ await this.write(profileKey, 'failed');
71
+ return true;
72
+ }
73
+ return false;
74
+ }
75
+
76
+ async markProbing(profileKey: string): Promise<void> {
77
+ await this.write(profileKey, 'probing');
78
+ }
79
+
80
+ async markPassed(profileKey: string): Promise<void> {
81
+ await this.write(profileKey, 'passed');
82
+ }
83
+
84
+ async markFailed(profileKey: string): Promise<void> {
85
+ await this.write(profileKey, 'failed');
86
+ }
87
+
88
+ async isFailed(profileKey: string): Promise<boolean> {
89
+ return (await this.getState(profileKey)) === 'failed';
90
+ }
91
+
92
+ /** Clear persisted/memory state for a profile (recover from stale FAILED). */
93
+ async clear(profileKey: string): Promise<void> {
94
+ const key = this.storageKey(profileKey);
95
+ this.memory.delete(key);
96
+ if (!this.kv) return;
97
+ try {
98
+ await this.kv.removeItem(key);
99
+ } catch {
100
+ /* best-effort */
101
+ }
102
+ }
103
+
104
+ private async write(profileKey: string, state: GuardState): Promise<void> {
105
+ const key = this.storageKey(profileKey);
106
+ const record: GuardRecord = {
107
+ state,
108
+ updatedAt: Date.now(),
109
+ sdkVersion: this.sdkVersion,
110
+ };
111
+ this.memory.set(key, record);
112
+ if (!this.kv) return;
113
+ try {
114
+ await this.kv.setItem(key, JSON.stringify(record));
115
+ } catch {
116
+ /* persistence best-effort */
117
+ }
118
+ }
119
+ }