@pose-tracker/pose-estimation-web 0.1.0 → 0.3.0

package/dist/index.d.cts

@@ -1,7 +1,7 @@
 import * as _tensorflow_tfjs from '@tensorflow/tfjs';
 
 /** Bumped with package.json version (keep in sync). */
-declare const SDK_VERSION = "0.1.0";
+declare const SDK_VERSION = "0.3.0";
 declare const SDK_NAME: "posetracker-web";
 
 /**
@@ -132,6 +132,8 @@ interface EngineBundleDescriptor {
     minSdkVersion: string;
     /** True when the version sent in `localVersions.engine` is already current. */
     upToDate?: boolean;
+    /** Which interpreter this descriptor is. Absent on older backends = v3. */
+    channel?: 'v3' | 'v4';
 }
 interface PoseRuntimePartDescriptor {
     /** Download URL of this part (`GET /api/sdk/pose-runtime?part=...&v=...`). */
@@ -172,6 +174,11 @@ interface ConfigureRequest {
         poseRuntime?: string | null;
         engine?: string | null;
     };
+    /**
+     * Which remote engine to download. Default / omitted = v3 (production).
+     * `v4` is opt-in (heuristic catalog: squat, shoulder_roll, …).
+     */
+    engineChannel?: 'v3' | 'v4';
 }
 interface PlanInfo {
     plan: string;
@@ -202,6 +209,13 @@ interface SdkManifest {
     expiresAt: number;
 }
 
+/** Opt-in remote engine channel. Default is production V3. */
+type EngineChannel = 'v3' | 'v4';
+/** Movements that exist only in the V4 catalog (not in V3 FSM / Strapi). */
+declare const V4_ONLY_EXERCISE_IDS: readonly ["shoulder_roll", "shoulder_deep_breath", "chair_forward_fold"];
+declare function normalizeEngineChannel(raw?: string | null): EngineChannel;
+declare function requiresEngineV4(exerciseId: string): boolean;
+
 /**
  * Handshake client: `configure(apiToken)` → Strapi `SdkManifest`.
  * Same endpoint as RN light; `targetPlatform: 'web'`, `sdkName: posetracker-web`.
@@ -213,6 +227,11 @@ interface ConfigureOptions {
     poseModelProfile?: PoseModelProfile;
     locale?: string;
     localVersions?: ConfigureRequest['localVersions'];
+    /**
+     * Opt-in V4 heuristic engine. Default `'v3'` (production FSM).
+     * Set `'v4'` at init — the handshake downloads `engine-v4.bundle.js`.
+     */
+    engine?: EngineChannel;
 }
 declare class ConfigureError extends Error {
     readonly code: 'invalid_token' | 'quota_exceeded' | 'network' | 'internal';
@@ -310,7 +329,7 @@ type PoseTrackerStatus = 'idle' | 'configuring' | 'downloading' | 'warming' | 'r
  * restarting the camera pipeline, via `configure()` or a preload retry.
  */
 type PoseTrackerMode = 'keypoints-only' | 'full-engine';
-type InitializationStep = 'configuring' | 'downloading' | 'warming' | 'accessing_webcam' | 'loading_pose_model' | 'ready';
+type InitializationStep = 'configuring' | 'downloading' | 'warming' | 'accessing_webcam' | 'loading_media' | 'loading_pose_model' | 'ready';
 interface InitializationEvent {
     type: 'initialization';
     /**
@@ -359,7 +378,7 @@ interface ErrorEvent {
     /** Unknown exercise id (WebView parity: `invalid_exercise`). */
      | 'invalid_exercise'
     /** jump_analysis requires `userHeightCm` (WebView parity). */
-     | 'jump_analysis_missing_height' | 'model_load_failed' | 'engine_load_failed' | 'integrity_check_failed' | 'webview_error' | 'device_too_slow' | 'backend_fallback_wasm' | 'camera_denied' | 'camera_unavailable' | 'internal';
+     | 'jump_analysis_missing_height' | 'model_load_failed' | 'engine_load_failed' | 'integrity_check_failed' | 'webview_error' | 'device_too_slow' | 'backend_fallback_wasm' | 'camera_denied' | 'camera_unavailable' | 'media_load_failed' | 'media_unavailable' | 'internal';
     message: string;
 }
 /**
@@ -716,6 +735,13 @@ interface PoseTrackerEngine {
     createSession(options: EngineSessionOptions, emit: EventSink): EngineSession;
     /** Engine >= 1.2.0: custom exercises shipped in the bundle (jump analysis…). */
     listCustomExercises?(): CustomExerciseDescriptor[];
+    /** Engine V4 catalog (opt-in channel). Absent on V3 bundles. */
+    listExercises?(): Array<{
+        id: string;
+        displayName: string;
+        type: string;
+        description?: string;
+    }>;
     /** Engine >= 1.2.0: start a custom exercise session (placement + jump_* events). */
     createCustomSession?(options: CustomSessionOptions, emit: EventSink): EngineSession;
 }
@@ -811,6 +837,52 @@ declare class UsageTracker {
     private enqueue;
 }
 
+/**
+ * Input source for pose estimation (camera / uploaded video / still image).
+ * Default remains camera for quickstart compatibility.
+ */
+type PoseSourceType = 'camera' | 'video' | 'image';
+type CameraFacingMode = 'user' | 'environment';
+interface CameraPoseSource {
+    type: 'camera';
+    facingMode?: CameraFacingMode;
+}
+/**
+ * Uploaded or remote video. `src` may be a URL string, File/Blob (object URL),
+ * or an existing HTMLVideoElement (SDK uses it as the frame source).
+ */
+interface VideoPoseSource {
+    type: 'video';
+    src: string | File | Blob | HTMLVideoElement;
+    /** Autoplay when attached. Default true. */
+    autoplay?: boolean;
+    /** Loop the file. Default true for continuous inference demos. */
+    loop?: boolean;
+    muted?: boolean;
+}
+/**
+ * Still image — single-shot estimate (re-run via `analyze()`).
+ */
+interface ImagePoseSource {
+    type: 'image';
+    src: string | File | Blob | HTMLImageElement | ImageBitmap;
+}
+type PoseSource = CameraPoseSource | VideoPoseSource | ImagePoseSource;
+/** Narrow helper for prop-style APIs (`source` + `sourceUrl` / `sourceFile`). */
+interface PoseSourceProps {
+    source?: PoseSourceType;
+    sourceUrl?: string;
+    sourceFile?: File | Blob;
+    facingMode?: CameraFacingMode;
+}
+/** Frame types accepted by detector adapters. */
+type PoseEstimateInput = HTMLVideoElement | HTMLImageElement | HTMLCanvasElement | ImageBitmap;
+declare function normalizePoseSource(source?: PoseSource | PoseSourceType | null, extras?: {
+    sourceUrl?: string;
+    sourceFile?: File | Blob;
+    facingMode?: CameraFacingMode;
+}): PoseSource;
+
 /**
  * Online pose-model catalog (web) — mirrors RN light `poseModels.ts`.
  *
@@ -954,6 +1026,20 @@ interface PoseTrackerClientOptions extends ConfigureOptions {
     onDiagnostic?: (message: string) => void;
     /** Alias of preload coldStart when using start(). Default full on camera screens. */
     coldStart?: ColdStartMode;
+    /**
+     * Input source. Default `{ type: 'camera' }` (webcam).
+     * Use `{ type: 'video', src }` or `{ type: 'image', src }` for file / URL / element.
+     */
+    source?: PoseSource | PoseSourceType;
+    /** Convenience with `source: 'video' | 'image'` (string / File / Blob). */
+    sourceUrl?: string;
+    sourceFile?: File | Blob;
+    /**
+     * Still-image inference. Default `single`: one full-frame SinglePose pass
+     * (no EMA / no ROI). `tracked`: treat analyze() as video frames — EMA +
+     * previous-pose crop so each shot can refine the last skeleton.
+     */
+    stillMode?: 'single' | 'tracked';
 }
 /** @deprecated Prefer PoseTrackerClientOptions — kept for vanilla DX. */
 type PoseTrackerOptions = PoseTrackerClientOptions;
@@ -989,6 +1075,10 @@ declare class PoseTrackerClient {
     private opts;
     private shell;
     private stream;
+    private poseSource;
+    private objectUrlRevoke;
+    private estimateInput;
+    private imageShotPending;
     private adapter;
     private resolved;
     private running;
@@ -1013,6 +1103,8 @@ declare class PoseTrackerClient {
     getAccelerationDiagnostics(): AccelerationDiagnostics;
     getResolvedModel(): ResolvedPoseModel;
     getColdStartMode(): ColdStartMode;
+    /** Active pose input source (`camera` | `video` | `image`). */
+    getSource(): PoseSource;
     addEventListener(listener: PoseTrackerEventListener): () => void;
     on(type: PoseTrackerEvent['type'] | '*', listener: PoseTrackerEventListener): () => void;
     addMessageListener(listener: ClassicMessageListener): () => void;
@@ -1024,14 +1116,35 @@ declare class PoseTrackerClient {
      */
     preload(options?: PreloadOptions): Promise<void>;
     warmup(options?: PreloadOptions): Promise<void>;
-    /** Mount + load model + open camera + start loop (camera-screen convenience). */
+    /**
+     * Mount + load model + open source + start inference.
+     * Camera / video stream continuously; image runs a single shot (re-run with {@link analyze}).
+     */
     start(): Promise<void>;
+    /**
+     * Switch input source. Stops the current media, attaches the new source, and
+     * restarts the loop when previously running.
+     */
+    setSource(source: PoseSource | PoseSourceType, extras?: {
+        sourceUrl?: string;
+        sourceFile?: File | Blob;
+        facingMode?: 'user' | 'environment';
+    }): Promise<void>;
+    /**
+     * Re-run pose estimation on the current still image (no-op for camera/video
+     * while the loop is already running — for those, call {@link start}).
+     */
+    analyze(): Promise<Pose | null>;
     stop(): void;
+    private stopMediaOnly;
     dispose(): Promise<void>;
     configure(apiToken?: string): Promise<boolean>;
     getAvailableExercises(): ExerciseConfig[];
+    getEngineChannel(): EngineChannel;
+    private v4ExerciseConfig;
     getAvailableCustomExercises(): CustomExerciseDescriptor[];
     startExercise(exerciseId: string, options?: StartExerciseOptions): void;
+    private beginEngineSession;
     private startCustomExercise;
     stopExercise(): void;
     getCurrentExerciseId(): string | null;
@@ -1042,8 +1155,13 @@ declare class PoseTrackerClient {
     private doPreload;
     private ensureColdStartMode;
     private ensureAdapter;
-    private ensureCamera;
+    private ensureSource;
+    private scheduleReveal;
+    private attachVideoSource;
+    private attachImageSource;
     private beginLoop;
+    private frameReady;
+    private runOneFrame;
     private loop;
     private resolveSkeletonFromOptions;
     private resolveManifestOnce;
@@ -1078,6 +1196,9 @@ interface Letterbox {
     drawH: number;
     vw: number;
     vh: number;
+    /** Window top-left in original media pixels (still-image video simulation). */
+    originX?: number;
+    originY?: number;
 }
 interface DetectorFrameResult {
     /** Raw model output used for skeleton mapping (MoveNet: Float32Array length 51). */
@@ -1095,18 +1216,24 @@ interface DetectorFrameResult {
     letterbox: Letterbox;
 }
 type Float32ArrayLike = ArrayLike<number>;
+
 interface PoseDetectorAdapter {
     readonly modelId: string;
     load(): Promise<void>;
     /**
-     * Run one frame. `video` must have non-zero dimensions.
+     * Run one frame. Input must have non-zero dimensions
+     * (HTMLVideoElement, HTMLImageElement, canvas, or ImageBitmap).
      * Returns null when the frame should be skipped.
      */
-    estimate(video: HTMLVideoElement, options: {
+    estimate(input: PoseEstimateInput, options: {
         facingMode: 'user' | 'environment';
         displayWidth: number;
         displayHeight: number;
+        /** EMA across frames. Default true. Stills may pass true to simulate a short clip. */
+        temporalSmooth?: boolean;
     }): Promise<DetectorFrameResult | null>;
+    /** Optional: clear temporal filters when the media source changes. */
+    resetTemporal?(): void;
     dispose(): void;
 }
 interface CreateAdapterContext {
@@ -1126,6 +1253,9 @@ declare class MoveNetGraphAdapter implements PoseDetectorAdapter {
     private offscreen;
     private offCtx;
     private smoothed;
+    private lastVideoKps;
+    private stillPass;
+    private inferLb;
     private letterbox;
     constructor(ctx: CreateAdapterContext);
     getBackend(): string | null;
@@ -1133,11 +1263,14 @@ declare class MoveNetGraphAdapter implements PoseDetectorAdapter {
     private poseCanvasCtx;
     private prepareInput;
     private smooth;
-    estimate(video: HTMLVideoElement, options: {
+    estimate(input: PoseEstimateInput, options: {
         facingMode: 'user' | 'environment';
         displayWidth: number;
         displayHeight: number;
+        temporalSmooth?: boolean;
     }): Promise<DetectorFrameResult | null>;
+    /** Clear EMA state (call when the still / clip changes). */
+    resetTemporal(): void;
     dispose(): void;
 }
 /** Map video keypoints → object-fit:cover display pixels for drawing. */
@@ -1162,11 +1295,14 @@ declare class BlazePoseAdapter implements PoseDetectorAdapter {
     constructor(_ctx: CreateAdapterContext);
     getBackend(): string | null;
     load(): Promise<void>;
-    estimate(video: HTMLVideoElement, options: {
+    estimate(input: PoseEstimateInput, options: {
         facingMode: 'user' | 'environment';
         displayWidth: number;
         displayHeight: number;
+        temporalSmooth?: boolean;
     }): Promise<DetectorFrameResult | null>;
+    /** Clear cached ROI / landmark filters (call when the still changes). */
+    resetTemporal(): void;
     dispose(): void;
 }
 
@@ -1266,6 +1402,8 @@ declare function drawSkeleton(ctx: CanvasRenderingContext2D, kps: DrawKeypoint[]
 interface DomShell {
     root: HTMLElement;
     video: HTMLVideoElement;
+    /** Still-image element for `source: image` mode. */
+    image: HTMLImageElement;
     canvas: HTMLCanvasElement;
     boot: HTMLElement;
     bootMsg: HTMLElement;
@@ -1274,10 +1412,12 @@ interface DomShell {
     placement: HTMLElement;
     setBootMessage(text: string, isError?: boolean): void;
     setCameraVisible(visible: boolean): void;
+    /** Show video preview, still image, or neither (boot only). */
+    setMediaMode(mode: 'video' | 'image' | 'none'): void;
     setWatermarkVisible(visible: boolean): void;
     setHud(text: string, debug: boolean): void;
     setPlacementVisible(visible: boolean, paddingPercent?: number): void;
-    applyMirror(facingMode: 'user' | 'environment'): void;
+    applyMirror(facingMode: 'user' | 'environment' | 'none'): void;
     destroy(): void;
 }
 declare function mountDomShell(host: HTMLElement, options?: {
@@ -1322,4 +1462,4 @@ declare function findExerciseByIdOrAlias<T extends {
     id: string;
 }>(exerciseId: string, exercises: readonly T[]): T | undefined;
 
-export { type AccelerationDiagnostics, type AccelerationState, type AngleValue, type AnglesEvent, BRAND_COLORS, BlazePoseAdapter, type BlazePoseModelType, COCO_KEYPOINT_NAMES, COMBINED_REFERENCE_EXERCISE_MESSAGE, type CameraOptions, type ClassicMessageListener, type ClassicNativeMessage, type CocoKeypointName, type ColdStartMode, ConfigureError, type ConfigureOptions, type ConfigureRequest, type CounterEvent, type CounterFormScore, type CustomExerciseDescriptor, type CustomSessionOptions, DEFAULT_BASE_URL, DEFAULT_FEATURES, DEFAULT_LOADING_TEXT, DEFAULT_MOVENET_LIGHTNING_URL, DEFAULT_SKELETON, DEFAULT_SKELETON_DEFINITION, type DetectorFrameResult, type DiagnosticListener, EXERCISE_ALIASES, type EngineBundleDescriptor, type EngineLoadResult, EngineLoader, type EngineLoaderOptions, type EngineSession, type EngineSessionFeatures, type EngineSessionOptions, type ErrorEvent, type EventSink, type ExerciseConfig, type ExerciseSummaryEvent, FREE_PLAN_FEATURES_MESSAGE, type FetchSkeletonOptions, type FileStore, type FormGrade, type FormScoreEvent, INVALID_TOKEN_MESSAGE, type InferenceRuntime, type InitialPosture, type InitializationEvent, type InitializationStep, type JumpCalibrationEvent, type JumpDiscardedEvent, type JumpHeightEvent, type JumpResultEvent, type JumpStartedEvent, type JumpSummaryEntry, type JumpSummaryEvent, type KeyValueStore, type Keypoint, type KeypointName, type KeypointsEvent, type MinGrade, type ModelDescriptor, type ModelFormat, type ModelsByProfile, MoveNetGraphAdapter, type MovementDefinition, type MovementRecommendation, type MovementStep, type MovementStepParameter, ONLINE_POSE_DETECTION_VERSION, ONLINE_TFJS_VERSION, POSETRACKER_LOGO_URL, type PerformanceWarningEvent, type PlanInfo, type Pose, PoseCamera, PoseCamera_mount, type PoseDetectionModule, type PoseDetectorAdapter, type PoseModelAlias, type PoseModelKind, type PoseModelProfile, type PoseRuntimeDescriptor, type PoseRuntimePartDescriptor, type PoseTracker, type PoseTrackerCallbacks, PoseTrackerClient, type PoseTrackerClientOptions, type PoseTrackerEngine, type PoseTrackerEvent, type PoseTrackerEventListener, type PoseTrackerEventType, type PoseTrackerFeatures, type PoseTrackerMode, type PoseTrackerOptions, type PoseTrackerStatus, type PostureEvent, type PreloadOptions, type ProgressionEvent, type QualityChangedEvent, type RecommendationsEvent, type ReferenceMovementDescriptor, type RepSummary, type ResolvePoseModelOptions, type ResolvedFeatures, type ResolvedPoseModel, type RuntimeDownloadProgressEvent, SDK_NAME, SDK_VERSION, type SdkManifest, type SkeletonAnglesStyle, type SkeletonCirclesStyle, type SkeletonDefinition, SkeletonFetchError, type SkeletonLinesStyle, type StartExerciseOptions, type StatsEvent, type TfModule, type TfjsCdnUrls, TrackError, type TrackRequest, type TrackResponse, UsageTracker, type UsageTrackerOptions, type VisibleHipsInfo, type WarningEvent, configure, createLocalFileStore, createLocalKeyValueStore, createMemoryFileStore, createPoseAdapter, createPoseTracker, createWebFileStore, defaultPoseDetectionCdnUrl, defaultTfjsCdnUrls, drawSkeleton, featureNotSupportedMessage, fetchSkeletonDefinition, findExerciseByIdOrAlias, freeBlockedFeatures, initWebGlBackend, isPaidPlan, mapVideoKeypointsToDisplay, mountDomShell, openCameraStream, resolveExerciseName, resolveFeatures, resolvePoseDetection, resolvePoseModel, resolveTf, shouldShowWatermark, stopMediaStream, toClassicNativeMessage };
+export { type AccelerationDiagnostics, type AccelerationState, type AngleValue, type AnglesEvent, BRAND_COLORS, BlazePoseAdapter, type BlazePoseModelType, COCO_KEYPOINT_NAMES, COMBINED_REFERENCE_EXERCISE_MESSAGE, type CameraFacingMode, type CameraOptions, type CameraPoseSource, type ClassicMessageListener, type ClassicNativeMessage, type CocoKeypointName, type ColdStartMode, ConfigureError, type ConfigureOptions, type ConfigureRequest, type CounterEvent, type CounterFormScore, type CustomExerciseDescriptor, type CustomSessionOptions, DEFAULT_BASE_URL, DEFAULT_FEATURES, DEFAULT_LOADING_TEXT, DEFAULT_MOVENET_LIGHTNING_URL, DEFAULT_SKELETON, DEFAULT_SKELETON_DEFINITION, type DetectorFrameResult, type DiagnosticListener, EXERCISE_ALIASES, type EngineBundleDescriptor, type EngineChannel, type EngineLoadResult, EngineLoader, type EngineLoaderOptions, type EngineSession, type EngineSessionFeatures, type EngineSessionOptions, type ErrorEvent, type EventSink, type ExerciseConfig, type ExerciseSummaryEvent, FREE_PLAN_FEATURES_MESSAGE, type FetchSkeletonOptions, type FileStore, type FormGrade, type FormScoreEvent, INVALID_TOKEN_MESSAGE, type ImagePoseSource, type InferenceRuntime, type InitialPosture, type InitializationEvent, type InitializationStep, type JumpCalibrationEvent, type JumpDiscardedEvent, type JumpHeightEvent, type JumpResultEvent, type JumpStartedEvent, type JumpSummaryEntry, type JumpSummaryEvent, type KeyValueStore, type Keypoint, type KeypointName, type KeypointsEvent, type MinGrade, type ModelDescriptor, type ModelFormat, type ModelsByProfile, MoveNetGraphAdapter, type MovementDefinition, type MovementRecommendation, type MovementStep, type MovementStepParameter, ONLINE_POSE_DETECTION_VERSION, ONLINE_TFJS_VERSION, POSETRACKER_LOGO_URL, type PerformanceWarningEvent, type PlanInfo, type Pose, PoseCamera, PoseCamera_mount, type PoseDetectionModule, type PoseDetectorAdapter, type PoseEstimateInput, type PoseModelAlias, type PoseModelKind, type PoseModelProfile, type PoseRuntimeDescriptor, type PoseRuntimePartDescriptor, type PoseSource, type PoseSourceProps, type PoseSourceType, type PoseTracker, type PoseTrackerCallbacks, PoseTrackerClient, type PoseTrackerClientOptions, type PoseTrackerEngine, type PoseTrackerEvent, type PoseTrackerEventListener, type PoseTrackerEventType, type PoseTrackerFeatures, type PoseTrackerMode, type PoseTrackerOptions, type PoseTrackerStatus, type PostureEvent, type PreloadOptions, type ProgressionEvent, type QualityChangedEvent, type RecommendationsEvent, type ReferenceMovementDescriptor, type RepSummary, type ResolvePoseModelOptions, type ResolvedFeatures, type ResolvedPoseModel, type RuntimeDownloadProgressEvent, SDK_NAME, SDK_VERSION, type SdkManifest, type SkeletonAnglesStyle, type SkeletonCirclesStyle, type SkeletonDefinition, SkeletonFetchError, type SkeletonLinesStyle, type StartExerciseOptions, type StatsEvent, type TfModule, type TfjsCdnUrls, TrackError, type TrackRequest, type TrackResponse, UsageTracker, type UsageTrackerOptions, V4_ONLY_EXERCISE_IDS, type VideoPoseSource, type VisibleHipsInfo, type WarningEvent, configure, createLocalFileStore, createLocalKeyValueStore, createMemoryFileStore, createPoseAdapter, createPoseTracker, createWebFileStore, defaultPoseDetectionCdnUrl, defaultTfjsCdnUrls, drawSkeleton, featureNotSupportedMessage, fetchSkeletonDefinition, findExerciseByIdOrAlias, freeBlockedFeatures, initWebGlBackend, isPaidPlan, mapVideoKeypointsToDisplay, mountDomShell, normalizeEngineChannel, normalizePoseSource, openCameraStream, requiresEngineV4, resolveExerciseName, resolveFeatures, resolvePoseDetection, resolvePoseModel, resolveTf, shouldShowWatermark, stopMediaStream, toClassicNativeMessage };