@viji-dev/core 0.4.6 → 0.5.1

package/dist/artist-global.d.ts

@@ -1211,14 +1211,16 @@ declare global {
       isEnding: boolean;
   }
 
-  const VERSION = "0.4.5";
+  const VERSION = "0.5.1";
 
   /**
-   * Real-time video API: drawable frame, dimensions, and computer-vision results
-   * (faces, hands, pose, segmentation). Activate individual CV features through `cv.*`.
+   * Real-time video API: drawable frame, dimensions, and the source-side
+   * properties of the connected stream. All computer-vision data and controls
+   * (faces, hands, pose, segmentation, the analysed frame, and feature
+   * enable/disable verbs) live on `viji.video.cv`.
    *
-   * When `isConnected` is `false`, `currentFrame` is `null`, all dimensions are `0`,
-   * and CV result fields hold their empty defaults.
+   * When `isConnected` is `false`, `currentFrame` is `null`, all dimensions are
+   * `0`, and every member of `cv` holds its empty default.
    */
   interface VideoAPI {
       /** `true` when a video stream is connected. When `false`, all other fields hold their default empty values. */
@@ -1237,6 +1239,78 @@ declare global {
        * use only when you need to read individual pixel values.
        */
       getFrameData: () => ImageData | null;
+      /**
+       * Computer-vision surface for this video stream. Holds both the data
+       * outputs (`faces`, `hands`, `pose`, `segmentation`, `analysedFrame`,
+       * `getAnalysedFrameData`) and the verbs that activate them
+       * (`enable*`/`disable*`/`getActiveFeatures`/`isProcessing`).
+       *
+       * Multiple features can be active simultaneously, but each adds CPU/GPU
+       * and WebGL-context cost; toggle only what you need.
+       *
+       * `analysedFrame` and the result fields refresh together once per host
+       * frame as an atomic snapshot: within a single `render()` call they are
+       * always paired.
+       */
+      cv: VideoCVAPI;
+  }
+
+  /**
+   * Computer-vision surface attached to a `VideoAPI`. Combines paired CV data
+   * outputs (the analysed frame and its detection / segmentation results) with
+   * the verbs that activate the underlying MediaPipe pipelines.
+   *
+   * Available only on the main video stream (`viji.video.cv`). Additional
+   * streams (`viji.videoStreams[i].cv`) and device videos
+   * (`viji.devices[i].video.cv`) do not run CV; their `cv` surface exists for
+   * API consistency but the data fields stay at their empty defaults and the
+   * `enable*` verbs are no-ops.
+   */
+  interface VideoCVAPI {
+      /**
+       * The exact video frame that produced the current `faces` / `hands` /
+       * `pose` / `segmentation` results: the frame MediaPipe actually analysed.
+       *
+       * Choose between `analysedFrame` and `viji.video.currentFrame` by asking
+       * whether the effect reads pixels from the displayed frame at CV-derived
+       * positions. Reach for `analysedFrame` when the answer is yes
+       * (compositing the segmentation mask onto the body, sampling skin tone
+       * under a face landmark, warping the face along its mesh, texture-mapped
+       * face filters): pairing the displayed frame with the CV results is
+       * required for those effects. Stay on `currentFrame` when the answer is
+       * no (drawing landmark dots, particles, debug overlays, geometry driven
+       * by CV positions): `analysedFrame` only refreshes when MediaPipe
+       * completes an inference, which is not every frame, so reaching for it
+       * without a reason makes the displayed video stutter or hold between
+       * inferences.
+       *
+       * Pairing guarantee: within a single `render()` call, `analysedFrame`
+       * is paired with the values of `faces`, `hands`, `pose`, `segmentation`
+       * read in the same call.
+       *
+       * `null` until the first CV inference completes after a feature is
+       * enabled, after the video disconnects, or after a CV-feature toggle
+       * clears state. A common pattern is
+       * `viji.video.cv.analysedFrame ?? viji.video.currentFrame` to fall back
+       * to the live feed during the brief startup window; expect a small
+       * visual hitch when the source switches as the first inference lands.
+       *
+       * Read-only: this `OffscreenCanvas` is owned by the engine. Do not
+       * mutate it (no `getContext`-and-paint, no `transferToImageBitmap`).
+       * Drawing it as a source (`ctx.drawImage`, `gl.texImage2D`,
+       * `p5.image(...)`) is the only supported usage.
+       */
+      analysedFrame: OffscreenCanvas | null;
+      /**
+       * Returns `analysedFrame` as raw RGBA `ImageData` for per-pixel CPU
+       * analysis, or `null` when no CV result has landed yet (or the stream is
+       * disconnected). Cached: re-extracted only when a new CV result arrives,
+       * so multiple readers within a render share one allocation. Slow
+       * compared to drawing `analysedFrame` directly. Use only when you need
+       * pixel values aligned with CV landmark positions (e.g. sampling skin
+       * colour at a face landmark).
+       */
+      getAnalysedFrameData: () => ImageData | null;
       /** Detected faces (empty array when face detection is off or no faces are visible). */
       faces: FaceData[];
       /** Detected hands (up to 2; empty array when hand tracking is off or no hands are visible). */
@@ -1246,50 +1320,42 @@ declare global {
       /** Body segmentation mask, or `null` when segmentation is off. */
       segmentation: SegmentationData | null;
       /**
-       * Computer-vision feature control. Each `enable*` method toggles a specific CV
-       * pipeline; results land in the corresponding `faces` / `hands` / `pose` /
-       * `segmentation` field on the next available frame. Multiple features can be
-       * active simultaneously, but each adds CPU/GPU and WebGL-context cost.
+       * Toggle face detection (bounding box, center, confidence, id).
+       * @returns Resolves once the underlying pipeline has finished switching.
        */
-      cv: {
-          /**
-           * Toggle face detection (bounding box, center, confidence, id).
-           * @returns Resolves once the underlying pipeline has finished switching.
-           */
-          enableFaceDetection(enabled: boolean): Promise<void>;
-          /**
-           * Toggle 468-point face mesh + head pose (`pitch`, `yaw`, `roll`). Implies
-           * face detection.
-           * @returns Resolves once the pipeline has finished switching.
-           */
-          enableFaceMesh(enabled: boolean): Promise<void>;
-          /**
-           * Toggle 7 expressions + 52 ARKit blendshape coefficients on each face.
-           * Implies face mesh.
-           * @returns Resolves once the pipeline has finished switching.
-           */
-          enableEmotionDetection(enabled: boolean): Promise<void>;
-          /**
-           * Toggle 21-point hand landmarks + ML-classified gesture confidences for up
-           * to two hands.
-           * @returns Resolves once the pipeline has finished switching.
-           */
-          enableHandTracking(enabled: boolean): Promise<void>;
-          /**
-           * Toggle 33-point BlazePose body landmarks (face / torso / arms / legs).
-           * @returns Resolves once the pipeline has finished switching.
-           */
-          enablePoseDetection(enabled: boolean): Promise<void>;
-          /**
-           * Toggle per-pixel person/background segmentation mask.
-           * @returns Resolves once the pipeline has finished switching.
-           */
-          enableBodySegmentation(enabled: boolean): Promise<void>;
-          /** Returns the list of CV features currently enabled on this stream. */
-          getActiveFeatures(): CVFeature[];
-          /** Returns `true` if the CV worker is actively processing frames. */
-          isProcessing(): boolean;
-      };
+      enableFaceDetection(enabled: boolean): Promise<void>;
+      /**
+       * Toggle 468-point face mesh + head pose (`pitch`, `yaw`, `roll`). Implies
+       * face detection.
+       * @returns Resolves once the pipeline has finished switching.
+       */
+      enableFaceMesh(enabled: boolean): Promise<void>;
+      /**
+       * Toggle 7 expressions + 52 ARKit blendshape coefficients on each face.
+       * Implies face mesh.
+       * @returns Resolves once the pipeline has finished switching.
+       */
+      enableEmotionDetection(enabled: boolean): Promise<void>;
+      /**
+       * Toggle 21-point hand landmarks + ML-classified gesture confidences for up
+       * to two hands.
+       * @returns Resolves once the pipeline has finished switching.
+       */
+      enableHandTracking(enabled: boolean): Promise<void>;
+      /**
+       * Toggle 33-point BlazePose body landmarks (face / torso / arms / legs).
+       * @returns Resolves once the pipeline has finished switching.
+       */
+      enablePoseDetection(enabled: boolean): Promise<void>;
+      /**
+       * Toggle per-pixel person/background segmentation mask.
+       * @returns Resolves once the pipeline has finished switching.
+       */
+      enableBodySegmentation(enabled: boolean): Promise<void>;
+      /** Returns the list of CV features currently enabled on this stream. */
+      getActiveFeatures(): CVFeature[];
+      /** Returns `true` if the CV worker is actively processing frames. */
+      isProcessing(): boolean;
   }
 
   /**

package/dist/artist-jsdoc.d.ts

@@ -265,7 +265,7 @@
  */
 
 /**
- * Real-time video API: drawable frame, dimensions, and computer-vision results (faces, hands, pose, segmentation). Activate individual CV features through `cv.*`. When `isConnected` is `false`, `currentFrame` is `null`, all dimensions are `0`, and CV result fields hold their empty defaults.
+ * Real-time video API: drawable frame, dimensions, and the source-side properties of the connected stream. All computer-vision data and controls (faces, hands, pose, segmentation, the analysed frame, and feature enable/disable verbs) live on `viji.video.cv`. When `isConnected` is `false`, `currentFrame` is `null`, all dimensions are `0`, and every member of `cv` holds its empty default.
  * @typedef {Object} VideoAPI
  * @property {boolean} isConnected - `true` when a video stream is connected. When `false`, all other fields hold their default empty values.
  * @property {OffscreenCanvas | ImageBitmap |null} currentFrame - Latest video frame, drawable directly with `ctx.drawImage()`. `null` when disconnected.
@@ -273,11 +273,7 @@
  * @property {number} frameHeight - Source video frame height in pixels. `0` when disconnected.
  * @property {number} frameRate - Source video frame rate in Hz (frames per second). `0` when disconnected.
  * @property {() => ImageData |null} getFrameData - Returns the latest frame as raw RGBA `ImageData` for per-pixel CPU analysis, or `null` when disconnected. Slow compared to drawing `currentFrame` directly; use only when you need to read individual pixel values.
- * @property {FaceData[]} faces - Detected faces (empty array when face detection is off or no faces are visible).
- * @property {HandData[]} hands - Detected hands (up to 2; empty array when hand tracking is off or no hands are visible).
- * @property {PoseData |null} pose - Detected body pose, or `null` when pose detection is off or no pose is visible.
- * @property {SegmentationData |null} segmentation - Body segmentation mask, or `null` when segmentation is off.
- * @property {Object} cv - Computer-vision feature control. Each `enable*` method toggles a specific CV pipeline; results land in the corresponding `faces` / `hands` / `pose` / `segmentation` field on the next available frame. Multiple features can be active simultaneously, but each adds CPU/GPU and WebGL-context cost.
+ * @property {VideoCVAPI} cv - Computer-vision surface for this video stream. Holds both the data outputs (`faces`, `hands`, `pose`, `segmentation`, `analysedFrame`, `getAnalysedFrameData`) and the verbs that activate them (`enable*`/`disable*`/`getActiveFeatures`/`isProcessing`). Multiple features can be active simultaneously, but each adds CPU/GPU and WebGL-context cost; toggle only what you need. `analysedFrame` and the result fields refresh together once per host frame as an atomic snapshot: within a single `render()` call they are always paired.
  */
 
 /**