@viji-dev/core 0.4.6 → 0.5.0

package/dist/index.d.ts

@@ -1863,14 +1863,16 @@ declare interface TouchPoint {
  */
 declare type TrackingState = 'TRACKING' | 'LOCKED' | 'BREAKDOWN' | 'LOST';
 
-export declare const VERSION = "0.3.29";
+export declare const VERSION = "0.5.0";
 
 /**
- * Real-time video API: drawable frame, dimensions, and computer-vision results
- * (faces, hands, pose, segmentation). Activate individual CV features through `cv.*`.
+ * Real-time video API: drawable frame, dimensions, and the source-side
+ * properties of the connected stream. All computer-vision data and controls
+ * (faces, hands, pose, segmentation, the analysed frame, and feature
+ * enable/disable verbs) live on `viji.video.cv`.
  *
- * When `isConnected` is `false`, `currentFrame` is `null`, all dimensions are `0`,
- * and CV result fields hold their empty defaults.
+ * When `isConnected` is `false`, `currentFrame` is `null`, all dimensions are
+ * `0`, and every member of `cv` holds its empty default.
  */
 export declare interface VideoAPI {
     /** `true` when a video stream is connected. When `false`, all other fields hold their default empty values. */
@@ -1889,6 +1891,70 @@ export declare interface VideoAPI {
      * use only when you need to read individual pixel values.
      */
     getFrameData: () => ImageData | null;
+    /**
+     * Computer-vision surface for this video stream. Holds both the data
+     * outputs (`faces`, `hands`, `pose`, `segmentation`, `analysedFrame`,
+     * `getAnalysedFrameData`) and the verbs that activate them
+     * (`enable*`/`disable*`/`getActiveFeatures`/`isProcessing`).
+     *
+     * Multiple features can be active simultaneously, but each adds CPU/GPU
+     * and WebGL-context cost — toggle only what you need.
+     *
+     * `analysedFrame` and the result fields refresh together once per host
+     * frame (atomic snapshot) — within a single `render()` call they are
+     * always paired.
+     */
+    cv: VideoCVAPI;
+}
+
+/**
+ * Computer-vision surface attached to a `VideoAPI`. Combines paired CV data
+ * outputs (the analysed frame and its detection / segmentation results) with
+ * the verbs that activate the underlying MediaPipe pipelines.
+ *
+ * Available only on the main video stream (`viji.video.cv`). Additional
+ * streams (`viji.videoStreams[i].cv`) and device videos
+ * (`viji.devices[i].video.cv`) do not run CV — their `cv` surface exists for
+ * API consistency but the data fields stay at their empty defaults and the
+ * `enable*` verbs are no-ops.
+ */
+export declare interface VideoCVAPI {
+    /**
+     * The exact video frame that produced the current `faces` / `hands` /
+     * `pose` / `segmentation` results — the frame MediaPipe actually analysed.
+     * Use this (not `viji.video.currentFrame`) when overlaying pixel-precise
+     * CV-driven effects (face-mesh masks, body-segmentation compositing,
+     * makeup, distortion). `currentFrame` is the just-arrived frame and runs
+     * ~15–40 ms ahead of the analysis; `analysedFrame` is the frame the
+     * landmarks/mask actually correspond to, so the two stay locked under
+     * fast head/body motion.
+     *
+     * Pairing guarantee: within a single `render()` call, `analysedFrame`
+     * is paired with the values of `faces`, `hands`, `pose`, `segmentation`
+     * you read in the same call. They refresh together once per host frame.
+     *
+     * `null` until the first CV inference completes after a feature is
+     * enabled, after the video disconnects, or after a CV-feature toggle
+     * clears state. A common pattern is
+     * `viji.video.cv.analysedFrame ?? viji.video.currentFrame` to fall back
+     * to the live feed during the brief startup window.
+     *
+     * Read-only: this `OffscreenCanvas` is owned by the engine. Do not
+     * mutate it (no `getContext`-and-paint, no `transferToImageBitmap`).
+     * Drawing it as a source (`ctx.drawImage`, `gl.texImage2D`,
+     * `p5.image(...)`) is the only supported usage.
+     */
+    analysedFrame: OffscreenCanvas | null;
+    /**
+     * Returns `analysedFrame` as raw RGBA `ImageData` for per-pixel CPU
+     * analysis, or `null` when no CV result has landed yet (or the stream is
+     * disconnected). Cached: re-extracted only when a new CV result arrives,
+     * so multiple readers within a render share one allocation. Slow
+     * compared to drawing `analysedFrame` directly — use only when you need
+     * pixel values aligned with CV landmark positions (e.g. sampling skin
+     * colour at a face landmark).
+     */
+    getAnalysedFrameData: () => ImageData | null;
     /** Detected faces (empty array when face detection is off or no faces are visible). */
     faces: FaceData[];
     /** Detected hands (up to 2; empty array when hand tracking is off or no hands are visible). */
@@ -1898,50 +1964,42 @@ export declare interface VideoAPI {
     /** Body segmentation mask, or `null` when segmentation is off. */
     segmentation: SegmentationData | null;
     /**
-     * Computer-vision feature control. Each `enable*` method toggles a specific CV
-     * pipeline; results land in the corresponding `faces` / `hands` / `pose` /
-     * `segmentation` field on the next available frame. Multiple features can be
-     * active simultaneously, but each adds CPU/GPU and WebGL-context cost.
+     * Toggle face detection (bounding box, center, confidence, id).
+     * @returns Resolves once the underlying pipeline has finished switching.
      */
-    cv: {
-        /**
-         * Toggle face detection (bounding box, center, confidence, id).
-         * @returns Resolves once the underlying pipeline has finished switching.
-         */
-        enableFaceDetection(enabled: boolean): Promise<void>;
-        /**
-         * Toggle 468-point face mesh + head pose (`pitch`, `yaw`, `roll`). Implies
-         * face detection.
-         * @returns Resolves once the pipeline has finished switching.
-         */
-        enableFaceMesh(enabled: boolean): Promise<void>;
-        /**
-         * Toggle 7 expressions + 52 ARKit blendshape coefficients on each face.
-         * Implies face mesh.
-         * @returns Resolves once the pipeline has finished switching.
-         */
-        enableEmotionDetection(enabled: boolean): Promise<void>;
-        /**
-         * Toggle 21-point hand landmarks + ML-classified gesture confidences for up
-         * to two hands.
-         * @returns Resolves once the pipeline has finished switching.
-         */
-        enableHandTracking(enabled: boolean): Promise<void>;
-        /**
-         * Toggle 33-point BlazePose body landmarks (face / torso / arms / legs).
-         * @returns Resolves once the pipeline has finished switching.
-         */
-        enablePoseDetection(enabled: boolean): Promise<void>;
-        /**
-         * Toggle per-pixel person/background segmentation mask.
-         * @returns Resolves once the pipeline has finished switching.
-         */
-        enableBodySegmentation(enabled: boolean): Promise<void>;
-        /** Returns the list of CV features currently enabled on this stream. */
-        getActiveFeatures(): CVFeature[];
-        /** Returns `true` if the CV worker is actively processing frames. */
-        isProcessing(): boolean;
-    };
+    enableFaceDetection(enabled: boolean): Promise<void>;
+    /**
+     * Toggle 468-point face mesh + head pose (`pitch`, `yaw`, `roll`). Implies
+     * face detection.
+     * @returns Resolves once the pipeline has finished switching.
+     */
+    enableFaceMesh(enabled: boolean): Promise<void>;
+    /**
+     * Toggle 7 expressions + 52 ARKit blendshape coefficients on each face.
+     * Implies face mesh.
+     * @returns Resolves once the pipeline has finished switching.
+     */
+    enableEmotionDetection(enabled: boolean): Promise<void>;
+    /**
+     * Toggle 21-point hand landmarks + ML-classified gesture confidences for up
+     * to two hands.
+     * @returns Resolves once the pipeline has finished switching.
+     */
+    enableHandTracking(enabled: boolean): Promise<void>;
+    /**
+     * Toggle 33-point BlazePose body landmarks (face / torso / arms / legs).
+     * @returns Resolves once the pipeline has finished switching.
+     */
+    enablePoseDetection(enabled: boolean): Promise<void>;
+    /**
+     * Toggle per-pixel person/background segmentation mask.
+     * @returns Resolves once the pipeline has finished switching.
+     */
+    enableBodySegmentation(enabled: boolean): Promise<void>;
+    /** Returns the list of CV features currently enabled on this stream. */
+    getActiveFeatures(): CVFeature[];
+    /** Returns `true` if the CV worker is actively processing frames. */
+    isProcessing(): boolean;
 }
 
 /**

package/dist/index.js

@@ -1,4 +1,4 @@
-import { A, V, a, b } from "./index-Oj_9v8r4.js";
+import { A, V, a, b } from "./index-Cp9G0z4E.js";
 export {
   A as AudioSystem,
   V as VERSION,

package/dist/shader-uniforms.js

@@ -374,6 +374,16 @@ export const shaderUniforms = {
     "category": "Video",
     "description": "Video frame rate in frames per second"
   },
+  "u_videoAnalysed": {
+    "type": "sampler2D",
+    "category": "Video",
+    "description": "The exact frame MediaPipe analysed; pair with u_face*/u_hand*/u_pose* uniforms for pixel-precise CV-driven effects (face mesh masks, segmentation overlays). Shares u_videoResolution."
+  },
+  "u_videoAnalysedAvailable": {
+    "type": "bool",
+    "category": "Video",
+    "description": "True after the first CV result lands; before that, u_videoAnalysed samples a 1x1 black fallback. Use to gate effects that require valid analysed pixels."
+  },
   "u_videoStreamCount": {
     "type": "int",
     "category": "Video",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@viji-dev/core",
-  "version": "0.4.6",
+  "version": "0.5.0",
   "description": "Universal execution engine for Viji Creative scenes",
   "type": "module",
   "main": "./dist/index.js",