@viji-dev/core 0.4.5 → 0.5.0

package/dist/index.d.ts

@@ -1863,14 +1863,16 @@ declare interface TouchPoint {
  */
 declare type TrackingState = 'TRACKING' | 'LOCKED' | 'BREAKDOWN' | 'LOST';
 
-export declare const VERSION = "0.3.29";
+export declare const VERSION = "0.5.0";
 
 /**
- * Real-time video API: drawable frame, dimensions, and computer-vision results
- * (faces, hands, pose, segmentation). Activate individual CV features through `cv.*`.
+ * Real-time video API: drawable frame, dimensions, and the source-side
+ * properties of the connected stream. All computer-vision data and controls
+ * (faces, hands, pose, segmentation, the analysed frame, and feature
+ * enable/disable verbs) live on `viji.video.cv`.
  *
- * When `isConnected` is `false`, `currentFrame` is `null`, all dimensions are `0`,
- * and CV result fields hold their empty defaults.
+ * When `isConnected` is `false`, `currentFrame` is `null`, all dimensions are
+ * `0`, and every member of `cv` holds its empty default.
  */
 export declare interface VideoAPI {
     /** `true` when a video stream is connected. When `false`, all other fields hold their default empty values. */
@@ -1889,6 +1891,70 @@ export declare interface VideoAPI {
      * use only when you need to read individual pixel values.
      */
     getFrameData: () => ImageData | null;
+    /**
+     * Computer-vision surface for this video stream. Holds both the data
+     * outputs (`faces`, `hands`, `pose`, `segmentation`, `analysedFrame`,
+     * `getAnalysedFrameData`) and the verbs that activate them
+     * (`enable*`/`disable*`/`getActiveFeatures`/`isProcessing`).
+     *
+     * Multiple features can be active simultaneously, but each adds CPU/GPU
+     * and WebGL-context cost — toggle only what you need.
+     *
+     * `analysedFrame` and the result fields refresh together once per host
+     * frame (atomic snapshot) — within a single `render()` call they are
+     * always paired.
+     */
+    cv: VideoCVAPI;
+}
+
+/**
+ * Computer-vision surface attached to a `VideoAPI`. Combines paired CV data
+ * outputs (the analysed frame and its detection / segmentation results) with
+ * the verbs that activate the underlying MediaPipe pipelines.
+ *
+ * Available only on the main video stream (`viji.video.cv`). Additional
+ * streams (`viji.videoStreams[i].cv`) and device videos
+ * (`viji.devices[i].video.cv`) do not run CV — their `cv` surface exists for
+ * API consistency but the data fields stay at their empty defaults and the
+ * `enable*` verbs are no-ops.
+ */
+export declare interface VideoCVAPI {
+    /**
+     * The exact video frame that produced the current `faces` / `hands` /
+     * `pose` / `segmentation` results — the frame MediaPipe actually analysed.
+     * Use this (not `viji.video.currentFrame`) when overlaying pixel-precise
+     * CV-driven effects (face-mesh masks, body-segmentation compositing,
+     * makeup, distortion). `currentFrame` is the just-arrived frame and runs
+     * ~15–40 ms ahead of the analysis; `analysedFrame` is the frame the
+     * landmarks/mask actually correspond to, so the two stay locked under
+     * fast head/body motion.
+     *
+     * Pairing guarantee: within a single `render()` call, `analysedFrame`
+     * is paired with the values of `faces`, `hands`, `pose`, `segmentation`
+     * you read in the same call. They refresh together once per host frame.
+     *
+     * `null` until the first CV inference completes after a feature is
+     * enabled, after the video disconnects, or after a CV-feature toggle
+     * clears state. A common pattern is
+     * `viji.video.cv.analysedFrame ?? viji.video.currentFrame` to fall back
+     * to the live feed during the brief startup window.
+     *
+     * Read-only: this `OffscreenCanvas` is owned by the engine. Do not
+     * mutate it (no `getContext`-and-paint, no `transferToImageBitmap`).
+     * Drawing it as a source (`ctx.drawImage`, `gl.texImage2D`,
+     * `p5.image(...)`) is the only supported usage.
+     */
+    analysedFrame: OffscreenCanvas | null;
+    /**
+     * Returns `analysedFrame` as raw RGBA `ImageData` for per-pixel CPU
+     * analysis, or `null` when no CV result has landed yet (or the stream is
+     * disconnected). Cached: re-extracted only when a new CV result arrives,
+     * so multiple readers within a render share one allocation. Slow
+     * compared to drawing `analysedFrame` directly — use only when you need
+     * pixel values aligned with CV landmark positions (e.g. sampling skin
+     * colour at a face landmark).
+     */
+    getAnalysedFrameData: () => ImageData | null;
     /** Detected faces (empty array when face detection is off or no faces are visible). */
     faces: FaceData[];
     /** Detected hands (up to 2; empty array when hand tracking is off or no hands are visible). */
@@ -1898,50 +1964,42 @@ export declare interface VideoAPI {
     /** Body segmentation mask, or `null` when segmentation is off. */
     segmentation: SegmentationData | null;
     /**
-     * Computer-vision feature control. Each `enable*` method toggles a specific CV
-     * pipeline; results land in the corresponding `faces` / `hands` / `pose` /
-     * `segmentation` field on the next available frame. Multiple features can be
-     * active simultaneously, but each adds CPU/GPU and WebGL-context cost.
+     * Toggle face detection (bounding box, center, confidence, id).
+     * @returns Resolves once the underlying pipeline has finished switching.
      */
-    cv: {
-        /**
-         * Toggle face detection (bounding box, center, confidence, id).
-         * @returns Resolves once the underlying pipeline has finished switching.
-         */
-        enableFaceDetection(enabled: boolean): Promise<void>;
-        /**
-         * Toggle 468-point face mesh + head pose (`pitch`, `yaw`, `roll`). Implies
-         * face detection.
-         * @returns Resolves once the pipeline has finished switching.
-         */
-        enableFaceMesh(enabled: boolean): Promise<void>;
-        /**
-         * Toggle 7 expressions + 52 ARKit blendshape coefficients on each face.
-         * Implies face mesh.
-         * @returns Resolves once the pipeline has finished switching.
-         */
-        enableEmotionDetection(enabled: boolean): Promise<void>;
-        /**
-         * Toggle 21-point hand landmarks + ML-classified gesture confidences for up
-         * to two hands.
-         * @returns Resolves once the pipeline has finished switching.
-         */
-        enableHandTracking(enabled: boolean): Promise<void>;
-        /**
-         * Toggle 33-point BlazePose body landmarks (face / torso / arms / legs).
-         * @returns Resolves once the pipeline has finished switching.
-         */
-        enablePoseDetection(enabled: boolean): Promise<void>;
-        /**
-         * Toggle per-pixel person/background segmentation mask.
-         * @returns Resolves once the pipeline has finished switching.
-         */
-        enableBodySegmentation(enabled: boolean): Promise<void>;
-        /** Returns the list of CV features currently enabled on this stream. */
-        getActiveFeatures(): CVFeature[];
-        /** Returns `true` if the CV worker is actively processing frames. */
-        isProcessing(): boolean;
-    };
+    enableFaceDetection(enabled: boolean): Promise<void>;
+    /**
+     * Toggle 468-point face mesh + head pose (`pitch`, `yaw`, `roll`). Implies
+     * face detection.
+     * @returns Resolves once the pipeline has finished switching.
+     */
+    enableFaceMesh(enabled: boolean): Promise<void>;
+    /**
+     * Toggle 7 expressions + 52 ARKit blendshape coefficients on each face.
+     * Implies face mesh.
+     * @returns Resolves once the pipeline has finished switching.
+     */
+    enableEmotionDetection(enabled: boolean): Promise<void>;
+    /**
+     * Toggle 21-point hand landmarks + ML-classified gesture confidences for up
+     * to two hands.
+     * @returns Resolves once the pipeline has finished switching.
+     */
+    enableHandTracking(enabled: boolean): Promise<void>;
+    /**
+     * Toggle 33-point BlazePose body landmarks (face / torso / arms / legs).
+     * @returns Resolves once the pipeline has finished switching.
+     */
+    enablePoseDetection(enabled: boolean): Promise<void>;
+    /**
+     * Toggle per-pixel person/background segmentation mask.
+     * @returns Resolves once the pipeline has finished switching.
+     */
+    enableBodySegmentation(enabled: boolean): Promise<void>;
+    /** Returns the list of CV features currently enabled on this stream. */
+    getActiveFeatures(): CVFeature[];
+    /** Returns `true` if the CV worker is actively processing frames. */
+    isProcessing(): boolean;
 }
 
 /**

package/dist/index.js

@@ -1,4 +1,4 @@
-import { A, V, a, b } from "./index-DUg1Foui.js";
+import { A, V, a, b } from "./index-Cp9G0z4E.js";
 export {
   A as AudioSystem,
   V as VERSION,

package/dist/shader-uniforms.js

@@ -374,6 +374,16 @@ export const shaderUniforms = {
     "category": "Video",
     "description": "Video frame rate in frames per second"
   },
+  "u_videoAnalysed": {
+    "type": "sampler2D",
+    "category": "Video",
+    "description": "The exact frame MediaPipe analysed; pair with u_face*/u_hand*/u_pose* uniforms for pixel-precise CV-driven effects (face mesh masks, segmentation overlays). Shares u_videoResolution."
+  },
+  "u_videoAnalysedAvailable": {
+    "type": "bool",
+    "category": "Video",
+    "description": "True after the first CV result lands; before that, u_videoAnalysed samples a 1x1 black fallback. Use to gate effects that require valid analysed pixels."
+  },
   "u_videoStreamCount": {
     "type": "int",
     "category": "Video",

package/package.json

@@ -1,97 +1,98 @@
-{
-  "name": "@viji-dev/core",
-  "version": "0.4.5",
-  "description": "Universal execution engine for Viji Creative scenes",
-  "type": "module",
-  "main": "./dist/index.js",
-  "module": "./dist/index.js",
-  "types": "./dist/index.d.ts",
-  "exports": {
-    ".": {
-      "import": "./dist/index.js",
-      "types": "./dist/index.d.ts"
-    },
-    "./artist-global": {
-      "types": "./dist/artist-global.d.ts"
-    },
-    "./artist-dts": {
-      "import": "./dist/artist-dts.js",
-      "types": "./dist/artist-dts.d.ts"
-    },
-    "./artist-jsdoc": {
-      "types": "./dist/artist-jsdoc.d.ts"
-    },
-    "./artist-js-ambient": {
-      "types": "./dist/artist-js-ambient.d.ts"
-    },
-    "./artist-dts-p5": {
-      "import": "./dist/artist-dts-p5.js",
-      "types": "./dist/artist-dts-p5.d.ts"
-    },
-    "./artist-global-p5": {
-      "types": "./dist/artist-global-p5.d.ts"
-    },
-    "./shader-uniforms": {
-      "import": "./dist/shader-uniforms.js",
-      "types": "./dist/shader-uniforms.d.ts"
-    },
-    "./docs-api": {
-      "import": "./dist/docs-api.js",
-      "types": "./dist/docs-api.d.ts"
-    }
-  },
-  "files": [
-    "dist/*.js",
-    "dist/*.d.ts",
-    "dist/*.map",
-    "dist/assets",
-    "README.md"
-  ],
-  "scripts": {
-    "prebuild": "node scripts/prebuild-clean.mjs",
-    "build": "vite build && node scripts/copy-tasks-assets.mjs && node scripts/build-artist-types.mjs && node scripts/build-docs-api.mjs",
-    "build:docs": "node scripts/build-docs-api.mjs",
-    "dev": "vite build --watch",
-    "test": "vitest",
-    "test:coverage": "vitest --coverage",
-    "type-check": "tsc --noEmit",
-    "lint": "eslint src --ext .ts"
-  },
-  "keywords": [
-    "viji",
-    "creative",
-    "webgl",
-    "audio-reactive",
-    "canvas",
-    "webworker"
-  ],
-  "author": "Viji Team",
-  "license": "SEE LICENSE IN LICENSE",
-  "devDependencies": {
-    "@types/node": "^20.0.0",
-    "@types/p5": "^1.7.6",
-    "@typescript-eslint/eslint-plugin": "^6.0.0",
-    "@typescript-eslint/parser": "^6.0.0",
-    "@vitest/coverage-v8": "^1.0.0",
-    "dts-bundle-generator": "^9.5.1",
-    "eslint": "^8.0.0",
-    "github-slugger": "^2.0.0",
-    "happy-dom": "^18.0.1",
-    "mdast-util-to-string": "^4.0.0",
-    "remark-parse": "^11.0.0",
-    "typescript": "^5.0.0",
-    "unified": "^11.0.5",
-    "vite": "^5.0.0",
-    "vite-plugin-dts": "^3.0.0",
-    "vitest": "^1.0.0"
-  },
-  "engines": {
-    "node": ">=18.0.0"
-  },
-  "dependencies": {
-    "@mediapipe/tasks-vision": "^0.10.20",
-    "essentia.js": "^0.1.3",
-    "fft.js": "^4.0.4",
-    "sucrase": "^3.35.1"
-  }
-}
+{
+  "name": "@viji-dev/core",
+  "version": "0.5.0",
+  "description": "Universal execution engine for Viji Creative scenes",
+  "type": "module",
+  "main": "./dist/index.js",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "import": "./dist/index.js",
+      "types": "./dist/index.d.ts"
+    },
+    "./artist-global": {
+      "types": "./dist/artist-global.d.ts"
+    },
+    "./artist-dts": {
+      "import": "./dist/artist-dts.js",
+      "types": "./dist/artist-dts.d.ts"
+    },
+    "./artist-jsdoc": {
+      "types": "./dist/artist-jsdoc.d.ts"
+    },
+    "./artist-js-ambient": {
+      "types": "./dist/artist-js-ambient.d.ts"
+    },
+    "./artist-dts-p5": {
+      "import": "./dist/artist-dts-p5.js",
+      "types": "./dist/artist-dts-p5.d.ts"
+    },
+    "./artist-global-p5": {
+      "types": "./dist/artist-global-p5.d.ts"
+    },
+    "./shader-uniforms": {
+      "import": "./dist/shader-uniforms.js",
+      "types": "./dist/shader-uniforms.d.ts"
+    },
+    "./docs-api": {
+      "import": "./dist/docs-api.js",
+      "types": "./dist/docs-api.d.ts"
+    }
+  },
+  "files": [
+    "dist/*.js",
+    "dist/*.d.ts",
+    "dist/*.map",
+    "dist/assets",
+    "README.md"
+  ],
+  "scripts": {
+    "prebuild": "node scripts/prebuild-clean.mjs",
+    "build": "vite build && node scripts/copy-tasks-assets.mjs && node scripts/build-artist-types.mjs && node scripts/build-docs-api.mjs",
+    "build:docs": "node scripts/build-docs-api.mjs",
+    "dev": "vite build --watch",
+    "integration": "vite --config integration-example/vite.config.ts",
+    "test": "vitest",
+    "test:coverage": "vitest --coverage",
+    "type-check": "tsc --noEmit",
+    "lint": "eslint src --ext .ts"
+  },
+  "keywords": [
+    "viji",
+    "creative",
+    "webgl",
+    "audio-reactive",
+    "canvas",
+    "webworker"
+  ],
+  "author": "Viji Team",
+  "license": "SEE LICENSE IN LICENSE",
+  "devDependencies": {
+    "@types/node": "^20.0.0",
+    "@types/p5": "^1.7.6",
+    "@typescript-eslint/eslint-plugin": "^6.0.0",
+    "@typescript-eslint/parser": "^6.0.0",
+    "@vitest/coverage-v8": "^1.0.0",
+    "dts-bundle-generator": "^9.5.1",
+    "eslint": "^8.0.0",
+    "github-slugger": "^2.0.0",
+    "happy-dom": "^18.0.1",
+    "mdast-util-to-string": "^4.0.0",
+    "remark-parse": "^11.0.0",
+    "typescript": "^5.0.0",
+    "unified": "^11.0.5",
+    "vite": "^5.0.0",
+    "vite-plugin-dts": "^3.0.0",
+    "vitest": "^1.0.0"
+  },
+  "engines": {
+    "node": ">=18.0.0"
+  },
+  "dependencies": {
+    "@mediapipe/tasks-vision": "^0.10.20",
+    "essentia.js": "^0.1.3",
+    "fft.js": "^4.0.4",
+    "sucrase": "^3.35.1"
+  }
+}