@viji-dev/core 0.9.0 → 0.11.0

package/dist/index.d.ts

@@ -422,6 +422,16 @@ export declare class AudioSystem {
      * Tags with streamIndex and conditionally includes beat data (main channel only).
      */
     private sendChannelResults;
+    /**
+     * Deliver an analysis message to one sink with PER-SINK typed-array
+     * copies. The host→worker relay transfers `frequencyData`/`waveformData`
+     * buffers (zero-copy receive, `collectTransferables` case
+     * 'audio-analysis-update') — posting the same message object to N sinks
+     * would let the first post DETACH the buffers and starve every other
+     * sink (and the replay cache). One copy per sink per analysis frame is
+     * the unavoidable cost of a transfer-per-receiver delivery.
+     */
+    private deliverAnalysisMessage;
     /**
      * Register a result-delivery sink (one per attached core's worker) and
      * replay current state to it in FIFO order: the latest main-channel
@@ -453,6 +463,41 @@ export declare class AudioSystem {
      * context alive in `Pending activities` and leaks ~4 MB per cycle.
      */
     resetAudioState(): Promise<void>;
+    private sourceFedIndices;
+    /**
+     * Register a source-fed device channel (no MediaStream). Emits the
+     * `audio-stream-setup` identity to all sinks and starts the staleness
+     * timer. Idempotent.
+     */
+    addDeviceAnalysisChannel(streamIndex: number, deviceId: string): void;
+    /**
+     * Feed a controller analysis packet into a source-fed channel. Instant
+     * values pass through; the smoothed companions are derived host-side via
+     * the channel's envelope followers to absorb network packet bunching
+     * (§3.9 light smoothing). Missing `frequencyData`/`waveform` surface as
+     * empty arrays (the controller omits them unless the scene reads them).
+     */
+    feedDeviceAnalysis(streamIndex: number, payload: {
+        volume?: {
+            current?: number;
+            peak?: number;
+        };
+        bands?: {
+            low?: number;
+            lowMid?: number;
+            mid?: number;
+            highMid?: number;
+            high?: number;
+        };
+        spectral?: {
+            brightness?: number;
+            flatness?: number;
+        };
+        frequencyData?: Uint8Array;
+        waveformData?: Float32Array;
+    }): void;
+    /** Remove a source-fed device channel (sends a disconnected state). */
+    removeDeviceAnalysisChannel(streamIndex: number): void;
     /**
      * Add an additional audio stream (lightweight analysis, no beat detection).
      * When `meta` is provided, the stream identity is recorded and an
@@ -996,6 +1041,59 @@ export declare type CVFeatureKey = typeof CV_FEATURE_KEYS[number];
  */
 declare type CVFrameRateMode = 'full' | 'half' | 'quarter' | 'eighth';
 
+/**
+ * A controller-produced analysis packet ingested via
+ * {@link VijiCore.setDeviceAnalysis} (remote analysis, Phase 3). The
+ * controller analyzes its own device's audio/camera and ships results — no
+ * media track for the analyzed signal crosses the wire.
+ *
+ * - `kind: 'audio'` — surfaces on `viji.devices[n].audio`. `payload` carries
+ *   instant `volume`/`bands`/`spectral`; the host derives the smoothed
+ *   companions and absorbs network jitter. Omit `frequencyData`/`waveform`
+ *   unless the device scene reads them (they are ~9 KB each).
+ * - `kind: 'cv'` — surfaces on `viji.devices[n].video.cv` (requires the
+ *   device video track to be sent for display — the CV surface carrier).
+ *   `payload` carries dequantized `faces`/`hands`/`pose` in the same shapes
+ *   as `viji.video.cv`. Segmentation is excluded in v1.
+ *
+ * `seq` is a monotonic per-(device,kind) counter for latest-wins ordering on
+ * an unreliable channel (every packet carries absolute values, so drops are
+ * harmless). `captureTs` is the capture timestamp of the analyzed media.
+ */
+export declare type DeviceAnalysisPacket = {
+    kind: 'audio';
+    seq?: number;
+    captureTs?: number;
+    payload: {
+        volume?: {
+            current?: number;
+            peak?: number;
+        };
+        bands?: {
+            low?: number;
+            lowMid?: number;
+            mid?: number;
+            highMid?: number;
+            high?: number;
+        };
+        spectral?: {
+            brightness?: number;
+            flatness?: number;
+        };
+        frequencyData?: Uint8Array;
+        waveformData?: Float32Array;
+    };
+} | {
+    kind: 'cv';
+    seq?: number;
+    captureTs?: number;
+    payload: {
+        faces?: FaceData[];
+        hands?: HandData[];
+        pose?: PoseData | null;
+    };
+};
+
 /**
  * Device motion data sourced from the platform's `DeviceMotionEvent`. All
  * accelerations are in metres per second squared (m/s²); rotation rates are
@@ -2692,7 +2790,7 @@ declare type TrackingState = 'TRACKING' | 'LOCKED' | 'BREAKDOWN' | 'LOST';
 /** Returned by `on*` listener registration calls; invoke to unsubscribe. */
 export declare type Unsubscribe = () => void;
 
-export declare const VERSION = "0.9.0";
+export declare const VERSION = "0.11.0";
 
 /**
  * Real-time video API: drawable frame, dimensions, and the source-side
@@ -2741,11 +2839,12 @@ export declare interface VideoAPI {
  * outputs (the analysed frame and its detection / segmentation results) with
  * the verbs that activate the underlying MediaPipe pipelines.
  *
- * Available only on the main video stream (`viji.video.cv`). Additional
- * streams (`viji.videoStreams[i].cv`) and device videos
- * (`viji.devices[i].video.cv`) do not run CV; their `cv` surface exists for
- * API consistency but the data fields stay at their empty defaults and the
- * `enable*` verbs are no-ops.
+ * Available on the main video stream (`viji.video.cv`) and on additional
+ * streams (`viji.videoStreams[i].cv`) — same surface, same verbs, activated
+ * per stream. `analysedFrame` is populated on the MAIN stream only; on other
+ * streams it stays `null` (use `currentFrame`). Device videos
+ * (`viji.devices[i].video.cv`) do not run CV yet; their `cv` surface exists
+ * for API consistency with the data fields at their empty defaults.
  */
 export declare interface VideoCVAPI {
     /**
@@ -3079,6 +3178,7 @@ export declare class VijiCore {
     private sceneRenderer;
     private sceneCapabilities;
     private sceneP5CanvasMode?;
+    private sceneConsumesAnalysedFrame;
     private sceneContextType?;
     private parametersInitialized;
     private parameterListeners;
@@ -3088,6 +3188,7 @@ export declare class VijiCore {
     private stateImportErrorListeners;
     private sceneRuntimeErrorListeners;
     private sceneConsoleListeners;
+    private deviceCVRequestListeners;
     private stats;
     private performanceSnapshotListeners;
     private capabilityTransitionListeners;
@@ -3501,8 +3602,14 @@ export declare class VijiCore {
      */
     setFrameRate(mode: FrameRateMode): Promise<void>;
     /**
-     * Sets the CV processing frame rate mode (relative to scene frame rate)
-     * @param mode - CV processing rate: 'full', 'half', 'quarter', or 'eighth' of scene rate
+     * Sets the CV processing frame rate mode, relative to the **video stream
+     * delivery rate** (the capture's ~30 fps), not the scene render rate —
+     * with shared engines, attached scenes may render at different rates, so
+     * the capture rate is the only coherent base.
+     *
+     * Engine-level write: visible to ALL cores sharing this core's media
+     * engines (same rule as the stream mutators).
+     * @param mode - CV processing rate: 'full', 'half', 'quarter', or 'eighth' of the capture rate
      */
     setCVFrameRate(mode: CVFrameRateMode): Promise<void>;
     /**
@@ -3604,6 +3711,51 @@ export declare class VijiCore {
      */
     private get audioEngine();
     private ensureAudioEngine;
+    /**
+     * Host-side video control namespace. The `cv` sub-namespace lets host
+     * code (as opposed to scene code) drive CV on the main video stream —
+     * the controller use case: a scene-less headless core analyzing its
+     * local camera and shipping results onward.
+     */
+    readonly video: {
+        cv: {
+            /**
+             * Enable or disable a CV feature on the main video stream from the
+             * host side. The host acts as one more consumer of the shared CV
+             * engine: inference runs the union of all consumers' desired sets,
+             * and host results arrive via {@link onResults} filtered to the
+             * host-enabled features.
+             *
+             * Engine-level write: the underlying inference is visible to ALL
+             * cores sharing this core's media engines (scenes still only receive
+             * features they enabled themselves).
+             *
+             * @param feature - The CV feature to toggle
+             * @param enabled - Whether the feature should run
+             */
+            enable: (feature: CVFeature, enabled: boolean) => void;
+            /**
+             * Subscribe to CV results on the host side, filtered to the features
+             * enabled via {@link enable}. Fires once per completed inference with
+             * the same result shapes the artist API exposes
+             * (`faces`/`hands`/`pose`/`segmentation`).
+             *
+             * Engine-level: one listener registry per shared engine group —
+             * subscribe once per app.
+             *
+             * @returns Unsubscribe function. Call to remove the listener.
+             */
+            onResults: (listener: (update: {
+                streamIndex: number;
+                results: {
+                    faces?: FaceData[];
+                    hands?: HandData[];
+                    pose?: PoseData | null;
+                    segmentation?: SegmentationData | null;
+                };
+            }) => void) => Unsubscribe;
+        };
+    };
     readonly audio: {
         /**
          * Beat control namespace
@@ -3911,6 +4063,13 @@ export declare class VijiCore {
      * Gets current performance statistics
      */
     getStats(): PerformanceStats;
+    /**
+     * Merge the shared CV engine's main-stream processing stats into
+     * `stats.cv`. The cv block is absent until CV state exists (no engine, or
+     * no features ever requested on the main stream) — same shape consumers
+     * saw when the worker produced it.
+     */
+    private mergeCVStats;
     /**
      * Pauses the scene render loop — stops the worker from rendering. Intended to
      * be driven from the host on `document.hidden`, so a hidden tab doesn't keep
@@ -3991,7 +4150,8 @@ export declare class VijiCore {
      */
     removeExternalDevice(deviceId: string): void;
     /**
-     * Get list of connected external devices (for host UI)
+     * Get list of connected external devices (for host UI). Engine-scoped:
+     * identical for every core sharing this core's media engines.
      * @returns Array of device metadata with id, name, and array index
      */
     getConnectedDevices(): Array<{
@@ -3999,6 +4159,32 @@ export declare class VijiCore {
         name: string;
         index: number;
     }>;
+    /**
+     * Ingest a controller-produced analysis packet for a registered device
+     * (remote analysis, Phase 3). `kind:'audio'` surfaces on
+     * `viji.devices[n].audio`; `kind:'cv'` surfaces on
+     * `viji.devices[n].video.cv` (requires the device video track to be sent
+     * for display — the CV surface carrier). Latest-wins by `seq`; unknown
+     * devices / malformed packets are dropped silently. Engine-level: visible
+     * to all cores sharing this core's media engines.
+     *
+     * @param deviceId Registered device identifier
+     * @param packet `{ kind, seq?, captureTs?, payload }`
+     */
+    setDeviceAnalysis(deviceId: string, packet: DeviceAnalysisPacket): void;
+    /**
+     * Listen for device CV feature requests. Fires when a host scene toggles a
+     * CV feature on a device stream (`viji.devices[n].video.cv.enableX`) —
+     * device CV is always remote, so the app must relay this to the device's
+     * controller over its own transport (the controller runs the inference and
+     * ships results back via {@link setDeviceAnalysis}). The core never owns
+     * the transport.
+     *
+     * Engine-level: one registry per shared engine group — subscribe once.
+     * @returns Unsubscribe function.
+     */
+    onDeviceCVRequest(listener: (deviceId: string, feature: CVFeature, enabled: boolean) => void): Unsubscribe;
+    private fireDeviceCVRequest;
     /**
      * Set video stream for an external device
      * Device camera will appear in viji.devices[].video

package/dist/index.js

@@ -1,4 +1,4 @@
-import { A as s, C as o, I as E, V as i, a as A, b as C } from "./index-CAx7Pxyl.js";
+import { A as s, C as o, I as E, V as i, a as A, b as C } from "./index-Dfhuv8fL.js";
 export {
   s as AudioSystem,
   o as CV_FEATURE_KEYS,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@viji-dev/core",
-  "version": "0.9.0",
+  "version": "0.11.0",
   "description": "Universal execution engine for Viji Creative scenes",
   "type": "module",
   "main": "./dist/index.js",
@@ -50,7 +50,7 @@
   "scripts": {
     "prebuild": "node scripts/prebuild-clean.mjs && node scripts/sync-version.mjs",
     "version": "npm run build && git add -A",
-    "build": "vite build && node scripts/copy-tasks-assets.mjs && node scripts/build-artist-types.mjs && node scripts/build-docs-api.mjs",
+    "build": "vite build && node scripts/copy-tasks-assets.mjs && node scripts/build-artist-types.mjs && node scripts/lint-artist-types-drift.mjs && node scripts/build-docs-api.mjs",
     "build:docs": "node scripts/build-docs-api.mjs",
     "lint:ai-prompts-drift": "node scripts/lint-ai-prompts-drift.mjs",
     "dev": "node scripts/sync-version.mjs && vite build --watch",