@viji-dev/core 0.8.17 → 0.10.0

package/dist/index.d.ts

@@ -307,7 +307,9 @@ export declare class AudioSystem {
     private debugLog;
     private fftSize;
     private audioStateBeat;
-    private sendAnalysisResults;
+    private analysisSinks;
+    private latestChannelResults;
+    private channelMeta;
     constructor(sendAnalysisResultsCallback?: (data: any) => void);
     /**
      * Get the current audio analysis state (for host-side usage)
@@ -420,6 +422,33 @@ export declare class AudioSystem {
      * Tags with streamIndex and conditionally includes beat data (main channel only).
      */
     private sendChannelResults;
+    /**
+     * Deliver an analysis message to one sink with PER-SINK typed-array
+     * copies. The host→worker relay transfers `frequencyData`/`waveformData`
+     * buffers (zero-copy receive, `collectTransferables` case
+     * 'audio-analysis-update') — posting the same message object to N sinks
+     * would let the first post DETACH the buffers and starve every other
+     * sink (and the replay cache). One copy per sink per analysis frame is
+     * the unavoidable cost of a transfer-per-receiver delivery.
+     */
+    private deliverAnalysisMessage;
+    /**
+     * Register a result-delivery sink (one per attached core's worker) and
+     * replay current state to it in FIFO order: the latest main-channel
+     * analysis, then per non-main channel its `audio-stream-setup` identity
+     * followed by the latest cached analysis. A mid-stream attach is thereby
+     * indistinguishable from a fresh connect on the receiving worker.
+     */
+    addSink(id: string, send: (message: {
+        type: string;
+        data: any;
+    }) => void): void;
+    /**
+     * Remove a previously registered result-delivery sink.
+     */
+    removeSink(id: string): void;
+    private buildStreamSetupMessage;
+    private emitStreamSetup;
     /**
      * Reset audio values to defaults (main channel + beat state)
      */
@@ -436,17 +465,25 @@ export declare class AudioSystem {
     resetAudioState(): Promise<void>;
     /**
      * Add an additional audio stream (lightweight analysis, no beat detection).
-     * @param streamIndex Global stream index (from VijiCore: AUDIO_ADDITIONAL_BASE + n or AUDIO_DEVICE_BASE + n)
+     * When `meta` is provided, the stream identity is recorded and an
+     * `audio-stream-setup` message is emitted to every sink (and replayed to
+     * sinks attached later).
+     * @param streamIndex Global stream index (AUDIO_ADDITIONAL_BASE + n or AUDIO_DEVICE_BASE + n)
      * @param stream The MediaStream to analyze
+     * @param meta Stream identity for the worker-side two-phase setup protocol
      */
-    addStream(streamIndex: number, stream: MediaStream): Promise<void>;
+    addStream(streamIndex: number, stream: MediaStream, meta?: {
+        streamType: 'additional' | 'device';
+        deviceId?: string;
+    }): Promise<void>;
     /**
      * Remove an additional audio stream.
      */
     removeStream(streamIndex: number): void;
     /**
-     * Tear down all additional channels and rebuild with corrected indices.
-     * Mirrors video's reinitializeAdditionalCoordinators pattern.
+     * Tear down all additional channels in the base range and rebuild with
+     * corrected indices, re-emitting `audio-stream-setup` for the re-indexed
+     * streams. Mirrors video's reinitializeAdditionalCoordinators pattern.
      */
     reinitializeAdditionalChannels(streams: MediaStream[], baseIndex: number): Promise<void>;
     /**
@@ -2665,7 +2702,7 @@ declare type TrackingState = 'TRACKING' | 'LOCKED' | 'BREAKDOWN' | 'LOST';
 /** Returned by `on*` listener registration calls; invoke to unsubscribe. */
 export declare type Unsubscribe = () => void;
 
-export declare const VERSION = "0.8.17";
+export declare const VERSION = "0.10.0";
 
 /**
  * Real-time video API: drawable frame, dimensions, and the source-side
@@ -2714,11 +2751,12 @@ export declare interface VideoAPI {
  * outputs (the analysed frame and its detection / segmentation results) with
  * the verbs that activate the underlying MediaPipe pipelines.
  *
- * Available only on the main video stream (`viji.video.cv`). Additional
- * streams (`viji.videoStreams[i].cv`) and device videos
- * (`viji.devices[i].video.cv`) do not run CV; their `cv` surface exists for
- * API consistency but the data fields stay at their empty defaults and the
- * `enable*` verbs are no-ops.
+ * Available on the main video stream (`viji.video.cv`) and on additional
+ * streams (`viji.videoStreams[i].cv`) — same surface, same verbs, activated
+ * per stream. `analysedFrame` is populated on the MAIN stream only; on other
+ * streams it stays `null` (use `currentFrame`). Device videos
+ * (`viji.devices[i].video.cv`) do not run CV yet; their `cv` surface exists
+ * for API consistency with the data fields at their empty defaults.
  */
 export declare interface VideoCVAPI {
     /**
@@ -3024,7 +3062,6 @@ export declare class VijiCore {
     private workerManager;
     private interactionManager;
     private deviceSensorManager;
-    private audioSystem;
     private isInitialized;
     private isDestroyed;
     private isInitializing;
@@ -3038,23 +3075,10 @@ export declare class VijiCore {
     private debugLog;
     private config;
     private isHeadless;
-    private currentAudioStream;
-    private static readonly ADDITIONAL_STREAM_BASE;
-    private static readonly DEVICE_VIDEO_BASE;
     private static readonly DIRECT_FRAME_BASE;
-    private static readonly AUDIO_ADDITIONAL_BASE;
-    private static readonly AUDIO_DEVICE_BASE;
-    private static readonly MAX_ADDITIONAL_AUDIO_STREAMS;
-    private static readonly MAX_ADDITIONAL_VIDEO_STREAMS;
-    private videoStream;
-    private videoStreams;
-    private audioStreams;
-    private deviceAudioStreamIndices;
-    private mainVideoCoordinator;
-    private additionalCoordinators;
+    private engineGroup;
+    private groupMember;
     private directFrameSlots;
-    private deviceVideoCoordinators;
-    private deviceVideoStreamIndices;
     private latestFrameBuffer;
     private autoCaptureEnabled;
     private eventSourceCore;
@@ -3066,6 +3090,7 @@ export declare class VijiCore {
     private sceneRenderer;
     private sceneCapabilities;
     private sceneP5CanvasMode?;
+    private sceneConsumesAnalysedFrame;
     private sceneContextType?;
     private parametersInitialized;
     private parameterListeners;
@@ -3367,27 +3392,33 @@ export declare class VijiCore {
         code: string;
     }) => void): void;
     /**
-     * Snapshot the full audio analysis + onset state for cross-instance
-     * transfer. Use for **same-process scene-switch** continuity (a fresh
-     * `VijiCore` can resume detection where the previous one left off).
+     * Snapshot the full audio analysis + onset state for **cross-device**
+     * transfer (e.g. host → second host, multi-machine sync). Pair with
+     * `importFullState` on the receiver, applying an NTP-derived `clockOffset`.
+     *
+     * NOT needed for same-process transitions: construct the successor core
+     * with `{ mediaSource: oldCore }` instead — the shared audio engine (and
+     * its live BPM lock / onset patterns) survives the swap with no
+     * export/import step and no warm-up gap.
      *
-     * For **cross-device** transfer, prefer `audio.onset.exportSessionState`
+     * For controller → host onset sync, prefer `audio.onset.exportSessionState`
      * — the controller's audio analysis state doesn't apply to the host's
      * audio source and would corrupt detection.
      *
      * Wall-clock fields are in this instance's `performance.now()` clock space.
-     * Receiver applies `clockOffset` on import (use `0` for same-process).
      *
-     * **Staleness caveat**: the audio block is robust to scene-load gaps under
-     * ~1 second. Longer gaps (>5s, e.g. cold cache + slow network) may cause
+     * **Staleness caveat**: the audio block is robust to transfer gaps under
+     * ~1 second. Longer gaps (>5s, e.g. slow network) may cause
      * stale-timestamp drift in `gridLockMemoryTime` / `warmupStartTimeMs`;
      * callers should treat that as a degraded path and skip the audio block.
      */
     exportFullState(): SerializedCoreState;
     /**
-     * Replace the audio analysis + onset state from a serialized payload.
-     * `clockOffset` is added to all sender-clocked fields (`0` for same-process,
-     * NTP-derived for cross-device). Mutation is synchronous.
+     * Replace the audio analysis + onset state from a serialized payload
+     * (**cross-device** transfer — for same-process transitions use
+     * `mediaSource` instead, see {@link exportFullState}).
+     * `clockOffset` is added to all sender-clocked fields (NTP-derived;
+     * `0` only when clocks are genuinely shared). Mutation is synchronous.
      *
      * Field-level events (`onModeChange`, `onMuteChange`) fire when imported
      * values differ from current — consumer-visible state stays consistent
@@ -3482,8 +3513,14 @@ export declare class VijiCore {
      */
     setFrameRate(mode: FrameRateMode): Promise<void>;
     /**
-     * Sets the CV processing frame rate mode (relative to scene frame rate)
-     * @param mode - CV processing rate: 'full', 'half', 'quarter', or 'eighth' of scene rate
+     * Sets the CV processing frame rate mode, relative to the **video stream
+     * delivery rate** (the capture's ~30 fps), not the scene render rate —
+     * with shared engines, attached scenes may render at different rates, so
+     * the capture rate is the only coherent base.
+     *
+     * Engine-level write: visible to ALL cores sharing this core's media
+     * engines (same rule as the stream mutators).
+     * @param mode - CV processing rate: 'full', 'half', 'quarter', or 'eighth' of the capture rate
      */
     setCVFrameRate(mode: CVFrameRateMode): Promise<void>;
     /**
@@ -3495,57 +3532,76 @@ export declare class VijiCore {
      */
     setAudioStream(audioStream: MediaStream | null): Promise<void>;
     /**
-     * Sets the main video stream (CV enabled)
+     * Sets the main video stream (CV enabled).
+     *
+     * Engine-level write: visible to ALL cores sharing this core's media
+     * engines — each attached core's worker gets its own fresh capture for the
+     * new stream. Capability change notifications fan out to all of them.
      */
     setVideoStream(stream: MediaStream | null): Promise<void>;
     /**
-     * Gets the main video stream
+     * Gets the main video stream (engine-scoped — identical for all cores
+     * sharing this core's media engines)
      */
     getVideoStream(): MediaStream | null;
     /**
      * Adds an additional video stream (no CV). Returns its index in viji.videoStreams[].
+     *
+     * Engine-level write: the stream appears in `viji.videoStreams[]` of EVERY
+     * core sharing this core's media engines.
      */
     addVideoStream(stream: MediaStream): Promise<number>;
     /**
-     * Removes an additional video stream by index.
+     * Removes an additional video stream by index. Stops the stream's tracks.
+     *
+     * Engine-level write: removal (and the resulting re-indexing) is visible to
+     * ALL cores sharing this core's media engines.
      */
     removeVideoStream(index: number): Promise<void>;
     /**
-     * Gets the number of additional video streams.
+     * Gets the number of additional video streams (engine-scoped).
      */
     getVideoStreamCount(): number;
     /**
      * Add an additional audio stream for lightweight analysis (no beat detection).
+     *
+     * Engine-level write: the stream appears in `viji.audioStreams[]` of EVERY
+     * core sharing this core's media engines.
      * @param stream MediaStream with audio tracks
      * @returns Index of the new stream (0-based within additional streams)
      * @throws VijiCoreError if limit reached (max 8)
      */
     addAudioStream(stream: MediaStream): Promise<number>;
     /**
-     * Remove an additional audio stream by index.
-     * Triggers re-indexing of remaining streams.
+     * Remove an additional audio stream by index. Stops the stream's tracks
+     * and triggers re-indexing of remaining streams.
+     *
+     * Engine-level write: removal is visible to ALL cores sharing this core's
+     * media engines.
      */
     removeAudioStream(index: number): Promise<void>;
     /**
-     * Gets the number of additional audio streams.
+     * Gets the number of additional audio streams (engine-scoped).
      */
     getAudioStreamCount(): number;
     /**
      * Set audio stream from an external device.
+     *
+     * Engine-level write: visible to ALL cores sharing this core's media
+     * engines. Device registration (`addExternalDevice`) stays per-core.
      * @param deviceId Device identifier (must be registered via addExternalDevice)
      * @param stream MediaStream with audio tracks from the device
      */
     setDeviceAudio(deviceId: string, stream: MediaStream): Promise<void>;
     /**
      * Clear audio stream from a device.
+     *
+     * Engine-level write: visible to ALL cores sharing this core's media engines.
      */
     clearDeviceAudio(deviceId: string): Promise<void>;
     /**
-     * Reinitializes all additional coordinators after array mutation.
-     */
-    private reinitializeAdditionalCoordinators;
-    /**
-     * Gets the current audio stream
+     * Gets the current audio stream (engine-scoped — identical for all cores
+     * sharing this core's media engines)
      */
     getAudioStream(): MediaStream | null;
     /**
@@ -3556,6 +3612,61 @@ export declare class VijiCore {
      * Gets the current interaction enabled state
      */
     getInteractionEnabled(): boolean;
+    /**
+     * Lazy audio-engine access. Read-only getters use `audioEngine` (never
+     * materializes — headless previews construct nothing); every other
+     * `core.audio.*` call uses `ensureAudioEngine()` (materialize-on-use:
+     * mutators, subscriptions, imports AND exports — a listener registered or
+     * a tap received before any stream exists must hit a live engine, see the
+     * constructor-started idle ticker in AudioSystem).
+     */
+    private get audioEngine();
+    private ensureAudioEngine;
+    /**
+     * Host-side video control namespace. The `cv` sub-namespace lets host
+     * code (as opposed to scene code) drive CV on the main video stream —
+     * the controller use case: a scene-less headless core analyzing its
+     * local camera and shipping results onward.
+     */
+    readonly video: {
+        cv: {
+            /**
+             * Enable or disable a CV feature on the main video stream from the
+             * host side. The host acts as one more consumer of the shared CV
+             * engine: inference runs the union of all consumers' desired sets,
+             * and host results arrive via {@link onResults} filtered to the
+             * host-enabled features.
+             *
+             * Engine-level write: the underlying inference is visible to ALL
+             * cores sharing this core's media engines (scenes still only receive
+             * features they enabled themselves).
+             *
+             * @param feature - The CV feature to toggle
+             * @param enabled - Whether the feature should run
+             */
+            enable: (feature: CVFeature, enabled: boolean) => void;
+            /**
+             * Subscribe to CV results on the host side, filtered to the features
+             * enabled via {@link enable}. Fires once per completed inference with
+             * the same result shapes the artist API exposes
+             * (`faces`/`hands`/`pose`/`segmentation`).
+             *
+             * Engine-level: one listener registry per shared engine group —
+             * subscribe once per app.
+             *
+             * @returns Unsubscribe function. Call to remove the listener.
+             */
+            onResults: (listener: (update: {
+                streamIndex: number;
+                results: {
+                    faces?: FaceData[];
+                    hands?: HandData[];
+                    pose?: PoseData | null;
+                    segmentation?: SegmentationData | null;
+                };
+            }) => void) => Unsubscribe;
+        };
+    };
     readonly audio: {
         /**
          * Beat control namespace
@@ -3637,6 +3748,11 @@ export declare class VijiCore {
              *
              * Idempotent transitions (already in target mode) do not fire.
              *
+             * Engine-level: the listener registry lives on the shared audio engine,
+             * so subscribing through several cores that share media engines targets
+             * ONE registry — subscribe once per app. The same function reference is
+             * deduped by identity; distinct closures doing the same work are not.
+             *
              * @returns Unsubscribe function. Call to remove the listener.
              */
             onModeChange: (listener: (ev: OnsetModeChangeEvent) => void) => Unsubscribe;
@@ -3656,6 +3772,9 @@ export declare class VijiCore {
              * when fields differ; this event does not because session-end is a
              * different category — about session lifecycle, not field state.
              *
+             * Engine-level: one registry per shared audio engine — subscribe once
+             * per app (see {@link onModeChange}).
+             *
              * @returns Unsubscribe function. Call to remove the listener.
              */
             onSessionEnd: (listener: (ev: OnsetSessionEndEvent) => void) => Unsubscribe;
@@ -3671,6 +3790,9 @@ export declare class VijiCore {
              * Idempotent calls (e.g. `setMuted(true)` on an already-muted instrument)
              * do not fire.
              *
+             * Engine-level: one registry per shared audio engine — subscribe once
+             * per app (see {@link onModeChange}).
+             *
              * @returns Unsubscribe function. Call to remove the listener.
              */
             onMuteChange: (listener: (ev: OnsetMuteChangeEvent) => void) => Unsubscribe;
@@ -3717,7 +3839,11 @@ export declare class VijiCore {
             importSessionState: (state: SerializedOnsetState, clockOffset: number) => void;
         };
         /**
-         * Advanced audio controls
+         * Advanced audio controls.
+         *
+         * Engine-level: these settings live on the shared audio engine and are
+         * visible to ALL cores sharing this core's media engines (same rule as
+         * the stream mutators).
          */
         advanced: {
             /**
@@ -3848,6 +3974,13 @@ export declare class VijiCore {
      * Gets current performance statistics
      */
     getStats(): PerformanceStats;
+    /**
+     * Merge the shared CV engine's main-stream processing stats into
+     * `stats.cv`. The cv block is absent until CV state exists (no engine, or
+     * no features ever requested on the main stream) — same shape consumers
+     * saw when the worker produced it.
+     */
+    private mergeCVStats;
     /**
      * Pauses the scene render loop — stops the worker from rendering. Intended to
      * be driven from the host on `document.hidden`, so a hidden tab doesn't keep
@@ -3939,12 +4072,20 @@ export declare class VijiCore {
     /**
      * Set video stream for an external device
      * Device camera will appear in viji.devices[].video
+     *
+     * Engine-level write: every core sharing this core's media engines gets
+     * its own capture of the device stream. Device registration
+     * (`addExternalDevice`) and sensor state stay per-core, so each attached
+     * core's host app registers the device on that core — the worker
+     * self-heals whichever of stream/registration arrives last.
      * @param deviceId - Device identifier
      * @param stream - MediaStream from device camera
      */
     setDeviceVideo(deviceId: string, stream: MediaStream): Promise<void>;
     /**
      * Clear video stream from device
+     *
+     * Engine-level write: visible to ALL cores sharing this core's media engines.
      * @param deviceId - Device identifier
      */
     clearDeviceVideo(deviceId: string): Promise<void>;
@@ -4004,8 +4145,27 @@ export declare interface VijiCoreConfig {
     videoStreams?: MediaStream[];
     /** Additional audio input streams (lightweight analysis, no beat detection) */
     audioStreams?: MediaStream[];
-    /** Disable input processing (for preview instances) */
-    noInputs?: boolean;
+    /**
+     * Share another core's media analysis engines instead of creating private
+     * ones. The new core attaches to the referenced core's engine group at
+     * construction time and detaches on `destroy()`; the engines (audio
+     * analysis, video capture) survive as long as ANY attached core is alive,
+     * so analysis state (BPM lock, onset patterns, stream connections) carries
+     * across core transitions with no export/import step.
+     *
+     * Mutually exclusive with `audioStream`, `videoStream`, `audioStreams`,
+     * and `videoStreams` — the engine group already owns the streams.
+     *
+     * Stream mutators (`setAudioStream`, `setVideoStream`, `addAudioStream`,
+     * `setDeviceAudio`, …) called on ANY attached core write to the shared
+     * engines and are visible to ALL attached cores.
+     *
+     * The referenced core must not be destroyed and must not have a fenced
+     * benchmark in flight (`BENCHMARK_NOT_ALLOWED`). The reference is resolved
+     * to the engine group at construction; it is not retained afterwards and
+     * is omitted from the `configuration` getter.
+     */
+    mediaSource?: VijiCore;
     /** Enable user interaction events (DOM event capture, disabled in headless by default) */
     allowUserInteraction?: boolean;
     /** Enable device sensor APIs (motion, orientation) */

package/dist/index.js

@@ -1,4 +1,4 @@
-import { A as s, C as o, I as E, V as i, a as A, b as C } from "./index-Cvk3ZfuY.js";
+import { A as s, C as o, I as E, V as i, a as A, b as C } from "./index-C-CGSJSn.js";
 export {
   s as AudioSystem,
   o as CV_FEATURE_KEYS,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@viji-dev/core",
-  "version": "0.8.17",
+  "version": "0.10.0",
   "description": "Universal execution engine for Viji Creative scenes",
   "type": "module",
   "main": "./dist/index.js",
@@ -49,7 +49,8 @@
   ],
   "scripts": {
     "prebuild": "node scripts/prebuild-clean.mjs && node scripts/sync-version.mjs",
-    "build": "vite build && node scripts/copy-tasks-assets.mjs && node scripts/build-artist-types.mjs && node scripts/build-docs-api.mjs",
+    "version": "npm run build && git add -A",
+    "build": "vite build && node scripts/copy-tasks-assets.mjs && node scripts/build-artist-types.mjs && node scripts/lint-artist-types-drift.mjs && node scripts/build-docs-api.mjs",
     "build:docs": "node scripts/build-docs-api.mjs",
     "lint:ai-prompts-drift": "node scripts/lint-ai-prompts-drift.mjs",
     "dev": "node scripts/sync-version.mjs && vite build --watch",