@viji-dev/core 0.3.19 → 0.3.21

package/dist/index.d.ts

@@ -1,13 +1,14 @@
-/* Excluded from this release type: AnalysisConfiguration */
-
 declare interface AudioAnalysisState {
     isConnected: boolean;
     currentBPM: number;
     confidence: number;
-    mode: 'auto' | 'manual';
-    tapCount: number;
     isLocked: boolean;
-    sensitivity: number;
+    trackingState: string;
+    onsetModes: {
+        kick: OnsetMode;
+        snare: OnsetMode;
+        hat: OnsetMode;
+    };
 }
 
 export declare interface AudioAPI {
@@ -36,6 +37,7 @@ export declare interface AudioAPI {
         any: number;
         kickSmoothed: number;
         snareSmoothed: number;
+        hatSmoothed: number;
         anySmoothed: number;
         triggers: {
             any: boolean;
@@ -49,19 +51,15 @@ export declare interface AudioAPI {
             strength: number;
         }>;
         bpm: number;
-        phase: number;
-        bar: number;
         confidence: number;
         isLocked: boolean;
     };
     spectral: {
         brightness: number;
         flatness: number;
-        flux: number;
     };
     getFrequencyData: () => Uint8Array;
-    getWaveform?: (samples?: number) => Float32Array;
-    getFFT?: (bins?: number, scale?: 'linear' | 'log') => Float32Array;
+    getWaveform: () => Float32Array;
 }
 
 /**
@@ -73,7 +71,7 @@ export declare interface AudioAPI {
  * - Beat detection with automatic BPM tracking
  * - Onset detection for audio-reactive triggers
  * - Smooth energy curves with envelope followers
- * - Tap tempo and manual BPM control
+ * - Per-instrument onset tap patterns with automatic pattern recognition
  * - Auto-gain normalization
  * - Spectral feature extraction
  * - AudioContext management and cleanup
@@ -88,28 +86,18 @@ export declare class AudioSystem {
     private analysisMode;
     private workletNode;
     private workletReady;
+    private workletRegistered;
     private currentSampleRate;
-    private pcmRing;
-    private pcmWriteIndex;
-    private pcmFilled;
-    private pcmHistorySeconds;
-    private lastTempoExtraction;
-    private tempoExtractionIntervalMs;
-    private analysisBackend;
     private bandNoiseFloor;
     /** Tracks the last non-zero BPM chosen for output to avoid dropping to 0 */
     private lastNonZeroBpm;
     /** Tracks which source provided the current BPM (pll | tempo | carry | default) */
     private lastBpmSource;
-    private aubioWarningLogged;
-    private useEssentiaTempo;
     private workletFrameCount;
-    private forceAnalyser;
-    private statusLogTimer;
-    private isAudioLab;
+    private lastFrameTime;
+    private stalenessTimer;
+    private static readonly STALENESS_THRESHOLD_MS;
     private analysisTicks;
-    private lastLoopWarn;
-    private verboseLabLogs;
     private lastPhaseLogTime;
     private onsetLogBuffer;
     private debugMode;
@@ -126,18 +114,12 @@ export declare class AudioSystem {
     private rawBandsPreGain;
     /** Last dt used in analysis loop (ms) */
     private lastDtMs;
-    /** Flag to disable auto-gain for debugging */
-    private debugDisableAutoGain;
     /** Flag to track if we've logged detection method (to avoid spam) */
     private readonly bandNames;
     private essentiaBandHistories;
     private readonly essentiaHistoryWindowMs;
-    private beatMode;
-    private manualBPM;
-    private tapHistory;
-    private tapTimeout;
+    private onsetTapManager;
     private envelopeFollowers;
-    private sensitivity;
     private beatDetectionEnabled;
     private onsetDetectionEnabled;
     private autoGainEnabled;
@@ -156,11 +138,6 @@ export declare class AudioSystem {
     private updateEssentiaBandHistories;
     private getEssentiaBandHistory;
     private getEssentiaAllHistories;
-    private logVerbose;
-    /**
-     * Start/stop periodic status logging. If maxSamples provided, stops after that many ticks.
-     */
-    private toggleStatusLogger;
     /**
      * Handle frames pushed from AudioWorklet
      */
@@ -185,14 +162,6 @@ export declare class AudioSystem {
      * Spectral features from magnitude spectrum
      */
     private calculateSpectralFeaturesFromMagnitude;
-    /**
-     * Append PCM frame to ring buffer for periodic Essentia tempo extraction
-     */
-    private appendPcmFrame;
-    /**
-     * Periodically estimate tempo using Essentia's RhythmExtractor2013 (offline chunk)
-     */
-    private runEssentiaTempoEstimate;
     /**
      * Run onset + beat detection pipeline and return BeatState
      */
@@ -202,7 +171,6 @@ export declare class AudioSystem {
      */
     private debugLog;
     private fftSize;
-    private smoothingTimeConstant;
     private analysisInterval;
     private analysisIntervalMs;
     private frequencyData;
@@ -215,11 +183,11 @@ export declare class AudioSystem {
     private fftPhase;
     private hannWindow;
     private audioState;
+    private lastWaveformFrame;
     private analysisLoopId;
     private isAnalysisRunning;
     private lastAnalysisTimestamp;
     private sendAnalysisResults;
-    private prevSpectralCentroid;
     constructor(sendAnalysisResultsCallback?: (data: any) => void);
     /**
      * Prepare FFT buffers and windowing for the selected fftSize
@@ -254,18 +222,16 @@ export declare class AudioSystem {
             any: number;
             kickSmoothed: number;
             snareSmoothed: number;
+            hatSmoothed: number;
             anySmoothed: number;
             events: BeatEvent[];
             bpm: number;
-            phase: number;
-            bar: number;
             confidence: number;
             isLocked: boolean;
         };
         spectral: {
             brightness: number;
             flatness: number;
-            flux: number;
         };
     };
     /**
@@ -282,7 +248,6 @@ export declare class AudioSystem {
      */
     handleAudioStreamUpdate(data: {
         audioStream?: MediaStream | null;
-        analysisConfig?: AnalysisConfiguration;
         timestamp: number;
     }): void;
     /**
@@ -293,10 +258,6 @@ export declare class AudioSystem {
      * Disconnect current audio stream and clean up resources
      */
     private disconnectAudioStream;
-    /**
-     * Update analysis configuration
-     */
-    private updateAnalysisConfig;
     /**
      * Initialize audio worklet for high-quality capture (complex STFT path)
      */
@@ -309,6 +270,8 @@ export declare class AudioSystem {
      * Stop the audio analysis loop
      */
     private stopAnalysisLoop;
+    private startStalenessTimer;
+    private stopStalenessTimer;
     /**
      * Pause audio analysis (for tests or temporary suspension)
      * The setInterval continues but performAnalysis() exits early
@@ -345,68 +308,29 @@ export declare class AudioSystem {
      */
     resetAudioState(): void;
     /**
-     * Get current analysis configuration
-     */
-    getAnalysisConfig(): {
-        fftSize: number;
-        smoothing: number;
-    };
-    /**
-     * Set analysis backend preference
-     */
-    setAnalysisBackend(backend: 'auto' | 'essentia' | 'custom' | 'aubio'): void;
-    getAnalysisBackend(): 'auto' | 'essentia' | 'custom' | 'aubio';
-    /**
-     * Force analyser path (skip worklet) for debugging
-     */
-    setForceAnalyser(enabled: boolean): void;
-    isForceAnalyser(): boolean;
-    /**
-     * Enable/disable Essentia tempo extraction (disabled by default due to WASM exception config)
-     */
-    setEssentiaTempoEnabled(enabled: boolean): void;
-    isEssentiaTempoEnabled(): boolean;
-    /**
-     * Set global sensitivity multiplier
-     * @param value - Sensitivity (0.5-2.0, default 1.0)
-     */
-    setSensitivity(value: number): void;
-    /**
-     * Get current sensitivity
+     * Record a tap for the specified instrument onset
      */
-    getSensitivity(): number;
+    tapOnset(instrument: InstrumentType): void;
     /**
-     * Tap tempo - record tap for manual BPM
+     * Clear tap pattern for an instrument, restoring auto-detection
      */
-    tapTempo(): void;
+    clearOnsetTap(instrument: InstrumentType): void;
     /**
-     * Calculate BPM from tap history
+     * Get the current onset mode for an instrument
      */
-    private calculateBPMFromTaps;
+    getOnsetMode(instrument: InstrumentType): OnsetMode_2;
     /**
-     * Start auto-clear timeout for tap tempo
+     * Get pattern info for an instrument (null if no pattern recognized)
      */
-    private startTapClearTimeout;
+    getOnsetPatternInfo(instrument: InstrumentType): PatternInfo | null;
     /**
-     * Clear tap tempo history
+     * Mute/unmute an instrument onset (suppresses output without destroying state)
      */
-    clearTaps(): void;
+    setOnsetMuted(instrument: InstrumentType, muted: boolean): void;
     /**
-     * Get tap count
+     * Check if an instrument onset is muted
      */
-    getTapCount(): number;
-    /**
-     * Set beat sync mode
-     */
-    setBeatMode(mode: 'auto' | 'manual'): void;
-    /**
-     * Get beat sync mode
-     */
-    getBeatMode(): 'auto' | 'manual';
-    /**
-     * Set manual BPM
-     */
-    setManualBPM(bpm: number): void;
+    isOnsetMuted(instrument: InstrumentType): boolean;
     /**
      * Get current BPM (manual or auto-detected)
      */
@@ -423,10 +347,6 @@ export declare class AudioSystem {
      * Set FFT size
      */
     setFFTSize(size: 2048 | 4096 | 8192): void;
-    /**
-     * Set smoothing time constant
-     */
-    setSmoothing(value: number): void;
     /**
      * Enable/disable auto-gain
      */
@@ -446,11 +366,13 @@ export declare class AudioSystem {
         isConnected: boolean;
         currentBPM: number;
         confidence: number;
-        mode: 'auto' | 'manual';
-        tapCount: number;
         isLocked: boolean;
-        sensitivity: number;
         trackingState: string;
+        onsetModes: {
+            kick: OnsetMode_2;
+            snare: OnsetMode_2;
+            hat: OnsetMode_2;
+        };
     };
     /**
      * Get current per-frame audio analysis data (for UI display)
@@ -482,18 +404,16 @@ export declare class AudioSystem {
             any: number;
             kickSmoothed: number;
             snareSmoothed: number;
+            hatSmoothed: number;
             anySmoothed: number;
             events: BeatEvent[];
             bpm: number;
-            phase: number;
-            bar: number;
             confidence: number;
             isLocked: boolean;
         };
         spectral: {
             brightness: number;
             flatness: number;
-            flux: number;
         };
     };
     /**
@@ -597,7 +517,6 @@ export declare class AudioSystem {
         };
         volumeGain: number;
         dtMs: number;
-        smoothingTimeConstant: number;
         analysisIntervalMs: number;
         events: BeatEvent[];
     };
@@ -677,7 +596,7 @@ declare interface CoreCapabilities {
     hasGeneral: boolean;
 }
 
-declare type CVFeature = 'faceDetection' | 'faceMesh' | 'handTracking' | 'poseDetection' | 'bodySegmentation';
+declare type CVFeature = 'faceDetection' | 'faceMesh' | 'handTracking' | 'poseDetection' | 'bodySegmentation' | 'emotionDetection';
 
 declare type CVFrameRateMode = 'full' | 'half' | 'quarter' | 'eighth';
 
@@ -743,6 +662,61 @@ declare interface DeviceState extends DeviceSensorState {
     video: VideoAPI | null;
 }
 
+declare interface FaceBlendshapes {
+    browDownLeft: number;
+    browDownRight: number;
+    browInnerUp: number;
+    browOuterUpLeft: number;
+    browOuterUpRight: number;
+    cheekPuff: number;
+    cheekSquintLeft: number;
+    cheekSquintRight: number;
+    eyeBlinkLeft: number;
+    eyeBlinkRight: number;
+    eyeLookDownLeft: number;
+    eyeLookDownRight: number;
+    eyeLookInLeft: number;
+    eyeLookInRight: number;
+    eyeLookOutLeft: number;
+    eyeLookOutRight: number;
+    eyeLookUpLeft: number;
+    eyeLookUpRight: number;
+    eyeSquintLeft: number;
+    eyeSquintRight: number;
+    eyeWideLeft: number;
+    eyeWideRight: number;
+    jawForward: number;
+    jawLeft: number;
+    jawOpen: number;
+    jawRight: number;
+    mouthClose: number;
+    mouthDimpleLeft: number;
+    mouthDimpleRight: number;
+    mouthFrownLeft: number;
+    mouthFrownRight: number;
+    mouthFunnel: number;
+    mouthLeft: number;
+    mouthLowerDownLeft: number;
+    mouthLowerDownRight: number;
+    mouthPressLeft: number;
+    mouthPressRight: number;
+    mouthPucker: number;
+    mouthRight: number;
+    mouthRollLower: number;
+    mouthRollUpper: number;
+    mouthShrugLower: number;
+    mouthShrugUpper: number;
+    mouthSmileLeft: number;
+    mouthSmileRight: number;
+    mouthStretchLeft: number;
+    mouthStretchRight: number;
+    mouthUpperUpLeft: number;
+    mouthUpperUpRight: number;
+    noseSneerLeft: number;
+    noseSneerRight: number;
+    tongueOut: number;
+}
+
 export declare interface FaceData {
     id: number;
     bounds: {
@@ -751,8 +725,12 @@ export declare interface FaceData {
         width: number;
         height: number;
     };
+    center: {
+        x: number;
+        y: number;
+    };
     confidence: number;
-    landmarks?: {
+    landmarks: {
         x: number;
         y: number;
         z?: number;
@@ -771,6 +749,7 @@ export declare interface FaceData {
         yaw: number;
         roll: number;
     };
+    blendshapes: FaceBlendshapes;
 }
 
 declare interface FrameRateInfo {
@@ -873,6 +852,8 @@ declare interface ImageParameter {
     category: ParameterCategory;
 }
 
+declare type InstrumentType = 'kick' | 'snare' | 'hat';
+
 export declare interface KeyboardAPI {
     isPressed(key: string): boolean;
     wasPressed(key: string): boolean;
@@ -952,6 +933,10 @@ declare interface NumberParameter {
     category: ParameterCategory;
 }
 
+declare type OnsetMode = 'auto' | 'tapping' | 'pattern';
+
+declare type OnsetMode_2 = 'auto' | 'tapping' | 'pattern';
+
 export declare interface ParameterAPI {
     define(groups: ParameterGroup[]): void;
     [key: string]: any;
@@ -993,6 +978,12 @@ export declare interface ParameterGroup {
 
 declare type ParameterValue = string | number | boolean | ImageBitmap | OffscreenCanvas | null;
 
+declare interface PatternInfo {
+    length: number;
+    intervals: number[];
+    confidence: number;
+}
+
 export declare interface PerformanceStats {
     frameTime: number;
     resolution: Resolution;
@@ -1245,6 +1236,7 @@ export declare interface VideoAPI {
     cv: {
         enableFaceDetection(enabled: boolean): Promise<void>;
         enableFaceMesh(enabled: boolean): Promise<void>;
+        enableEmotionDetection(enabled: boolean): Promise<void>;
         enableHandTracking(enabled: boolean): Promise<void>;
         enablePoseDetection(enabled: boolean): Promise<void>;
         enableBodySegmentation(enabled: boolean): Promise<void>;
@@ -1256,10 +1248,11 @@ export declare interface VideoAPI {
 /**
  * Video stream categorization type
  * - main: Primary video with CV processing (viji.video)
- * - additional: Host-provided streams without CV (viji.streams[])
+ * - additional: Host-provided MediaStreams without CV (viji.streams[])
+ * - directFrame: Injected frames from compositor pipeline (viji.streams[], after additional)
  * - device: Device-provided camera streams (device.video)
  */
-export declare type VideoStreamType = 'main' | 'additional' | 'device';
+export declare type VideoStreamType = 'main' | 'additional' | 'directFrame' | 'device';
 
 export declare interface VijiAPI {
     canvas: OffscreenCanvas;
@@ -1315,6 +1308,9 @@ export declare class VijiCore {
     private config;
     private isHeadless;
     private currentAudioStream;
+    private static readonly ADDITIONAL_STREAM_BASE;
+    private static readonly DEVICE_VIDEO_BASE;
+    private static readonly DIRECT_FRAME_BASE;
     private videoStream;
     private videoStreams;
     private mainVideoCoordinator;
@@ -1351,9 +1347,10 @@ export declare class VijiCore {
      */
     private updateFrameBuffer;
     /**
-     * Get latest frame (transfers ownership, zero-copy)
+     * Get latest frame (transfers ownership, zero-copy).
+     * Internal: consumed by getLatestFramesFromSources().
      */
-    getLatestFrame(): ImageBitmap | null;
+    private getLatestFrame;
     /**
      * Get latest frames from all linked sources
      */
@@ -1385,11 +1382,6 @@ export declare class VijiCore {
      * No-op for headless cores.
      */
     hide(): void;
-    /**
-     * Select audio analysis backend
-     */
-    setAudioAnalysisBackend(backend: 'auto' | 'essentia' | 'custom' | 'aubio'): void;
-    getAudioAnalysisBackend(): 'auto' | 'essentia' | 'custom' | 'aubio';
     /**
      * Initializes the core components in sequence
      */
@@ -1470,11 +1462,6 @@ export declare class VijiCore {
      * Inject touch event
      */
     injectTouchEvent(data: TouchEventData): void;
-    /**
-     * Worker-side stream index offset for direct frame slots.
-     * Accounts for main video stream and additional media streams.
-     */
-    private get directFrameStartIndex();
     /**
      * Ensure the worker has enough direct frame slots prepared.
      * Auto-grows as needed; skips if already sufficient.
@@ -1486,11 +1473,6 @@ export declare class VijiCore {
      * Auto-prepares slots as needed. Transfers ownership of the bitmap (zero-copy).
      */
     injectFrame(bitmap: ImageBitmap, streamIndex: number): void;
-    /**
-     * Inject frames for all direct-frame stream slots at once (compositor pipeline).
-     * Auto-prepares slots on first call. Transfers ownership of all bitmaps (zero-copy).
-     */
-    injectFrames(bitmaps: ImageBitmap[]): void;
     /**
      * Link this core to receive events from a source core
      * @param syncResolution Smart default: true for headless, false for visible
@@ -1602,35 +1584,19 @@ export declare class VijiCore {
      */
     getVideoStream(): MediaStream | null;
     /**
-     * Sets all additional video streams (no CV)
-     */
-    setVideoStreams(streams: MediaStream[]): Promise<void>;
-    /**
-     * Gets all additional video streams
-     */
-    getVideoStreams(): MediaStream[];
-    /**
-     * Gets video stream at specific index
-     */
-    getVideoStreamAt(index: number): MediaStream | null;
-    /**
-     * Adds a video stream
+     * Adds an additional video stream (no CV). Returns its index in viji.streams[].
      */
     addVideoStream(stream: MediaStream): Promise<number>;
     /**
-     * Removes video stream at index
+     * Removes an additional video stream by index.
      */
-    removeVideoStreamAt(index: number): Promise<void>;
+    removeVideoStream(index: number): Promise<void>;
     /**
-     * Removes video stream by reference
+     * Gets the number of additional video streams.
      */
-    removeVideoStream(stream: MediaStream): Promise<boolean>;
+    getVideoStreamCount(): number;
     /**
-     * Updates video stream at index
-     */
-    setVideoStreamAt(index: number, stream: MediaStream): Promise<void>;
-    /**
-     * Reinitializes all additional coordinators
+     * Reinitializes all additional coordinators after array mutation.
      */
     private reinitializeAdditionalCoordinators;
     /**
@@ -1646,54 +1612,59 @@ export declare class VijiCore {
      */
     getInteractionEnabled(): boolean;
     readonly audio: {
-        /**
-         * Set global audio sensitivity (0.5-2.0, default 1.0)
-         */
-        setSensitivity: (value: number) => void;
-        /**
-         * Get current sensitivity
-         */
-        getSensitivity: () => number;
         /**
          * Beat control namespace
          */
         beat: {
             /**
-             * Tap tempo - record tap for manual BPM
+             * Get current BPM (auto-detected)
              */
-            tap: () => void;
+            getBPM: () => number;
             /**
-             * Clear tap tempo history
+             * Nudge beat phase
              */
-            clearTaps: () => void;
+            nudge: (amount: number) => void;
             /**
-             * Get tap count
+             * Reset beat phase to next beat
              */
-            getTapCount: () => number;
+            resetPhase: () => void;
+        };
+        /**
+         * Per-instrument onset tap control
+         */
+        onset: {
             /**
-             * Set beat sync mode
+             * Tap an onset for a specific instrument.
+             * First tap switches the instrument from auto to tapping mode.
+             * If a repeating pattern is recognized, it continues after tapping stops.
              */
-            setMode: (mode: "auto" | "manual") => void;
+            tap: (instrument: "kick" | "snare" | "hat") => void;
             /**
-             * Get beat sync mode
+             * Clear the tap pattern for an instrument, restoring auto-detection
              */
-            getMode: () => "auto" | "manual";
+            clear: (instrument: "kick" | "snare" | "hat") => void;
             /**
-             * Set manual BPM
+             * Get onset mode for an instrument: 'auto' | 'tapping' | 'pattern'
              */
-            setBPM: (bpm: number) => void;
+            getMode: (instrument: "kick" | "snare" | "hat") => string;
             /**
-             * Get current BPM (manual or auto-detected)
+             * Get recognized pattern info for an instrument, or null
              */
-            getBPM: () => number;
+            getPatternInfo: (instrument: "kick" | "snare" | "hat") => {
+                length: number;
+                intervals: number[];
+                confidence: number;
+            } | null;
             /**
-             * Nudge beat phase
+             * Mute/unmute an instrument onset.
+             * Suppresses all output (auto, tap, pattern) without destroying state.
+             * Unmuting resumes pattern replay in-phase.
              */
-            nudge: (amount: number) => void;
+            setMuted: (instrument: "kick" | "snare" | "hat", muted: boolean) => void;
             /**
-             * Reset beat phase to next beat
+             * Check if an instrument onset is muted
              */
-            resetPhase: () => void;
+            isMuted: (instrument: "kick" | "snare" | "hat") => boolean;
         };
         /**
          * Advanced audio controls
@@ -1703,22 +1674,14 @@ export declare class VijiCore {
              * Set FFT size (2048, 4096, or 8192)
              */
             setFFTSize: (size: 2048 | 4096 | 8192) => void;
-            /**
-             * Set smoothing time constant (0-1)
-             */
-            setSmoothing: (value: number) => void;
             /**
              * Enable/disable auto-gain
              */
             setAutoGain: (enabled: boolean) => void;
             /**
-             * Enable/disable beat detection
-             */
-            setBeatDetection: (enabled: boolean) => void;
-            /**
-             * Enable/disable onset detection
+             * Enable/disable audio analysis debug mode (independent from core debug mode)
              */
-            setOnsetDetection: (enabled: boolean) => void;
+            setDebugMode: (enabled: boolean) => void;
         };
         /**
          * Get audio analysis state (metadata)
@@ -1755,18 +1718,16 @@ export declare class VijiCore {
                 any: number;
                 kickSmoothed: number;
                 snareSmoothed: number;
+                hatSmoothed: number;
                 anySmoothed: number;
                 events: BeatEvent[];
                 bpm: number;
-                phase: number;
-                bar: number;
                 confidence: number;
                 isLocked: boolean;
             };
             spectral: {
                 brightness: number;
                 flatness: number;
-                flux: number;
             };
         } | {
             isConnected: false;
@@ -1794,6 +1755,7 @@ export declare class VijiCore {
                 any: number;
                 kickSmoothed: number;
                 snareSmoothed: number;
+                hatSmoothed: number;
                 anySmoothed: number;
                 triggers: {
                     any: boolean;
@@ -1802,15 +1764,12 @@ export declare class VijiCore {
                     hat: boolean;
                 };
                 bpm: number;
-                phase: number;
-                bar: number;
                 confidence: number;
                 isLocked: false;
             };
             spectral: {
                 brightness: number;
                 flatness: number;
-                flux: number;
             };
         };
     };
@@ -1925,10 +1884,6 @@ export declare interface VijiCoreConfig {
     videoStream?: MediaStream;
     /** Additional video input streams (no CV processing) */
     videoStreams?: MediaStream[];
-    /** Audio analysis configuration */
-    analysisConfig?: AnalysisConfiguration;
-    /** Initial parameter values */
-    parameters?: ParameterGroup[];
     /** Disable input processing (for preview instances) */
     noInputs?: boolean;
     /** Enable user interaction events (DOM event capture, disabled in headless by default) */