@viji-dev/core 0.3.25 → 0.3.27

package/dist/index.d.ts

@@ -62,6 +62,33 @@ export declare interface AudioAPI {
     getWaveform: () => Float32Array;
 }
 
+declare interface AudioStreamAPI {
+    isConnected: boolean;
+    volume: {
+        current: number;
+        peak: number;
+        smoothed: number;
+    };
+    bands: {
+        low: number;
+        lowMid: number;
+        mid: number;
+        highMid: number;
+        high: number;
+        lowSmoothed: number;
+        lowMidSmoothed: number;
+        midSmoothed: number;
+        highMidSmoothed: number;
+        highSmoothed: number;
+    };
+    spectral: {
+        brightness: number;
+        flatness: number;
+    };
+    getFrequencyData: () => Uint8Array;
+    getWaveform: () => Float32Array;
+}
+
 /**
  * Manages audio analysis and processing for the host side.
  * Handles audio stream processing and provides real-time audio analysis.
@@ -80,24 +107,17 @@ export declare interface AudioAPI {
  */
 export declare class AudioSystem {
     private audioContext;
-    private analyser;
-    private mediaStreamSource;
-    private currentStream;
-    private analysisMode;
-    private workletNode;
-    private workletReady;
     private workletRegistered;
-    private currentSampleRate;
-    private bandNoiseFloor;
+    private mainChannel;
+    private additionalChannels;
+    private analysisInterval;
+    private analysisIntervalMs;
+    private stalenessTimer;
+    private static readonly STALENESS_THRESHOLD_MS;
     /** Tracks the last non-zero BPM chosen for output to avoid dropping to 0 */
     private lastNonZeroBpm;
     /** Tracks which source provided the current BPM (pll | tempo | carry | default) */
     private lastBpmSource;
-    private workletFrameCount;
-    private lastFrameTime;
-    private stalenessTimer;
-    private static readonly STALENESS_THRESHOLD_MS;
-    private analysisTicks;
     private lastPhaseLogTime;
     private onsetLogBuffer;
     private debugMode;
@@ -108,18 +128,13 @@ export declare class AudioSystem {
     private pll;
     private stateManager;
     private lastKickDetected;
-    private volumeAutoGain;
-    private bandAutoGain;
-    /** Raw band snapshot before auto-gain (for debug) */
+    /** Raw band snapshot before auto-gain (for debug, main channel only) */
     private rawBandsPreGain;
-    /** Last dt used in analysis loop (ms) */
-    private lastDtMs;
-    /** Flag to track if we've logged detection method (to avoid spam) */
     private readonly bandNames;
     private essentiaBandHistories;
     private readonly essentiaHistoryWindowMs;
     private onsetTapManager;
-    private envelopeFollowers;
+    private beatEnvelopeFollowers;
     private beatDetectionEnabled;
     private onsetDetectionEnabled;
     private autoGainEnabled;
@@ -139,29 +154,42 @@ export declare class AudioSystem {
     private getEssentiaBandHistory;
     private getEssentiaAllHistories;
     /**
-     * Handle frames pushed from AudioWorklet
+     * Handle frames pushed from AudioWorklet for main channel
      */
-    private handleWorkletFrame;
+    private handleMainWorkletFrame;
     /**
-     * Unified analysis pipeline (worklet and analyser paths)
+     * Handle frames pushed from AudioWorklet for an additional channel
      */
-    private analyzeFrame;
+    private handleAdditionalWorkletFrame;
     /**
-     * Compute FFT and derived arrays
+     * Common analysis pipeline for any channel (FFT, volume, bands, auto-gain, spectral, smoothing).
+     * Returns raw band energies (pre-auto-gain) for optional beat processing by the caller.
      */
-    private computeFFT;
+    private analyzeChannelFrame;
+    /**
+     * Full analysis for the main channel (common analysis + beat pipeline).
+     */
+    private analyzeMainFrame;
+    /**
+     * Lightweight analysis for additional/device channels (no beat detection).
+     */
+    private analyzeAdditionalFrame;
+    /**
+     * Compute FFT on a specific channel's buffers
+     */
+    private computeChannelFFT;
     /**
      * Calculate RMS/peak from float PCM frame (-1..1)
      */
     private calculateVolumeMetrics;
     /**
-     * Calculate perceptual/log-ish bands (pre-gain), normalized by max magnitude
+     * Calculate perceptual/log-ish bands on a specific channel
      */
-    private calculateFrequencyBandsFromMagnitude;
+    private calculateChannelBands;
     /**
-     * Spectral features from magnitude spectrum
+     * Spectral features on a specific channel
      */
-    private calculateSpectralFeaturesFromMagnitude;
+    private calculateChannelSpectral;
     /**
      * Run onset + beat detection pipeline and return BeatState
      */
@@ -171,28 +199,9 @@ export declare class AudioSystem {
      */
     private debugLog;
     private fftSize;
-    private analysisInterval;
-    private analysisIntervalMs;
-    private frequencyData;
-    private timeDomainData;
-    private fftEngine;
-    private fftInput;
-    private fftOutput;
-    private fftMagnitude;
-    private fftMagnitudeDb;
-    private fftPhase;
-    private hannWindow;
-    private audioState;
-    private lastWaveformFrame;
-    private analysisLoopId;
-    private isAnalysisRunning;
-    private lastAnalysisTimestamp;
+    private audioStateBeat;
     private sendAnalysisResults;
     constructor(sendAnalysisResultsCallback?: (data: any) => void);
-    /**
-     * Prepare FFT buffers and windowing for the selected fftSize
-     */
-    private refreshFFTResources;
     /**
      * Get the current audio analysis state (for host-side usage)
      */
@@ -215,20 +224,6 @@ export declare class AudioSystem {
             highMidSmoothed: number;
             highSmoothed: number;
         };
-        beat: {
-            kick: number;
-            snare: number;
-            hat: number;
-            any: number;
-            kickSmoothed: number;
-            snareSmoothed: number;
-            hatSmoothed: number;
-            anySmoothed: number;
-            events: BeatEvent[];
-            bpm: number;
-            confidence: number;
-            isLocked: boolean;
-        };
         spectral: {
             brightness: number;
             flatness: number;
@@ -244,37 +239,49 @@ export declare class AudioSystem {
      */
     waitForEssentia(timeoutMs?: number): Promise<boolean>;
     /**
-     * Handle audio stream update (called from VijiCore)
+     * Handle audio stream update for main stream (called from VijiCore)
      */
     handleAudioStreamUpdate(data: {
         audioStream?: MediaStream | null;
         timestamp: number;
     }): void;
     /**
-     * Set the audio stream for analysis
+     * Ensure the shared AudioContext is created and resumed.
+     */
+    private ensureAudioContext;
+    /**
+     * Connect a channel to Web Audio nodes (source, worklet/analyser).
+     * Used for both main and additional channels.
+     */
+    private connectChannel;
+    /**
+     * Set the main audio stream for analysis (preserves original public API surface).
      */
     private setAudioStream;
     /**
-     * Disconnect current audio stream and clean up resources
+     * Disconnect the main audio stream (does NOT close AudioContext -- additional channels may still be active).
      */
-    private disconnectAudioStream;
+    private disconnectMainStream;
     /**
-     * Initialize audio worklet for high-quality capture (complex STFT path)
+     * Set up an AudioWorklet node for a specific channel.
+     * Registers the processor module once (shared), creates a new WorkletNode per channel.
      */
-    private setupAudioWorklet;
+    private setupChannelWorklet;
     /**
-     * Start the audio analysis loop at high speed (8ms intervals for transient capture)
+     * Ensure the shared analysis interval is running (for analyser-mode channels).
      */
-    private startAnalysisLoop;
+    private ensureAnalysisLoop;
     /**
-     * Stop the audio analysis loop
+     * Stop the shared analysis loop.
      */
     private stopAnalysisLoop;
+    /**
+     * Shared staleness timer: checks all channels for stale data.
+     */
     private startStalenessTimer;
     private stopStalenessTimer;
     /**
      * Pause audio analysis (for tests or temporary suspension)
-     * The setInterval continues but performAnalysis() exits early
      */
     pauseAnalysis(): void;
     /**
@@ -282,31 +289,47 @@ export declare class AudioSystem {
      */
     resumeAnalysis(): void;
     /**
-     * Perform audio analysis (called every frame)
-     * Uses industry-grade 4-layer architecture:
-     * Layer 1: MultiOnsetDetection (per-band onset detection)
-     * Layer 2: TempoInduction (dual-method BPM detection)
-     * Layer 3: PhaseLockedLoop (stable phase tracking)
-     * Layer 4: BeatStateManager (state machine + confidence)
+     * Shared analysis loop callback: iterates main + additional channels in analyser mode.
      */
     private performAnalysis;
-    private applyAutoGain;
+    private applyChannelAutoGain;
     /**
-     * Update smooth versions of frequency bands
+     * Update smooth versions of frequency bands on a specific channel
      */
-    private updateSmoothBands;
+    private updateChannelSmoothBands;
     /**
-     * Send analysis results to worker
+     * Send analysis results for a specific channel to the worker.
+     * Tags with streamIndex and conditionally includes beat data (main channel only).
      */
-    private sendAnalysisResultsToWorker;
+    private sendChannelResults;
     /**
-     * Reset audio values to defaults
+     * Reset audio values to defaults (main channel + beat state)
      */
     private resetAudioValues;
     /**
-     * Reset all audio state (called when destroying)
+     * Reset all audio state (called when destroying).
+     * Disconnects all channels, closes AudioContext, resets all modules.
      */
     resetAudioState(): void;
+    /**
+     * Add an additional audio stream (lightweight analysis, no beat detection).
+     * @param streamIndex Global stream index (from VijiCore: AUDIO_ADDITIONAL_BASE + n or AUDIO_DEVICE_BASE + n)
+     * @param stream The MediaStream to analyze
+     */
+    addStream(streamIndex: number, stream: MediaStream): Promise<void>;
+    /**
+     * Remove an additional audio stream.
+     */
+    removeStream(streamIndex: number): void;
+    /**
+     * Tear down all additional channels and rebuild with corrected indices.
+     * Mirrors video's reinitializeAdditionalCoordinators pattern.
+     */
+    reinitializeAdditionalChannels(streams: MediaStream[], baseIndex: number): Promise<void>;
+    /**
+     * Get the number of additional audio channels.
+     */
+    getChannelCount(): number;
     /**
      * Record a tap for the specified instrument onset
      */
@@ -675,6 +698,8 @@ declare interface DeviceState extends DeviceSensorState {
     name: string;
     /** Device camera video (null if not available) */
     video: VideoAPI | null;
+    /** Device audio stream (null if not available) */
+    audio: AudioStreamAPI | null;
 }
 
 declare interface FaceBlendshapes {
@@ -801,54 +826,14 @@ export declare interface HandData {
         y: number;
         z: number;
     };
-    fingers: {
-        thumb: {
-            tip: {
-                x: number;
-                y: number;
-                z: number;
-            };
-            extended: boolean;
-        };
-        index: {
-            tip: {
-                x: number;
-                y: number;
-                z: number;
-            };
-            extended: boolean;
-        };
-        middle: {
-            tip: {
-                x: number;
-                y: number;
-                z: number;
-            };
-            extended: boolean;
-        };
-        ring: {
-            tip: {
-                x: number;
-                y: number;
-                z: number;
-            };
-            extended: boolean;
-        };
-        pinky: {
-            tip: {
-                x: number;
-                y: number;
-                z: number;
-            };
-            extended: boolean;
-        };
-    };
     gestures: {
         fist: number;
         openPalm: number;
         peace: number;
         thumbsUp: number;
+        thumbsDown: number;
         pointing: number;
+        iLoveYou: number;
     };
 }
 
@@ -1251,8 +1236,8 @@ export declare interface VideoAPI {
 /**
  * Video stream categorization type
  * - main: Primary video with CV processing (viji.video)
- * - additional: Host-provided MediaStreams without CV (viji.streams[])
- * - directFrame: Injected frames from compositor pipeline (viji.streams[], after additional)
+ * - additional: Host-provided MediaStreams without CV (viji.videoStreams[])
+ * - directFrame: Injected frames from compositor pipeline (viji.videoStreams[], after additional)
  * - device: Device-provided camera streams (device.video)
  */
 export declare type VideoStreamType = 'main' | 'additional' | 'directFrame' | 'device';
@@ -1269,7 +1254,8 @@ export declare interface VijiAPI {
     fps: number;
     audio: AudioAPI;
     video: VideoAPI;
-    streams: VideoAPI[];
+    videoStreams: VideoAPI[];
+    audioStreams: AudioStreamAPI[];
     mouse: MouseAPI;
     keyboard: KeyboardAPI;
     touches: TouchAPI;
@@ -1317,8 +1303,14 @@ export declare class VijiCore {
     private static readonly ADDITIONAL_STREAM_BASE;
     private static readonly DEVICE_VIDEO_BASE;
     private static readonly DIRECT_FRAME_BASE;
+    private static readonly AUDIO_ADDITIONAL_BASE;
+    private static readonly AUDIO_DEVICE_BASE;
+    private static readonly MAX_ADDITIONAL_AUDIO_STREAMS;
+    private static readonly MAX_ADDITIONAL_VIDEO_STREAMS;
     private videoStream;
     private videoStreams;
+    private audioStreams;
+    private deviceAudioStreamIndices;
     private mainVideoCoordinator;
     private additionalCoordinators;
     private directFrameSlots;
@@ -1590,7 +1582,7 @@ export declare class VijiCore {
      */
     getVideoStream(): MediaStream | null;
     /**
-     * Adds an additional video stream (no CV). Returns its index in viji.streams[].
+     * Adds an additional video stream (no CV). Returns its index in viji.videoStreams[].
      */
     addVideoStream(stream: MediaStream): Promise<number>;
     /**
@@ -1601,6 +1593,32 @@ export declare class VijiCore {
      * Gets the number of additional video streams.
      */
     getVideoStreamCount(): number;
+    /**
+     * Add an additional audio stream for lightweight analysis (no beat detection).
+     * @param stream MediaStream with audio tracks
+     * @returns Index of the new stream (0-based within additional streams)
+     * @throws VijiCoreError if limit reached (max 8)
+     */
+    addAudioStream(stream: MediaStream): Promise<number>;
+    /**
+     * Remove an additional audio stream by index.
+     * Triggers re-indexing of remaining streams.
+     */
+    removeAudioStream(index: number): Promise<void>;
+    /**
+     * Gets the number of additional audio streams.
+     */
+    getAudioStreamCount(): number;
+    /**
+     * Set audio stream from an external device.
+     * @param deviceId Device identifier (must be registered via addExternalDevice)
+     * @param stream MediaStream with audio tracks from the device
+     */
+    setDeviceAudio(deviceId: string, stream: MediaStream): Promise<void>;
+    /**
+     * Clear audio stream from a device.
+     */
+    clearDeviceAudio(deviceId: string): Promise<void>;
     /**
      * Reinitializes all additional coordinators after array mutation.
      */
@@ -1890,6 +1908,8 @@ export declare interface VijiCoreConfig {
     videoStream?: MediaStream;
     /** Additional video input streams (no CV processing) */
     videoStreams?: MediaStream[];
+    /** Additional audio input streams (lightweight analysis, no beat detection) */
+    audioStreams?: MediaStream[];
     /** Disable input processing (for preview instances) */
     noInputs?: boolean;
     /** Enable user interaction events (DOM event capture, disabled in headless by default) */

package/dist/index.js

@@ -1,4 +1,4 @@
-import { A, V, a, b } from "./index-C6fMmKQj.js";
+import { A, V, a, b } from "./index-9YDi9UBP.js";
 export {
   A as AudioSystem,
   V as VERSION,