@viji-dev/core 0.2.20 → 0.3.0

package/dist/index.d.ts

@@ -1,27 +1,643 @@
-export declare interface AnalysisConfiguration {
-    fftSize?: number;
-    smoothing?: number;
-    frequencyBands?: FrequencyBand[];
-    beatDetection?: boolean;
-    onsetDetection?: boolean;
+/* Excluded from this release type: AnalysisConfiguration */
+
+declare interface AudioAnalysisState {
+    isConnected: boolean;
+    currentBPM: number;
+    confidence: number;
+    mode: 'auto' | 'manual';
+    tapCount: number;
+    isLocked: boolean;
+    sensitivity: number;
 }
 
 export declare interface AudioAPI {
     isConnected: boolean;
     volume: {
-        rms: number;
+        current: number;
         peak: number;
+        smoothed: number;
+    };
+    bands: {
+        low: number;
+        lowMid: number;
+        mid: number;
+        highMid: number;
+        high: number;
+        lowSmoothed: number;
+        lowMidSmoothed: number;
+        midSmoothed: number;
+        highMidSmoothed: number;
+        highSmoothed: number;
     };
-    beat?: {
-        isKick: boolean;
+    beat: {
+        kick: number;
+        snare: number;
+        hat: number;
+        any: number;
+        kickSmoothed: number;
+        snareSmoothed: number;
+        anySmoothed: number;
+        triggers: {
+            any: boolean;
+            kick: boolean;
+            snare: boolean;
+            hat: boolean;
+        };
+        events: Array<{
+            type: 'kick' | 'snare' | 'hat';
+            time: number;
+            strength: number;
+        }>;
+        bpm: number;
+        phase: number;
+        bar: number;
         confidence: number;
+        isLocked: boolean;
+    };
+    spectral: {
+        brightness: number;
+        flatness: number;
+        flux: number;
     };
-    bands: Record<string, number>;
     getFrequencyData: () => Uint8Array;
+    getWaveform?: (samples?: number) => Float32Array;
+    getFFT?: (bins?: number, scale?: 'linear' | 'log') => Float32Array;
+}
+
+/**
+ * Manages audio analysis and processing for the host side.
+ * Handles audio stream processing and provides real-time audio analysis.
+ *
+ * This system provides:
+ * - Real-time audio analysis (RMS, peak, frequency bands)
+ * - Beat detection with automatic BPM tracking
+ * - Onset detection for audio-reactive triggers
+ * - Smooth energy curves with envelope followers
+ * - Tap tempo and manual BPM control
+ * - Auto-gain normalization
+ * - Spectral feature extraction
+ * - AudioContext management and cleanup
+ * - Raw FFT data access for custom analysis
+ * - Analysis results sent to worker for artist API
+ */
+export declare class AudioSystem {
+    private audioContext;
+    private analyser;
+    private mediaStreamSource;
+    private currentStream;
+    private analysisMode;
+    private workletNode;
+    private workletReady;
+    private currentSampleRate;
+    private pcmRing;
+    private pcmWriteIndex;
+    private pcmFilled;
+    private pcmHistorySeconds;
+    private lastTempoExtraction;
+    private tempoExtractionIntervalMs;
+    private analysisBackend;
+    private bandNoiseFloor;
+    /** Tracks the last non-zero BPM chosen for output to avoid dropping to 0 */
+    private lastNonZeroBpm;
+    /** Tracks which source provided the current BPM (pll | tempo | carry | default) */
+    private lastBpmSource;
+    private aubioWarningLogged;
+    private useEssentiaTempo;
+    private workletFrameCount;
+    private forceAnalyser;
+    private statusLogTimer;
+    private isAudioLab;
+    private analysisTicks;
+    private lastLoopWarn;
+    private verboseLabLogs;
+    private lastPhaseLogTime;
+    private onsetLogBuffer;
+    private debugMode;
+    private diagnosticLogger;
+    private onsetDetection;
+    private essentiaOnsetDetection;
+    private tempoInduction;
+    private pll;
+    private stateManager;
+    private lastKickDetected;
+    private volumeAutoGain;
+    private bandAutoGain;
+    /** Raw band snapshot before auto-gain (for debug) */
+    private rawBandsPreGain;
+    /** Last dt used in analysis loop (ms) */
+    private lastDtMs;
+    /** Flag to disable auto-gain for debugging */
+    private debugDisableAutoGain;
+    /** Flag to track if we've logged detection method (to avoid spam) */
+    private readonly bandNames;
+    private essentiaBandHistories;
+    private readonly essentiaHistoryWindowMs;
+    private beatMode;
+    private manualBPM;
+    private tapHistory;
+    private tapTimeout;
+    private envelopeFollowers;
+    private sensitivity;
+    private beatDetectionEnabled;
+    private onsetDetectionEnabled;
+    private autoGainEnabled;
+    /**
+     * Enable or disable comprehensive debug logging for all layers
+     * Enables enhanced logging in: MultiOnsetDetection, BeatStateManager
+     */
+    setDebugMode(enabled: boolean): void;
+    /**
+     * Reset Essentia band onset histories (used to feed tempo induction when using the Essentia backend)
+     */
+    private resetEssentiaBandHistories;
+    /**
+     * Update Essentia band onset histories and trim old entries
+     */
+    private updateEssentiaBandHistories;
+    private getEssentiaBandHistory;
+    private getEssentiaAllHistories;
+    private logVerbose;
+    /**
+     * Start/stop periodic status logging. If maxSamples provided, stops after that many ticks.
+     */
+    private toggleStatusLogger;
+    /**
+     * Handle frames pushed from AudioWorklet
+     */
+    private handleWorkletFrame;
+    /**
+     * Unified analysis pipeline (worklet and analyser paths)
+     */
+    private analyzeFrame;
+    /**
+     * Compute FFT and derived arrays
+     */
+    private computeFFT;
+    /**
+     * Calculate RMS/peak from float PCM frame (-1..1)
+     */
+    private calculateVolumeMetrics;
+    /**
+     * Calculate perceptual/log-ish bands (pre-gain), normalized by max magnitude
+     */
+    private calculateFrequencyBandsFromMagnitude;
+    /**
+     * Spectral features from magnitude spectrum
+     */
+    private calculateSpectralFeaturesFromMagnitude;
+    /**
+     * Append PCM frame to ring buffer for periodic Essentia tempo extraction
+     */
+    private appendPcmFrame;
+    /**
+     * Periodically estimate tempo using Essentia's RhythmExtractor2013 (offline chunk)
+     */
+    private runEssentiaTempoEstimate;
+    /**
+     * Run onset + beat detection pipeline and return BeatState
+     */
+    private runBeatPipeline;
+    /**
+     * Debug logging helper
+     */
+    private debugLog;
+    private fftSize;
+    private smoothingTimeConstant;
+    private analysisInterval;
+    private analysisIntervalMs;
+    private frequencyData;
+    private timeDomainData;
+    private fftEngine;
+    private fftInput;
+    private fftOutput;
+    private fftMagnitude;
+    private fftMagnitudeDb;
+    private fftPhase;
+    private hannWindow;
+    private audioState;
+    private analysisLoopId;
+    private isAnalysisRunning;
+    private lastAnalysisTimestamp;
+    private sendAnalysisResults;
+    private prevSpectralCentroid;
+    constructor(sendAnalysisResultsCallback?: (data: any) => void);
+    /**
+     * Prepare FFT buffers and windowing for the selected fftSize
+     */
+    private refreshFFTResources;
+    /**
+     * Get the current audio analysis state (for host-side usage)
+     */
+    getAudioState(): {
+        isConnected: boolean;
+        volume: {
+            current: number;
+            peak: number;
+            smoothed: number;
+        };
+        bands: {
+            low: number;
+            lowMid: number;
+            mid: number;
+            highMid: number;
+            high: number;
+            lowSmoothed: number;
+            lowMidSmoothed: number;
+            midSmoothed: number;
+            highMidSmoothed: number;
+            highSmoothed: number;
+        };
+        beat: {
+            kick: number;
+            snare: number;
+            hat: number;
+            any: number;
+            kickSmoothed: number;
+            snareSmoothed: number;
+            anySmoothed: number;
+            events: BeatEvent[];
+            bpm: number;
+            phase: number;
+            bar: number;
+            confidence: number;
+            isLocked: boolean;
+        };
+        spectral: {
+            brightness: number;
+            flatness: number;
+            flux: number;
+        };
+    };
+    /**
+     * Initialize Essentia.js (async WASM loading)
+     */
+    private initializeEssentia;
+    /**
+     * Wait for Essentia.js to initialize (for testing/benchmarking)
+     * Returns immediately if already initialized or if initialization failed
+     */
+    waitForEssentia(timeoutMs?: number): Promise<boolean>;
+    /**
+     * Handle audio stream update (called from VijiCore)
+     */
+    handleAudioStreamUpdate(data: {
+        audioStream?: MediaStream | null;
+        analysisConfig?: AnalysisConfiguration;
+        timestamp: number;
+    }): void;
+    /**
+     * Set the audio stream for analysis
+     */
+    private setAudioStream;
+    /**
+     * Disconnect current audio stream and clean up resources
+     */
+    private disconnectAudioStream;
+    /**
+     * Update analysis configuration
+     */
+    private updateAnalysisConfig;
+    /**
+     * Initialize audio worklet for high-quality capture (complex STFT path)
+     */
+    private setupAudioWorklet;
+    /**
+     * Start the audio analysis loop at high speed (8ms intervals for transient capture)
+     */
+    private startAnalysisLoop;
+    /**
+     * Stop the audio analysis loop
+     */
+    private stopAnalysisLoop;
+    /**
+     * Pause audio analysis (for tests or temporary suspension)
+     * The setInterval continues but performAnalysis() exits early
+     */
+    pauseAnalysis(): void;
+    /**
+     * Resume audio analysis after pause
+     */
+    resumeAnalysis(): void;
+    /**
+     * Perform audio analysis (called every frame)
+     * Uses industry-grade 4-layer architecture:
+     * Layer 1: MultiOnsetDetection (per-band onset detection)
+     * Layer 2: TempoInduction (dual-method BPM detection)
+     * Layer 3: PhaseLockedLoop (stable phase tracking)
+     * Layer 4: BeatStateManager (state machine + confidence)
+     */
+    private performAnalysis;
+    private applyAutoGain;
+    /**
+     * Update smooth versions of frequency bands
+     */
+    private updateSmoothBands;
+    /**
+     * Send analysis results to worker
+     */
+    private sendAnalysisResultsToWorker;
+    /**
+     * Reset audio values to defaults
+     */
+    private resetAudioValues;
+    /**
+     * Reset all audio state (called when destroying)
+     */
+    resetAudioState(): void;
+    /**
+     * Get current analysis configuration
+     */
+    getAnalysisConfig(): {
+        fftSize: number;
+        smoothing: number;
+    };
+    /**
+     * Set analysis backend preference
+     */
+    setAnalysisBackend(backend: 'auto' | 'essentia' | 'custom' | 'aubio'): void;
+    getAnalysisBackend(): 'auto' | 'essentia' | 'custom' | 'aubio';
+    /**
+     * Force analyser path (skip worklet) for debugging
+     */
+    setForceAnalyser(enabled: boolean): void;
+    isForceAnalyser(): boolean;
+    /**
+     * Enable/disable Essentia tempo extraction (disabled by default due to WASM exception config)
+     */
+    setEssentiaTempoEnabled(enabled: boolean): void;
+    isEssentiaTempoEnabled(): boolean;
+    /**
+     * Set global sensitivity multiplier
+     * @param value - Sensitivity (0.5-2.0, default 1.0)
+     */
+    setSensitivity(value: number): void;
+    /**
+     * Get current sensitivity
+     */
+    getSensitivity(): number;
+    /**
+     * Tap tempo - record tap for manual BPM
+     */
+    tapTempo(): void;
+    /**
+     * Calculate BPM from tap history
+     */
+    private calculateBPMFromTaps;
+    /**
+     * Start auto-clear timeout for tap tempo
+     */
+    private startTapClearTimeout;
+    /**
+     * Clear tap tempo history
+     */
+    clearTaps(): void;
+    /**
+     * Get tap count
+     */
+    getTapCount(): number;
+    /**
+     * Set beat sync mode
+     */
+    setBeatMode(mode: 'auto' | 'manual'): void;
+    /**
+     * Get beat sync mode
+     */
+    getBeatMode(): 'auto' | 'manual';
+    /**
+     * Set manual BPM
+     */
+    setManualBPM(bpm: number): void;
+    /**
+     * Get current BPM (manual or auto-detected)
+     */
+    getCurrentBPM(): number;
+    /**
+     * Nudge beat phase
+     */
+    nudgeBeatPhase(amount: number): void;
+    /**
+     * Reset beat phase to next beat
+     */
+    resetBeatPhase(): void;
+    /**
+     * Set FFT size
+     */
+    setFFTSize(size: 2048 | 4096 | 8192): void;
+    /**
+     * Set smoothing time constant
+     */
+    setSmoothing(value: number): void;
+    /**
+     * Enable/disable auto-gain
+     */
+    setAutoGain(enabled: boolean): void;
+    /**
+     * Enable/disable beat detection
+     */
+    setBeatDetection(enabled: boolean): void;
+    /**
+     * Enable/disable onset detection
+     */
+    setOnsetDetection(enabled: boolean): void;
+    /**
+     * Get complete audio analysis state
+     */
+    getState(): {
+        isConnected: boolean;
+        currentBPM: number;
+        confidence: number;
+        mode: 'auto' | 'manual';
+        tapCount: number;
+        isLocked: boolean;
+        sensitivity: number;
+        trackingState: string;
+    };
+    /**
+     * Get current per-frame audio analysis data (for UI display)
+     * Returns a snapshot of the latest audio analysis values
+     */
+    getCurrentAudioData(): {
+        isConnected: boolean;
+        volume: {
+            current: number;
+            peak: number;
+            smoothed: number;
+        };
+        bands: {
+            low: number;
+            lowMid: number;
+            mid: number;
+            highMid: number;
+            high: number;
+            lowSmoothed: number;
+            lowMidSmoothed: number;
+            midSmoothed: number;
+            highMidSmoothed: number;
+            highSmoothed: number;
+        };
+        beat: {
+            kick: number;
+            snare: number;
+            hat: number;
+            any: number;
+            kickSmoothed: number;
+            snareSmoothed: number;
+            anySmoothed: number;
+            events: BeatEvent[];
+            bpm: number;
+            phase: number;
+            bar: number;
+            confidence: number;
+            isLocked: boolean;
+        };
+        spectral: {
+            brightness: number;
+            flatness: number;
+            flux: number;
+        };
+    };
+    /**
+     * Get enhanced debug snapshot from all layers
+     */
+    getEnhancedDebugSnapshot(): {
+        onset: {
+            currentBPM: number;
+            debugMode: boolean;
+            totalOnsets: number;
+            currentCluster: {
+                primaryBands: string[];
+                ageMssinceStart: number;
+            } | null;
+            bandStates: Record<string, {
+                lastOnsetTime: number;
+                onsetCount: number;
+                cutoff: number;
+                minEnergy: number;
+            }>;
+        };
+        tempo: {
+            bpm: number;
+            confidence: number;
+        };
+        pll: PLLState;
+        state: {
+            trackingState: TrackingState;
+            isLocked: boolean;
+        };
+    };
+    /**
+     * Get comprehensive debug info for testing/debugging
+     * Exposes internal state of all analysis layers
+     */
+    getDebugInfo(): {
+        onset: Record<BandName, {
+            energy: number;
+            prevEnergy: number;
+            prevEnergy2: number;
+            minEnergy: number;
+            flux: number;
+            cutoff: number;
+            minThreshold: number;
+            minFlux: number;
+            wouldTrigger: boolean;
+            reason: string;
+            debounceMs: number;
+            lastOnsetAgo: number;
+            baselineAlpha?: number;
+            dtMs?: number;
+            fluxPrevOnly?: number;
+            fluxBaseline?: number;
+            initialized?: boolean;
+            lastFluxStored?: number;
+            fluxCalcNow?: number;
+        }>;
+        tempo: {
+            currentBPM: number;
+            confidence: number;
+            method: string;
+            anchorBand: string | null;
+            methodAgreement: number;
+            bpmHistory: number[];
+            envelopeLength: number;
+            lastAutocorrBPM: number;
+            lastIOIBPM: number;
+        };
+        pll: {
+            bpm: number;
+            phase: number;
+            bar: number;
+            driftRate: number;
+            state: PLLState;
+        };
+        state: {
+            trackingState: TrackingState;
+            isLocked: boolean;
+        };
+        levels: {
+            low: number;
+            lowMid: number;
+            mid: number;
+            highMid: number;
+            high: number;
+            volume: number;
+        };
+        rawBands: {
+            low: number;
+            lowMid: number;
+            mid: number;
+            highMid: number;
+            high: number;
+        };
+        bandGains: {
+            low: number;
+            lowMid: number;
+            mid: number;
+            highMid: number;
+            high: number;
+        };
+        volumeGain: number;
+        dtMs: number;
+        smoothingTimeConstant: number;
+        analysisIntervalMs: number;
+        events: BeatEvent[];
+    };
+}
+
+/**
+ * Multi-band onset detection with hybrid peak-decay + spectral flux approach
+ *
+ * Industry-grade onset detection that:
+ * - Uses 5 frequency bands for precise instrument detection
+ * - Combines peak-decay threshold (proven reliable) with spectral flux (transient detection)
+ * - Maintains per-band onset history for tempo pattern analysis
+ * - Is frame-rate independent (dt-aware)
+ * - Prevents double-triggering with adaptive cooldown
+ */
+/**
+ * Band names for the 5-band analysis
+ */
+declare type BandName = 'low' | 'lowMid' | 'mid' | 'highMid' | 'high';
+
+/**
+ * Beat event (discrete, timestamped)
+ */
+declare interface BeatEvent {
+    type: 'kick' | 'snare' | 'hat';
+    time: number;
+    strength: number;
+    confidence?: number;
+    phase?: number;
+    bar?: number;
+    isLayered?: boolean;
+    isPredicted?: boolean;
+    bpm?: number;
+    bands?: string[];
 }
 
 export declare interface CaptureFrameOptions {
-    /** MIME type for output, e.g., 'image/png', 'image/jpeg', 'image/webp' */
+    /** Output format: 'blob' for encoded image, 'bitmap' for GPU-friendly ImageBitmap */
+    format?: 'blob' | 'bitmap';
+    /** MIME type for blob output (ignored for bitmap), e.g., 'image/png', 'image/jpeg', 'image/webp' */
     type?: string;
     /**
      * Target resolution.
@@ -65,6 +681,67 @@ declare type CVFeature = 'faceDetection' | 'faceMesh' | 'handTracking' | 'poseDe
 
 declare type CVFrameRateMode = 'full' | 'half' | 'quarter' | 'eighth';
 
+/**
+ * DeviceMotionEvent acceleration data
+ * Matches native DeviceMotionEvent.acceleration structure
+ */
+declare interface DeviceMotionData {
+    /** Acceleration without gravity (m/s²) */
+    acceleration: {
+        x: number | null;
+        y: number | null;
+        z: number | null;
+    } | null;
+    /** Acceleration including gravity (m/s²) */
+    accelerationIncludingGravity: {
+        x: number | null;
+        y: number | null;
+        z: number | null;
+    } | null;
+    /** Rotation rate (degrees/second) */
+    rotationRate: {
+        alpha: number | null;
+        beta: number | null;
+        gamma: number | null;
+    } | null;
+    /** Interval between updates (milliseconds) */
+    interval: number;
+}
+
+/**
+ * DeviceOrientationEvent data
+ * Matches native DeviceOrientationEvent structure
+ */
+declare interface DeviceOrientationData {
+    /** Rotation around Z-axis (0-360 degrees, compass heading) */
+    alpha: number | null;
+    /** Rotation around X-axis (-180 to 180 degrees, front-to-back tilt) */
+    beta: number | null;
+    /** Rotation around Y-axis (-90 to 90 degrees, left-to-right tilt) */
+    gamma: number | null;
+    /** True if using magnetometer (compass) for absolute orientation */
+    absolute: boolean;
+}
+
+/**
+ * Complete device sensor state (internal device - no id/name)
+ */
+declare interface DeviceSensorState {
+    motion: DeviceMotionData | null;
+    orientation: DeviceOrientationData | null;
+    geolocation: GeolocationData | null;
+}
+
+/**
+ * External device state (includes id and name)
+ */
+declare interface DeviceState extends DeviceSensorState {
+    /** Unique device identifier */
+    id: string;
+    /** User-friendly device name */
+    name: string;
+}
+
 export declare interface FaceData {
     id: number;
     bounds: {
@@ -109,6 +786,27 @@ export declare interface FrequencyBand {
     max: number;
 }
 
+/**
+ * Geolocation API data
+ * Matches native GeolocationPosition.coords structure
+ */
+declare interface GeolocationData {
+    /** Latitude in decimal degrees */
+    latitude: number | null;
+    /** Longitude in decimal degrees */
+    longitude: number | null;
+    /** Altitude in meters above sea level */
+    altitude: number | null;
+    /** Accuracy of latitude/longitude in meters */
+    accuracy: number | null;
+    /** Accuracy of altitude in meters */
+    altitudeAccuracy: number | null;
+    /** Direction of travel (0-360 degrees, 0=North, 90=East) */
+    heading: number | null;
+    /** Speed in meters per second */
+    speed: number | null;
+}
+
 export declare interface HandData {
     id: number;
     handedness: 'left' | 'right';
@@ -210,6 +908,16 @@ export declare interface KeyboardAPI {
     meta: boolean;
 }
 
+export declare interface KeyboardEventData {
+    type: 'keydown' | 'keyup';
+    key: string;
+    code: string;
+    shiftKey: boolean;
+    ctrlKey: boolean;
+    altKey: boolean;
+    metaKey: boolean;
+}
+
 export declare interface MouseAPI {
     x: number;
     y: number;
@@ -232,6 +940,17 @@ export declare interface MouseAPI {
     wasMoved: boolean;
 }
 
+export declare interface MouseEventData {
+    x: number;
+    y: number;
+    buttons: number;
+    deltaX?: number;
+    deltaY?: number;
+    wheelDeltaX?: number;
+    wheelDeltaY?: number;
+    isInCanvas?: boolean;
+}
+
 declare interface NumberConfig {
     min?: number;
     max?: number;
@@ -312,6 +1031,35 @@ export declare interface PerformanceStats {
     };
 }
 
+/**
+ * Phase-Locked Loop for stable beat phase tracking
+ *
+ * Industry-grade phase tracking that:
+ * - Maintains expected beat times based on current BPM
+ * - Compares actual onsets to expected times (phase error)
+ * - Corrects phase gradually to avoid jumps
+ * - Handles breakdowns by extrapolating phase
+ * - Detects tempo drift in DJ sets
+ * - Provides stable bar position (0-3 for 4/4 time)
+ */
+/**
+ * PLL state for external access
+ */
+declare interface PLLState {
+    /** Current phase (0-1, where 0 = beat) */
+    phase: number;
+    /** Current bar position (0-3 for 4/4 time) */
+    bar: number;
+    /** Current BPM being tracked */
+    bpm: number;
+    /** Time since last confirmed beat (ms) */
+    timeSinceLastBeat: number;
+    /** Whether tempo is drifting (DJ transition) */
+    isDrifting: boolean;
+    /** Drift rate (BPM per second, positive = speeding up) */
+    driftRate: number;
+}
+
 declare interface PoseData {
     confidence: number;
     landmarks: {
@@ -439,6 +1187,20 @@ export declare interface TouchAPI {
     gestures: TouchGestureAPI;
 }
 
+export declare interface TouchEventData {
+    type: 'touchstart' | 'touchmove' | 'touchend' | 'touchcancel';
+    touches: Array<{
+        identifier: number;
+        clientX: number;
+        clientY: number;
+        pressure: number;
+        radiusX: number;
+        radiusY: number;
+        rotationAngle: number;
+        force: number;
+    }>;
+}
+
 declare interface TouchGestureAPI {
     isPinching: boolean;
     isRotating: boolean;
@@ -481,6 +1243,12 @@ declare interface TouchPoint {
     isEnding: boolean;
 }
 
+/**
+ * State machine states
+ * NOTE: LEARNING state removed - system now uses continuous rolling window adaptation
+ */
+declare type TrackingState = 'TRACKING' | 'LOCKED' | 'BREAKDOWN' | 'LOST';
+
 export declare const VERSION = "0.2.20";
 
 export declare interface VideoAPI {
@@ -518,9 +1286,12 @@ export declare interface VijiAPI {
     fps: number;
     audio: AudioAPI;
     video: VideoAPI;
+    streams: VideoAPI[];
     mouse: MouseAPI;
     keyboard: KeyboardAPI;
     touches: TouchAPI;
+    device: DeviceSensorState;
+    devices: DeviceState[];
     slider: (defaultValue: number, config: SliderConfig) => SliderParameter;
     color: (defaultValue: string, config: ColorConfig) => ColorParameter;
     toggle: (defaultValue: boolean, config: ToggleConfig) => ToggleParameter;
@@ -541,8 +1312,8 @@ export declare class VijiCore {
     private iframeManager;
     private workerManager;
     private interactionManager;
+    private deviceSensorManager;
     private audioSystem;
-    private videoCoordinator;
     private isInitialized;
     private isDestroyed;
     private isInitializing;
@@ -554,8 +1325,19 @@ export declare class VijiCore {
      */
     private debugLog;
     private config;
+    private isHeadless;
     private currentAudioStream;
-    private currentVideoStream;
+    private videoStream;
+    private videoStreams;
+    private mainVideoCoordinator;
+    private additionalCoordinators;
+    private directFrameSlots;
+    private latestFrameBuffer;
+    private autoCaptureEnabled;
+    private eventSourceCore;
+    private frameSourceCores;
+    private eventSourceCleanup;
+    private resizeSyncEnabled;
     private currentInteractionEnabled;
     private parameterGroups;
     private parameterValues;
@@ -567,10 +1349,33 @@ export declare class VijiCore {
     private stats;
     constructor(config: VijiCoreConfig);
     /**
-     * Capture current scene frame as an image Blob.
-     * Resolution can be a scale (0-1+) or explicit width/height with center-crop.
+     * Capture current scene frame
+     * @param options.format 'blob' = encoded image, 'bitmap' = GPU-friendly ImageBitmap
      */
+    captureFrame(options: {
+        format: 'bitmap';
+    } & Partial<CaptureFrameOptions>): Promise<ImageBitmap>;
     captureFrame(options?: CaptureFrameOptions): Promise<Blob>;
+    /**
+     * Update frame buffer (internal, called by worker message)
+     */
+    private updateFrameBuffer;
+    /**
+     * Get latest frame (transfers ownership, zero-copy)
+     */
+    getLatestFrame(): ImageBitmap | null;
+    /**
+     * Get latest frames from all linked sources
+     */
+    getLatestFramesFromSources(): (ImageBitmap | null)[];
+    /**
+     * Enable auto-capture with optional format (internal, called by linkFrameSources)
+     */
+    private enableAutoCapture;
+    /**
+     * Disable auto-capture (internal)
+     */
+    private disableAutoCapture;
     /**
      * Enable or disable debug logging
      */
@@ -579,6 +1384,11 @@ export declare class VijiCore {
      * Get current debug mode status
      */
     getDebugMode(): boolean;
+    /**
+     * Select audio analysis backend
+     */
+    setAudioAnalysisBackend(backend: 'auto' | 'essentia' | 'custom' | 'aubio'): void;
+    getAudioAnalysisBackend(): 'auto' | 'essentia' | 'custom' | 'aubio';
     /**
      * Initializes the core components in sequence
      */
@@ -591,6 +1401,14 @@ export declare class VijiCore {
      * Sets up the interaction system for Phase 7
      */
     private setupInteractionSystem;
+    /**
+     * Sets up device sensor system for internal device sensors
+     */
+    private setupDeviceSensors;
+    /**
+     * Syncs device sensor state to worker
+     */
+    private syncDeviceStateToWorker;
     /**
      * Sets up communication between components
      */
@@ -639,6 +1457,39 @@ export declare class VijiCore {
      * Get current core capabilities (what's currently active)
      */
     getCapabilities(): CoreCapabilities;
+    /**
+     * Inject mouse event (for headless cores or event forwarding)
+     */
+    injectMouseEvent(data: MouseEventData): void;
+    /**
+     * Inject keyboard event
+     */
+    injectKeyboardEvent(data: KeyboardEventData): void;
+    /**
+     * Inject touch event
+     */
+    injectTouchEvent(data: TouchEventData): void;
+    /**
+     * Inject frames directly (compositor pipeline)
+     */
+    injectFrames(bitmaps: ImageBitmap[]): void;
+    /**
+     * Link this core to receive events from a source core
+     * @param syncResolution Smart default: true for headless, false for visible
+     */
+    linkEventSource(sourceCore: VijiCore, syncResolution?: boolean): void;
+    /**
+     * Unlink from event source
+     */
+    unlinkEventSource(): void;
+    /**
+     * Link source cores that this core will pull frames from
+     */
+    linkFrameSources(...sourceCores: VijiCore[]): void;
+    /**
+     * Unlink all frame sources
+     */
+    unlinkFrameSources(): void;
     /**
      * Get parameter groups filtered by active capabilities
      */
@@ -725,17 +1576,49 @@ export declare class VijiCore {
      */
     setAudioStream(audioStream: MediaStream | null): Promise<void>;
     /**
-     * Sets the video stream for processing
+     * Sets the main video stream (CV enabled)
      */
-    setVideoStream(videoStream: MediaStream | null): Promise<void>;
+    setVideoStream(stream: MediaStream | null): Promise<void>;
     /**
-     * Gets the current audio stream
+     * Gets the main video stream
      */
-    getAudioStream(): MediaStream | null;
+    getVideoStream(): MediaStream | null;
     /**
-     * Gets the current video stream
+     * Sets all additional video streams (no CV)
      */
-    getVideoStream(): MediaStream | null;
+    setVideoStreams(streams: MediaStream[]): Promise<void>;
+    /**
+     * Gets all additional video streams
+     */
+    getVideoStreams(): MediaStream[];
+    /**
+     * Gets video stream at specific index
+     */
+    getVideoStreamAt(index: number): MediaStream | null;
+    /**
+     * Adds a video stream
+     */
+    addVideoStream(stream: MediaStream): Promise<number>;
+    /**
+     * Removes video stream at index
+     */
+    removeVideoStreamAt(index: number): Promise<void>;
+    /**
+     * Removes video stream by reference
+     */
+    removeVideoStream(stream: MediaStream): Promise<boolean>;
+    /**
+     * Updates video stream at index
+     */
+    setVideoStreamAt(index: number, stream: MediaStream): Promise<void>;
+    /**
+     * Reinitializes all additional coordinators
+     */
+    private reinitializeAdditionalCoordinators;
+    /**
+     * Gets the current audio stream
+     */
+    getAudioStream(): MediaStream | null;
     /**
      * Enables or disables user interactions (mouse, keyboard, touch) at runtime
      */
@@ -744,17 +1627,183 @@ export declare class VijiCore {
      * Gets the current interaction enabled state
      */
     getInteractionEnabled(): boolean;
+    readonly audio: {
+        /**
+         * Set global audio sensitivity (0.5-2.0, default 1.0)
+         */
+        setSensitivity: (value: number) => void;
+        /**
+         * Get current sensitivity
+         */
+        getSensitivity: () => number;
+        /**
+         * Beat control namespace
+         */
+        beat: {
+            /**
+             * Tap tempo - record tap for manual BPM
+             */
+            tap: () => void;
+            /**
+             * Clear tap tempo history
+             */
+            clearTaps: () => void;
+            /**
+             * Get tap count
+             */
+            getTapCount: () => number;
+            /**
+             * Set beat sync mode
+             */
+            setMode: (mode: "auto" | "manual") => void;
+            /**
+             * Get beat sync mode
+             */
+            getMode: () => "auto" | "manual";
+            /**
+             * Set manual BPM
+             */
+            setBPM: (bpm: number) => void;
+            /**
+             * Get current BPM (manual or auto-detected)
+             */
+            getBPM: () => number;
+            /**
+             * Nudge beat phase
+             */
+            nudge: (amount: number) => void;
+            /**
+             * Reset beat phase to next beat
+             */
+            resetPhase: () => void;
+        };
+        /**
+         * Advanced audio controls
+         */
+        advanced: {
+            /**
+             * Set FFT size (2048, 4096, or 8192)
+             */
+            setFFTSize: (size: 2048 | 4096 | 8192) => void;
+            /**
+             * Set smoothing time constant (0-1)
+             */
+            setSmoothing: (value: number) => void;
+            /**
+             * Enable/disable auto-gain
+             */
+            setAutoGain: (enabled: boolean) => void;
+            /**
+             * Enable/disable beat detection
+             */
+            setBeatDetection: (enabled: boolean) => void;
+            /**
+             * Enable/disable onset detection
+             */
+            setOnsetDetection: (enabled: boolean) => void;
+        };
+        /**
+         * Get audio analysis state (metadata)
+         */
+        getState: () => AudioAnalysisState;
+        /**
+         * Get current per-frame audio analysis data
+         * Returns volume, bands, beat energies, triggers, and spectral features
+         * Useful for UI displays and visualizations
+         */
+        getCurrentAudioData: () => {
+            isConnected: boolean;
+            volume: {
+                current: number;
+                peak: number;
+                smoothed: number;
+            };
+            bands: {
+                low: number;
+                lowMid: number;
+                mid: number;
+                highMid: number;
+                high: number;
+                lowSmoothed: number;
+                lowMidSmoothed: number;
+                midSmoothed: number;
+                highMidSmoothed: number;
+                highSmoothed: number;
+            };
+            beat: {
+                kick: number;
+                snare: number;
+                hat: number;
+                any: number;
+                kickSmoothed: number;
+                snareSmoothed: number;
+                anySmoothed: number;
+                events: BeatEvent[];
+                bpm: number;
+                phase: number;
+                bar: number;
+                confidence: number;
+                isLocked: boolean;
+            };
+            spectral: {
+                brightness: number;
+                flatness: number;
+                flux: number;
+            };
+        } | {
+            isConnected: false;
+            volume: {
+                current: number;
+                peak: number;
+                smoothed: number;
+            };
+            bands: {
+                low: number;
+                lowMid: number;
+                mid: number;
+                highMid: number;
+                high: number;
+                lowSmoothed: number;
+                lowMidSmoothed: number;
+                midSmoothed: number;
+                highMidSmoothed: number;
+                highSmoothed: number;
+            };
+            beat: {
+                kick: number;
+                snare: number;
+                hat: number;
+                any: number;
+                kickSmoothed: number;
+                snareSmoothed: number;
+                anySmoothed: number;
+                triggers: {
+                    any: boolean;
+                    kick: boolean;
+                    snare: boolean;
+                    hat: boolean;
+                };
+                bpm: number;
+                phase: number;
+                bar: number;
+                confidence: number;
+                isLocked: false;
+            };
+            spectral: {
+                brightness: number;
+                flatness: number;
+                flux: number;
+            };
+        };
+    };
     /**
-     * Updates audio analysis configuration
-     */
-    setAudioAnalysisConfig(config: {
-        fftSize?: number;
-        smoothing?: number;
-    }): Promise<void>;
-    /**
-     * Updates the canvas resolution by scale
+     * Updates the canvas resolution (scale or explicit dimensions)
+     * Works in both headless and normal modes
      */
-    setResolution(scale: number): void;
+    setResolution(resolution: number | {
+        width: number;
+        height: number;
+    }): void;
     /**
      * Detects the screen refresh rate
      */
@@ -767,6 +1816,42 @@ export declare class VijiCore {
      * Gets current performance statistics
      */
     getStats(): PerformanceStats;
+    /**
+     * Register a new external device
+     * @param device Device metadata (id and name)
+     * @returns Device index in the devices array
+     */
+    addExternalDevice(device: {
+        id: string;
+        name: string;
+    }): number;
+    /**
+     * Update sensor data for an external device
+     * @param update Device sensor update (must include id)
+     * @returns true if updated, false if device not found (not registered or removed)
+     */
+    updateExternalDevice(update: {
+        id: string;
+        name?: string;
+        motion?: DeviceMotionData | null;
+        orientation?: DeviceOrientationData | null;
+        geolocation?: GeolocationData | null;
+    }): boolean;
+    /**
+     * Remove/ban an external device
+     * Further updates for this device will be ignored
+     * @param deviceId Device identifier to remove
+     */
+    removeExternalDevice(deviceId: string): void;
+    /**
+     * Get list of connected external devices (for host UI)
+     * @returns Array of device metadata with id, name, and array index
+     */
+    getConnectedDevices(): Array<{
+        id: string;
+        name: string;
+        index: number;
+    }>;
     /**
      * Checks if the core is ready for use
      */
@@ -794,26 +1879,33 @@ export declare class VijiCore {
 }
 
 export declare interface VijiCoreConfig {
-    /** Container element where the scene will be rendered */
-    hostContainer: HTMLElement;
+    /** Container element where the scene will be rendered (optional for headless mode) */
+    hostContainer?: HTMLElement;
     /** Artist scene code to execute */
     sceneCode: string;
     /** Frame rate mode - 'full' for every animation frame, 'half' for every second frame */
     frameRateMode?: FrameRateMode;
-    /** Enable automatic performance optimization */
-    autoOptimize?: boolean;
+    /** Resolution: scale (number) or explicit dimensions (object). Required for headless, must be object form */
+    resolution?: number | {
+        width: number;
+        height: number;
+    };
     /** Audio input stream */
     audioStream?: MediaStream;
-    /** Video input stream */
+    /** Video input stream (main stream, CV enabled) */
     videoStream?: MediaStream;
+    /** Additional video input streams (no CV processing) */
+    videoStreams?: MediaStream[];
     /** Audio analysis configuration */
     analysisConfig?: AnalysisConfiguration;
     /** Initial parameter values */
     parameters?: ParameterGroup[];
     /** Disable input processing (for preview instances) */
     noInputs?: boolean;
-    /** Enable user interaction events */
+    /** Enable user interaction events (DOM event capture, disabled in headless by default) */
     allowUserInteraction?: boolean;
+    /** Enable device sensor APIs (motion, orientation, geolocation) */
+    allowDeviceInteraction?: boolean;
 }
 
 export declare class VijiCoreError extends Error {