@viji-dev/core 0.2.19 → 0.3.0

package/dist/assets/{viji.worker-C8mrsLDG.js → viji.worker-b3XR7zKX.js}

@@ -1241,6 +1241,8 @@ class VideoSystem {
   cvScratchContext = null;
   // Debug logging control
   debugMode = false;
+  // GPU-only mode flag (for shader compositor only - P5/Canvas need OffscreenCanvas)
+  directGPUMode = false;
   /**
    * Enable or disable debug logging
    */
@@ -1286,9 +1288,13 @@ class VideoSystem {
    */
   getVideoAPI() {
     const cvResults = this.cvSystem.getResults();
+    const currentFrame = this.videoState.currentFrame;
+    if (this.directGPUMode && currentFrame instanceof ImageBitmap) {
+      this.videoState.currentFrame = null;
+    }
     return {
       isConnected: this.videoState.isConnected,
-      currentFrame: this.videoState.currentFrame,
+      currentFrame,
       frameWidth: this.videoState.frameWidth,
       frameHeight: this.videoState.frameHeight,
       frameRate: this.videoState.frameRate,
@@ -1526,6 +1532,54 @@ class VideoSystem {
   getCVStats() {
     return this.cvSystem.getStats();
   }
+  /**
+   * Initialize for direct frame injection (no MediaStream)
+   * Enables GPU-only mode for zero-copy pipeline (ImageBitmaps are pre-flipped at capture)
+   */
+  initializeForDirectFrames(consumerRendererType) {
+    this.disconnectVideo();
+    this.directGPUMode = consumerRendererType === "shader";
+    if (this.directGPUMode) {
+      this.debugLog("VideoSystem initialized in direct GPU mode (zero-copy, pre-flipped ImageBitmaps)");
+    } else {
+      this.offscreenCanvas = new OffscreenCanvas(1920, 1080);
+      this.ctx = this.offscreenCanvas.getContext("2d", { willReadFrequently: true });
+      if (!this.ctx) {
+        throw new Error("Failed to get 2D context for direct frames");
+      }
+      this.videoState.currentFrame = this.offscreenCanvas;
+      this.debugLog("VideoSystem initialized with canvas (P5/Canvas consumer)");
+    }
+    this.videoState.isConnected = false;
+  }
+  /**
+   * Handle directly injected frame (zero-copy)
+   */
+  handleDirectFrame(data) {
+    if (this.directGPUMode) {
+      this.videoState.currentFrame = data.imageBitmap;
+      this.videoState.frameWidth = data.imageBitmap.width;
+      this.videoState.frameHeight = data.imageBitmap.height;
+      this.videoState.isConnected = true;
+    } else {
+      if (!this.offscreenCanvas || !this.ctx) {
+        this.debugLog("[Compositor] Initializing canvas for direct frames");
+        this.offscreenCanvas = new OffscreenCanvas(data.imageBitmap.width, data.imageBitmap.height);
+        this.ctx = this.offscreenCanvas.getContext("2d", { willReadFrequently: true });
+      }
+      if (this.offscreenCanvas.width !== data.imageBitmap.width || this.offscreenCanvas.height !== data.imageBitmap.height) {
+        this.offscreenCanvas.width = data.imageBitmap.width;
+        this.offscreenCanvas.height = data.imageBitmap.height;
+      }
+      this.ctx.drawImage(data.imageBitmap, 0, 0);
+      this.videoState.currentFrame = this.offscreenCanvas;
+      this.videoState.frameWidth = data.imageBitmap.width;
+      this.videoState.frameHeight = data.imageBitmap.height;
+      this.videoState.isConnected = true;
+      this.processCurrentFrame(data.timestamp);
+      data.imageBitmap.close();
+    }
+  }
 }
 class P5WorkerAdapter {
   constructor(offscreenCanvas, _vijiAPI, sceneCode) {
@@ -1770,6 +1824,49 @@ class P5WorkerAdapter {
       return null;
     }
   }
+  /**
+   * Wrap video frames in P5.js-compatible format
+   * P5.js expects images to have {canvas, elt, width, height} structure
+   * This wrapping is done per-frame for P5 scenes only, keeping the artist API unchanged
+   */
+  wrapVideoFramesForP5(vijiAPI) {
+    if (vijiAPI.video?.currentFrame instanceof OffscreenCanvas) {
+      const canvas = vijiAPI.video.currentFrame;
+      vijiAPI.video.currentFrame = {
+        canvas,
+        elt: canvas,
+        width: canvas.width,
+        height: canvas.height
+      };
+    } else if (vijiAPI.video?.currentFrame instanceof ImageBitmap) {
+      const bitmap = vijiAPI.video.currentFrame;
+      vijiAPI.video.currentFrame = {
+        elt: bitmap,
+        width: bitmap.width,
+        height: bitmap.height
+      };
+    }
+    if (Array.isArray(vijiAPI.streams)) {
+      for (const stream of vijiAPI.streams) {
+        if (stream?.currentFrame instanceof OffscreenCanvas) {
+          const canvas = stream.currentFrame;
+          stream.currentFrame = {
+            canvas,
+            elt: canvas,
+            width: canvas.width,
+            height: canvas.height
+          };
+        } else if (stream?.currentFrame instanceof ImageBitmap) {
+          const bitmap = stream.currentFrame;
+          stream.currentFrame = {
+            elt: bitmap,
+            width: bitmap.width,
+            height: bitmap.height
+          };
+        }
+      }
+    }
+  }
   /**
    * Add .p5 property to image parameters for P5.js-specific rendering
    * This allows artists to use p5.image() while keeping .value for native canvas API
@@ -1810,6 +1907,7 @@ class P5WorkerAdapter {
     }
     try {
       this.addP5PropertyToImageParameters(parameterObjects);
+      this.wrapVideoFramesForP5(vijiAPI);
       if (!this.artistSetupComplete && this.setupFn) {
         this.setupFn(vijiAPI, this.p5Instance);
         this.artistSetupComplete = true;
@@ -2032,6 +2130,8 @@ class ShaderWorkerAdapter {
       this.gl = gl;
     }
   }
+  static MAX_STREAMS = 8;
+  // Maximum number of compositor input streams
   gl;
   program = null;
   uniformLocations = /* @__PURE__ */ new Map();
@@ -2048,6 +2148,8 @@ class ShaderWorkerAdapter {
   audioFFTTexture = null;
   videoTexture = null;
   segmentationTexture = null;
+  // Multi-stream textures
+  streamTextures = [];
   // Backbuffer support (ping-pong framebuffers)
   backbufferFramebuffer = null;
   backbufferTexture = null;
@@ -2271,6 +2373,33 @@ uniform sampler2D u_video; // Current video frame as texture
 uniform vec2 u_videoResolution; // Video frame width and height in pixels
 uniform float u_videoFrameRate; // Video frame rate in frames per second
 
+// Multi-Stream Compositor Support (using individual uniforms due to WebGL 1.0 limitations)
+uniform int u_streamCount; // Number of available compositor input streams (0-8)
+uniform sampler2D u_stream0; // Stream 0 texture
+uniform sampler2D u_stream1; // Stream 1 texture
+uniform sampler2D u_stream2; // Stream 2 texture
+uniform sampler2D u_stream3; // Stream 3 texture
+uniform sampler2D u_stream4; // Stream 4 texture
+uniform sampler2D u_stream5; // Stream 5 texture
+uniform sampler2D u_stream6; // Stream 6 texture
+uniform sampler2D u_stream7; // Stream 7 texture
+uniform vec2 u_stream0Resolution; // Stream 0 resolution
+uniform vec2 u_stream1Resolution; // Stream 1 resolution
+uniform vec2 u_stream2Resolution; // Stream 2 resolution
+uniform vec2 u_stream3Resolution; // Stream 3 resolution
+uniform vec2 u_stream4Resolution; // Stream 4 resolution
+uniform vec2 u_stream5Resolution; // Stream 5 resolution
+uniform vec2 u_stream6Resolution; // Stream 6 resolution
+uniform vec2 u_stream7Resolution; // Stream 7 resolution
+uniform bool u_stream0Connected; // Stream 0 connection status
+uniform bool u_stream1Connected; // Stream 1 connection status
+uniform bool u_stream2Connected; // Stream 2 connection status
+uniform bool u_stream3Connected; // Stream 3 connection status
+uniform bool u_stream4Connected; // Stream 4 connection status
+uniform bool u_stream5Connected; // Stream 5 connection status
+uniform bool u_stream6Connected; // Stream 6 connection status
+uniform bool u_stream7Connected; // Stream 7 connection status
+
 // CV - Face Detection
 uniform int u_faceCount; // Number of detected faces (0-1)
 uniform vec4 u_face0Bounds; // First face bounding box (x, y, width, height)
@@ -2364,13 +2493,22 @@ ${error}`);
     if (!this.program) return;
     const gl = this.gl;
     const numUniforms = gl.getProgramParameter(this.program, gl.ACTIVE_UNIFORMS);
+    const streamUniforms = [];
     for (let i = 0; i < numUniforms; i++) {
       const info = gl.getActiveUniform(this.program, i);
       if (info) {
         const location = gl.getUniformLocation(this.program, info.name);
         this.uniformLocations.set(info.name, location);
+        if (info.name.includes("stream") || info.name.includes("u_stream")) {
+          streamUniforms.push(info.name);
+        }
       }
     }
+    if (streamUniforms.length > 0) {
+      console.log("[ShaderAdapter] Stream-related uniforms found:", streamUniforms);
+    } else {
+      console.log("[ShaderAdapter] NO stream-related uniforms found in shader!");
+    }
   }
   /**
    * Reserve texture units for special textures
@@ -2379,6 +2517,9 @@ ${error}`);
     this.textureUnits.set("u_audioFFT", this.nextTextureUnit++);
     this.textureUnits.set("u_video", this.nextTextureUnit++);
     this.textureUnits.set("u_segmentationMask", this.nextTextureUnit++);
+    for (let i = 0; i < ShaderWorkerAdapter.MAX_STREAMS; i++) {
+      this.textureUnits.set(`u_stream${i}`, this.nextTextureUnit++);
+    }
     if (this.backbufferEnabled) {
       this.textureUnits.set("backbuffer", this.nextTextureUnit++);
     }
@@ -2490,16 +2631,16 @@ ${error}`);
       }
     }
     const audio = viji.audio;
-    this.setUniform("u_audioVolume", "float", audio.volume?.rms || 0);
+    this.setUniform("u_audioVolume", "float", audio.volume?.current || 0);
     this.setUniform("u_audioPeak", "float", audio.volume?.peak || 0);
-    this.setUniform("u_audioBass", "float", audio.bands?.bass || 0);
+    this.setUniform("u_audioBass", "float", audio.bands?.low || 0);
     this.setUniform("u_audioMid", "float", audio.bands?.mid || 0);
-    this.setUniform("u_audioTreble", "float", audio.bands?.treble || 0);
-    this.setUniform("u_audioSubBass", "float", audio.bands?.subBass || 0);
+    this.setUniform("u_audioTreble", "float", audio.bands?.high || 0);
+    this.setUniform("u_audioSubBass", "float", audio.bands?.low || 0);
     this.setUniform("u_audioLowMid", "float", audio.bands?.lowMid || 0);
     this.setUniform("u_audioHighMid", "float", audio.bands?.highMid || 0);
-    this.setUniform("u_audioPresence", "float", audio.bands?.presence || 0);
-    this.setUniform("u_audioBrilliance", "float", audio.bands?.brilliance || 0);
+    this.setUniform("u_audioPresence", "float", audio.bands?.highMid || 0);
+    this.setUniform("u_audioBrilliance", "float", audio.bands?.high || 0);
     if (audio.isConnected) {
       this.updateAudioFFTTexture(audio.getFrequencyData());
     }
@@ -2512,6 +2653,25 @@ ${error}`);
       this.setUniform("u_videoResolution", "vec2", [0, 0]);
       this.setUniform("u_videoFrameRate", "float", 0);
     }
+    const streams = viji.streams || [];
+    const streamCount = Math.min(streams.length, ShaderWorkerAdapter.MAX_STREAMS);
+    this.setUniform("u_streamCount", "int", streamCount);
+    for (let i = 0; i < ShaderWorkerAdapter.MAX_STREAMS; i++) {
+      const connectedUniform = `u_stream${i}Connected`;
+      const resolutionUniform = `u_stream${i}Resolution`;
+      if (i < streamCount && streams[i]?.isConnected && streams[i]?.currentFrame) {
+        this.updateStreamTexture(i, streams[i].currentFrame);
+        this.setUniform(
+          resolutionUniform,
+          "vec2",
+          [streams[i].frameWidth, streams[i].frameHeight]
+        );
+        this.setUniform(connectedUniform, "bool", true);
+      } else {
+        this.setUniform(resolutionUniform, "vec2", [0, 0]);
+        this.setUniform(connectedUniform, "bool", false);
+      }
+    }
     const faces = video.faces || [];
     this.setUniform("u_faceCount", "int", faces.length);
     if (faces.length > 0) {
@@ -2588,7 +2748,12 @@ ${error}`);
   updateParameterUniforms(parameterObjects) {
     for (const param of this.parameters) {
       const paramObj = parameterObjects.get(param.uniformName);
-      if (!paramObj) continue;
+      if (!paramObj) {
+        if (Math.random() < 0.01) {
+          console.log(`[ShaderAdapter] Parameter '${param.uniformName}' not found in parameterObjects`);
+        }
+        continue;
+      }
       const value = paramObj.value;
       switch (param.type) {
         case "slider":
@@ -2620,6 +2785,9 @@ ${error}`);
   setUniform(name, type, value) {
     const location = this.uniformLocations.get(name);
     if (location === null || location === void 0) {
+      if (name.includes("[") && Math.random() < 0.01) {
+        console.log(`[ShaderAdapter] Uniform '${name}' not found (location is ${location})`);
+      }
       return;
     }
     const gl = this.gl;
@@ -2691,6 +2859,7 @@ ${error}`);
   }
   /**
    * Update video texture
+   * Supports both OffscreenCanvas and ImageBitmap
    */
   updateVideoTexture(videoFrame) {
     const gl = this.gl;
@@ -2700,6 +2869,10 @@ ${error}`);
     }
     gl.activeTexture(gl.TEXTURE0 + unit);
     gl.bindTexture(gl.TEXTURE_2D, this.videoTexture);
+    const shouldFlip = videoFrame instanceof OffscreenCanvas;
+    if (shouldFlip) {
+      gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, true);
+    }
     gl.texImage2D(
       gl.TEXTURE_2D,
       0,
@@ -2708,6 +2881,9 @@ ${error}`);
       gl.UNSIGNED_BYTE,
       videoFrame
     );
+    if (shouldFlip) {
+      gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, false);
+    }
     gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
     gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR);
     gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
@@ -2716,6 +2892,49 @@ ${error}`);
     if (location) {
       gl.uniform1i(location, unit);
     }
+    if (videoFrame instanceof ImageBitmap) {
+      videoFrame.close();
+    }
+  }
+  /**
+   * Update compositor stream texture at specified index
+   * Supports both OffscreenCanvas and ImageBitmap for zero-copy pipeline
+   */
+  updateStreamTexture(index, streamFrame) {
+    const gl = this.gl;
+    const uniformName = `u_stream${index}`;
+    const unit = this.textureUnits.get(uniformName);
+    if (!this.streamTextures[index]) {
+      this.streamTextures[index] = gl.createTexture();
+    }
+    gl.activeTexture(gl.TEXTURE0 + unit);
+    gl.bindTexture(gl.TEXTURE_2D, this.streamTextures[index]);
+    const shouldFlip = streamFrame instanceof OffscreenCanvas;
+    if (shouldFlip) {
+      gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, true);
+    }
+    gl.texImage2D(
+      gl.TEXTURE_2D,
+      0,
+      gl.RGBA,
+      gl.RGBA,
+      gl.UNSIGNED_BYTE,
+      streamFrame
+    );
+    if (shouldFlip) {
+      gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, false);
+    }
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR);
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
+    const location = this.uniformLocations.get(uniformName);
+    if (location) {
+      gl.uniform1i(location, unit);
+    }
+    if (streamFrame instanceof ImageBitmap) {
+      streamFrame.close();
+    }
   }
   /**
    * Update segmentation mask texture
@@ -2800,6 +3019,32 @@ ${error}`);
   getParameterDefinitions() {
     return this.parameters;
   }
+  /**
+   * Cleanup resources
+   */
+  destroy() {
+    const gl = this.gl;
+    if (this.program) {
+      gl.deleteProgram(this.program);
+      this.program = null;
+    }
+    if (this.audioFFTTexture) gl.deleteTexture(this.audioFFTTexture);
+    if (this.videoTexture) gl.deleteTexture(this.videoTexture);
+    if (this.segmentationTexture) gl.deleteTexture(this.segmentationTexture);
+    for (const texture of this.streamTextures) {
+      if (texture) gl.deleteTexture(texture);
+    }
+    this.streamTextures = [];
+    for (const texture of this.textures.values()) {
+      if (texture) gl.deleteTexture(texture);
+    }
+    this.textures.clear();
+    if (this.backbufferFramebuffer) gl.deleteFramebuffer(this.backbufferFramebuffer);
+    if (this.backbufferTexture) gl.deleteTexture(this.backbufferTexture);
+    if (this.currentFramebuffer) gl.deleteFramebuffer(this.currentFramebuffer);
+    if (this.currentTexture) gl.deleteTexture(this.currentTexture);
+    if (this.quadBuffer) gl.deleteBuffer(this.quadBuffer);
+  }
 }
 class VijiWorkerRuntime {
   canvas = null;
@@ -2820,12 +3065,14 @@ class VijiWorkerRuntime {
   // Shader adapter for shader mode
   shaderAdapter = null;
   rendererType = "native";
+  // Pending capture requests (queue to handle multiple simultaneous requests)
+  pendingCaptures = [];
   /**
    * Enable or disable debug logging
    */
   setDebugMode(enabled) {
     this.debugMode = enabled;
-    if (this.videoSystem) this.videoSystem.setDebugMode(enabled);
+    this.videoSystems.forEach((vs) => vs?.setDebugMode(enabled));
     if (this.parameterSystem && "setDebugMode" in this.parameterSystem) {
       this.parameterSystem.setDebugMode(enabled);
     }
@@ -2850,24 +3097,61 @@ class VijiWorkerRuntime {
   parameterSystem;
   // Interaction system (Phase 7)
   interactionSystem;
-  // Video system (Phase 10) - worker-side video processing
-  videoSystem;
+  // Video systems (multi-stream) - index 0 = main with CV, 1+ = additional without CV
+  videoSystems = [];
+  hasMainVideoStream = false;
+  // Track if videoSystems[0] is main stream
+  // Auto-capture for frame sources
+  autoCaptureEnabled = false;
+  autoCaptureFormat = { flipY: true };
+  // Default: flip for WebGL compatibility
   // Audio state (Phase 5) - receives analysis results from host
   audioState = {
     isConnected: false,
-    volume: { rms: 0, peak: 0 },
+    volume: { current: 0, peak: 0, smoothed: 0 },
     bands: {
-      bass: 0,
-      mid: 0,
-      treble: 0,
-      subBass: 0,
+      low: 0,
       lowMid: 0,
+      mid: 0,
       highMid: 0,
-      presence: 0,
-      brilliance: 0
+      high: 0,
+      lowSmoothed: 0,
+      lowMidSmoothed: 0,
+      midSmoothed: 0,
+      highMidSmoothed: 0,
+      highSmoothed: 0
+    },
+    beat: {
+      kick: 0,
+      snare: 0,
+      hat: 0,
+      any: 0,
+      kickSmoothed: 0,
+      snareSmoothed: 0,
+      anySmoothed: 0,
+      triggers: { any: false, kick: false, snare: false, hat: false },
+      bpm: 120,
+      phase: 0,
+      bar: 0,
+      confidence: 0,
+      isLocked: false
+    },
+    spectral: {
+      brightness: 0,
+      flatness: 0,
+      flux: 0
     },
     frequencyData: new Uint8Array(0)
   };
+  // Device sensor state (internal device + external devices)
+  deviceState = {
+    device: {
+      motion: null,
+      orientation: null,
+      geolocation: null
+    },
+    devices: []
+  };
   // Video state is now managed by the worker-side VideoSystem
   // Artist API object
   viji = {
@@ -2885,6 +3169,7 @@ class VijiWorkerRuntime {
     fps: 60,
     // Audio API (Phase 5) - will be set in constructor
     audio: {},
+    // Main video stream (index 0, CV enabled)
     video: {
       isConnected: false,
       currentFrame: null,
@@ -2893,12 +3178,24 @@ class VijiWorkerRuntime {
       frameRate: 0,
       getFrameData: () => null,
       faces: [],
-      hands: []
+      hands: [],
+      pose: null,
+      segmentation: null,
+      cv: {}
     },
+    // Additional video streams (index 1+, no CV)
+    streams: [],
     // Interaction APIs will be added during construction
     mouse: {},
     keyboard: {},
     touches: {},
+    // Device sensor APIs (internal device + external devices)
+    device: {
+      motion: null,
+      orientation: null,
+      geolocation: null
+    },
+    devices: [],
     // Parameter helper functions (return parameter objects) - delegate to parameter system
     slider: (defaultValue, config) => {
       return this.parameterSystem.createSliderParameter(defaultValue, config);
@@ -2947,9 +3244,7 @@ class VijiWorkerRuntime {
       this.postMessage(type, data);
     });
     this.interactionSystem = new InteractionSystem();
-    this.videoSystem = new VideoSystem();
     Object.assign(this.viji, this.interactionSystem.getInteractionAPIs());
-    Object.assign(this.viji.video, this.videoSystem.getVideoAPI());
     this.viji.audio = {
       ...this.audioState,
       getFrequencyData: () => this.audioState.frequencyData
@@ -3055,16 +3350,38 @@ class VijiWorkerRuntime {
     this.interactionSystem.resetInteractionState();
     this.audioState = {
       isConnected: false,
-      volume: { rms: 0, peak: 0 },
+      volume: { current: 0, peak: 0, smoothed: 0 },
       bands: {
-        bass: 0,
-        mid: 0,
-        treble: 0,
-        subBass: 0,
+        low: 0,
         lowMid: 0,
+        mid: 0,
         highMid: 0,
-        presence: 0,
-        brilliance: 0
+        high: 0,
+        lowSmoothed: 0,
+        lowMidSmoothed: 0,
+        midSmoothed: 0,
+        highMidSmoothed: 0,
+        highSmoothed: 0
+      },
+      beat: {
+        kick: 0,
+        snare: 0,
+        hat: 0,
+        any: 0,
+        kickSmoothed: 0,
+        snareSmoothed: 0,
+        anySmoothed: 0,
+        triggers: { any: false, kick: false, snare: false, hat: false },
+        bpm: 120,
+        phase: 0,
+        bar: 0,
+        confidence: 0,
+        isLocked: false
+      },
+      spectral: {
+        brightness: 0,
+        flatness: 0,
+        flux: 0
       },
       frequencyData: new Uint8Array(0)
     };
@@ -3072,8 +3389,37 @@ class VijiWorkerRuntime {
       ...this.audioState,
       getFrequencyData: () => this.audioState.frequencyData
     };
-    this.videoSystem.resetVideoState();
-    Object.assign(this.viji.video, this.videoSystem.getVideoAPI());
+    this.videoSystems.forEach((vs) => vs.resetVideoState());
+    if (this.videoSystems[0]) {
+      Object.assign(this.viji.video, this.videoSystems[0].getVideoAPI());
+    }
+    this.updateVijiStreams();
+  }
+  /**
+   * Updates viji.streams from videoSystems array
+   * Slices based on whether there's a main stream at index 0
+   */
+  updateVijiStreams() {
+    const startIndex = this.hasMainVideoStream ? 1 : 0;
+    const freshStreams = this.videoSystems.slice(startIndex).map((vs) => {
+      const api = vs?.getVideoAPI() || {
+        isConnected: false,
+        currentFrame: null,
+        frameWidth: 0,
+        frameHeight: 0,
+        frameRate: 0,
+        getFrameData: () => null
+      };
+      return api;
+    });
+    this.viji.streams.length = freshStreams.length;
+    for (let i = 0; i < freshStreams.length; i++) {
+      if (this.viji.streams[i]) {
+        Object.assign(this.viji.streams[i], freshStreams[i]);
+      } else {
+        this.viji.streams[i] = freshStreams[i];
+      }
+    }
   }
   // Send all parameters (from helper functions) to host
   sendAllParametersToHost() {
@@ -3125,6 +3471,21 @@ class VijiWorkerRuntime {
         case "video-config-update":
           this.handleVideoConfigUpdate(message);
           break;
+        case "video-streams-prepare":
+          this.handleVideoStreamsPrepare(message);
+          break;
+        case "video-frame-direct":
+          this.handleVideoFrameDirect(message);
+          break;
+        case "enable-auto-capture":
+          this.autoCaptureEnabled = message.data.enabled;
+          if (message.data.flipY !== void 0) {
+            this.autoCaptureFormat.flipY = message.data.flipY;
+            this.debugLog(`[AutoCapture] ${message.data.enabled ? "ENABLED" : "DISABLED"} with flipY=${message.data.flipY}`);
+          } else {
+            this.debugLog(`[AutoCapture] ${message.data.enabled ? "ENABLED" : "DISABLED"}`);
+          }
+          break;
         case "mouse-update":
           this.handleMouseUpdate(message);
           break;
@@ -3137,6 +3498,9 @@ class VijiWorkerRuntime {
         case "interaction-enabled":
           this.handleInteractionEnabled(message);
           break;
+        case "device-state-update":
+          this.handleDeviceStateUpdate(message);
+          break;
         case "performance-update":
           this.handlePerformanceUpdate(message);
           break;
@@ -3183,8 +3547,8 @@ class VijiWorkerRuntime {
   handleCVFrameRateUpdate(message) {
     if (message.data && message.data.mode) {
       const sceneProcessingFPS = this.frameRateMode === "full" ? this.screenRefreshRate : this.screenRefreshRate / 2;
-      if (this.videoSystem) {
-        this.videoSystem.handleVideoConfigUpdate({
+      if (this.videoSystems[0]) {
+        this.videoSystems[0].handleVideoConfigUpdate({
           cvFrameRate: {
             mode: message.data.mode,
             sceneTargetFPS: sceneProcessingFPS
@@ -3207,7 +3571,12 @@ class VijiWorkerRuntime {
         const totalTime = this.effectiveFrameTimes[this.effectiveFrameTimes.length - 1] - this.effectiveFrameTimes[0];
         const frameCount = this.effectiveFrameTimes.length - 1;
         const effectiveRefreshRate = Math.round(frameCount / totalTime * 1e3);
-        const cvStats = this.videoSystem.getCVStats();
+        const cvStats = this.videoSystems[0]?.getCVStats() || {
+          activeFeatures: [],
+          processingTime: 0,
+          actualFPS: 0,
+          isProcessing: false
+        };
         this.postMessage("performance-update", {
           effectiveRefreshRate,
           frameRateMode: this.frameRateMode,
@@ -3265,10 +3634,23 @@ class VijiWorkerRuntime {
     this.debugLog("Stream update:", message.data);
   }
   handleAudioAnalysisUpdate(message) {
+    const events = message.data.beat.events || [];
+    const triggers = {
+      kick: events.some((e) => e.type === "kick"),
+      snare: events.some((e) => e.type === "snare"),
+      hat: events.some((e) => e.type === "hat"),
+      any: events.length > 0
+    };
     this.audioState = {
       isConnected: message.data.isConnected,
       volume: message.data.volume,
       bands: message.data.bands,
+      beat: {
+        ...message.data.beat,
+        triggers
+        // Add derived triggers (reliable!)
+      },
+      spectral: message.data.spectral,
       frequencyData: new Uint8Array(message.data.frequencyData)
     };
     this.viji.audio = {
@@ -3277,44 +3659,113 @@ class VijiWorkerRuntime {
     };
   }
   handleVideoCanvasSetup(message) {
-    this.videoSystem.handleCanvasSetup({
+    const { streamIndex, isMain } = message.data;
+    const index = streamIndex || 0;
+    const videoSystem = new VideoSystem();
+    videoSystem.setDebugMode(this.debugMode);
+    videoSystem.handleCanvasSetup({
       offscreenCanvas: message.data.offscreenCanvas,
       width: message.data.width,
       height: message.data.height,
       timestamp: message.data.timestamp
     });
-    Object.assign(this.viji.video, this.videoSystem.getVideoAPI());
+    this.videoSystems[index] = videoSystem;
+    this.debugLog(`Video system setup at index ${index}, isMain: ${isMain}`);
+    if (index === 0) {
+      Object.assign(this.viji.video, videoSystem.getVideoAPI());
+    } else {
+      this.updateVijiStreams();
+    }
   }
   handleVideoFrameUpdate(message) {
-    this.videoSystem.handleFrameUpdate({
-      imageBitmap: message.data.imageBitmap,
-      timestamp: message.data.timestamp
-    });
-    Object.assign(this.viji.video, this.videoSystem.getVideoAPI());
+    const index = message.data.streamIndex || 0;
+    if (this.videoSystems[index]) {
+      this.videoSystems[index].handleFrameUpdate({
+        imageBitmap: message.data.imageBitmap,
+        timestamp: message.data.timestamp
+      });
+      if (index === 0) {
+        Object.assign(this.viji.video, this.videoSystems[index].getVideoAPI());
+      }
+    }
   }
   handleVideoConfigUpdate(message) {
-    this.videoSystem.handleVideoConfigUpdate({
-      ...message.data.targetFrameRate && { targetFrameRate: message.data.targetFrameRate },
-      ...message.data.cvConfig && { cvConfig: message.data.cvConfig },
-      ...message.data.width && { width: message.data.width },
-      ...message.data.height && { height: message.data.height },
-      ...message.data.disconnect && { disconnect: message.data.disconnect },
-      timestamp: message.data.timestamp
-    });
-    Object.assign(this.viji.video, this.videoSystem.getVideoAPI());
+    const index = message.data.streamIndex || 0;
+    if (this.videoSystems[index]) {
+      this.videoSystems[index].handleVideoConfigUpdate({
+        ...message.data.targetFrameRate && { targetFrameRate: message.data.targetFrameRate },
+        ...message.data.cvConfig && { cvConfig: message.data.cvConfig },
+        ...message.data.width && { width: message.data.width },
+        ...message.data.height && { height: message.data.height },
+        ...message.data.disconnect && { disconnect: message.data.disconnect },
+        timestamp: message.data.timestamp
+      });
+      if (index === 0) {
+        Object.assign(this.viji.video, this.videoSystems[index].getVideoAPI());
+      }
+    }
+  }
+  handleVideoStreamsPrepare(message) {
+    const { mainStream, mediaStreamCount, directFrameCount } = message.data;
+    const totalStreams = (mainStream ? 1 : 0) + mediaStreamCount + directFrameCount;
+    this.hasMainVideoStream = mainStream;
+    this.debugLog(`[Compositor] Preparing video streams: main=${mainStream}, media=${mediaStreamCount}, direct=${directFrameCount}, total=${totalStreams}`);
+    while (this.videoSystems.length < totalStreams) {
+      this.videoSystems.push(new VideoSystem());
+    }
+    const directFrameStartIndex = (mainStream ? 1 : 0) + mediaStreamCount;
+    for (let i = 0; i < directFrameCount; i++) {
+      const index = directFrameStartIndex + i;
+      if (!this.videoSystems[index]) {
+        this.videoSystems[index] = new VideoSystem();
+      }
+      this.videoSystems[index].setDebugMode(this.debugMode);
+      this.videoSystems[index].initializeForDirectFrames(this.rendererType);
+    }
+    this.updateVijiStreams();
+    this.debugLog(`Prepared ${totalStreams} video systems (${directFrameCount} direct frames)`);
+  }
+  handleVideoFrameDirect(message) {
+    const index = message.data.streamIndex || 0;
+    if (!this.videoSystems[index]) {
+      this.debugLog(`[Compositor] Creating new VideoSystem at index ${index} for direct frames`);
+      this.videoSystems[index] = new VideoSystem();
+      this.videoSystems[index].setDebugMode(this.debugMode);
+      this.videoSystems[index].initializeForDirectFrames(this.rendererType);
+    }
+    this.videoSystems[index].handleDirectFrame(message.data);
   }
   handlePerformanceUpdate(message) {
     this.debugLog("Performance update:", message.data);
   }
   /**
    * Handle capture-frame request from host.
-   * Produces an ArrayBuffer (image bytes) to send back as transferable.
+   * Defers capture to immediately after the next render completes to avoid race conditions.
    */
   async handleCaptureFrame(message) {
+    this.pendingCaptures.push(message);
+    this.debugLog(`Capture queued: ${message.data.format || "blob"} (${this.pendingCaptures.length} pending)`);
+  }
+  /**
+   * Execute a capture frame request immediately after render completes.
+   * This ensures we capture a fully rendered frame, avoiding race conditions.
+   */
+  async executeCaptureFrame(message) {
     try {
       if (!this.canvas) {
         throw new Error("Canvas not initialized");
       }
+      const format = message.data.format || "blob";
+      if (format === "bitmap") {
+        const bitmap = this.canvas.transferToImageBitmap();
+        self.postMessage({
+          type: "capture-frame-result",
+          id: message.id,
+          timestamp: Date.now(),
+          data: { bitmap }
+        }, [bitmap]);
+        return;
+      }
       const mimeType = message.data.type || "image/jpeg";
       const srcWidth = this.canvas.width;
       const srcHeight = this.canvas.height;
@@ -3361,10 +3812,10 @@ class VijiWorkerRuntime {
         if (!sourceCtx) throw new Error("Failed to create source context");
         const imageData = new ImageData(flippedPixels, srcWidth, srcHeight);
         sourceCtx.putImageData(imageData, 0, 0);
-        this.debugLog("Captured frame from WebGL using readPixels");
+        this.debugLog("Captured frame from WebGL using readPixels (post-render)");
       } else {
         sourceCanvas = this.canvas;
-        this.debugLog("Captured frame from 2D canvas");
+        this.debugLog("Captured frame from 2D canvas (post-render)");
       }
       const temp = new OffscreenCanvas(targetWidth, targetHeight);
       const tctx = temp.getContext("2d");
@@ -3406,6 +3857,7 @@ class VijiWorkerRuntime {
     if (!this.isRunning) return;
     const currentTime = performance.now();
     this.interactionSystem.frameStart();
+    this.updateVijiStreams();
     this.viji.fps = this.frameRateMode === "full" ? this.screenRefreshRate : this.screenRefreshRate / 2;
     let shouldRender = true;
     if (this.frameRateMode === "half") {
@@ -3439,6 +3891,38 @@ class VijiWorkerRuntime {
           stack: error.stack
         });
       }
+      if (this.pendingCaptures.length > 0) {
+        const captures = [...this.pendingCaptures];
+        this.pendingCaptures = [];
+        for (const captureMsg of captures) {
+          this.executeCaptureFrame(captureMsg).catch((error) => {
+            console.error("Capture execution error:", error);
+          });
+        }
+      }
+      if (this.autoCaptureEnabled && this.canvas) {
+        try {
+          const ctx = this.canvas.getContext("2d") || this.canvas.getContext("webgl2") || this.canvas.getContext("webgl");
+          if (ctx) {
+            const options = this.autoCaptureFormat.flipY ? { imageOrientation: "flipY" } : {};
+            createImageBitmap(this.canvas, options).then((bitmap) => {
+              self.postMessage({
+                type: "auto-capture-result",
+                timestamp: Date.now(),
+                data: { bitmap, timestamp: performance.now() }
+              }, [bitmap]);
+            }).catch((err) => {
+              console.warn("[AutoCapture] ImageBitmap creation failed:", err);
+            });
+          } else {
+            if (this.debugMode && this.frameCount % 60 === 0) {
+              this.debugLog("[AutoCapture] No context yet, skipping capture");
+            }
+          }
+        } catch (error) {
+          console.warn("[AutoCapture] Failed:", error);
+        }
+      }
     }
     this.reportPerformanceStats(currentTime);
     requestAnimationFrame(() => this.renderFrame());
@@ -3464,6 +3948,11 @@ class VijiWorkerRuntime {
   handleInteractionEnabled(message) {
     this.interactionSystem.setInteractionEnabled(message.data.enabled);
   }
+  handleDeviceStateUpdate(message) {
+    this.deviceState = message.data;
+    this.viji.device = this.deviceState.device;
+    this.viji.devices = this.deviceState.devices;
+  }
 }
 class SceneAnalyzer {
   /**
@@ -3508,8 +3997,10 @@ async function setSceneCode(sceneCode) {
       runtime.sendAllParametersToHost();
     } else {
       const functionBody = sceneCode + '\nif (typeof render === "function") {\n  return render;\n}\nthrow new Error("Scene code must define a render function");';
-      const sceneFunction = new Function("viji", functionBody);
-      renderFunction = sceneFunction(runtime.viji);
+      const AsyncFunction = Object.getPrototypeOf(async function() {
+      }).constructor;
+      const sceneFunction = new AsyncFunction("viji", functionBody);
+      renderFunction = await sceneFunction(runtime.viji);
       self.renderFunction = renderFunction;
       runtime.sendAllParametersToHost();
     }
@@ -3527,4 +4018,4 @@ async function setSceneCode(sceneCode) {
   }
 }
 self.setSceneCode = setSceneCode;
-//# sourceMappingURL=viji.worker-C8mrsLDG.js.map
+//# sourceMappingURL=viji.worker-b3XR7zKX.js.map