@viji-dev/core 0.3.1 → 0.3.3

package/dist/assets/{viji.worker-Be0jZvYj.js → viji.worker-Cx1WpxdF.js}

@@ -1243,6 +1243,9 @@ class VideoSystem {
   debugMode = false;
   // GPU-only mode flag (for shader compositor only - P5/Canvas need OffscreenCanvas)
   directGPUMode = false;
+  // Stream categorization
+  streamType = "additional";
+  deviceId;
   /**
    * Enable or disable debug logging
    */
@@ -1283,6 +1286,25 @@ class VideoSystem {
   constructor() {
     this.cvSystem = new CVSystem();
   }
+  /**
+   * Set stream type and optional device ID
+   */
+  setStreamType(type, deviceId) {
+    this.streamType = type;
+    this.deviceId = deviceId;
+  }
+  /**
+   * Get stream type
+   */
+  getStreamType() {
+    return this.streamType;
+  }
+  /**
+   * Get device ID (for device streams)
+   */
+  getDeviceId() {
+    return this.deviceId;
+  }
   /**
    * Get the video API for inclusion in the viji object
    */
@@ -2134,6 +2156,8 @@ class ShaderWorkerAdapter {
   // Maximum number of compositor input streams
   static MAX_EXTERNAL_DEVICES = 8;
   // Maximum number of external devices
+  static MAX_DEVICE_VIDEOS = 8;
+  // Maximum number of device video streams
   gl;
   program = null;
   uniformLocations = /* @__PURE__ */ new Map();
@@ -2152,6 +2176,8 @@ class ShaderWorkerAdapter {
   segmentationTexture = null;
   // Multi-stream textures
   streamTextures = [];
+  // Device video textures
+  deviceTextures = new Array(ShaderWorkerAdapter.MAX_DEVICE_VIDEOS).fill(null);
   // Backbuffer support (ping-pong framebuffers)
   backbufferFramebuffer = null;
   backbufferTexture = null;
@@ -2402,6 +2428,33 @@ uniform bool u_stream5Connected; // Stream 5 connection status
 uniform bool u_stream6Connected; // Stream 6 connection status
 uniform bool u_stream7Connected; // Stream 7 connection status
 
+// Device Video Support (device cameras)
+uniform int u_deviceCount; // Number of device videos (0-8)
+uniform sampler2D u_device0; // Device 0 camera texture
+uniform sampler2D u_device1; // Device 1 camera texture
+uniform sampler2D u_device2; // Device 2 camera texture
+uniform sampler2D u_device3; // Device 3 camera texture
+uniform sampler2D u_device4; // Device 4 camera texture
+uniform sampler2D u_device5; // Device 5 camera texture
+uniform sampler2D u_device6; // Device 6 camera texture
+uniform sampler2D u_device7; // Device 7 camera texture
+uniform vec2 u_device0Resolution; // Device 0 resolution
+uniform vec2 u_device1Resolution; // Device 1 resolution
+uniform vec2 u_device2Resolution; // Device 2 resolution
+uniform vec2 u_device3Resolution; // Device 3 resolution
+uniform vec2 u_device4Resolution; // Device 4 resolution
+uniform vec2 u_device5Resolution; // Device 5 resolution
+uniform vec2 u_device6Resolution; // Device 6 resolution
+uniform vec2 u_device7Resolution; // Device 7 resolution
+uniform bool u_device0Connected; // Device 0 connection status
+uniform bool u_device1Connected; // Device 1 connection status
+uniform bool u_device2Connected; // Device 2 connection status
+uniform bool u_device3Connected; // Device 3 connection status
+uniform bool u_device4Connected; // Device 4 connection status
+uniform bool u_device5Connected; // Device 5 connection status
+uniform bool u_device6Connected; // Device 6 connection status
+uniform bool u_device7Connected; // Device 7 connection status
+
 // CV - Face Detection
 uniform int u_faceCount; // Number of detected faces (0-1)
 uniform vec4 u_face0Bounds; // First face bounding box (x, y, width, height)
@@ -2726,6 +2779,25 @@ ${error}`);
         this.setUniform(connectedUniform, "bool", false);
       }
     }
+    const devices = viji.devices || [];
+    const deviceCount = Math.min(devices.length, ShaderWorkerAdapter.MAX_DEVICE_VIDEOS);
+    this.setUniform("u_deviceCount", "int", deviceCount);
+    for (let i = 0; i < ShaderWorkerAdapter.MAX_DEVICE_VIDEOS; i++) {
+      const connectedUniform = `u_device${i}Connected`;
+      const resolutionUniform = `u_device${i}Resolution`;
+      if (i < deviceCount && devices[i]?.video?.isConnected && devices[i].video.currentFrame) {
+        this.updateDeviceTexture(i, devices[i].video.currentFrame);
+        this.setUniform(
+          resolutionUniform,
+          "vec2",
+          [devices[i].video.frameWidth, devices[i].video.frameHeight]
+        );
+        this.setUniform(connectedUniform, "bool", true);
+      } else {
+        this.setUniform(resolutionUniform, "vec2", [0, 0]);
+        this.setUniform(connectedUniform, "bool", false);
+      }
+    }
     const faces = video.faces || [];
     this.setUniform("u_faceCount", "int", faces.length);
     if (faces.length > 0) {
@@ -3118,6 +3190,42 @@ ${error}`);
       streamFrame.close();
     }
   }
+  /**
+   * Update device video texture
+   */
+  updateDeviceTexture(index, frame) {
+    if (!this.gl || index >= ShaderWorkerAdapter.MAX_DEVICE_VIDEOS) return;
+    if (!this.deviceTextures[index]) {
+      this.deviceTextures[index] = this.gl.createTexture();
+    }
+    const texture = this.deviceTextures[index];
+    if (!texture) return;
+    const textureUnit = 8 + index;
+    this.gl.activeTexture(this.gl.TEXTURE0 + textureUnit);
+    this.gl.bindTexture(this.gl.TEXTURE_2D, texture);
+    const shouldFlip = frame instanceof OffscreenCanvas;
+    if (shouldFlip) {
+      this.gl.pixelStorei(this.gl.UNPACK_FLIP_Y_WEBGL, true);
+    }
+    this.gl.texImage2D(
+      this.gl.TEXTURE_2D,
+      0,
+      this.gl.RGBA,
+      this.gl.RGBA,
+      this.gl.UNSIGNED_BYTE,
+      frame
+    );
+    if (shouldFlip) {
+      this.gl.pixelStorei(this.gl.UNPACK_FLIP_Y_WEBGL, false);
+    }
+    this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_MIN_FILTER, this.gl.LINEAR);
+    this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_MAG_FILTER, this.gl.LINEAR);
+    this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_WRAP_S, this.gl.CLAMP_TO_EDGE);
+    this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_WRAP_T, this.gl.CLAMP_TO_EDGE);
+    if (frame instanceof ImageBitmap) {
+      frame.close();
+    }
+  }
   /**
    * Update segmentation mask texture
    */
@@ -3217,6 +3325,10 @@ ${error}`);
       if (texture) gl.deleteTexture(texture);
     }
     this.streamTextures = [];
+    for (const texture of this.deviceTextures) {
+      if (texture) gl.deleteTexture(texture);
+    }
+    this.deviceTextures = [];
     for (const texture of this.textures.values()) {
       if (texture) gl.deleteTexture(texture);
     }
@@ -3281,8 +3393,6 @@ class VijiWorkerRuntime {
   interactionSystem;
   // Video systems (multi-stream) - index 0 = main with CV, 1+ = additional without CV
   videoSystems = [];
-  hasMainVideoStream = false;
-  // Track if videoSystems[0] is main stream
   // Auto-capture for frame sources
   autoCaptureEnabled = false;
   autoCaptureFormat = { flipY: true };
@@ -3579,28 +3689,13 @@ class VijiWorkerRuntime {
   }
   /**
    * Updates viji.streams from videoSystems array
-   * Slices based on whether there's a main stream at index 0
+   * Filters only 'additional' type streams (excludes main and device streams)
    */
   updateVijiStreams() {
-    const startIndex = this.hasMainVideoStream ? 1 : 0;
-    const freshStreams = this.videoSystems.slice(startIndex).map((vs) => {
-      const api = vs?.getVideoAPI() || {
-        isConnected: false,
-        currentFrame: null,
-        frameWidth: 0,
-        frameHeight: 0,
-        frameRate: 0,
-        getFrameData: () => null
-      };
-      return api;
-    });
+    const freshStreams = this.videoSystems.filter((vs) => vs && vs.getStreamType() === "additional").map((vs) => vs.getVideoAPI());
     this.viji.streams.length = freshStreams.length;
     for (let i = 0; i < freshStreams.length; i++) {
-      if (this.viji.streams[i]) {
-        Object.assign(this.viji.streams[i], freshStreams[i]);
-      } else {
-        this.viji.streams[i] = freshStreams[i];
-      }
+      this.viji.streams[i] = freshStreams[i];
     }
   }
   // Send all parameters (from helper functions) to host
@@ -3841,10 +3936,11 @@ class VijiWorkerRuntime {
     };
   }
   handleVideoCanvasSetup(message) {
-    const { streamIndex, isMain } = message.data;
+    const { streamIndex, streamType, deviceId } = message.data;
     const index = streamIndex || 0;
     const videoSystem = new VideoSystem();
     videoSystem.setDebugMode(this.debugMode);
+    videoSystem.setStreamType(streamType, deviceId);
     videoSystem.handleCanvasSetup({
       offscreenCanvas: message.data.offscreenCanvas,
       width: message.data.width,
@@ -3852,29 +3948,34 @@ class VijiWorkerRuntime {
       timestamp: message.data.timestamp
     });
     this.videoSystems[index] = videoSystem;
-    this.debugLog(`Video system setup at index ${index}, isMain: ${isMain}`);
-    if (index === 0) {
-      Object.assign(this.viji.video, videoSystem.getVideoAPI());
-    } else {
-      this.updateVijiStreams();
+    this.debugLog(`Video system setup at index ${index}, type: ${streamType}${deviceId ? `, deviceId: ${deviceId}` : ""}`);
+    switch (streamType) {
+      case "main":
+        Object.assign(this.viji.video, videoSystem.getVideoAPI());
+        break;
+      case "additional":
+        this.updateVijiStreams();
+        break;
     }
   }
   handleVideoFrameUpdate(message) {
     const index = message.data.streamIndex || 0;
-    if (this.videoSystems[index]) {
-      this.videoSystems[index].handleFrameUpdate({
+    const vs = this.videoSystems[index];
+    if (vs) {
+      vs.handleFrameUpdate({
         imageBitmap: message.data.imageBitmap,
         timestamp: message.data.timestamp
       });
-      if (index === 0) {
-        Object.assign(this.viji.video, this.videoSystems[index].getVideoAPI());
+      if (vs.getStreamType() === "main") {
+        Object.assign(this.viji.video, vs.getVideoAPI());
       }
     }
   }
   handleVideoConfigUpdate(message) {
     const index = message.data.streamIndex || 0;
-    if (this.videoSystems[index]) {
-      this.videoSystems[index].handleVideoConfigUpdate({
+    const vs = this.videoSystems[index];
+    if (vs) {
+      vs.handleVideoConfigUpdate({
         ...message.data.targetFrameRate && { targetFrameRate: message.data.targetFrameRate },
         ...message.data.cvConfig && { cvConfig: message.data.cvConfig },
         ...message.data.width && { width: message.data.width },
@@ -3882,15 +3983,14 @@ class VijiWorkerRuntime {
         ...message.data.disconnect && { disconnect: message.data.disconnect },
         timestamp: message.data.timestamp
       });
-      if (index === 0) {
-        Object.assign(this.viji.video, this.videoSystems[index].getVideoAPI());
+      if (vs.getStreamType() === "main") {
+        Object.assign(this.viji.video, vs.getVideoAPI());
       }
     }
   }
   handleVideoStreamsPrepare(message) {
     const { mainStream, mediaStreamCount, directFrameCount } = message.data;
     const totalStreams = (mainStream ? 1 : 0) + mediaStreamCount + directFrameCount;
-    this.hasMainVideoStream = mainStream;
     this.debugLog(`[Compositor] Preparing video streams: main=${mainStream}, media=${mediaStreamCount}, direct=${directFrameCount}, total=${totalStreams}`);
     while (this.videoSystems.length < totalStreams) {
       this.videoSystems.push(new VideoSystem());
@@ -4127,7 +4227,16 @@ class VijiWorkerRuntime {
   handleDeviceStateUpdate(message) {
     this.deviceState = message.data;
     this.viji.device = this.deviceState.device;
-    this.viji.devices = this.deviceState.devices;
+    const devicesWithVideo = this.deviceState.devices.map((device) => {
+      const videoSystem = this.videoSystems.find(
+        (vs) => vs && vs.getStreamType() === "device" && vs.getDeviceId() === device.id
+      );
+      return {
+        ...device,
+        video: videoSystem ? videoSystem.getVideoAPI() : null
+      };
+    });
+    this.viji.devices = devicesWithVideo;
   }
 }
 class SceneAnalyzer {
@@ -4194,4 +4303,4 @@ async function setSceneCode(sceneCode) {
   }
 }
 self.setSceneCode = setSceneCode;
-//# sourceMappingURL=viji.worker-Be0jZvYj.js.map
+//# sourceMappingURL=viji.worker-Cx1WpxdF.js.map