@viji-dev/core 0.3.0 → 0.3.2

package/dist/assets/{viji.worker-b3XR7zKX.js → viji.worker-BGYUuaFF.js}

@@ -1243,6 +1243,9 @@ class VideoSystem {
   debugMode = false;
   // GPU-only mode flag (for shader compositor only - P5/Canvas need OffscreenCanvas)
   directGPUMode = false;
+  // Stream categorization
+  streamType = "additional";
+  deviceId;
   /**
    * Enable or disable debug logging
    */
@@ -1283,6 +1286,25 @@ class VideoSystem {
   constructor() {
     this.cvSystem = new CVSystem();
   }
+  /**
+   * Set stream type and optional device ID
+   */
+  setStreamType(type, deviceId) {
+    this.streamType = type;
+    this.deviceId = deviceId;
+  }
+  /**
+   * Get stream type
+   */
+  getStreamType() {
+    return this.streamType;
+  }
+  /**
+   * Get device ID (for device streams)
+   */
+  getDeviceId() {
+    return this.deviceId;
+  }
   /**
    * Get the video API for inclusion in the viji object
    */
@@ -2132,6 +2154,10 @@ class ShaderWorkerAdapter {
   }
   static MAX_STREAMS = 8;
   // Maximum number of compositor input streams
+  static MAX_EXTERNAL_DEVICES = 8;
+  // Maximum number of external devices
+  static MAX_DEVICE_VIDEOS = 8;
+  // Maximum number of device video streams
   gl;
   program = null;
   uniformLocations = /* @__PURE__ */ new Map();
@@ -2150,6 +2176,8 @@ class ShaderWorkerAdapter {
   segmentationTexture = null;
   // Multi-stream textures
   streamTextures = [];
+  // Device video textures
+  deviceTextures = new Array(ShaderWorkerAdapter.MAX_DEVICE_VIDEOS).fill(null);
   // Backbuffer support (ping-pong framebuffers)
   backbufferFramebuffer = null;
   backbufferTexture = null;
@@ -2400,6 +2428,33 @@ uniform bool u_stream5Connected; // Stream 5 connection status
 uniform bool u_stream6Connected; // Stream 6 connection status
 uniform bool u_stream7Connected; // Stream 7 connection status
 
+// Device Video Support (device cameras)
+uniform int u_deviceCount; // Number of device videos (0-8)
+uniform sampler2D u_device0; // Device 0 camera texture
+uniform sampler2D u_device1; // Device 1 camera texture
+uniform sampler2D u_device2; // Device 2 camera texture
+uniform sampler2D u_device3; // Device 3 camera texture
+uniform sampler2D u_device4; // Device 4 camera texture
+uniform sampler2D u_device5; // Device 5 camera texture
+uniform sampler2D u_device6; // Device 6 camera texture
+uniform sampler2D u_device7; // Device 7 camera texture
+uniform vec2 u_device0Resolution; // Device 0 resolution
+uniform vec2 u_device1Resolution; // Device 1 resolution
+uniform vec2 u_device2Resolution; // Device 2 resolution
+uniform vec2 u_device3Resolution; // Device 3 resolution
+uniform vec2 u_device4Resolution; // Device 4 resolution
+uniform vec2 u_device5Resolution; // Device 5 resolution
+uniform vec2 u_device6Resolution; // Device 6 resolution
+uniform vec2 u_device7Resolution; // Device 7 resolution
+uniform bool u_device0Connected; // Device 0 connection status
+uniform bool u_device1Connected; // Device 1 connection status
+uniform bool u_device2Connected; // Device 2 connection status
+uniform bool u_device3Connected; // Device 3 connection status
+uniform bool u_device4Connected; // Device 4 connection status
+uniform bool u_device5Connected; // Device 5 connection status
+uniform bool u_device6Connected; // Device 6 connection status
+uniform bool u_device7Connected; // Device 7 connection status
+
 // CV - Face Detection
 uniform int u_faceCount; // Number of detected faces (0-1)
 uniform vec4 u_face0Bounds; // First face bounding box (x, y, width, height)
@@ -2431,6 +2486,58 @@ uniform vec2 u_rightAnklePosition; // Right ankle landmark position in pixels
 uniform sampler2D u_segmentationMask; // Body segmentation mask texture (0=background, 1=person)
 uniform vec2 u_segmentationRes; // Segmentation mask resolution in pixels
 
+// Device Sensors - Internal Device (viji.device)
+uniform vec3 u_deviceAcceleration; // Acceleration without gravity (m/s²) - x, y, z
+uniform vec3 u_deviceAccelerationGravity; // Acceleration with gravity (m/s²) - x, y, z
+uniform vec3 u_deviceRotationRate; // Rotation rate (deg/s) - alpha, beta, gamma
+uniform vec3 u_deviceOrientation; // Device orientation (degrees) - alpha, beta, gamma
+uniform bool u_deviceOrientationAbsolute; // True if orientation uses magnetometer (compass)
+uniform vec2 u_deviceLocation; // Geolocation (degrees) - latitude, longitude
+uniform vec4 u_deviceLocationExt; // Extended geolocation - altitude(m), accuracy(m), heading(deg), speed(m/s)
+
+// Device Sensors - External Devices (viji.devices[0-7])
+uniform int u_externalDeviceCount; // Number of connected external devices (0-8)
+uniform vec3 u_device0Acceleration; // Device 0 acceleration without gravity (m/s²)
+uniform vec3 u_device0AccelerationGravity; // Device 0 acceleration with gravity (m/s²)
+uniform vec3 u_device0RotationRate; // Device 0 rotation rate (deg/s)
+uniform vec3 u_device0Orientation; // Device 0 orientation (degrees) - alpha, beta, gamma
+uniform vec2 u_device0Location; // Device 0 location - latitude, longitude
+uniform vec3 u_device1Acceleration; // Device 1 acceleration without gravity (m/s²)
+uniform vec3 u_device1AccelerationGravity; // Device 1 acceleration with gravity (m/s²)
+uniform vec3 u_device1RotationRate; // Device 1 rotation rate (deg/s)
+uniform vec3 u_device1Orientation; // Device 1 orientation (degrees) - alpha, beta, gamma
+uniform vec2 u_device1Location; // Device 1 location - latitude, longitude
+uniform vec3 u_device2Acceleration; // Device 2 acceleration without gravity (m/s²)
+uniform vec3 u_device2AccelerationGravity; // Device 2 acceleration with gravity (m/s²)
+uniform vec3 u_device2RotationRate; // Device 2 rotation rate (deg/s)
+uniform vec3 u_device2Orientation; // Device 2 orientation (degrees) - alpha, beta, gamma
+uniform vec2 u_device2Location; // Device 2 location - latitude, longitude
+uniform vec3 u_device3Acceleration; // Device 3 acceleration without gravity (m/s²)
+uniform vec3 u_device3AccelerationGravity; // Device 3 acceleration with gravity (m/s²)
+uniform vec3 u_device3RotationRate; // Device 3 rotation rate (deg/s)
+uniform vec3 u_device3Orientation; // Device 3 orientation (degrees) - alpha, beta, gamma
+uniform vec2 u_device3Location; // Device 3 location - latitude, longitude
+uniform vec3 u_device4Acceleration; // Device 4 acceleration without gravity (m/s²)
+uniform vec3 u_device4AccelerationGravity; // Device 4 acceleration with gravity (m/s²)
+uniform vec3 u_device4RotationRate; // Device 4 rotation rate (deg/s)
+uniform vec3 u_device4Orientation; // Device 4 orientation (degrees) - alpha, beta, gamma
+uniform vec2 u_device4Location; // Device 4 location - latitude, longitude
+uniform vec3 u_device5Acceleration; // Device 5 acceleration without gravity (m/s²)
+uniform vec3 u_device5AccelerationGravity; // Device 5 acceleration with gravity (m/s²)
+uniform vec3 u_device5RotationRate; // Device 5 rotation rate (deg/s)
+uniform vec3 u_device5Orientation; // Device 5 orientation (degrees) - alpha, beta, gamma
+uniform vec2 u_device5Location; // Device 5 location - latitude, longitude
+uniform vec3 u_device6Acceleration; // Device 6 acceleration without gravity (m/s²)
+uniform vec3 u_device6AccelerationGravity; // Device 6 acceleration with gravity (m/s²)
+uniform vec3 u_device6RotationRate; // Device 6 rotation rate (deg/s)
+uniform vec3 u_device6Orientation; // Device 6 orientation (degrees) - alpha, beta, gamma
+uniform vec2 u_device6Location; // Device 6 location - latitude, longitude
+uniform vec3 u_device7Acceleration; // Device 7 acceleration without gravity (m/s²)
+uniform vec3 u_device7AccelerationGravity; // Device 7 acceleration with gravity (m/s²)
+uniform vec3 u_device7RotationRate; // Device 7 rotation rate (deg/s)
+uniform vec3 u_device7Orientation; // Device 7 orientation (degrees) - alpha, beta, gamma
+uniform vec2 u_device7Location; // Device 7 location - latitude, longitude
+
 // Backbuffer (previous frame feedback)
 ${this.backbufferEnabled ? "uniform sampler2D backbuffer; // Previous frame texture for feedback effects" : "// backbuffer not enabled"}
 `;
@@ -2672,6 +2779,25 @@ ${error}`);
         this.setUniform(connectedUniform, "bool", false);
       }
     }
+    const devices = viji.devices || [];
+    const deviceCount = Math.min(devices.length, ShaderWorkerAdapter.MAX_DEVICE_VIDEOS);
+    this.setUniform("u_deviceCount", "int", deviceCount);
+    for (let i = 0; i < ShaderWorkerAdapter.MAX_DEVICE_VIDEOS; i++) {
+      const connectedUniform = `u_device${i}Connected`;
+      const resolutionUniform = `u_device${i}Resolution`;
+      if (i < deviceCount && devices[i]?.video?.isConnected && devices[i].video.currentFrame) {
+        this.updateDeviceTexture(i, devices[i].video.currentFrame);
+        this.setUniform(
+          resolutionUniform,
+          "vec2",
+          [devices[i].video.frameWidth, devices[i].video.frameHeight]
+        );
+        this.setUniform(connectedUniform, "bool", true);
+      } else {
+        this.setUniform(resolutionUniform, "vec2", [0, 0]);
+        this.setUniform(connectedUniform, "bool", false);
+      }
+    }
     const faces = video.faces || [];
     this.setUniform("u_faceCount", "int", faces.length);
     if (faces.length > 0) {
@@ -2741,6 +2867,134 @@ ${error}`);
     } else {
       this.setUniform("u_segmentationRes", "vec2", [0, 0]);
     }
+    const internalDevice = viji.device;
+    if (internalDevice) {
+      const motion = internalDevice.motion;
+      if (motion?.acceleration) {
+        this.setUniform("u_deviceAcceleration", "vec3", [
+          motion.acceleration.x ?? 0,
+          motion.acceleration.y ?? 0,
+          motion.acceleration.z ?? 0
+        ]);
+      } else {
+        this.setUniform("u_deviceAcceleration", "vec3", [0, 0, 0]);
+      }
+      if (motion?.accelerationIncludingGravity) {
+        this.setUniform("u_deviceAccelerationGravity", "vec3", [
+          motion.accelerationIncludingGravity.x ?? 0,
+          motion.accelerationIncludingGravity.y ?? 0,
+          motion.accelerationIncludingGravity.z ?? 0
+        ]);
+      } else {
+        this.setUniform("u_deviceAccelerationGravity", "vec3", [0, 0, 0]);
+      }
+      if (motion?.rotationRate) {
+        this.setUniform("u_deviceRotationRate", "vec3", [
+          motion.rotationRate.alpha ?? 0,
+          motion.rotationRate.beta ?? 0,
+          motion.rotationRate.gamma ?? 0
+        ]);
+      } else {
+        this.setUniform("u_deviceRotationRate", "vec3", [0, 0, 0]);
+      }
+      const orientation = internalDevice.orientation;
+      if (orientation) {
+        this.setUniform("u_deviceOrientation", "vec3", [
+          orientation.alpha ?? 0,
+          orientation.beta ?? 0,
+          orientation.gamma ?? 0
+        ]);
+        this.setUniform("u_deviceOrientationAbsolute", "bool", orientation.absolute);
+      } else {
+        this.setUniform("u_deviceOrientation", "vec3", [0, 0, 0]);
+        this.setUniform("u_deviceOrientationAbsolute", "bool", false);
+      }
+      const geolocation = internalDevice.geolocation;
+      if (geolocation) {
+        this.setUniform("u_deviceLocation", "vec2", [
+          geolocation.latitude ?? 0,
+          geolocation.longitude ?? 0
+        ]);
+        this.setUniform("u_deviceLocationExt", "vec4", [
+          geolocation.altitude ?? 0,
+          geolocation.accuracy ?? 0,
+          geolocation.heading ?? 0,
+          geolocation.speed ?? 0
+        ]);
+      } else {
+        this.setUniform("u_deviceLocation", "vec2", [0, 0]);
+        this.setUniform("u_deviceLocationExt", "vec4", [0, 0, 0, 0]);
+      }
+    } else {
+      this.setUniform("u_deviceAcceleration", "vec3", [0, 0, 0]);
+      this.setUniform("u_deviceAccelerationGravity", "vec3", [0, 0, 0]);
+      this.setUniform("u_deviceRotationRate", "vec3", [0, 0, 0]);
+      this.setUniform("u_deviceOrientation", "vec3", [0, 0, 0]);
+      this.setUniform("u_deviceOrientationAbsolute", "bool", false);
+      this.setUniform("u_deviceLocation", "vec2", [0, 0]);
+      this.setUniform("u_deviceLocationExt", "vec4", [0, 0, 0, 0]);
+    }
+    const externalDevices = viji.devices || [];
+    const externalDeviceCount = Math.min(externalDevices.length, ShaderWorkerAdapter.MAX_EXTERNAL_DEVICES);
+    this.setUniform("u_externalDeviceCount", "int", externalDeviceCount);
+    for (let i = 0; i < ShaderWorkerAdapter.MAX_EXTERNAL_DEVICES; i++) {
+      if (i < externalDeviceCount) {
+        const device = externalDevices[i];
+        const motion = device.motion;
+        if (motion?.acceleration) {
+          this.setUniform(`u_device${i}Acceleration`, "vec3", [
+            motion.acceleration.x ?? 0,
+            motion.acceleration.y ?? 0,
+            motion.acceleration.z ?? 0
+          ]);
+        } else {
+          this.setUniform(`u_device${i}Acceleration`, "vec3", [0, 0, 0]);
+        }
+        if (motion?.accelerationIncludingGravity) {
+          this.setUniform(`u_device${i}AccelerationGravity`, "vec3", [
+            motion.accelerationIncludingGravity.x ?? 0,
+            motion.accelerationIncludingGravity.y ?? 0,
+            motion.accelerationIncludingGravity.z ?? 0
+          ]);
+        } else {
+          this.setUniform(`u_device${i}AccelerationGravity`, "vec3", [0, 0, 0]);
+        }
+        if (motion?.rotationRate) {
+          this.setUniform(`u_device${i}RotationRate`, "vec3", [
+            motion.rotationRate.alpha ?? 0,
+            motion.rotationRate.beta ?? 0,
+            motion.rotationRate.gamma ?? 0
+          ]);
+        } else {
+          this.setUniform(`u_device${i}RotationRate`, "vec3", [0, 0, 0]);
+        }
+        const orientation = device.orientation;
+        if (orientation) {
+          this.setUniform(`u_device${i}Orientation`, "vec3", [
+            orientation.alpha ?? 0,
+            orientation.beta ?? 0,
+            orientation.gamma ?? 0
+          ]);
+        } else {
+          this.setUniform(`u_device${i}Orientation`, "vec3", [0, 0, 0]);
+        }
+        const geolocation = device.geolocation;
+        if (geolocation) {
+          this.setUniform(`u_device${i}Location`, "vec2", [
+            geolocation.latitude ?? 0,
+            geolocation.longitude ?? 0
+          ]);
+        } else {
+          this.setUniform(`u_device${i}Location`, "vec2", [0, 0]);
+        }
+      } else {
+        this.setUniform(`u_device${i}Acceleration`, "vec3", [0, 0, 0]);
+        this.setUniform(`u_device${i}AccelerationGravity`, "vec3", [0, 0, 0]);
+        this.setUniform(`u_device${i}RotationRate`, "vec3", [0, 0, 0]);
+        this.setUniform(`u_device${i}Orientation`, "vec3", [0, 0, 0]);
+        this.setUniform(`u_device${i}Location`, "vec2", [0, 0]);
+      }
+    }
   }
   /**
    * Update parameter uniforms from parameter objects
@@ -2936,6 +3190,42 @@ ${error}`);
       streamFrame.close();
     }
   }
+  /**
+   * Update device video texture
+   */
+  updateDeviceTexture(index, frame) {
+    if (!this.gl || index >= ShaderWorkerAdapter.MAX_DEVICE_VIDEOS) return;
+    if (!this.deviceTextures[index]) {
+      this.deviceTextures[index] = this.gl.createTexture();
+    }
+    const texture = this.deviceTextures[index];
+    if (!texture) return;
+    const textureUnit = 8 + index;
+    this.gl.activeTexture(this.gl.TEXTURE0 + textureUnit);
+    this.gl.bindTexture(this.gl.TEXTURE_2D, texture);
+    const shouldFlip = frame instanceof OffscreenCanvas;
+    if (shouldFlip) {
+      this.gl.pixelStorei(this.gl.UNPACK_FLIP_Y_WEBGL, true);
+    }
+    this.gl.texImage2D(
+      this.gl.TEXTURE_2D,
+      0,
+      this.gl.RGBA,
+      this.gl.RGBA,
+      this.gl.UNSIGNED_BYTE,
+      frame
+    );
+    if (shouldFlip) {
+      this.gl.pixelStorei(this.gl.UNPACK_FLIP_Y_WEBGL, false);
+    }
+    this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_MIN_FILTER, this.gl.LINEAR);
+    this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_MAG_FILTER, this.gl.LINEAR);
+    this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_WRAP_S, this.gl.CLAMP_TO_EDGE);
+    this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_WRAP_T, this.gl.CLAMP_TO_EDGE);
+    if (frame instanceof ImageBitmap) {
+      frame.close();
+    }
+  }
   /**
    * Update segmentation mask texture
    */
@@ -3035,6 +3325,10 @@ ${error}`);
       if (texture) gl.deleteTexture(texture);
     }
     this.streamTextures = [];
+    for (const texture of this.deviceTextures) {
+      if (texture) gl.deleteTexture(texture);
+    }
+    this.deviceTextures = [];
     for (const texture of this.textures.values()) {
       if (texture) gl.deleteTexture(texture);
     }
@@ -3099,8 +3393,6 @@ class VijiWorkerRuntime {
   interactionSystem;
   // Video systems (multi-stream) - index 0 = main with CV, 1+ = additional without CV
   videoSystems = [];
-  hasMainVideoStream = false;
-  // Track if videoSystems[0] is main stream
   // Auto-capture for frame sources
   autoCaptureEnabled = false;
   autoCaptureFormat = { flipY: true };
@@ -3397,28 +3689,13 @@ class VijiWorkerRuntime {
   }
   /**
    * Updates viji.streams from videoSystems array
-   * Slices based on whether there's a main stream at index 0
+   * Filters only 'additional' type streams (excludes main and device streams)
    */
   updateVijiStreams() {
-    const startIndex = this.hasMainVideoStream ? 1 : 0;
-    const freshStreams = this.videoSystems.slice(startIndex).map((vs) => {
-      const api = vs?.getVideoAPI() || {
-        isConnected: false,
-        currentFrame: null,
-        frameWidth: 0,
-        frameHeight: 0,
-        frameRate: 0,
-        getFrameData: () => null
-      };
-      return api;
-    });
+    const freshStreams = this.videoSystems.filter((vs) => vs && vs.getStreamType() === "additional").map((vs) => vs.getVideoAPI());
     this.viji.streams.length = freshStreams.length;
     for (let i = 0; i < freshStreams.length; i++) {
-      if (this.viji.streams[i]) {
-        Object.assign(this.viji.streams[i], freshStreams[i]);
-      } else {
-        this.viji.streams[i] = freshStreams[i];
-      }
+      this.viji.streams[i] = freshStreams[i];
     }
   }
   // Send all parameters (from helper functions) to host
@@ -3659,10 +3936,11 @@ class VijiWorkerRuntime {
     };
   }
   handleVideoCanvasSetup(message) {
-    const { streamIndex, isMain } = message.data;
+    const { streamIndex, streamType, deviceId } = message.data;
     const index = streamIndex || 0;
     const videoSystem = new VideoSystem();
     videoSystem.setDebugMode(this.debugMode);
+    videoSystem.setStreamType(streamType, deviceId);
     videoSystem.handleCanvasSetup({
       offscreenCanvas: message.data.offscreenCanvas,
       width: message.data.width,
@@ -3670,11 +3948,14 @@ class VijiWorkerRuntime {
       timestamp: message.data.timestamp
     });
     this.videoSystems[index] = videoSystem;
-    this.debugLog(`Video system setup at index ${index}, isMain: ${isMain}`);
-    if (index === 0) {
-      Object.assign(this.viji.video, videoSystem.getVideoAPI());
-    } else {
-      this.updateVijiStreams();
+    this.debugLog(`Video system setup at index ${index}, type: ${streamType}${deviceId ? `, deviceId: ${deviceId}` : ""}`);
+    switch (streamType) {
+      case "main":
+        Object.assign(this.viji.video, videoSystem.getVideoAPI());
+        break;
+      case "additional":
+        this.updateVijiStreams();
+        break;
     }
   }
   handleVideoFrameUpdate(message) {
@@ -3708,7 +3989,6 @@ class VijiWorkerRuntime {
   handleVideoStreamsPrepare(message) {
     const { mainStream, mediaStreamCount, directFrameCount } = message.data;
     const totalStreams = (mainStream ? 1 : 0) + mediaStreamCount + directFrameCount;
-    this.hasMainVideoStream = mainStream;
     this.debugLog(`[Compositor] Preparing video streams: main=${mainStream}, media=${mediaStreamCount}, direct=${directFrameCount}, total=${totalStreams}`);
     while (this.videoSystems.length < totalStreams) {
       this.videoSystems.push(new VideoSystem());
@@ -3822,18 +4102,12 @@ class VijiWorkerRuntime {
       if (!tctx) throw new Error("Failed to get 2D context");
       tctx.drawImage(sourceCanvas, sx, sy, sWidth, sHeight, 0, 0, targetWidth, targetHeight);
       const blob = await temp.convertToBlob({ type: mimeType });
-      const arrayBuffer = await blob.arrayBuffer();
       self.postMessage({
         type: "capture-frame-result",
         id: message.id,
         timestamp: Date.now(),
-        data: {
-          mimeType,
-          buffer: arrayBuffer,
-          width: targetWidth,
-          height: targetHeight
-        }
-      }, [arrayBuffer]);
+        data: { blob }
+      });
     } catch (error) {
       this.postMessage("error", {
         id: message.id,
@@ -3951,7 +4225,16 @@ class VijiWorkerRuntime {
   handleDeviceStateUpdate(message) {
     this.deviceState = message.data;
     this.viji.device = this.deviceState.device;
-    this.viji.devices = this.deviceState.devices;
+    const devicesWithVideo = this.deviceState.devices.map((device) => {
+      const videoSystem = this.videoSystems.find(
+        (vs) => vs && vs.getStreamType() === "device" && vs.getDeviceId() === device.id
+      );
+      return {
+        ...device,
+        video: videoSystem ? videoSystem.getVideoAPI() : null
+      };
+    });
+    this.viji.devices = devicesWithVideo;
   }
 }
 class SceneAnalyzer {
@@ -4018,4 +4301,4 @@ async function setSceneCode(sceneCode) {
   }
 }
 self.setSceneCode = setSceneCode;
-//# sourceMappingURL=viji.worker-b3XR7zKX.js.map
+//# sourceMappingURL=viji.worker-BGYUuaFF.js.map