mujoco-react 12.1.2 → 12.2.0

package/dist/{chunk-CV6LBHC4.js → chunk-SP72GTOD.js}

@@ -1,4 +1,4 @@
-import * as THREE from 'three';
+import * as THREE2 from 'three';
 import { useThree } from '@react-three/fiber';
 import { useMemo, useEffect } from 'react';
 import { jsxs, Fragment, jsx } from 'react/jsx-runtime';
@@ -174,6 +174,64 @@ async function dataUrlToPolicyImageTensor(dataUrl, options) {
     resolved
   );
 }
+function createMetricDepthReader(width, height, texture) {
+  const target = new THREE2.WebGLRenderTarget(width, height, { depthBuffer: false });
+  const material = new THREE2.ShaderMaterial({
+    uniforms: { depthTexture: { value: texture } },
+    vertexShader: `varying vec2 vUv;
+      void main() { vUv = uv; gl_Position = vec4(position.xy, 0.0, 1.0); }`,
+    fragmentShader: `#include <packing>
+      uniform sampler2D depthTexture;
+      varying vec2 vUv;
+      void main() { gl_FragColor = packDepthToRGBA(texture2D(depthTexture, vUv).r); }`,
+    depthTest: false,
+    depthWrite: false,
+    blending: THREE2.NoBlending,
+    toneMapped: false
+  });
+  const geometry = new THREE2.PlaneGeometry(2, 2);
+  const scene = new THREE2.Scene();
+  scene.add(new THREE2.Mesh(geometry, material));
+  const screenCamera = new THREE2.Camera();
+  const packed = new Uint8Array(width * height * 4);
+  return {
+    read(renderer, camera, flipX) {
+      renderer.setRenderTarget(target);
+      renderer.clear();
+      renderer.render(scene, screenCamera);
+      renderer.readRenderTargetPixels(target, 0, 0, width, height, packed);
+      const result = new Float32Array(width * height);
+      const inverse = camera.projectionMatrixInverse.elements;
+      for (let y = 0; y < height; y++) {
+        for (let x = 0; x < width; x++) {
+          const sx = flipX ? width - x - 1 : x;
+          const sy = height - y - 1;
+          const offset = (sy * width + sx) * 4;
+          const r = packed[offset], g = packed[offset + 1], b = packed[offset + 2], a = packed[offset + 3];
+          if (r === 255 && g === 255 && b === 255 && a === 255) {
+            result[y * width + x] = Infinity;
+            continue;
+          }
+          const d = r / 256 + g / 65536 + b / 16777216 + a / (255 * 16777216);
+          const nx = (sx + 0.5) / width * 2 - 1;
+          const ny = (sy + 0.5) / height * 2 - 1;
+          const nz = d * 2 - 1;
+          const z = inverse[2] * nx + inverse[6] * ny + inverse[10] * nz + inverse[14];
+          const w = inverse[3] * nx + inverse[7] * ny + inverse[11] * nz + inverse[15];
+          result[y * width + x] = -z / w;
+        }
+      }
+      return result;
+    },
+    dispose() {
+      target.dispose();
+      material.dispose();
+      geometry.dispose();
+    }
+  };
+}
+
+// src/rendering/cameraFrameCapture.ts
 var CAMERA_FRAME_CAPTURE_RENDER_USER_DATA_KEY = "mujocoReactCameraFrameCaptureRender";
 var CAMERA_FRAME_CAPTURE_PRE_RENDER_USER_DATA_KEY = "mujocoReactCameraFrameCapturePreRender";
 var CAPTURE_EXCLUDE_KEY = "mujoco.capture.exclude";
@@ -229,7 +287,7 @@ function createIsolatedRenderer(sourceRenderer, width, height, options) {
 function createUncachedIsolatedRenderer(sourceRenderer, width, height, options) {
   const isolation = getRenderIsolationOptions(options);
   const canvas = document.createElement("canvas");
-  const renderer = new THREE.WebGLRenderer({
+  const renderer = new THREE2.WebGLRenderer({
     canvas,
     antialias: isolation.antialias ?? false,
     alpha: isolation.alpha ?? false,
@@ -245,13 +303,13 @@ function createUncachedIsolatedRenderer(sourceRenderer, width, height, options)
 }
 function toVector3(value, fallback) {
   if (!value) return fallback.clone();
-  return value instanceof THREE.Vector3 ? value.clone() : new THREE.Vector3(value[0], value[1], value[2]);
+  return value instanceof THREE2.Vector3 ? value.clone() : new THREE2.Vector3(value[0], value[1], value[2]);
 }
 function applyCameraPose(camera, options, fallbackCamera) {
   camera.position.copy(toVector3(options.position, fallbackCamera.position));
   camera.up.copy(toVector3(options.up, fallbackCamera.up));
   if (options.quaternion) {
-    if (options.quaternion instanceof THREE.Quaternion) {
+    if (options.quaternion instanceof THREE2.Quaternion) {
       camera.quaternion.copy(options.quaternion);
     } else {
       camera.quaternion.set(
@@ -262,7 +320,7 @@ function applyCameraPose(camera, options, fallbackCamera) {
       );
     }
   } else if (options.lookAt) {
-    camera.lookAt(toVector3(options.lookAt, new THREE.Vector3()));
+    camera.lookAt(toVector3(options.lookAt, new THREE2.Vector3()));
   } else {
     camera.quaternion.copy(fallbackCamera.quaternion);
   }
@@ -270,7 +328,7 @@ function applyCameraPose(camera, options, fallbackCamera) {
 }
 function applyProjectionMatrix(camera, projectionMatrix) {
   if (!projectionMatrix) return;
-  if (projectionMatrix instanceof THREE.Matrix4) {
+  if (projectionMatrix instanceof THREE2.Matrix4) {
     camera.projectionMatrix.copy(projectionMatrix);
   } else {
     camera.projectionMatrix.fromArray(projectionMatrix);
@@ -278,14 +336,19 @@ function applyProjectionMatrix(camera, projectionMatrix) {
   camera.projectionMatrixInverse.copy(camera.projectionMatrix).invert();
 }
 function createCaptureCamera(options, fallbackCamera, width, height) {
-  const camera = options.camera ? options.camera.clone() : fallbackCamera instanceof THREE.PerspectiveCamera ? fallbackCamera.clone() : new THREE.PerspectiveCamera(45, width / height, 0.01, 100);
-  if (camera instanceof THREE.PerspectiveCamera) {
+  const camera = options.camera ? options.camera.clone() : fallbackCamera instanceof THREE2.PerspectiveCamera ? fallbackCamera.clone() : new THREE2.PerspectiveCamera(45, width / height, 0.01, 100);
+  if (camera instanceof THREE2.PerspectiveCamera) {
     camera.aspect = width / height;
     camera.fov = options.fov ?? camera.fov;
     camera.near = options.near ?? camera.near;
     camera.far = options.far ?? camera.far;
     camera.updateProjectionMatrix();
   }
+  if (camera instanceof THREE2.OrthographicCamera) {
+    camera.near = options.near ?? camera.near;
+    camera.far = options.far ?? camera.far;
+    camera.updateProjectionMatrix();
+  }
   applyProjectionMatrix(camera, options.projectionMatrix);
   applyCameraPose(camera, options, fallbackCamera);
   return camera;
@@ -305,20 +368,25 @@ function prepareCaptureCamera(camera, options, fallbackCamera, width, height) {
   if (options.camera) {
     camera.copy(options.camera);
   }
-  if (camera instanceof THREE.PerspectiveCamera) {
+  if (camera instanceof THREE2.PerspectiveCamera) {
     camera.aspect = width / height;
     camera.fov = options.fov ?? camera.fov;
     camera.near = options.near ?? camera.near;
     camera.far = options.far ?? camera.far;
     camera.updateProjectionMatrix();
   }
+  if (camera instanceof THREE2.OrthographicCamera) {
+    camera.near = options.near ?? camera.near;
+    camera.far = options.far ?? camera.far;
+    camera.updateProjectionMatrix();
+  }
   applyProjectionMatrix(camera, options.projectionMatrix);
   applyCameraPose(camera, options, fallbackCamera);
 }
 function readRenderTargetToCanvas(renderer, target, canvas, context, pixels, imageData, width, height, outputColorSpace, flipX = false, readback = true) {
   if (readback) renderer.readRenderTargetPixels(target, 0, 0, width, height, pixels);
   const rowBytes = width * 4;
-  const encodeSrgb = outputColorSpace === THREE.SRGBColorSpace;
+  const encodeSrgb = outputColorSpace === THREE2.SRGBColorSpace;
   for (let y = 0; y < height; y += 1) {
     const sourceStart = (height - y - 1) * rowBytes;
     const targetStart = y * rowBytes;
@@ -421,9 +489,9 @@ function getCameraFrameCaptureSource(options) {
   return { kind: "fallback-camera" };
 }
 function saveRendererState(renderer) {
-  const viewport = new THREE.Vector4();
-  const scissor = new THREE.Vector4();
-  const clearColor = new THREE.Color();
+  const viewport = new THREE2.Vector4();
+  const scissor = new THREE2.Vector4();
+  const clearColor = new THREE2.Color();
   renderer.getViewport(viewport);
   renderer.getScissor(scissor);
   renderer.getClearColor(clearColor);
@@ -474,7 +542,7 @@ function applyCaptureVisualOverrides(renderer, scene, options) {
   const previousSceneState = saveSceneVisualState(scene);
   if (hasOwn(overrides, "sceneBackground")) {
     const background = overrides.sceneBackground;
-    scene.background = background === false ? null : typeof background === "string" || typeof background === "number" ? new THREE.Color(background) : background ?? null;
+    scene.background = background === false ? null : typeof background === "string" || typeof background === "number" ? new THREE2.Color(background) : background ?? null;
   }
   if (hasOwn(overrides, "sceneEnvironment")) {
     const environment = overrides.sceneEnvironment;
@@ -517,9 +585,9 @@ function createCameraFrameCaptureSession(renderer, scene, fallbackCamera, option
   const isolatedRenderer = createIsolatedRenderer(renderer, width, height, options);
   const sessionRenderer = isolatedRenderer?.renderer ?? renderer;
   const camera = createCaptureCamera(options, fallbackCamera, width, height);
-  const target = new THREE.WebGLRenderTarget(width, height, {
-    format: THREE.RGBAFormat,
-    type: THREE.UnsignedByteType
+  const target = new THREE2.WebGLRenderTarget(width, height, {
+    format: THREE2.RGBAFormat,
+    type: THREE2.UnsignedByteType
   });
   const canvas = document.createElement("canvas");
   canvas.width = width;
@@ -532,12 +600,14 @@ function createCameraFrameCaptureSession(renderer, scene, fallbackCamera, option
   const drawContext = context;
   const pixels = new Uint8Array(width * height * 4);
   const imageData = drawContext.createImageData(width, height);
+  let depthReader;
   let inFlight = false, disposed = false;
   function assertAvailable() {
     if (disposed) throw new Error("Camera capture session is disposed");
     if (inFlight) throw new Error("Camera capture session is busy; await the current capture");
   }
   function releaseResources() {
+    depthReader?.dispose();
     target.dispose();
     if (isolatedRenderer && !isolatedRenderer.cached) isolatedRenderer.renderer.dispose();
   }
@@ -576,7 +646,7 @@ function createCameraFrameCaptureSession(renderer, scene, fallbackCamera, option
       sessionRenderer.xr.enabled = false;
       if (captureOptions.background !== void 0) {
         sessionRenderer.setClearColor(
-          new THREE.Color(captureOptions.background),
+          new THREE2.Color(captureOptions.background),
           captureOptions.backgroundAlpha ?? previousState.clearAlpha
         );
       } else if (captureOptions.backgroundAlpha !== void 0) {
@@ -592,8 +662,6 @@ function createCameraFrameCaptureSession(renderer, scene, fallbackCamera, option
   function renderCaptureToTarget(captureOptions, readback) {
     withCaptureState(captureOptions, () => {
       sessionRenderer.setRenderTarget(target);
-      sessionRenderer.setViewport(0, 0, width, height);
-      sessionRenderer.setScissor(0, 0, width, height);
       sessionRenderer.setScissorTest(false);
       sessionRenderer.clear();
       sessionRenderer.render(scene, camera);
@@ -641,6 +709,53 @@ function createCameraFrameCaptureSession(renderer, scene, fallbackCamera, option
       source: getCameraFrameCaptureSource(captureOptions)
     };
   }
+  function captureRGBD(nextOptions = {}) {
+    assertAvailable();
+    const captureOptions = resolveCaptureOptions(nextOptions);
+    if (getCaptureRenderer(scene)) {
+      throw new Error("Metric RGB-D capture requires the standard Three renderer; custom capture renderers do not provide aligned depth.");
+    }
+    if (sessionRenderer.capabilities.logarithmicDepthBuffer || sessionRenderer.capabilities.reversedDepthBuffer) {
+      throw new Error("Metric RGB-D capture does not support logarithmic or reversed depth buffers.");
+    }
+    if (!camera.projectionMatrix.elements.every(Number.isFinite) || !Number.isFinite(camera.projectionMatrix.determinant()) || camera.projectionMatrix.determinant() === 0) {
+      throw new Error("Metric RGB-D capture requires a finite, invertible projection matrix.");
+    }
+    if (!depthReader) {
+      target.dispose();
+      target.depthTexture = new THREE2.DepthTexture(width, height, THREE2.UnsignedIntType);
+      depthReader = createMetricDepthReader(width, height, target.depthTexture);
+    }
+    const rgb = new Uint8Array(width * height * 3);
+    let depth;
+    renderCaptureToTarget(captureOptions, () => {
+      sessionRenderer.readRenderTargetPixels(target, 0, 0, width, height, pixels);
+      for (let y = 0; y < height; y++) {
+        for (let x = 0; x < width; x++) {
+          const sx = captureOptions.flipX ? width - x - 1 : x;
+          const src = ((height - y - 1) * width + sx) * 4;
+          const dst = (y * width + x) * 3;
+          for (let c = 0; c < 3; c++) {
+            rgb[dst + c] = linearByteToSrgbByte(pixels[src + c]);
+          }
+        }
+      }
+      depth = depthReader.read(sessionRenderer, camera, captureOptions.flipX ?? false);
+    });
+    return {
+      rgb,
+      depth,
+      width,
+      height,
+      source: getCameraFrameCaptureSource(captureOptions),
+      origin: "top-left",
+      depthUnits: "meters",
+      depthConvention: "camera-z",
+      flipX: captureOptions.flipX ?? false,
+      projectionMatrix: camera.projectionMatrix.toArray(),
+      cameraWorldMatrix: camera.matrixWorld.toArray()
+    };
+  }
   function captureTensor(nextOptions = {}) {
     const result = capturePixels(nextOptions);
     const tensor = pixelsToPolicyImageTensor(pixels, {
@@ -730,6 +845,7 @@ function createCameraFrameCaptureSession(renderer, scene, fallbackCamera, option
     captureAsync,
     capturePixels,
     captureTensor,
+    captureRGBD,
     captureDataUrl(nextOptions = {}) {
       const type = nextOptions.type ?? options.type ?? "image/png";
       const result = capture(nextOptions);
@@ -834,6 +950,14 @@ function captureCameraFrameTensor(renderer, scene, fallbackCamera, options = {})
     session.dispose();
   }
 }
+function captureCameraRGBD(renderer, scene, fallbackCamera, options = {}) {
+  const session = createCameraFrameCaptureSession(renderer, scene, fallbackCamera, options);
+  try {
+    return session.captureRGBD();
+  } finally {
+    session.dispose();
+  }
+}
 var DEFAULT_BACKGROUND = "#181a1f";
 function ScenarioLighting({
   preset = "studio",
@@ -1007,7 +1131,7 @@ function useVisualScenarioEffects({
       gl.toneMappingExposure = scenario.camera?.exposure ?? 1;
     }
     if (applyBackground) {
-      scene.background = new THREE.Color(
+      scene.background = new THREE2.Color(
         background ?? getScenarioBackground(scenario.lighting)
       );
     }
@@ -1347,21 +1471,21 @@ function SplatPlaceholder() {
         transparent: true,
         opacity: 0.06,
         wireframe: true,
-        side: THREE.DoubleSide
+        side: THREE2.DoubleSide
       }
     )
   ] }) });
 }
 function createScenarioFog(scenario, background, fogNear, fogFar) {
   if (scenario.lighting === "low-light") {
-    return new THREE.Fog(
+    return new THREE2.Fog(
       background ?? getScenarioBackground(scenario.lighting),
       fogNear ?? 2.5,
       fogFar ?? 9
     );
   }
   if (scenario.lighting === "warehouse") {
-    return new THREE.Fog(
+    return new THREE2.Fog(
       background ?? getScenarioBackground(scenario.lighting),
       fogNear ?? 5,
       fogFar ?? 16
@@ -1373,7 +1497,7 @@ function applyScenarioMaterials(scene, scenario, snapshots, materialFilter) {
   const materials = scenario.materials;
   if (!materials) return;
   scene.traverse((object) => {
-    if (!(object instanceof THREE.Mesh)) {
+    if (!(object instanceof THREE2.Mesh)) {
       return;
     }
     for (const material of normalizeMaterials(object.material)) {
@@ -1412,7 +1536,7 @@ function normalizeMaterials(material) {
   return Array.isArray(material) ? material : [material];
 }
 function getMutableScenarioMaterial(material) {
-  if (material instanceof THREE.MeshStandardMaterial || material instanceof THREE.MeshPhysicalMaterial) {
+  if (material instanceof THREE2.MeshStandardMaterial || material instanceof THREE2.MeshPhysicalMaterial) {
     return material;
   }
   return null;
@@ -1438,6 +1562,7 @@ function clamp01(value) {
  *
  * Helpers for turning browser camera captures into policy image tensors.
  */
+/** @license SPDX-License-Identifier: Apache-2.0 */
 /**
  * @license
  * SPDX-License-Identifier: Apache-2.0
@@ -1445,6 +1570,6 @@ function clamp01(value) {
  * Offscreen camera-frame capture for R3F/MuJoCo scenes.
  */
 
-export { CAMERA_FRAME_CAPTURE_PRE_RENDER_USER_DATA_KEY, CAMERA_FRAME_CAPTURE_RENDER_USER_DATA_KEY, CAPTURE_EXCLUDE_KEY, ModelActuators, ModelBodies, ModelCameras, ModelGeoms, ModelJoints, ModelKeyframes, ModelResources, ModelSensors, ModelSites, ScenarioLighting, SplatEnvironment, SplatEnvironmentReadinessStatus, VisualScenarioEffects, captureCameraFrame, captureCameraFrameBlob, captureCameraFrameTensor, createCameraFrameCaptureSession, createCaptureCamera, createPairedSplatEnvironment, createSparkSplatViewerUrl, createSplatEnvironmentUserData, createSplatSceneConfig, createVisualScenarioExecutionContext, dataUrlToPolicyImageTensor, getContact, getScenarioBackground, getScenarioCameraPosition, getSplatEnvironmentReadiness, imageDataToPolicyImageTensor, pixelsToPolicyImageTensor, prepareCaptureCamera, registerModelResources, renderCameraFrameToCanvas, useSplatEnvironment, useSplatSceneConfig, useVisualScenarioEffects, useVisualScenarioExecutionContext, withContacts, withSplatEnvironment };
-//# sourceMappingURL=chunk-CV6LBHC4.js.map
-//# sourceMappingURL=chunk-CV6LBHC4.js.map
+export { CAMERA_FRAME_CAPTURE_PRE_RENDER_USER_DATA_KEY, CAMERA_FRAME_CAPTURE_RENDER_USER_DATA_KEY, CAPTURE_EXCLUDE_KEY, ModelActuators, ModelBodies, ModelCameras, ModelGeoms, ModelJoints, ModelKeyframes, ModelResources, ModelSensors, ModelSites, ScenarioLighting, SplatEnvironment, SplatEnvironmentReadinessStatus, VisualScenarioEffects, captureCameraFrame, captureCameraFrameBlob, captureCameraFrameTensor, captureCameraRGBD, createCameraFrameCaptureSession, createCaptureCamera, createPairedSplatEnvironment, createSparkSplatViewerUrl, createSplatEnvironmentUserData, createSplatSceneConfig, createVisualScenarioExecutionContext, dataUrlToPolicyImageTensor, getContact, getScenarioBackground, getScenarioCameraPosition, getSplatEnvironmentReadiness, imageDataToPolicyImageTensor, pixelsToPolicyImageTensor, prepareCaptureCamera, registerModelResources, renderCameraFrameToCanvas, useSplatEnvironment, useSplatSceneConfig, useVisualScenarioEffects, useVisualScenarioExecutionContext, withContacts, withSplatEnvironment };
+//# sourceMappingURL=chunk-SP72GTOD.js.map
+//# sourceMappingURL=chunk-SP72GTOD.js.map