mujoco-react 9.4.0 → 9.5.0

package/README.md

@@ -297,6 +297,16 @@ function Scene() {
 
 `SparkSplatEnvironment` currently renders `.spz` assets. Use the renderer-agnostic
 `SplatEnvironment` for `.ply`/`.splat` metadata or when wiring a different renderer.
+Tune live rendering and snapshots separately with `renderTuning` and
+`captureTuning`:
+
+```tsx
+<SparkSplatEnvironment
+  {...splat.props}
+  renderTuning={{ lodSplatScale: 0.75, minSortIntervalMs: 50 }}
+  captureTuning={{ lodSplatScale: 1.4, lodRenderScale: 0.45, maxWarmupFrames: 6 }}
+/>
+```
 
 ## Write Controllers
 
@@ -850,6 +860,7 @@ Visualization overlays:
 | `showSites` | `boolean?` | `false` | Site markers |
 | `showJoints` | `boolean?` | `false` | Joint axes |
 | `showContacts` | `boolean?` | `false` | Contact force vectors |
+| `showCameras` | `boolean?` | `false` | MuJoCo camera positions, frustums, and forward rays |
 | `showCOM` | `boolean?` | `false` | Center of mass markers |
 | `showInertia` | `boolean?` | `false` | Inertia ellipsoids |
 | `showTendons` | `boolean?` | `false` | Tendon paths |
@@ -858,6 +869,8 @@ Visualization overlays:
 | `contactColor` | `string?` | `"#ff4444"` | Color for contact force arrows |
 | `comColor` | `string?` | `"#ff0000"` | Color for COM markers |
 
+Camera debug overlays use the live MuJoCo `cam_xpos` / `cam_xmat` frame, so the frustum matches mounted camera captures and follows parent body motion.
+
 ### `<TendonRenderer />`
 
 Renders tendons as tube geometry from wrap paths.
@@ -1198,12 +1211,15 @@ function DatasetRecorder() {
 recent recording.
 
 Use `resolveMountedCameraFrameSource()` when dataset feature names need to map
-to named MuJoCo cameras, sites, or bodies before recording. The helper first
-checks exact names and aliases, then falls back to normalized/prefix/suffix
-matches such as `left_wrist` -> `left_wrist_camera_optical_frame`, and
-token-contained imported-model names such as `observation.images.head` ->
-`robot_head_camera`. It returns both the capture selector and the
-mounted-source provenance that should be stored beside the dataset:
+to named MuJoCo cameras, sites, or bodies before recording. The helper honors
+aliases first, then prefers camera matches over sites and bodies, then falls
+back to exact body/site names. This keeps streams such as `wrist` mapped to a
+real `<camera name="wrist_cam">` even when the model also has a body named
+`wrist`. It also handles normalized/prefix/suffix matches such as
+`left_wrist` -> `left_wrist_camera_optical_frame`, and token-contained
+imported-model names such as `observation.images.head` ->
+`robot_head_camera`. It returns both the capture selector and the mounted-source
+provenance that should be stored beside the dataset:
 
 ```tsx
 const resolved = resolveMountedCameraFrameSource("head", {

package/dist/{chunk-VDSEPZYQ.js → chunk-6MOK6ZWB.js}

@@ -1,6 +1,6 @@
+import * as THREE from 'three';
 import { useThree } from '@react-three/fiber';
 import { useMemo, useEffect } from 'react';
-import * as THREE from 'three';
 import { jsxs, Fragment, jsx } from 'react/jsx-runtime';
 
 // src/types.ts
@@ -85,6 +85,393 @@ var SplatEnvironmentReadinessStatus = {
   UnsupportedFormat: "unsupported-format",
   Ready: "ready"
 };
+var CAMERA_FRAME_CAPTURE_RENDER_USER_DATA_KEY = "mujocoReactCameraFrameCaptureRender";
+var CAPTURE_EXCLUDE_KEY = "mujoco.capture.exclude";
+function toVector3(value, fallback) {
+  if (!value) return fallback.clone();
+  return value instanceof THREE.Vector3 ? value.clone() : new THREE.Vector3(value[0], value[1], value[2]);
+}
+function applyCameraPose(camera, options, fallbackCamera) {
+  camera.position.copy(toVector3(options.position, fallbackCamera.position));
+  camera.up.copy(toVector3(options.up, fallbackCamera.up));
+  if (options.quaternion) {
+    if (options.quaternion instanceof THREE.Quaternion) {
+      camera.quaternion.copy(options.quaternion);
+    } else {
+      camera.quaternion.set(
+        options.quaternion[0],
+        options.quaternion[1],
+        options.quaternion[2],
+        options.quaternion[3]
+      );
+    }
+  } else if (options.lookAt) {
+    camera.lookAt(toVector3(options.lookAt, new THREE.Vector3()));
+  } else {
+    camera.quaternion.copy(fallbackCamera.quaternion);
+  }
+  camera.updateMatrixWorld();
+}
+function createCaptureCamera(options, fallbackCamera, width, height) {
+  const camera = options.camera ? options.camera.clone() : fallbackCamera instanceof THREE.PerspectiveCamera ? fallbackCamera.clone() : new THREE.PerspectiveCamera(45, width / height, 0.01, 100);
+  if (camera instanceof THREE.PerspectiveCamera) {
+    camera.aspect = width / height;
+    camera.fov = options.fov ?? camera.fov;
+    camera.near = options.near ?? camera.near;
+    camera.far = options.far ?? camera.far;
+    camera.updateProjectionMatrix();
+  }
+  applyCameraPose(camera, options, fallbackCamera);
+  return camera;
+}
+function getCaptureDimensions(renderer, options) {
+  const width = Math.max(
+    1,
+    Math.floor(options.width ?? renderer.domElement.width)
+  );
+  const height = Math.max(
+    1,
+    Math.floor(options.height ?? renderer.domElement.height)
+  );
+  return { width, height };
+}
+function prepareCaptureCamera(camera, options, fallbackCamera, width, height) {
+  if (options.camera) {
+    camera.copy(options.camera);
+  }
+  if (camera instanceof THREE.PerspectiveCamera) {
+    camera.aspect = width / height;
+    camera.fov = options.fov ?? camera.fov;
+    camera.near = options.near ?? camera.near;
+    camera.far = options.far ?? camera.far;
+    camera.updateProjectionMatrix();
+  }
+  applyCameraPose(camera, options, fallbackCamera);
+}
+function readRenderTargetToCanvas(renderer, target, canvas, context, pixels, imageData, width, height, outputColorSpace) {
+  renderer.readRenderTargetPixels(target, 0, 0, width, height, pixels);
+  const rowBytes = width * 4;
+  const encodeSrgb = outputColorSpace === THREE.SRGBColorSpace;
+  for (let y = 0; y < height; y += 1) {
+    const sourceStart = (height - y - 1) * rowBytes;
+    const targetStart = y * rowBytes;
+    const row = pixels.subarray(sourceStart, sourceStart + rowBytes);
+    if (!encodeSrgb) {
+      imageData.data.set(row, targetStart);
+      continue;
+    }
+    for (let x = 0; x < rowBytes; x += 4) {
+      const pixelOffset = targetStart + x;
+      imageData.data[pixelOffset] = linearByteToSrgbByte(row[x]);
+      imageData.data[pixelOffset + 1] = linearByteToSrgbByte(row[x + 1]);
+      imageData.data[pixelOffset + 2] = linearByteToSrgbByte(row[x + 2]);
+      imageData.data[pixelOffset + 3] = row[x + 3];
+    }
+  }
+  context.putImageData(imageData, 0, 0);
+  return canvas;
+}
+function linearByteToSrgbByte(value) {
+  const normalized = value / 255;
+  const encoded = normalized <= 31308e-7 ? normalized * 12.92 : 1.055 * Math.pow(normalized, 1 / 2.4) - 0.055;
+  return Math.min(255, Math.max(0, Math.round(encoded * 255)));
+}
+function readPixelsToCanvas(pixels, context, imageData, width, height, flipY = true) {
+  const rowBytes = width * 4;
+  for (let y = 0; y < height; y += 1) {
+    const sourceY = flipY ? height - y - 1 : y;
+    const sourceStart = sourceY * rowBytes;
+    const targetStart = y * rowBytes;
+    imageData.data.set(
+      pixels.subarray(sourceStart, sourceStart + rowBytes),
+      targetStart
+    );
+  }
+  context.putImageData(imageData, 0, 0);
+}
+function hideExcludedCaptureObjects(scene) {
+  const hidden = [];
+  scene.traverse((object) => {
+    if (!object.visible) return;
+    if (!object.userData[CAPTURE_EXCLUDE_KEY]) return;
+    hidden.push({ object, visible: object.visible });
+    object.visible = false;
+  });
+  return hidden;
+}
+function restoreObjectVisibility(hidden) {
+  for (const { object, visible } of hidden) {
+    object.visible = visible;
+  }
+}
+function getCameraFrameCaptureSource(options) {
+  if (options.source) return options.source;
+  if (options.cameraName) {
+    return { kind: "mujoco-camera", cameraName: options.cameraName };
+  }
+  if (options.siteName) {
+    return { kind: "mujoco-site", siteName: options.siteName };
+  }
+  if (options.bodyName) {
+    return { kind: "mujoco-body", bodyName: options.bodyName };
+  }
+  if (options.camera) return { kind: "custom-camera" };
+  if (options.position || options.lookAt || options.quaternion) {
+    return { kind: "explicit-pose" };
+  }
+  return { kind: "fallback-camera" };
+}
+function saveRendererState(renderer) {
+  const viewport = new THREE.Vector4();
+  const scissor = new THREE.Vector4();
+  const clearColor = new THREE.Color();
+  renderer.getViewport(viewport);
+  renderer.getScissor(scissor);
+  renderer.getClearColor(clearColor);
+  return {
+    target: renderer.getRenderTarget(),
+    xrEnabled: renderer.xr.enabled,
+    viewport,
+    scissor,
+    scissorTest: renderer.getScissorTest(),
+    clearColor,
+    clearAlpha: renderer.getClearAlpha(),
+    autoClear: renderer.autoClear
+  };
+}
+function restoreRendererState(renderer, state) {
+  renderer.setRenderTarget(state.target);
+  renderer.xr.enabled = state.xrEnabled;
+  renderer.setViewport(state.viewport);
+  renderer.setScissor(state.scissor);
+  renderer.setScissorTest(state.scissorTest);
+  renderer.setClearColor(state.clearColor, state.clearAlpha);
+  renderer.autoClear = state.autoClear;
+}
+function getCaptureRenderer(scene) {
+  const renderers = [];
+  scene.traverse((object) => {
+    if (renderers.length) return;
+    const render = object.userData[CAMERA_FRAME_CAPTURE_RENDER_USER_DATA_KEY];
+    if (typeof render === "function") renderers.push(render);
+  });
+  return renderers[0] ?? null;
+}
+function createCameraFrameCaptureSession(renderer, scene, fallbackCamera, options = {}) {
+  const { width, height } = getCaptureDimensions(renderer, options);
+  const camera = createCaptureCamera(options, fallbackCamera, width, height);
+  const target = new THREE.WebGLRenderTarget(width, height, {
+    format: THREE.RGBAFormat,
+    type: THREE.UnsignedByteType
+  });
+  const canvas = document.createElement("canvas");
+  canvas.width = width;
+  canvas.height = height;
+  const context = canvas.getContext("2d");
+  if (!context) {
+    target.dispose();
+    throw new Error("Unable to create a 2D canvas for camera frame capture.");
+  }
+  const drawContext = context;
+  const pixels = new Uint8Array(width * height * 4);
+  const imageData = drawContext.createImageData(width, height);
+  function resolveCaptureOptions(nextOptions = {}) {
+    const captureOptions = { ...options, ...nextOptions };
+    const nextDimensions = getCaptureDimensions(renderer, captureOptions);
+    if (nextDimensions.width !== width || nextDimensions.height !== height) {
+      throw new Error(
+        "Camera frame capture sessions require stable width and height."
+      );
+    }
+    prepareCaptureCamera(
+      camera,
+      captureOptions,
+      fallbackCamera,
+      width,
+      height
+    );
+    return captureOptions;
+  }
+  function renderPreparedCapture(captureOptions) {
+    const previousState = saveRendererState(renderer);
+    const hidden = hideExcludedCaptureObjects(scene);
+    scene.updateMatrixWorld(true);
+    try {
+      renderer.xr.enabled = false;
+      renderer.setRenderTarget(target);
+      renderer.setViewport(0, 0, width, height);
+      renderer.setScissor(0, 0, width, height);
+      renderer.setScissorTest(false);
+      renderer.clear();
+      renderer.render(scene, camera);
+      readRenderTargetToCanvas(
+        renderer,
+        target,
+        canvas,
+        drawContext,
+        pixels,
+        imageData,
+        width,
+        height,
+        renderer.outputColorSpace
+      );
+      return {
+        canvas,
+        camera,
+        width,
+        height,
+        source: getCameraFrameCaptureSource(captureOptions)
+      };
+    } finally {
+      restoreObjectVisibility(hidden);
+      restoreRendererState(renderer, previousState);
+    }
+  }
+  function capture(nextOptions = {}) {
+    return renderPreparedCapture(resolveCaptureOptions(nextOptions));
+  }
+  async function captureAsync(nextOptions = {}) {
+    const captureOptions = resolveCaptureOptions(nextOptions);
+    scene.updateMatrixWorld(true);
+    const captureRenderer = getCaptureRenderer(scene);
+    if (captureRenderer) {
+      const previousState = saveRendererState(renderer);
+      const hidden = hideExcludedCaptureObjects(scene);
+      try {
+        renderer.xr.enabled = false;
+        const captureResult = await captureRenderer({
+          renderer,
+          scene,
+          camera,
+          target,
+          width,
+          height
+        });
+        if (captureResult) {
+          const captureWidth = captureResult.width ?? width;
+          const captureHeight = captureResult.height ?? height;
+          if (captureWidth !== width || captureHeight !== height) {
+            throw new Error(
+              "Camera frame capture renderer returned unexpected dimensions."
+            );
+          }
+          readPixelsToCanvas(
+            captureResult.pixels,
+            drawContext,
+            imageData,
+            width,
+            height,
+            captureResult.flipY ?? true
+          );
+          return {
+            canvas,
+            camera,
+            width,
+            height,
+            source: getCameraFrameCaptureSource(captureOptions)
+          };
+        }
+      } finally {
+        restoreObjectVisibility(hidden);
+        restoreRendererState(renderer, previousState);
+      }
+    }
+    return renderPreparedCapture(captureOptions);
+  }
+  return {
+    width,
+    height,
+    capture,
+    captureAsync,
+    captureDataUrl(nextOptions = {}) {
+      const type = nextOptions.type ?? options.type ?? "image/png";
+      const result = capture(nextOptions);
+      return {
+        ...result,
+        dataUrl: result.canvas.toDataURL(
+          type,
+          nextOptions.quality ?? options.quality
+        ),
+        type
+      };
+    },
+    async captureDataUrlAsync(nextOptions = {}) {
+      const type = nextOptions.type ?? options.type ?? "image/png";
+      const result = await captureAsync(nextOptions);
+      return {
+        ...result,
+        dataUrl: result.canvas.toDataURL(
+          type,
+          nextOptions.quality ?? options.quality
+        ),
+        type
+      };
+    },
+    async captureBlob(nextOptions = {}) {
+      const type = nextOptions.type ?? options.type ?? "image/png";
+      const result = await captureAsync(nextOptions);
+      const blob = await new Promise((resolve, reject) => {
+        result.canvas.toBlob(
+          (nextBlob) => {
+            if (nextBlob) resolve(nextBlob);
+            else reject(new Error("Camera frame capture did not produce a Blob."));
+          },
+          type,
+          nextOptions.quality ?? options.quality
+        );
+      });
+      return { ...result, blob, type };
+    },
+    dispose() {
+      target.dispose();
+    }
+  };
+}
+function renderCameraFrameToCanvas(renderer, scene, fallbackCamera, options = {}) {
+  const session = createCameraFrameCaptureSession(
+    renderer,
+    scene,
+    fallbackCamera,
+    options
+  );
+  try {
+    return session.capture();
+  } finally {
+    session.dispose();
+  }
+}
+async function captureCameraFrame(renderer, scene, fallbackCamera, options = {}) {
+  const type = options.type ?? "image/png";
+  const session = createCameraFrameCaptureSession(
+    renderer,
+    scene,
+    fallbackCamera,
+    options
+  );
+  try {
+    const result = await session.captureAsync();
+    return {
+      ...result,
+      dataUrl: result.canvas.toDataURL(type, options.quality),
+      type
+    };
+  } finally {
+    session.dispose();
+  }
+}
+async function captureCameraFrameBlob(renderer, scene, fallbackCamera, options = {}) {
+  const session = createCameraFrameCaptureSession(
+    renderer,
+    scene,
+    fallbackCamera,
+    options
+  );
+  try {
+    return await session.captureBlob();
+  } finally {
+    session.dispose();
+  }
+}
 var DEFAULT_BACKGROUND = "#181a1f";
 function ScenarioLighting({
   preset = "studio",
@@ -683,7 +1070,13 @@ function clamp01(value) {
  * @license
  * SPDX-License-Identifier: Apache-2.0
  */
+/**
+ * @license
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Offscreen camera-frame capture for R3F/MuJoCo scenes.
+ */
 
-export { RobotActuators, RobotBodies, RobotCameras, RobotGeoms, RobotJoints, RobotKeyframes, RobotResources, RobotSensors, RobotSites, ScenarioLighting, SplatEnvironment, SplatEnvironmentReadinessStatus, VisualScenarioEffects, createPairedSplatEnvironment, createSparkSplatViewerUrl, createSplatEnvironmentUserData, createSplatSceneConfig, createVisualScenarioExecutionContext, getContact, getScenarioBackground, getScenarioCameraPosition, getSplatEnvironmentReadiness, registerRobotResources, useSplatEnvironment, useSplatSceneConfig, useVisualScenarioEffects, useVisualScenarioExecutionContext, withContacts, withSplatEnvironment };
-//# sourceMappingURL=chunk-VDSEPZYQ.js.map
-//# sourceMappingURL=chunk-VDSEPZYQ.js.map
+export { CAMERA_FRAME_CAPTURE_RENDER_USER_DATA_KEY, CAPTURE_EXCLUDE_KEY, RobotActuators, RobotBodies, RobotCameras, RobotGeoms, RobotJoints, RobotKeyframes, RobotResources, RobotSensors, RobotSites, ScenarioLighting, SplatEnvironment, SplatEnvironmentReadinessStatus, VisualScenarioEffects, captureCameraFrame, captureCameraFrameBlob, createCameraFrameCaptureSession, createPairedSplatEnvironment, createSparkSplatViewerUrl, createSplatEnvironmentUserData, createSplatSceneConfig, createVisualScenarioExecutionContext, getContact, getScenarioBackground, getScenarioCameraPosition, getSplatEnvironmentReadiness, registerRobotResources, renderCameraFrameToCanvas, useSplatEnvironment, useSplatSceneConfig, useVisualScenarioEffects, useVisualScenarioExecutionContext, withContacts, withSplatEnvironment };
+//# sourceMappingURL=chunk-6MOK6ZWB.js.map
+//# sourceMappingURL=chunk-6MOK6ZWB.js.map