mujoco-react 10.2.1 → 10.4.0

package/src/components/SceneRenderer.tsx

@@ -9,7 +9,7 @@ import { useCallback, useEffect, useMemo, useRef } from 'react';
 import * as THREE from 'three';
 import { GeomBuilder } from '../rendering/GeomBuilder';
 import { CAMERA_FRAME_CAPTURE_PRE_RENDER_USER_DATA_KEY } from '../rendering/cameraFrameCapture';
-import { MujocoModel } from '../types';
+import { MujocoModel, MujocoRenderOptions } from '../types';
 import { getName } from '../core/SceneLoader';
 import { useMujocoContext } from '../core/MujocoSimProvider';
 
@@ -17,7 +17,18 @@ import { useMujocoContext } from '../core/MujocoSimProvider';
  * SceneRenderer — creates and syncs MuJoCo body meshes every frame.
  * Accepts standard R3F group props (position, rotation, scale, visible, etc.).
  */
-export function SceneRenderer(props: Omit<ThreeElements['group'], 'ref'>) {
+export interface SceneRendererProps extends Omit<ThreeElements['group'], 'ref'> {
+  renderOptions?: MujocoRenderOptions;
+}
+
+function getRenderOptionsKey(renderOptions: MujocoRenderOptions | undefined) {
+  const smoothing = renderOptions?.meshNormalSmoothing;
+  if (!smoothing) return 'default';
+  if (smoothing === true) return 'meshNormalSmoothing:true';
+  return `meshNormalSmoothing:${smoothing.tolerance ?? 'default'}`;
+}
+
+export function SceneRenderer({ renderOptions, ...props }: SceneRendererProps) {
   const {
     mjModelRef,
     mjDataRef,
@@ -31,11 +42,13 @@ export function SceneRenderer(props: Omit<ThreeElements['group'], 'ref'>) {
   const groupRef = useRef<THREE.Group>(null);
   const bodyRefs = useRef<(THREE.Group | null)[]>([]);
   const prevModelRef = useRef<MujocoModel | null>(null);
+  const prevRenderOptionsKeyRef = useRef<string | null>(null);
+  const renderOptionsKey = getRenderOptionsKey(renderOptions);
 
   const geomBuilder = useMemo(() => {
     if (status !== 'ready') return null;
-    return new GeomBuilder(mujocoRef.current);
-  }, [status, mujocoRef]);
+    return new GeomBuilder(mujocoRef.current, renderOptions);
+  }, [status, mujocoRef, renderOptionsKey]);
 
   // Build body groups when model loads
   useEffect(() => {
@@ -45,8 +58,14 @@ export function SceneRenderer(props: Omit<ThreeElements['group'], 'ref'>) {
     if (!model || !group) return;
 
     // Skip if model hasn't changed
-    if (prevModelRef.current === model) return;
+    if (
+      prevModelRef.current === model &&
+      prevRenderOptionsKeyRef.current === renderOptionsKey
+    ) {
+      return;
+    }
     prevModelRef.current = model;
+    prevRenderOptionsKeyRef.current = renderOptionsKey;
 
     // Clear previous bodies
     while (group.children.length > 0) {
@@ -73,7 +92,7 @@ export function SceneRenderer(props: Omit<ThreeElements['group'], 'ref'>) {
       refs.push(bodyGroup);
     }
     bodyRefs.current = refs;
-  }, [status, geomBuilder, mjModelRef]);
+  }, [status, geomBuilder, mjModelRef, renderOptionsKey]);
 
   const syncBodiesToData = useCallback(() => {
     const data = mjDataRef.current;

package/src/core/MujocoCanvas.tsx

@@ -4,7 +4,7 @@
  */
 
 import { Canvas } from '@react-three/fiber';
-import { forwardRef, useEffect } from 'react';
+import { forwardRef, useEffect, useRef } from 'react';
 import { useMujocoWasm } from './MujocoProvider';
 import { MujocoSimProvider } from './MujocoSimProvider';
 import { MujocoCanvasProps, MujocoSimAPI } from '../types';
@@ -31,6 +31,7 @@ export const MujocoCanvas = forwardRef<MujocoSimAPI, MujocoCanvasProps>(
       paused,
       speed,
       interpolate,
+      renderOptions,
       loadingFallback,
       children,
       ...canvasProps
@@ -38,12 +39,14 @@ export const MujocoCanvas = forwardRef<MujocoSimAPI, MujocoCanvasProps>(
     ref
   ) {
     const { mujoco, status: wasmStatus, error: wasmError } = useMujocoWasm();
+    const onErrorRef = useRef(onError);
+    onErrorRef.current = onError;
 
     useEffect(() => {
-      if (wasmStatus === 'error' && onError) {
-        onError(new Error(wasmError ?? 'WASM load failed'));
+      if (wasmStatus === 'error') {
+        onErrorRef.current?.(new Error(wasmError ?? 'WASM load failed'));
       }
-    }, [wasmStatus, wasmError, onError]);
+    }, [wasmStatus, wasmError]);
 
     if (wasmStatus === 'loading' || !mujoco) {
       return loadingFallback ? (
@@ -71,6 +74,7 @@ export const MujocoCanvas = forwardRef<MujocoSimAPI, MujocoCanvasProps>(
           paused={paused}
           speed={speed}
           interpolate={interpolate}
+          renderOptions={renderOptions}
         >
           {children}
         </MujocoSimProvider>

package/src/core/MujocoPhysics.tsx

@@ -3,7 +3,7 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-import { forwardRef, useEffect } from 'react';
+import { forwardRef, useEffect, useRef } from 'react';
 import { useMujocoWasm } from './MujocoProvider';
 import { MujocoSimProvider } from './MujocoSimProvider';
 import type {
@@ -62,12 +62,14 @@ export interface MujocoPhysicsProps {
 export const MujocoPhysics = forwardRef<MujocoSimAPI, MujocoPhysicsProps>(
   function MujocoPhysics({ onError, children, ...props }, ref) {
     const { mujoco, status: wasmStatus, error: wasmError } = useMujocoWasm();
+    const onErrorRef = useRef(onError);
+    onErrorRef.current = onError;
 
     useEffect(() => {
-      if (wasmStatus === 'error' && onError) {
-        onError(new Error(wasmError ?? 'WASM load failed'));
+      if (wasmStatus === 'error') {
+        onErrorRef.current?.(new Error(wasmError ?? 'WASM load failed'));
       }
-    }, [wasmStatus, wasmError, onError]);
+    }, [wasmStatus, wasmError]);
 
     if (wasmStatus === 'error' || wasmStatus === 'loading' || !mujoco) {
       return null;

package/src/core/MujocoProvider.tsx

@@ -96,6 +96,8 @@ export function MujocoProvider({
   const [error, setError] = useState<string | null>(null);
   const moduleRef = useRef<MujocoModule | null>(null);
   const isMounted = useRef(true);
+  const onErrorRef = useRef(onError);
+  onErrorRef.current = onError;
 
   useEffect(() => {
     isMounted.current = true;
@@ -105,7 +107,7 @@ export function MujocoProvider({
       const err = new Error('MujocoProvider wasmVariant="threaded" requires a threadedLoader from @mujoco/mujoco/mt');
       setError(err.message);
       setStatus('error');
-      onError?.(err);
+      onErrorRef.current?.(err);
       return;
     }
     let selectedWasmUrl = wasmUrl ?? defaultMujocoWasmUrl;
@@ -115,7 +117,7 @@ export function MujocoProvider({
         const err = new Error('MujocoProvider wasmVariant="threaded" requires mtWasmUrl from @mujoco/mujoco/mt/mujoco.wasm?url');
         setError(err.message);
         setStatus('error');
-        onError?.(err);
+        onErrorRef.current?.(err);
         return;
       }
       selectedWasmUrl = mtWasmUrl;
@@ -152,14 +154,14 @@ export function MujocoProvider({
           const msg = err.message || 'Failed to init spatial simulation';
           setError(msg);
           setStatus('error');
-          onError?.(new Error(msg));
+          onErrorRef.current?.(new Error(msg));
         }
       });
 
     return () => {
       isMounted.current = false;
     };
-  }, [wasmUrl, mtWasmUrl, threadedLoader, wasmVariant, timeout, onError]);
+  }, [wasmUrl, mtWasmUrl, threadedLoader, wasmVariant, timeout]);
 
   return (
     <MujocoContext.Provider

package/src/core/MujocoSimProvider.tsx

@@ -37,6 +37,7 @@ import {
   LoadFromFilesOptions,
   LocalMujocoFile,
   ModelOptions,
+  MujocoRenderOptions,
   MujocoSimAPI,
   PhysicsStepCallback,
   RayHit,
@@ -241,6 +242,143 @@ function applyMountedCameraPoseOffsets(
   };
 }
 
+function resolveMujocoCameraCompatibilityOptions(
+  options: CameraFrameCaptureOptions
+) {
+  const compatibility = options.mujocoCameraCompatibility;
+  if (!compatibility) return null;
+  if (compatibility === true) {
+    return {
+      useResolution: true,
+      useIntrinsics: true,
+      useClipping: true,
+      preserveAspect: true,
+      preferResolution: false,
+    };
+  }
+  return {
+    useResolution: compatibility.useResolution ?? true,
+    useIntrinsics: compatibility.useIntrinsics ?? true,
+    useClipping: compatibility.useClipping ?? true,
+    preserveAspect: compatibility.preserveAspect ?? true,
+    preferResolution: compatibility.preferResolution ?? false,
+  };
+}
+
+function mujocoVisualClip(model: MujocoModel) {
+  const map = (model as unknown as {
+    vis?: { map?: { znear?: number; zfar?: number } };
+  }).vis?.map;
+  const near = typeof map?.znear === 'number' && map.znear > 0
+    ? map.znear
+    : undefined;
+  const far = typeof map?.zfar === 'number' && map.zfar > 0
+    ? map.zfar
+    : undefined;
+  return { near, far };
+}
+
+function mujocoCameraResolution(
+  model: MujocoModel,
+  cameraId: number
+): { width?: number; height?: number } {
+  const resolution = model.cam_resolution;
+  if (!resolution) return {};
+  const width = Number(resolution[cameraId * 2]);
+  const height = Number(resolution[cameraId * 2 + 1]);
+  return {
+    width: Number.isFinite(width) && width > 0 ? width : undefined,
+    height: Number.isFinite(height) && height > 0 ? height : undefined,
+  };
+}
+
+function mujocoCameraProjectionMatrix(
+  model: MujocoModel,
+  cameraId: number,
+  width: number | undefined,
+  height: number | undefined,
+  near: number | undefined,
+  far: number | undefined
+): THREE.Matrix4 | undefined {
+  const intrinsic = model.cam_intrinsic;
+  const sensorSize = model.cam_sensorsize;
+  if (!intrinsic || !sensorSize || !width || !height) return undefined;
+
+  const intrinsicOffset = cameraId * 4;
+  const sensorOffset = cameraId * 2;
+  const focalX = Number(intrinsic[intrinsicOffset]);
+  const focalY = Number(intrinsic[intrinsicOffset + 1]);
+  const principalX = Number(intrinsic[intrinsicOffset + 2]);
+  const principalY = Number(intrinsic[intrinsicOffset + 3]);
+  const sensorWidth = Number(sensorSize[sensorOffset]);
+  const sensorHeight = Number(sensorSize[sensorOffset + 1]);
+  if (
+    !Number.isFinite(focalX) ||
+    !Number.isFinite(focalY) ||
+    !Number.isFinite(principalX) ||
+    !Number.isFinite(principalY) ||
+    !Number.isFinite(sensorWidth) ||
+    !Number.isFinite(sensorHeight) ||
+    focalX <= 0 ||
+    focalY <= 0 ||
+    sensorWidth <= 0 ||
+    sensorHeight <= 0
+  ) {
+    return undefined;
+  }
+
+  const fx = focalX / sensorWidth * width;
+  const fy = focalY / sensorHeight * height;
+  const cx = width * (0.5 + principalX / sensorWidth);
+  const cy = height * (0.5 + principalY / sensorHeight);
+  const znear = near ?? 0.01;
+  const zfar = far ?? 100;
+
+  return new THREE.Matrix4().set(
+    2 * fx / width, 0, 1 - 2 * cx / width, 0,
+    0, 2 * fy / height, 2 * cy / height - 1, 0,
+    0, 0, -(zfar + znear) / (zfar - znear), -2 * zfar * znear / (zfar - znear),
+    0, 0, -1, 0
+  );
+}
+
+function resolveMujocoCameraCaptureDimensions(
+  requested: CameraFrameCaptureOptions,
+  cameraResolution: { width?: number; height?: number },
+  compatibility: NonNullable<ReturnType<typeof resolveMujocoCameraCompatibilityOptions>>
+) {
+  if (!compatibility.useResolution) {
+    return {
+      width: requested.width,
+      height: requested.height,
+    };
+  }
+
+  if (compatibility.preferResolution) {
+    return {
+      width: cameraResolution.width ?? requested.width,
+      height: cameraResolution.height ?? requested.height,
+    };
+  }
+
+  let width = requested.width ?? cameraResolution.width;
+  let height = requested.height ?? cameraResolution.height;
+
+  if (
+    compatibility.preserveAspect &&
+    cameraResolution.width &&
+    cameraResolution.height
+  ) {
+    if (requested.width !== undefined && requested.height === undefined) {
+      height = requested.width * cameraResolution.height / cameraResolution.width;
+    } else if (requested.height !== undefined && requested.width === undefined) {
+      width = requested.height * cameraResolution.width / cameraResolution.height;
+    }
+  }
+
+  return { width, height };
+}
+
 function countMountedCameraSelectors(options: CameraFrameCaptureOptions) {
   return Number(Boolean(options.cameraName)) +
     Number(Boolean(options.siteName)) +
@@ -406,6 +544,7 @@ interface MujocoSimProviderProps {
   paused?: boolean;
   speed?: number;
   interpolate?: boolean;
+  renderOptions?: MujocoRenderOptions;
   children: React.ReactNode;
 }
 
@@ -423,6 +562,7 @@ export function MujocoSimProvider({
   paused,
   speed,
   interpolate,
+  renderOptions,
   children,
 }: MujocoSimProviderProps) {
   const { gl, camera, scene } = useThree();
@@ -462,14 +602,12 @@ export function MujocoSimProvider({
   const hiddenBodiesRef = useRef(new Set<string>());
   const bodyReloadTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
 
-  useEffect(() => { configRef.current = config; }, [config]);
-  useEffect(() => { mujocoRef.current = mujoco; }, [mujoco]);
-
-  // Sync declarative props to refs
-  useEffect(() => { pausedRef.current = paused ?? false; }, [paused]);
-  useEffect(() => { speedRef.current = speed ?? 1; }, [speed]);
-  useEffect(() => { substepsRef.current = substeps ?? 1; }, [substeps]);
-  useEffect(() => { interpolateRef.current = interpolate ?? false; }, [interpolate]);
+  configRef.current = config;
+  mujocoRef.current = mujoco;
+  pausedRef.current = paused ?? false;
+  speedRef.current = speed ?? 1;
+  substepsRef.current = substeps ?? 1;
+  interpolateRef.current = interpolate ?? false;
 
   // Sync gravity prop
   useEffect(() => {
@@ -1037,11 +1175,33 @@ export function MujocoSimProvider({
     for (let i = 0; i < ncam; i += 1) {
       const posOffset = i * 3;
       const quatOffset = i * 4;
+      const intrinsicOffset = i * 4;
+      const resolutionOffset = i * 2;
       result.push({
         id: i,
         name: getName(model, nameAddresses[i]),
         bodyId: model.cam_bodyid?.[i] ?? -1,
         fov: model.cam_fovy?.[i] ?? null,
+        resolution: model.cam_resolution
+          ? [
+            model.cam_resolution[resolutionOffset],
+            model.cam_resolution[resolutionOffset + 1],
+          ]
+          : null,
+        sensorSize: model.cam_sensorsize
+          ? [
+            model.cam_sensorsize[resolutionOffset],
+            model.cam_sensorsize[resolutionOffset + 1],
+          ]
+          : null,
+        intrinsic: model.cam_intrinsic
+          ? [
+            model.cam_intrinsic[intrinsicOffset],
+            model.cam_intrinsic[intrinsicOffset + 1],
+            model.cam_intrinsic[intrinsicOffset + 2],
+            model.cam_intrinsic[intrinsicOffset + 3],
+          ]
+          : null,
         position: model.cam_pos
           ? vector3FromArray(model.cam_pos, posOffset)
           : null,
@@ -1087,11 +1247,31 @@ export function MujocoSimProvider({
         }
 
         const pose = applyMountedCameraPoseOffsets(options, position, quaternion);
+        const compatibility = resolveMujocoCameraCompatibilityOptions(options);
+        const cameraResolution = compatibility?.useResolution
+          ? mujocoCameraResolution(model, cameraId)
+          : { width: undefined, height: undefined };
+        const clip = compatibility?.useClipping
+          ? mujocoVisualClip(model)
+          : { near: undefined, far: undefined };
+        const { width, height } = compatibility
+          ? resolveMujocoCameraCaptureDimensions(options, cameraResolution, compatibility)
+          : { width: options.width, height: options.height };
+        const near = options.near ?? clip.near;
+        const far = options.far ?? clip.far;
+        const projectionMatrix = compatibility?.useIntrinsics
+          ? mujocoCameraProjectionMatrix(model, cameraId, width, height, near, far)
+          : undefined;
 
         return {
           ...baseOptions,
+          width,
+          height,
           ...pose,
           fov: options.fov ?? model.cam_fovy?.[cameraId],
+          near,
+          far,
+          projectionMatrix: options.projectionMatrix ?? projectionMatrix,
           source: { kind: 'mujoco-camera', cameraName: options.cameraName },
         };
       }
@@ -1759,7 +1939,7 @@ export function MujocoSimProvider({
 
   return (
     <MujocoSimContext.Provider value={contextValue}>
-      <SceneRenderer />
+      <SceneRenderer renderOptions={renderOptions} />
       {children}
     </MujocoSimContext.Provider>
   );

package/src/index.ts

@@ -320,6 +320,7 @@ export type {
   IkGizmoProps,
   IkGizmoDragInput,
   DragInteractionProps,
+  DebugVirtualCamera,
   DebugProps,
   SceneLightsProps,
   ScenarioLightingPreset,

package/src/rendering/GeomBuilder.ts

@@ -5,9 +5,12 @@
 
 
 import * as THREE from 'three';
+import { mergeVertices } from 'three/examples/jsm/utils/BufferGeometryUtils.js';
 import { CapsuleGeometry } from './CapsuleGeometry';
 import { getName } from '../core/SceneLoader';
-import { MujocoModel, MujocoModule } from '../types';
+import { MujocoModel, MujocoModule, MujocoRenderOptions } from '../types';
+
+const DEFAULT_MESH_NORMAL_SMOOTHING_TOLERANCE = 1e-4;
 
 /**
  * GeomBuilder
@@ -20,9 +23,18 @@ import { MujocoModel, MujocoModule } from '../types';
 export class GeomBuilder {
     private mujoco: MujocoModule; 
     private textureCache = new Map<number, THREE.Texture>();
+    private renderOptions?: MujocoRenderOptions;
 
-    constructor(mujoco: MujocoModule) {
+    constructor(mujoco: MujocoModule, renderOptions?: MujocoRenderOptions) {
         this.mujoco = mujoco;
+        this.renderOptions = renderOptions;
+    }
+
+    private getMeshNormalSmoothingTolerance(): number | null {
+        const smoothing = this.renderOptions?.meshNormalSmoothing;
+        if (!smoothing) return null;
+        if (smoothing === true) return DEFAULT_MESH_NORMAL_SMOOTHING_TOLERANCE;
+        return smoothing.tolerance ?? DEFAULT_MESH_NORMAL_SMOOTHING_TOLERANCE;
     }
 
     private getMaterialTexture(mjModel: MujocoModel, matId: number): THREE.Texture | null {
@@ -148,6 +160,10 @@ export class GeomBuilder {
             geo.setAttribute('position', new THREE.Float32BufferAttribute(mjModel.mesh_vert.subarray(vAdr * 3, (vAdr + vNum) * 3), 3));
             // 'index' = faces (triangles connecting vertices)
             geo.setIndex(Array.from(mjModel.mesh_face.subarray(fAdr * 3, (fAdr + fNum) * 3)));
+            const smoothingTolerance = this.getMeshNormalSmoothingTolerance();
+            if (smoothingTolerance !== null) {
+                geo = mergeVertices(geo, smoothingTolerance);
+            }
             geo.computeVertexNormals(); // Auto-calculate smooth lighting normals
         }