mujoco-react 10.2.0 → 10.3.0

package/dist/index.js

@@ -1,11 +1,12 @@
-import { withContacts, getContact, captureCameraFrame, captureCameraFrameBlob, createCameraFrameCaptureSession, CAMERA_FRAME_CAPTURE_PRE_RENDER_USER_DATA_KEY, CAPTURE_EXCLUDE_KEY } from './chunk-3BMNRSS2.js';
-export { CAMERA_FRAME_CAPTURE_PRE_RENDER_USER_DATA_KEY, CAMERA_FRAME_CAPTURE_RENDER_USER_DATA_KEY, CAPTURE_EXCLUDE_KEY, ModelActuators, ModelBodies, ModelCameras, ModelGeoms, ModelJoints, ModelKeyframes, ModelResources, ModelSensors, ModelSites, ScenarioLighting, SplatEnvironment, SplatEnvironmentReadinessStatus, VisualScenarioEffects, captureCameraFrame, captureCameraFrameBlob, createCameraFrameCaptureSession, createPairedSplatEnvironment, createSparkSplatViewerUrl, createSplatEnvironmentUserData, createSplatSceneConfig, createVisualScenarioExecutionContext, getContact, getScenarioBackground, getScenarioCameraPosition, getSplatEnvironmentReadiness, registerModelResources, renderCameraFrameToCanvas, useSplatEnvironment, useSplatSceneConfig, useVisualScenarioEffects, useVisualScenarioExecutionContext, withSplatEnvironment } from './chunk-3BMNRSS2.js';
+import { withContacts, getContact, captureCameraFrame, captureCameraFrameBlob, createCameraFrameCaptureSession, CAMERA_FRAME_CAPTURE_PRE_RENDER_USER_DATA_KEY, CAPTURE_EXCLUDE_KEY } from './chunk-6AZEFI6A.js';
+export { CAMERA_FRAME_CAPTURE_PRE_RENDER_USER_DATA_KEY, CAMERA_FRAME_CAPTURE_RENDER_USER_DATA_KEY, CAPTURE_EXCLUDE_KEY, ModelActuators, ModelBodies, ModelCameras, ModelGeoms, ModelJoints, ModelKeyframes, ModelResources, ModelSensors, ModelSites, ScenarioLighting, SplatEnvironment, SplatEnvironmentReadinessStatus, VisualScenarioEffects, captureCameraFrame, captureCameraFrameBlob, createCameraFrameCaptureSession, createPairedSplatEnvironment, createSparkSplatViewerUrl, createSplatEnvironmentUserData, createSplatSceneConfig, createVisualScenarioExecutionContext, getContact, getScenarioBackground, getScenarioCameraPosition, getSplatEnvironmentReadiness, registerModelResources, renderCameraFrameToCanvas, useSplatEnvironment, useSplatSceneConfig, useVisualScenarioEffects, useVisualScenarioExecutionContext, withSplatEnvironment } from './chunk-6AZEFI6A.js';
 import loadMujoco from '@mujoco/mujoco';
 import defaultMujocoWasmUrl from '@mujoco/mujoco/mujoco.wasm?url';
-import { createContext, forwardRef, useEffect, useContext, useState, useRef, useCallback, useMemo, useLayoutEffect } from 'react';
+import { createContext, forwardRef, useRef, useEffect, useContext, useState, useCallback, useMemo, useLayoutEffect } from 'react';
 import { jsx, jsxs } from 'react/jsx-runtime';
 import { Canvas, useThree, useFrame } from '@react-three/fiber';
 import * as THREE11 from 'three';
+import { mergeVertices } from 'three/examples/jsm/utils/BufferGeometryUtils.js';
 import { PivotControls } from '@react-three/drei';
 
 var MujocoContext = createContext({
@@ -43,6 +44,8 @@ function MujocoProvider({
   const [error, setError] = useState(null);
   const moduleRef = useRef(null);
   const isMounted = useRef(true);
+  const onErrorRef = useRef(onError);
+  onErrorRef.current = onError;
   useEffect(() => {
     isMounted.current = true;
     const variant = resolveWasmVariant(wasmVariant, threadedLoader, mtWasmUrl);
@@ -50,7 +53,7 @@ function MujocoProvider({
       const err = new Error('MujocoProvider wasmVariant="threaded" requires a threadedLoader from @mujoco/mujoco/mt');
       setError(err.message);
       setStatus("error");
-      onError?.(err);
+      onErrorRef.current?.(err);
       return;
     }
     let selectedWasmUrl = wasmUrl ?? defaultMujocoWasmUrl;
@@ -59,7 +62,7 @@ function MujocoProvider({
         const err = new Error('MujocoProvider wasmVariant="threaded" requires mtWasmUrl from @mujoco/mujoco/mt/mujoco.wasm?url');
         setError(err.message);
         setStatus("error");
-        onError?.(err);
+        onErrorRef.current?.(err);
         return;
       }
       selectedWasmUrl = mtWasmUrl;
@@ -90,13 +93,13 @@ function MujocoProvider({
         const msg = err.message || "Failed to init spatial simulation";
         setError(msg);
         setStatus("error");
-        onError?.(new Error(msg));
+        onErrorRef.current?.(new Error(msg));
       }
     });
     return () => {
       isMounted.current = false;
     };
-  }, [wasmUrl, mtWasmUrl, threadedLoader, wasmVariant, timeout, onError]);
+  }, [wasmUrl, mtWasmUrl, threadedLoader, wasmVariant, timeout]);
   return /* @__PURE__ */ jsx(
     MujocoContext.Provider,
     {
@@ -891,11 +894,20 @@ function collectDependencyPaths(xmlString, currentFile, parser) {
 }
 
 // src/rendering/GeomBuilder.ts
+var DEFAULT_MESH_NORMAL_SMOOTHING_TOLERANCE = 1e-4;
 var GeomBuilder = class {
   mujoco;
   textureCache = /* @__PURE__ */ new Map();
-  constructor(mujoco) {
+  renderOptions;
+  constructor(mujoco, renderOptions) {
     this.mujoco = mujoco;
+    this.renderOptions = renderOptions;
+  }
+  getMeshNormalSmoothingTolerance() {
+    const smoothing = this.renderOptions?.meshNormalSmoothing;
+    if (!smoothing) return null;
+    if (smoothing === true) return DEFAULT_MESH_NORMAL_SMOOTHING_TOLERANCE;
+    return smoothing.tolerance ?? DEFAULT_MESH_NORMAL_SMOOTHING_TOLERANCE;
   }
   getMaterialTexture(mjModel, matId) {
     if (matId < 0 || !mjModel.mat_texid || !mjModel.tex_data) return null;
@@ -990,6 +1002,10 @@ var GeomBuilder = class {
       geo = new THREE11.BufferGeometry();
       geo.setAttribute("position", new THREE11.Float32BufferAttribute(mjModel.mesh_vert.subarray(vAdr * 3, (vAdr + vNum) * 3), 3));
       geo.setIndex(Array.from(mjModel.mesh_face.subarray(fAdr * 3, (fAdr + fNum) * 3)));
+      const smoothingTolerance = this.getMeshNormalSmoothingTolerance();
+      if (smoothingTolerance !== null) {
+        geo = mergeVertices(geo, smoothingTolerance);
+      }
       geo.computeVertexNormals();
     }
     if (geo) {
@@ -1020,7 +1036,13 @@ var GeomBuilder = class {
     return null;
   }
 };
-function SceneRenderer(props) {
+function getRenderOptionsKey(renderOptions) {
+  const smoothing = renderOptions?.meshNormalSmoothing;
+  if (!smoothing) return "default";
+  if (smoothing === true) return "meshNormalSmoothing:true";
+  return `meshNormalSmoothing:${smoothing.tolerance ?? "default"}`;
+}
+function SceneRenderer({ renderOptions, ...props }) {
   const {
     mjModelRef,
     mjDataRef,
@@ -1034,17 +1056,22 @@ function SceneRenderer(props) {
   const groupRef = useRef(null);
   const bodyRefs = useRef([]);
   const prevModelRef = useRef(null);
+  const prevRenderOptionsKeyRef = useRef(null);
+  const renderOptionsKey = getRenderOptionsKey(renderOptions);
   const geomBuilder = useMemo(() => {
     if (status !== "ready") return null;
-    return new GeomBuilder(mujocoRef.current);
-  }, [status, mujocoRef]);
+    return new GeomBuilder(mujocoRef.current, renderOptions);
+  }, [status, mujocoRef, renderOptionsKey]);
   useEffect(() => {
     if (status !== "ready" || !geomBuilder) return;
     const model = mjModelRef.current;
     const group = groupRef.current;
     if (!model || !group) return;
-    if (prevModelRef.current === model) return;
+    if (prevModelRef.current === model && prevRenderOptionsKeyRef.current === renderOptionsKey) {
+      return;
+    }
     prevModelRef.current = model;
+    prevRenderOptionsKeyRef.current = renderOptionsKey;
     while (group.children.length > 0) {
       group.remove(group.children[0]);
     }
@@ -1065,7 +1092,7 @@ function SceneRenderer(props) {
       refs.push(bodyGroup);
     }
     bodyRefs.current = refs;
-  }, [status, geomBuilder, mjModelRef]);
+  }, [status, geomBuilder, mjModelRef, renderOptionsKey]);
   const syncBodiesToData = useCallback(() => {
     const data = mjDataRef.current;
     if (!data) return;
@@ -1930,6 +1957,106 @@ function applyMountedCameraPoseOffsets(options, position, quaternion) {
     ]
   };
 }
+function resolveMujocoCameraCompatibilityOptions(options) {
+  const compatibility = options.mujocoCameraCompatibility;
+  if (!compatibility) return null;
+  if (compatibility === true) {
+    return {
+      useResolution: true,
+      useIntrinsics: true,
+      useClipping: true,
+      preserveAspect: true,
+      preferResolution: false
+    };
+  }
+  return {
+    useResolution: compatibility.useResolution ?? true,
+    useIntrinsics: compatibility.useIntrinsics ?? true,
+    useClipping: compatibility.useClipping ?? true,
+    preserveAspect: compatibility.preserveAspect ?? true,
+    preferResolution: compatibility.preferResolution ?? false
+  };
+}
+function mujocoVisualClip(model) {
+  const map = model.vis?.map;
+  const near = typeof map?.znear === "number" && map.znear > 0 ? map.znear : void 0;
+  const far = typeof map?.zfar === "number" && map.zfar > 0 ? map.zfar : void 0;
+  return { near, far };
+}
+function mujocoCameraResolution(model, cameraId) {
+  const resolution = model.cam_resolution;
+  if (!resolution) return {};
+  const width = Number(resolution[cameraId * 2]);
+  const height = Number(resolution[cameraId * 2 + 1]);
+  return {
+    width: Number.isFinite(width) && width > 0 ? width : void 0,
+    height: Number.isFinite(height) && height > 0 ? height : void 0
+  };
+}
+function mujocoCameraProjectionMatrix(model, cameraId, width, height, near, far) {
+  const intrinsic = model.cam_intrinsic;
+  const sensorSize = model.cam_sensorsize;
+  if (!intrinsic || !sensorSize || !width || !height) return void 0;
+  const intrinsicOffset = cameraId * 4;
+  const sensorOffset = cameraId * 2;
+  const focalX = Number(intrinsic[intrinsicOffset]);
+  const focalY = Number(intrinsic[intrinsicOffset + 1]);
+  const principalX = Number(intrinsic[intrinsicOffset + 2]);
+  const principalY = Number(intrinsic[intrinsicOffset + 3]);
+  const sensorWidth = Number(sensorSize[sensorOffset]);
+  const sensorHeight = Number(sensorSize[sensorOffset + 1]);
+  if (!Number.isFinite(focalX) || !Number.isFinite(focalY) || !Number.isFinite(principalX) || !Number.isFinite(principalY) || !Number.isFinite(sensorWidth) || !Number.isFinite(sensorHeight) || focalX <= 0 || focalY <= 0 || sensorWidth <= 0 || sensorHeight <= 0) {
+    return void 0;
+  }
+  const fx = focalX / sensorWidth * width;
+  const fy = focalY / sensorHeight * height;
+  const cx = width * (0.5 + principalX / sensorWidth);
+  const cy = height * (0.5 + principalY / sensorHeight);
+  const znear = near ?? 0.01;
+  const zfar = far ?? 100;
+  return new THREE11.Matrix4().set(
+    2 * fx / width,
+    0,
+    1 - 2 * cx / width,
+    0,
+    0,
+    2 * fy / height,
+    2 * cy / height - 1,
+    0,
+    0,
+    0,
+    -(zfar + znear) / (zfar - znear),
+    -2 * zfar * znear / (zfar - znear),
+    0,
+    0,
+    -1,
+    0
+  );
+}
+function resolveMujocoCameraCaptureDimensions(requested, cameraResolution, compatibility) {
+  if (!compatibility.useResolution) {
+    return {
+      width: requested.width,
+      height: requested.height
+    };
+  }
+  if (compatibility.preferResolution) {
+    return {
+      width: cameraResolution.width ?? requested.width,
+      height: cameraResolution.height ?? requested.height
+    };
+  }
+  let width = requested.width ?? cameraResolution.width;
+  let height = requested.height ?? cameraResolution.height;
+  if (compatibility.preserveAspect && cameraResolution.width && cameraResolution.height) {
+    if (requested.width !== void 0 && requested.height === void 0) {
+      height = requested.width * cameraResolution.height / cameraResolution.width;
+    } else if (requested.height !== void 0 && requested.width === void 0) {
+      width = requested.height * cameraResolution.width / cameraResolution.height;
+    }
+  }
+  return { width, height };
+}
 function countMountedCameraSelectors(options) {
   return Number(Boolean(options.cameraName)) + Number(Boolean(options.siteName)) + Number(Boolean(options.bodyName));
 }
@@ -2033,6 +2160,7 @@ function MujocoSimProvider({
   paused,
   speed,
   interpolate,
+  renderOptions,
   children
 }) {
   const { gl, camera, scene } = useThree();
@@ -2067,24 +2195,12 @@ function MujocoSimProvider({
   const bodyRegistryRef = useRef(/* @__PURE__ */ new Map());
   const hiddenBodiesRef = useRef(/* @__PURE__ */ new Set());
   const bodyReloadTimerRef = useRef(null);
-  useEffect(() => {
-    configRef.current = config;
-  }, [config]);
-  useEffect(() => {
-    mujocoRef.current = mujoco;
-  }, [mujoco]);
-  useEffect(() => {
-    pausedRef.current = paused ?? false;
-  }, [paused]);
-  useEffect(() => {
-    speedRef.current = speed ?? 1;
-  }, [speed]);
-  useEffect(() => {
-    substepsRef.current = substeps ?? 1;
-  }, [substeps]);
-  useEffect(() => {
-    interpolateRef.current = interpolate ?? false;
-  }, [interpolate]);
+  configRef.current = config;
+  mujocoRef.current = mujoco;
+  pausedRef.current = paused ?? false;
+  speedRef.current = speed ?? 1;
+  substepsRef.current = substeps ?? 1;
+  interpolateRef.current = interpolate ?? false;
   useEffect(() => {
     if (!gravity) return;
     const model = mjModelRef.current;
@@ -2590,11 +2706,27 @@ function MujocoSimProvider({
     for (let i = 0; i < ncam; i += 1) {
       const posOffset = i * 3;
       const quatOffset = i * 4;
+      const intrinsicOffset = i * 4;
+      const resolutionOffset = i * 2;
       result.push({
         id: i,
         name: getName(model, nameAddresses[i]),
         bodyId: model.cam_bodyid?.[i] ?? -1,
         fov: model.cam_fovy?.[i] ?? null,
+        resolution: model.cam_resolution ? [
+          model.cam_resolution[resolutionOffset],
+          model.cam_resolution[resolutionOffset + 1]
+        ] : null,
+        sensorSize: model.cam_sensorsize ? [
+          model.cam_sensorsize[resolutionOffset],
+          model.cam_sensorsize[resolutionOffset + 1]
+        ] : null,
+        intrinsic: model.cam_intrinsic ? [
+          model.cam_intrinsic[intrinsicOffset],
+          model.cam_intrinsic[intrinsicOffset + 1],
+          model.cam_intrinsic[intrinsicOffset + 2],
+          model.cam_intrinsic[intrinsicOffset + 3]
+        ] : null,
         position: model.cam_pos ? vector3FromArray(model.cam_pos, posOffset) : null,
         quaternion: model.cam_quat ? quaternionFromMujocoQuat(model.cam_quat, quatOffset) : null
       });
@@ -2622,10 +2754,22 @@ function MujocoSimProvider({
           );
         }
         const pose = applyMountedCameraPoseOffsets(options, position, quaternion);
+        const compatibility = resolveMujocoCameraCompatibilityOptions(options);
+        const cameraResolution = compatibility?.useResolution ? mujocoCameraResolution(model, cameraId) : { width: void 0, height: void 0 };
+        const clip = compatibility?.useClipping ? mujocoVisualClip(model) : { near: void 0, far: void 0 };
+        const { width, height } = compatibility ? resolveMujocoCameraCaptureDimensions(options, cameraResolution, compatibility) : { width: options.width, height: options.height };
+        const near = options.near ?? clip.near;
+        const far = options.far ?? clip.far;
+        const projectionMatrix = compatibility?.useIntrinsics ? mujocoCameraProjectionMatrix(model, cameraId, width, height, near, far) : void 0;
         return {
           ...baseOptions,
+          width,
+          height,
           ...pose,
           fov: options.fov ?? model.cam_fovy?.[cameraId],
+          near,
+          far,
+          projectionMatrix: options.projectionMatrix ?? projectionMatrix,
           source: { kind: "mujoco-camera", cameraName: options.cameraName }
         };
       }
@@ -3271,7 +3415,7 @@ function MujocoSimProvider({
     [api, status, requestBodyReload]
   );
   return /* @__PURE__ */ jsxs(MujocoSimContext.Provider, { value: contextValue, children: [
-    /* @__PURE__ */ jsx(SceneRenderer, {}),
+    /* @__PURE__ */ jsx(SceneRenderer, { renderOptions }),
     children
   ] });
 }
@@ -3289,16 +3433,19 @@ var MujocoCanvas = forwardRef(
     paused,
     speed,
     interpolate,
+    renderOptions,
     loadingFallback,
     children,
     ...canvasProps
   }, ref) {
     const { mujoco, status: wasmStatus, error: wasmError } = useMujocoWasm();
+    const onErrorRef = useRef(onError);
+    onErrorRef.current = onError;
     useEffect(() => {
-      if (wasmStatus === "error" && onError) {
-        onError(new Error(wasmError ?? "WASM load failed"));
+      if (wasmStatus === "error") {
+        onErrorRef.current?.(new Error(wasmError ?? "WASM load failed"));
       }
-    }, [wasmStatus, wasmError, onError]);
+    }, [wasmStatus, wasmError]);
     if (wasmStatus === "loading" || !mujoco) {
       return loadingFallback ? /* @__PURE__ */ jsx(Canvas, { ...canvasProps, children: loadingFallback }) : null;
     }
@@ -3321,6 +3468,7 @@ var MujocoCanvas = forwardRef(
         paused,
         speed,
         interpolate,
+        renderOptions,
         children
       }
     ) });
@@ -3329,11 +3477,13 @@ var MujocoCanvas = forwardRef(
 var MujocoPhysics = forwardRef(
   function MujocoPhysics2({ onError, children, ...props }, ref) {
     const { mujoco, status: wasmStatus, error: wasmError } = useMujocoWasm();
+    const onErrorRef = useRef(onError);
+    onErrorRef.current = onError;
     useEffect(() => {
-      if (wasmStatus === "error" && onError) {
-        onError(new Error(wasmError ?? "WASM load failed"));
+      if (wasmStatus === "error") {
+        onErrorRef.current?.(new Error(wasmError ?? "WASM load failed"));
       }
-    }, [wasmStatus, wasmError, onError]);
+    }, [wasmStatus, wasmError]);
     if (wasmStatus === "error" || wasmStatus === "loading" || !mujoco) {
       return null;
     }
@@ -6561,7 +6711,8 @@ function usePolicy(config) {
       const shouldInfer = !inFlightRef.current && (!queuedAction || actionQueueRef.current.length <= prefetchThreshold);
       if (!shouldInfer) return;
       const observation = cfg.onObservation({ model, data });
-      const result = cfg.infer ? cfg.infer({ observation, model, data }) : observation;
+      const queuedActions = actionQueueRef.current.length;
+      const result = cfg.infer ? cfg.infer({ observation, model, data, queuedActions }) : observation;
       if (isPromiseLike(result)) {
         const epoch = epochRef.current;
         inFlightRef.current = true;
@@ -6592,11 +6743,11 @@ function usePolicy(config) {
             cfg.queueStrategy ?? "append"
           );
         } else {
-          const [action, ...queuedActions] = actions;
+          const [action, ...queuedActions2] = actions;
           if (!action) return;
           enqueuePolicyActions(
             actionQueueRef.current,
-            queuedActions,
+            queuedActions2,
             observation,
             cfg.queueStrategy ?? "append"
           );
@@ -6739,7 +6890,7 @@ function useRemotePolicy(config) {
   const remoteEpochRef = useRef(0);
   const policy = usePolicy({
     ...config,
-    infer: async ({ observation, model, data }) => {
+    infer: async ({ observation, model, data, queuedActions }) => {
       const cfg = configRef.current;
       abortController(abortControllerRef.current, createAbortError("Remote policy request was superseded."));
       const controller = new AbortController();
@@ -6751,6 +6902,7 @@ function useRemotePolicy(config) {
         observation,
         model,
         data,
+        queuedActions,
         reset: requestIndex === 0,
         requestIndex,
         signal