mujoco-react 10.2.1 → 10.4.0

package/src/rendering/cameraFrameCapture.ts

@@ -80,6 +80,15 @@ type RendererState = {
   clearColor: THREE.Color;
   clearAlpha: number;
   autoClear: boolean;
+  shadowMapEnabled: boolean;
+  toneMapping: THREE.WebGLRenderer['toneMapping'];
+  outputColorSpace: THREE.WebGLRenderer['outputColorSpace'];
+};
+
+type SceneVisualState = {
+  background: THREE.Scene['background'];
+  environment: THREE.Scene['environment'];
+  fog: THREE.Scene['fog'];
 };
 
 type VisibilityState = {
@@ -89,6 +98,107 @@ type VisibilityState = {
 
 type CameraFrameCapturePreRender = () => void;
 
+const isolatedRendererCache = new WeakMap<
+  THREE.WebGLRenderer,
+  Map<string, THREE.WebGLRenderer>
+>();
+
+function shouldUseRenderIsolation(
+  options: CameraFrameCaptureOptions
+): boolean {
+  return options.renderIsolation === true || (
+    typeof options.renderIsolation === 'object' &&
+    options.renderIsolation.enabled !== false
+  );
+}
+
+function getRenderIsolationOptions(
+  options: CameraFrameCaptureOptions
+) {
+  return typeof options.renderIsolation === 'object'
+    ? options.renderIsolation
+    : {};
+}
+
+function getRenderIsolationCacheKey(
+  width: number,
+  height: number,
+  options: CameraFrameCaptureOptions
+) {
+  const isolation = getRenderIsolationOptions(options);
+  return JSON.stringify({
+    width,
+    height,
+    antialias: isolation.antialias ?? false,
+    alpha: isolation.alpha ?? false,
+    preserveDrawingBuffer: isolation.preserveDrawingBuffer ?? false,
+    powerPreference: isolation.powerPreference ?? null,
+  });
+}
+
+function createIsolatedRenderer(
+  sourceRenderer: THREE.WebGLRenderer,
+  width: number,
+  height: number,
+  options: CameraFrameCaptureOptions
+): { renderer: THREE.WebGLRenderer; cached: boolean } | null {
+  if (!shouldUseRenderIsolation(options)) return null;
+
+  const isolation = getRenderIsolationOptions(options);
+  if (isolation.cache !== false) {
+    const cacheKey = getRenderIsolationCacheKey(width, height, options);
+    let rendererCache = isolatedRendererCache.get(sourceRenderer);
+    if (!rendererCache) {
+      rendererCache = new Map();
+      isolatedRendererCache.set(sourceRenderer, rendererCache);
+    }
+    const cachedRenderer = rendererCache.get(cacheKey);
+    if (cachedRenderer) {
+      cachedRenderer.outputColorSpace = sourceRenderer.outputColorSpace;
+      cachedRenderer.toneMapping = sourceRenderer.toneMapping;
+      cachedRenderer.shadowMap.enabled = false;
+      return { renderer: cachedRenderer, cached: true };
+    }
+    const createdRenderer = createUncachedIsolatedRenderer(
+      sourceRenderer,
+      width,
+      height,
+      options
+    );
+    rendererCache.set(cacheKey, createdRenderer);
+    return { renderer: createdRenderer, cached: true };
+  }
+
+  return {
+    renderer: createUncachedIsolatedRenderer(sourceRenderer, width, height, options),
+    cached: false,
+  };
+}
+
+function createUncachedIsolatedRenderer(
+  sourceRenderer: THREE.WebGLRenderer,
+  width: number,
+  height: number,
+  options: CameraFrameCaptureOptions
+) {
+  const isolation = getRenderIsolationOptions(options);
+  const canvas = document.createElement('canvas');
+  const renderer = new THREE.WebGLRenderer({
+    canvas,
+    antialias: isolation.antialias ?? false,
+    alpha: isolation.alpha ?? false,
+    preserveDrawingBuffer: isolation.preserveDrawingBuffer ?? false,
+    powerPreference: isolation.powerPreference,
+  });
+
+  renderer.setPixelRatio(1);
+  renderer.setSize(width, height, false);
+  renderer.outputColorSpace = sourceRenderer.outputColorSpace;
+  renderer.toneMapping = sourceRenderer.toneMapping;
+  renderer.shadowMap.enabled = false;
+  return renderer;
+}
+
 function toVector3(
   value: CameraFrameCaptureVector3 | undefined,
   fallback: THREE.Vector3
@@ -127,6 +237,19 @@ function applyCameraPose(
   camera.updateMatrixWorld();
 }
 
+function applyProjectionMatrix(
+  camera: THREE.Camera,
+  projectionMatrix: CameraFrameCaptureOptions['projectionMatrix'] | undefined
+) {
+  if (!projectionMatrix) return;
+  if (projectionMatrix instanceof THREE.Matrix4) {
+    camera.projectionMatrix.copy(projectionMatrix);
+  } else {
+    camera.projectionMatrix.fromArray(projectionMatrix);
+  }
+  camera.projectionMatrixInverse.copy(camera.projectionMatrix).invert();
+}
+
 function createCaptureCamera(
   options: CameraFrameCaptureOptions,
   fallbackCamera: THREE.Camera,
@@ -146,6 +269,7 @@ function createCaptureCamera(
     camera.far = options.far ?? camera.far;
     camera.updateProjectionMatrix();
   }
+  applyProjectionMatrix(camera, options.projectionMatrix);
 
   applyCameraPose(camera, options, fallbackCamera);
   return camera;
@@ -184,6 +308,7 @@ function prepareCaptureCamera(
     camera.far = options.far ?? camera.far;
     camera.updateProjectionMatrix();
   }
+  applyProjectionMatrix(camera, options.projectionMatrix);
 
   applyCameraPose(camera, options, fallbackCamera);
 }
@@ -362,6 +487,9 @@ function saveRendererState(renderer: THREE.WebGLRenderer): RendererState {
     clearColor,
     clearAlpha: renderer.getClearAlpha(),
     autoClear: renderer.autoClear,
+    shadowMapEnabled: renderer.shadowMap.enabled,
+    toneMapping: renderer.toneMapping,
+    outputColorSpace: renderer.outputColorSpace,
   };
 }
 
@@ -376,6 +504,64 @@ function restoreRendererState(
   renderer.setScissorTest(state.scissorTest);
   renderer.setClearColor(state.clearColor, state.clearAlpha);
   renderer.autoClear = state.autoClear;
+  renderer.shadowMap.enabled = state.shadowMapEnabled;
+  renderer.toneMapping = state.toneMapping;
+  renderer.outputColorSpace = state.outputColorSpace;
+}
+
+function saveSceneVisualState(scene: THREE.Scene): SceneVisualState {
+  return {
+    background: scene.background,
+    environment: scene.environment,
+    fog: scene.fog,
+  };
+}
+
+function restoreSceneVisualState(scene: THREE.Scene, state: SceneVisualState) {
+  scene.background = state.background;
+  scene.environment = state.environment;
+  scene.fog = state.fog;
+}
+
+function hasOwn<T extends object>(object: T, key: keyof T) {
+  return Object.prototype.hasOwnProperty.call(object, key);
+}
+
+function applyCaptureVisualOverrides(
+  renderer: THREE.WebGLRenderer,
+  scene: THREE.Scene,
+  options: CameraFrameCaptureOptions
+): SceneVisualState | null {
+  const overrides = options.visualOverrides;
+  if (!overrides) return null;
+
+  const previousSceneState = saveSceneVisualState(scene);
+  if (hasOwn(overrides, 'sceneBackground')) {
+    const background = overrides.sceneBackground;
+    scene.background = background === false ? null : (
+      typeof background === 'string' || typeof background === 'number'
+        ? new THREE.Color(background)
+        : background ?? null
+    );
+  }
+  if (hasOwn(overrides, 'sceneEnvironment')) {
+    const environment = overrides.sceneEnvironment;
+    scene.environment = environment === false ? null : environment ?? null;
+  }
+  if (hasOwn(overrides, 'sceneFog')) {
+    const fog = overrides.sceneFog;
+    scene.fog = fog === false ? null : fog ?? null;
+  }
+  if (overrides.shadows !== undefined) {
+    renderer.shadowMap.enabled = overrides.shadows;
+  }
+  if (overrides.toneMapping !== undefined) {
+    renderer.toneMapping = overrides.toneMapping;
+  }
+  if (overrides.outputColorSpace !== undefined) {
+    renderer.outputColorSpace = overrides.outputColorSpace;
+  }
+  return previousSceneState;
 }
 
 function getCaptureRenderer(
@@ -410,6 +596,8 @@ export function createCameraFrameCaptureSession(
   options: CameraFrameCaptureOptions = {}
 ): CameraFrameCaptureSession {
   const { width, height } = getCaptureDimensions(renderer, options);
+  const isolatedRenderer = createIsolatedRenderer(renderer, width, height, options);
+  const sessionRenderer = isolatedRenderer?.renderer ?? renderer;
   const camera = createCaptureCamera(options, fallbackCamera, width, height);
   const target = new THREE.WebGLRenderTarget(width, height, {
     format: THREE.RGBAFormat,
@@ -430,6 +618,13 @@ export function createCameraFrameCaptureSession(
 
   function resolveCaptureOptions(nextOptions: CameraFrameCaptureOptions = {}) {
     const captureOptions = { ...options, ...nextOptions };
+    if (
+      shouldUseRenderIsolation(captureOptions) !== shouldUseRenderIsolation(options)
+    ) {
+      throw new Error(
+        'Camera frame capture sessions require stable renderIsolation settings.'
+      );
+    }
     const nextDimensions = getCaptureDimensions(renderer, captureOptions);
     if (
       nextDimensions.width !== width ||
@@ -452,7 +647,12 @@ export function createCameraFrameCaptureSession(
   }
 
   function renderPreparedCapture(captureOptions: CameraFrameCaptureOptions) {
-    const previousState = saveRendererState(renderer);
+    const previousState = saveRendererState(sessionRenderer);
+    const previousSceneState = applyCaptureVisualOverrides(
+      sessionRenderer,
+      scene,
+      captureOptions
+    );
     const hidden = [
       ...hideExcludedCaptureObjects(scene),
       ...hideCaptureGeomGroups(scene, captureOptions),
@@ -461,23 +661,23 @@ export function createCameraFrameCaptureSession(
     runCapturePreRenderHooks(scene);
     scene.updateMatrixWorld(true);
     try {
-      renderer.xr.enabled = false;
-      renderer.setRenderTarget(target);
-      renderer.setViewport(0, 0, width, height);
-      renderer.setScissor(0, 0, width, height);
-      renderer.setScissorTest(false);
+      sessionRenderer.xr.enabled = false;
+      sessionRenderer.setRenderTarget(target);
+      sessionRenderer.setViewport(0, 0, width, height);
+      sessionRenderer.setScissor(0, 0, width, height);
+      sessionRenderer.setScissorTest(false);
       if (captureOptions.background !== undefined) {
-        renderer.setClearColor(
+        sessionRenderer.setClearColor(
           new THREE.Color(captureOptions.background),
           captureOptions.backgroundAlpha ?? previousState.clearAlpha
         );
       } else if (captureOptions.backgroundAlpha !== undefined) {
-        renderer.setClearColor(previousState.clearColor, captureOptions.backgroundAlpha);
+        sessionRenderer.setClearColor(previousState.clearColor, captureOptions.backgroundAlpha);
       }
-      renderer.clear();
-      renderer.render(scene, camera);
+      sessionRenderer.clear();
+      sessionRenderer.render(scene, camera);
       readRenderTargetToCanvas(
-        renderer,
+        sessionRenderer,
         target,
         canvas,
         drawContext,
@@ -485,7 +685,7 @@ export function createCameraFrameCaptureSession(
         imageData,
         width,
         height,
-        renderer.outputColorSpace,
+        sessionRenderer.outputColorSpace,
         captureOptions.flipX ?? false
       );
       return {
@@ -497,7 +697,8 @@ export function createCameraFrameCaptureSession(
       };
     } finally {
       restoreObjectVisibility(hidden);
-      restoreRendererState(renderer, previousState);
+      if (previousSceneState) restoreSceneVisualState(scene, previousSceneState);
+      restoreRendererState(sessionRenderer, previousState);
     }
   }
 
@@ -511,23 +712,28 @@ export function createCameraFrameCaptureSession(
     scene.updateMatrixWorld(true);
     const captureRenderer = getCaptureRenderer(scene);
     if (captureRenderer) {
-      const previousState = saveRendererState(renderer);
+      const previousState = saveRendererState(sessionRenderer);
+      const previousSceneState = applyCaptureVisualOverrides(
+        sessionRenderer,
+        scene,
+        captureOptions
+      );
       const hidden = [
         ...hideExcludedCaptureObjects(scene),
         ...hideCaptureGeomGroups(scene, captureOptions),
       ];
       try {
-        renderer.xr.enabled = false;
+        sessionRenderer.xr.enabled = false;
         if (captureOptions.background !== undefined) {
-          renderer.setClearColor(
+          sessionRenderer.setClearColor(
             new THREE.Color(captureOptions.background),
             captureOptions.backgroundAlpha ?? previousState.clearAlpha
           );
         } else if (captureOptions.backgroundAlpha !== undefined) {
-          renderer.setClearColor(previousState.clearColor, captureOptions.backgroundAlpha);
+          sessionRenderer.setClearColor(previousState.clearColor, captureOptions.backgroundAlpha);
         }
         const captureResult = await captureRenderer({
-          renderer,
+          renderer: sessionRenderer,
           scene,
           camera,
           target,
@@ -561,7 +767,8 @@ export function createCameraFrameCaptureSession(
         }
       } finally {
         restoreObjectVisibility(hidden);
-        restoreRendererState(renderer, previousState);
+        if (previousSceneState) restoreSceneVisualState(scene, previousSceneState);
+        restoreRendererState(sessionRenderer, previousState);
       }
     }
     return renderPreparedCapture(captureOptions);
@@ -613,6 +820,9 @@ export function createCameraFrameCaptureSession(
     },
     dispose() {
       target.dispose();
+      if (isolatedRenderer && !isolatedRenderer.cached) {
+        isolatedRenderer.renderer.dispose();
+      }
     },
   };
 }

package/src/spark.tsx

@@ -381,6 +381,10 @@ export function SparkSplatEnvironment({
   const onErrorRef = useRef(onError);
   const [status, setStatus] = useState<SparkSplatStatus>('idle');
   const { gl, invalidate } = useThree();
+  onStatusChangeRef.current = onStatusChange;
+  onLoadRef.current = onLoad;
+  onErrorRef.current = onError;
+
   const metadata = useSplatEnvironment({
     environment,
     scenario,
@@ -413,18 +417,6 @@ export function SparkSplatEnvironment({
     ]
   );
 
-  useEffect(() => {
-    onStatusChangeRef.current = onStatusChange;
-  }, [onStatusChange]);
-
-  useEffect(() => {
-    onLoadRef.current = onLoad;
-  }, [onLoad]);
-
-  useEffect(() => {
-    onErrorRef.current = onError;
-  }, [onError]);
-
   useEffect(() => {
     let disposed = false;
     ensureSparkDisposeRejectionHandler();

package/src/types.ts

@@ -328,6 +328,9 @@ export interface MujocoModel {
   cam_pos?: Float64Array;
   cam_quat?: Float64Array;
   cam_fovy?: Float64Array;
+  cam_intrinsic?: Float64Array;
+  cam_resolution?: Int32Array;
+  cam_sensorsize?: Float64Array;
 
   // Tendon
   ten_wrapadr: Int32Array;
@@ -723,6 +726,9 @@ export interface CameraInfo {
   name: string;
   bodyId: number;
   fov: number | null;
+  resolution: [number, number] | null;
+  sensorSize: [number, number] | null;
+  intrinsic: [number, number, number, number] | null;
   position: [number, number, number] | null;
   quaternion: [number, number, number, number] | null;
 }
@@ -1021,11 +1027,28 @@ export interface ObservationHandle {
 
 // ---- Debug Component (spec 6.1) ----
 
+export interface DebugVirtualCamera {
+  name?: string;
+  position?: CameraFrameCaptureVector3;
+  lookAt?: CameraFrameCaptureVector3;
+  up?: CameraFrameCaptureVector3;
+  quaternion?: THREE.Quaternion | readonly [number, number, number, number];
+  fov?: number;
+  width?: number;
+  height?: number;
+  frustumDepth?: number;
+  markerScale?: number;
+  color?: THREE.ColorRepresentation;
+  aimColor?: THREE.ColorRepresentation;
+}
+
 export interface DebugProps {
   showGeoms?: boolean;
   showSites?: boolean;
   showJoints?: boolean;
   showCameras?: boolean;
+  /** Additional explicit virtual camera poses to visualize alongside MuJoCo XML cameras. */
+  virtualCameras?: readonly DebugVirtualCamera[];
   showContacts?: boolean;
   showCOM?: boolean;
   showInertia?: boolean;
@@ -1505,6 +1528,51 @@ export type CameraFrameCaptureQuaternion =
   | THREE.Quaternion
   | readonly [number, number, number, number];
 
+export interface CameraFrameVisualOverrides {
+  /**
+   * Override `scene.background` for this capture only.
+   * Use `null` or `false` to render without the viewer scene background.
+   */
+  sceneBackground?: THREE.Scene['background'] | THREE.ColorRepresentation | null | false;
+  /**
+   * Override `scene.environment` for this capture only.
+   * Use `null` or `false` to remove viewer environment lighting/maps.
+   */
+  sceneEnvironment?: THREE.Scene['environment'] | null | false;
+  /**
+   * Override `scene.fog` for this capture only.
+   * Use `null` or `false` to remove viewer fog.
+   */
+  sceneFog?: THREE.Scene['fog'] | null | false;
+  /** Override `renderer.shadowMap.enabled` while capturing. */
+  shadows?: boolean;
+  /** Override renderer tone mapping while capturing. */
+  toneMapping?: THREE.WebGLRenderer['toneMapping'];
+  /** Override renderer output color space while capturing. */
+  outputColorSpace?: THREE.WebGLRenderer['outputColorSpace'];
+}
+
+export interface CameraFrameRenderIsolationOptions {
+  /**
+   * Use an independent offscreen WebGLRenderer for this capture.
+   *
+   * This prevents viewer renderer settings such as antialiasing, shadow-map
+   * configuration, tone mapping, and environment setup from leaking into
+   * policy/training images. Leave unset for the historical shared-renderer path.
+   */
+  enabled?: boolean;
+  /** Offscreen renderer antialiasing. Defaults to false for deterministic policy captures. */
+  antialias?: boolean;
+  /** Offscreen renderer alpha buffer. Defaults to false, matching Three.js WebGLRenderer. */
+  alpha?: boolean;
+  /** Offscreen renderer preserveDrawingBuffer flag. Defaults to false. */
+  preserveDrawingBuffer?: boolean;
+  /** Offscreen renderer power preference. Defaults to the browser's renderer default. */
+  powerPreference?: WebGLPowerPreference;
+  /** Reuse an offscreen renderer for matching capture dimensions/options. Defaults to true. */
+  cache?: boolean;
+}
+
 export interface CameraFrameCaptureOptions {
   /** Existing Three camera to clone before applying pose overrides. */
   camera?: THREE.Camera;
@@ -1529,8 +1597,36 @@ export interface CameraFrameCaptureOptions {
   fov?: number;
   near?: number;
   far?: number;
+  /**
+   * Explicit projection matrix for offscreen capture. This is useful when a
+   * MuJoCo camera has calibrated intrinsics that cannot be represented by a
+   * symmetric Three.js PerspectiveCamera fov alone.
+   */
+  projectionMatrix?: THREE.Matrix4 | readonly number[];
   /** Provenance for the camera pose used by the capture. Usually set by the MuJoCo provider. */
   source?: CameraFrameCaptureSource;
+  /**
+   * When resolving a named MuJoCo camera, derive Three capture settings from
+   * MuJoCo camera metadata where available: cam_resolution, cam_fovy,
+   * cam_intrinsic/cam_sensorsize, and visual map near/far clipping.
+   */
+  mujocoCameraCompatibility?: boolean | {
+    useResolution?: boolean;
+    useIntrinsics?: boolean;
+    useClipping?: boolean;
+    /**
+     * When a MuJoCo camera has `resolution` metadata and the caller supplies
+     * only width or only height, derive the missing dimension from the MuJoCo
+     * camera aspect ratio.
+     */
+    preserveAspect?: boolean;
+    /**
+     * Prefer the MuJoCo camera's configured resolution over width/height
+     * provided by the caller. Leave false when a policy wants fixed-size
+     * payloads while still preserving the camera aspect ratio.
+     */
+    preferResolution?: boolean;
+  };
   /** Hide rendered Three objects whose MuJoCo geom group is in this list. */
   hiddenGeomGroups?: readonly number[];
   /** When provided, only rendered Three objects whose MuJoCo geom group is in this list are visible. */
@@ -1541,6 +1637,15 @@ export interface CameraFrameCaptureOptions {
   background?: THREE.ColorRepresentation;
   /** Optional clear alpha for this capture only. Defaults to the renderer's current clear alpha. */
   backgroundAlpha?: number;
+  /** Temporary scene/renderer visual overrides applied only for this offscreen capture. */
+  visualOverrides?: CameraFrameVisualOverrides;
+  /**
+   * Render this capture with a separate offscreen WebGLRenderer.
+   *
+   * This is useful for policy or training captures that should remain canonical
+   * while the interactive viewer uses richer visual effects.
+   */
+  renderIsolation?: boolean | CameraFrameRenderIsolationOptions;
   /** Mirror the captured image horizontally after rendering. Useful when matching policy datasets with mirrored camera frames. */
   flipX?: boolean;
 }
@@ -1658,6 +1763,21 @@ export interface CameraFrameSequenceRecorderAPI {
   reset: () => void;
 }
 
+export type MujocoMeshNormalSmoothing =
+  | boolean
+  | {
+    /** Vertex merge tolerance used before recomputing mesh normals. Defaults to `1e-4`. */
+    tolerance?: number;
+  };
+
+export interface MujocoRenderOptions {
+  /**
+   * Smooth mesh normals by welding duplicate vertices before recomputing normals.
+   * Useful for faceted STL visuals; keep off for exact policy-render parity.
+   */
+  meshNormalSmoothing?: MujocoMeshNormalSmoothing;
+}
+
 // ---- Canvas Props ----
 
 export type MujocoCanvasProps = Omit<CanvasProps, 'onError'> & {
@@ -1675,6 +1795,7 @@ export type MujocoCanvasProps = Omit<CanvasProps, 'onError'> & {
   paused?: boolean;
   speed?: number;
   interpolate?: boolean;
+  renderOptions?: MujocoRenderOptions;
 };
 
 // ---- Hook Return Types ----