mujoco-react 8.10.0 → 9.0.0

package/src/components/VisualScenario.tsx

@@ -3,20 +3,26 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
+import { useThree } from '@react-three/fiber';
 import type { ThreeElements } from '@react-three/fiber';
 import type { ReactNode } from 'react';
-import { useMemo } from 'react';
+import { useEffect, useMemo } from 'react';
 import * as THREE from 'three';
 import type {
   PairedSplatEnvironmentConfig,
+  ScenarioMaterialConfig,
+  SceneConfig,
   SplatCollisionProxyConfig,
   SplatEnvironmentMetadata,
   SplatEnvironmentMetadataInput,
   SplatFormat,
+  SplatRendererKind,
+  SplatSceneInput,
   ScenarioLightingPreset,
   ScenarioLightingProps,
   SplatEnvironmentProps,
   VisualScenarioConfig,
+  VisualScenarioEffectsProps,
 } from '../types';
 
 const DEFAULT_BACKGROUND = '#181a1f';
@@ -104,6 +110,99 @@ export function getScenarioCameraPosition(
   ];
 }
 
+export function VisualScenarioEffects(props: VisualScenarioEffectsProps) {
+  useVisualScenarioEffects(props);
+  return null;
+}
+
+export function useVisualScenarioEffects({
+  scenario,
+  enabled = true,
+  applyBackground = true,
+  applyFog = true,
+  applyRenderer = true,
+  applyMaterials = true,
+  background,
+  fogNear,
+  fogFar,
+  materialFilter,
+}: VisualScenarioEffectsProps) {
+  const { gl, scene, invalidate } = useThree();
+
+  useEffect(() => {
+    if (!enabled || !scenario) {
+      return undefined;
+    }
+
+    const previousExposure = gl.toneMappingExposure;
+    const previousBackground = scene.background;
+    const previousFog = scene.fog;
+    const materialSnapshots = new Map<
+      THREE.Material,
+      {
+        color?: THREE.Color;
+        roughness?: number;
+        metalness?: number;
+      }
+    >();
+
+    if (applyRenderer) {
+      gl.toneMappingExposure = scenario.camera?.exposure ?? 1;
+    }
+
+    if (applyBackground) {
+      scene.background = new THREE.Color(
+        background ?? getScenarioBackground(scenario.lighting)
+      );
+    }
+
+    if (applyFog) {
+      scene.fog = createScenarioFog(scenario, background, fogNear, fogFar);
+    }
+
+    if (applyMaterials && scenario.materials) {
+      applyScenarioMaterials(scene, scenario, materialSnapshots, materialFilter);
+    }
+
+    invalidate();
+
+    return () => {
+      gl.toneMappingExposure = previousExposure;
+      scene.background = previousBackground;
+      scene.fog = previousFog;
+
+      for (const [material, snapshot] of materialSnapshots) {
+        const mutable = getMutableScenarioMaterial(material);
+        if (!mutable) continue;
+        if (snapshot.color) mutable.color.copy(snapshot.color);
+        if (typeof snapshot.roughness === 'number') {
+          mutable.roughness = snapshot.roughness;
+        }
+        if (typeof snapshot.metalness === 'number') {
+          mutable.metalness = snapshot.metalness;
+        }
+        mutable.needsUpdate = true;
+      }
+
+      invalidate();
+    };
+  }, [
+    applyBackground,
+    applyFog,
+    applyMaterials,
+    applyRenderer,
+    background,
+    enabled,
+    fogFar,
+    fogNear,
+    gl,
+    invalidate,
+    materialFilter,
+    scenario,
+    scene,
+  ]);
+}
+
 /**
  * Renderer-agnostic Gaussian splat environment boundary.
  *
@@ -113,6 +212,8 @@ export function getScenarioCameraPosition(
  */
 export function SplatEnvironment({
   environment,
+  scenario,
+  renderer,
   src,
   format,
   collisionProxy,
@@ -123,6 +224,8 @@ export function SplatEnvironment({
 }: SplatEnvironmentProps) {
   const metadata = useSplatEnvironment({
     environment,
+    scenario,
+    renderer,
     src,
     format,
     collisionProxy: collisionProxyMetadata,
@@ -149,13 +252,31 @@ export function SplatEnvironment({
 
 export function useSplatEnvironment({
   environment,
+  scenario,
+  renderer,
   src,
   format,
   collisionProxy,
 }: SplatEnvironmentMetadataInput): SplatEnvironmentMetadata {
-  const resolvedSrc = src ?? environment?.splat.src;
-  const resolvedFormat = format ?? environment?.splat.format ?? 'spz';
-  const resolvedCollisionProxy = collisionProxy ?? environment?.collisionProxy;
+  const scenarioEnvironment = useMemo(
+    () =>
+      environment ??
+      (scenario
+        ? createPairedSplatEnvironment(scenario, { renderer })
+        : undefined),
+    [environment, renderer, scenario]
+  );
+  const resolvedSrc = src ?? scenarioEnvironment?.splat.src ?? scenario?.splat?.src;
+  const resolvedFormat =
+    format ??
+    scenarioEnvironment?.splat.format ??
+    scenario?.splat?.format ??
+    'spz';
+  const resolvedCollisionProxy =
+    collisionProxy ??
+    scenarioEnvironment?.collisionProxy ??
+    scenario?.splat?.collisionProxy ??
+    undefined;
 
   return useMemo(
     () => ({
@@ -163,16 +284,110 @@ export function useSplatEnvironment({
       format: resolvedFormat,
       collisionProxy: resolvedCollisionProxy,
       userData: createSplatEnvironmentUserData({
-        environment,
+        environment: scenarioEnvironment,
         src: resolvedSrc,
         format: resolvedFormat,
         collisionProxy: resolvedCollisionProxy,
       }),
     }),
-    [environment, resolvedSrc, resolvedFormat, resolvedCollisionProxy]
+    [scenarioEnvironment, resolvedSrc, resolvedFormat, resolvedCollisionProxy]
   );
 }
 
+/**
+ * Convert a generic visual scenario splat block into a paired visual/physics
+ * environment config. Returns undefined until both the splat asset and MJCF
+ * collision proxy are present.
+ */
+export function createPairedSplatEnvironment(
+  scenario: Pick<VisualScenarioConfig, 'id' | 'label' | 'environment' | 'splat'>,
+  options: {
+    id?: string;
+    label?: string;
+    description?: string;
+    renderer?: SplatRendererKind;
+  } = {}
+): PairedSplatEnvironmentConfig | undefined {
+  const splat = scenario.splat;
+  const collisionProxy = splat?.collisionProxy;
+
+  if (!splat?.enabled || !splat.src || !collisionProxy?.xmlPath) {
+    return undefined;
+  }
+
+  return {
+    id: options.id ?? scenario.id ?? 'splat-environment',
+    label: options.label ?? scenario.label ?? 'Gaussian splat environment',
+    description:
+      options.description ??
+      (scenario.environment
+        ? `Visual ${scenario.environment} splat paired with MJCF collision proxy.`
+        : undefined),
+    splat: {
+      src: splat.src,
+      format: splat.format ?? 'spz',
+      renderer: options.renderer,
+    },
+    collisionProxy: {
+      ...collisionProxy,
+      xmlPath: collisionProxy.xmlPath,
+    },
+  };
+}
+
+function isPairedSplatEnvironment(input: SplatSceneInput): input is PairedSplatEnvironmentConfig {
+  return !!input && 'collisionProxy' in input && 'splat' in input;
+}
+
+function sceneRelativePath(sceneConfig: SceneConfig, path: string): string {
+  const src = sceneConfig.src;
+  if (!src) return path;
+
+  const base = src.endsWith('/') ? src : src + '/';
+  if (path.startsWith(base)) return path.slice(base.length);
+  return path;
+}
+
+function uniquePaths(paths: readonly string[]): string[] {
+  const seen = new Set<string>();
+  const result: string[] = [];
+  for (const path of paths) {
+    if (seen.has(path)) continue;
+    seen.add(path);
+    result.push(path);
+  }
+  return result;
+}
+
+/**
+ * Compose a MuJoCo scene config with a paired splat collision proxy.
+ *
+ * This keeps the common hybrid setup declarative:
+ * robot XML remains `sceneFile`, the `.spz` remains a visual-only layer, and
+ * the paired MJCF collision proxy is added to `environmentFiles`.
+ */
+export function withSplatEnvironment(
+  sceneConfig: SceneConfig,
+  input: SplatSceneInput,
+  options: { renderer?: SplatRendererKind } = {}
+): SceneConfig {
+  const environment = isPairedSplatEnvironment(input)
+    ? input
+    : input
+      ? createPairedSplatEnvironment(input, options)
+      : undefined;
+  const xmlPath = environment?.collisionProxy.xmlPath;
+  if (!xmlPath) return sceneConfig;
+
+  return {
+    ...sceneConfig,
+    environmentFiles: uniquePaths([
+      ...(sceneConfig.environmentFiles ?? []),
+      sceneRelativePath(sceneConfig, xmlPath),
+    ]),
+  };
+}
+
 export function createSplatEnvironmentUserData({
   environment,
   src,
@@ -226,4 +441,126 @@ function SplatPlaceholder() {
   );
 }
 
+function createScenarioFog(
+  scenario: VisualScenarioConfig,
+  background: THREE.ColorRepresentation | undefined,
+  fogNear: number | undefined,
+  fogFar: number | undefined
+) {
+  if (scenario.lighting === 'low-light') {
+    return new THREE.Fog(
+      background ?? getScenarioBackground(scenario.lighting),
+      fogNear ?? 2.5,
+      fogFar ?? 9
+    );
+  }
+
+  if (scenario.lighting === 'warehouse') {
+    return new THREE.Fog(
+      background ?? getScenarioBackground(scenario.lighting),
+      fogNear ?? 5,
+      fogFar ?? 16
+    );
+  }
+
+  return null;
+}
+
+function applyScenarioMaterials(
+  scene: THREE.Scene,
+  scenario: VisualScenarioConfig,
+  snapshots: Map<
+    THREE.Material,
+    {
+      color?: THREE.Color;
+      roughness?: number;
+      metalness?: number;
+    }
+  >,
+  materialFilter: VisualScenarioEffectsProps['materialFilter']
+) {
+  const materials = scenario.materials;
+  if (!materials) return;
+
+  scene.traverse((object) => {
+    if (!(object instanceof THREE.Mesh)) {
+      return;
+    }
+
+    for (const material of normalizeMaterials(object.material)) {
+      const mutable = getMutableScenarioMaterial(material);
+      if (!mutable) continue;
+      if (materialFilter && !materialFilter({ object, material })) continue;
+
+      if (!snapshots.has(material)) {
+        snapshots.set(material, {
+          color: mutable.color.clone(),
+          roughness: mutable.roughness,
+          metalness: mutable.metalness,
+        });
+      }
+
+      applyScenarioMaterial(mutable, object, scenario, materials);
+    }
+  });
+}
+
+function applyScenarioMaterial(
+  material: THREE.MeshStandardMaterial | THREE.MeshPhysicalMaterial,
+  object: THREE.Object3D,
+  scenario: VisualScenarioConfig,
+  materials: ScenarioMaterialConfig
+) {
+  const seed = scenario.seed ?? 0;
+  const objectKey = `${scenario.id ?? 'scenario'}:${object.name}:${material.name}:${seed}`;
+  const variation = hashToUnitInterval(objectKey);
+
+  if (materials.randomizeObjectColors) {
+    material.color.setHSL(variation, 0.38, 0.42);
+  }
+
+  if (materials.randomizeTableMaterial) {
+    material.roughness = clamp01(
+      materials.roughness ?? 0.35 + variation * 0.45
+    );
+    material.metalness = clamp01(
+      materials.metalness ?? variation * 0.12
+    );
+  }
+
+  material.needsUpdate = true;
+}
+
+function normalizeMaterials(
+  material: THREE.Material | THREE.Material[]
+): THREE.Material[] {
+  return Array.isArray(material) ? material : [material];
+}
+
+function getMutableScenarioMaterial(
+  material: THREE.Material
+): THREE.MeshStandardMaterial | THREE.MeshPhysicalMaterial | null {
+  if (
+    material instanceof THREE.MeshStandardMaterial ||
+    material instanceof THREE.MeshPhysicalMaterial
+  ) {
+    return material;
+  }
+
+  return null;
+}
+
+function hashToUnitInterval(value: string) {
+  let hash = 2166136261;
+  for (let index = 0; index < value.length; index += 1) {
+    hash ^= value.charCodeAt(index);
+    hash = Math.imul(hash, 16777619);
+  }
+  return (hash >>> 0) / 4294967295;
+}
+
+function clamp01(value: number) {
+  return Math.max(0, Math.min(1, value));
+}
+
 export type SplatCollisionProxy = ReactNode | ThreeElements['group'];

package/src/core/MujocoCanvas.tsx

@@ -31,6 +31,7 @@ export const MujocoCanvas = forwardRef<MujocoSimAPI, MujocoCanvasProps>(
       paused,
       speed,
       interpolate,
+      loadingFallback,
       children,
       ...canvasProps
     },
@@ -44,7 +45,13 @@ export const MujocoCanvas = forwardRef<MujocoSimAPI, MujocoCanvasProps>(
       }
     }, [wasmStatus, wasmError, onError]);
 
-    if (wasmStatus === 'error' || wasmStatus === 'loading' || !mujoco) {
+    if (wasmStatus === 'loading' || !mujoco) {
+      return loadingFallback ? (
+        <Canvas {...canvasProps}>{loadingFallback}</Canvas>
+      ) : null;
+    }
+
+    if (wasmStatus === 'error') {
       return null;
     }
 

package/src/core/MujocoPhysics.tsx

@@ -6,19 +6,25 @@
 import { forwardRef, useEffect } from 'react';
 import { useMujocoWasm } from './MujocoProvider';
 import { MujocoSimProvider } from './MujocoSimProvider';
-import type { MujocoSimAPI, SceneConfig } from '../types';
+import type {
+  MujocoSimAPI,
+  ReadyCallbackInput,
+  SceneConfig,
+  SelectionCallbackInput,
+  StepCallbackInput,
+} from '../types';
 
 export interface MujocoPhysicsProps {
   /** Scene/robot configuration. */
   config: SceneConfig;
   /** Fires when model is loaded and API is ready. */
-  onReady?: (api: MujocoSimAPI) => void;
+  onReady?: (input: ReadyCallbackInput) => void;
   /** Fires on scene load failure. */
   onError?: (error: Error) => void;
   /** Called each physics step. */
-  onStep?: (time: number) => void;
+  onStep?: (input: StepCallbackInput) => void;
   /** Called on body double-click selection. */
-  onSelection?: (bodyId: number, name: string) => void;
+  onSelection?: (input: SelectionCallbackInput) => void;
   /** Override model gravity. */
   gravity?: [number, number, number];
   /** Override model.opt.timestep. */

package/src/core/MujocoSimProvider.tsx

@@ -31,11 +31,14 @@ import {
   MujocoSimAPI,
   PhysicsStepCallback,
   RayHit,
+  ReadyCallbackInput,
   SceneConfig,
   SceneObject,
+  SelectionCallbackInput,
   SensorInfo,
   SiteInfo,
   StateSnapshot,
+  StepCallbackInput,
   XmlPatch,
 } from '../types';
 import {
@@ -117,7 +120,7 @@ export interface MujocoSimContextValue {
   interpolateRef: React.RefObject<boolean>;
   interpolationStateRef: React.RefObject<BodyInterpolationState>;
   onSelectionRef: React.RefObject<
-    ((bodyId: number, name: string) => void) | undefined
+    ((input: SelectionCallbackInput) => void) | undefined
   >;
   beforeStepCallbacks: React.RefObject<Set<PhysicsStepCallback>>;
   afterStepCallbacks: React.RefObject<Set<PhysicsStepCallback>>;
@@ -197,7 +200,7 @@ export function useBeforePhysicsStep(callback: PhysicsStepCallback) {
   callbackRef.current = callback;
 
   useEffect(() => {
-    const wrapped: PhysicsStepCallback = (model, data) => callbackRef.current(model, data);
+    const wrapped: PhysicsStepCallback = (input) => callbackRef.current(input);
     beforeStepCallbacks.current.add(wrapped);
     return () => { beforeStepCallbacks.current.delete(wrapped); };
   }, [beforeStepCallbacks]);
@@ -209,7 +212,7 @@ export function useAfterPhysicsStep(callback: PhysicsStepCallback) {
   callbackRef.current = callback;
 
   useEffect(() => {
-    const wrapped: PhysicsStepCallback = (model, data) => callbackRef.current(model, data);
+    const wrapped: PhysicsStepCallback = (input) => callbackRef.current(input);
     afterStepCallbacks.current.add(wrapped);
     return () => { afterStepCallbacks.current.delete(wrapped); };
   }, [afterStepCallbacks]);
@@ -219,10 +222,10 @@ interface MujocoSimProviderProps {
   mujoco: MujocoModule;
   config: SceneConfig;
   apiRef?: React.ForwardedRef<MujocoSimAPI>;
-  onReady?: (api: MujocoSimAPI) => void;
+  onReady?: (input: ReadyCallbackInput) => void;
   onError?: (error: Error) => void;
-  onStep?: (time: number) => void;
-  onSelection?: (bodyId: number, name: string) => void;
+  onStep?: (input: StepCallbackInput) => void;
+  onSelection?: (input: SelectionCallbackInput) => void;
   // Declarative physics config props
   gravity?: [number, number, number];
   timestep?: number;
@@ -380,7 +383,7 @@ export function MujocoSimProvider({
   useEffect(() => {
     if (status === 'ready') {
       const api = apiRef.current;
-      if (onReady) onReady(api);
+      if (onReady) onReady({ api });
       // Assign the forwarded ref
       if (externalApiRef) {
         if (typeof externalApiRef === 'function') {
@@ -409,7 +412,7 @@ export function MujocoSimProvider({
 
     // Before-step callbacks
     for (const cb of beforeStepCallbacks.current) {
-      cb(model, data);
+      cb({ model, data });
     }
 
     const numSubsteps = substepsRef.current;
@@ -466,17 +469,17 @@ export function MujocoSimProvider({
       interpolationStateRef.current.valid = true;
 
       if (!stepped) {
-        onStepRef.current?.(data.time);
+        onStepRef.current?.({ time: data.time, model, data });
         return;
       }
     }
 
     // After-step callbacks
     for (const cb of afterStepCallbacks.current) {
-      cb(model, data);
+      cb({ model, data });
     }
 
-    onStepRef.current?.(data.time);
+    onStepRef.current?.({ time: data.time, model, data });
   }, -1);
 
   function ensureInterpolationBuffers(model: MujocoModel) {
@@ -515,7 +518,7 @@ export function MujocoSimProvider({
       }
     }
 
-    configRef.current.onReset?.(model, data);
+    configRef.current.onReset?.({ model, data });
     mujoco.mj_forward(model, data);
 
     // Notify composable plugins (e.g. IkController)