mujoco-react 8.10.0 → 8.11.0

package/src/components/VisualScenario.tsx

@@ -3,20 +3,26 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
+import { useThree } from '@react-three/fiber';
 import type { ThreeElements } from '@react-three/fiber';
 import type { ReactNode } from 'react';
-import { useMemo } from 'react';
+import { useEffect, useMemo } from 'react';
 import * as THREE from 'three';
 import type {
   PairedSplatEnvironmentConfig,
+  ScenarioMaterialConfig,
+  SceneConfig,
   SplatCollisionProxyConfig,
   SplatEnvironmentMetadata,
   SplatEnvironmentMetadataInput,
   SplatFormat,
+  SplatRendererKind,
+  SplatSceneInput,
   ScenarioLightingPreset,
   ScenarioLightingProps,
   SplatEnvironmentProps,
   VisualScenarioConfig,
+  VisualScenarioEffectsProps,
 } from '../types';
 
 const DEFAULT_BACKGROUND = '#181a1f';
@@ -104,6 +110,99 @@ export function getScenarioCameraPosition(
   ];
 }
 
+export function VisualScenarioEffects(props: VisualScenarioEffectsProps) {
+  useVisualScenarioEffects(props);
+  return null;
+}
+
+export function useVisualScenarioEffects({
+  scenario,
+  enabled = true,
+  applyBackground = true,
+  applyFog = true,
+  applyRenderer = true,
+  applyMaterials = true,
+  background,
+  fogNear,
+  fogFar,
+  materialFilter,
+}: VisualScenarioEffectsProps) {
+  const { gl, scene, invalidate } = useThree();
+
+  useEffect(() => {
+    if (!enabled || !scenario) {
+      return undefined;
+    }
+
+    const previousExposure = gl.toneMappingExposure;
+    const previousBackground = scene.background;
+    const previousFog = scene.fog;
+    const materialSnapshots = new Map<
+      THREE.Material,
+      {
+        color?: THREE.Color;
+        roughness?: number;
+        metalness?: number;
+      }
+    >();
+
+    if (applyRenderer) {
+      gl.toneMappingExposure = scenario.camera?.exposure ?? 1;
+    }
+
+    if (applyBackground) {
+      scene.background = new THREE.Color(
+        background ?? getScenarioBackground(scenario.lighting)
+      );
+    }
+
+    if (applyFog) {
+      scene.fog = createScenarioFog(scenario, background, fogNear, fogFar);
+    }
+
+    if (applyMaterials && scenario.materials) {
+      applyScenarioMaterials(scene, scenario, materialSnapshots, materialFilter);
+    }
+
+    invalidate();
+
+    return () => {
+      gl.toneMappingExposure = previousExposure;
+      scene.background = previousBackground;
+      scene.fog = previousFog;
+
+      for (const [material, snapshot] of materialSnapshots) {
+        const mutable = getMutableScenarioMaterial(material);
+        if (!mutable) continue;
+        if (snapshot.color) mutable.color.copy(snapshot.color);
+        if (typeof snapshot.roughness === 'number') {
+          mutable.roughness = snapshot.roughness;
+        }
+        if (typeof snapshot.metalness === 'number') {
+          mutable.metalness = snapshot.metalness;
+        }
+        mutable.needsUpdate = true;
+      }
+
+      invalidate();
+    };
+  }, [
+    applyBackground,
+    applyFog,
+    applyMaterials,
+    applyRenderer,
+    background,
+    enabled,
+    fogFar,
+    fogNear,
+    gl,
+    invalidate,
+    materialFilter,
+    scenario,
+    scene,
+  ]);
+}
+
 /**
  * Renderer-agnostic Gaussian splat environment boundary.
  *
@@ -113,6 +212,8 @@ export function getScenarioCameraPosition(
  */
 export function SplatEnvironment({
   environment,
+  scenario,
+  renderer,
   src,
   format,
   collisionProxy,
@@ -123,6 +224,8 @@ export function SplatEnvironment({
 }: SplatEnvironmentProps) {
   const metadata = useSplatEnvironment({
     environment,
+    scenario,
+    renderer,
     src,
     format,
     collisionProxy: collisionProxyMetadata,
@@ -149,13 +252,31 @@ export function SplatEnvironment({
 
 export function useSplatEnvironment({
   environment,
+  scenario,
+  renderer,
   src,
   format,
   collisionProxy,
 }: SplatEnvironmentMetadataInput): SplatEnvironmentMetadata {
-  const resolvedSrc = src ?? environment?.splat.src;
-  const resolvedFormat = format ?? environment?.splat.format ?? 'spz';
-  const resolvedCollisionProxy = collisionProxy ?? environment?.collisionProxy;
+  const scenarioEnvironment = useMemo(
+    () =>
+      environment ??
+      (scenario
+        ? createPairedSplatEnvironment(scenario, { renderer })
+        : undefined),
+    [environment, renderer, scenario]
+  );
+  const resolvedSrc = src ?? scenarioEnvironment?.splat.src ?? scenario?.splat?.src;
+  const resolvedFormat =
+    format ??
+    scenarioEnvironment?.splat.format ??
+    scenario?.splat?.format ??
+    'spz';
+  const resolvedCollisionProxy =
+    collisionProxy ??
+    scenarioEnvironment?.collisionProxy ??
+    scenario?.splat?.collisionProxy ??
+    undefined;
 
   return useMemo(
     () => ({
@@ -163,16 +284,110 @@ export function useSplatEnvironment({
       format: resolvedFormat,
       collisionProxy: resolvedCollisionProxy,
       userData: createSplatEnvironmentUserData({
-        environment,
+        environment: scenarioEnvironment,
         src: resolvedSrc,
         format: resolvedFormat,
         collisionProxy: resolvedCollisionProxy,
       }),
     }),
-    [environment, resolvedSrc, resolvedFormat, resolvedCollisionProxy]
+    [scenarioEnvironment, resolvedSrc, resolvedFormat, resolvedCollisionProxy]
   );
 }
 
+/**
+ * Convert a generic visual scenario splat block into a paired visual/physics
+ * environment config. Returns undefined until both the splat asset and MJCF
+ * collision proxy are present.
+ */
+export function createPairedSplatEnvironment(
+  scenario: Pick<VisualScenarioConfig, 'id' | 'label' | 'environment' | 'splat'>,
+  options: {
+    id?: string;
+    label?: string;
+    description?: string;
+    renderer?: SplatRendererKind;
+  } = {}
+): PairedSplatEnvironmentConfig | undefined {
+  const splat = scenario.splat;
+  const collisionProxy = splat?.collisionProxy;
+
+  if (!splat?.enabled || !splat.src || !collisionProxy?.xmlPath) {
+    return undefined;
+  }
+
+  return {
+    id: options.id ?? scenario.id ?? 'splat-environment',
+    label: options.label ?? scenario.label ?? 'Gaussian splat environment',
+    description:
+      options.description ??
+      (scenario.environment
+        ? `Visual ${scenario.environment} splat paired with MJCF collision proxy.`
+        : undefined),
+    splat: {
+      src: splat.src,
+      format: splat.format ?? 'spz',
+      renderer: options.renderer,
+    },
+    collisionProxy: {
+      ...collisionProxy,
+      xmlPath: collisionProxy.xmlPath,
+    },
+  };
+}
+
+function isPairedSplatEnvironment(input: SplatSceneInput): input is PairedSplatEnvironmentConfig {
+  return !!input && 'collisionProxy' in input && 'splat' in input;
+}
+
+function sceneRelativePath(sceneConfig: SceneConfig, path: string): string {
+  const src = sceneConfig.src;
+  if (!src) return path;
+
+  const base = src.endsWith('/') ? src : src + '/';
+  if (path.startsWith(base)) return path.slice(base.length);
+  return path;
+}
+
+function uniquePaths(paths: readonly string[]): string[] {
+  const seen = new Set<string>();
+  const result: string[] = [];
+  for (const path of paths) {
+    if (seen.has(path)) continue;
+    seen.add(path);
+    result.push(path);
+  }
+  return result;
+}
+
+/**
+ * Compose a MuJoCo scene config with a paired splat collision proxy.
+ *
+ * This keeps the common hybrid setup declarative:
+ * robot XML remains `sceneFile`, the `.spz` remains a visual-only layer, and
+ * the paired MJCF collision proxy is added to `environmentFiles`.
+ */
+export function withSplatEnvironment(
+  sceneConfig: SceneConfig,
+  input: SplatSceneInput,
+  options: { renderer?: SplatRendererKind } = {}
+): SceneConfig {
+  const environment = isPairedSplatEnvironment(input)
+    ? input
+    : input
+      ? createPairedSplatEnvironment(input, options)
+      : undefined;
+  const xmlPath = environment?.collisionProxy.xmlPath;
+  if (!xmlPath) return sceneConfig;
+
+  return {
+    ...sceneConfig,
+    environmentFiles: uniquePaths([
+      ...(sceneConfig.environmentFiles ?? []),
+      sceneRelativePath(sceneConfig, xmlPath),
+    ]),
+  };
+}
+
 export function createSplatEnvironmentUserData({
   environment,
   src,
@@ -226,4 +441,126 @@ function SplatPlaceholder() {
   );
 }
 
+function createScenarioFog(
+  scenario: VisualScenarioConfig,
+  background: THREE.ColorRepresentation | undefined,
+  fogNear: number | undefined,
+  fogFar: number | undefined
+) {
+  if (scenario.lighting === 'low-light') {
+    return new THREE.Fog(
+      background ?? getScenarioBackground(scenario.lighting),
+      fogNear ?? 2.5,
+      fogFar ?? 9
+    );
+  }
+
+  if (scenario.lighting === 'warehouse') {
+    return new THREE.Fog(
+      background ?? getScenarioBackground(scenario.lighting),
+      fogNear ?? 5,
+      fogFar ?? 16
+    );
+  }
+
+  return null;
+}
+
+function applyScenarioMaterials(
+  scene: THREE.Scene,
+  scenario: VisualScenarioConfig,
+  snapshots: Map<
+    THREE.Material,
+    {
+      color?: THREE.Color;
+      roughness?: number;
+      metalness?: number;
+    }
+  >,
+  materialFilter: VisualScenarioEffectsProps['materialFilter']
+) {
+  const materials = scenario.materials;
+  if (!materials) return;
+
+  scene.traverse((object) => {
+    if (!(object instanceof THREE.Mesh)) {
+      return;
+    }
+
+    for (const material of normalizeMaterials(object.material)) {
+      const mutable = getMutableScenarioMaterial(material);
+      if (!mutable) continue;
+      if (materialFilter && !materialFilter(object, material)) continue;
+
+      if (!snapshots.has(material)) {
+        snapshots.set(material, {
+          color: mutable.color.clone(),
+          roughness: mutable.roughness,
+          metalness: mutable.metalness,
+        });
+      }
+
+      applyScenarioMaterial(mutable, object, scenario, materials);
+    }
+  });
+}
+
+function applyScenarioMaterial(
+  material: THREE.MeshStandardMaterial | THREE.MeshPhysicalMaterial,
+  object: THREE.Object3D,
+  scenario: VisualScenarioConfig,
+  materials: ScenarioMaterialConfig
+) {
+  const seed = scenario.seed ?? 0;
+  const objectKey = `${scenario.id ?? 'scenario'}:${object.name}:${material.name}:${seed}`;
+  const variation = hashToUnitInterval(objectKey);
+
+  if (materials.randomizeObjectColors) {
+    material.color.setHSL(variation, 0.38, 0.42);
+  }
+
+  if (materials.randomizeTableMaterial) {
+    material.roughness = clamp01(
+      materials.roughness ?? 0.35 + variation * 0.45
+    );
+    material.metalness = clamp01(
+      materials.metalness ?? variation * 0.12
+    );
+  }
+
+  material.needsUpdate = true;
+}
+
+function normalizeMaterials(
+  material: THREE.Material | THREE.Material[]
+): THREE.Material[] {
+  return Array.isArray(material) ? material : [material];
+}
+
+function getMutableScenarioMaterial(
+  material: THREE.Material
+): THREE.MeshStandardMaterial | THREE.MeshPhysicalMaterial | null {
+  if (
+    material instanceof THREE.MeshStandardMaterial ||
+    material instanceof THREE.MeshPhysicalMaterial
+  ) {
+    return material;
+  }
+
+  return null;
+}
+
+function hashToUnitInterval(value: string) {
+  let hash = 2166136261;
+  for (let index = 0; index < value.length; index += 1) {
+    hash ^= value.charCodeAt(index);
+    hash = Math.imul(hash, 16777619);
+  }
+  return (hash >>> 0) / 4294967295;
+}
+
+function clamp01(value: number) {
+  return Math.max(0, Math.min(1, value));
+}
+
 export type SplatCollisionProxy = ReactNode | ThreeElements['group'];

package/src/core/MujocoCanvas.tsx

@@ -31,6 +31,7 @@ export const MujocoCanvas = forwardRef<MujocoSimAPI, MujocoCanvasProps>(
       paused,
       speed,
       interpolate,
+      loadingFallback,
       children,
       ...canvasProps
     },
@@ -44,7 +45,13 @@ export const MujocoCanvas = forwardRef<MujocoSimAPI, MujocoCanvasProps>(
       }
     }, [wasmStatus, wasmError, onError]);
 
-    if (wasmStatus === 'error' || wasmStatus === 'loading' || !mujoco) {
+    if (wasmStatus === 'loading' || !mujoco) {
+      return loadingFallback ? (
+        <Canvas {...canvasProps}>{loadingFallback}</Canvas>
+      ) : null;
+    }
+
+    if (wasmStatus === 'error') {
       return null;
     }
 

package/src/core/SceneLoader.ts

@@ -477,8 +477,9 @@ function sceneObjectToXml(obj: SceneObject): string {
   const solref = obj.solref ? ` solref="${obj.solref}"` : '';
   const solimp = obj.solimp ? ` solimp="${obj.solimp}"` : '';
   const condim = obj.condim ? ` condim="${obj.condim}"` : '';
+  const group = obj.group !== undefined ? ` group="${obj.group}"` : '';
   // Always set contype/conaffinity=1 so objects collide regardless of model defaults
-  return `<body name="${obj.name}" pos="${pos}">${joint}<geom type="${obj.type}" size="${size}" rgba="${rgba}" contype="1" conaffinity="1"${mass}${friction}${solref}${solimp}${condim}/></body>`;
+  return `<body name="${obj.name}" pos="${pos}">${joint}<geom type="${obj.type}" size="${size}" rgba="${rgba}" contype="1" conaffinity="1"${mass}${friction}${solref}${solimp}${condim}${group}/></body>`;
 }
 
 /** Create virtual directory structure for a file path. */
@@ -527,6 +528,22 @@ function localFilePath(file: LocalMujocoFile): string {
   return normalizeVfsPath(file.webkitRelativePath || file.name);
 }
 
+function dirname(path: string): string {
+  const normalized = normalizeVfsPath(path);
+  const idx = normalized.lastIndexOf('/');
+  return idx === -1 ? '' : normalized.slice(0, idx + 1);
+}
+
+function relativeVfsPath(fromDir: string, targetPath: string): string {
+  const from = normalizeVfsPath(fromDir).split('/').filter(Boolean);
+  const target = normalizeVfsPath(targetPath).split('/').filter(Boolean);
+  while (from.length && target.length && from[0] === target[0]) {
+    from.shift();
+    target.shift();
+  }
+  return [...from.map(() => '..'), ...target].join('/') || '.';
+}
+
 function inferSceneFile(files: readonly LocalMujocoFile[], options?: LoadFromFilesOptions): string {
   if (options?.sceneFile) return normalizeVfsPath(options.sceneFile);
 
@@ -551,6 +568,7 @@ export function createSceneConfigFromFiles(
     src: '',
     sceneFile: inferSceneFile(fileArray, options),
     files: fileArray,
+    environmentFiles: options.environmentFiles?.map(normalizeVfsPath),
     homeJoints: options.homeJoints,
     xmlPatches: options.xmlPatches,
     sceneObjects: options.sceneObjects,
@@ -558,6 +576,137 @@ export function createSceneConfigFromFiles(
   };
 }
 
+const ENVIRONMENT_MERGE_SECTIONS = [
+  'asset',
+  'worldbody',
+  'contact',
+  'equality',
+  'tendon',
+  'sensor',
+  'keyframe',
+  'custom',
+  'extension',
+] as const;
+
+function directChild(parent: Element, tagName: string): Element | null {
+  const lower = tagName.toLowerCase();
+  for (const child of Array.from(parent.children)) {
+    if (child.tagName.toLowerCase() === lower) return child;
+  }
+  return null;
+}
+
+function ensureTopLevelSection(doc: XMLDocument, tagName: string): Element {
+  const root = doc.documentElement;
+  const existing = directChild(root, tagName);
+  if (existing) return existing;
+
+  const section = doc.createElement(tagName);
+  if (tagName === 'asset') {
+    const worldbody = directChild(root, 'worldbody');
+    if (worldbody) root.insertBefore(section, worldbody);
+    else root.appendChild(section);
+  } else {
+    root.appendChild(section);
+  }
+  return section;
+}
+
+function readCompilerDirs(doc: XMLDocument) {
+  const compiler = directChild(doc.documentElement, 'compiler');
+  const assetDir = compiler?.getAttribute('assetdir') || '';
+  return {
+    meshDir: compiler?.getAttribute('meshdir') || assetDir,
+    textureDir: compiler?.getAttribute('texturedir') || assetDir,
+  };
+}
+
+function isExternalPath(path: string): boolean {
+  return /^[a-z]+:\/\//i.test(path) || path.startsWith('package://') || path.startsWith('/');
+}
+
+function fileReferencePrefix(el: Element, compilerDirs: ReturnType<typeof readCompilerDirs>): string {
+  const tag = el.tagName.toLowerCase();
+  if (tag === 'mesh') return compilerDirs.meshDir ? compilerDirs.meshDir + '/' : '';
+  if (tag === 'texture' || tag === 'hfield') return compilerDirs.textureDir ? compilerDirs.textureDir + '/' : '';
+  return '';
+}
+
+function rewriteFileReferencesForMerge(node: Element, sourceFile: string, targetFile: string, sourceDoc: XMLDocument) {
+  const sourceDir = dirname(sourceFile);
+  const targetDir = dirname(targetFile);
+  const compilerDirs = readCompilerDirs(sourceDoc);
+  node.querySelectorAll('[file], [filename]').forEach((el) => {
+    const attr = el.hasAttribute('file') ? 'file' : 'filename';
+    const value = el.getAttribute(attr);
+    if (!value || isExternalPath(value)) return;
+
+    const sourceRelativePath = normalizeVfsPath(fileReferencePrefix(el, compilerDirs) + value);
+    const resolvedPath = normalizeVfsPath(sourceDir + sourceRelativePath);
+    el.setAttribute(attr, relativeVfsPath(targetDir, resolvedPath));
+  });
+}
+
+function hasParseError(doc: XMLDocument): boolean {
+  return doc.getElementsByTagName('parsererror').length > 0;
+}
+
+function composeEnvironmentXml(
+  sceneXml: string,
+  config: SceneConfig,
+  parser: DOMParser,
+  environmentXmlByPath: Map<string, string>
+): string {
+  const environmentFiles = config.environmentFiles?.map(normalizeVfsPath) ?? [];
+  if (!environmentFiles.length) return sceneXml;
+
+  const sceneDoc = parser.parseFromString(sceneXml, 'text/xml');
+  if (hasParseError(sceneDoc)) {
+    console.warn(`Could not compose environments: failed to parse ${config.sceneFile}`);
+    return sceneXml;
+  }
+
+  for (const environmentFile of environmentFiles) {
+    const environmentXml = environmentXmlByPath.get(environmentFile);
+    if (!environmentXml) {
+      console.warn(`Environment XML not found: ${environmentFile}`);
+      continue;
+    }
+
+    const environmentDoc = parser.parseFromString(environmentXml, 'text/xml');
+    if (hasParseError(environmentDoc)) {
+      console.warn(`Skipping environment XML with parse errors: ${environmentFile}`);
+      continue;
+    }
+
+    for (const sectionName of ENVIRONMENT_MERGE_SECTIONS) {
+      const environmentSection = directChild(environmentDoc.documentElement, sectionName);
+      if (!environmentSection?.children.length) continue;
+
+      const targetSection = ensureTopLevelSection(sceneDoc, sectionName);
+      for (const child of Array.from(environmentSection.children)) {
+        const imported = sceneDoc.importNode(child, true) as Element;
+        rewriteFileReferencesForMerge(imported, environmentFile, config.sceneFile, environmentDoc);
+        targetSection.appendChild(imported);
+      }
+    }
+  }
+
+  return new XMLSerializer().serializeToString(sceneDoc);
+}
+
+function findTextByConfiguredPath(textByPath: Map<string, string>, configuredPath: string): string | undefined {
+  const normalized = normalizeVfsPath(configuredPath);
+  const direct = textByPath.get(normalized);
+  if (direct) return direct;
+
+  const suffix = '/' + normalized;
+  for (const [path, text] of textByPath) {
+    if (path.endsWith(suffix) || path === normalized.split('/').pop()) return text;
+  }
+  return undefined;
+}
+
 function applyXmlPatches(text: string, fname: string, config: SceneConfig): string {
   let result = text;
   for (const patch of config.xmlPatches ?? []) {
@@ -627,10 +776,25 @@ async function loadSceneFromFiles(
     if (isModelTextFile(path)) {
       const text = applyXmlPatches(await file.text(), path, config);
       textByPath.set(path, text);
-      mujoco.FS.writeFile(`/working/${path}`, text);
     } else {
       mujoco.FS.writeFile(`/working/${path}`, new Uint8Array(await file.arrayBuffer()));
+      written.add(path);
     }
+  }
+
+  const environmentXmlByPath = new Map<string, string>();
+  for (const environmentFile of config.environmentFiles?.map(normalizeVfsPath) ?? []) {
+    const environmentXml = findTextByConfiguredPath(textByPath, environmentFile);
+    if (environmentXml) environmentXmlByPath.set(environmentFile, environmentXml);
+  }
+
+  for (const [path, text] of textByPath) {
+    const composedText = path === config.sceneFile
+      ? composeEnvironmentXml(text, config, parser, environmentXmlByPath)
+      : text;
+    textByPath.set(path, composedText);
+    ensureDir(mujoco, path);
+    mujoco.FS.writeFile(`/working/${path}`, composedText);
     written.add(path);
   }
 
@@ -689,6 +853,18 @@ export async function loadScene(
 
   const baseUrl = config.src.endsWith('/') ? config.src : config.src + '/';
 
+  const environmentXmlByPath = new Map<string, string>();
+  const environmentFiles = config.environmentFiles?.map(normalizeVfsPath) ?? [];
+  for (const environmentFile of environmentFiles) {
+    onProgress?.(`Downloading ${environmentFile}...`);
+    const res = await fetch(baseUrl + environmentFile);
+    if (!res.ok) {
+      console.warn(`Failed to fetch environment XML ${environmentFile}: ${res.status} ${res.statusText}`);
+      continue;
+    }
+    environmentXmlByPath.set(environmentFile, applyXmlPatches(await res.text(), environmentFile, config));
+  }
+
   const downloaded = new Set<string>();
   const xmlQueue: string[] = [config.sceneFile];
   const assetFiles: string[] = [];
@@ -714,7 +890,10 @@ export async function loadScene(
       continue;
     }
 
-    const text = applyXmlPatches(await res.text(), fname, config);
+    const patchedText = applyXmlPatches(await res.text(), fname, config);
+    const text = fname === config.sceneFile
+      ? composeEnvironmentXml(patchedText, config, parser, environmentXmlByPath)
+      : patchedText;
 
     ensureDir(mujoco, fname);
     mujoco.FS.writeFile(`/working/${fname}`, text);