mujoco-react 8.10.0 → 9.0.0

package/README.md

@@ -145,20 +145,42 @@ Use it as a child of `<MujocoCanvas>`:
 
 Gaussian splats are visual context; MuJoCo XML remains the source of physics, contacts, and task fixtures. Pair each splat asset with collision proxy metadata so scene variants, rollouts, and datasets preserve both sides of the environment.
 
-Use the renderer-agnostic boundary from the main package:
+Use `VisualScenarioEffects` when the same MuJoCo task should render under
+different camera exposure, fog/background, and deterministic material variants:
 
 ```tsx
-import { SplatEnvironment } from "mujoco-react";
-
-<SplatEnvironment
-  src="/models/lab/scene.spz"
-  format="spz"
-  collisionProxyMetadata={{
-    xmlPath: "/models/lab/collision.xml",
-    status: "validated",
-    primitives: ["plane", "box"],
-  }}
-/>;
+import { ScenarioLighting, VisualScenarioEffects } from "mujoco-react";
+
+<MujocoCanvas config={sceneConfig}>
+  <VisualScenarioEffects
+    scenario={scenario}
+    materialFilter={({ object }) => object.name.startsWith("prop_")}
+  />
+  <ScenarioLighting preset={scenario.lighting} />
+</MujocoCanvas>;
+```
+
+Use the renderer-agnostic boundary from the main package. If your app stores
+visual scenarios as data, pass the scenario directly; the component resolves the
+splat asset and paired MJCF collision proxy metadata for you.
+
+```tsx
+import { SplatEnvironment, withSplatEnvironment } from "mujoco-react";
+
+<SplatEnvironment scenario={scenario} renderer="custom" />;
+```
+
+For MuJoCo + 3DGS composition, derive the collision environment from the same
+splat metadata and pass the resulting config to `<MujocoCanvas>`:
+
+```tsx
+const sceneConfig = withSplatEnvironment(
+  {
+    src: "/models/xlerobot/",
+    sceneFile: "xlerobot.xml",
+  },
+  kitchenScenario
+);
 ```
 
 For first-class Spark rendering, install Spark and import the optional adapter:
@@ -168,30 +190,33 @@ npm install @sparkjsdev/spark
 ```
 
 ```tsx
-import { SparkSplatEnvironment } from "mujoco-react/spark";
-
-<MujocoCanvas config={sceneConfig} gl={{ preserveDrawingBuffer: true }}>
-  <SparkSplatEnvironment
-    src="/models/lab/scene.spz"
-    format="spz"
-    collisionProxyMetadata={{
-      xmlPath: "/models/lab/collision.xml",
-      status: "validated",
-      primitives: ["plane", "box"],
-    }}
-    hideGroundMeshes
-    onStatusChange={(status) => console.log(status)}
-  />
-</MujocoCanvas>;
+import {
+  SparkSplatEnvironment,
+  useSparkSplatLifecycle,
+} from "mujoco-react/spark";
+
+function Scene() {
+  const splat = useSparkSplatLifecycle();
+
+  return (
+    <MujocoCanvas config={sceneConfig} gl={{ preserveDrawingBuffer: true }}>
+      <SparkSplatEnvironment scenario={scenario} hideGroundMeshes {...splat.props} />
+      <StatusBadge status={splat.status} error={splat.error} />
+    </MujocoCanvas>
+  );
+}
 ```
 
+`SparkSplatEnvironment` currently renders `.spz` assets. Use the renderer-agnostic
+`SplatEnvironment` for `.ply`/`.splat` metadata or when wiring a different renderer.
+
 ## Write Controllers
 
 ```tsx
 import { useBeforePhysicsStep } from "mujoco-react";
 
 function MyController() {
-  useBeforePhysicsStep((_model, data) => {
+  useBeforePhysicsStep(({ data }) => {
     data.ctrl[0] = Math.sin(data.time);
   });
 
@@ -259,7 +284,7 @@ import type { ControlGroupInfo } from "mujoco-react";
 function HoldTcpPose() {
   const armRef = useRef<ControlGroupInfo | null>(null);
 
-  useBeforePhysicsStep((model, data) => {
+  useBeforePhysicsStep(({ model, data }) => {
     armRef.current ??= resolveControlGroup(model, { siteName: RobotSites.franka.tcp });
     if (!armRef.current) return;
 
@@ -280,7 +305,7 @@ Build policy-ready observation vectors from common MuJoCo state without hard-cod
 import { buildObservation, useBeforePhysicsStep } from "mujoco-react";
 
 function PolicyDriver() {
-  useBeforePhysicsStep((model, data) => {
+  useBeforePhysicsStep(({ model, data }) => {
     const obs = buildObservation(model, data, {
       qpos: true,
       qvel: true,
@@ -336,7 +361,7 @@ function useWebSocketControls(url: string) {
   }, [url]);
 
   // Apply incoming actuator controls each physics step.
-  useBeforePhysicsStep((model, data) => {
+  useBeforePhysicsStep(({ model, data }) => {
     const ctrl = latestCtrlRef.current;
     if (!ctrl) return;
     for (let i = 0; i < Math.min(ctrl.length, model.nu); i++) {
@@ -345,7 +370,7 @@ function useWebSocketControls(url: string) {
   });
 
   // Send simulation feedback back after physics.
-  useAfterPhysicsStep((model, data) => {
+  useAfterPhysicsStep(({ data }) => {
     const ws = wsRef.current;
     if (!ws || ws.readyState !== WebSocket.OPEN) return;
 
@@ -408,8 +433,8 @@ The built-in `useIkController()` uses Damped Least-Squares. Pass `ikSolveFn` to
 import { RobotSites } from "mujoco-react";
 import type { IKSolveFn } from "mujoco-react";
 
-const myIK: IKSolveFn = (pos, quat, currentQ) => {
-  return myAnalyticalSolver(pos, currentQ); // return joint angles or null
+const myIK: IKSolveFn = ({ position, currentQ }) => {
+  return myAnalyticalSolver(position, currentQ); // return joint angles or null
 };
 
 const ik = useIkController({ siteName: RobotSites.franka.tcp, ikSolveFn: myIK });
@@ -497,13 +522,24 @@ interface SceneConfig {
   src: string;                      // Base URL for model files
   sceneFile: string;                // Entry XML/URDF file, e.g. "scene.xml"
   files?: File[];                   // Local files for browser upload workflows
+  environmentFiles?: string[];      // Static MJCF environment XMLs merged before compile
   sceneObjects?: SceneObject[];     // Objects injected into scene XML at load time
   homeJoints?: number[];            // Initial joint positions
   xmlPatches?: XmlPatch[];          // Patches applied to XML files during loading
-  onReset?: (model, data) => void;  // Called during reset after mj_resetData
+  onReset?: ({ model, data }) => void;  // Called during reset after mj_resetData
 }
 ```
 
+Use `environmentFiles` to compose reusable physics/collision layers with a robot model. For Gaussian splat scenes, keep the `.spz` as a parallel visual layer and point `environmentFiles` at the paired MJCF proxy scene:
+
+```tsx
+const kitchenRobot: SceneConfig = {
+  src: "/models/xlerobot/",
+  sceneFile: "xlerobot.xml",
+  environmentFiles: ["splats/tabletop/scene.xml"],
+};
+```
+
 ### Local Files and URDF
 
 Load browser-selected MJCF or URDF files directly. Folder uploads preserve `webkitRelativePath`, and flat uploads fall back to matching referenced mesh/texture assets by basename:
@@ -600,10 +636,10 @@ Thin wrapper around R3F `<Canvas>`. Accepts all R3F Canvas props plus:
 | Prop | Type | Description |
 |------|------|-------------|
 | `config` | `SceneConfig` | **Required.** Scene/robot configuration |
-| `onReady` | `(api: MujocoSimAPI) => void` | Fires when model is loaded |
+| `onReady` | `({ api }) => void` | Fires when model is loaded |
 | `onError` | `(error: Error) => void` | Fires on scene load failure |
-| `onStep` | `(time: number) => void` | Called each physics step |
-| `onSelection` | `(bodyId: number, name: string) => void` | Called on double-click |
+| `onStep` | `({ time, model, data }) => void` | Called each physics step |
+| `onSelection` | `({ bodyId, name }) => void` | Called on double-click |
 | `gravity` | `[number, number, number]` | Override model gravity |
 | `timestep` | `number` | Override model.opt.timestep |
 | `substeps` | `number` | mj_step calls per frame |
@@ -628,10 +664,10 @@ Physics provider for use inside your own R3F `<Canvas>`. Same physics props as `
 | Prop | Type | Description |
 |------|------|-------------|
 | `config` | `SceneConfig` | **Required.** Scene/robot configuration |
-| `onReady` | `(api: MujocoSimAPI) => void` | Fires when model is loaded |
+| `onReady` | `({ api }) => void` | Fires when model is loaded |
 | `onError` | `(error: Error) => void` | Fires on scene load failure |
-| `onStep` | `(time: number) => void` | Called each physics step |
-| `onSelection` | `(bodyId: number, name: string) => void` | Called on double-click |
+| `onStep` | `({ time, model, data }) => void` | Called each physics step |
+| `onSelection` | `({ bodyId, name }) => void` | Called on double-click |
 | `gravity` | `[number, number, number]` | Override model gravity |
 | `timestep` | `number` | Override model.opt.timestep |
 | `substeps` | `number` | mj_step calls per frame |
@@ -679,7 +715,7 @@ drei PivotControls gizmo that tracks a MuJoCo site and drives IK on drag. Requir
 | `controller` | `IkContextValue` | **required** | Controller from `useIkController()` |
 | `siteName` | `string?` | controller's site | MuJoCo site to track |
 | `scale` | `number?` | `0.18` | Gizmo handle scale |
-| `onDrag` | `(pos, quat) => void` | -- | Custom drag handler (disables auto-IK) |
+| `onDrag` | `({ position, quaternion }) => void` | -- | Custom drag handler (disables auto-IK) |
 
 ### `<DragInteraction />`
 
@@ -790,7 +826,7 @@ if (mujoco) {
 Run logic **before** `mj_step` each frame. Write to `data.ctrl`, apply forces, drive automation.
 
 ```tsx
-useBeforePhysicsStep((model, data) => {
+useBeforePhysicsStep(({ data }) => {
   data.ctrl[0] = Math.sin(data.time);
 });
 ```
@@ -905,7 +941,8 @@ const obs = useObservation({ qpos: true, qvel: true, projectedGravity: "torso" }
 const policy = usePolicy({
   frequency: 50,
   onObservation: () => obs.readValues(),
-  onAction: (action, model, data) => applyAction(action, data),
+  infer: ({ observation }) => policySession.run(observation),
+  onAction: ({ action, data }) => applyAction(action, data),
 });
 ```
 

package/dist/chunk-33CV6HSV.js

@@ -0,0 +1,400 @@
+import { useThree } from '@react-three/fiber';
+import { useEffect, useMemo } from 'react';
+import * as THREE from 'three';
+import { jsxs, Fragment, jsx } from 'react/jsx-runtime';
+
+// src/components/VisualScenario.tsx
+var DEFAULT_BACKGROUND = "#181a1f";
+function ScenarioLighting({
+  preset = "studio",
+  castShadow = true,
+  intensity = 1
+}) {
+  if (preset === "warehouse") {
+    return /* @__PURE__ */ jsxs(Fragment, { children: [
+      /* @__PURE__ */ jsx("ambientLight", { intensity: 0.18 * intensity }),
+      /* @__PURE__ */ jsx(
+        "directionalLight",
+        {
+          position: [3.5, -2, 5],
+          intensity: 2.2 * intensity,
+          castShadow
+        }
+      ),
+      /* @__PURE__ */ jsx("directionalLight", { position: [-2, 1.5, 2.5], intensity: 0.25 * intensity })
+    ] });
+  }
+  if (preset === "low-light") {
+    return /* @__PURE__ */ jsxs(Fragment, { children: [
+      /* @__PURE__ */ jsx("ambientLight", { intensity: 0.08 * intensity }),
+      /* @__PURE__ */ jsx(
+        "directionalLight",
+        {
+          position: [2, -2, 3],
+          intensity: 0.75 * intensity,
+          castShadow
+        }
+      ),
+      /* @__PURE__ */ jsx("pointLight", { position: [-0.5, -0.8, 1.3], intensity: 0.6 * intensity })
+    ] });
+  }
+  if (preset === "splat") {
+    return /* @__PURE__ */ jsxs(Fragment, { children: [
+      /* @__PURE__ */ jsx("ambientLight", { intensity: 0.42 * intensity }),
+      /* @__PURE__ */ jsx(
+        "directionalLight",
+        {
+          position: [1.8, -2.4, 3.5],
+          intensity: 1.2 * intensity,
+          castShadow
+        }
+      ),
+      /* @__PURE__ */ jsx("pointLight", { position: [0.4, 0.2, 1.4], intensity: 0.35 * intensity })
+    ] });
+  }
+  return /* @__PURE__ */ jsxs(Fragment, { children: [
+    /* @__PURE__ */ jsx("ambientLight", { intensity: 0.35 * intensity }),
+    /* @__PURE__ */ jsx(
+      "directionalLight",
+      {
+        position: [2.5, -3, 4],
+        intensity: 1.6 * intensity,
+        castShadow
+      }
+    )
+  ] });
+}
+function getScenarioBackground(preset, fallback = DEFAULT_BACKGROUND) {
+  if (preset === "warehouse") return "#20242b";
+  if (preset === "low-light") return "#0f1115";
+  if (preset === "splat") return "#1b1f24";
+  return fallback;
+}
+function getScenarioCameraPosition(basePosition, scenario) {
+  const [x, y, z] = basePosition;
+  const jitter = scenario?.camera?.jitter ?? 0;
+  return [
+    Number((x + jitter * 0.6).toFixed(3)),
+    Number((y - jitter * 0.4).toFixed(3)),
+    Number((z + jitter * 0.25).toFixed(3))
+  ];
+}
+function VisualScenarioEffects(props) {
+  useVisualScenarioEffects(props);
+  return null;
+}
+function useVisualScenarioEffects({
+  scenario,
+  enabled = true,
+  applyBackground = true,
+  applyFog = true,
+  applyRenderer = true,
+  applyMaterials = true,
+  background,
+  fogNear,
+  fogFar,
+  materialFilter
+}) {
+  const { gl, scene, invalidate } = useThree();
+  useEffect(() => {
+    if (!enabled || !scenario) {
+      return void 0;
+    }
+    const previousExposure = gl.toneMappingExposure;
+    const previousBackground = scene.background;
+    const previousFog = scene.fog;
+    const materialSnapshots = /* @__PURE__ */ new Map();
+    if (applyRenderer) {
+      gl.toneMappingExposure = scenario.camera?.exposure ?? 1;
+    }
+    if (applyBackground) {
+      scene.background = new THREE.Color(
+        background ?? getScenarioBackground(scenario.lighting)
+      );
+    }
+    if (applyFog) {
+      scene.fog = createScenarioFog(scenario, background, fogNear, fogFar);
+    }
+    if (applyMaterials && scenario.materials) {
+      applyScenarioMaterials(scene, scenario, materialSnapshots, materialFilter);
+    }
+    invalidate();
+    return () => {
+      gl.toneMappingExposure = previousExposure;
+      scene.background = previousBackground;
+      scene.fog = previousFog;
+      for (const [material, snapshot] of materialSnapshots) {
+        const mutable = getMutableScenarioMaterial(material);
+        if (!mutable) continue;
+        if (snapshot.color) mutable.color.copy(snapshot.color);
+        if (typeof snapshot.roughness === "number") {
+          mutable.roughness = snapshot.roughness;
+        }
+        if (typeof snapshot.metalness === "number") {
+          mutable.metalness = snapshot.metalness;
+        }
+        mutable.needsUpdate = true;
+      }
+      invalidate();
+    };
+  }, [
+    applyBackground,
+    applyFog,
+    applyMaterials,
+    applyRenderer,
+    background,
+    enabled,
+    fogFar,
+    fogNear,
+    gl,
+    invalidate,
+    materialFilter,
+    scenario,
+    scene
+  ]);
+}
+function SplatEnvironment({
+  environment,
+  scenario,
+  renderer,
+  src,
+  format,
+  collisionProxy,
+  collisionProxyMetadata,
+  children,
+  showPlaceholder = true,
+  ...groupProps
+}) {
+  const metadata = useSplatEnvironment({
+    environment,
+    scenario,
+    renderer,
+    src,
+    format,
+    collisionProxy: collisionProxyMetadata
+  });
+  const existingUserData = typeof groupProps.userData === "object" && groupProps.userData !== null ? groupProps.userData : {};
+  return /* @__PURE__ */ jsxs(
+    "group",
+    {
+      ...groupProps,
+      userData: {
+        ...existingUserData,
+        ...metadata.userData
+      },
+      children: [
+        children,
+        children || !showPlaceholder ? null : /* @__PURE__ */ jsx(SplatPlaceholder, {}),
+        collisionProxy
+      ]
+    }
+  );
+}
+function useSplatEnvironment({
+  environment,
+  scenario,
+  renderer,
+  src,
+  format,
+  collisionProxy
+}) {
+  const scenarioEnvironment = useMemo(
+    () => environment ?? (scenario ? createPairedSplatEnvironment(scenario, { renderer }) : void 0),
+    [environment, renderer, scenario]
+  );
+  const resolvedSrc = src ?? scenarioEnvironment?.splat.src ?? scenario?.splat?.src;
+  const resolvedFormat = format ?? scenarioEnvironment?.splat.format ?? scenario?.splat?.format ?? "spz";
+  const resolvedCollisionProxy = collisionProxy ?? scenarioEnvironment?.collisionProxy ?? scenario?.splat?.collisionProxy ?? void 0;
+  return useMemo(
+    () => ({
+      src: resolvedSrc,
+      format: resolvedFormat,
+      collisionProxy: resolvedCollisionProxy,
+      userData: createSplatEnvironmentUserData({
+        environment: scenarioEnvironment,
+        src: resolvedSrc,
+        format: resolvedFormat,
+        collisionProxy: resolvedCollisionProxy
+      })
+    }),
+    [scenarioEnvironment, resolvedSrc, resolvedFormat, resolvedCollisionProxy]
+  );
+}
+function createPairedSplatEnvironment(scenario, options = {}) {
+  const splat = scenario.splat;
+  const collisionProxy = splat?.collisionProxy;
+  if (!splat?.enabled || !splat.src || !collisionProxy?.xmlPath) {
+    return void 0;
+  }
+  return {
+    id: options.id ?? scenario.id ?? "splat-environment",
+    label: options.label ?? scenario.label ?? "Gaussian splat environment",
+    description: options.description ?? (scenario.environment ? `Visual ${scenario.environment} splat paired with MJCF collision proxy.` : void 0),
+    splat: {
+      src: splat.src,
+      format: splat.format ?? "spz",
+      renderer: options.renderer
+    },
+    collisionProxy: {
+      ...collisionProxy,
+      xmlPath: collisionProxy.xmlPath
+    }
+  };
+}
+function isPairedSplatEnvironment(input) {
+  return !!input && "collisionProxy" in input && "splat" in input;
+}
+function sceneRelativePath(sceneConfig, path) {
+  const src = sceneConfig.src;
+  if (!src) return path;
+  const base = src.endsWith("/") ? src : src + "/";
+  if (path.startsWith(base)) return path.slice(base.length);
+  return path;
+}
+function uniquePaths(paths) {
+  const seen = /* @__PURE__ */ new Set();
+  const result = [];
+  for (const path of paths) {
+    if (seen.has(path)) continue;
+    seen.add(path);
+    result.push(path);
+  }
+  return result;
+}
+function withSplatEnvironment(sceneConfig, input, options = {}) {
+  const environment = isPairedSplatEnvironment(input) ? input : input ? createPairedSplatEnvironment(input, options) : void 0;
+  const xmlPath = environment?.collisionProxy.xmlPath;
+  if (!xmlPath) return sceneConfig;
+  return {
+    ...sceneConfig,
+    environmentFiles: uniquePaths([
+      ...sceneConfig.environmentFiles ?? [],
+      sceneRelativePath(sceneConfig, xmlPath)
+    ])
+  };
+}
+function createSplatEnvironmentUserData({
+  environment,
+  src,
+  format = "spz",
+  collisionProxy
+}) {
+  return {
+    role: "splat-environment",
+    environmentId: environment?.id,
+    environmentLabel: environment?.label,
+    splatSrc: src,
+    splatFormat: format,
+    splatRenderer: environment?.splat.renderer,
+    collisionProxyStatus: collisionProxy?.status ?? "missing",
+    collisionProxyXmlPath: collisionProxy?.xmlPath,
+    collisionProxyPrimitives: collisionProxy?.primitives ?? []
+  };
+}
+function createSparkSplatViewerUrl({
+  viewerUrl,
+  splatSrc
+}) {
+  const url = new URL(viewerUrl, "http://mujoco-react.local");
+  url.searchParams.set("splat", splatSrc);
+  return viewerUrl.startsWith("http") ? url.toString() : `${url.pathname}${url.search}`;
+}
+function SplatPlaceholder() {
+  return /* @__PURE__ */ jsx("group", { children: /* @__PURE__ */ jsxs("mesh", { position: [0, 0, 1.2], children: [
+    /* @__PURE__ */ jsx("boxGeometry", { args: [2.4, 2.4, 2.4] }),
+    /* @__PURE__ */ jsx(
+      "meshBasicMaterial",
+      {
+        color: "#8b8b8b",
+        transparent: true,
+        opacity: 0.06,
+        wireframe: true,
+        side: THREE.DoubleSide
+      }
+    )
+  ] }) });
+}
+function createScenarioFog(scenario, background, fogNear, fogFar) {
+  if (scenario.lighting === "low-light") {
+    return new THREE.Fog(
+      background ?? getScenarioBackground(scenario.lighting),
+      fogNear ?? 2.5,
+      fogFar ?? 9
+    );
+  }
+  if (scenario.lighting === "warehouse") {
+    return new THREE.Fog(
+      background ?? getScenarioBackground(scenario.lighting),
+      fogNear ?? 5,
+      fogFar ?? 16
+    );
+  }
+  return null;
+}
+function applyScenarioMaterials(scene, scenario, snapshots, materialFilter) {
+  const materials = scenario.materials;
+  if (!materials) return;
+  scene.traverse((object) => {
+    if (!(object instanceof THREE.Mesh)) {
+      return;
+    }
+    for (const material of normalizeMaterials(object.material)) {
+      const mutable = getMutableScenarioMaterial(material);
+      if (!mutable) continue;
+      if (materialFilter && !materialFilter({ object, material })) continue;
+      if (!snapshots.has(material)) {
+        snapshots.set(material, {
+          color: mutable.color.clone(),
+          roughness: mutable.roughness,
+          metalness: mutable.metalness
+        });
+      }
+      applyScenarioMaterial(mutable, object, scenario, materials);
+    }
+  });
+}
+function applyScenarioMaterial(material, object, scenario, materials) {
+  const seed = scenario.seed ?? 0;
+  const objectKey = `${scenario.id ?? "scenario"}:${object.name}:${material.name}:${seed}`;
+  const variation = hashToUnitInterval(objectKey);
+  if (materials.randomizeObjectColors) {
+    material.color.setHSL(variation, 0.38, 0.42);
+  }
+  if (materials.randomizeTableMaterial) {
+    material.roughness = clamp01(
+      materials.roughness ?? 0.35 + variation * 0.45
+    );
+    material.metalness = clamp01(
+      materials.metalness ?? variation * 0.12
+    );
+  }
+  material.needsUpdate = true;
+}
+function normalizeMaterials(material) {
+  return Array.isArray(material) ? material : [material];
+}
+function getMutableScenarioMaterial(material) {
+  if (material instanceof THREE.MeshStandardMaterial || material instanceof THREE.MeshPhysicalMaterial) {
+    return material;
+  }
+  return null;
+}
+function hashToUnitInterval(value) {
+  let hash = 2166136261;
+  for (let index = 0; index < value.length; index += 1) {
+    hash ^= value.charCodeAt(index);
+    hash = Math.imul(hash, 16777619);
+  }
+  return (hash >>> 0) / 4294967295;
+}
+function clamp01(value) {
+  return Math.max(0, Math.min(1, value));
+}
+/**
+ * @license
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+export { ScenarioLighting, SplatEnvironment, VisualScenarioEffects, createPairedSplatEnvironment, createSparkSplatViewerUrl, createSplatEnvironmentUserData, getScenarioBackground, getScenarioCameraPosition, useSplatEnvironment, useVisualScenarioEffects, withSplatEnvironment };
+//# sourceMappingURL=chunk-33CV6HSV.js.map
+//# sourceMappingURL=chunk-33CV6HSV.js.map