mujoco-react 8.10.0 → 9.0.0

package/src/hooks/useTrajectoryPlayer.ts

@@ -8,7 +8,7 @@
 import { useCallback, useRef } from 'react';
 import { useFrame } from '@react-three/fiber';
 import { useMujocoContext, useBeforePhysicsStep } from '../core/MujocoSimProvider';
-import type { PlaybackState, TrajectoryFrame, TrajectoryInput } from '../types';
+import type { PlaybackState, TrajectoryStateChangeInput, TrajectoryFrame, TrajectoryInput } from '../types';
 
 export interface TrajectoryPlayerOptions {
   fps?: number;
@@ -16,7 +16,7 @@ export interface TrajectoryPlayerOptions {
   loop?: boolean;
   mode?: 'kinematic' | 'physics';
   onComplete?: () => void;
-  onStateChange?: (state: PlaybackState) => void;
+  onStateChange?: (input: TrajectoryStateChangeInput) => void;
 }
 
 /** Check if input is TrajectoryFrame[] (vs number[][]) */
@@ -74,7 +74,7 @@ export function useTrajectoryPlayer(
   const setState = useCallback((next: PlaybackState) => {
     if (stateRef.current === next) return;
     stateRef.current = next;
-    optionsRef.current.onStateChange?.(next);
+    optionsRef.current.onStateChange?.({ state: next });
   }, []);
 
   const play = useCallback(() => {
@@ -181,7 +181,7 @@ export function useTrajectoryPlayer(
   });
 
   // --- Physics mode: set ctrl values each physics step ---
-  useBeforePhysicsStep((model, data) => {
+  useBeforePhysicsStep(({ model, data }) => {
     if (stateRef.current !== 'playing') return;
     if ((optionsRef.current.mode ?? 'kinematic') !== 'physics') return;
 

package/src/hooks/useTrajectoryRecorder.ts

@@ -22,7 +22,7 @@ export function useTrajectoryRecorder(options: RecorderOptions = {}) {
   const framesRef = useRef<TrajectoryFrame[]>([]);
   const fields = options.fields ?? ['qpos'];
 
-  useAfterPhysicsStep((_model, data) => {
+  useAfterPhysicsStep(({ data }) => {
     if (!recordingRef.current) return;
 
     const frame: TrajectoryFrame = {

package/src/index.ts

@@ -44,11 +44,15 @@ export { SceneLights } from './components/SceneLights';
 export {
   ScenarioLighting,
   SplatEnvironment,
+  VisualScenarioEffects,
+  createPairedSplatEnvironment,
   createSparkSplatViewerUrl,
   createSplatEnvironmentUserData,
   getScenarioBackground,
   getScenarioCameraPosition,
   useSplatEnvironment,
+  useVisualScenarioEffects,
+  withSplatEnvironment,
 } from './components/VisualScenario';
 export { Debug } from './components/Debug';
 export { TendonRenderer } from './components/TendonRenderer';
@@ -73,6 +77,20 @@ export { useTrajectoryPlayer } from './hooks/useTrajectoryPlayer';
 export { useTrajectoryRecorder } from './hooks/useTrajectoryRecorder';
 export { useGamepad } from './hooks/useGamepad';
 export { useVideoRecorder } from './hooks/useVideoRecorder';
+export {
+  captureFrame,
+  captureFrameBlob,
+  useFrameCapture,
+} from './hooks/useFrameCapture';
+export type {
+  FrameCaptureAPI,
+  FrameCaptureBlobResult,
+  FrameCaptureOptions,
+  FrameCaptureResult,
+  FrameCaptureStatus,
+  FrameCaptureTarget,
+  FrameCaptureTargetRef,
+} from './hooks/useFrameCapture';
 export { useCtrlNoise } from './hooks/useCtrlNoise';
 export { useBodyMeshes } from './hooks/useBodyMeshes';
 export { useSelectionHighlight } from './hooks/useSelectionHighlight';
@@ -92,8 +110,14 @@ export type {
   IkConfig,
   IkContextValue,
   IKSolveFn,
+  IkSolveInput,
   // Callbacks
   PhysicsStepCallback,
+  PhysicsStepInput,
+  ResetCallbackInput,
+  ReadyCallbackInput,
+  StepCallbackInput,
+  SelectionCallbackInput,
   // State management
   StateSnapshot,
   // Model introspection
@@ -124,6 +148,10 @@ export type {
   KeyboardTeleopConfig,
   // Policy
   PolicyConfig,
+  PolicyVector,
+  PolicyObservationInput,
+  PolicyInferenceInput,
+  PolicyActionInput,
   // Observations
   ObservationConfig,
   ObservationHandle,
@@ -133,6 +161,7 @@ export type {
   // Component props
   BodyProps,
   IkGizmoProps,
+  IkGizmoDragInput,
   DragInteractionProps,
   DebugProps,
   SceneLightsProps,
@@ -147,10 +176,16 @@ export type {
   PairedSplatEnvironmentConfig,
   SplatEnvironmentMetadataInput,
   SplatEnvironmentMetadata,
+  SplatSceneInput,
   VisualScenarioConfig,
   ScenarioLightingProps,
+  ScenarioMaterialConfig,
   SplatEnvironmentProps,
+  VisualScenarioEffectsProps,
+  VisualScenarioMaterialFilterInput,
   TrajectoryPlayerProps,
+  TrajectoryFrameCallbackInput,
+  TrajectoryStateChangeInput,
   ContactListenerProps,
   // API
   MujocoSimAPI,

package/src/spark.tsx

@@ -4,7 +4,13 @@
  */
 
 import { useThree } from '@react-three/fiber';
-import { useEffect, useRef, useState } from 'react';
+import {
+  useCallback,
+  useEffect,
+  useMemo,
+  useRef,
+  useState,
+} from 'react';
 import * as THREE from 'three';
 import {
   SplatEnvironment,
@@ -17,9 +23,22 @@ import type {
 type SparkModule = typeof import('@sparkjsdev/spark');
 type SparkRendererInstance = InstanceType<SparkModule['SparkRenderer']>;
 type SparkSplatMeshInstance = InstanceType<SparkModule['SplatMesh']>;
+type SparkDisposable = {
+  dispose?: () => unknown;
+};
 
 export type SparkSplatStatus = 'idle' | 'loading' | 'ready' | 'error';
 
+export interface SparkSplatLifecycle {
+  status: SparkSplatStatus;
+  error: Error | null;
+  isLoading: boolean;
+  isReady: boolean;
+  isError: boolean;
+  props: Pick<SparkSplatEnvironmentProps, 'onStatusChange' | 'onError'>;
+  reset: () => void;
+}
+
 export interface SparkSplatEnvironmentProps extends SplatEnvironmentProps {
   /** Enable Spark LoD handling for large splat assets. Default: true. */
   lod?: boolean | 'quality';
@@ -34,6 +53,75 @@ export interface SparkSplatEnvironmentProps extends SplatEnvironmentProps {
   onError?: (error: Error) => void;
 }
 
+/**
+ * Tracks Spark 3DGS loading state for UI that wraps `SparkSplatEnvironment`.
+ *
+ * Use the returned `props` with `<SparkSplatEnvironment {...lifecycle.props} />`
+ * to avoid repeating status/error state in app code.
+ */
+export function useSparkSplatLifecycle({
+  enabled = true,
+  initialStatus,
+  onError,
+  onStatusChange,
+}: {
+  enabled?: boolean;
+  initialStatus?: SparkSplatStatus;
+  onError?: (error: Error) => void;
+  onStatusChange?: (status: SparkSplatStatus) => void;
+} = {}): SparkSplatLifecycle {
+  const [status, setStatus] = useState<SparkSplatStatus>(
+    initialStatus ?? (enabled ? 'loading' : 'idle')
+  );
+  const [error, setError] = useState<Error | null>(null);
+
+  useEffect(() => {
+    setStatus(enabled ? initialStatus ?? 'loading' : 'idle');
+    setError(null);
+  }, [enabled, initialStatus]);
+
+  const handleStatusChange = useCallback(
+    (nextStatus: SparkSplatStatus) => {
+      setStatus(nextStatus);
+      if (nextStatus !== 'error') {
+        setError(null);
+      }
+      onStatusChange?.(nextStatus);
+    },
+    [onStatusChange]
+  );
+
+  const handleError = useCallback(
+    (nextError: Error) => {
+      setError(nextError);
+      setStatus('error');
+      onError?.(nextError);
+    },
+    [onError]
+  );
+
+  const reset = useCallback(() => {
+    setStatus(enabled ? initialStatus ?? 'loading' : 'idle');
+    setError(null);
+  }, [enabled, initialStatus]);
+
+  return useMemo(
+    () => ({
+      status,
+      error,
+      isLoading: status === 'loading',
+      isReady: status === 'ready',
+      isError: status === 'error',
+      props: {
+        onStatusChange: handleStatusChange,
+        onError: handleError,
+      },
+      reset,
+    }),
+    [error, handleError, handleStatusChange, reset, status]
+  );
+}
+
 /**
  * Optional SparkJS-backed Gaussian splat renderer for React Three Fiber scenes.
  *
@@ -42,6 +130,8 @@ export interface SparkSplatEnvironmentProps extends SplatEnvironmentProps {
  */
 export function SparkSplatEnvironment({
   environment,
+  scenario,
+  renderer = 'spark',
   src,
   format,
   collisionProxy,
@@ -66,6 +156,8 @@ export function SparkSplatEnvironment({
   const { gl, invalidate } = useThree();
   const metadata = useSplatEnvironment({
     environment,
+    scenario,
+    renderer,
     src,
     format,
     collisionProxy: collisionProxyMetadata,
@@ -99,11 +191,20 @@ export function SparkSplatEnvironment({
     }
 
     async function loadSplat() {
-      if (!metadata.src || metadata.format !== 'spz') {
+      if (!metadata.src) {
         setLifecycleStatus('idle');
         return;
       }
 
+      if (metadata.format !== 'spz') {
+        const unsupportedFormatError = new Error(
+          `SparkSplatEnvironment only supports .spz assets; received "${metadata.format}".`
+        );
+        setLifecycleStatus('error');
+        onErrorRef.current?.(unsupportedFormatError);
+        return;
+      }
+
       setLifecycleStatus('loading');
 
       try {
@@ -166,13 +267,13 @@ export function SparkSplatEnvironment({
 
       if (meshRef.current) {
         groupRef.current?.remove(meshRef.current);
-        meshRef.current.dispose?.();
+        safelyDisposeSparkResource(meshRef.current);
         meshRef.current = null;
       }
 
       if (sparkRef.current) {
         groupRef.current?.remove(sparkRef.current);
-        sparkRef.current.dispose?.();
+        safelyDisposeSparkResource(sparkRef.current);
         sparkRef.current = null;
       }
     };
@@ -189,6 +290,8 @@ export function SparkSplatEnvironment({
     <SplatEnvironment
       {...groupProps}
       environment={environment}
+      scenario={scenario}
+      renderer={renderer}
       src={metadata.src}
       format={metadata.format}
       collisionProxyMetadata={metadata.collisionProxy}
@@ -200,3 +303,34 @@ export function SparkSplatEnvironment({
     </SplatEnvironment>
   );
 }
+
+function safelyDisposeSparkResource(resource: SparkDisposable) {
+  try {
+    const result = resource.dispose?.();
+    if (isPromiseLike(result)) {
+      void Promise.resolve(result).catch(handleSparkDisposeError);
+    }
+  } catch (error) {
+    handleSparkDisposeError(error);
+  }
+}
+
+function isPromiseLike(value: unknown): value is PromiseLike<unknown> {
+  return (
+    typeof value === 'object' &&
+    value !== null &&
+    'then' in value &&
+    typeof (value as { then?: unknown }).then === 'function'
+  );
+}
+
+function handleSparkDisposeError(error: unknown) {
+  if (
+    error instanceof Error &&
+    error.message.toLowerCase().includes('worker terminate')
+  ) {
+    return;
+  }
+
+  console.warn('[mujoco-react] Spark resource disposal failed.', error);
+}

package/src/types.ts

@@ -429,6 +429,8 @@ export interface SceneObject {
   solref?: string;
   solimp?: string;
   condim?: number;
+  /** MuJoCo geom group. Group 3 is conventionally used for collision-only helper geoms. */
+  group?: number;
 }
 
 export interface XmlPatch {
@@ -443,10 +445,12 @@ export type LocalMujocoFile = File;
 export interface LoadFromFilesOptions {
   /** Entry MJCF/URDF file. Inferred from scene.xml, model.xml, robot.xml, or the first XML/URDF file when omitted. */
   sceneFile?: string;
+  /** Additional MJCF environment XML files merged into the entry scene before MuJoCo compilation. */
+  environmentFiles?: string[];
   homeJoints?: number[];
   xmlPatches?: XmlPatch[];
   sceneObjects?: SceneObject[];
-  onReset?: (model: MujocoModel, data: MujocoData) => void;
+  onReset?: (input: ResetCallbackInput) => void;
 }
 
 export interface SceneConfig {
@@ -456,10 +460,18 @@ export interface SceneConfig {
   sceneFile: string;
   /** Browser-selected files for local MJCF/URDF loading. Preserves webkitRelativePath when available. */
   files?: readonly LocalMujocoFile[];
+  /**
+   * Additional MJCF environment XML files merged into the entry scene before compilation.
+   *
+   * Use this for static collision/physics layers such as a Gaussian-splat
+   * environment's proxy `scene.xml`; render the splat itself as a separate
+   * visual layer.
+   */
+  environmentFiles?: string[];
   sceneObjects?: SceneObject[];
   homeJoints?: number[];
   xmlPatches?: XmlPatch[];
-  onReset?: (model: MujocoModel, data: MujocoData) => void;
+  onReset?: (input: ResetCallbackInput) => void;
 }
 
 // ---- IK Controller Config ----
@@ -501,7 +513,7 @@ export interface IkContextValue {
   setIkEnabled: (enabled: boolean) => void;
   moveTarget: (pos: THREE.Vector3, duration?: number) => void;
   syncTargetToSite: () => void;
-  solveIK: (pos: THREE.Vector3, quat: THREE.Quaternion, currentQ: number[]) => number[] | null;
+  solveIK: (input: IkSolveInput) => number[] | null;
   getGizmoStats: () => { pos: THREE.Vector3; rot: THREE.Euler } | null;
 }
 
@@ -524,12 +536,16 @@ export interface PhysicsConfig {
 // ---- IK ----
 
 export type IKSolveFn = (
-  pos: THREE.Vector3,
-  quat: THREE.Quaternion,
-  currentQ: number[],
-  context?: IKSolveContext
+  input: IkSolveInput
 ) => number[] | null;
 
+export interface IkSolveInput {
+  position: THREE.Vector3;
+  quaternion: THREE.Quaternion;
+  currentQ: number[];
+  context?: IKSolveContext;
+}
+
 export interface IKSolveContext {
   model: MujocoModel;
   data: MujocoData;
@@ -539,10 +555,29 @@ export interface IKSolveContext {
 
 // ---- Callbacks ----
 
-export type PhysicsStepCallback = (
-  model: MujocoModel,
-  data: MujocoData
-) => void;
+export interface PhysicsStepInput {
+  model: MujocoModel;
+  data: MujocoData;
+}
+
+export interface ResetCallbackInput extends PhysicsStepInput {}
+
+export interface ReadyCallbackInput {
+  api: MujocoSimAPI;
+}
+
+export interface StepCallbackInput {
+  time: number;
+  model: MujocoModel;
+  data: MujocoData;
+}
+
+export interface SelectionCallbackInput {
+  bodyId: number;
+  name: string;
+}
+
+export type PhysicsStepCallback = (input: PhysicsStepInput) => void;
 
 // ---- State Management (spec 4.1) ----
 
@@ -709,10 +744,28 @@ export interface KeyboardTeleopConfig {
 
 // ---- Policy (spec 10.1) ----
 
+export type PolicyVector = Float32Array | Float64Array | number[];
+
+export interface PolicyObservationInput {
+  model: MujocoModel;
+  data: MujocoData;
+}
+
+export interface PolicyInferenceInput extends PolicyObservationInput {
+  observation: PolicyVector;
+}
+
+export interface PolicyActionInput extends PolicyInferenceInput {
+  action: PolicyVector;
+}
+
 export interface PolicyConfig {
   frequency: number;
-  onObservation: (model: MujocoModel, data: MujocoData) => Float32Array | Float64Array | number[];
-  onAction: (action: Float32Array | Float64Array | number[], model: MujocoModel, data: MujocoData) => void;
+  enabled?: boolean;
+  onObservation: (input: PolicyObservationInput) => PolicyVector;
+  /** Run policy inference. Omit to pass observations directly to `onAction` for custom inline controllers. */
+  infer?: (input: PolicyInferenceInput) => PolicyVector;
+  onAction: (input: PolicyActionInput) => void;
 }
 
 // ---- Observation Builder ----
@@ -778,7 +831,12 @@ export interface IkGizmoProps {
   controller: IkContextValue;
   siteName?: string;
   scale?: number;
-  onDrag?: (position: THREE.Vector3, quaternion: THREE.Quaternion) => void;
+  onDrag?: (input: IkGizmoDragInput) => void;
+}
+
+export interface IkGizmoDragInput {
+  position: THREE.Vector3;
+  quaternion: THREE.Quaternion;
 }
 
 export interface DragInteractionProps {
@@ -805,6 +863,13 @@ export interface ScenarioCameraConfig {
   blur?: number;
 }
 
+export interface ScenarioMaterialConfig {
+  randomizeObjectColors?: boolean;
+  randomizeTableMaterial?: boolean;
+  roughness?: number;
+  metalness?: number;
+}
+
 export interface SplatAssetConfig {
   src: string;
   /** Common browser-friendly splat format. Renderer-specific loaders may accept more. */
@@ -819,7 +884,7 @@ export interface SplatScenarioConfig {
   format?: SplatFormat;
   src?: string;
   requiresCollisionProxy?: boolean;
-  collisionProxy?: SplatCollisionProxyConfig;
+  collisionProxy?: SplatCollisionProxyConfig | null;
 }
 
 export interface SplatCollisionProxyConfig {
@@ -845,6 +910,8 @@ export interface PairedSplatEnvironmentConfig {
 
 export interface SplatEnvironmentMetadataInput {
   environment?: PairedSplatEnvironmentConfig;
+  scenario?: VisualScenarioConfig;
+  renderer?: SplatRendererKind;
   src?: string;
   format?: SplatFormat;
   collisionProxy?: SplatCollisionProxyConfig;
@@ -857,6 +924,12 @@ export interface SplatEnvironmentMetadata {
   userData: Record<string, unknown>;
 }
 
+export type SplatSceneInput =
+  | PairedSplatEnvironmentConfig
+  | VisualScenarioConfig
+  | undefined
+  | null;
+
 export interface VisualScenarioConfig {
   id?: string;
   label?: string;
@@ -864,7 +937,8 @@ export interface VisualScenarioConfig {
   lighting?: ScenarioLightingPreset;
   environment?: string;
   camera?: ScenarioCameraConfig;
-  splat?: SplatScenarioConfig;
+  materials?: ScenarioMaterialConfig;
+  splat?: SplatScenarioConfig | null;
 }
 
 export interface ScenarioLightingProps {
@@ -875,6 +949,8 @@ export interface ScenarioLightingProps {
 
 export interface SplatEnvironmentProps extends Omit<ThreeElements['group'], 'ref'> {
   environment?: PairedSplatEnvironmentConfig;
+  scenario?: VisualScenarioConfig;
+  renderer?: SplatRendererKind;
   src?: string;
   format?: SplatFormat;
   collisionProxy?: ReactNode;
@@ -882,6 +958,24 @@ export interface SplatEnvironmentProps extends Omit<ThreeElements['group'], 'ref
   showPlaceholder?: boolean;
 }
 
+export interface VisualScenarioEffectsProps {
+  scenario?: VisualScenarioConfig;
+  enabled?: boolean;
+  applyBackground?: boolean;
+  applyFog?: boolean;
+  applyRenderer?: boolean;
+  applyMaterials?: boolean;
+  background?: THREE.ColorRepresentation;
+  fogNear?: number;
+  fogFar?: number;
+  materialFilter?: (input: VisualScenarioMaterialFilterInput) => boolean;
+}
+
+export interface VisualScenarioMaterialFilterInput {
+  object: THREE.Object3D;
+  material: THREE.Material;
+}
+
 export type TrajectoryInput = TrajectoryFrame[] | number[][];
 
 export interface TrajectoryPlayerProps {
@@ -891,9 +985,18 @@ export interface TrajectoryPlayerProps {
   loop?: boolean;
   playing?: boolean;
   mode?: 'kinematic' | 'physics';
-  onFrame?: (frameIdx: number) => void;
+  onFrame?: (input: TrajectoryFrameCallbackInput) => void;
   onComplete?: () => void;
-  onStateChange?: (state: PlaybackState) => void;
+  onStateChange?: (input: TrajectoryStateChangeInput) => void;
+}
+
+export interface TrajectoryFrameCallbackInput {
+  frameIndex: number;
+  frame: TrajectoryFrame | number[] | undefined;
+}
+
+export interface TrajectoryStateChangeInput {
+  state: PlaybackState;
 }
 
 export interface SelectionHighlightProps {
@@ -920,6 +1023,8 @@ export interface BodyProps {
   solref?: string;
   solimp?: string;
   condim?: number;
+  /** MuJoCo geom group. Group 3 is conventionally used for collision-only helper geoms. */
+  group?: number;
   children?: ReactNode;
 }
 
@@ -1017,10 +1122,12 @@ export interface MujocoSimAPI {
 
 export type MujocoCanvasProps = Omit<CanvasProps, 'onError'> & {
   config: SceneConfig;
-  onReady?: (api: MujocoSimAPI) => void;
+  /** R3F content rendered while the MuJoCo WASM module is still loading. */
+  loadingFallback?: ReactNode;
+  onReady?: (input: ReadyCallbackInput) => void;
   onError?: (error: Error) => void;
-  onStep?: (time: number) => void;
-  onSelection?: (bodyId: number, name: string) => void;
+  onStep?: (input: StepCallbackInput) => void;
+  onSelection?: (input: SelectionCallbackInput) => void;
   // Declarative physics config (spec 1.1)
   gravity?: [number, number, number];
   timestep?: number;