mujoco-react 10.4.0 → 10.6.0

package/src/components/CameraView.tsx

@@ -0,0 +1,245 @@
+/**
+ * @license
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Live on-screen camera viewports for MuJoCo scenes. Each view renders the
+ * shared scene from a named MuJoCo camera/site/body into a `gl.scissor` region
+ * tracking a DOM element — no GPU readback, no PNG encoding.
+ *
+ * While at least one viewport is mounted the canvas switches from R3F's
+ * automatic render to a managed render loop (main scene full-frame, then each
+ * viewport). This is incompatible with `EffectComposer`/postprocessing or other
+ * custom render loops; use the offscreen capture APIs in those setups instead.
+ */
+
+import {
+  createContext,
+  type CSSProperties,
+  type MutableRefObject,
+  type ReactNode,
+  type RefObject,
+  useCallback,
+  useContext,
+  useEffect,
+  useMemo,
+  useRef,
+  useState,
+} from 'react';
+import { useFrame, useThree } from '@react-three/fiber';
+import type * as THREE from 'three';
+import { useMujoco } from '../core/MujocoSimProvider';
+import {
+  createCaptureCamera,
+  prepareCaptureCamera,
+} from '../rendering/cameraFrameCapture';
+import type {
+  Bodies,
+  Cameras,
+  CameraFrameCaptureOptions,
+  Sites,
+} from '../types';
+
+/** Camera selection + pose options for a live viewport. */
+export type CameraViewportOptions = Pick<
+  CameraFrameCaptureOptions,
+  | 'camera'
+  | 'cameraName'
+  | 'siteName'
+  | 'bodyName'
+  | 'position'
+  | 'quaternion'
+  | 'lookAt'
+  | 'up'
+  | 'positionOffset'
+  | 'quaternionOffset'
+  | 'fov'
+  | 'near'
+  | 'far'
+  | 'projectionMatrix'
+  | 'mujocoCameraCompatibility'
+>;
+
+interface CameraViewportDescriptor {
+  getElement: () => HTMLElement | null;
+  getOptions: () => CameraViewportOptions;
+  camera: THREE.Camera | null;
+}
+
+interface CameraViewportRegistry {
+  register: (descriptor: CameraViewportDescriptor) => () => void;
+}
+
+const CameraViewportRegistryContext = createContext<CameraViewportRegistry | null>(null);
+
+let nextViewportId = 0;
+
+const VIEWPORT_RENDER_PRIORITY = 1;
+
+function CameraViewportRenderer({
+  viewportsRef,
+}: {
+  viewportsRef: MutableRefObject<Map<number, CameraViewportDescriptor>>;
+}) {
+  const gl = useThree((state) => state.gl);
+  const scene = useThree((state) => state.scene);
+  const mainCamera = useThree((state) => state.camera);
+  const mujoco = useMujoco();
+
+  useFrame(() => {
+    const drawWidth = gl.domElement.width;
+    const drawHeight = gl.domElement.height;
+
+    // We own the render now: draw the main scene full-frame first.
+    gl.setScissorTest(false);
+    gl.setViewport(0, 0, drawWidth, drawHeight);
+    gl.render(scene, mainCamera);
+
+    const api = mujoco.api;
+    if (!api || viewportsRef.current.size === 0) return;
+
+    const dpr = gl.getPixelRatio();
+    const canvasRect = gl.domElement.getBoundingClientRect();
+
+    for (const descriptor of viewportsRef.current.values()) {
+      const element = descriptor.getElement();
+      if (!element) continue;
+
+      const rect = element.getBoundingClientRect();
+      const isOffscreen =
+        rect.bottom < canvasRect.top ||
+        rect.top > canvasRect.bottom ||
+        rect.right < canvasRect.left ||
+        rect.left > canvasRect.right;
+      if (isOffscreen) continue;
+
+      const width = Math.floor(rect.width * dpr);
+      const height = Math.floor(rect.height * dpr);
+      if (width <= 0 || height <= 0) continue;
+
+      const left = Math.floor((rect.left - canvasRect.left) * dpr);
+      const bottom = Math.floor((canvasRect.bottom - rect.bottom) * dpr);
+
+      let resolved: CameraFrameCaptureOptions;
+      try {
+        resolved = api.resolveCameraCaptureOptions(descriptor.getOptions());
+      } catch {
+        continue;
+      }
+
+      if (!descriptor.camera) {
+        descriptor.camera = createCaptureCamera(resolved, mainCamera, width, height);
+      } else {
+        prepareCaptureCamera(descriptor.camera, resolved, mainCamera, width, height);
+      }
+
+      gl.setViewport(left, bottom, width, height);
+      gl.setScissor(left, bottom, width, height);
+      gl.setScissorTest(true);
+      gl.render(scene, descriptor.camera);
+    }
+
+    gl.setScissorTest(false);
+    gl.setViewport(0, 0, drawWidth, drawHeight);
+  }, VIEWPORT_RENDER_PRIORITY);
+
+  return null;
+}
+
+/**
+ * Provides the live-viewport registry and mounts the managed render loop only
+ * while at least one viewport is active. Mounted internally by the MuJoCo
+ * provider; you do not need to add it yourself.
+ */
+export function CameraViewportProvider({ children }: { children?: ReactNode }) {
+  const viewportsRef = useRef<Map<number, CameraViewportDescriptor>>(new Map());
+  const [count, setCount] = useState(0);
+
+  const register = useCallback((descriptor: CameraViewportDescriptor) => {
+    const id = nextViewportId++;
+    viewportsRef.current.set(id, descriptor);
+    setCount((value) => value + 1);
+    return () => {
+      viewportsRef.current.delete(id);
+      setCount((value) => value - 1);
+    };
+  }, []);
+
+  const value = useMemo<CameraViewportRegistry>(() => ({ register }), [register]);
+
+  return (
+    <CameraViewportRegistryContext.Provider value={value}>
+      {children}
+      {count > 0 && <CameraViewportRenderer viewportsRef={viewportsRef} />}
+    </CameraViewportRegistryContext.Provider>
+  );
+}
+
+/**
+ * Render the live MuJoCo scene from a named camera into the region covered by
+ * `elementRef`'s DOM element. Call this inside `<MujocoCanvas>` with a ref to a
+ * DOM element you position anywhere (it does not need to be in the R3F tree).
+ */
+export function useCameraViewport<T extends HTMLElement = HTMLElement>(
+  elementRef: RefObject<T | null>,
+  options: CameraViewportOptions
+) {
+  const registry = useContext(CameraViewportRegistryContext);
+  if (!registry) {
+    throw new Error('useCameraViewport must be used inside <MujocoCanvas>.');
+  }
+
+  const optionsRef = useRef(options);
+  optionsRef.current = options;
+
+  useEffect(() => {
+    const descriptor: CameraViewportDescriptor = {
+      getElement: () => elementRef.current,
+      getOptions: () => optionsRef.current,
+      camera: null,
+    };
+    return registry.register(descriptor);
+  }, [registry, elementRef]);
+}
+
+export interface CameraViewProps extends CameraViewportOptions {
+  className?: string;
+  style?: CSSProperties;
+}
+
+/**
+ * Drop-in live camera pane. Renders an absolutely-positioned overlay `<div>`
+ * over the canvas showing the selected MuJoCo camera. Position it with
+ * `style`/`className` (the canvas's parent should be positioned).
+ */
+export function CameraView({ className, style, ...options }: CameraViewProps) {
+  const gl = useThree((state) => state.gl);
+  const elementRef = useRef<HTMLDivElement | null>(null);
+  if (!elementRef.current && typeof document !== 'undefined') {
+    elementRef.current = document.createElement('div');
+  }
+
+  useEffect(() => {
+    const element = elementRef.current;
+    const parent = gl.domElement.parentElement;
+    if (!element || !parent) return;
+    element.style.position = 'absolute';
+    element.style.overflow = 'hidden';
+    parent.appendChild(element);
+    return () => {
+      parent.removeChild(element);
+    };
+  }, [gl]);
+
+  useEffect(() => {
+    const element = elementRef.current;
+    if (!element) return;
+    element.className = className ?? '';
+    if (style) Object.assign(element.style, style);
+  }, [className, style]);
+
+  useCameraViewport(elementRef, options);
+
+  return null;
+}
+
+export type { Bodies, Cameras, Sites };

package/src/core/GenericIK.ts

@@ -13,9 +13,13 @@ export interface GenericIKOptions {
     epsilon: number;
     posWeight: number;
     rotWeight: number;
+    jointLimits?: ReadonlyArray<readonly [number, number] | null | undefined>;
 }
 
-const DEFAULTS: GenericIKOptions = {
+type ResolvedGenericIKOptions = Required<Omit<GenericIKOptions, 'jointLimits'>> &
+    Pick<GenericIKOptions, 'jointLimits'>;
+
+const DEFAULTS: Required<Omit<GenericIKOptions, 'jointLimits'>> = {
     maxIterations: 50,
     damping: 0.01,
     tolerance: 1e-3,
@@ -24,7 +28,7 @@ const DEFAULTS: GenericIKOptions = {
     rotWeight: 0.3,
 };
 
-function resolveOptions(opts?: Partial<GenericIKOptions>): GenericIKOptions {
+function resolveOptions(opts?: Partial<GenericIKOptions>): ResolvedGenericIKOptions {
     return {
         maxIterations: opts?.maxIterations ?? DEFAULTS.maxIterations,
         damping: opts?.damping ?? DEFAULTS.damping,
@@ -32,6 +36,7 @@ function resolveOptions(opts?: Partial<GenericIKOptions>): GenericIKOptions {
         epsilon: opts?.epsilon ?? DEFAULTS.epsilon,
         posWeight: opts?.posWeight ?? DEFAULTS.posWeight,
         rotWeight: opts?.rotWeight ?? DEFAULTS.rotWeight,
+        jointLimits: opts?.jointLimits,
     };
 }
 
@@ -96,6 +101,14 @@ export class GenericIK {
 
         let bestQ: number[] | null = null;
         let bestErr = Infinity;
+        // Stop early once the error stops improving. The solver always returns the
+        // best configuration it has seen, so when it stalls or diverges (an
+        // unreachable target, e.g. a near-singular wrist orientation) further
+        // iterations can't beat bestQ — they only burn (1 + n) mj_forward calls
+        // each and feed jitter into ctrl. This is what bounds the cost of dragging
+        // the target into an unreachable region.
+        const patience = 4;
+        let noImprove = 0;
 
         if (n === 0) return null;
 
@@ -133,14 +146,20 @@ export class GenericIK {
             );
 
             // Track best solution
-            if (errNorm < bestErr) {
+            if (errNorm < bestErr - 1e-9) {
                 bestErr = errNorm;
                 bestQ = Array.from(q);
+                noImprove = 0;
+            } else {
+                noImprove++;
             }
 
             // Converged
             if (errNorm < o.tolerance) break;
 
+            // Stalled or diverging — bestQ can no longer improve, so stop.
+            if (noImprove >= patience) break;
+
             // Compute Jacobian via finite differences
             for (let j = 0; j < n; j++) {
                 const adr = qposAdr[j];

package/src/core/MujocoSimProvider.tsx

@@ -16,6 +16,7 @@ import {
 import * as THREE from 'three';
 import { MujocoData, MujocoModel, MujocoModule, getContact, withContacts } from '../types';
 import { SceneRenderer } from '../components/SceneRenderer';
+import { CameraViewportProvider } from '../components/CameraView';
 import {
   ActuatedJointInfo,
   ActuatorInfo,
@@ -59,6 +60,7 @@ import {
   captureCameraFrame,
   captureCameraFrameBlob,
   createCameraFrameCaptureSession,
+  type CameraFrameCaptureTensorOptions,
 } from '../rendering/cameraFrameCapture';
 import {
   getCameraFrameCaptureSourceTarget,
@@ -1558,6 +1560,35 @@ export function MujocoSimProvider({
     [camera, gl, resolveCameraCaptureOptions, scene]
   );
 
+  const createCameraFrameCaptureSessionApi = useCallback(
+    (options: CameraFrameCaptureOptions = {}) =>
+      createCameraFrameCaptureSession(
+        gl,
+        scene,
+        camera,
+        resolveCameraCaptureOptions(options)
+      ),
+    [camera, gl, resolveCameraCaptureOptions, scene]
+  );
+
+  const captureCameraFrameTensorApi = useCallback(
+    (options: CameraFrameCaptureTensorOptions = {}) => {
+      const resolved: CameraFrameCaptureTensorOptions = {
+        ...resolveCameraCaptureOptions(options),
+        channels: options.channels,
+        layout: options.layout,
+        range: options.range,
+      };
+      const session = createCameraFrameCaptureSession(gl, scene, camera, resolved);
+      try {
+        return session.captureTensor(resolved);
+      } finally {
+        session.dispose();
+      }
+    },
+    [camera, gl, resolveCameraCaptureOptions, scene]
+  );
+
   const recordCameraSequenceApi = useCallback(
     async (
       options: CameraFrameSequenceOptions
@@ -1882,6 +1913,9 @@ export function MujocoSimProvider({
       captureFrameBlob: captureFrameBlobApi,
       captureCameraFrame: captureCameraFrameApi,
       captureCameraFrameBlob: captureCameraFrameBlobApi,
+      captureCameraFrameTensor: captureCameraFrameTensorApi,
+      createCameraFrameCaptureSession: createCameraFrameCaptureSessionApi,
+      resolveCameraCaptureOptions,
       recordCameraSequence: recordCameraSequenceApi,
       project2DTo3D,
       projectImagePointTo3D,
@@ -1903,6 +1937,8 @@ export function MujocoSimProvider({
       loadFromFilesApi, addBodyApi, removeBodyApi, recompileApi,
       getCanvas, getCanvasSnapshot, captureFrameApi, captureFrameBlobApi,
       captureCameraFrameApi, captureCameraFrameBlobApi,
+      captureCameraFrameTensorApi, createCameraFrameCaptureSessionApi,
+      resolveCameraCaptureOptions,
       recordCameraSequenceApi,
       project2DTo3D,
       projectImagePointTo3D,
@@ -1940,7 +1976,7 @@ export function MujocoSimProvider({
   return (
     <MujocoSimContext.Provider value={contextValue}>
       <SceneRenderer renderOptions={renderOptions} />
-      {children}
+      <CameraViewportProvider>{children}</CameraViewportProvider>
     </MujocoSimContext.Provider>
   );
 }

package/src/core/SceneLoader.ts

@@ -496,9 +496,10 @@ function sceneObjectToXml(obj: SceneObject): string {
   const solref = obj.solref ? ` solref="${obj.solref}"` : '';
   const solimp = obj.solimp ? ` solimp="${obj.solimp}"` : '';
   const condim = obj.condim ? ` condim="${obj.condim}"` : '';
+  const contype = obj.contype ?? 1;
+  const conaffinity = obj.conaffinity ?? 1;
   const group = obj.group !== undefined ? ` group="${obj.group}"` : '';
-  // Always set contype/conaffinity=1 so objects collide regardless of model defaults
-  return `<body name="${obj.name}" pos="${pos}">${joint}<geom name="${geomName}" type="${obj.type}" size="${size}" rgba="${rgba}" contype="1" conaffinity="1"${mass}${friction}${solref}${solimp}${condim}${group}/></body>`;
+  return `<body name="${obj.name}" pos="${pos}">${joint}<geom name="${geomName}" type="${obj.type}" size="${size}" rgba="${rgba}" contype="${contype}" conaffinity="${conaffinity}"${mass}${friction}${solref}${solimp}${condim}${group}/></body>`;
 }
 
 /** Create virtual directory structure for a file path. */

package/src/hooks/useCameraStream.ts

@@ -0,0 +1,115 @@
+/**
+ * @license
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Stream a live MuJoCo camera into a DOM `<canvas>`. Each frame the scene is
+ * rendered offscreen from the selected camera and blitted into the canvas, so
+ * it composites normally in the DOM (works inside opaque panels) and does NOT
+ * take over the render loop. Prefer this over `useCameraViewport` for camera
+ * tiles embedded in HTML UI; use `useCameraViewport` for transparent overlays
+ * on a full-bleed canvas.
+ *
+ * Uses the async capture path so Gaussian-splat environments render through
+ * their dedicated capture renderer — streaming a splat scene at full rate does
+ * not disturb the main view's splat sort.
+ */
+
+import { useEffect, useRef } from 'react';
+import type { RefObject } from 'react';
+import { useFrame } from '@react-three/fiber';
+import { useMujoco } from '../core/MujocoSimProvider';
+import type { CameraFrameCaptureSession } from '../rendering/cameraFrameCapture';
+import type { CameraFrameCaptureOptions } from '../types';
+
+export interface CameraStreamOptions extends CameraFrameCaptureOptions {
+  /**
+   * Optional cap on updates per second. Omit to stream as fast as captures
+   * complete (one capture is in flight at a time regardless).
+   */
+  fps?: number;
+  /** Pause updates without unmounting. */
+  paused?: boolean;
+}
+
+function streamSignature(options: CameraStreamOptions): string {
+  return JSON.stringify({
+    cameraName: options.cameraName,
+    siteName: options.siteName,
+    bodyName: options.bodyName,
+    width: options.width,
+    height: options.height,
+    renderIsolation: options.renderIsolation ?? false,
+  });
+}
+
+/**
+ * Render the live scene from a MuJoCo camera/site/body into `canvasRef`'s
+ * `<canvas>` every frame (throttled to `fps`). Call inside `<MujocoCanvas>`;
+ * the canvas itself can live anywhere in the DOM.
+ */
+export function useCameraStream(
+  canvasRef: RefObject<HTMLCanvasElement | null>,
+  options: CameraStreamOptions
+) {
+  const mujoco = useMujoco();
+  const sessionRef = useRef<CameraFrameCaptureSession | null>(null);
+  const signatureRef = useRef<string>('');
+  const optionsRef = useRef(options);
+  optionsRef.current = options;
+  const elapsedRef = useRef(0);
+  const inFlightRef = useRef(false);
+  const mountedRef = useRef(true);
+
+  useEffect(() => {
+    mountedRef.current = true;
+    return () => {
+      mountedRef.current = false;
+      sessionRef.current?.dispose();
+      sessionRef.current = null;
+      signatureRef.current = '';
+    };
+  }, []);
+
+  useFrame((_state, delta) => {
+    const api = mujoco.api;
+    if (!api || !canvasRef.current) return;
+
+    const opts = optionsRef.current;
+    if (opts.paused) return;
+
+    if (opts.fps && opts.fps > 0) {
+      elapsedRef.current += delta;
+      if (elapsedRef.current < 1 / opts.fps) return;
+    }
+    // One capture in flight at a time — naturally rate-limits to capture speed.
+    if (inFlightRef.current) return;
+    elapsedRef.current = 0;
+
+    const signature = streamSignature(opts);
+    if (!sessionRef.current || signatureRef.current !== signature) {
+      sessionRef.current?.dispose();
+      sessionRef.current = api.createCameraFrameCaptureSession(opts);
+      signatureRef.current = signature;
+    }
+    const session = sessionRef.current;
+
+    inFlightRef.current = true;
+    session
+      .captureAsync(api.resolveCameraCaptureOptions(opts))
+      .then((frame) => {
+        const canvas = canvasRef.current;
+        if (!mountedRef.current || !canvas) return;
+        const ctx = canvas.getContext('2d');
+        if (!ctx) return;
+        if (canvas.width !== frame.width || canvas.height !== frame.height) {
+          canvas.width = frame.width;
+          canvas.height = frame.height;
+        }
+        ctx.drawImage(frame.canvas, 0, 0);
+      })
+      .catch(() => {})
+      .finally(() => {
+        inFlightRef.current = false;
+      });
+  });
+}

package/src/hooks/useIkController.ts

@@ -10,11 +10,31 @@ import { createControllerHook } from '../core/createController';
 import { useMujocoContext, useBeforePhysicsStep } from '../core/MujocoSimProvider';
 import { GenericIK } from '../core/GenericIK';
 import { createContiguousControlGroup, findSiteByName, resolveControlGroup } from '../core/SceneLoader';
-import type { ControlGroupInfo, IkConfig, IkContextValue, IKSolveFn, IkSolveInput, MujocoData } from '../types';
+import type {
+  ControlGroupInfo,
+  IkConfig,
+  IkContextValue,
+  IKSolveFn,
+  IkSolveInput,
+  IkTargetPosition,
+  MujocoData,
+} from '../types';
 
 // Preallocated temp for syncGizmoToSite
 const _syncMat4 = new THREE.Matrix4();
 
+function toIkVector3(value: IkSolveInput['position']): THREE.Vector3 {
+  return 'x' in value
+    ? new THREE.Vector3(value.x, value.y, value.z)
+    : new THREE.Vector3(value[0], value[1], value[2]);
+}
+
+function toIkQuaternion(value: IkSolveInput['quaternion']): THREE.Quaternion {
+  return 'x' in value
+    ? new THREE.Quaternion(value.x, value.y, value.z, value.w)
+    : new THREE.Quaternion(value[0], value[1], value[2], value[3]);
+}
+
 function syncGizmoToSite(data: MujocoData, siteId: number, target: THREE.Group) {
   if (siteId === -1) return;
   const sitePos = data.site_xpos.subarray(siteId * 3, siteId * 3 + 3);
@@ -86,17 +106,28 @@ export const useIkController = createControllerHook<IkConfig, IkContextValue>(
     const ikSolveFn = useCallback(
       ({ position, quaternion, currentQ, context }: IkSolveInput): number[] | null => {
         if (!config) return null;
-        if (config.ikSolveFn) return config.ikSolveFn({ position, quaternion, currentQ, context });
+        const targetPosition = toIkVector3(position);
+        const targetQuaternion = toIkQuaternion(quaternion);
+        const normalizedInput: IkSolveInput = {
+          position: targetPosition,
+          quaternion: targetQuaternion,
+          currentQ,
+          context,
+        };
+        if (config.ikSolveFn) return config.ikSolveFn(normalizedInput);
         const model = mjModelRef.current;
         const data = mjDataRef.current;
         const controlGroup = controlGroupRef.current;
         if (!model || !data || !controlGroup || siteIdRef.current === -1) return null;
         return genericIkRef.current.solve(
           model, data, siteIdRef.current, controlGroup.qposAdr,
-          position, quaternion, currentQ,
+          targetPosition, targetQuaternion, currentQ,
           {
             damping: config.damping,
             epsilon: config.epsilon,
+            jointLimits: config.jointLimits ?? controlGroup.joints.map((joint) => (
+              joint.limited ? joint.range : joint.ctrlRange
+            )),
             maxIterations: config.maxIterations,
             posWeight: config.posWeight,
             rotWeight: config.rotWeight,
@@ -215,12 +246,12 @@ export const useIkController = createControllerHook<IkConfig, IkContextValue>(
     );
 
     const moveTarget = useCallback(
-      (pos: THREE.Vector3, duration = 0) => {
+      (pos: IkTargetPosition, duration = 0) => {
         if (!ikEnabledRef.current) setIkEnabled(true);
         const target = ikTargetRef.current;
         if (!target) return;
 
-        const targetPos = pos.clone();
+        const targetPos = toIkVector3(pos);
         const targetRot = new THREE.Quaternion().setFromEuler(
           new THREE.Euler(Math.PI, 0, 0),
         );