mujoco-react 10.3.0 → 10.5.0

package/src/components/Debug.tsx

@@ -13,7 +13,7 @@ import { useMujocoContext } from '../core/MujocoSimProvider';
 import { getName } from '../core/SceneLoader';
 import { CAPTURE_EXCLUDE_KEY } from '../rendering/cameraFrameCapture';
 import { getContact, withContacts } from '../types';
-import type { DebugProps } from '../types';
+import type { CameraFrameCaptureVector3, DebugProps, DebugVirtualCamera } from '../types';
 
 const JOINT_COLORS: Record<number, number> = {
   0: 0xff0000, // free - red
@@ -41,6 +41,170 @@ type CameraDebugObject = THREE.Group & {
   };
 };
 
+function toVector3(
+  value: CameraFrameCaptureVector3 | undefined,
+  fallback: THREE.Vector3
+) {
+  if (!value) return fallback.clone();
+  return value instanceof THREE.Vector3
+    ? value.clone()
+    : new THREE.Vector3(value[0], value[1], value[2]);
+}
+
+function createCameraLabel(text: string, color: THREE.ColorRepresentation) {
+  const canvas = document.createElement('canvas');
+  canvas.width = 256;
+  canvas.height = 64;
+  const ctx = canvas.getContext('2d')!;
+  ctx.fillStyle = new THREE.Color(color).getStyle();
+  ctx.font = 'bold 32px monospace';
+  ctx.textAlign = 'center';
+  ctx.fillText(text, 128, 42);
+  const texture = new THREE.CanvasTexture(canvas);
+  const sprite = new THREE.Sprite(
+    new THREE.SpriteMaterial({
+      map: texture,
+      depthTest: false,
+      transparent: true,
+    })
+  );
+  sprite.position.set(0, 0.014, 0.01);
+  sprite.scale.set(0.05, 0.012, 1);
+  sprite.renderOrder = 999;
+  return sprite;
+}
+
+function createVirtualCameraDebugObject(
+  camera: DebugVirtualCamera,
+  index: number
+) {
+  const color = camera.color ?? '#ff3d71';
+  const aimColor = camera.aimColor ?? '#38bdf8';
+  const markerScale = camera.markerScale ?? 1;
+  const cameraPosition = toVector3(camera.position, new THREE.Vector3());
+  const configuredUp = toVector3(camera.up, new THREE.Vector3(0, 0, 1)).normalize();
+  const cameraQuaternion = new THREE.Quaternion();
+  const forward = new THREE.Vector3();
+
+  if (camera.quaternion) {
+    if (camera.quaternion instanceof THREE.Quaternion) {
+      cameraQuaternion.copy(camera.quaternion);
+    } else {
+      cameraQuaternion.set(
+        camera.quaternion[0],
+        camera.quaternion[1],
+        camera.quaternion[2],
+        camera.quaternion[3]
+      );
+    }
+    forward.set(0, 0, -1).applyQuaternion(cameraQuaternion).normalize();
+  } else {
+    const target = toVector3(
+      camera.lookAt,
+      cameraPosition.clone().add(new THREE.Vector3(0, 0, -1))
+    );
+    forward.copy(target).sub(cameraPosition);
+    if (forward.lengthSq() < 1e-8) forward.set(0, 0, -1);
+    forward.normalize();
+    cameraQuaternion.setFromRotationMatrix(
+      new THREE.Matrix4().lookAt(cameraPosition, target, configuredUp)
+    );
+  }
+
+  const target = camera.lookAt
+    ? toVector3(camera.lookAt, cameraPosition.clone().add(forward))
+    : cameraPosition.clone().addScaledVector(forward, 0.4);
+  const distanceToTarget = Math.max(target.distanceTo(cameraPosition), 0.001);
+  const depth = camera.frustumDepth ?? Math.min(Math.max(distanceToTarget * 0.42, 0.16), 0.45);
+  const fov = camera.fov ?? 50;
+  const aspect = (camera.width ?? 640) / (camera.height ?? 480);
+
+  const right = forward.clone().cross(configuredUp);
+  if (right.lengthSq() < 1e-8) right.set(1, 0, 0);
+  right.normalize();
+  const orthogonalUp = right.clone().cross(forward).normalize();
+  const frustumHeight = 2 * Math.tan(THREE.MathUtils.degToRad(fov) / 2) * depth;
+  const frustumWidth = frustumHeight * aspect;
+  const center = cameraPosition.clone().addScaledVector(forward, depth);
+  const halfRight = right.clone().multiplyScalar(frustumWidth / 2);
+  const halfUp = orthogonalUp.clone().multiplyScalar(frustumHeight / 2);
+
+  const topLeft = center.clone().sub(halfRight).add(halfUp);
+  const topRight = center.clone().add(halfRight).add(halfUp);
+  const bottomRight = center.clone().add(halfRight).sub(halfUp);
+  const bottomLeft = center.clone().sub(halfRight).sub(halfUp);
+  const frustumPoints = [
+    cameraPosition, topLeft,
+    cameraPosition, topRight,
+    cameraPosition, bottomRight,
+    cameraPosition, bottomLeft,
+    topLeft, topRight,
+    topRight, bottomRight,
+    bottomRight, bottomLeft,
+    bottomLeft, topLeft,
+  ];
+
+  const group = new THREE.Group();
+  group.name = camera.name ?? `virtual-camera-${index}`;
+  group.renderOrder = 999;
+  group.frustumCulled = false;
+
+  const frustum = new THREE.LineSegments(
+    new THREE.BufferGeometry().setFromPoints(frustumPoints),
+    new THREE.LineBasicMaterial({
+      color,
+      transparent: true,
+      opacity: 0.9,
+      depthTest: false,
+    })
+  );
+  frustum.renderOrder = 999;
+  frustum.frustumCulled = false;
+  group.add(frustum);
+
+  const aim = new THREE.LineSegments(
+    new THREE.BufferGeometry().setFromPoints([cameraPosition, target]),
+    new THREE.LineBasicMaterial({
+      color: aimColor,
+      transparent: true,
+      opacity: 0.95,
+      depthTest: false,
+    })
+  );
+  aim.renderOrder = 999;
+  aim.frustumCulled = false;
+  group.add(aim);
+
+  const markerGroup = new THREE.Group();
+  markerGroup.position.copy(cameraPosition);
+  markerGroup.quaternion.copy(cameraQuaternion);
+  markerGroup.renderOrder = 999;
+  markerGroup.frustumCulled = false;
+  markerGroup.add(new THREE.Mesh(
+    new THREE.BoxGeometry(0.045 * markerScale, 0.028 * markerScale, 0.022 * markerScale),
+    new THREE.MeshBasicMaterial({ color, depthTest: false })
+  ));
+  const lens = new THREE.Mesh(
+    new THREE.BoxGeometry(0.025 * markerScale, 0.018 * markerScale, 0.014 * markerScale),
+    new THREE.MeshBasicMaterial({ color: aimColor, depthTest: false })
+  );
+  lens.position.set(0, 0, -0.021 * markerScale);
+  markerGroup.add(lens);
+  if (camera.name) markerGroup.add(createCameraLabel(camera.name, color));
+  group.add(markerGroup);
+
+  const targetMarker = new THREE.Mesh(
+    new THREE.SphereGeometry(0.018 * markerScale, 16, 10),
+    new THREE.MeshBasicMaterial({ color: aimColor, depthTest: false })
+  );
+  targetMarker.position.copy(target);
+  targetMarker.renderOrder = 999;
+  targetMarker.frustumCulled = false;
+  group.add(targetMarker);
+
+  return group;
+}
+
 /**
  * Declarative debug visualization component.
  * Renders wireframe geoms, site markers, joint axes, contact forces, COM markers, etc.
@@ -50,6 +214,7 @@ export function Debug({
   showSites = false,
   showJoints = false,
   showCameras = false,
+  virtualCameras = [],
   showContacts = false,
   showCOM = false,
   showInertia = false,
@@ -69,6 +234,7 @@ export function Debug({
     const sites: THREE.Object3D[] = [];
     const joints: THREE.Object3D[] = [];
     const cameras: CameraDebugObject[] = [];
+    const virtualCameraObjects: THREE.Object3D[] = [];
     const comMarkers: THREE.Object3D[] = [];
 
     // Wireframe geoms
@@ -270,6 +436,10 @@ export function Debug({
       }
     }
 
+    for (let i = 0; i < virtualCameras.length; i += 1) {
+      virtualCameraObjects.push(createVirtualCameraDebugObject(virtualCameras[i], i));
+    }
+
     // COM markers
     if (showCOM) {
       for (let i = 1; i < model.nbody; i++) {
@@ -281,8 +451,8 @@ export function Debug({
       }
     }
 
-    return { geoms, sites, joints, cameras, comMarkers };
-  }, [status, mjModelRef, showGeoms, showSites, showJoints, showCameras, showCOM]);
+    return { geoms, sites, joints, cameras, virtualCameraObjects, comMarkers };
+  }, [status, mjModelRef, showGeoms, showSites, showJoints, showCameras, virtualCameras, showCOM]);
 
   // Add/remove debug objects from scene
   useEffect(() => {
@@ -294,6 +464,7 @@ export function Debug({
       ...debugGeometry.sites,
       ...debugGeometry.joints,
       ...debugGeometry.cameras,
+      ...debugGeometry.virtualCameraObjects,
       ...debugGeometry.comMarkers,
     ];
     for (const obj of allObjects) group.add(obj);

package/src/core/GenericIK.ts

@@ -13,9 +13,13 @@ export interface GenericIKOptions {
     epsilon: number;
     posWeight: number;
     rotWeight: number;
+    jointLimits?: ReadonlyArray<readonly [number, number] | null | undefined>;
 }
 
-const DEFAULTS: GenericIKOptions = {
+type ResolvedGenericIKOptions = Required<Omit<GenericIKOptions, 'jointLimits'>> &
+    Pick<GenericIKOptions, 'jointLimits'>;
+
+const DEFAULTS: Required<Omit<GenericIKOptions, 'jointLimits'>> = {
     maxIterations: 50,
     damping: 0.01,
     tolerance: 1e-3,
@@ -24,7 +28,7 @@ const DEFAULTS: GenericIKOptions = {
     rotWeight: 0.3,
 };
 
-function resolveOptions(opts?: Partial<GenericIKOptions>): GenericIKOptions {
+function resolveOptions(opts?: Partial<GenericIKOptions>): ResolvedGenericIKOptions {
     return {
         maxIterations: opts?.maxIterations ?? DEFAULTS.maxIterations,
         damping: opts?.damping ?? DEFAULTS.damping,
@@ -32,9 +36,17 @@ function resolveOptions(opts?: Partial<GenericIKOptions>): GenericIKOptions {
         epsilon: opts?.epsilon ?? DEFAULTS.epsilon,
         posWeight: opts?.posWeight ?? DEFAULTS.posWeight,
         rotWeight: opts?.rotWeight ?? DEFAULTS.rotWeight,
+        jointLimits: opts?.jointLimits,
     };
 }
 
+function clampJoint(value: number, limit: readonly [number, number] | null | undefined) {
+    if (!limit) return value;
+    const [min, max] = limit;
+    if (!Number.isFinite(min) || !Number.isFinite(max) || min >= max) return value;
+    return Math.max(min, Math.min(max, value));
+}
+
 /**
  * Generic Damped Least-Squares IK solver.
  * Uses finite-difference Jacobian via MuJoCo's mj_forward.
@@ -81,7 +93,7 @@ export class GenericIK {
 
         // Working joint angles — start from current configuration
         const q = new Float64Array(n);
-        for (let i = 0; i < n; i++) q[i] = currentQ[i];
+        for (let i = 0; i < n; i++) q[i] = clampJoint(currentQ[i], o.jointLimits?.[i]);
 
         // Pre-allocate work arrays
         const J = new Float64Array(6 * n);       // 6×n Jacobian (row-major)
@@ -196,7 +208,7 @@ export class GenericIK {
             }
 
             // Update joints
-            for (let i = 0; i < n; i++) q[i] += dq[i];
+            for (let i = 0; i < n; i++) q[i] = clampJoint(q[i] + dq[i], o.jointLimits?.[i]);
         }
 
         // Restore original qpos

package/src/core/MujocoSimProvider.tsx

@@ -16,6 +16,7 @@ import {
 import * as THREE from 'three';
 import { MujocoData, MujocoModel, MujocoModule, getContact, withContacts } from '../types';
 import { SceneRenderer } from '../components/SceneRenderer';
+import { CameraViewportProvider } from '../components/CameraView';
 import {
   ActuatedJointInfo,
   ActuatorInfo,
@@ -59,6 +60,7 @@ import {
   captureCameraFrame,
   captureCameraFrameBlob,
   createCameraFrameCaptureSession,
+  type CameraFrameCaptureTensorOptions,
 } from '../rendering/cameraFrameCapture';
 import {
   getCameraFrameCaptureSourceTarget,
@@ -1558,6 +1560,35 @@ export function MujocoSimProvider({
     [camera, gl, resolveCameraCaptureOptions, scene]
   );
 
+  const createCameraFrameCaptureSessionApi = useCallback(
+    (options: CameraFrameCaptureOptions = {}) =>
+      createCameraFrameCaptureSession(
+        gl,
+        scene,
+        camera,
+        resolveCameraCaptureOptions(options)
+      ),
+    [camera, gl, resolveCameraCaptureOptions, scene]
+  );
+
+  const captureCameraFrameTensorApi = useCallback(
+    (options: CameraFrameCaptureTensorOptions = {}) => {
+      const resolved: CameraFrameCaptureTensorOptions = {
+        ...resolveCameraCaptureOptions(options),
+        channels: options.channels,
+        layout: options.layout,
+        range: options.range,
+      };
+      const session = createCameraFrameCaptureSession(gl, scene, camera, resolved);
+      try {
+        return session.captureTensor(resolved);
+      } finally {
+        session.dispose();
+      }
+    },
+    [camera, gl, resolveCameraCaptureOptions, scene]
+  );
+
   const recordCameraSequenceApi = useCallback(
     async (
       options: CameraFrameSequenceOptions
@@ -1882,6 +1913,9 @@ export function MujocoSimProvider({
       captureFrameBlob: captureFrameBlobApi,
       captureCameraFrame: captureCameraFrameApi,
       captureCameraFrameBlob: captureCameraFrameBlobApi,
+      captureCameraFrameTensor: captureCameraFrameTensorApi,
+      createCameraFrameCaptureSession: createCameraFrameCaptureSessionApi,
+      resolveCameraCaptureOptions,
       recordCameraSequence: recordCameraSequenceApi,
       project2DTo3D,
       projectImagePointTo3D,
@@ -1903,6 +1937,8 @@ export function MujocoSimProvider({
       loadFromFilesApi, addBodyApi, removeBodyApi, recompileApi,
       getCanvas, getCanvasSnapshot, captureFrameApi, captureFrameBlobApi,
       captureCameraFrameApi, captureCameraFrameBlobApi,
+      captureCameraFrameTensorApi, createCameraFrameCaptureSessionApi,
+      resolveCameraCaptureOptions,
       recordCameraSequenceApi,
       project2DTo3D,
       projectImagePointTo3D,
@@ -1940,7 +1976,7 @@ export function MujocoSimProvider({
   return (
     <MujocoSimContext.Provider value={contextValue}>
       <SceneRenderer renderOptions={renderOptions} />
-      {children}
+      <CameraViewportProvider>{children}</CameraViewportProvider>
     </MujocoSimContext.Provider>
   );
 }

package/src/core/SceneLoader.ts

@@ -496,9 +496,10 @@ function sceneObjectToXml(obj: SceneObject): string {
   const solref = obj.solref ? ` solref="${obj.solref}"` : '';
   const solimp = obj.solimp ? ` solimp="${obj.solimp}"` : '';
   const condim = obj.condim ? ` condim="${obj.condim}"` : '';
+  const contype = obj.contype ?? 1;
+  const conaffinity = obj.conaffinity ?? 1;
   const group = obj.group !== undefined ? ` group="${obj.group}"` : '';
-  // Always set contype/conaffinity=1 so objects collide regardless of model defaults
-  return `<body name="${obj.name}" pos="${pos}">${joint}<geom name="${geomName}" type="${obj.type}" size="${size}" rgba="${rgba}" contype="1" conaffinity="1"${mass}${friction}${solref}${solimp}${condim}${group}/></body>`;
+  return `<body name="${obj.name}" pos="${pos}">${joint}<geom name="${geomName}" type="${obj.type}" size="${size}" rgba="${rgba}" contype="${contype}" conaffinity="${conaffinity}"${mass}${friction}${solref}${solimp}${condim}${group}/></body>`;
 }
 
 /** Create virtual directory structure for a file path. */

package/src/hooks/useCameraStream.ts

@@ -0,0 +1,115 @@
+/**
+ * @license
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Stream a live MuJoCo camera into a DOM `<canvas>`. Each frame the scene is
+ * rendered offscreen from the selected camera and blitted into the canvas, so
+ * it composites normally in the DOM (works inside opaque panels) and does NOT
+ * take over the render loop. Prefer this over `useCameraViewport` for camera
+ * tiles embedded in HTML UI; use `useCameraViewport` for transparent overlays
+ * on a full-bleed canvas.
+ *
+ * Uses the async capture path so Gaussian-splat environments render through
+ * their dedicated capture renderer — streaming a splat scene at full rate does
+ * not disturb the main view's splat sort.
+ */
+
+import { useEffect, useRef } from 'react';
+import type { RefObject } from 'react';
+import { useFrame } from '@react-three/fiber';
+import { useMujoco } from '../core/MujocoSimProvider';
+import type { CameraFrameCaptureSession } from '../rendering/cameraFrameCapture';
+import type { CameraFrameCaptureOptions } from '../types';
+
+export interface CameraStreamOptions extends CameraFrameCaptureOptions {
+  /**
+   * Optional cap on updates per second. Omit to stream as fast as captures
+   * complete (one capture is in flight at a time regardless).
+   */
+  fps?: number;
+  /** Pause updates without unmounting. */
+  paused?: boolean;
+}
+
+function streamSignature(options: CameraStreamOptions): string {
+  return JSON.stringify({
+    cameraName: options.cameraName,
+    siteName: options.siteName,
+    bodyName: options.bodyName,
+    width: options.width,
+    height: options.height,
+    renderIsolation: options.renderIsolation ?? false,
+  });
+}
+
+/**
+ * Render the live scene from a MuJoCo camera/site/body into `canvasRef`'s
+ * `<canvas>` every frame (throttled to `fps`). Call inside `<MujocoCanvas>`;
+ * the canvas itself can live anywhere in the DOM.
+ */
+export function useCameraStream(
+  canvasRef: RefObject<HTMLCanvasElement | null>,
+  options: CameraStreamOptions
+) {
+  const mujoco = useMujoco();
+  const sessionRef = useRef<CameraFrameCaptureSession | null>(null);
+  const signatureRef = useRef<string>('');
+  const optionsRef = useRef(options);
+  optionsRef.current = options;
+  const elapsedRef = useRef(0);
+  const inFlightRef = useRef(false);
+  const mountedRef = useRef(true);
+
+  useEffect(() => {
+    mountedRef.current = true;
+    return () => {
+      mountedRef.current = false;
+      sessionRef.current?.dispose();
+      sessionRef.current = null;
+      signatureRef.current = '';
+    };
+  }, []);
+
+  useFrame((_state, delta) => {
+    const api = mujoco.api;
+    if (!api || !canvasRef.current) return;
+
+    const opts = optionsRef.current;
+    if (opts.paused) return;
+
+    if (opts.fps && opts.fps > 0) {
+      elapsedRef.current += delta;
+      if (elapsedRef.current < 1 / opts.fps) return;
+    }
+    // One capture in flight at a time — naturally rate-limits to capture speed.
+    if (inFlightRef.current) return;
+    elapsedRef.current = 0;
+
+    const signature = streamSignature(opts);
+    if (!sessionRef.current || signatureRef.current !== signature) {
+      sessionRef.current?.dispose();
+      sessionRef.current = api.createCameraFrameCaptureSession(opts);
+      signatureRef.current = signature;
+    }
+    const session = sessionRef.current;
+
+    inFlightRef.current = true;
+    session
+      .captureAsync(api.resolveCameraCaptureOptions(opts))
+      .then((frame) => {
+        const canvas = canvasRef.current;
+        if (!mountedRef.current || !canvas) return;
+        const ctx = canvas.getContext('2d');
+        if (!ctx) return;
+        if (canvas.width !== frame.width || canvas.height !== frame.height) {
+          canvas.width = frame.width;
+          canvas.height = frame.height;
+        }
+        ctx.drawImage(frame.canvas, 0, 0);
+      })
+      .catch(() => {})
+      .finally(() => {
+        inFlightRef.current = false;
+      });
+  });
+}

package/src/hooks/useIkController.ts

@@ -97,6 +97,9 @@ export const useIkController = createControllerHook<IkConfig, IkContextValue>(
           {
             damping: config.damping,
             epsilon: config.epsilon,
+            jointLimits: config.jointLimits ?? controlGroup.joints.map((joint) => (
+              joint.limited ? joint.range : joint.ctrlRange
+            )),
             maxIterations: config.maxIterations,
             posWeight: config.posWeight,
             rotWeight: config.rotWeight,