mujoco-react 10.4.0 → 10.5.0

package/src/core/GenericIK.ts

@@ -13,9 +13,13 @@ export interface GenericIKOptions {
     epsilon: number;
     posWeight: number;
     rotWeight: number;
+    jointLimits?: ReadonlyArray<readonly [number, number] | null | undefined>;
 }
 
-const DEFAULTS: GenericIKOptions = {
+type ResolvedGenericIKOptions = Required<Omit<GenericIKOptions, 'jointLimits'>> &
+    Pick<GenericIKOptions, 'jointLimits'>;
+
+const DEFAULTS: Required<Omit<GenericIKOptions, 'jointLimits'>> = {
     maxIterations: 50,
     damping: 0.01,
     tolerance: 1e-3,
@@ -24,7 +28,7 @@ const DEFAULTS: GenericIKOptions = {
     rotWeight: 0.3,
 };
 
-function resolveOptions(opts?: Partial<GenericIKOptions>): GenericIKOptions {
+function resolveOptions(opts?: Partial<GenericIKOptions>): ResolvedGenericIKOptions {
     return {
         maxIterations: opts?.maxIterations ?? DEFAULTS.maxIterations,
         damping: opts?.damping ?? DEFAULTS.damping,
@@ -32,9 +36,17 @@ function resolveOptions(opts?: Partial<GenericIKOptions>): GenericIKOptions {
         epsilon: opts?.epsilon ?? DEFAULTS.epsilon,
         posWeight: opts?.posWeight ?? DEFAULTS.posWeight,
         rotWeight: opts?.rotWeight ?? DEFAULTS.rotWeight,
+        jointLimits: opts?.jointLimits,
     };
 }
 
+function clampJoint(value: number, limit: readonly [number, number] | null | undefined) {
+    if (!limit) return value;
+    const [min, max] = limit;
+    if (!Number.isFinite(min) || !Number.isFinite(max) || min >= max) return value;
+    return Math.max(min, Math.min(max, value));
+}
+
 /**
  * Generic Damped Least-Squares IK solver.
  * Uses finite-difference Jacobian via MuJoCo's mj_forward.
@@ -81,7 +93,7 @@ export class GenericIK {
 
         // Working joint angles — start from current configuration
         const q = new Float64Array(n);
-        for (let i = 0; i < n; i++) q[i] = currentQ[i];
+        for (let i = 0; i < n; i++) q[i] = clampJoint(currentQ[i], o.jointLimits?.[i]);
 
         // Pre-allocate work arrays
         const J = new Float64Array(6 * n);       // 6×n Jacobian (row-major)
@@ -196,7 +208,7 @@ export class GenericIK {
             }
 
             // Update joints
-            for (let i = 0; i < n; i++) q[i] += dq[i];
+            for (let i = 0; i < n; i++) q[i] = clampJoint(q[i] + dq[i], o.jointLimits?.[i]);
         }
 
         // Restore original qpos

package/src/core/MujocoSimProvider.tsx

@@ -16,6 +16,7 @@ import {
 import * as THREE from 'three';
 import { MujocoData, MujocoModel, MujocoModule, getContact, withContacts } from '../types';
 import { SceneRenderer } from '../components/SceneRenderer';
+import { CameraViewportProvider } from '../components/CameraView';
 import {
   ActuatedJointInfo,
   ActuatorInfo,
@@ -59,6 +60,7 @@ import {
   captureCameraFrame,
   captureCameraFrameBlob,
   createCameraFrameCaptureSession,
+  type CameraFrameCaptureTensorOptions,
 } from '../rendering/cameraFrameCapture';
 import {
   getCameraFrameCaptureSourceTarget,
@@ -1558,6 +1560,35 @@ export function MujocoSimProvider({
     [camera, gl, resolveCameraCaptureOptions, scene]
   );
 
+  const createCameraFrameCaptureSessionApi = useCallback(
+    (options: CameraFrameCaptureOptions = {}) =>
+      createCameraFrameCaptureSession(
+        gl,
+        scene,
+        camera,
+        resolveCameraCaptureOptions(options)
+      ),
+    [camera, gl, resolveCameraCaptureOptions, scene]
+  );
+
+  const captureCameraFrameTensorApi = useCallback(
+    (options: CameraFrameCaptureTensorOptions = {}) => {
+      const resolved: CameraFrameCaptureTensorOptions = {
+        ...resolveCameraCaptureOptions(options),
+        channels: options.channels,
+        layout: options.layout,
+        range: options.range,
+      };
+      const session = createCameraFrameCaptureSession(gl, scene, camera, resolved);
+      try {
+        return session.captureTensor(resolved);
+      } finally {
+        session.dispose();
+      }
+    },
+    [camera, gl, resolveCameraCaptureOptions, scene]
+  );
+
   const recordCameraSequenceApi = useCallback(
     async (
       options: CameraFrameSequenceOptions
@@ -1882,6 +1913,9 @@ export function MujocoSimProvider({
       captureFrameBlob: captureFrameBlobApi,
       captureCameraFrame: captureCameraFrameApi,
       captureCameraFrameBlob: captureCameraFrameBlobApi,
+      captureCameraFrameTensor: captureCameraFrameTensorApi,
+      createCameraFrameCaptureSession: createCameraFrameCaptureSessionApi,
+      resolveCameraCaptureOptions,
       recordCameraSequence: recordCameraSequenceApi,
       project2DTo3D,
       projectImagePointTo3D,
@@ -1903,6 +1937,8 @@ export function MujocoSimProvider({
       loadFromFilesApi, addBodyApi, removeBodyApi, recompileApi,
       getCanvas, getCanvasSnapshot, captureFrameApi, captureFrameBlobApi,
       captureCameraFrameApi, captureCameraFrameBlobApi,
+      captureCameraFrameTensorApi, createCameraFrameCaptureSessionApi,
+      resolveCameraCaptureOptions,
       recordCameraSequenceApi,
       project2DTo3D,
       projectImagePointTo3D,
@@ -1940,7 +1976,7 @@ export function MujocoSimProvider({
   return (
     <MujocoSimContext.Provider value={contextValue}>
       <SceneRenderer renderOptions={renderOptions} />
-      {children}
+      <CameraViewportProvider>{children}</CameraViewportProvider>
     </MujocoSimContext.Provider>
   );
 }

package/src/core/SceneLoader.ts

@@ -496,9 +496,10 @@ function sceneObjectToXml(obj: SceneObject): string {
   const solref = obj.solref ? ` solref="${obj.solref}"` : '';
   const solimp = obj.solimp ? ` solimp="${obj.solimp}"` : '';
   const condim = obj.condim ? ` condim="${obj.condim}"` : '';
+  const contype = obj.contype ?? 1;
+  const conaffinity = obj.conaffinity ?? 1;
   const group = obj.group !== undefined ? ` group="${obj.group}"` : '';
-  // Always set contype/conaffinity=1 so objects collide regardless of model defaults
-  return `<body name="${obj.name}" pos="${pos}">${joint}<geom name="${geomName}" type="${obj.type}" size="${size}" rgba="${rgba}" contype="1" conaffinity="1"${mass}${friction}${solref}${solimp}${condim}${group}/></body>`;
+  return `<body name="${obj.name}" pos="${pos}">${joint}<geom name="${geomName}" type="${obj.type}" size="${size}" rgba="${rgba}" contype="${contype}" conaffinity="${conaffinity}"${mass}${friction}${solref}${solimp}${condim}${group}/></body>`;
 }
 
 /** Create virtual directory structure for a file path. */

package/src/hooks/useCameraStream.ts

@@ -0,0 +1,115 @@
+/**
+ * @license
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Stream a live MuJoCo camera into a DOM `<canvas>`. Each frame the scene is
+ * rendered offscreen from the selected camera and blitted into the canvas, so
+ * it composites normally in the DOM (works inside opaque panels) and does NOT
+ * take over the render loop. Prefer this over `useCameraViewport` for camera
+ * tiles embedded in HTML UI; use `useCameraViewport` for transparent overlays
+ * on a full-bleed canvas.
+ *
+ * Uses the async capture path so Gaussian-splat environments render through
+ * their dedicated capture renderer — streaming a splat scene at full rate does
+ * not disturb the main view's splat sort.
+ */
+
+import { useEffect, useRef } from 'react';
+import type { RefObject } from 'react';
+import { useFrame } from '@react-three/fiber';
+import { useMujoco } from '../core/MujocoSimProvider';
+import type { CameraFrameCaptureSession } from '../rendering/cameraFrameCapture';
+import type { CameraFrameCaptureOptions } from '../types';
+
+export interface CameraStreamOptions extends CameraFrameCaptureOptions {
+  /**
+   * Optional cap on updates per second. Omit to stream as fast as captures
+   * complete (one capture is in flight at a time regardless).
+   */
+  fps?: number;
+  /** Pause updates without unmounting. */
+  paused?: boolean;
+}
+
+function streamSignature(options: CameraStreamOptions): string {
+  return JSON.stringify({
+    cameraName: options.cameraName,
+    siteName: options.siteName,
+    bodyName: options.bodyName,
+    width: options.width,
+    height: options.height,
+    renderIsolation: options.renderIsolation ?? false,
+  });
+}
+
+/**
+ * Render the live scene from a MuJoCo camera/site/body into `canvasRef`'s
+ * `<canvas>` every frame (throttled to `fps`). Call inside `<MujocoCanvas>`;
+ * the canvas itself can live anywhere in the DOM.
+ */
+export function useCameraStream(
+  canvasRef: RefObject<HTMLCanvasElement | null>,
+  options: CameraStreamOptions
+) {
+  const mujoco = useMujoco();
+  const sessionRef = useRef<CameraFrameCaptureSession | null>(null);
+  const signatureRef = useRef<string>('');
+  const optionsRef = useRef(options);
+  optionsRef.current = options;
+  const elapsedRef = useRef(0);
+  const inFlightRef = useRef(false);
+  const mountedRef = useRef(true);
+
+  useEffect(() => {
+    mountedRef.current = true;
+    return () => {
+      mountedRef.current = false;
+      sessionRef.current?.dispose();
+      sessionRef.current = null;
+      signatureRef.current = '';
+    };
+  }, []);
+
+  useFrame((_state, delta) => {
+    const api = mujoco.api;
+    if (!api || !canvasRef.current) return;
+
+    const opts = optionsRef.current;
+    if (opts.paused) return;
+
+    if (opts.fps && opts.fps > 0) {
+      elapsedRef.current += delta;
+      if (elapsedRef.current < 1 / opts.fps) return;
+    }
+    // One capture in flight at a time — naturally rate-limits to capture speed.
+    if (inFlightRef.current) return;
+    elapsedRef.current = 0;
+
+    const signature = streamSignature(opts);
+    if (!sessionRef.current || signatureRef.current !== signature) {
+      sessionRef.current?.dispose();
+      sessionRef.current = api.createCameraFrameCaptureSession(opts);
+      signatureRef.current = signature;
+    }
+    const session = sessionRef.current;
+
+    inFlightRef.current = true;
+    session
+      .captureAsync(api.resolveCameraCaptureOptions(opts))
+      .then((frame) => {
+        const canvas = canvasRef.current;
+        if (!mountedRef.current || !canvas) return;
+        const ctx = canvas.getContext('2d');
+        if (!ctx) return;
+        if (canvas.width !== frame.width || canvas.height !== frame.height) {
+          canvas.width = frame.width;
+          canvas.height = frame.height;
+        }
+        ctx.drawImage(frame.canvas, 0, 0);
+      })
+      .catch(() => {})
+      .finally(() => {
+        inFlightRef.current = false;
+      });
+  });
+}

package/src/hooks/useIkController.ts

@@ -97,6 +97,9 @@ export const useIkController = createControllerHook<IkConfig, IkContextValue>(
           {
             damping: config.damping,
             epsilon: config.epsilon,
+            jointLimits: config.jointLimits ?? controlGroup.joints.map((joint) => (
+              joint.limited ? joint.range : joint.ctrlRange
+            )),
             maxIterations: config.maxIterations,
             posWeight: config.posWeight,
             rotWeight: config.rotWeight,

package/src/hooks/usePolicyCameraTensors.ts

@@ -0,0 +1,215 @@
+/**
+ * @license
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Capture policy observation tensors directly from Three/MuJoCo cameras,
+ * skipping the data-URL/PNG round-trip. Sessions are created once per camera
+ * and reused every step, so live inference and dataset recording read straight
+ * from the GPU into Float32 tensors.
+ */
+
+import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
+import { useMujoco } from '../core/MujocoSimProvider';
+import { createPolicyCameraFrameCapturePlanFromApi } from '../policyCameraFrames';
+import type { MountedPolicyCameraFrameCaptureOptions } from './usePolicyCameraFrames';
+import type {
+  CameraFrameCaptureSession,
+  CameraFrameCaptureTensorOptions,
+  CameraFrameTensorResult,
+} from '../rendering/cameraFrameCapture';
+import type { FrameCaptureStatus } from '../types';
+
+export interface PolicyCameraTensorStream extends CameraFrameCaptureTensorOptions {
+  /** Payload key this stream's tensor is stored under. */
+  key: string;
+  /** Additional payload keys that should reference the same tensor. */
+  aliases?: readonly string[];
+}
+
+export interface PolicyCameraTensorsOptions {
+  streams: PolicyCameraTensorStream[];
+  /** Also expose tensors under `observation.images.<key>` aliases. Defaults to `false`. */
+  includeObservationImageAliases?: boolean;
+}
+
+export interface PolicyCameraTensorsResult {
+  tensors: Record<string, CameraFrameTensorResult>;
+  sourceSummary: string;
+  capturedAt: number;
+}
+
+export interface PolicyCameraTensorsAPI {
+  status: FrameCaptureStatus;
+  error: Error | null;
+  isCapturing: boolean;
+  /** Synchronously render and convert every stream into a policy image tensor. */
+  capture: () => PolicyCameraTensorsResult;
+  reset: () => void;
+}
+
+export type MountedPolicyCameraTensorOptions = MountedPolicyCameraFrameCaptureOptions & {
+  tensor?: Pick<
+    CameraFrameCaptureTensorOptions,
+    'width' | 'height' | 'channels' | 'layout' | 'range'
+  >;
+};
+
+type SessionEntry = {
+  session: CameraFrameCaptureSession;
+  signature: string;
+};
+
+function sessionSignature(stream: PolicyCameraTensorStream): string {
+  return JSON.stringify({
+    width: stream.width,
+    height: stream.height,
+    channels: stream.channels,
+    renderIsolation: stream.renderIsolation ?? false,
+    cameraName: stream.cameraName,
+    siteName: stream.siteName,
+    bodyName: stream.bodyName,
+  });
+}
+
+function addTensorAliases(
+  tensors: Record<string, CameraFrameTensorResult>,
+  stream: PolicyCameraTensorStream,
+  tensor: CameraFrameTensorResult,
+  includeObservationImageAliases: boolean
+) {
+  const keys = new Set<string>([stream.key, ...(stream.aliases ?? [])]);
+  if (includeObservationImageAliases) {
+    for (const base of [stream.key, ...(stream.aliases ?? [])]) {
+      keys.add(`observation.images.${base}`);
+    }
+  }
+  for (const key of keys) tensors[key] = tensor;
+}
+
+export function usePolicyCameraTensors(
+  options: PolicyCameraTensorsOptions
+): PolicyCameraTensorsAPI {
+  const mujoco = useMujoco();
+  const [status, setStatus] = useState<FrameCaptureStatus>('idle');
+  const [error, setError] = useState<Error | null>(null);
+  const sessionsRef = useRef<Map<string, SessionEntry>>(new Map());
+
+  const disposeSessions = useCallback(() => {
+    for (const { session } of sessionsRef.current.values()) session.dispose();
+    sessionsRef.current.clear();
+  }, []);
+
+  useEffect(() => disposeSessions, [disposeSessions]);
+
+  const reset = useCallback(() => {
+    setStatus('idle');
+    setError(null);
+  }, []);
+
+  const capture = useCallback((): PolicyCameraTensorsResult => {
+    const api = mujoco.api;
+    if (!api) {
+      throw new Error('MuJoCo scene is not ready for policy camera tensor capture.');
+    }
+
+    setStatus('capturing');
+    setError(null);
+
+    try {
+      const sessions = sessionsRef.current;
+      const seen = new Set<string>();
+      const tensors: Record<string, CameraFrameTensorResult> = {};
+      const sourceParts: string[] = [];
+
+      for (const stream of options.streams) {
+        seen.add(stream.key);
+        const resolved: CameraFrameCaptureTensorOptions = {
+          ...api.resolveCameraCaptureOptions(stream),
+          channels: stream.channels,
+          layout: stream.layout,
+          range: stream.range,
+        };
+
+        const signature = sessionSignature(stream);
+        let entry = sessions.get(stream.key);
+        if (!entry || entry.signature !== signature) {
+          entry?.session.dispose();
+          entry = {
+            session: api.createCameraFrameCaptureSession(resolved),
+            signature,
+          };
+          sessions.set(stream.key, entry);
+        }
+
+        const tensor = entry.session.captureTensor(resolved);
+        addTensorAliases(
+          tensors,
+          stream,
+          tensor,
+          options.includeObservationImageAliases ?? false
+        );
+        sourceParts.push(`${stream.key}:${tensor.source.kind}`);
+      }
+
+      // Drop sessions for streams that are no longer requested.
+      for (const key of [...sessions.keys()]) {
+        if (!seen.has(key)) {
+          sessions.get(key)?.session.dispose();
+          sessions.delete(key);
+        }
+      }
+
+      setStatus('captured');
+      return {
+        tensors,
+        sourceSummary: sourceParts.join(' + ') || 'not used by policy',
+        capturedAt: Date.now(),
+      };
+    } catch (nextError) {
+      const captureError =
+        nextError instanceof Error
+          ? nextError
+          : new Error('Unable to capture policy camera tensors.');
+      setError(captureError);
+      setStatus('error');
+      throw captureError;
+    }
+  }, [mujoco.api, options.includeObservationImageAliases, options.streams]);
+
+  return {
+    status,
+    error,
+    isCapturing: status === 'capturing',
+    capture,
+    reset,
+  };
+}
+
+export function usePolicyCameraTensorsFromMountedStreams(
+  options: MountedPolicyCameraTensorOptions
+): PolicyCameraTensorsAPI {
+  const mujoco = useMujoco();
+  const tensorOptions = options.tensor;
+  const mountedOptions = useMemo<PolicyCameraTensorsOptions>(() => {
+    const api = mujoco.api;
+    if (!api) {
+      return {
+        streams: [],
+        includeObservationImageAliases: options.includeObservationImageAliases ?? false,
+      };
+    }
+
+    const plan = createPolicyCameraFrameCapturePlanFromApi(api, options);
+    return {
+      streams: plan.streams.map(({ key, aliases, ...stream }) => ({
+        ...stream,
+        ...tensorOptions,
+        key,
+        aliases,
+      })),
+      includeObservationImageAliases: plan.includeObservationImageAliases ?? false,
+    };
+  }, [mujoco.api, options, tensorOptions]);
+
+  return usePolicyCameraTensors(mountedOptions);
+}

package/src/index.ts

@@ -34,6 +34,8 @@ export type { ControllerOptions, ControllerComponent } from './core/createContro
 
 // IK controller hook
 export { useIkController } from './hooks/useIkController';
+export { GenericIK } from './core/GenericIK';
+export type { GenericIKOptions } from './core/GenericIK';
 
 // Components
 export { Body } from './components/Body';
@@ -74,6 +76,10 @@ export type {
   SplatCollisionProxyPreviewVector3,
   UseSplatCollisionProxyGeomsOptions,
 } from './components/SplatCollisionProxyPreview';
+export { CameraView, useCameraViewport } from './components/CameraView';
+export type { CameraViewProps, CameraViewportOptions } from './components/CameraView';
+export { useCameraStream } from './hooks/useCameraStream';
+export type { CameraStreamOptions } from './hooks/useCameraStream';
 export { Debug } from './components/Debug';
 export { TendonRenderer } from './components/TendonRenderer';
 export { FlexRenderer } from './components/FlexRenderer';
@@ -91,6 +97,13 @@ export { useBodyState } from './hooks/useBodyState';
 export { useBodyPose, useGeomPose, useSitePose } from './hooks/usePose';
 export type { PoseReadout, PoseResourceKind } from './hooks/usePose';
 export { useCtrl } from './hooks/useCtrl';
+export { controlGroup, useControlGroup } from './hooks/useControlGroup';
+export type {
+  ControlGroup,
+  ControlGroupHandle,
+  ControlGroupSetOptions,
+  UseControlGroupOptions,
+} from './hooks/useControlGroup';
 export { useControlWriter } from './hooks/useControlWriter';
 export type {
   ControlWriterConflict,
@@ -130,6 +143,17 @@ export type {
   MountedPolicyCameraFrameCaptureAPI,
   MountedPolicyCameraFrameCaptureOptions,
 } from './hooks/usePolicyCameraFrames';
+export {
+  usePolicyCameraTensors,
+  usePolicyCameraTensorsFromMountedStreams,
+} from './hooks/usePolicyCameraTensors';
+export type {
+  MountedPolicyCameraTensorOptions,
+  PolicyCameraTensorsAPI,
+  PolicyCameraTensorsOptions,
+  PolicyCameraTensorsResult,
+  PolicyCameraTensorStream,
+} from './hooks/usePolicyCameraTensors';
 export { useCameraSequenceRecorder } from './hooks/useCameraSequenceRecorder';
 export { useMountedCameraSequenceRecorder } from './hooks/useMountedCameraSequenceRecorder';
 export type {
@@ -145,9 +169,16 @@ export {
   CAPTURE_EXCLUDE_KEY,
   captureCameraFrame,
   captureCameraFrameBlob,
+  captureCameraFrameTensor,
   createCameraFrameCaptureSession,
   renderCameraFrameToCanvas,
 } from './rendering/cameraFrameCapture';
+export type {
+  CameraFrameCaptureSession,
+  CameraFrameCaptureTensorOptions,
+  CameraFramePixelsResult,
+  CameraFrameTensorResult,
+} from './rendering/cameraFrameCapture';
 export {
   imagePointToNdc,
   projectImagePointTo3D,
@@ -172,6 +203,11 @@ export {
   readNamedObservation,
   sitePositionField,
 } from './policyObservation';
+export {
+  dataUrlToPolicyImageTensor,
+  imageDataToPolicyImageTensor,
+  pixelsToPolicyImageTensor,
+} from './policyImageTensors';
 export type {
   CreatePolicyCameraFrameCapturePlanOptions,
   PolicyCameraFrameCapturePlan,
@@ -191,6 +227,14 @@ export type {
   NamedObservationOptions,
   NamedObservationResult,
 } from './policyObservation';
+export type {
+  PolicyImageTensorLayout,
+  PolicyImageTensorOptions,
+  PolicyImageTensorPixelOptions,
+  PolicyImageTensorRange,
+  PolicyImageTensorResult,
+  PolicyImageTensorSourceOrigin,
+} from './policyImageTensors';
 export {
   createMountedCameraFrameSequenceManifest,
   createMountedCameraFrameSequenceReadiness,