mujoco-react 9.4.0 → 9.6.0

package/src/components/Debug.tsx

@@ -11,6 +11,7 @@ import type { ThreeElements } from '@react-three/fiber';
 import * as THREE from 'three';
 import { useMujocoContext } from '../core/MujocoSimProvider';
 import { getName } from '../core/SceneLoader';
+import { CAPTURE_EXCLUDE_KEY } from '../rendering/cameraFrameCapture';
 import { getContact, withContacts } from '../types';
 import type { DebugProps } from '../types';
 
@@ -25,9 +26,20 @@ const JOINT_COLORS: Record<number, number> = {
 const _v3a = new THREE.Vector3();
 const _v3b = new THREE.Vector3();
 const _quat = new THREE.Quaternion();
+const _cameraMatrix = new THREE.Matrix4();
 const _contactPos = new THREE.Vector3();
 const _contactNormal = new THREE.Vector3();
 const MAX_CONTACT_ARROWS = 50;
+const CAMERA_DEBUG_LENGTH = 0.12;
+const CAMERA_DEBUG_FRUSTUM_DEPTH = 0.08;
+
+type CameraDebugObject = THREE.Group & {
+  userData: {
+    cameraId: number;
+    frustum: THREE.LineSegments;
+    label?: THREE.Sprite;
+  };
+};
 
 /**
  * Declarative debug visualization component.
@@ -37,6 +49,7 @@ export function Debug({
   showGeoms = false,
   showSites = false,
   showJoints = false,
+  showCameras = false,
   showContacts = false,
   showCOM = false,
   showInertia = false,
@@ -55,6 +68,7 @@ export function Debug({
     const geoms: THREE.Object3D[] = [];
     const sites: THREE.Object3D[] = [];
     const joints: THREE.Object3D[] = [];
+    const cameras: CameraDebugObject[] = [];
     const comMarkers: THREE.Object3D[] = [];
 
     // Wireframe geoms
@@ -179,6 +193,83 @@ export function Debug({
       }
     }
 
+    if (showCameras && model.ncam && model.name_camadr) {
+      for (let i = 0; i < model.ncam; i++) {
+        const group = new THREE.Group() as CameraDebugObject;
+        group.userData.cameraId = i;
+        group.renderOrder = 999;
+        group.frustumCulled = false;
+
+        const marker = new THREE.Mesh(
+          new THREE.BoxGeometry(0.014, 0.009, 0.006),
+          new THREE.MeshBasicMaterial({ color: 0x38bdf8, depthTest: false })
+        );
+        marker.renderOrder = 999;
+        marker.frustumCulled = false;
+        group.add(marker);
+
+        const forward = new THREE.ArrowHelper(
+          new THREE.Vector3(0, 0, -1),
+          new THREE.Vector3(),
+          CAMERA_DEBUG_LENGTH,
+          0x38bdf8,
+          CAMERA_DEBUG_LENGTH * 0.24,
+          CAMERA_DEBUG_LENGTH * 0.11
+        );
+        forward.renderOrder = 999;
+        forward.frustumCulled = false;
+        forward.line.material = new THREE.LineBasicMaterial({
+          color: 0x38bdf8,
+          depthTest: false,
+        });
+        (forward.cone.material as THREE.MeshBasicMaterial).depthTest = false;
+        group.add(forward);
+
+        const frustumGeometry = new THREE.BufferGeometry();
+        frustumGeometry.setAttribute(
+          'position',
+          new THREE.Float32BufferAttribute(new Float32Array(8 * 2 * 3), 3)
+        );
+        const frustum = new THREE.LineSegments(
+          frustumGeometry,
+          new THREE.LineBasicMaterial({
+            color: 0x38bdf8,
+            transparent: true,
+            opacity: 0.8,
+            depthTest: false,
+          })
+        );
+        frustum.renderOrder = 999;
+        frustum.frustumCulled = false;
+        group.userData.frustum = frustum;
+        group.add(frustum);
+
+        const canvas = document.createElement('canvas');
+        canvas.width = 256;
+        canvas.height = 64;
+        const ctx = canvas.getContext('2d')!;
+        ctx.fillStyle = '#38bdf8';
+        ctx.font = 'bold 32px monospace';
+        ctx.textAlign = 'center';
+        ctx.fillText(getName(model, model.name_camadr[i]), 128, 42);
+        const texture = new THREE.CanvasTexture(canvas);
+        const sprite = new THREE.Sprite(
+          new THREE.SpriteMaterial({
+            map: texture,
+            depthTest: false,
+            transparent: true,
+          })
+        );
+        sprite.position.set(0, 0.014, 0.01);
+        sprite.scale.set(0.04, 0.01, 1);
+        sprite.renderOrder = 999;
+        group.userData.label = sprite;
+        group.add(sprite);
+
+        cameras.push(group);
+      }
+    }
+
     // COM markers
     if (showCOM) {
       for (let i = 1; i < model.nbody; i++) {
@@ -190,8 +281,8 @@ export function Debug({
       }
     }
 
-    return { geoms, sites, joints, comMarkers };
-  }, [status, mjModelRef, showGeoms, showSites, showJoints, showCOM]);
+    return { geoms, sites, joints, cameras, comMarkers };
+  }, [status, mjModelRef, showGeoms, showSites, showJoints, showCameras, showCOM]);
 
   // Add/remove debug objects from scene
   useEffect(() => {
@@ -202,6 +293,7 @@ export function Debug({
       ...debugGeometry.geoms,
       ...debugGeometry.sites,
       ...debugGeometry.joints,
+      ...debugGeometry.cameras,
       ...debugGeometry.comMarkers,
     ];
     for (const obj of allObjects) group.add(obj);
@@ -281,6 +373,59 @@ export function Debug({
       }
     }
 
+    const camXpos = data.cam_xpos;
+    const camXmat = data.cam_xmat;
+    if (camXpos && camXmat) {
+      for (const group of debugGeometry.cameras) {
+        const cameraId = group.userData.cameraId;
+        const i3 = cameraId * 3;
+        const i9 = cameraId * 9;
+        group.position.set(
+          camXpos[i3],
+          camXpos[i3 + 1],
+          camXpos[i3 + 2]
+        );
+        _cameraMatrix.set(
+          camXmat[i9], camXmat[i9 + 1], camXmat[i9 + 2], 0,
+          camXmat[i9 + 3], camXmat[i9 + 4], camXmat[i9 + 5], 0,
+          camXmat[i9 + 6], camXmat[i9 + 7], camXmat[i9 + 8], 0,
+          0, 0, 0, 1
+        );
+        group.quaternion.setFromRotationMatrix(_cameraMatrix);
+
+        const fovy = model.cam_fovy?.[cameraId] ?? 45;
+        const halfHeight = Math.tan(THREE.MathUtils.degToRad(fovy) / 2) *
+          CAMERA_DEBUG_FRUSTUM_DEPTH;
+        const halfWidth = halfHeight * 4 / 3;
+        const positions = group.userData.frustum.geometry.attributes.position;
+        const array = positions.array as Float32Array;
+        const points = [
+          [0, 0, 0],
+          [-halfWidth, halfHeight, -CAMERA_DEBUG_FRUSTUM_DEPTH],
+          [0, 0, 0],
+          [halfWidth, halfHeight, -CAMERA_DEBUG_FRUSTUM_DEPTH],
+          [0, 0, 0],
+          [halfWidth, -halfHeight, -CAMERA_DEBUG_FRUSTUM_DEPTH],
+          [0, 0, 0],
+          [-halfWidth, -halfHeight, -CAMERA_DEBUG_FRUSTUM_DEPTH],
+          [-halfWidth, halfHeight, -CAMERA_DEBUG_FRUSTUM_DEPTH],
+          [halfWidth, halfHeight, -CAMERA_DEBUG_FRUSTUM_DEPTH],
+          [halfWidth, halfHeight, -CAMERA_DEBUG_FRUSTUM_DEPTH],
+          [halfWidth, -halfHeight, -CAMERA_DEBUG_FRUSTUM_DEPTH],
+          [halfWidth, -halfHeight, -CAMERA_DEBUG_FRUSTUM_DEPTH],
+          [-halfWidth, -halfHeight, -CAMERA_DEBUG_FRUSTUM_DEPTH],
+          [-halfWidth, -halfHeight, -CAMERA_DEBUG_FRUSTUM_DEPTH],
+          [-halfWidth, halfHeight, -CAMERA_DEBUG_FRUSTUM_DEPTH],
+        ];
+        for (let i = 0; i < points.length; i += 1) {
+          array[i * 3] = points[i][0];
+          array[i * 3 + 1] = points[i][1];
+          array[i * 3 + 2] = points[i][2];
+        }
+        positions.needsUpdate = true;
+      }
+    }
+
     // Update COM markers
     for (const mesh of debugGeometry.comMarkers) {
       const bid = mesh.userData.bodyId;
@@ -358,7 +503,13 @@ export function Debug({
   if (status !== 'ready') return null;
 
   return (
-    <group {...groupProps}>
+    <group
+      {...groupProps}
+      userData={{
+        ...groupProps.userData,
+        [CAPTURE_EXCLUDE_KEY]: true,
+      }}
+    >
       <group ref={groupRef} />
       {showContacts && <group ref={contactGroupRef} />}
     </group>

package/src/components/DragInteraction.tsx

@@ -8,6 +8,7 @@ import type { ThreeElements } from '@react-three/fiber';
 import { useEffect, useRef } from 'react';
 import * as THREE from 'three';
 import { useMujocoContext, useBeforePhysicsStep } from '../core/MujocoSimProvider';
+import { CAPTURE_EXCLUDE_KEY } from '../rendering/cameraFrameCapture';
 import type { DragInteractionProps } from '../types';
 
 // Preallocated temps to avoid GC pressure
@@ -60,6 +61,7 @@ export function DragInteraction({
       0.1,
       0xff4444,
     );
+    arrow.userData[CAPTURE_EXCLUDE_KEY] = true;
     arrow.visible = false;
     // Make arrow semi-transparent
     (arrow.line.material as THREE.LineBasicMaterial).transparent = true;

package/src/components/IkGizmo.tsx

@@ -9,6 +9,7 @@ import { useEffect, useRef } from 'react';
 import * as THREE from 'three';
 import { useMujocoContext } from '../core/MujocoSimProvider';
 import { findSiteByName } from '../core/SceneLoader';
+import { CAPTURE_EXCLUDE_KEY } from '../rendering/cameraFrameCapture';
 import type { IkGizmoProps } from '../types';
 
 // Preallocated temps to avoid GC pressure in useFrame
@@ -83,7 +84,10 @@ export function IkGizmo({ controller, siteName, scale = 0.18, onDrag }: IkGizmoP
   if (status !== 'ready') return null;
 
   return (
-    <group ref={wrapperRef}>
+    <group
+      ref={wrapperRef}
+      userData={{ [CAPTURE_EXCLUDE_KEY]: true }}
+    >
       <PivotControls
         ref={pivotRef}
         autoTransform

package/src/core/MujocoSimProvider.tsx

@@ -148,12 +148,12 @@ function vector3FromArray(values: ArrayLike<number>, offset: number): [number, n
   return [values[offset], values[offset + 1], values[offset + 2]];
 }
 
-function quaternionFromArray(values: ArrayLike<number>, offset: number): [number, number, number, number] {
+function quaternionFromMujocoQuat(values: ArrayLike<number>, offset: number): [number, number, number, number] {
   return [
-    values[offset],
     values[offset + 1],
     values[offset + 2],
     values[offset + 3],
+    values[offset],
   ];
 }
 
@@ -993,7 +993,7 @@ export function MujocoSimProvider({
           ? vector3FromArray(model.cam_pos, posOffset)
           : null,
         quaternion: model.cam_quat
-          ? quaternionFromArray(model.cam_quat, quatOffset)
+          ? quaternionFromMujocoQuat(model.cam_quat, quatOffset)
           : null,
       });
     }
@@ -1024,7 +1024,7 @@ export function MujocoSimProvider({
         const quaternion = data.cam_xmat
           ? quaternionFromXmat(data.cam_xmat, cameraId * 9)
           : model.cam_quat
-            ? quaternionFromArray(model.cam_quat, cameraId * 4)
+            ? quaternionFromMujocoQuat(model.cam_quat, cameraId * 4)
             : undefined;
 
         if (!position || !quaternion) {
@@ -1442,7 +1442,7 @@ export function MujocoSimProvider({
           const cameraFrames: Record<string, CameraFrameCaptureResult> = {};
           for (const { key, captureOptions, mountedSource, session } of captureSessions) {
             const resolvedCaptureOptions = resolveCameraCaptureOptions(captureOptions);
-            const cameraFrame = session.captureDataUrl({
+            const cameraFrame = await session.captureDataUrlAsync({
               ...resolvedCaptureOptions,
               source: mountedSource ?? resolvedCaptureOptions.source,
             });

package/src/hooks/useIkController.ts

@@ -94,7 +94,14 @@ export const useIkController = createControllerHook<IkConfig, IkContextValue>(
         return genericIkRef.current.solve(
           model, data, siteIdRef.current, controlGroup.qposAdr,
           position, quaternion, currentQ,
-          { damping: config.damping, maxIterations: config.maxIterations },
+          {
+            damping: config.damping,
+            epsilon: config.epsilon,
+            maxIterations: config.maxIterations,
+            posWeight: config.posWeight,
+            rotWeight: config.rotWeight,
+            tolerance: config.tolerance,
+          },
         );
       },
       [config, mjModelRef, mjDataRef],

package/src/hooks/useKeyboardIkTarget.ts

@@ -0,0 +1,170 @@
+/**
+ * @license
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { useFrame } from '@react-three/fiber';
+import { useEffect, useMemo, useRef } from 'react';
+import * as THREE from 'three';
+import type { KeyboardIkTargetAction, KeyboardIkTargetBinding, KeyboardIkTargetConfig } from '../types';
+
+const DEFAULT_TRANSLATE_SPEED = 0.25;
+const DEFAULT_ROTATE_SPEED = 1.0;
+
+const _translation = new THREE.Vector3();
+const _axis = new THREE.Vector3();
+const _quat = new THREE.Quaternion();
+
+function actionSign(action: KeyboardIkTargetAction): 1 | -1 {
+  return action.endsWith('+') ? 1 : -1;
+}
+
+function actionBase(action: KeyboardIkTargetAction) {
+  return action.slice(0, -1);
+}
+
+function applyRotation(
+  target: THREE.Object3D,
+  action: KeyboardIkTargetAction,
+  amount: number,
+  frame: 'world' | 'target',
+) {
+  const base = actionBase(action);
+  if (base === 'pitch') {
+    _axis.set(1, 0, 0);
+  } else if (base === 'yaw') {
+    _axis.set(0, 1, 0);
+  } else if (base === 'roll') {
+    _axis.set(0, 0, 1);
+  } else {
+    return;
+  }
+
+  _quat.setFromAxisAngle(_axis, amount);
+  if (frame === 'target') {
+    target.quaternion.multiply(_quat);
+  } else {
+    target.quaternion.premultiply(_quat);
+  }
+}
+
+function addTranslation(action: KeyboardIkTargetAction, amount: number) {
+  switch (action) {
+    case 'x+':
+      _translation.x += amount;
+      break;
+    case 'x-':
+      _translation.x -= amount;
+      break;
+    case 'y+':
+      _translation.y += amount;
+      break;
+    case 'y-':
+      _translation.y -= amount;
+      break;
+    case 'z+':
+      _translation.z += amount;
+      break;
+    case 'z-':
+      _translation.z -= amount;
+      break;
+  }
+}
+
+/**
+ * Moves an existing IK target from keyboard input.
+ *
+ * This hook is intentionally robot-agnostic: it only edits the target owned by
+ * `useIkController`, then lets the normal IK solver write robot controls.
+ */
+export function useKeyboardIkTarget(config: KeyboardIkTargetConfig | null) {
+  const pressedRef = useRef(new Set<string>());
+  const wasActiveRef = useRef(false);
+  const configRef = useRef(config);
+  configRef.current = config;
+
+  const boundCodes = useMemo(() => {
+    return new Set((config?.bindings ?? []).map((binding) => binding.code));
+  }, [config?.bindings]);
+
+  useEffect(() => {
+    const onKeyDown = (event: KeyboardEvent) => {
+      const current = configRef.current;
+      if (!current || current.enabled === false || !boundCodes.has(event.code)) return;
+      if (current.preventDefault !== false) event.preventDefault();
+      pressedRef.current.add(event.code);
+    };
+    const onKeyUp = (event: KeyboardEvent) => {
+      const current = configRef.current;
+      if (boundCodes.has(event.code) && current?.preventDefault !== false) {
+        event.preventDefault();
+      }
+      pressedRef.current.delete(event.code);
+    };
+    const onBlur = () => {
+      pressedRef.current.clear();
+      wasActiveRef.current = false;
+    };
+
+    window.addEventListener('keydown', onKeyDown);
+    window.addEventListener('keyup', onKeyUp);
+    window.addEventListener('blur', onBlur);
+
+    return () => {
+      window.removeEventListener('keydown', onKeyDown);
+      window.removeEventListener('keyup', onKeyUp);
+      window.removeEventListener('blur', onBlur);
+    };
+  }, [boundCodes]);
+
+  useFrame((_state, delta) => {
+    const current = configRef.current;
+    const controller = current?.controller;
+    if (!current || current.enabled === false || !controller) {
+      wasActiveRef.current = false;
+      return;
+    }
+
+    const activeBindings: KeyboardIkTargetBinding[] = [];
+    for (const binding of current.bindings) {
+      if (pressedRef.current.has(binding.code)) activeBindings.push(binding);
+    }
+
+    if (activeBindings.length === 0) {
+      wasActiveRef.current = false;
+      return;
+    }
+
+    if (!wasActiveRef.current) {
+      if (current.syncOnStart !== false) controller.syncTargetToSite();
+      if (current.autoEnableIk !== false && !controller.ikEnabledRef.current) {
+        controller.setIkEnabled(true);
+      }
+    }
+    wasActiveRef.current = true;
+
+    const target = controller.ikTargetRef.current;
+    if (!target) return;
+
+    const frame = current.frame ?? 'world';
+    _translation.set(0, 0, 0);
+
+    for (const binding of activeBindings) {
+      const translateSpeed = binding.translateSpeed ?? current.translateSpeed ?? DEFAULT_TRANSLATE_SPEED;
+      const rotateSpeed = binding.rotateSpeed ?? current.rotateSpeed ?? DEFAULT_ROTATE_SPEED;
+      const amount = actionSign(binding.action) * delta;
+      const base = actionBase(binding.action);
+
+      if (base === 'x' || base === 'y' || base === 'z') {
+        addTranslation(binding.action, translateSpeed * delta);
+      } else {
+        applyRotation(target, binding.action, rotateSpeed * amount, frame);
+      }
+    }
+
+    if (_translation.lengthSq() > 0) {
+      if (frame === 'target') _translation.applyQuaternion(target.quaternion);
+      target.position.add(_translation);
+    }
+  });
+}

package/src/index.ts

@@ -91,6 +91,7 @@ export { useBodyState } from './hooks/useBodyState';
 export { useCtrl } from './hooks/useCtrl';
 export { useContacts, useContactEvents } from './hooks/useContacts';
 export { useKeyboardTeleop } from './hooks/useKeyboardTeleop';
+export { useKeyboardIkTarget } from './hooks/useKeyboardIkTarget';
 export { usePolicy } from './hooks/usePolicy';
 export { useObservation } from './hooks/useObservation';
 export { useTrajectoryPlayer } from './hooks/useTrajectoryPlayer';
@@ -113,6 +114,7 @@ export type {
   MountedCameraSequenceRecordResult,
 } from './hooks/useMountedCameraSequenceRecorder';
 export {
+  CAPTURE_EXCLUDE_KEY,
   captureCameraFrame,
   captureCameraFrameBlob,
   createCameraFrameCaptureSession,
@@ -210,6 +212,9 @@ export type {
   // Keyboard teleop
   KeyBinding,
   KeyboardTeleopConfig,
+  KeyboardIkTargetAction,
+  KeyboardIkTargetBinding,
+  KeyboardIkTargetConfig,
   // Policy
   PolicyConfig,
   PolicyVector,