mujoco-react 10.3.0 → 10.4.0

package/dist/spark.d.ts

@@ -1,6 +1,6 @@
 import * as react_jsx_runtime from 'react/jsx-runtime';
 import * as _sparkjsdev_spark from '@sparkjsdev/spark';
-import { o as SplatEnvironmentProps, r as PairedSplatEnvironmentConfig, S as SceneConfig, u as SplatEnvironmentReadiness, p as VisualScenarioConfig } from './types-BOhNDICK.js';
+import { o as SplatEnvironmentProps, r as PairedSplatEnvironmentConfig, S as SceneConfig, u as SplatEnvironmentReadiness, p as VisualScenarioConfig } from './types-CdFZCYmy.js';
 import 'react';
 import '@react-three/fiber';
 import 'three';

package/dist/spark.js

@@ -1,4 +1,4 @@
-import { useSplatSceneConfig, useSplatEnvironment, SplatEnvironment, CAMERA_FRAME_CAPTURE_RENDER_USER_DATA_KEY } from './chunk-6AZEFI6A.js';
+import { useSplatSceneConfig, useSplatEnvironment, SplatEnvironment, CAMERA_FRAME_CAPTURE_RENDER_USER_DATA_KEY } from './chunk-FBXXXPLQ.js';
 import { useThree } from '@react-three/fiber';
 import { useMemo, useState, useEffect, useCallback, useRef } from 'react';
 import * as THREE from 'three';

package/dist/{types-BOhNDICK.d.ts → types-CdFZCYmy.d.ts}

@@ -801,11 +801,27 @@ interface ObservationHandle {
     /** Read just the vector values for policy inference. */
     readValues(): Float32Array | Float64Array;
 }
+interface DebugVirtualCamera {
+    name?: string;
+    position?: CameraFrameCaptureVector3;
+    lookAt?: CameraFrameCaptureVector3;
+    up?: CameraFrameCaptureVector3;
+    quaternion?: THREE.Quaternion | readonly [number, number, number, number];
+    fov?: number;
+    width?: number;
+    height?: number;
+    frustumDepth?: number;
+    markerScale?: number;
+    color?: THREE.ColorRepresentation;
+    aimColor?: THREE.ColorRepresentation;
+}
 interface DebugProps {
     showGeoms?: boolean;
     showSites?: boolean;
     showJoints?: boolean;
     showCameras?: boolean;
+    /** Additional explicit virtual camera poses to visualize alongside MuJoCo XML cameras. */
+    virtualCameras?: readonly DebugVirtualCamera[];
     showContacts?: boolean;
     showCOM?: boolean;
     showInertia?: boolean;
@@ -1482,4 +1498,4 @@ interface ArrayJointStateResult {
     velocity: React__default.RefObject<Float64Array>;
 }
 
-export { type ScalarJointStateResult as $, type ActuatedJointInfo as A, type BodyProps as B, type ControlGroupInfo as C, type DragInteractionProps as D, type VisualScenarioExecutionContext as E, type ScenarioLightingPreset as F, type SplatEnvironmentMetadataInput as G, type SplatEnvironmentMetadata as H, type IkConfig as I, type SplatSceneInput as J, type DebugProps as K, type GeomInfo as L, type MujocoContextValue as M, type ContactListenerProps as N, type ObservationConfig as O, type PhysicsStepCallback as P, type ActuatorInfo as Q, type ReadyCallbackInput as R, type SceneConfig as S, type TrajectoryPlayerProps as T, type Sites as U, type VisualScenarioEffectsProps as V, type SitePositionResult as W, type Sensors as X, type SensorHandle as Y, type SensorInfo as Z, type Joints as _, type MujocoCanvasProps as a, type Keyframes as a$, type ArrayJointStateResult as a0, type JointStateOptions as a1, type JointStateResult as a2, type Bodies as a3, type BodyStateResult as a4, type Geoms as a5, type Actuators as a6, type CtrlHandle as a7, type ContactInfo as a8, type KeyboardTeleopConfig as a9, type PolicyCameraFrameCaptureAPI as aA, type CameraFrameSequenceRecorderAPI as aB, type CameraFrameCaptureResult as aC, type CameraFrameCaptureBlobResult as aD, type ImagePointCoordinateSpace as aE, type ImagePointProjectionOptions as aF, type ImagePointProjectionResult as aG, type PolicyVector as aH, type BodyInfo as aI, type CameraFrameCaptureQuaternion as aJ, type CameraFrameCaptureVector3 as aK, type CameraFrameSequenceCameraSummary as aL, type CameraFrameSequenceFrame as aM, type CameraFrameSequenceSampleInput as aN, type CameraFrameSequenceStepInput as aO, type CameraInfo as aP, type ControlJointInfo as aQ, type FrameCaptureTarget as aR, type FrameCaptureTargetRef as aS, type IKSolveFn as aT, type IkGizmoDragInput as aU, type IkSolveInput as aV, type JointInfo as aW, type JointStateKind as aX, type KeyBinding as aY, type KeyboardIkTargetAction as aZ, type KeyboardIkTargetBinding as a_, type KeyboardIkTargetConfig as aa, type PolicyConfig as ab, type PolicyAPI as ac, type RemotePolicyConfig as ad, type RemotePolicyAPI as ae, type ObservationHandle as af, type ObservationOutput as ag, type TrajectoryInput as ah, type TrajectoryStateChangeInput as ai, type PlaybackState as aj, type TrajectoryFrame as ak, type FrameCaptureOptions as al, type FrameCaptureResult as am, type FrameCaptureBlobResult as an, type FrameCaptureAPI as ao, type CameraFrameCaptureOptions as ap, type CameraFrameCaptureAPI as aq, type Cameras as ar, type CameraFrameSequenceCamera as as, type CameraFrameCaptureSource as at, type CameraFrameSequenceOptions as au, type CameraFrameSequenceResult as av, type PolicyCameraFrameStream as aw, type PolicyCameraFrameCaptureOptions as ax, type PolicyCameraFrameCaptureResult as ay, type FrameCaptureStatus as az, type MujocoSimAPI as b, ModelActuators as b0, ModelBodies as b1, ModelCameras as b2, ModelGeoms as b3, ModelJoints as b4, ModelKeyframes as b5, type ModelOptions as b6, type ModelResource as b7, ModelResources as b8, ModelSensors as b9, type ScenarioMaterialConfig as bA, type SceneMarker as bB, type SceneObject as bC, type SensorResult as bD, type SiteInfo as bE, type SplatAssetConfig as bF, type SplatScenarioConfig as bG, type StateSnapshot as bH, type TrajectoryData as bI, type TrajectoryFrameCallbackInput as bJ, type VisualScenarioMaterialFilterInput as bK, type XmlPatch as bL, getContact as bM, registerModelResources as bN, ModelSites as ba, type Models as bb, type MujocoContact as bc, type MujocoContactArray as bd, type MujocoFrameCaptureOptions as be, type ObservationLayoutItem as bf, type PhysicsConfig as bg, type PhysicsStepInput as bh, type PolicyActionChunk as bi, type PolicyActionInput as bj, type PolicyInferenceInput as bk, type PolicyInferenceOutput as bl, type PolicyInferenceResult as bm, type PolicyObservationInput as bn, type RayHit as bo, type Register as bp, type RegisteredModelMap as bq, type RemotePolicyRequestInfo as br, type RemotePolicyRequestInput as bs, type RemotePolicyResponseInfo as bt, type RemotePolicyStatus as bu, type ResetCallbackInput as bv, type ResolvedScenarioCameraConfig as bw, type ResolvedScenarioMaterialConfig as bx, type ResourceSelector as by, type ScenarioCameraConfig as bz, type StepCallbackInput as c, type SelectionCallbackInput as d, type MujocoModule as e, type MujocoRenderOptions as f, type MujocoModel as g, type MujocoData as h, type ControlGroupSelector as i, type ObservationResult as j, type IkContextValue as k, type IkGizmoProps as l, type SceneLightsProps as m, type ScenarioLightingProps as n, type SplatEnvironmentProps as o, type VisualScenarioConfig as p, type SplatRendererKind as q, type PairedSplatEnvironmentConfig as r, type SplatFormat as s, type SplatCollisionProxyConfig as t, type SplatEnvironmentReadiness as u, type SplatCollisionPrimitive as v, SplatEnvironmentReadinessStatus as w, type SplatSceneConfigInput as x, type SplatSceneConfigState as y, type VisualScenarioExecutionContextInput as z };
+export { type ScalarJointStateResult as $, type ActuatedJointInfo as A, type BodyProps as B, type ControlGroupInfo as C, type DragInteractionProps as D, type VisualScenarioExecutionContext as E, type ScenarioLightingPreset as F, type SplatEnvironmentMetadataInput as G, type SplatEnvironmentMetadata as H, type IkConfig as I, type SplatSceneInput as J, type DebugProps as K, type GeomInfo as L, type MujocoContextValue as M, type ContactListenerProps as N, type ObservationConfig as O, type PhysicsStepCallback as P, type ActuatorInfo as Q, type ReadyCallbackInput as R, type SceneConfig as S, type TrajectoryPlayerProps as T, type Sites as U, type VisualScenarioEffectsProps as V, type SitePositionResult as W, type Sensors as X, type SensorHandle as Y, type SensorInfo as Z, type Joints as _, type MujocoCanvasProps as a, type KeyboardIkTargetBinding as a$, type ArrayJointStateResult as a0, type JointStateOptions as a1, type JointStateResult as a2, type Bodies as a3, type BodyStateResult as a4, type Geoms as a5, type Actuators as a6, type CtrlHandle as a7, type ContactInfo as a8, type KeyboardTeleopConfig as a9, type PolicyCameraFrameCaptureAPI as aA, type CameraFrameSequenceRecorderAPI as aB, type CameraFrameCaptureResult as aC, type CameraFrameCaptureBlobResult as aD, type ImagePointCoordinateSpace as aE, type ImagePointProjectionOptions as aF, type ImagePointProjectionResult as aG, type PolicyVector as aH, type BodyInfo as aI, type CameraFrameCaptureQuaternion as aJ, type CameraFrameCaptureVector3 as aK, type CameraFrameSequenceCameraSummary as aL, type CameraFrameSequenceFrame as aM, type CameraFrameSequenceSampleInput as aN, type CameraFrameSequenceStepInput as aO, type CameraInfo as aP, type ControlJointInfo as aQ, type DebugVirtualCamera as aR, type FrameCaptureTarget as aS, type FrameCaptureTargetRef as aT, type IKSolveFn as aU, type IkGizmoDragInput as aV, type IkSolveInput as aW, type JointInfo as aX, type JointStateKind as aY, type KeyBinding as aZ, type KeyboardIkTargetAction as a_, type KeyboardIkTargetConfig as aa, type PolicyConfig as ab, type PolicyAPI as ac, type RemotePolicyConfig as ad, type RemotePolicyAPI as ae, type ObservationHandle as af, type ObservationOutput as ag, type TrajectoryInput as ah, type TrajectoryStateChangeInput as ai, type PlaybackState as aj, type TrajectoryFrame as ak, type FrameCaptureOptions as al, type FrameCaptureResult as am, type FrameCaptureBlobResult as an, type FrameCaptureAPI as ao, type CameraFrameCaptureOptions as ap, type CameraFrameCaptureAPI as aq, type Cameras as ar, type CameraFrameSequenceCamera as as, type CameraFrameCaptureSource as at, type CameraFrameSequenceOptions as au, type CameraFrameSequenceResult as av, type PolicyCameraFrameStream as aw, type PolicyCameraFrameCaptureOptions as ax, type PolicyCameraFrameCaptureResult as ay, type FrameCaptureStatus as az, type MujocoSimAPI as b, type Keyframes as b0, ModelActuators as b1, ModelBodies as b2, ModelCameras as b3, ModelGeoms as b4, ModelJoints as b5, ModelKeyframes as b6, type ModelOptions as b7, type ModelResource as b8, ModelResources as b9, type ScenarioCameraConfig as bA, type ScenarioMaterialConfig as bB, type SceneMarker as bC, type SceneObject as bD, type SensorResult as bE, type SiteInfo as bF, type SplatAssetConfig as bG, type SplatScenarioConfig as bH, type StateSnapshot as bI, type TrajectoryData as bJ, type TrajectoryFrameCallbackInput as bK, type VisualScenarioMaterialFilterInput as bL, type XmlPatch as bM, getContact as bN, registerModelResources as bO, ModelSensors as ba, ModelSites as bb, type Models as bc, type MujocoContact as bd, type MujocoContactArray as be, type MujocoFrameCaptureOptions as bf, type ObservationLayoutItem as bg, type PhysicsConfig as bh, type PhysicsStepInput as bi, type PolicyActionChunk as bj, type PolicyActionInput as bk, type PolicyInferenceInput as bl, type PolicyInferenceOutput as bm, type PolicyInferenceResult as bn, type PolicyObservationInput as bo, type RayHit as bp, type Register as bq, type RegisteredModelMap as br, type RemotePolicyRequestInfo as bs, type RemotePolicyRequestInput as bt, type RemotePolicyResponseInfo as bu, type RemotePolicyStatus as bv, type ResetCallbackInput as bw, type ResolvedScenarioCameraConfig as bx, type ResolvedScenarioMaterialConfig as by, type ResourceSelector as bz, type StepCallbackInput as c, type SelectionCallbackInput as d, type MujocoModule as e, type MujocoRenderOptions as f, type MujocoModel as g, type MujocoData as h, type ControlGroupSelector as i, type ObservationResult as j, type IkContextValue as k, type IkGizmoProps as l, type SceneLightsProps as m, type ScenarioLightingProps as n, type SplatEnvironmentProps as o, type VisualScenarioConfig as p, type SplatRendererKind as q, type PairedSplatEnvironmentConfig as r, type SplatFormat as s, type SplatCollisionProxyConfig as t, type SplatEnvironmentReadiness as u, type SplatCollisionPrimitive as v, SplatEnvironmentReadinessStatus as w, type SplatSceneConfigInput as x, type SplatSceneConfigState as y, type VisualScenarioExecutionContextInput as z };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mujoco-react",
-  "version": "10.3.0",
+  "version": "10.4.0",
   "description": "Composable React Three Fiber building blocks for MuJoCo WASM simulations",
   "type": "module",
   "main": "./dist/index.js",

package/src/components/Debug.tsx

@@ -13,7 +13,7 @@ import { useMujocoContext } from '../core/MujocoSimProvider';
 import { getName } from '../core/SceneLoader';
 import { CAPTURE_EXCLUDE_KEY } from '../rendering/cameraFrameCapture';
 import { getContact, withContacts } from '../types';
-import type { DebugProps } from '../types';
+import type { CameraFrameCaptureVector3, DebugProps, DebugVirtualCamera } from '../types';
 
 const JOINT_COLORS: Record<number, number> = {
   0: 0xff0000, // free - red
@@ -41,6 +41,170 @@ type CameraDebugObject = THREE.Group & {
   };
 };
 
+function toVector3(
+  value: CameraFrameCaptureVector3 | undefined,
+  fallback: THREE.Vector3
+) {
+  if (!value) return fallback.clone();
+  return value instanceof THREE.Vector3
+    ? value.clone()
+    : new THREE.Vector3(value[0], value[1], value[2]);
+}
+
+function createCameraLabel(text: string, color: THREE.ColorRepresentation) {
+  const canvas = document.createElement('canvas');
+  canvas.width = 256;
+  canvas.height = 64;
+  const ctx = canvas.getContext('2d')!;
+  ctx.fillStyle = new THREE.Color(color).getStyle();
+  ctx.font = 'bold 32px monospace';
+  ctx.textAlign = 'center';
+  ctx.fillText(text, 128, 42);
+  const texture = new THREE.CanvasTexture(canvas);
+  const sprite = new THREE.Sprite(
+    new THREE.SpriteMaterial({
+      map: texture,
+      depthTest: false,
+      transparent: true,
+    })
+  );
+  sprite.position.set(0, 0.014, 0.01);
+  sprite.scale.set(0.05, 0.012, 1);
+  sprite.renderOrder = 999;
+  return sprite;
+}
+
+function createVirtualCameraDebugObject(
+  camera: DebugVirtualCamera,
+  index: number
+) {
+  const color = camera.color ?? '#ff3d71';
+  const aimColor = camera.aimColor ?? '#38bdf8';
+  const markerScale = camera.markerScale ?? 1;
+  const cameraPosition = toVector3(camera.position, new THREE.Vector3());
+  const configuredUp = toVector3(camera.up, new THREE.Vector3(0, 0, 1)).normalize();
+  const cameraQuaternion = new THREE.Quaternion();
+  const forward = new THREE.Vector3();
+
+  if (camera.quaternion) {
+    if (camera.quaternion instanceof THREE.Quaternion) {
+      cameraQuaternion.copy(camera.quaternion);
+    } else {
+      cameraQuaternion.set(
+        camera.quaternion[0],
+        camera.quaternion[1],
+        camera.quaternion[2],
+        camera.quaternion[3]
+      );
+    }
+    forward.set(0, 0, -1).applyQuaternion(cameraQuaternion).normalize();
+  } else {
+    const target = toVector3(
+      camera.lookAt,
+      cameraPosition.clone().add(new THREE.Vector3(0, 0, -1))
+    );
+    forward.copy(target).sub(cameraPosition);
+    if (forward.lengthSq() < 1e-8) forward.set(0, 0, -1);
+    forward.normalize();
+    cameraQuaternion.setFromRotationMatrix(
+      new THREE.Matrix4().lookAt(cameraPosition, target, configuredUp)
+    );
+  }
+
+  const target = camera.lookAt
+    ? toVector3(camera.lookAt, cameraPosition.clone().add(forward))
+    : cameraPosition.clone().addScaledVector(forward, 0.4);
+  const distanceToTarget = Math.max(target.distanceTo(cameraPosition), 0.001);
+  const depth = camera.frustumDepth ?? Math.min(Math.max(distanceToTarget * 0.42, 0.16), 0.45);
+  const fov = camera.fov ?? 50;
+  const aspect = (camera.width ?? 640) / (camera.height ?? 480);
+
+  const right = forward.clone().cross(configuredUp);
+  if (right.lengthSq() < 1e-8) right.set(1, 0, 0);
+  right.normalize();
+  const orthogonalUp = right.clone().cross(forward).normalize();
+  const frustumHeight = 2 * Math.tan(THREE.MathUtils.degToRad(fov) / 2) * depth;
+  const frustumWidth = frustumHeight * aspect;
+  const center = cameraPosition.clone().addScaledVector(forward, depth);
+  const halfRight = right.clone().multiplyScalar(frustumWidth / 2);
+  const halfUp = orthogonalUp.clone().multiplyScalar(frustumHeight / 2);
+
+  const topLeft = center.clone().sub(halfRight).add(halfUp);
+  const topRight = center.clone().add(halfRight).add(halfUp);
+  const bottomRight = center.clone().add(halfRight).sub(halfUp);
+  const bottomLeft = center.clone().sub(halfRight).sub(halfUp);
+  const frustumPoints = [
+    cameraPosition, topLeft,
+    cameraPosition, topRight,
+    cameraPosition, bottomRight,
+    cameraPosition, bottomLeft,
+    topLeft, topRight,
+    topRight, bottomRight,
+    bottomRight, bottomLeft,
+    bottomLeft, topLeft,
+  ];
+
+  const group = new THREE.Group();
+  group.name = camera.name ?? `virtual-camera-${index}`;
+  group.renderOrder = 999;
+  group.frustumCulled = false;
+
+  const frustum = new THREE.LineSegments(
+    new THREE.BufferGeometry().setFromPoints(frustumPoints),
+    new THREE.LineBasicMaterial({
+      color,
+      transparent: true,
+      opacity: 0.9,
+      depthTest: false,
+    })
+  );
+  frustum.renderOrder = 999;
+  frustum.frustumCulled = false;
+  group.add(frustum);
+
+  const aim = new THREE.LineSegments(
+    new THREE.BufferGeometry().setFromPoints([cameraPosition, target]),
+    new THREE.LineBasicMaterial({
+      color: aimColor,
+      transparent: true,
+      opacity: 0.95,
+      depthTest: false,
+    })
+  );
+  aim.renderOrder = 999;
+  aim.frustumCulled = false;
+  group.add(aim);
+
+  const markerGroup = new THREE.Group();
+  markerGroup.position.copy(cameraPosition);
+  markerGroup.quaternion.copy(cameraQuaternion);
+  markerGroup.renderOrder = 999;
+  markerGroup.frustumCulled = false;
+  markerGroup.add(new THREE.Mesh(
+    new THREE.BoxGeometry(0.045 * markerScale, 0.028 * markerScale, 0.022 * markerScale),
+    new THREE.MeshBasicMaterial({ color, depthTest: false })
+  ));
+  const lens = new THREE.Mesh(
+    new THREE.BoxGeometry(0.025 * markerScale, 0.018 * markerScale, 0.014 * markerScale),
+    new THREE.MeshBasicMaterial({ color: aimColor, depthTest: false })
+  );
+  lens.position.set(0, 0, -0.021 * markerScale);
+  markerGroup.add(lens);
+  if (camera.name) markerGroup.add(createCameraLabel(camera.name, color));
+  group.add(markerGroup);
+
+  const targetMarker = new THREE.Mesh(
+    new THREE.SphereGeometry(0.018 * markerScale, 16, 10),
+    new THREE.MeshBasicMaterial({ color: aimColor, depthTest: false })
+  );
+  targetMarker.position.copy(target);
+  targetMarker.renderOrder = 999;
+  targetMarker.frustumCulled = false;
+  group.add(targetMarker);
+
+  return group;
+}
+
 /**
  * Declarative debug visualization component.
  * Renders wireframe geoms, site markers, joint axes, contact forces, COM markers, etc.
@@ -50,6 +214,7 @@ export function Debug({
   showSites = false,
   showJoints = false,
   showCameras = false,
+  virtualCameras = [],
   showContacts = false,
   showCOM = false,
   showInertia = false,
@@ -69,6 +234,7 @@ export function Debug({
     const sites: THREE.Object3D[] = [];
     const joints: THREE.Object3D[] = [];
     const cameras: CameraDebugObject[] = [];
+    const virtualCameraObjects: THREE.Object3D[] = [];
     const comMarkers: THREE.Object3D[] = [];
 
     // Wireframe geoms
@@ -270,6 +436,10 @@ export function Debug({
       }
     }
 
+    for (let i = 0; i < virtualCameras.length; i += 1) {
+      virtualCameraObjects.push(createVirtualCameraDebugObject(virtualCameras[i], i));
+    }
+
     // COM markers
     if (showCOM) {
       for (let i = 1; i < model.nbody; i++) {
@@ -281,8 +451,8 @@ export function Debug({
       }
     }
 
-    return { geoms, sites, joints, cameras, comMarkers };
-  }, [status, mjModelRef, showGeoms, showSites, showJoints, showCameras, showCOM]);
+    return { geoms, sites, joints, cameras, virtualCameraObjects, comMarkers };
+  }, [status, mjModelRef, showGeoms, showSites, showJoints, showCameras, virtualCameras, showCOM]);
 
   // Add/remove debug objects from scene
   useEffect(() => {
@@ -294,6 +464,7 @@ export function Debug({
       ...debugGeometry.sites,
       ...debugGeometry.joints,
       ...debugGeometry.cameras,
+      ...debugGeometry.virtualCameraObjects,
       ...debugGeometry.comMarkers,
     ];
     for (const obj of allObjects) group.add(obj);

package/src/index.ts

@@ -320,6 +320,7 @@ export type {
   IkGizmoProps,
   IkGizmoDragInput,
   DragInteractionProps,
+  DebugVirtualCamera,
   DebugProps,
   SceneLightsProps,
   ScenarioLightingPreset,

package/src/types.ts

@@ -1027,11 +1027,28 @@ export interface ObservationHandle {
 
 // ---- Debug Component (spec 6.1) ----
 
+export interface DebugVirtualCamera {
+  name?: string;
+  position?: CameraFrameCaptureVector3;
+  lookAt?: CameraFrameCaptureVector3;
+  up?: CameraFrameCaptureVector3;
+  quaternion?: THREE.Quaternion | readonly [number, number, number, number];
+  fov?: number;
+  width?: number;
+  height?: number;
+  frustumDepth?: number;
+  markerScale?: number;
+  color?: THREE.ColorRepresentation;
+  aimColor?: THREE.ColorRepresentation;
+}
+
 export interface DebugProps {
   showGeoms?: boolean;
   showSites?: boolean;
   showJoints?: boolean;
   showCameras?: boolean;
+  /** Additional explicit virtual camera poses to visualize alongside MuJoCo XML cameras. */
+  virtualCameras?: readonly DebugVirtualCamera[];
   showContacts?: boolean;
   showCOM?: boolean;
   showInertia?: boolean;