mujoco-react 10.0.1 → 10.2.0

package/dist/spark.d.ts

@@ -1,6 +1,6 @@
 import * as react_jsx_runtime from 'react/jsx-runtime';
 import * as _sparkjsdev_spark from '@sparkjsdev/spark';
-import { n as SplatEnvironmentProps, q as PairedSplatEnvironmentConfig, S as SceneConfig, t as SplatEnvironmentReadiness, o as VisualScenarioConfig } from './types-BaSMqJHT.js';
+import { n as SplatEnvironmentProps, q as PairedSplatEnvironmentConfig, S as SceneConfig, t as SplatEnvironmentReadiness, o as VisualScenarioConfig } from './types-CLD5K3JD.js';
 import 'react';
 import '@react-three/fiber';
 import 'three';

package/dist/spark.js

@@ -1,4 +1,4 @@
-import { useSplatSceneConfig, useSplatEnvironment, SplatEnvironment, CAMERA_FRAME_CAPTURE_RENDER_USER_DATA_KEY } from './chunk-QTCAVQS6.js';
+import { useSplatSceneConfig, useSplatEnvironment, SplatEnvironment, CAMERA_FRAME_CAPTURE_RENDER_USER_DATA_KEY } from './chunk-3BMNRSS2.js';
 import { useThree } from '@react-three/fiber';
 import { useMemo, useState, useEffect, useCallback, useRef } from 'react';
 import * as THREE from 'three';

package/dist/{types-BaSMqJHT.d.ts → types-CLD5K3JD.d.ts}

@@ -177,7 +177,15 @@ interface MujocoModel {
     geom_contype: Int32Array;
     geom_conaffinity: Int32Array;
     geom_friction: Float64Array;
+    mat_texid: Int32Array;
+    mat_texrepeat: Float32Array;
+    mat_texuniform: Uint8Array;
     mat_rgba: Float32Array;
+    tex_adr: Int32Array;
+    tex_data: Uint8Array;
+    tex_height: Int32Array;
+    tex_nchannel: Int32Array;
+    tex_width: Int32Array;
     mesh_vertadr: Int32Array;
     mesh_vertnum: Int32Array;
     mesh_faceadr: Int32Array;
@@ -313,6 +321,8 @@ interface MujocoModule {
 }
 interface SceneObject {
     name: string;
+    /** MuJoCo geom name. Defaults to `${name}_geom` for generated objects. */
+    geomName?: string;
     type: 'box' | 'sphere' | 'cylinder';
     size: [number, number, number];
     position: [number, number, number];
@@ -563,6 +573,35 @@ interface RayHit {
     geomId: number;
     distance: number;
 }
+type ImagePointCoordinateSpace = 'normalized' | 'normalized-1000' | 'pixel' | 'ndc';
+interface ImagePointProjectionOptions extends CameraFrameCaptureOptions {
+    /** X coordinate in the selected coordinate space. Defaults to normalized 0..1. */
+    x: number;
+    /** Y coordinate in the selected coordinate space. Defaults to normalized 0..1 with origin at top-left. */
+    y: number;
+    /**
+     * Coordinate convention for x/y:
+     * - normalized: 0..1 image coordinates, top-left origin
+     * - normalized-1000: 0..1000 detector coordinates, top-left origin
+     * - pixel: pixel coordinates, top-left origin
+     * - ndc: Three.js normalized device coordinates, -1..1
+     */
+    coordinateSpace?: ImagePointCoordinateSpace;
+    /** Image width for pixel coordinates. Falls back to `width` or renderer canvas width. */
+    imageWidth?: number;
+    /** Image height for pixel coordinates. Falls back to `height` or renderer canvas height. */
+    imageHeight?: number;
+    /** Ignore hits farther than this distance from the camera ray origin. */
+    maxDistance?: number;
+}
+interface ImagePointProjectionResult extends RayHit {
+    /** NDC coordinates used for raycasting. */
+    ndc: [number, number];
+    /** Image dimensions used when interpreting pixel coordinates. */
+    imageSize: [number, number];
+    /** Camera pose provenance, matching camera-frame capture results. */
+    source: CameraFrameCaptureSource;
+}
 interface ModelOptions {
     timestep: number;
     gravity: [number, number, number];
@@ -598,16 +637,123 @@ interface PolicyObservationInput {
 interface PolicyInferenceInput extends PolicyObservationInput {
     observation: PolicyVector;
 }
+type PolicyActionChunk = readonly PolicyVector[];
+type PolicyInferenceOutput = PolicyVector | PolicyActionChunk;
+type PolicyInferenceResult = PolicyInferenceOutput | Promise<PolicyInferenceOutput>;
 interface PolicyActionInput extends PolicyInferenceInput {
     action: PolicyVector;
 }
+interface PolicyAPI {
+    readonly isRunning: boolean;
+    start: () => void;
+    stop: () => void;
+    clearQueue: () => void;
+    reset: () => void;
+    readonly inFlight: boolean;
+    readonly queuedActions: number;
+    readonly lastObservation: PolicyVector | null;
+    readonly lastAction: PolicyVector | null;
+    readonly lastError: unknown;
+}
 interface PolicyConfig {
     frequency: number;
     enabled?: boolean;
+    /** Start async inference while this many queued actions remain. Defaults to 0. */
+    prefetchThreshold?: number;
+    /**
+     * How async action chunks update the queue.
+     * - append preserves legacy FIFO behavior.
+     * - replace is useful for receding-horizon policies where a fresh chunk should supersede stale queued actions.
+     */
+    queueStrategy?: 'append' | 'replace';
+    /**
+     * Clear queued actions and ignore in-flight async results when `stop()` is called.
+     * Defaults to false so callers can choose pause/resume behavior explicitly.
+     */
+    clearQueueOnStop?: boolean;
     onObservation: (input: PolicyObservationInput) => PolicyVector;
     /** Run policy inference. Omit to pass observations directly to `onAction` for custom inline controllers. */
-    infer?: (input: PolicyInferenceInput) => PolicyVector;
+    infer?: (input: PolicyInferenceInput) => PolicyInferenceResult;
     onAction: (input: PolicyActionInput) => void;
+    /** Called when async inference rejects. */
+    onError?: (error: unknown) => void;
+}
+interface RemotePolicyRequestInput extends PolicyInferenceInput {
+    /** True for the first request after hook construction or `reset()`. */
+    reset: boolean;
+    /** Zero-based request index since construction or `reset()`. */
+    requestIndex: number;
+    /** Aborts when the request is no longer needed, e.g. after pause/reset. */
+    signal: AbortSignal;
+}
+interface RemotePolicyRequestInfo extends RemotePolicyRequestInput {
+    body: unknown;
+    requestStartedAt: number;
+}
+interface RemotePolicyResponseInfo extends RemotePolicyRequestInfo {
+    response: Response;
+    responseBody: unknown;
+    responseFinishedAt: number;
+    requestMs: number;
+}
+type RemotePolicyStatus = 'idle' | 'requesting' | 'ready' | 'error' | 'aborted';
+interface RemotePolicyConfig extends Omit<PolicyConfig, 'infer'> {
+    endpoint: string | URL;
+    method?: string;
+    headers?: HeadersInit;
+    credentials?: RequestCredentials;
+    /** Additional external cancellation signal for remote inference requests. */
+    signal?: AbortSignal;
+    /**
+     * Abort the active HTTP request when `stop()` or `reset()` is called.
+     * Defaults to true so paused policies stop consuming server work.
+     */
+    abortOnStop?: boolean;
+    fetcher?: typeof fetch;
+    requestInit?: Omit<RequestInit, 'body' | 'headers' | 'method' | 'credentials' | 'signal'>;
+    buildRequest?: (input: RemotePolicyRequestInput) => unknown | Promise<unknown>;
+    readResponse?: (response: Response) => unknown | Promise<unknown>;
+    parseResponse?: (responseBody: unknown, info: RemotePolicyResponseInfo) => PolicyInferenceResult;
+    onRequest?: (info: RemotePolicyRequestInfo) => void;
+    onResponse?: (info: RemotePolicyResponseInfo) => void;
+}
+interface RemotePolicyAPI extends PolicyAPI {
+    abort: (reason?: unknown) => void;
+    readonly remoteStatus: RemotePolicyStatus;
+    readonly requestCount: number;
+    readonly responseCount: number;
+    readonly lastRequestBody: unknown;
+    readonly lastResponseBody: unknown;
+    readonly lastHttpStatus: number | null;
+    readonly lastRequestMs: number | null;
+}
+interface PolicyCameraFrameStream extends CameraFrameCaptureOptions {
+    /** Image key used in policy payloads, e.g. `image`, `front`, or `wrist_cam`. */
+    key: string;
+    /** Additional payload keys that should receive the same data URL. */
+    aliases?: readonly string[];
+}
+interface PolicyCameraFrameCaptureOptions {
+    streams: readonly PolicyCameraFrameStream[];
+    /**
+     * Include `observation.images.${key}` for every captured stream.
+     * Defaults to true because LeRobot-style policies usually use these names.
+     */
+    includeObservationImageAliases?: boolean;
+}
+interface PolicyCameraFrameCaptureResult {
+    frames: Record<string, CameraFrameCaptureResult>;
+    images: Record<string, string>;
+    /** Human-readable source summary for UI/debug telemetry. */
+    sourceSummary: string;
+    capturedAt: number;
+}
+interface PolicyCameraFrameCaptureAPI {
+    status: FrameCaptureStatus;
+    error: Error | null;
+    isCapturing: boolean;
+    capture: (options?: Partial<PolicyCameraFrameCaptureOptions>) => Promise<PolicyCameraFrameCaptureResult>;
+    reset: () => void;
 }
 type ObservationOutput = 'float32' | 'float64';
 interface ObservationConfig {
@@ -977,6 +1123,7 @@ interface MujocoSimAPI {
         bodyId: number;
         geomId: number;
     } | null;
+    projectImagePointTo3D(options: ImagePointProjectionOptions): ImagePointProjectionResult | null;
     setBodyMass(name: Bodies, mass: number): void;
     setGeomFriction(name: Geoms, friction: [number, number, number]): void;
     setGeomSize(name: Geoms, size: [number, number, number]): void;
@@ -1026,6 +1173,10 @@ interface CameraFrameCaptureOptions {
     lookAt?: CameraFrameCaptureVector3;
     quaternion?: CameraFrameCaptureQuaternion;
     up?: CameraFrameCaptureVector3;
+    /** Local-space offset applied after resolving a mounted MuJoCo camera/site/body pose. */
+    positionOffset?: CameraFrameCaptureVector3;
+    /** Local-space rotation applied after resolving a mounted MuJoCo camera/site/body pose. Array values use Three.js order: [x, y, z, w]. */
+    quaternionOffset?: CameraFrameCaptureQuaternion;
     width?: number;
     height?: number;
     type?: string;
@@ -1035,6 +1186,16 @@ interface CameraFrameCaptureOptions {
     far?: number;
     /** Provenance for the camera pose used by the capture. Usually set by the MuJoCo provider. */
     source?: CameraFrameCaptureSource;
+    /** Hide rendered Three objects whose MuJoCo geom group is in this list. */
+    hiddenGeomGroups?: readonly number[];
+    /** When provided, only rendered Three objects whose MuJoCo geom group is in this list are visible. */
+    visibleGeomGroups?: readonly number[];
+    /** Hide rendered Three objects whose MuJoCo geom name is in this list. */
+    hiddenGeomNames?: readonly string[];
+    /** Optional clear color for this capture only. Defaults to the renderer's current clear color. */
+    background?: THREE.ColorRepresentation;
+    /** Optional clear alpha for this capture only. Defaults to the renderer's current clear alpha. */
+    backgroundAlpha?: number;
 }
 type CameraFrameCaptureSource = {
     kind: 'mujoco-camera';
@@ -1219,4 +1380,4 @@ interface ArrayJointStateResult {
     velocity: React__default.RefObject<Float64Array>;
 }
 
-export { type ArrayJointStateResult as $, type ActuatedJointInfo as A, type BodyProps as B, type ControlGroupInfo as C, type DragInteractionProps as D, type ScenarioLightingPreset as E, type SplatEnvironmentMetadataInput as F, type SplatEnvironmentMetadata as G, type SplatSceneInput as H, type IkConfig as I, type DebugProps as J, type GeomInfo as K, type ContactListenerProps as L, type MujocoContextValue as M, type ActuatorInfo as N, type ObservationConfig as O, type PhysicsStepCallback as P, type Sites as Q, type ReadyCallbackInput as R, type SceneConfig as S, type TrajectoryPlayerProps as T, type SitePositionResult as U, type VisualScenarioEffectsProps as V, type Sensors as W, type SensorHandle as X, type SensorInfo as Y, type Joints as Z, type ScalarJointStateResult as _, type MujocoCanvasProps as a, type Models as a$, type JointStateOptions as a0, type JointStateResult as a1, type Bodies as a2, type BodyStateResult as a3, type Actuators as a4, type CtrlHandle as a5, type ContactInfo as a6, type KeyboardTeleopConfig as a7, type KeyboardIkTargetConfig as a8, type PolicyConfig as a9, type CameraFrameSequenceStepInput as aA, type CameraInfo as aB, type ControlJointInfo as aC, type FrameCaptureStatus as aD, type FrameCaptureTarget as aE, type FrameCaptureTargetRef as aF, type Geoms as aG, type IKSolveFn as aH, type IkGizmoDragInput as aI, type IkSolveInput as aJ, type JointInfo as aK, type JointStateKind as aL, type KeyBinding as aM, type KeyboardIkTargetAction as aN, type KeyboardIkTargetBinding as aO, type Keyframes as aP, ModelActuators as aQ, ModelBodies as aR, ModelCameras as aS, ModelGeoms as aT, ModelJoints as aU, ModelKeyframes as aV, type ModelOptions as aW, type ModelResource as aX, ModelResources as aY, ModelSensors as aZ, ModelSites as a_, type PolicyVector as aa, type ObservationHandle as ab, type TrajectoryInput as ac, type TrajectoryStateChangeInput as ad, type PlaybackState as ae, type TrajectoryFrame as af, type FrameCaptureOptions as ag, type FrameCaptureResult as ah, type FrameCaptureBlobResult as ai, type FrameCaptureAPI as aj, type CameraFrameCaptureOptions as ak, type CameraFrameCaptureAPI as al, type CameraFrameSequenceRecorderAPI as am, type Cameras as an, type CameraFrameSequenceCamera as ao, type CameraFrameCaptureSource as ap, type CameraFrameSequenceOptions as aq, type CameraFrameSequenceResult as ar, type CameraFrameCaptureResult as as, type CameraFrameCaptureBlobResult as at, type BodyInfo as au, type CameraFrameCaptureQuaternion as av, type CameraFrameCaptureVector3 as aw, type CameraFrameSequenceCameraSummary as ax, type CameraFrameSequenceFrame as ay, type CameraFrameSequenceSampleInput as az, type MujocoSimAPI as b, type MujocoContact as b0, type MujocoContactArray as b1, type MujocoFrameCaptureOptions as b2, type ObservationLayoutItem as b3, type ObservationOutput as b4, type PhysicsConfig as b5, type PhysicsStepInput as b6, type PolicyActionInput as b7, type PolicyInferenceInput as b8, type PolicyObservationInput as b9, type RayHit as ba, type Register as bb, type RegisteredModelMap as bc, type ResetCallbackInput as bd, type ResolvedScenarioCameraConfig as be, type ResolvedScenarioMaterialConfig as bf, type ResourceSelector as bg, type ScenarioCameraConfig as bh, type ScenarioMaterialConfig as bi, type SceneMarker as bj, type SceneObject as bk, type SensorResult as bl, type SiteInfo as bm, type SplatAssetConfig as bn, type SplatScenarioConfig as bo, type StateSnapshot as bp, type TrajectoryData as bq, type TrajectoryFrameCallbackInput as br, type VisualScenarioMaterialFilterInput as bs, type XmlPatch as bt, getContact as bu, registerModelResources as bv, type StepCallbackInput as c, type SelectionCallbackInput as d, type MujocoModule as e, type MujocoModel as f, type MujocoData as g, type ControlGroupSelector as h, type ObservationResult as i, type IkContextValue as j, type IkGizmoProps as k, type SceneLightsProps as l, type ScenarioLightingProps as m, type SplatEnvironmentProps as n, type VisualScenarioConfig as o, type SplatRendererKind as p, type PairedSplatEnvironmentConfig as q, type SplatFormat as r, type SplatCollisionProxyConfig as s, type SplatEnvironmentReadiness as t, type SplatCollisionPrimitive as u, SplatEnvironmentReadinessStatus as v, type SplatSceneConfigInput as w, type SplatSceneConfigState as x, type VisualScenarioExecutionContextInput as y, type VisualScenarioExecutionContext as z };
+export { type ArrayJointStateResult as $, type ActuatedJointInfo as A, type BodyProps as B, type ControlGroupInfo as C, type DragInteractionProps as D, type ScenarioLightingPreset as E, type SplatEnvironmentMetadataInput as F, type SplatEnvironmentMetadata as G, type SplatSceneInput as H, type IkConfig as I, type DebugProps as J, type GeomInfo as K, type ContactListenerProps as L, type MujocoContextValue as M, type ActuatorInfo as N, type ObservationConfig as O, type PhysicsStepCallback as P, type Sites as Q, type ReadyCallbackInput as R, type SceneConfig as S, type TrajectoryPlayerProps as T, type SitePositionResult as U, type VisualScenarioEffectsProps as V, type Sensors as W, type SensorHandle as X, type SensorInfo as Y, type Joints as Z, type ScalarJointStateResult as _, type MujocoCanvasProps as a, ModelActuators as a$, type JointStateOptions as a0, type JointStateResult as a1, type Bodies as a2, type BodyStateResult as a3, type Geoms as a4, type Actuators as a5, type CtrlHandle as a6, type ContactInfo as a7, type KeyboardTeleopConfig as a8, type KeyboardIkTargetConfig as a9, type CameraFrameSequenceRecorderAPI as aA, type CameraFrameCaptureResult as aB, type CameraFrameCaptureBlobResult as aC, type ImagePointCoordinateSpace as aD, type ImagePointProjectionOptions as aE, type ImagePointProjectionResult as aF, type PolicyVector as aG, type BodyInfo as aH, type CameraFrameCaptureQuaternion as aI, type CameraFrameCaptureVector3 as aJ, type CameraFrameSequenceCameraSummary as aK, type CameraFrameSequenceFrame as aL, type CameraFrameSequenceSampleInput as aM, type CameraFrameSequenceStepInput as aN, type CameraInfo as aO, type ControlJointInfo as aP, type FrameCaptureTarget as aQ, type FrameCaptureTargetRef as aR, type IKSolveFn as aS, type IkGizmoDragInput as aT, type IkSolveInput as aU, type JointInfo as aV, type JointStateKind as aW, type KeyBinding as aX, type KeyboardIkTargetAction as aY, type KeyboardIkTargetBinding as aZ, type Keyframes as a_, type PolicyConfig as aa, type PolicyAPI as ab, type RemotePolicyConfig as ac, type RemotePolicyAPI as ad, type ObservationHandle as ae, type ObservationOutput as af, type TrajectoryInput as ag, type TrajectoryStateChangeInput as ah, type PlaybackState as ai, type TrajectoryFrame as aj, type FrameCaptureOptions as ak, type FrameCaptureResult as al, type FrameCaptureBlobResult as am, type FrameCaptureAPI as an, type CameraFrameCaptureOptions as ao, type CameraFrameCaptureAPI as ap, type Cameras as aq, type CameraFrameSequenceCamera as ar, type CameraFrameCaptureSource as as, type CameraFrameSequenceOptions as at, type CameraFrameSequenceResult as au, type PolicyCameraFrameStream as av, type PolicyCameraFrameCaptureOptions as aw, type PolicyCameraFrameCaptureResult as ax, type FrameCaptureStatus as ay, type PolicyCameraFrameCaptureAPI as az, type MujocoSimAPI as b, ModelBodies as b0, ModelCameras as b1, ModelGeoms as b2, ModelJoints as b3, ModelKeyframes as b4, type ModelOptions as b5, type ModelResource as b6, ModelResources as b7, ModelSensors as b8, ModelSites as b9, type SceneMarker as bA, type SceneObject as bB, type SensorResult as bC, type SiteInfo as bD, type SplatAssetConfig as bE, type SplatScenarioConfig as bF, type StateSnapshot as bG, type TrajectoryData as bH, type TrajectoryFrameCallbackInput as bI, type VisualScenarioMaterialFilterInput as bJ, type XmlPatch as bK, getContact as bL, registerModelResources as bM, type Models as ba, type MujocoContact as bb, type MujocoContactArray as bc, type MujocoFrameCaptureOptions as bd, type ObservationLayoutItem as be, type PhysicsConfig as bf, type PhysicsStepInput as bg, type PolicyActionChunk as bh, type PolicyActionInput as bi, type PolicyInferenceInput as bj, type PolicyInferenceOutput as bk, type PolicyInferenceResult as bl, type PolicyObservationInput as bm, type RayHit as bn, type Register as bo, type RegisteredModelMap as bp, type RemotePolicyRequestInfo as bq, type RemotePolicyRequestInput as br, type RemotePolicyResponseInfo as bs, type RemotePolicyStatus as bt, type ResetCallbackInput as bu, type ResolvedScenarioCameraConfig as bv, type ResolvedScenarioMaterialConfig as bw, type ResourceSelector as bx, type ScenarioCameraConfig as by, type ScenarioMaterialConfig as bz, type StepCallbackInput as c, type SelectionCallbackInput as d, type MujocoModule as e, type MujocoModel as f, type MujocoData as g, type ControlGroupSelector as h, type ObservationResult as i, type IkContextValue as j, type IkGizmoProps as k, type SceneLightsProps as l, type ScenarioLightingProps as m, type SplatEnvironmentProps as n, type VisualScenarioConfig as o, type SplatRendererKind as p, type PairedSplatEnvironmentConfig as q, type SplatFormat as r, type SplatCollisionProxyConfig as s, type SplatEnvironmentReadiness as t, type SplatCollisionPrimitive as u, SplatEnvironmentReadinessStatus as v, type SplatSceneConfigInput as w, type SplatSceneConfigState as x, type VisualScenarioExecutionContextInput as y, type VisualScenarioExecutionContext as z };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mujoco-react",
-  "version": "10.0.1",
+  "version": "10.2.0",
   "description": "Composable React Three Fiber building blocks for MuJoCo WASM simulations",
   "type": "module",
   "main": "./dist/index.js",

package/src/components/SceneRenderer.tsx

@@ -5,9 +5,10 @@
 
 import { useFrame } from '@react-three/fiber';
 import type { ThreeElements } from '@react-three/fiber';
-import { useEffect, useMemo, useRef } from 'react';
+import { useCallback, useEffect, useMemo, useRef } from 'react';
 import * as THREE from 'three';
 import { GeomBuilder } from '../rendering/GeomBuilder';
+import { CAMERA_FRAME_CAPTURE_PRE_RENDER_USER_DATA_KEY } from '../rendering/cameraFrameCapture';
 import { MujocoModel } from '../types';
 import { getName } from '../core/SceneLoader';
 import { useMujocoContext } from '../core/MujocoSimProvider';
@@ -74,8 +75,7 @@ export function SceneRenderer(props: Omit<ThreeElements['group'], 'ref'>) {
     bodyRefs.current = refs;
   }, [status, geomBuilder, mjModelRef]);
 
-  // Sync body positions from mjData every frame
-  useFrame(() => {
+  const syncBodiesToData = useCallback(() => {
     const data = mjDataRef.current;
     if (!data) return;
     const bodies = bodyRefs.current;
@@ -121,12 +121,19 @@ export function SceneRenderer(props: Omit<ThreeElements['group'], 'ref'>) {
         );
       }
     }
-  });
+  }, [interpolateRef, interpolationStateRef, mjDataRef]);
+
+  // Sync body positions from mjData every frame
+  useFrame(syncBodiesToData);
 
   return (
     <group
       {...props}
       ref={groupRef}
+      userData={{
+        ...props.userData,
+        [CAMERA_FRAME_CAPTURE_PRE_RENDER_USER_DATA_KEY]: syncBodiesToData,
+      }}
       onDoubleClick={(e) => {
         if (typeof props.onDoubleClick === 'function') props.onDoubleClick(e);
         e.stopPropagation();

package/src/core/MujocoSimProvider.tsx

@@ -31,6 +31,8 @@ import {
   ControlGroupSelector,
   ContactInfo,
   GeomInfo,
+  ImagePointProjectionOptions,
+  ImagePointProjectionResult,
   JointInfo,
   LoadFromFilesOptions,
   LocalMujocoFile,
@@ -61,6 +63,7 @@ import {
   getCameraFrameCaptureSourceTarget,
   isMountedCameraFrameCaptureSource,
 } from '../rendering/cameraFrameSource';
+import { projectImagePointTo3D as projectImagePointTo3DFromScene } from '../rendering/imageProjection';
 import {
   loadScene,
   createSceneConfigFromFiles,
@@ -188,6 +191,56 @@ function omitResolvedCameraSelectors(
   return rest;
 }
 
+function vector3FromCaptureValue(value: NonNullable<CameraFrameCaptureOptions['positionOffset']>) {
+  return value instanceof THREE.Vector3
+    ? value.clone()
+    : new THREE.Vector3(value[0], value[1], value[2]);
+}
+
+function quaternionFromCaptureValue(value: NonNullable<CameraFrameCaptureOptions['quaternionOffset']>) {
+  return value instanceof THREE.Quaternion
+    ? value.clone()
+    : new THREE.Quaternion(value[0], value[1], value[2], value[3]);
+}
+
+function applyMountedCameraPoseOffsets(
+  options: CameraFrameCaptureOptions,
+  position: [number, number, number],
+  quaternion: [number, number, number, number]
+) {
+  const resolvedPosition = new THREE.Vector3(position[0], position[1], position[2]);
+  const resolvedQuaternion = new THREE.Quaternion(
+    quaternion[0],
+    quaternion[1],
+    quaternion[2],
+    quaternion[3]
+  );
+
+  if (options.positionOffset) {
+    resolvedPosition.add(
+      vector3FromCaptureValue(options.positionOffset).applyQuaternion(resolvedQuaternion)
+    );
+  }
+
+  if (options.quaternionOffset) {
+    resolvedQuaternion.multiply(quaternionFromCaptureValue(options.quaternionOffset)).normalize();
+  }
+
+  return {
+    position: [
+      resolvedPosition.x,
+      resolvedPosition.y,
+      resolvedPosition.z,
+    ] as [number, number, number],
+    quaternion: [
+      resolvedQuaternion.x,
+      resolvedQuaternion.y,
+      resolvedQuaternion.z,
+      resolvedQuaternion.w,
+    ] as [number, number, number, number],
+  };
+}
+
 function countMountedCameraSelectors(options: CameraFrameCaptureOptions) {
   return Number(Boolean(options.cameraName)) +
     Number(Boolean(options.siteName)) +
@@ -1033,10 +1086,11 @@ export function MujocoSimProvider({
           );
         }
 
+        const pose = applyMountedCameraPoseOffsets(options, position, quaternion);
+
         return {
           ...baseOptions,
-          position,
-          quaternion,
+          ...pose,
           fov: options.fov ?? model.cam_fovy?.[cameraId],
           source: { kind: 'mujoco-camera', cameraName: options.cameraName },
         };
@@ -1048,10 +1102,15 @@ export function MujocoSimProvider({
           throw new Error(`MuJoCo site "${options.siteName}" was not found.`);
         }
 
+        const pose = applyMountedCameraPoseOffsets(
+          options,
+          vector3FromArray(data.site_xpos, siteId * 3),
+          quaternionFromXmat(data.site_xmat, siteId * 9)
+        );
+
         return {
           ...baseOptions,
-          position: vector3FromArray(data.site_xpos, siteId * 3),
-          quaternion: quaternionFromXmat(data.site_xmat, siteId * 9),
+          ...pose,
           source: { kind: 'mujoco-site', siteName: options.siteName },
         };
       }
@@ -1067,10 +1126,15 @@ export function MujocoSimProvider({
           );
         }
 
+        const pose = applyMountedCameraPoseOffsets(
+          options,
+          vector3FromArray(data.xpos, bodyId * 3),
+          quaternionFromXmat(data.xmat, bodyId * 9)
+        );
+
         return {
           ...baseOptions,
-          position: vector3FromArray(data.xpos, bodyId * 3),
-          quaternion: quaternionFromXmat(data.xmat, bodyId * 9),
+          ...pose,
           source: { kind: 'mujoco-body', bodyName: options.bodyName },
         };
       }
@@ -1529,6 +1593,31 @@ export function MujocoSimProvider({
     [camera, gl]
   );
 
+  const projectImagePointTo3D = useCallback(
+    (options: ImagePointProjectionOptions): ImagePointProjectionResult | null => {
+      const {
+        x,
+        y,
+        coordinateSpace,
+        imageWidth,
+        imageHeight,
+        maxDistance,
+        ...captureOptions
+      } = options;
+      const resolvedCaptureOptions = resolveCameraCaptureOptions(captureOptions);
+      return projectImagePointTo3DFromScene(gl, scene, camera, {
+        ...resolvedCaptureOptions,
+        x,
+        y,
+        coordinateSpace,
+        imageWidth,
+        imageHeight,
+        maxDistance,
+      });
+    },
+    [camera, gl, resolveCameraCaptureOptions, scene]
+  );
+
   // --- Domain randomization ---
 
   const setBodyMass = useCallback((name: string, mass: number): void => {
@@ -1615,6 +1704,7 @@ export function MujocoSimProvider({
       captureCameraFrameBlob: captureCameraFrameBlobApi,
       recordCameraSequence: recordCameraSequenceApi,
       project2DTo3D,
+      projectImagePointTo3D,
       setBodyMass,
       setGeomFriction,
       setGeomSize,
@@ -1635,6 +1725,7 @@ export function MujocoSimProvider({
       captureCameraFrameApi, captureCameraFrameBlobApi,
       recordCameraSequenceApi,
       project2DTo3D,
+      projectImagePointTo3D,
       setBodyMass, setGeomFriction, setGeomSize,
     ]
   );

package/src/core/SceneLoader.ts

@@ -482,8 +482,14 @@ export function createContiguousControlGroup(mjModel: MujocoModel, count: number
  */
 function sceneObjectToXml(obj: SceneObject): string {
   const joint = obj.freejoint ? '<freejoint/>' : '';
+  const geomName = obj.geomName ?? `${obj.name}_geom`;
   const pos = obj.position.map((v) => v.toFixed(3)).join(' ');
-  const size = obj.size.map((v) => v.toFixed(3)).join(' ');
+  const sizeValues = obj.type === 'sphere'
+    ? obj.size.slice(0, 1)
+    : obj.type === 'cylinder'
+      ? obj.size.slice(0, 2)
+      : obj.size;
+  const size = sizeValues.map((v) => v.toFixed(3)).join(' ');
   const rgba = obj.rgba.join(' ');
   const mass = obj.mass ? ` mass="${obj.mass}"` : '';
   const friction = obj.friction ? ` friction="${obj.friction}"` : '';
@@ -492,7 +498,7 @@ function sceneObjectToXml(obj: SceneObject): string {
   const condim = obj.condim ? ` condim="${obj.condim}"` : '';
   const group = obj.group !== undefined ? ` group="${obj.group}"` : '';
   // Always set contype/conaffinity=1 so objects collide regardless of model defaults
-  return `<body name="${obj.name}" pos="${pos}">${joint}<geom type="${obj.type}" size="${size}" rgba="${rgba}" contype="1" conaffinity="1"${mass}${friction}${solref}${solimp}${condim}${group}/></body>`;
+  return `<body name="${obj.name}" pos="${pos}">${joint}<geom name="${geomName}" type="${obj.type}" size="${size}" rgba="${rgba}" contype="1" conaffinity="1"${mass}${friction}${solref}${solimp}${condim}${group}/></body>`;
 }
 
 /** Create virtual directory structure for a file path. */

package/src/hooks/useContactHistory.ts

@@ -0,0 +1,155 @@
+/**
+ * @license
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Bounded contact history for rollout verification and debugging.
+ */
+
+import { useCallback, useRef } from 'react';
+import type { RefObject } from 'react';
+import { useAfterPhysicsStep } from '../core/MujocoSimProvider';
+import { getName } from '../core/SceneLoader';
+import { getContact, withContacts } from '../types';
+import type { ContactInfo, MujocoModel } from '../types';
+
+export interface ContactHistoryEntry extends ContactInfo {
+  time: number;
+  body1: number;
+  body1Name: string;
+  body2: number;
+  body2Name: string;
+}
+
+export interface ContactHistoryOptions {
+  maxLength?: number;
+  bodyNames?: readonly string[];
+  geomNames?: readonly string[];
+  includeWorldBody?: boolean;
+}
+
+export interface ContactHistoryHandle {
+  entries: RefObject<ContactHistoryEntry[]>;
+  clear: () => void;
+  countPair: (nameA: string, nameB: string) => number;
+}
+
+const geomNameCacheByModel = new WeakMap<MujocoModel, Map<number, string>>();
+const bodyNameCacheByModel = new WeakMap<MujocoModel, Map<number, string>>();
+
+function getCachedName(
+  cacheByModel: WeakMap<MujocoModel, Map<number, string>>,
+  model: MujocoModel,
+  id: number,
+  address: number
+) {
+  if (id < 0) return '';
+  let cache = cacheByModel.get(model);
+  if (!cache) {
+    cache = new Map();
+    cacheByModel.set(model, cache);
+  }
+  let name = cache.get(id);
+  if (name === undefined) {
+    name = getName(model, address);
+    cache.set(id, name);
+  }
+  return name;
+}
+
+function matchesFilter(
+  entry: ContactHistoryEntry,
+  bodyNames: readonly string[] | undefined,
+  geomNames: readonly string[] | undefined,
+  includeWorldBody: boolean
+) {
+  if (!includeWorldBody && (entry.body1 === 0 || entry.body2 === 0)) return false;
+  if (
+    bodyNames &&
+    !bodyNames.includes(entry.body1Name) &&
+    !bodyNames.includes(entry.body2Name)
+  ) {
+    return false;
+  }
+  if (
+    geomNames &&
+    !geomNames.includes(entry.geom1Name) &&
+    !geomNames.includes(entry.geom2Name)
+  ) {
+    return false;
+  }
+  return true;
+}
+
+export function useContactHistory(options: ContactHistoryOptions = {}): ContactHistoryHandle {
+  const entriesRef = useRef<ContactHistoryEntry[]>([]);
+  const optionsRef = useRef(options);
+  optionsRef.current = options;
+
+  const clear = useCallback(() => {
+    entriesRef.current = [];
+  }, []);
+
+  const countPair = useCallback((nameA: string, nameB: string) => {
+    let count = 0;
+    for (const entry of entriesRef.current) {
+      const matchesBodies =
+        (entry.body1Name === nameA && entry.body2Name === nameB) ||
+        (entry.body1Name === nameB && entry.body2Name === nameA);
+      const matchesGeoms =
+        (entry.geom1Name === nameA && entry.geom2Name === nameB) ||
+        (entry.geom1Name === nameB && entry.geom2Name === nameA);
+      if (matchesBodies || matchesGeoms) count += 1;
+    }
+    return count;
+  }, []);
+
+  useAfterPhysicsStep(({ model, data }) => {
+    if ((data.ncon ?? 0) <= 0) return;
+
+    const {
+      maxLength = 2000,
+      bodyNames,
+      geomNames,
+      includeWorldBody = false,
+    } = optionsRef.current;
+    if (maxLength <= 0) return;
+
+    const nextEntries: ContactHistoryEntry[] = [];
+    withContacts(data, (contacts) => {
+      for (let index = 0; index < data.ncon; index += 1) {
+        const contact = getContact(contacts, index);
+        if (!contact) break;
+        const body1 = model.geom_bodyid[contact.geom1] ?? -1;
+        const body2 = model.geom_bodyid[contact.geom2] ?? -1;
+        const entry: ContactHistoryEntry = {
+          geom1: contact.geom1,
+          geom2: contact.geom2,
+          geom1Name: getCachedName(geomNameCacheByModel, model, contact.geom1, model.name_geomadr[contact.geom1]),
+          geom2Name: getCachedName(geomNameCacheByModel, model, contact.geom2, model.name_geomadr[contact.geom2]),
+          body1,
+          body2,
+          body1Name: body1 >= 0 ? getCachedName(bodyNameCacheByModel, model, body1, model.name_bodyadr[body1]) : '',
+          body2Name: body2 >= 0 ? getCachedName(bodyNameCacheByModel, model, body2, model.name_bodyadr[body2]) : '',
+          pos: [contact.pos[0], contact.pos[1], contact.pos[2]],
+          depth: contact.dist,
+          time: data.time,
+        };
+        if (matchesFilter(entry, bodyNames, geomNames, includeWorldBody)) {
+          nextEntries.push(entry);
+        }
+      }
+    });
+
+    if (nextEntries.length === 0) return;
+    entriesRef.current.push(...nextEntries);
+    if (entriesRef.current.length > maxLength) {
+      entriesRef.current.splice(0, entriesRef.current.length - maxLength);
+    }
+  });
+
+  return {
+    entries: entriesRef,
+    clear,
+    countPair,
+  };
+}