mujoco-react 10.1.0 → 10.2.1

package/dist/spark.d.ts

@@ -1,6 +1,6 @@
 import * as react_jsx_runtime from 'react/jsx-runtime';
 import * as _sparkjsdev_spark from '@sparkjsdev/spark';
-import { n as SplatEnvironmentProps, q as PairedSplatEnvironmentConfig, S as SceneConfig, t as SplatEnvironmentReadiness, o as VisualScenarioConfig } from './types-BHBNJubg.js';
+import { n as SplatEnvironmentProps, q as PairedSplatEnvironmentConfig, S as SceneConfig, t as SplatEnvironmentReadiness, o as VisualScenarioConfig } from './types-B-J8fpPP.js';
 import 'react';
 import '@react-three/fiber';
 import 'three';

package/dist/spark.js

@@ -1,4 +1,4 @@
-import { useSplatSceneConfig, useSplatEnvironment, SplatEnvironment, CAMERA_FRAME_CAPTURE_RENDER_USER_DATA_KEY } from './chunk-FEKBKHEN.js';
+import { useSplatSceneConfig, useSplatEnvironment, SplatEnvironment, CAMERA_FRAME_CAPTURE_RENDER_USER_DATA_KEY } from './chunk-CYDGWNKQ.js';
 import { useThree } from '@react-three/fiber';
 import { useMemo, useState, useEffect, useCallback, useRef } from 'react';
 import * as THREE from 'three';

package/dist/{types-BHBNJubg.d.ts → types-B-J8fpPP.d.ts}

@@ -573,6 +573,35 @@ interface RayHit {
     geomId: number;
     distance: number;
 }
+type ImagePointCoordinateSpace = 'normalized' | 'normalized-1000' | 'pixel' | 'ndc';
+interface ImagePointProjectionOptions extends CameraFrameCaptureOptions {
+    /** X coordinate in the selected coordinate space. Defaults to normalized 0..1. */
+    x: number;
+    /** Y coordinate in the selected coordinate space. Defaults to normalized 0..1 with origin at top-left. */
+    y: number;
+    /**
+     * Coordinate convention for x/y:
+     * - normalized: 0..1 image coordinates, top-left origin
+     * - normalized-1000: 0..1000 detector coordinates, top-left origin
+     * - pixel: pixel coordinates, top-left origin
+     * - ndc: Three.js normalized device coordinates, -1..1
+     */
+    coordinateSpace?: ImagePointCoordinateSpace;
+    /** Image width for pixel coordinates. Falls back to `width` or renderer canvas width. */
+    imageWidth?: number;
+    /** Image height for pixel coordinates. Falls back to `height` or renderer canvas height. */
+    imageHeight?: number;
+    /** Ignore hits farther than this distance from the camera ray origin. */
+    maxDistance?: number;
+}
+interface ImagePointProjectionResult extends RayHit {
+    /** NDC coordinates used for raycasting. */
+    ndc: [number, number];
+    /** Image dimensions used when interpreting pixel coordinates. */
+    imageSize: [number, number];
+    /** Camera pose provenance, matching camera-frame capture results. */
+    source: CameraFrameCaptureSource;
+}
 interface ModelOptions {
     timestep: number;
     gravity: [number, number, number];
@@ -607,6 +636,8 @@ interface PolicyObservationInput {
 }
 interface PolicyInferenceInput extends PolicyObservationInput {
     observation: PolicyVector;
+    /** Number of actions still queued locally when inference is requested. */
+    queuedActions?: number;
 }
 type PolicyActionChunk = readonly PolicyVector[];
 type PolicyInferenceOutput = PolicyVector | PolicyActionChunk;
@@ -1094,6 +1125,7 @@ interface MujocoSimAPI {
         bodyId: number;
         geomId: number;
     } | null;
+    projectImagePointTo3D(options: ImagePointProjectionOptions): ImagePointProjectionResult | null;
     setBodyMass(name: Bodies, mass: number): void;
     setGeomFriction(name: Geoms, friction: [number, number, number]): void;
     setGeomSize(name: Geoms, size: [number, number, number]): void;
@@ -1166,6 +1198,8 @@ interface CameraFrameCaptureOptions {
     background?: THREE.ColorRepresentation;
     /** Optional clear alpha for this capture only. Defaults to the renderer's current clear alpha. */
     backgroundAlpha?: number;
+    /** Mirror the captured image horizontally after rendering. Useful when matching policy datasets with mirrored camera frames. */
+    flipX?: boolean;
 }
 type CameraFrameCaptureSource = {
     kind: 'mujoco-camera';
@@ -1350,4 +1384,4 @@ interface ArrayJointStateResult {
     velocity: React__default.RefObject<Float64Array>;
 }
 
-export { type ArrayJointStateResult as $, type ActuatedJointInfo as A, type BodyProps as B, type ControlGroupInfo as C, type DragInteractionProps as D, type ScenarioLightingPreset as E, type SplatEnvironmentMetadataInput as F, type SplatEnvironmentMetadata as G, type SplatSceneInput as H, type IkConfig as I, type DebugProps as J, type GeomInfo as K, type ContactListenerProps as L, type MujocoContextValue as M, type ActuatorInfo as N, type ObservationConfig as O, type PhysicsStepCallback as P, type Sites as Q, type ReadyCallbackInput as R, type SceneConfig as S, type TrajectoryPlayerProps as T, type SitePositionResult as U, type VisualScenarioEffectsProps as V, type Sensors as W, type SensorHandle as X, type SensorInfo as Y, type Joints as Z, type ScalarJointStateResult as _, type MujocoCanvasProps as a, ModelGeoms as a$, type JointStateOptions as a0, type JointStateResult as a1, type Bodies as a2, type BodyStateResult as a3, type Geoms as a4, type Actuators as a5, type CtrlHandle as a6, type ContactInfo as a7, type KeyboardTeleopConfig as a8, type KeyboardIkTargetConfig as a9, type CameraFrameSequenceRecorderAPI as aA, type CameraFrameCaptureResult as aB, type CameraFrameCaptureBlobResult as aC, type PolicyVector as aD, type BodyInfo as aE, type CameraFrameCaptureQuaternion as aF, type CameraFrameCaptureVector3 as aG, type CameraFrameSequenceCameraSummary as aH, type CameraFrameSequenceFrame as aI, type CameraFrameSequenceSampleInput as aJ, type CameraFrameSequenceStepInput as aK, type CameraInfo as aL, type ControlJointInfo as aM, type FrameCaptureTarget as aN, type FrameCaptureTargetRef as aO, type IKSolveFn as aP, type IkGizmoDragInput as aQ, type IkSolveInput as aR, type JointInfo as aS, type JointStateKind as aT, type KeyBinding as aU, type KeyboardIkTargetAction as aV, type KeyboardIkTargetBinding as aW, type Keyframes as aX, ModelActuators as aY, ModelBodies as aZ, ModelCameras as a_, type PolicyConfig as aa, type PolicyAPI as ab, type RemotePolicyConfig as ac, type RemotePolicyAPI as ad, type ObservationHandle as ae, type ObservationOutput as af, type TrajectoryInput as ag, type TrajectoryStateChangeInput as ah, type PlaybackState as ai, type TrajectoryFrame as aj, type FrameCaptureOptions as ak, type FrameCaptureResult as al, type FrameCaptureBlobResult as am, type FrameCaptureAPI as an, type CameraFrameCaptureOptions as ao, type CameraFrameCaptureAPI as ap, type Cameras as aq, type CameraFrameSequenceCamera as ar, type CameraFrameCaptureSource as as, type CameraFrameSequenceOptions as at, type CameraFrameSequenceResult as au, type PolicyCameraFrameStream as av, type PolicyCameraFrameCaptureOptions as aw, type PolicyCameraFrameCaptureResult as ax, type FrameCaptureStatus as ay, type PolicyCameraFrameCaptureAPI as az, type MujocoSimAPI as b, ModelJoints as b0, ModelKeyframes as b1, type ModelOptions as b2, type ModelResource as b3, ModelResources as b4, ModelSensors as b5, ModelSites as b6, type Models as b7, type MujocoContact as b8, type MujocoContactArray as b9, type SiteInfo as bA, type SplatAssetConfig as bB, type SplatScenarioConfig as bC, type StateSnapshot as bD, type TrajectoryData as bE, type TrajectoryFrameCallbackInput as bF, type VisualScenarioMaterialFilterInput as bG, type XmlPatch as bH, getContact as bI, registerModelResources as bJ, type MujocoFrameCaptureOptions as ba, type ObservationLayoutItem as bb, type PhysicsConfig as bc, type PhysicsStepInput as bd, type PolicyActionChunk as be, type PolicyActionInput as bf, type PolicyInferenceInput as bg, type PolicyInferenceOutput as bh, type PolicyInferenceResult as bi, type PolicyObservationInput as bj, type RayHit as bk, type Register as bl, type RegisteredModelMap as bm, type RemotePolicyRequestInfo as bn, type RemotePolicyRequestInput as bo, type RemotePolicyResponseInfo as bp, type RemotePolicyStatus as bq, type ResetCallbackInput as br, type ResolvedScenarioCameraConfig as bs, type ResolvedScenarioMaterialConfig as bt, type ResourceSelector as bu, type ScenarioCameraConfig as bv, type ScenarioMaterialConfig as bw, type SceneMarker as bx, type SceneObject as by, type SensorResult as bz, type StepCallbackInput as c, type SelectionCallbackInput as d, type MujocoModule as e, type MujocoModel as f, type MujocoData as g, type ControlGroupSelector as h, type ObservationResult as i, type IkContextValue as j, type IkGizmoProps as k, type SceneLightsProps as l, type ScenarioLightingProps as m, type SplatEnvironmentProps as n, type VisualScenarioConfig as o, type SplatRendererKind as p, type PairedSplatEnvironmentConfig as q, type SplatFormat as r, type SplatCollisionProxyConfig as s, type SplatEnvironmentReadiness as t, type SplatCollisionPrimitive as u, SplatEnvironmentReadinessStatus as v, type SplatSceneConfigInput as w, type SplatSceneConfigState as x, type VisualScenarioExecutionContextInput as y, type VisualScenarioExecutionContext as z };
+export { type ArrayJointStateResult as $, type ActuatedJointInfo as A, type BodyProps as B, type ControlGroupInfo as C, type DragInteractionProps as D, type ScenarioLightingPreset as E, type SplatEnvironmentMetadataInput as F, type SplatEnvironmentMetadata as G, type SplatSceneInput as H, type IkConfig as I, type DebugProps as J, type GeomInfo as K, type ContactListenerProps as L, type MujocoContextValue as M, type ActuatorInfo as N, type ObservationConfig as O, type PhysicsStepCallback as P, type Sites as Q, type ReadyCallbackInput as R, type SceneConfig as S, type TrajectoryPlayerProps as T, type SitePositionResult as U, type VisualScenarioEffectsProps as V, type Sensors as W, type SensorHandle as X, type SensorInfo as Y, type Joints as Z, type ScalarJointStateResult as _, type MujocoCanvasProps as a, ModelActuators as a$, type JointStateOptions as a0, type JointStateResult as a1, type Bodies as a2, type BodyStateResult as a3, type Geoms as a4, type Actuators as a5, type CtrlHandle as a6, type ContactInfo as a7, type KeyboardTeleopConfig as a8, type KeyboardIkTargetConfig as a9, type CameraFrameSequenceRecorderAPI as aA, type CameraFrameCaptureResult as aB, type CameraFrameCaptureBlobResult as aC, type ImagePointCoordinateSpace as aD, type ImagePointProjectionOptions as aE, type ImagePointProjectionResult as aF, type PolicyVector as aG, type BodyInfo as aH, type CameraFrameCaptureQuaternion as aI, type CameraFrameCaptureVector3 as aJ, type CameraFrameSequenceCameraSummary as aK, type CameraFrameSequenceFrame as aL, type CameraFrameSequenceSampleInput as aM, type CameraFrameSequenceStepInput as aN, type CameraInfo as aO, type ControlJointInfo as aP, type FrameCaptureTarget as aQ, type FrameCaptureTargetRef as aR, type IKSolveFn as aS, type IkGizmoDragInput as aT, type IkSolveInput as aU, type JointInfo as aV, type JointStateKind as aW, type KeyBinding as aX, type KeyboardIkTargetAction as aY, type KeyboardIkTargetBinding as aZ, type Keyframes as a_, type PolicyConfig as aa, type PolicyAPI as ab, type RemotePolicyConfig as ac, type RemotePolicyAPI as ad, type ObservationHandle as ae, type ObservationOutput as af, type TrajectoryInput as ag, type TrajectoryStateChangeInput as ah, type PlaybackState as ai, type TrajectoryFrame as aj, type FrameCaptureOptions as ak, type FrameCaptureResult as al, type FrameCaptureBlobResult as am, type FrameCaptureAPI as an, type CameraFrameCaptureOptions as ao, type CameraFrameCaptureAPI as ap, type Cameras as aq, type CameraFrameSequenceCamera as ar, type CameraFrameCaptureSource as as, type CameraFrameSequenceOptions as at, type CameraFrameSequenceResult as au, type PolicyCameraFrameStream as av, type PolicyCameraFrameCaptureOptions as aw, type PolicyCameraFrameCaptureResult as ax, type FrameCaptureStatus as ay, type PolicyCameraFrameCaptureAPI as az, type MujocoSimAPI as b, ModelBodies as b0, ModelCameras as b1, ModelGeoms as b2, ModelJoints as b3, ModelKeyframes as b4, type ModelOptions as b5, type ModelResource as b6, ModelResources as b7, ModelSensors as b8, ModelSites as b9, type SceneMarker as bA, type SceneObject as bB, type SensorResult as bC, type SiteInfo as bD, type SplatAssetConfig as bE, type SplatScenarioConfig as bF, type StateSnapshot as bG, type TrajectoryData as bH, type TrajectoryFrameCallbackInput as bI, type VisualScenarioMaterialFilterInput as bJ, type XmlPatch as bK, getContact as bL, registerModelResources as bM, type Models as ba, type MujocoContact as bb, type MujocoContactArray as bc, type MujocoFrameCaptureOptions as bd, type ObservationLayoutItem as be, type PhysicsConfig as bf, type PhysicsStepInput as bg, type PolicyActionChunk as bh, type PolicyActionInput as bi, type PolicyInferenceInput as bj, type PolicyInferenceOutput as bk, type PolicyInferenceResult as bl, type PolicyObservationInput as bm, type RayHit as bn, type Register as bo, type RegisteredModelMap as bp, type RemotePolicyRequestInfo as bq, type RemotePolicyRequestInput as br, type RemotePolicyResponseInfo as bs, type RemotePolicyStatus as bt, type ResetCallbackInput as bu, type ResolvedScenarioCameraConfig as bv, type ResolvedScenarioMaterialConfig as bw, type ResourceSelector as bx, type ScenarioCameraConfig as by, type ScenarioMaterialConfig as bz, type StepCallbackInput as c, type SelectionCallbackInput as d, type MujocoModule as e, type MujocoModel as f, type MujocoData as g, type ControlGroupSelector as h, type ObservationResult as i, type IkContextValue as j, type IkGizmoProps as k, type SceneLightsProps as l, type ScenarioLightingProps as m, type SplatEnvironmentProps as n, type VisualScenarioConfig as o, type SplatRendererKind as p, type PairedSplatEnvironmentConfig as q, type SplatFormat as r, type SplatCollisionProxyConfig as s, type SplatEnvironmentReadiness as t, type SplatCollisionPrimitive as u, SplatEnvironmentReadinessStatus as v, type SplatSceneConfigInput as w, type SplatSceneConfigState as x, type VisualScenarioExecutionContextInput as y, type VisualScenarioExecutionContext as z };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mujoco-react",
-  "version": "10.1.0",
+  "version": "10.2.1",
   "description": "Composable React Three Fiber building blocks for MuJoCo WASM simulations",
   "type": "module",
   "main": "./dist/index.js",

package/src/core/MujocoSimProvider.tsx

@@ -31,6 +31,8 @@ import {
   ControlGroupSelector,
   ContactInfo,
   GeomInfo,
+  ImagePointProjectionOptions,
+  ImagePointProjectionResult,
   JointInfo,
   LoadFromFilesOptions,
   LocalMujocoFile,
@@ -61,6 +63,7 @@ import {
   getCameraFrameCaptureSourceTarget,
   isMountedCameraFrameCaptureSource,
 } from '../rendering/cameraFrameSource';
+import { projectImagePointTo3D as projectImagePointTo3DFromScene } from '../rendering/imageProjection';
 import {
   loadScene,
   createSceneConfigFromFiles,
@@ -1590,6 +1593,31 @@ export function MujocoSimProvider({
     [camera, gl]
   );
 
+  const projectImagePointTo3D = useCallback(
+    (options: ImagePointProjectionOptions): ImagePointProjectionResult | null => {
+      const {
+        x,
+        y,
+        coordinateSpace,
+        imageWidth,
+        imageHeight,
+        maxDistance,
+        ...captureOptions
+      } = options;
+      const resolvedCaptureOptions = resolveCameraCaptureOptions(captureOptions);
+      return projectImagePointTo3DFromScene(gl, scene, camera, {
+        ...resolvedCaptureOptions,
+        x,
+        y,
+        coordinateSpace,
+        imageWidth,
+        imageHeight,
+        maxDistance,
+      });
+    },
+    [camera, gl, resolveCameraCaptureOptions, scene]
+  );
+
   // --- Domain randomization ---
 
   const setBodyMass = useCallback((name: string, mass: number): void => {
@@ -1676,6 +1704,7 @@ export function MujocoSimProvider({
       captureCameraFrameBlob: captureCameraFrameBlobApi,
       recordCameraSequence: recordCameraSequenceApi,
       project2DTo3D,
+      projectImagePointTo3D,
       setBodyMass,
       setGeomFriction,
       setGeomSize,
@@ -1696,6 +1725,7 @@ export function MujocoSimProvider({
       captureCameraFrameApi, captureCameraFrameBlobApi,
       recordCameraSequenceApi,
       project2DTo3D,
+      projectImagePointTo3D,
       setBodyMass, setGeomFriction, setGeomSize,
     ]
   );

package/src/hooks/usePolicy.ts

@@ -111,7 +111,10 @@ export function usePolicy(config: PolicyConfig): PolicyAPI {
 
       // Build observation
       const observation = cfg.onObservation({ model, data });
-      const result = cfg.infer ? cfg.infer({ observation, model, data }) : observation;
+      const queuedActions = actionQueueRef.current.length;
+      const result = cfg.infer
+        ? cfg.infer({ observation, model, data, queuedActions })
+        : observation;
 
       if (isPromiseLike(result)) {
         const epoch = epochRef.current;

package/src/hooks/useRemotePolicy.ts

@@ -163,7 +163,7 @@ export function useRemotePolicy(config: RemotePolicyConfig): RemotePolicyAPI {
 
   const policy = usePolicy({
     ...config,
-    infer: async ({ observation, model, data }) => {
+    infer: async ({ observation, model, data, queuedActions }) => {
       const cfg = configRef.current;
       abortController(abortControllerRef.current, createAbortError('Remote policy request was superseded.'));
       const controller = new AbortController();
@@ -175,6 +175,7 @@ export function useRemotePolicy(config: RemotePolicyConfig): RemotePolicyAPI {
         observation,
         model,
         data,
+        queuedActions,
         reset: requestIndex === 0,
         requestIndex,
         signal,

package/src/index.ts

@@ -148,6 +148,10 @@ export {
   createCameraFrameCaptureSession,
   renderCameraFrameToCanvas,
 } from './rendering/cameraFrameCapture';
+export {
+  imagePointToNdc,
+  projectImagePointTo3D,
+} from './rendering/imageProjection';
 export {
   capturePolicyCameraFrames,
   capturePolicyCameraFramesFromMountedStreams,
@@ -269,6 +273,9 @@ export type {
   ContactInfo,
   // Raycast
   RayHit,
+  ImagePointCoordinateSpace,
+  ImagePointProjectionOptions,
+  ImagePointProjectionResult,
   // Model options
   ModelOptions,
   // Trajectory

package/src/rendering/cameraFrameCapture.ts

@@ -60,6 +60,7 @@ export type CameraFrameCaptureRenderResult = {
   width?: number;
   height?: number;
   flipY?: boolean;
+  flipX?: boolean;
 };
 
 type CameraFrameCaptureRender = (
@@ -196,7 +197,8 @@ function readRenderTargetToCanvas(
   imageData: ImageData,
   width: number,
   height: number,
-  outputColorSpace: string
+  outputColorSpace: string,
+  flipX = false
 ) {
   renderer.readRenderTargetPixels(target, 0, 0, width, height, pixels);
 
@@ -206,17 +208,25 @@ function readRenderTargetToCanvas(
     const sourceStart = (height - y - 1) * rowBytes;
     const targetStart = y * rowBytes;
     const row = pixels.subarray(sourceStart, sourceStart + rowBytes);
-    if (!encodeSrgb) {
+    if (!encodeSrgb && !flipX) {
       imageData.data.set(row, targetStart);
       continue;
     }
 
-    for (let x = 0; x < rowBytes; x += 4) {
-      const pixelOffset = targetStart + x;
-      imageData.data[pixelOffset] = linearByteToSrgbByte(row[x]);
-      imageData.data[pixelOffset + 1] = linearByteToSrgbByte(row[x + 1]);
-      imageData.data[pixelOffset + 2] = linearByteToSrgbByte(row[x + 2]);
-      imageData.data[pixelOffset + 3] = row[x + 3];
+    for (let x = 0; x < width; x += 1) {
+      const sourceX = flipX ? width - x - 1 : x;
+      const sourceOffset = sourceX * 4;
+      const targetOffset = targetStart + x * 4;
+      imageData.data[targetOffset] = encodeSrgb
+        ? linearByteToSrgbByte(row[sourceOffset])
+        : row[sourceOffset];
+      imageData.data[targetOffset + 1] = encodeSrgb
+        ? linearByteToSrgbByte(row[sourceOffset + 1])
+        : row[sourceOffset + 1];
+      imageData.data[targetOffset + 2] = encodeSrgb
+        ? linearByteToSrgbByte(row[sourceOffset + 2])
+        : row[sourceOffset + 2];
+      imageData.data[targetOffset + 3] = row[sourceOffset + 3];
     }
   }
   context.putImageData(imageData, 0, 0);
@@ -238,17 +248,30 @@ function readPixelsToCanvas(
   imageData: ImageData,
   width: number,
   height: number,
-  flipY = true
+  flipY = true,
+  flipX = false
 ) {
   const rowBytes = width * 4;
   for (let y = 0; y < height; y += 1) {
     const sourceY = flipY ? height - y - 1 : y;
     const sourceStart = sourceY * rowBytes;
     const targetStart = y * rowBytes;
-    imageData.data.set(
-      pixels.subarray(sourceStart, sourceStart + rowBytes),
-      targetStart
-    );
+    if (!flipX) {
+      imageData.data.set(
+        pixels.subarray(sourceStart, sourceStart + rowBytes),
+        targetStart
+      );
+      continue;
+    }
+    for (let x = 0; x < width; x += 1) {
+      const sourceX = width - x - 1;
+      const sourceOffset = sourceStart + sourceX * 4;
+      const targetOffset = targetStart + x * 4;
+      imageData.data[targetOffset] = pixels[sourceOffset];
+      imageData.data[targetOffset + 1] = pixels[sourceOffset + 1];
+      imageData.data[targetOffset + 2] = pixels[sourceOffset + 2];
+      imageData.data[targetOffset + 3] = pixels[sourceOffset + 3];
+    }
   }
   context.putImageData(imageData, 0, 0);
 }
@@ -462,7 +485,8 @@ export function createCameraFrameCaptureSession(
         imageData,
         width,
         height,
-        renderer.outputColorSpace
+        renderer.outputColorSpace,
+        captureOptions.flipX ?? false
       );
       return {
         canvas,
@@ -524,7 +548,8 @@ export function createCameraFrameCaptureSession(
             imageData,
             width,
             height,
-            captureResult.flipY ?? true
+            captureResult.flipY ?? true,
+            captureResult.flipX ?? captureOptions.flipX ?? false
           );
           return {
             canvas,

package/src/rendering/imageProjection.ts

@@ -0,0 +1,186 @@
+/**
+ * @license
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Project detector/image coordinates from a camera view into the rendered MuJoCo scene.
+ */
+
+import * as THREE from 'three';
+import type {
+  CameraFrameCaptureOptions,
+  CameraFrameCaptureSource,
+  ImagePointCoordinateSpace,
+  ImagePointProjectionOptions,
+  ImagePointProjectionResult,
+} from '../types';
+import { CAPTURE_EXCLUDE_KEY } from './cameraFrameCapture';
+
+const _raycaster = new THREE.Raycaster();
+const _ndc = new THREE.Vector2();
+
+function toVector3(
+  value: CameraFrameCaptureOptions['position'] | undefined,
+  fallback: THREE.Vector3
+) {
+  if (!value) return fallback.clone();
+  return value instanceof THREE.Vector3
+    ? value.clone()
+    : new THREE.Vector3(value[0], value[1], value[2]);
+}
+
+function applyCameraPose(
+  camera: THREE.Camera,
+  options: CameraFrameCaptureOptions,
+  fallbackCamera: THREE.Camera
+) {
+  camera.position.copy(toVector3(options.position, fallbackCamera.position));
+  camera.up.copy(toVector3(options.up, fallbackCamera.up));
+
+  if (options.quaternion) {
+    if (options.quaternion instanceof THREE.Quaternion) {
+      camera.quaternion.copy(options.quaternion);
+    } else {
+      camera.quaternion.set(
+        options.quaternion[0],
+        options.quaternion[1],
+        options.quaternion[2],
+        options.quaternion[3]
+      );
+    }
+  } else if (options.lookAt) {
+    camera.lookAt(toVector3(options.lookAt, new THREE.Vector3()));
+  } else {
+    camera.quaternion.copy(fallbackCamera.quaternion);
+  }
+
+  camera.updateMatrixWorld();
+}
+
+function createProjectionCamera(
+  fallbackCamera: THREE.Camera,
+  options: CameraFrameCaptureOptions,
+  width: number,
+  height: number
+) {
+  const camera = options.camera
+    ? options.camera.clone()
+    : fallbackCamera instanceof THREE.PerspectiveCamera
+      ? fallbackCamera.clone()
+      : new THREE.PerspectiveCamera(45, width / height, 0.01, 100);
+
+  if (camera instanceof THREE.PerspectiveCamera) {
+    camera.aspect = width / height;
+    camera.fov = options.fov ?? camera.fov;
+    camera.near = options.near ?? camera.near;
+    camera.far = options.far ?? camera.far;
+    camera.updateProjectionMatrix();
+  }
+
+  applyCameraPose(camera, options, fallbackCamera);
+  return camera;
+}
+
+function getProjectionSource(options: CameraFrameCaptureOptions): CameraFrameCaptureSource {
+  if (options.source) return options.source;
+  if (options.cameraName) return { kind: 'mujoco-camera', cameraName: options.cameraName };
+  if (options.siteName) return { kind: 'mujoco-site', siteName: options.siteName };
+  if (options.bodyName) return { kind: 'mujoco-body', bodyName: options.bodyName };
+  if (options.camera) return { kind: 'custom-camera' };
+  if (options.position || options.lookAt || options.quaternion) return { kind: 'explicit-pose' };
+  return { kind: 'fallback-camera' };
+}
+
+function imageSize(
+  renderer: THREE.WebGLRenderer,
+  options: ImagePointProjectionOptions
+): [number, number] {
+  return [
+    Math.max(1, Math.floor(options.imageWidth ?? options.width ?? renderer.domElement.width)),
+    Math.max(1, Math.floor(options.imageHeight ?? options.height ?? renderer.domElement.height)),
+  ];
+}
+
+export function imagePointToNdc(
+  x: number,
+  y: number,
+  coordinateSpace: ImagePointCoordinateSpace = 'normalized',
+  width = 1,
+  height = 1
+): [number, number] {
+  if (coordinateSpace === 'ndc') return [x, y];
+  if (coordinateSpace === 'normalized-1000') {
+    return [(x / 1000) * 2 - 1, 1 - (y / 1000) * 2];
+  }
+  if (coordinateSpace === 'pixel') {
+    return [(x / width) * 2 - 1, 1 - (y / height) * 2];
+  }
+  return [x * 2 - 1, 1 - y * 2];
+}
+
+function isProjectionCandidate(object: THREE.Object3D, options: ImagePointProjectionOptions) {
+  if (!object.visible) return false;
+  if (object.userData[CAPTURE_EXCLUDE_KEY]) return false;
+
+  const geomGroup = object.userData.geomGroup;
+  const geomName = object.userData.geomName;
+  if (options.hiddenGeomNames && typeof geomName === 'string' && options.hiddenGeomNames.includes(geomName)) {
+    return false;
+  }
+  if (options.hiddenGeomGroups && typeof geomGroup === 'number' && options.hiddenGeomGroups.includes(geomGroup)) {
+    return false;
+  }
+  if (options.visibleGeomGroups && typeof geomGroup === 'number' && !options.visibleGeomGroups.includes(geomGroup)) {
+    return false;
+  }
+  return true;
+}
+
+function findBodyId(object: THREE.Object3D) {
+  let current: THREE.Object3D | null = object;
+  while (current && current.userData.bodyID === undefined && current.parent) {
+    current = current.parent;
+  }
+  return typeof current?.userData.bodyID === 'number' ? current.userData.bodyID : -1;
+}
+
+export function projectImagePointTo3D(
+  renderer: THREE.WebGLRenderer,
+  scene: THREE.Scene,
+  fallbackCamera: THREE.Camera,
+  options: ImagePointProjectionOptions
+): ImagePointProjectionResult | null {
+  const [width, height] = imageSize(renderer, options);
+  const [ndcX, ndcY] = imagePointToNdc(
+    options.x,
+    options.y,
+    options.coordinateSpace,
+    width,
+    height
+  );
+  const projectionCamera = createProjectionCamera(fallbackCamera, options, width, height);
+
+  scene.updateMatrixWorld(true);
+  _ndc.set(ndcX, ndcY);
+  _raycaster.setFromCamera(_ndc, projectionCamera);
+  _raycaster.far = options.maxDistance ?? Infinity;
+
+  const objects: THREE.Object3D[] = [];
+  scene.traverse((object) => {
+    if ((object as THREE.Mesh).isMesh && isProjectionCandidate(object, options)) {
+      objects.push(object);
+    }
+  });
+
+  const [hit] = _raycaster.intersectObjects(objects, true);
+  if (!hit) return null;
+
+  return {
+    point: hit.point.clone(),
+    bodyId: findBodyId(hit.object),
+    geomId: typeof hit.object.userData.geomID === 'number' ? hit.object.userData.geomID : -1,
+    distance: hit.distance,
+    ndc: [ndcX, ndcY],
+    imageSize: [width, height],
+    source: getProjectionSource(options),
+  };
+}

package/src/types.ts

@@ -747,6 +747,42 @@ export interface RayHit {
   distance: number;
 }
 
+export type ImagePointCoordinateSpace =
+  | 'normalized'
+  | 'normalized-1000'
+  | 'pixel'
+  | 'ndc';
+
+export interface ImagePointProjectionOptions extends CameraFrameCaptureOptions {
+  /** X coordinate in the selected coordinate space. Defaults to normalized 0..1. */
+  x: number;
+  /** Y coordinate in the selected coordinate space. Defaults to normalized 0..1 with origin at top-left. */
+  y: number;
+  /**
+   * Coordinate convention for x/y:
+   * - normalized: 0..1 image coordinates, top-left origin
+   * - normalized-1000: 0..1000 detector coordinates, top-left origin
+   * - pixel: pixel coordinates, top-left origin
+   * - ndc: Three.js normalized device coordinates, -1..1
+   */
+  coordinateSpace?: ImagePointCoordinateSpace;
+  /** Image width for pixel coordinates. Falls back to `width` or renderer canvas width. */
+  imageWidth?: number;
+  /** Image height for pixel coordinates. Falls back to `height` or renderer canvas height. */
+  imageHeight?: number;
+  /** Ignore hits farther than this distance from the camera ray origin. */
+  maxDistance?: number;
+}
+
+export interface ImagePointProjectionResult extends RayHit {
+  /** NDC coordinates used for raycasting. */
+  ndc: [number, number];
+  /** Image dimensions used when interpreting pixel coordinates. */
+  imageSize: [number, number];
+  /** Camera pose provenance, matching camera-frame capture results. */
+  source: CameraFrameCaptureSource;
+}
+
 // ---- Model Options (spec 5.3) ----
 
 export interface ModelOptions {
@@ -797,6 +833,8 @@ export interface PolicyObservationInput {
 
 export interface PolicyInferenceInput extends PolicyObservationInput {
   observation: PolicyVector;
+  /** Number of actions still queued locally when inference is requested. */
+  queuedActions?: number;
 }
 
 export type PolicyActionChunk = readonly PolicyVector[];
@@ -1404,6 +1442,7 @@ export interface MujocoSimAPI {
     cameraPos: THREE.Vector3,
     lookAt: THREE.Vector3
   ): { point: THREE.Vector3; bodyId: number; geomId: number } | null;
+  projectImagePointTo3D(options: ImagePointProjectionOptions): ImagePointProjectionResult | null;
 
   // Domain randomization (spec 10.3)
   setBodyMass(name: Bodies, mass: number): void;
@@ -1502,6 +1541,8 @@ export interface CameraFrameCaptureOptions {
   background?: THREE.ColorRepresentation;
   /** Optional clear alpha for this capture only. Defaults to the renderer's current clear alpha. */
   backgroundAlpha?: number;
+  /** Mirror the captured image horizontally after rendering. Useful when matching policy datasets with mirrored camera frames. */
+  flipX?: boolean;
 }
 
 export type CameraFrameCaptureSource =