mujoco-react 12.1.2 → 12.2.0

package/dist/onnx.d.ts

@@ -1,5 +1,5 @@
 import * as ort from 'onnxruntime-web';
-import { P as PolicyActionChunk } from './types-BvMb-EIC.js';
+import { P as PolicyActionChunk } from './types-CF-BZXtW.js';
 import 'react';
 import '@react-three/fiber';
 import 'three';

package/dist/spark.d.ts

@@ -1,6 +1,6 @@
 import * as react_jsx_runtime from 'react/jsx-runtime';
 import * as _sparkjsdev_spark from '@sparkjsdev/spark';
-import { S as SplatEnvironmentProps, a as PairedSplatEnvironmentConfig, b as SceneConfig, c as SplatEnvironmentReadiness, V as VisualScenarioConfig } from './types-BvMb-EIC.js';
+import { S as SplatEnvironmentProps, a as PairedSplatEnvironmentConfig, b as SceneConfig, c as SplatEnvironmentReadiness, V as VisualScenarioConfig } from './types-CF-BZXtW.js';
 import 'react';
 import '@react-three/fiber';
 import 'three';

package/dist/spark.js

@@ -1,4 +1,4 @@
-import { useSplatSceneConfig, useSplatEnvironment, SplatEnvironment, CAMERA_FRAME_CAPTURE_RENDER_USER_DATA_KEY } from './chunk-CV6LBHC4.js';
+import { useSplatSceneConfig, useSplatEnvironment, SplatEnvironment, CAMERA_FRAME_CAPTURE_RENDER_USER_DATA_KEY } from './chunk-SP72GTOD.js';
 import { useThree } from '@react-three/fiber';
 import { useMemo, useState, useEffect, useCallback, useRef } from 'react';
 import * as THREE from 'three';

package/dist/{types-BvMb-EIC.d.ts → types-CF-BZXtW.d.ts}

@@ -124,6 +124,25 @@ interface CameraFramePixelsResult {
     height: number;
     source: CameraFrameCaptureSource;
 }
+/** An owned, aligned RGB-D observation from one render. Scene units must be meters. */
+interface CameraRGBDResult {
+    /** Interleaved sRGB bytes (RGB), row-major, top-left origin. */
+    rgb: Uint8Array;
+    /** Positive camera-axis distance (-view Z) in meters, not Euclidean range.
+     * Background / no depth-writing surface is +Infinity. Same pixel order as rgb. */
+    depth: Float32Array;
+    width: number;
+    height: number;
+    source: CameraFrameCaptureSource;
+    origin: 'top-left';
+    depthUnits: 'meters';
+    depthConvention: 'camera-z';
+    /** Horizontal mirroring applied to both arrays. Undo before unprojecting. */
+    flipX: boolean;
+    /** Owned column-major matrices. Camera coordinates: +X right, +Y up, -Z forward. */
+    projectionMatrix: number[];
+    cameraWorldMatrix: number[];
+}
 interface CameraFrameTensorResult extends PolicyImageTensorResult {
     camera: THREE.Camera;
     source: CameraFrameCaptureSource;
@@ -156,6 +175,8 @@ interface CameraFrameCaptureSession {
     capturePixels(options?: CameraFrameCaptureOptions): CameraFramePixelsResult;
     /** Render straight into a normalized policy image tensor (no canvas/PNG encode). */
     captureTensor(options?: CameraFrameCaptureTensorOptions): CameraFrameTensorResult;
+    /** Synchronous RGB + metric depth from one render. Returned arrays are owned. */
+    captureRGBD(options?: CameraFrameCaptureOptions): CameraRGBDResult;
     dispose(): void;
 }
 declare const CAMERA_FRAME_CAPTURE_RENDER_USER_DATA_KEY = "mujocoReactCameraFrameCaptureRender";
@@ -178,6 +199,8 @@ declare function captureCameraFrameBlob(renderer: THREE.WebGLRenderer, scene: TH
  * `session.captureTensor()` so the render target and buffers are reused.
  */
 declare function captureCameraFrameTensor(renderer: THREE.WebGLRenderer, scene: THREE.Scene, fallbackCamera: THREE.Camera, options?: CameraFrameCaptureTensorOptions): CameraFrameTensorResult;
+/** One-shot aligned RGB-D observation. Use a session for repeated observations. */
+declare function captureCameraRGBD(renderer: THREE.WebGLRenderer, scene: THREE.Scene, fallbackCamera: THREE.Camera, options?: CameraFrameCaptureOptions): CameraRGBDResult;
 
 /**
  * @license
@@ -1401,6 +1424,8 @@ interface MujocoSimAPI {
     captureFrameBlob(options?: MujocoFrameCaptureOptions): Promise<FrameCaptureBlobResult>;
     captureCameraFrame(options?: CameraFrameCaptureOptions): Promise<CameraFrameCaptureResult>;
     captureCameraFrameBlob(options?: CameraFrameCaptureOptions): Promise<CameraFrameCaptureBlobResult>;
+    /** Aligned sRGB + camera-axis depth in meters from one render. */
+    captureCameraRGBD(options?: CameraFrameCaptureOptions): CameraRGBDResult;
     /** Capture a camera frame straight into a policy image tensor (no canvas/PNG encode). */
     captureCameraFrameTensor(options?: CameraFrameCaptureTensorOptions): CameraFrameTensorResult;
     /**
@@ -1621,6 +1646,7 @@ interface CameraFrameCaptureAPI {
     isCapturing: boolean;
     capture: (options?: CameraFrameCaptureOptions) => Promise<CameraFrameCaptureResult>;
     captureBlob: (options?: CameraFrameCaptureOptions) => Promise<CameraFrameCaptureBlobResult>;
+    captureRGBD: (options?: CameraFrameCaptureOptions) => Promise<CameraRGBDResult>;
     reset: () => void;
 }
 interface CameraFrameSequenceCamera extends CameraFrameCaptureOptions {
@@ -1787,4 +1813,4 @@ interface ArrayJointStateResult {
     velocity: React__default.RefObject<Float64Array>;
 }
 
-export { type SensorHandle as $, type ActuatedJointInfo as A, type BodyProps as B, type ControlGroupInfo as C, type DragInteractionProps as D, type SplatSceneConfigState as E, type VisualScenarioExecutionContextInput as F, type VisualScenarioExecutionContext as G, type ScenarioLightingPreset as H, type IkConfig as I, type SplatEnvironmentMetadataInput as J, type SplatEnvironmentMetadata as K, type SplatSceneInput as L, type MujocoContextValue as M, type CameraFrameCaptureOptions as N, type ObservationConfig as O, type PolicyActionChunk as P, type DebugProps as Q, type ReadyCallbackInput as R, type SplatEnvironmentProps as S, type GeomInfo as T, type ContactListenerProps as U, type VisualScenarioConfig as V, type TrajectoryPlayerProps as W, type ActuatorInfo as X, type Sites as Y, type SitePositionResult as Z, type Sensors as _, type PairedSplatEnvironmentConfig as a, type ControlJointInfo as a$, type SensorInfo as a0, type Joints as a1, type ScalarJointStateResult as a2, type ArrayJointStateResult as a3, type JointStateOptions as a4, type JointStateResult as a5, type Bodies as a6, type BodyStateResult as a7, type Geoms as a8, type Actuators as a9, type PolicyCameraFrameCaptureResult as aA, type FrameCaptureStatus as aB, type PolicyCameraFrameCaptureAPI as aC, type CameraFrameCaptureTensorOptions as aD, type CameraFrameTensorResult as aE, type CameraFrameSequenceRecorderAPI as aF, type ImagePointCoordinateSpace as aG, type ImagePointProjectionOptions as aH, type ImagePointProjectionResult as aI, type PolicyVector as aJ, type ActuatorControlValue as aK, type AdvanceOptions as aL, type BodyInfo as aM, CAMERA_FRAME_CAPTURE_PRE_RENDER_USER_DATA_KEY as aN, CAMERA_FRAME_CAPTURE_RENDER_USER_DATA_KEY as aO, CAPTURE_EXCLUDE_KEY as aP, type CameraFrameCaptureBlobResult as aQ, type CameraFrameCaptureQuaternion as aR, type CameraFrameCaptureResult as aS, type CameraFrameCaptureSession as aT, type CameraFrameCaptureVector3 as aU, type CameraFramePixelsResult as aV, type CameraFrameSequenceCameraSummary as aW, type CameraFrameSequenceFrame as aX, type CameraFrameSequenceSampleInput as aY, type CameraFrameSequenceStepInput as aZ, type CameraInfo as a_, type CtrlHandle as aa, type ContactInfo as ab, type KeyboardTeleopConfig as ac, type KeyboardIkTargetConfig as ad, type PolicyConfig as ae, type PolicyAPI as af, type RemotePolicyConfig as ag, type RemotePolicyAPI as ah, type ObservationHandle as ai, type ObservationOutput as aj, type TrajectoryInput as ak, type TrajectoryStateChangeInput as al, type PlaybackState as am, type TrajectoryFrame as an, type FrameCaptureOptions as ao, type FrameCaptureResult as ap, type FrameCaptureBlobResult as aq, type FrameCaptureAPI as ar, type CameraFrameCaptureAPI as as, type Cameras as at, type CameraFrameSequenceCamera as au, type CameraFrameCaptureSource as av, type CameraFrameSequenceOptions as aw, type CameraFrameSequenceResult as ax, type PolicyCameraFrameStream as ay, type PolicyCameraFrameCaptureOptions as az, type SceneConfig as b, type SiteInfo as b$, type DebugVirtualCamera as b0, type FrameCaptureTarget as b1, type FrameCaptureTargetRef as b2, type IKSolveFn as b3, type IkGizmoDragInput as b4, type IkSolveInput as b5, type JointInfo as b6, type JointStateKind as b7, type KeyBinding as b8, type KeyboardIkTargetAction as b9, type PolicyImageTensorResult as bA, type PolicyImageTensorSourceOrigin as bB, type PolicyInferenceInput as bC, type PolicyInferenceOutput as bD, type PolicyInferenceResult as bE, type PolicyObservationInput as bF, type RayHit as bG, type Register as bH, type RegisteredModelMap as bI, type RemotePolicyRequestInfo as bJ, type RemotePolicyRequestInput as bK, type RemotePolicyResponseInfo as bL, type RemotePolicyStatus as bM, type ResetCallbackInput as bN, type ResolvedScenarioCameraConfig as bO, type ResolvedScenarioMaterialConfig as bP, type ResourceSelector as bQ, type RuntimeMetrics as bR, type RuntimeOptions as bS, type RuntimeSnapshot as bT, type RuntimeState as bU, type ScenarioCameraConfig as bV, type ScenarioMaterialConfig as bW, type SceneMarker as bX, type SceneObject as bY, type SensorResult as bZ, type SimulationRuntime as b_, type KeyboardIkTargetBinding as ba, type Keyframes as bb, ModelActuators as bc, ModelBodies as bd, ModelCameras as be, ModelGeoms as bf, ModelJoints as bg, ModelKeyframes as bh, type ModelOptions as bi, type ModelResource as bj, ModelResources as bk, ModelSensors as bl, ModelSites as bm, type Models as bn, type MujocoContact as bo, type MujocoContactArray as bp, type MujocoFrameCaptureOptions as bq, type ObservationLayoutItem as br, type PhysicsConfig as bs, type PhysicsPerformance as bt, type PhysicsStepInput as bu, type PolicyActionInput as bv, type PolicyImageTensorLayout as bw, type PolicyImageTensorOptions as bx, type PolicyImageTensorPixelOptions as by, type PolicyImageTensorRange as bz, type SplatEnvironmentReadiness as c, type SplatAssetConfig as c0, type SplatScenarioConfig as c1, type StateSnapshot as c2, type TrajectoryData as c3, type TrajectoryFrameCallbackInput as c4, type VisualScenarioMaterialFilterInput as c5, type WorkerPort as c6, type XmlPatch as c7, captureCameraFrame as c8, captureCameraFrameBlob as c9, captureCameraFrameTensor as ca, createCameraFrameCaptureSession as cb, dataUrlToPolicyImageTensor as cc, getContact as cd, imageDataToPolicyImageTensor as ce, pixelsToPolicyImageTensor as cf, registerModelResources as cg, renderCameraFrameToCanvas as ch, type MujocoCanvasProps as d, type MujocoSimAPI as e, type StepCallbackInput as f, type SelectionCallbackInput as g, type MujocoIntegrator as h, type MujocoModule as i, type MujocoRenderOptions as j, type PhysicsStepCallback as k, type MujocoModel as l, type MujocoData as m, type ControlGroupSelector as n, type ObservationResult as o, type IkContextValue as p, type IkGizmoProps as q, type SceneLightsProps as r, type ScenarioLightingProps as s, type VisualScenarioEffectsProps as t, type SplatRendererKind as u, type SplatFormat as v, type SplatCollisionProxyConfig as w, type SplatCollisionPrimitive as x, SplatEnvironmentReadinessStatus as y, type SplatSceneConfigInput as z };
+export { type SensorHandle as $, type ActuatedJointInfo as A, type BodyProps as B, type ControlGroupInfo as C, type DragInteractionProps as D, type SplatSceneConfigState as E, type VisualScenarioExecutionContextInput as F, type VisualScenarioExecutionContext as G, type ScenarioLightingPreset as H, type IkConfig as I, type SplatEnvironmentMetadataInput as J, type SplatEnvironmentMetadata as K, type SplatSceneInput as L, type MujocoContextValue as M, type CameraFrameCaptureOptions as N, type ObservationConfig as O, type PolicyActionChunk as P, type DebugProps as Q, type ReadyCallbackInput as R, type SplatEnvironmentProps as S, type GeomInfo as T, type ContactListenerProps as U, type VisualScenarioConfig as V, type TrajectoryPlayerProps as W, type ActuatorInfo as X, type Sites as Y, type SitePositionResult as Z, type Sensors as _, type PairedSplatEnvironmentConfig as a, type CameraRGBDResult as a$, type SensorInfo as a0, type Joints as a1, type ScalarJointStateResult as a2, type ArrayJointStateResult as a3, type JointStateOptions as a4, type JointStateResult as a5, type Bodies as a6, type BodyStateResult as a7, type Geoms as a8, type Actuators as a9, type PolicyCameraFrameCaptureResult as aA, type FrameCaptureStatus as aB, type PolicyCameraFrameCaptureAPI as aC, type CameraFrameCaptureTensorOptions as aD, type CameraFrameTensorResult as aE, type CameraFrameSequenceRecorderAPI as aF, type ImagePointCoordinateSpace as aG, type ImagePointProjectionOptions as aH, type ImagePointProjectionResult as aI, type PolicyVector as aJ, type ActuatorControlValue as aK, type AdvanceOptions as aL, type BodyInfo as aM, CAMERA_FRAME_CAPTURE_PRE_RENDER_USER_DATA_KEY as aN, CAMERA_FRAME_CAPTURE_RENDER_USER_DATA_KEY as aO, CAPTURE_EXCLUDE_KEY as aP, type CameraFrameCaptureBlobResult as aQ, type CameraFrameCaptureQuaternion as aR, type CameraFrameCaptureResult as aS, type CameraFrameCaptureSession as aT, type CameraFrameCaptureVector3 as aU, type CameraFramePixelsResult as aV, type CameraFrameSequenceCameraSummary as aW, type CameraFrameSequenceFrame as aX, type CameraFrameSequenceSampleInput as aY, type CameraFrameSequenceStepInput as aZ, type CameraInfo as a_, type CtrlHandle as aa, type ContactInfo as ab, type KeyboardTeleopConfig as ac, type KeyboardIkTargetConfig as ad, type PolicyConfig as ae, type PolicyAPI as af, type RemotePolicyConfig as ag, type RemotePolicyAPI as ah, type ObservationHandle as ai, type ObservationOutput as aj, type TrajectoryInput as ak, type TrajectoryStateChangeInput as al, type PlaybackState as am, type TrajectoryFrame as an, type FrameCaptureOptions as ao, type FrameCaptureResult as ap, type FrameCaptureBlobResult as aq, type FrameCaptureAPI as ar, type CameraFrameCaptureAPI as as, type Cameras as at, type CameraFrameSequenceCamera as au, type CameraFrameCaptureSource as av, type CameraFrameSequenceOptions as aw, type CameraFrameSequenceResult as ax, type PolicyCameraFrameStream as ay, type PolicyCameraFrameCaptureOptions as az, type SceneConfig as b, type SimulationRuntime as b$, type ControlJointInfo as b0, type DebugVirtualCamera as b1, type FrameCaptureTarget as b2, type FrameCaptureTargetRef as b3, type IKSolveFn as b4, type IkGizmoDragInput as b5, type IkSolveInput as b6, type JointInfo as b7, type JointStateKind as b8, type KeyBinding as b9, type PolicyImageTensorRange as bA, type PolicyImageTensorResult as bB, type PolicyImageTensorSourceOrigin as bC, type PolicyInferenceInput as bD, type PolicyInferenceOutput as bE, type PolicyInferenceResult as bF, type PolicyObservationInput as bG, type RayHit as bH, type Register as bI, type RegisteredModelMap as bJ, type RemotePolicyRequestInfo as bK, type RemotePolicyRequestInput as bL, type RemotePolicyResponseInfo as bM, type RemotePolicyStatus as bN, type ResetCallbackInput as bO, type ResolvedScenarioCameraConfig as bP, type ResolvedScenarioMaterialConfig as bQ, type ResourceSelector as bR, type RuntimeMetrics as bS, type RuntimeOptions as bT, type RuntimeSnapshot as bU, type RuntimeState as bV, type ScenarioCameraConfig as bW, type ScenarioMaterialConfig as bX, type SceneMarker as bY, type SceneObject as bZ, type SensorResult as b_, type KeyboardIkTargetAction as ba, type KeyboardIkTargetBinding as bb, type Keyframes as bc, ModelActuators as bd, ModelBodies as be, ModelCameras as bf, ModelGeoms as bg, ModelJoints as bh, ModelKeyframes as bi, type ModelOptions as bj, type ModelResource as bk, ModelResources as bl, ModelSensors as bm, ModelSites as bn, type Models as bo, type MujocoContact as bp, type MujocoContactArray as bq, type MujocoFrameCaptureOptions as br, type ObservationLayoutItem as bs, type PhysicsConfig as bt, type PhysicsPerformance as bu, type PhysicsStepInput as bv, type PolicyActionInput as bw, type PolicyImageTensorLayout as bx, type PolicyImageTensorOptions as by, type PolicyImageTensorPixelOptions as bz, type SplatEnvironmentReadiness as c, type SiteInfo as c0, type SplatAssetConfig as c1, type SplatScenarioConfig as c2, type StateSnapshot as c3, type TrajectoryData as c4, type TrajectoryFrameCallbackInput as c5, type VisualScenarioMaterialFilterInput as c6, type WorkerPort as c7, type XmlPatch as c8, captureCameraFrame as c9, captureCameraFrameBlob as ca, captureCameraFrameTensor as cb, captureCameraRGBD as cc, createCameraFrameCaptureSession as cd, dataUrlToPolicyImageTensor as ce, getContact as cf, imageDataToPolicyImageTensor as cg, pixelsToPolicyImageTensor as ch, registerModelResources as ci, renderCameraFrameToCanvas as cj, type MujocoCanvasProps as d, type MujocoSimAPI as e, type StepCallbackInput as f, type SelectionCallbackInput as g, type MujocoIntegrator as h, type MujocoModule as i, type MujocoRenderOptions as j, type PhysicsStepCallback as k, type MujocoModel as l, type MujocoData as m, type ControlGroupSelector as n, type ObservationResult as o, type IkContextValue as p, type IkGizmoProps as q, type SceneLightsProps as r, type ScenarioLightingProps as s, type VisualScenarioEffectsProps as t, type SplatRendererKind as u, type SplatFormat as v, type SplatCollisionProxyConfig as w, type SplatCollisionPrimitive as x, SplatEnvironmentReadinessStatus as y, type SplatSceneConfigInput as z };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mujoco-react",
-  "version": "12.1.2",
+  "version": "12.2.0",
   "description": "Composable React Three Fiber building blocks for MuJoCo WASM simulations",
   "type": "module",
   "main": "./dist/index.js",

package/src/core/MujocoSimProvider.tsx

@@ -67,6 +67,7 @@ import {
 import {
   captureCameraFrame,
   captureCameraFrameBlob,
+  captureCameraRGBD,
   createCameraFrameCaptureSession,
   type CameraFrameCaptureTensorOptions,
 } from '../rendering/cameraFrameCapture';
@@ -1583,6 +1584,12 @@ export function MujocoSimProvider({
     [camera, gl, resolveCameraCaptureOptions, scene]
   );
 
+  const captureCameraRGBDApi = useCallback(
+    (options: CameraFrameCaptureOptions = {}) =>
+      captureCameraRGBD(gl, scene, camera, resolveCameraCaptureOptions(options)),
+    [camera, gl, resolveCameraCaptureOptions, scene]
+  );
+
   const createCameraFrameCaptureSessionApi = useCallback(
     (options: CameraFrameCaptureOptions = {}) =>
       createCameraFrameCaptureSession(
@@ -1950,6 +1957,7 @@ export function MujocoSimProvider({
       captureCameraFrame: captureCameraFrameApi,
       captureCameraFrameBlob: captureCameraFrameBlobApi,
       captureCameraFrameTensor: captureCameraFrameTensorApi,
+      captureCameraRGBD: captureCameraRGBDApi,
       createCameraFrameCaptureSession: createCameraFrameCaptureSessionApi,
       resolveCameraCaptureOptions,
       recordCameraSequence: recordCameraSequenceApi,
@@ -1973,7 +1981,7 @@ export function MujocoSimProvider({
       loadFromFilesApi, addBodyApi, removeBodyApi, recompileApi,
       getCanvas, getCanvasSnapshot, captureFrameApi, captureFrameBlobApi,
       captureCameraFrameApi, captureCameraFrameBlobApi,
-      captureCameraFrameTensorApi, createCameraFrameCaptureSessionApi,
+      captureCameraFrameTensorApi, captureCameraRGBDApi, createCameraFrameCaptureSessionApi,
       resolveCameraCaptureOptions,
       recordCameraSequenceApi,
       project2DTo3D,

package/src/hooks/useCameraFrameCapture.ts

@@ -83,12 +83,29 @@ export function useCameraFrameCapture(
     [defaultOptions, mujoco.api]
   );
 
+  async function captureRGBD(options: CameraFrameCaptureOptions = {}) {
+    setStatus('capturing');
+    setError(null);
+    try {
+      if (!mujoco.api) throw new Error('MuJoCo scene is not ready for camera frame capture.');
+      const result = mujoco.api.captureCameraRGBD({ ...defaultOptions, ...options });
+      setStatus('captured');
+      return result;
+    } catch (nextError) {
+      const error = nextError instanceof Error ? nextError : new Error('Unable to capture RGB-D.');
+      setError(error);
+      setStatus('error');
+      throw error;
+    }
+  }
+
   return {
     status,
     error,
     isCapturing: status === 'capturing',
     capture,
     captureBlob,
+    captureRGBD,
     reset,
   };
 }

package/src/index.ts

@@ -172,6 +172,7 @@ export {
   captureCameraFrame,
   captureCameraFrameBlob,
   captureCameraFrameTensor,
+  captureCameraRGBD,
   createCameraFrameCaptureSession,
   renderCameraFrameToCanvas,
 } from './rendering/cameraFrameCapture';
@@ -179,6 +180,7 @@ export type {
   CameraFrameCaptureSession,
   CameraFrameCaptureTensorOptions,
   CameraFramePixelsResult,
+  CameraRGBDResult,
   CameraFrameTensorResult,
 } from './rendering/cameraFrameCapture';
 export {

package/src/rendering/cameraFrameCapture.ts

@@ -6,6 +6,7 @@
  */
 
 import * as THREE from 'three';
+import { createMetricDepthReader } from './metricDepth';
 import type {
   CameraFrameCaptureBlobResult,
   CameraFrameCaptureOptions,
@@ -32,6 +33,26 @@ export interface CameraFramePixelsResult {
   source: CameraFrameCaptureSource;
 }
 
+/** An owned, aligned RGB-D observation from one render. Scene units must be meters. */
+export interface CameraRGBDResult {
+  /** Interleaved sRGB bytes (RGB), row-major, top-left origin. */
+  rgb: Uint8Array;
+  /** Positive camera-axis distance (-view Z) in meters, not Euclidean range.
+   * Background / no depth-writing surface is +Infinity. Same pixel order as rgb. */
+  depth: Float32Array;
+  width: number;
+  height: number;
+  source: CameraFrameCaptureSource;
+  origin: 'top-left';
+  depthUnits: 'meters';
+  depthConvention: 'camera-z';
+  /** Horizontal mirroring applied to both arrays. Undo before unprojecting. */
+  flipX: boolean;
+  /** Owned column-major matrices. Camera coordinates: +X right, +Y up, -Z forward. */
+  projectionMatrix: number[];
+  cameraWorldMatrix: number[];
+}
+
 export interface CameraFrameTensorResult extends PolicyImageTensorResult {
   camera: THREE.Camera;
   source: CameraFrameCaptureSource;
@@ -67,6 +88,8 @@ export interface CameraFrameCaptureSession {
   capturePixels(options?: CameraFrameCaptureOptions): CameraFramePixelsResult;
   /** Render straight into a normalized policy image tensor (no canvas/PNG encode). */
   captureTensor(options?: CameraFrameCaptureTensorOptions): CameraFrameTensorResult;
+  /** Synchronous RGB + metric depth from one render. Returned arrays are owned. */
+  captureRGBD(options?: CameraFrameCaptureOptions): CameraRGBDResult;
   dispose(): void;
 }
 
@@ -306,6 +329,11 @@ export function createCaptureCamera(
     camera.far = options.far ?? camera.far;
     camera.updateProjectionMatrix();
   }
+  if (camera instanceof THREE.OrthographicCamera) {
+    camera.near = options.near ?? camera.near;
+    camera.far = options.far ?? camera.far;
+    camera.updateProjectionMatrix();
+  }
   applyProjectionMatrix(camera, options.projectionMatrix);
 
   applyCameraPose(camera, options, fallbackCamera);
@@ -345,6 +373,11 @@ export function prepareCaptureCamera(
     camera.far = options.far ?? camera.far;
     camera.updateProjectionMatrix();
   }
+  if (camera instanceof THREE.OrthographicCamera) {
+    camera.near = options.near ?? camera.near;
+    camera.far = options.far ?? camera.far;
+    camera.updateProjectionMatrix();
+  }
   applyProjectionMatrix(camera, options.projectionMatrix);
 
   applyCameraPose(camera, options, fallbackCamera);
@@ -653,12 +686,14 @@ export function createCameraFrameCaptureSession(
 
   const pixels = new Uint8Array(width * height * 4);
   const imageData = drawContext.createImageData(width, height);
+  let depthReader: ReturnType<typeof createMetricDepthReader> | undefined;
   let inFlight = false, disposed = false;
   function assertAvailable() {
     if (disposed) throw new Error('Camera capture session is disposed');
     if (inFlight) throw new Error('Camera capture session is busy; await the current capture');
   }
   function releaseResources() {
+    depthReader?.dispose();
     target.dispose();
     if (isolatedRenderer && !isolatedRenderer.cached) isolatedRenderer.renderer.dispose();
   }
@@ -727,8 +762,7 @@ export function createCameraFrameCaptureSession(
   ) {
     withCaptureState(captureOptions, () => {
       sessionRenderer.setRenderTarget(target);
-      sessionRenderer.setViewport(0, 0, width, height);
-      sessionRenderer.setScissor(0, 0, width, height);
+      // The render target installs its pixel viewport; setters would apply DPR twice.
       sessionRenderer.setScissorTest(false);
       sessionRenderer.clear();
       sessionRenderer.render(scene, camera);
@@ -780,6 +814,51 @@ export function createCameraFrameCaptureSession(
     };
   }
 
+  function captureRGBD(nextOptions: CameraFrameCaptureOptions = {}): CameraRGBDResult {
+    assertAvailable();
+    const captureOptions = resolveCaptureOptions(nextOptions);
+    if (getCaptureRenderer(scene)) {
+      throw new Error('Metric RGB-D capture requires the standard Three renderer; custom capture renderers do not provide aligned depth.');
+    }
+    if (sessionRenderer.capabilities.logarithmicDepthBuffer || sessionRenderer.capabilities.reversedDepthBuffer) {
+      throw new Error('Metric RGB-D capture does not support logarithmic or reversed depth buffers.');
+    }
+    if (!camera.projectionMatrix.elements.every(Number.isFinite) ||
+        !Number.isFinite(camera.projectionMatrix.determinant()) || camera.projectionMatrix.determinant() === 0) {
+      throw new Error('Metric RGB-D capture requires a finite, invertible projection matrix.');
+    }
+    if (!depthReader) {
+      // Recreate the framebuffer if this session has already rendered RGB-only frames.
+      target.dispose();
+      target.depthTexture = new THREE.DepthTexture(width, height, THREE.UnsignedIntType);
+      depthReader = createMetricDepthReader(width, height, target.depthTexture);
+    }
+    const rgb = new Uint8Array(width * height * 3);
+    let depth!: Float32Array;
+    renderCaptureToTarget(captureOptions, () => {
+      sessionRenderer.readRenderTargetPixels(target, 0, 0, width, height, pixels);
+      for (let y = 0; y < height; y++) {
+        for (let x = 0; x < width; x++) {
+          const sx = captureOptions.flipX ? width - x - 1 : x;
+          const src = ((height - y - 1) * width + sx) * 4;
+          const dst = (y * width + x) * 3;
+          for (let c = 0; c < 3; c++) {
+            rgb[dst + c] = linearByteToSrgbByte(pixels[src + c]);
+          }
+        }
+      }
+      depth = depthReader!.read(sessionRenderer, camera, captureOptions.flipX ?? false);
+    });
+    return {
+      rgb, depth, width, height,
+      source: getCameraFrameCaptureSource(captureOptions),
+      origin: 'top-left', depthUnits: 'meters', depthConvention: 'camera-z',
+      flipX: captureOptions.flipX ?? false,
+      projectionMatrix: camera.projectionMatrix.toArray(),
+      cameraWorldMatrix: camera.matrixWorld.toArray(),
+    };
+  }
+
   function captureTensor(
     nextOptions: CameraFrameCaptureTensorOptions = {}
   ): CameraFrameTensorResult {
@@ -857,6 +936,7 @@ export function createCameraFrameCaptureSession(
     captureAsync,
     capturePixels,
     captureTensor,
+    captureRGBD,
     captureDataUrl(nextOptions = {}) {
       const type = nextOptions.type ?? options.type ?? 'image/png';
       const result = capture(nextOptions);
@@ -991,3 +1071,17 @@ export function captureCameraFrameTensor(
     session.dispose();
   }
 }
+/** One-shot aligned RGB-D observation. Use a session for repeated observations. */
+export function captureCameraRGBD(
+  renderer: THREE.WebGLRenderer,
+  scene: THREE.Scene,
+  fallbackCamera: THREE.Camera,
+  options: CameraFrameCaptureOptions = {}
+): CameraRGBDResult {
+  const session = createCameraFrameCaptureSession(renderer, scene, fallbackCamera, options);
+  try {
+    return session.captureRGBD();
+  } finally {
+    session.dispose();
+  }
+}

package/src/rendering/metricDepth.ts

@@ -0,0 +1,56 @@
+/** @license SPDX-License-Identifier: Apache-2.0 */
+import * as THREE from 'three';
+
+/** Internal GPU readback adapter. Read the real depth attachment, preserving
+ * alpha tests, displacement, skinning, and depthWrite from the RGB render. */
+export function createMetricDepthReader(width: number, height: number, texture: THREE.DepthTexture) {
+  const target = new THREE.WebGLRenderTarget(width, height, { depthBuffer: false });
+  const material = new THREE.ShaderMaterial({
+    uniforms: { depthTexture: { value: texture } },
+    vertexShader: `varying vec2 vUv;
+      void main() { vUv = uv; gl_Position = vec4(position.xy, 0.0, 1.0); }`,
+    fragmentShader: `#include <packing>
+      uniform sampler2D depthTexture;
+      varying vec2 vUv;
+      void main() { gl_FragColor = packDepthToRGBA(texture2D(depthTexture, vUv).r); }`,
+    depthTest: false, depthWrite: false, blending: THREE.NoBlending,
+    toneMapped: false,
+  });
+  const geometry = new THREE.PlaneGeometry(2, 2);
+  const scene = new THREE.Scene();
+  scene.add(new THREE.Mesh(geometry, material));
+  const screenCamera = new THREE.Camera();
+  const packed = new Uint8Array(width * height * 4);
+  return {
+    read(renderer: THREE.WebGLRenderer, camera: THREE.Camera, flipX: boolean) {
+      renderer.setRenderTarget(target);
+      renderer.clear();
+      renderer.render(scene, screenCamera);
+      renderer.readRenderTargetPixels(target, 0, 0, width, height, packed);
+      const result = new Float32Array(width * height);
+      const inverse = camera.projectionMatrixInverse.elements;
+      for (let y = 0; y < height; y++) {
+        for (let x = 0; x < width; x++) {
+          const sx = flipX ? width - x - 1 : x;
+          const sy = height - y - 1;
+          const offset = (sy * width + sx) * 4;
+          const r = packed[offset], g = packed[offset + 1], b = packed[offset + 2], a = packed[offset + 3];
+          if (r === 255 && g === 255 && b === 255 && a === 255) {
+            result[y * width + x] = Infinity;
+            continue;
+          }
+          // Mirrors Three's unpackRGBAToDepth (packing.glsl.js).
+          const d = r / 256 + g / 65536 + b / 16777216 + a / (255 * 16777216);
+          const nx = (sx + 0.5) / width * 2 - 1;
+          const ny = (sy + 0.5) / height * 2 - 1;
+          const nz = d * 2 - 1;
+          const z = inverse[2] * nx + inverse[6] * ny + inverse[10] * nz + inverse[14];
+          const w = inverse[3] * nx + inverse[7] * ny + inverse[11] * nz + inverse[15];
+          result[y * width + x] = -z / w;
+        }
+      }
+      return result;
+    },
+    dispose() { target.dispose(); material.dispose(); geometry.dispose(); },
+  };
+}

package/src/types.ts

@@ -13,6 +13,7 @@ import type {
   CameraFrameCaptureSession,
   CameraFrameCaptureTensorOptions,
   CameraFrameTensorResult,
+  CameraRGBDResult,
 } from './rendering/cameraFrameCapture';
 
 // ---- Register (type-safe named resources) ----
@@ -1548,6 +1549,8 @@ export interface MujocoSimAPI {
   captureFrameBlob(options?: MujocoFrameCaptureOptions): Promise<FrameCaptureBlobResult>;
   captureCameraFrame(options?: CameraFrameCaptureOptions): Promise<CameraFrameCaptureResult>;
   captureCameraFrameBlob(options?: CameraFrameCaptureOptions): Promise<CameraFrameCaptureBlobResult>;
+  /** Aligned sRGB + camera-axis depth in meters from one render. */
+  captureCameraRGBD(options?: CameraFrameCaptureOptions): CameraRGBDResult;
   /** Capture a camera frame straight into a policy image tensor (no canvas/PNG encode). */
   captureCameraFrameTensor(options?: CameraFrameCaptureTensorOptions): CameraFrameTensorResult;
   /**
@@ -1796,6 +1799,7 @@ export interface CameraFrameCaptureAPI {
   captureBlob: (
     options?: CameraFrameCaptureOptions
   ) => Promise<CameraFrameCaptureBlobResult>;
+  captureRGBD: (options?: CameraFrameCaptureOptions) => Promise<CameraRGBDResult>;
   reset: () => void;
 }