mujoco-react 9.1.0 → 9.2.0

package/dist/spark.d.ts

@@ -1,6 +1,6 @@
 import * as react_jsx_runtime from 'react/jsx-runtime';
 import * as _sparkjsdev_spark from '@sparkjsdev/spark';
-import { n as SplatEnvironmentProps } from './types-C5gTvR7b.js';
+import { n as SplatEnvironmentProps } from './types-S8ggQY2n.js';
 import 'react';
 import '@react-three/fiber';
 import 'three';

package/dist/{types-C5gTvR7b.d.ts → types-S8ggQY2n.d.ts}

@@ -841,6 +841,9 @@ interface MujocoSimAPI {
     getCanvasSnapshot(width?: number, height?: number, mimeType?: string): string;
     captureFrame(options?: MujocoFrameCaptureOptions): Promise<FrameCaptureResult>;
     captureFrameBlob(options?: MujocoFrameCaptureOptions): Promise<FrameCaptureBlobResult>;
+    captureCameraFrame(options?: CameraFrameCaptureOptions): Promise<CameraFrameCaptureResult>;
+    captureCameraFrameBlob(options?: CameraFrameCaptureOptions): Promise<CameraFrameCaptureBlobResult>;
+    recordCameraSequence(options: CameraFrameSequenceOptions): Promise<CameraFrameSequenceResult>;
     project2DTo3D(x: number, y: number, cameraPos: THREE.Vector3, lookAt: THREE.Vector3): {
         point: THREE.Vector3;
         bodyId: number;
@@ -880,6 +883,74 @@ interface FrameCaptureAPI {
     captureBlob: (options?: FrameCaptureOptions) => Promise<FrameCaptureBlobResult>;
     reset: () => void;
 }
+type CameraFrameCaptureVector3 = THREE.Vector3 | readonly [number, number, number];
+type CameraFrameCaptureQuaternion = THREE.Quaternion | readonly [number, number, number, number];
+interface CameraFrameCaptureOptions {
+    camera?: THREE.Camera;
+    position?: CameraFrameCaptureVector3;
+    lookAt?: CameraFrameCaptureVector3;
+    quaternion?: CameraFrameCaptureQuaternion;
+    up?: CameraFrameCaptureVector3;
+    width?: number;
+    height?: number;
+    type?: string;
+    quality?: number;
+    fov?: number;
+    near?: number;
+    far?: number;
+}
+interface CameraFrameCaptureResult {
+    canvas: HTMLCanvasElement;
+    camera: THREE.Camera;
+    dataUrl: string;
+    type: string;
+    width: number;
+    height: number;
+}
+interface CameraFrameCaptureBlobResult {
+    canvas: HTMLCanvasElement;
+    camera: THREE.Camera;
+    blob: Blob;
+    type: string;
+    width: number;
+    height: number;
+}
+interface CameraFrameCaptureAPI {
+    status: FrameCaptureStatus;
+    error: Error | null;
+    isCapturing: boolean;
+    capture: (options?: CameraFrameCaptureOptions) => Promise<CameraFrameCaptureResult>;
+    captureBlob: (options?: CameraFrameCaptureOptions) => Promise<CameraFrameCaptureBlobResult>;
+    reset: () => void;
+}
+interface CameraFrameSequenceCamera extends CameraFrameCaptureOptions {
+    key: string;
+}
+interface CameraFrameSequenceFrame {
+    frameIndex: number;
+    time: number;
+    cameras: Record<string, CameraFrameCaptureResult>;
+}
+interface CameraFrameSequenceOptions {
+    cameras: readonly CameraFrameSequenceCamera[];
+    frames: number;
+    stepsPerFrame?: number;
+    reset?: boolean;
+    captureInitialFrame?: boolean;
+    onFrame?: (frame: CameraFrameSequenceFrame) => void | Promise<void>;
+}
+interface CameraFrameSequenceResult {
+    frames: CameraFrameSequenceFrame[];
+    cameraKeys: string[];
+    frameCount: number;
+}
+interface CameraFrameSequenceRecorderAPI {
+    status: FrameCaptureStatus;
+    error: Error | null;
+    isRecording: boolean;
+    record: (options: CameraFrameSequenceOptions) => Promise<CameraFrameSequenceResult>;
+    reset: () => void;
+}
 type MujocoCanvasProps = Omit<CanvasProps, 'onError'> & {
     config: SceneConfig;
     /** R3F content rendered while the MuJoCo WASM module is still loading. */
@@ -938,4 +1009,4 @@ interface JointStateResult {
     velocity: React__default.RefObject<number | Float64Array>;
 }
 
-export { type PolicyConfig as $, type ActuatedJointInfo as A, type BodyProps as B, type ControlGroupInfo as C, type DragInteractionProps as D, type ActuatorInfo as E, type Sites as F, type GeomInfo as G, type SitePositionResult as H, type IkConfig as I, type Sensors as J, type SensorHandle as K, type SensorInfo as L, type MujocoContextValue as M, type Joints as N, type ObservationConfig as O, type PhysicsStepCallback as P, type JointStateResult as Q, type ReadyCallbackInput as R, type SceneConfig as S, type TrajectoryPlayerProps as T, type Bodies as U, type VisualScenarioEffectsProps as V, type BodyStateResult as W, type Actuators as X, type CtrlHandle as Y, type ContactInfo as Z, type KeyboardTeleopConfig as _, type MujocoCanvasProps as a, type PolicyVector as a0, type ObservationHandle as a1, type TrajectoryInput as a2, type TrajectoryStateChangeInput as a3, type PlaybackState as a4, type TrajectoryFrame as a5, type FrameCaptureOptions as a6, type FrameCaptureResult as a7, type FrameCaptureBlobResult as a8, type FrameCaptureAPI as a9, type ResetCallbackInput as aA, type ResourceSelector as aB, RobotActuators as aC, RobotBodies as aD, RobotGeoms as aE, RobotJoints as aF, RobotKeyframes as aG, type RobotResource as aH, RobotResources as aI, RobotSensors as aJ, RobotSites as aK, type Robots as aL, type ScenarioCameraConfig as aM, type ScenarioMaterialConfig as aN, type SceneMarker as aO, type SceneObject as aP, type SensorResult as aQ, type SiteInfo as aR, type SplatAssetConfig as aS, type SplatScenarioConfig as aT, type StateSnapshot as aU, type TrajectoryData as aV, type TrajectoryFrameCallbackInput as aW, type VisualScenarioMaterialFilterInput as aX, type XmlPatch as aY, getContact as aZ, registerRobotResources as a_, type BodyInfo as aa, type ControlJointInfo as ab, type FrameCaptureStatus as ac, type FrameCaptureTarget as ad, type FrameCaptureTargetRef as ae, type Geoms as af, type IKSolveFn as ag, type IkGizmoDragInput as ah, type IkSolveInput as ai, type JointInfo as aj, type KeyBinding as ak, type Keyframes as al, type ModelOptions as am, type MujocoContact as an, type MujocoContactArray as ao, type MujocoFrameCaptureOptions as ap, type ObservationLayoutItem as aq, type ObservationOutput as ar, type PhysicsConfig as as, type PhysicsStepInput as at, type PolicyActionInput as au, type PolicyInferenceInput as av, type PolicyObservationInput as aw, type RayHit as ax, type Register as ay, type RegisteredRobotMap as az, type MujocoSimAPI as b, type StepCallbackInput as c, type SelectionCallbackInput as d, type MujocoModule as e, type MujocoModel as f, type MujocoData as g, type ControlGroupSelector as h, type ObservationResult as i, type IkContextValue as j, type IkGizmoProps as k, type SceneLightsProps as l, type ScenarioLightingProps as m, type SplatEnvironmentProps as n, type VisualScenarioConfig as o, type SplatRendererKind as p, type PairedSplatEnvironmentConfig as q, type SplatFormat as r, type SplatCollisionProxyConfig as s, type SplatCollisionPrimitive as t, type ScenarioLightingPreset as u, type SplatEnvironmentMetadataInput as v, type SplatEnvironmentMetadata as w, type SplatSceneInput as x, type DebugProps as y, type ContactListenerProps as z };
+export { type PolicyConfig as $, type ActuatedJointInfo as A, type BodyProps as B, type ControlGroupInfo as C, type DragInteractionProps as D, type ActuatorInfo as E, type Sites as F, type GeomInfo as G, type SitePositionResult as H, type IkConfig as I, type Sensors as J, type SensorHandle as K, type SensorInfo as L, type MujocoContextValue as M, type Joints as N, type ObservationConfig as O, type PhysicsStepCallback as P, type JointStateResult as Q, type ReadyCallbackInput as R, type SceneConfig as S, type TrajectoryPlayerProps as T, type Bodies as U, type VisualScenarioEffectsProps as V, type BodyStateResult as W, type Actuators as X, type CtrlHandle as Y, type ContactInfo as Z, type KeyboardTeleopConfig as _, type MujocoCanvasProps as a, type SensorResult as a$, type PolicyVector as a0, type ObservationHandle as a1, type TrajectoryInput as a2, type TrajectoryStateChangeInput as a3, type PlaybackState as a4, type TrajectoryFrame as a5, type FrameCaptureOptions as a6, type FrameCaptureResult as a7, type FrameCaptureBlobResult as a8, type FrameCaptureAPI as a9, type MujocoFrameCaptureOptions as aA, type ObservationLayoutItem as aB, type ObservationOutput as aC, type PhysicsConfig as aD, type PhysicsStepInput as aE, type PolicyActionInput as aF, type PolicyInferenceInput as aG, type PolicyObservationInput as aH, type RayHit as aI, type Register as aJ, type RegisteredRobotMap as aK, type ResetCallbackInput as aL, type ResourceSelector as aM, RobotActuators as aN, RobotBodies as aO, RobotGeoms as aP, RobotJoints as aQ, RobotKeyframes as aR, type RobotResource as aS, RobotResources as aT, RobotSensors as aU, RobotSites as aV, type Robots as aW, type ScenarioCameraConfig as aX, type ScenarioMaterialConfig as aY, type SceneMarker as aZ, type SceneObject as a_, type CameraFrameCaptureOptions as aa, type CameraFrameCaptureAPI as ab, type CameraFrameSequenceRecorderAPI as ac, type CameraFrameCaptureResult as ad, type CameraFrameCaptureBlobResult as ae, type BodyInfo as af, type CameraFrameCaptureQuaternion as ag, type CameraFrameCaptureVector3 as ah, type CameraFrameSequenceCamera as ai, type CameraFrameSequenceFrame as aj, type CameraFrameSequenceOptions as ak, type CameraFrameSequenceResult as al, type ControlJointInfo as am, type FrameCaptureStatus as an, type FrameCaptureTarget as ao, type FrameCaptureTargetRef as ap, type Geoms as aq, type IKSolveFn as ar, type IkGizmoDragInput as as, type IkSolveInput as at, type JointInfo as au, type KeyBinding as av, type Keyframes as aw, type ModelOptions as ax, type MujocoContact as ay, type MujocoContactArray as az, type MujocoSimAPI as b, type SiteInfo as b0, type SplatAssetConfig as b1, type SplatScenarioConfig as b2, type StateSnapshot as b3, type TrajectoryData as b4, type TrajectoryFrameCallbackInput as b5, type VisualScenarioMaterialFilterInput as b6, type XmlPatch as b7, getContact as b8, registerRobotResources as b9, type StepCallbackInput as c, type SelectionCallbackInput as d, type MujocoModule as e, type MujocoModel as f, type MujocoData as g, type ControlGroupSelector as h, type ObservationResult as i, type IkContextValue as j, type IkGizmoProps as k, type SceneLightsProps as l, type ScenarioLightingProps as m, type SplatEnvironmentProps as n, type VisualScenarioConfig as o, type SplatRendererKind as p, type PairedSplatEnvironmentConfig as q, type SplatFormat as r, type SplatCollisionProxyConfig as s, type SplatCollisionPrimitive as t, type ScenarioLightingPreset as u, type SplatEnvironmentMetadataInput as v, type SplatEnvironmentMetadata as w, type SplatSceneInput as x, type DebugProps as y, type ContactListenerProps as z };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mujoco-react",
-  "version": "9.1.0",
+  "version": "9.2.0",
   "description": "Composable React Three Fiber building blocks for MuJoCo WASM simulations",
   "type": "module",
   "main": "./dist/index.js",

package/src/core/MujocoSimProvider.tsx

@@ -20,6 +20,10 @@ import {
   ActuatedJointInfo,
   ActuatorInfo,
   BodyInfo,
+  CameraFrameCaptureResult,
+  CameraFrameSequenceFrame,
+  CameraFrameSequenceOptions,
+  CameraFrameSequenceResult,
   ControlGroupInfo,
   ControlGroupSelector,
   ContactInfo,
@@ -45,6 +49,10 @@ import {
   captureFrame as captureCanvasFrame,
   captureFrameBlob as captureCanvasFrameBlob,
 } from '../hooks/useFrameCapture';
+import {
+  captureCameraFrame,
+  captureCameraFrameBlob,
+} from '../rendering/cameraFrameCapture';
 import {
   loadScene,
   createSceneConfigFromFiles,
@@ -110,6 +118,12 @@ const _rayGeomId = new Int32Array(1);
 const _projRaycaster = new THREE.Raycaster();
 const _projNdc = new THREE.Vector2();
 
+function waitForNextAnimationFrame() {
+  return new Promise<void>((resolve) => {
+    requestAnimationFrame(() => resolve());
+  });
+}
+
 // ---- Internal context types ----
 
 export interface MujocoSimContextValue {
@@ -256,7 +270,7 @@ export function MujocoSimProvider({
   interpolate,
   children,
 }: MujocoSimProviderProps) {
-  const { gl, camera } = useThree();
+  const { gl, camera, scene } = useThree();
   const [status, setStatus] = useState<'loading' | 'ready' | 'error'>('loading');
 
   // --- Refs ---
@@ -548,6 +562,30 @@ export function MujocoSimProvider({
     stepsToRunRef.current = n;
   }, []);
 
+  const stepImmediately = useCallback((steps = 1) => {
+    const model = mjModelRef.current;
+    const data = mjDataRef.current;
+    if (!model || !data) return false;
+
+    for (let stepIndex = 0; stepIndex < steps; stepIndex += 1) {
+      for (let i = 0; i < model.nv; i += 1) {
+        data.qfrc_applied[i] = 0;
+      }
+      for (const cb of beforeStepCallbacks.current) {
+        cb({ model, data });
+      }
+      mujoco.mj_step(model, data);
+      for (const cb of afterStepCallbacks.current) {
+        cb({ model, data });
+      }
+      onStepRef.current?.({ time: data.time, model, data });
+    }
+
+    physicsAccumulatorRef.current = 0;
+    interpolationStateRef.current.valid = false;
+    return true;
+  }, [mujoco]);
+
   const getTime = useCallback((): number => {
     return mjDataRef.current?.time ?? 0;
   }, []);
@@ -1041,6 +1079,81 @@ export function MujocoSimProvider({
     [gl]
   );
 
+  const captureCameraFrameApi = useCallback(
+    (options = {}) => {
+      return captureCameraFrame(gl, scene, camera, options);
+    },
+    [camera, gl, scene]
+  );
+
+  const captureCameraFrameBlobApi = useCallback(
+    (options = {}) => {
+      return captureCameraFrameBlob(gl, scene, camera, options);
+    },
+    [camera, gl, scene]
+  );
+
+  const recordCameraSequenceApi = useCallback(
+    async (
+      options: CameraFrameSequenceOptions
+    ): Promise<CameraFrameSequenceResult> => {
+      const frameCount = Math.max(0, Math.floor(options.frames));
+      const stepsPerFrame = Math.max(1, Math.floor(options.stepsPerFrame ?? 1));
+      const cameras = options.cameras;
+      const frames: CameraFrameSequenceFrame[] = [];
+      const wasPaused = pausedRef.current;
+
+      if (frameCount === 0 || cameras.length === 0) {
+        return {
+          frames,
+          cameraKeys: cameras.map((sequenceCamera) => sequenceCamera.key),
+          frameCount: 0,
+        };
+      }
+
+      try {
+        pausedRef.current = true;
+        stepsToRunRef.current = 0;
+        if (options.reset) reset();
+
+        for (let frameIndex = 0; frameIndex < frameCount; frameIndex += 1) {
+          if (frameIndex > 0 || options.captureInitialFrame === false) {
+            stepImmediately(stepsPerFrame);
+          }
+          await waitForNextAnimationFrame();
+
+          const cameraFrames: Record<string, CameraFrameCaptureResult> = {};
+          for (const sequenceCamera of cameras) {
+            const { key, ...captureOptions } = sequenceCamera;
+            cameraFrames[key] = await captureCameraFrame(
+              gl,
+              scene,
+              camera,
+              captureOptions
+            );
+          }
+
+          const frame = {
+            frameIndex,
+            time: getTime(),
+            cameras: cameraFrames,
+          };
+          frames.push(frame);
+          await options.onFrame?.(frame);
+        }
+      } finally {
+        pausedRef.current = wasPaused;
+      }
+
+      return {
+        frames,
+        cameraKeys: cameras.map((sequenceCamera) => sequenceCamera.key),
+        frameCount: frames.length,
+      };
+    },
+    [camera, getTime, gl, reset, scene, stepImmediately]
+  );
+
   const project2DTo3D = useCallback(
     (x: number, y: number, cameraPos: THREE.Vector3, lookAt: THREE.Vector3): { point: THREE.Vector3; bodyId: number; geomId: number } | null => {
       const virtCam = (camera as THREE.PerspectiveCamera).clone();
@@ -1154,6 +1267,9 @@ export function MujocoSimProvider({
       getCanvasSnapshot,
       captureFrame: captureFrameApi,
       captureFrameBlob: captureFrameBlobApi,
+      captureCameraFrame: captureCameraFrameApi,
+      captureCameraFrameBlob: captureCameraFrameBlobApi,
+      recordCameraSequence: recordCameraSequenceApi,
       project2DTo3D,
       setBodyMass,
       setGeomFriction,
@@ -1172,6 +1288,8 @@ export function MujocoSimProvider({
       raycast, getKeyframeNames, getKeyframeCount, loadSceneApi,
       loadFromFilesApi, addBodyApi, removeBodyApi, recompileApi,
       getCanvas, getCanvasSnapshot, captureFrameApi, captureFrameBlobApi,
+      captureCameraFrameApi, captureCameraFrameBlobApi,
+      recordCameraSequenceApi,
       project2DTo3D,
       setBodyMass, setGeomFriction, setGeomSize,
     ]

package/src/core/createController.tsx

@@ -43,8 +43,10 @@ export type ControllerComponent<TConfig> = React.FC<{
  * const MyController = createController<{ speed: number }>(
  *   { name: 'my-controller', defaultConfig: { speed: 1.0 } },
  *   function MyControllerImpl({ config }) {
+ *     const wheel = useCtrl(RobotActuators.mobile.leftWheel);
+ *
  *     useBeforePhysicsStep(({ data }) => {
- *       data.ctrl[0] = config.speed;
+ *       wheel.write(config.speed);
  *     });
  *     return null;
  *   },
@@ -100,9 +102,11 @@ export function createController<TConfig>(
  *   { name: 'useMyController', defaultConfig: { gain: 1.0 } },
  *   function useMyControllerImpl(config) {
  *     // config is MyConfig | null — hooks must be called unconditionally
+ *     const joint = useCtrl(config?.actuator ?? RobotActuators.franka.actuator1);
+ *
  *     useBeforePhysicsStep(({ data }) => {
  *       if (!config) return;
- *       data.ctrl[0] = config.gain * Math.sin(data.time);
+ *       joint.write(config.gain * Math.sin(data.time));
  *     });
  *     if (!config) return null;
  *     return { /* value *\/ };

package/src/hooks/useCameraFrameCapture.ts

@@ -0,0 +1,94 @@
+/**
+ * @license
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * React state wrapper around MuJoCo/R3F offscreen camera-frame capture.
+ */
+
+import { useCallback, useState } from 'react';
+import { useMujoco } from '../core/MujocoSimProvider';
+import type {
+  CameraFrameCaptureAPI,
+  CameraFrameCaptureOptions,
+  FrameCaptureStatus,
+} from '../types';
+
+export function useCameraFrameCapture(
+  defaultOptions: CameraFrameCaptureOptions = {}
+): CameraFrameCaptureAPI {
+  const mujoco = useMujoco();
+  const [status, setStatus] = useState<FrameCaptureStatus>('idle');
+  const [error, setError] = useState<Error | null>(null);
+
+  const reset = useCallback(() => {
+    setStatus('idle');
+    setError(null);
+  }, []);
+
+  const capture = useCallback(
+    async (options: CameraFrameCaptureOptions = {}) => {
+      if (!mujoco.api) {
+        throw new Error('MuJoCo scene is not ready for camera frame capture.');
+      }
+
+      setStatus('capturing');
+      setError(null);
+
+      try {
+        const result = await mujoco.api.captureCameraFrame({
+          ...defaultOptions,
+          ...options,
+        });
+        setStatus('captured');
+        return result;
+      } catch (nextError) {
+        const error =
+          nextError instanceof Error
+            ? nextError
+            : new Error('Unable to capture the requested camera frame.');
+        setError(error);
+        setStatus('error');
+        throw error;
+      }
+    },
+    [defaultOptions, mujoco.api]
+  );
+
+  const captureBlob = useCallback(
+    async (options: CameraFrameCaptureOptions = {}) => {
+      if (!mujoco.api) {
+        throw new Error('MuJoCo scene is not ready for camera frame capture.');
+      }
+
+      setStatus('capturing');
+      setError(null);
+
+      try {
+        const result = await mujoco.api.captureCameraFrameBlob({
+          ...defaultOptions,
+          ...options,
+        });
+        setStatus('captured');
+        return result;
+      } catch (nextError) {
+        const error =
+          nextError instanceof Error
+            ? nextError
+            : new Error('Unable to capture the requested camera frame.');
+        setError(error);
+        setStatus('error');
+        throw error;
+      }
+    },
+    [defaultOptions, mujoco.api]
+  );
+
+  return {
+    status,
+    error,
+    isCapturing: status === 'capturing',
+    capture,
+    captureBlob,
+    reset,
+  };
+}

package/src/hooks/useCameraSequenceRecorder.ts

@@ -0,0 +1,59 @@
+/**
+ * @license
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * React state wrapper around fixed-camera simulation sequence recording.
+ */
+
+import { useCallback, useState } from 'react';
+import { useMujoco } from '../core/MujocoSimProvider';
+import type {
+  CameraFrameSequenceOptions,
+  CameraFrameSequenceRecorderAPI,
+  FrameCaptureStatus,
+} from '../types';
+
+export function useCameraSequenceRecorder(): CameraFrameSequenceRecorderAPI {
+  const mujoco = useMujoco();
+  const [status, setStatus] = useState<FrameCaptureStatus>('idle');
+  const [error, setError] = useState<Error | null>(null);
+
+  const reset = useCallback(() => {
+    setStatus('idle');
+    setError(null);
+  }, []);
+
+  const record = useCallback(
+    async (options: CameraFrameSequenceOptions) => {
+      if (!mujoco.api) {
+        throw new Error('MuJoCo scene is not ready for camera sequence recording.');
+      }
+
+      setStatus('capturing');
+      setError(null);
+
+      try {
+        const result = await mujoco.api.recordCameraSequence(options);
+        setStatus('captured');
+        return result;
+      } catch (nextError) {
+        const error =
+          nextError instanceof Error
+            ? nextError
+            : new Error('Unable to record the requested camera sequence.');
+        setError(error);
+        setStatus('error');
+        throw error;
+      }
+    },
+    [mujoco.api]
+  );
+
+  return {
+    status,
+    error,
+    isRecording: status === 'capturing',
+    record,
+    reset,
+  };
+}

package/src/index.ts

@@ -82,6 +82,13 @@ export {
   captureFrameBlob,
   useFrameCapture,
 } from './hooks/useFrameCapture';
+export { useCameraFrameCapture } from './hooks/useCameraFrameCapture';
+export { useCameraSequenceRecorder } from './hooks/useCameraSequenceRecorder';
+export {
+  captureCameraFrame,
+  captureCameraFrameBlob,
+  renderCameraFrameToCanvas,
+} from './rendering/cameraFrameCapture';
 export { useCtrlNoise } from './hooks/useCtrlNoise';
 export { useBodyMeshes } from './hooks/useBodyMeshes';
 export { useSelectionHighlight } from './hooks/useSelectionHighlight';
@@ -181,6 +188,17 @@ export type {
   // API
   MujocoSimAPI,
   MujocoFrameCaptureOptions,
+  CameraFrameCaptureAPI,
+  CameraFrameCaptureBlobResult,
+  CameraFrameCaptureOptions,
+  CameraFrameCaptureQuaternion,
+  CameraFrameCaptureResult,
+  CameraFrameCaptureVector3,
+  CameraFrameSequenceCamera,
+  CameraFrameSequenceFrame,
+  CameraFrameSequenceOptions,
+  CameraFrameSequenceRecorderAPI,
+  CameraFrameSequenceResult,
   MujocoCanvasProps,
   MujocoContextValue,
   // Hook return types

package/src/rendering/cameraFrameCapture.ts

@@ -0,0 +1,184 @@
+/**
+ * @license
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Offscreen camera-frame capture for R3F/MuJoCo scenes.
+ */
+
+import * as THREE from 'three';
+import type {
+  CameraFrameCaptureBlobResult,
+  CameraFrameCaptureOptions,
+  CameraFrameCaptureResult,
+  CameraFrameCaptureVector3,
+} from '../types';
+
+function toVector3(
+  value: CameraFrameCaptureVector3 | undefined,
+  fallback: THREE.Vector3
+): THREE.Vector3 {
+  if (!value) return fallback.clone();
+  return value instanceof THREE.Vector3
+    ? value.clone()
+    : new THREE.Vector3(value[0], value[1], value[2]);
+}
+
+function applyCameraPose(
+  camera: THREE.Camera,
+  options: CameraFrameCaptureOptions,
+  fallbackCamera: THREE.Camera
+) {
+  camera.position.copy(toVector3(options.position, fallbackCamera.position));
+  camera.up.copy(toVector3(options.up, fallbackCamera.up));
+
+  if (options.quaternion) {
+    if (options.quaternion instanceof THREE.Quaternion) {
+      camera.quaternion.copy(options.quaternion);
+    } else {
+      camera.quaternion.set(
+        options.quaternion[0],
+        options.quaternion[1],
+        options.quaternion[2],
+        options.quaternion[3]
+      );
+    }
+  } else if (options.lookAt) {
+    camera.lookAt(toVector3(options.lookAt, new THREE.Vector3()));
+  } else {
+    camera.quaternion.copy(fallbackCamera.quaternion);
+  }
+
+  camera.updateMatrixWorld();
+}
+
+function createCaptureCamera(
+  options: CameraFrameCaptureOptions,
+  fallbackCamera: THREE.Camera,
+  width: number,
+  height: number
+): THREE.Camera {
+  const camera = options.camera
+    ? options.camera.clone()
+    : fallbackCamera instanceof THREE.PerspectiveCamera
+      ? fallbackCamera.clone()
+      : new THREE.PerspectiveCamera(45, width / height, 0.01, 100);
+
+  if (camera instanceof THREE.PerspectiveCamera) {
+    camera.aspect = width / height;
+    camera.fov = options.fov ?? camera.fov;
+    camera.near = options.near ?? camera.near;
+    camera.far = options.far ?? camera.far;
+    camera.updateProjectionMatrix();
+  }
+
+  applyCameraPose(camera, options, fallbackCamera);
+  return camera;
+}
+
+function readRenderTargetToCanvas(
+  renderer: THREE.WebGLRenderer,
+  target: THREE.WebGLRenderTarget,
+  width: number,
+  height: number
+) {
+  const pixels = new Uint8Array(width * height * 4);
+  renderer.readRenderTargetPixels(target, 0, 0, width, height, pixels);
+
+  const canvas = document.createElement('canvas');
+  canvas.width = width;
+  canvas.height = height;
+  const context = canvas.getContext('2d');
+  if (!context) {
+    throw new Error('Unable to create a 2D canvas for camera frame capture.');
+  }
+
+  const imageData = context.createImageData(width, height);
+  const rowBytes = width * 4;
+  for (let y = 0; y < height; y += 1) {
+    const sourceStart = (height - y - 1) * rowBytes;
+    const targetStart = y * rowBytes;
+    imageData.data.set(
+      pixels.subarray(sourceStart, sourceStart + rowBytes),
+      targetStart
+    );
+  }
+  context.putImageData(imageData, 0, 0);
+  return canvas;
+}
+
+export function renderCameraFrameToCanvas(
+  renderer: THREE.WebGLRenderer,
+  scene: THREE.Scene,
+  fallbackCamera: THREE.Camera,
+  options: CameraFrameCaptureOptions = {}
+) {
+  const width = Math.max(1, Math.floor(options.width ?? renderer.domElement.width));
+  const height = Math.max(1, Math.floor(options.height ?? renderer.domElement.height));
+  const camera = createCaptureCamera(options, fallbackCamera, width, height);
+  const target = new THREE.WebGLRenderTarget(width, height, {
+    format: THREE.RGBAFormat,
+    type: THREE.UnsignedByteType,
+  });
+  const previousTarget = renderer.getRenderTarget();
+  const previousXrEnabled = renderer.xr.enabled;
+
+  scene.updateMatrixWorld(true);
+  try {
+    renderer.xr.enabled = false;
+    renderer.setRenderTarget(target);
+    renderer.clear();
+    renderer.render(scene, camera);
+    const canvas = readRenderTargetToCanvas(renderer, target, width, height);
+    return { canvas, camera, width, height };
+  } finally {
+    renderer.setRenderTarget(previousTarget);
+    renderer.xr.enabled = previousXrEnabled;
+    target.dispose();
+  }
+}
+
+export async function captureCameraFrame(
+  renderer: THREE.WebGLRenderer,
+  scene: THREE.Scene,
+  fallbackCamera: THREE.Camera,
+  options: CameraFrameCaptureOptions = {}
+): Promise<CameraFrameCaptureResult> {
+  const type = options.type ?? 'image/png';
+  const result = renderCameraFrameToCanvas(
+    renderer,
+    scene,
+    fallbackCamera,
+    options
+  );
+  return {
+    ...result,
+    dataUrl: result.canvas.toDataURL(type, options.quality),
+    type,
+  };
+}
+
+export async function captureCameraFrameBlob(
+  renderer: THREE.WebGLRenderer,
+  scene: THREE.Scene,
+  fallbackCamera: THREE.Camera,
+  options: CameraFrameCaptureOptions = {}
+): Promise<CameraFrameCaptureBlobResult> {
+  const type = options.type ?? 'image/png';
+  const result = renderCameraFrameToCanvas(
+    renderer,
+    scene,
+    fallbackCamera,
+    options
+  );
+  const blob = await new Promise<Blob>((resolve, reject) => {
+    result.canvas.toBlob(
+      (nextBlob) => {
+        if (nextBlob) resolve(nextBlob);
+        else reject(new Error('Camera frame capture did not produce a Blob.'));
+      },
+      type,
+      options.quality
+    );
+  });
+  return { ...result, blob, type };
+}