mujoco-react 9.0.0 → 9.2.0

package/README.md

@@ -213,18 +213,20 @@ function Scene() {
 ## Write Controllers
 
 ```tsx
-import { useBeforePhysicsStep } from "mujoco-react";
+import { RobotActuators, useBeforePhysicsStep, useCtrl } from "mujoco-react";
 
 function MyController() {
+  const shoulder = useCtrl(RobotActuators.franka.actuator1);
+
   useBeforePhysicsStep(({ data }) => {
-    data.ctrl[0] = Math.sin(data.time);
+    shoulder.write(Math.sin(data.time));
   });
 
   return null;
 }
 ```
 
-Controllers are just React children that read sensors, write `data.ctrl`, apply forces, or call the `MujocoSimAPI` at physics-step time.
+Controllers are just React children that read sensors, write named controls, apply forces, or call the `MujocoSimAPI` at physics-step time.
 
 With generated resource values, reusable controllers can be scoped to one robot without hand-typing names:
 
@@ -388,15 +390,20 @@ function useWebSocketControls(url: string) {
 For reusable controllers with typed config, default merging, and children, use the `createController` factory:
 
 ```tsx
-import { createController, useCtrl, useBeforePhysicsStep } from "mujoco-react";
+import {
+  RobotActuators,
+  createController,
+  useBeforePhysicsStep,
+  useCtrl,
+} from "mujoco-react";
 
 export const MyController = createController<{ gain: number }>(
   { name: "MyController", defaultConfig: { gain: 1.0 } },
   ({ config, children }) => {
-    const shoulder = useCtrl("shoulder");
+    const shoulder = useCtrl(RobotActuators.franka.actuator1);
 
-    useBeforePhysicsStep(() => {
-      shoulder.write(config.gain * Math.sin(Date.now() / 1000));
+    useBeforePhysicsStep(({ data }) => {
+      shoulder.write(config.gain * Math.sin(data.time));
     });
 
     return <>{children}</>;
@@ -826,8 +833,12 @@ if (mujoco) {
 Run logic **before** `mj_step` each frame. Write to `data.ctrl`, apply forces, drive automation.
 
 ```tsx
+import { RobotActuators, useBeforePhysicsStep, useCtrl } from "mujoco-react";
+
+const shoulder = useCtrl(RobotActuators.franka.actuator1);
+
 useBeforePhysicsStep(({ data }) => {
-  data.ctrl[0] = Math.sin(data.time);
+  shoulder.write(Math.sin(data.time));
 });
 ```
 
@@ -861,8 +872,8 @@ Read sensor values by name. Returns a `SensorHandle` with `read()`, `dim`, and `
 ```tsx
 import { RobotSensors, useSensor } from "mujoco-react";
 
-const force = useSensor(RobotSensors.franka.force_sensor);
-// force.read() -> Float64Array, force.dim -> number
+const imu = useSensor(RobotSensors.g1["imu-torso-angular-velocity"]);
+// imu.read() -> Float64Array, imu.dim -> number
 ```
 
 ### `useBodyState(name)`
@@ -983,6 +994,27 @@ const video = useVideoRecorder({ fps: 30, mimeType: "video/webm" });
 // video.start(), video.stop() -> returns Blob
 ```
 
+### Frame Capture
+
+Capture dataset/debug stills from the rendered MuJoCo canvas:
+
+```tsx
+const apiRef = useRef<MujocoSimAPI>(null);
+
+const frame = await apiRef.current?.captureFrame({ type: "image/png" });
+const blob = await apiRef.current?.captureFrameBlob({ type: "image/png" });
+```
+
+Use `useFrameCapture()` or the standalone `captureFrame()` helpers when you own
+the canvas or want to capture a custom container.
+
+Use `captureCameraFrame()` / `captureCameraFrameBlob()` when dataset generation
+needs a fixed offscreen camera pose or resolution without moving the user's
+interactive viewport.
+
+Use `recordCameraSequence()` / `useCameraSequenceRecorder()` to step policy
+rollouts and capture synchronized frames from one or more fixed camera configs.
+
 ### `useCtrlNoise(config)`
 
 Apply Gaussian noise to controls for robustness testing:
@@ -1104,7 +1136,10 @@ The full API object available via `ref` or `useMujoco()` (when `isReady`):
 |--------|-------------|
 | `raycast(origin, direction, maxDist?)` | Physics raycast via `mj_ray` |
 | `project2DTo3D(x, y, camPos, lookAt)` | Screen-to-world raycast (returns bodyId + geomId) |
+| `getCanvas()` | Return the underlying R3F canvas element |
 | `getCanvasSnapshot(w?, h?, mime?)` | Base64 screenshot |
+| `captureFrame(options?)` | Capture the canvas as a data URL |
+| `captureFrameBlob(options?)` | Capture the canvas as a Blob |
 
 ### Scene Management
 

package/dist/index.d.ts

@@ -1,8 +1,7 @@
 import * as react_jsx_runtime from 'react/jsx-runtime';
-import { M as MujocoContextValue, a as MujocoCanvasProps, b as MujocoSimAPI, S as SceneConfig, R as ReadyCallbackInput, c as StepCallbackInput, d as SelectionCallbackInput, e as MujocoModule, P as PhysicsStepCallback, f as MujocoModel, g as MujocoData, C as ControlGroupInfo, A as ActuatedJointInfo, h as ControlGroupSelector, O as ObservationConfig, i as ObservationResult, I as IkConfig, j as IkContextValue, B as BodyProps, k as IkGizmoProps, D as DragInteractionProps, l as SceneLightsProps, m as ScenarioLightingProps, n as SplatEnvironmentProps, V as VisualScenarioEffectsProps, o as VisualScenarioConfig, p as SplatRendererKind, q as PairedSplatEnvironmentConfig, r as SplatFormat, s as SplatCollisionProxyConfig, t as SplatCollisionPrimitive, u as ScenarioLightingPreset, v as SplatEnvironmentMetadataInput, w as SplatEnvironmentMetadata, x as SplatSceneInput, y as DebugProps, G as GeomInfo, z as ContactListenerProps, T as TrajectoryPlayerProps, E as ActuatorInfo, F as Sites, H as SitePositionResult, J as Sensors, K as SensorHandle, L as SensorInfo, N as Joints, Q as JointStateResult, U as Bodies, W as BodyStateResult, X as Actuators, Y as CtrlHandle, Z as ContactInfo, _ as KeyboardTeleopConfig, $ as PolicyConfig, a0 as PolicyVector, a1 as ObservationHandle, a2 as TrajectoryInput, a3 as TrajectoryStateChangeInput, a4 as PlaybackState, a5 as TrajectoryFrame } from './types-izZlUweI.js';
-export { a6 as BodyInfo, a7 as ControlJointInfo, a8 as Geoms, a9 as IKSolveFn, aa as IkGizmoDragInput, ab as IkSolveInput, ac as JointInfo, ad as KeyBinding, ae as Keyframes, af as ModelOptions, ag as MujocoContact, ah as MujocoContactArray, ai as ObservationLayoutItem, aj as ObservationOutput, ak as PhysicsConfig, al as PhysicsStepInput, am as PolicyActionInput, an as PolicyInferenceInput, ao as PolicyObservationInput, ap as RayHit, aq as Register, ar as RegisteredRobotMap, as as ResetCallbackInput, at as ResourceSelector, au as RobotActuators, av as RobotBodies, aw as RobotGeoms, ax as RobotJoints, ay as RobotKeyframes, az as RobotResource, aA as RobotResources, aB as RobotSensors, aC as RobotSites, aD as Robots, aE as ScenarioCameraConfig, aF as ScenarioMaterialConfig, aG as SceneMarker, aH as SceneObject, aI as SensorResult, aJ as SiteInfo, aK as SplatAssetConfig, aL as SplatScenarioConfig, aM as StateSnapshot, aN as TrajectoryData, aO as TrajectoryFrameCallbackInput, aP as VisualScenarioMaterialFilterInput, aQ as XmlPatch, aR as getContact, aS as registerRobotResources } from './types-izZlUweI.js';
+import { M as MujocoContextValue, a as MujocoCanvasProps, b as MujocoSimAPI, S as SceneConfig, R as ReadyCallbackInput, c as StepCallbackInput, d as SelectionCallbackInput, e as MujocoModule, P as PhysicsStepCallback, f as MujocoModel, g as MujocoData, C as ControlGroupInfo, A as ActuatedJointInfo, h as ControlGroupSelector, O as ObservationConfig, i as ObservationResult, I as IkConfig, j as IkContextValue, B as BodyProps, k as IkGizmoProps, D as DragInteractionProps, l as SceneLightsProps, m as ScenarioLightingProps, n as SplatEnvironmentProps, V as VisualScenarioEffectsProps, o as VisualScenarioConfig, p as SplatRendererKind, q as PairedSplatEnvironmentConfig, r as SplatFormat, s as SplatCollisionProxyConfig, t as SplatCollisionPrimitive, u as ScenarioLightingPreset, v as SplatEnvironmentMetadataInput, w as SplatEnvironmentMetadata, x as SplatSceneInput, y as DebugProps, G as GeomInfo, z as ContactListenerProps, T as TrajectoryPlayerProps, E as ActuatorInfo, F as Sites, H as SitePositionResult, J as Sensors, K as SensorHandle, L as SensorInfo, N as Joints, Q as JointStateResult, U as Bodies, W as BodyStateResult, X as Actuators, Y as CtrlHandle, Z as ContactInfo, _ as KeyboardTeleopConfig, $ as PolicyConfig, a0 as PolicyVector, a1 as ObservationHandle, a2 as TrajectoryInput, a3 as TrajectoryStateChangeInput, a4 as PlaybackState, a5 as TrajectoryFrame, a6 as FrameCaptureOptions, a7 as FrameCaptureResult, a8 as FrameCaptureBlobResult, a9 as FrameCaptureAPI, aa as CameraFrameCaptureOptions, ab as CameraFrameCaptureAPI, ac as CameraFrameSequenceRecorderAPI, ad as CameraFrameCaptureResult, ae as CameraFrameCaptureBlobResult } from './types-S8ggQY2n.js';
+export { af as BodyInfo, ag as CameraFrameCaptureQuaternion, ah as CameraFrameCaptureVector3, ai as CameraFrameSequenceCamera, aj as CameraFrameSequenceFrame, ak as CameraFrameSequenceOptions, al as CameraFrameSequenceResult, am as ControlJointInfo, an as FrameCaptureStatus, ao as FrameCaptureTarget, ap as FrameCaptureTargetRef, aq as Geoms, ar as IKSolveFn, as as IkGizmoDragInput, at as IkSolveInput, au as JointInfo, av as KeyBinding, aw as Keyframes, ax as ModelOptions, ay as MujocoContact, az as MujocoContactArray, aA as MujocoFrameCaptureOptions, aB as ObservationLayoutItem, aC as ObservationOutput, aD as PhysicsConfig, aE as PhysicsStepInput, aF as PolicyActionInput, aG as PolicyInferenceInput, aH as PolicyObservationInput, aI as RayHit, aJ as Register, aK as RegisteredRobotMap, aL as ResetCallbackInput, aM as ResourceSelector, aN as RobotActuators, aO as RobotBodies, aP as RobotGeoms, aQ as RobotJoints, aR as RobotKeyframes, aS as RobotResource, aT as RobotResources, aU as RobotSensors, aV as RobotSites, aW as Robots, aX as ScenarioCameraConfig, aY as ScenarioMaterialConfig, aZ as SceneMarker, a_ as SceneObject, a$ as SensorResult, b0 as SiteInfo, b1 as SplatAssetConfig, b2 as SplatScenarioConfig, b3 as StateSnapshot, b4 as TrajectoryData, b5 as TrajectoryFrameCallbackInput, b6 as VisualScenarioMaterialFilterInput, b7 as XmlPatch, b8 as getContact, b9 as registerRobotResources } from './types-S8ggQY2n.js';
 import * as React$1 from 'react';
-import React__default from 'react';
 import { ThreeElements } from '@react-three/fiber';
 import * as THREE from 'three';
 
@@ -243,8 +242,10 @@ type ControllerComponent<TConfig> = React.FC<{
  * const MyController = createController<{ speed: number }>(
  *   { name: 'my-controller', defaultConfig: { speed: 1.0 } },
  *   function MyControllerImpl({ config }) {
+ *     const wheel = useCtrl(RobotActuators.mobile.leftWheel);
+ *
  *     useBeforePhysicsStep(({ data }) => {
- *       data.ctrl[0] = config.speed;
+ *       wheel.write(config.speed);
  *     });
  *     return null;
  *   },
@@ -270,9 +271,11 @@ declare function createController<TConfig>(options: ControllerOptions<TConfig>,
  *   { name: 'useMyController', defaultConfig: { gain: 1.0 } },
  *   function useMyControllerImpl(config) {
  *     // config is MyConfig | null — hooks must be called unconditionally
+ *     const joint = useCtrl(config?.actuator ?? RobotActuators.franka.actuator1);
+ *
  *     useBeforePhysicsStep(({ data }) => {
  *       if (!config) return;
- *       data.ctrl[0] = config.gain * Math.sin(data.time);
+ *       joint.write(config.gain * Math.sin(data.time));
  *     });
  *     if (!config) return null;
  *     return { /* value *\/ };
@@ -758,33 +761,6 @@ declare function useVideoRecorder(options?: VideoRecorderOptions): {
  * useFrameCapture — still-frame capture for canvas-backed MuJoCo/R3F scenes.
  */
 
-type FrameCaptureStatus = 'idle' | 'capturing' | 'captured' | 'error';
-type FrameCaptureTarget = HTMLCanvasElement | HTMLElement | null | undefined;
-type FrameCaptureTargetRef = React__default.RefObject<HTMLCanvasElement | HTMLElement | null>;
-interface FrameCaptureOptions {
-    target?: FrameCaptureTarget | FrameCaptureTargetRef;
-    type?: string;
-    quality?: number;
-    waitForAnimationFrame?: boolean;
-}
-interface FrameCaptureResult {
-    canvas: HTMLCanvasElement;
-    dataUrl: string;
-    type: string;
-}
-interface FrameCaptureBlobResult {
-    canvas: HTMLCanvasElement;
-    blob: Blob;
-    type: string;
-}
-interface FrameCaptureAPI {
-    status: FrameCaptureStatus;
-    error: Error | null;
-    isCapturing: boolean;
-    capture: (options?: FrameCaptureOptions) => Promise<FrameCaptureResult>;
-    captureBlob: (options?: FrameCaptureOptions) => Promise<FrameCaptureBlobResult>;
-    reset: () => void;
-}
 /**
  * Capture the current canvas frame as a data URL.
  *
@@ -801,6 +777,40 @@ declare function captureFrameBlob(options: FrameCaptureOptions): Promise<FrameCa
  */
 declare function useFrameCapture(defaultOptions?: FrameCaptureOptions): FrameCaptureAPI;
 
+/**
+ * @license
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * React state wrapper around MuJoCo/R3F offscreen camera-frame capture.
+ */
+
+declare function useCameraFrameCapture(defaultOptions?: CameraFrameCaptureOptions): CameraFrameCaptureAPI;
+
+/**
+ * @license
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * React state wrapper around fixed-camera simulation sequence recording.
+ */
+
+declare function useCameraSequenceRecorder(): CameraFrameSequenceRecorderAPI;
+
+/**
+ * @license
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Offscreen camera-frame capture for R3F/MuJoCo scenes.
+ */
+
+declare function renderCameraFrameToCanvas(renderer: THREE.WebGLRenderer, scene: THREE.Scene, fallbackCamera: THREE.Camera, options?: CameraFrameCaptureOptions): {
+    canvas: HTMLCanvasElement;
+    camera: THREE.Camera;
+    width: number;
+    height: number;
+};
+declare function captureCameraFrame(renderer: THREE.WebGLRenderer, scene: THREE.Scene, fallbackCamera: THREE.Camera, options?: CameraFrameCaptureOptions): Promise<CameraFrameCaptureResult>;
+declare function captureCameraFrameBlob(renderer: THREE.WebGLRenderer, scene: THREE.Scene, fallbackCamera: THREE.Camera, options?: CameraFrameCaptureOptions): Promise<CameraFrameCaptureBlobResult>;
+
 /**
  * @license
  * SPDX-License-Identifier: Apache-2.0
@@ -896,4 +906,4 @@ interface CameraAnimationAPI {
  */
 declare function useCameraAnimation(): CameraAnimationAPI;
 
-export { ActuatedJointInfo, ActuatorInfo, Actuators, Bodies, Body, BodyProps, BodyStateResult, type CameraAnimationAPI, ContactInfo, ContactListener, ContactListenerProps, ContactMarkers, ControlGroupInfo, ControlGroupSelector, type ControllerComponent, type ControllerOptions, CtrlHandle, Debug, DebugProps, DragInteraction, DragInteractionProps, FlexRenderer, type FrameCaptureAPI, type FrameCaptureBlobResult, type FrameCaptureOptions, type FrameCaptureResult, type FrameCaptureStatus, type FrameCaptureTarget, type FrameCaptureTargetRef, GeomInfo, IkConfig, IkContextValue, IkGizmo, IkGizmoProps, InstancedGeomRenderer, JointStateResult, Joints, KeyboardTeleopConfig, MujocoCanvas, MujocoCanvasProps, MujocoContextValue, MujocoData, type MujocoLoader, type MujocoLoaderOptions, MujocoModel, MujocoModule, MujocoPhysics, type MujocoPhysicsProps, MujocoProvider, type MujocoProviderProps, MujocoSimAPI, MujocoSimProvider, type MujocoWasmVariant, ObservationConfig, ObservationHandle, ObservationResult, PairedSplatEnvironmentConfig, PhysicsStepCallback, PlaybackState, PolicyConfig, PolicyVector, ReadyCallbackInput, ScenarioLighting, ScenarioLightingPreset, ScenarioLightingProps, SceneConfig, SceneLights, SceneLightsProps, SelectionCallbackInput, SensorHandle, SensorInfo, Sensors, SitePositionResult, Sites, SplatCollisionPrimitive, SplatCollisionProxyConfig, SplatEnvironment, SplatEnvironmentMetadata, SplatEnvironmentMetadataInput, SplatEnvironmentProps, SplatFormat, SplatRendererKind, SplatSceneInput, StepCallbackInput, TendonRenderer, TrajectoryFrame, TrajectoryInput, TrajectoryPlayer, TrajectoryPlayerProps, TrajectoryStateChangeInput, VisualScenarioConfig, VisualScenarioEffects, VisualScenarioEffectsProps, buildObservation, captureFrame, captureFrameBlob, createContiguousControlGroup, createController, createControllerHook, createPairedSplatEnvironment, createSparkSplatViewerUrl, createSplatEnvironmentUserData, findActuatorByName, findBodyByName, findGeomByName, findJointByName, findKeyframeByName, findSensorByName, findSiteByName, findTendonByName, getActuatedJoints, getControlMap, getName, getScenarioBackground, getScenarioCameraPosition, loadScene, resolveControlGroup, useActuators, useAfterPhysicsStep, useBeforePhysicsStep, useBodyMeshes, useBodyState, useCameraAnimation, useContactEvents, useContacts, useCtrl, useCtrlNoise, useFrameCapture, useGamepad, useGravityCompensation, useIkController, useJointState, useKeyboardTeleop, useMujoco, useMujocoWasm, useObservation, usePolicy, useSceneLights, useSelectionHighlight, useSensor, useSensors, useSitePosition, useSplatEnvironment, useTrajectoryPlayer, useTrajectoryRecorder, useVideoRecorder, useVisualScenarioEffects, withSplatEnvironment };
+export { ActuatedJointInfo, ActuatorInfo, Actuators, Bodies, Body, BodyProps, BodyStateResult, type CameraAnimationAPI, CameraFrameCaptureAPI, CameraFrameCaptureBlobResult, CameraFrameCaptureOptions, CameraFrameCaptureResult, CameraFrameSequenceRecorderAPI, ContactInfo, ContactListener, ContactListenerProps, ContactMarkers, ControlGroupInfo, ControlGroupSelector, type ControllerComponent, type ControllerOptions, CtrlHandle, Debug, DebugProps, DragInteraction, DragInteractionProps, FlexRenderer, FrameCaptureAPI, FrameCaptureBlobResult, FrameCaptureOptions, FrameCaptureResult, GeomInfo, IkConfig, IkContextValue, IkGizmo, IkGizmoProps, InstancedGeomRenderer, JointStateResult, Joints, KeyboardTeleopConfig, MujocoCanvas, MujocoCanvasProps, MujocoContextValue, MujocoData, type MujocoLoader, type MujocoLoaderOptions, MujocoModel, MujocoModule, MujocoPhysics, type MujocoPhysicsProps, MujocoProvider, type MujocoProviderProps, MujocoSimAPI, MujocoSimProvider, type MujocoWasmVariant, ObservationConfig, ObservationHandle, ObservationResult, PairedSplatEnvironmentConfig, PhysicsStepCallback, PlaybackState, PolicyConfig, PolicyVector, ReadyCallbackInput, ScenarioLighting, ScenarioLightingPreset, ScenarioLightingProps, SceneConfig, SceneLights, SceneLightsProps, SelectionCallbackInput, SensorHandle, SensorInfo, Sensors, SitePositionResult, Sites, SplatCollisionPrimitive, SplatCollisionProxyConfig, SplatEnvironment, SplatEnvironmentMetadata, SplatEnvironmentMetadataInput, SplatEnvironmentProps, SplatFormat, SplatRendererKind, SplatSceneInput, StepCallbackInput, TendonRenderer, TrajectoryFrame, TrajectoryInput, TrajectoryPlayer, TrajectoryPlayerProps, TrajectoryStateChangeInput, VisualScenarioConfig, VisualScenarioEffects, VisualScenarioEffectsProps, buildObservation, captureCameraFrame, captureCameraFrameBlob, captureFrame, captureFrameBlob, createContiguousControlGroup, createController, createControllerHook, createPairedSplatEnvironment, createSparkSplatViewerUrl, createSplatEnvironmentUserData, findActuatorByName, findBodyByName, findGeomByName, findJointByName, findKeyframeByName, findSensorByName, findSiteByName, findTendonByName, getActuatedJoints, getControlMap, getName, getScenarioBackground, getScenarioCameraPosition, loadScene, renderCameraFrameToCanvas, resolveControlGroup, useActuators, useAfterPhysicsStep, useBeforePhysicsStep, useBodyMeshes, useBodyState, useCameraAnimation, useCameraFrameCapture, useCameraSequenceRecorder, useContactEvents, useContacts, useCtrl, useCtrlNoise, useFrameCapture, useGamepad, useGravityCompensation, useIkController, useJointState, useKeyboardTeleop, useMujoco, useMujocoWasm, useObservation, usePolicy, useSceneLights, useSelectionHighlight, useSensor, useSensors, useSitePosition, useSplatEnvironment, useTrajectoryPlayer, useTrajectoryRecorder, useVideoRecorder, useVisualScenarioEffects, withSplatEnvironment };