mujoco-react 10.4.0 → 10.5.0

package/README.md

@@ -1,5 +1,5 @@
 <p align="center">
-  <img src="docs/images/mj2.gif" alt="mujoco-react demo" width="100%" />
+  <img width="941" height="598" alt="Screenshot 2026-06-24 at 5 44 13 PM" src="https://github.com/user-attachments/assets/09999f02-c093-473e-aaee-519fe2e4cdd4" />
 </p>
 
 # mujoco-react
@@ -1095,29 +1095,20 @@ const video = useVideoRecorder({ fps: 30, mimeType: "video/webm" });
 // video.start(), video.stop() -> returns Blob
 ```
 
-### Frame Capture
+### Camera Frames, Streams, and Tensors
 
-Capture dataset/debug stills from the rendered MuJoCo canvas:
+A camera pose — the viewer camera, a MuJoCo `cameraName`/`siteName`/`bodyName`, or
+an explicit `position`+`lookAt` — can be turned into three outputs: a **snapshot**
+(PNG/JPEG), a **live on-screen stream**, or a **policy tensor**.
 
-```tsx
-const apiRef = useRef<MujocoSimAPI>(null);
-
-const frame = await apiRef.current?.captureFrame({ type: "image/png" });
-const blob = await apiRef.current?.captureFrameBlob({ type: "image/png" });
-```
-
-Use `useFrameCapture()` or the standalone `captureFrame()` helpers when you own
-the canvas or want to capture a custom container.
-
-Use `captureCameraFrame()` / `captureCameraFrameBlob()` when dataset generation
-needs an offscreen camera render at a stable resolution without moving the
-user's interactive viewport. Pass `cameraName`, `siteName`, or `bodyName` to
-record true MuJoCo-mounted camera frames; the returned image includes
-`source.kind` so dataset pipelines can reject fallback or synthetic fixed poses.
-For named MuJoCo cameras, set `mujocoCameraCompatibility` when you want the
-Three.js offscreen camera to inherit the MJCF camera's `resolution`, `fovy`,
-near/far clipping, and calibrated intrinsics when the WASM model exposes
-`cam_intrinsic` plus `cam_sensorsize`:
+**Snapshots.** `captureFrame()`/`captureFrameBlob()` grab the live canvas (or use
+`useFrameCapture()` / the standalone `captureFrame()` when you own the canvas).
+`captureCameraFrame()`/`captureCameraFrameBlob()` render a chosen camera offscreen
+at a stable resolution without moving the viewport; pass `cameraName`/`siteName`/
+`bodyName` for true mounted frames (`source.kind` lets dataset pipelines reject
+fallback/synthetic poses). Set `mujocoCameraCompatibility` to inherit a MJCF
+camera's `resolution`, `fovy`, clipping, and calibrated intrinsics; use
+`visualOverrides`/`renderIsolation` for canonical training captures.
 
 ```tsx
 const frame = await apiRef.current?.captureCameraFrame({
@@ -1128,138 +1119,84 @@ const frame = await apiRef.current?.captureCameraFrame({
 });
 ```
 
-This is still rendered by Three.js. It is useful for browser policy payloads and
-dataset debugging until you have native MuJoCo framebuffer bindings available.
-For canonical policy/training captures, use `visualOverrides` to temporarily
-override scene background, environment, fog, shadow maps, tone mapping, or color
-space, and `renderIsolation` to render with an independent offscreen
-`WebGLRenderer` instead of inheriting viewer renderer state.
-
-Use `recordCameraSequence()` / `useCameraSequenceRecorder()` to step policy
-rollouts and capture synchronized per-camera frames from one or more MuJoCo
-camera configs. Sequence recording requires mounted MuJoCo camera, site, or
-body selectors by default; use still capture APIs for synthetic debug poses.
-
-For LeRobot-style datasets, prefer the named-camera wrapper. It resolves task
-camera keys to MuJoCo cameras/sites/bodies, records the sequence, and returns
-the plan and readiness summary alongside frame provenance:
+**Live streams.** `useCameraStream(canvasRef, options)` renders a camera into a
+DOM `<canvas>` every frame (offscreen render → blit: no PNG round-trip, no render-
+loop takeover, splat scenes stream safely). Call it inside `<MujocoCanvas>`; the
+canvas can live anywhere in your DOM:
 
 ```tsx
-const sequence = await recordMountedCameraFrameSequence(api, {
-  cameraKeys: ["head", "left_wrist", "right_wrist"],
-  aliases: {
-    head: [{ siteName: "head_camera_rgb_optical_frame" }],
-    left_wrist: [{ siteName: "left_wrist_camera_optical_frame" }],
-    right_wrist: [{ siteName: "right_wrist_camera_optical_frame" }],
-  },
-  defaults: {
-    width: 640,
-    height: 480,
-    type: "image/png",
-    fov: 45,
-    near: 0.01,
-    far: 100,
-  },
-  frames: 16,
-  stepsPerFrame: 1,
-  retainFrames: false,
-  requireMountedSources: true,
-  onFrame: ({ frameIndex, cameras }) => {
-    queueLeRobotImages(frameIndex, cameras);
-  },
-});
-
-sequence.readiness.ready; // true when every requested stream resolved
-sequence.plan.missingKeys; // unresolved task cameras, if requireAll is false
-sequence.cameraSummaries.head.source; // mounted source provenance
-
-const manifest = createMountedCameraFrameSequenceManifest(sequence);
-manifest.streamSummaries.head.complete; // per-camera frame coverage
-manifest.status; // "complete", "partial", or "missing"
+function WristStream({ canvasRef }) {
+  useCameraStream(canvasRef, { cameraName: "wrist_cam", width: 256, height: 192 });
+  return null;
+}
 ```
 
-`recordMountedCameraFrameSequence()` requires all requested `cameraKeys` by
-default so dataset recording cannot silently omit a camera stream. Set
-`requireAll: false` only for exploratory tooling that can tolerate partial
-camera coverage.
+For a transparent picture-in-picture overlay, use `<CameraView>` (or
+`useCameraViewport` to track your own element). It scissors into the main canvas —
+cheaper, but it takes over the render loop while mounted (incompatible with
+postprocessing, occluded by opaque DOM), so prefer `useCameraStream` for panel
+tiles:
 
-Use `createMountedCameraFrameSequenceManifest()` after recording to persist a
-stable dataset-facing manifest with readiness, source targets, dimensions,
-first/last frame indices, timestamps, and per-camera missing-frame counts.
+```tsx
+<MujocoCanvas config={config}>
+  <CameraView cameraName="wrist_cam" style={{ right: 16, bottom: 16, width: 240, height: 180 }} />
+</MujocoCanvas>
+```
 
-Inside `<MujocoCanvas>` children, `useMountedCameraSequenceRecorder()` exposes
-the same planning and recording surface with React status/error/result state.
-Use `checkReadiness()` before recording when the UI needs a preflight gate for
-LeRobot camera streams:
+**Policy tensors.** For in-browser inference, capture straight into a
+`Float32Array` — no canvas, no PNG. `usePolicyCameraTensors` keeps one reusable
+session per camera and re-aims it to the live pose each step:
 
 ```tsx
-function DatasetRecorder() {
-  const recorder = useMountedCameraSequenceRecorder({
-    defaults: { width: 640, height: 480, type: "image/png" },
-    aliases: {
-      head: { cameraName: "head" },
-      left_wrist: { siteName: "left_wrist_camera_optical_frame" },
-      right_wrist: { siteName: "right_wrist_camera_optical_frame" },
-    },
-  });
-
-  async function recordDatasetEpisode() {
-    const cameraKeys = ["head", "left_wrist", "right_wrist"];
-    const readiness = recorder.checkReadiness(cameraKeys);
-    if (!readiness.ready) return;
-
-    await recorder.record({
-      cameraKeys,
-      frames: 16,
-      retainFrames: false,
-      onFrame: ({ frameIndex, cameras }) => {
-        queueLeRobotImages(frameIndex, cameras);
-      },
-    });
-  }
+const cams = usePolicyCameraTensors({
+  streams: [
+    { key: "wrist", cameraName: "wrist_cam", width: 96, height: 96, layout: "CHW" },
+    { key: "front", cameraName: "front",     width: 96, height: 96, layout: "CHW" },
+  ],
+});
 
-  return (
-    <button disabled={recorder.isRecording} onClick={recordDatasetEpisode}>
-      Record camera streams
-    </button>
-  );
-}
+useAfterPhysicsStep(() => {
+  const { tensors } = cams.capture();
+  const wrist = new ort.Tensor("float32", tensors.wrist.data, [1, ...tensors.wrist.shape]);
+});
 ```
 
-`recorder.readiness` keeps the latest preflight result, and
-`recorder.result?.readiness` keeps the readiness that shipped with the most
-recent recording.
+Use `usePolicyCameraTensorsFromMountedStreams` for dataset-name resolution,
+`captureCameraFrameTensor()` for one-offs, or `createCameraFrameCaptureSession()`
++ `pixelsToPolicyImageTensor()` for lower-level control. The optional
+`mujoco-react/onnx` entry point wraps ONNX Runtime Web: `createOnnxPolicySession()`
+loads a manifest plus model, and `onnxTensorToPolicyActionChunk()` decodes the
+output into action chunks.
 
-Use `resolveMountedCameraFrameSource()` when dataset feature names need to map
-to named MuJoCo cameras, sites, or bodies before recording. The helper honors
-aliases first, then prefers camera matches over sites and bodies, then falls
-back to exact body/site names. This keeps streams such as `wrist` mapped to a
-real `<camera name="wrist_cam">` even when the model also has a body named
-`wrist`. It also handles normalized/prefix/suffix matches such as
-`left_wrist` -> `left_wrist_camera_optical_frame`, and token-contained
-imported-model names such as `observation.images.head` ->
-`robot_head_camera`. It returns both the capture selector and the mounted-source
-provenance that should be stored beside the dataset:
+**Dataset recording.** `recordMountedCameraFrameSequence()` steps a rollout and
+captures synchronized per-camera frames, resolving LeRobot-style task camera keys
+to MuJoCo cameras/sites/bodies. It requires every requested `cameraKey` by default
+(set `requireAll: false` for exploratory tooling) and returns a plan, readiness,
+and per-camera source provenance:
 
 ```tsx
-const resolved = resolveMountedCameraFrameSource("head", {
-  cameras: api.getCameras(),
-  sites: api.getSites(),
-  bodies: api.getBodies(),
-});
-
-if (!resolved) throw new Error("head does not resolve to a MuJoCo source");
-
-await api.recordCameraSequence({
+const sequence = await recordMountedCameraFrameSequence(api, {
+  cameraKeys: ["head", "left_wrist", "right_wrist"],
+  aliases: {
+    head: [{ siteName: "head_camera_rgb_optical_frame" }],
+    left_wrist: [{ siteName: "left_wrist_camera_optical_frame" }],
+    right_wrist: [{ siteName: "right_wrist_camera_optical_frame" }],
+  },
+  defaults: { width: 640, height: 480, type: "image/png", fov: 45 },
   frames: 16,
-  requireMountedSources: true,
-  cameras: [{ key: "head", width: 640, height: 480, ...resolved.selector }],
+  onFrame: ({ frameIndex, cameras }) => queueLeRobotImages(frameIndex, cameras),
 });
+
+const manifest = createMountedCameraFrameSequenceManifest(sequence); // dataset-facing manifest
 ```
 
-Pass `aliases` when multiple MuJoCo resources could match a dataset stream key
-or when the model uses names that do not share a normalized prefix/suffix with
-the LeRobot camera feature.
+Inside `<MujocoCanvas>`, `useMountedCameraSequenceRecorder()` exposes the same
+planning/recording with React status and a `checkReadiness()` preflight gate.
+`resolveMountedCameraFrameSource()` maps a single dataset feature name to a MuJoCo
+source (aliases first, then camera > site > body, then normalized prefix/suffix
+matches) when you need the selector before recording. The lower-level
+`recordCameraSequence()` / `useCameraSequenceRecorder()` take explicit camera
+configs.
 
 ### `useCtrlNoise(config)`
 

package/dist/{chunk-FBXXXPLQ.js → chunk-KHZ5U36J.js}

@@ -85,6 +85,95 @@ var SplatEnvironmentReadinessStatus = {
   UnsupportedFormat: "unsupported-format",
   Ready: "ready"
 };
+
+// src/policyImageTensors.ts
+function resolveTensorOptions(options) {
+  return {
+    channels: 3,
+    layout: "CHW",
+    range: [0, 1],
+    ...options
+  };
+}
+function normalizeChannel(value, range) {
+  const [min, max] = range;
+  if (min === 0 && max === 255) return value;
+  return min + value / 255 * (max - min);
+}
+function pixelsToPolicyImageTensor(pixels, options) {
+  const resolved = resolveTensorOptions(options);
+  const { width, height, channels, layout, range } = resolved;
+  const expected = width * height * 4;
+  if (pixels.length < expected) {
+    throw new Error(
+      `Pixel buffer of length ${pixels.length} is too small for ${width}x${height} RGBA data (${expected} bytes).`
+    );
+  }
+  const flipY = options.sourceOrigin === "bottom-left";
+  const flipX = options.flipX ?? false;
+  const pixelCount = width * height;
+  const data = new Float32Array(pixelCount * channels);
+  for (let y = 0; y < height; y += 1) {
+    const sourceY = flipY ? height - y - 1 : y;
+    for (let x = 0; x < width; x += 1) {
+      const sourceX = flipX ? width - x - 1 : x;
+      const source = (sourceY * width + sourceX) * 4;
+      const target = y * width + x;
+      for (let channel = 0; channel < channels; channel += 1) {
+        const value = normalizeChannel(pixels[source + channel], range);
+        if (layout === "CHW") {
+          data[channel * pixelCount + target] = value;
+        } else {
+          data[target * channels + channel] = value;
+        }
+      }
+    }
+  }
+  return {
+    data,
+    shape: layout === "CHW" ? [channels, height, width] : [height, width, channels],
+    width,
+    height,
+    channels,
+    layout,
+    range
+  };
+}
+function imageDataToPolicyImageTensor(imageData, options) {
+  const resolved = resolveTensorOptions(options);
+  if (imageData.width !== resolved.width || imageData.height !== resolved.height) {
+    throw new Error(
+      `ImageData size ${imageData.width}x${imageData.height} does not match tensor size ${resolved.width}x${resolved.height}.`
+    );
+  }
+  return pixelsToPolicyImageTensor(imageData.data, {
+    ...resolved,
+    sourceOrigin: "top-left"
+  });
+}
+async function decodeImageSource(dataUrl) {
+  const image = new Image();
+  image.decoding = "async";
+  image.src = dataUrl;
+  await image.decode();
+  return image;
+}
+async function dataUrlToPolicyImageTensor(dataUrl, options) {
+  const resolved = resolveTensorOptions(options);
+  const image = await decodeImageSource(dataUrl);
+  const canvas = document.createElement("canvas");
+  canvas.width = resolved.width;
+  canvas.height = resolved.height;
+  const context = canvas.getContext("2d", { willReadFrequently: true });
+  if (!context) {
+    throw new Error("Unable to create a 2D canvas context for policy image tensor conversion.");
+  }
+  context.drawImage(image, 0, 0, resolved.width, resolved.height);
+  return imageDataToPolicyImageTensor(
+    context.getImageData(0, 0, resolved.width, resolved.height),
+    resolved
+  );
+}
 var CAMERA_FRAME_CAPTURE_RENDER_USER_DATA_KEY = "mujocoReactCameraFrameCaptureRender";
 var CAMERA_FRAME_CAPTURE_PRE_RENDER_USER_DATA_KEY = "mujocoReactCameraFrameCapturePreRender";
 var CAPTURE_EXCLUDE_KEY = "mujoco.capture.exclude";
@@ -465,7 +554,7 @@ function createCameraFrameCaptureSession(renderer, scene, fallbackCamera, option
     );
     return captureOptions;
   }
-  function renderPreparedCapture(captureOptions) {
+  function renderCaptureToTarget(captureOptions, readback) {
     const previousState = saveRendererState(sessionRenderer);
     const previousSceneState = applyCaptureVisualOverrides(
       sessionRenderer,
@@ -494,6 +583,15 @@ function createCameraFrameCaptureSession(renderer, scene, fallbackCamera, option
       }
       sessionRenderer.clear();
       sessionRenderer.render(scene, camera);
+      readback();
+    } finally {
+      restoreObjectVisibility(hidden);
+      if (previousSceneState) restoreSceneVisualState(scene, previousSceneState);
+      restoreRendererState(sessionRenderer, previousState);
+    }
+  }
+  function renderPreparedCapture(captureOptions) {
+    renderCaptureToTarget(captureOptions, () => {
       readRenderTargetToCanvas(
         sessionRenderer,
         target,
@@ -506,22 +604,44 @@ function createCameraFrameCaptureSession(renderer, scene, fallbackCamera, option
         sessionRenderer.outputColorSpace,
         captureOptions.flipX ?? false
       );
-      return {
-        canvas,
-        camera,
-        width,
-        height,
-        source: getCameraFrameCaptureSource(captureOptions)
-      };
-    } finally {
-      restoreObjectVisibility(hidden);
-      if (previousSceneState) restoreSceneVisualState(scene, previousSceneState);
-      restoreRendererState(sessionRenderer, previousState);
-    }
+    });
+    return {
+      canvas,
+      camera,
+      width,
+      height,
+      source: getCameraFrameCaptureSource(captureOptions)
+    };
   }
   function capture(nextOptions = {}) {
     return renderPreparedCapture(resolveCaptureOptions(nextOptions));
   }
+  function capturePixels(nextOptions = {}) {
+    const captureOptions = resolveCaptureOptions(nextOptions);
+    renderCaptureToTarget(captureOptions, () => {
+      sessionRenderer.readRenderTargetPixels(target, 0, 0, width, height, pixels);
+    });
+    return {
+      pixels,
+      camera,
+      width,
+      height,
+      source: getCameraFrameCaptureSource(captureOptions)
+    };
+  }
+  function captureTensor(nextOptions = {}) {
+    const result = capturePixels(nextOptions);
+    const tensor = pixelsToPolicyImageTensor(pixels, {
+      width,
+      height,
+      channels: nextOptions.channels,
+      layout: nextOptions.layout,
+      range: nextOptions.range,
+      sourceOrigin: "bottom-left",
+      flipX: nextOptions.flipX
+    });
+    return { ...tensor, camera, source: result.source };
+  }
   async function captureAsync(nextOptions = {}) {
     const captureOptions = resolveCaptureOptions(nextOptions);
     runCapturePreRenderHooks(scene);
@@ -594,6 +714,8 @@ function createCameraFrameCaptureSession(renderer, scene, fallbackCamera, option
     height,
     capture,
     captureAsync,
+    capturePixels,
+    captureTensor,
     captureDataUrl(nextOptions = {}) {
       const type = nextOptions.type ?? options.type ?? "image/png";
       const result = capture(nextOptions);
@@ -686,6 +808,19 @@ async function captureCameraFrameBlob(renderer, scene, fallbackCamera, options =
     session.dispose();
   }
 }
+function captureCameraFrameTensor(renderer, scene, fallbackCamera, options = {}) {
+  const session = createCameraFrameCaptureSession(
+    renderer,
+    scene,
+    fallbackCamera,
+    options
+  );
+  try {
+    return session.captureTensor(options);
+  } finally {
+    session.dispose();
+  }
+}
 var DEFAULT_BACKGROUND = "#181a1f";
 function ScenarioLighting({
   preset = "studio",
@@ -1284,6 +1419,12 @@ function clamp01(value) {
  * @license
  * SPDX-License-Identifier: Apache-2.0
  */
+/**
+ * @license
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Helpers for turning browser camera captures into policy image tensors.
+ */
 /**
  * @license
  * SPDX-License-Identifier: Apache-2.0
@@ -1291,6 +1432,6 @@ function clamp01(value) {
  * Offscreen camera-frame capture for R3F/MuJoCo scenes.
  */
 
-export { CAMERA_FRAME_CAPTURE_PRE_RENDER_USER_DATA_KEY, CAMERA_FRAME_CAPTURE_RENDER_USER_DATA_KEY, CAPTURE_EXCLUDE_KEY, ModelActuators, ModelBodies, ModelCameras, ModelGeoms, ModelJoints, ModelKeyframes, ModelResources, ModelSensors, ModelSites, ScenarioLighting, SplatEnvironment, SplatEnvironmentReadinessStatus, VisualScenarioEffects, captureCameraFrame, captureCameraFrameBlob, createCameraFrameCaptureSession, createPairedSplatEnvironment, createSparkSplatViewerUrl, createSplatEnvironmentUserData, createSplatSceneConfig, createVisualScenarioExecutionContext, getContact, getScenarioBackground, getScenarioCameraPosition, getSplatEnvironmentReadiness, registerModelResources, renderCameraFrameToCanvas, useSplatEnvironment, useSplatSceneConfig, useVisualScenarioEffects, useVisualScenarioExecutionContext, withContacts, withSplatEnvironment };
-//# sourceMappingURL=chunk-FBXXXPLQ.js.map
-//# sourceMappingURL=chunk-FBXXXPLQ.js.map
+export { CAMERA_FRAME_CAPTURE_PRE_RENDER_USER_DATA_KEY, CAMERA_FRAME_CAPTURE_RENDER_USER_DATA_KEY, CAPTURE_EXCLUDE_KEY, ModelActuators, ModelBodies, ModelCameras, ModelGeoms, ModelJoints, ModelKeyframes, ModelResources, ModelSensors, ModelSites, ScenarioLighting, SplatEnvironment, SplatEnvironmentReadinessStatus, VisualScenarioEffects, captureCameraFrame, captureCameraFrameBlob, captureCameraFrameTensor, createCameraFrameCaptureSession, createCaptureCamera, createPairedSplatEnvironment, createSparkSplatViewerUrl, createSplatEnvironmentUserData, createSplatSceneConfig, createVisualScenarioExecutionContext, dataUrlToPolicyImageTensor, getContact, getScenarioBackground, getScenarioCameraPosition, getSplatEnvironmentReadiness, imageDataToPolicyImageTensor, pixelsToPolicyImageTensor, prepareCaptureCamera, registerModelResources, renderCameraFrameToCanvas, useSplatEnvironment, useSplatSceneConfig, useVisualScenarioEffects, useVisualScenarioExecutionContext, withContacts, withSplatEnvironment };
+//# sourceMappingURL=chunk-KHZ5U36J.js.map
+//# sourceMappingURL=chunk-KHZ5U36J.js.map