mujoco-react 10.2.0 → 10.3.0

package/README.md

@@ -748,6 +748,7 @@ Thin wrapper around R3F `<Canvas>`. Accepts all R3F Canvas props plus:
 | `substeps` | `number` | mj_step calls per frame |
 | `paused` | `boolean` | Declarative pause |
 | `speed` | `number` | Simulation speed multiplier |
+| `renderOptions` | `MujocoRenderOptions` | Optional render-time geometry settings such as `meshNormalSmoothing` |
 
 ### `<MujocoPhysics>`
 
@@ -1112,6 +1113,26 @@ needs an offscreen camera render at a stable resolution without moving the
 user's interactive viewport. Pass `cameraName`, `siteName`, or `bodyName` to
 record true MuJoCo-mounted camera frames; the returned image includes
 `source.kind` so dataset pipelines can reject fallback or synthetic fixed poses.
+For named MuJoCo cameras, set `mujocoCameraCompatibility` when you want the
+Three.js offscreen camera to inherit the MJCF camera's `resolution`, `fovy`,
+near/far clipping, and calibrated intrinsics when the WASM model exposes
+`cam_intrinsic` plus `cam_sensorsize`:
+
+```tsx
+const frame = await apiRef.current?.captureCameraFrame({
+  cameraName: "front_camera",
+  width: 640,
+  type: "image/jpeg",
+  mujocoCameraCompatibility: true,
+});
+```
+
+This is still rendered by Three.js. It is useful for browser policy payloads and
+dataset debugging until you have native MuJoCo framebuffer bindings available.
+For canonical policy/training captures, use `visualOverrides` to temporarily
+override scene background, environment, fog, shadow maps, tone mapping, or color
+space, and `renderIsolation` to render with an independent offscreen
+`WebGLRenderer` instead of inheriting viewer renderer state.
 
 Use `recordCameraSequence()` / `useCameraSequenceRecorder()` to step policy
 rollouts and capture synchronized per-camera frames from one or more MuJoCo

package/dist/{chunk-3BMNRSS2.js → chunk-6AZEFI6A.js}

@@ -88,6 +88,72 @@ var SplatEnvironmentReadinessStatus = {
 var CAMERA_FRAME_CAPTURE_RENDER_USER_DATA_KEY = "mujocoReactCameraFrameCaptureRender";
 var CAMERA_FRAME_CAPTURE_PRE_RENDER_USER_DATA_KEY = "mujocoReactCameraFrameCapturePreRender";
 var CAPTURE_EXCLUDE_KEY = "mujoco.capture.exclude";
+var isolatedRendererCache = /* @__PURE__ */ new WeakMap();
+function shouldUseRenderIsolation(options) {
+  return options.renderIsolation === true || typeof options.renderIsolation === "object" && options.renderIsolation.enabled !== false;
+}
+function getRenderIsolationOptions(options) {
+  return typeof options.renderIsolation === "object" ? options.renderIsolation : {};
+}
+function getRenderIsolationCacheKey(width, height, options) {
+  const isolation = getRenderIsolationOptions(options);
+  return JSON.stringify({
+    width,
+    height,
+    antialias: isolation.antialias ?? false,
+    alpha: isolation.alpha ?? false,
+    preserveDrawingBuffer: isolation.preserveDrawingBuffer ?? false,
+    powerPreference: isolation.powerPreference ?? null
+  });
+}
+function createIsolatedRenderer(sourceRenderer, width, height, options) {
+  if (!shouldUseRenderIsolation(options)) return null;
+  const isolation = getRenderIsolationOptions(options);
+  if (isolation.cache !== false) {
+    const cacheKey = getRenderIsolationCacheKey(width, height, options);
+    let rendererCache = isolatedRendererCache.get(sourceRenderer);
+    if (!rendererCache) {
+      rendererCache = /* @__PURE__ */ new Map();
+      isolatedRendererCache.set(sourceRenderer, rendererCache);
+    }
+    const cachedRenderer = rendererCache.get(cacheKey);
+    if (cachedRenderer) {
+      cachedRenderer.outputColorSpace = sourceRenderer.outputColorSpace;
+      cachedRenderer.toneMapping = sourceRenderer.toneMapping;
+      cachedRenderer.shadowMap.enabled = false;
+      return { renderer: cachedRenderer, cached: true };
+    }
+    const createdRenderer = createUncachedIsolatedRenderer(
+      sourceRenderer,
+      width,
+      height,
+      options
+    );
+    rendererCache.set(cacheKey, createdRenderer);
+    return { renderer: createdRenderer, cached: true };
+  }
+  return {
+    renderer: createUncachedIsolatedRenderer(sourceRenderer, width, height, options),
+    cached: false
+  };
+}
+function createUncachedIsolatedRenderer(sourceRenderer, width, height, options) {
+  const isolation = getRenderIsolationOptions(options);
+  const canvas = document.createElement("canvas");
+  const renderer = new THREE.WebGLRenderer({
+    canvas,
+    antialias: isolation.antialias ?? false,
+    alpha: isolation.alpha ?? false,
+    preserveDrawingBuffer: isolation.preserveDrawingBuffer ?? false,
+    powerPreference: isolation.powerPreference
+  });
+  renderer.setPixelRatio(1);
+  renderer.setSize(width, height, false);
+  renderer.outputColorSpace = sourceRenderer.outputColorSpace;
+  renderer.toneMapping = sourceRenderer.toneMapping;
+  renderer.shadowMap.enabled = false;
+  return renderer;
+}
 function toVector3(value, fallback) {
   if (!value) return fallback.clone();
   return value instanceof THREE.Vector3 ? value.clone() : new THREE.Vector3(value[0], value[1], value[2]);
@@ -113,6 +179,15 @@ function applyCameraPose(camera, options, fallbackCamera) {
   }
   camera.updateMatrixWorld();
 }
+function applyProjectionMatrix(camera, projectionMatrix) {
+  if (!projectionMatrix) return;
+  if (projectionMatrix instanceof THREE.Matrix4) {
+    camera.projectionMatrix.copy(projectionMatrix);
+  } else {
+    camera.projectionMatrix.fromArray(projectionMatrix);
+  }
+  camera.projectionMatrixInverse.copy(camera.projectionMatrix).invert();
+}
 function createCaptureCamera(options, fallbackCamera, width, height) {
   const camera = options.camera ? options.camera.clone() : fallbackCamera instanceof THREE.PerspectiveCamera ? fallbackCamera.clone() : new THREE.PerspectiveCamera(45, width / height, 0.01, 100);
   if (camera instanceof THREE.PerspectiveCamera) {
@@ -122,6 +197,7 @@ function createCaptureCamera(options, fallbackCamera, width, height) {
     camera.far = options.far ?? camera.far;
     camera.updateProjectionMatrix();
   }
+  applyProjectionMatrix(camera, options.projectionMatrix);
   applyCameraPose(camera, options, fallbackCamera);
   return camera;
 }
@@ -147,9 +223,10 @@ function prepareCaptureCamera(camera, options, fallbackCamera, width, height) {
     camera.far = options.far ?? camera.far;
     camera.updateProjectionMatrix();
   }
+  applyProjectionMatrix(camera, options.projectionMatrix);
   applyCameraPose(camera, options, fallbackCamera);
 }
-function readRenderTargetToCanvas(renderer, target, canvas, context, pixels, imageData, width, height, outputColorSpace) {
+function readRenderTargetToCanvas(renderer, target, canvas, context, pixels, imageData, width, height, outputColorSpace, flipX = false) {
   renderer.readRenderTargetPixels(target, 0, 0, width, height, pixels);
   const rowBytes = width * 4;
   const encodeSrgb = outputColorSpace === THREE.SRGBColorSpace;
@@ -157,16 +234,18 @@ function readRenderTargetToCanvas(renderer, target, canvas, context, pixels, ima
     const sourceStart = (height - y - 1) * rowBytes;
     const targetStart = y * rowBytes;
     const row = pixels.subarray(sourceStart, sourceStart + rowBytes);
-    if (!encodeSrgb) {
+    if (!encodeSrgb && !flipX) {
       imageData.data.set(row, targetStart);
       continue;
     }
-    for (let x = 0; x < rowBytes; x += 4) {
-      const pixelOffset = targetStart + x;
-      imageData.data[pixelOffset] = linearByteToSrgbByte(row[x]);
-      imageData.data[pixelOffset + 1] = linearByteToSrgbByte(row[x + 1]);
-      imageData.data[pixelOffset + 2] = linearByteToSrgbByte(row[x + 2]);
-      imageData.data[pixelOffset + 3] = row[x + 3];
+    for (let x = 0; x < width; x += 1) {
+      const sourceX = flipX ? width - x - 1 : x;
+      const sourceOffset = sourceX * 4;
+      const targetOffset = targetStart + x * 4;
+      imageData.data[targetOffset] = encodeSrgb ? linearByteToSrgbByte(row[sourceOffset]) : row[sourceOffset];
+      imageData.data[targetOffset + 1] = encodeSrgb ? linearByteToSrgbByte(row[sourceOffset + 1]) : row[sourceOffset + 1];
+      imageData.data[targetOffset + 2] = encodeSrgb ? linearByteToSrgbByte(row[sourceOffset + 2]) : row[sourceOffset + 2];
+      imageData.data[targetOffset + 3] = row[sourceOffset + 3];
     }
   }
   context.putImageData(imageData, 0, 0);
@@ -177,16 +256,28 @@ function linearByteToSrgbByte(value) {
   const encoded = normalized <= 31308e-7 ? normalized * 12.92 : 1.055 * Math.pow(normalized, 1 / 2.4) - 0.055;
   return Math.min(255, Math.max(0, Math.round(encoded * 255)));
 }
-function readPixelsToCanvas(pixels, context, imageData, width, height, flipY = true) {
+function readPixelsToCanvas(pixels, context, imageData, width, height, flipY = true, flipX = false) {
   const rowBytes = width * 4;
   for (let y = 0; y < height; y += 1) {
     const sourceY = flipY ? height - y - 1 : y;
     const sourceStart = sourceY * rowBytes;
     const targetStart = y * rowBytes;
-    imageData.data.set(
-      pixels.subarray(sourceStart, sourceStart + rowBytes),
-      targetStart
-    );
+    if (!flipX) {
+      imageData.data.set(
+        pixels.subarray(sourceStart, sourceStart + rowBytes),
+        targetStart
+      );
+      continue;
+    }
+    for (let x = 0; x < width; x += 1) {
+      const sourceX = width - x - 1;
+      const sourceOffset = sourceStart + sourceX * 4;
+      const targetOffset = targetStart + x * 4;
+      imageData.data[targetOffset] = pixels[sourceOffset];
+      imageData.data[targetOffset + 1] = pixels[sourceOffset + 1];
+      imageData.data[targetOffset + 2] = pixels[sourceOffset + 2];
+      imageData.data[targetOffset + 3] = pixels[sourceOffset + 3];
+    }
   }
   context.putImageData(imageData, 0, 0);
 }
@@ -255,7 +346,10 @@ function saveRendererState(renderer) {
     scissorTest: renderer.getScissorTest(),
     clearColor,
     clearAlpha: renderer.getClearAlpha(),
-    autoClear: renderer.autoClear
+    autoClear: renderer.autoClear,
+    shadowMapEnabled: renderer.shadowMap.enabled,
+    toneMapping: renderer.toneMapping,
+    outputColorSpace: renderer.outputColorSpace
   };
 }
 function restoreRendererState(renderer, state) {
@@ -266,6 +360,51 @@ function restoreRendererState(renderer, state) {
   renderer.setScissorTest(state.scissorTest);
   renderer.setClearColor(state.clearColor, state.clearAlpha);
   renderer.autoClear = state.autoClear;
+  renderer.shadowMap.enabled = state.shadowMapEnabled;
+  renderer.toneMapping = state.toneMapping;
+  renderer.outputColorSpace = state.outputColorSpace;
+}
+function saveSceneVisualState(scene) {
+  return {
+    background: scene.background,
+    environment: scene.environment,
+    fog: scene.fog
+  };
+}
+function restoreSceneVisualState(scene, state) {
+  scene.background = state.background;
+  scene.environment = state.environment;
+  scene.fog = state.fog;
+}
+function hasOwn(object, key) {
+  return Object.prototype.hasOwnProperty.call(object, key);
+}
+function applyCaptureVisualOverrides(renderer, scene, options) {
+  const overrides = options.visualOverrides;
+  if (!overrides) return null;
+  const previousSceneState = saveSceneVisualState(scene);
+  if (hasOwn(overrides, "sceneBackground")) {
+    const background = overrides.sceneBackground;
+    scene.background = background === false ? null : typeof background === "string" || typeof background === "number" ? new THREE.Color(background) : background ?? null;
+  }
+  if (hasOwn(overrides, "sceneEnvironment")) {
+    const environment = overrides.sceneEnvironment;
+    scene.environment = environment === false ? null : environment ?? null;
+  }
+  if (hasOwn(overrides, "sceneFog")) {
+    const fog = overrides.sceneFog;
+    scene.fog = fog === false ? null : fog ?? null;
+  }
+  if (overrides.shadows !== void 0) {
+    renderer.shadowMap.enabled = overrides.shadows;
+  }
+  if (overrides.toneMapping !== void 0) {
+    renderer.toneMapping = overrides.toneMapping;
+  }
+  if (overrides.outputColorSpace !== void 0) {
+    renderer.outputColorSpace = overrides.outputColorSpace;
+  }
+  return previousSceneState;
 }
 function getCaptureRenderer(scene) {
   const renderers = [];
@@ -286,6 +425,8 @@ function runCapturePreRenderHooks(scene) {
 }
 function createCameraFrameCaptureSession(renderer, scene, fallbackCamera, options = {}) {
   const { width, height } = getCaptureDimensions(renderer, options);
+  const isolatedRenderer = createIsolatedRenderer(renderer, width, height, options);
+  const sessionRenderer = isolatedRenderer?.renderer ?? renderer;
   const camera = createCaptureCamera(options, fallbackCamera, width, height);
   const target = new THREE.WebGLRenderTarget(width, height, {
     format: THREE.RGBAFormat,
@@ -304,6 +445,11 @@ function createCameraFrameCaptureSession(renderer, scene, fallbackCamera, option
   const imageData = drawContext.createImageData(width, height);
   function resolveCaptureOptions(nextOptions = {}) {
     const captureOptions = { ...options, ...nextOptions };
+    if (shouldUseRenderIsolation(captureOptions) !== shouldUseRenderIsolation(options)) {
+      throw new Error(
+        "Camera frame capture sessions require stable renderIsolation settings."
+      );
+    }
     const nextDimensions = getCaptureDimensions(renderer, captureOptions);
     if (nextDimensions.width !== width || nextDimensions.height !== height) {
       throw new Error(
@@ -320,7 +466,12 @@ function createCameraFrameCaptureSession(renderer, scene, fallbackCamera, option
     return captureOptions;
   }
   function renderPreparedCapture(captureOptions) {
-    const previousState = saveRendererState(renderer);
+    const previousState = saveRendererState(sessionRenderer);
+    const previousSceneState = applyCaptureVisualOverrides(
+      sessionRenderer,
+      scene,
+      captureOptions
+    );
     const hidden = [
       ...hideExcludedCaptureObjects(scene),
       ...hideCaptureGeomGroups(scene, captureOptions)
@@ -328,23 +479,23 @@ function createCameraFrameCaptureSession(renderer, scene, fallbackCamera, option
     runCapturePreRenderHooks(scene);
     scene.updateMatrixWorld(true);
     try {
-      renderer.xr.enabled = false;
-      renderer.setRenderTarget(target);
-      renderer.setViewport(0, 0, width, height);
-      renderer.setScissor(0, 0, width, height);
-      renderer.setScissorTest(false);
+      sessionRenderer.xr.enabled = false;
+      sessionRenderer.setRenderTarget(target);
+      sessionRenderer.setViewport(0, 0, width, height);
+      sessionRenderer.setScissor(0, 0, width, height);
+      sessionRenderer.setScissorTest(false);
       if (captureOptions.background !== void 0) {
-        renderer.setClearColor(
+        sessionRenderer.setClearColor(
           new THREE.Color(captureOptions.background),
           captureOptions.backgroundAlpha ?? previousState.clearAlpha
         );
       } else if (captureOptions.backgroundAlpha !== void 0) {
-        renderer.setClearColor(previousState.clearColor, captureOptions.backgroundAlpha);
+        sessionRenderer.setClearColor(previousState.clearColor, captureOptions.backgroundAlpha);
       }
-      renderer.clear();
-      renderer.render(scene, camera);
+      sessionRenderer.clear();
+      sessionRenderer.render(scene, camera);
       readRenderTargetToCanvas(
-        renderer,
+        sessionRenderer,
         target,
         canvas,
         drawContext,
@@ -352,7 +503,8 @@ function createCameraFrameCaptureSession(renderer, scene, fallbackCamera, option
         imageData,
         width,
         height,
-        renderer.outputColorSpace
+        sessionRenderer.outputColorSpace,
+        captureOptions.flipX ?? false
       );
       return {
         canvas,
@@ -363,7 +515,8 @@ function createCameraFrameCaptureSession(renderer, scene, fallbackCamera, option
       };
     } finally {
       restoreObjectVisibility(hidden);
-      restoreRendererState(renderer, previousState);
+      if (previousSceneState) restoreSceneVisualState(scene, previousSceneState);
+      restoreRendererState(sessionRenderer, previousState);
     }
   }
   function capture(nextOptions = {}) {
@@ -375,23 +528,28 @@ function createCameraFrameCaptureSession(renderer, scene, fallbackCamera, option
     scene.updateMatrixWorld(true);
     const captureRenderer = getCaptureRenderer(scene);
     if (captureRenderer) {
-      const previousState = saveRendererState(renderer);
+      const previousState = saveRendererState(sessionRenderer);
+      const previousSceneState = applyCaptureVisualOverrides(
+        sessionRenderer,
+        scene,
+        captureOptions
+      );
       const hidden = [
         ...hideExcludedCaptureObjects(scene),
         ...hideCaptureGeomGroups(scene, captureOptions)
       ];
       try {
-        renderer.xr.enabled = false;
+        sessionRenderer.xr.enabled = false;
         if (captureOptions.background !== void 0) {
-          renderer.setClearColor(
+          sessionRenderer.setClearColor(
             new THREE.Color(captureOptions.background),
             captureOptions.backgroundAlpha ?? previousState.clearAlpha
           );
         } else if (captureOptions.backgroundAlpha !== void 0) {
-          renderer.setClearColor(previousState.clearColor, captureOptions.backgroundAlpha);
+          sessionRenderer.setClearColor(previousState.clearColor, captureOptions.backgroundAlpha);
         }
         const captureResult = await captureRenderer({
-          renderer,
+          renderer: sessionRenderer,
           scene,
           camera,
           target,
@@ -412,7 +570,8 @@ function createCameraFrameCaptureSession(renderer, scene, fallbackCamera, option
             imageData,
             width,
             height,
-            captureResult.flipY ?? true
+            captureResult.flipY ?? true,
+            captureResult.flipX ?? captureOptions.flipX ?? false
           );
           return {
             canvas,
@@ -424,7 +583,8 @@ function createCameraFrameCaptureSession(renderer, scene, fallbackCamera, option
         }
       } finally {
         restoreObjectVisibility(hidden);
-        restoreRendererState(renderer, previousState);
+        if (previousSceneState) restoreSceneVisualState(scene, previousSceneState);
+        restoreRendererState(sessionRenderer, previousState);
       }
     }
     return renderPreparedCapture(captureOptions);
@@ -475,6 +635,9 @@ function createCameraFrameCaptureSession(renderer, scene, fallbackCamera, option
     },
     dispose() {
       target.dispose();
+      if (isolatedRenderer && !isolatedRenderer.cached) {
+        isolatedRenderer.renderer.dispose();
+      }
     }
   };
 }
@@ -1129,5 +1292,5 @@ function clamp01(value) {
  */
 
 export { CAMERA_FRAME_CAPTURE_PRE_RENDER_USER_DATA_KEY, CAMERA_FRAME_CAPTURE_RENDER_USER_DATA_KEY, CAPTURE_EXCLUDE_KEY, ModelActuators, ModelBodies, ModelCameras, ModelGeoms, ModelJoints, ModelKeyframes, ModelResources, ModelSensors, ModelSites, ScenarioLighting, SplatEnvironment, SplatEnvironmentReadinessStatus, VisualScenarioEffects, captureCameraFrame, captureCameraFrameBlob, createCameraFrameCaptureSession, createPairedSplatEnvironment, createSparkSplatViewerUrl, createSplatEnvironmentUserData, createSplatSceneConfig, createVisualScenarioExecutionContext, getContact, getScenarioBackground, getScenarioCameraPosition, getSplatEnvironmentReadiness, registerModelResources, renderCameraFrameToCanvas, useSplatEnvironment, useSplatSceneConfig, useVisualScenarioEffects, useVisualScenarioExecutionContext, withContacts, withSplatEnvironment };
-//# sourceMappingURL=chunk-3BMNRSS2.js.map
-//# sourceMappingURL=chunk-3BMNRSS2.js.map
+//# sourceMappingURL=chunk-6AZEFI6A.js.map
+//# sourceMappingURL=chunk-6AZEFI6A.js.map