mujoco-react 9.3.0 → 9.5.0

package/dist/{chunk-T3GVZJ4F.js → chunk-6MOK6ZWB.js}

@@ -1,6 +1,6 @@
-import { useThree } from '@react-three/fiber';
-import { useEffect, useMemo } from 'react';
 import * as THREE from 'three';
+import { useThree } from '@react-three/fiber';
+import { useMemo, useEffect } from 'react';
 import { jsxs, Fragment, jsx } from 'react/jsx-runtime';
 
 // src/types.ts
@@ -85,6 +85,393 @@ var SplatEnvironmentReadinessStatus = {
   UnsupportedFormat: "unsupported-format",
   Ready: "ready"
 };
+var CAMERA_FRAME_CAPTURE_RENDER_USER_DATA_KEY = "mujocoReactCameraFrameCaptureRender";
+var CAPTURE_EXCLUDE_KEY = "mujoco.capture.exclude";
+function toVector3(value, fallback) {
+  if (!value) return fallback.clone();
+  return value instanceof THREE.Vector3 ? value.clone() : new THREE.Vector3(value[0], value[1], value[2]);
+}
+function applyCameraPose(camera, options, fallbackCamera) {
+  camera.position.copy(toVector3(options.position, fallbackCamera.position));
+  camera.up.copy(toVector3(options.up, fallbackCamera.up));
+  if (options.quaternion) {
+    if (options.quaternion instanceof THREE.Quaternion) {
+      camera.quaternion.copy(options.quaternion);
+    } else {
+      camera.quaternion.set(
+        options.quaternion[0],
+        options.quaternion[1],
+        options.quaternion[2],
+        options.quaternion[3]
+      );
+    }
+  } else if (options.lookAt) {
+    camera.lookAt(toVector3(options.lookAt, new THREE.Vector3()));
+  } else {
+    camera.quaternion.copy(fallbackCamera.quaternion);
+  }
+  camera.updateMatrixWorld();
+}
+function createCaptureCamera(options, fallbackCamera, width, height) {
+  const camera = options.camera ? options.camera.clone() : fallbackCamera instanceof THREE.PerspectiveCamera ? fallbackCamera.clone() : new THREE.PerspectiveCamera(45, width / height, 0.01, 100);
+  if (camera instanceof THREE.PerspectiveCamera) {
+    camera.aspect = width / height;
+    camera.fov = options.fov ?? camera.fov;
+    camera.near = options.near ?? camera.near;
+    camera.far = options.far ?? camera.far;
+    camera.updateProjectionMatrix();
+  }
+  applyCameraPose(camera, options, fallbackCamera);
+  return camera;
+}
+function getCaptureDimensions(renderer, options) {
+  const width = Math.max(
+    1,
+    Math.floor(options.width ?? renderer.domElement.width)
+  );
+  const height = Math.max(
+    1,
+    Math.floor(options.height ?? renderer.domElement.height)
+  );
+  return { width, height };
+}
+function prepareCaptureCamera(camera, options, fallbackCamera, width, height) {
+  if (options.camera) {
+    camera.copy(options.camera);
+  }
+  if (camera instanceof THREE.PerspectiveCamera) {
+    camera.aspect = width / height;
+    camera.fov = options.fov ?? camera.fov;
+    camera.near = options.near ?? camera.near;
+    camera.far = options.far ?? camera.far;
+    camera.updateProjectionMatrix();
+  }
+  applyCameraPose(camera, options, fallbackCamera);
+}
+function readRenderTargetToCanvas(renderer, target, canvas, context, pixels, imageData, width, height, outputColorSpace) {
+  renderer.readRenderTargetPixels(target, 0, 0, width, height, pixels);
+  const rowBytes = width * 4;
+  const encodeSrgb = outputColorSpace === THREE.SRGBColorSpace;
+  for (let y = 0; y < height; y += 1) {
+    const sourceStart = (height - y - 1) * rowBytes;
+    const targetStart = y * rowBytes;
+    const row = pixels.subarray(sourceStart, sourceStart + rowBytes);
+    if (!encodeSrgb) {
+      imageData.data.set(row, targetStart);
+      continue;
+    }
+    for (let x = 0; x < rowBytes; x += 4) {
+      const pixelOffset = targetStart + x;
+      imageData.data[pixelOffset] = linearByteToSrgbByte(row[x]);
+      imageData.data[pixelOffset + 1] = linearByteToSrgbByte(row[x + 1]);
+      imageData.data[pixelOffset + 2] = linearByteToSrgbByte(row[x + 2]);
+      imageData.data[pixelOffset + 3] = row[x + 3];
+    }
+  }
+  context.putImageData(imageData, 0, 0);
+  return canvas;
+}
+function linearByteToSrgbByte(value) {
+  const normalized = value / 255;
+  const encoded = normalized <= 31308e-7 ? normalized * 12.92 : 1.055 * Math.pow(normalized, 1 / 2.4) - 0.055;
+  return Math.min(255, Math.max(0, Math.round(encoded * 255)));
+}
+function readPixelsToCanvas(pixels, context, imageData, width, height, flipY = true) {
+  const rowBytes = width * 4;
+  for (let y = 0; y < height; y += 1) {
+    const sourceY = flipY ? height - y - 1 : y;
+    const sourceStart = sourceY * rowBytes;
+    const targetStart = y * rowBytes;
+    imageData.data.set(
+      pixels.subarray(sourceStart, sourceStart + rowBytes),
+      targetStart
+    );
+  }
+  context.putImageData(imageData, 0, 0);
+}
+function hideExcludedCaptureObjects(scene) {
+  const hidden = [];
+  scene.traverse((object) => {
+    if (!object.visible) return;
+    if (!object.userData[CAPTURE_EXCLUDE_KEY]) return;
+    hidden.push({ object, visible: object.visible });
+    object.visible = false;
+  });
+  return hidden;
+}
+function restoreObjectVisibility(hidden) {
+  for (const { object, visible } of hidden) {
+    object.visible = visible;
+  }
+}
+function getCameraFrameCaptureSource(options) {
+  if (options.source) return options.source;
+  if (options.cameraName) {
+    return { kind: "mujoco-camera", cameraName: options.cameraName };
+  }
+  if (options.siteName) {
+    return { kind: "mujoco-site", siteName: options.siteName };
+  }
+  if (options.bodyName) {
+    return { kind: "mujoco-body", bodyName: options.bodyName };
+  }
+  if (options.camera) return { kind: "custom-camera" };
+  if (options.position || options.lookAt || options.quaternion) {
+    return { kind: "explicit-pose" };
+  }
+  return { kind: "fallback-camera" };
+}
+function saveRendererState(renderer) {
+  const viewport = new THREE.Vector4();
+  const scissor = new THREE.Vector4();
+  const clearColor = new THREE.Color();
+  renderer.getViewport(viewport);
+  renderer.getScissor(scissor);
+  renderer.getClearColor(clearColor);
+  return {
+    target: renderer.getRenderTarget(),
+    xrEnabled: renderer.xr.enabled,
+    viewport,
+    scissor,
+    scissorTest: renderer.getScissorTest(),
+    clearColor,
+    clearAlpha: renderer.getClearAlpha(),
+    autoClear: renderer.autoClear
+  };
+}
+function restoreRendererState(renderer, state) {
+  renderer.setRenderTarget(state.target);
+  renderer.xr.enabled = state.xrEnabled;
+  renderer.setViewport(state.viewport);
+  renderer.setScissor(state.scissor);
+  renderer.setScissorTest(state.scissorTest);
+  renderer.setClearColor(state.clearColor, state.clearAlpha);
+  renderer.autoClear = state.autoClear;
+}
+function getCaptureRenderer(scene) {
+  const renderers = [];
+  scene.traverse((object) => {
+    if (renderers.length) return;
+    const render = object.userData[CAMERA_FRAME_CAPTURE_RENDER_USER_DATA_KEY];
+    if (typeof render === "function") renderers.push(render);
+  });
+  return renderers[0] ?? null;
+}
+function createCameraFrameCaptureSession(renderer, scene, fallbackCamera, options = {}) {
+  const { width, height } = getCaptureDimensions(renderer, options);
+  const camera = createCaptureCamera(options, fallbackCamera, width, height);
+  const target = new THREE.WebGLRenderTarget(width, height, {
+    format: THREE.RGBAFormat,
+    type: THREE.UnsignedByteType
+  });
+  const canvas = document.createElement("canvas");
+  canvas.width = width;
+  canvas.height = height;
+  const context = canvas.getContext("2d");
+  if (!context) {
+    target.dispose();
+    throw new Error("Unable to create a 2D canvas for camera frame capture.");
+  }
+  const drawContext = context;
+  const pixels = new Uint8Array(width * height * 4);
+  const imageData = drawContext.createImageData(width, height);
+  function resolveCaptureOptions(nextOptions = {}) {
+    const captureOptions = { ...options, ...nextOptions };
+    const nextDimensions = getCaptureDimensions(renderer, captureOptions);
+    if (nextDimensions.width !== width || nextDimensions.height !== height) {
+      throw new Error(
+        "Camera frame capture sessions require stable width and height."
+      );
+    }
+    prepareCaptureCamera(
+      camera,
+      captureOptions,
+      fallbackCamera,
+      width,
+      height
+    );
+    return captureOptions;
+  }
+  function renderPreparedCapture(captureOptions) {
+    const previousState = saveRendererState(renderer);
+    const hidden = hideExcludedCaptureObjects(scene);
+    scene.updateMatrixWorld(true);
+    try {
+      renderer.xr.enabled = false;
+      renderer.setRenderTarget(target);
+      renderer.setViewport(0, 0, width, height);
+      renderer.setScissor(0, 0, width, height);
+      renderer.setScissorTest(false);
+      renderer.clear();
+      renderer.render(scene, camera);
+      readRenderTargetToCanvas(
+        renderer,
+        target,
+        canvas,
+        drawContext,
+        pixels,
+        imageData,
+        width,
+        height,
+        renderer.outputColorSpace
+      );
+      return {
+        canvas,
+        camera,
+        width,
+        height,
+        source: getCameraFrameCaptureSource(captureOptions)
+      };
+    } finally {
+      restoreObjectVisibility(hidden);
+      restoreRendererState(renderer, previousState);
+    }
+  }
+  function capture(nextOptions = {}) {
+    return renderPreparedCapture(resolveCaptureOptions(nextOptions));
+  }
+  async function captureAsync(nextOptions = {}) {
+    const captureOptions = resolveCaptureOptions(nextOptions);
+    scene.updateMatrixWorld(true);
+    const captureRenderer = getCaptureRenderer(scene);
+    if (captureRenderer) {
+      const previousState = saveRendererState(renderer);
+      const hidden = hideExcludedCaptureObjects(scene);
+      try {
+        renderer.xr.enabled = false;
+        const captureResult = await captureRenderer({
+          renderer,
+          scene,
+          camera,
+          target,
+          width,
+          height
+        });
+        if (captureResult) {
+          const captureWidth = captureResult.width ?? width;
+          const captureHeight = captureResult.height ?? height;
+          if (captureWidth !== width || captureHeight !== height) {
+            throw new Error(
+              "Camera frame capture renderer returned unexpected dimensions."
+            );
+          }
+          readPixelsToCanvas(
+            captureResult.pixels,
+            drawContext,
+            imageData,
+            width,
+            height,
+            captureResult.flipY ?? true
+          );
+          return {
+            canvas,
+            camera,
+            width,
+            height,
+            source: getCameraFrameCaptureSource(captureOptions)
+          };
+        }
+      } finally {
+        restoreObjectVisibility(hidden);
+        restoreRendererState(renderer, previousState);
+      }
+    }
+    return renderPreparedCapture(captureOptions);
+  }
+  return {
+    width,
+    height,
+    capture,
+    captureAsync,
+    captureDataUrl(nextOptions = {}) {
+      const type = nextOptions.type ?? options.type ?? "image/png";
+      const result = capture(nextOptions);
+      return {
+        ...result,
+        dataUrl: result.canvas.toDataURL(
+          type,
+          nextOptions.quality ?? options.quality
+        ),
+        type
+      };
+    },
+    async captureDataUrlAsync(nextOptions = {}) {
+      const type = nextOptions.type ?? options.type ?? "image/png";
+      const result = await captureAsync(nextOptions);
+      return {
+        ...result,
+        dataUrl: result.canvas.toDataURL(
+          type,
+          nextOptions.quality ?? options.quality
+        ),
+        type
+      };
+    },
+    async captureBlob(nextOptions = {}) {
+      const type = nextOptions.type ?? options.type ?? "image/png";
+      const result = await captureAsync(nextOptions);
+      const blob = await new Promise((resolve, reject) => {
+        result.canvas.toBlob(
+          (nextBlob) => {
+            if (nextBlob) resolve(nextBlob);
+            else reject(new Error("Camera frame capture did not produce a Blob."));
+          },
+          type,
+          nextOptions.quality ?? options.quality
+        );
+      });
+      return { ...result, blob, type };
+    },
+    dispose() {
+      target.dispose();
+    }
+  };
+}
+function renderCameraFrameToCanvas(renderer, scene, fallbackCamera, options = {}) {
+  const session = createCameraFrameCaptureSession(
+    renderer,
+    scene,
+    fallbackCamera,
+    options
+  );
+  try {
+    return session.capture();
+  } finally {
+    session.dispose();
+  }
+}
+async function captureCameraFrame(renderer, scene, fallbackCamera, options = {}) {
+  const type = options.type ?? "image/png";
+  const session = createCameraFrameCaptureSession(
+    renderer,
+    scene,
+    fallbackCamera,
+    options
+  );
+  try {
+    const result = await session.captureAsync();
+    return {
+      ...result,
+      dataUrl: result.canvas.toDataURL(type, options.quality),
+      type
+    };
+  } finally {
+    session.dispose();
+  }
+}
+async function captureCameraFrameBlob(renderer, scene, fallbackCamera, options = {}) {
+  const session = createCameraFrameCaptureSession(
+    renderer,
+    scene,
+    fallbackCamera,
+    options
+  );
+  try {
+    return await session.captureBlob();
+  } finally {
+    session.dispose();
+  }
+}
 var DEFAULT_BACKGROUND = "#181a1f";
 function ScenarioLighting({
   preset = "studio",
@@ -160,6 +547,75 @@ function getScenarioCameraPosition(basePosition, scenario) {
     Number((z + jitter * 0.25).toFixed(3))
   ];
 }
+function useVisualScenarioExecutionContext({
+  scenario,
+  environment,
+  renderer,
+  variantId,
+  enabled
+}) {
+  return useMemo(
+    () => createVisualScenarioExecutionContext({
+      scenario,
+      environment,
+      renderer,
+      variantId,
+      enabled
+    }),
+    [enabled, environment, renderer, scenario, variantId]
+  );
+}
+function createVisualScenarioExecutionContext({
+  scenario,
+  environment,
+  renderer,
+  variantId,
+  enabled = true
+}) {
+  const pairedEnvironment = environment ?? (scenario ? createPairedSplatEnvironment(scenario, { renderer }) : void 0);
+  const splat = scenario?.splat;
+  const collisionProxy = pairedEnvironment?.collisionProxy ?? splat?.collisionProxy ?? void 0;
+  const readiness = getSplatEnvironmentReadiness({
+    environment: pairedEnvironment,
+    scenario,
+    renderer,
+    enabled
+  });
+  const format = pairedEnvironment?.splat.format ?? splat?.format ?? readiness.format ?? "spz";
+  return {
+    scenarioId: scenario?.id ?? pairedEnvironment?.id ?? "visual-scenario",
+    scenarioLabel: scenario?.label ?? pairedEnvironment?.label ?? "Visual scenario",
+    variantId,
+    seed: scenario?.seed ?? 0,
+    lighting: scenario?.lighting ?? "studio",
+    environment: scenario?.environment,
+    camera: {
+      jitter: scenario?.camera?.jitter ?? 0,
+      exposure: scenario?.camera?.exposure ?? 1,
+      noise: scenario?.camera?.noise ?? 0,
+      blur: scenario?.camera?.blur ?? 0
+    },
+    materials: {
+      randomizeObjectColors: Boolean(
+        scenario?.materials?.randomizeObjectColors
+      ),
+      randomizeTableMaterial: Boolean(
+        scenario?.materials?.randomizeTableMaterial
+      ),
+      roughness: scenario?.materials?.roughness,
+      metalness: scenario?.materials?.metalness
+    },
+    splatEnabled: Boolean(splat?.enabled || pairedEnvironment),
+    splatSrc: pairedEnvironment?.splat.src ?? splat?.src,
+    splatFormat: format,
+    splatRenderer: renderer ?? pairedEnvironment?.splat.renderer,
+    collisionProxyXmlPath: collisionProxy?.xmlPath,
+    collisionProxyStatus: collisionProxy?.status,
+    collisionProxyPrimitives: collisionProxy?.primitives ?? [],
+    readiness,
+    transformSource: "visualScenario.camera"
+  };
+}
 function VisualScenarioEffects(props) {
   useVisualScenarioEffects(props);
   return null;
@@ -335,33 +791,39 @@ function useSplatSceneConfig({
   enabled = true,
   renderer
 }) {
-  const resolvedEnvironment = useMemo(
-    () => enabled ? environment ?? (scenario ? createPairedSplatEnvironment(scenario, { renderer }) : void 0) : void 0,
-    [enabled, environment, renderer, scenario]
-  );
-  const readiness = useMemo(
-    () => getSplatEnvironmentReadiness({
-      environment: resolvedEnvironment,
-      scenario,
-      renderer,
-      enabled
-    }),
-    [enabled, renderer, resolvedEnvironment, scenario]
-  );
-  const resolvedSceneConfig = useMemo(
-    () => resolvedEnvironment ? withSplatEnvironment(sceneConfig, resolvedEnvironment) : sceneConfig,
-    [resolvedEnvironment, sceneConfig]
-  );
   return useMemo(
-    () => ({
-      environment: resolvedEnvironment,
-      sceneConfig: resolvedSceneConfig,
-      enabled: enabled && readiness.status !== SplatEnvironmentReadinessStatus.Disabled,
-      readiness
+    () => createSplatSceneConfig({
+      sceneConfig,
+      scenario,
+      environment,
+      enabled,
+      renderer
     }),
-    [enabled, readiness, resolvedEnvironment, resolvedSceneConfig]
+    [enabled, environment, renderer, scenario, sceneConfig]
   );
 }
+function createSplatSceneConfig({
+  sceneConfig,
+  scenario,
+  environment,
+  enabled = true,
+  renderer
+}) {
+  const resolvedEnvironment = enabled ? environment ?? (scenario ? createPairedSplatEnvironment(scenario, { renderer }) : void 0) : void 0;
+  const readiness = getSplatEnvironmentReadiness({
+    environment: resolvedEnvironment,
+    scenario,
+    renderer,
+    enabled
+  });
+  const resolvedSceneConfig = resolvedEnvironment ? withSplatEnvironment(sceneConfig, resolvedEnvironment, { renderer }) : sceneConfig;
+  return {
+    environment: resolvedEnvironment,
+    sceneConfig: resolvedSceneConfig,
+    enabled: enabled && readiness.status !== SplatEnvironmentReadinessStatus.Disabled,
+    readiness
+  };
+}
 function getSplatEnvironmentReadiness({
   environment,
   scenario,
@@ -434,7 +896,7 @@ function getSplatEnvironmentReadiness({
 function createPairedSplatEnvironment(scenario, options = {}) {
   const splat = scenario.splat;
   const collisionProxy = splat?.collisionProxy;
-  if (!splat?.enabled || !splat.src || !collisionProxy?.xmlPath) {
+  if (!splat?.enabled || !splat.src) {
     return void 0;
   }
   return {
@@ -446,14 +908,14 @@ function createPairedSplatEnvironment(scenario, options = {}) {
       format: splat.format ?? "spz",
       renderer: options.renderer
     },
-    collisionProxy: {
+    collisionProxy: collisionProxy?.xmlPath ? {
       ...collisionProxy,
       xmlPath: collisionProxy.xmlPath
-    }
+    } : void 0
   };
 }
 function isPairedSplatEnvironment(input) {
-  return !!input && "collisionProxy" in input && "splat" in input;
+  return !!input && "splat" in input && !!input.splat && !("enabled" in input.splat);
 }
 function sceneRelativePath(sceneConfig, path) {
   const src = sceneConfig.src;
@@ -474,7 +936,7 @@ function uniquePaths(paths) {
 }
 function withSplatEnvironment(sceneConfig, input, options = {}) {
   const environment = isPairedSplatEnvironment(input) ? input : input ? createPairedSplatEnvironment(input, options) : void 0;
-  const xmlPath = environment?.collisionProxy.xmlPath;
+  const xmlPath = environment?.collisionProxy?.xmlPath;
   if (!xmlPath) return sceneConfig;
   return {
     ...sceneConfig,
@@ -608,7 +1070,13 @@ function clamp01(value) {
  * @license
  * SPDX-License-Identifier: Apache-2.0
  */
+/**
+ * @license
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Offscreen camera-frame capture for R3F/MuJoCo scenes.
+ */
 
-export { RobotActuators, RobotBodies, RobotCameras, RobotGeoms, RobotJoints, RobotKeyframes, RobotResources, RobotSensors, RobotSites, ScenarioLighting, SplatEnvironment, SplatEnvironmentReadinessStatus, VisualScenarioEffects, createPairedSplatEnvironment, createSparkSplatViewerUrl, createSplatEnvironmentUserData, getContact, getScenarioBackground, getScenarioCameraPosition, getSplatEnvironmentReadiness, registerRobotResources, useSplatEnvironment, useSplatSceneConfig, useVisualScenarioEffects, withContacts, withSplatEnvironment };
-//# sourceMappingURL=chunk-T3GVZJ4F.js.map
-//# sourceMappingURL=chunk-T3GVZJ4F.js.map
+export { CAMERA_FRAME_CAPTURE_RENDER_USER_DATA_KEY, CAPTURE_EXCLUDE_KEY, RobotActuators, RobotBodies, RobotCameras, RobotGeoms, RobotJoints, RobotKeyframes, RobotResources, RobotSensors, RobotSites, ScenarioLighting, SplatEnvironment, SplatEnvironmentReadinessStatus, VisualScenarioEffects, captureCameraFrame, captureCameraFrameBlob, createCameraFrameCaptureSession, createPairedSplatEnvironment, createSparkSplatViewerUrl, createSplatEnvironmentUserData, createSplatSceneConfig, createVisualScenarioExecutionContext, getContact, getScenarioBackground, getScenarioCameraPosition, getSplatEnvironmentReadiness, registerRobotResources, renderCameraFrameToCanvas, useSplatEnvironment, useSplatSceneConfig, useVisualScenarioEffects, useVisualScenarioExecutionContext, withContacts, withSplatEnvironment };
+//# sourceMappingURL=chunk-6MOK6ZWB.js.map
+//# sourceMappingURL=chunk-6MOK6ZWB.js.map