mujoco-react 9.2.0 → 9.4.0

package/src/hooks/useMountedCameraSequenceRecorder.ts

@@ -0,0 +1,155 @@
+/**
+ * @license
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * React state wrapper for named MuJoCo camera/site/body sequence recording.
+ */
+
+import { useCallback, useState } from 'react';
+import { useMujoco } from '../core/MujocoSimProvider';
+import {
+  createMountedCameraFrameSequenceReadiness,
+  createMountedCameraFrameSequencePlanFromApi,
+  recordMountedCameraFrameSequence,
+  type MountedCameraFrameSequencePlan,
+  type MountedCameraFrameSequencePlanOptions,
+  type MountedCameraFrameSequenceReadiness,
+  type MountedCameraFrameSequenceRecordOptions,
+  type MountedCameraFrameSequenceRecordResult,
+} from '../rendering/cameraFrameSource';
+import type {
+  CameraFrameSequenceRecorderAPI,
+  FrameCaptureStatus,
+} from '../types';
+
+export type MountedCameraSequencePlanOptions =
+  MountedCameraFrameSequencePlanOptions;
+export type MountedCameraSequenceRecordOptions =
+  MountedCameraFrameSequenceRecordOptions;
+export type MountedCameraSequenceRecordResult =
+  MountedCameraFrameSequenceRecordResult;
+export type MountedCameraSequenceReadiness =
+  MountedCameraFrameSequenceReadiness;
+
+export interface MountedCameraSequenceRecorderAPI
+  extends Omit<CameraFrameSequenceRecorderAPI, 'record'> {
+  plan: MountedCameraFrameSequencePlan | null;
+  readiness: MountedCameraSequenceReadiness | null;
+  result: MountedCameraSequenceRecordResult | null;
+  createPlan: (
+    cameraKeys: readonly string[],
+    options?: MountedCameraSequencePlanOptions
+  ) => MountedCameraFrameSequencePlan;
+  checkReadiness: (
+    cameraKeys: readonly string[],
+    options?: MountedCameraSequencePlanOptions
+  ) => MountedCameraSequenceReadiness;
+  record: (
+    options: MountedCameraSequenceRecordOptions
+  ) => Promise<MountedCameraSequenceRecordResult>;
+}
+
+export function useMountedCameraSequenceRecorder(
+  defaultOptions: MountedCameraSequencePlanOptions = {}
+): MountedCameraSequenceRecorderAPI {
+  const mujoco = useMujoco();
+  const [status, setStatus] = useState<FrameCaptureStatus>('idle');
+  const [error, setError] = useState<Error | null>(null);
+  const [plan, setPlan] = useState<MountedCameraFrameSequencePlan | null>(null);
+  const [readiness, setReadiness] =
+    useState<MountedCameraSequenceReadiness | null>(null);
+  const [result, setResult] = useState<MountedCameraSequenceRecordResult | null>(
+    null
+  );
+
+  const reset = useCallback(() => {
+    setStatus('idle');
+    setError(null);
+    setPlan(null);
+    setReadiness(null);
+    setResult(null);
+  }, []);
+
+  const createPlan = useCallback(
+    (
+      cameraKeys: readonly string[],
+      options: MountedCameraSequencePlanOptions = {}
+    ) => {
+      if (!mujoco.api) {
+        throw new Error('MuJoCo scene is not ready for mounted camera sequence planning.');
+      }
+
+      const nextPlan = createMountedCameraFrameSequencePlanFromApi(
+        mujoco.api,
+        cameraKeys,
+        {
+          ...defaultOptions,
+          ...options,
+        }
+      );
+      setPlan(nextPlan);
+      setReadiness(null);
+      return nextPlan;
+    },
+    [defaultOptions, mujoco.api]
+  );
+
+  const checkReadiness = useCallback(
+    (
+      cameraKeys: readonly string[],
+      options: MountedCameraSequencePlanOptions = {}
+    ) => {
+      const nextPlan = createPlan(cameraKeys, options);
+      const nextReadiness = createMountedCameraFrameSequenceReadiness(nextPlan);
+      setReadiness(nextReadiness);
+      return nextReadiness;
+    },
+    [createPlan]
+  );
+
+  const record = useCallback(
+    async (options: MountedCameraSequenceRecordOptions) => {
+      if (!mujoco.api) {
+        throw new Error('MuJoCo scene is not ready for mounted camera sequence recording.');
+      }
+
+      setStatus('capturing');
+      setError(null);
+      setResult(null);
+
+      try {
+        const nextResult = await recordMountedCameraFrameSequence(mujoco.api, {
+          ...defaultOptions,
+          ...options,
+        });
+        setPlan(nextResult.plan);
+        setReadiness(nextResult.readiness);
+        setResult(nextResult);
+        setStatus('captured');
+        return nextResult;
+      } catch (nextError) {
+        const error =
+          nextError instanceof Error
+            ? nextError
+            : new Error('Unable to record the requested mounted camera sequence.');
+        setError(error);
+        setStatus('error');
+        throw error;
+      }
+    },
+    [defaultOptions, mujoco.api]
+  );
+
+  return {
+    status,
+    error,
+    plan,
+    readiness,
+    result,
+    isRecording: status === 'capturing',
+    createPlan,
+    checkReadiness,
+    record,
+    reset,
+  };
+}

package/src/index.ts

@@ -48,12 +48,32 @@ export {
   createPairedSplatEnvironment,
   createSparkSplatViewerUrl,
   createSplatEnvironmentUserData,
+  createSplatSceneConfig,
+  createVisualScenarioExecutionContext,
+  getSplatEnvironmentReadiness,
   getScenarioBackground,
   getScenarioCameraPosition,
   useSplatEnvironment,
+  useSplatSceneConfig,
+  useVisualScenarioExecutionContext,
   useVisualScenarioEffects,
   withSplatEnvironment,
 } from './components/VisualScenario';
+export {
+  canFetchSplatCollisionProxyXml,
+  fetchSplatCollisionProxyXml,
+  parseSplatCollisionProxyGeoms,
+  SplatCollisionProxyPreview,
+  useSplatCollisionProxyGeoms,
+} from './components/SplatCollisionProxyPreview';
+export type {
+  SplatCollisionProxyGeomPreview,
+  SplatCollisionProxyGeomsState,
+  SplatCollisionProxyPreviewProps,
+  SplatCollisionProxyPreviewStatus,
+  SplatCollisionProxyPreviewVector3,
+  UseSplatCollisionProxyGeomsOptions,
+} from './components/SplatCollisionProxyPreview';
 export { Debug } from './components/Debug';
 export { TendonRenderer } from './components/TendonRenderer';
 export { FlexRenderer } from './components/FlexRenderer';
@@ -84,11 +104,56 @@ export {
 } from './hooks/useFrameCapture';
 export { useCameraFrameCapture } from './hooks/useCameraFrameCapture';
 export { useCameraSequenceRecorder } from './hooks/useCameraSequenceRecorder';
+export { useMountedCameraSequenceRecorder } from './hooks/useMountedCameraSequenceRecorder';
+export type {
+  MountedCameraSequencePlanOptions,
+  MountedCameraSequenceRecorderAPI,
+  MountedCameraSequenceReadiness,
+  MountedCameraSequenceRecordOptions,
+  MountedCameraSequenceRecordResult,
+} from './hooks/useMountedCameraSequenceRecorder';
 export {
   captureCameraFrame,
   captureCameraFrameBlob,
+  createCameraFrameCaptureSession,
   renderCameraFrameToCanvas,
 } from './rendering/cameraFrameCapture';
+export {
+  createMountedCameraFrameSequenceManifest,
+  createMountedCameraFrameSequenceReadiness,
+  createMountedCameraFrameSourceSuggestions,
+  MountedCameraFrameSequenceManifestStatus,
+  MountedCameraFrameSequenceReadinessStatus,
+  MountedCameraFrameSourceSuggestionMatch,
+  createMountedCameraFrameSequencePlanFromApi,
+  createMountedCameraFrameSequencePlan,
+  getCameraFrameCaptureSourceTarget,
+  getMountedCameraFrameCaptureSource,
+  isMountedCameraFrameCaptureSource,
+  recordMountedCameraFrameSequence,
+  resolveMountedCameraFrameSource,
+} from './rendering/cameraFrameSource';
+export type {
+  CameraFrameMountSelector,
+  CreateMountedCameraFrameSequenceManifestOptions,
+  CreateMountedCameraFrameSequencePlanOptions,
+  MountedCameraFrameCaptureSource,
+  MountedCameraFrameSequenceManifest,
+  MountedCameraFrameSequencePlanOptions,
+  MountedCameraFrameSequenceRecorderTarget,
+  MountedCameraFrameSequenceCameraOptions,
+  MountedCameraFrameSequenceDefaults,
+  MountedCameraFrameSequencePlan,
+  MountedCameraFrameSequenceReadiness,
+  MountedCameraFrameSequenceRecordOptions,
+  MountedCameraFrameSequenceRecordResult,
+  MountedCameraFrameSequenceSourceReadiness,
+  MountedCameraFrameSequenceStreamSummary,
+  MountedCameraFrameSourceSuggestion,
+  NamedCameraFrameResource,
+  ResolveMountedCameraFrameSourceOptions,
+  ResolvedMountedCameraFrameSource,
+} from './rendering/cameraFrameSource';
 export { useCtrlNoise } from './hooks/useCtrlNoise';
 export { useBodyMeshes } from './hooks/useBodyMeshes';
 export { useSelectionHighlight } from './hooks/useSelectionHighlight';
@@ -130,6 +195,7 @@ export type {
   SiteInfo,
   ActuatorInfo,
   SensorInfo,
+  CameraInfo,
   // Contacts
   ContactInfo,
   // Raycast
@@ -172,8 +238,15 @@ export type {
   SplatScenarioConfig,
   SplatCollisionProxyConfig,
   PairedSplatEnvironmentConfig,
+  SplatEnvironmentReadiness,
   SplatEnvironmentMetadataInput,
   SplatEnvironmentMetadata,
+  VisualScenarioExecutionContext,
+  VisualScenarioExecutionContextInput,
+  ResolvedScenarioCameraConfig,
+  ResolvedScenarioMaterialConfig,
+  SplatSceneConfigInput,
+  SplatSceneConfigState,
   SplatSceneInput,
   VisualScenarioConfig,
   ScenarioLightingProps,
@@ -193,12 +266,16 @@ export type {
   CameraFrameCaptureOptions,
   CameraFrameCaptureQuaternion,
   CameraFrameCaptureResult,
+  CameraFrameCaptureSource,
   CameraFrameCaptureVector3,
   CameraFrameSequenceCamera,
+  CameraFrameSequenceCameraSummary,
   CameraFrameSequenceFrame,
   CameraFrameSequenceOptions,
   CameraFrameSequenceRecorderAPI,
   CameraFrameSequenceResult,
+  CameraFrameSequenceSampleInput,
+  CameraFrameSequenceStepInput,
   MujocoCanvasProps,
   MujocoContextValue,
   // Hook return types
@@ -227,6 +304,7 @@ export type {
   Sites,
   Geoms,
   Keyframes,
+  Cameras,
 } from './types';
 
 export {
@@ -239,6 +317,8 @@ export {
   RobotSites,
   RobotGeoms,
   RobotKeyframes,
+  RobotCameras,
+  SplatEnvironmentReadinessStatus,
 } from './types';
 
 // Re-export MuJoCo types for convenience

package/src/rendering/cameraFrameCapture.ts

@@ -10,9 +10,25 @@ import type {
   CameraFrameCaptureBlobResult,
   CameraFrameCaptureOptions,
   CameraFrameCaptureResult,
+  CameraFrameCaptureSource,
   CameraFrameCaptureVector3,
 } from '../types';
 
+export interface CameraFrameCaptureSession {
+  readonly width: number;
+  readonly height: number;
+  capture(options?: CameraFrameCaptureOptions): {
+    canvas: HTMLCanvasElement;
+    camera: THREE.Camera;
+    width: number;
+    height: number;
+    source: CameraFrameCaptureSource;
+  };
+  captureDataUrl(options?: CameraFrameCaptureOptions): CameraFrameCaptureResult;
+  captureBlob(options?: CameraFrameCaptureOptions): Promise<CameraFrameCaptureBlobResult>;
+  dispose(): void;
+}
+
 function toVector3(
   value: CameraFrameCaptureVector3 | undefined,
   fallback: THREE.Vector3
@@ -75,24 +91,55 @@ function createCaptureCamera(
   return camera;
 }
 
+function getCaptureDimensions(
+  renderer: THREE.WebGLRenderer,
+  options: CameraFrameCaptureOptions
+) {
+  const width = Math.max(
+    1,
+    Math.floor(options.width ?? renderer.domElement.width)
+  );
+  const height = Math.max(
+    1,
+    Math.floor(options.height ?? renderer.domElement.height)
+  );
+  return { width, height };
+}
+
+function prepareCaptureCamera(
+  camera: THREE.Camera,
+  options: CameraFrameCaptureOptions,
+  fallbackCamera: THREE.Camera,
+  width: number,
+  height: number
+) {
+  if (options.camera) {
+    camera.copy(options.camera);
+  }
+
+  if (camera instanceof THREE.PerspectiveCamera) {
+    camera.aspect = width / height;
+    camera.fov = options.fov ?? camera.fov;
+    camera.near = options.near ?? camera.near;
+    camera.far = options.far ?? camera.far;
+    camera.updateProjectionMatrix();
+  }
+
+  applyCameraPose(camera, options, fallbackCamera);
+}
+
 function readRenderTargetToCanvas(
   renderer: THREE.WebGLRenderer,
   target: THREE.WebGLRenderTarget,
+  canvas: HTMLCanvasElement,
+  context: CanvasRenderingContext2D,
+  pixels: Uint8Array,
+  imageData: ImageData,
   width: number,
   height: number
 ) {
-  const pixels = new Uint8Array(width * height * 4);
   renderer.readRenderTargetPixels(target, 0, 0, width, height, pixels);
 
-  const canvas = document.createElement('canvas');
-  canvas.width = width;
-  canvas.height = height;
-  const context = canvas.getContext('2d');
-  if (!context) {
-    throw new Error('Unable to create a 2D canvas for camera frame capture.');
-  }
-
-  const imageData = context.createImageData(width, height);
   const rowBytes = width * 4;
   for (let y = 0; y < height; y += 1) {
     const sourceStart = (height - y - 1) * rowBytes;
@@ -106,34 +153,156 @@ function readRenderTargetToCanvas(
   return canvas;
 }
 
-export function renderCameraFrameToCanvas(
+function getCameraFrameCaptureSource(
+  options: CameraFrameCaptureOptions
+): CameraFrameCaptureSource {
+  if (options.source) return options.source;
+  if (options.cameraName) {
+    return { kind: 'mujoco-camera', cameraName: options.cameraName };
+  }
+  if (options.siteName) {
+    return { kind: 'mujoco-site', siteName: options.siteName };
+  }
+  if (options.bodyName) {
+    return { kind: 'mujoco-body', bodyName: options.bodyName };
+  }
+  if (options.camera) return { kind: 'custom-camera' };
+  if (options.position || options.lookAt || options.quaternion) {
+    return { kind: 'explicit-pose' };
+  }
+  return { kind: 'fallback-camera' };
+}
+
+export function createCameraFrameCaptureSession(
   renderer: THREE.WebGLRenderer,
   scene: THREE.Scene,
   fallbackCamera: THREE.Camera,
   options: CameraFrameCaptureOptions = {}
-) {
-  const width = Math.max(1, Math.floor(options.width ?? renderer.domElement.width));
-  const height = Math.max(1, Math.floor(options.height ?? renderer.domElement.height));
+): CameraFrameCaptureSession {
+  const { width, height } = getCaptureDimensions(renderer, options);
   const camera = createCaptureCamera(options, fallbackCamera, width, height);
   const target = new THREE.WebGLRenderTarget(width, height, {
     format: THREE.RGBAFormat,
     type: THREE.UnsignedByteType,
   });
-  const previousTarget = renderer.getRenderTarget();
-  const previousXrEnabled = renderer.xr.enabled;
+  const canvas = document.createElement('canvas');
+  canvas.width = width;
+  canvas.height = height;
+  const context = canvas.getContext('2d');
+  if (!context) {
+    target.dispose();
+    throw new Error('Unable to create a 2D canvas for camera frame capture.');
+  }
+  const drawContext = context;
+
+  const pixels = new Uint8Array(width * height * 4);
+  const imageData = drawContext.createImageData(width, height);
+
+  function capture(nextOptions: CameraFrameCaptureOptions = {}) {
+    const captureOptions = { ...options, ...nextOptions };
+    const nextDimensions = getCaptureDimensions(renderer, captureOptions);
+    if (
+      nextDimensions.width !== width ||
+      nextDimensions.height !== height
+    ) {
+      throw new Error(
+        'Camera frame capture sessions require stable width and height.'
+      );
+    }
+
+    prepareCaptureCamera(
+      camera,
+      captureOptions,
+      fallbackCamera,
+      width,
+      height
+    );
+
+    const previousTarget = renderer.getRenderTarget();
+    const previousXrEnabled = renderer.xr.enabled;
 
-  scene.updateMatrixWorld(true);
+    scene.updateMatrixWorld(true);
+    try {
+      renderer.xr.enabled = false;
+      renderer.setRenderTarget(target);
+      renderer.clear();
+      renderer.render(scene, camera);
+      readRenderTargetToCanvas(
+        renderer,
+        target,
+        canvas,
+        drawContext,
+        pixels,
+        imageData,
+        width,
+        height
+      );
+      return {
+        canvas,
+        camera,
+        width,
+        height,
+        source: getCameraFrameCaptureSource(captureOptions),
+      };
+    } finally {
+      renderer.setRenderTarget(previousTarget);
+      renderer.xr.enabled = previousXrEnabled;
+    }
+  }
+
+  return {
+    width,
+    height,
+    capture,
+    captureDataUrl(nextOptions = {}) {
+      const type = nextOptions.type ?? options.type ?? 'image/png';
+      const result = capture(nextOptions);
+      return {
+        ...result,
+        dataUrl: result.canvas.toDataURL(
+          type,
+          nextOptions.quality ?? options.quality
+        ),
+        type,
+      };
+    },
+    async captureBlob(nextOptions = {}) {
+      const type = nextOptions.type ?? options.type ?? 'image/png';
+      const result = capture(nextOptions);
+      const blob = await new Promise<Blob>((resolve, reject) => {
+        result.canvas.toBlob(
+          (nextBlob) => {
+            if (nextBlob) resolve(nextBlob);
+            else reject(new Error('Camera frame capture did not produce a Blob.'));
+          },
+          type,
+          nextOptions.quality ?? options.quality
+        );
+      });
+      return { ...result, blob, type };
+    },
+    dispose() {
+      target.dispose();
+    },
+  };
+}
+
+export function renderCameraFrameToCanvas(
+  renderer: THREE.WebGLRenderer,
+  scene: THREE.Scene,
+  fallbackCamera: THREE.Camera,
+  options: CameraFrameCaptureOptions = {}
+) {
+  const session = createCameraFrameCaptureSession(
+    renderer,
+    scene,
+    fallbackCamera,
+    options
+  );
   try {
-    renderer.xr.enabled = false;
-    renderer.setRenderTarget(target);
-    renderer.clear();
-    renderer.render(scene, camera);
-    const canvas = readRenderTargetToCanvas(renderer, target, width, height);
-    return { canvas, camera, width, height };
+    return session.capture();
   } finally {
-    renderer.setRenderTarget(previousTarget);
-    renderer.xr.enabled = previousXrEnabled;
-    target.dispose();
+    session.dispose();
   }
 }