mujoco-react 9.3.0 → 9.5.0

package/src/components/SplatCollisionProxyPreview.tsx

@@ -0,0 +1,350 @@
+/**
+ * @license
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import type { ThreeElements } from '@react-three/fiber';
+import { useEffect, useMemo, useState } from 'react';
+import type { SplatCollisionProxyConfig } from '../types';
+
+export type SplatCollisionProxyPreviewVector3 = [number, number, number];
+
+export interface SplatCollisionProxyGeomPreview {
+  id: string;
+  type: 'box' | 'plane' | 'sphere' | 'capsule' | 'mesh';
+  position: SplatCollisionProxyPreviewVector3;
+  size: number[];
+}
+
+export interface SplatCollisionProxyPreviewProps
+  extends Omit<ThreeElements['group'], 'ref'> {
+  collisionProxy?: SplatCollisionProxyConfig | null;
+  xmlText?: string;
+  fetchXml?: (xmlPath: string) => Promise<string>;
+  color?: string;
+  opacity?: number;
+  planeColor?: string;
+  planeOpacity?: number;
+}
+
+export type SplatCollisionProxyPreviewStatus =
+  | 'idle'
+  | 'loading'
+  | 'ready'
+  | 'error';
+
+export interface UseSplatCollisionProxyGeomsOptions {
+  collisionProxy?: SplatCollisionProxyConfig | null;
+  xmlText?: string;
+  fetchXml?: (xmlPath: string) => Promise<string>;
+  enabled?: boolean;
+}
+
+export interface SplatCollisionProxyGeomsState {
+  geoms: SplatCollisionProxyGeomPreview[];
+  status: SplatCollisionProxyPreviewStatus;
+  error: Error | null;
+  xmlPath?: string;
+}
+
+export function SplatCollisionProxyPreview({
+  collisionProxy,
+  xmlText,
+  fetchXml = fetchSplatCollisionProxyXml,
+  color = '#60a5fa',
+  opacity = 0.12,
+  planeColor = '#94a3b8',
+  planeOpacity = 0.08,
+  children,
+  ...groupProps
+}: SplatCollisionProxyPreviewProps) {
+  const { geoms } = useSplatCollisionProxyGeoms({
+    collisionProxy,
+    xmlText,
+    fetchXml,
+  });
+
+  if (geoms.length === 0 && !children) return null;
+
+  return (
+    <group
+      {...groupProps}
+      userData={{
+        kind: 'splat-collision-proxy-preview',
+        ...groupProps.userData,
+      }}
+    >
+      {geoms.map((geom) => (
+        <SplatCollisionProxyGeom
+          key={geom.id}
+          geom={geom}
+          color={color}
+          opacity={opacity}
+          planeColor={planeColor}
+          planeOpacity={planeOpacity}
+        />
+      ))}
+      {children}
+    </group>
+  );
+}
+
+export function useSplatCollisionProxyGeoms({
+  collisionProxy,
+  xmlText,
+  fetchXml = fetchSplatCollisionProxyXml,
+  enabled = true,
+}: UseSplatCollisionProxyGeomsOptions): SplatCollisionProxyGeomsState {
+  const [loadedXmlText, setLoadedXmlText] = useState<string | null>(null);
+  const [status, setStatus] =
+    useState<SplatCollisionProxyPreviewStatus>('idle');
+  const [error, setError] = useState<Error | null>(null);
+  const xmlPath = collisionProxy?.xmlPath;
+
+  useEffect(() => {
+    let cancelled = false;
+
+    if (!enabled) {
+      setLoadedXmlText(null);
+      setStatus('idle');
+      setError(null);
+      return undefined;
+    }
+
+    if (xmlText) {
+      setLoadedXmlText(xmlText);
+      setStatus('ready');
+      setError(null);
+      return undefined;
+    }
+
+    if (!xmlPath || !canFetchSplatCollisionProxyXml(xmlPath)) {
+      setLoadedXmlText(null);
+      setStatus('idle');
+      setError(null);
+      return undefined;
+    }
+    const fetchPath = xmlPath;
+
+    async function loadProxyXml() {
+      setStatus('loading');
+      setError(null);
+
+      try {
+        const nextXmlText = await fetchXml(fetchPath);
+        if (!cancelled) {
+          setLoadedXmlText(nextXmlText);
+          setStatus('ready');
+        }
+      } catch (nextError) {
+        if (!cancelled) {
+          setLoadedXmlText(null);
+          setStatus('error');
+          setError(
+            nextError instanceof Error
+              ? nextError
+              : new Error('Unable to load collision proxy XML.')
+          );
+        }
+      }
+    }
+
+    void loadProxyXml();
+
+    return () => {
+      cancelled = true;
+    };
+  }, [enabled, fetchXml, xmlPath, xmlText]);
+
+  const geoms = useMemo(
+    () => (loadedXmlText ? parseSplatCollisionProxyGeoms(loadedXmlText) : []),
+    [loadedXmlText]
+  );
+
+  return useMemo(
+    () => ({
+      geoms,
+      status,
+      error,
+      xmlPath,
+    }),
+    [error, geoms, status, xmlPath]
+  );
+}
+
+export async function fetchSplatCollisionProxyXml(xmlPath: string) {
+  const response = await fetch(xmlPath);
+  if (!response.ok) {
+    throw new Error(`Unable to load collision proxy XML (${response.status}).`);
+  }
+  return response.text();
+}
+
+export function canFetchSplatCollisionProxyXml(xmlPath: string) {
+  return (
+    xmlPath.startsWith('/') ||
+    xmlPath.startsWith('http://') ||
+    xmlPath.startsWith('https://')
+  );
+}
+
+export function parseSplatCollisionProxyGeoms(
+  xmlText: string
+): SplatCollisionProxyGeomPreview[] {
+  const parser = typeof DOMParser === 'undefined' ? null : new DOMParser();
+  if (!parser) return [];
+
+  const document = parser.parseFromString(xmlText, 'application/xml');
+  if (document.querySelector('parsererror')) return [];
+
+  const bodyPositions = new Map<Element, SplatCollisionProxyPreviewVector3>();
+
+  for (const body of Array.from(document.querySelectorAll('body'))) {
+    const parentBody = body.parentElement?.closest('body');
+    const parentPosition: SplatCollisionProxyPreviewVector3 = parentBody
+      ? bodyPositions.get(parentBody) ?? [0, 0, 0]
+      : [0, 0, 0];
+    bodyPositions.set(
+      body,
+      addProxyVectors(parentPosition, parseProxyVector(body.getAttribute('pos')))
+    );
+  }
+
+  return Array.from(document.querySelectorAll('geom'))
+    .map((geom, index): SplatCollisionProxyGeomPreview | null => {
+      const type = getCollisionProxyGeomType(geom);
+      if (!type) return null;
+      const parentBody = geom.closest('body');
+      const bodyPosition: SplatCollisionProxyPreviewVector3 = parentBody
+        ? bodyPositions.get(parentBody) ?? [0, 0, 0]
+        : [0, 0, 0];
+      const position = addProxyVectors(
+        bodyPosition,
+        parseProxyVector(geom.getAttribute('pos'))
+      );
+      const size = parseNumberList(geom.getAttribute('size'));
+      return {
+        id: geom.getAttribute('name') ?? `${type}-${index}`,
+        type,
+        position,
+        size,
+      };
+    })
+    .filter((geom): geom is SplatCollisionProxyGeomPreview => Boolean(geom));
+}
+
+function SplatCollisionProxyGeom({
+  geom,
+  color,
+  opacity,
+  planeColor,
+  planeOpacity,
+}: {
+  geom: SplatCollisionProxyGeomPreview;
+  color: string;
+  opacity: number;
+  planeColor: string;
+  planeOpacity: number;
+}) {
+  if (geom.type === 'sphere') {
+    return (
+      <mesh position={geom.position}>
+        <sphereGeometry args={[geom.size[0] ?? 0.1, 16, 8]} />
+        <SplatCollisionProxyMaterial color={color} opacity={opacity} />
+      </mesh>
+    );
+  }
+
+  if (geom.type === 'plane') {
+    const width = geom.size[0] && geom.size[0] > 0 ? geom.size[0] * 2 : 4;
+    const height = geom.size[1] && geom.size[1] > 0 ? geom.size[1] * 2 : 4;
+    return (
+      <mesh position={geom.position}>
+        <boxGeometry args={[width, height, 0.02]} />
+        <SplatCollisionProxyMaterial color={planeColor} opacity={planeOpacity} />
+      </mesh>
+    );
+  }
+
+  const size = getCollisionProxyBoxSize(geom);
+  return (
+    <mesh position={geom.position}>
+      <boxGeometry args={size} />
+      <SplatCollisionProxyMaterial color={color} opacity={opacity} />
+    </mesh>
+  );
+}
+
+function SplatCollisionProxyMaterial({
+  color,
+  opacity,
+}: {
+  color: string;
+  opacity: number;
+}) {
+  return (
+    <meshBasicMaterial
+      color={color}
+      transparent
+      opacity={opacity}
+      wireframe
+    />
+  );
+}
+
+function getCollisionProxyGeomType(
+  geom: Element
+): SplatCollisionProxyGeomPreview['type'] | null {
+  const type = geom.getAttribute('type') ?? 'sphere';
+  if (
+    type === 'box' ||
+    type === 'plane' ||
+    type === 'sphere' ||
+    type === 'capsule' ||
+    type === 'mesh'
+  ) {
+    return type;
+  }
+  return null;
+}
+
+function getCollisionProxyBoxSize(
+  geom: SplatCollisionProxyGeomPreview
+): SplatCollisionProxyPreviewVector3 {
+  if (geom.type === 'capsule') {
+    const radius = geom.size[0] ?? 0.05;
+    const halfLength = geom.size[1] ?? radius;
+    return [radius * 2, radius * 2, Math.max(radius * 2, halfLength * 2)];
+  }
+
+  if (geom.type === 'mesh') return [0.2, 0.2, 0.2];
+
+  return [
+    (geom.size[0] ?? 0.1) * 2,
+    (geom.size[1] ?? geom.size[0] ?? 0.1) * 2,
+    (geom.size[2] ?? geom.size[0] ?? 0.1) * 2,
+  ];
+}
+
+function parseProxyVector(
+  value: string | null
+): SplatCollisionProxyPreviewVector3 {
+  const values = parseNumberList(value);
+  return [values[0] ?? 0, values[1] ?? 0, values[2] ?? 0];
+}
+
+function parseNumberList(value: string | null) {
+  if (!value) return [];
+  return value
+    .trim()
+    .split(/\s+/)
+    .map((part) => Number(part))
+    .filter((part) => Number.isFinite(part));
+}
+
+function addProxyVectors(
+  a: SplatCollisionProxyPreviewVector3,
+  b: SplatCollisionProxyPreviewVector3
+): SplatCollisionProxyPreviewVector3 {
+  return [a[0] + b[0], a[1] + b[1], a[2] + b[2]];
+}

package/src/components/VisualScenario.tsx

@@ -26,6 +26,8 @@ import type {
   ScenarioLightingProps,
   SplatEnvironmentProps,
   VisualScenarioConfig,
+  VisualScenarioExecutionContext,
+  VisualScenarioExecutionContextInput,
   VisualScenarioEffectsProps,
 } from '../types';
 
@@ -114,6 +116,84 @@ export function getScenarioCameraPosition(
   ];
 }
 
+export function useVisualScenarioExecutionContext({
+  scenario,
+  environment,
+  renderer,
+  variantId,
+  enabled,
+}: VisualScenarioExecutionContextInput): VisualScenarioExecutionContext {
+  return useMemo(
+    () =>
+      createVisualScenarioExecutionContext({
+        scenario,
+        environment,
+        renderer,
+        variantId,
+        enabled,
+      }),
+    [enabled, environment, renderer, scenario, variantId]
+  );
+}
+
+export function createVisualScenarioExecutionContext({
+  scenario,
+  environment,
+  renderer,
+  variantId,
+  enabled = true,
+}: VisualScenarioExecutionContextInput): VisualScenarioExecutionContext {
+  const pairedEnvironment =
+    environment ??
+    (scenario ? createPairedSplatEnvironment(scenario, { renderer }) : undefined);
+  const splat = scenario?.splat;
+  const collisionProxy =
+    pairedEnvironment?.collisionProxy ?? splat?.collisionProxy ?? undefined;
+  const readiness = getSplatEnvironmentReadiness({
+    environment: pairedEnvironment,
+    scenario,
+    renderer,
+    enabled,
+  });
+  const format =
+    pairedEnvironment?.splat.format ?? splat?.format ?? readiness.format ?? 'spz';
+
+  return {
+    scenarioId: scenario?.id ?? pairedEnvironment?.id ?? 'visual-scenario',
+    scenarioLabel:
+      scenario?.label ?? pairedEnvironment?.label ?? 'Visual scenario',
+    variantId,
+    seed: scenario?.seed ?? 0,
+    lighting: scenario?.lighting ?? 'studio',
+    environment: scenario?.environment,
+    camera: {
+      jitter: scenario?.camera?.jitter ?? 0,
+      exposure: scenario?.camera?.exposure ?? 1,
+      noise: scenario?.camera?.noise ?? 0,
+      blur: scenario?.camera?.blur ?? 0,
+    },
+    materials: {
+      randomizeObjectColors: Boolean(
+        scenario?.materials?.randomizeObjectColors
+      ),
+      randomizeTableMaterial: Boolean(
+        scenario?.materials?.randomizeTableMaterial
+      ),
+      roughness: scenario?.materials?.roughness,
+      metalness: scenario?.materials?.metalness,
+    },
+    splatEnabled: Boolean(splat?.enabled || pairedEnvironment),
+    splatSrc: pairedEnvironment?.splat.src ?? splat?.src,
+    splatFormat: format,
+    splatRenderer: renderer ?? pairedEnvironment?.splat.renderer,
+    collisionProxyXmlPath: collisionProxy?.xmlPath,
+    collisionProxyStatus: collisionProxy?.status,
+    collisionProxyPrimitives: collisionProxy?.primitives ?? [],
+    readiness,
+    transformSource: 'visualScenario.camera',
+  };
+}
+
 export function VisualScenarioEffects(props: VisualScenarioEffectsProps) {
   useVisualScenarioEffects(props);
   return null;
@@ -341,43 +421,55 @@ export function useSplatSceneConfig({
   enabled = true,
   renderer,
 }: SplatSceneConfigInput): SplatSceneConfigState {
-  const resolvedEnvironment = useMemo(
-    () =>
-      enabled
-        ? environment ??
-          (scenario
-            ? createPairedSplatEnvironment(scenario, { renderer })
-            : undefined)
-        : undefined,
-    [enabled, environment, renderer, scenario]
-  );
-  const readiness = useMemo(
+  return useMemo(
     () =>
-      getSplatEnvironmentReadiness({
-        environment: resolvedEnvironment,
+      createSplatSceneConfig({
+        sceneConfig,
         scenario,
-        renderer,
+        environment,
         enabled,
+        renderer,
       }),
-    [enabled, renderer, resolvedEnvironment, scenario]
-  );
-  const resolvedSceneConfig = useMemo(
-    () =>
-      resolvedEnvironment
-        ? withSplatEnvironment(sceneConfig, resolvedEnvironment)
-        : sceneConfig,
-    [resolvedEnvironment, sceneConfig]
+    [enabled, environment, renderer, scenario, sceneConfig]
   );
+}
 
-  return useMemo(
-    () => ({
-      environment: resolvedEnvironment,
-      sceneConfig: resolvedSceneConfig,
-      enabled: enabled && readiness.status !== SplatEnvironmentReadinessStatus.Disabled,
-      readiness,
-    }),
-    [enabled, readiness, resolvedEnvironment, resolvedSceneConfig]
-  );
+/**
+ * Resolve a visual scenario's paired splat environment without requiring React.
+ *
+ * Use this in codegen, import validators, backend handoff metadata, or app code
+ * that needs the same behavior as `useSplatSceneConfig` outside a component.
+ */
+export function createSplatSceneConfig({
+  sceneConfig,
+  scenario,
+  environment,
+  enabled = true,
+  renderer,
+}: SplatSceneConfigInput): SplatSceneConfigState {
+  const resolvedEnvironment = enabled
+    ? environment ??
+      (scenario
+        ? createPairedSplatEnvironment(scenario, { renderer })
+        : undefined)
+    : undefined;
+  const readiness = getSplatEnvironmentReadiness({
+    environment: resolvedEnvironment,
+    scenario,
+    renderer,
+    enabled,
+  });
+  const resolvedSceneConfig = resolvedEnvironment
+    ? withSplatEnvironment(sceneConfig, resolvedEnvironment, { renderer })
+    : sceneConfig;
+
+  return {
+    environment: resolvedEnvironment,
+    sceneConfig: resolvedSceneConfig,
+    enabled:
+      enabled && readiness.status !== SplatEnvironmentReadinessStatus.Disabled,
+    readiness,
+  };
 }
 
 export function getSplatEnvironmentReadiness({
@@ -468,9 +560,9 @@ export function getSplatEnvironmentReadiness({
 }
 
 /**
- * Convert a generic visual scenario splat block into a paired visual/physics
- * environment config. Returns undefined until both the splat asset and MJCF
- * collision proxy are present.
+ * Convert a generic visual scenario splat block into a composable splat
+ * environment config. Visual-only splats are valid; readiness reports whether
+ * a paired MJCF collision proxy is required before training/physics handoff.
  */
 export function createPairedSplatEnvironment(
   scenario: Pick<VisualScenarioConfig, 'id' | 'label' | 'environment' | 'splat'>,
@@ -484,7 +576,7 @@ export function createPairedSplatEnvironment(
   const splat = scenario.splat;
   const collisionProxy = splat?.collisionProxy;
 
-  if (!splat?.enabled || !splat.src || !collisionProxy?.xmlPath) {
+  if (!splat?.enabled || !splat.src) {
     return undefined;
   }
 
@@ -501,15 +593,22 @@ export function createPairedSplatEnvironment(
       format: splat.format ?? 'spz',
       renderer: options.renderer,
     },
-    collisionProxy: {
-      ...collisionProxy,
-      xmlPath: collisionProxy.xmlPath,
-    },
+    collisionProxy: collisionProxy?.xmlPath
+      ? {
+          ...collisionProxy,
+          xmlPath: collisionProxy.xmlPath,
+        }
+      : undefined,
   };
 }
 
 function isPairedSplatEnvironment(input: SplatSceneInput): input is PairedSplatEnvironmentConfig {
-  return !!input && 'collisionProxy' in input && 'splat' in input;
+  return (
+    !!input &&
+    'splat' in input &&
+    !!input.splat &&
+    !('enabled' in input.splat)
+  );
 }
 
 function sceneRelativePath(sceneConfig: SceneConfig, path: string): string {
@@ -549,7 +648,7 @@ export function withSplatEnvironment(
     : input
       ? createPairedSplatEnvironment(input, options)
       : undefined;
-  const xmlPath = environment?.collisionProxy.xmlPath;
+  const xmlPath = environment?.collisionProxy?.xmlPath;
   if (!xmlPath) return sceneConfig;
 
   return {

package/src/core/MujocoSimProvider.tsx

@@ -148,12 +148,12 @@ function vector3FromArray(values: ArrayLike<number>, offset: number): [number, n
   return [values[offset], values[offset + 1], values[offset + 2]];
 }
 
-function quaternionFromArray(values: ArrayLike<number>, offset: number): [number, number, number, number] {
+function quaternionFromMujocoQuat(values: ArrayLike<number>, offset: number): [number, number, number, number] {
   return [
-    values[offset],
     values[offset + 1],
     values[offset + 2],
     values[offset + 3],
+    values[offset],
   ];
 }
 
@@ -993,7 +993,7 @@ export function MujocoSimProvider({
           ? vector3FromArray(model.cam_pos, posOffset)
           : null,
         quaternion: model.cam_quat
-          ? quaternionFromArray(model.cam_quat, quatOffset)
+          ? quaternionFromMujocoQuat(model.cam_quat, quatOffset)
           : null,
       });
     }
@@ -1024,7 +1024,7 @@ export function MujocoSimProvider({
         const quaternion = data.cam_xmat
           ? quaternionFromXmat(data.cam_xmat, cameraId * 9)
           : model.cam_quat
-            ? quaternionFromArray(model.cam_quat, cameraId * 4)
+            ? quaternionFromMujocoQuat(model.cam_quat, cameraId * 4)
             : undefined;
 
         if (!position || !quaternion) {
@@ -1442,7 +1442,7 @@ export function MujocoSimProvider({
           const cameraFrames: Record<string, CameraFrameCaptureResult> = {};
           for (const { key, captureOptions, mountedSource, session } of captureSessions) {
             const resolvedCaptureOptions = resolveCameraCaptureOptions(captureOptions);
-            const cameraFrame = session.captureDataUrl({
+            const cameraFrame = await session.captureDataUrlAsync({
               ...resolvedCaptureOptions,
               source: mountedSource ?? resolvedCaptureOptions.source,
             });
@@ -1471,7 +1471,7 @@ export function MujocoSimProvider({
 
           const frame = {
             frameIndex,
-            time: getTime(),
+            time: data.time,
             cameras: cameraFrames,
           };
           if (retainFrames) {