mujoco-react 9.3.0 → 9.4.0

package/src/rendering/cameraFrameSource.ts

@@ -56,6 +56,27 @@ export interface ResolvedMountedCameraFrameSource {
   source: MountedCameraFrameCaptureSource;
 }
 
+export const MountedCameraFrameSourceSuggestionMatch = {
+  Direct: 'direct',
+  Alias: 'alias',
+  Normalized: 'normalized',
+  Prefix: 'prefix',
+  Suffix: 'suffix',
+  Contains: 'contains',
+} as const;
+
+export type MountedCameraFrameSourceSuggestionMatch =
+  (typeof MountedCameraFrameSourceSuggestionMatch)[keyof typeof MountedCameraFrameSourceSuggestionMatch];
+
+export interface MountedCameraFrameSourceSuggestion {
+  key: string;
+  selector: CameraFrameMountSelector;
+  source: MountedCameraFrameCaptureSource;
+  resourceName: string;
+  resourceKind: MountedCameraFrameCaptureSource['kind'];
+  match: MountedCameraFrameSourceSuggestionMatch;
+}
+
 export type MountedCameraFrameSequenceDefaults = Omit<
   CameraFrameSequenceCamera,
   'key' | 'cameraName' | 'siteName' | 'bodyName' | 'source'
@@ -123,6 +144,57 @@ export interface MountedCameraFrameSequenceRecordResult
   readiness: MountedCameraFrameSequenceReadiness;
 }
 
+export const MountedCameraFrameSequenceManifestStatus = {
+  Complete: 'complete',
+  Partial: 'partial',
+  Missing: 'missing',
+} as const;
+
+export type MountedCameraFrameSequenceManifestStatus =
+  (typeof MountedCameraFrameSequenceManifestStatus)[keyof typeof MountedCameraFrameSequenceManifestStatus];
+
+export interface MountedCameraFrameSequenceStreamSummary {
+  key: string;
+  ready: boolean;
+  complete: boolean;
+  status: MountedCameraFrameSequenceManifestStatus;
+  source?: CameraFrameCaptureSource;
+  selector?: CameraFrameMountSelector;
+  target?: string;
+  width?: number;
+  height?: number;
+  expectedFrameCount: number;
+  recordedFrameCount: number;
+  missingFrameCount: number;
+  firstFrameIndex: number | null;
+  lastFrameIndex: number | null;
+  firstTimestamp: number | null;
+  lastTimestamp: number | null;
+  message: string;
+}
+
+export interface CreateMountedCameraFrameSequenceManifestOptions {
+  expectedFrameCount?: number;
+  cameraKeys?: readonly string[];
+}
+
+export interface MountedCameraFrameSequenceManifest {
+  schema: 'mujoco-react/mounted-camera-frame-sequence-manifest@1';
+  ready: boolean;
+  complete: boolean;
+  status: MountedCameraFrameSequenceManifestStatus;
+  cameraKeys: string[];
+  resolvedKeys: string[];
+  missingKeys: string[];
+  expectedFrameCount: number;
+  recordedFrameCount: number;
+  missingFrameCount: number;
+  streamSummaries: Record<string, MountedCameraFrameSequenceStreamSummary>;
+  streams: MountedCameraFrameSequenceStreamSummary[];
+  readiness: MountedCameraFrameSequenceReadiness;
+  message: string;
+}
+
 export type MountedCameraFrameSequenceRecorderTarget = Pick<
   MujocoSimAPI,
   'getCameras' | 'getSites' | 'getBodies' | 'recordCameraSequence'
@@ -147,6 +219,16 @@ function createNameSet(
   );
 }
 
+function createResourceNames(
+  resources:
+    | readonly (string | NamedCameraFrameResource | null | undefined)[]
+    | undefined
+) {
+  return (resources ?? [])
+    .map((resource) => getResourceName(resource))
+    .filter((name): name is string => Boolean(name));
+}
+
 function normalizeAliasCandidates(
   value: CameraFrameMountSelector | readonly CameraFrameMountSelector[] | undefined
 ) {
@@ -160,6 +242,35 @@ function countMountedSelectors(selector: CameraFrameMountSelector) {
     Number(Boolean(selector.bodyName));
 }
 
+function normalizeCameraSourceName(value: string) {
+  return value
+    .trim()
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, '_')
+    .replace(/^_+|_+$/g, '');
+}
+
+function createCameraSourceKeyVariants(key: string) {
+  const candidates = [
+    key,
+    key.startsWith('observation.images.')
+      ? key.slice('observation.images.'.length)
+      : '',
+    key.includes('.') ? key.split('.').at(-1) ?? '' : '',
+    key.includes('/') ? key.split('/').at(-1) ?? '' : '',
+  ];
+  return candidates
+    .map((candidate) => candidate.trim())
+    .filter((candidate, index, items) => candidate && items.indexOf(candidate) === index);
+}
+
+function getSelectorKey(selector: CameraFrameMountSelector) {
+  if (selector.cameraName) return `camera:${selector.cameraName}`;
+  if (selector.siteName) return `site:${selector.siteName}`;
+  if (selector.bodyName) return `body:${selector.bodyName}`;
+  return null;
+}
+
 export function getMountedCameraFrameCaptureSource(
   selector: CameraFrameMountSelector
 ): MountedCameraFrameCaptureSource | null {
@@ -197,6 +308,66 @@ export function getCameraFrameCaptureSourceTarget(
   return 'fallback camera';
 }
 
+function createMountedCameraFrameSourceSuggestion(
+  key: string,
+  selector: CameraFrameMountSelector,
+  resourceName: string,
+  match: MountedCameraFrameSourceSuggestionMatch
+): MountedCameraFrameSourceSuggestion | null {
+  const source = getMountedCameraFrameCaptureSource(selector);
+  if (!source) return null;
+  return {
+    key,
+    selector,
+    source,
+    resourceName,
+    resourceKind: source.kind,
+    match,
+  };
+}
+
+function addMountedCameraFrameSourceSuggestion(
+  suggestions: MountedCameraFrameSourceSuggestion[],
+  seen: Set<string>,
+  suggestion: MountedCameraFrameSourceSuggestion | null
+) {
+  if (!suggestion) return;
+  const selectorKey = getSelectorKey(suggestion.selector);
+  if (!selectorKey || seen.has(selectorKey)) return;
+  seen.add(selectorKey);
+  suggestions.push(suggestion);
+}
+
+function getCameraFrameResourceMatch(
+  key: string,
+  resourceName: string
+): MountedCameraFrameSourceSuggestionMatch | null {
+  if (resourceName === key) return MountedCameraFrameSourceSuggestionMatch.Direct;
+
+  const normalizedResource = normalizeCameraSourceName(resourceName);
+  if (!normalizedResource) return null;
+
+  for (const variant of createCameraSourceKeyVariants(key)) {
+    if (resourceName === variant) return MountedCameraFrameSourceSuggestionMatch.Direct;
+
+    const normalizedKey = normalizeCameraSourceName(variant);
+    if (!normalizedKey) continue;
+    if (normalizedResource === normalizedKey) {
+      return MountedCameraFrameSourceSuggestionMatch.Normalized;
+    }
+    if (normalizedResource.startsWith(`${normalizedKey}_`)) {
+      return MountedCameraFrameSourceSuggestionMatch.Prefix;
+    }
+    if (normalizedResource.endsWith(`_${normalizedKey}`)) {
+      return MountedCameraFrameSourceSuggestionMatch.Suffix;
+    }
+    if (normalizedResource.includes(`_${normalizedKey}_`)) {
+      return MountedCameraFrameSourceSuggestionMatch.Contains;
+    }
+  }
+  return null;
+}
+
 function isSelectorMounted(
   selector: CameraFrameMountSelector,
   cameraNames: Set<string>,
@@ -211,6 +382,82 @@ function isSelectorMounted(
   );
 }
 
+export function createMountedCameraFrameSourceSuggestions(
+  key: string,
+  options: ResolveMountedCameraFrameSourceOptions
+): MountedCameraFrameSourceSuggestion[] {
+  const cameraNames = createNameSet(options.cameras);
+  const siteNames = createNameSet(options.sites);
+  const bodyNames = createNameSet(options.bodies);
+  const suggestions: MountedCameraFrameSourceSuggestion[] = [];
+  const seen = new Set<string>();
+
+  for (const selector of normalizeAliasCandidates(options.aliases?.[key])) {
+    if (!isSelectorMounted(selector, cameraNames, siteNames, bodyNames)) {
+      continue;
+    }
+    const source = getMountedCameraFrameCaptureSource(selector);
+    if (!source) continue;
+    addMountedCameraFrameSourceSuggestion(
+      suggestions,
+      seen,
+      createMountedCameraFrameSourceSuggestion(
+        key,
+        selector,
+        getCameraFrameCaptureSourceTarget(source),
+        MountedCameraFrameSourceSuggestionMatch.Alias
+      )
+    );
+  }
+
+  for (const cameraName of createResourceNames(options.cameras)) {
+    const match = getCameraFrameResourceMatch(key, cameraName);
+    if (!match) continue;
+    addMountedCameraFrameSourceSuggestion(
+      suggestions,
+      seen,
+      createMountedCameraFrameSourceSuggestion(
+        key,
+        { cameraName },
+        cameraName,
+        match
+      )
+    );
+  }
+
+  for (const siteName of createResourceNames(options.sites)) {
+    const match = getCameraFrameResourceMatch(key, siteName);
+    if (!match) continue;
+    addMountedCameraFrameSourceSuggestion(
+      suggestions,
+      seen,
+      createMountedCameraFrameSourceSuggestion(
+        key,
+        { siteName },
+        siteName,
+        match
+      )
+    );
+  }
+
+  for (const bodyName of createResourceNames(options.bodies)) {
+    const match = getCameraFrameResourceMatch(key, bodyName);
+    if (!match) continue;
+    addMountedCameraFrameSourceSuggestion(
+      suggestions,
+      seen,
+      createMountedCameraFrameSourceSuggestion(
+        key,
+        { bodyName },
+        bodyName,
+        match
+      )
+    );
+  }
+
+  return suggestions;
+}
+
 export function resolveMountedCameraFrameSource(
   key: string,
   options: ResolveMountedCameraFrameSourceOptions
@@ -235,6 +482,15 @@ export function resolveMountedCameraFrameSource(
     return { key, selector, source };
   }
 
+  const [suggestion] = createMountedCameraFrameSourceSuggestions(key, options);
+  if (suggestion) {
+    return {
+      key,
+      selector: suggestion.selector,
+      source: suggestion.source,
+    };
+  }
+
   if (options.allowAliasFallback) {
     for (const selector of aliasCandidates) {
       const source = getMountedCameraFrameCaptureSource(selector);
@@ -335,6 +591,122 @@ export function createMountedCameraFrameSequenceReadiness(
   };
 }
 
+function normalizeFrameCount(frameCount: number | undefined) {
+  return Number.isFinite(frameCount) && frameCount !== undefined
+    ? Math.max(0, Math.floor(frameCount))
+    : 0;
+}
+
+export function createMountedCameraFrameSequenceManifest(
+  result: MountedCameraFrameSequenceRecordResult,
+  options: CreateMountedCameraFrameSequenceManifestOptions = {}
+): MountedCameraFrameSequenceManifest {
+  const cameraKeys = [
+    ...(options.cameraKeys ??
+      result.readiness.cameraKeys ??
+      result.plan.cameraKeys ??
+      result.cameraKeys),
+  ];
+  const expectedFrameCount = normalizeFrameCount(
+    options.expectedFrameCount ?? result.frameCount
+  );
+  const recordedFrameCount = normalizeFrameCount(result.frameCount);
+  const streamSummaries: Record<string, MountedCameraFrameSequenceStreamSummary> =
+    {};
+  const streams: MountedCameraFrameSequenceStreamSummary[] = [];
+  let missingFrameCount = 0;
+  let completeStreamCount = 0;
+  let resolvedOrRecordedStreamCount = 0;
+
+  for (const key of cameraKeys) {
+    const summary = result.cameraSummaries[key];
+    const readiness = result.readiness.cameras[key];
+    const source = summary?.source ?? readiness?.source;
+    const ready = readiness?.ready ?? Boolean(summary);
+    const recorded = normalizeFrameCount(summary?.frameCount);
+    const missing = Math.max(expectedFrameCount - recorded, 0);
+    const complete = ready && missing === 0;
+    const status = complete
+      ? MountedCameraFrameSequenceManifestStatus.Complete
+      : ready || recorded > 0
+        ? MountedCameraFrameSequenceManifestStatus.Partial
+        : MountedCameraFrameSequenceManifestStatus.Missing;
+    const target = source
+      ? getCameraFrameCaptureSourceTarget(source)
+      : readiness?.message
+        ? undefined
+        : 'missing MuJoCo camera';
+    const message = complete
+      ? `Camera stream "${key}" recorded ${recorded} of ${expectedFrameCount} frame${
+          expectedFrameCount === 1 ? '' : 's'
+        }.`
+      : ready || recorded > 0
+        ? `Camera stream "${key}" recorded ${recorded} of ${expectedFrameCount} frame${
+            expectedFrameCount === 1 ? '' : 's'
+          }.`
+        : readiness?.message ??
+          `Camera stream "${key}" did not record any frames.`;
+    const stream = {
+      key,
+      ready,
+      complete,
+      status,
+      source,
+      selector: readiness?.selector,
+      target,
+      width: summary?.width,
+      height: summary?.height,
+      expectedFrameCount,
+      recordedFrameCount: recorded,
+      missingFrameCount: missing,
+      firstFrameIndex: summary?.firstFrameIndex ?? null,
+      lastFrameIndex: summary?.lastFrameIndex ?? null,
+      firstTimestamp: summary?.firstTimestamp ?? null,
+      lastTimestamp: summary?.lastTimestamp ?? null,
+      message,
+    };
+
+    streamSummaries[key] = stream;
+    streams.push(stream);
+    missingFrameCount += missing;
+    if (complete) completeStreamCount += 1;
+    if (ready || recorded > 0) resolvedOrRecordedStreamCount += 1;
+  }
+
+  const complete =
+    result.readiness.ready &&
+    streams.length === completeStreamCount &&
+    missingFrameCount === 0;
+  const status = complete
+    ? MountedCameraFrameSequenceManifestStatus.Complete
+    : resolvedOrRecordedStreamCount > 0
+      ? MountedCameraFrameSequenceManifestStatus.Partial
+      : MountedCameraFrameSequenceManifestStatus.Missing;
+
+  return {
+    schema: 'mujoco-react/mounted-camera-frame-sequence-manifest@1',
+    ready: result.readiness.ready,
+    complete,
+    status,
+    cameraKeys,
+    resolvedKeys: [...result.readiness.resolvedKeys],
+    missingKeys: [...result.readiness.missingKeys],
+    expectedFrameCount,
+    recordedFrameCount,
+    missingFrameCount,
+    streamSummaries,
+    streams,
+    readiness: result.readiness,
+    message: complete
+      ? `All ${cameraKeys.length} camera stream${
+          cameraKeys.length === 1 ? '' : 's'
+        } recorded ${expectedFrameCount} frame${
+          expectedFrameCount === 1 ? '' : 's'
+        }.`
+      : `Mounted camera sequence coverage is ${status}.`,
+  };
+}
+
 export function createMountedCameraFrameSequencePlanFromApi(
   api: MountedCameraFrameSequenceRecorderTarget,
   cameraKeys: readonly string[],

package/src/types.ts

@@ -928,8 +928,8 @@ export interface PairedSplatEnvironmentConfig {
   description?: string;
   /** Visual-only Gaussian splat asset. */
   splat: SplatAssetConfig;
-  /** MJCF/XML contact geometry paired with the visual splat. */
-  collisionProxy: SplatCollisionProxyConfig & { xmlPath: string };
+  /** Optional MJCF/XML contact geometry paired with the visual splat. */
+  collisionProxy?: SplatCollisionProxyConfig & { xmlPath: string };
 }
 
 export const SplatEnvironmentReadinessStatus = {
@@ -970,6 +970,48 @@ export interface SplatEnvironmentMetadata {
   userData: Record<string, unknown>;
 }
 
+export interface ResolvedScenarioCameraConfig {
+  jitter: number;
+  exposure: number;
+  noise: number;
+  blur: number;
+}
+
+export interface ResolvedScenarioMaterialConfig {
+  randomizeObjectColors: boolean;
+  randomizeTableMaterial: boolean;
+  roughness?: number;
+  metalness?: number;
+}
+
+export interface VisualScenarioExecutionContext {
+  scenarioId: string;
+  scenarioLabel: string;
+  variantId?: string;
+  seed: number;
+  lighting: ScenarioLightingPreset;
+  environment?: string;
+  camera: ResolvedScenarioCameraConfig;
+  materials: ResolvedScenarioMaterialConfig;
+  splatEnabled: boolean;
+  splatSrc?: string;
+  splatFormat: SplatFormat;
+  splatRenderer?: SplatRendererKind;
+  collisionProxyXmlPath?: string;
+  collisionProxyStatus?: SplatCollisionProxyConfig['status'];
+  collisionProxyPrimitives: SplatCollisionPrimitive[];
+  readiness: SplatEnvironmentReadiness;
+  transformSource: 'visualScenario.camera';
+}
+
+export interface VisualScenarioExecutionContextInput {
+  scenario?: VisualScenarioConfig;
+  environment?: PairedSplatEnvironmentConfig;
+  renderer?: SplatRendererKind;
+  variantId?: string;
+  enabled?: boolean;
+}
+
 export type SplatSceneInput =
   | PairedSplatEnvironmentConfig
   | VisualScenarioConfig

package/src/vite.ts

@@ -158,7 +158,7 @@ async function scanModel(
   seen: Set<string>,
   names: Record<RegisterKey, Set<string>>
 ) {
-  const normalized = path.normalize(filePath);
+  const normalized = path.resolve(filePath);
   if (seen.has(normalized)) return;
   seen.add(normalized);
 
@@ -174,7 +174,7 @@ async function scanModel(
 
   for (const includePath of collectIncludePaths(xml)) {
     const next = path.resolve(path.dirname(normalized), includePath);
-    if (next.startsWith(root)) await scanModel(next, root, seen, names);
+    if (isPathInsideRoot(next, root)) await scanModel(next, root, seen, names);
   }
 }
 
@@ -346,7 +346,7 @@ function shouldInjectRegisterImport(id: string, root: string, generatedRegister:
   if (file.includes(`${path.sep}node_modules${path.sep}`)) return false;
   const absolute = path.resolve(file);
   if (absolute === generatedRegister) return false;
-  return absolute.startsWith(root);
+  return isPathInsideRoot(absolute, root);
 }
 
 function renderGeneratedImport(id: string, generatedRegister: string): string {
@@ -395,5 +395,10 @@ function shouldRegenerate(file: string, watchedFiles: string[], models: readonly
   if (watchedFiles.includes(absolute)) return true;
   if (!MODEL_EXTENSIONS.has(path.extname(absolute).toLowerCase())) return false;
   const modelDirs = models.map((model) => path.dirname(path.resolve(root, model.file)));
-  return modelDirs.some((dir) => absolute.startsWith(dir));
+  return modelDirs.some((dir) => isPathInsideRoot(absolute, dir));
+}
+
+function isPathInsideRoot(filePath: string, root: string): boolean {
+  const relative = path.relative(path.resolve(root), path.resolve(filePath));
+  return relative === '' || (relative !== '' && !relative.startsWith('..') && !path.isAbsolute(relative));
 }