mujoco-react 10.2.0 → 10.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -60,6 +60,7 @@ export type CameraFrameCaptureRenderResult = {
60
60
  width?: number;
61
61
  height?: number;
62
62
  flipY?: boolean;
63
+ flipX?: boolean;
63
64
  };
64
65
 
65
66
  type CameraFrameCaptureRender = (
@@ -79,6 +80,15 @@ type RendererState = {
79
80
  clearColor: THREE.Color;
80
81
  clearAlpha: number;
81
82
  autoClear: boolean;
83
+ shadowMapEnabled: boolean;
84
+ toneMapping: THREE.WebGLRenderer['toneMapping'];
85
+ outputColorSpace: THREE.WebGLRenderer['outputColorSpace'];
86
+ };
87
+
88
+ type SceneVisualState = {
89
+ background: THREE.Scene['background'];
90
+ environment: THREE.Scene['environment'];
91
+ fog: THREE.Scene['fog'];
82
92
  };
83
93
 
84
94
  type VisibilityState = {
@@ -88,6 +98,107 @@ type VisibilityState = {
88
98
 
89
99
  type CameraFrameCapturePreRender = () => void;
90
100
 
101
+ const isolatedRendererCache = new WeakMap<
102
+ THREE.WebGLRenderer,
103
+ Map<string, THREE.WebGLRenderer>
104
+ >();
105
+
106
+ function shouldUseRenderIsolation(
107
+ options: CameraFrameCaptureOptions
108
+ ): boolean {
109
+ return options.renderIsolation === true || (
110
+ typeof options.renderIsolation === 'object' &&
111
+ options.renderIsolation.enabled !== false
112
+ );
113
+ }
114
+
115
+ function getRenderIsolationOptions(
116
+ options: CameraFrameCaptureOptions
117
+ ) {
118
+ return typeof options.renderIsolation === 'object'
119
+ ? options.renderIsolation
120
+ : {};
121
+ }
122
+
123
+ function getRenderIsolationCacheKey(
124
+ width: number,
125
+ height: number,
126
+ options: CameraFrameCaptureOptions
127
+ ) {
128
+ const isolation = getRenderIsolationOptions(options);
129
+ return JSON.stringify({
130
+ width,
131
+ height,
132
+ antialias: isolation.antialias ?? false,
133
+ alpha: isolation.alpha ?? false,
134
+ preserveDrawingBuffer: isolation.preserveDrawingBuffer ?? false,
135
+ powerPreference: isolation.powerPreference ?? null,
136
+ });
137
+ }
138
+
139
+ function createIsolatedRenderer(
140
+ sourceRenderer: THREE.WebGLRenderer,
141
+ width: number,
142
+ height: number,
143
+ options: CameraFrameCaptureOptions
144
+ ): { renderer: THREE.WebGLRenderer; cached: boolean } | null {
145
+ if (!shouldUseRenderIsolation(options)) return null;
146
+
147
+ const isolation = getRenderIsolationOptions(options);
148
+ if (isolation.cache !== false) {
149
+ const cacheKey = getRenderIsolationCacheKey(width, height, options);
150
+ let rendererCache = isolatedRendererCache.get(sourceRenderer);
151
+ if (!rendererCache) {
152
+ rendererCache = new Map();
153
+ isolatedRendererCache.set(sourceRenderer, rendererCache);
154
+ }
155
+ const cachedRenderer = rendererCache.get(cacheKey);
156
+ if (cachedRenderer) {
157
+ cachedRenderer.outputColorSpace = sourceRenderer.outputColorSpace;
158
+ cachedRenderer.toneMapping = sourceRenderer.toneMapping;
159
+ cachedRenderer.shadowMap.enabled = false;
160
+ return { renderer: cachedRenderer, cached: true };
161
+ }
162
+ const createdRenderer = createUncachedIsolatedRenderer(
163
+ sourceRenderer,
164
+ width,
165
+ height,
166
+ options
167
+ );
168
+ rendererCache.set(cacheKey, createdRenderer);
169
+ return { renderer: createdRenderer, cached: true };
170
+ }
171
+
172
+ return {
173
+ renderer: createUncachedIsolatedRenderer(sourceRenderer, width, height, options),
174
+ cached: false,
175
+ };
176
+ }
177
+
178
+ function createUncachedIsolatedRenderer(
179
+ sourceRenderer: THREE.WebGLRenderer,
180
+ width: number,
181
+ height: number,
182
+ options: CameraFrameCaptureOptions
183
+ ) {
184
+ const isolation = getRenderIsolationOptions(options);
185
+ const canvas = document.createElement('canvas');
186
+ const renderer = new THREE.WebGLRenderer({
187
+ canvas,
188
+ antialias: isolation.antialias ?? false,
189
+ alpha: isolation.alpha ?? false,
190
+ preserveDrawingBuffer: isolation.preserveDrawingBuffer ?? false,
191
+ powerPreference: isolation.powerPreference,
192
+ });
193
+
194
+ renderer.setPixelRatio(1);
195
+ renderer.setSize(width, height, false);
196
+ renderer.outputColorSpace = sourceRenderer.outputColorSpace;
197
+ renderer.toneMapping = sourceRenderer.toneMapping;
198
+ renderer.shadowMap.enabled = false;
199
+ return renderer;
200
+ }
201
+
91
202
  function toVector3(
92
203
  value: CameraFrameCaptureVector3 | undefined,
93
204
  fallback: THREE.Vector3
@@ -126,6 +237,19 @@ function applyCameraPose(
126
237
  camera.updateMatrixWorld();
127
238
  }
128
239
 
240
+ function applyProjectionMatrix(
241
+ camera: THREE.Camera,
242
+ projectionMatrix: CameraFrameCaptureOptions['projectionMatrix'] | undefined
243
+ ) {
244
+ if (!projectionMatrix) return;
245
+ if (projectionMatrix instanceof THREE.Matrix4) {
246
+ camera.projectionMatrix.copy(projectionMatrix);
247
+ } else {
248
+ camera.projectionMatrix.fromArray(projectionMatrix);
249
+ }
250
+ camera.projectionMatrixInverse.copy(camera.projectionMatrix).invert();
251
+ }
252
+
129
253
  function createCaptureCamera(
130
254
  options: CameraFrameCaptureOptions,
131
255
  fallbackCamera: THREE.Camera,
@@ -145,6 +269,7 @@ function createCaptureCamera(
145
269
  camera.far = options.far ?? camera.far;
146
270
  camera.updateProjectionMatrix();
147
271
  }
272
+ applyProjectionMatrix(camera, options.projectionMatrix);
148
273
 
149
274
  applyCameraPose(camera, options, fallbackCamera);
150
275
  return camera;
@@ -183,6 +308,7 @@ function prepareCaptureCamera(
183
308
  camera.far = options.far ?? camera.far;
184
309
  camera.updateProjectionMatrix();
185
310
  }
311
+ applyProjectionMatrix(camera, options.projectionMatrix);
186
312
 
187
313
  applyCameraPose(camera, options, fallbackCamera);
188
314
  }
@@ -196,7 +322,8 @@ function readRenderTargetToCanvas(
196
322
  imageData: ImageData,
197
323
  width: number,
198
324
  height: number,
199
- outputColorSpace: string
325
+ outputColorSpace: string,
326
+ flipX = false
200
327
  ) {
201
328
  renderer.readRenderTargetPixels(target, 0, 0, width, height, pixels);
202
329
 
@@ -206,17 +333,25 @@ function readRenderTargetToCanvas(
206
333
  const sourceStart = (height - y - 1) * rowBytes;
207
334
  const targetStart = y * rowBytes;
208
335
  const row = pixels.subarray(sourceStart, sourceStart + rowBytes);
209
- if (!encodeSrgb) {
336
+ if (!encodeSrgb && !flipX) {
210
337
  imageData.data.set(row, targetStart);
211
338
  continue;
212
339
  }
213
340
 
214
- for (let x = 0; x < rowBytes; x += 4) {
215
- const pixelOffset = targetStart + x;
216
- imageData.data[pixelOffset] = linearByteToSrgbByte(row[x]);
217
- imageData.data[pixelOffset + 1] = linearByteToSrgbByte(row[x + 1]);
218
- imageData.data[pixelOffset + 2] = linearByteToSrgbByte(row[x + 2]);
219
- imageData.data[pixelOffset + 3] = row[x + 3];
341
+ for (let x = 0; x < width; x += 1) {
342
+ const sourceX = flipX ? width - x - 1 : x;
343
+ const sourceOffset = sourceX * 4;
344
+ const targetOffset = targetStart + x * 4;
345
+ imageData.data[targetOffset] = encodeSrgb
346
+ ? linearByteToSrgbByte(row[sourceOffset])
347
+ : row[sourceOffset];
348
+ imageData.data[targetOffset + 1] = encodeSrgb
349
+ ? linearByteToSrgbByte(row[sourceOffset + 1])
350
+ : row[sourceOffset + 1];
351
+ imageData.data[targetOffset + 2] = encodeSrgb
352
+ ? linearByteToSrgbByte(row[sourceOffset + 2])
353
+ : row[sourceOffset + 2];
354
+ imageData.data[targetOffset + 3] = row[sourceOffset + 3];
220
355
  }
221
356
  }
222
357
  context.putImageData(imageData, 0, 0);
@@ -238,17 +373,30 @@ function readPixelsToCanvas(
238
373
  imageData: ImageData,
239
374
  width: number,
240
375
  height: number,
241
- flipY = true
376
+ flipY = true,
377
+ flipX = false
242
378
  ) {
243
379
  const rowBytes = width * 4;
244
380
  for (let y = 0; y < height; y += 1) {
245
381
  const sourceY = flipY ? height - y - 1 : y;
246
382
  const sourceStart = sourceY * rowBytes;
247
383
  const targetStart = y * rowBytes;
248
- imageData.data.set(
249
- pixels.subarray(sourceStart, sourceStart + rowBytes),
250
- targetStart
251
- );
384
+ if (!flipX) {
385
+ imageData.data.set(
386
+ pixels.subarray(sourceStart, sourceStart + rowBytes),
387
+ targetStart
388
+ );
389
+ continue;
390
+ }
391
+ for (let x = 0; x < width; x += 1) {
392
+ const sourceX = width - x - 1;
393
+ const sourceOffset = sourceStart + sourceX * 4;
394
+ const targetOffset = targetStart + x * 4;
395
+ imageData.data[targetOffset] = pixels[sourceOffset];
396
+ imageData.data[targetOffset + 1] = pixels[sourceOffset + 1];
397
+ imageData.data[targetOffset + 2] = pixels[sourceOffset + 2];
398
+ imageData.data[targetOffset + 3] = pixels[sourceOffset + 3];
399
+ }
252
400
  }
253
401
  context.putImageData(imageData, 0, 0);
254
402
  }
@@ -339,6 +487,9 @@ function saveRendererState(renderer: THREE.WebGLRenderer): RendererState {
339
487
  clearColor,
340
488
  clearAlpha: renderer.getClearAlpha(),
341
489
  autoClear: renderer.autoClear,
490
+ shadowMapEnabled: renderer.shadowMap.enabled,
491
+ toneMapping: renderer.toneMapping,
492
+ outputColorSpace: renderer.outputColorSpace,
342
493
  };
343
494
  }
344
495
 
@@ -353,6 +504,64 @@ function restoreRendererState(
353
504
  renderer.setScissorTest(state.scissorTest);
354
505
  renderer.setClearColor(state.clearColor, state.clearAlpha);
355
506
  renderer.autoClear = state.autoClear;
507
+ renderer.shadowMap.enabled = state.shadowMapEnabled;
508
+ renderer.toneMapping = state.toneMapping;
509
+ renderer.outputColorSpace = state.outputColorSpace;
510
+ }
511
+
512
+ function saveSceneVisualState(scene: THREE.Scene): SceneVisualState {
513
+ return {
514
+ background: scene.background,
515
+ environment: scene.environment,
516
+ fog: scene.fog,
517
+ };
518
+ }
519
+
520
+ function restoreSceneVisualState(scene: THREE.Scene, state: SceneVisualState) {
521
+ scene.background = state.background;
522
+ scene.environment = state.environment;
523
+ scene.fog = state.fog;
524
+ }
525
+
526
+ function hasOwn<T extends object>(object: T, key: keyof T) {
527
+ return Object.prototype.hasOwnProperty.call(object, key);
528
+ }
529
+
530
+ function applyCaptureVisualOverrides(
531
+ renderer: THREE.WebGLRenderer,
532
+ scene: THREE.Scene,
533
+ options: CameraFrameCaptureOptions
534
+ ): SceneVisualState | null {
535
+ const overrides = options.visualOverrides;
536
+ if (!overrides) return null;
537
+
538
+ const previousSceneState = saveSceneVisualState(scene);
539
+ if (hasOwn(overrides, 'sceneBackground')) {
540
+ const background = overrides.sceneBackground;
541
+ scene.background = background === false ? null : (
542
+ typeof background === 'string' || typeof background === 'number'
543
+ ? new THREE.Color(background)
544
+ : background ?? null
545
+ );
546
+ }
547
+ if (hasOwn(overrides, 'sceneEnvironment')) {
548
+ const environment = overrides.sceneEnvironment;
549
+ scene.environment = environment === false ? null : environment ?? null;
550
+ }
551
+ if (hasOwn(overrides, 'sceneFog')) {
552
+ const fog = overrides.sceneFog;
553
+ scene.fog = fog === false ? null : fog ?? null;
554
+ }
555
+ if (overrides.shadows !== undefined) {
556
+ renderer.shadowMap.enabled = overrides.shadows;
557
+ }
558
+ if (overrides.toneMapping !== undefined) {
559
+ renderer.toneMapping = overrides.toneMapping;
560
+ }
561
+ if (overrides.outputColorSpace !== undefined) {
562
+ renderer.outputColorSpace = overrides.outputColorSpace;
563
+ }
564
+ return previousSceneState;
356
565
  }
357
566
 
358
567
  function getCaptureRenderer(
@@ -387,6 +596,8 @@ export function createCameraFrameCaptureSession(
387
596
  options: CameraFrameCaptureOptions = {}
388
597
  ): CameraFrameCaptureSession {
389
598
  const { width, height } = getCaptureDimensions(renderer, options);
599
+ const isolatedRenderer = createIsolatedRenderer(renderer, width, height, options);
600
+ const sessionRenderer = isolatedRenderer?.renderer ?? renderer;
390
601
  const camera = createCaptureCamera(options, fallbackCamera, width, height);
391
602
  const target = new THREE.WebGLRenderTarget(width, height, {
392
603
  format: THREE.RGBAFormat,
@@ -407,6 +618,13 @@ export function createCameraFrameCaptureSession(
407
618
 
408
619
  function resolveCaptureOptions(nextOptions: CameraFrameCaptureOptions = {}) {
409
620
  const captureOptions = { ...options, ...nextOptions };
621
+ if (
622
+ shouldUseRenderIsolation(captureOptions) !== shouldUseRenderIsolation(options)
623
+ ) {
624
+ throw new Error(
625
+ 'Camera frame capture sessions require stable renderIsolation settings.'
626
+ );
627
+ }
410
628
  const nextDimensions = getCaptureDimensions(renderer, captureOptions);
411
629
  if (
412
630
  nextDimensions.width !== width ||
@@ -429,7 +647,12 @@ export function createCameraFrameCaptureSession(
429
647
  }
430
648
 
431
649
  function renderPreparedCapture(captureOptions: CameraFrameCaptureOptions) {
432
- const previousState = saveRendererState(renderer);
650
+ const previousState = saveRendererState(sessionRenderer);
651
+ const previousSceneState = applyCaptureVisualOverrides(
652
+ sessionRenderer,
653
+ scene,
654
+ captureOptions
655
+ );
433
656
  const hidden = [
434
657
  ...hideExcludedCaptureObjects(scene),
435
658
  ...hideCaptureGeomGroups(scene, captureOptions),
@@ -438,23 +661,23 @@ export function createCameraFrameCaptureSession(
438
661
  runCapturePreRenderHooks(scene);
439
662
  scene.updateMatrixWorld(true);
440
663
  try {
441
- renderer.xr.enabled = false;
442
- renderer.setRenderTarget(target);
443
- renderer.setViewport(0, 0, width, height);
444
- renderer.setScissor(0, 0, width, height);
445
- renderer.setScissorTest(false);
664
+ sessionRenderer.xr.enabled = false;
665
+ sessionRenderer.setRenderTarget(target);
666
+ sessionRenderer.setViewport(0, 0, width, height);
667
+ sessionRenderer.setScissor(0, 0, width, height);
668
+ sessionRenderer.setScissorTest(false);
446
669
  if (captureOptions.background !== undefined) {
447
- renderer.setClearColor(
670
+ sessionRenderer.setClearColor(
448
671
  new THREE.Color(captureOptions.background),
449
672
  captureOptions.backgroundAlpha ?? previousState.clearAlpha
450
673
  );
451
674
  } else if (captureOptions.backgroundAlpha !== undefined) {
452
- renderer.setClearColor(previousState.clearColor, captureOptions.backgroundAlpha);
675
+ sessionRenderer.setClearColor(previousState.clearColor, captureOptions.backgroundAlpha);
453
676
  }
454
- renderer.clear();
455
- renderer.render(scene, camera);
677
+ sessionRenderer.clear();
678
+ sessionRenderer.render(scene, camera);
456
679
  readRenderTargetToCanvas(
457
- renderer,
680
+ sessionRenderer,
458
681
  target,
459
682
  canvas,
460
683
  drawContext,
@@ -462,7 +685,8 @@ export function createCameraFrameCaptureSession(
462
685
  imageData,
463
686
  width,
464
687
  height,
465
- renderer.outputColorSpace
688
+ sessionRenderer.outputColorSpace,
689
+ captureOptions.flipX ?? false
466
690
  );
467
691
  return {
468
692
  canvas,
@@ -473,7 +697,8 @@ export function createCameraFrameCaptureSession(
473
697
  };
474
698
  } finally {
475
699
  restoreObjectVisibility(hidden);
476
- restoreRendererState(renderer, previousState);
700
+ if (previousSceneState) restoreSceneVisualState(scene, previousSceneState);
701
+ restoreRendererState(sessionRenderer, previousState);
477
702
  }
478
703
  }
479
704
 
@@ -487,23 +712,28 @@ export function createCameraFrameCaptureSession(
487
712
  scene.updateMatrixWorld(true);
488
713
  const captureRenderer = getCaptureRenderer(scene);
489
714
  if (captureRenderer) {
490
- const previousState = saveRendererState(renderer);
715
+ const previousState = saveRendererState(sessionRenderer);
716
+ const previousSceneState = applyCaptureVisualOverrides(
717
+ sessionRenderer,
718
+ scene,
719
+ captureOptions
720
+ );
491
721
  const hidden = [
492
722
  ...hideExcludedCaptureObjects(scene),
493
723
  ...hideCaptureGeomGroups(scene, captureOptions),
494
724
  ];
495
725
  try {
496
- renderer.xr.enabled = false;
726
+ sessionRenderer.xr.enabled = false;
497
727
  if (captureOptions.background !== undefined) {
498
- renderer.setClearColor(
728
+ sessionRenderer.setClearColor(
499
729
  new THREE.Color(captureOptions.background),
500
730
  captureOptions.backgroundAlpha ?? previousState.clearAlpha
501
731
  );
502
732
  } else if (captureOptions.backgroundAlpha !== undefined) {
503
- renderer.setClearColor(previousState.clearColor, captureOptions.backgroundAlpha);
733
+ sessionRenderer.setClearColor(previousState.clearColor, captureOptions.backgroundAlpha);
504
734
  }
505
735
  const captureResult = await captureRenderer({
506
- renderer,
736
+ renderer: sessionRenderer,
507
737
  scene,
508
738
  camera,
509
739
  target,
@@ -524,7 +754,8 @@ export function createCameraFrameCaptureSession(
524
754
  imageData,
525
755
  width,
526
756
  height,
527
- captureResult.flipY ?? true
757
+ captureResult.flipY ?? true,
758
+ captureResult.flipX ?? captureOptions.flipX ?? false
528
759
  );
529
760
  return {
530
761
  canvas,
@@ -536,7 +767,8 @@ export function createCameraFrameCaptureSession(
536
767
  }
537
768
  } finally {
538
769
  restoreObjectVisibility(hidden);
539
- restoreRendererState(renderer, previousState);
770
+ if (previousSceneState) restoreSceneVisualState(scene, previousSceneState);
771
+ restoreRendererState(sessionRenderer, previousState);
540
772
  }
541
773
  }
542
774
  return renderPreparedCapture(captureOptions);
@@ -588,6 +820,9 @@ export function createCameraFrameCaptureSession(
588
820
  },
589
821
  dispose() {
590
822
  target.dispose();
823
+ if (isolatedRenderer && !isolatedRenderer.cached) {
824
+ isolatedRenderer.renderer.dispose();
825
+ }
591
826
  },
592
827
  };
593
828
  }
package/src/spark.tsx CHANGED
@@ -381,6 +381,10 @@ export function SparkSplatEnvironment({
381
381
  const onErrorRef = useRef(onError);
382
382
  const [status, setStatus] = useState<SparkSplatStatus>('idle');
383
383
  const { gl, invalidate } = useThree();
384
+ onStatusChangeRef.current = onStatusChange;
385
+ onLoadRef.current = onLoad;
386
+ onErrorRef.current = onError;
387
+
384
388
  const metadata = useSplatEnvironment({
385
389
  environment,
386
390
  scenario,
@@ -413,18 +417,6 @@ export function SparkSplatEnvironment({
413
417
  ]
414
418
  );
415
419
 
416
- useEffect(() => {
417
- onStatusChangeRef.current = onStatusChange;
418
- }, [onStatusChange]);
419
-
420
- useEffect(() => {
421
- onLoadRef.current = onLoad;
422
- }, [onLoad]);
423
-
424
- useEffect(() => {
425
- onErrorRef.current = onError;
426
- }, [onError]);
427
-
428
420
  useEffect(() => {
429
421
  let disposed = false;
430
422
  ensureSparkDisposeRejectionHandler();
package/src/types.ts CHANGED
@@ -328,6 +328,9 @@ export interface MujocoModel {
328
328
  cam_pos?: Float64Array;
329
329
  cam_quat?: Float64Array;
330
330
  cam_fovy?: Float64Array;
331
+ cam_intrinsic?: Float64Array;
332
+ cam_resolution?: Int32Array;
333
+ cam_sensorsize?: Float64Array;
331
334
 
332
335
  // Tendon
333
336
  ten_wrapadr: Int32Array;
@@ -723,6 +726,9 @@ export interface CameraInfo {
723
726
  name: string;
724
727
  bodyId: number;
725
728
  fov: number | null;
729
+ resolution: [number, number] | null;
730
+ sensorSize: [number, number] | null;
731
+ intrinsic: [number, number, number, number] | null;
726
732
  position: [number, number, number] | null;
727
733
  quaternion: [number, number, number, number] | null;
728
734
  }
@@ -833,6 +839,8 @@ export interface PolicyObservationInput {
833
839
 
834
840
  export interface PolicyInferenceInput extends PolicyObservationInput {
835
841
  observation: PolicyVector;
842
+ /** Number of actions still queued locally when inference is requested. */
843
+ queuedActions?: number;
836
844
  }
837
845
 
838
846
  export type PolicyActionChunk = readonly PolicyVector[];
@@ -1503,6 +1511,51 @@ export type CameraFrameCaptureQuaternion =
1503
1511
  | THREE.Quaternion
1504
1512
  | readonly [number, number, number, number];
1505
1513
 
1514
+ export interface CameraFrameVisualOverrides {
1515
+ /**
1516
+ * Override `scene.background` for this capture only.
1517
+ * Use `null` or `false` to render without the viewer scene background.
1518
+ */
1519
+ sceneBackground?: THREE.Scene['background'] | THREE.ColorRepresentation | null | false;
1520
+ /**
1521
+ * Override `scene.environment` for this capture only.
1522
+ * Use `null` or `false` to remove viewer environment lighting/maps.
1523
+ */
1524
+ sceneEnvironment?: THREE.Scene['environment'] | null | false;
1525
+ /**
1526
+ * Override `scene.fog` for this capture only.
1527
+ * Use `null` or `false` to remove viewer fog.
1528
+ */
1529
+ sceneFog?: THREE.Scene['fog'] | null | false;
1530
+ /** Override `renderer.shadowMap.enabled` while capturing. */
1531
+ shadows?: boolean;
1532
+ /** Override renderer tone mapping while capturing. */
1533
+ toneMapping?: THREE.WebGLRenderer['toneMapping'];
1534
+ /** Override renderer output color space while capturing. */
1535
+ outputColorSpace?: THREE.WebGLRenderer['outputColorSpace'];
1536
+ }
1537
+
1538
+ export interface CameraFrameRenderIsolationOptions {
1539
+ /**
1540
+ * Use an independent offscreen WebGLRenderer for this capture.
1541
+ *
1542
+ * This prevents viewer renderer settings such as antialiasing, shadow-map
1543
+ * configuration, tone mapping, and environment setup from leaking into
1544
+ * policy/training images. Leave unset for the historical shared-renderer path.
1545
+ */
1546
+ enabled?: boolean;
1547
+ /** Offscreen renderer antialiasing. Defaults to false for deterministic policy captures. */
1548
+ antialias?: boolean;
1549
+ /** Offscreen renderer alpha buffer. Defaults to false, matching Three.js WebGLRenderer. */
1550
+ alpha?: boolean;
1551
+ /** Offscreen renderer preserveDrawingBuffer flag. Defaults to false. */
1552
+ preserveDrawingBuffer?: boolean;
1553
+ /** Offscreen renderer power preference. Defaults to the browser's renderer default. */
1554
+ powerPreference?: WebGLPowerPreference;
1555
+ /** Reuse an offscreen renderer for matching capture dimensions/options. Defaults to true. */
1556
+ cache?: boolean;
1557
+ }
1558
+
1506
1559
  export interface CameraFrameCaptureOptions {
1507
1560
  /** Existing Three camera to clone before applying pose overrides. */
1508
1561
  camera?: THREE.Camera;
@@ -1527,8 +1580,36 @@ export interface CameraFrameCaptureOptions {
1527
1580
  fov?: number;
1528
1581
  near?: number;
1529
1582
  far?: number;
1583
+ /**
1584
+ * Explicit projection matrix for offscreen capture. This is useful when a
1585
+ * MuJoCo camera has calibrated intrinsics that cannot be represented by a
1586
+ * symmetric Three.js PerspectiveCamera fov alone.
1587
+ */
1588
+ projectionMatrix?: THREE.Matrix4 | readonly number[];
1530
1589
  /** Provenance for the camera pose used by the capture. Usually set by the MuJoCo provider. */
1531
1590
  source?: CameraFrameCaptureSource;
1591
+ /**
1592
+ * When resolving a named MuJoCo camera, derive Three capture settings from
1593
+ * MuJoCo camera metadata where available: cam_resolution, cam_fovy,
1594
+ * cam_intrinsic/cam_sensorsize, and visual map near/far clipping.
1595
+ */
1596
+ mujocoCameraCompatibility?: boolean | {
1597
+ useResolution?: boolean;
1598
+ useIntrinsics?: boolean;
1599
+ useClipping?: boolean;
1600
+ /**
1601
+ * When a MuJoCo camera has `resolution` metadata and the caller supplies
1602
+ * only width or only height, derive the missing dimension from the MuJoCo
1603
+ * camera aspect ratio.
1604
+ */
1605
+ preserveAspect?: boolean;
1606
+ /**
1607
+ * Prefer the MuJoCo camera's configured resolution over width/height
1608
+ * provided by the caller. Leave false when a policy wants fixed-size
1609
+ * payloads while still preserving the camera aspect ratio.
1610
+ */
1611
+ preferResolution?: boolean;
1612
+ };
1532
1613
  /** Hide rendered Three objects whose MuJoCo geom group is in this list. */
1533
1614
  hiddenGeomGroups?: readonly number[];
1534
1615
  /** When provided, only rendered Three objects whose MuJoCo geom group is in this list are visible. */
@@ -1539,6 +1620,17 @@ export interface CameraFrameCaptureOptions {
1539
1620
  background?: THREE.ColorRepresentation;
1540
1621
  /** Optional clear alpha for this capture only. Defaults to the renderer's current clear alpha. */
1541
1622
  backgroundAlpha?: number;
1623
+ /** Temporary scene/renderer visual overrides applied only for this offscreen capture. */
1624
+ visualOverrides?: CameraFrameVisualOverrides;
1625
+ /**
1626
+ * Render this capture with a separate offscreen WebGLRenderer.
1627
+ *
1628
+ * This is useful for policy or training captures that should remain canonical
1629
+ * while the interactive viewer uses richer visual effects.
1630
+ */
1631
+ renderIsolation?: boolean | CameraFrameRenderIsolationOptions;
1632
+ /** Mirror the captured image horizontally after rendering. Useful when matching policy datasets with mirrored camera frames. */
1633
+ flipX?: boolean;
1542
1634
  }
1543
1635
 
1544
1636
  export type CameraFrameCaptureSource =
@@ -1654,6 +1746,21 @@ export interface CameraFrameSequenceRecorderAPI {
1654
1746
  reset: () => void;
1655
1747
  }
1656
1748
 
1749
+ export type MujocoMeshNormalSmoothing =
1750
+ | boolean
1751
+ | {
1752
+ /** Vertex merge tolerance used before recomputing mesh normals. Defaults to `1e-4`. */
1753
+ tolerance?: number;
1754
+ };
1755
+
1756
+ export interface MujocoRenderOptions {
1757
+ /**
1758
+ * Smooth mesh normals by welding duplicate vertices before recomputing normals.
1759
+ * Useful for faceted STL visuals; keep off for exact policy-render parity.
1760
+ */
1761
+ meshNormalSmoothing?: MujocoMeshNormalSmoothing;
1762
+ }
1763
+
1657
1764
  // ---- Canvas Props ----
1658
1765
 
1659
1766
  export type MujocoCanvasProps = Omit<CanvasProps, 'onError'> & {
@@ -1671,6 +1778,7 @@ export type MujocoCanvasProps = Omit<CanvasProps, 'onError'> & {
1671
1778
  paused?: boolean;
1672
1779
  speed?: number;
1673
1780
  interpolate?: boolean;
1781
+ renderOptions?: MujocoRenderOptions;
1674
1782
  };
1675
1783
 
1676
1784
  // ---- Hook Return Types ----