mujoco-react 10.2.0 → 10.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -0
- package/dist/{chunk-3BMNRSS2.js → chunk-6AZEFI6A.js} +199 -36
- package/dist/chunk-6AZEFI6A.js.map +1 -0
- package/dist/index.d.ts +4 -3
- package/dist/index.js +194 -42
- package/dist/index.js.map +1 -1
- package/dist/spark.d.ts +1 -1
- package/dist/spark.js +4 -10
- package/dist/spark.js.map +1 -1
- package/dist/{types-CLD5K3JD.d.ts → types-BOhNDICK.d.ts} +103 -1
- package/package.json +1 -1
- package/src/components/SceneRenderer.tsx +25 -6
- package/src/core/MujocoCanvas.tsx +8 -4
- package/src/core/MujocoPhysics.tsx +6 -4
- package/src/core/MujocoProvider.tsx +6 -4
- package/src/core/MujocoSimProvider.tsx +189 -9
- package/src/hooks/usePolicy.ts +4 -1
- package/src/hooks/useRemotePolicy.ts +2 -1
- package/src/rendering/GeomBuilder.ts +18 -2
- package/src/rendering/cameraFrameCapture.ts +268 -33
- package/src/spark.tsx +4 -12
- package/src/types.ts +108 -0
- package/dist/chunk-3BMNRSS2.js.map +0 -1
|
@@ -60,6 +60,7 @@ export type CameraFrameCaptureRenderResult = {
|
|
|
60
60
|
width?: number;
|
|
61
61
|
height?: number;
|
|
62
62
|
flipY?: boolean;
|
|
63
|
+
flipX?: boolean;
|
|
63
64
|
};
|
|
64
65
|
|
|
65
66
|
type CameraFrameCaptureRender = (
|
|
@@ -79,6 +80,15 @@ type RendererState = {
|
|
|
79
80
|
clearColor: THREE.Color;
|
|
80
81
|
clearAlpha: number;
|
|
81
82
|
autoClear: boolean;
|
|
83
|
+
shadowMapEnabled: boolean;
|
|
84
|
+
toneMapping: THREE.WebGLRenderer['toneMapping'];
|
|
85
|
+
outputColorSpace: THREE.WebGLRenderer['outputColorSpace'];
|
|
86
|
+
};
|
|
87
|
+
|
|
88
|
+
type SceneVisualState = {
|
|
89
|
+
background: THREE.Scene['background'];
|
|
90
|
+
environment: THREE.Scene['environment'];
|
|
91
|
+
fog: THREE.Scene['fog'];
|
|
82
92
|
};
|
|
83
93
|
|
|
84
94
|
type VisibilityState = {
|
|
@@ -88,6 +98,107 @@ type VisibilityState = {
|
|
|
88
98
|
|
|
89
99
|
type CameraFrameCapturePreRender = () => void;
|
|
90
100
|
|
|
101
|
+
const isolatedRendererCache = new WeakMap<
|
|
102
|
+
THREE.WebGLRenderer,
|
|
103
|
+
Map<string, THREE.WebGLRenderer>
|
|
104
|
+
>();
|
|
105
|
+
|
|
106
|
+
function shouldUseRenderIsolation(
|
|
107
|
+
options: CameraFrameCaptureOptions
|
|
108
|
+
): boolean {
|
|
109
|
+
return options.renderIsolation === true || (
|
|
110
|
+
typeof options.renderIsolation === 'object' &&
|
|
111
|
+
options.renderIsolation.enabled !== false
|
|
112
|
+
);
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
function getRenderIsolationOptions(
|
|
116
|
+
options: CameraFrameCaptureOptions
|
|
117
|
+
) {
|
|
118
|
+
return typeof options.renderIsolation === 'object'
|
|
119
|
+
? options.renderIsolation
|
|
120
|
+
: {};
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
function getRenderIsolationCacheKey(
|
|
124
|
+
width: number,
|
|
125
|
+
height: number,
|
|
126
|
+
options: CameraFrameCaptureOptions
|
|
127
|
+
) {
|
|
128
|
+
const isolation = getRenderIsolationOptions(options);
|
|
129
|
+
return JSON.stringify({
|
|
130
|
+
width,
|
|
131
|
+
height,
|
|
132
|
+
antialias: isolation.antialias ?? false,
|
|
133
|
+
alpha: isolation.alpha ?? false,
|
|
134
|
+
preserveDrawingBuffer: isolation.preserveDrawingBuffer ?? false,
|
|
135
|
+
powerPreference: isolation.powerPreference ?? null,
|
|
136
|
+
});
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
function createIsolatedRenderer(
|
|
140
|
+
sourceRenderer: THREE.WebGLRenderer,
|
|
141
|
+
width: number,
|
|
142
|
+
height: number,
|
|
143
|
+
options: CameraFrameCaptureOptions
|
|
144
|
+
): { renderer: THREE.WebGLRenderer; cached: boolean } | null {
|
|
145
|
+
if (!shouldUseRenderIsolation(options)) return null;
|
|
146
|
+
|
|
147
|
+
const isolation = getRenderIsolationOptions(options);
|
|
148
|
+
if (isolation.cache !== false) {
|
|
149
|
+
const cacheKey = getRenderIsolationCacheKey(width, height, options);
|
|
150
|
+
let rendererCache = isolatedRendererCache.get(sourceRenderer);
|
|
151
|
+
if (!rendererCache) {
|
|
152
|
+
rendererCache = new Map();
|
|
153
|
+
isolatedRendererCache.set(sourceRenderer, rendererCache);
|
|
154
|
+
}
|
|
155
|
+
const cachedRenderer = rendererCache.get(cacheKey);
|
|
156
|
+
if (cachedRenderer) {
|
|
157
|
+
cachedRenderer.outputColorSpace = sourceRenderer.outputColorSpace;
|
|
158
|
+
cachedRenderer.toneMapping = sourceRenderer.toneMapping;
|
|
159
|
+
cachedRenderer.shadowMap.enabled = false;
|
|
160
|
+
return { renderer: cachedRenderer, cached: true };
|
|
161
|
+
}
|
|
162
|
+
const createdRenderer = createUncachedIsolatedRenderer(
|
|
163
|
+
sourceRenderer,
|
|
164
|
+
width,
|
|
165
|
+
height,
|
|
166
|
+
options
|
|
167
|
+
);
|
|
168
|
+
rendererCache.set(cacheKey, createdRenderer);
|
|
169
|
+
return { renderer: createdRenderer, cached: true };
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
return {
|
|
173
|
+
renderer: createUncachedIsolatedRenderer(sourceRenderer, width, height, options),
|
|
174
|
+
cached: false,
|
|
175
|
+
};
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
function createUncachedIsolatedRenderer(
|
|
179
|
+
sourceRenderer: THREE.WebGLRenderer,
|
|
180
|
+
width: number,
|
|
181
|
+
height: number,
|
|
182
|
+
options: CameraFrameCaptureOptions
|
|
183
|
+
) {
|
|
184
|
+
const isolation = getRenderIsolationOptions(options);
|
|
185
|
+
const canvas = document.createElement('canvas');
|
|
186
|
+
const renderer = new THREE.WebGLRenderer({
|
|
187
|
+
canvas,
|
|
188
|
+
antialias: isolation.antialias ?? false,
|
|
189
|
+
alpha: isolation.alpha ?? false,
|
|
190
|
+
preserveDrawingBuffer: isolation.preserveDrawingBuffer ?? false,
|
|
191
|
+
powerPreference: isolation.powerPreference,
|
|
192
|
+
});
|
|
193
|
+
|
|
194
|
+
renderer.setPixelRatio(1);
|
|
195
|
+
renderer.setSize(width, height, false);
|
|
196
|
+
renderer.outputColorSpace = sourceRenderer.outputColorSpace;
|
|
197
|
+
renderer.toneMapping = sourceRenderer.toneMapping;
|
|
198
|
+
renderer.shadowMap.enabled = false;
|
|
199
|
+
return renderer;
|
|
200
|
+
}
|
|
201
|
+
|
|
91
202
|
function toVector3(
|
|
92
203
|
value: CameraFrameCaptureVector3 | undefined,
|
|
93
204
|
fallback: THREE.Vector3
|
|
@@ -126,6 +237,19 @@ function applyCameraPose(
|
|
|
126
237
|
camera.updateMatrixWorld();
|
|
127
238
|
}
|
|
128
239
|
|
|
240
|
+
function applyProjectionMatrix(
|
|
241
|
+
camera: THREE.Camera,
|
|
242
|
+
projectionMatrix: CameraFrameCaptureOptions['projectionMatrix'] | undefined
|
|
243
|
+
) {
|
|
244
|
+
if (!projectionMatrix) return;
|
|
245
|
+
if (projectionMatrix instanceof THREE.Matrix4) {
|
|
246
|
+
camera.projectionMatrix.copy(projectionMatrix);
|
|
247
|
+
} else {
|
|
248
|
+
camera.projectionMatrix.fromArray(projectionMatrix);
|
|
249
|
+
}
|
|
250
|
+
camera.projectionMatrixInverse.copy(camera.projectionMatrix).invert();
|
|
251
|
+
}
|
|
252
|
+
|
|
129
253
|
function createCaptureCamera(
|
|
130
254
|
options: CameraFrameCaptureOptions,
|
|
131
255
|
fallbackCamera: THREE.Camera,
|
|
@@ -145,6 +269,7 @@ function createCaptureCamera(
|
|
|
145
269
|
camera.far = options.far ?? camera.far;
|
|
146
270
|
camera.updateProjectionMatrix();
|
|
147
271
|
}
|
|
272
|
+
applyProjectionMatrix(camera, options.projectionMatrix);
|
|
148
273
|
|
|
149
274
|
applyCameraPose(camera, options, fallbackCamera);
|
|
150
275
|
return camera;
|
|
@@ -183,6 +308,7 @@ function prepareCaptureCamera(
|
|
|
183
308
|
camera.far = options.far ?? camera.far;
|
|
184
309
|
camera.updateProjectionMatrix();
|
|
185
310
|
}
|
|
311
|
+
applyProjectionMatrix(camera, options.projectionMatrix);
|
|
186
312
|
|
|
187
313
|
applyCameraPose(camera, options, fallbackCamera);
|
|
188
314
|
}
|
|
@@ -196,7 +322,8 @@ function readRenderTargetToCanvas(
|
|
|
196
322
|
imageData: ImageData,
|
|
197
323
|
width: number,
|
|
198
324
|
height: number,
|
|
199
|
-
outputColorSpace: string
|
|
325
|
+
outputColorSpace: string,
|
|
326
|
+
flipX = false
|
|
200
327
|
) {
|
|
201
328
|
renderer.readRenderTargetPixels(target, 0, 0, width, height, pixels);
|
|
202
329
|
|
|
@@ -206,17 +333,25 @@ function readRenderTargetToCanvas(
|
|
|
206
333
|
const sourceStart = (height - y - 1) * rowBytes;
|
|
207
334
|
const targetStart = y * rowBytes;
|
|
208
335
|
const row = pixels.subarray(sourceStart, sourceStart + rowBytes);
|
|
209
|
-
if (!encodeSrgb) {
|
|
336
|
+
if (!encodeSrgb && !flipX) {
|
|
210
337
|
imageData.data.set(row, targetStart);
|
|
211
338
|
continue;
|
|
212
339
|
}
|
|
213
340
|
|
|
214
|
-
for (let x = 0; x <
|
|
215
|
-
const
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
imageData.data[
|
|
219
|
-
|
|
341
|
+
for (let x = 0; x < width; x += 1) {
|
|
342
|
+
const sourceX = flipX ? width - x - 1 : x;
|
|
343
|
+
const sourceOffset = sourceX * 4;
|
|
344
|
+
const targetOffset = targetStart + x * 4;
|
|
345
|
+
imageData.data[targetOffset] = encodeSrgb
|
|
346
|
+
? linearByteToSrgbByte(row[sourceOffset])
|
|
347
|
+
: row[sourceOffset];
|
|
348
|
+
imageData.data[targetOffset + 1] = encodeSrgb
|
|
349
|
+
? linearByteToSrgbByte(row[sourceOffset + 1])
|
|
350
|
+
: row[sourceOffset + 1];
|
|
351
|
+
imageData.data[targetOffset + 2] = encodeSrgb
|
|
352
|
+
? linearByteToSrgbByte(row[sourceOffset + 2])
|
|
353
|
+
: row[sourceOffset + 2];
|
|
354
|
+
imageData.data[targetOffset + 3] = row[sourceOffset + 3];
|
|
220
355
|
}
|
|
221
356
|
}
|
|
222
357
|
context.putImageData(imageData, 0, 0);
|
|
@@ -238,17 +373,30 @@ function readPixelsToCanvas(
|
|
|
238
373
|
imageData: ImageData,
|
|
239
374
|
width: number,
|
|
240
375
|
height: number,
|
|
241
|
-
flipY = true
|
|
376
|
+
flipY = true,
|
|
377
|
+
flipX = false
|
|
242
378
|
) {
|
|
243
379
|
const rowBytes = width * 4;
|
|
244
380
|
for (let y = 0; y < height; y += 1) {
|
|
245
381
|
const sourceY = flipY ? height - y - 1 : y;
|
|
246
382
|
const sourceStart = sourceY * rowBytes;
|
|
247
383
|
const targetStart = y * rowBytes;
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
384
|
+
if (!flipX) {
|
|
385
|
+
imageData.data.set(
|
|
386
|
+
pixels.subarray(sourceStart, sourceStart + rowBytes),
|
|
387
|
+
targetStart
|
|
388
|
+
);
|
|
389
|
+
continue;
|
|
390
|
+
}
|
|
391
|
+
for (let x = 0; x < width; x += 1) {
|
|
392
|
+
const sourceX = width - x - 1;
|
|
393
|
+
const sourceOffset = sourceStart + sourceX * 4;
|
|
394
|
+
const targetOffset = targetStart + x * 4;
|
|
395
|
+
imageData.data[targetOffset] = pixels[sourceOffset];
|
|
396
|
+
imageData.data[targetOffset + 1] = pixels[sourceOffset + 1];
|
|
397
|
+
imageData.data[targetOffset + 2] = pixels[sourceOffset + 2];
|
|
398
|
+
imageData.data[targetOffset + 3] = pixels[sourceOffset + 3];
|
|
399
|
+
}
|
|
252
400
|
}
|
|
253
401
|
context.putImageData(imageData, 0, 0);
|
|
254
402
|
}
|
|
@@ -339,6 +487,9 @@ function saveRendererState(renderer: THREE.WebGLRenderer): RendererState {
|
|
|
339
487
|
clearColor,
|
|
340
488
|
clearAlpha: renderer.getClearAlpha(),
|
|
341
489
|
autoClear: renderer.autoClear,
|
|
490
|
+
shadowMapEnabled: renderer.shadowMap.enabled,
|
|
491
|
+
toneMapping: renderer.toneMapping,
|
|
492
|
+
outputColorSpace: renderer.outputColorSpace,
|
|
342
493
|
};
|
|
343
494
|
}
|
|
344
495
|
|
|
@@ -353,6 +504,64 @@ function restoreRendererState(
|
|
|
353
504
|
renderer.setScissorTest(state.scissorTest);
|
|
354
505
|
renderer.setClearColor(state.clearColor, state.clearAlpha);
|
|
355
506
|
renderer.autoClear = state.autoClear;
|
|
507
|
+
renderer.shadowMap.enabled = state.shadowMapEnabled;
|
|
508
|
+
renderer.toneMapping = state.toneMapping;
|
|
509
|
+
renderer.outputColorSpace = state.outputColorSpace;
|
|
510
|
+
}
|
|
511
|
+
|
|
512
|
+
function saveSceneVisualState(scene: THREE.Scene): SceneVisualState {
|
|
513
|
+
return {
|
|
514
|
+
background: scene.background,
|
|
515
|
+
environment: scene.environment,
|
|
516
|
+
fog: scene.fog,
|
|
517
|
+
};
|
|
518
|
+
}
|
|
519
|
+
|
|
520
|
+
function restoreSceneVisualState(scene: THREE.Scene, state: SceneVisualState) {
|
|
521
|
+
scene.background = state.background;
|
|
522
|
+
scene.environment = state.environment;
|
|
523
|
+
scene.fog = state.fog;
|
|
524
|
+
}
|
|
525
|
+
|
|
526
|
+
function hasOwn<T extends object>(object: T, key: keyof T) {
|
|
527
|
+
return Object.prototype.hasOwnProperty.call(object, key);
|
|
528
|
+
}
|
|
529
|
+
|
|
530
|
+
function applyCaptureVisualOverrides(
|
|
531
|
+
renderer: THREE.WebGLRenderer,
|
|
532
|
+
scene: THREE.Scene,
|
|
533
|
+
options: CameraFrameCaptureOptions
|
|
534
|
+
): SceneVisualState | null {
|
|
535
|
+
const overrides = options.visualOverrides;
|
|
536
|
+
if (!overrides) return null;
|
|
537
|
+
|
|
538
|
+
const previousSceneState = saveSceneVisualState(scene);
|
|
539
|
+
if (hasOwn(overrides, 'sceneBackground')) {
|
|
540
|
+
const background = overrides.sceneBackground;
|
|
541
|
+
scene.background = background === false ? null : (
|
|
542
|
+
typeof background === 'string' || typeof background === 'number'
|
|
543
|
+
? new THREE.Color(background)
|
|
544
|
+
: background ?? null
|
|
545
|
+
);
|
|
546
|
+
}
|
|
547
|
+
if (hasOwn(overrides, 'sceneEnvironment')) {
|
|
548
|
+
const environment = overrides.sceneEnvironment;
|
|
549
|
+
scene.environment = environment === false ? null : environment ?? null;
|
|
550
|
+
}
|
|
551
|
+
if (hasOwn(overrides, 'sceneFog')) {
|
|
552
|
+
const fog = overrides.sceneFog;
|
|
553
|
+
scene.fog = fog === false ? null : fog ?? null;
|
|
554
|
+
}
|
|
555
|
+
if (overrides.shadows !== undefined) {
|
|
556
|
+
renderer.shadowMap.enabled = overrides.shadows;
|
|
557
|
+
}
|
|
558
|
+
if (overrides.toneMapping !== undefined) {
|
|
559
|
+
renderer.toneMapping = overrides.toneMapping;
|
|
560
|
+
}
|
|
561
|
+
if (overrides.outputColorSpace !== undefined) {
|
|
562
|
+
renderer.outputColorSpace = overrides.outputColorSpace;
|
|
563
|
+
}
|
|
564
|
+
return previousSceneState;
|
|
356
565
|
}
|
|
357
566
|
|
|
358
567
|
function getCaptureRenderer(
|
|
@@ -387,6 +596,8 @@ export function createCameraFrameCaptureSession(
|
|
|
387
596
|
options: CameraFrameCaptureOptions = {}
|
|
388
597
|
): CameraFrameCaptureSession {
|
|
389
598
|
const { width, height } = getCaptureDimensions(renderer, options);
|
|
599
|
+
const isolatedRenderer = createIsolatedRenderer(renderer, width, height, options);
|
|
600
|
+
const sessionRenderer = isolatedRenderer?.renderer ?? renderer;
|
|
390
601
|
const camera = createCaptureCamera(options, fallbackCamera, width, height);
|
|
391
602
|
const target = new THREE.WebGLRenderTarget(width, height, {
|
|
392
603
|
format: THREE.RGBAFormat,
|
|
@@ -407,6 +618,13 @@ export function createCameraFrameCaptureSession(
|
|
|
407
618
|
|
|
408
619
|
function resolveCaptureOptions(nextOptions: CameraFrameCaptureOptions = {}) {
|
|
409
620
|
const captureOptions = { ...options, ...nextOptions };
|
|
621
|
+
if (
|
|
622
|
+
shouldUseRenderIsolation(captureOptions) !== shouldUseRenderIsolation(options)
|
|
623
|
+
) {
|
|
624
|
+
throw new Error(
|
|
625
|
+
'Camera frame capture sessions require stable renderIsolation settings.'
|
|
626
|
+
);
|
|
627
|
+
}
|
|
410
628
|
const nextDimensions = getCaptureDimensions(renderer, captureOptions);
|
|
411
629
|
if (
|
|
412
630
|
nextDimensions.width !== width ||
|
|
@@ -429,7 +647,12 @@ export function createCameraFrameCaptureSession(
|
|
|
429
647
|
}
|
|
430
648
|
|
|
431
649
|
function renderPreparedCapture(captureOptions: CameraFrameCaptureOptions) {
|
|
432
|
-
const previousState = saveRendererState(
|
|
650
|
+
const previousState = saveRendererState(sessionRenderer);
|
|
651
|
+
const previousSceneState = applyCaptureVisualOverrides(
|
|
652
|
+
sessionRenderer,
|
|
653
|
+
scene,
|
|
654
|
+
captureOptions
|
|
655
|
+
);
|
|
433
656
|
const hidden = [
|
|
434
657
|
...hideExcludedCaptureObjects(scene),
|
|
435
658
|
...hideCaptureGeomGroups(scene, captureOptions),
|
|
@@ -438,23 +661,23 @@ export function createCameraFrameCaptureSession(
|
|
|
438
661
|
runCapturePreRenderHooks(scene);
|
|
439
662
|
scene.updateMatrixWorld(true);
|
|
440
663
|
try {
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
664
|
+
sessionRenderer.xr.enabled = false;
|
|
665
|
+
sessionRenderer.setRenderTarget(target);
|
|
666
|
+
sessionRenderer.setViewport(0, 0, width, height);
|
|
667
|
+
sessionRenderer.setScissor(0, 0, width, height);
|
|
668
|
+
sessionRenderer.setScissorTest(false);
|
|
446
669
|
if (captureOptions.background !== undefined) {
|
|
447
|
-
|
|
670
|
+
sessionRenderer.setClearColor(
|
|
448
671
|
new THREE.Color(captureOptions.background),
|
|
449
672
|
captureOptions.backgroundAlpha ?? previousState.clearAlpha
|
|
450
673
|
);
|
|
451
674
|
} else if (captureOptions.backgroundAlpha !== undefined) {
|
|
452
|
-
|
|
675
|
+
sessionRenderer.setClearColor(previousState.clearColor, captureOptions.backgroundAlpha);
|
|
453
676
|
}
|
|
454
|
-
|
|
455
|
-
|
|
677
|
+
sessionRenderer.clear();
|
|
678
|
+
sessionRenderer.render(scene, camera);
|
|
456
679
|
readRenderTargetToCanvas(
|
|
457
|
-
|
|
680
|
+
sessionRenderer,
|
|
458
681
|
target,
|
|
459
682
|
canvas,
|
|
460
683
|
drawContext,
|
|
@@ -462,7 +685,8 @@ export function createCameraFrameCaptureSession(
|
|
|
462
685
|
imageData,
|
|
463
686
|
width,
|
|
464
687
|
height,
|
|
465
|
-
|
|
688
|
+
sessionRenderer.outputColorSpace,
|
|
689
|
+
captureOptions.flipX ?? false
|
|
466
690
|
);
|
|
467
691
|
return {
|
|
468
692
|
canvas,
|
|
@@ -473,7 +697,8 @@ export function createCameraFrameCaptureSession(
|
|
|
473
697
|
};
|
|
474
698
|
} finally {
|
|
475
699
|
restoreObjectVisibility(hidden);
|
|
476
|
-
|
|
700
|
+
if (previousSceneState) restoreSceneVisualState(scene, previousSceneState);
|
|
701
|
+
restoreRendererState(sessionRenderer, previousState);
|
|
477
702
|
}
|
|
478
703
|
}
|
|
479
704
|
|
|
@@ -487,23 +712,28 @@ export function createCameraFrameCaptureSession(
|
|
|
487
712
|
scene.updateMatrixWorld(true);
|
|
488
713
|
const captureRenderer = getCaptureRenderer(scene);
|
|
489
714
|
if (captureRenderer) {
|
|
490
|
-
const previousState = saveRendererState(
|
|
715
|
+
const previousState = saveRendererState(sessionRenderer);
|
|
716
|
+
const previousSceneState = applyCaptureVisualOverrides(
|
|
717
|
+
sessionRenderer,
|
|
718
|
+
scene,
|
|
719
|
+
captureOptions
|
|
720
|
+
);
|
|
491
721
|
const hidden = [
|
|
492
722
|
...hideExcludedCaptureObjects(scene),
|
|
493
723
|
...hideCaptureGeomGroups(scene, captureOptions),
|
|
494
724
|
];
|
|
495
725
|
try {
|
|
496
|
-
|
|
726
|
+
sessionRenderer.xr.enabled = false;
|
|
497
727
|
if (captureOptions.background !== undefined) {
|
|
498
|
-
|
|
728
|
+
sessionRenderer.setClearColor(
|
|
499
729
|
new THREE.Color(captureOptions.background),
|
|
500
730
|
captureOptions.backgroundAlpha ?? previousState.clearAlpha
|
|
501
731
|
);
|
|
502
732
|
} else if (captureOptions.backgroundAlpha !== undefined) {
|
|
503
|
-
|
|
733
|
+
sessionRenderer.setClearColor(previousState.clearColor, captureOptions.backgroundAlpha);
|
|
504
734
|
}
|
|
505
735
|
const captureResult = await captureRenderer({
|
|
506
|
-
renderer,
|
|
736
|
+
renderer: sessionRenderer,
|
|
507
737
|
scene,
|
|
508
738
|
camera,
|
|
509
739
|
target,
|
|
@@ -524,7 +754,8 @@ export function createCameraFrameCaptureSession(
|
|
|
524
754
|
imageData,
|
|
525
755
|
width,
|
|
526
756
|
height,
|
|
527
|
-
captureResult.flipY ?? true
|
|
757
|
+
captureResult.flipY ?? true,
|
|
758
|
+
captureResult.flipX ?? captureOptions.flipX ?? false
|
|
528
759
|
);
|
|
529
760
|
return {
|
|
530
761
|
canvas,
|
|
@@ -536,7 +767,8 @@ export function createCameraFrameCaptureSession(
|
|
|
536
767
|
}
|
|
537
768
|
} finally {
|
|
538
769
|
restoreObjectVisibility(hidden);
|
|
539
|
-
|
|
770
|
+
if (previousSceneState) restoreSceneVisualState(scene, previousSceneState);
|
|
771
|
+
restoreRendererState(sessionRenderer, previousState);
|
|
540
772
|
}
|
|
541
773
|
}
|
|
542
774
|
return renderPreparedCapture(captureOptions);
|
|
@@ -588,6 +820,9 @@ export function createCameraFrameCaptureSession(
|
|
|
588
820
|
},
|
|
589
821
|
dispose() {
|
|
590
822
|
target.dispose();
|
|
823
|
+
if (isolatedRenderer && !isolatedRenderer.cached) {
|
|
824
|
+
isolatedRenderer.renderer.dispose();
|
|
825
|
+
}
|
|
591
826
|
},
|
|
592
827
|
};
|
|
593
828
|
}
|
package/src/spark.tsx
CHANGED
|
@@ -381,6 +381,10 @@ export function SparkSplatEnvironment({
|
|
|
381
381
|
const onErrorRef = useRef(onError);
|
|
382
382
|
const [status, setStatus] = useState<SparkSplatStatus>('idle');
|
|
383
383
|
const { gl, invalidate } = useThree();
|
|
384
|
+
onStatusChangeRef.current = onStatusChange;
|
|
385
|
+
onLoadRef.current = onLoad;
|
|
386
|
+
onErrorRef.current = onError;
|
|
387
|
+
|
|
384
388
|
const metadata = useSplatEnvironment({
|
|
385
389
|
environment,
|
|
386
390
|
scenario,
|
|
@@ -413,18 +417,6 @@ export function SparkSplatEnvironment({
|
|
|
413
417
|
]
|
|
414
418
|
);
|
|
415
419
|
|
|
416
|
-
useEffect(() => {
|
|
417
|
-
onStatusChangeRef.current = onStatusChange;
|
|
418
|
-
}, [onStatusChange]);
|
|
419
|
-
|
|
420
|
-
useEffect(() => {
|
|
421
|
-
onLoadRef.current = onLoad;
|
|
422
|
-
}, [onLoad]);
|
|
423
|
-
|
|
424
|
-
useEffect(() => {
|
|
425
|
-
onErrorRef.current = onError;
|
|
426
|
-
}, [onError]);
|
|
427
|
-
|
|
428
420
|
useEffect(() => {
|
|
429
421
|
let disposed = false;
|
|
430
422
|
ensureSparkDisposeRejectionHandler();
|
package/src/types.ts
CHANGED
|
@@ -328,6 +328,9 @@ export interface MujocoModel {
|
|
|
328
328
|
cam_pos?: Float64Array;
|
|
329
329
|
cam_quat?: Float64Array;
|
|
330
330
|
cam_fovy?: Float64Array;
|
|
331
|
+
cam_intrinsic?: Float64Array;
|
|
332
|
+
cam_resolution?: Int32Array;
|
|
333
|
+
cam_sensorsize?: Float64Array;
|
|
331
334
|
|
|
332
335
|
// Tendon
|
|
333
336
|
ten_wrapadr: Int32Array;
|
|
@@ -723,6 +726,9 @@ export interface CameraInfo {
|
|
|
723
726
|
name: string;
|
|
724
727
|
bodyId: number;
|
|
725
728
|
fov: number | null;
|
|
729
|
+
resolution: [number, number] | null;
|
|
730
|
+
sensorSize: [number, number] | null;
|
|
731
|
+
intrinsic: [number, number, number, number] | null;
|
|
726
732
|
position: [number, number, number] | null;
|
|
727
733
|
quaternion: [number, number, number, number] | null;
|
|
728
734
|
}
|
|
@@ -833,6 +839,8 @@ export interface PolicyObservationInput {
|
|
|
833
839
|
|
|
834
840
|
export interface PolicyInferenceInput extends PolicyObservationInput {
|
|
835
841
|
observation: PolicyVector;
|
|
842
|
+
/** Number of actions still queued locally when inference is requested. */
|
|
843
|
+
queuedActions?: number;
|
|
836
844
|
}
|
|
837
845
|
|
|
838
846
|
export type PolicyActionChunk = readonly PolicyVector[];
|
|
@@ -1503,6 +1511,51 @@ export type CameraFrameCaptureQuaternion =
|
|
|
1503
1511
|
| THREE.Quaternion
|
|
1504
1512
|
| readonly [number, number, number, number];
|
|
1505
1513
|
|
|
1514
|
+
export interface CameraFrameVisualOverrides {
|
|
1515
|
+
/**
|
|
1516
|
+
* Override `scene.background` for this capture only.
|
|
1517
|
+
* Use `null` or `false` to render without the viewer scene background.
|
|
1518
|
+
*/
|
|
1519
|
+
sceneBackground?: THREE.Scene['background'] | THREE.ColorRepresentation | null | false;
|
|
1520
|
+
/**
|
|
1521
|
+
* Override `scene.environment` for this capture only.
|
|
1522
|
+
* Use `null` or `false` to remove viewer environment lighting/maps.
|
|
1523
|
+
*/
|
|
1524
|
+
sceneEnvironment?: THREE.Scene['environment'] | null | false;
|
|
1525
|
+
/**
|
|
1526
|
+
* Override `scene.fog` for this capture only.
|
|
1527
|
+
* Use `null` or `false` to remove viewer fog.
|
|
1528
|
+
*/
|
|
1529
|
+
sceneFog?: THREE.Scene['fog'] | null | false;
|
|
1530
|
+
/** Override `renderer.shadowMap.enabled` while capturing. */
|
|
1531
|
+
shadows?: boolean;
|
|
1532
|
+
/** Override renderer tone mapping while capturing. */
|
|
1533
|
+
toneMapping?: THREE.WebGLRenderer['toneMapping'];
|
|
1534
|
+
/** Override renderer output color space while capturing. */
|
|
1535
|
+
outputColorSpace?: THREE.WebGLRenderer['outputColorSpace'];
|
|
1536
|
+
}
|
|
1537
|
+
|
|
1538
|
+
export interface CameraFrameRenderIsolationOptions {
|
|
1539
|
+
/**
|
|
1540
|
+
* Use an independent offscreen WebGLRenderer for this capture.
|
|
1541
|
+
*
|
|
1542
|
+
* This prevents viewer renderer settings such as antialiasing, shadow-map
|
|
1543
|
+
* configuration, tone mapping, and environment setup from leaking into
|
|
1544
|
+
* policy/training images. Leave unset for the historical shared-renderer path.
|
|
1545
|
+
*/
|
|
1546
|
+
enabled?: boolean;
|
|
1547
|
+
/** Offscreen renderer antialiasing. Defaults to false for deterministic policy captures. */
|
|
1548
|
+
antialias?: boolean;
|
|
1549
|
+
/** Offscreen renderer alpha buffer. Defaults to false, matching Three.js WebGLRenderer. */
|
|
1550
|
+
alpha?: boolean;
|
|
1551
|
+
/** Offscreen renderer preserveDrawingBuffer flag. Defaults to false. */
|
|
1552
|
+
preserveDrawingBuffer?: boolean;
|
|
1553
|
+
/** Offscreen renderer power preference. Defaults to the browser's renderer default. */
|
|
1554
|
+
powerPreference?: WebGLPowerPreference;
|
|
1555
|
+
/** Reuse an offscreen renderer for matching capture dimensions/options. Defaults to true. */
|
|
1556
|
+
cache?: boolean;
|
|
1557
|
+
}
|
|
1558
|
+
|
|
1506
1559
|
export interface CameraFrameCaptureOptions {
|
|
1507
1560
|
/** Existing Three camera to clone before applying pose overrides. */
|
|
1508
1561
|
camera?: THREE.Camera;
|
|
@@ -1527,8 +1580,36 @@ export interface CameraFrameCaptureOptions {
|
|
|
1527
1580
|
fov?: number;
|
|
1528
1581
|
near?: number;
|
|
1529
1582
|
far?: number;
|
|
1583
|
+
/**
|
|
1584
|
+
* Explicit projection matrix for offscreen capture. This is useful when a
|
|
1585
|
+
* MuJoCo camera has calibrated intrinsics that cannot be represented by a
|
|
1586
|
+
* symmetric Three.js PerspectiveCamera fov alone.
|
|
1587
|
+
*/
|
|
1588
|
+
projectionMatrix?: THREE.Matrix4 | readonly number[];
|
|
1530
1589
|
/** Provenance for the camera pose used by the capture. Usually set by the MuJoCo provider. */
|
|
1531
1590
|
source?: CameraFrameCaptureSource;
|
|
1591
|
+
/**
|
|
1592
|
+
* When resolving a named MuJoCo camera, derive Three capture settings from
|
|
1593
|
+
* MuJoCo camera metadata where available: cam_resolution, cam_fovy,
|
|
1594
|
+
* cam_intrinsic/cam_sensorsize, and visual map near/far clipping.
|
|
1595
|
+
*/
|
|
1596
|
+
mujocoCameraCompatibility?: boolean | {
|
|
1597
|
+
useResolution?: boolean;
|
|
1598
|
+
useIntrinsics?: boolean;
|
|
1599
|
+
useClipping?: boolean;
|
|
1600
|
+
/**
|
|
1601
|
+
* When a MuJoCo camera has `resolution` metadata and the caller supplies
|
|
1602
|
+
* only width or only height, derive the missing dimension from the MuJoCo
|
|
1603
|
+
* camera aspect ratio.
|
|
1604
|
+
*/
|
|
1605
|
+
preserveAspect?: boolean;
|
|
1606
|
+
/**
|
|
1607
|
+
* Prefer the MuJoCo camera's configured resolution over width/height
|
|
1608
|
+
* provided by the caller. Leave false when a policy wants fixed-size
|
|
1609
|
+
* payloads while still preserving the camera aspect ratio.
|
|
1610
|
+
*/
|
|
1611
|
+
preferResolution?: boolean;
|
|
1612
|
+
};
|
|
1532
1613
|
/** Hide rendered Three objects whose MuJoCo geom group is in this list. */
|
|
1533
1614
|
hiddenGeomGroups?: readonly number[];
|
|
1534
1615
|
/** When provided, only rendered Three objects whose MuJoCo geom group is in this list are visible. */
|
|
@@ -1539,6 +1620,17 @@ export interface CameraFrameCaptureOptions {
|
|
|
1539
1620
|
background?: THREE.ColorRepresentation;
|
|
1540
1621
|
/** Optional clear alpha for this capture only. Defaults to the renderer's current clear alpha. */
|
|
1541
1622
|
backgroundAlpha?: number;
|
|
1623
|
+
/** Temporary scene/renderer visual overrides applied only for this offscreen capture. */
|
|
1624
|
+
visualOverrides?: CameraFrameVisualOverrides;
|
|
1625
|
+
/**
|
|
1626
|
+
* Render this capture with a separate offscreen WebGLRenderer.
|
|
1627
|
+
*
|
|
1628
|
+
* This is useful for policy or training captures that should remain canonical
|
|
1629
|
+
* while the interactive viewer uses richer visual effects.
|
|
1630
|
+
*/
|
|
1631
|
+
renderIsolation?: boolean | CameraFrameRenderIsolationOptions;
|
|
1632
|
+
/** Mirror the captured image horizontally after rendering. Useful when matching policy datasets with mirrored camera frames. */
|
|
1633
|
+
flipX?: boolean;
|
|
1542
1634
|
}
|
|
1543
1635
|
|
|
1544
1636
|
export type CameraFrameCaptureSource =
|
|
@@ -1654,6 +1746,21 @@ export interface CameraFrameSequenceRecorderAPI {
|
|
|
1654
1746
|
reset: () => void;
|
|
1655
1747
|
}
|
|
1656
1748
|
|
|
1749
|
+
export type MujocoMeshNormalSmoothing =
|
|
1750
|
+
| boolean
|
|
1751
|
+
| {
|
|
1752
|
+
/** Vertex merge tolerance used before recomputing mesh normals. Defaults to `1e-4`. */
|
|
1753
|
+
tolerance?: number;
|
|
1754
|
+
};
|
|
1755
|
+
|
|
1756
|
+
export interface MujocoRenderOptions {
|
|
1757
|
+
/**
|
|
1758
|
+
* Smooth mesh normals by welding duplicate vertices before recomputing normals.
|
|
1759
|
+
* Useful for faceted STL visuals; keep off for exact policy-render parity.
|
|
1760
|
+
*/
|
|
1761
|
+
meshNormalSmoothing?: MujocoMeshNormalSmoothing;
|
|
1762
|
+
}
|
|
1763
|
+
|
|
1657
1764
|
// ---- Canvas Props ----
|
|
1658
1765
|
|
|
1659
1766
|
export type MujocoCanvasProps = Omit<CanvasProps, 'onError'> & {
|
|
@@ -1671,6 +1778,7 @@ export type MujocoCanvasProps = Omit<CanvasProps, 'onError'> & {
|
|
|
1671
1778
|
paused?: boolean;
|
|
1672
1779
|
speed?: number;
|
|
1673
1780
|
interpolate?: boolean;
|
|
1781
|
+
renderOptions?: MujocoRenderOptions;
|
|
1674
1782
|
};
|
|
1675
1783
|
|
|
1676
1784
|
// ---- Hook Return Types ----
|