mujoco-react 12.1.1 → 12.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +200 -180
- package/dist/{chunk-XD3AY7XZ.js → chunk-SP72GTOD.js} +192 -92
- package/dist/chunk-SP72GTOD.js.map +1 -0
- package/dist/index.d.ts +2 -2
- package/dist/index.js +25 -3
- package/dist/index.js.map +1 -1
- package/dist/onnx.d.ts +1 -1
- package/dist/spark.d.ts +1 -1
- package/dist/spark.js +25 -16
- package/dist/spark.js.map +1 -1
- package/dist/{types-CSi3fIVs.d.ts → types-CF-BZXtW.d.ts} +32 -1
- package/package.json +2 -2
- package/src/core/MujocoSimProvider.tsx +9 -1
- package/src/hooks/useCameraFrameCapture.ts +17 -0
- package/src/hooks/useCameraStream.ts +1 -1
- package/src/index.ts +2 -0
- package/src/rendering/cameraFrameCapture.ts +141 -68
- package/src/rendering/metricDepth.ts +56 -0
- package/src/spark.tsx +24 -15
- package/src/types.ts +9 -0
- package/dist/chunk-XD3AY7XZ.js.map +0 -1
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import * as
|
|
1
|
+
import * as THREE2 from 'three';
|
|
2
2
|
import { useThree } from '@react-three/fiber';
|
|
3
3
|
import { useMemo, useEffect } from 'react';
|
|
4
4
|
import { jsxs, Fragment, jsx } from 'react/jsx-runtime';
|
|
@@ -174,6 +174,64 @@ async function dataUrlToPolicyImageTensor(dataUrl, options) {
|
|
|
174
174
|
resolved
|
|
175
175
|
);
|
|
176
176
|
}
|
|
177
|
+
function createMetricDepthReader(width, height, texture) {
|
|
178
|
+
const target = new THREE2.WebGLRenderTarget(width, height, { depthBuffer: false });
|
|
179
|
+
const material = new THREE2.ShaderMaterial({
|
|
180
|
+
uniforms: { depthTexture: { value: texture } },
|
|
181
|
+
vertexShader: `varying vec2 vUv;
|
|
182
|
+
void main() { vUv = uv; gl_Position = vec4(position.xy, 0.0, 1.0); }`,
|
|
183
|
+
fragmentShader: `#include <packing>
|
|
184
|
+
uniform sampler2D depthTexture;
|
|
185
|
+
varying vec2 vUv;
|
|
186
|
+
void main() { gl_FragColor = packDepthToRGBA(texture2D(depthTexture, vUv).r); }`,
|
|
187
|
+
depthTest: false,
|
|
188
|
+
depthWrite: false,
|
|
189
|
+
blending: THREE2.NoBlending,
|
|
190
|
+
toneMapped: false
|
|
191
|
+
});
|
|
192
|
+
const geometry = new THREE2.PlaneGeometry(2, 2);
|
|
193
|
+
const scene = new THREE2.Scene();
|
|
194
|
+
scene.add(new THREE2.Mesh(geometry, material));
|
|
195
|
+
const screenCamera = new THREE2.Camera();
|
|
196
|
+
const packed = new Uint8Array(width * height * 4);
|
|
197
|
+
return {
|
|
198
|
+
read(renderer, camera, flipX) {
|
|
199
|
+
renderer.setRenderTarget(target);
|
|
200
|
+
renderer.clear();
|
|
201
|
+
renderer.render(scene, screenCamera);
|
|
202
|
+
renderer.readRenderTargetPixels(target, 0, 0, width, height, packed);
|
|
203
|
+
const result = new Float32Array(width * height);
|
|
204
|
+
const inverse = camera.projectionMatrixInverse.elements;
|
|
205
|
+
for (let y = 0; y < height; y++) {
|
|
206
|
+
for (let x = 0; x < width; x++) {
|
|
207
|
+
const sx = flipX ? width - x - 1 : x;
|
|
208
|
+
const sy = height - y - 1;
|
|
209
|
+
const offset = (sy * width + sx) * 4;
|
|
210
|
+
const r = packed[offset], g = packed[offset + 1], b = packed[offset + 2], a = packed[offset + 3];
|
|
211
|
+
if (r === 255 && g === 255 && b === 255 && a === 255) {
|
|
212
|
+
result[y * width + x] = Infinity;
|
|
213
|
+
continue;
|
|
214
|
+
}
|
|
215
|
+
const d = r / 256 + g / 65536 + b / 16777216 + a / (255 * 16777216);
|
|
216
|
+
const nx = (sx + 0.5) / width * 2 - 1;
|
|
217
|
+
const ny = (sy + 0.5) / height * 2 - 1;
|
|
218
|
+
const nz = d * 2 - 1;
|
|
219
|
+
const z = inverse[2] * nx + inverse[6] * ny + inverse[10] * nz + inverse[14];
|
|
220
|
+
const w = inverse[3] * nx + inverse[7] * ny + inverse[11] * nz + inverse[15];
|
|
221
|
+
result[y * width + x] = -z / w;
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
return result;
|
|
225
|
+
},
|
|
226
|
+
dispose() {
|
|
227
|
+
target.dispose();
|
|
228
|
+
material.dispose();
|
|
229
|
+
geometry.dispose();
|
|
230
|
+
}
|
|
231
|
+
};
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
// src/rendering/cameraFrameCapture.ts
|
|
177
235
|
var CAMERA_FRAME_CAPTURE_RENDER_USER_DATA_KEY = "mujocoReactCameraFrameCaptureRender";
|
|
178
236
|
var CAMERA_FRAME_CAPTURE_PRE_RENDER_USER_DATA_KEY = "mujocoReactCameraFrameCapturePreRender";
|
|
179
237
|
var CAPTURE_EXCLUDE_KEY = "mujoco.capture.exclude";
|
|
@@ -229,7 +287,7 @@ function createIsolatedRenderer(sourceRenderer, width, height, options) {
|
|
|
229
287
|
function createUncachedIsolatedRenderer(sourceRenderer, width, height, options) {
|
|
230
288
|
const isolation = getRenderIsolationOptions(options);
|
|
231
289
|
const canvas = document.createElement("canvas");
|
|
232
|
-
const renderer = new
|
|
290
|
+
const renderer = new THREE2.WebGLRenderer({
|
|
233
291
|
canvas,
|
|
234
292
|
antialias: isolation.antialias ?? false,
|
|
235
293
|
alpha: isolation.alpha ?? false,
|
|
@@ -245,13 +303,13 @@ function createUncachedIsolatedRenderer(sourceRenderer, width, height, options)
|
|
|
245
303
|
}
|
|
246
304
|
function toVector3(value, fallback) {
|
|
247
305
|
if (!value) return fallback.clone();
|
|
248
|
-
return value instanceof
|
|
306
|
+
return value instanceof THREE2.Vector3 ? value.clone() : new THREE2.Vector3(value[0], value[1], value[2]);
|
|
249
307
|
}
|
|
250
308
|
function applyCameraPose(camera, options, fallbackCamera) {
|
|
251
309
|
camera.position.copy(toVector3(options.position, fallbackCamera.position));
|
|
252
310
|
camera.up.copy(toVector3(options.up, fallbackCamera.up));
|
|
253
311
|
if (options.quaternion) {
|
|
254
|
-
if (options.quaternion instanceof
|
|
312
|
+
if (options.quaternion instanceof THREE2.Quaternion) {
|
|
255
313
|
camera.quaternion.copy(options.quaternion);
|
|
256
314
|
} else {
|
|
257
315
|
camera.quaternion.set(
|
|
@@ -262,7 +320,7 @@ function applyCameraPose(camera, options, fallbackCamera) {
|
|
|
262
320
|
);
|
|
263
321
|
}
|
|
264
322
|
} else if (options.lookAt) {
|
|
265
|
-
camera.lookAt(toVector3(options.lookAt, new
|
|
323
|
+
camera.lookAt(toVector3(options.lookAt, new THREE2.Vector3()));
|
|
266
324
|
} else {
|
|
267
325
|
camera.quaternion.copy(fallbackCamera.quaternion);
|
|
268
326
|
}
|
|
@@ -270,7 +328,7 @@ function applyCameraPose(camera, options, fallbackCamera) {
|
|
|
270
328
|
}
|
|
271
329
|
function applyProjectionMatrix(camera, projectionMatrix) {
|
|
272
330
|
if (!projectionMatrix) return;
|
|
273
|
-
if (projectionMatrix instanceof
|
|
331
|
+
if (projectionMatrix instanceof THREE2.Matrix4) {
|
|
274
332
|
camera.projectionMatrix.copy(projectionMatrix);
|
|
275
333
|
} else {
|
|
276
334
|
camera.projectionMatrix.fromArray(projectionMatrix);
|
|
@@ -278,14 +336,19 @@ function applyProjectionMatrix(camera, projectionMatrix) {
|
|
|
278
336
|
camera.projectionMatrixInverse.copy(camera.projectionMatrix).invert();
|
|
279
337
|
}
|
|
280
338
|
function createCaptureCamera(options, fallbackCamera, width, height) {
|
|
281
|
-
const camera = options.camera ? options.camera.clone() : fallbackCamera instanceof
|
|
282
|
-
if (camera instanceof
|
|
339
|
+
const camera = options.camera ? options.camera.clone() : fallbackCamera instanceof THREE2.PerspectiveCamera ? fallbackCamera.clone() : new THREE2.PerspectiveCamera(45, width / height, 0.01, 100);
|
|
340
|
+
if (camera instanceof THREE2.PerspectiveCamera) {
|
|
283
341
|
camera.aspect = width / height;
|
|
284
342
|
camera.fov = options.fov ?? camera.fov;
|
|
285
343
|
camera.near = options.near ?? camera.near;
|
|
286
344
|
camera.far = options.far ?? camera.far;
|
|
287
345
|
camera.updateProjectionMatrix();
|
|
288
346
|
}
|
|
347
|
+
if (camera instanceof THREE2.OrthographicCamera) {
|
|
348
|
+
camera.near = options.near ?? camera.near;
|
|
349
|
+
camera.far = options.far ?? camera.far;
|
|
350
|
+
camera.updateProjectionMatrix();
|
|
351
|
+
}
|
|
289
352
|
applyProjectionMatrix(camera, options.projectionMatrix);
|
|
290
353
|
applyCameraPose(camera, options, fallbackCamera);
|
|
291
354
|
return camera;
|
|
@@ -305,20 +368,25 @@ function prepareCaptureCamera(camera, options, fallbackCamera, width, height) {
|
|
|
305
368
|
if (options.camera) {
|
|
306
369
|
camera.copy(options.camera);
|
|
307
370
|
}
|
|
308
|
-
if (camera instanceof
|
|
371
|
+
if (camera instanceof THREE2.PerspectiveCamera) {
|
|
309
372
|
camera.aspect = width / height;
|
|
310
373
|
camera.fov = options.fov ?? camera.fov;
|
|
311
374
|
camera.near = options.near ?? camera.near;
|
|
312
375
|
camera.far = options.far ?? camera.far;
|
|
313
376
|
camera.updateProjectionMatrix();
|
|
314
377
|
}
|
|
378
|
+
if (camera instanceof THREE2.OrthographicCamera) {
|
|
379
|
+
camera.near = options.near ?? camera.near;
|
|
380
|
+
camera.far = options.far ?? camera.far;
|
|
381
|
+
camera.updateProjectionMatrix();
|
|
382
|
+
}
|
|
315
383
|
applyProjectionMatrix(camera, options.projectionMatrix);
|
|
316
384
|
applyCameraPose(camera, options, fallbackCamera);
|
|
317
385
|
}
|
|
318
386
|
function readRenderTargetToCanvas(renderer, target, canvas, context, pixels, imageData, width, height, outputColorSpace, flipX = false, readback = true) {
|
|
319
387
|
if (readback) renderer.readRenderTargetPixels(target, 0, 0, width, height, pixels);
|
|
320
388
|
const rowBytes = width * 4;
|
|
321
|
-
const encodeSrgb = outputColorSpace ===
|
|
389
|
+
const encodeSrgb = outputColorSpace === THREE2.SRGBColorSpace;
|
|
322
390
|
for (let y = 0; y < height; y += 1) {
|
|
323
391
|
const sourceStart = (height - y - 1) * rowBytes;
|
|
324
392
|
const targetStart = y * rowBytes;
|
|
@@ -421,9 +489,9 @@ function getCameraFrameCaptureSource(options) {
|
|
|
421
489
|
return { kind: "fallback-camera" };
|
|
422
490
|
}
|
|
423
491
|
function saveRendererState(renderer) {
|
|
424
|
-
const viewport = new
|
|
425
|
-
const scissor = new
|
|
426
|
-
const clearColor = new
|
|
492
|
+
const viewport = new THREE2.Vector4();
|
|
493
|
+
const scissor = new THREE2.Vector4();
|
|
494
|
+
const clearColor = new THREE2.Color();
|
|
427
495
|
renderer.getViewport(viewport);
|
|
428
496
|
renderer.getScissor(scissor);
|
|
429
497
|
renderer.getClearColor(clearColor);
|
|
@@ -474,7 +542,7 @@ function applyCaptureVisualOverrides(renderer, scene, options) {
|
|
|
474
542
|
const previousSceneState = saveSceneVisualState(scene);
|
|
475
543
|
if (hasOwn(overrides, "sceneBackground")) {
|
|
476
544
|
const background = overrides.sceneBackground;
|
|
477
|
-
scene.background = background === false ? null : typeof background === "string" || typeof background === "number" ? new
|
|
545
|
+
scene.background = background === false ? null : typeof background === "string" || typeof background === "number" ? new THREE2.Color(background) : background ?? null;
|
|
478
546
|
}
|
|
479
547
|
if (hasOwn(overrides, "sceneEnvironment")) {
|
|
480
548
|
const environment = overrides.sceneEnvironment;
|
|
@@ -517,9 +585,9 @@ function createCameraFrameCaptureSession(renderer, scene, fallbackCamera, option
|
|
|
517
585
|
const isolatedRenderer = createIsolatedRenderer(renderer, width, height, options);
|
|
518
586
|
const sessionRenderer = isolatedRenderer?.renderer ?? renderer;
|
|
519
587
|
const camera = createCaptureCamera(options, fallbackCamera, width, height);
|
|
520
|
-
const target = new
|
|
521
|
-
format:
|
|
522
|
-
type:
|
|
588
|
+
const target = new THREE2.WebGLRenderTarget(width, height, {
|
|
589
|
+
format: THREE2.RGBAFormat,
|
|
590
|
+
type: THREE2.UnsignedByteType
|
|
523
591
|
});
|
|
524
592
|
const canvas = document.createElement("canvas");
|
|
525
593
|
canvas.width = width;
|
|
@@ -532,12 +600,14 @@ function createCameraFrameCaptureSession(renderer, scene, fallbackCamera, option
|
|
|
532
600
|
const drawContext = context;
|
|
533
601
|
const pixels = new Uint8Array(width * height * 4);
|
|
534
602
|
const imageData = drawContext.createImageData(width, height);
|
|
603
|
+
let depthReader;
|
|
535
604
|
let inFlight = false, disposed = false;
|
|
536
605
|
function assertAvailable() {
|
|
537
606
|
if (disposed) throw new Error("Camera capture session is disposed");
|
|
538
607
|
if (inFlight) throw new Error("Camera capture session is busy; await the current capture");
|
|
539
608
|
}
|
|
540
609
|
function releaseResources() {
|
|
610
|
+
depthReader?.dispose();
|
|
541
611
|
target.dispose();
|
|
542
612
|
if (isolatedRenderer && !isolatedRenderer.cached) isolatedRenderer.renderer.dispose();
|
|
543
613
|
}
|
|
@@ -563,42 +633,41 @@ function createCameraFrameCaptureSession(renderer, scene, fallbackCamera, option
|
|
|
563
633
|
);
|
|
564
634
|
return captureOptions;
|
|
565
635
|
}
|
|
566
|
-
function
|
|
636
|
+
function withCaptureState(captureOptions, render) {
|
|
567
637
|
const previousState = saveRendererState(sessionRenderer);
|
|
568
|
-
const previousSceneState = applyCaptureVisualOverrides(
|
|
569
|
-
sessionRenderer,
|
|
570
|
-
scene,
|
|
571
|
-
captureOptions
|
|
572
|
-
);
|
|
638
|
+
const previousSceneState = applyCaptureVisualOverrides(sessionRenderer, scene, captureOptions);
|
|
573
639
|
const hidden = [
|
|
574
640
|
...hideExcludedCaptureObjects(scene),
|
|
575
641
|
...hideCaptureGeomGroups(scene, captureOptions)
|
|
576
642
|
];
|
|
577
|
-
runCapturePreRenderHooks(scene);
|
|
578
|
-
scene.updateMatrixWorld(true);
|
|
579
643
|
try {
|
|
644
|
+
runCapturePreRenderHooks(scene);
|
|
645
|
+
scene.updateMatrixWorld(true);
|
|
580
646
|
sessionRenderer.xr.enabled = false;
|
|
581
|
-
sessionRenderer.setRenderTarget(target);
|
|
582
|
-
sessionRenderer.setViewport(0, 0, width, height);
|
|
583
|
-
sessionRenderer.setScissor(0, 0, width, height);
|
|
584
|
-
sessionRenderer.setScissorTest(false);
|
|
585
647
|
if (captureOptions.background !== void 0) {
|
|
586
648
|
sessionRenderer.setClearColor(
|
|
587
|
-
new
|
|
649
|
+
new THREE2.Color(captureOptions.background),
|
|
588
650
|
captureOptions.backgroundAlpha ?? previousState.clearAlpha
|
|
589
651
|
);
|
|
590
652
|
} else if (captureOptions.backgroundAlpha !== void 0) {
|
|
591
653
|
sessionRenderer.setClearColor(previousState.clearColor, captureOptions.backgroundAlpha);
|
|
592
654
|
}
|
|
593
|
-
|
|
594
|
-
sessionRenderer.render(scene, camera);
|
|
595
|
-
readback();
|
|
655
|
+
render();
|
|
596
656
|
} finally {
|
|
597
657
|
restoreObjectVisibility(hidden);
|
|
598
658
|
if (previousSceneState) restoreSceneVisualState(scene, previousSceneState);
|
|
599
659
|
restoreRendererState(sessionRenderer, previousState);
|
|
600
660
|
}
|
|
601
661
|
}
|
|
662
|
+
function renderCaptureToTarget(captureOptions, readback) {
|
|
663
|
+
withCaptureState(captureOptions, () => {
|
|
664
|
+
sessionRenderer.setRenderTarget(target);
|
|
665
|
+
sessionRenderer.setScissorTest(false);
|
|
666
|
+
sessionRenderer.clear();
|
|
667
|
+
sessionRenderer.render(scene, camera);
|
|
668
|
+
readback();
|
|
669
|
+
});
|
|
670
|
+
}
|
|
602
671
|
function renderPreparedCapture(captureOptions) {
|
|
603
672
|
renderCaptureToTarget(captureOptions, () => {
|
|
604
673
|
readRenderTargetToCanvas(
|
|
@@ -640,6 +709,53 @@ function createCameraFrameCaptureSession(renderer, scene, fallbackCamera, option
|
|
|
640
709
|
source: getCameraFrameCaptureSource(captureOptions)
|
|
641
710
|
};
|
|
642
711
|
}
|
|
712
|
+
function captureRGBD(nextOptions = {}) {
|
|
713
|
+
assertAvailable();
|
|
714
|
+
const captureOptions = resolveCaptureOptions(nextOptions);
|
|
715
|
+
if (getCaptureRenderer(scene)) {
|
|
716
|
+
throw new Error("Metric RGB-D capture requires the standard Three renderer; custom capture renderers do not provide aligned depth.");
|
|
717
|
+
}
|
|
718
|
+
if (sessionRenderer.capabilities.logarithmicDepthBuffer || sessionRenderer.capabilities.reversedDepthBuffer) {
|
|
719
|
+
throw new Error("Metric RGB-D capture does not support logarithmic or reversed depth buffers.");
|
|
720
|
+
}
|
|
721
|
+
if (!camera.projectionMatrix.elements.every(Number.isFinite) || !Number.isFinite(camera.projectionMatrix.determinant()) || camera.projectionMatrix.determinant() === 0) {
|
|
722
|
+
throw new Error("Metric RGB-D capture requires a finite, invertible projection matrix.");
|
|
723
|
+
}
|
|
724
|
+
if (!depthReader) {
|
|
725
|
+
target.dispose();
|
|
726
|
+
target.depthTexture = new THREE2.DepthTexture(width, height, THREE2.UnsignedIntType);
|
|
727
|
+
depthReader = createMetricDepthReader(width, height, target.depthTexture);
|
|
728
|
+
}
|
|
729
|
+
const rgb = new Uint8Array(width * height * 3);
|
|
730
|
+
let depth;
|
|
731
|
+
renderCaptureToTarget(captureOptions, () => {
|
|
732
|
+
sessionRenderer.readRenderTargetPixels(target, 0, 0, width, height, pixels);
|
|
733
|
+
for (let y = 0; y < height; y++) {
|
|
734
|
+
for (let x = 0; x < width; x++) {
|
|
735
|
+
const sx = captureOptions.flipX ? width - x - 1 : x;
|
|
736
|
+
const src = ((height - y - 1) * width + sx) * 4;
|
|
737
|
+
const dst = (y * width + x) * 3;
|
|
738
|
+
for (let c = 0; c < 3; c++) {
|
|
739
|
+
rgb[dst + c] = linearByteToSrgbByte(pixels[src + c]);
|
|
740
|
+
}
|
|
741
|
+
}
|
|
742
|
+
}
|
|
743
|
+
depth = depthReader.read(sessionRenderer, camera, captureOptions.flipX ?? false);
|
|
744
|
+
});
|
|
745
|
+
return {
|
|
746
|
+
rgb,
|
|
747
|
+
depth,
|
|
748
|
+
width,
|
|
749
|
+
height,
|
|
750
|
+
source: getCameraFrameCaptureSource(captureOptions),
|
|
751
|
+
origin: "top-left",
|
|
752
|
+
depthUnits: "meters",
|
|
753
|
+
depthConvention: "camera-z",
|
|
754
|
+
flipX: captureOptions.flipX ?? false,
|
|
755
|
+
projectionMatrix: camera.projectionMatrix.toArray(),
|
|
756
|
+
cameraWorldMatrix: camera.matrixWorld.toArray()
|
|
757
|
+
};
|
|
758
|
+
}
|
|
643
759
|
function captureTensor(nextOptions = {}) {
|
|
644
760
|
const result = capturePixels(nextOptions);
|
|
645
761
|
const tensor = pixelsToPolicyImageTensor(pixels, {
|
|
@@ -669,63 +785,37 @@ function createCameraFrameCaptureSession(renderer, scene, fallbackCamera, option
|
|
|
669
785
|
scene.updateMatrixWorld(true);
|
|
670
786
|
const captureRenderer = getCaptureRenderer(scene);
|
|
671
787
|
if (captureRenderer) {
|
|
672
|
-
|
|
673
|
-
const
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
captureOptions
|
|
677
|
-
);
|
|
678
|
-
const hidden = [
|
|
679
|
-
...hideExcludedCaptureObjects(scene),
|
|
680
|
-
...hideCaptureGeomGroups(scene, captureOptions)
|
|
681
|
-
];
|
|
682
|
-
try {
|
|
683
|
-
sessionRenderer.xr.enabled = false;
|
|
684
|
-
if (captureOptions.background !== void 0) {
|
|
685
|
-
sessionRenderer.setClearColor(
|
|
686
|
-
new THREE.Color(captureOptions.background),
|
|
687
|
-
captureOptions.backgroundAlpha ?? previousState.clearAlpha
|
|
688
|
-
);
|
|
689
|
-
} else if (captureOptions.backgroundAlpha !== void 0) {
|
|
690
|
-
sessionRenderer.setClearColor(previousState.clearColor, captureOptions.backgroundAlpha);
|
|
691
|
-
}
|
|
692
|
-
const captureResult = await captureRenderer({
|
|
788
|
+
let pending;
|
|
789
|
+
const renderWithCaptureState = (render) => withCaptureState(captureOptions, render);
|
|
790
|
+
renderWithCaptureState(() => {
|
|
791
|
+
pending = captureRenderer({
|
|
693
792
|
renderer: sessionRenderer,
|
|
694
793
|
scene,
|
|
695
794
|
camera,
|
|
696
795
|
target,
|
|
697
796
|
width,
|
|
698
|
-
height
|
|
797
|
+
height,
|
|
798
|
+
withCaptureState: renderWithCaptureState,
|
|
799
|
+
renderMode: captureOptions.renderMode ?? "snapshot"
|
|
699
800
|
});
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
}
|
|
708
|
-
readPixelsToCanvas(
|
|
709
|
-
captureResult.pixels,
|
|
710
|
-
drawContext,
|
|
711
|
-
imageData,
|
|
712
|
-
width,
|
|
713
|
-
height,
|
|
714
|
-
captureResult.flipY ?? true,
|
|
715
|
-
captureResult.flipX ?? captureOptions.flipX ?? false
|
|
716
|
-
);
|
|
717
|
-
return {
|
|
718
|
-
canvas,
|
|
719
|
-
camera,
|
|
720
|
-
width,
|
|
721
|
-
height,
|
|
722
|
-
source: getCameraFrameCaptureSource(captureOptions)
|
|
723
|
-
};
|
|
801
|
+
});
|
|
802
|
+
const captureResult = await pending;
|
|
803
|
+
if (captureResult) {
|
|
804
|
+
const captureWidth = captureResult.width ?? width;
|
|
805
|
+
const captureHeight = captureResult.height ?? height;
|
|
806
|
+
if (captureWidth !== width || captureHeight !== height) {
|
|
807
|
+
throw new Error("Camera frame capture renderer returned unexpected dimensions.");
|
|
724
808
|
}
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
809
|
+
readPixelsToCanvas(
|
|
810
|
+
captureResult.pixels,
|
|
811
|
+
drawContext,
|
|
812
|
+
imageData,
|
|
813
|
+
width,
|
|
814
|
+
height,
|
|
815
|
+
captureResult.flipY ?? true,
|
|
816
|
+
captureResult.flipX ?? captureOptions.flipX ?? false
|
|
817
|
+
);
|
|
818
|
+
return { canvas, camera, width, height, source: getCameraFrameCaptureSource(captureOptions) };
|
|
729
819
|
}
|
|
730
820
|
}
|
|
731
821
|
let readback;
|
|
@@ -755,6 +845,7 @@ function createCameraFrameCaptureSession(renderer, scene, fallbackCamera, option
|
|
|
755
845
|
captureAsync,
|
|
756
846
|
capturePixels,
|
|
757
847
|
captureTensor,
|
|
848
|
+
captureRGBD,
|
|
758
849
|
captureDataUrl(nextOptions = {}) {
|
|
759
850
|
const type = nextOptions.type ?? options.type ?? "image/png";
|
|
760
851
|
const result = capture(nextOptions);
|
|
@@ -859,6 +950,14 @@ function captureCameraFrameTensor(renderer, scene, fallbackCamera, options = {})
|
|
|
859
950
|
session.dispose();
|
|
860
951
|
}
|
|
861
952
|
}
|
|
953
|
+
function captureCameraRGBD(renderer, scene, fallbackCamera, options = {}) {
|
|
954
|
+
const session = createCameraFrameCaptureSession(renderer, scene, fallbackCamera, options);
|
|
955
|
+
try {
|
|
956
|
+
return session.captureRGBD();
|
|
957
|
+
} finally {
|
|
958
|
+
session.dispose();
|
|
959
|
+
}
|
|
960
|
+
}
|
|
862
961
|
var DEFAULT_BACKGROUND = "#181a1f";
|
|
863
962
|
function ScenarioLighting({
|
|
864
963
|
preset = "studio",
|
|
@@ -1032,7 +1131,7 @@ function useVisualScenarioEffects({
|
|
|
1032
1131
|
gl.toneMappingExposure = scenario.camera?.exposure ?? 1;
|
|
1033
1132
|
}
|
|
1034
1133
|
if (applyBackground) {
|
|
1035
|
-
scene.background = new
|
|
1134
|
+
scene.background = new THREE2.Color(
|
|
1036
1135
|
background ?? getScenarioBackground(scenario.lighting)
|
|
1037
1136
|
);
|
|
1038
1137
|
}
|
|
@@ -1372,21 +1471,21 @@ function SplatPlaceholder() {
|
|
|
1372
1471
|
transparent: true,
|
|
1373
1472
|
opacity: 0.06,
|
|
1374
1473
|
wireframe: true,
|
|
1375
|
-
side:
|
|
1474
|
+
side: THREE2.DoubleSide
|
|
1376
1475
|
}
|
|
1377
1476
|
)
|
|
1378
1477
|
] }) });
|
|
1379
1478
|
}
|
|
1380
1479
|
function createScenarioFog(scenario, background, fogNear, fogFar) {
|
|
1381
1480
|
if (scenario.lighting === "low-light") {
|
|
1382
|
-
return new
|
|
1481
|
+
return new THREE2.Fog(
|
|
1383
1482
|
background ?? getScenarioBackground(scenario.lighting),
|
|
1384
1483
|
fogNear ?? 2.5,
|
|
1385
1484
|
fogFar ?? 9
|
|
1386
1485
|
);
|
|
1387
1486
|
}
|
|
1388
1487
|
if (scenario.lighting === "warehouse") {
|
|
1389
|
-
return new
|
|
1488
|
+
return new THREE2.Fog(
|
|
1390
1489
|
background ?? getScenarioBackground(scenario.lighting),
|
|
1391
1490
|
fogNear ?? 5,
|
|
1392
1491
|
fogFar ?? 16
|
|
@@ -1398,7 +1497,7 @@ function applyScenarioMaterials(scene, scenario, snapshots, materialFilter) {
|
|
|
1398
1497
|
const materials = scenario.materials;
|
|
1399
1498
|
if (!materials) return;
|
|
1400
1499
|
scene.traverse((object) => {
|
|
1401
|
-
if (!(object instanceof
|
|
1500
|
+
if (!(object instanceof THREE2.Mesh)) {
|
|
1402
1501
|
return;
|
|
1403
1502
|
}
|
|
1404
1503
|
for (const material of normalizeMaterials(object.material)) {
|
|
@@ -1437,7 +1536,7 @@ function normalizeMaterials(material) {
|
|
|
1437
1536
|
return Array.isArray(material) ? material : [material];
|
|
1438
1537
|
}
|
|
1439
1538
|
function getMutableScenarioMaterial(material) {
|
|
1440
|
-
if (material instanceof
|
|
1539
|
+
if (material instanceof THREE2.MeshStandardMaterial || material instanceof THREE2.MeshPhysicalMaterial) {
|
|
1441
1540
|
return material;
|
|
1442
1541
|
}
|
|
1443
1542
|
return null;
|
|
@@ -1463,6 +1562,7 @@ function clamp01(value) {
|
|
|
1463
1562
|
*
|
|
1464
1563
|
* Helpers for turning browser camera captures into policy image tensors.
|
|
1465
1564
|
*/
|
|
1565
|
+
/** @license SPDX-License-Identifier: Apache-2.0 */
|
|
1466
1566
|
/**
|
|
1467
1567
|
* @license
|
|
1468
1568
|
* SPDX-License-Identifier: Apache-2.0
|
|
@@ -1470,6 +1570,6 @@ function clamp01(value) {
|
|
|
1470
1570
|
* Offscreen camera-frame capture for R3F/MuJoCo scenes.
|
|
1471
1571
|
*/
|
|
1472
1572
|
|
|
1473
|
-
export { CAMERA_FRAME_CAPTURE_PRE_RENDER_USER_DATA_KEY, CAMERA_FRAME_CAPTURE_RENDER_USER_DATA_KEY, CAPTURE_EXCLUDE_KEY, ModelActuators, ModelBodies, ModelCameras, ModelGeoms, ModelJoints, ModelKeyframes, ModelResources, ModelSensors, ModelSites, ScenarioLighting, SplatEnvironment, SplatEnvironmentReadinessStatus, VisualScenarioEffects, captureCameraFrame, captureCameraFrameBlob, captureCameraFrameTensor, createCameraFrameCaptureSession, createCaptureCamera, createPairedSplatEnvironment, createSparkSplatViewerUrl, createSplatEnvironmentUserData, createSplatSceneConfig, createVisualScenarioExecutionContext, dataUrlToPolicyImageTensor, getContact, getScenarioBackground, getScenarioCameraPosition, getSplatEnvironmentReadiness, imageDataToPolicyImageTensor, pixelsToPolicyImageTensor, prepareCaptureCamera, registerModelResources, renderCameraFrameToCanvas, useSplatEnvironment, useSplatSceneConfig, useVisualScenarioEffects, useVisualScenarioExecutionContext, withContacts, withSplatEnvironment };
|
|
1474
|
-
//# sourceMappingURL=chunk-
|
|
1475
|
-
//# sourceMappingURL=chunk-
|
|
1573
|
+
export { CAMERA_FRAME_CAPTURE_PRE_RENDER_USER_DATA_KEY, CAMERA_FRAME_CAPTURE_RENDER_USER_DATA_KEY, CAPTURE_EXCLUDE_KEY, ModelActuators, ModelBodies, ModelCameras, ModelGeoms, ModelJoints, ModelKeyframes, ModelResources, ModelSensors, ModelSites, ScenarioLighting, SplatEnvironment, SplatEnvironmentReadinessStatus, VisualScenarioEffects, captureCameraFrame, captureCameraFrameBlob, captureCameraFrameTensor, captureCameraRGBD, createCameraFrameCaptureSession, createCaptureCamera, createPairedSplatEnvironment, createSparkSplatViewerUrl, createSplatEnvironmentUserData, createSplatSceneConfig, createVisualScenarioExecutionContext, dataUrlToPolicyImageTensor, getContact, getScenarioBackground, getScenarioCameraPosition, getSplatEnvironmentReadiness, imageDataToPolicyImageTensor, pixelsToPolicyImageTensor, prepareCaptureCamera, registerModelResources, renderCameraFrameToCanvas, useSplatEnvironment, useSplatSceneConfig, useVisualScenarioEffects, useVisualScenarioExecutionContext, withContacts, withSplatEnvironment };
|
|
1574
|
+
//# sourceMappingURL=chunk-SP72GTOD.js.map
|
|
1575
|
+
//# sourceMappingURL=chunk-SP72GTOD.js.map
|