mujoco-react 12.2.1 → 12.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -0
- package/dist/{chunk-OIMH4U5U.js → chunk-FVSVETBC.js} +8 -3
- package/dist/{chunk-OIMH4U5U.js.map → chunk-FVSVETBC.js.map} +1 -1
- package/dist/index.d.ts +3 -3
- package/dist/index.js +467 -214
- package/dist/index.js.map +1 -1
- package/dist/onnx.d.ts +1 -1
- package/dist/spark.d.ts +1 -1
- package/dist/spark.js +1 -1
- package/dist/{types-CF-BZXtW.d.ts → types-1ONW9gUW.d.ts} +10 -1
- package/package.json +2 -2
- package/src/components/SceneRenderer.tsx +38 -16
- package/src/index.ts +1 -0
- package/src/rendering/GeomBuilder.ts +34 -14
- package/src/rendering/cameraFrameCapture.ts +9 -3
- package/src/rendering/mujocoClassicMaterial.ts +203 -0
- package/src/types.ts +7 -0
package/dist/onnx.d.ts
CHANGED
package/dist/spark.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import * as react_jsx_runtime from 'react/jsx-runtime';
|
|
2
2
|
import * as _sparkjsdev_spark from '@sparkjsdev/spark';
|
|
3
|
-
import { S as SplatEnvironmentProps, a as PairedSplatEnvironmentConfig, b as SceneConfig, c as SplatEnvironmentReadiness, V as VisualScenarioConfig } from './types-
|
|
3
|
+
import { S as SplatEnvironmentProps, a as PairedSplatEnvironmentConfig, b as SceneConfig, c as SplatEnvironmentReadiness, V as VisualScenarioConfig } from './types-1ONW9gUW.js';
|
|
4
4
|
import 'react';
|
|
5
5
|
import '@react-three/fiber';
|
|
6
6
|
import 'three';
|
package/dist/spark.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { useSplatSceneConfig, useSplatEnvironment, SplatEnvironment, CAMERA_FRAME_CAPTURE_RENDER_USER_DATA_KEY } from './chunk-
|
|
1
|
+
import { useSplatSceneConfig, useSplatEnvironment, SplatEnvironment, CAMERA_FRAME_CAPTURE_RENDER_USER_DATA_KEY } from './chunk-FVSVETBC.js';
|
|
2
2
|
import { useThree } from '@react-three/fiber';
|
|
3
3
|
import { useMemo, useState, useEffect, useCallback, useRef } from 'react';
|
|
4
4
|
import * as THREE from 'three';
|
|
@@ -126,8 +126,10 @@ interface CameraFramePixelsResult {
|
|
|
126
126
|
}
|
|
127
127
|
/** An owned, aligned RGB-D observation from one render. Scene units must be meters. */
|
|
128
128
|
interface CameraRGBDResult {
|
|
129
|
-
/** Interleaved
|
|
129
|
+
/** Interleaved RGB bytes, row-major, top-left origin. sRGB by default. */
|
|
130
130
|
rgb: Uint8Array;
|
|
131
|
+
/** Transfer function for RGB bytes. Missing on older observations means sRGB. */
|
|
132
|
+
rgbColorSpace?: 'srgb' | 'linear-srgb';
|
|
131
133
|
/** Positive camera-axis distance (-view Z) in meters, not Euclidean range.
|
|
132
134
|
* Background / no depth-writing surface is +Infinity. Same pixel order as rgb. */
|
|
133
135
|
depth: Float32Array;
|
|
@@ -1526,6 +1528,9 @@ interface CameraFrameRenderIsolationOptions {
|
|
|
1526
1528
|
cache?: boolean;
|
|
1527
1529
|
}
|
|
1528
1530
|
interface CameraFrameCaptureOptions {
|
|
1531
|
+
/** RGB-D byte transfer function. Defaults to sRGB. Linear readback preserves
|
|
1532
|
+
* unencoded target bytes, as used by MuJoCo's classic offscreen renderer. */
|
|
1533
|
+
rgbColorSpace?: 'srgb' | 'linear-srgb';
|
|
1529
1534
|
/** Snapshot (default) waits for splat preparation. Preview reuses the latest
|
|
1530
1535
|
* sorted splats while updating in the background for responsive live motion.
|
|
1531
1536
|
* useCameraStream defaults to preview; policy/dataset captures stay snapshot.
|
|
@@ -1718,6 +1723,10 @@ type MujocoMeshNormalSmoothing = boolean | {
|
|
|
1718
1723
|
tolerance?: number;
|
|
1719
1724
|
};
|
|
1720
1725
|
interface MujocoRenderOptions {
|
|
1726
|
+
/** Opt into classic MuJoCo material/lighting equations for native camera research.
|
|
1727
|
+
* Standard PBR rendering remains the default. Classic currently excludes
|
|
1728
|
+
* shadows, planar reflections, and non-spot/directional lights. */
|
|
1729
|
+
appearance?: 'standard' | 'mujoco-classic';
|
|
1721
1730
|
/**
|
|
1722
1731
|
* Smooth mesh normals by welding duplicate vertices before recomputing normals.
|
|
1723
1732
|
* Useful for faceted STL visuals; keep off for exact policy-render parity.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "mujoco-react",
|
|
3
|
-
"version": "12.
|
|
3
|
+
"version": "12.3.0",
|
|
4
4
|
"description": "Composable React Three Fiber building blocks for MuJoCo WASM simulations",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -66,7 +66,7 @@
|
|
|
66
66
|
"build": "tsup",
|
|
67
67
|
"dev": "tsup --watch",
|
|
68
68
|
"typecheck": "tsc --noEmit",
|
|
69
|
-
"test:runtime": "tsup --config tests/tsup.config.ts && node --test .test-dist/runtime.js .test-dist/simulation-runtime.js .test-dist/camera-capture.js"
|
|
69
|
+
"test:runtime": "tsup --config tests/tsup.config.ts && node --test .test-dist/runtime.js .test-dist/simulation-runtime.js .test-dist/camera-capture.js .test-dist/classic-appearance.js"
|
|
70
70
|
},
|
|
71
71
|
"peerDependencies": {
|
|
72
72
|
"@react-three/drei": ">=9",
|
|
@@ -23,9 +23,9 @@ export interface SceneRendererProps extends Omit<ThreeElements['group'], 'ref'>
|
|
|
23
23
|
|
|
24
24
|
function getRenderOptionsKey(renderOptions: MujocoRenderOptions | undefined) {
|
|
25
25
|
const smoothing = renderOptions?.meshNormalSmoothing;
|
|
26
|
-
if (!smoothing) return '
|
|
27
|
-
if (smoothing === true) return 'meshNormalSmoothing:true
|
|
28
|
-
return
|
|
26
|
+
if (!smoothing) return renderOptions?.appearance ?? 'standard';
|
|
27
|
+
if (smoothing === true) return `${renderOptions?.appearance ?? 'standard'}:meshNormalSmoothing:true`;
|
|
28
|
+
return `${renderOptions?.appearance ?? 'standard'}:meshNormalSmoothing:${smoothing.tolerance ?? 'default'}`;
|
|
29
29
|
}
|
|
30
30
|
|
|
31
31
|
export function SceneRenderer({ renderOptions, ...props }: SceneRendererProps) {
|
|
@@ -78,24 +78,46 @@ export function SceneRenderer({ renderOptions, ...props }: SceneRendererProps) {
|
|
|
78
78
|
|
|
79
79
|
// Create body groups with geometry
|
|
80
80
|
const refs: (THREE.Group | null)[] = [];
|
|
81
|
-
|
|
82
|
-
const
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
81
|
+
const dispose = () => {
|
|
82
|
+
for (const body of refs) {
|
|
83
|
+
body?.parent?.remove(body);
|
|
84
|
+
body?.traverse(object => {
|
|
85
|
+
const mesh = object as THREE.Mesh;
|
|
86
|
+
if (!mesh.isMesh) return;
|
|
87
|
+
mesh.geometry.dispose();
|
|
88
|
+
for (const material of Array.isArray(mesh.material) ? mesh.material : [mesh.material]) material.dispose();
|
|
89
|
+
});
|
|
90
|
+
}
|
|
91
|
+
geomBuilder.dispose();
|
|
92
|
+
prevModelRef.current = null;
|
|
93
|
+
prevRenderOptionsKeyRef.current = null;
|
|
94
|
+
prevHiddenVersionRef.current = null;
|
|
95
|
+
if (bodyRefs.current === refs) bodyRefs.current = [];
|
|
96
|
+
};
|
|
97
|
+
try {
|
|
98
|
+
for (let i = 0; i < model.nbody; i++) {
|
|
99
|
+
const bodyGroup = new THREE.Group();
|
|
100
|
+
refs.push(bodyGroup);
|
|
101
|
+
bodyGroup.userData.bodyID = i;
|
|
102
|
+
const bodyName = getName(model, model.name_bodyadr[i]);
|
|
103
|
+
|
|
104
|
+
if (!hiddenBodiesRef.current.has(bodyName)) {
|
|
105
|
+
for (let g = 0; g < model.ngeom; g++) {
|
|
106
|
+
if (model.geom_bodyid[g] === i) {
|
|
107
|
+
const mesh = geomBuilder.create(model, g, mjDataRef.current ?? undefined);
|
|
108
|
+
if (mesh) bodyGroup.add(mesh);
|
|
109
|
+
}
|
|
91
110
|
}
|
|
92
111
|
}
|
|
93
|
-
}
|
|
94
112
|
|
|
95
|
-
|
|
96
|
-
|
|
113
|
+
group.add(bodyGroup);
|
|
114
|
+
}
|
|
115
|
+
} catch (error) {
|
|
116
|
+
dispose();
|
|
117
|
+
throw error;
|
|
97
118
|
}
|
|
98
119
|
bodyRefs.current = refs;
|
|
120
|
+
return dispose;
|
|
99
121
|
}, [status, geomBuilder, mjModelRef, renderOptionsKey, hiddenBodiesRef, hiddenBodiesVersion]);
|
|
100
122
|
|
|
101
123
|
const syncBodiesToData = useCallback(() => {
|
package/src/index.ts
CHANGED
|
@@ -6,9 +6,10 @@
|
|
|
6
6
|
|
|
7
7
|
import * as THREE from 'three';
|
|
8
8
|
import { mergeVertices } from 'three/examples/jsm/utils/BufferGeometryUtils.js';
|
|
9
|
+
import { createMujocoClassicMaterial, createClassicPlaneGeometry } from './mujocoClassicMaterial';
|
|
9
10
|
import { CapsuleGeometry } from './CapsuleGeometry';
|
|
10
11
|
import { getName } from '../core/SceneLoader';
|
|
11
|
-
import { MujocoModel, MujocoModule, MujocoRenderOptions } from '../types';
|
|
12
|
+
import { MujocoModel, MujocoModule, MujocoRenderOptions, MujocoData } from '../types';
|
|
12
13
|
|
|
13
14
|
const DEFAULT_MESH_NORMAL_SMOOTHING_TOLERANCE = 1e-4;
|
|
14
15
|
|
|
@@ -93,7 +94,15 @@ export class GeomBuilder {
|
|
|
93
94
|
* Creates a Three.js Object3D (usually a Mesh) for a specific geometry in the MuJoCo model.
|
|
94
95
|
* Returns null if the geometry shouldn't be rendered (e.g., invisible collision triggers).
|
|
95
96
|
*/
|
|
96
|
-
|
|
97
|
+
dispose() {
|
|
98
|
+
for (const texture of this.textureCache.values()) texture.dispose();
|
|
99
|
+
this.textureCache.clear();
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
create(mjModel: MujocoModel, g: number, data?: MujocoData): THREE.Object3D | null {
|
|
103
|
+
const classic = this.renderOptions?.appearance === 'mujoco-classic';
|
|
104
|
+
if (classic && !data) throw new Error('Classic appearance requires live MuJoCo data');
|
|
105
|
+
if (classic && this.renderOptions?.meshNormalSmoothing) throw new Error('Classic appearance uses MuJoCo mesh normals; meshNormalSmoothing is not supported');
|
|
97
106
|
// 1. Check if this geom is meant to be visible
|
|
98
107
|
// Group 3 in MuJoCo is conventionally used for invisible 'helper' geoms.
|
|
99
108
|
if (mjModel.geom_group[g] === 3) return null;
|
|
@@ -108,7 +117,7 @@ export class GeomBuilder {
|
|
|
108
117
|
// Sometimes color is on the geom itself, sometimes it uses a shared material definition.
|
|
109
118
|
const matId = mjModel.geom_matid[g];
|
|
110
119
|
const color = new THREE.Color(0xffffff);
|
|
111
|
-
const map = this.getMaterialTexture(mjModel, matId);
|
|
120
|
+
const map = classic ? null : this.getMaterialTexture(mjModel, matId);
|
|
112
121
|
let opacity = 1.0;
|
|
113
122
|
|
|
114
123
|
if (matId >= 0) {
|
|
@@ -132,7 +141,7 @@ export class GeomBuilder {
|
|
|
132
141
|
|
|
133
142
|
if (type === getVal(MG.mjGEOM_PLANE)) {
|
|
134
143
|
// Planes are infinite in MuJoCo, but Three needs finite UVs for textured captures.
|
|
135
|
-
geo = new THREE.PlaneGeometry(size[0] * 2 || 5, size[1] * 2 || 5);
|
|
144
|
+
geo = classic ? createClassicPlaneGeometry(mjModel, g) : new THREE.PlaneGeometry(size[0] * 2 || 5, size[1] * 2 || 5);
|
|
136
145
|
} else if (type === getVal(MG.mjGEOM_SPHERE)) {
|
|
137
146
|
geo = new THREE.SphereGeometry(size[0], 24, 24);
|
|
138
147
|
} else if (type === getVal(MG.mjGEOM_CAPSULE)) {
|
|
@@ -156,27 +165,37 @@ export class GeomBuilder {
|
|
|
156
165
|
const fNum = mjModel.mesh_facenum[mId];
|
|
157
166
|
|
|
158
167
|
const tAdr = mjModel.mesh_texcoordadr?.[mId] ?? -1;
|
|
159
|
-
if (map && tAdr >= 0 && mjModel.mesh_texcoord && mjModel.mesh_facetexcoord) {
|
|
168
|
+
if (classic || (map && tAdr >= 0 && mjModel.mesh_texcoord && mjModel.mesh_facetexcoord)) {
|
|
160
169
|
// Textured mesh. MuJoCo indexes texcoords per face corner
|
|
161
170
|
// (mesh_facetexcoord), independently of the vertex indices, so
|
|
162
171
|
// the geometry must be de-indexed to carry a 'uv' attribute.
|
|
163
172
|
const corners = fNum * 3;
|
|
164
173
|
const positions = new Float32Array(corners * 3);
|
|
165
|
-
const
|
|
174
|
+
const hasUV = tAdr >= 0 && mjModel.mesh_texcoord && mjModel.mesh_facetexcoord;
|
|
175
|
+
const uvs = hasUV ? new Float32Array(corners * 2) : undefined;
|
|
176
|
+
const normalModel = mjModel as MujocoModel & { mesh_normaladr: Int32Array; mesh_facenormal: Int32Array };
|
|
177
|
+
const normals = classic ? new Float32Array(corners * 3) : undefined;
|
|
178
|
+
if (classic && (!normalModel.mesh_normaladr || !normalModel.mesh_facenormal)) throw new Error('Classic appearance requires MuJoCo mesh face normals');
|
|
166
179
|
for (let c = 0; c < corners; c++) {
|
|
167
180
|
const v = vAdr + mjModel.mesh_face[fAdr * 3 + c];
|
|
168
181
|
positions[c * 3] = mjModel.mesh_vert[v * 3];
|
|
169
182
|
positions[c * 3 + 1] = mjModel.mesh_vert[v * 3 + 1];
|
|
170
183
|
positions[c * 3 + 2] = mjModel.mesh_vert[v * 3 + 2];
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
184
|
+
if (uvs) {
|
|
185
|
+
const t = tAdr + mjModel.mesh_facetexcoord![fAdr * 3 + c];
|
|
186
|
+
uvs[c * 2] = mjModel.mesh_texcoord![t * 2];
|
|
187
|
+
uvs[c * 2 + 1] = 1 - mjModel.mesh_texcoord![t * 2 + 1];
|
|
188
|
+
}
|
|
189
|
+
if (normals) {
|
|
190
|
+
const n = normalModel.mesh_normaladr[mId] + normalModel.mesh_facenormal[fAdr * 3 + c];
|
|
191
|
+
normals.set(mjModel.mesh_normal.subarray(n * 3, n * 3 + 3), c * 3);
|
|
192
|
+
}
|
|
175
193
|
}
|
|
176
194
|
geo = new THREE.BufferGeometry();
|
|
177
195
|
geo.setAttribute('position', new THREE.Float32BufferAttribute(positions, 3));
|
|
178
|
-
geo.setAttribute('uv', new THREE.Float32BufferAttribute(uvs, 2));
|
|
179
|
-
geo.
|
|
196
|
+
if (uvs) geo.setAttribute('uv', new THREE.Float32BufferAttribute(uvs, 2));
|
|
197
|
+
if (normals) geo.setAttribute('normal', new THREE.Float32BufferAttribute(normals, 3));
|
|
198
|
+
else geo.computeVertexNormals();
|
|
180
199
|
} else {
|
|
181
200
|
geo = new THREE.BufferGeometry();
|
|
182
201
|
// 'position' attribute = vertices
|
|
@@ -194,12 +213,12 @@ export class GeomBuilder {
|
|
|
194
213
|
// 5. Construct the final Mesh
|
|
195
214
|
if (geo) {
|
|
196
215
|
const isPlane = type === getVal(MG.mjGEOM_PLANE);
|
|
197
|
-
const materialMap = isPlane && map ? map.clone() : map;
|
|
216
|
+
const materialMap = !classic && isPlane && map ? map.clone() : map;
|
|
198
217
|
if (isPlane && materialMap) {
|
|
199
218
|
materialMap.repeat.multiplyScalar(2.5);
|
|
200
219
|
materialMap.needsUpdate = true;
|
|
201
220
|
}
|
|
202
|
-
const mesh = new THREE.Mesh(geo, new THREE.MeshStandardMaterial({
|
|
221
|
+
const mesh = new THREE.Mesh(geo, classic ? createMujocoClassicMaterial(mjModel, data!, g) : new THREE.MeshStandardMaterial({
|
|
203
222
|
color,
|
|
204
223
|
map: materialMap,
|
|
205
224
|
transparent: opacity < 1,
|
|
@@ -207,6 +226,7 @@ export class GeomBuilder {
|
|
|
207
226
|
roughness: 0.6,
|
|
208
227
|
metalness: 0
|
|
209
228
|
}));
|
|
229
|
+
if (materialMap && materialMap !== map) mesh.material.addEventListener('dispose', () => materialMap.dispose());
|
|
210
230
|
mesh.castShadow = type !== getVal(MG.mjGEOM_PLANE);
|
|
211
231
|
mesh.receiveShadow = true;
|
|
212
232
|
|
|
@@ -36,8 +36,10 @@ export interface CameraFramePixelsResult {
|
|
|
36
36
|
|
|
37
37
|
/** An owned, aligned RGB-D observation from one render. Scene units must be meters. */
|
|
38
38
|
export interface CameraRGBDResult {
|
|
39
|
-
/** Interleaved
|
|
39
|
+
/** Interleaved RGB bytes, row-major, top-left origin. sRGB by default. */
|
|
40
40
|
rgb: Uint8Array;
|
|
41
|
+
/** Transfer function for RGB bytes. Missing on older observations means sRGB. */
|
|
42
|
+
rgbColorSpace?: 'srgb' | 'linear-srgb';
|
|
41
43
|
/** Positive camera-axis distance (-view Z) in meters, not Euclidean range.
|
|
42
44
|
* Background / no depth-writing surface is +Infinity. Same pixel order as rgb. */
|
|
43
45
|
depth: Float32Array;
|
|
@@ -826,6 +828,10 @@ export function createCameraFrameCaptureSession(
|
|
|
826
828
|
target.depthTexture = new THREE.DepthTexture(width, height, THREE.UnsignedIntType);
|
|
827
829
|
depthReader = createMetricDepthReader(width, height, target.depthTexture);
|
|
828
830
|
}
|
|
831
|
+
const rgbColorSpace = nextOptions.rgbColorSpace ?? options.rgbColorSpace ?? 'srgb';
|
|
832
|
+
if (rgbColorSpace !== 'srgb' && rgbColorSpace !== 'linear-srgb') {
|
|
833
|
+
throw new Error('RGB-D capture requires srgb or linear-srgb RGB bytes');
|
|
834
|
+
}
|
|
829
835
|
const rgb = new Uint8Array(width * height * 3);
|
|
830
836
|
let depth!: Float32Array;
|
|
831
837
|
renderCaptureToTarget(captureOptions, () => {
|
|
@@ -836,14 +842,14 @@ export function createCameraFrameCaptureSession(
|
|
|
836
842
|
const src = ((height - y - 1) * width + sx) * 4;
|
|
837
843
|
const dst = (y * width + x) * 3;
|
|
838
844
|
for (let c = 0; c < 3; c++) {
|
|
839
|
-
rgb[dst + c] = linearByteToSrgbByte(pixels[src + c]);
|
|
845
|
+
rgb[dst + c] = rgbColorSpace === 'srgb' ? linearByteToSrgbByte(pixels[src + c]) : pixels[src + c];
|
|
840
846
|
}
|
|
841
847
|
}
|
|
842
848
|
}
|
|
843
849
|
depth = depthReader!.read(sessionRenderer, camera, captureOptions.flipX ?? false);
|
|
844
850
|
});
|
|
845
851
|
return {
|
|
846
|
-
rgb, depth, width, height,
|
|
852
|
+
rgb, rgbColorSpace, depth, width, height,
|
|
847
853
|
source: getCameraFrameCaptureSource(captureOptions),
|
|
848
854
|
origin: 'top-left', depthUnits: 'meters', depthConvention: 'camera-z',
|
|
849
855
|
flipX: captureOptions.flipX ?? false,
|
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
/** @license SPDX-License-Identifier: Apache-2.0 */
|
|
2
|
+
import * as THREE from 'three';
|
|
3
|
+
import type { MujocoData, MujocoModel } from '../types';
|
|
4
|
+
|
|
5
|
+
const MAX_LIGHTS = 8;
|
|
6
|
+
const vertexShader = `
|
|
7
|
+
uniform vec4 diffuse;
|
|
8
|
+
uniform float emission, specular, shininess;
|
|
9
|
+
uniform int lightCount;
|
|
10
|
+
uniform float lightMode[8];
|
|
11
|
+
uniform vec3 lightPos[8], lightDir[8], lightAmbient[8], lightDiffuse[8], lightSpecular[8], lightAttenuation[8];
|
|
12
|
+
uniform float lightCutoff[8], lightExponent[8];
|
|
13
|
+
uniform bool explicitUV;
|
|
14
|
+
uniform vec2 repeatScale;
|
|
15
|
+
uniform vec3 cubeScale;
|
|
16
|
+
varying vec3 cubeCoord;
|
|
17
|
+
varying vec4 litColor;
|
|
18
|
+
varying vec2 texCoord;
|
|
19
|
+
void main() {
|
|
20
|
+
vec4 p = modelViewMatrix * vec4(position, 1.0);
|
|
21
|
+
vec3 n = normalize(normalMatrix * normal);
|
|
22
|
+
vec3 v = normalize(-p.xyz);
|
|
23
|
+
vec3 rgb = diffuse.rgb * emission;
|
|
24
|
+
if (lightCount == 0) rgb += diffuse.rgb * 0.3;
|
|
25
|
+
for (int i=0; i<8; i++) {
|
|
26
|
+
if (i>=lightCount) break;
|
|
27
|
+
vec3 l;
|
|
28
|
+
float attenuation = 1.0;
|
|
29
|
+
if (lightMode[i] > 1.5) l = vec3(0.0, 0.0, 1.0);
|
|
30
|
+
else if (lightMode[i] > 0.5) l = normalize(-(viewMatrix * vec4(lightDir[i],0.0)).xyz);
|
|
31
|
+
else {
|
|
32
|
+
vec3 delta = (viewMatrix * vec4(lightPos[i],1.0)).xyz - p.xyz;
|
|
33
|
+
float distance = length(delta);
|
|
34
|
+
l = normalize(delta);
|
|
35
|
+
vec3 a = lightAttenuation[i];
|
|
36
|
+
attenuation = 1.0 / max(1e-8, a.x+a.y*distance+a.z*distance*distance);
|
|
37
|
+
vec3 direction = normalize((viewMatrix * vec4(lightDir[i],0.0)).xyz);
|
|
38
|
+
float spot = dot(-l,direction);
|
|
39
|
+
if(lightCutoff[i] >= -1.0) attenuation *= spot >= lightCutoff[i] ? (lightExponent[i] > 0.0 ? pow(max(0.0,spot),lightExponent[i]) : 1.0) : 0.0;
|
|
40
|
+
}
|
|
41
|
+
float lambert = max(0.0,dot(n,l));
|
|
42
|
+
vec3 h = normalize(l+v);
|
|
43
|
+
float shine = lambert > 0.0 && specular > 0.0 ? (shininess > 0.0 ? pow(max(0.0,dot(n,h)),shininess) : 1.0) : 0.0;
|
|
44
|
+
rgb += attenuation*(diffuse.rgb*(lightAmbient[i]+lambert*lightDiffuse[i])+specular*shine*lightSpecular[i]);
|
|
45
|
+
}
|
|
46
|
+
litColor = vec4(clamp(rgb,0.0,1.0),diffuse.a);
|
|
47
|
+
cubeCoord = position*cubeScale;
|
|
48
|
+
texCoord = explicitUV ? vec2(uv.x,1.0-uv.y) : vec2(.5*repeatScale.x*position.x-.5,-.5*repeatScale.y*position.y-.5);
|
|
49
|
+
gl_Position = projectionMatrix*p;
|
|
50
|
+
}`;
|
|
51
|
+
const fragmentShader = `
|
|
52
|
+
uniform bool useMap;
|
|
53
|
+
uniform sampler2D colorMap;
|
|
54
|
+
uniform bool useCube;
|
|
55
|
+
uniform samplerCube cubeMap;
|
|
56
|
+
varying vec3 cubeCoord;
|
|
57
|
+
varying vec4 litColor;
|
|
58
|
+
varying vec2 texCoord;
|
|
59
|
+
void main() {
|
|
60
|
+
gl_FragColor = litColor * (useMap ? texture2D(colorMap,texCoord) : (useCube ? textureCube(cubeMap,cubeCoord) : vec4(1.0)));
|
|
61
|
+
}`;
|
|
62
|
+
|
|
63
|
+
interface ClassicVisualModel {
|
|
64
|
+
vis: { headlight: { active: number; ambient: Float32Array; diffuse: Float32Array; specular: Float32Array } };
|
|
65
|
+
tex_type: Int32Array;
|
|
66
|
+
tex_colorspace: Int32Array;
|
|
67
|
+
mat_emission: Float32Array;
|
|
68
|
+
mat_specular: Float32Array;
|
|
69
|
+
mat_shininess: Float32Array;
|
|
70
|
+
light_ambient: Float32Array;
|
|
71
|
+
}
|
|
72
|
+
interface ClassicLightData { light_xpos: Float64Array; light_xdir: Float64Array }
|
|
73
|
+
|
|
74
|
+
const textureCache = new WeakMap<MujocoModel, Map<number, { texture: THREE.Texture; references: number }>>();
|
|
75
|
+
function acquireTexture(model: MujocoModel, id: number) {
|
|
76
|
+
const m = model as MujocoModel & ClassicVisualModel;
|
|
77
|
+
let cache = textureCache.get(model);
|
|
78
|
+
if (!cache) { cache = new Map(); textureCache.set(model, cache); }
|
|
79
|
+
let entry = cache.get(id);
|
|
80
|
+
if (!entry) {
|
|
81
|
+
const width = model.tex_width[id], height = model.tex_height[id], channels = model.tex_nchannel[id];
|
|
82
|
+
if (channels !== 3 && channels !== 4) throw new Error('Classic RGB textures require three or four channels');
|
|
83
|
+
const bytes = model.tex_data.subarray(model.tex_adr[id],model.tex_adr[id]+width*height*channels);
|
|
84
|
+
const rgba = new Uint8Array(width*height*4);
|
|
85
|
+
for(let i=0;i<width*height;i++) {
|
|
86
|
+
rgba.set(bytes.subarray(i*channels,i*channels+3),i*4);
|
|
87
|
+
rgba[i*4+3] = channels === 4 ? bytes[i*channels+3] : 255;
|
|
88
|
+
}
|
|
89
|
+
let texture: THREE.Texture;
|
|
90
|
+
if(m.tex_type[id] === 0) {
|
|
91
|
+
texture = new THREE.DataTexture(rgba,width,height,THREE.RGBAFormat);
|
|
92
|
+
texture.colorSpace = m.tex_colorspace[id] === 2 ? THREE.SRGBColorSpace : THREE.LinearSRGBColorSpace;
|
|
93
|
+
texture.wrapS = texture.wrapT = THREE.RepeatWrapping;
|
|
94
|
+
texture.magFilter = THREE.LinearFilter;
|
|
95
|
+
texture.minFilter = THREE.LinearMipmapLinearFilter;
|
|
96
|
+
texture.generateMipmaps = true;
|
|
97
|
+
} else if(m.tex_type[id] === 1) {
|
|
98
|
+
if(height !== 6*width) throw new Error('Classic cube texture must contain six square faces');
|
|
99
|
+
const faces = Array.from({length:6},(_,i)=>new THREE.DataTexture(rgba.subarray(i*width*width*4,(i+1)*width*width*4),width,width,THREE.RGBAFormat));
|
|
100
|
+
// WebGLTextures supports DataTexture cube faces; older @types/three
|
|
101
|
+
// narrows this constructor to DOM images only.
|
|
102
|
+
texture = new THREE.CubeTexture(faces as unknown as HTMLImageElement[]);
|
|
103
|
+
// MuJoCo's classic cube upload uses a linear RGB internal format.
|
|
104
|
+
texture.colorSpace = THREE.LinearSRGBColorSpace;
|
|
105
|
+
} else throw new Error('Classic materials require a 2D or cube RGB texture');
|
|
106
|
+
texture.flipY=false; texture.needsUpdate=true;
|
|
107
|
+
entry={texture,references:0}; cache.set(id,entry);
|
|
108
|
+
}
|
|
109
|
+
entry.references++;
|
|
110
|
+
const retained=entry;
|
|
111
|
+
return { texture:entry.texture, release() {
|
|
112
|
+
if(--retained.references===0) { retained.texture.dispose(); cache!.delete(id); }
|
|
113
|
+
}};
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/** Finite planes use the same grid spacing as MuJoCo's classic display lists. */
|
|
117
|
+
export function createClassicPlaneGeometry(model: MujocoModel, geom: number) {
|
|
118
|
+
const size = model.geom_size.subarray(geom*3,geom*3+3);
|
|
119
|
+
if(size[0]<=0 || size[1]<=0) throw new Error('Classic appearance currently requires finite plane extents');
|
|
120
|
+
const axis = (half:number) => {
|
|
121
|
+
const step=Math.max(size[2],half/98);
|
|
122
|
+
let count=Math.floor(half/step); count-=count%2;
|
|
123
|
+
const pad=half-count*step; count+=2;
|
|
124
|
+
const grid:number[]=[];
|
|
125
|
+
for(let i=0;i<count;i++) {
|
|
126
|
+
const left=i===0 ? -half : -half+pad+(i-1)*2*step;
|
|
127
|
+
const right=left+(i===0 || i===count-1 ? pad : 2*step);
|
|
128
|
+
grid[i]=left;grid[i+1]=Math.max(left,right);
|
|
129
|
+
}
|
|
130
|
+
return grid;
|
|
131
|
+
};
|
|
132
|
+
const x=axis(size[0]),y=axis(size[1]),positions:number[]=[],normals:number[]=[],indices:number[]=[];
|
|
133
|
+
for(const py of y)for(const px of x){positions.push(px,py,0);normals.push(0,0,1);}
|
|
134
|
+
for(let row=0;row<y.length-1;row++)for(let col=0;col<x.length-1;col++) {
|
|
135
|
+
const a=row*x.length+col,b=a+1,c=a+x.length,d=c+1;
|
|
136
|
+
indices.push(a,b,d,a,d,c);
|
|
137
|
+
}
|
|
138
|
+
const geometry=new THREE.BufferGeometry();
|
|
139
|
+
geometry.setAttribute('position',new THREE.Float32BufferAttribute(positions,3));
|
|
140
|
+
geometry.setAttribute('normal',new THREE.Float32BufferAttribute(normals,3));
|
|
141
|
+
geometry.setIndex(indices);return geometry;
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
/** Opt-in classic-OpenGL equations. Produces native display-code RGB values;
|
|
145
|
+
* request linear RGB-D readback for comparisons. Shadows and planar
|
|
146
|
+
* reflections are outside this material adapter's scope. */
|
|
147
|
+
export function createMujocoClassicMaterial(model: MujocoModel, data: MujocoData,
|
|
148
|
+
geom: number): THREE.ShaderMaterial {
|
|
149
|
+
const m = model as MujocoModel & ClassicVisualModel, d = data as MujocoData & ClassicLightData;
|
|
150
|
+
for (const name of ['tex_type','tex_colorspace','mat_emission','mat_specular','mat_shininess','light_ambient'] as const) {
|
|
151
|
+
if(!m[name] || typeof m[name].length !== 'number') throw new Error(`Classic appearance requires model.${name}`);
|
|
152
|
+
}
|
|
153
|
+
if(!m.vis?.headlight || !d.light_xpos || !d.light_xdir) throw new Error('Classic appearance requires MuJoCo visual/light state');
|
|
154
|
+
const mat = model.geom_matid[geom], rgba = Array.from(model.geom_rgba.slice(4*geom,4*geom+4));
|
|
155
|
+
if (mat >= 0 && rgba.every((v,i)=>v === [0.5,0.5,0.5,1][i])) rgba.splice(0,4,...model.mat_rgba.slice(4*mat,4*mat+4));
|
|
156
|
+
const roles = model.mat_texid.length / Math.max(1,model.mat_rgba.length/4);
|
|
157
|
+
const textureId = mat >= 0 ? model.mat_texid[mat*roles+1] : -1;
|
|
158
|
+
const retained = textureId >= 0 ? acquireTexture(model,textureId) : undefined;
|
|
159
|
+
const texture = retained?.texture ?? null;
|
|
160
|
+
const vectors = () => Array.from({length:MAX_LIGHTS},()=>new THREE.Vector3());
|
|
161
|
+
const uniforms = {
|
|
162
|
+
diffuse:{value:new THREE.Vector4(...rgba as [number,number,number,number])},
|
|
163
|
+
emission:{value:mat>=0?m.mat_emission[mat]:0}, specular:{value:mat>=0?m.mat_specular[mat]:.5},
|
|
164
|
+
shininess:{value:128*(mat>=0?m.mat_shininess[mat]:.5)},
|
|
165
|
+
lightCount:{value:0},lightMode:{value:new Float32Array(MAX_LIGHTS)},
|
|
166
|
+
lightPos:{value:vectors()},lightDir:{value:vectors()},lightAmbient:{value:vectors()},lightDiffuse:{value:vectors()},lightSpecular:{value:vectors()},lightAttenuation:{value:vectors()},
|
|
167
|
+
lightCutoff:{value:new Float32Array(MAX_LIGHTS)},lightExponent:{value:new Float32Array(MAX_LIGHTS)},
|
|
168
|
+
explicitUV:{value:Number(model.geom_type[geom])===7 && (model.mesh_texcoordadr?.[model.geom_dataid[geom]]??-1)>=0},
|
|
169
|
+
repeatScale:{value:new THREE.Vector2(mat>=0?model.mat_texrepeat[mat*2]:1,mat>=0?model.mat_texrepeat[mat*2+1]:1)},
|
|
170
|
+
useMap:{value:!!texture && m.tex_type[textureId]===0},colorMap:{value:texture && m.tex_type[textureId]===0 ? texture : null},
|
|
171
|
+
useCube:{value:!!texture && m.tex_type[textureId]===1},cubeMap:{value:texture && m.tex_type[textureId]===1 ? texture : null},
|
|
172
|
+
cubeScale:{value:new THREE.Vector3(1,1,1)},
|
|
173
|
+
};
|
|
174
|
+
if(mat>=0 && !model.mat_texuniform[mat]) {
|
|
175
|
+
const size=model.geom_size;
|
|
176
|
+
uniforms.repeatScale.value.x/=size[geom*3]||1;uniforms.repeatScale.value.y/=size[geom*3+1]||1;
|
|
177
|
+
}
|
|
178
|
+
const isMeshOrPlane=Number(model.geom_type[geom])===7 || Number(model.geom_type[geom])===0;
|
|
179
|
+
const uniform=mat>=0 && model.mat_texuniform[mat];
|
|
180
|
+
for(let i=0;i<3;i++) uniforms.cubeScale.value.setComponent(i,isMeshOrPlane ? (uniform ? model.geom_size[geom*3+i] : 1) : (uniform ? 1 : 1/(model.geom_size[geom*3+i]||1)));
|
|
181
|
+
const material=new THREE.ShaderMaterial({uniforms,vertexShader,fragmentShader,toneMapped:false,side:THREE.DoubleSide,transparent:rgba[3]<1,depthWrite:rgba[3]>=1});
|
|
182
|
+
const head=m.vis.headlight;
|
|
183
|
+
material.onBeforeRender=()=>{
|
|
184
|
+
let count=0;
|
|
185
|
+
const add=(mode:number,pos:ArrayLike<number>,dir:ArrayLike<number>,ambient:ArrayLike<number>,diffuse:ArrayLike<number>,specular:ArrayLike<number>,attenuation:ArrayLike<number>,cutoff:number,exponent:number)=>{
|
|
186
|
+
if(count>=MAX_LIGHTS)throw new Error('Classic appearance supports at most eight lights');
|
|
187
|
+
uniforms.lightMode.value[count]=mode;
|
|
188
|
+
for(const [uniform,value] of [[uniforms.lightPos,pos],[uniforms.lightDir,dir],[uniforms.lightAmbient,ambient],[uniforms.lightDiffuse,diffuse],[uniforms.lightSpecular,specular],[uniforms.lightAttenuation,attenuation]] as const)uniform.value[count].fromArray(value);
|
|
189
|
+
uniforms.lightCutoff.value[count]=cutoff >= 180 ? -2 : Math.cos(cutoff*Math.PI/180);uniforms.lightExponent.value[count]=exponent;count++;
|
|
190
|
+
};
|
|
191
|
+
if(head.active)add(2,[0,0,0],[0,0,-1],head.ambient,head.diffuse,head.specular,[1,0,0],180,0);
|
|
192
|
+
for(let i=0;i<model.nlight;i++)if(m.light_active[i]){
|
|
193
|
+
const type=m.light_type[i];
|
|
194
|
+
if(type!==0&&type!==1)throw new Error('Classic appearance requires spot or directional MJCF lights');
|
|
195
|
+
const slice=(a:ArrayLike<number>)=>[a[i*3],a[i*3+1],a[i*3+2]];
|
|
196
|
+
add(type,slice(d.light_xpos),slice(d.light_xdir),slice(m.light_ambient),slice(m.light_diffuse),slice(m.light_specular),slice(m.light_attenuation),m.light_cutoff[i],m.light_exponent[i]);
|
|
197
|
+
}
|
|
198
|
+
uniforms.lightCount.value=count;material.uniformsNeedUpdate=true;
|
|
199
|
+
};
|
|
200
|
+
let released=false;
|
|
201
|
+
material.addEventListener('dispose',()=>{if(!released){retained?.release();released=true;}});
|
|
202
|
+
return material;
|
|
203
|
+
}
|
package/src/types.ts
CHANGED
|
@@ -1680,6 +1680,9 @@ export interface CameraFrameRenderIsolationOptions {
|
|
|
1680
1680
|
}
|
|
1681
1681
|
|
|
1682
1682
|
export interface CameraFrameCaptureOptions {
|
|
1683
|
+
/** RGB-D byte transfer function. Defaults to sRGB. Linear readback preserves
|
|
1684
|
+
* unencoded target bytes, as used by MuJoCo's classic offscreen renderer. */
|
|
1685
|
+
rgbColorSpace?: 'srgb' | 'linear-srgb';
|
|
1683
1686
|
/** Snapshot (default) waits for splat preparation. Preview reuses the latest
|
|
1684
1687
|
* sorted splats while updating in the background for responsive live motion.
|
|
1685
1688
|
* useCameraStream defaults to preview; policy/dataset captures stay snapshot.
|
|
@@ -1883,6 +1886,10 @@ export type MujocoMeshNormalSmoothing =
|
|
|
1883
1886
|
};
|
|
1884
1887
|
|
|
1885
1888
|
export interface MujocoRenderOptions {
|
|
1889
|
+
/** Opt into classic MuJoCo material/lighting equations for native camera research.
|
|
1890
|
+
* Standard PBR rendering remains the default. Classic currently excludes
|
|
1891
|
+
* shadows, planar reflections, and non-spot/directional lights. */
|
|
1892
|
+
appearance?: 'standard' | 'mujoco-classic';
|
|
1886
1893
|
/**
|
|
1887
1894
|
* Smooth mesh normals by welding duplicate vertices before recomputing normals.
|
|
1888
1895
|
* Useful for faceted STL visuals; keep off for exact policy-render parity.
|