mujoco-react 9.1.0 → 9.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +121 -17
- package/dist/{chunk-33CV6HSV.js → chunk-T3GVZJ4F.js} +222 -8
- package/dist/chunk-T3GVZJ4F.js.map +1 -0
- package/dist/index.d.ts +198 -6
- package/dist/index.js +1109 -216
- package/dist/index.js.map +1 -1
- package/dist/spark.d.ts +24 -2
- package/dist/spark.js +89 -3
- package/dist/spark.js.map +1 -1
- package/dist/{types-C5gTvR7b.d.ts → types-oxbxOkAx.d.ts} +190 -2
- package/dist/vite.d.ts +1 -1
- package/dist/vite.js +6 -3
- package/dist/vite.js.map +1 -1
- package/package.json +1 -1
- package/src/components/VisualScenario.tsx +178 -1
- package/src/core/MujocoSimProvider.tsx +473 -11
- package/src/core/SceneLoader.ts +13 -0
- package/src/core/createController.tsx +6 -2
- package/src/hooks/useCameraFrameCapture.ts +94 -0
- package/src/hooks/useCameraSequenceRecorder.ts +59 -0
- package/src/hooks/useMountedCameraSequenceRecorder.ts +107 -0
- package/src/index.ts +67 -0
- package/src/rendering/cameraFrameCapture.ts +353 -0
- package/src/rendering/cameraFrameSource.ts +375 -0
- package/src/spark.tsx +144 -0
- package/src/types.ts +212 -2
- package/src/vite.ts +5 -2
- package/dist/chunk-33CV6HSV.js.map +0 -1
package/dist/index.js
CHANGED
|
@@ -1,10 +1,11 @@
|
|
|
1
|
-
|
|
1
|
+
import { withContacts, getContact } from './chunk-T3GVZJ4F.js';
|
|
2
|
+
export { RobotActuators, RobotBodies, RobotCameras, RobotGeoms, RobotJoints, RobotKeyframes, RobotResources, RobotSensors, RobotSites, ScenarioLighting, SplatEnvironment, SplatEnvironmentReadinessStatus, VisualScenarioEffects, createPairedSplatEnvironment, createSparkSplatViewerUrl, createSplatEnvironmentUserData, getContact, getScenarioBackground, getScenarioCameraPosition, getSplatEnvironmentReadiness, registerRobotResources, useSplatEnvironment, useSplatSceneConfig, useVisualScenarioEffects, withSplatEnvironment } from './chunk-T3GVZJ4F.js';
|
|
2
3
|
import loadMujoco from '@mujoco/mujoco';
|
|
3
4
|
import defaultMujocoWasmUrl from '@mujoco/mujoco/mujoco.wasm?url';
|
|
4
5
|
import { createContext, forwardRef, useEffect, useContext, useState, useRef, useCallback, useMemo, useLayoutEffect } from 'react';
|
|
5
6
|
import { jsx, jsxs } from 'react/jsx-runtime';
|
|
6
7
|
import { Canvas, useThree, useFrame } from '@react-three/fiber';
|
|
7
|
-
import * as
|
|
8
|
+
import * as THREE12 from 'three';
|
|
8
9
|
import { PivotControls } from '@react-three/drei';
|
|
9
10
|
|
|
10
11
|
var MujocoContext = createContext({
|
|
@@ -104,90 +105,16 @@ function MujocoProvider({
|
|
|
104
105
|
}
|
|
105
106
|
);
|
|
106
107
|
}
|
|
107
|
-
|
|
108
|
-
// src/types.ts
|
|
109
|
-
var runtimeRobotResources = {};
|
|
110
|
-
var REGISTER_RESOURCE_KEYS = ["actuators", "sensors", "bodies", "joints", "sites", "geoms", "keyframes"];
|
|
111
|
-
function createEmptyRuntimeResources() {
|
|
112
|
-
return {
|
|
113
|
-
actuators: {},
|
|
114
|
-
sensors: {},
|
|
115
|
-
bodies: {},
|
|
116
|
-
joints: {},
|
|
117
|
-
sites: {},
|
|
118
|
-
geoms: {},
|
|
119
|
-
keyframes: {}
|
|
120
|
-
};
|
|
121
|
-
}
|
|
122
|
-
function registerRobotResources(resources) {
|
|
123
|
-
for (const [robot, robotResources] of Object.entries(resources)) {
|
|
124
|
-
const existing = runtimeRobotResources[robot] ?? createEmptyRuntimeResources();
|
|
125
|
-
for (const key of REGISTER_RESOURCE_KEYS) {
|
|
126
|
-
existing[key] = { ...existing[key], ...robotResources[key] ?? {} };
|
|
127
|
-
}
|
|
128
|
-
runtimeRobotResources[robot] = existing;
|
|
129
|
-
}
|
|
130
|
-
}
|
|
131
|
-
function createResourceCategory(key) {
|
|
132
|
-
return new Proxy({}, {
|
|
133
|
-
get(_target, robot) {
|
|
134
|
-
if (typeof robot !== "string") return void 0;
|
|
135
|
-
return runtimeRobotResources[robot]?.[key] ?? {};
|
|
136
|
-
},
|
|
137
|
-
ownKeys() {
|
|
138
|
-
return Reflect.ownKeys(runtimeRobotResources);
|
|
139
|
-
},
|
|
140
|
-
getOwnPropertyDescriptor(_target, robot) {
|
|
141
|
-
if (typeof robot !== "string" || !(robot in runtimeRobotResources)) return void 0;
|
|
142
|
-
return { enumerable: true, configurable: true };
|
|
143
|
-
}
|
|
144
|
-
});
|
|
145
|
-
}
|
|
146
|
-
var RobotResources = new Proxy(runtimeRobotResources, {
|
|
147
|
-
get(target, robot) {
|
|
148
|
-
if (typeof robot !== "string") return void 0;
|
|
149
|
-
return target[robot] ?? createEmptyRuntimeResources();
|
|
150
|
-
},
|
|
151
|
-
ownKeys(target) {
|
|
152
|
-
return Reflect.ownKeys(target);
|
|
153
|
-
},
|
|
154
|
-
getOwnPropertyDescriptor(target, robot) {
|
|
155
|
-
if (typeof robot !== "string" || !(robot in target)) return void 0;
|
|
156
|
-
return { enumerable: true, configurable: true };
|
|
157
|
-
}
|
|
158
|
-
});
|
|
159
|
-
var RobotActuators = createResourceCategory("actuators");
|
|
160
|
-
var RobotSensors = createResourceCategory("sensors");
|
|
161
|
-
var RobotBodies = createResourceCategory("bodies");
|
|
162
|
-
var RobotJoints = createResourceCategory("joints");
|
|
163
|
-
var RobotSites = createResourceCategory("sites");
|
|
164
|
-
var RobotGeoms = createResourceCategory("geoms");
|
|
165
|
-
var RobotKeyframes = createResourceCategory("keyframes");
|
|
166
|
-
function getContact(contacts, i) {
|
|
167
|
-
try {
|
|
168
|
-
return contacts.get(i);
|
|
169
|
-
} catch {
|
|
170
|
-
return void 0;
|
|
171
|
-
}
|
|
172
|
-
}
|
|
173
|
-
function withContacts(data, read) {
|
|
174
|
-
const contacts = data.contact;
|
|
175
|
-
try {
|
|
176
|
-
return read(contacts);
|
|
177
|
-
} finally {
|
|
178
|
-
contacts.delete?.();
|
|
179
|
-
}
|
|
180
|
-
}
|
|
181
|
-
var CapsuleGeometry = class extends THREE11.BufferGeometry {
|
|
108
|
+
var CapsuleGeometry = class extends THREE12.BufferGeometry {
|
|
182
109
|
parameters;
|
|
183
110
|
constructor(radius = 1, length = 1, capSegments = 4, radialSegments = 8) {
|
|
184
111
|
super();
|
|
185
112
|
this.type = "CapsuleGeometry";
|
|
186
113
|
this.parameters = { radius, length, capSegments, radialSegments };
|
|
187
|
-
const path = new
|
|
114
|
+
const path = new THREE12.Path();
|
|
188
115
|
path.absarc(0, -length / 2, radius, Math.PI * 1.5, 0, false);
|
|
189
116
|
path.absarc(0, length / 2, radius, 0, Math.PI * 0.5, false);
|
|
190
|
-
const latheGeometry = new
|
|
117
|
+
const latheGeometry = new THREE12.LatheGeometry(path.getPoints(capSegments), radialSegments);
|
|
191
118
|
const self = this;
|
|
192
119
|
self.setIndex(latheGeometry.getIndex());
|
|
193
120
|
self.setAttribute("position", latheGeometry.getAttribute("position"));
|
|
@@ -195,27 +122,27 @@ var CapsuleGeometry = class extends THREE11.BufferGeometry {
|
|
|
195
122
|
self.setAttribute("uv", latheGeometry.getAttribute("uv"));
|
|
196
123
|
}
|
|
197
124
|
};
|
|
198
|
-
var Reflector = class extends
|
|
125
|
+
var Reflector = class extends THREE12.Mesh {
|
|
199
126
|
isReflector = true;
|
|
200
127
|
camera;
|
|
201
|
-
reflectorPlane = new
|
|
202
|
-
normal = new
|
|
203
|
-
reflectorWorldPosition = new
|
|
204
|
-
cameraWorldPosition = new
|
|
205
|
-
rotationMatrix = new
|
|
206
|
-
lookAtPosition = new
|
|
207
|
-
clipPlane = new
|
|
208
|
-
view = new
|
|
209
|
-
target = new
|
|
210
|
-
q = new
|
|
211
|
-
textureMatrix = new
|
|
128
|
+
reflectorPlane = new THREE12.Plane();
|
|
129
|
+
normal = new THREE12.Vector3();
|
|
130
|
+
reflectorWorldPosition = new THREE12.Vector3();
|
|
131
|
+
cameraWorldPosition = new THREE12.Vector3();
|
|
132
|
+
rotationMatrix = new THREE12.Matrix4();
|
|
133
|
+
lookAtPosition = new THREE12.Vector3(0, 0, -1);
|
|
134
|
+
clipPlane = new THREE12.Vector4();
|
|
135
|
+
view = new THREE12.Vector3();
|
|
136
|
+
target = new THREE12.Vector3();
|
|
137
|
+
q = new THREE12.Vector4();
|
|
138
|
+
textureMatrix = new THREE12.Matrix4();
|
|
212
139
|
virtualCamera;
|
|
213
140
|
renderTarget;
|
|
214
141
|
constructor(geometry, options = {}) {
|
|
215
142
|
super(geometry);
|
|
216
143
|
this.type = "Reflector";
|
|
217
|
-
this.camera = new
|
|
218
|
-
const color = options.color !== void 0 ? new
|
|
144
|
+
this.camera = new THREE12.PerspectiveCamera();
|
|
145
|
+
const color = options.color !== void 0 ? new THREE12.Color(options.color) : new THREE12.Color(8355711);
|
|
219
146
|
const textureWidth = options.textureWidth || 512;
|
|
220
147
|
const textureHeight = options.textureHeight || 512;
|
|
221
148
|
const clipBias = options.clipBias || 0;
|
|
@@ -223,11 +150,11 @@ var Reflector = class extends THREE11.Mesh {
|
|
|
223
150
|
const blendTexture = options.texture || void 0;
|
|
224
151
|
const mixStrength = options.mixStrength !== void 0 ? options.mixStrength : 0.25;
|
|
225
152
|
this.virtualCamera = this.camera;
|
|
226
|
-
this.renderTarget = new
|
|
153
|
+
this.renderTarget = new THREE12.WebGLRenderTarget(textureWidth, textureHeight, {
|
|
227
154
|
samples: multisample,
|
|
228
|
-
type:
|
|
155
|
+
type: THREE12.HalfFloatType
|
|
229
156
|
});
|
|
230
|
-
this.material = new
|
|
157
|
+
this.material = new THREE12.MeshPhysicalMaterial({
|
|
231
158
|
map: blendTexture,
|
|
232
159
|
color,
|
|
233
160
|
roughness: 0.5,
|
|
@@ -353,7 +280,7 @@ var GeomBuilder = class {
|
|
|
353
280
|
const pos = mjModel.geom_pos.subarray(g * 3, g * 3 + 3);
|
|
354
281
|
const quat = mjModel.geom_quat.subarray(g * 4, g * 4 + 4);
|
|
355
282
|
const matId = mjModel.geom_matid[g];
|
|
356
|
-
const color = new
|
|
283
|
+
const color = new THREE12.Color(16777215);
|
|
357
284
|
let opacity = 1;
|
|
358
285
|
if (matId >= 0) {
|
|
359
286
|
const rgba = mjModel.mat_rgba.subarray(matId * 4, matId * 4 + 4);
|
|
@@ -368,16 +295,16 @@ var GeomBuilder = class {
|
|
|
368
295
|
let geo = null;
|
|
369
296
|
const getVal = (v) => v?.value ?? v;
|
|
370
297
|
if (type === getVal(MG.mjGEOM_PLANE)) {
|
|
371
|
-
geo = new
|
|
298
|
+
geo = new THREE12.PlaneGeometry(size[0] * 2 || 5, size[1] * 2 || 5);
|
|
372
299
|
} else if (type === getVal(MG.mjGEOM_SPHERE)) {
|
|
373
|
-
geo = new
|
|
300
|
+
geo = new THREE12.SphereGeometry(size[0], 24, 24);
|
|
374
301
|
} else if (type === getVal(MG.mjGEOM_CAPSULE)) {
|
|
375
302
|
geo = new CapsuleGeometry(size[0], size[1] * 2, 24, 12);
|
|
376
303
|
geo.rotateX(Math.PI / 2);
|
|
377
304
|
} else if (type === getVal(MG.mjGEOM_BOX)) {
|
|
378
|
-
geo = new
|
|
305
|
+
geo = new THREE12.BoxGeometry(size[0] * 2, size[1] * 2, size[2] * 2);
|
|
379
306
|
} else if (type === getVal(MG.mjGEOM_CYLINDER)) {
|
|
380
|
-
geo = new
|
|
307
|
+
geo = new THREE12.CylinderGeometry(size[0], size[0], size[1] * 2, 24);
|
|
381
308
|
geo.rotateX(Math.PI / 2);
|
|
382
309
|
} else if (type === getVal(MG.mjGEOM_MESH)) {
|
|
383
310
|
const mId = mjModel.geom_dataid[g];
|
|
@@ -385,8 +312,8 @@ var GeomBuilder = class {
|
|
|
385
312
|
const vNum = mjModel.mesh_vertnum[mId];
|
|
386
313
|
const fAdr = mjModel.mesh_faceadr[mId];
|
|
387
314
|
const fNum = mjModel.mesh_facenum[mId];
|
|
388
|
-
geo = new
|
|
389
|
-
geo.setAttribute("position", new
|
|
315
|
+
geo = new THREE12.BufferGeometry();
|
|
316
|
+
geo.setAttribute("position", new THREE12.Float32BufferAttribute(mjModel.mesh_vert.subarray(vAdr * 3, (vAdr + vNum) * 3), 3));
|
|
390
317
|
geo.setIndex(Array.from(mjModel.mesh_face.subarray(fAdr * 3, (fAdr + fNum) * 3)));
|
|
391
318
|
geo.computeVertexNormals();
|
|
392
319
|
}
|
|
@@ -401,7 +328,7 @@ var GeomBuilder = class {
|
|
|
401
328
|
mixStrength: 0.25
|
|
402
329
|
});
|
|
403
330
|
} else {
|
|
404
|
-
mesh = new
|
|
331
|
+
mesh = new THREE12.Mesh(geo, new THREE12.MeshStandardMaterial({
|
|
405
332
|
color,
|
|
406
333
|
transparent: opacity < 1,
|
|
407
334
|
opacity,
|
|
@@ -480,6 +407,15 @@ function findSensorByName(mjModel, name) {
|
|
|
480
407
|
}
|
|
481
408
|
return -1;
|
|
482
409
|
}
|
|
410
|
+
function findCameraByName(mjModel, name) {
|
|
411
|
+
const ncam = mjModel.ncam ?? 0;
|
|
412
|
+
const addresses = mjModel.name_camadr;
|
|
413
|
+
if (!addresses) return -1;
|
|
414
|
+
for (let i = 0; i < ncam; i++) {
|
|
415
|
+
if (getName(mjModel, addresses[i]) === name) return i;
|
|
416
|
+
}
|
|
417
|
+
return -1;
|
|
418
|
+
}
|
|
483
419
|
function findTendonByName(mjModel, name) {
|
|
484
420
|
for (let i = 0; i < (mjModel.ntendon ?? 0); i++) {
|
|
485
421
|
if (getName(mjModel, mjModel.name_tendonadr[i]) === name) return i;
|
|
@@ -1206,7 +1142,7 @@ function SceneRenderer(props) {
|
|
|
1206
1142
|
}
|
|
1207
1143
|
const refs = [];
|
|
1208
1144
|
for (let i = 0; i < model.nbody; i++) {
|
|
1209
|
-
const bodyGroup = new
|
|
1145
|
+
const bodyGroup = new THREE12.Group();
|
|
1210
1146
|
bodyGroup.userData.bodyID = i;
|
|
1211
1147
|
const bodyName = getName(model, model.name_bodyadr[i]);
|
|
1212
1148
|
if (!hiddenBodiesRef.current.has(bodyName)) {
|
|
@@ -1235,9 +1171,9 @@ function SceneRenderer(props) {
|
|
|
1235
1171
|
const alpha = interpolation.alpha;
|
|
1236
1172
|
const i3 = i * 3;
|
|
1237
1173
|
ref.position.set(
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1174
|
+
THREE12.MathUtils.lerp(interpolation.previousXpos[i3], interpolation.currentXpos[i3], alpha),
|
|
1175
|
+
THREE12.MathUtils.lerp(interpolation.previousXpos[i3 + 1], interpolation.currentXpos[i3 + 1], alpha),
|
|
1176
|
+
THREE12.MathUtils.lerp(interpolation.previousXpos[i3 + 2], interpolation.currentXpos[i3 + 2], alpha)
|
|
1241
1177
|
);
|
|
1242
1178
|
const i4 = i * 4;
|
|
1243
1179
|
_previousQuat.set(
|
|
@@ -1292,8 +1228,8 @@ function SceneRenderer(props) {
|
|
|
1292
1228
|
}
|
|
1293
1229
|
);
|
|
1294
1230
|
}
|
|
1295
|
-
var _previousQuat = new
|
|
1296
|
-
var _currentQuat = new
|
|
1231
|
+
var _previousQuat = new THREE12.Quaternion();
|
|
1232
|
+
var _currentQuat = new THREE12.Quaternion();
|
|
1297
1233
|
function isTargetRef(target) {
|
|
1298
1234
|
return Boolean(target && typeof target === "object" && "current" in target);
|
|
1299
1235
|
}
|
|
@@ -1402,6 +1338,412 @@ function useFrameCapture(defaultOptions = {}) {
|
|
|
1402
1338
|
reset
|
|
1403
1339
|
};
|
|
1404
1340
|
}
|
|
1341
|
+
function toVector3(value, fallback) {
|
|
1342
|
+
if (!value) return fallback.clone();
|
|
1343
|
+
return value instanceof THREE12.Vector3 ? value.clone() : new THREE12.Vector3(value[0], value[1], value[2]);
|
|
1344
|
+
}
|
|
1345
|
+
function applyCameraPose(camera, options, fallbackCamera) {
|
|
1346
|
+
camera.position.copy(toVector3(options.position, fallbackCamera.position));
|
|
1347
|
+
camera.up.copy(toVector3(options.up, fallbackCamera.up));
|
|
1348
|
+
if (options.quaternion) {
|
|
1349
|
+
if (options.quaternion instanceof THREE12.Quaternion) {
|
|
1350
|
+
camera.quaternion.copy(options.quaternion);
|
|
1351
|
+
} else {
|
|
1352
|
+
camera.quaternion.set(
|
|
1353
|
+
options.quaternion[0],
|
|
1354
|
+
options.quaternion[1],
|
|
1355
|
+
options.quaternion[2],
|
|
1356
|
+
options.quaternion[3]
|
|
1357
|
+
);
|
|
1358
|
+
}
|
|
1359
|
+
} else if (options.lookAt) {
|
|
1360
|
+
camera.lookAt(toVector3(options.lookAt, new THREE12.Vector3()));
|
|
1361
|
+
} else {
|
|
1362
|
+
camera.quaternion.copy(fallbackCamera.quaternion);
|
|
1363
|
+
}
|
|
1364
|
+
camera.updateMatrixWorld();
|
|
1365
|
+
}
|
|
1366
|
+
function createCaptureCamera(options, fallbackCamera, width, height) {
|
|
1367
|
+
const camera = options.camera ? options.camera.clone() : fallbackCamera instanceof THREE12.PerspectiveCamera ? fallbackCamera.clone() : new THREE12.PerspectiveCamera(45, width / height, 0.01, 100);
|
|
1368
|
+
if (camera instanceof THREE12.PerspectiveCamera) {
|
|
1369
|
+
camera.aspect = width / height;
|
|
1370
|
+
camera.fov = options.fov ?? camera.fov;
|
|
1371
|
+
camera.near = options.near ?? camera.near;
|
|
1372
|
+
camera.far = options.far ?? camera.far;
|
|
1373
|
+
camera.updateProjectionMatrix();
|
|
1374
|
+
}
|
|
1375
|
+
applyCameraPose(camera, options, fallbackCamera);
|
|
1376
|
+
return camera;
|
|
1377
|
+
}
|
|
1378
|
+
function getCaptureDimensions(renderer, options) {
|
|
1379
|
+
const width = Math.max(
|
|
1380
|
+
1,
|
|
1381
|
+
Math.floor(options.width ?? renderer.domElement.width)
|
|
1382
|
+
);
|
|
1383
|
+
const height = Math.max(
|
|
1384
|
+
1,
|
|
1385
|
+
Math.floor(options.height ?? renderer.domElement.height)
|
|
1386
|
+
);
|
|
1387
|
+
return { width, height };
|
|
1388
|
+
}
|
|
1389
|
+
function prepareCaptureCamera(camera, options, fallbackCamera, width, height) {
|
|
1390
|
+
if (options.camera) {
|
|
1391
|
+
camera.copy(options.camera);
|
|
1392
|
+
}
|
|
1393
|
+
if (camera instanceof THREE12.PerspectiveCamera) {
|
|
1394
|
+
camera.aspect = width / height;
|
|
1395
|
+
camera.fov = options.fov ?? camera.fov;
|
|
1396
|
+
camera.near = options.near ?? camera.near;
|
|
1397
|
+
camera.far = options.far ?? camera.far;
|
|
1398
|
+
camera.updateProjectionMatrix();
|
|
1399
|
+
}
|
|
1400
|
+
applyCameraPose(camera, options, fallbackCamera);
|
|
1401
|
+
}
|
|
1402
|
+
function readRenderTargetToCanvas(renderer, target, canvas, context, pixels, imageData, width, height) {
|
|
1403
|
+
renderer.readRenderTargetPixels(target, 0, 0, width, height, pixels);
|
|
1404
|
+
const rowBytes = width * 4;
|
|
1405
|
+
for (let y = 0; y < height; y += 1) {
|
|
1406
|
+
const sourceStart = (height - y - 1) * rowBytes;
|
|
1407
|
+
const targetStart = y * rowBytes;
|
|
1408
|
+
imageData.data.set(
|
|
1409
|
+
pixels.subarray(sourceStart, sourceStart + rowBytes),
|
|
1410
|
+
targetStart
|
|
1411
|
+
);
|
|
1412
|
+
}
|
|
1413
|
+
context.putImageData(imageData, 0, 0);
|
|
1414
|
+
return canvas;
|
|
1415
|
+
}
|
|
1416
|
+
function getCameraFrameCaptureSource(options) {
|
|
1417
|
+
if (options.source) return options.source;
|
|
1418
|
+
if (options.cameraName) {
|
|
1419
|
+
return { kind: "mujoco-camera", cameraName: options.cameraName };
|
|
1420
|
+
}
|
|
1421
|
+
if (options.siteName) {
|
|
1422
|
+
return { kind: "mujoco-site", siteName: options.siteName };
|
|
1423
|
+
}
|
|
1424
|
+
if (options.bodyName) {
|
|
1425
|
+
return { kind: "mujoco-body", bodyName: options.bodyName };
|
|
1426
|
+
}
|
|
1427
|
+
if (options.camera) return { kind: "custom-camera" };
|
|
1428
|
+
if (options.position || options.lookAt || options.quaternion) {
|
|
1429
|
+
return { kind: "explicit-pose" };
|
|
1430
|
+
}
|
|
1431
|
+
return { kind: "fallback-camera" };
|
|
1432
|
+
}
|
|
1433
|
+
function createCameraFrameCaptureSession(renderer, scene, fallbackCamera, options = {}) {
|
|
1434
|
+
const { width, height } = getCaptureDimensions(renderer, options);
|
|
1435
|
+
const camera = createCaptureCamera(options, fallbackCamera, width, height);
|
|
1436
|
+
const target = new THREE12.WebGLRenderTarget(width, height, {
|
|
1437
|
+
format: THREE12.RGBAFormat,
|
|
1438
|
+
type: THREE12.UnsignedByteType
|
|
1439
|
+
});
|
|
1440
|
+
const canvas = document.createElement("canvas");
|
|
1441
|
+
canvas.width = width;
|
|
1442
|
+
canvas.height = height;
|
|
1443
|
+
const context = canvas.getContext("2d");
|
|
1444
|
+
if (!context) {
|
|
1445
|
+
target.dispose();
|
|
1446
|
+
throw new Error("Unable to create a 2D canvas for camera frame capture.");
|
|
1447
|
+
}
|
|
1448
|
+
const drawContext = context;
|
|
1449
|
+
const pixels = new Uint8Array(width * height * 4);
|
|
1450
|
+
const imageData = drawContext.createImageData(width, height);
|
|
1451
|
+
function capture(nextOptions = {}) {
|
|
1452
|
+
const captureOptions = { ...options, ...nextOptions };
|
|
1453
|
+
const nextDimensions = getCaptureDimensions(renderer, captureOptions);
|
|
1454
|
+
if (nextDimensions.width !== width || nextDimensions.height !== height) {
|
|
1455
|
+
throw new Error(
|
|
1456
|
+
"Camera frame capture sessions require stable width and height."
|
|
1457
|
+
);
|
|
1458
|
+
}
|
|
1459
|
+
prepareCaptureCamera(
|
|
1460
|
+
camera,
|
|
1461
|
+
captureOptions,
|
|
1462
|
+
fallbackCamera,
|
|
1463
|
+
width,
|
|
1464
|
+
height
|
|
1465
|
+
);
|
|
1466
|
+
const previousTarget = renderer.getRenderTarget();
|
|
1467
|
+
const previousXrEnabled = renderer.xr.enabled;
|
|
1468
|
+
scene.updateMatrixWorld(true);
|
|
1469
|
+
try {
|
|
1470
|
+
renderer.xr.enabled = false;
|
|
1471
|
+
renderer.setRenderTarget(target);
|
|
1472
|
+
renderer.clear();
|
|
1473
|
+
renderer.render(scene, camera);
|
|
1474
|
+
readRenderTargetToCanvas(
|
|
1475
|
+
renderer,
|
|
1476
|
+
target,
|
|
1477
|
+
canvas,
|
|
1478
|
+
drawContext,
|
|
1479
|
+
pixels,
|
|
1480
|
+
imageData,
|
|
1481
|
+
width,
|
|
1482
|
+
height
|
|
1483
|
+
);
|
|
1484
|
+
return {
|
|
1485
|
+
canvas,
|
|
1486
|
+
camera,
|
|
1487
|
+
width,
|
|
1488
|
+
height,
|
|
1489
|
+
source: getCameraFrameCaptureSource(captureOptions)
|
|
1490
|
+
};
|
|
1491
|
+
} finally {
|
|
1492
|
+
renderer.setRenderTarget(previousTarget);
|
|
1493
|
+
renderer.xr.enabled = previousXrEnabled;
|
|
1494
|
+
}
|
|
1495
|
+
}
|
|
1496
|
+
return {
|
|
1497
|
+
width,
|
|
1498
|
+
height,
|
|
1499
|
+
capture,
|
|
1500
|
+
captureDataUrl(nextOptions = {}) {
|
|
1501
|
+
const type = nextOptions.type ?? options.type ?? "image/png";
|
|
1502
|
+
const result = capture(nextOptions);
|
|
1503
|
+
return {
|
|
1504
|
+
...result,
|
|
1505
|
+
dataUrl: result.canvas.toDataURL(
|
|
1506
|
+
type,
|
|
1507
|
+
nextOptions.quality ?? options.quality
|
|
1508
|
+
),
|
|
1509
|
+
type
|
|
1510
|
+
};
|
|
1511
|
+
},
|
|
1512
|
+
async captureBlob(nextOptions = {}) {
|
|
1513
|
+
const type = nextOptions.type ?? options.type ?? "image/png";
|
|
1514
|
+
const result = capture(nextOptions);
|
|
1515
|
+
const blob = await new Promise((resolve, reject) => {
|
|
1516
|
+
result.canvas.toBlob(
|
|
1517
|
+
(nextBlob) => {
|
|
1518
|
+
if (nextBlob) resolve(nextBlob);
|
|
1519
|
+
else reject(new Error("Camera frame capture did not produce a Blob."));
|
|
1520
|
+
},
|
|
1521
|
+
type,
|
|
1522
|
+
nextOptions.quality ?? options.quality
|
|
1523
|
+
);
|
|
1524
|
+
});
|
|
1525
|
+
return { ...result, blob, type };
|
|
1526
|
+
},
|
|
1527
|
+
dispose() {
|
|
1528
|
+
target.dispose();
|
|
1529
|
+
}
|
|
1530
|
+
};
|
|
1531
|
+
}
|
|
1532
|
+
function renderCameraFrameToCanvas(renderer, scene, fallbackCamera, options = {}) {
|
|
1533
|
+
const session = createCameraFrameCaptureSession(
|
|
1534
|
+
renderer,
|
|
1535
|
+
scene,
|
|
1536
|
+
fallbackCamera,
|
|
1537
|
+
options
|
|
1538
|
+
);
|
|
1539
|
+
try {
|
|
1540
|
+
return session.capture();
|
|
1541
|
+
} finally {
|
|
1542
|
+
session.dispose();
|
|
1543
|
+
}
|
|
1544
|
+
}
|
|
1545
|
+
async function captureCameraFrame(renderer, scene, fallbackCamera, options = {}) {
|
|
1546
|
+
const type = options.type ?? "image/png";
|
|
1547
|
+
const result = renderCameraFrameToCanvas(
|
|
1548
|
+
renderer,
|
|
1549
|
+
scene,
|
|
1550
|
+
fallbackCamera,
|
|
1551
|
+
options
|
|
1552
|
+
);
|
|
1553
|
+
return {
|
|
1554
|
+
...result,
|
|
1555
|
+
dataUrl: result.canvas.toDataURL(type, options.quality),
|
|
1556
|
+
type
|
|
1557
|
+
};
|
|
1558
|
+
}
|
|
1559
|
+
async function captureCameraFrameBlob(renderer, scene, fallbackCamera, options = {}) {
|
|
1560
|
+
const type = options.type ?? "image/png";
|
|
1561
|
+
const result = renderCameraFrameToCanvas(
|
|
1562
|
+
renderer,
|
|
1563
|
+
scene,
|
|
1564
|
+
fallbackCamera,
|
|
1565
|
+
options
|
|
1566
|
+
);
|
|
1567
|
+
const blob = await new Promise((resolve, reject) => {
|
|
1568
|
+
result.canvas.toBlob(
|
|
1569
|
+
(nextBlob) => {
|
|
1570
|
+
if (nextBlob) resolve(nextBlob);
|
|
1571
|
+
else reject(new Error("Camera frame capture did not produce a Blob."));
|
|
1572
|
+
},
|
|
1573
|
+
type,
|
|
1574
|
+
options.quality
|
|
1575
|
+
);
|
|
1576
|
+
});
|
|
1577
|
+
return { ...result, blob, type };
|
|
1578
|
+
}
|
|
1579
|
+
|
|
1580
|
+
// src/rendering/cameraFrameSource.ts
|
|
1581
|
+
var MountedCameraFrameSequenceReadinessStatus = {
|
|
1582
|
+
Ready: "ready",
|
|
1583
|
+
Partial: "partial",
|
|
1584
|
+
Missing: "missing"
|
|
1585
|
+
};
|
|
1586
|
+
function getResourceName(resource) {
|
|
1587
|
+
if (!resource) return null;
|
|
1588
|
+
return typeof resource === "string" ? resource : resource.name ?? null;
|
|
1589
|
+
}
|
|
1590
|
+
function createNameSet(resources) {
|
|
1591
|
+
return new Set(
|
|
1592
|
+
(resources ?? []).map((resource) => getResourceName(resource)).filter((name) => Boolean(name))
|
|
1593
|
+
);
|
|
1594
|
+
}
|
|
1595
|
+
function normalizeAliasCandidates(value) {
|
|
1596
|
+
if (!value) return [];
|
|
1597
|
+
return Array.isArray(value) ? value : [value];
|
|
1598
|
+
}
|
|
1599
|
+
function countMountedSelectors(selector) {
|
|
1600
|
+
return Number(Boolean(selector.cameraName)) + Number(Boolean(selector.siteName)) + Number(Boolean(selector.bodyName));
|
|
1601
|
+
}
|
|
1602
|
+
function getMountedCameraFrameCaptureSource(selector) {
|
|
1603
|
+
if (countMountedSelectors(selector) !== 1) return null;
|
|
1604
|
+
if (selector.cameraName) {
|
|
1605
|
+
return { kind: "mujoco-camera", cameraName: selector.cameraName };
|
|
1606
|
+
}
|
|
1607
|
+
if (selector.siteName) {
|
|
1608
|
+
return { kind: "mujoco-site", siteName: selector.siteName };
|
|
1609
|
+
}
|
|
1610
|
+
if (selector.bodyName) {
|
|
1611
|
+
return { kind: "mujoco-body", bodyName: selector.bodyName };
|
|
1612
|
+
}
|
|
1613
|
+
return null;
|
|
1614
|
+
}
|
|
1615
|
+
function isMountedCameraFrameCaptureSource(source) {
|
|
1616
|
+
return source.kind === "mujoco-camera" || source.kind === "mujoco-site" || source.kind === "mujoco-body";
|
|
1617
|
+
}
|
|
1618
|
+
function getCameraFrameCaptureSourceTarget(source) {
|
|
1619
|
+
if (source.kind === "mujoco-camera") return source.cameraName;
|
|
1620
|
+
if (source.kind === "mujoco-site") return source.siteName;
|
|
1621
|
+
if (source.kind === "mujoco-body") return source.bodyName;
|
|
1622
|
+
if (source.kind === "custom-camera") return "custom camera";
|
|
1623
|
+
if (source.kind === "explicit-pose") return "explicit pose";
|
|
1624
|
+
return "fallback camera";
|
|
1625
|
+
}
|
|
1626
|
+
function isSelectorMounted(selector, cameraNames, siteNames, bodyNames) {
|
|
1627
|
+
if (countMountedSelectors(selector) !== 1) return false;
|
|
1628
|
+
return (selector.cameraName ? cameraNames.has(selector.cameraName) : false) || (selector.siteName ? siteNames.has(selector.siteName) : false) || (selector.bodyName ? bodyNames.has(selector.bodyName) : false);
|
|
1629
|
+
}
|
|
1630
|
+
function resolveMountedCameraFrameSource(key, options) {
|
|
1631
|
+
const cameraNames = createNameSet(options.cameras);
|
|
1632
|
+
const siteNames = createNameSet(options.sites);
|
|
1633
|
+
const bodyNames = createNameSet(options.bodies);
|
|
1634
|
+
const directCandidates = [
|
|
1635
|
+
{ cameraName: key },
|
|
1636
|
+
{ siteName: key },
|
|
1637
|
+
{ bodyName: key }
|
|
1638
|
+
];
|
|
1639
|
+
const aliasCandidates = normalizeAliasCandidates(options.aliases?.[key]);
|
|
1640
|
+
const candidates = [...directCandidates, ...aliasCandidates];
|
|
1641
|
+
for (const selector of candidates) {
|
|
1642
|
+
if (!isSelectorMounted(selector, cameraNames, siteNames, bodyNames)) {
|
|
1643
|
+
continue;
|
|
1644
|
+
}
|
|
1645
|
+
const source = getMountedCameraFrameCaptureSource(selector);
|
|
1646
|
+
if (!source) continue;
|
|
1647
|
+
return { key, selector, source };
|
|
1648
|
+
}
|
|
1649
|
+
if (options.allowAliasFallback) {
|
|
1650
|
+
for (const selector of aliasCandidates) {
|
|
1651
|
+
const source = getMountedCameraFrameCaptureSource(selector);
|
|
1652
|
+
if (!source) continue;
|
|
1653
|
+
return { key, selector, source };
|
|
1654
|
+
}
|
|
1655
|
+
}
|
|
1656
|
+
return null;
|
|
1657
|
+
}
|
|
1658
|
+
function createMountedCameraFrameSequencePlan(cameraKeys, options) {
|
|
1659
|
+
const cameras = [];
|
|
1660
|
+
const resolved = {};
|
|
1661
|
+
const missingKeys = [];
|
|
1662
|
+
for (const key of cameraKeys) {
|
|
1663
|
+
const mountedSource = resolveMountedCameraFrameSource(key, options);
|
|
1664
|
+
if (!mountedSource) {
|
|
1665
|
+
missingKeys.push(key);
|
|
1666
|
+
continue;
|
|
1667
|
+
}
|
|
1668
|
+
resolved[key] = mountedSource;
|
|
1669
|
+
cameras.push({
|
|
1670
|
+
key,
|
|
1671
|
+
...options.defaults,
|
|
1672
|
+
...options.cameraOptions?.[key],
|
|
1673
|
+
...mountedSource.selector,
|
|
1674
|
+
source: mountedSource.source
|
|
1675
|
+
});
|
|
1676
|
+
}
|
|
1677
|
+
if (options.requireAll && missingKeys.length > 0) {
|
|
1678
|
+
throw new Error(
|
|
1679
|
+
`Unable to resolve mounted MuJoCo camera source${missingKeys.length === 1 ? "" : "s"} for ${missingKeys.join(", ")}.`
|
|
1680
|
+
);
|
|
1681
|
+
}
|
|
1682
|
+
return {
|
|
1683
|
+
cameraKeys: [...cameraKeys],
|
|
1684
|
+
cameras,
|
|
1685
|
+
resolved,
|
|
1686
|
+
missingKeys
|
|
1687
|
+
};
|
|
1688
|
+
}
|
|
1689
|
+
function createMountedCameraFrameSequenceReadiness(plan) {
|
|
1690
|
+
const cameras = {};
|
|
1691
|
+
const resolvedKeys = plan.cameraKeys.filter((key) => Boolean(plan.resolved[key]));
|
|
1692
|
+
for (const key of plan.cameraKeys) {
|
|
1693
|
+
const resolved = plan.resolved[key];
|
|
1694
|
+
cameras[key] = resolved ? {
|
|
1695
|
+
key,
|
|
1696
|
+
ready: true,
|
|
1697
|
+
selector: resolved.selector,
|
|
1698
|
+
source: resolved.source,
|
|
1699
|
+
message: `Camera stream "${key}" resolves to ${resolved.source.kind}:${getCameraFrameCaptureSourceTarget(resolved.source)}.`
|
|
1700
|
+
} : {
|
|
1701
|
+
key,
|
|
1702
|
+
ready: false,
|
|
1703
|
+
message: `Camera stream "${key}" does not resolve to a mounted MuJoCo camera, site, or body.`
|
|
1704
|
+
};
|
|
1705
|
+
}
|
|
1706
|
+
const missingKeys = [...plan.missingKeys];
|
|
1707
|
+
const ready = missingKeys.length === 0;
|
|
1708
|
+
const status = ready ? MountedCameraFrameSequenceReadinessStatus.Ready : resolvedKeys.length > 0 ? MountedCameraFrameSequenceReadinessStatus.Partial : MountedCameraFrameSequenceReadinessStatus.Missing;
|
|
1709
|
+
return {
|
|
1710
|
+
ready,
|
|
1711
|
+
status,
|
|
1712
|
+
cameraKeys: [...plan.cameraKeys],
|
|
1713
|
+
resolvedKeys,
|
|
1714
|
+
missingKeys,
|
|
1715
|
+
cameras,
|
|
1716
|
+
message: ready ? `All ${plan.cameraKeys.length} requested camera stream${plan.cameraKeys.length === 1 ? "" : "s"} resolve to mounted MuJoCo sources.` : `Missing mounted MuJoCo source${missingKeys.length === 1 ? "" : "s"} for ${missingKeys.join(", ")}.`
|
|
1717
|
+
};
|
|
1718
|
+
}
|
|
1719
|
+
function createMountedCameraFrameSequencePlanFromApi(api, cameraKeys, options = {}) {
|
|
1720
|
+
return createMountedCameraFrameSequencePlan(cameraKeys, {
|
|
1721
|
+
...options,
|
|
1722
|
+
cameras: api.getCameras(),
|
|
1723
|
+
sites: api.getSites(),
|
|
1724
|
+
bodies: api.getBodies()
|
|
1725
|
+
});
|
|
1726
|
+
}
|
|
1727
|
+
async function recordMountedCameraFrameSequence(api, options) {
|
|
1728
|
+
const { cameraKeys, ...restOptions } = options;
|
|
1729
|
+
const requireAll = restOptions.requireAll ?? restOptions.requireMountedSources ?? true;
|
|
1730
|
+
const plan = createMountedCameraFrameSequencePlanFromApi(
|
|
1731
|
+
api,
|
|
1732
|
+
cameraKeys,
|
|
1733
|
+
{ ...restOptions, requireAll }
|
|
1734
|
+
);
|
|
1735
|
+
const readiness = createMountedCameraFrameSequenceReadiness(plan);
|
|
1736
|
+
const result = await api.recordCameraSequence({
|
|
1737
|
+
...restOptions,
|
|
1738
|
+
cameras: plan.cameras,
|
|
1739
|
+
requireMountedSources: restOptions.requireMountedSources ?? true
|
|
1740
|
+
});
|
|
1741
|
+
return {
|
|
1742
|
+
...result,
|
|
1743
|
+
plan,
|
|
1744
|
+
readiness
|
|
1745
|
+
};
|
|
1746
|
+
}
|
|
1405
1747
|
var JOINT_TYPE_NAMES2 = ["free", "ball", "slide", "hinge"];
|
|
1406
1748
|
var GEOM_TYPE_NAMES = ["plane", "hfield", "sphere", "capsule", "ellipsoid", "cylinder", "box", "mesh"];
|
|
1407
1749
|
var SENSOR_TYPE_NAMES = {
|
|
@@ -1477,8 +1819,79 @@ var _applyPoint = new Float64Array(3);
|
|
|
1477
1819
|
var _rayPnt = new Float64Array(3);
|
|
1478
1820
|
var _rayVec = new Float64Array(3);
|
|
1479
1821
|
var _rayGeomId = new Int32Array(1);
|
|
1480
|
-
var _projRaycaster = new
|
|
1481
|
-
var _projNdc = new
|
|
1822
|
+
var _projRaycaster = new THREE12.Raycaster();
|
|
1823
|
+
var _projNdc = new THREE12.Vector2();
|
|
1824
|
+
function waitForNextAnimationFrame2() {
|
|
1825
|
+
return new Promise((resolve) => {
|
|
1826
|
+
requestAnimationFrame(() => resolve());
|
|
1827
|
+
});
|
|
1828
|
+
}
|
|
1829
|
+
function throwIfCameraSequenceAborted(signal) {
|
|
1830
|
+
if (!signal?.aborted) return;
|
|
1831
|
+
if (typeof signal.reason === "object" && signal.reason instanceof Error) {
|
|
1832
|
+
throw signal.reason;
|
|
1833
|
+
}
|
|
1834
|
+
throw new DOMException("Camera sequence recording was aborted.", "AbortError");
|
|
1835
|
+
}
|
|
1836
|
+
function vector3FromArray(values, offset) {
|
|
1837
|
+
return [values[offset], values[offset + 1], values[offset + 2]];
|
|
1838
|
+
}
|
|
1839
|
+
function quaternionFromArray(values, offset) {
|
|
1840
|
+
return [
|
|
1841
|
+
values[offset],
|
|
1842
|
+
values[offset + 1],
|
|
1843
|
+
values[offset + 2],
|
|
1844
|
+
values[offset + 3]
|
|
1845
|
+
];
|
|
1846
|
+
}
|
|
1847
|
+
function quaternionFromXmat(values, offset) {
|
|
1848
|
+
const matrix = new THREE12.Matrix4();
|
|
1849
|
+
matrix.set(
|
|
1850
|
+
values[offset],
|
|
1851
|
+
values[offset + 1],
|
|
1852
|
+
values[offset + 2],
|
|
1853
|
+
0,
|
|
1854
|
+
values[offset + 3],
|
|
1855
|
+
values[offset + 4],
|
|
1856
|
+
values[offset + 5],
|
|
1857
|
+
0,
|
|
1858
|
+
values[offset + 6],
|
|
1859
|
+
values[offset + 7],
|
|
1860
|
+
values[offset + 8],
|
|
1861
|
+
0,
|
|
1862
|
+
0,
|
|
1863
|
+
0,
|
|
1864
|
+
0,
|
|
1865
|
+
1
|
|
1866
|
+
);
|
|
1867
|
+
const quaternion = new THREE12.Quaternion().setFromRotationMatrix(matrix);
|
|
1868
|
+
return [quaternion.x, quaternion.y, quaternion.z, quaternion.w];
|
|
1869
|
+
}
|
|
1870
|
+
function omitResolvedCameraSelectors(options) {
|
|
1871
|
+
const { cameraName, siteName, bodyName, ...rest } = options;
|
|
1872
|
+
return rest;
|
|
1873
|
+
}
|
|
1874
|
+
function countMountedCameraSelectors(options) {
|
|
1875
|
+
return Number(Boolean(options.cameraName)) + Number(Boolean(options.siteName)) + Number(Boolean(options.bodyName));
|
|
1876
|
+
}
|
|
1877
|
+
function assertMatchingMountedCameraSource(key, requested, source) {
|
|
1878
|
+
const selectorCount = countMountedCameraSelectors(requested);
|
|
1879
|
+
if (selectorCount !== 1) {
|
|
1880
|
+
throw new Error(
|
|
1881
|
+
`Camera sequence stream "${key}" must provide exactly one mounted MuJoCo cameraName, siteName, or bodyName selector.`
|
|
1882
|
+
);
|
|
1883
|
+
}
|
|
1884
|
+
if (!isMountedCameraFrameCaptureSource(source)) {
|
|
1885
|
+
throw new Error(
|
|
1886
|
+
`Camera sequence stream "${key}" resolved to ${source.kind}; use a MuJoCo-mounted camera, site, or body selector for sequence recording.`
|
|
1887
|
+
);
|
|
1888
|
+
}
|
|
1889
|
+
if (requested.cameraName && (source.kind !== "mujoco-camera" || source.cameraName !== requested.cameraName) || requested.siteName && (source.kind !== "mujoco-site" || source.siteName !== requested.siteName) || requested.bodyName && (source.kind !== "mujoco-body" || source.bodyName !== requested.bodyName)) {
|
|
1890
|
+
throw new Error(
|
|
1891
|
+
`Camera sequence stream "${key}" resolved to ${source.kind}:${getCameraFrameCaptureSourceTarget(source)} instead of the requested mounted selector.`
|
|
1892
|
+
);
|
|
1893
|
+
}
|
|
1894
|
+
}
|
|
1482
1895
|
var MujocoSimContext = createContext(null);
|
|
1483
1896
|
function useMujocoContext() {
|
|
1484
1897
|
const ctx = useContext(MujocoSimContext);
|
|
@@ -1563,7 +1976,7 @@ function MujocoSimProvider({
|
|
|
1563
1976
|
interpolate,
|
|
1564
1977
|
children
|
|
1565
1978
|
}) {
|
|
1566
|
-
const { gl, camera } = useThree();
|
|
1979
|
+
const { gl, camera, scene } = useThree();
|
|
1567
1980
|
const [status, setStatus] = useState("loading");
|
|
1568
1981
|
const mjModelRef = useRef(null);
|
|
1569
1982
|
const mjDataRef = useRef(null);
|
|
@@ -1598,6 +2011,9 @@ function MujocoSimProvider({
|
|
|
1598
2011
|
useEffect(() => {
|
|
1599
2012
|
configRef.current = config;
|
|
1600
2013
|
}, [config]);
|
|
2014
|
+
useEffect(() => {
|
|
2015
|
+
mujocoRef.current = mujoco;
|
|
2016
|
+
}, [mujoco]);
|
|
1601
2017
|
useEffect(() => {
|
|
1602
2018
|
pausedRef.current = paused ?? false;
|
|
1603
2019
|
}, [paused]);
|
|
@@ -1645,6 +2061,7 @@ function MujocoSimProvider({
|
|
|
1645
2061
|
result.mjData.delete();
|
|
1646
2062
|
return;
|
|
1647
2063
|
}
|
|
2064
|
+
mujocoRef.current = mujoco;
|
|
1648
2065
|
mjModelRef.current = result.mjModel;
|
|
1649
2066
|
mjDataRef.current = result.mjData;
|
|
1650
2067
|
physicsAccumulatorRef.current = 0;
|
|
@@ -1710,7 +2127,7 @@ function MujocoSimProvider({
|
|
|
1710
2127
|
if (!interpolateRef.current) {
|
|
1711
2128
|
if (stepsToRunRef.current > 0) {
|
|
1712
2129
|
for (let s = 0; s < stepsToRunRef.current; s++) {
|
|
1713
|
-
|
|
2130
|
+
mujocoRef.current.mj_step(model, data);
|
|
1714
2131
|
}
|
|
1715
2132
|
stepsToRunRef.current = 0;
|
|
1716
2133
|
} else {
|
|
@@ -1719,7 +2136,7 @@ function MujocoSimProvider({
|
|
|
1719
2136
|
const frameTime = clampedDelta * speedRef.current;
|
|
1720
2137
|
while (data.time - startSimTime < frameTime) {
|
|
1721
2138
|
for (let s = 0; s < numSubsteps; s++) {
|
|
1722
|
-
|
|
2139
|
+
mujocoRef.current.mj_step(model, data);
|
|
1723
2140
|
}
|
|
1724
2141
|
}
|
|
1725
2142
|
}
|
|
@@ -1727,7 +2144,7 @@ function MujocoSimProvider({
|
|
|
1727
2144
|
ensureInterpolationBuffers(model);
|
|
1728
2145
|
copyBodyPose(data, interpolationStateRef.current.previousXpos, interpolationStateRef.current.previousXquat);
|
|
1729
2146
|
for (let s = 0; s < stepsToRunRef.current; s++) {
|
|
1730
|
-
|
|
2147
|
+
mujocoRef.current.mj_step(model, data);
|
|
1731
2148
|
}
|
|
1732
2149
|
copyBodyPose(data, interpolationStateRef.current.currentXpos, interpolationStateRef.current.currentXquat);
|
|
1733
2150
|
interpolationStateRef.current.alpha = 1;
|
|
@@ -1742,7 +2159,7 @@ function MujocoSimProvider({
|
|
|
1742
2159
|
while (physicsAccumulatorRef.current >= stepDt) {
|
|
1743
2160
|
copyBodyPose(data, interpolationStateRef.current.previousXpos, interpolationStateRef.current.previousXquat);
|
|
1744
2161
|
for (let s = 0; s < numSubsteps; s++) {
|
|
1745
|
-
|
|
2162
|
+
mujocoRef.current.mj_step(model, data);
|
|
1746
2163
|
}
|
|
1747
2164
|
copyBodyPose(data, interpolationStateRef.current.currentXpos, interpolationStateRef.current.currentXquat);
|
|
1748
2165
|
physicsAccumulatorRef.current -= stepDt;
|
|
@@ -1781,7 +2198,7 @@ function MujocoSimProvider({
|
|
|
1781
2198
|
const model = mjModelRef.current;
|
|
1782
2199
|
const data = mjDataRef.current;
|
|
1783
2200
|
if (!model || !data) return;
|
|
1784
|
-
|
|
2201
|
+
mujocoRef.current.mj_resetData(model, data);
|
|
1785
2202
|
const homeJoints = configRef.current.homeJoints;
|
|
1786
2203
|
if (homeJoints) {
|
|
1787
2204
|
const homeCount = Math.min(homeJoints.length, model.nu);
|
|
@@ -1794,7 +2211,7 @@ function MujocoSimProvider({
|
|
|
1794
2211
|
}
|
|
1795
2212
|
}
|
|
1796
2213
|
configRef.current.onReset?.({ model, data });
|
|
1797
|
-
|
|
2214
|
+
mujocoRef.current.mj_forward(model, data);
|
|
1798
2215
|
for (const cb of resetCallbacks.current) {
|
|
1799
2216
|
cb();
|
|
1800
2217
|
}
|
|
@@ -1812,6 +2229,27 @@ function MujocoSimProvider({
|
|
|
1812
2229
|
const step = useCallback((n = 1) => {
|
|
1813
2230
|
stepsToRunRef.current = n;
|
|
1814
2231
|
}, []);
|
|
2232
|
+
useCallback((steps = 1) => {
|
|
2233
|
+
const model = mjModelRef.current;
|
|
2234
|
+
const data = mjDataRef.current;
|
|
2235
|
+
if (!model || !data) return false;
|
|
2236
|
+
for (let stepIndex = 0; stepIndex < steps; stepIndex += 1) {
|
|
2237
|
+
for (let i = 0; i < model.nv; i += 1) {
|
|
2238
|
+
data.qfrc_applied[i] = 0;
|
|
2239
|
+
}
|
|
2240
|
+
for (const cb of beforeStepCallbacks.current) {
|
|
2241
|
+
cb({ model, data });
|
|
2242
|
+
}
|
|
2243
|
+
mujocoRef.current.mj_step(model, data);
|
|
2244
|
+
for (const cb of afterStepCallbacks.current) {
|
|
2245
|
+
cb({ model, data });
|
|
2246
|
+
}
|
|
2247
|
+
onStepRef.current?.({ time: data.time, model, data });
|
|
2248
|
+
}
|
|
2249
|
+
physicsAccumulatorRef.current = 0;
|
|
2250
|
+
interpolationStateRef.current.valid = false;
|
|
2251
|
+
return true;
|
|
2252
|
+
}, [mujoco]);
|
|
1815
2253
|
const getTime = useCallback(() => {
|
|
1816
2254
|
return mjDataRef.current?.time ?? 0;
|
|
1817
2255
|
}, []);
|
|
@@ -1840,7 +2278,7 @@ function MujocoSimProvider({
|
|
|
1840
2278
|
data.ctrl.set(snapshot.ctrl);
|
|
1841
2279
|
if (snapshot.act.length > 0) data.act.set(snapshot.act);
|
|
1842
2280
|
data.qfrc_applied.set(snapshot.qfrc_applied);
|
|
1843
|
-
|
|
2281
|
+
mujocoRef.current.mj_forward(model, data);
|
|
1844
2282
|
}, [mujoco]);
|
|
1845
2283
|
const setQpos = useCallback((values) => {
|
|
1846
2284
|
const model = mjModelRef.current;
|
|
@@ -1848,7 +2286,7 @@ function MujocoSimProvider({
|
|
|
1848
2286
|
if (!model || !data) return;
|
|
1849
2287
|
const arr = values instanceof Float64Array ? values : new Float64Array(values);
|
|
1850
2288
|
data.qpos.set(arr.subarray(0, Math.min(arr.length, model.nq)));
|
|
1851
|
-
|
|
2289
|
+
mujocoRef.current.mj_forward(model, data);
|
|
1852
2290
|
}, [mujoco]);
|
|
1853
2291
|
const setQvel = useCallback((values) => {
|
|
1854
2292
|
const data = mjDataRef.current;
|
|
@@ -2083,6 +2521,88 @@ function MujocoSimProvider({
|
|
|
2083
2521
|
}
|
|
2084
2522
|
return result;
|
|
2085
2523
|
}, []);
|
|
2524
|
+
const getCameras = useCallback(() => {
|
|
2525
|
+
const model = mjModelRef.current;
|
|
2526
|
+
if (!model) return [];
|
|
2527
|
+
const ncam = model.ncam ?? 0;
|
|
2528
|
+
const nameAddresses = model.name_camadr;
|
|
2529
|
+
if (!ncam || !nameAddresses) return [];
|
|
2530
|
+
const result = [];
|
|
2531
|
+
for (let i = 0; i < ncam; i += 1) {
|
|
2532
|
+
const posOffset = i * 3;
|
|
2533
|
+
const quatOffset = i * 4;
|
|
2534
|
+
result.push({
|
|
2535
|
+
id: i,
|
|
2536
|
+
name: getName(model, nameAddresses[i]),
|
|
2537
|
+
bodyId: model.cam_bodyid?.[i] ?? -1,
|
|
2538
|
+
fov: model.cam_fovy?.[i] ?? null,
|
|
2539
|
+
position: model.cam_pos ? vector3FromArray(model.cam_pos, posOffset) : null,
|
|
2540
|
+
quaternion: model.cam_quat ? quaternionFromArray(model.cam_quat, quatOffset) : null
|
|
2541
|
+
});
|
|
2542
|
+
}
|
|
2543
|
+
return result;
|
|
2544
|
+
}, []);
|
|
2545
|
+
const resolveCameraCaptureOptions = useCallback(
|
|
2546
|
+
(options = {}) => {
|
|
2547
|
+
const model = mjModelRef.current;
|
|
2548
|
+
const data = mjDataRef.current;
|
|
2549
|
+
if (!model || !data) {
|
|
2550
|
+
return options;
|
|
2551
|
+
}
|
|
2552
|
+
const baseOptions = omitResolvedCameraSelectors(options);
|
|
2553
|
+
if (options.cameraName) {
|
|
2554
|
+
const cameraId = findCameraByName(model, options.cameraName);
|
|
2555
|
+
if (cameraId < 0) {
|
|
2556
|
+
throw new Error(`MuJoCo camera "${options.cameraName}" was not found.`);
|
|
2557
|
+
}
|
|
2558
|
+
const position = data.cam_xpos ? vector3FromArray(data.cam_xpos, cameraId * 3) : model.cam_pos ? vector3FromArray(model.cam_pos, cameraId * 3) : void 0;
|
|
2559
|
+
const quaternion = data.cam_xmat ? quaternionFromXmat(data.cam_xmat, cameraId * 9) : model.cam_quat ? quaternionFromArray(model.cam_quat, cameraId * 4) : void 0;
|
|
2560
|
+
if (!position || !quaternion) {
|
|
2561
|
+
throw new Error(
|
|
2562
|
+
`MuJoCo camera "${options.cameraName}" does not expose a capture pose.`
|
|
2563
|
+
);
|
|
2564
|
+
}
|
|
2565
|
+
return {
|
|
2566
|
+
...baseOptions,
|
|
2567
|
+
position,
|
|
2568
|
+
quaternion,
|
|
2569
|
+
fov: options.fov ?? model.cam_fovy?.[cameraId],
|
|
2570
|
+
source: { kind: "mujoco-camera", cameraName: options.cameraName }
|
|
2571
|
+
};
|
|
2572
|
+
}
|
|
2573
|
+
if (options.siteName) {
|
|
2574
|
+
const siteId = findSiteByName(model, options.siteName);
|
|
2575
|
+
if (siteId < 0) {
|
|
2576
|
+
throw new Error(`MuJoCo site "${options.siteName}" was not found.`);
|
|
2577
|
+
}
|
|
2578
|
+
return {
|
|
2579
|
+
...baseOptions,
|
|
2580
|
+
position: vector3FromArray(data.site_xpos, siteId * 3),
|
|
2581
|
+
quaternion: quaternionFromXmat(data.site_xmat, siteId * 9),
|
|
2582
|
+
source: { kind: "mujoco-site", siteName: options.siteName }
|
|
2583
|
+
};
|
|
2584
|
+
}
|
|
2585
|
+
if (options.bodyName) {
|
|
2586
|
+
const bodyId = findBodyByName(model, options.bodyName);
|
|
2587
|
+
if (bodyId < 0) {
|
|
2588
|
+
throw new Error(`MuJoCo body "${options.bodyName}" was not found.`);
|
|
2589
|
+
}
|
|
2590
|
+
if (!data.xmat) {
|
|
2591
|
+
throw new Error(
|
|
2592
|
+
`MuJoCo body "${options.bodyName}" does not expose world orientation data.`
|
|
2593
|
+
);
|
|
2594
|
+
}
|
|
2595
|
+
return {
|
|
2596
|
+
...baseOptions,
|
|
2597
|
+
position: vector3FromArray(data.xpos, bodyId * 3),
|
|
2598
|
+
quaternion: quaternionFromXmat(data.xmat, bodyId * 9),
|
|
2599
|
+
source: { kind: "mujoco-body", bodyName: options.bodyName }
|
|
2600
|
+
};
|
|
2601
|
+
}
|
|
2602
|
+
return options;
|
|
2603
|
+
},
|
|
2604
|
+
[]
|
|
2605
|
+
);
|
|
2086
2606
|
const getModelOption = useCallback(() => {
|
|
2087
2607
|
const model = mjModelRef.current;
|
|
2088
2608
|
if (!model?.opt) return { timestep: 2e-3, gravity: [0, 0, -9.81], integrator: 0 };
|
|
@@ -2122,7 +2642,7 @@ function MujocoSimProvider({
|
|
|
2122
2642
|
const geomId = _rayGeomId[0];
|
|
2123
2643
|
const bodyId = geomId >= 0 ? model.geom_bodyid[geomId] : -1;
|
|
2124
2644
|
return {
|
|
2125
|
-
point: new
|
|
2645
|
+
point: new THREE12.Vector3(
|
|
2126
2646
|
origin.x + dir.x * dist,
|
|
2127
2647
|
origin.y + dir.y * dist,
|
|
2128
2648
|
origin.z + dir.z * dist
|
|
@@ -2159,7 +2679,7 @@ function MujocoSimProvider({
|
|
|
2159
2679
|
const qvelOffset = keyId * model.nv;
|
|
2160
2680
|
for (let i = 0; i < model.nv; i++) data.qvel[i] = model.key_qvel[qvelOffset + i];
|
|
2161
2681
|
}
|
|
2162
|
-
|
|
2682
|
+
mujocoRef.current.mj_forward(model, data);
|
|
2163
2683
|
for (const cb of resetCallbacks.current) {
|
|
2164
2684
|
cb();
|
|
2165
2685
|
}
|
|
@@ -2268,6 +2788,190 @@ function MujocoSimProvider({
|
|
|
2268
2788
|
},
|
|
2269
2789
|
[gl]
|
|
2270
2790
|
);
|
|
2791
|
+
const captureCameraFrameApi = useCallback(
|
|
2792
|
+
(options = {}) => {
|
|
2793
|
+
return captureCameraFrame(
|
|
2794
|
+
gl,
|
|
2795
|
+
scene,
|
|
2796
|
+
camera,
|
|
2797
|
+
resolveCameraCaptureOptions(options)
|
|
2798
|
+
);
|
|
2799
|
+
},
|
|
2800
|
+
[camera, gl, resolveCameraCaptureOptions, scene]
|
|
2801
|
+
);
|
|
2802
|
+
const captureCameraFrameBlobApi = useCallback(
|
|
2803
|
+
(options = {}) => {
|
|
2804
|
+
return captureCameraFrameBlob(
|
|
2805
|
+
gl,
|
|
2806
|
+
scene,
|
|
2807
|
+
camera,
|
|
2808
|
+
resolveCameraCaptureOptions(options)
|
|
2809
|
+
);
|
|
2810
|
+
},
|
|
2811
|
+
[camera, gl, resolveCameraCaptureOptions, scene]
|
|
2812
|
+
);
|
|
2813
|
+
const recordCameraSequenceApi = useCallback(
|
|
2814
|
+
async (options) => {
|
|
2815
|
+
const frameCount = Math.max(0, Math.floor(options.frames));
|
|
2816
|
+
const stepsPerFrame = Math.max(0, Math.floor(options.stepsPerFrame ?? 1));
|
|
2817
|
+
const cameras = options.cameras;
|
|
2818
|
+
const frames = [];
|
|
2819
|
+
const cameraSummaries = {};
|
|
2820
|
+
const wasPaused = pausedRef.current;
|
|
2821
|
+
const retainFrames = options.retainFrames ?? true;
|
|
2822
|
+
const requireMountedSources = options.requireMountedSources ?? true;
|
|
2823
|
+
let recordedFrameCount = 0;
|
|
2824
|
+
async function stepCameraSequence(frameIndex, steps) {
|
|
2825
|
+
const model = mjModelRef.current;
|
|
2826
|
+
const data = mjDataRef.current;
|
|
2827
|
+
if (!model || !data) {
|
|
2828
|
+
throw new Error("MuJoCo scene is not ready for camera sequence stepping.");
|
|
2829
|
+
}
|
|
2830
|
+
for (let stepIndex = 0; stepIndex < steps; stepIndex += 1) {
|
|
2831
|
+
for (let i = 0; i < model.nv; i += 1) {
|
|
2832
|
+
data.qfrc_applied[i] = 0;
|
|
2833
|
+
}
|
|
2834
|
+
await options.onBeforeStep?.({
|
|
2835
|
+
frameIndex,
|
|
2836
|
+
stepIndex,
|
|
2837
|
+
time: data.time,
|
|
2838
|
+
model,
|
|
2839
|
+
data
|
|
2840
|
+
});
|
|
2841
|
+
for (const cb of beforeStepCallbacks.current) {
|
|
2842
|
+
cb({ model, data });
|
|
2843
|
+
}
|
|
2844
|
+
mujocoRef.current.mj_step(model, data);
|
|
2845
|
+
for (const cb of afterStepCallbacks.current) {
|
|
2846
|
+
cb({ model, data });
|
|
2847
|
+
}
|
|
2848
|
+
onStepRef.current?.({ time: data.time, model, data });
|
|
2849
|
+
await options.onAfterStep?.({
|
|
2850
|
+
frameIndex,
|
|
2851
|
+
stepIndex,
|
|
2852
|
+
time: data.time,
|
|
2853
|
+
model,
|
|
2854
|
+
data
|
|
2855
|
+
});
|
|
2856
|
+
}
|
|
2857
|
+
physicsAccumulatorRef.current = 0;
|
|
2858
|
+
interpolationStateRef.current.valid = false;
|
|
2859
|
+
}
|
|
2860
|
+
if (frameCount === 0 || cameras.length === 0) {
|
|
2861
|
+
return {
|
|
2862
|
+
frames,
|
|
2863
|
+
cameraKeys: cameras.map((sequenceCamera) => sequenceCamera.key),
|
|
2864
|
+
cameraSummaries,
|
|
2865
|
+
frameCount: 0
|
|
2866
|
+
};
|
|
2867
|
+
}
|
|
2868
|
+
throwIfCameraSequenceAborted(options.signal);
|
|
2869
|
+
for (let attempt = 0; attempt < 30; attempt += 1) {
|
|
2870
|
+
if (mjModelRef.current && mjDataRef.current) break;
|
|
2871
|
+
await waitForNextAnimationFrame2();
|
|
2872
|
+
throwIfCameraSequenceAborted(options.signal);
|
|
2873
|
+
}
|
|
2874
|
+
if (!mjModelRef.current || !mjDataRef.current) {
|
|
2875
|
+
throw new Error("MuJoCo scene is not ready for camera sequence recording.");
|
|
2876
|
+
}
|
|
2877
|
+
const captureSessions = cameras.map((sequenceCamera) => {
|
|
2878
|
+
const { key, ...captureOptions } = sequenceCamera;
|
|
2879
|
+
const initialCaptureOptions = resolveCameraCaptureOptions(captureOptions);
|
|
2880
|
+
const mountedSource = initialCaptureOptions.source;
|
|
2881
|
+
if (requireMountedSources) {
|
|
2882
|
+
assertMatchingMountedCameraSource(
|
|
2883
|
+
key,
|
|
2884
|
+
captureOptions,
|
|
2885
|
+
mountedSource ?? { kind: "fallback-camera" }
|
|
2886
|
+
);
|
|
2887
|
+
}
|
|
2888
|
+
return {
|
|
2889
|
+
key,
|
|
2890
|
+
captureOptions,
|
|
2891
|
+
mountedSource,
|
|
2892
|
+
session: createCameraFrameCaptureSession(
|
|
2893
|
+
gl,
|
|
2894
|
+
scene,
|
|
2895
|
+
camera,
|
|
2896
|
+
initialCaptureOptions
|
|
2897
|
+
)
|
|
2898
|
+
};
|
|
2899
|
+
});
|
|
2900
|
+
try {
|
|
2901
|
+
pausedRef.current = true;
|
|
2902
|
+
stepsToRunRef.current = 0;
|
|
2903
|
+
if (options.reset) reset();
|
|
2904
|
+
for (let frameIndex = 0; frameIndex < frameCount; frameIndex += 1) {
|
|
2905
|
+
throwIfCameraSequenceAborted(options.signal);
|
|
2906
|
+
if (stepsPerFrame > 0 && (frameIndex > 0 || options.captureInitialFrame === false)) {
|
|
2907
|
+
await stepCameraSequence(frameIndex, stepsPerFrame);
|
|
2908
|
+
}
|
|
2909
|
+
await waitForNextAnimationFrame2();
|
|
2910
|
+
throwIfCameraSequenceAborted(options.signal);
|
|
2911
|
+
const model = mjModelRef.current;
|
|
2912
|
+
const data = mjDataRef.current;
|
|
2913
|
+
if (!model || !data) {
|
|
2914
|
+
throw new Error("MuJoCo scene is not ready for camera sequence sampling.");
|
|
2915
|
+
}
|
|
2916
|
+
await options.onSample?.({
|
|
2917
|
+
frameIndex,
|
|
2918
|
+
time: data.time,
|
|
2919
|
+
model,
|
|
2920
|
+
data
|
|
2921
|
+
});
|
|
2922
|
+
const cameraFrames = {};
|
|
2923
|
+
for (const { key, captureOptions, mountedSource, session } of captureSessions) {
|
|
2924
|
+
const resolvedCaptureOptions = resolveCameraCaptureOptions(captureOptions);
|
|
2925
|
+
const cameraFrame = session.captureDataUrl({
|
|
2926
|
+
...resolvedCaptureOptions,
|
|
2927
|
+
source: mountedSource ?? resolvedCaptureOptions.source
|
|
2928
|
+
});
|
|
2929
|
+
if (requireMountedSources) {
|
|
2930
|
+
assertMatchingMountedCameraSource(
|
|
2931
|
+
key,
|
|
2932
|
+
captureOptions,
|
|
2933
|
+
cameraFrame.source
|
|
2934
|
+
);
|
|
2935
|
+
}
|
|
2936
|
+
cameraSummaries[key] = {
|
|
2937
|
+
key,
|
|
2938
|
+
width: cameraFrame.width,
|
|
2939
|
+
height: cameraFrame.height,
|
|
2940
|
+
source: cameraFrame.source,
|
|
2941
|
+
frameCount: (cameraSummaries[key]?.frameCount ?? 0) + 1,
|
|
2942
|
+
firstFrameIndex: cameraSummaries[key]?.firstFrameIndex ?? frameIndex,
|
|
2943
|
+
lastFrameIndex: frameIndex,
|
|
2944
|
+
firstTimestamp: cameraSummaries[key]?.firstTimestamp ?? data.time,
|
|
2945
|
+
lastTimestamp: data.time
|
|
2946
|
+
};
|
|
2947
|
+
cameraFrames[key] = cameraFrame;
|
|
2948
|
+
}
|
|
2949
|
+
const frame = {
|
|
2950
|
+
frameIndex,
|
|
2951
|
+
time: getTime(),
|
|
2952
|
+
cameras: cameraFrames
|
|
2953
|
+
};
|
|
2954
|
+
if (retainFrames) {
|
|
2955
|
+
frames.push(frame);
|
|
2956
|
+
}
|
|
2957
|
+
recordedFrameCount += 1;
|
|
2958
|
+
await options.onFrame?.(frame);
|
|
2959
|
+
}
|
|
2960
|
+
} finally {
|
|
2961
|
+
for (const { session } of captureSessions) {
|
|
2962
|
+
session.dispose();
|
|
2963
|
+
}
|
|
2964
|
+
pausedRef.current = wasPaused;
|
|
2965
|
+
}
|
|
2966
|
+
return {
|
|
2967
|
+
frames,
|
|
2968
|
+
cameraKeys: cameras.map((sequenceCamera) => sequenceCamera.key),
|
|
2969
|
+
cameraSummaries,
|
|
2970
|
+
frameCount: recordedFrameCount
|
|
2971
|
+
};
|
|
2972
|
+
},
|
|
2973
|
+
[camera, getTime, gl, mujoco, reset, resolveCameraCaptureOptions, scene]
|
|
2974
|
+
);
|
|
2271
2975
|
const project2DTo3D = useCallback(
|
|
2272
2976
|
(x, y, cameraPos, lookAt) => {
|
|
2273
2977
|
const virtCam = camera.clone();
|
|
@@ -2278,9 +2982,9 @@ function MujocoSimProvider({
|
|
|
2278
2982
|
_projNdc.set(x * 2 - 1, -(y * 2 - 1));
|
|
2279
2983
|
_projRaycaster.setFromCamera(_projNdc, virtCam);
|
|
2280
2984
|
const objects = [];
|
|
2281
|
-
const
|
|
2282
|
-
if (
|
|
2283
|
-
|
|
2985
|
+
const scene2 = camera.parent;
|
|
2986
|
+
if (scene2) {
|
|
2987
|
+
scene2.traverse((c) => {
|
|
2284
2988
|
if (c.isMesh) objects.push(c);
|
|
2285
2989
|
});
|
|
2286
2990
|
}
|
|
@@ -2363,6 +3067,7 @@ function MujocoSimProvider({
|
|
|
2363
3067
|
getSites,
|
|
2364
3068
|
getActuators: getActuatorsApi,
|
|
2365
3069
|
getSensors,
|
|
3070
|
+
getCameras,
|
|
2366
3071
|
getModelOption,
|
|
2367
3072
|
setGravity,
|
|
2368
3073
|
setTimestep: setTimestepApi,
|
|
@@ -2378,6 +3083,9 @@ function MujocoSimProvider({
|
|
|
2378
3083
|
getCanvasSnapshot,
|
|
2379
3084
|
captureFrame: captureFrameApi,
|
|
2380
3085
|
captureFrameBlob: captureFrameBlobApi,
|
|
3086
|
+
captureCameraFrame: captureCameraFrameApi,
|
|
3087
|
+
captureCameraFrameBlob: captureCameraFrameBlobApi,
|
|
3088
|
+
recordCameraSequence: recordCameraSequenceApi,
|
|
2381
3089
|
project2DTo3D,
|
|
2382
3090
|
setBodyMass,
|
|
2383
3091
|
setGeomFriction,
|
|
@@ -2418,6 +3126,7 @@ function MujocoSimProvider({
|
|
|
2418
3126
|
getSites,
|
|
2419
3127
|
getActuatorsApi,
|
|
2420
3128
|
getSensors,
|
|
3129
|
+
getCameras,
|
|
2421
3130
|
getModelOption,
|
|
2422
3131
|
setGravity,
|
|
2423
3132
|
setTimestepApi,
|
|
@@ -2433,6 +3142,9 @@ function MujocoSimProvider({
|
|
|
2433
3142
|
getCanvasSnapshot,
|
|
2434
3143
|
captureFrameApi,
|
|
2435
3144
|
captureFrameBlobApi,
|
|
3145
|
+
captureCameraFrameApi,
|
|
3146
|
+
captureCameraFrameBlobApi,
|
|
3147
|
+
recordCameraSequenceApi,
|
|
2436
3148
|
project2DTo3D,
|
|
2437
3149
|
setBodyMass,
|
|
2438
3150
|
setGeomFriction,
|
|
@@ -2899,7 +3611,7 @@ function solve6x6(A, b, x) {
|
|
|
2899
3611
|
}
|
|
2900
3612
|
|
|
2901
3613
|
// src/hooks/useIkController.ts
|
|
2902
|
-
var _syncMat4 = new
|
|
3614
|
+
var _syncMat4 = new THREE12.Matrix4();
|
|
2903
3615
|
function syncGizmoToSite(data, siteId, target) {
|
|
2904
3616
|
if (siteId === -1) return;
|
|
2905
3617
|
const sitePos = data.site_xpos.subarray(siteId * 3, siteId * 3 + 3);
|
|
@@ -2931,7 +3643,7 @@ var useIkController = createControllerHook(
|
|
|
2931
3643
|
const { mjModelRef, mjDataRef, mujocoRef, resetCallbacks, status } = useMujocoContext();
|
|
2932
3644
|
const ikEnabledRef = useRef(false);
|
|
2933
3645
|
const ikCalculatingRef = useRef(false);
|
|
2934
|
-
const ikTargetRef = useRef(new
|
|
3646
|
+
const ikTargetRef = useRef(new THREE12.Group());
|
|
2935
3647
|
const siteIdRef = useRef(-1);
|
|
2936
3648
|
const controlGroupRef = useRef(null);
|
|
2937
3649
|
const genericIkRef = useRef(new GenericIK(mujocoRef.current));
|
|
@@ -2939,10 +3651,10 @@ var useIkController = createControllerHook(
|
|
|
2939
3651
|
const needsInitialSync = useRef(true);
|
|
2940
3652
|
const gizmoAnimRef = useRef({
|
|
2941
3653
|
active: false,
|
|
2942
|
-
startPos: new
|
|
2943
|
-
endPos: new
|
|
2944
|
-
startRot: new
|
|
2945
|
-
endRot: new
|
|
3654
|
+
startPos: new THREE12.Vector3(),
|
|
3655
|
+
endPos: new THREE12.Vector3(),
|
|
3656
|
+
startRot: new THREE12.Quaternion(),
|
|
3657
|
+
endRot: new THREE12.Quaternion(),
|
|
2946
3658
|
startTime: 0,
|
|
2947
3659
|
duration: 1e3
|
|
2948
3660
|
});
|
|
@@ -3087,8 +3799,8 @@ var useIkController = createControllerHook(
|
|
|
3087
3799
|
const target = ikTargetRef.current;
|
|
3088
3800
|
if (!target) return;
|
|
3089
3801
|
const targetPos = pos.clone();
|
|
3090
|
-
const targetRot = new
|
|
3091
|
-
new
|
|
3802
|
+
const targetRot = new THREE12.Quaternion().setFromEuler(
|
|
3803
|
+
new THREE12.Euler(Math.PI, 0, 0)
|
|
3092
3804
|
);
|
|
3093
3805
|
if (duration > 0) {
|
|
3094
3806
|
const ga = gizmoAnimRef.current;
|
|
@@ -3113,7 +3825,7 @@ var useIkController = createControllerHook(
|
|
|
3113
3825
|
if (!ikCalculatingRef.current || !target) return null;
|
|
3114
3826
|
return {
|
|
3115
3827
|
pos: target.position.clone(),
|
|
3116
|
-
rot: new
|
|
3828
|
+
rot: new THREE12.Euler().setFromQuaternion(target.quaternion)
|
|
3117
3829
|
};
|
|
3118
3830
|
},
|
|
3119
3831
|
[]
|
|
@@ -3209,10 +3921,10 @@ function Body({
|
|
|
3209
3921
|
if (!hasChildren) return null;
|
|
3210
3922
|
return /* @__PURE__ */ jsx("group", { ref: groupRef, children });
|
|
3211
3923
|
}
|
|
3212
|
-
var _mat4 = new
|
|
3213
|
-
var _pos = new
|
|
3214
|
-
var _quat = new
|
|
3215
|
-
var _scale = new
|
|
3924
|
+
var _mat4 = new THREE12.Matrix4();
|
|
3925
|
+
var _pos = new THREE12.Vector3();
|
|
3926
|
+
var _quat = new THREE12.Quaternion();
|
|
3927
|
+
var _scale = new THREE12.Vector3(1, 1, 1);
|
|
3216
3928
|
function IkGizmo({ controller, siteName, scale = 0.18, onDrag }) {
|
|
3217
3929
|
const { mjModelRef, mjDataRef, status } = useMujocoContext();
|
|
3218
3930
|
const { ikTargetRef, siteIdRef, ikEnabledRef, setIkEnabled } = controller;
|
|
@@ -3304,7 +4016,7 @@ function IkGizmo({ controller, siteName, scale = 0.18, onDrag }) {
|
|
|
3304
4016
|
}
|
|
3305
4017
|
) });
|
|
3306
4018
|
}
|
|
3307
|
-
var _dummy = new
|
|
4019
|
+
var _dummy = new THREE12.Object3D();
|
|
3308
4020
|
function ContactMarkers({
|
|
3309
4021
|
maxContacts = 100,
|
|
3310
4022
|
radius = 8e-3,
|
|
@@ -3348,11 +4060,11 @@ function ContactMarkers({
|
|
|
3348
4060
|
var _force = new Float64Array(3);
|
|
3349
4061
|
var _torque = new Float64Array(3);
|
|
3350
4062
|
var _point = new Float64Array(3);
|
|
3351
|
-
var _bodyPos = new
|
|
3352
|
-
var _bodyQuat = new
|
|
3353
|
-
var _worldHit = new
|
|
3354
|
-
var _raycaster = new
|
|
3355
|
-
var _mouse = new
|
|
4063
|
+
var _bodyPos = new THREE12.Vector3();
|
|
4064
|
+
var _bodyQuat = new THREE12.Quaternion();
|
|
4065
|
+
var _worldHit = new THREE12.Vector3();
|
|
4066
|
+
var _raycaster = new THREE12.Raycaster();
|
|
4067
|
+
var _mouse = new THREE12.Vector2();
|
|
3356
4068
|
function DragInteraction({
|
|
3357
4069
|
stiffness = 250,
|
|
3358
4070
|
showArrow = true,
|
|
@@ -3363,16 +4075,16 @@ function DragInteraction({
|
|
|
3363
4075
|
const draggingRef = useRef(false);
|
|
3364
4076
|
const bodyIdRef = useRef(-1);
|
|
3365
4077
|
const grabDistanceRef = useRef(0);
|
|
3366
|
-
const localHitRef = useRef(new
|
|
3367
|
-
const grabWorldRef = useRef(new
|
|
3368
|
-
const mouseWorldRef = useRef(new
|
|
4078
|
+
const localHitRef = useRef(new THREE12.Vector3());
|
|
4079
|
+
const grabWorldRef = useRef(new THREE12.Vector3());
|
|
4080
|
+
const mouseWorldRef = useRef(new THREE12.Vector3());
|
|
3369
4081
|
const arrowRef = useRef(null);
|
|
3370
4082
|
const groupRef = useRef(null);
|
|
3371
4083
|
useEffect(() => {
|
|
3372
4084
|
if (!showArrow || !groupRef.current) return;
|
|
3373
|
-
const arrow = new
|
|
3374
|
-
new
|
|
3375
|
-
new
|
|
4085
|
+
const arrow = new THREE12.ArrowHelper(
|
|
4086
|
+
new THREE12.Vector3(0, 1, 0),
|
|
4087
|
+
new THREE12.Vector3(),
|
|
3376
4088
|
0.1,
|
|
3377
4089
|
16729156
|
|
3378
4090
|
);
|
|
@@ -3554,7 +4266,7 @@ function useSceneLights(intensity = 1) {
|
|
|
3554
4266
|
const dr = lightDiffuse ? lightDiffuse[3 * i] : 1;
|
|
3555
4267
|
const dg = lightDiffuse ? lightDiffuse[3 * i + 1] : 1;
|
|
3556
4268
|
const db = lightDiffuse ? lightDiffuse[3 * i + 2] : 1;
|
|
3557
|
-
const color = new
|
|
4269
|
+
const color = new THREE12.Color(dr, dg, db);
|
|
3558
4270
|
const px = lightPos[3 * i];
|
|
3559
4271
|
const py = lightPos[3 * i + 1];
|
|
3560
4272
|
const pz = lightPos[3 * i + 2];
|
|
@@ -3562,7 +4274,7 @@ function useSceneLights(intensity = 1) {
|
|
|
3562
4274
|
const dy = lightDir[3 * i + 1];
|
|
3563
4275
|
const dz = lightDir[3 * i + 2];
|
|
3564
4276
|
if (isDirectional) {
|
|
3565
|
-
const light = new
|
|
4277
|
+
const light = new THREE12.DirectionalLight(color, finalIntensity);
|
|
3566
4278
|
light.position.set(px, py, pz);
|
|
3567
4279
|
light.target.position.set(px + dx, py + dy, pz + dz);
|
|
3568
4280
|
light.castShadow = castShadow;
|
|
@@ -3585,7 +4297,7 @@ function useSceneLights(intensity = 1) {
|
|
|
3585
4297
|
const cutoff = lightCutoff ? lightCutoff[i] : 45;
|
|
3586
4298
|
const exponent = lightExponent ? lightExponent[i] : 10;
|
|
3587
4299
|
const angle = cutoff * Math.PI / 180;
|
|
3588
|
-
const light = new
|
|
4300
|
+
const light = new THREE12.SpotLight(color, finalIntensity, 0, angle, exponent / 128);
|
|
3589
4301
|
light.position.set(px, py, pz);
|
|
3590
4302
|
light.target.position.set(px + dx, py + dy, pz + dz);
|
|
3591
4303
|
light.castShadow = castShadow;
|
|
@@ -3643,11 +4355,11 @@ var JOINT_COLORS = {
|
|
|
3643
4355
|
3: 16776960
|
|
3644
4356
|
// hinge - yellow
|
|
3645
4357
|
};
|
|
3646
|
-
var _v3a = new
|
|
3647
|
-
new
|
|
3648
|
-
var _quat2 = new
|
|
3649
|
-
var _contactPos = new
|
|
3650
|
-
var _contactNormal = new
|
|
4358
|
+
var _v3a = new THREE12.Vector3();
|
|
4359
|
+
new THREE12.Vector3();
|
|
4360
|
+
var _quat2 = new THREE12.Quaternion();
|
|
4361
|
+
var _contactPos = new THREE12.Vector3();
|
|
4362
|
+
var _contactNormal = new THREE12.Vector3();
|
|
3651
4363
|
var MAX_CONTACT_ARROWS = 50;
|
|
3652
4364
|
function Debug({
|
|
3653
4365
|
showGeoms = false,
|
|
@@ -3676,21 +4388,21 @@ function Debug({
|
|
|
3676
4388
|
let geometry = null;
|
|
3677
4389
|
switch (type) {
|
|
3678
4390
|
case 2:
|
|
3679
|
-
geometry = new
|
|
4391
|
+
geometry = new THREE12.SphereGeometry(s[3 * i], 12, 8);
|
|
3680
4392
|
break;
|
|
3681
4393
|
case 3:
|
|
3682
|
-
geometry = new
|
|
4394
|
+
geometry = new THREE12.CapsuleGeometry(s[3 * i], s[3 * i + 1] * 2, 6, 8);
|
|
3683
4395
|
break;
|
|
3684
4396
|
case 5:
|
|
3685
|
-
geometry = new
|
|
4397
|
+
geometry = new THREE12.CylinderGeometry(s[3 * i], s[3 * i], s[3 * i + 1] * 2, 12);
|
|
3686
4398
|
break;
|
|
3687
4399
|
case 6:
|
|
3688
|
-
geometry = new
|
|
4400
|
+
geometry = new THREE12.BoxGeometry(s[3 * i] * 2, s[3 * i + 1] * 2, s[3 * i + 2] * 2);
|
|
3689
4401
|
break;
|
|
3690
4402
|
}
|
|
3691
4403
|
if (geometry) {
|
|
3692
|
-
const mat = new
|
|
3693
|
-
const mesh = new
|
|
4404
|
+
const mat = new THREE12.MeshBasicMaterial({ color: 65280, wireframe: true, transparent: true, opacity: 0.3 });
|
|
4405
|
+
const mesh = new THREE12.Mesh(geometry, mat);
|
|
3694
4406
|
mesh.userData.geomId = i;
|
|
3695
4407
|
mesh.userData.bodyId = model.geom_bodyid[i];
|
|
3696
4408
|
geoms.push(mesh);
|
|
@@ -3713,9 +4425,9 @@ function Debug({
|
|
|
3713
4425
|
}
|
|
3714
4426
|
if (maxGeomSize > 0) radius = maxGeomSize * 0.15;
|
|
3715
4427
|
}
|
|
3716
|
-
const geometry = new
|
|
3717
|
-
const mat = new
|
|
3718
|
-
const mesh = new
|
|
4428
|
+
const geometry = new THREE12.OctahedronGeometry(radius);
|
|
4429
|
+
const mat = new THREE12.MeshBasicMaterial({ color: 16711935, depthTest: false });
|
|
4430
|
+
const mesh = new THREE12.Mesh(geometry, mat);
|
|
3719
4431
|
mesh.renderOrder = 999;
|
|
3720
4432
|
mesh.frustumCulled = false;
|
|
3721
4433
|
mesh.userData.siteId = i;
|
|
@@ -3727,9 +4439,9 @@ function Debug({
|
|
|
3727
4439
|
ctx.font = "bold 36px monospace";
|
|
3728
4440
|
ctx.textAlign = "center";
|
|
3729
4441
|
ctx.fillText(getName(model, model.name_siteadr[i]), 128, 42);
|
|
3730
|
-
const tex = new
|
|
3731
|
-
const spriteMat = new
|
|
3732
|
-
const sprite = new
|
|
4442
|
+
const tex = new THREE12.CanvasTexture(canvas);
|
|
4443
|
+
const spriteMat = new THREE12.SpriteMaterial({ map: tex, depthTest: false, transparent: true });
|
|
4444
|
+
const sprite = new THREE12.Sprite(spriteMat);
|
|
3733
4445
|
const labelScale = radius * 15;
|
|
3734
4446
|
sprite.scale.set(labelScale, labelScale * 0.25, 1);
|
|
3735
4447
|
sprite.position.y = radius * 2;
|
|
@@ -3752,9 +4464,9 @@ function Debug({
|
|
|
3752
4464
|
}
|
|
3753
4465
|
}
|
|
3754
4466
|
const arrowLen = Math.max(maxGeomSize * 0.8, 0.05);
|
|
3755
|
-
const arrow = new
|
|
3756
|
-
new
|
|
3757
|
-
new
|
|
4467
|
+
const arrow = new THREE12.ArrowHelper(
|
|
4468
|
+
new THREE12.Vector3(0, 0, 1),
|
|
4469
|
+
new THREE12.Vector3(),
|
|
3758
4470
|
arrowLen,
|
|
3759
4471
|
color,
|
|
3760
4472
|
arrowLen * 0.25,
|
|
@@ -3762,7 +4474,7 @@ function Debug({
|
|
|
3762
4474
|
);
|
|
3763
4475
|
arrow.renderOrder = 999;
|
|
3764
4476
|
arrow.frustumCulled = false;
|
|
3765
|
-
arrow.line.material = new
|
|
4477
|
+
arrow.line.material = new THREE12.LineBasicMaterial({ color, depthTest: false });
|
|
3766
4478
|
arrow.cone.material.depthTest = false;
|
|
3767
4479
|
arrow.line.renderOrder = 999;
|
|
3768
4480
|
arrow.line.frustumCulled = false;
|
|
@@ -3777,9 +4489,9 @@ function Debug({
|
|
|
3777
4489
|
}
|
|
3778
4490
|
if (showCOM) {
|
|
3779
4491
|
for (let i = 1; i < model.nbody; i++) {
|
|
3780
|
-
const geometry = new
|
|
3781
|
-
const mat = new
|
|
3782
|
-
const mesh = new
|
|
4492
|
+
const geometry = new THREE12.SphereGeometry(5e-3, 6, 6);
|
|
4493
|
+
const mat = new THREE12.MeshBasicMaterial({ color: 16711680 });
|
|
4494
|
+
const mesh = new THREE12.Mesh(geometry, mat);
|
|
3783
4495
|
mesh.userData.bodyId = i;
|
|
3784
4496
|
comMarkers.push(mesh);
|
|
3785
4497
|
}
|
|
@@ -3870,9 +4582,9 @@ function Debug({
|
|
|
3870
4582
|
contactPoolInitRef.current = true;
|
|
3871
4583
|
const pool = [];
|
|
3872
4584
|
for (let i = 0; i < MAX_CONTACT_ARROWS; i++) {
|
|
3873
|
-
const arrow = new
|
|
3874
|
-
new
|
|
3875
|
-
new
|
|
4585
|
+
const arrow = new THREE12.ArrowHelper(
|
|
4586
|
+
new THREE12.Vector3(0, 1, 0),
|
|
4587
|
+
new THREE12.Vector3(),
|
|
3876
4588
|
0.1,
|
|
3877
4589
|
16729156,
|
|
3878
4590
|
0.03,
|
|
@@ -3927,9 +4639,9 @@ function Debug({
|
|
|
3927
4639
|
showContacts && /* @__PURE__ */ jsx("group", { ref: contactGroupRef })
|
|
3928
4640
|
] });
|
|
3929
4641
|
}
|
|
3930
|
-
var DEFAULT_TENDON_COLOR = new
|
|
4642
|
+
var DEFAULT_TENDON_COLOR = new THREE12.Color(0.3, 0.3, 0.8);
|
|
3931
4643
|
var DEFAULT_TENDON_WIDTH = 2e-3;
|
|
3932
|
-
new
|
|
4644
|
+
new THREE12.Vector3();
|
|
3933
4645
|
function TendonRenderer(props) {
|
|
3934
4646
|
const { mjModelRef, mjDataRef, status } = useMujocoContext();
|
|
3935
4647
|
const groupRef = useRef(null);
|
|
@@ -3943,7 +4655,7 @@ function TendonRenderer(props) {
|
|
|
3943
4655
|
if (!model || !data || !group) return;
|
|
3944
4656
|
const ntendon = model.ntendon ?? 0;
|
|
3945
4657
|
if (ntendon === 0) return;
|
|
3946
|
-
const material = new
|
|
4658
|
+
const material = new THREE12.MeshStandardMaterial({
|
|
3947
4659
|
color: DEFAULT_TENDON_COLOR,
|
|
3948
4660
|
roughness: 0.6,
|
|
3949
4661
|
metalness: 0.1
|
|
@@ -3958,11 +4670,11 @@ function TendonRenderer(props) {
|
|
|
3958
4670
|
curves.push(null);
|
|
3959
4671
|
continue;
|
|
3960
4672
|
}
|
|
3961
|
-
const points = Array.from({ length: wrapNum }, () => new
|
|
3962
|
-
const curve = new
|
|
4673
|
+
const points = Array.from({ length: wrapNum }, () => new THREE12.Vector3());
|
|
4674
|
+
const curve = new THREE12.CatmullRomCurve3(points, false);
|
|
3963
4675
|
const segments = Math.max(wrapNum * 2, 4);
|
|
3964
|
-
const geometry = new
|
|
3965
|
-
const mesh = new
|
|
4676
|
+
const geometry = new THREE12.TubeGeometry(curve, segments, DEFAULT_TENDON_WIDTH, 6, false);
|
|
4677
|
+
const mesh = new THREE12.Mesh(geometry, material);
|
|
3966
4678
|
mesh.frustumCulled = false;
|
|
3967
4679
|
group.add(mesh);
|
|
3968
4680
|
meshes.push(mesh);
|
|
@@ -4017,11 +4729,11 @@ function TendonRenderer(props) {
|
|
|
4017
4729
|
if (curve.points.length !== validCount) {
|
|
4018
4730
|
curve.points.length = validCount;
|
|
4019
4731
|
while (curve.points.length < validCount) {
|
|
4020
|
-
curve.points.push(new
|
|
4732
|
+
curve.points.push(new THREE12.Vector3());
|
|
4021
4733
|
}
|
|
4022
4734
|
}
|
|
4023
4735
|
mesh.geometry.dispose();
|
|
4024
|
-
mesh.geometry = new
|
|
4736
|
+
mesh.geometry = new THREE12.TubeGeometry(
|
|
4025
4737
|
curve,
|
|
4026
4738
|
Math.max(validCount * 2, 4),
|
|
4027
4739
|
DEFAULT_TENDON_WIDTH,
|
|
@@ -4048,24 +4760,24 @@ function FlexRenderer(props) {
|
|
|
4048
4760
|
const vertAdr = model.flex_vertadr[f];
|
|
4049
4761
|
const vertNum = model.flex_vertnum[f];
|
|
4050
4762
|
if (vertNum === 0) continue;
|
|
4051
|
-
const geometry = new
|
|
4763
|
+
const geometry = new THREE12.BufferGeometry();
|
|
4052
4764
|
const positions = new Float32Array(vertNum * 3);
|
|
4053
|
-
geometry.setAttribute("position", new
|
|
4765
|
+
geometry.setAttribute("position", new THREE12.BufferAttribute(positions, 3));
|
|
4054
4766
|
geometry.computeVertexNormals();
|
|
4055
|
-
let color = new
|
|
4767
|
+
let color = new THREE12.Color(0.5, 0.5, 0.5);
|
|
4056
4768
|
if (model.flex_rgba) {
|
|
4057
|
-
color = new
|
|
4769
|
+
color = new THREE12.Color(
|
|
4058
4770
|
model.flex_rgba[4 * f],
|
|
4059
4771
|
model.flex_rgba[4 * f + 1],
|
|
4060
4772
|
model.flex_rgba[4 * f + 2]
|
|
4061
4773
|
);
|
|
4062
4774
|
}
|
|
4063
|
-
const material = new
|
|
4775
|
+
const material = new THREE12.MeshStandardMaterial({
|
|
4064
4776
|
color,
|
|
4065
4777
|
roughness: 0.7,
|
|
4066
|
-
side:
|
|
4778
|
+
side: THREE12.DoubleSide
|
|
4067
4779
|
});
|
|
4068
|
-
const mesh = new
|
|
4780
|
+
const mesh = new THREE12.Mesh(geometry, material);
|
|
4069
4781
|
mesh.userData.flexId = f;
|
|
4070
4782
|
mesh.userData.vertAdr = vertAdr;
|
|
4071
4783
|
mesh.userData.vertNum = vertNum;
|
|
@@ -4101,7 +4813,7 @@ function FlexRenderer(props) {
|
|
|
4101
4813
|
return /* @__PURE__ */ jsx("group", { ...props, ref: groupRef });
|
|
4102
4814
|
}
|
|
4103
4815
|
var GEOM_TYPE_NAMES2 = ["plane", "hfield", "sphere", "capsule", "ellipsoid", "cylinder", "box", "mesh"];
|
|
4104
|
-
var _matrix = new
|
|
4816
|
+
var _matrix = new THREE12.Matrix4();
|
|
4105
4817
|
function getGeomInfo(model, geomId) {
|
|
4106
4818
|
const size = model.geom_size.subarray(geomId * 3, geomId * 3 + 3);
|
|
4107
4819
|
const type = model.geom_type[geomId];
|
|
@@ -4123,10 +4835,10 @@ function geomSignature(model, geomId) {
|
|
|
4123
4835
|
return [type, size, mat, data, rgba].join("|");
|
|
4124
4836
|
}
|
|
4125
4837
|
function firstMesh(object) {
|
|
4126
|
-
if (object instanceof
|
|
4838
|
+
if (object instanceof THREE12.Mesh) return object;
|
|
4127
4839
|
let mesh = null;
|
|
4128
4840
|
object.traverse((child) => {
|
|
4129
|
-
if (!mesh && child instanceof
|
|
4841
|
+
if (!mesh && child instanceof THREE12.Mesh) mesh = child;
|
|
4130
4842
|
});
|
|
4131
4843
|
return mesh;
|
|
4132
4844
|
}
|
|
@@ -4555,12 +5267,12 @@ function useActuators() {
|
|
|
4555
5267
|
return actuators;
|
|
4556
5268
|
}, [status, mjModelRef]);
|
|
4557
5269
|
}
|
|
4558
|
-
var _mat42 = new
|
|
5270
|
+
var _mat42 = new THREE12.Matrix4();
|
|
4559
5271
|
function useSitePosition(siteName) {
|
|
4560
5272
|
const { mjModelRef, mjDataRef, status } = useMujocoContext();
|
|
4561
5273
|
const siteIdRef = useRef(-1);
|
|
4562
|
-
const positionRef = useRef(new
|
|
4563
|
-
const quaternionRef = useRef(new
|
|
5274
|
+
const positionRef = useRef(new THREE12.Vector3());
|
|
5275
|
+
const quaternionRef = useRef(new THREE12.Quaternion());
|
|
4564
5276
|
useEffect(() => {
|
|
4565
5277
|
const model = mjModelRef.current;
|
|
4566
5278
|
if (!model || status !== "ready") {
|
|
@@ -4751,10 +5463,10 @@ function useJointState(name) {
|
|
|
4751
5463
|
function useBodyState(name) {
|
|
4752
5464
|
const { mjModelRef, status } = useMujocoContext();
|
|
4753
5465
|
const bodyIdRef = useRef(-1);
|
|
4754
|
-
const position = useRef(new
|
|
4755
|
-
const quaternion = useRef(new
|
|
4756
|
-
const linearVelocity = useRef(new
|
|
4757
|
-
const angularVelocity = useRef(new
|
|
5466
|
+
const position = useRef(new THREE12.Vector3());
|
|
5467
|
+
const quaternion = useRef(new THREE12.Quaternion());
|
|
5468
|
+
const linearVelocity = useRef(new THREE12.Vector3());
|
|
5469
|
+
const angularVelocity = useRef(new THREE12.Vector3());
|
|
4758
5470
|
useEffect(() => {
|
|
4759
5471
|
const model = mjModelRef.current;
|
|
4760
5472
|
if (!model || status !== "ready") return;
|
|
@@ -5108,6 +5820,157 @@ function useVideoRecorder(options = {}) {
|
|
|
5108
5820
|
}
|
|
5109
5821
|
};
|
|
5110
5822
|
}
|
|
5823
|
+
function useCameraFrameCapture(defaultOptions = {}) {
|
|
5824
|
+
const mujoco = useMujoco();
|
|
5825
|
+
const [status, setStatus] = useState("idle");
|
|
5826
|
+
const [error, setError] = useState(null);
|
|
5827
|
+
const reset = useCallback(() => {
|
|
5828
|
+
setStatus("idle");
|
|
5829
|
+
setError(null);
|
|
5830
|
+
}, []);
|
|
5831
|
+
const capture = useCallback(
|
|
5832
|
+
async (options = {}) => {
|
|
5833
|
+
if (!mujoco.api) {
|
|
5834
|
+
throw new Error("MuJoCo scene is not ready for camera frame capture.");
|
|
5835
|
+
}
|
|
5836
|
+
setStatus("capturing");
|
|
5837
|
+
setError(null);
|
|
5838
|
+
try {
|
|
5839
|
+
const result = await mujoco.api.captureCameraFrame({
|
|
5840
|
+
...defaultOptions,
|
|
5841
|
+
...options
|
|
5842
|
+
});
|
|
5843
|
+
setStatus("captured");
|
|
5844
|
+
return result;
|
|
5845
|
+
} catch (nextError) {
|
|
5846
|
+
const error2 = nextError instanceof Error ? nextError : new Error("Unable to capture the requested camera frame.");
|
|
5847
|
+
setError(error2);
|
|
5848
|
+
setStatus("error");
|
|
5849
|
+
throw error2;
|
|
5850
|
+
}
|
|
5851
|
+
},
|
|
5852
|
+
[defaultOptions, mujoco.api]
|
|
5853
|
+
);
|
|
5854
|
+
const captureBlob = useCallback(
|
|
5855
|
+
async (options = {}) => {
|
|
5856
|
+
if (!mujoco.api) {
|
|
5857
|
+
throw new Error("MuJoCo scene is not ready for camera frame capture.");
|
|
5858
|
+
}
|
|
5859
|
+
setStatus("capturing");
|
|
5860
|
+
setError(null);
|
|
5861
|
+
try {
|
|
5862
|
+
const result = await mujoco.api.captureCameraFrameBlob({
|
|
5863
|
+
...defaultOptions,
|
|
5864
|
+
...options
|
|
5865
|
+
});
|
|
5866
|
+
setStatus("captured");
|
|
5867
|
+
return result;
|
|
5868
|
+
} catch (nextError) {
|
|
5869
|
+
const error2 = nextError instanceof Error ? nextError : new Error("Unable to capture the requested camera frame.");
|
|
5870
|
+
setError(error2);
|
|
5871
|
+
setStatus("error");
|
|
5872
|
+
throw error2;
|
|
5873
|
+
}
|
|
5874
|
+
},
|
|
5875
|
+
[defaultOptions, mujoco.api]
|
|
5876
|
+
);
|
|
5877
|
+
return {
|
|
5878
|
+
status,
|
|
5879
|
+
error,
|
|
5880
|
+
isCapturing: status === "capturing",
|
|
5881
|
+
capture,
|
|
5882
|
+
captureBlob,
|
|
5883
|
+
reset
|
|
5884
|
+
};
|
|
5885
|
+
}
|
|
5886
|
+
function useCameraSequenceRecorder() {
|
|
5887
|
+
const mujoco = useMujoco();
|
|
5888
|
+
const [status, setStatus] = useState("idle");
|
|
5889
|
+
const [error, setError] = useState(null);
|
|
5890
|
+
const reset = useCallback(() => {
|
|
5891
|
+
setStatus("idle");
|
|
5892
|
+
setError(null);
|
|
5893
|
+
}, []);
|
|
5894
|
+
const record = useCallback(
|
|
5895
|
+
async (options) => {
|
|
5896
|
+
if (!mujoco.api) {
|
|
5897
|
+
throw new Error("MuJoCo scene is not ready for camera sequence recording.");
|
|
5898
|
+
}
|
|
5899
|
+
setStatus("capturing");
|
|
5900
|
+
setError(null);
|
|
5901
|
+
try {
|
|
5902
|
+
const result = await mujoco.api.recordCameraSequence(options);
|
|
5903
|
+
setStatus("captured");
|
|
5904
|
+
return result;
|
|
5905
|
+
} catch (nextError) {
|
|
5906
|
+
const error2 = nextError instanceof Error ? nextError : new Error("Unable to record the requested camera sequence.");
|
|
5907
|
+
setError(error2);
|
|
5908
|
+
setStatus("error");
|
|
5909
|
+
throw error2;
|
|
5910
|
+
}
|
|
5911
|
+
},
|
|
5912
|
+
[mujoco.api]
|
|
5913
|
+
);
|
|
5914
|
+
return {
|
|
5915
|
+
status,
|
|
5916
|
+
error,
|
|
5917
|
+
isRecording: status === "capturing",
|
|
5918
|
+
record,
|
|
5919
|
+
reset
|
|
5920
|
+
};
|
|
5921
|
+
}
|
|
5922
|
+
function useMountedCameraSequenceRecorder(defaultOptions = {}) {
|
|
5923
|
+
const mujoco = useMujoco();
|
|
5924
|
+
const [status, setStatus] = useState("idle");
|
|
5925
|
+
const [error, setError] = useState(null);
|
|
5926
|
+
const reset = useCallback(() => {
|
|
5927
|
+
setStatus("idle");
|
|
5928
|
+
setError(null);
|
|
5929
|
+
}, []);
|
|
5930
|
+
const createPlan = useCallback(
|
|
5931
|
+
(cameraKeys, options = {}) => {
|
|
5932
|
+
if (!mujoco.api) {
|
|
5933
|
+
throw new Error("MuJoCo scene is not ready for mounted camera sequence planning.");
|
|
5934
|
+
}
|
|
5935
|
+
return createMountedCameraFrameSequencePlanFromApi(mujoco.api, cameraKeys, {
|
|
5936
|
+
...defaultOptions,
|
|
5937
|
+
...options
|
|
5938
|
+
});
|
|
5939
|
+
},
|
|
5940
|
+
[defaultOptions, mujoco.api]
|
|
5941
|
+
);
|
|
5942
|
+
const record = useCallback(
|
|
5943
|
+
async (options) => {
|
|
5944
|
+
if (!mujoco.api) {
|
|
5945
|
+
throw new Error("MuJoCo scene is not ready for mounted camera sequence recording.");
|
|
5946
|
+
}
|
|
5947
|
+
setStatus("capturing");
|
|
5948
|
+
setError(null);
|
|
5949
|
+
try {
|
|
5950
|
+
const result = await recordMountedCameraFrameSequence(mujoco.api, {
|
|
5951
|
+
...defaultOptions,
|
|
5952
|
+
...options
|
|
5953
|
+
});
|
|
5954
|
+
setStatus("captured");
|
|
5955
|
+
return result;
|
|
5956
|
+
} catch (nextError) {
|
|
5957
|
+
const error2 = nextError instanceof Error ? nextError : new Error("Unable to record the requested mounted camera sequence.");
|
|
5958
|
+
setError(error2);
|
|
5959
|
+
setStatus("error");
|
|
5960
|
+
throw error2;
|
|
5961
|
+
}
|
|
5962
|
+
},
|
|
5963
|
+
[defaultOptions, mujoco.api]
|
|
5964
|
+
);
|
|
5965
|
+
return {
|
|
5966
|
+
status,
|
|
5967
|
+
error,
|
|
5968
|
+
isRecording: status === "capturing",
|
|
5969
|
+
createPlan,
|
|
5970
|
+
record,
|
|
5971
|
+
reset
|
|
5972
|
+
};
|
|
5973
|
+
}
|
|
5111
5974
|
function useCtrlNoise(config = {}) {
|
|
5112
5975
|
const { mjModelRef } = useMujocoContext();
|
|
5113
5976
|
const configRef = useRef(config);
|
|
@@ -5159,7 +6022,7 @@ function useSelectionHighlight(bodyId, options = {}) {
|
|
|
5159
6022
|
}
|
|
5160
6023
|
}
|
|
5161
6024
|
prevRef.current = [];
|
|
5162
|
-
const highlightColor = new
|
|
6025
|
+
const highlightColor = new THREE12.Color(color);
|
|
5163
6026
|
for (const mesh of meshes) {
|
|
5164
6027
|
const mat = mesh.material;
|
|
5165
6028
|
if (mat.emissive) {
|
|
@@ -5186,15 +6049,15 @@ function useSelectionHighlight(bodyId, options = {}) {
|
|
|
5186
6049
|
}
|
|
5187
6050
|
function useCameraAnimation() {
|
|
5188
6051
|
const { camera } = useThree();
|
|
5189
|
-
const orbitTargetRef = useRef(new
|
|
6052
|
+
const orbitTargetRef = useRef(new THREE12.Vector3(0, 0, 0));
|
|
5190
6053
|
const cameraAnimRef = useRef({
|
|
5191
6054
|
active: false,
|
|
5192
|
-
startPos: new
|
|
5193
|
-
endPos: new
|
|
5194
|
-
startRot: new
|
|
5195
|
-
endRot: new
|
|
5196
|
-
startTarget: new
|
|
5197
|
-
endTarget: new
|
|
6055
|
+
startPos: new THREE12.Vector3(),
|
|
6056
|
+
endPos: new THREE12.Vector3(),
|
|
6057
|
+
startRot: new THREE12.Quaternion(),
|
|
6058
|
+
endRot: new THREE12.Quaternion(),
|
|
6059
|
+
startTarget: new THREE12.Vector3(),
|
|
6060
|
+
endTarget: new THREE12.Vector3(),
|
|
5198
6061
|
startTime: 0,
|
|
5199
6062
|
duration: 0,
|
|
5200
6063
|
resolve: null
|
|
@@ -5266,6 +6129,18 @@ function useCameraAnimation() {
|
|
|
5266
6129
|
*
|
|
5267
6130
|
* useFrameCapture — still-frame capture for canvas-backed MuJoCo/R3F scenes.
|
|
5268
6131
|
*/
|
|
6132
|
+
/**
|
|
6133
|
+
* @license
|
|
6134
|
+
* SPDX-License-Identifier: Apache-2.0
|
|
6135
|
+
*
|
|
6136
|
+
* Offscreen camera-frame capture for R3F/MuJoCo scenes.
|
|
6137
|
+
*/
|
|
6138
|
+
/**
|
|
6139
|
+
* @license
|
|
6140
|
+
* SPDX-License-Identifier: Apache-2.0
|
|
6141
|
+
*
|
|
6142
|
+
* Helpers for resolving dataset camera streams to mounted MuJoCo resources.
|
|
6143
|
+
*/
|
|
5269
6144
|
/**
|
|
5270
6145
|
* @license
|
|
5271
6146
|
* SPDX-License-Identifier: Apache-2.0
|
|
@@ -5399,6 +6274,24 @@ function useCameraAnimation() {
|
|
|
5399
6274
|
*
|
|
5400
6275
|
* useVideoRecorder — canvas video recording hook (spec 13.3)
|
|
5401
6276
|
*/
|
|
6277
|
+
/**
|
|
6278
|
+
* @license
|
|
6279
|
+
* SPDX-License-Identifier: Apache-2.0
|
|
6280
|
+
*
|
|
6281
|
+
* React state wrapper around MuJoCo/R3F offscreen camera-frame capture.
|
|
6282
|
+
*/
|
|
6283
|
+
/**
|
|
6284
|
+
* @license
|
|
6285
|
+
* SPDX-License-Identifier: Apache-2.0
|
|
6286
|
+
*
|
|
6287
|
+
* React state wrapper around fixed-camera simulation sequence recording.
|
|
6288
|
+
*/
|
|
6289
|
+
/**
|
|
6290
|
+
* @license
|
|
6291
|
+
* SPDX-License-Identifier: Apache-2.0
|
|
6292
|
+
*
|
|
6293
|
+
* React state wrapper for named MuJoCo camera/site/body sequence recording.
|
|
6294
|
+
*/
|
|
5402
6295
|
/**
|
|
5403
6296
|
* @license
|
|
5404
6297
|
* SPDX-License-Identifier: Apache-2.0
|
|
@@ -5429,6 +6322,6 @@ function useCameraAnimation() {
|
|
|
5429
6322
|
* useCameraAnimation — composable camera animation hook.
|
|
5430
6323
|
*/
|
|
5431
6324
|
|
|
5432
|
-
export { Body, ContactListener, ContactMarkers, Debug, DragInteraction, FlexRenderer, IkGizmo, InstancedGeomRenderer, MujocoCanvas, MujocoPhysics, MujocoProvider, MujocoSimProvider,
|
|
6325
|
+
export { Body, ContactListener, ContactMarkers, Debug, DragInteraction, FlexRenderer, IkGizmo, InstancedGeomRenderer, MountedCameraFrameSequenceReadinessStatus, MujocoCanvas, MujocoPhysics, MujocoProvider, MujocoSimProvider, SceneLights, TendonRenderer, TrajectoryPlayer, buildObservation, captureCameraFrame, captureCameraFrameBlob, captureFrame, captureFrameBlob, createCameraFrameCaptureSession, createContiguousControlGroup, createController, createControllerHook, createMountedCameraFrameSequencePlan, createMountedCameraFrameSequencePlanFromApi, createMountedCameraFrameSequenceReadiness, findActuatorByName, findBodyByName, findGeomByName, findJointByName, findKeyframeByName, findSensorByName, findSiteByName, findTendonByName, getActuatedJoints, getCameraFrameCaptureSourceTarget, getControlMap, getMountedCameraFrameCaptureSource, getName, isMountedCameraFrameCaptureSource, loadScene, recordMountedCameraFrameSequence, renderCameraFrameToCanvas, resolveControlGroup, resolveMountedCameraFrameSource, useActuators, useAfterPhysicsStep, useBeforePhysicsStep, useBodyMeshes, useBodyState, useCameraAnimation, useCameraFrameCapture, useCameraSequenceRecorder, useContactEvents, useContacts, useCtrl, useCtrlNoise, useFrameCapture, useGamepad, useGravityCompensation, useIkController, useJointState, useKeyboardTeleop, useMountedCameraSequenceRecorder, useMujoco, useMujocoWasm, useObservation, usePolicy, useSceneLights, useSelectionHighlight, useSensor, useSensors, useSitePosition, useTrajectoryPlayer, useTrajectoryRecorder, useVideoRecorder };
|
|
5433
6326
|
//# sourceMappingURL=index.js.map
|
|
5434
6327
|
//# sourceMappingURL=index.js.map
|