mujoco-react 9.1.0 → 9.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +121 -17
- package/dist/{chunk-33CV6HSV.js → chunk-T3GVZJ4F.js} +222 -8
- package/dist/chunk-T3GVZJ4F.js.map +1 -0
- package/dist/index.d.ts +198 -6
- package/dist/index.js +1109 -216
- package/dist/index.js.map +1 -1
- package/dist/spark.d.ts +24 -2
- package/dist/spark.js +89 -3
- package/dist/spark.js.map +1 -1
- package/dist/{types-C5gTvR7b.d.ts → types-oxbxOkAx.d.ts} +190 -2
- package/dist/vite.d.ts +1 -1
- package/dist/vite.js +6 -3
- package/dist/vite.js.map +1 -1
- package/package.json +1 -1
- package/src/components/VisualScenario.tsx +178 -1
- package/src/core/MujocoSimProvider.tsx +473 -11
- package/src/core/SceneLoader.ts +13 -0
- package/src/core/createController.tsx +6 -2
- package/src/hooks/useCameraFrameCapture.ts +94 -0
- package/src/hooks/useCameraSequenceRecorder.ts +59 -0
- package/src/hooks/useMountedCameraSequenceRecorder.ts +107 -0
- package/src/index.ts +67 -0
- package/src/rendering/cameraFrameCapture.ts +353 -0
- package/src/rendering/cameraFrameSource.ts +375 -0
- package/src/spark.tsx +144 -0
- package/src/types.ts +212 -2
- package/src/vite.ts +5 -2
- package/dist/chunk-33CV6HSV.js.map +0 -1
|
@@ -20,6 +20,13 @@ import {
|
|
|
20
20
|
ActuatedJointInfo,
|
|
21
21
|
ActuatorInfo,
|
|
22
22
|
BodyInfo,
|
|
23
|
+
CameraFrameCaptureOptions,
|
|
24
|
+
CameraFrameCaptureResult,
|
|
25
|
+
CameraFrameCaptureSource,
|
|
26
|
+
CameraFrameSequenceFrame,
|
|
27
|
+
CameraFrameSequenceOptions,
|
|
28
|
+
CameraFrameSequenceResult,
|
|
29
|
+
CameraInfo,
|
|
23
30
|
ControlGroupInfo,
|
|
24
31
|
ControlGroupSelector,
|
|
25
32
|
ContactInfo,
|
|
@@ -45,14 +52,25 @@ import {
|
|
|
45
52
|
captureFrame as captureCanvasFrame,
|
|
46
53
|
captureFrameBlob as captureCanvasFrameBlob,
|
|
47
54
|
} from '../hooks/useFrameCapture';
|
|
55
|
+
import {
|
|
56
|
+
captureCameraFrame,
|
|
57
|
+
captureCameraFrameBlob,
|
|
58
|
+
createCameraFrameCaptureSession,
|
|
59
|
+
} from '../rendering/cameraFrameCapture';
|
|
60
|
+
import {
|
|
61
|
+
getCameraFrameCaptureSourceTarget,
|
|
62
|
+
isMountedCameraFrameCaptureSource,
|
|
63
|
+
} from '../rendering/cameraFrameSource';
|
|
48
64
|
import {
|
|
49
65
|
loadScene,
|
|
50
66
|
createSceneConfigFromFiles,
|
|
51
67
|
findKeyframeByName,
|
|
52
68
|
findBodyByName,
|
|
69
|
+
findSiteByName,
|
|
53
70
|
findGeomByName,
|
|
54
71
|
findSensorByName,
|
|
55
72
|
findActuatorByName,
|
|
73
|
+
findCameraByName,
|
|
56
74
|
getActuatedScalarQposAdr,
|
|
57
75
|
getActuatedJoints as getActuatedJointsFromModel,
|
|
58
76
|
getControlMap as getControlMapFromModel,
|
|
@@ -110,6 +128,104 @@ const _rayGeomId = new Int32Array(1);
|
|
|
110
128
|
const _projRaycaster = new THREE.Raycaster();
|
|
111
129
|
const _projNdc = new THREE.Vector2();
|
|
112
130
|
|
|
131
|
+
function waitForNextAnimationFrame() {
|
|
132
|
+
return new Promise<void>((resolve) => {
|
|
133
|
+
requestAnimationFrame(() => resolve());
|
|
134
|
+
});
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
function throwIfCameraSequenceAborted(signal: AbortSignal | undefined) {
|
|
138
|
+
if (!signal?.aborted) return;
|
|
139
|
+
|
|
140
|
+
if (typeof signal.reason === 'object' && signal.reason instanceof Error) {
|
|
141
|
+
throw signal.reason;
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
throw new DOMException('Camera sequence recording was aborted.', 'AbortError');
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
function vector3FromArray(values: ArrayLike<number>, offset: number): [number, number, number] {
|
|
148
|
+
return [values[offset], values[offset + 1], values[offset + 2]];
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
function quaternionFromArray(values: ArrayLike<number>, offset: number): [number, number, number, number] {
|
|
152
|
+
return [
|
|
153
|
+
values[offset],
|
|
154
|
+
values[offset + 1],
|
|
155
|
+
values[offset + 2],
|
|
156
|
+
values[offset + 3],
|
|
157
|
+
];
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
function quaternionFromXmat(values: ArrayLike<number>, offset: number): [number, number, number, number] {
|
|
161
|
+
const matrix = new THREE.Matrix4();
|
|
162
|
+
matrix.set(
|
|
163
|
+
values[offset],
|
|
164
|
+
values[offset + 1],
|
|
165
|
+
values[offset + 2],
|
|
166
|
+
0,
|
|
167
|
+
values[offset + 3],
|
|
168
|
+
values[offset + 4],
|
|
169
|
+
values[offset + 5],
|
|
170
|
+
0,
|
|
171
|
+
values[offset + 6],
|
|
172
|
+
values[offset + 7],
|
|
173
|
+
values[offset + 8],
|
|
174
|
+
0,
|
|
175
|
+
0,
|
|
176
|
+
0,
|
|
177
|
+
0,
|
|
178
|
+
1
|
|
179
|
+
);
|
|
180
|
+
const quaternion = new THREE.Quaternion().setFromRotationMatrix(matrix);
|
|
181
|
+
return [quaternion.x, quaternion.y, quaternion.z, quaternion.w];
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
function omitResolvedCameraSelectors(
|
|
185
|
+
options: CameraFrameCaptureOptions
|
|
186
|
+
): CameraFrameCaptureOptions {
|
|
187
|
+
const { cameraName, siteName, bodyName, ...rest } = options;
|
|
188
|
+
return rest;
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
function countMountedCameraSelectors(options: CameraFrameCaptureOptions) {
|
|
192
|
+
return Number(Boolean(options.cameraName)) +
|
|
193
|
+
Number(Boolean(options.siteName)) +
|
|
194
|
+
Number(Boolean(options.bodyName));
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
function assertMatchingMountedCameraSource(
|
|
198
|
+
key: string,
|
|
199
|
+
requested: CameraFrameCaptureOptions,
|
|
200
|
+
source: CameraFrameCaptureSource
|
|
201
|
+
) {
|
|
202
|
+
const selectorCount = countMountedCameraSelectors(requested);
|
|
203
|
+
if (selectorCount !== 1) {
|
|
204
|
+
throw new Error(
|
|
205
|
+
`Camera sequence stream "${key}" must provide exactly one mounted MuJoCo cameraName, siteName, or bodyName selector.`
|
|
206
|
+
);
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
if (!isMountedCameraFrameCaptureSource(source)) {
|
|
210
|
+
throw new Error(
|
|
211
|
+
`Camera sequence stream "${key}" resolved to ${source.kind}; use a MuJoCo-mounted camera, site, or body selector for sequence recording.`
|
|
212
|
+
);
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
if (
|
|
216
|
+
(requested.cameraName &&
|
|
217
|
+
(source.kind !== 'mujoco-camera' || source.cameraName !== requested.cameraName)) ||
|
|
218
|
+
(requested.siteName &&
|
|
219
|
+
(source.kind !== 'mujoco-site' || source.siteName !== requested.siteName)) ||
|
|
220
|
+
(requested.bodyName &&
|
|
221
|
+
(source.kind !== 'mujoco-body' || source.bodyName !== requested.bodyName))
|
|
222
|
+
) {
|
|
223
|
+
throw new Error(
|
|
224
|
+
`Camera sequence stream "${key}" resolved to ${source.kind}:${getCameraFrameCaptureSourceTarget(source)} instead of the requested mounted selector.`
|
|
225
|
+
);
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
|
|
113
229
|
// ---- Internal context types ----
|
|
114
230
|
|
|
115
231
|
export interface MujocoSimContextValue {
|
|
@@ -256,7 +372,7 @@ export function MujocoSimProvider({
|
|
|
256
372
|
interpolate,
|
|
257
373
|
children,
|
|
258
374
|
}: MujocoSimProviderProps) {
|
|
259
|
-
const { gl, camera } = useThree();
|
|
375
|
+
const { gl, camera, scene } = useThree();
|
|
260
376
|
const [status, setStatus] = useState<'loading' | 'ready' | 'error'>('loading');
|
|
261
377
|
|
|
262
378
|
// --- Refs ---
|
|
@@ -294,6 +410,7 @@ export function MujocoSimProvider({
|
|
|
294
410
|
const bodyReloadTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
|
|
295
411
|
|
|
296
412
|
useEffect(() => { configRef.current = config; }, [config]);
|
|
413
|
+
useEffect(() => { mujocoRef.current = mujoco; }, [mujoco]);
|
|
297
414
|
|
|
298
415
|
// Sync declarative props to refs
|
|
299
416
|
useEffect(() => { pausedRef.current = paused ?? false; }, [paused]);
|
|
@@ -345,6 +462,7 @@ export function MujocoSimProvider({
|
|
|
345
462
|
return;
|
|
346
463
|
}
|
|
347
464
|
|
|
465
|
+
mujocoRef.current = mujoco;
|
|
348
466
|
mjModelRef.current = result.mjModel;
|
|
349
467
|
mjDataRef.current = result.mjData;
|
|
350
468
|
physicsAccumulatorRef.current = 0;
|
|
@@ -424,7 +542,7 @@ export function MujocoSimProvider({
|
|
|
424
542
|
// Step physics with substeps
|
|
425
543
|
if (stepsToRunRef.current > 0) {
|
|
426
544
|
for (let s = 0; s < stepsToRunRef.current; s++) {
|
|
427
|
-
|
|
545
|
+
mujocoRef.current.mj_step(model, data);
|
|
428
546
|
}
|
|
429
547
|
stepsToRunRef.current = 0;
|
|
430
548
|
} else {
|
|
@@ -433,7 +551,7 @@ export function MujocoSimProvider({
|
|
|
433
551
|
const frameTime = clampedDelta * speedRef.current;
|
|
434
552
|
while (data.time - startSimTime < frameTime) {
|
|
435
553
|
for (let s = 0; s < numSubsteps; s++) {
|
|
436
|
-
|
|
554
|
+
mujocoRef.current.mj_step(model, data);
|
|
437
555
|
}
|
|
438
556
|
}
|
|
439
557
|
}
|
|
@@ -441,7 +559,7 @@ export function MujocoSimProvider({
|
|
|
441
559
|
ensureInterpolationBuffers(model);
|
|
442
560
|
copyBodyPose(data, interpolationStateRef.current.previousXpos, interpolationStateRef.current.previousXquat);
|
|
443
561
|
for (let s = 0; s < stepsToRunRef.current; s++) {
|
|
444
|
-
|
|
562
|
+
mujocoRef.current.mj_step(model, data);
|
|
445
563
|
}
|
|
446
564
|
copyBodyPose(data, interpolationStateRef.current.currentXpos, interpolationStateRef.current.currentXquat);
|
|
447
565
|
interpolationStateRef.current.alpha = 1;
|
|
@@ -457,7 +575,7 @@ export function MujocoSimProvider({
|
|
|
457
575
|
while (physicsAccumulatorRef.current >= stepDt) {
|
|
458
576
|
copyBodyPose(data, interpolationStateRef.current.previousXpos, interpolationStateRef.current.previousXquat);
|
|
459
577
|
for (let s = 0; s < numSubsteps; s++) {
|
|
460
|
-
|
|
578
|
+
mujocoRef.current.mj_step(model, data);
|
|
461
579
|
}
|
|
462
580
|
copyBodyPose(data, interpolationStateRef.current.currentXpos, interpolationStateRef.current.currentXquat);
|
|
463
581
|
physicsAccumulatorRef.current -= stepDt;
|
|
@@ -508,7 +626,7 @@ export function MujocoSimProvider({
|
|
|
508
626
|
const data = mjDataRef.current;
|
|
509
627
|
if (!model || !data) return;
|
|
510
628
|
|
|
511
|
-
|
|
629
|
+
mujocoRef.current.mj_resetData(model, data);
|
|
512
630
|
|
|
513
631
|
const homeJoints = configRef.current.homeJoints;
|
|
514
632
|
if (homeJoints) {
|
|
@@ -523,7 +641,7 @@ export function MujocoSimProvider({
|
|
|
523
641
|
}
|
|
524
642
|
|
|
525
643
|
configRef.current.onReset?.({ model, data });
|
|
526
|
-
|
|
644
|
+
mujocoRef.current.mj_forward(model, data);
|
|
527
645
|
|
|
528
646
|
// Notify composable plugins (e.g. IkController)
|
|
529
647
|
for (const cb of resetCallbacks.current) {
|
|
@@ -548,6 +666,30 @@ export function MujocoSimProvider({
|
|
|
548
666
|
stepsToRunRef.current = n;
|
|
549
667
|
}, []);
|
|
550
668
|
|
|
669
|
+
const stepImmediately = useCallback((steps = 1) => {
|
|
670
|
+
const model = mjModelRef.current;
|
|
671
|
+
const data = mjDataRef.current;
|
|
672
|
+
if (!model || !data) return false;
|
|
673
|
+
|
|
674
|
+
for (let stepIndex = 0; stepIndex < steps; stepIndex += 1) {
|
|
675
|
+
for (let i = 0; i < model.nv; i += 1) {
|
|
676
|
+
data.qfrc_applied[i] = 0;
|
|
677
|
+
}
|
|
678
|
+
for (const cb of beforeStepCallbacks.current) {
|
|
679
|
+
cb({ model, data });
|
|
680
|
+
}
|
|
681
|
+
mujocoRef.current.mj_step(model, data);
|
|
682
|
+
for (const cb of afterStepCallbacks.current) {
|
|
683
|
+
cb({ model, data });
|
|
684
|
+
}
|
|
685
|
+
onStepRef.current?.({ time: data.time, model, data });
|
|
686
|
+
}
|
|
687
|
+
|
|
688
|
+
physicsAccumulatorRef.current = 0;
|
|
689
|
+
interpolationStateRef.current.valid = false;
|
|
690
|
+
return true;
|
|
691
|
+
}, [mujoco]);
|
|
692
|
+
|
|
551
693
|
const getTime = useCallback((): number => {
|
|
552
694
|
return mjDataRef.current?.time ?? 0;
|
|
553
695
|
}, []);
|
|
@@ -579,7 +721,7 @@ export function MujocoSimProvider({
|
|
|
579
721
|
data.ctrl.set(snapshot.ctrl);
|
|
580
722
|
if (snapshot.act.length > 0) data.act.set(snapshot.act);
|
|
581
723
|
data.qfrc_applied.set(snapshot.qfrc_applied);
|
|
582
|
-
|
|
724
|
+
mujocoRef.current.mj_forward(model, data);
|
|
583
725
|
}, [mujoco]);
|
|
584
726
|
|
|
585
727
|
const setQpos = useCallback((values: Float64Array | number[]) => {
|
|
@@ -588,7 +730,7 @@ export function MujocoSimProvider({
|
|
|
588
730
|
if (!model || !data) return;
|
|
589
731
|
const arr = values instanceof Float64Array ? values : new Float64Array(values);
|
|
590
732
|
data.qpos.set(arr.subarray(0, Math.min(arr.length, model.nq)));
|
|
591
|
-
|
|
733
|
+
mujocoRef.current.mj_forward(model, data);
|
|
592
734
|
}, [mujoco]);
|
|
593
735
|
|
|
594
736
|
const setQvel = useCallback((values: Float64Array | number[]) => {
|
|
@@ -831,6 +973,113 @@ export function MujocoSimProvider({
|
|
|
831
973
|
return result;
|
|
832
974
|
}, []);
|
|
833
975
|
|
|
976
|
+
const getCameras = useCallback((): CameraInfo[] => {
|
|
977
|
+
const model = mjModelRef.current;
|
|
978
|
+
if (!model) return [];
|
|
979
|
+
const ncam = model.ncam ?? 0;
|
|
980
|
+
const nameAddresses = model.name_camadr;
|
|
981
|
+
if (!ncam || !nameAddresses) return [];
|
|
982
|
+
|
|
983
|
+
const result: CameraInfo[] = [];
|
|
984
|
+
for (let i = 0; i < ncam; i += 1) {
|
|
985
|
+
const posOffset = i * 3;
|
|
986
|
+
const quatOffset = i * 4;
|
|
987
|
+
result.push({
|
|
988
|
+
id: i,
|
|
989
|
+
name: getName(model, nameAddresses[i]),
|
|
990
|
+
bodyId: model.cam_bodyid?.[i] ?? -1,
|
|
991
|
+
fov: model.cam_fovy?.[i] ?? null,
|
|
992
|
+
position: model.cam_pos
|
|
993
|
+
? vector3FromArray(model.cam_pos, posOffset)
|
|
994
|
+
: null,
|
|
995
|
+
quaternion: model.cam_quat
|
|
996
|
+
? quaternionFromArray(model.cam_quat, quatOffset)
|
|
997
|
+
: null,
|
|
998
|
+
});
|
|
999
|
+
}
|
|
1000
|
+
return result;
|
|
1001
|
+
}, []);
|
|
1002
|
+
|
|
1003
|
+
const resolveCameraCaptureOptions = useCallback(
|
|
1004
|
+
(options: CameraFrameCaptureOptions = {}): CameraFrameCaptureOptions => {
|
|
1005
|
+
const model = mjModelRef.current;
|
|
1006
|
+
const data = mjDataRef.current;
|
|
1007
|
+
if (!model || !data) {
|
|
1008
|
+
return options;
|
|
1009
|
+
}
|
|
1010
|
+
|
|
1011
|
+
const baseOptions = omitResolvedCameraSelectors(options);
|
|
1012
|
+
|
|
1013
|
+
if (options.cameraName) {
|
|
1014
|
+
const cameraId = findCameraByName(model, options.cameraName);
|
|
1015
|
+
if (cameraId < 0) {
|
|
1016
|
+
throw new Error(`MuJoCo camera "${options.cameraName}" was not found.`);
|
|
1017
|
+
}
|
|
1018
|
+
|
|
1019
|
+
const position = data.cam_xpos
|
|
1020
|
+
? vector3FromArray(data.cam_xpos, cameraId * 3)
|
|
1021
|
+
: model.cam_pos
|
|
1022
|
+
? vector3FromArray(model.cam_pos, cameraId * 3)
|
|
1023
|
+
: undefined;
|
|
1024
|
+
const quaternion = data.cam_xmat
|
|
1025
|
+
? quaternionFromXmat(data.cam_xmat, cameraId * 9)
|
|
1026
|
+
: model.cam_quat
|
|
1027
|
+
? quaternionFromArray(model.cam_quat, cameraId * 4)
|
|
1028
|
+
: undefined;
|
|
1029
|
+
|
|
1030
|
+
if (!position || !quaternion) {
|
|
1031
|
+
throw new Error(
|
|
1032
|
+
`MuJoCo camera "${options.cameraName}" does not expose a capture pose.`
|
|
1033
|
+
);
|
|
1034
|
+
}
|
|
1035
|
+
|
|
1036
|
+
return {
|
|
1037
|
+
...baseOptions,
|
|
1038
|
+
position,
|
|
1039
|
+
quaternion,
|
|
1040
|
+
fov: options.fov ?? model.cam_fovy?.[cameraId],
|
|
1041
|
+
source: { kind: 'mujoco-camera', cameraName: options.cameraName },
|
|
1042
|
+
};
|
|
1043
|
+
}
|
|
1044
|
+
|
|
1045
|
+
if (options.siteName) {
|
|
1046
|
+
const siteId = findSiteByName(model, options.siteName);
|
|
1047
|
+
if (siteId < 0) {
|
|
1048
|
+
throw new Error(`MuJoCo site "${options.siteName}" was not found.`);
|
|
1049
|
+
}
|
|
1050
|
+
|
|
1051
|
+
return {
|
|
1052
|
+
...baseOptions,
|
|
1053
|
+
position: vector3FromArray(data.site_xpos, siteId * 3),
|
|
1054
|
+
quaternion: quaternionFromXmat(data.site_xmat, siteId * 9),
|
|
1055
|
+
source: { kind: 'mujoco-site', siteName: options.siteName },
|
|
1056
|
+
};
|
|
1057
|
+
}
|
|
1058
|
+
|
|
1059
|
+
if (options.bodyName) {
|
|
1060
|
+
const bodyId = findBodyByName(model, options.bodyName);
|
|
1061
|
+
if (bodyId < 0) {
|
|
1062
|
+
throw new Error(`MuJoCo body "${options.bodyName}" was not found.`);
|
|
1063
|
+
}
|
|
1064
|
+
if (!data.xmat) {
|
|
1065
|
+
throw new Error(
|
|
1066
|
+
`MuJoCo body "${options.bodyName}" does not expose world orientation data.`
|
|
1067
|
+
);
|
|
1068
|
+
}
|
|
1069
|
+
|
|
1070
|
+
return {
|
|
1071
|
+
...baseOptions,
|
|
1072
|
+
position: vector3FromArray(data.xpos, bodyId * 3),
|
|
1073
|
+
quaternion: quaternionFromXmat(data.xmat, bodyId * 9),
|
|
1074
|
+
source: { kind: 'mujoco-body', bodyName: options.bodyName },
|
|
1075
|
+
};
|
|
1076
|
+
}
|
|
1077
|
+
|
|
1078
|
+
return options;
|
|
1079
|
+
},
|
|
1080
|
+
[]
|
|
1081
|
+
);
|
|
1082
|
+
|
|
834
1083
|
const getModelOption = useCallback((): ModelOptions => {
|
|
835
1084
|
const model = mjModelRef.current;
|
|
836
1085
|
if (!model?.opt) return { timestep: 0.002, gravity: [0, 0, -9.81], integrator: 0 };
|
|
@@ -913,7 +1162,7 @@ export function MujocoSimProvider({
|
|
|
913
1162
|
for (let i = 0; i < model.nv; i++) data.qvel[i] = model.key_qvel[qvelOffset + i];
|
|
914
1163
|
}
|
|
915
1164
|
|
|
916
|
-
|
|
1165
|
+
mujocoRef.current.mj_forward(model, data);
|
|
917
1166
|
|
|
918
1167
|
// Notify composable plugins
|
|
919
1168
|
for (const cb of resetCallbacks.current) {
|
|
@@ -1041,6 +1290,213 @@ export function MujocoSimProvider({
|
|
|
1041
1290
|
[gl]
|
|
1042
1291
|
);
|
|
1043
1292
|
|
|
1293
|
+
const captureCameraFrameApi = useCallback(
|
|
1294
|
+
(options = {}) => {
|
|
1295
|
+
return captureCameraFrame(
|
|
1296
|
+
gl,
|
|
1297
|
+
scene,
|
|
1298
|
+
camera,
|
|
1299
|
+
resolveCameraCaptureOptions(options)
|
|
1300
|
+
);
|
|
1301
|
+
},
|
|
1302
|
+
[camera, gl, resolveCameraCaptureOptions, scene]
|
|
1303
|
+
);
|
|
1304
|
+
|
|
1305
|
+
const captureCameraFrameBlobApi = useCallback(
|
|
1306
|
+
(options = {}) => {
|
|
1307
|
+
return captureCameraFrameBlob(
|
|
1308
|
+
gl,
|
|
1309
|
+
scene,
|
|
1310
|
+
camera,
|
|
1311
|
+
resolveCameraCaptureOptions(options)
|
|
1312
|
+
);
|
|
1313
|
+
},
|
|
1314
|
+
[camera, gl, resolveCameraCaptureOptions, scene]
|
|
1315
|
+
);
|
|
1316
|
+
|
|
1317
|
+
const recordCameraSequenceApi = useCallback(
|
|
1318
|
+
async (
|
|
1319
|
+
options: CameraFrameSequenceOptions
|
|
1320
|
+
): Promise<CameraFrameSequenceResult> => {
|
|
1321
|
+
const frameCount = Math.max(0, Math.floor(options.frames));
|
|
1322
|
+
const stepsPerFrame = Math.max(0, Math.floor(options.stepsPerFrame ?? 1));
|
|
1323
|
+
const cameras = options.cameras;
|
|
1324
|
+
const frames: CameraFrameSequenceFrame[] = [];
|
|
1325
|
+
const cameraSummaries: CameraFrameSequenceResult['cameraSummaries'] = {};
|
|
1326
|
+
const wasPaused = pausedRef.current;
|
|
1327
|
+
const retainFrames = options.retainFrames ?? true;
|
|
1328
|
+
const requireMountedSources = options.requireMountedSources ?? true;
|
|
1329
|
+
let recordedFrameCount = 0;
|
|
1330
|
+
|
|
1331
|
+
async function stepCameraSequence(frameIndex: number, steps: number) {
|
|
1332
|
+
const model = mjModelRef.current;
|
|
1333
|
+
const data = mjDataRef.current;
|
|
1334
|
+
if (!model || !data) {
|
|
1335
|
+
throw new Error('MuJoCo scene is not ready for camera sequence stepping.');
|
|
1336
|
+
}
|
|
1337
|
+
|
|
1338
|
+
for (let stepIndex = 0; stepIndex < steps; stepIndex += 1) {
|
|
1339
|
+
for (let i = 0; i < model.nv; i += 1) {
|
|
1340
|
+
data.qfrc_applied[i] = 0;
|
|
1341
|
+
}
|
|
1342
|
+
await options.onBeforeStep?.({
|
|
1343
|
+
frameIndex,
|
|
1344
|
+
stepIndex,
|
|
1345
|
+
time: data.time,
|
|
1346
|
+
model,
|
|
1347
|
+
data,
|
|
1348
|
+
});
|
|
1349
|
+
for (const cb of beforeStepCallbacks.current) {
|
|
1350
|
+
cb({ model, data });
|
|
1351
|
+
}
|
|
1352
|
+
mujocoRef.current.mj_step(model, data);
|
|
1353
|
+
for (const cb of afterStepCallbacks.current) {
|
|
1354
|
+
cb({ model, data });
|
|
1355
|
+
}
|
|
1356
|
+
onStepRef.current?.({ time: data.time, model, data });
|
|
1357
|
+
await options.onAfterStep?.({
|
|
1358
|
+
frameIndex,
|
|
1359
|
+
stepIndex,
|
|
1360
|
+
time: data.time,
|
|
1361
|
+
model,
|
|
1362
|
+
data,
|
|
1363
|
+
});
|
|
1364
|
+
}
|
|
1365
|
+
|
|
1366
|
+
physicsAccumulatorRef.current = 0;
|
|
1367
|
+
interpolationStateRef.current.valid = false;
|
|
1368
|
+
}
|
|
1369
|
+
|
|
1370
|
+
if (frameCount === 0 || cameras.length === 0) {
|
|
1371
|
+
return {
|
|
1372
|
+
frames,
|
|
1373
|
+
cameraKeys: cameras.map((sequenceCamera) => sequenceCamera.key),
|
|
1374
|
+
cameraSummaries,
|
|
1375
|
+
frameCount: 0,
|
|
1376
|
+
};
|
|
1377
|
+
}
|
|
1378
|
+
|
|
1379
|
+
throwIfCameraSequenceAborted(options.signal);
|
|
1380
|
+
|
|
1381
|
+
for (let attempt = 0; attempt < 30; attempt += 1) {
|
|
1382
|
+
if (mjModelRef.current && mjDataRef.current) break;
|
|
1383
|
+
await waitForNextAnimationFrame();
|
|
1384
|
+
throwIfCameraSequenceAborted(options.signal);
|
|
1385
|
+
}
|
|
1386
|
+
if (!mjModelRef.current || !mjDataRef.current) {
|
|
1387
|
+
throw new Error('MuJoCo scene is not ready for camera sequence recording.');
|
|
1388
|
+
}
|
|
1389
|
+
|
|
1390
|
+
const captureSessions = cameras.map((sequenceCamera) => {
|
|
1391
|
+
const { key, ...captureOptions } = sequenceCamera;
|
|
1392
|
+
const initialCaptureOptions = resolveCameraCaptureOptions(captureOptions);
|
|
1393
|
+
const mountedSource = initialCaptureOptions.source;
|
|
1394
|
+
if (requireMountedSources) {
|
|
1395
|
+
assertMatchingMountedCameraSource(
|
|
1396
|
+
key,
|
|
1397
|
+
captureOptions,
|
|
1398
|
+
mountedSource ?? { kind: 'fallback-camera' }
|
|
1399
|
+
);
|
|
1400
|
+
}
|
|
1401
|
+
return {
|
|
1402
|
+
key,
|
|
1403
|
+
captureOptions,
|
|
1404
|
+
mountedSource,
|
|
1405
|
+
session: createCameraFrameCaptureSession(
|
|
1406
|
+
gl,
|
|
1407
|
+
scene,
|
|
1408
|
+
camera,
|
|
1409
|
+
initialCaptureOptions
|
|
1410
|
+
),
|
|
1411
|
+
};
|
|
1412
|
+
});
|
|
1413
|
+
|
|
1414
|
+
try {
|
|
1415
|
+
pausedRef.current = true;
|
|
1416
|
+
stepsToRunRef.current = 0;
|
|
1417
|
+
if (options.reset) reset();
|
|
1418
|
+
|
|
1419
|
+
for (let frameIndex = 0; frameIndex < frameCount; frameIndex += 1) {
|
|
1420
|
+
throwIfCameraSequenceAborted(options.signal);
|
|
1421
|
+
if (
|
|
1422
|
+
stepsPerFrame > 0 &&
|
|
1423
|
+
(frameIndex > 0 || options.captureInitialFrame === false)
|
|
1424
|
+
) {
|
|
1425
|
+
await stepCameraSequence(frameIndex, stepsPerFrame);
|
|
1426
|
+
}
|
|
1427
|
+
await waitForNextAnimationFrame();
|
|
1428
|
+
throwIfCameraSequenceAborted(options.signal);
|
|
1429
|
+
|
|
1430
|
+
const model = mjModelRef.current;
|
|
1431
|
+
const data = mjDataRef.current;
|
|
1432
|
+
if (!model || !data) {
|
|
1433
|
+
throw new Error('MuJoCo scene is not ready for camera sequence sampling.');
|
|
1434
|
+
}
|
|
1435
|
+
await options.onSample?.({
|
|
1436
|
+
frameIndex,
|
|
1437
|
+
time: data.time,
|
|
1438
|
+
model,
|
|
1439
|
+
data,
|
|
1440
|
+
});
|
|
1441
|
+
|
|
1442
|
+
const cameraFrames: Record<string, CameraFrameCaptureResult> = {};
|
|
1443
|
+
for (const { key, captureOptions, mountedSource, session } of captureSessions) {
|
|
1444
|
+
const resolvedCaptureOptions = resolveCameraCaptureOptions(captureOptions);
|
|
1445
|
+
const cameraFrame = session.captureDataUrl({
|
|
1446
|
+
...resolvedCaptureOptions,
|
|
1447
|
+
source: mountedSource ?? resolvedCaptureOptions.source,
|
|
1448
|
+
});
|
|
1449
|
+
if (requireMountedSources) {
|
|
1450
|
+
assertMatchingMountedCameraSource(
|
|
1451
|
+
key,
|
|
1452
|
+
captureOptions,
|
|
1453
|
+
cameraFrame.source
|
|
1454
|
+
);
|
|
1455
|
+
}
|
|
1456
|
+
cameraSummaries[key] = {
|
|
1457
|
+
key,
|
|
1458
|
+
width: cameraFrame.width,
|
|
1459
|
+
height: cameraFrame.height,
|
|
1460
|
+
source: cameraFrame.source,
|
|
1461
|
+
frameCount: (cameraSummaries[key]?.frameCount ?? 0) + 1,
|
|
1462
|
+
firstFrameIndex:
|
|
1463
|
+
cameraSummaries[key]?.firstFrameIndex ?? frameIndex,
|
|
1464
|
+
lastFrameIndex: frameIndex,
|
|
1465
|
+
firstTimestamp:
|
|
1466
|
+
cameraSummaries[key]?.firstTimestamp ?? data.time,
|
|
1467
|
+
lastTimestamp: data.time,
|
|
1468
|
+
};
|
|
1469
|
+
cameraFrames[key] = cameraFrame;
|
|
1470
|
+
}
|
|
1471
|
+
|
|
1472
|
+
const frame = {
|
|
1473
|
+
frameIndex,
|
|
1474
|
+
time: getTime(),
|
|
1475
|
+
cameras: cameraFrames,
|
|
1476
|
+
};
|
|
1477
|
+
if (retainFrames) {
|
|
1478
|
+
frames.push(frame);
|
|
1479
|
+
}
|
|
1480
|
+
recordedFrameCount += 1;
|
|
1481
|
+
await options.onFrame?.(frame);
|
|
1482
|
+
}
|
|
1483
|
+
} finally {
|
|
1484
|
+
for (const { session } of captureSessions) {
|
|
1485
|
+
session.dispose();
|
|
1486
|
+
}
|
|
1487
|
+
pausedRef.current = wasPaused;
|
|
1488
|
+
}
|
|
1489
|
+
|
|
1490
|
+
return {
|
|
1491
|
+
frames,
|
|
1492
|
+
cameraKeys: cameras.map((sequenceCamera) => sequenceCamera.key),
|
|
1493
|
+
cameraSummaries,
|
|
1494
|
+
frameCount: recordedFrameCount,
|
|
1495
|
+
};
|
|
1496
|
+
},
|
|
1497
|
+
[camera, getTime, gl, mujoco, reset, resolveCameraCaptureOptions, scene]
|
|
1498
|
+
);
|
|
1499
|
+
|
|
1044
1500
|
const project2DTo3D = useCallback(
|
|
1045
1501
|
(x: number, y: number, cameraPos: THREE.Vector3, lookAt: THREE.Vector3): { point: THREE.Vector3; bodyId: number; geomId: number } | null => {
|
|
1046
1502
|
const virtCam = (camera as THREE.PerspectiveCamera).clone();
|
|
@@ -1139,6 +1595,7 @@ export function MujocoSimProvider({
|
|
|
1139
1595
|
getSites,
|
|
1140
1596
|
getActuators: getActuatorsApi,
|
|
1141
1597
|
getSensors,
|
|
1598
|
+
getCameras,
|
|
1142
1599
|
getModelOption,
|
|
1143
1600
|
setGravity,
|
|
1144
1601
|
setTimestep: setTimestepApi,
|
|
@@ -1154,6 +1611,9 @@ export function MujocoSimProvider({
|
|
|
1154
1611
|
getCanvasSnapshot,
|
|
1155
1612
|
captureFrame: captureFrameApi,
|
|
1156
1613
|
captureFrameBlob: captureFrameBlobApi,
|
|
1614
|
+
captureCameraFrame: captureCameraFrameApi,
|
|
1615
|
+
captureCameraFrameBlob: captureCameraFrameBlobApi,
|
|
1616
|
+
recordCameraSequence: recordCameraSequenceApi,
|
|
1157
1617
|
project2DTo3D,
|
|
1158
1618
|
setBodyMass,
|
|
1159
1619
|
setGeomFriction,
|
|
@@ -1168,10 +1628,12 @@ export function MujocoSimProvider({
|
|
|
1168
1628
|
getControlMapApi, getActuatedJointsApi, resolveControlGroupApi,
|
|
1169
1629
|
applyForce, applyTorqueApi, setExternalForce, applyGeneralizedForce,
|
|
1170
1630
|
getSensorData, getContacts, getBodies, getJoints, getGeoms, getSites,
|
|
1171
|
-
getActuatorsApi, getSensors, getModelOption, setGravity, setTimestepApi,
|
|
1631
|
+
getActuatorsApi, getSensors, getCameras, getModelOption, setGravity, setTimestepApi,
|
|
1172
1632
|
raycast, getKeyframeNames, getKeyframeCount, loadSceneApi,
|
|
1173
1633
|
loadFromFilesApi, addBodyApi, removeBodyApi, recompileApi,
|
|
1174
1634
|
getCanvas, getCanvasSnapshot, captureFrameApi, captureFrameBlobApi,
|
|
1635
|
+
captureCameraFrameApi, captureCameraFrameBlobApi,
|
|
1636
|
+
recordCameraSequenceApi,
|
|
1175
1637
|
project2DTo3D,
|
|
1176
1638
|
setBodyMass, setGeomFriction, setGeomSize,
|
|
1177
1639
|
]
|
package/src/core/SceneLoader.ts
CHANGED
|
@@ -110,6 +110,19 @@ export function findSensorByName(mjModel: MujocoModel, name: string): number {
|
|
|
110
110
|
return -1;
|
|
111
111
|
}
|
|
112
112
|
|
|
113
|
+
/**
|
|
114
|
+
* Find a camera by name in the MuJoCo model. Returns -1 if not found.
|
|
115
|
+
*/
|
|
116
|
+
export function findCameraByName(mjModel: MujocoModel, name: string): number {
|
|
117
|
+
const ncam = mjModel.ncam ?? 0;
|
|
118
|
+
const addresses = mjModel.name_camadr;
|
|
119
|
+
if (!addresses) return -1;
|
|
120
|
+
for (let i = 0; i < ncam; i++) {
|
|
121
|
+
if (getName(mjModel, addresses[i]) === name) return i;
|
|
122
|
+
}
|
|
123
|
+
return -1;
|
|
124
|
+
}
|
|
125
|
+
|
|
113
126
|
/**
|
|
114
127
|
* Find a tendon by name in the MuJoCo model. Returns -1 if not found.
|
|
115
128
|
*/
|
|
@@ -43,8 +43,10 @@ export type ControllerComponent<TConfig> = React.FC<{
|
|
|
43
43
|
* const MyController = createController<{ speed: number }>(
|
|
44
44
|
* { name: 'my-controller', defaultConfig: { speed: 1.0 } },
|
|
45
45
|
* function MyControllerImpl({ config }) {
|
|
46
|
+
* const wheel = useCtrl(RobotActuators.mobile.leftWheel);
|
|
47
|
+
*
|
|
46
48
|
* useBeforePhysicsStep(({ data }) => {
|
|
47
|
-
*
|
|
49
|
+
* wheel.write(config.speed);
|
|
48
50
|
* });
|
|
49
51
|
* return null;
|
|
50
52
|
* },
|
|
@@ -100,9 +102,11 @@ export function createController<TConfig>(
|
|
|
100
102
|
* { name: 'useMyController', defaultConfig: { gain: 1.0 } },
|
|
101
103
|
* function useMyControllerImpl(config) {
|
|
102
104
|
* // config is MyConfig | null — hooks must be called unconditionally
|
|
105
|
+
* const joint = useCtrl(config?.actuator ?? RobotActuators.franka.actuator1);
|
|
106
|
+
*
|
|
103
107
|
* useBeforePhysicsStep(({ data }) => {
|
|
104
108
|
* if (!config) return;
|
|
105
|
-
*
|
|
109
|
+
* joint.write(config.gain * Math.sin(data.time));
|
|
106
110
|
* });
|
|
107
111
|
* if (!config) return null;
|
|
108
112
|
* return { /* value *\/ };
|