mujoco-react 9.1.0 → 9.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -20,6 +20,13 @@ import {
20
20
  ActuatedJointInfo,
21
21
  ActuatorInfo,
22
22
  BodyInfo,
23
+ CameraFrameCaptureOptions,
24
+ CameraFrameCaptureResult,
25
+ CameraFrameCaptureSource,
26
+ CameraFrameSequenceFrame,
27
+ CameraFrameSequenceOptions,
28
+ CameraFrameSequenceResult,
29
+ CameraInfo,
23
30
  ControlGroupInfo,
24
31
  ControlGroupSelector,
25
32
  ContactInfo,
@@ -45,14 +52,25 @@ import {
45
52
  captureFrame as captureCanvasFrame,
46
53
  captureFrameBlob as captureCanvasFrameBlob,
47
54
  } from '../hooks/useFrameCapture';
55
+ import {
56
+ captureCameraFrame,
57
+ captureCameraFrameBlob,
58
+ createCameraFrameCaptureSession,
59
+ } from '../rendering/cameraFrameCapture';
60
+ import {
61
+ getCameraFrameCaptureSourceTarget,
62
+ isMountedCameraFrameCaptureSource,
63
+ } from '../rendering/cameraFrameSource';
48
64
  import {
49
65
  loadScene,
50
66
  createSceneConfigFromFiles,
51
67
  findKeyframeByName,
52
68
  findBodyByName,
69
+ findSiteByName,
53
70
  findGeomByName,
54
71
  findSensorByName,
55
72
  findActuatorByName,
73
+ findCameraByName,
56
74
  getActuatedScalarQposAdr,
57
75
  getActuatedJoints as getActuatedJointsFromModel,
58
76
  getControlMap as getControlMapFromModel,
@@ -110,6 +128,104 @@ const _rayGeomId = new Int32Array(1);
110
128
  const _projRaycaster = new THREE.Raycaster();
111
129
  const _projNdc = new THREE.Vector2();
112
130
 
131
+ function waitForNextAnimationFrame() {
132
+ return new Promise<void>((resolve) => {
133
+ requestAnimationFrame(() => resolve());
134
+ });
135
+ }
136
+
137
+ function throwIfCameraSequenceAborted(signal: AbortSignal | undefined) {
138
+ if (!signal?.aborted) return;
139
+
140
+ if (typeof signal.reason === 'object' && signal.reason instanceof Error) {
141
+ throw signal.reason;
142
+ }
143
+
144
+ throw new DOMException('Camera sequence recording was aborted.', 'AbortError');
145
+ }
146
+
147
+ function vector3FromArray(values: ArrayLike<number>, offset: number): [number, number, number] {
148
+ return [values[offset], values[offset + 1], values[offset + 2]];
149
+ }
150
+
151
+ function quaternionFromArray(values: ArrayLike<number>, offset: number): [number, number, number, number] {
152
+ return [
153
+ values[offset],
154
+ values[offset + 1],
155
+ values[offset + 2],
156
+ values[offset + 3],
157
+ ];
158
+ }
159
+
160
+ function quaternionFromXmat(values: ArrayLike<number>, offset: number): [number, number, number, number] {
161
+ const matrix = new THREE.Matrix4();
162
+ matrix.set(
163
+ values[offset],
164
+ values[offset + 1],
165
+ values[offset + 2],
166
+ 0,
167
+ values[offset + 3],
168
+ values[offset + 4],
169
+ values[offset + 5],
170
+ 0,
171
+ values[offset + 6],
172
+ values[offset + 7],
173
+ values[offset + 8],
174
+ 0,
175
+ 0,
176
+ 0,
177
+ 0,
178
+ 1
179
+ );
180
+ const quaternion = new THREE.Quaternion().setFromRotationMatrix(matrix);
181
+ return [quaternion.x, quaternion.y, quaternion.z, quaternion.w];
182
+ }
183
+
184
+ function omitResolvedCameraSelectors(
185
+ options: CameraFrameCaptureOptions
186
+ ): CameraFrameCaptureOptions {
187
+ const { cameraName, siteName, bodyName, ...rest } = options;
188
+ return rest;
189
+ }
190
+
191
+ function countMountedCameraSelectors(options: CameraFrameCaptureOptions) {
192
+ return Number(Boolean(options.cameraName)) +
193
+ Number(Boolean(options.siteName)) +
194
+ Number(Boolean(options.bodyName));
195
+ }
196
+
197
+ function assertMatchingMountedCameraSource(
198
+ key: string,
199
+ requested: CameraFrameCaptureOptions,
200
+ source: CameraFrameCaptureSource
201
+ ) {
202
+ const selectorCount = countMountedCameraSelectors(requested);
203
+ if (selectorCount !== 1) {
204
+ throw new Error(
205
+ `Camera sequence stream "${key}" must provide exactly one mounted MuJoCo cameraName, siteName, or bodyName selector.`
206
+ );
207
+ }
208
+
209
+ if (!isMountedCameraFrameCaptureSource(source)) {
210
+ throw new Error(
211
+ `Camera sequence stream "${key}" resolved to ${source.kind}; use a MuJoCo-mounted camera, site, or body selector for sequence recording.`
212
+ );
213
+ }
214
+
215
+ if (
216
+ (requested.cameraName &&
217
+ (source.kind !== 'mujoco-camera' || source.cameraName !== requested.cameraName)) ||
218
+ (requested.siteName &&
219
+ (source.kind !== 'mujoco-site' || source.siteName !== requested.siteName)) ||
220
+ (requested.bodyName &&
221
+ (source.kind !== 'mujoco-body' || source.bodyName !== requested.bodyName))
222
+ ) {
223
+ throw new Error(
224
+ `Camera sequence stream "${key}" resolved to ${source.kind}:${getCameraFrameCaptureSourceTarget(source)} instead of the requested mounted selector.`
225
+ );
226
+ }
227
+ }
228
+
113
229
  // ---- Internal context types ----
114
230
 
115
231
  export interface MujocoSimContextValue {
@@ -256,7 +372,7 @@ export function MujocoSimProvider({
256
372
  interpolate,
257
373
  children,
258
374
  }: MujocoSimProviderProps) {
259
- const { gl, camera } = useThree();
375
+ const { gl, camera, scene } = useThree();
260
376
  const [status, setStatus] = useState<'loading' | 'ready' | 'error'>('loading');
261
377
 
262
378
  // --- Refs ---
@@ -294,6 +410,7 @@ export function MujocoSimProvider({
294
410
  const bodyReloadTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
295
411
 
296
412
  useEffect(() => { configRef.current = config; }, [config]);
413
+ useEffect(() => { mujocoRef.current = mujoco; }, [mujoco]);
297
414
 
298
415
  // Sync declarative props to refs
299
416
  useEffect(() => { pausedRef.current = paused ?? false; }, [paused]);
@@ -345,6 +462,7 @@ export function MujocoSimProvider({
345
462
  return;
346
463
  }
347
464
 
465
+ mujocoRef.current = mujoco;
348
466
  mjModelRef.current = result.mjModel;
349
467
  mjDataRef.current = result.mjData;
350
468
  physicsAccumulatorRef.current = 0;
@@ -424,7 +542,7 @@ export function MujocoSimProvider({
424
542
  // Step physics with substeps
425
543
  if (stepsToRunRef.current > 0) {
426
544
  for (let s = 0; s < stepsToRunRef.current; s++) {
427
- mujoco.mj_step(model, data);
545
+ mujocoRef.current.mj_step(model, data);
428
546
  }
429
547
  stepsToRunRef.current = 0;
430
548
  } else {
@@ -433,7 +551,7 @@ export function MujocoSimProvider({
433
551
  const frameTime = clampedDelta * speedRef.current;
434
552
  while (data.time - startSimTime < frameTime) {
435
553
  for (let s = 0; s < numSubsteps; s++) {
436
- mujoco.mj_step(model, data);
554
+ mujocoRef.current.mj_step(model, data);
437
555
  }
438
556
  }
439
557
  }
@@ -441,7 +559,7 @@ export function MujocoSimProvider({
441
559
  ensureInterpolationBuffers(model);
442
560
  copyBodyPose(data, interpolationStateRef.current.previousXpos, interpolationStateRef.current.previousXquat);
443
561
  for (let s = 0; s < stepsToRunRef.current; s++) {
444
- mujoco.mj_step(model, data);
562
+ mujocoRef.current.mj_step(model, data);
445
563
  }
446
564
  copyBodyPose(data, interpolationStateRef.current.currentXpos, interpolationStateRef.current.currentXquat);
447
565
  interpolationStateRef.current.alpha = 1;
@@ -457,7 +575,7 @@ export function MujocoSimProvider({
457
575
  while (physicsAccumulatorRef.current >= stepDt) {
458
576
  copyBodyPose(data, interpolationStateRef.current.previousXpos, interpolationStateRef.current.previousXquat);
459
577
  for (let s = 0; s < numSubsteps; s++) {
460
- mujoco.mj_step(model, data);
578
+ mujocoRef.current.mj_step(model, data);
461
579
  }
462
580
  copyBodyPose(data, interpolationStateRef.current.currentXpos, interpolationStateRef.current.currentXquat);
463
581
  physicsAccumulatorRef.current -= stepDt;
@@ -508,7 +626,7 @@ export function MujocoSimProvider({
508
626
  const data = mjDataRef.current;
509
627
  if (!model || !data) return;
510
628
 
511
- mujoco.mj_resetData(model, data);
629
+ mujocoRef.current.mj_resetData(model, data);
512
630
 
513
631
  const homeJoints = configRef.current.homeJoints;
514
632
  if (homeJoints) {
@@ -523,7 +641,7 @@ export function MujocoSimProvider({
523
641
  }
524
642
 
525
643
  configRef.current.onReset?.({ model, data });
526
- mujoco.mj_forward(model, data);
644
+ mujocoRef.current.mj_forward(model, data);
527
645
 
528
646
  // Notify composable plugins (e.g. IkController)
529
647
  for (const cb of resetCallbacks.current) {
@@ -548,6 +666,30 @@ export function MujocoSimProvider({
548
666
  stepsToRunRef.current = n;
549
667
  }, []);
550
668
 
669
+ const stepImmediately = useCallback((steps = 1) => {
670
+ const model = mjModelRef.current;
671
+ const data = mjDataRef.current;
672
+ if (!model || !data) return false;
673
+
674
+ for (let stepIndex = 0; stepIndex < steps; stepIndex += 1) {
675
+ for (let i = 0; i < model.nv; i += 1) {
676
+ data.qfrc_applied[i] = 0;
677
+ }
678
+ for (const cb of beforeStepCallbacks.current) {
679
+ cb({ model, data });
680
+ }
681
+ mujocoRef.current.mj_step(model, data);
682
+ for (const cb of afterStepCallbacks.current) {
683
+ cb({ model, data });
684
+ }
685
+ onStepRef.current?.({ time: data.time, model, data });
686
+ }
687
+
688
+ physicsAccumulatorRef.current = 0;
689
+ interpolationStateRef.current.valid = false;
690
+ return true;
691
+ }, [mujoco]);
692
+
551
693
  const getTime = useCallback((): number => {
552
694
  return mjDataRef.current?.time ?? 0;
553
695
  }, []);
@@ -579,7 +721,7 @@ export function MujocoSimProvider({
579
721
  data.ctrl.set(snapshot.ctrl);
580
722
  if (snapshot.act.length > 0) data.act.set(snapshot.act);
581
723
  data.qfrc_applied.set(snapshot.qfrc_applied);
582
- mujoco.mj_forward(model, data);
724
+ mujocoRef.current.mj_forward(model, data);
583
725
  }, [mujoco]);
584
726
 
585
727
  const setQpos = useCallback((values: Float64Array | number[]) => {
@@ -588,7 +730,7 @@ export function MujocoSimProvider({
588
730
  if (!model || !data) return;
589
731
  const arr = values instanceof Float64Array ? values : new Float64Array(values);
590
732
  data.qpos.set(arr.subarray(0, Math.min(arr.length, model.nq)));
591
- mujoco.mj_forward(model, data);
733
+ mujocoRef.current.mj_forward(model, data);
592
734
  }, [mujoco]);
593
735
 
594
736
  const setQvel = useCallback((values: Float64Array | number[]) => {
@@ -831,6 +973,113 @@ export function MujocoSimProvider({
831
973
  return result;
832
974
  }, []);
833
975
 
976
+ const getCameras = useCallback((): CameraInfo[] => {
977
+ const model = mjModelRef.current;
978
+ if (!model) return [];
979
+ const ncam = model.ncam ?? 0;
980
+ const nameAddresses = model.name_camadr;
981
+ if (!ncam || !nameAddresses) return [];
982
+
983
+ const result: CameraInfo[] = [];
984
+ for (let i = 0; i < ncam; i += 1) {
985
+ const posOffset = i * 3;
986
+ const quatOffset = i * 4;
987
+ result.push({
988
+ id: i,
989
+ name: getName(model, nameAddresses[i]),
990
+ bodyId: model.cam_bodyid?.[i] ?? -1,
991
+ fov: model.cam_fovy?.[i] ?? null,
992
+ position: model.cam_pos
993
+ ? vector3FromArray(model.cam_pos, posOffset)
994
+ : null,
995
+ quaternion: model.cam_quat
996
+ ? quaternionFromArray(model.cam_quat, quatOffset)
997
+ : null,
998
+ });
999
+ }
1000
+ return result;
1001
+ }, []);
1002
+
1003
+ const resolveCameraCaptureOptions = useCallback(
1004
+ (options: CameraFrameCaptureOptions = {}): CameraFrameCaptureOptions => {
1005
+ const model = mjModelRef.current;
1006
+ const data = mjDataRef.current;
1007
+ if (!model || !data) {
1008
+ return options;
1009
+ }
1010
+
1011
+ const baseOptions = omitResolvedCameraSelectors(options);
1012
+
1013
+ if (options.cameraName) {
1014
+ const cameraId = findCameraByName(model, options.cameraName);
1015
+ if (cameraId < 0) {
1016
+ throw new Error(`MuJoCo camera "${options.cameraName}" was not found.`);
1017
+ }
1018
+
1019
+ const position = data.cam_xpos
1020
+ ? vector3FromArray(data.cam_xpos, cameraId * 3)
1021
+ : model.cam_pos
1022
+ ? vector3FromArray(model.cam_pos, cameraId * 3)
1023
+ : undefined;
1024
+ const quaternion = data.cam_xmat
1025
+ ? quaternionFromXmat(data.cam_xmat, cameraId * 9)
1026
+ : model.cam_quat
1027
+ ? quaternionFromArray(model.cam_quat, cameraId * 4)
1028
+ : undefined;
1029
+
1030
+ if (!position || !quaternion) {
1031
+ throw new Error(
1032
+ `MuJoCo camera "${options.cameraName}" does not expose a capture pose.`
1033
+ );
1034
+ }
1035
+
1036
+ return {
1037
+ ...baseOptions,
1038
+ position,
1039
+ quaternion,
1040
+ fov: options.fov ?? model.cam_fovy?.[cameraId],
1041
+ source: { kind: 'mujoco-camera', cameraName: options.cameraName },
1042
+ };
1043
+ }
1044
+
1045
+ if (options.siteName) {
1046
+ const siteId = findSiteByName(model, options.siteName);
1047
+ if (siteId < 0) {
1048
+ throw new Error(`MuJoCo site "${options.siteName}" was not found.`);
1049
+ }
1050
+
1051
+ return {
1052
+ ...baseOptions,
1053
+ position: vector3FromArray(data.site_xpos, siteId * 3),
1054
+ quaternion: quaternionFromXmat(data.site_xmat, siteId * 9),
1055
+ source: { kind: 'mujoco-site', siteName: options.siteName },
1056
+ };
1057
+ }
1058
+
1059
+ if (options.bodyName) {
1060
+ const bodyId = findBodyByName(model, options.bodyName);
1061
+ if (bodyId < 0) {
1062
+ throw new Error(`MuJoCo body "${options.bodyName}" was not found.`);
1063
+ }
1064
+ if (!data.xmat) {
1065
+ throw new Error(
1066
+ `MuJoCo body "${options.bodyName}" does not expose world orientation data.`
1067
+ );
1068
+ }
1069
+
1070
+ return {
1071
+ ...baseOptions,
1072
+ position: vector3FromArray(data.xpos, bodyId * 3),
1073
+ quaternion: quaternionFromXmat(data.xmat, bodyId * 9),
1074
+ source: { kind: 'mujoco-body', bodyName: options.bodyName },
1075
+ };
1076
+ }
1077
+
1078
+ return options;
1079
+ },
1080
+ []
1081
+ );
1082
+
834
1083
  const getModelOption = useCallback((): ModelOptions => {
835
1084
  const model = mjModelRef.current;
836
1085
  if (!model?.opt) return { timestep: 0.002, gravity: [0, 0, -9.81], integrator: 0 };
@@ -913,7 +1162,7 @@ export function MujocoSimProvider({
913
1162
  for (let i = 0; i < model.nv; i++) data.qvel[i] = model.key_qvel[qvelOffset + i];
914
1163
  }
915
1164
 
916
- mujoco.mj_forward(model, data);
1165
+ mujocoRef.current.mj_forward(model, data);
917
1166
 
918
1167
  // Notify composable plugins
919
1168
  for (const cb of resetCallbacks.current) {
@@ -1041,6 +1290,213 @@ export function MujocoSimProvider({
1041
1290
  [gl]
1042
1291
  );
1043
1292
 
1293
+ const captureCameraFrameApi = useCallback(
1294
+ (options = {}) => {
1295
+ return captureCameraFrame(
1296
+ gl,
1297
+ scene,
1298
+ camera,
1299
+ resolveCameraCaptureOptions(options)
1300
+ );
1301
+ },
1302
+ [camera, gl, resolveCameraCaptureOptions, scene]
1303
+ );
1304
+
1305
+ const captureCameraFrameBlobApi = useCallback(
1306
+ (options = {}) => {
1307
+ return captureCameraFrameBlob(
1308
+ gl,
1309
+ scene,
1310
+ camera,
1311
+ resolveCameraCaptureOptions(options)
1312
+ );
1313
+ },
1314
+ [camera, gl, resolveCameraCaptureOptions, scene]
1315
+ );
1316
+
1317
+ const recordCameraSequenceApi = useCallback(
1318
+ async (
1319
+ options: CameraFrameSequenceOptions
1320
+ ): Promise<CameraFrameSequenceResult> => {
1321
+ const frameCount = Math.max(0, Math.floor(options.frames));
1322
+ const stepsPerFrame = Math.max(0, Math.floor(options.stepsPerFrame ?? 1));
1323
+ const cameras = options.cameras;
1324
+ const frames: CameraFrameSequenceFrame[] = [];
1325
+ const cameraSummaries: CameraFrameSequenceResult['cameraSummaries'] = {};
1326
+ const wasPaused = pausedRef.current;
1327
+ const retainFrames = options.retainFrames ?? true;
1328
+ const requireMountedSources = options.requireMountedSources ?? true;
1329
+ let recordedFrameCount = 0;
1330
+
1331
+ async function stepCameraSequence(frameIndex: number, steps: number) {
1332
+ const model = mjModelRef.current;
1333
+ const data = mjDataRef.current;
1334
+ if (!model || !data) {
1335
+ throw new Error('MuJoCo scene is not ready for camera sequence stepping.');
1336
+ }
1337
+
1338
+ for (let stepIndex = 0; stepIndex < steps; stepIndex += 1) {
1339
+ for (let i = 0; i < model.nv; i += 1) {
1340
+ data.qfrc_applied[i] = 0;
1341
+ }
1342
+ await options.onBeforeStep?.({
1343
+ frameIndex,
1344
+ stepIndex,
1345
+ time: data.time,
1346
+ model,
1347
+ data,
1348
+ });
1349
+ for (const cb of beforeStepCallbacks.current) {
1350
+ cb({ model, data });
1351
+ }
1352
+ mujocoRef.current.mj_step(model, data);
1353
+ for (const cb of afterStepCallbacks.current) {
1354
+ cb({ model, data });
1355
+ }
1356
+ onStepRef.current?.({ time: data.time, model, data });
1357
+ await options.onAfterStep?.({
1358
+ frameIndex,
1359
+ stepIndex,
1360
+ time: data.time,
1361
+ model,
1362
+ data,
1363
+ });
1364
+ }
1365
+
1366
+ physicsAccumulatorRef.current = 0;
1367
+ interpolationStateRef.current.valid = false;
1368
+ }
1369
+
1370
+ if (frameCount === 0 || cameras.length === 0) {
1371
+ return {
1372
+ frames,
1373
+ cameraKeys: cameras.map((sequenceCamera) => sequenceCamera.key),
1374
+ cameraSummaries,
1375
+ frameCount: 0,
1376
+ };
1377
+ }
1378
+
1379
+ throwIfCameraSequenceAborted(options.signal);
1380
+
1381
+ for (let attempt = 0; attempt < 30; attempt += 1) {
1382
+ if (mjModelRef.current && mjDataRef.current) break;
1383
+ await waitForNextAnimationFrame();
1384
+ throwIfCameraSequenceAborted(options.signal);
1385
+ }
1386
+ if (!mjModelRef.current || !mjDataRef.current) {
1387
+ throw new Error('MuJoCo scene is not ready for camera sequence recording.');
1388
+ }
1389
+
1390
+ const captureSessions = cameras.map((sequenceCamera) => {
1391
+ const { key, ...captureOptions } = sequenceCamera;
1392
+ const initialCaptureOptions = resolveCameraCaptureOptions(captureOptions);
1393
+ const mountedSource = initialCaptureOptions.source;
1394
+ if (requireMountedSources) {
1395
+ assertMatchingMountedCameraSource(
1396
+ key,
1397
+ captureOptions,
1398
+ mountedSource ?? { kind: 'fallback-camera' }
1399
+ );
1400
+ }
1401
+ return {
1402
+ key,
1403
+ captureOptions,
1404
+ mountedSource,
1405
+ session: createCameraFrameCaptureSession(
1406
+ gl,
1407
+ scene,
1408
+ camera,
1409
+ initialCaptureOptions
1410
+ ),
1411
+ };
1412
+ });
1413
+
1414
+ try {
1415
+ pausedRef.current = true;
1416
+ stepsToRunRef.current = 0;
1417
+ if (options.reset) reset();
1418
+
1419
+ for (let frameIndex = 0; frameIndex < frameCount; frameIndex += 1) {
1420
+ throwIfCameraSequenceAborted(options.signal);
1421
+ if (
1422
+ stepsPerFrame > 0 &&
1423
+ (frameIndex > 0 || options.captureInitialFrame === false)
1424
+ ) {
1425
+ await stepCameraSequence(frameIndex, stepsPerFrame);
1426
+ }
1427
+ await waitForNextAnimationFrame();
1428
+ throwIfCameraSequenceAborted(options.signal);
1429
+
1430
+ const model = mjModelRef.current;
1431
+ const data = mjDataRef.current;
1432
+ if (!model || !data) {
1433
+ throw new Error('MuJoCo scene is not ready for camera sequence sampling.');
1434
+ }
1435
+ await options.onSample?.({
1436
+ frameIndex,
1437
+ time: data.time,
1438
+ model,
1439
+ data,
1440
+ });
1441
+
1442
+ const cameraFrames: Record<string, CameraFrameCaptureResult> = {};
1443
+ for (const { key, captureOptions, mountedSource, session } of captureSessions) {
1444
+ const resolvedCaptureOptions = resolveCameraCaptureOptions(captureOptions);
1445
+ const cameraFrame = session.captureDataUrl({
1446
+ ...resolvedCaptureOptions,
1447
+ source: mountedSource ?? resolvedCaptureOptions.source,
1448
+ });
1449
+ if (requireMountedSources) {
1450
+ assertMatchingMountedCameraSource(
1451
+ key,
1452
+ captureOptions,
1453
+ cameraFrame.source
1454
+ );
1455
+ }
1456
+ cameraSummaries[key] = {
1457
+ key,
1458
+ width: cameraFrame.width,
1459
+ height: cameraFrame.height,
1460
+ source: cameraFrame.source,
1461
+ frameCount: (cameraSummaries[key]?.frameCount ?? 0) + 1,
1462
+ firstFrameIndex:
1463
+ cameraSummaries[key]?.firstFrameIndex ?? frameIndex,
1464
+ lastFrameIndex: frameIndex,
1465
+ firstTimestamp:
1466
+ cameraSummaries[key]?.firstTimestamp ?? data.time,
1467
+ lastTimestamp: data.time,
1468
+ };
1469
+ cameraFrames[key] = cameraFrame;
1470
+ }
1471
+
1472
+ const frame = {
1473
+ frameIndex,
1474
+ time: getTime(),
1475
+ cameras: cameraFrames,
1476
+ };
1477
+ if (retainFrames) {
1478
+ frames.push(frame);
1479
+ }
1480
+ recordedFrameCount += 1;
1481
+ await options.onFrame?.(frame);
1482
+ }
1483
+ } finally {
1484
+ for (const { session } of captureSessions) {
1485
+ session.dispose();
1486
+ }
1487
+ pausedRef.current = wasPaused;
1488
+ }
1489
+
1490
+ return {
1491
+ frames,
1492
+ cameraKeys: cameras.map((sequenceCamera) => sequenceCamera.key),
1493
+ cameraSummaries,
1494
+ frameCount: recordedFrameCount,
1495
+ };
1496
+ },
1497
+ [camera, getTime, gl, mujoco, reset, resolveCameraCaptureOptions, scene]
1498
+ );
1499
+
1044
1500
  const project2DTo3D = useCallback(
1045
1501
  (x: number, y: number, cameraPos: THREE.Vector3, lookAt: THREE.Vector3): { point: THREE.Vector3; bodyId: number; geomId: number } | null => {
1046
1502
  const virtCam = (camera as THREE.PerspectiveCamera).clone();
@@ -1139,6 +1595,7 @@ export function MujocoSimProvider({
1139
1595
  getSites,
1140
1596
  getActuators: getActuatorsApi,
1141
1597
  getSensors,
1598
+ getCameras,
1142
1599
  getModelOption,
1143
1600
  setGravity,
1144
1601
  setTimestep: setTimestepApi,
@@ -1154,6 +1611,9 @@ export function MujocoSimProvider({
1154
1611
  getCanvasSnapshot,
1155
1612
  captureFrame: captureFrameApi,
1156
1613
  captureFrameBlob: captureFrameBlobApi,
1614
+ captureCameraFrame: captureCameraFrameApi,
1615
+ captureCameraFrameBlob: captureCameraFrameBlobApi,
1616
+ recordCameraSequence: recordCameraSequenceApi,
1157
1617
  project2DTo3D,
1158
1618
  setBodyMass,
1159
1619
  setGeomFriction,
@@ -1168,10 +1628,12 @@ export function MujocoSimProvider({
1168
1628
  getControlMapApi, getActuatedJointsApi, resolveControlGroupApi,
1169
1629
  applyForce, applyTorqueApi, setExternalForce, applyGeneralizedForce,
1170
1630
  getSensorData, getContacts, getBodies, getJoints, getGeoms, getSites,
1171
- getActuatorsApi, getSensors, getModelOption, setGravity, setTimestepApi,
1631
+ getActuatorsApi, getSensors, getCameras, getModelOption, setGravity, setTimestepApi,
1172
1632
  raycast, getKeyframeNames, getKeyframeCount, loadSceneApi,
1173
1633
  loadFromFilesApi, addBodyApi, removeBodyApi, recompileApi,
1174
1634
  getCanvas, getCanvasSnapshot, captureFrameApi, captureFrameBlobApi,
1635
+ captureCameraFrameApi, captureCameraFrameBlobApi,
1636
+ recordCameraSequenceApi,
1175
1637
  project2DTo3D,
1176
1638
  setBodyMass, setGeomFriction, setGeomSize,
1177
1639
  ]
@@ -110,6 +110,19 @@ export function findSensorByName(mjModel: MujocoModel, name: string): number {
110
110
  return -1;
111
111
  }
112
112
 
113
+ /**
114
+ * Find a camera by name in the MuJoCo model. Returns -1 if not found.
115
+ */
116
+ export function findCameraByName(mjModel: MujocoModel, name: string): number {
117
+ const ncam = mjModel.ncam ?? 0;
118
+ const addresses = mjModel.name_camadr;
119
+ if (!addresses) return -1;
120
+ for (let i = 0; i < ncam; i++) {
121
+ if (getName(mjModel, addresses[i]) === name) return i;
122
+ }
123
+ return -1;
124
+ }
125
+
113
126
  /**
114
127
  * Find a tendon by name in the MuJoCo model. Returns -1 if not found.
115
128
  */
@@ -43,8 +43,10 @@ export type ControllerComponent<TConfig> = React.FC<{
43
43
  * const MyController = createController<{ speed: number }>(
44
44
  * { name: 'my-controller', defaultConfig: { speed: 1.0 } },
45
45
  * function MyControllerImpl({ config }) {
46
+ * const wheel = useCtrl(RobotActuators.mobile.leftWheel);
47
+ *
46
48
  * useBeforePhysicsStep(({ data }) => {
47
- * data.ctrl[0] = config.speed;
49
+ * wheel.write(config.speed);
48
50
  * });
49
51
  * return null;
50
52
  * },
@@ -100,9 +102,11 @@ export function createController<TConfig>(
100
102
  * { name: 'useMyController', defaultConfig: { gain: 1.0 } },
101
103
  * function useMyControllerImpl(config) {
102
104
  * // config is MyConfig | null — hooks must be called unconditionally
105
+ * const joint = useCtrl(config?.actuator ?? RobotActuators.franka.actuator1);
106
+ *
103
107
  * useBeforePhysicsStep(({ data }) => {
104
108
  * if (!config) return;
105
- * data.ctrl[0] = config.gain * Math.sin(data.time);
109
+ * joint.write(config.gain * Math.sin(data.time));
106
110
  * });
107
111
  * if (!config) return null;
108
112
  * return { /* value *\/ };