mujoco-react 12.0.0 → 12.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -1,5 +1,6 @@
1
- import { withContacts, getContact, captureCameraFrame, captureCameraFrameBlob, createCameraFrameCaptureSession, CAMERA_FRAME_CAPTURE_PRE_RENDER_USER_DATA_KEY, CAPTURE_EXCLUDE_KEY, createCaptureCamera, prepareCaptureCamera } from './chunk-7BCIHJMW.js';
2
- export { CAMERA_FRAME_CAPTURE_PRE_RENDER_USER_DATA_KEY, CAMERA_FRAME_CAPTURE_RENDER_USER_DATA_KEY, CAPTURE_EXCLUDE_KEY, ModelActuators, ModelBodies, ModelCameras, ModelGeoms, ModelJoints, ModelKeyframes, ModelResources, ModelSensors, ModelSites, ScenarioLighting, SplatEnvironment, SplatEnvironmentReadinessStatus, VisualScenarioEffects, captureCameraFrame, captureCameraFrameBlob, captureCameraFrameTensor, createCameraFrameCaptureSession, createPairedSplatEnvironment, createSparkSplatViewerUrl, createSplatEnvironmentUserData, createSplatSceneConfig, createVisualScenarioExecutionContext, dataUrlToPolicyImageTensor, getContact, getScenarioBackground, getScenarioCameraPosition, getSplatEnvironmentReadiness, imageDataToPolicyImageTensor, pixelsToPolicyImageTensor, registerModelResources, renderCameraFrameToCanvas, useSplatEnvironment, useSplatSceneConfig, useVisualScenarioEffects, useVisualScenarioExecutionContext, withSplatEnvironment } from './chunk-7BCIHJMW.js';
1
+ import { displayFields } from './chunk-U2Y2BODT.js';
2
+ import { withContacts, getContact, captureCameraFrame, captureCameraFrameBlob, createCameraFrameCaptureSession, CAMERA_FRAME_CAPTURE_PRE_RENDER_USER_DATA_KEY, CAPTURE_EXCLUDE_KEY, createCaptureCamera, prepareCaptureCamera } from './chunk-XD3AY7XZ.js';
3
+ export { CAMERA_FRAME_CAPTURE_PRE_RENDER_USER_DATA_KEY, CAMERA_FRAME_CAPTURE_RENDER_USER_DATA_KEY, CAPTURE_EXCLUDE_KEY, ModelActuators, ModelBodies, ModelCameras, ModelGeoms, ModelJoints, ModelKeyframes, ModelResources, ModelSensors, ModelSites, ScenarioLighting, SplatEnvironment, SplatEnvironmentReadinessStatus, VisualScenarioEffects, captureCameraFrame, captureCameraFrameBlob, captureCameraFrameTensor, createCameraFrameCaptureSession, createPairedSplatEnvironment, createSparkSplatViewerUrl, createSplatEnvironmentUserData, createSplatSceneConfig, createVisualScenarioExecutionContext, dataUrlToPolicyImageTensor, getContact, getScenarioBackground, getScenarioCameraPosition, getSplatEnvironmentReadiness, imageDataToPolicyImageTensor, pixelsToPolicyImageTensor, registerModelResources, renderCameraFrameToCanvas, useSplatEnvironment, useSplatSceneConfig, useVisualScenarioEffects, useVisualScenarioExecutionContext, withSplatEnvironment } from './chunk-XD3AY7XZ.js';
3
4
  import loadMujoco from '@mujoco/mujoco';
4
5
  import defaultMujocoWasmUrl from '@mujoco/mujoco/mujoco.wasm?url';
5
6
  import { createContext, forwardRef, useRef, useEffect, useContext, useState, useCallback, useMemo, useLayoutEffect, useId } from 'react';
@@ -975,939 +976,1322 @@ function collectDependencyPaths(xmlString, currentFile, parser) {
975
976
  });
976
977
  return paths;
977
978
  }
978
- var CapsuleGeometry = class extends THREE12.BufferGeometry {
979
- parameters;
980
- constructor(radius = 1, length = 1, capSegments = 4, radialSegments = 8) {
981
- super();
982
- this.type = "CapsuleGeometry";
983
- this.parameters = { radius, length, capSegments, radialSegments };
984
- const path = new THREE12.Path();
985
- path.absarc(0, -length / 2, radius, Math.PI * 1.5, 0, false);
986
- path.absarc(0, length / 2, radius, 0, Math.PI * 0.5, false);
987
- const latheGeometry = new THREE12.LatheGeometry(path.getPoints(capSegments), radialSegments);
988
- const self = this;
989
- self.setIndex(latheGeometry.getIndex());
990
- self.setAttribute("position", latheGeometry.getAttribute("position"));
991
- self.setAttribute("normal", latheGeometry.getAttribute("normal"));
992
- self.setAttribute("uv", latheGeometry.getAttribute("uv"));
993
- }
994
- };
995
979
 
996
- // src/rendering/GeomBuilder.ts
997
- var DEFAULT_MESH_NORMAL_SMOOTHING_TOLERANCE = 1e-4;
998
- var GeomBuilder = class {
999
- mujoco;
1000
- textureCache = /* @__PURE__ */ new Map();
1001
- renderOptions;
1002
- constructor(mujoco, renderOptions) {
1003
- this.mujoco = mujoco;
1004
- this.renderOptions = renderOptions;
980
+ // src/core/sceneIdentity.ts
981
+ var fileIds = /* @__PURE__ */ new WeakMap();
982
+ var nextFileId = 0;
983
+ function sceneIdentity(config) {
984
+ return JSON.stringify([
985
+ config.src,
986
+ config.sceneFile,
987
+ config.environmentFiles ?? [],
988
+ config.sceneObjects ?? [],
989
+ config.homeJoints ?? [],
990
+ config.xmlPatches ?? [],
991
+ (config.files ?? []).map((file) => {
992
+ if (!fileIds.has(file)) fileIds.set(file, ++nextFileId);
993
+ return fileIds.get(file);
994
+ })
995
+ ], (_key, value) => value && typeof value === "object" && !Array.isArray(value) ? Object.fromEntries(Object.entries(value).sort(([a], [b]) => a.localeCompare(b))) : value);
996
+ }
997
+
998
+ // src/core/stepBudget.ts
999
+ function runStepBudget(requested, step, budgetMs = 8, now2 = () => performance.now()) {
1000
+ if (!Number.isFinite(budgetMs) || budgetMs <= 0) throw new Error("physicsBudgetMs must be positive and finite");
1001
+ const start = now2();
1002
+ let steps = 0;
1003
+ while (steps < requested && steps < 100) {
1004
+ step();
1005
+ steps++;
1006
+ if (now2() - start >= budgetMs) break;
1007
+ }
1008
+ return { steps, physicsMs: now2() - start, budgetExceeded: steps < requested };
1009
+ }
1010
+
1011
+ // src/rendering/cameraFrameSource.ts
1012
+ var MountedCameraFrameSourceSuggestionMatch = {
1013
+ Direct: "direct",
1014
+ Alias: "alias",
1015
+ Normalized: "normalized",
1016
+ Prefix: "prefix",
1017
+ Suffix: "suffix",
1018
+ Contains: "contains"
1019
+ };
1020
+ var MountedCameraFrameSequenceReadinessStatus = {
1021
+ Ready: "ready",
1022
+ Partial: "partial",
1023
+ Missing: "missing"
1024
+ };
1025
+ var MountedCameraFrameSequenceManifestStatus = {
1026
+ Complete: "complete",
1027
+ Partial: "partial",
1028
+ Missing: "missing"
1029
+ };
1030
+ function getResourceName(resource) {
1031
+ if (!resource) return null;
1032
+ return typeof resource === "string" ? resource : resource.name ?? null;
1033
+ }
1034
+ function createNameSet(resources) {
1035
+ return new Set(
1036
+ (resources ?? []).map((resource) => getResourceName(resource)).filter((name) => Boolean(name))
1037
+ );
1038
+ }
1039
+ function createResourceNames(resources) {
1040
+ return (resources ?? []).map((resource) => getResourceName(resource)).filter((name) => Boolean(name));
1041
+ }
1042
+ function normalizeAliasCandidates(value) {
1043
+ if (!value) return [];
1044
+ return Array.isArray(value) ? value : [value];
1045
+ }
1046
+ function countMountedSelectors(selector) {
1047
+ return Number(Boolean(selector.cameraName)) + Number(Boolean(selector.siteName)) + Number(Boolean(selector.bodyName));
1048
+ }
1049
+ function normalizeCameraSourceName(value) {
1050
+ return value.trim().toLowerCase().replace(/[^a-z0-9]+/g, "_").replace(/^_+|_+$/g, "");
1051
+ }
1052
+ function createCameraSourceKeyVariants(key) {
1053
+ const candidates = [
1054
+ key,
1055
+ key.startsWith("observation.images.") ? key.slice("observation.images.".length) : "",
1056
+ key.includes(".") ? key.split(".").at(-1) ?? "" : "",
1057
+ key.includes("/") ? key.split("/").at(-1) ?? "" : ""
1058
+ ];
1059
+ return candidates.map((candidate) => candidate.trim()).filter((candidate, index, items) => candidate && items.indexOf(candidate) === index);
1060
+ }
1061
+ function getSelectorKey(selector) {
1062
+ if (selector.cameraName) return `camera:${selector.cameraName}`;
1063
+ if (selector.siteName) return `site:${selector.siteName}`;
1064
+ if (selector.bodyName) return `body:${selector.bodyName}`;
1065
+ return null;
1066
+ }
1067
+ function getMountedCameraFrameCaptureSource(selector) {
1068
+ if (countMountedSelectors(selector) !== 1) return null;
1069
+ if (selector.cameraName) {
1070
+ return { kind: "mujoco-camera", cameraName: selector.cameraName };
1005
1071
  }
1006
- getMeshNormalSmoothingTolerance() {
1007
- const smoothing = this.renderOptions?.meshNormalSmoothing;
1008
- if (!smoothing) return null;
1009
- if (smoothing === true) return DEFAULT_MESH_NORMAL_SMOOTHING_TOLERANCE;
1010
- return smoothing.tolerance ?? DEFAULT_MESH_NORMAL_SMOOTHING_TOLERANCE;
1072
+ if (selector.siteName) {
1073
+ return { kind: "mujoco-site", siteName: selector.siteName };
1011
1074
  }
1012
- getMaterialTexture(mjModel, matId) {
1013
- if (matId < 0 || !mjModel.mat_texid || !mjModel.tex_data) return null;
1014
- const materialCount = Math.max(1, Math.floor(mjModel.mat_rgba.length / 4));
1015
- const textureRoles = Math.max(1, Math.floor(mjModel.mat_texid.length / materialCount));
1016
- let texId = -1;
1017
- for (let role = 0; role < textureRoles; role += 1) {
1018
- const candidate = mjModel.mat_texid[matId * textureRoles + role];
1019
- if (candidate >= 0) {
1020
- texId = candidate;
1021
- break;
1022
- }
1075
+ if (selector.bodyName) {
1076
+ return { kind: "mujoco-body", bodyName: selector.bodyName };
1077
+ }
1078
+ return null;
1079
+ }
1080
+ function isMountedCameraFrameCaptureSource(source) {
1081
+ return source.kind === "mujoco-camera" || source.kind === "mujoco-site" || source.kind === "mujoco-body";
1082
+ }
1083
+ function getCameraFrameCaptureSourceTarget(source) {
1084
+ if (source.kind === "mujoco-camera") return source.cameraName;
1085
+ if (source.kind === "mujoco-site") return source.siteName;
1086
+ if (source.kind === "mujoco-body") return source.bodyName;
1087
+ if (source.kind === "custom-camera") return "custom camera";
1088
+ if (source.kind === "explicit-pose") return "explicit pose";
1089
+ return "fallback camera";
1090
+ }
1091
+ function createMountedCameraFrameSourceSuggestion(key, selector, resourceName, match) {
1092
+ const source = getMountedCameraFrameCaptureSource(selector);
1093
+ if (!source) return null;
1094
+ return {
1095
+ key,
1096
+ selector,
1097
+ source,
1098
+ resourceName,
1099
+ resourceKind: source.kind,
1100
+ match
1101
+ };
1102
+ }
1103
+ function addMountedCameraFrameSourceSuggestion(suggestions, seen, suggestion) {
1104
+ if (!suggestion) return;
1105
+ const selectorKey = getSelectorKey(suggestion.selector);
1106
+ if (!selectorKey || seen.has(selectorKey)) return;
1107
+ seen.add(selectorKey);
1108
+ suggestions.push(suggestion);
1109
+ }
1110
+ function getCameraFrameResourceMatch(key, resourceName) {
1111
+ if (resourceName === key) return MountedCameraFrameSourceSuggestionMatch.Direct;
1112
+ const normalizedResource = normalizeCameraSourceName(resourceName);
1113
+ if (!normalizedResource) return null;
1114
+ for (const variant of createCameraSourceKeyVariants(key)) {
1115
+ if (resourceName === variant) return MountedCameraFrameSourceSuggestionMatch.Direct;
1116
+ const normalizedKey = normalizeCameraSourceName(variant);
1117
+ if (!normalizedKey) continue;
1118
+ if (normalizedResource === normalizedKey) {
1119
+ return MountedCameraFrameSourceSuggestionMatch.Normalized;
1023
1120
  }
1024
- if (texId < 0) return null;
1025
- const cached = this.textureCache.get(texId);
1026
- if (cached) return cached;
1027
- const width = Number(mjModel.tex_width[texId]);
1028
- const height = Number(mjModel.tex_height[texId]);
1029
- const channels = Number(mjModel.tex_nchannel[texId]);
1030
- const offset = Number(mjModel.tex_adr[texId]);
1031
- if (width <= 0 || height <= 0 || channels <= 0 || offset < 0) return null;
1032
- const source = mjModel.tex_data.subarray(offset, offset + width * height * channels);
1033
- const rgba = new Uint8Array(width * height * 4);
1034
- for (let i = 0, j = 0; i < width * height; i += 1, j += channels) {
1035
- const r = source[j] ?? 255;
1036
- const g = channels > 1 ? source[j + 1] : r;
1037
- const b = channels > 2 ? source[j + 2] : r;
1038
- const a = channels > 3 ? source[j + 3] : 255;
1039
- const out = i * 4;
1040
- rgba[out] = r;
1041
- rgba[out + 1] = g;
1042
- rgba[out + 2] = b;
1043
- rgba[out + 3] = a;
1121
+ if (normalizedResource.startsWith(`${normalizedKey}_`)) {
1122
+ return MountedCameraFrameSourceSuggestionMatch.Prefix;
1123
+ }
1124
+ if (normalizedResource.endsWith(`_${normalizedKey}`)) {
1125
+ return MountedCameraFrameSourceSuggestionMatch.Suffix;
1126
+ }
1127
+ if (normalizedResource.includes(`_${normalizedKey}_`)) {
1128
+ return MountedCameraFrameSourceSuggestionMatch.Contains;
1044
1129
  }
1045
- const texture = new THREE12.DataTexture(rgba, width, height, THREE12.RGBAFormat);
1046
- texture.colorSpace = THREE12.LinearSRGBColorSpace;
1047
- texture.wrapS = THREE12.RepeatWrapping;
1048
- texture.wrapT = THREE12.RepeatWrapping;
1049
- texture.flipY = true;
1050
- const repeatOffset = matId * 2;
1051
- const repeatS = mjModel.mat_texrepeat?.[repeatOffset] ?? 1;
1052
- const repeatT = mjModel.mat_texrepeat?.[repeatOffset + 1] ?? 1;
1053
- texture.repeat.set(repeatS || 1, repeatT || 1);
1054
- texture.needsUpdate = true;
1055
- this.textureCache.set(texId, texture);
1056
- return texture;
1057
1130
  }
1058
- /**
1059
- * Creates a Three.js Object3D (usually a Mesh) for a specific geometry in the MuJoCo model.
1060
- * Returns null if the geometry shouldn't be rendered (e.g., invisible collision triggers).
1061
- */
1062
- create(mjModel, g) {
1063
- if (mjModel.geom_group[g] === 3) return null;
1064
- const type = mjModel.geom_type[g];
1065
- const size = mjModel.geom_size.subarray(g * 3, g * 3 + 3);
1066
- const pos = mjModel.geom_pos.subarray(g * 3, g * 3 + 3);
1067
- const quat = mjModel.geom_quat.subarray(g * 4, g * 4 + 4);
1068
- const matId = mjModel.geom_matid[g];
1069
- const color = new THREE12.Color(16777215);
1070
- const map = this.getMaterialTexture(mjModel, matId);
1071
- let opacity = 1;
1072
- if (matId >= 0) {
1073
- const rgba = mjModel.mat_rgba.subarray(matId * 4, matId * 4 + 4);
1074
- color.setRGB(rgba[0], rgba[1], rgba[2]);
1075
- opacity = rgba[3];
1076
- } else {
1077
- const rgba = mjModel.geom_rgba.subarray(g * 4, g * 4 + 4);
1078
- color.setRGB(rgba[0], rgba[1], rgba[2]);
1079
- opacity = rgba[3];
1131
+ return null;
1132
+ }
1133
+ function isSelectorMounted(selector, cameraNames, siteNames, bodyNames) {
1134
+ if (countMountedSelectors(selector) !== 1) return false;
1135
+ return (selector.cameraName ? cameraNames.has(selector.cameraName) : false) || (selector.siteName ? siteNames.has(selector.siteName) : false) || (selector.bodyName ? bodyNames.has(selector.bodyName) : false);
1136
+ }
1137
+ function createMountedCameraFrameSourceSuggestions(key, options) {
1138
+ const cameraNames = createNameSet(options.cameras);
1139
+ const siteNames = createNameSet(options.sites);
1140
+ const bodyNames = createNameSet(options.bodies);
1141
+ const suggestions = [];
1142
+ const seen = /* @__PURE__ */ new Set();
1143
+ for (const selector of normalizeAliasCandidates(options.aliases?.[key])) {
1144
+ if (!isSelectorMounted(selector, cameraNames, siteNames, bodyNames)) {
1145
+ continue;
1080
1146
  }
1081
- const MG = this.mujoco.mjtGeom;
1082
- let geo = null;
1083
- const getVal = (v) => v?.value ?? v;
1084
- if (type === getVal(MG.mjGEOM_PLANE)) {
1085
- geo = new THREE12.PlaneGeometry(size[0] * 2 || 5, size[1] * 2 || 5);
1086
- } else if (type === getVal(MG.mjGEOM_SPHERE)) {
1087
- geo = new THREE12.SphereGeometry(size[0], 24, 24);
1088
- } else if (type === getVal(MG.mjGEOM_CAPSULE)) {
1089
- geo = new CapsuleGeometry(size[0], size[1] * 2, 24, 12);
1090
- geo.rotateX(Math.PI / 2);
1091
- } else if (type === getVal(MG.mjGEOM_BOX)) {
1092
- geo = new THREE12.BoxGeometry(size[0] * 2, size[1] * 2, size[2] * 2);
1093
- } else if (type === getVal(MG.mjGEOM_CYLINDER)) {
1094
- geo = new THREE12.CylinderGeometry(size[0], size[0], size[1] * 2, 24);
1095
- geo.rotateX(Math.PI / 2);
1096
- } else if (type === getVal(MG.mjGEOM_MESH)) {
1097
- const mId = mjModel.geom_dataid[g];
1098
- const vAdr = mjModel.mesh_vertadr[mId];
1099
- const vNum = mjModel.mesh_vertnum[mId];
1100
- const fAdr = mjModel.mesh_faceadr[mId];
1101
- const fNum = mjModel.mesh_facenum[mId];
1102
- const tAdr = mjModel.mesh_texcoordadr?.[mId] ?? -1;
1103
- if (map && tAdr >= 0 && mjModel.mesh_texcoord && mjModel.mesh_facetexcoord) {
1104
- const corners = fNum * 3;
1105
- const positions = new Float32Array(corners * 3);
1106
- const uvs = new Float32Array(corners * 2);
1107
- for (let c = 0; c < corners; c++) {
1108
- const v = vAdr + mjModel.mesh_face[fAdr * 3 + c];
1109
- positions[c * 3] = mjModel.mesh_vert[v * 3];
1110
- positions[c * 3 + 1] = mjModel.mesh_vert[v * 3 + 1];
1111
- positions[c * 3 + 2] = mjModel.mesh_vert[v * 3 + 2];
1112
- const t = tAdr + mjModel.mesh_facetexcoord[fAdr * 3 + c];
1113
- uvs[c * 2] = mjModel.mesh_texcoord[t * 2];
1114
- uvs[c * 2 + 1] = 1 - mjModel.mesh_texcoord[t * 2 + 1];
1115
- }
1116
- geo = new THREE12.BufferGeometry();
1117
- geo.setAttribute("position", new THREE12.Float32BufferAttribute(positions, 3));
1118
- geo.setAttribute("uv", new THREE12.Float32BufferAttribute(uvs, 2));
1119
- geo.computeVertexNormals();
1120
- } else {
1121
- geo = new THREE12.BufferGeometry();
1122
- geo.setAttribute("position", new THREE12.Float32BufferAttribute(mjModel.mesh_vert.subarray(vAdr * 3, (vAdr + vNum) * 3), 3));
1123
- geo.setIndex(Array.from(mjModel.mesh_face.subarray(fAdr * 3, (fAdr + fNum) * 3)));
1124
- const smoothingTolerance = this.getMeshNormalSmoothingTolerance();
1125
- if (smoothingTolerance !== null) {
1126
- geo = mergeVertices(geo, smoothingTolerance);
1127
- }
1128
- geo.computeVertexNormals();
1129
- }
1130
- }
1131
- if (geo) {
1132
- const isPlane = type === getVal(MG.mjGEOM_PLANE);
1133
- const materialMap = isPlane && map ? map.clone() : map;
1134
- if (isPlane && materialMap) {
1135
- materialMap.repeat.multiplyScalar(2.5);
1136
- materialMap.needsUpdate = true;
1137
- }
1138
- const mesh = new THREE12.Mesh(geo, new THREE12.MeshStandardMaterial({
1139
- color,
1140
- map: materialMap,
1141
- transparent: opacity < 1,
1142
- opacity,
1143
- roughness: 0.6,
1144
- metalness: 0
1145
- }));
1146
- mesh.castShadow = type !== getVal(MG.mjGEOM_PLANE);
1147
- mesh.receiveShadow = true;
1148
- mesh.position.set(pos[0], pos[1], pos[2]);
1149
- mesh.quaternion.set(quat[1], quat[2], quat[3], quat[0]);
1150
- mesh.userData.bodyID = mjModel.geom_bodyid[g];
1151
- mesh.userData.geomID = g;
1152
- mesh.userData.geomGroup = mjModel.geom_group[g];
1153
- mesh.userData.geomName = getName(mjModel, mjModel.name_geomadr[g]);
1154
- return mesh;
1155
- }
1156
- return null;
1147
+ const source = getMountedCameraFrameCaptureSource(selector);
1148
+ if (!source) continue;
1149
+ addMountedCameraFrameSourceSuggestion(
1150
+ suggestions,
1151
+ seen,
1152
+ createMountedCameraFrameSourceSuggestion(
1153
+ key,
1154
+ selector,
1155
+ getCameraFrameCaptureSourceTarget(source),
1156
+ MountedCameraFrameSourceSuggestionMatch.Alias
1157
+ )
1158
+ );
1157
1159
  }
1158
- };
1159
- function getRenderOptionsKey(renderOptions) {
1160
- const smoothing = renderOptions?.meshNormalSmoothing;
1161
- if (!smoothing) return "default";
1162
- if (smoothing === true) return "meshNormalSmoothing:true";
1163
- return `meshNormalSmoothing:${smoothing.tolerance ?? "default"}`;
1160
+ for (const cameraName of createResourceNames(options.cameras)) {
1161
+ const match = getCameraFrameResourceMatch(key, cameraName);
1162
+ if (!match) continue;
1163
+ addMountedCameraFrameSourceSuggestion(
1164
+ suggestions,
1165
+ seen,
1166
+ createMountedCameraFrameSourceSuggestion(
1167
+ key,
1168
+ { cameraName },
1169
+ cameraName,
1170
+ match
1171
+ )
1172
+ );
1173
+ }
1174
+ for (const siteName of createResourceNames(options.sites)) {
1175
+ const match = getCameraFrameResourceMatch(key, siteName);
1176
+ if (!match) continue;
1177
+ addMountedCameraFrameSourceSuggestion(
1178
+ suggestions,
1179
+ seen,
1180
+ createMountedCameraFrameSourceSuggestion(
1181
+ key,
1182
+ { siteName },
1183
+ siteName,
1184
+ match
1185
+ )
1186
+ );
1187
+ }
1188
+ for (const bodyName of createResourceNames(options.bodies)) {
1189
+ const match = getCameraFrameResourceMatch(key, bodyName);
1190
+ if (!match) continue;
1191
+ addMountedCameraFrameSourceSuggestion(
1192
+ suggestions,
1193
+ seen,
1194
+ createMountedCameraFrameSourceSuggestion(
1195
+ key,
1196
+ { bodyName },
1197
+ bodyName,
1198
+ match
1199
+ )
1200
+ );
1201
+ }
1202
+ return suggestions;
1164
1203
  }
1165
- function SceneRenderer({ renderOptions, ...props }) {
1166
- const {
1167
- mjModelRef,
1168
- mjDataRef,
1169
- mujocoRef,
1170
- onSelectionRef,
1171
- hiddenBodiesRef,
1172
- hiddenBodiesVersion,
1173
- interpolateRef,
1174
- interpolationStateRef,
1175
- status
1176
- } = useMujocoContext();
1177
- const groupRef = useRef(null);
1178
- const bodyRefs = useRef([]);
1179
- const prevModelRef = useRef(null);
1180
- const prevRenderOptionsKeyRef = useRef(null);
1181
- const prevHiddenVersionRef = useRef(null);
1182
- const renderOptionsKey = getRenderOptionsKey(renderOptions);
1183
- const geomBuilder = useMemo(() => {
1184
- if (status !== "ready") return null;
1185
- return new GeomBuilder(mujocoRef.current, renderOptions);
1186
- }, [status, mujocoRef, renderOptionsKey]);
1187
- useEffect(() => {
1188
- if (status !== "ready" || !geomBuilder) return;
1189
- const model = mjModelRef.current;
1190
- const group = groupRef.current;
1191
- if (!model || !group) return;
1192
- if (prevModelRef.current === model && prevRenderOptionsKeyRef.current === renderOptionsKey && prevHiddenVersionRef.current === hiddenBodiesVersion) {
1193
- return;
1194
- }
1195
- prevModelRef.current = model;
1196
- prevRenderOptionsKeyRef.current = renderOptionsKey;
1197
- prevHiddenVersionRef.current = hiddenBodiesVersion;
1198
- while (group.children.length > 0) {
1199
- group.remove(group.children[0]);
1200
- }
1201
- const refs = [];
1202
- for (let i = 0; i < model.nbody; i++) {
1203
- const bodyGroup = new THREE12.Group();
1204
- bodyGroup.userData.bodyID = i;
1205
- const bodyName = getName(model, model.name_bodyadr[i]);
1206
- if (!hiddenBodiesRef.current.has(bodyName)) {
1207
- for (let g = 0; g < model.ngeom; g++) {
1208
- if (model.geom_bodyid[g] === i) {
1209
- const mesh = geomBuilder.create(model, g);
1210
- if (mesh) bodyGroup.add(mesh);
1211
- }
1212
- }
1213
- }
1214
- group.add(bodyGroup);
1215
- refs.push(bodyGroup);
1216
- }
1217
- bodyRefs.current = refs;
1218
- }, [status, geomBuilder, mjModelRef, renderOptionsKey, hiddenBodiesRef, hiddenBodiesVersion]);
1219
- const syncBodiesToData = useCallback(() => {
1220
- const data = mjDataRef.current;
1221
- if (!data) return;
1222
- const bodies = bodyRefs.current;
1223
- const interpolation = interpolationStateRef.current;
1224
- const useInterpolation = interpolateRef.current && interpolation.valid;
1225
- for (let i = 0; i < bodies.length; i++) {
1226
- const ref = bodies[i];
1227
- if (!ref) continue;
1228
- if (useInterpolation) {
1229
- const alpha = interpolation.alpha;
1230
- const i3 = i * 3;
1231
- ref.position.set(
1232
- THREE12.MathUtils.lerp(interpolation.previousXpos[i3], interpolation.currentXpos[i3], alpha),
1233
- THREE12.MathUtils.lerp(interpolation.previousXpos[i3 + 1], interpolation.currentXpos[i3 + 1], alpha),
1234
- THREE12.MathUtils.lerp(interpolation.previousXpos[i3 + 2], interpolation.currentXpos[i3 + 2], alpha)
1235
- );
1236
- const i4 = i * 4;
1237
- _previousQuat.set(
1238
- interpolation.previousXquat[i4 + 1],
1239
- interpolation.previousXquat[i4 + 2],
1240
- interpolation.previousXquat[i4 + 3],
1241
- interpolation.previousXquat[i4]
1242
- );
1243
- _currentQuat.set(
1244
- interpolation.currentXquat[i4 + 1],
1245
- interpolation.currentXquat[i4 + 2],
1246
- interpolation.currentXquat[i4 + 3],
1247
- interpolation.currentXquat[i4]
1248
- );
1249
- ref.quaternion.copy(_previousQuat).slerp(_currentQuat, alpha);
1250
- } else {
1251
- ref.position.set(
1252
- data.xpos[i * 3],
1253
- data.xpos[i * 3 + 1],
1254
- data.xpos[i * 3 + 2]
1255
- );
1256
- ref.quaternion.set(
1257
- data.xquat[i * 4 + 1],
1258
- data.xquat[i * 4 + 2],
1259
- data.xquat[i * 4 + 3],
1260
- data.xquat[i * 4]
1261
- );
1262
- }
1263
- }
1264
- }, [interpolateRef, interpolationStateRef, mjDataRef]);
1265
- useFrame(syncBodiesToData);
1266
- return /* @__PURE__ */ jsx(
1267
- "group",
1268
- {
1269
- ...props,
1270
- ref: groupRef,
1271
- userData: {
1272
- ...props.userData,
1273
- [CAMERA_FRAME_CAPTURE_PRE_RENDER_USER_DATA_KEY]: syncBodiesToData
1274
- },
1275
- onDoubleClick: (e) => {
1276
- if (typeof props.onDoubleClick === "function") props.onDoubleClick(e);
1277
- e.stopPropagation();
1278
- let obj = e.object;
1279
- while (obj && obj.userData.bodyID === void 0 && obj.parent) {
1280
- obj = obj.parent;
1281
- }
1282
- const bodyID = obj?.userData.bodyID;
1283
- if (typeof bodyID === "number" && bodyID > 0) {
1284
- const model = mjModelRef.current;
1285
- if (model && bodyID < model.nbody && onSelectionRef.current) {
1286
- const name = getName(model, model.name_bodyadr[bodyID]);
1287
- onSelectionRef.current({ bodyId: bodyID, name });
1288
- }
1289
- }
1290
- }
1204
+ function resolveMountedCameraFrameSource(key, options) {
1205
+ const cameraNames = createNameSet(options.cameras);
1206
+ const siteNames = createNameSet(options.sites);
1207
+ const bodyNames = createNameSet(options.bodies);
1208
+ const directCandidates = [
1209
+ { cameraName: key },
1210
+ { siteName: key },
1211
+ { bodyName: key }
1212
+ ];
1213
+ const aliasCandidates = normalizeAliasCandidates(options.aliases?.[key]);
1214
+ for (const selector of aliasCandidates) {
1215
+ if (!isSelectorMounted(selector, cameraNames, siteNames, bodyNames)) {
1216
+ continue;
1291
1217
  }
1292
- );
1293
- }
1294
- var _previousQuat = new THREE12.Quaternion();
1295
- var _currentQuat = new THREE12.Quaternion();
1296
- var CameraViewportRegistryContext = createContext(null);
1297
- var nextViewportId = 0;
1298
- var VIEWPORT_RENDER_PRIORITY = 1;
1299
- function CameraViewportRenderer({
1300
- viewportsRef
1301
- }) {
1302
- const gl = useThree((state) => state.gl);
1303
- const scene = useThree((state) => state.scene);
1304
- const mainCamera = useThree((state) => state.camera);
1305
- const mujoco = useMujoco();
1306
- useFrame(() => {
1307
- const drawWidth = gl.domElement.width;
1308
- const drawHeight = gl.domElement.height;
1309
- gl.setScissorTest(false);
1310
- gl.setViewport(0, 0, drawWidth, drawHeight);
1311
- gl.render(scene, mainCamera);
1312
- const api = mujoco.api;
1313
- if (!api || viewportsRef.current.size === 0) return;
1314
- const dpr = gl.getPixelRatio();
1315
- const canvasRect = gl.domElement.getBoundingClientRect();
1316
- for (const descriptor of viewportsRef.current.values()) {
1317
- const element = descriptor.getElement();
1318
- if (!element) continue;
1319
- const rect = element.getBoundingClientRect();
1320
- const isOffscreen = rect.bottom < canvasRect.top || rect.top > canvasRect.bottom || rect.right < canvasRect.left || rect.left > canvasRect.right;
1321
- if (isOffscreen) continue;
1322
- const width = Math.floor(rect.width * dpr);
1323
- const height = Math.floor(rect.height * dpr);
1324
- if (width <= 0 || height <= 0) continue;
1325
- const left = Math.floor((rect.left - canvasRect.left) * dpr);
1326
- const bottom = Math.floor((canvasRect.bottom - rect.bottom) * dpr);
1327
- let resolved;
1328
- try {
1329
- resolved = api.resolveCameraCaptureOptions(descriptor.getOptions());
1330
- } catch {
1331
- continue;
1332
- }
1333
- if (!descriptor.camera) {
1334
- descriptor.camera = createCaptureCamera(resolved, mainCamera, width, height);
1335
- } else {
1336
- prepareCaptureCamera(descriptor.camera, resolved, mainCamera, width, height);
1337
- }
1338
- gl.setViewport(left, bottom, width, height);
1339
- gl.setScissor(left, bottom, width, height);
1340
- gl.setScissorTest(true);
1341
- gl.render(scene, descriptor.camera);
1218
+ const source = getMountedCameraFrameCaptureSource(selector);
1219
+ if (!source) continue;
1220
+ return { key, selector, source };
1221
+ }
1222
+ const [suggestion] = createMountedCameraFrameSourceSuggestions(key, options);
1223
+ if (suggestion) {
1224
+ return {
1225
+ key,
1226
+ selector: suggestion.selector,
1227
+ source: suggestion.source
1228
+ };
1229
+ }
1230
+ if (options.allowAliasFallback) {
1231
+ for (const selector of aliasCandidates) {
1232
+ const source = getMountedCameraFrameCaptureSource(selector);
1233
+ if (!source) continue;
1234
+ return { key, selector, source };
1342
1235
  }
1343
- gl.setScissorTest(false);
1344
- gl.setViewport(0, 0, drawWidth, drawHeight);
1345
- }, VIEWPORT_RENDER_PRIORITY);
1236
+ }
1237
+ for (const selector of directCandidates) {
1238
+ if (!isSelectorMounted(selector, cameraNames, siteNames, bodyNames)) {
1239
+ continue;
1240
+ }
1241
+ const source = getMountedCameraFrameCaptureSource(selector);
1242
+ if (!source) continue;
1243
+ return { key, selector, source };
1244
+ }
1346
1245
  return null;
1347
1246
  }
1348
- function CameraViewportProvider({ children }) {
1349
- const viewportsRef = useRef(/* @__PURE__ */ new Map());
1350
- const [count, setCount] = useState(0);
1351
- const register = useCallback((descriptor) => {
1352
- const id = nextViewportId++;
1353
- viewportsRef.current.set(id, descriptor);
1354
- setCount((value2) => value2 + 1);
1355
- return () => {
1356
- viewportsRef.current.delete(id);
1357
- setCount((value2) => value2 - 1);
1358
- };
1359
- }, []);
1360
- const value = useMemo(() => ({ register }), [register]);
1361
- return /* @__PURE__ */ jsxs(CameraViewportRegistryContext.Provider, { value, children: [
1362
- children,
1363
- count > 0 && /* @__PURE__ */ jsx(CameraViewportRenderer, { viewportsRef })
1364
- ] });
1365
- }
1366
- function useCameraViewport(elementRef, options) {
1367
- const registry = useContext(CameraViewportRegistryContext);
1368
- if (!registry) {
1369
- throw new Error("useCameraViewport must be used inside <MujocoCanvas>.");
1247
+ function createMountedCameraFrameSequencePlan(cameraKeys, options) {
1248
+ const cameras = [];
1249
+ const resolved = {};
1250
+ const missingKeys = [];
1251
+ for (const key of cameraKeys) {
1252
+ const mountedSource = resolveMountedCameraFrameSource(key, options);
1253
+ if (!mountedSource) {
1254
+ missingKeys.push(key);
1255
+ continue;
1256
+ }
1257
+ resolved[key] = mountedSource;
1258
+ cameras.push({
1259
+ key,
1260
+ ...options.defaults,
1261
+ ...options.cameraOptions?.[key],
1262
+ ...mountedSource.selector,
1263
+ source: mountedSource.source
1264
+ });
1370
1265
  }
1371
- const optionsRef = useRef(options);
1372
- optionsRef.current = options;
1373
- useEffect(() => {
1374
- const descriptor = {
1375
- getElement: () => elementRef.current,
1376
- getOptions: () => optionsRef.current,
1377
- camera: null
1378
- };
1379
- return registry.register(descriptor);
1380
- }, [registry, elementRef]);
1266
+ if (options.requireAll && missingKeys.length > 0) {
1267
+ throw new Error(
1268
+ `Unable to resolve mounted MuJoCo camera source${missingKeys.length === 1 ? "" : "s"} for ${missingKeys.join(", ")}.`
1269
+ );
1270
+ }
1271
+ return {
1272
+ cameraKeys: [...cameraKeys],
1273
+ cameras,
1274
+ resolved,
1275
+ missingKeys
1276
+ };
1381
1277
  }
1382
- function CameraView({ className, style, ...options }) {
1383
- const gl = useThree((state) => state.gl);
1384
- const elementRef = useRef(null);
1385
- if (!elementRef.current && typeof document !== "undefined") {
1386
- elementRef.current = document.createElement("div");
1278
+ function createMountedCameraFrameSequenceReadiness(plan) {
1279
+ const cameras = {};
1280
+ const resolvedKeys = plan.cameraKeys.filter((key) => Boolean(plan.resolved[key]));
1281
+ for (const key of plan.cameraKeys) {
1282
+ const resolved = plan.resolved[key];
1283
+ cameras[key] = resolved ? {
1284
+ key,
1285
+ ready: true,
1286
+ selector: resolved.selector,
1287
+ source: resolved.source,
1288
+ message: `Camera stream "${key}" resolves to ${resolved.source.kind}:${getCameraFrameCaptureSourceTarget(resolved.source)}.`
1289
+ } : {
1290
+ key,
1291
+ ready: false,
1292
+ message: `Camera stream "${key}" does not resolve to a mounted MuJoCo camera, site, or body.`
1293
+ };
1387
1294
  }
1388
- useEffect(() => {
1389
- const element = elementRef.current;
1390
- const parent = gl.domElement.parentElement;
1391
- if (!element || !parent) return;
1392
- element.style.position = "absolute";
1393
- element.style.overflow = "hidden";
1394
- parent.appendChild(element);
1395
- return () => {
1396
- parent.removeChild(element);
1295
+ const missingKeys = [...plan.missingKeys];
1296
+ const ready = missingKeys.length === 0;
1297
+ const status = ready ? MountedCameraFrameSequenceReadinessStatus.Ready : resolvedKeys.length > 0 ? MountedCameraFrameSequenceReadinessStatus.Partial : MountedCameraFrameSequenceReadinessStatus.Missing;
1298
+ return {
1299
+ ready,
1300
+ status,
1301
+ cameraKeys: [...plan.cameraKeys],
1302
+ resolvedKeys,
1303
+ missingKeys,
1304
+ cameras,
1305
+ message: ready ? `All ${plan.cameraKeys.length} requested camera stream${plan.cameraKeys.length === 1 ? "" : "s"} resolve to mounted MuJoCo sources.` : `Missing mounted MuJoCo source${missingKeys.length === 1 ? "" : "s"} for ${missingKeys.join(", ")}.`
1306
+ };
1307
+ }
1308
+ function normalizeFrameCount(frameCount) {
1309
+ return Number.isFinite(frameCount) && frameCount !== void 0 ? Math.max(0, Math.floor(frameCount)) : 0;
1310
+ }
1311
+ function createMountedCameraFrameSequenceManifest(result, options = {}) {
1312
+ const cameraKeys = [
1313
+ ...options.cameraKeys ?? result.readiness.cameraKeys ?? result.plan.cameraKeys ?? result.cameraKeys
1314
+ ];
1315
+ const expectedFrameCount = normalizeFrameCount(
1316
+ options.expectedFrameCount ?? result.frameCount
1317
+ );
1318
+ const recordedFrameCount = normalizeFrameCount(result.frameCount);
1319
+ const streamSummaries = {};
1320
+ const streams = [];
1321
+ let missingFrameCount = 0;
1322
+ let completeStreamCount = 0;
1323
+ let resolvedOrRecordedStreamCount = 0;
1324
+ for (const key of cameraKeys) {
1325
+ const summary = result.cameraSummaries[key];
1326
+ const readiness = result.readiness.cameras[key];
1327
+ const source = summary?.source ?? readiness?.source;
1328
+ const ready = readiness?.ready ?? Boolean(summary);
1329
+ const recorded = normalizeFrameCount(summary?.frameCount);
1330
+ const missing = Math.max(expectedFrameCount - recorded, 0);
1331
+ const complete2 = ready && missing === 0;
1332
+ const status2 = complete2 ? MountedCameraFrameSequenceManifestStatus.Complete : ready || recorded > 0 ? MountedCameraFrameSequenceManifestStatus.Partial : MountedCameraFrameSequenceManifestStatus.Missing;
1333
+ const target = source ? getCameraFrameCaptureSourceTarget(source) : readiness?.message ? void 0 : "missing MuJoCo camera";
1334
+ const message = complete2 ? `Camera stream "${key}" recorded ${recorded} of ${expectedFrameCount} frame${expectedFrameCount === 1 ? "" : "s"}.` : ready || recorded > 0 ? `Camera stream "${key}" recorded ${recorded} of ${expectedFrameCount} frame${expectedFrameCount === 1 ? "" : "s"}.` : readiness?.message ?? `Camera stream "${key}" did not record any frames.`;
1335
+ const stream = {
1336
+ key,
1337
+ ready,
1338
+ complete: complete2,
1339
+ status: status2,
1340
+ source,
1341
+ selector: readiness?.selector,
1342
+ target,
1343
+ width: summary?.width,
1344
+ height: summary?.height,
1345
+ expectedFrameCount,
1346
+ recordedFrameCount: recorded,
1347
+ missingFrameCount: missing,
1348
+ firstFrameIndex: summary?.firstFrameIndex ?? null,
1349
+ lastFrameIndex: summary?.lastFrameIndex ?? null,
1350
+ firstTimestamp: summary?.firstTimestamp ?? null,
1351
+ lastTimestamp: summary?.lastTimestamp ?? null,
1352
+ message
1397
1353
  };
1398
- }, [gl]);
1399
- useEffect(() => {
1400
- const element = elementRef.current;
1401
- if (!element) return;
1402
- element.className = className ?? "";
1403
- if (style) Object.assign(element.style, style);
1404
- }, [className, style]);
1405
- useCameraViewport(elementRef, options);
1406
- return null;
1354
+ streamSummaries[key] = stream;
1355
+ streams.push(stream);
1356
+ missingFrameCount += missing;
1357
+ if (complete2) completeStreamCount += 1;
1358
+ if (ready || recorded > 0) resolvedOrRecordedStreamCount += 1;
1359
+ }
1360
+ const complete = result.readiness.ready && streams.length === completeStreamCount && missingFrameCount === 0;
1361
+ const status = complete ? MountedCameraFrameSequenceManifestStatus.Complete : resolvedOrRecordedStreamCount > 0 ? MountedCameraFrameSequenceManifestStatus.Partial : MountedCameraFrameSequenceManifestStatus.Missing;
1362
+ return {
1363
+ schema: "mujoco-react/mounted-camera-frame-sequence-manifest@1",
1364
+ ready: result.readiness.ready,
1365
+ complete,
1366
+ status,
1367
+ cameraKeys,
1368
+ resolvedKeys: [...result.readiness.resolvedKeys],
1369
+ missingKeys: [...result.readiness.missingKeys],
1370
+ expectedFrameCount,
1371
+ recordedFrameCount,
1372
+ missingFrameCount,
1373
+ streamSummaries,
1374
+ streams,
1375
+ readiness: result.readiness,
1376
+ message: complete ? `All ${cameraKeys.length} camera stream${cameraKeys.length === 1 ? "" : "s"} recorded ${expectedFrameCount} frame${expectedFrameCount === 1 ? "" : "s"}.` : `Mounted camera sequence coverage is ${status}.`
1377
+ };
1407
1378
  }
1408
- function isTargetRef(target) {
1409
- return Boolean(target && typeof target === "object" && "current" in target);
1379
+ function createMountedCameraFrameSequencePlanFromApi(api, cameraKeys, options = {}) {
1380
+ return createMountedCameraFrameSequencePlan(cameraKeys, {
1381
+ ...options,
1382
+ cameras: api.getCameras(),
1383
+ sites: api.getSites(),
1384
+ bodies: api.getBodies()
1385
+ });
1410
1386
  }
1411
- function resolveCanvasTarget(target) {
1412
- const resolvedTarget = isTargetRef(target) ? target.current : target;
1413
- if (!resolvedTarget) {
1414
- throw new Error("No frame capture target is available.");
1387
+ async function recordMountedCameraFrameSequence(api, options) {
1388
+ const { cameraKeys, ...restOptions } = options;
1389
+ const requireAll = restOptions.requireAll ?? restOptions.requireMountedSources ?? true;
1390
+ const plan = createMountedCameraFrameSequencePlanFromApi(
1391
+ api,
1392
+ cameraKeys,
1393
+ { ...restOptions, requireAll }
1394
+ );
1395
+ const readiness = createMountedCameraFrameSequenceReadiness(plan);
1396
+ const result = await api.recordCameraSequence({
1397
+ ...restOptions,
1398
+ cameras: plan.cameras,
1399
+ requireMountedSources: restOptions.requireMountedSources ?? true
1400
+ });
1401
+ return {
1402
+ ...result,
1403
+ plan,
1404
+ readiness
1405
+ };
1406
+ }
1407
+
1408
+ // src/policyCameraFrames.ts
1409
+ function addPolicyImageAliases(images, stream, frame, includeObservationImageAliases) {
1410
+ const keys = /* @__PURE__ */ new Set();
1411
+ keys.add(stream.key);
1412
+ for (const alias of stream.aliases ?? []) keys.add(alias);
1413
+ if (includeObservationImageAliases) {
1414
+ keys.add(`observation.images.${stream.key}`);
1415
+ for (const alias of stream.aliases ?? []) {
1416
+ keys.add(`observation.images.${alias}`);
1417
+ }
1415
1418
  }
1416
- if (resolvedTarget instanceof HTMLCanvasElement) {
1417
- return resolvedTarget;
1419
+ for (const key of keys) {
1420
+ images[key] = frame.dataUrl;
1421
+ }
1422
+ }
1423
+ function describeFrameSource(key, frame) {
1424
+ return `${key}:${frame.source.kind}`;
1425
+ }
1426
+ function hasExplicitPolicyCameraSource(options) {
1427
+ return Boolean(
1428
+ options?.camera || options?.position || options?.quaternion || options?.source
1429
+ );
1430
+ }
1431
+ function createPolicyCameraFrameCapturePlan(options) {
1432
+ const {
1433
+ cameraKeys,
1434
+ defaults,
1435
+ streamOptions,
1436
+ includeObservationImageAliases,
1437
+ requireAll,
1438
+ ...sourceOptions
1439
+ } = options;
1440
+ const mountedPlan = createMountedCameraFrameSequencePlan(cameraKeys, {
1441
+ ...sourceOptions,
1442
+ defaults,
1443
+ cameraOptions: streamOptions
1444
+ });
1445
+ const streams = [];
1446
+ const missingKeys = new Set(mountedPlan.missingKeys);
1447
+ for (const key of cameraKeys) {
1448
+ const perStreamOptions = streamOptions?.[key];
1449
+ if (hasExplicitPolicyCameraSource(perStreamOptions)) {
1450
+ missingKeys.delete(key);
1451
+ streams.push({
1452
+ ...defaults,
1453
+ ...perStreamOptions,
1454
+ key,
1455
+ aliases: perStreamOptions?.aliases
1456
+ });
1457
+ continue;
1458
+ }
1459
+ const mountedCamera = mountedPlan.cameras.find((camera) => camera.key === key);
1460
+ if (!mountedCamera) continue;
1461
+ const { key: _mountedKey, ...captureOptions } = mountedCamera;
1462
+ streams.push({
1463
+ ...captureOptions,
1464
+ key,
1465
+ aliases: perStreamOptions?.aliases
1466
+ });
1418
1467
  }
1419
- const canvas = resolvedTarget.querySelector("canvas");
1420
- if (!canvas) {
1421
- throw new Error("Frame capture target does not contain a canvas.");
1468
+ const result = {
1469
+ cameraKeys: [...cameraKeys],
1470
+ streams,
1471
+ includeObservationImageAliases,
1472
+ mountedPlan,
1473
+ missingKeys: [...missingKeys]
1474
+ };
1475
+ if (requireAll && result.missingKeys.length > 0) {
1476
+ throw new Error(
1477
+ `Unable to resolve policy camera stream${result.missingKeys.length === 1 ? "" : "s"} for ${result.missingKeys.join(", ")}.`
1478
+ );
1422
1479
  }
1423
- return canvas;
1480
+ return result;
1424
1481
  }
1425
- function waitForNextAnimationFrame() {
1426
- return new Promise((resolve) => {
1427
- requestAnimationFrame(() => resolve());
1482
+ function createPolicyCameraFrameCapturePlanFromApi(api, options) {
1483
+ return createPolicyCameraFrameCapturePlan({
1484
+ ...options,
1485
+ cameras: api.getCameras(),
1486
+ sites: api.getSites(),
1487
+ bodies: api.getBodies()
1428
1488
  });
1429
1489
  }
1430
- async function captureFrame(options) {
1431
- const type = options.type ?? "image/png";
1432
- const canvas = resolveCanvasTarget(options.target);
1433
- if (options.waitForAnimationFrame ?? true) {
1434
- await waitForNextAnimationFrame();
1490
+ async function capturePolicyCameraFrames(target, options) {
1491
+ const includeObservationImageAliases = options.includeObservationImageAliases ?? true;
1492
+ const entries = await Promise.all(
1493
+ options.streams.map(async ({ key, aliases, ...captureOptions }) => {
1494
+ const frame = await target.captureCameraFrame(captureOptions);
1495
+ return [key, { frame, aliases }];
1496
+ })
1497
+ );
1498
+ const frames = {};
1499
+ const images = {};
1500
+ const sourceParts = [];
1501
+ for (const [key, { frame, aliases }] of entries) {
1502
+ const stream = { key, aliases };
1503
+ frames[key] = frame;
1504
+ addPolicyImageAliases(
1505
+ images,
1506
+ stream,
1507
+ frame,
1508
+ includeObservationImageAliases
1509
+ );
1510
+ sourceParts.push(describeFrameSource(key, frame));
1435
1511
  }
1436
1512
  return {
1437
- canvas,
1438
- dataUrl: canvas.toDataURL(type, options.quality),
1439
- type
1513
+ frames,
1514
+ images,
1515
+ sourceSummary: sourceParts.length > 0 ? sourceParts.join(" + ") : "not used by policy",
1516
+ capturedAt: Date.now()
1440
1517
  };
1441
1518
  }
1442
- async function captureFrameBlob(options) {
1443
- const type = options.type ?? "image/png";
1444
- const canvas = resolveCanvasTarget(options.target);
1445
- if (options.waitForAnimationFrame ?? true) {
1446
- await waitForNextAnimationFrame();
1447
- }
1448
- const blob = await new Promise((resolve, reject) => {
1449
- canvas.toBlob(
1450
- (nextBlob) => {
1451
- if (nextBlob) {
1452
- resolve(nextBlob);
1453
- } else {
1454
- reject(new Error("Canvas frame capture did not produce a Blob."));
1455
- }
1456
- },
1457
- type,
1458
- options.quality
1459
- );
1460
- });
1461
- return { canvas, blob, type };
1519
+ async function capturePolicyCameraFramesFromMountedStreams(target, options) {
1520
+ const plan = createPolicyCameraFrameCapturePlanFromApi(target, options);
1521
+ const result = await capturePolicyCameraFrames(target, plan);
1522
+ return { ...result, plan };
1462
1523
  }
1463
- function useFrameCapture(defaultOptions = {}) {
1464
- const [status, setStatus] = useState("idle");
1465
- const [error, setError] = useState(null);
1466
- const reset = useCallback(() => {
1467
- setStatus("idle");
1468
- setError(null);
1469
- }, []);
1470
- const capture = useCallback(
1471
- async (options = {}) => {
1472
- setStatus("capturing");
1473
- setError(null);
1524
+
1525
+ // src/runtime/client.ts
1526
+ var owners = /* @__PURE__ */ new WeakMap();
1527
+ var isWorkerOwned = (model) => Boolean(model && owners.has(model));
1528
+ function closeModelRuntime(model) {
1529
+ if (model) owners.get(model)?.close();
1530
+ }
1531
+ function runtimeContacts(model) {
1532
+ return owners.get(model)?.contacts();
1533
+ }
1534
+ function assertMainThreadOwnership(model) {
1535
+ if (isWorkerOwned(model)) throw new Error("Worker runtime owns this simulation. Use runtime operations or dispose it before using viewer physics operations.");
1536
+ }
1537
+ var abortError = () => new DOMException("Simulation operation cancelled", "AbortError");
1538
+ var RuntimeClient = class {
1539
+ constructor(worker, apply, release, capture) {
1540
+ this.worker = worker;
1541
+ this.apply = apply;
1542
+ this.release = release;
1543
+ this.capture = capture;
1544
+ worker.onmessage = (event) => {
1545
+ const response = event.data;
1546
+ if (this.closed || response.id !== this.pending?.id) return;
1547
+ if (response.display) {
1548
+ try {
1549
+ this.apply(response.display);
1550
+ const f = response.display.fields;
1551
+ this.latest = {
1552
+ time: response.display.time,
1553
+ qpos: f.qpos,
1554
+ qvel: f.qvel,
1555
+ ctrl: f.ctrl,
1556
+ sensors: f.sensordata,
1557
+ contacts: response.done ? response.display.contacts : this.latest.contacts
1558
+ };
1559
+ } catch (error) {
1560
+ this.close(error instanceof Error ? error : new Error(String(error)));
1561
+ return;
1562
+ }
1563
+ }
1564
+ if (!response.done) this.worker.postMessage({ kind: "ack", id: response.id });
1565
+ if (response.done) {
1566
+ const pending = this.pending;
1567
+ this.pending = null;
1568
+ if (response.error) pending.reject(new Error(response.error));
1569
+ else pending.resolve(response);
1570
+ }
1571
+ };
1572
+ worker.onerror = (event) => this.close(new Error(event.message || "Simulation worker failed"));
1573
+ worker.onmessageerror = () => this.close(new Error("Cannot read simulation worker response"));
1574
+ }
1575
+ pending = null;
1576
+ nextId = 0;
1577
+ closed = false;
1578
+ busy = false;
1579
+ flight = Promise.resolve();
1580
+ generation = 0;
1581
+ latest = { time: 0, qpos: new Float64Array(), qvel: new Float64Array(), ctrl: new Float64Array(), sensors: new Float64Array(), contacts: [] };
1582
+ statistics = { steps: 0, physicsMs: 0, elapsedMs: 0, realTimeFactor: 0, captureMs: 0, busy: false };
1583
+ modelId = crypto.randomUUID();
1584
+ get state() {
1585
+ return {
1586
+ ...this.latest,
1587
+ qpos: this.latest.qpos.slice(),
1588
+ qvel: this.latest.qvel.slice(),
1589
+ ctrl: this.latest.ctrl.slice(),
1590
+ sensors: this.latest.sensors.slice(),
1591
+ contacts: this.latest.contacts.map((c) => ({ ...c, pos: [...c.pos] }))
1592
+ };
1593
+ }
1594
+ get metrics() {
1595
+ return { ...this.statistics, busy: this.busy };
1596
+ }
1597
+ request(request) {
1598
+ if (this.closed) return Promise.reject(new Error("Simulation runtime is disposed"));
1599
+ return new Promise((resolve, reject) => {
1600
+ const id = ++this.nextId;
1601
+ this.pending = { id, resolve, reject };
1474
1602
  try {
1475
- const result = await captureFrame({ ...defaultOptions, ...options });
1476
- setStatus("captured");
1477
- return result;
1478
- } catch (nextError) {
1479
- const error2 = nextError instanceof Error ? nextError : new Error("Unable to capture the current canvas frame.");
1480
- setError(error2);
1481
- setStatus("error");
1482
- throw error2;
1603
+ this.worker.postMessage({ ...request, id });
1604
+ } catch (error) {
1605
+ this.pending = null;
1606
+ reject(error instanceof Error ? error : new Error(String(error)));
1483
1607
  }
1484
- },
1485
- [defaultOptions]
1486
- );
1487
- const captureBlob = useCallback(
1488
- async (options = {}) => {
1489
- setStatus("capturing");
1490
- setError(null);
1608
+ });
1609
+ }
1610
+ exclusive(operation) {
1611
+ if (this.closed) return Promise.reject(new Error("Simulation runtime is disposed"));
1612
+ if (this.busy) return Promise.reject(new Error("Runtime is busy; await the current operation (work is not queued)"));
1613
+ this.busy = true;
1614
+ const result = operation().finally(() => {
1615
+ this.busy = false;
1616
+ });
1617
+ this.flight = result.catch(() => {
1618
+ });
1619
+ return result;
1620
+ }
1621
+ initialize(initial) {
1622
+ return this.exclusive(async () => {
1623
+ await this.request({ kind: "initialize", initial });
1624
+ });
1625
+ }
1626
+ advance(options) {
1627
+ return this.exclusive(async () => {
1628
+ if (options.signal?.aborted) throw abortError();
1629
+ const started = performance.now(), simStart = this.latest.time;
1630
+ const abort = () => {
1631
+ this.worker.postMessage({ kind: "cancel", id: ++this.nextId });
1632
+ };
1633
+ options.signal?.addEventListener("abort", abort, { once: true });
1491
1634
  try {
1492
- const result = await captureFrameBlob({
1493
- ...defaultOptions,
1494
- ...options
1635
+ const result = await this.request({
1636
+ kind: "advance",
1637
+ steps: options.steps,
1638
+ controls: options.controls ? Float64Array.from(options.controls) : void 0
1495
1639
  });
1496
- setStatus("captured");
1497
- return result;
1498
- } catch (nextError) {
1499
- const error2 = nextError instanceof Error ? nextError : new Error("Unable to capture the current canvas frame.");
1500
- setError(error2);
1501
- setStatus("error");
1502
- throw error2;
1640
+ const elapsedMs = performance.now() - started;
1641
+ this.statistics = {
1642
+ ...this.statistics,
1643
+ steps: result.steps ?? 0,
1644
+ physicsMs: result.physicsMs ?? 0,
1645
+ elapsedMs,
1646
+ realTimeFactor: (this.latest.time - simStart) * 1e3 / Math.max(elapsedMs, 1e-3)
1647
+ };
1648
+ if (result.cancelled || options.signal?.aborted) throw abortError();
1649
+ return this.state;
1650
+ } finally {
1651
+ options.signal?.removeEventListener("abort", abort);
1503
1652
  }
1504
- },
1505
- [defaultOptions]
1506
- );
1507
- return {
1508
- status,
1509
- error,
1510
- isCapturing: status === "capturing",
1511
- capture,
1512
- captureBlob,
1513
- reset
1514
- };
1515
- }
1516
-
1517
- // src/rendering/cameraFrameSource.ts
1518
- var MountedCameraFrameSourceSuggestionMatch = {
1519
- Direct: "direct",
1520
- Alias: "alias",
1521
- Normalized: "normalized",
1522
- Prefix: "prefix",
1523
- Suffix: "suffix",
1524
- Contains: "contains"
1525
- };
1526
- var MountedCameraFrameSequenceReadinessStatus = {
1527
- Ready: "ready",
1528
- Partial: "partial",
1529
- Missing: "missing"
1530
- };
1531
- var MountedCameraFrameSequenceManifestStatus = {
1532
- Complete: "complete",
1533
- Partial: "partial",
1534
- Missing: "missing"
1535
- };
1536
- function getResourceName(resource) {
1537
- if (!resource) return null;
1538
- return typeof resource === "string" ? resource : resource.name ?? null;
1539
- }
1540
- function createNameSet(resources) {
1541
- return new Set(
1542
- (resources ?? []).map((resource) => getResourceName(resource)).filter((name) => Boolean(name))
1543
- );
1544
- }
1545
- function createResourceNames(resources) {
1546
- return (resources ?? []).map((resource) => getResourceName(resource)).filter((name) => Boolean(name));
1547
- }
1548
- function normalizeAliasCandidates(value) {
1549
- if (!value) return [];
1550
- return Array.isArray(value) ? value : [value];
1551
- }
1552
- function countMountedSelectors(selector) {
1553
- return Number(Boolean(selector.cameraName)) + Number(Boolean(selector.siteName)) + Number(Boolean(selector.bodyName));
1554
- }
1555
- function normalizeCameraSourceName(value) {
1556
- return value.trim().toLowerCase().replace(/[^a-z0-9]+/g, "_").replace(/^_+|_+$/g, "");
1557
- }
1558
- function createCameraSourceKeyVariants(key) {
1559
- const candidates = [
1560
- key,
1561
- key.startsWith("observation.images.") ? key.slice("observation.images.".length) : "",
1562
- key.includes(".") ? key.split(".").at(-1) ?? "" : "",
1563
- key.includes("/") ? key.split("/").at(-1) ?? "" : ""
1564
- ];
1565
- return candidates.map((candidate) => candidate.trim()).filter((candidate, index, items) => candidate && items.indexOf(candidate) === index);
1566
- }
1567
- function getSelectorKey(selector) {
1568
- if (selector.cameraName) return `camera:${selector.cameraName}`;
1569
- if (selector.siteName) return `site:${selector.siteName}`;
1570
- if (selector.bodyName) return `body:${selector.bodyName}`;
1571
- return null;
1572
- }
1573
- function getMountedCameraFrameCaptureSource(selector) {
1574
- if (countMountedSelectors(selector) !== 1) return null;
1575
- if (selector.cameraName) {
1576
- return { kind: "mujoco-camera", cameraName: selector.cameraName };
1653
+ });
1654
+ }
1655
+ observe(cameras) {
1656
+ return this.exclusive(async () => {
1657
+ const generation = this.generation;
1658
+ await this.request({ kind: "snapshot" });
1659
+ const state = this.state, started = performance.now();
1660
+ const frames = cameras ? await this.capture?.(cameras) : void 0;
1661
+ if (cameras && !frames) throw new Error("Camera observations require an attached canvas");
1662
+ this.statistics.captureMs = performance.now() - started;
1663
+ if (generation !== this.generation || this.closed) throw abortError();
1664
+ return { ...state, ...frames ? { cameras: frames } : {} };
1665
+ });
1666
+ }
1667
+ saveState() {
1668
+ return this.exclusive(async () => {
1669
+ const result = await this.request({ kind: "snapshot" });
1670
+ const state = result.display.integration;
1671
+ return { modelId: this.modelId, signature: state.signature, values: state.values.slice() };
1672
+ });
1577
1673
  }
1578
- if (selector.siteName) {
1579
- return { kind: "mujoco-site", siteName: selector.siteName };
1674
+ restoreState(snapshot) {
1675
+ return this.exclusive(async () => {
1676
+ if (snapshot.modelId !== this.modelId) throw new Error("Snapshot belongs to a different runtime");
1677
+ await this.request({ kind: "restore", state: { signature: snapshot.signature, values: snapshot.values } });
1678
+ return this.state;
1679
+ });
1580
1680
  }
1581
- if (selector.bodyName) {
1582
- return { kind: "mujoco-body", bodyName: selector.bodyName };
1681
+ async cancel() {
1682
+ if (this.closed) return;
1683
+ this.generation++;
1684
+ this.worker.postMessage({ kind: "cancel", id: ++this.nextId });
1685
+ await this.flight;
1686
+ }
1687
+ async reset() {
1688
+ await this.cancel();
1689
+ return this.exclusive(async () => {
1690
+ await this.request({ kind: "reset" });
1691
+ return this.state;
1692
+ });
1583
1693
  }
1584
- return null;
1585
- }
1586
- function isMountedCameraFrameCaptureSource(source) {
1587
- return source.kind === "mujoco-camera" || source.kind === "mujoco-site" || source.kind === "mujoco-body";
1588
- }
1589
- function getCameraFrameCaptureSourceTarget(source) {
1590
- if (source.kind === "mujoco-camera") return source.cameraName;
1591
- if (source.kind === "mujoco-site") return source.siteName;
1592
- if (source.kind === "mujoco-body") return source.bodyName;
1593
- if (source.kind === "custom-camera") return "custom camera";
1594
- if (source.kind === "explicit-pose") return "explicit pose";
1595
- return "fallback camera";
1596
- }
1597
- function createMountedCameraFrameSourceSuggestion(key, selector, resourceName, match) {
1598
- const source = getMountedCameraFrameCaptureSource(selector);
1599
- if (!source) return null;
1600
- return {
1601
- key,
1602
- selector,
1603
- source,
1604
- resourceName,
1605
- resourceKind: source.kind,
1606
- match
1694
+ async dispose() {
1695
+ await this.cancel();
1696
+ this.close();
1697
+ }
1698
+ close(error = new Error("Simulation runtime disposed")) {
1699
+ if (this.closed) return;
1700
+ this.closed = true;
1701
+ this.generation++;
1702
+ this.worker.terminate();
1703
+ this.pending?.reject(error);
1704
+ this.pending = null;
1705
+ this.release();
1706
+ }
1707
+ };
1708
+ async function attachSimulationRuntime(module, api, waitForDisplay, invalidate, options = {}) {
1709
+ const model = api.mjModelRef.current, data = api.mjDataRef.current;
1710
+ if (!model || !data) throw new Error("Scene must be ready before creating a runtime");
1711
+ assertMainThreadOwnership(model);
1712
+ api.setPaused(true);
1713
+ const state = api.saveState().integration;
1714
+ const initialState = { signature: state.signature, values: state.values };
1715
+ const mj = module;
1716
+ const path = `/runtime-${crypto.randomUUID()}.mjb`;
1717
+ let binary;
1718
+ try {
1719
+ mj.mj_saveModel(model, path, null);
1720
+ binary = mj.FS.readFile(path);
1721
+ } finally {
1722
+ try {
1723
+ mj.FS.unlink(path);
1724
+ } catch {
1725
+ }
1726
+ }
1727
+ const worker = options.createWorker?.() ?? new Worker(new URL("./physics.worker.js", import.meta.url), { type: "module" });
1728
+ let complete;
1729
+ const apply = (display) => {
1730
+ if (api.mjModelRef.current !== model) throw new Error("Simulation model changed");
1731
+ if (display.integration) {
1732
+ module.mj_setState(model, data, Array.from(display.integration.values), display.integration.signature);
1733
+ complete = display;
1734
+ }
1735
+ for (const field of displayFields) {
1736
+ const target = data[field];
1737
+ if (!(target instanceof Float64Array || target instanceof Int32Array) || target.length !== display.fields[field].length) throw new Error(`Invalid display field ${field}`);
1738
+ target.set(display.fields[field]);
1739
+ }
1740
+ data.time = display.time;
1741
+ invalidate();
1607
1742
  };
1743
+ const client = new RuntimeClient(worker, apply, () => {
1744
+ if (api.mjModelRef.current === model && complete) apply(complete);
1745
+ owners.delete(model);
1746
+ }, async (cameras) => {
1747
+ await waitForDisplay();
1748
+ return capturePolicyCameraFrames(api, cameras);
1749
+ });
1750
+ owners.set(model, { close: () => client.close(), contacts: () => client.state.contacts });
1751
+ try {
1752
+ await client.initialize({ model: binary, state: initialState });
1753
+ return client;
1754
+ } catch (error) {
1755
+ client.close();
1756
+ throw error;
1757
+ }
1608
1758
  }
1609
- function addMountedCameraFrameSourceSuggestion(suggestions, seen, suggestion) {
1610
- if (!suggestion) return;
1611
- const selectorKey = getSelectorKey(suggestion.selector);
1612
- if (!selectorKey || seen.has(selectorKey)) return;
1613
- seen.add(selectorKey);
1614
- suggestions.push(suggestion);
1759
+ var CapsuleGeometry = class extends THREE12.BufferGeometry {
1760
+ parameters;
1761
+ constructor(radius = 1, length = 1, capSegments = 4, radialSegments = 8) {
1762
+ super();
1763
+ this.type = "CapsuleGeometry";
1764
+ this.parameters = { radius, length, capSegments, radialSegments };
1765
+ const path = new THREE12.Path();
1766
+ path.absarc(0, -length / 2, radius, Math.PI * 1.5, 0, false);
1767
+ path.absarc(0, length / 2, radius, 0, Math.PI * 0.5, false);
1768
+ const latheGeometry = new THREE12.LatheGeometry(path.getPoints(capSegments), radialSegments);
1769
+ const self = this;
1770
+ self.setIndex(latheGeometry.getIndex());
1771
+ self.setAttribute("position", latheGeometry.getAttribute("position"));
1772
+ self.setAttribute("normal", latheGeometry.getAttribute("normal"));
1773
+ self.setAttribute("uv", latheGeometry.getAttribute("uv"));
1774
+ }
1775
+ };
1776
+
1777
+ // src/rendering/GeomBuilder.ts
1778
+ var DEFAULT_MESH_NORMAL_SMOOTHING_TOLERANCE = 1e-4;
1779
+ var GeomBuilder = class {
1780
+ mujoco;
1781
+ textureCache = /* @__PURE__ */ new Map();
1782
+ renderOptions;
1783
+ constructor(mujoco, renderOptions) {
1784
+ this.mujoco = mujoco;
1785
+ this.renderOptions = renderOptions;
1786
+ }
1787
+ getMeshNormalSmoothingTolerance() {
1788
+ const smoothing = this.renderOptions?.meshNormalSmoothing;
1789
+ if (!smoothing) return null;
1790
+ if (smoothing === true) return DEFAULT_MESH_NORMAL_SMOOTHING_TOLERANCE;
1791
+ return smoothing.tolerance ?? DEFAULT_MESH_NORMAL_SMOOTHING_TOLERANCE;
1792
+ }
1793
+ getMaterialTexture(mjModel, matId) {
1794
+ if (matId < 0 || !mjModel.mat_texid || !mjModel.tex_data) return null;
1795
+ const materialCount = Math.max(1, Math.floor(mjModel.mat_rgba.length / 4));
1796
+ const textureRoles = Math.max(1, Math.floor(mjModel.mat_texid.length / materialCount));
1797
+ let texId = -1;
1798
+ for (let role = 0; role < textureRoles; role += 1) {
1799
+ const candidate = mjModel.mat_texid[matId * textureRoles + role];
1800
+ if (candidate >= 0) {
1801
+ texId = candidate;
1802
+ break;
1803
+ }
1804
+ }
1805
+ if (texId < 0) return null;
1806
+ const cached = this.textureCache.get(texId);
1807
+ if (cached) return cached;
1808
+ const width = Number(mjModel.tex_width[texId]);
1809
+ const height = Number(mjModel.tex_height[texId]);
1810
+ const channels = Number(mjModel.tex_nchannel[texId]);
1811
+ const offset = Number(mjModel.tex_adr[texId]);
1812
+ if (width <= 0 || height <= 0 || channels <= 0 || offset < 0) return null;
1813
+ const source = mjModel.tex_data.subarray(offset, offset + width * height * channels);
1814
+ const rgba = new Uint8Array(width * height * 4);
1815
+ for (let i = 0, j = 0; i < width * height; i += 1, j += channels) {
1816
+ const r = source[j] ?? 255;
1817
+ const g = channels > 1 ? source[j + 1] : r;
1818
+ const b = channels > 2 ? source[j + 2] : r;
1819
+ const a = channels > 3 ? source[j + 3] : 255;
1820
+ const out = i * 4;
1821
+ rgba[out] = r;
1822
+ rgba[out + 1] = g;
1823
+ rgba[out + 2] = b;
1824
+ rgba[out + 3] = a;
1825
+ }
1826
+ const texture = new THREE12.DataTexture(rgba, width, height, THREE12.RGBAFormat);
1827
+ texture.colorSpace = THREE12.LinearSRGBColorSpace;
1828
+ texture.wrapS = THREE12.RepeatWrapping;
1829
+ texture.wrapT = THREE12.RepeatWrapping;
1830
+ texture.flipY = true;
1831
+ const repeatOffset = matId * 2;
1832
+ const repeatS = mjModel.mat_texrepeat?.[repeatOffset] ?? 1;
1833
+ const repeatT = mjModel.mat_texrepeat?.[repeatOffset + 1] ?? 1;
1834
+ texture.repeat.set(repeatS || 1, repeatT || 1);
1835
+ texture.needsUpdate = true;
1836
+ this.textureCache.set(texId, texture);
1837
+ return texture;
1838
+ }
1839
+ /**
1840
+ * Creates a Three.js Object3D (usually a Mesh) for a specific geometry in the MuJoCo model.
1841
+ * Returns null if the geometry shouldn't be rendered (e.g., invisible collision triggers).
1842
+ */
1843
+ create(mjModel, g) {
1844
+ if (mjModel.geom_group[g] === 3) return null;
1845
+ const type = mjModel.geom_type[g];
1846
+ const size = mjModel.geom_size.subarray(g * 3, g * 3 + 3);
1847
+ const pos = mjModel.geom_pos.subarray(g * 3, g * 3 + 3);
1848
+ const quat = mjModel.geom_quat.subarray(g * 4, g * 4 + 4);
1849
+ const matId = mjModel.geom_matid[g];
1850
+ const color = new THREE12.Color(16777215);
1851
+ const map = this.getMaterialTexture(mjModel, matId);
1852
+ let opacity = 1;
1853
+ if (matId >= 0) {
1854
+ const rgba = mjModel.mat_rgba.subarray(matId * 4, matId * 4 + 4);
1855
+ color.setRGB(rgba[0], rgba[1], rgba[2]);
1856
+ opacity = rgba[3];
1857
+ } else {
1858
+ const rgba = mjModel.geom_rgba.subarray(g * 4, g * 4 + 4);
1859
+ color.setRGB(rgba[0], rgba[1], rgba[2]);
1860
+ opacity = rgba[3];
1861
+ }
1862
+ const MG = this.mujoco.mjtGeom;
1863
+ let geo = null;
1864
+ const getVal = (v) => v?.value ?? v;
1865
+ if (type === getVal(MG.mjGEOM_PLANE)) {
1866
+ geo = new THREE12.PlaneGeometry(size[0] * 2 || 5, size[1] * 2 || 5);
1867
+ } else if (type === getVal(MG.mjGEOM_SPHERE)) {
1868
+ geo = new THREE12.SphereGeometry(size[0], 24, 24);
1869
+ } else if (type === getVal(MG.mjGEOM_CAPSULE)) {
1870
+ geo = new CapsuleGeometry(size[0], size[1] * 2, 24, 12);
1871
+ geo.rotateX(Math.PI / 2);
1872
+ } else if (type === getVal(MG.mjGEOM_BOX)) {
1873
+ geo = new THREE12.BoxGeometry(size[0] * 2, size[1] * 2, size[2] * 2);
1874
+ } else if (type === getVal(MG.mjGEOM_CYLINDER)) {
1875
+ geo = new THREE12.CylinderGeometry(size[0], size[0], size[1] * 2, 24);
1876
+ geo.rotateX(Math.PI / 2);
1877
+ } else if (type === getVal(MG.mjGEOM_MESH)) {
1878
+ const mId = mjModel.geom_dataid[g];
1879
+ const vAdr = mjModel.mesh_vertadr[mId];
1880
+ const vNum = mjModel.mesh_vertnum[mId];
1881
+ const fAdr = mjModel.mesh_faceadr[mId];
1882
+ const fNum = mjModel.mesh_facenum[mId];
1883
+ const tAdr = mjModel.mesh_texcoordadr?.[mId] ?? -1;
1884
+ if (map && tAdr >= 0 && mjModel.mesh_texcoord && mjModel.mesh_facetexcoord) {
1885
+ const corners = fNum * 3;
1886
+ const positions = new Float32Array(corners * 3);
1887
+ const uvs = new Float32Array(corners * 2);
1888
+ for (let c = 0; c < corners; c++) {
1889
+ const v = vAdr + mjModel.mesh_face[fAdr * 3 + c];
1890
+ positions[c * 3] = mjModel.mesh_vert[v * 3];
1891
+ positions[c * 3 + 1] = mjModel.mesh_vert[v * 3 + 1];
1892
+ positions[c * 3 + 2] = mjModel.mesh_vert[v * 3 + 2];
1893
+ const t = tAdr + mjModel.mesh_facetexcoord[fAdr * 3 + c];
1894
+ uvs[c * 2] = mjModel.mesh_texcoord[t * 2];
1895
+ uvs[c * 2 + 1] = 1 - mjModel.mesh_texcoord[t * 2 + 1];
1896
+ }
1897
+ geo = new THREE12.BufferGeometry();
1898
+ geo.setAttribute("position", new THREE12.Float32BufferAttribute(positions, 3));
1899
+ geo.setAttribute("uv", new THREE12.Float32BufferAttribute(uvs, 2));
1900
+ geo.computeVertexNormals();
1901
+ } else {
1902
+ geo = new THREE12.BufferGeometry();
1903
+ geo.setAttribute("position", new THREE12.Float32BufferAttribute(mjModel.mesh_vert.subarray(vAdr * 3, (vAdr + vNum) * 3), 3));
1904
+ geo.setIndex(Array.from(mjModel.mesh_face.subarray(fAdr * 3, (fAdr + fNum) * 3)));
1905
+ const smoothingTolerance = this.getMeshNormalSmoothingTolerance();
1906
+ if (smoothingTolerance !== null) {
1907
+ geo = mergeVertices(geo, smoothingTolerance);
1908
+ }
1909
+ geo.computeVertexNormals();
1910
+ }
1911
+ }
1912
+ if (geo) {
1913
+ const isPlane = type === getVal(MG.mjGEOM_PLANE);
1914
+ const materialMap = isPlane && map ? map.clone() : map;
1915
+ if (isPlane && materialMap) {
1916
+ materialMap.repeat.multiplyScalar(2.5);
1917
+ materialMap.needsUpdate = true;
1918
+ }
1919
+ const mesh = new THREE12.Mesh(geo, new THREE12.MeshStandardMaterial({
1920
+ color,
1921
+ map: materialMap,
1922
+ transparent: opacity < 1,
1923
+ opacity,
1924
+ roughness: 0.6,
1925
+ metalness: 0
1926
+ }));
1927
+ mesh.castShadow = type !== getVal(MG.mjGEOM_PLANE);
1928
+ mesh.receiveShadow = true;
1929
+ mesh.position.set(pos[0], pos[1], pos[2]);
1930
+ mesh.quaternion.set(quat[1], quat[2], quat[3], quat[0]);
1931
+ mesh.userData.bodyID = mjModel.geom_bodyid[g];
1932
+ mesh.userData.geomID = g;
1933
+ mesh.userData.geomGroup = mjModel.geom_group[g];
1934
+ mesh.userData.geomName = getName(mjModel, mjModel.name_geomadr[g]);
1935
+ return mesh;
1936
+ }
1937
+ return null;
1938
+ }
1939
+ };
1940
+ function getRenderOptionsKey(renderOptions) {
1941
+ const smoothing = renderOptions?.meshNormalSmoothing;
1942
+ if (!smoothing) return "default";
1943
+ if (smoothing === true) return "meshNormalSmoothing:true";
1944
+ return `meshNormalSmoothing:${smoothing.tolerance ?? "default"}`;
1615
1945
  }
1616
- function getCameraFrameResourceMatch(key, resourceName) {
1617
- if (resourceName === key) return MountedCameraFrameSourceSuggestionMatch.Direct;
1618
- const normalizedResource = normalizeCameraSourceName(resourceName);
1619
- if (!normalizedResource) return null;
1620
- for (const variant of createCameraSourceKeyVariants(key)) {
1621
- if (resourceName === variant) return MountedCameraFrameSourceSuggestionMatch.Direct;
1622
- const normalizedKey = normalizeCameraSourceName(variant);
1623
- if (!normalizedKey) continue;
1624
- if (normalizedResource === normalizedKey) {
1625
- return MountedCameraFrameSourceSuggestionMatch.Normalized;
1946
+ function SceneRenderer({ renderOptions, ...props }) {
1947
+ const {
1948
+ mjModelRef,
1949
+ mjDataRef,
1950
+ mujocoRef,
1951
+ onSelectionRef,
1952
+ hiddenBodiesRef,
1953
+ hiddenBodiesVersion,
1954
+ interpolateRef,
1955
+ interpolationStateRef,
1956
+ status
1957
+ } = useMujocoContext();
1958
+ const groupRef = useRef(null);
1959
+ const bodyRefs = useRef([]);
1960
+ const prevModelRef = useRef(null);
1961
+ const prevRenderOptionsKeyRef = useRef(null);
1962
+ const prevHiddenVersionRef = useRef(null);
1963
+ const renderOptionsKey = getRenderOptionsKey(renderOptions);
1964
+ const geomBuilder = useMemo(() => {
1965
+ if (status !== "ready") return null;
1966
+ return new GeomBuilder(mujocoRef.current, renderOptions);
1967
+ }, [status, mujocoRef, renderOptionsKey]);
1968
+ useEffect(() => {
1969
+ if (status !== "ready" || !geomBuilder) return;
1970
+ const model = mjModelRef.current;
1971
+ const group = groupRef.current;
1972
+ if (!model || !group) return;
1973
+ if (prevModelRef.current === model && prevRenderOptionsKeyRef.current === renderOptionsKey && prevHiddenVersionRef.current === hiddenBodiesVersion) {
1974
+ return;
1626
1975
  }
1627
- if (normalizedResource.startsWith(`${normalizedKey}_`)) {
1628
- return MountedCameraFrameSourceSuggestionMatch.Prefix;
1976
+ prevModelRef.current = model;
1977
+ prevRenderOptionsKeyRef.current = renderOptionsKey;
1978
+ prevHiddenVersionRef.current = hiddenBodiesVersion;
1979
+ while (group.children.length > 0) {
1980
+ group.remove(group.children[0]);
1629
1981
  }
1630
- if (normalizedResource.endsWith(`_${normalizedKey}`)) {
1631
- return MountedCameraFrameSourceSuggestionMatch.Suffix;
1982
+ const refs = [];
1983
+ for (let i = 0; i < model.nbody; i++) {
1984
+ const bodyGroup = new THREE12.Group();
1985
+ bodyGroup.userData.bodyID = i;
1986
+ const bodyName = getName(model, model.name_bodyadr[i]);
1987
+ if (!hiddenBodiesRef.current.has(bodyName)) {
1988
+ for (let g = 0; g < model.ngeom; g++) {
1989
+ if (model.geom_bodyid[g] === i) {
1990
+ const mesh = geomBuilder.create(model, g);
1991
+ if (mesh) bodyGroup.add(mesh);
1992
+ }
1993
+ }
1994
+ }
1995
+ group.add(bodyGroup);
1996
+ refs.push(bodyGroup);
1632
1997
  }
1633
- if (normalizedResource.includes(`_${normalizedKey}_`)) {
1634
- return MountedCameraFrameSourceSuggestionMatch.Contains;
1998
+ bodyRefs.current = refs;
1999
+ }, [status, geomBuilder, mjModelRef, renderOptionsKey, hiddenBodiesRef, hiddenBodiesVersion]);
2000
+ const syncBodiesToData = useCallback(() => {
2001
+ const data = mjDataRef.current;
2002
+ if (!data) return;
2003
+ const bodies = bodyRefs.current;
2004
+ const interpolation = interpolationStateRef.current;
2005
+ const useInterpolation = interpolateRef.current && interpolation.valid;
2006
+ for (let i = 0; i < bodies.length; i++) {
2007
+ const ref = bodies[i];
2008
+ if (!ref) continue;
2009
+ if (useInterpolation) {
2010
+ const alpha = interpolation.alpha;
2011
+ const i3 = i * 3;
2012
+ ref.position.set(
2013
+ THREE12.MathUtils.lerp(interpolation.previousXpos[i3], interpolation.currentXpos[i3], alpha),
2014
+ THREE12.MathUtils.lerp(interpolation.previousXpos[i3 + 1], interpolation.currentXpos[i3 + 1], alpha),
2015
+ THREE12.MathUtils.lerp(interpolation.previousXpos[i3 + 2], interpolation.currentXpos[i3 + 2], alpha)
2016
+ );
2017
+ const i4 = i * 4;
2018
+ _previousQuat.set(
2019
+ interpolation.previousXquat[i4 + 1],
2020
+ interpolation.previousXquat[i4 + 2],
2021
+ interpolation.previousXquat[i4 + 3],
2022
+ interpolation.previousXquat[i4]
2023
+ );
2024
+ _currentQuat.set(
2025
+ interpolation.currentXquat[i4 + 1],
2026
+ interpolation.currentXquat[i4 + 2],
2027
+ interpolation.currentXquat[i4 + 3],
2028
+ interpolation.currentXquat[i4]
2029
+ );
2030
+ ref.quaternion.copy(_previousQuat).slerp(_currentQuat, alpha);
2031
+ } else {
2032
+ ref.position.set(
2033
+ data.xpos[i * 3],
2034
+ data.xpos[i * 3 + 1],
2035
+ data.xpos[i * 3 + 2]
2036
+ );
2037
+ ref.quaternion.set(
2038
+ data.xquat[i * 4 + 1],
2039
+ data.xquat[i * 4 + 2],
2040
+ data.xquat[i * 4 + 3],
2041
+ data.xquat[i * 4]
2042
+ );
2043
+ }
1635
2044
  }
1636
- }
1637
- return null;
1638
- }
1639
- function isSelectorMounted(selector, cameraNames, siteNames, bodyNames) {
1640
- if (countMountedSelectors(selector) !== 1) return false;
1641
- return (selector.cameraName ? cameraNames.has(selector.cameraName) : false) || (selector.siteName ? siteNames.has(selector.siteName) : false) || (selector.bodyName ? bodyNames.has(selector.bodyName) : false);
2045
+ }, [interpolateRef, interpolationStateRef, mjDataRef]);
2046
+ useFrame(syncBodiesToData);
2047
+ return /* @__PURE__ */ jsx(
2048
+ "group",
2049
+ {
2050
+ ...props,
2051
+ ref: groupRef,
2052
+ userData: {
2053
+ ...props.userData,
2054
+ [CAMERA_FRAME_CAPTURE_PRE_RENDER_USER_DATA_KEY]: syncBodiesToData
2055
+ },
2056
+ onDoubleClick: (e) => {
2057
+ if (typeof props.onDoubleClick === "function") props.onDoubleClick(e);
2058
+ e.stopPropagation();
2059
+ let obj = e.object;
2060
+ while (obj && obj.userData.bodyID === void 0 && obj.parent) {
2061
+ obj = obj.parent;
2062
+ }
2063
+ const bodyID = obj?.userData.bodyID;
2064
+ if (typeof bodyID === "number" && bodyID > 0) {
2065
+ const model = mjModelRef.current;
2066
+ if (model && bodyID < model.nbody && onSelectionRef.current) {
2067
+ const name = getName(model, model.name_bodyadr[bodyID]);
2068
+ onSelectionRef.current({ bodyId: bodyID, name });
2069
+ }
2070
+ }
2071
+ }
2072
+ }
2073
+ );
1642
2074
  }
1643
- function createMountedCameraFrameSourceSuggestions(key, options) {
1644
- const cameraNames = createNameSet(options.cameras);
1645
- const siteNames = createNameSet(options.sites);
1646
- const bodyNames = createNameSet(options.bodies);
1647
- const suggestions = [];
1648
- const seen = /* @__PURE__ */ new Set();
1649
- for (const selector of normalizeAliasCandidates(options.aliases?.[key])) {
1650
- if (!isSelectorMounted(selector, cameraNames, siteNames, bodyNames)) {
1651
- continue;
2075
+ var _previousQuat = new THREE12.Quaternion();
2076
+ var _currentQuat = new THREE12.Quaternion();
2077
+ var CameraViewportRegistryContext = createContext(null);
2078
+ var nextViewportId = 0;
2079
+ var VIEWPORT_RENDER_PRIORITY = 1;
2080
+ function CameraViewportRenderer({
2081
+ viewportsRef
2082
+ }) {
2083
+ const gl = useThree((state) => state.gl);
2084
+ const scene = useThree((state) => state.scene);
2085
+ const mainCamera = useThree((state) => state.camera);
2086
+ const mujoco = useMujoco();
2087
+ useFrame(() => {
2088
+ const dpr = gl.getPixelRatio();
2089
+ const drawWidth = gl.domElement.width / dpr;
2090
+ const drawHeight = gl.domElement.height / dpr;
2091
+ gl.setScissorTest(false);
2092
+ gl.setViewport(0, 0, drawWidth, drawHeight);
2093
+ gl.render(scene, mainCamera);
2094
+ const api = mujoco.api;
2095
+ if (!api || viewportsRef.current.size === 0) return;
2096
+ const canvasRect = gl.domElement.getBoundingClientRect();
2097
+ for (const descriptor of viewportsRef.current.values()) {
2098
+ const element = descriptor.getElement();
2099
+ if (!element) continue;
2100
+ const rect = element.getBoundingClientRect();
2101
+ const isOffscreen = rect.bottom < canvasRect.top || rect.top > canvasRect.bottom || rect.right < canvasRect.left || rect.left > canvasRect.right;
2102
+ if (isOffscreen) continue;
2103
+ const width = rect.width;
2104
+ const height = rect.height;
2105
+ if (width <= 0 || height <= 0) continue;
2106
+ const left = rect.left - canvasRect.left;
2107
+ const bottom = canvasRect.bottom - rect.bottom;
2108
+ let resolved;
2109
+ try {
2110
+ resolved = api.resolveCameraCaptureOptions(descriptor.getOptions());
2111
+ } catch {
2112
+ continue;
2113
+ }
2114
+ if (!descriptor.camera) {
2115
+ descriptor.camera = createCaptureCamera(resolved, mainCamera, width, height);
2116
+ } else {
2117
+ prepareCaptureCamera(descriptor.camera, resolved, mainCamera, width, height);
2118
+ }
2119
+ gl.setViewport(left, bottom, width, height);
2120
+ gl.setScissor(left, bottom, width, height);
2121
+ gl.setScissorTest(true);
2122
+ gl.render(scene, descriptor.camera);
1652
2123
  }
1653
- const source = getMountedCameraFrameCaptureSource(selector);
1654
- if (!source) continue;
1655
- addMountedCameraFrameSourceSuggestion(
1656
- suggestions,
1657
- seen,
1658
- createMountedCameraFrameSourceSuggestion(
1659
- key,
1660
- selector,
1661
- getCameraFrameCaptureSourceTarget(source),
1662
- MountedCameraFrameSourceSuggestionMatch.Alias
1663
- )
1664
- );
1665
- }
1666
- for (const cameraName of createResourceNames(options.cameras)) {
1667
- const match = getCameraFrameResourceMatch(key, cameraName);
1668
- if (!match) continue;
1669
- addMountedCameraFrameSourceSuggestion(
1670
- suggestions,
1671
- seen,
1672
- createMountedCameraFrameSourceSuggestion(
1673
- key,
1674
- { cameraName },
1675
- cameraName,
1676
- match
1677
- )
1678
- );
1679
- }
1680
- for (const siteName of createResourceNames(options.sites)) {
1681
- const match = getCameraFrameResourceMatch(key, siteName);
1682
- if (!match) continue;
1683
- addMountedCameraFrameSourceSuggestion(
1684
- suggestions,
1685
- seen,
1686
- createMountedCameraFrameSourceSuggestion(
1687
- key,
1688
- { siteName },
1689
- siteName,
1690
- match
1691
- )
1692
- );
1693
- }
1694
- for (const bodyName of createResourceNames(options.bodies)) {
1695
- const match = getCameraFrameResourceMatch(key, bodyName);
1696
- if (!match) continue;
1697
- addMountedCameraFrameSourceSuggestion(
1698
- suggestions,
1699
- seen,
1700
- createMountedCameraFrameSourceSuggestion(
1701
- key,
1702
- { bodyName },
1703
- bodyName,
1704
- match
1705
- )
1706
- );
1707
- }
1708
- return suggestions;
2124
+ gl.setScissorTest(false);
2125
+ gl.setViewport(0, 0, drawWidth, drawHeight);
2126
+ }, VIEWPORT_RENDER_PRIORITY);
2127
+ return null;
2128
+ }
2129
+ function CameraViewportProvider({ children }) {
2130
+ const viewportsRef = useRef(/* @__PURE__ */ new Map());
2131
+ const [count, setCount] = useState(0);
2132
+ const register = useCallback((descriptor) => {
2133
+ const id = nextViewportId++;
2134
+ viewportsRef.current.set(id, descriptor);
2135
+ setCount((value2) => value2 + 1);
2136
+ return () => {
2137
+ viewportsRef.current.delete(id);
2138
+ setCount((value2) => value2 - 1);
2139
+ };
2140
+ }, []);
2141
+ const value = useMemo(() => ({ register }), [register]);
2142
+ return /* @__PURE__ */ jsxs(CameraViewportRegistryContext.Provider, { value, children: [
2143
+ children,
2144
+ count > 0 && /* @__PURE__ */ jsx(CameraViewportRenderer, { viewportsRef })
2145
+ ] });
1709
2146
  }
1710
- function resolveMountedCameraFrameSource(key, options) {
1711
- const cameraNames = createNameSet(options.cameras);
1712
- const siteNames = createNameSet(options.sites);
1713
- const bodyNames = createNameSet(options.bodies);
1714
- const directCandidates = [
1715
- { cameraName: key },
1716
- { siteName: key },
1717
- { bodyName: key }
1718
- ];
1719
- const aliasCandidates = normalizeAliasCandidates(options.aliases?.[key]);
1720
- for (const selector of aliasCandidates) {
1721
- if (!isSelectorMounted(selector, cameraNames, siteNames, bodyNames)) {
1722
- continue;
1723
- }
1724
- const source = getMountedCameraFrameCaptureSource(selector);
1725
- if (!source) continue;
1726
- return { key, selector, source };
2147
+ function useCameraViewport(elementRef, options) {
2148
+ const registry = useContext(CameraViewportRegistryContext);
2149
+ if (!registry) {
2150
+ throw new Error("useCameraViewport must be used inside <MujocoCanvas>.");
1727
2151
  }
1728
- const [suggestion] = createMountedCameraFrameSourceSuggestions(key, options);
1729
- if (suggestion) {
1730
- return {
1731
- key,
1732
- selector: suggestion.selector,
1733
- source: suggestion.source
2152
+ const optionsRef = useRef(options);
2153
+ optionsRef.current = options;
2154
+ useEffect(() => {
2155
+ const descriptor = {
2156
+ getElement: () => elementRef.current,
2157
+ getOptions: () => optionsRef.current,
2158
+ camera: null
1734
2159
  };
2160
+ return registry.register(descriptor);
2161
+ }, [registry, elementRef]);
2162
+ }
2163
+ function CameraView({ className, style, ...options }) {
2164
+ const gl = useThree((state) => state.gl);
2165
+ const elementRef = useRef(null);
2166
+ if (!elementRef.current && typeof document !== "undefined") {
2167
+ elementRef.current = document.createElement("div");
1735
2168
  }
1736
- if (options.allowAliasFallback) {
1737
- for (const selector of aliasCandidates) {
1738
- const source = getMountedCameraFrameCaptureSource(selector);
1739
- if (!source) continue;
1740
- return { key, selector, source };
1741
- }
1742
- }
1743
- for (const selector of directCandidates) {
1744
- if (!isSelectorMounted(selector, cameraNames, siteNames, bodyNames)) {
1745
- continue;
1746
- }
1747
- const source = getMountedCameraFrameCaptureSource(selector);
1748
- if (!source) continue;
1749
- return { key, selector, source };
1750
- }
2169
+ useEffect(() => {
2170
+ const element = elementRef.current;
2171
+ const parent = gl.domElement.parentElement;
2172
+ if (!element || !parent) return;
2173
+ element.style.position = "absolute";
2174
+ element.style.overflow = "hidden";
2175
+ parent.appendChild(element);
2176
+ return () => {
2177
+ parent.removeChild(element);
2178
+ };
2179
+ }, [gl]);
2180
+ useEffect(() => {
2181
+ const element = elementRef.current;
2182
+ if (!element) return;
2183
+ element.className = className ?? "";
2184
+ if (style) Object.assign(element.style, style);
2185
+ }, [className, style]);
2186
+ useCameraViewport(elementRef, options);
1751
2187
  return null;
1752
2188
  }
1753
- function createMountedCameraFrameSequencePlan(cameraKeys, options) {
1754
- const cameras = [];
1755
- const resolved = {};
1756
- const missingKeys = [];
1757
- for (const key of cameraKeys) {
1758
- const mountedSource = resolveMountedCameraFrameSource(key, options);
1759
- if (!mountedSource) {
1760
- missingKeys.push(key);
1761
- continue;
1762
- }
1763
- resolved[key] = mountedSource;
1764
- cameras.push({
1765
- key,
1766
- ...options.defaults,
1767
- ...options.cameraOptions?.[key],
1768
- ...mountedSource.selector,
1769
- source: mountedSource.source
1770
- });
2189
+ function isTargetRef(target) {
2190
+ return Boolean(target && typeof target === "object" && "current" in target);
2191
+ }
2192
+ function resolveCanvasTarget(target) {
2193
+ const resolvedTarget = isTargetRef(target) ? target.current : target;
2194
+ if (!resolvedTarget) {
2195
+ throw new Error("No frame capture target is available.");
1771
2196
  }
1772
- if (options.requireAll && missingKeys.length > 0) {
1773
- throw new Error(
1774
- `Unable to resolve mounted MuJoCo camera source${missingKeys.length === 1 ? "" : "s"} for ${missingKeys.join(", ")}.`
1775
- );
2197
+ if (resolvedTarget instanceof HTMLCanvasElement) {
2198
+ return resolvedTarget;
1776
2199
  }
1777
- return {
1778
- cameraKeys: [...cameraKeys],
1779
- cameras,
1780
- resolved,
1781
- missingKeys
1782
- };
1783
- }
1784
- function createMountedCameraFrameSequenceReadiness(plan) {
1785
- const cameras = {};
1786
- const resolvedKeys = plan.cameraKeys.filter((key) => Boolean(plan.resolved[key]));
1787
- for (const key of plan.cameraKeys) {
1788
- const resolved = plan.resolved[key];
1789
- cameras[key] = resolved ? {
1790
- key,
1791
- ready: true,
1792
- selector: resolved.selector,
1793
- source: resolved.source,
1794
- message: `Camera stream "${key}" resolves to ${resolved.source.kind}:${getCameraFrameCaptureSourceTarget(resolved.source)}.`
1795
- } : {
1796
- key,
1797
- ready: false,
1798
- message: `Camera stream "${key}" does not resolve to a mounted MuJoCo camera, site, or body.`
1799
- };
2200
+ const canvas = resolvedTarget.querySelector("canvas");
2201
+ if (!canvas) {
2202
+ throw new Error("Frame capture target does not contain a canvas.");
1800
2203
  }
1801
- const missingKeys = [...plan.missingKeys];
1802
- const ready = missingKeys.length === 0;
1803
- const status = ready ? MountedCameraFrameSequenceReadinessStatus.Ready : resolvedKeys.length > 0 ? MountedCameraFrameSequenceReadinessStatus.Partial : MountedCameraFrameSequenceReadinessStatus.Missing;
1804
- return {
1805
- ready,
1806
- status,
1807
- cameraKeys: [...plan.cameraKeys],
1808
- resolvedKeys,
1809
- missingKeys,
1810
- cameras,
1811
- message: ready ? `All ${plan.cameraKeys.length} requested camera stream${plan.cameraKeys.length === 1 ? "" : "s"} resolve to mounted MuJoCo sources.` : `Missing mounted MuJoCo source${missingKeys.length === 1 ? "" : "s"} for ${missingKeys.join(", ")}.`
1812
- };
2204
+ return canvas;
1813
2205
  }
1814
- function normalizeFrameCount(frameCount) {
1815
- return Number.isFinite(frameCount) && frameCount !== void 0 ? Math.max(0, Math.floor(frameCount)) : 0;
2206
+ function waitForNextAnimationFrame() {
2207
+ return new Promise((resolve) => {
2208
+ requestAnimationFrame(() => resolve());
2209
+ });
1816
2210
  }
1817
- function createMountedCameraFrameSequenceManifest(result, options = {}) {
1818
- const cameraKeys = [
1819
- ...options.cameraKeys ?? result.readiness.cameraKeys ?? result.plan.cameraKeys ?? result.cameraKeys
1820
- ];
1821
- const expectedFrameCount = normalizeFrameCount(
1822
- options.expectedFrameCount ?? result.frameCount
1823
- );
1824
- const recordedFrameCount = normalizeFrameCount(result.frameCount);
1825
- const streamSummaries = {};
1826
- const streams = [];
1827
- let missingFrameCount = 0;
1828
- let completeStreamCount = 0;
1829
- let resolvedOrRecordedStreamCount = 0;
1830
- for (const key of cameraKeys) {
1831
- const summary = result.cameraSummaries[key];
1832
- const readiness = result.readiness.cameras[key];
1833
- const source = summary?.source ?? readiness?.source;
1834
- const ready = readiness?.ready ?? Boolean(summary);
1835
- const recorded = normalizeFrameCount(summary?.frameCount);
1836
- const missing = Math.max(expectedFrameCount - recorded, 0);
1837
- const complete2 = ready && missing === 0;
1838
- const status2 = complete2 ? MountedCameraFrameSequenceManifestStatus.Complete : ready || recorded > 0 ? MountedCameraFrameSequenceManifestStatus.Partial : MountedCameraFrameSequenceManifestStatus.Missing;
1839
- const target = source ? getCameraFrameCaptureSourceTarget(source) : readiness?.message ? void 0 : "missing MuJoCo camera";
1840
- const message = complete2 ? `Camera stream "${key}" recorded ${recorded} of ${expectedFrameCount} frame${expectedFrameCount === 1 ? "" : "s"}.` : ready || recorded > 0 ? `Camera stream "${key}" recorded ${recorded} of ${expectedFrameCount} frame${expectedFrameCount === 1 ? "" : "s"}.` : readiness?.message ?? `Camera stream "${key}" did not record any frames.`;
1841
- const stream = {
1842
- key,
1843
- ready,
1844
- complete: complete2,
1845
- status: status2,
1846
- source,
1847
- selector: readiness?.selector,
1848
- target,
1849
- width: summary?.width,
1850
- height: summary?.height,
1851
- expectedFrameCount,
1852
- recordedFrameCount: recorded,
1853
- missingFrameCount: missing,
1854
- firstFrameIndex: summary?.firstFrameIndex ?? null,
1855
- lastFrameIndex: summary?.lastFrameIndex ?? null,
1856
- firstTimestamp: summary?.firstTimestamp ?? null,
1857
- lastTimestamp: summary?.lastTimestamp ?? null,
1858
- message
1859
- };
1860
- streamSummaries[key] = stream;
1861
- streams.push(stream);
1862
- missingFrameCount += missing;
1863
- if (complete2) completeStreamCount += 1;
1864
- if (ready || recorded > 0) resolvedOrRecordedStreamCount += 1;
2211
+ async function captureFrame(options) {
2212
+ const type = options.type ?? "image/png";
2213
+ const canvas = resolveCanvasTarget(options.target);
2214
+ if (options.waitForAnimationFrame ?? true) {
2215
+ await waitForNextAnimationFrame();
1865
2216
  }
1866
- const complete = result.readiness.ready && streams.length === completeStreamCount && missingFrameCount === 0;
1867
- const status = complete ? MountedCameraFrameSequenceManifestStatus.Complete : resolvedOrRecordedStreamCount > 0 ? MountedCameraFrameSequenceManifestStatus.Partial : MountedCameraFrameSequenceManifestStatus.Missing;
1868
2217
  return {
1869
- schema: "mujoco-react/mounted-camera-frame-sequence-manifest@1",
1870
- ready: result.readiness.ready,
1871
- complete,
1872
- status,
1873
- cameraKeys,
1874
- resolvedKeys: [...result.readiness.resolvedKeys],
1875
- missingKeys: [...result.readiness.missingKeys],
1876
- expectedFrameCount,
1877
- recordedFrameCount,
1878
- missingFrameCount,
1879
- streamSummaries,
1880
- streams,
1881
- readiness: result.readiness,
1882
- message: complete ? `All ${cameraKeys.length} camera stream${cameraKeys.length === 1 ? "" : "s"} recorded ${expectedFrameCount} frame${expectedFrameCount === 1 ? "" : "s"}.` : `Mounted camera sequence coverage is ${status}.`
2218
+ canvas,
2219
+ dataUrl: canvas.toDataURL(type, options.quality),
2220
+ type
1883
2221
  };
1884
2222
  }
1885
- function createMountedCameraFrameSequencePlanFromApi(api, cameraKeys, options = {}) {
1886
- return createMountedCameraFrameSequencePlan(cameraKeys, {
1887
- ...options,
1888
- cameras: api.getCameras(),
1889
- sites: api.getSites(),
1890
- bodies: api.getBodies()
1891
- });
1892
- }
1893
- async function recordMountedCameraFrameSequence(api, options) {
1894
- const { cameraKeys, ...restOptions } = options;
1895
- const requireAll = restOptions.requireAll ?? restOptions.requireMountedSources ?? true;
1896
- const plan = createMountedCameraFrameSequencePlanFromApi(
1897
- api,
1898
- cameraKeys,
1899
- { ...restOptions, requireAll }
1900
- );
1901
- const readiness = createMountedCameraFrameSequenceReadiness(plan);
1902
- const result = await api.recordCameraSequence({
1903
- ...restOptions,
1904
- cameras: plan.cameras,
1905
- requireMountedSources: restOptions.requireMountedSources ?? true
2223
+ async function captureFrameBlob(options) {
2224
+ const type = options.type ?? "image/png";
2225
+ const canvas = resolveCanvasTarget(options.target);
2226
+ if (options.waitForAnimationFrame ?? true) {
2227
+ await waitForNextAnimationFrame();
2228
+ }
2229
+ const blob = await new Promise((resolve, reject) => {
2230
+ canvas.toBlob(
2231
+ (nextBlob) => {
2232
+ if (nextBlob) {
2233
+ resolve(nextBlob);
2234
+ } else {
2235
+ reject(new Error("Canvas frame capture did not produce a Blob."));
2236
+ }
2237
+ },
2238
+ type,
2239
+ options.quality
2240
+ );
1906
2241
  });
2242
+ return { canvas, blob, type };
2243
+ }
2244
+ function useFrameCapture(defaultOptions = {}) {
2245
+ const [status, setStatus] = useState("idle");
2246
+ const [error, setError] = useState(null);
2247
+ const reset = useCallback(() => {
2248
+ setStatus("idle");
2249
+ setError(null);
2250
+ }, []);
2251
+ const capture = useCallback(
2252
+ async (options = {}) => {
2253
+ setStatus("capturing");
2254
+ setError(null);
2255
+ try {
2256
+ const result = await captureFrame({ ...defaultOptions, ...options });
2257
+ setStatus("captured");
2258
+ return result;
2259
+ } catch (nextError) {
2260
+ const error2 = nextError instanceof Error ? nextError : new Error("Unable to capture the current canvas frame.");
2261
+ setError(error2);
2262
+ setStatus("error");
2263
+ throw error2;
2264
+ }
2265
+ },
2266
+ [defaultOptions]
2267
+ );
2268
+ const captureBlob = useCallback(
2269
+ async (options = {}) => {
2270
+ setStatus("capturing");
2271
+ setError(null);
2272
+ try {
2273
+ const result = await captureFrameBlob({
2274
+ ...defaultOptions,
2275
+ ...options
2276
+ });
2277
+ setStatus("captured");
2278
+ return result;
2279
+ } catch (nextError) {
2280
+ const error2 = nextError instanceof Error ? nextError : new Error("Unable to capture the current canvas frame.");
2281
+ setError(error2);
2282
+ setStatus("error");
2283
+ throw error2;
2284
+ }
2285
+ },
2286
+ [defaultOptions]
2287
+ );
1907
2288
  return {
1908
- ...result,
1909
- plan,
1910
- readiness
2289
+ status,
2290
+ error,
2291
+ isCapturing: status === "capturing",
2292
+ capture,
2293
+ captureBlob,
2294
+ reset
1911
2295
  };
1912
2296
  }
1913
2297
  var _raycaster = new THREE12.Raycaster();
@@ -2358,10 +2742,11 @@ function useMujoco() {
2358
2742
  };
2359
2743
  }
2360
2744
  function useBeforePhysicsStep(callback) {
2361
- const { beforeStepCallbacks } = useMujocoContext();
2745
+ const { beforeStepCallbacks, mjModelRef } = useMujocoContext();
2362
2746
  const callbackRef = useRef(callback);
2363
2747
  callbackRef.current = callback;
2364
2748
  useEffect(() => {
2749
+ assertMainThreadOwnership(mjModelRef.current);
2365
2750
  const wrapped = (input) => callbackRef.current(input);
2366
2751
  beforeStepCallbacks.current.add(wrapped);
2367
2752
  return () => {
@@ -2370,10 +2755,11 @@ function useBeforePhysicsStep(callback) {
2370
2755
  }, [beforeStepCallbacks]);
2371
2756
  }
2372
2757
  function useAfterPhysicsStep(callback) {
2373
- const { afterStepCallbacks } = useMujocoContext();
2758
+ const { afterStepCallbacks, mjModelRef } = useMujocoContext();
2374
2759
  const callbackRef = useRef(callback);
2375
2760
  callbackRef.current = callback;
2376
2761
  useEffect(() => {
2762
+ assertMainThreadOwnership(mjModelRef.current);
2377
2763
  const wrapped = (input) => callbackRef.current(input);
2378
2764
  afterStepCallbacks.current.add(wrapped);
2379
2765
  return () => {
@@ -2393,6 +2779,7 @@ function MujocoSimProvider({
2393
2779
  timestep,
2394
2780
  integrator,
2395
2781
  substeps,
2782
+ physicsBudgetMs = 8,
2396
2783
  paused,
2397
2784
  speed,
2398
2785
  interpolate,
@@ -2400,7 +2787,32 @@ function MujocoSimProvider({
2400
2787
  hiddenBodies,
2401
2788
  children
2402
2789
  }) {
2403
- const { gl, camera, scene } = useThree();
2790
+ const { gl, camera, scene, invalidate } = useThree();
2791
+ const modelIdentity = sceneIdentity(config);
2792
+ const performanceRef = useRef({ steps: 0, physicsMs: 0, realTimeFactor: 0, backlogSteps: 0, budgetExceeded: false });
2793
+ const displayWaiters = useRef(/* @__PURE__ */ new Set());
2794
+ useFrame(() => {
2795
+ if (displayWaiters.current.size) {
2796
+ const waiters = [...displayWaiters.current];
2797
+ displayWaiters.current.clear();
2798
+ queueMicrotask(() => {
2799
+ scene.updateMatrixWorld(true);
2800
+ for (const ready of waiters) ready();
2801
+ });
2802
+ }
2803
+ }, 0);
2804
+ const waitForDisplay = () => new Promise((resolve, reject) => {
2805
+ const ready = () => {
2806
+ clearTimeout(timer);
2807
+ resolve();
2808
+ };
2809
+ const timer = setTimeout(() => {
2810
+ displayWaiters.current.delete(ready);
2811
+ reject(new Error("Canvas did not produce an observation frame within 5 seconds"));
2812
+ }, 5e3);
2813
+ displayWaiters.current.add(ready);
2814
+ invalidate();
2815
+ });
2404
2816
  const [status, setStatus] = useState("loading");
2405
2817
  const mjModelRef = useRef(null);
2406
2818
  const mjDataRef = useRef(null);
@@ -2436,18 +2848,23 @@ function MujocoSimProvider({
2436
2848
  const bodyReloadTimerRef = useRef(null);
2437
2849
  configRef.current = config;
2438
2850
  mujocoRef.current = mujoco;
2439
- pausedRef.current = paused ?? false;
2440
- speedRef.current = speed ?? 1;
2851
+ useEffect(() => {
2852
+ pausedRef.current = paused ?? false;
2853
+ }, [paused]);
2854
+ useEffect(() => {
2855
+ speedRef.current = speed ?? 1;
2856
+ }, [speed]);
2441
2857
  substepsRef.current = substeps ?? 1;
2442
2858
  interpolateRef.current = interpolate ?? false;
2443
2859
  useEffect(() => {
2444
2860
  if (!gravity) return;
2445
2861
  const model = mjModelRef.current;
2446
2862
  if (!model?.opt?.gravity) return;
2863
+ assertMainThreadOwnership(model);
2447
2864
  model.opt.gravity[0] = gravity[0];
2448
2865
  model.opt.gravity[1] = gravity[1];
2449
2866
  model.opt.gravity[2] = gravity[2];
2450
- }, [gravity]);
2867
+ }, [gravity?.[0], gravity?.[1], gravity?.[2]]);
2451
2868
  const integratorRef = useRef(integrator);
2452
2869
  integratorRef.current = integrator;
2453
2870
  const applyIntegrator = (model) => {
@@ -2455,12 +2872,16 @@ function MujocoSimProvider({
2455
2872
  if (value !== void 0) model.opt.integrator = { Euler: 0, RK4: 1, implicit: 2, implicitfast: 3, discrete: 4 }[value];
2456
2873
  };
2457
2874
  useEffect(() => {
2458
- if (mjModelRef.current) applyIntegrator(mjModelRef.current);
2875
+ if (mjModelRef.current) {
2876
+ assertMainThreadOwnership(mjModelRef.current);
2877
+ applyIntegrator(mjModelRef.current);
2878
+ }
2459
2879
  }, [integrator]);
2460
2880
  useEffect(() => {
2461
2881
  if (timestep === void 0) return;
2462
2882
  const model = mjModelRef.current;
2463
2883
  if (!model?.opt) return;
2884
+ assertMainThreadOwnership(model);
2464
2885
  model.opt.timestep = timestep;
2465
2886
  }, [timestep]);
2466
2887
  const hiddenBodiesKey = (hiddenBodies ?? []).join("\0");
@@ -2498,6 +2919,7 @@ function MujocoSimProvider({
2498
2919
  }
2499
2920
  useEffect(() => {
2500
2921
  let disposed = false;
2922
+ setStatus("loading");
2501
2923
  (async () => {
2502
2924
  try {
2503
2925
  const result = await loadScene(mujoco, buildMergedConfig(config));
@@ -2532,6 +2954,7 @@ function MujocoSimProvider({
2532
2954
  })();
2533
2955
  return () => {
2534
2956
  disposed = true;
2957
+ closeModelRuntime(mjModelRef.current);
2535
2958
  mjDataRef.current?.delete();
2536
2959
  mjModelRef.current?.delete();
2537
2960
  mjModelRef.current = null;
@@ -2543,7 +2966,7 @@ function MujocoSimProvider({
2543
2966
  } catch {
2544
2967
  }
2545
2968
  };
2546
- }, [mujoco, config]);
2969
+ }, [mujoco, modelIdentity]);
2547
2970
  useEffect(() => {
2548
2971
  if (status === "ready") {
2549
2972
  const api2 = apiRef.current;
@@ -2561,66 +2984,43 @@ function MujocoSimProvider({
2561
2984
  const model = mjModelRef.current;
2562
2985
  const data = mjDataRef.current;
2563
2986
  if (!model || !data) return;
2564
- const shouldStep = !pausedRef.current || stepsToRunRef.current > 0;
2565
- if (!shouldStep) return;
2566
- for (let i = 0; i < model.nv; i++) {
2567
- data.qfrc_applied[i] = 0;
2568
- }
2569
- for (const cb of beforeStepCallbacks.current) {
2570
- cb({ model, data });
2987
+ if (isWorkerOwned(model)) {
2988
+ interpolationStateRef.current.valid = false;
2989
+ return;
2571
2990
  }
2572
- const numSubsteps = substepsRef.current;
2573
- if (!interpolateRef.current) {
2574
- if (stepsToRunRef.current > 0) {
2575
- for (let s = 0; s < stepsToRunRef.current; s++) {
2576
- mujocoRef.current.mj_step(model, data);
2577
- }
2578
- stepsToRunRef.current = 0;
2579
- } else {
2580
- const startSimTime = data.time;
2581
- const clampedDelta = Math.min(delta, 1 / 15);
2582
- const frameTime = clampedDelta * speedRef.current;
2583
- while (data.time - startSimTime < frameTime) {
2584
- for (let s = 0; s < numSubsteps; s++) {
2585
- mujocoRef.current.mj_step(model, data);
2586
- }
2587
- }
2588
- }
2589
- } else if (stepsToRunRef.current > 0) {
2991
+ const queued = stepsToRunRef.current > 0;
2992
+ if (pausedRef.current && !queued) return;
2993
+ const startTime = data.time;
2994
+ const dt = model.opt.timestep;
2995
+ if (!Number.isFinite(dt) || dt <= 0) return;
2996
+ const substeps2 = Math.max(1, Math.floor(substepsRef.current));
2997
+ const frameTime = Math.max(0, Math.min(delta, 1 / 15) * speedRef.current);
2998
+ if (!queued && interpolateRef.current) physicsAccumulatorRef.current += frameTime;
2999
+ const desired = queued ? stepsToRunRef.current : interpolateRef.current ? Math.floor(physicsAccumulatorRef.current / (dt * substeps2)) * substeps2 : Math.ceil(frameTime / (dt * substeps2)) * substeps2;
3000
+ for (let i = 0; i < model.nv; i++) data.qfrc_applied[i] = 0;
3001
+ for (const cb of beforeStepCallbacks.current) cb({ model, data });
3002
+ if (interpolateRef.current) {
2590
3003
  ensureInterpolationBuffers(model);
2591
- copyBodyPose(data, interpolationStateRef.current.previousXpos, interpolationStateRef.current.previousXquat);
2592
- for (let s = 0; s < stepsToRunRef.current; s++) {
2593
- mujocoRef.current.mj_step(model, data);
2594
- }
3004
+ if (!interpolationStateRef.current.valid) copyBodyPose(data, interpolationStateRef.current.previousXpos, interpolationStateRef.current.previousXquat);
3005
+ }
3006
+ const measured = runStepBudget(desired, () => {
3007
+ if (interpolateRef.current) copyBodyPose(data, interpolationStateRef.current.previousXpos, interpolationStateRef.current.previousXquat);
3008
+ mujocoRef.current.mj_step(model, data);
3009
+ }, physicsBudgetMs);
3010
+ if (queued) stepsToRunRef.current -= measured.steps;
3011
+ performanceRef.current = {
3012
+ ...measured,
3013
+ backlogSteps: Math.max(0, desired - measured.steps),
3014
+ realTimeFactor: (data.time - startTime) / Math.max(delta, 1e-6)
3015
+ };
3016
+ if (interpolateRef.current) {
2595
3017
  copyBodyPose(data, interpolationStateRef.current.currentXpos, interpolationStateRef.current.currentXquat);
2596
- interpolationStateRef.current.alpha = 1;
2597
- interpolationStateRef.current.valid = true;
2598
- stepsToRunRef.current = 0;
2599
- } else {
2600
- ensureInterpolationBuffers(model);
2601
- const clampedDelta = Math.min(delta, 1 / 15);
2602
- physicsAccumulatorRef.current += clampedDelta * speedRef.current;
2603
- const stepDt = Math.max((model.opt?.timestep ?? 2e-3) * Math.max(1, numSubsteps), 1e-6);
2604
- let stepped = false;
2605
- while (physicsAccumulatorRef.current >= stepDt) {
2606
- copyBodyPose(data, interpolationStateRef.current.previousXpos, interpolationStateRef.current.previousXquat);
2607
- for (let s = 0; s < numSubsteps; s++) {
2608
- mujocoRef.current.mj_step(model, data);
2609
- }
2610
- copyBodyPose(data, interpolationStateRef.current.currentXpos, interpolationStateRef.current.currentXquat);
2611
- physicsAccumulatorRef.current -= stepDt;
2612
- stepped = true;
2613
- }
2614
- if (!interpolationStateRef.current.valid) {
2615
- copyBodyPose(data, interpolationStateRef.current.previousXpos, interpolationStateRef.current.previousXquat);
2616
- copyBodyPose(data, interpolationStateRef.current.currentXpos, interpolationStateRef.current.currentXquat);
2617
- }
2618
- interpolationStateRef.current.alpha = Math.min(Math.max(physicsAccumulatorRef.current / stepDt, 0), 1);
3018
+ if (!queued) {
3019
+ physicsAccumulatorRef.current -= measured.steps * dt;
3020
+ if (measured.budgetExceeded) physicsAccumulatorRef.current %= dt * substeps2;
3021
+ } else physicsAccumulatorRef.current = 0;
3022
+ interpolationStateRef.current.alpha = queued ? 1 : Math.min(1, physicsAccumulatorRef.current / dt);
2619
3023
  interpolationStateRef.current.valid = true;
2620
- if (!stepped) {
2621
- onStepRef.current?.({ time: data.time, model, data });
2622
- return;
2623
- }
2624
3024
  }
2625
3025
  for (const cb of afterStepCallbacks.current) {
2626
3026
  cb({ model, data });
@@ -2673,6 +3073,7 @@ function MujocoSimProvider({
2673
3073
  pausedRef.current = p;
2674
3074
  }, []);
2675
3075
  const step = useCallback((n = 1) => {
3076
+ if (!Number.isSafeInteger(n) || n <= 0) throw new Error("Step count must be a positive safe integer");
2676
3077
  stepsToRunRef.current = n;
2677
3078
  }, []);
2678
3079
  useCallback((steps = 1) => {
@@ -3091,6 +3492,7 @@ function MujocoSimProvider({
3091
3492
  const model = mjModelRef.current;
3092
3493
  const data = mjDataRef.current;
3093
3494
  if (!model || !data) return null;
3495
+ assertMainThreadOwnership(model);
3094
3496
  _rayPnt[0] = origin.x;
3095
3497
  _rayPnt[1] = origin.y;
3096
3498
  _rayPnt[2] = origin.z;
@@ -3162,6 +3564,7 @@ function MujocoSimProvider({
3162
3564
  const loadSceneApi = useCallback(async (newConfig) => {
3163
3565
  const gen = ++loadGenRef.current;
3164
3566
  try {
3567
+ closeModelRuntime(mjModelRef.current);
3165
3568
  mjDataRef.current?.delete();
3166
3569
  mjModelRef.current?.delete();
3167
3570
  mjModelRef.current = null;
@@ -3317,6 +3720,7 @@ function MujocoSimProvider({
3317
3720
  if (!model || !data) {
3318
3721
  throw new Error("MuJoCo scene is not ready for camera sequence stepping.");
3319
3722
  }
3723
+ let sliceStarted = performance.now();
3320
3724
  for (let stepIndex = 0; stepIndex < steps; stepIndex += 1) {
3321
3725
  for (let i = 0; i < model.nv; i += 1) {
3322
3726
  data.qfrc_applied[i] = 0;
@@ -3336,6 +3740,11 @@ function MujocoSimProvider({
3336
3740
  cb({ model, data });
3337
3741
  }
3338
3742
  onStepRef.current?.({ time: data.time, model, data });
3743
+ if (performance.now() - sliceStarted >= 8) {
3744
+ await new Promise((resolve) => setTimeout(resolve, 0));
3745
+ throwIfCameraSequenceAborted(options.signal);
3746
+ sliceStarted = performance.now();
3747
+ }
3339
3748
  await options.onAfterStep?.({
3340
3749
  frameIndex,
3341
3750
  stepIndex,
@@ -3543,7 +3952,14 @@ function MujocoSimProvider({
3543
3952
  model.geom_size[id * 3 + 2] = size[2];
3544
3953
  }, []);
3545
3954
  const api = useMemo(
3546
- () => ({
3955
+ () => guardRuntimeOwnership({
3956
+ createRuntime: (options) => {
3957
+ if (beforeStepCallbacks.current.size || afterStepCallbacks.current.size) {
3958
+ return Promise.reject(new Error("Unmount physics callbacks/controllers before creating a worker runtime; use runtime.advance({controls}) for policy actions."));
3959
+ }
3960
+ return attachSimulationRuntime(mujocoRef.current, apiRef.current, waitForDisplay, invalidate, options);
3961
+ },
3962
+ getPerformance: () => ({ ...performanceRef.current }),
3547
3963
  get status() {
3548
3964
  return status;
3549
3965
  },
@@ -3574,7 +3990,10 @@ function MujocoSimProvider({
3574
3990
  setExternalForce,
3575
3991
  applyGeneralizedForce,
3576
3992
  getSensorData,
3577
- getContacts,
3993
+ getContacts: () => {
3994
+ const model = mjModelRef.current;
3995
+ return model && isWorkerOwned(model) ? runtimeContacts(model) : getContacts();
3996
+ },
3578
3997
  getBodies,
3579
3998
  getJoints,
3580
3999
  getGeoms,
@@ -3610,7 +4029,7 @@ function MujocoSimProvider({
3610
4029
  setGeomSize,
3611
4030
  mjModelRef,
3612
4031
  mjDataRef
3613
- }),
4032
+ }, () => mjModelRef.current),
3614
4033
  [
3615
4034
  status,
3616
4035
  reset,
@@ -3705,6 +4124,45 @@ function MujocoSimProvider({
3705
4124
  /* @__PURE__ */ jsx(CameraViewportProvider, { children })
3706
4125
  ] });
3707
4126
  }
4127
+ var runtimeMutators = /* @__PURE__ */ new Set([
4128
+ "reset",
4129
+ "setSpeed",
4130
+ "togglePause",
4131
+ "setPaused",
4132
+ "step",
4133
+ "applyKeyframe",
4134
+ "saveState",
4135
+ "restoreState",
4136
+ "setQpos",
4137
+ "setQvel",
4138
+ "setCtrl",
4139
+ "applyForce",
4140
+ "applyTorque",
4141
+ "setExternalForce",
4142
+ "applyGeneralizedForce",
4143
+ "setGravity",
4144
+ "setTimestep",
4145
+ "setBodyMass",
4146
+ "setGeomFriction",
4147
+ "setGeomSize",
4148
+ "loadScene",
4149
+ "loadFromFiles",
4150
+ "addBody",
4151
+ "removeBody",
4152
+ "recompile",
4153
+ "recordCameraSequence"
4154
+ ]);
4155
+ function guardRuntimeOwnership(api, model) {
4156
+ for (const key of runtimeMutators) {
4157
+ const record = api;
4158
+ const method = record[key];
4159
+ record[key] = (...args) => {
4160
+ assertMainThreadOwnership(model());
4161
+ return method(...args);
4162
+ };
4163
+ }
4164
+ return api;
4165
+ }
3708
4166
  var MujocoCanvas = forwardRef(
3709
4167
  function MujocoCanvas2({
3710
4168
  config,
@@ -3717,6 +4175,7 @@ var MujocoCanvas = forwardRef(
3717
4175
  timestep,
3718
4176
  integrator,
3719
4177
  substeps,
4178
+ physicsBudgetMs,
3720
4179
  paused,
3721
4180
  speed,
3722
4181
  interpolate,
@@ -3754,6 +4213,7 @@ var MujocoCanvas = forwardRef(
3754
4213
  timestep,
3755
4214
  integrator,
3756
4215
  substeps,
4216
+ physicsBudgetMs,
3757
4217
  paused,
3758
4218
  speed,
3759
4219
  interpolate,
@@ -5248,10 +5708,10 @@ function useCameraStream(canvasRef, options) {
5248
5708
  if (!api || !canvasRef.current) return;
5249
5709
  const opts = optionsRef.current;
5250
5710
  if (opts.paused) return;
5251
- if (opts.fps && opts.fps > 0) {
5252
- elapsedRef.current += delta;
5253
- if (elapsedRef.current < 1 / opts.fps) return;
5254
- }
5711
+ const fps = opts.fps ?? 10;
5712
+ if (!Number.isFinite(fps) || fps <= 0) throw new Error("Camera stream fps must be positive and finite");
5713
+ elapsedRef.current += delta;
5714
+ if (elapsedRef.current < 1 / fps) return;
5255
5715
  if (inFlightRef.current) return;
5256
5716
  elapsedRef.current = 0;
5257
5717
  const signature = streamSignature(opts);
@@ -5272,7 +5732,10 @@ function useCameraStream(canvasRef, options) {
5272
5732
  canvas.height = frame.height;
5273
5733
  }
5274
5734
  ctx.drawImage(frame.canvas, 0, 0);
5275
- }).catch(() => {
5735
+ }).catch((error) => {
5736
+ const failure = error instanceof Error ? error : new Error(String(error));
5737
+ if (optionsRef.current.onError) optionsRef.current.onError(failure);
5738
+ else console.error("Camera stream capture failed", failure);
5276
5739
  }).finally(() => {
5277
5740
  inFlightRef.current = false;
5278
5741
  });
@@ -7407,7 +7870,9 @@ function isPolicyActionChunk(value) {
7407
7870
  function toPolicyActions(output) {
7408
7871
  return isPolicyActionChunk(output) ? [...output] : [output];
7409
7872
  }
7410
- function enqueuePolicyActions(queue, actions, observation, strategy) {
7873
+ function enqueuePolicyActions(queue, actions, observation, strategy, limit = 256) {
7874
+ if (!Number.isSafeInteger(limit) || limit < 1) throw new Error("maxQueuedActions must be a positive safe integer");
7875
+ if ((strategy === "replace" ? 0 : queue.length) + actions.length > limit) throw new Error(`Policy action queue exceeds maxQueuedActions (${limit})`);
7411
7876
  if (strategy === "replace") {
7412
7877
  queue.splice(0, queue.length);
7413
7878
  }
@@ -7428,7 +7893,6 @@ function usePolicy(config) {
7428
7893
  const clearQueue = useCallback(() => {
7429
7894
  epochRef.current += 1;
7430
7895
  actionQueueRef.current.splice(0, actionQueueRef.current.length);
7431
- inFlightRef.current = false;
7432
7896
  lastErrorRef.current = null;
7433
7897
  }, []);
7434
7898
  const reset = useCallback(() => {
@@ -7437,6 +7901,13 @@ function usePolicy(config) {
7437
7901
  lastObservationRef.current = null;
7438
7902
  lastActionRef.current = null;
7439
7903
  }, [clearQueue]);
7904
+ useEffect(() => {
7905
+ isRunningRef.current = configRef.current.enabled ?? true;
7906
+ return () => {
7907
+ epochRef.current++;
7908
+ isRunningRef.current = false;
7909
+ };
7910
+ }, []);
7440
7911
  useBeforePhysicsStep(({ model, data }) => {
7441
7912
  if (!isRunningRef.current) return;
7442
7913
  const cfg = configRef.current;
@@ -7469,7 +7940,8 @@ function usePolicy(config) {
7469
7940
  actionQueueRef.current,
7470
7941
  toPolicyActions(output),
7471
7942
  observation,
7472
- cfg.queueStrategy ?? "append"
7943
+ cfg.queueStrategy ?? "append",
7944
+ cfg.maxQueuedActions
7473
7945
  );
7474
7946
  lastErrorRef.current = null;
7475
7947
  }).catch((error) => {
@@ -7477,7 +7949,6 @@ function usePolicy(config) {
7477
7949
  lastErrorRef.current = error;
7478
7950
  cfg.onError?.(error);
7479
7951
  }).finally(() => {
7480
- if (epoch !== epochRef.current) return;
7481
7952
  inFlightRef.current = false;
7482
7953
  });
7483
7954
  } else {
@@ -7487,7 +7958,8 @@ function usePolicy(config) {
7487
7958
  actionQueueRef.current,
7488
7959
  actions,
7489
7960
  observation,
7490
- cfg.queueStrategy ?? "append"
7961
+ cfg.queueStrategy ?? "append",
7962
+ cfg.maxQueuedActions
7491
7963
  );
7492
7964
  } else {
7493
7965
  const [action, ...queuedActions2] = actions;
@@ -7496,7 +7968,8 @@ function usePolicy(config) {
7496
7968
  actionQueueRef.current,
7497
7969
  queuedActions2,
7498
7970
  observation,
7499
- cfg.queueStrategy ?? "append"
7971
+ cfg.queueStrategy ?? "append",
7972
+ cfg.maxQueuedActions
7500
7973
  );
7501
7974
  cfg.onAction({ action, observation, model, data });
7502
7975
  lastActionRef.current = action;
@@ -8197,125 +8670,6 @@ function useCameraFrameCapture(defaultOptions = {}) {
8197
8670
  reset
8198
8671
  };
8199
8672
  }
8200
-
8201
- // src/policyCameraFrames.ts
8202
- function addPolicyImageAliases(images, stream, frame, includeObservationImageAliases) {
8203
- const keys = /* @__PURE__ */ new Set();
8204
- keys.add(stream.key);
8205
- for (const alias of stream.aliases ?? []) keys.add(alias);
8206
- if (includeObservationImageAliases) {
8207
- keys.add(`observation.images.${stream.key}`);
8208
- for (const alias of stream.aliases ?? []) {
8209
- keys.add(`observation.images.${alias}`);
8210
- }
8211
- }
8212
- for (const key of keys) {
8213
- images[key] = frame.dataUrl;
8214
- }
8215
- }
8216
- function describeFrameSource(key, frame) {
8217
- return `${key}:${frame.source.kind}`;
8218
- }
8219
- function hasExplicitPolicyCameraSource(options) {
8220
- return Boolean(
8221
- options?.camera || options?.position || options?.quaternion || options?.source
8222
- );
8223
- }
8224
- function createPolicyCameraFrameCapturePlan(options) {
8225
- const {
8226
- cameraKeys,
8227
- defaults,
8228
- streamOptions,
8229
- includeObservationImageAliases,
8230
- requireAll,
8231
- ...sourceOptions
8232
- } = options;
8233
- const mountedPlan = createMountedCameraFrameSequencePlan(cameraKeys, {
8234
- ...sourceOptions,
8235
- defaults,
8236
- cameraOptions: streamOptions
8237
- });
8238
- const streams = [];
8239
- const missingKeys = new Set(mountedPlan.missingKeys);
8240
- for (const key of cameraKeys) {
8241
- const perStreamOptions = streamOptions?.[key];
8242
- if (hasExplicitPolicyCameraSource(perStreamOptions)) {
8243
- missingKeys.delete(key);
8244
- streams.push({
8245
- ...defaults,
8246
- ...perStreamOptions,
8247
- key,
8248
- aliases: perStreamOptions?.aliases
8249
- });
8250
- continue;
8251
- }
8252
- const mountedCamera = mountedPlan.cameras.find((camera) => camera.key === key);
8253
- if (!mountedCamera) continue;
8254
- const { key: _mountedKey, ...captureOptions } = mountedCamera;
8255
- streams.push({
8256
- ...captureOptions,
8257
- key,
8258
- aliases: perStreamOptions?.aliases
8259
- });
8260
- }
8261
- const result = {
8262
- cameraKeys: [...cameraKeys],
8263
- streams,
8264
- includeObservationImageAliases,
8265
- mountedPlan,
8266
- missingKeys: [...missingKeys]
8267
- };
8268
- if (requireAll && result.missingKeys.length > 0) {
8269
- throw new Error(
8270
- `Unable to resolve policy camera stream${result.missingKeys.length === 1 ? "" : "s"} for ${result.missingKeys.join(", ")}.`
8271
- );
8272
- }
8273
- return result;
8274
- }
8275
- function createPolicyCameraFrameCapturePlanFromApi(api, options) {
8276
- return createPolicyCameraFrameCapturePlan({
8277
- ...options,
8278
- cameras: api.getCameras(),
8279
- sites: api.getSites(),
8280
- bodies: api.getBodies()
8281
- });
8282
- }
8283
- async function capturePolicyCameraFrames(target, options) {
8284
- const includeObservationImageAliases = options.includeObservationImageAliases ?? true;
8285
- const entries = await Promise.all(
8286
- options.streams.map(async ({ key, aliases, ...captureOptions }) => {
8287
- const frame = await target.captureCameraFrame(captureOptions);
8288
- return [key, { frame, aliases }];
8289
- })
8290
- );
8291
- const frames = {};
8292
- const images = {};
8293
- const sourceParts = [];
8294
- for (const [key, { frame, aliases }] of entries) {
8295
- const stream = { key, aliases };
8296
- frames[key] = frame;
8297
- addPolicyImageAliases(
8298
- images,
8299
- stream,
8300
- frame,
8301
- includeObservationImageAliases
8302
- );
8303
- sourceParts.push(describeFrameSource(key, frame));
8304
- }
8305
- return {
8306
- frames,
8307
- images,
8308
- sourceSummary: sourceParts.length > 0 ? sourceParts.join(" + ") : "not used by policy",
8309
- capturedAt: Date.now()
8310
- };
8311
- }
8312
- async function capturePolicyCameraFramesFromMountedStreams(target, options) {
8313
- const plan = createPolicyCameraFrameCapturePlanFromApi(target, options);
8314
- const result = await capturePolicyCameraFrames(target, plan);
8315
- return { ...result, plan };
8316
- }
8317
-
8318
- // src/hooks/usePolicyCameraFrames.ts
8319
8673
  function mergePolicyCameraFrameCaptureOptions(defaultOptions, options) {
8320
8674
  return {
8321
8675
  ...defaultOptions,
@@ -8838,6 +9192,18 @@ function useCameraAnimation() {
8838
9192
  * @license
8839
9193
  * SPDX-License-Identifier: Apache-2.0
8840
9194
  */
9195
+ /**
9196
+ * @license
9197
+ * SPDX-License-Identifier: Apache-2.0
9198
+ *
9199
+ * Helpers for resolving dataset camera streams to mounted MuJoCo resources.
9200
+ */
9201
+ /**
9202
+ * @license
9203
+ * SPDX-License-Identifier: Apache-2.0
9204
+ *
9205
+ * Helpers for turning Three/MuJoCo camera captures into policy image payloads.
9206
+ */
8841
9207
  /**
8842
9208
  * @license
8843
9209
  * SPDX-License-Identifier: Apache-2.0
@@ -8861,12 +9227,6 @@ function useCameraAnimation() {
8861
9227
  *
8862
9228
  * useFrameCapture — still-frame capture for canvas-backed MuJoCo/R3F scenes.
8863
9229
  */
8864
- /**
8865
- * @license
8866
- * SPDX-License-Identifier: Apache-2.0
8867
- *
8868
- * Helpers for resolving dataset camera streams to mounted MuJoCo resources.
8869
- */
8870
9230
  /**
8871
9231
  * @license
8872
9232
  * SPDX-License-Identifier: Apache-2.0
@@ -8907,16 +9267,9 @@ function useCameraAnimation() {
8907
9267
  * @license
8908
9268
  * SPDX-License-Identifier: Apache-2.0
8909
9269
  *
8910
- * Stream a live MuJoCo camera into a DOM `<canvas>`. Each frame the scene is
8911
- * rendered offscreen from the selected camera and blitted into the canvas, so
8912
- * it composites normally in the DOM (works inside opaque panels) and does NOT
8913
- * take over the render loop. Prefer this over `useCameraViewport` for camera
8914
- * tiles embedded in HTML UI; use `useCameraViewport` for transparent overlays
8915
- * on a full-bleed canvas.
8916
- *
8917
- * Uses the async capture path so Gaussian-splat environments render through
8918
- * their dedicated capture renderer — streaming a splat scene at full rate does
8919
- * not disturb the main view's splat sort.
9270
+ * Readback stream for DOM compositing/encoding. For interactive camera previews,
9271
+ * use CameraView/useCameraViewport: they render on the GPU with no readback.
9272
+ * This path defaults to 10 FPS and permits only one capture in flight.
8920
9273
  */
8921
9274
  /**
8922
9275
  * @license
@@ -9071,12 +9424,6 @@ function useCameraAnimation() {
9071
9424
  *
9072
9425
  * React state wrapper around MuJoCo/R3F offscreen camera-frame capture.
9073
9426
  */
9074
- /**
9075
- * @license
9076
- * SPDX-License-Identifier: Apache-2.0
9077
- *
9078
- * Helpers for turning Three/MuJoCo camera captures into policy image payloads.
9079
- */
9080
9427
  /**
9081
9428
  * @license
9082
9429
  * SPDX-License-Identifier: Apache-2.0