mujoco-react 12.0.0 → 12.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -1,5 +1,6 @@
1
- import { withContacts, getContact, captureCameraFrame, captureCameraFrameBlob, createCameraFrameCaptureSession, CAMERA_FRAME_CAPTURE_PRE_RENDER_USER_DATA_KEY, CAPTURE_EXCLUDE_KEY, createCaptureCamera, prepareCaptureCamera } from './chunk-7BCIHJMW.js';
2
- export { CAMERA_FRAME_CAPTURE_PRE_RENDER_USER_DATA_KEY, CAMERA_FRAME_CAPTURE_RENDER_USER_DATA_KEY, CAPTURE_EXCLUDE_KEY, ModelActuators, ModelBodies, ModelCameras, ModelGeoms, ModelJoints, ModelKeyframes, ModelResources, ModelSensors, ModelSites, ScenarioLighting, SplatEnvironment, SplatEnvironmentReadinessStatus, VisualScenarioEffects, captureCameraFrame, captureCameraFrameBlob, captureCameraFrameTensor, createCameraFrameCaptureSession, createPairedSplatEnvironment, createSparkSplatViewerUrl, createSplatEnvironmentUserData, createSplatSceneConfig, createVisualScenarioExecutionContext, dataUrlToPolicyImageTensor, getContact, getScenarioBackground, getScenarioCameraPosition, getSplatEnvironmentReadiness, imageDataToPolicyImageTensor, pixelsToPolicyImageTensor, registerModelResources, renderCameraFrameToCanvas, useSplatEnvironment, useSplatSceneConfig, useVisualScenarioEffects, useVisualScenarioExecutionContext, withSplatEnvironment } from './chunk-7BCIHJMW.js';
1
+ import { displayFields } from './chunk-U2Y2BODT.js';
2
+ import { withContacts, getContact, captureCameraFrame, captureCameraFrameBlob, createCameraFrameCaptureSession, CAMERA_FRAME_CAPTURE_PRE_RENDER_USER_DATA_KEY, CAPTURE_EXCLUDE_KEY, createCaptureCamera, prepareCaptureCamera } from './chunk-XD3AY7XZ.js';
3
+ export { CAMERA_FRAME_CAPTURE_PRE_RENDER_USER_DATA_KEY, CAMERA_FRAME_CAPTURE_RENDER_USER_DATA_KEY, CAPTURE_EXCLUDE_KEY, ModelActuators, ModelBodies, ModelCameras, ModelGeoms, ModelJoints, ModelKeyframes, ModelResources, ModelSensors, ModelSites, ScenarioLighting, SplatEnvironment, SplatEnvironmentReadinessStatus, VisualScenarioEffects, captureCameraFrame, captureCameraFrameBlob, captureCameraFrameTensor, createCameraFrameCaptureSession, createPairedSplatEnvironment, createSparkSplatViewerUrl, createSplatEnvironmentUserData, createSplatSceneConfig, createVisualScenarioExecutionContext, dataUrlToPolicyImageTensor, getContact, getScenarioBackground, getScenarioCameraPosition, getSplatEnvironmentReadiness, imageDataToPolicyImageTensor, pixelsToPolicyImageTensor, registerModelResources, renderCameraFrameToCanvas, useSplatEnvironment, useSplatSceneConfig, useVisualScenarioEffects, useVisualScenarioExecutionContext, withSplatEnvironment } from './chunk-XD3AY7XZ.js';
3
4
  import loadMujoco from '@mujoco/mujoco';
4
5
  import defaultMujocoWasmUrl from '@mujoco/mujoco/mujoco.wasm?url';
5
6
  import { createContext, forwardRef, useRef, useEffect, useContext, useState, useCallback, useMemo, useLayoutEffect, useId } from 'react';
@@ -975,939 +976,1332 @@ function collectDependencyPaths(xmlString, currentFile, parser) {
975
976
  });
976
977
  return paths;
977
978
  }
978
- var CapsuleGeometry = class extends THREE12.BufferGeometry {
979
- parameters;
980
- constructor(radius = 1, length = 1, capSegments = 4, radialSegments = 8) {
981
- super();
982
- this.type = "CapsuleGeometry";
983
- this.parameters = { radius, length, capSegments, radialSegments };
984
- const path = new THREE12.Path();
985
- path.absarc(0, -length / 2, radius, Math.PI * 1.5, 0, false);
986
- path.absarc(0, length / 2, radius, 0, Math.PI * 0.5, false);
987
- const latheGeometry = new THREE12.LatheGeometry(path.getPoints(capSegments), radialSegments);
988
- const self = this;
989
- self.setIndex(latheGeometry.getIndex());
990
- self.setAttribute("position", latheGeometry.getAttribute("position"));
991
- self.setAttribute("normal", latheGeometry.getAttribute("normal"));
992
- self.setAttribute("uv", latheGeometry.getAttribute("uv"));
993
- }
994
- };
995
979
 
996
- // src/rendering/GeomBuilder.ts
997
- var DEFAULT_MESH_NORMAL_SMOOTHING_TOLERANCE = 1e-4;
998
- var GeomBuilder = class {
999
- mujoco;
1000
- textureCache = /* @__PURE__ */ new Map();
1001
- renderOptions;
1002
- constructor(mujoco, renderOptions) {
1003
- this.mujoco = mujoco;
1004
- this.renderOptions = renderOptions;
980
+ // src/core/sceneIdentity.ts
981
+ var fileIds = /* @__PURE__ */ new WeakMap();
982
+ var nextFileId = 0;
983
+ function sceneIdentity(config) {
984
+ return JSON.stringify([
985
+ config.src,
986
+ config.sceneFile,
987
+ config.environmentFiles ?? [],
988
+ config.sceneObjects ?? [],
989
+ config.homeJoints ?? [],
990
+ config.xmlPatches ?? [],
991
+ (config.files ?? []).map((file) => {
992
+ if (!fileIds.has(file)) fileIds.set(file, ++nextFileId);
993
+ return fileIds.get(file);
994
+ })
995
+ ], (_key, value) => value && typeof value === "object" && !Array.isArray(value) ? Object.fromEntries(Object.entries(value).sort(([a], [b]) => a.localeCompare(b))) : value);
996
+ }
997
+
998
+ // src/core/stepBudget.ts
999
+ function runStepBudget(requested, step, budgetMs = 8, now2 = () => performance.now()) {
1000
+ if (!Number.isFinite(budgetMs) || budgetMs <= 0) throw new Error("physicsBudgetMs must be positive and finite");
1001
+ const start = now2();
1002
+ let steps = 0;
1003
+ while (steps < requested && steps < 100) {
1004
+ step();
1005
+ steps++;
1006
+ if (now2() - start >= budgetMs) break;
1007
+ }
1008
+ return { steps, physicsMs: now2() - start, budgetExceeded: steps < requested };
1009
+ }
1010
+
1011
+ // src/rendering/cameraFrameSource.ts
1012
+ var MountedCameraFrameSourceSuggestionMatch = {
1013
+ Direct: "direct",
1014
+ Alias: "alias",
1015
+ Normalized: "normalized",
1016
+ Prefix: "prefix",
1017
+ Suffix: "suffix",
1018
+ Contains: "contains"
1019
+ };
1020
+ var MountedCameraFrameSequenceReadinessStatus = {
1021
+ Ready: "ready",
1022
+ Partial: "partial",
1023
+ Missing: "missing"
1024
+ };
1025
+ var MountedCameraFrameSequenceManifestStatus = {
1026
+ Complete: "complete",
1027
+ Partial: "partial",
1028
+ Missing: "missing"
1029
+ };
1030
+ function getResourceName(resource) {
1031
+ if (!resource) return null;
1032
+ return typeof resource === "string" ? resource : resource.name ?? null;
1033
+ }
1034
+ function createNameSet(resources) {
1035
+ return new Set(
1036
+ (resources ?? []).map((resource) => getResourceName(resource)).filter((name) => Boolean(name))
1037
+ );
1038
+ }
1039
+ function createResourceNames(resources) {
1040
+ return (resources ?? []).map((resource) => getResourceName(resource)).filter((name) => Boolean(name));
1041
+ }
1042
+ function normalizeAliasCandidates(value) {
1043
+ if (!value) return [];
1044
+ return Array.isArray(value) ? value : [value];
1045
+ }
1046
+ function countMountedSelectors(selector) {
1047
+ return Number(Boolean(selector.cameraName)) + Number(Boolean(selector.siteName)) + Number(Boolean(selector.bodyName));
1048
+ }
1049
+ function normalizeCameraSourceName(value) {
1050
+ return value.trim().toLowerCase().replace(/[^a-z0-9]+/g, "_").replace(/^_+|_+$/g, "");
1051
+ }
1052
+ function createCameraSourceKeyVariants(key) {
1053
+ const candidates = [
1054
+ key,
1055
+ key.startsWith("observation.images.") ? key.slice("observation.images.".length) : "",
1056
+ key.includes(".") ? key.split(".").at(-1) ?? "" : "",
1057
+ key.includes("/") ? key.split("/").at(-1) ?? "" : ""
1058
+ ];
1059
+ return candidates.map((candidate) => candidate.trim()).filter((candidate, index, items) => candidate && items.indexOf(candidate) === index);
1060
+ }
1061
+ function getSelectorKey(selector) {
1062
+ if (selector.cameraName) return `camera:${selector.cameraName}`;
1063
+ if (selector.siteName) return `site:${selector.siteName}`;
1064
+ if (selector.bodyName) return `body:${selector.bodyName}`;
1065
+ return null;
1066
+ }
1067
+ function getMountedCameraFrameCaptureSource(selector) {
1068
+ if (countMountedSelectors(selector) !== 1) return null;
1069
+ if (selector.cameraName) {
1070
+ return { kind: "mujoco-camera", cameraName: selector.cameraName };
1005
1071
  }
1006
- getMeshNormalSmoothingTolerance() {
1007
- const smoothing = this.renderOptions?.meshNormalSmoothing;
1008
- if (!smoothing) return null;
1009
- if (smoothing === true) return DEFAULT_MESH_NORMAL_SMOOTHING_TOLERANCE;
1010
- return smoothing.tolerance ?? DEFAULT_MESH_NORMAL_SMOOTHING_TOLERANCE;
1072
+ if (selector.siteName) {
1073
+ return { kind: "mujoco-site", siteName: selector.siteName };
1011
1074
  }
1012
- getMaterialTexture(mjModel, matId) {
1013
- if (matId < 0 || !mjModel.mat_texid || !mjModel.tex_data) return null;
1014
- const materialCount = Math.max(1, Math.floor(mjModel.mat_rgba.length / 4));
1015
- const textureRoles = Math.max(1, Math.floor(mjModel.mat_texid.length / materialCount));
1016
- let texId = -1;
1017
- for (let role = 0; role < textureRoles; role += 1) {
1018
- const candidate = mjModel.mat_texid[matId * textureRoles + role];
1019
- if (candidate >= 0) {
1020
- texId = candidate;
1021
- break;
1022
- }
1075
+ if (selector.bodyName) {
1076
+ return { kind: "mujoco-body", bodyName: selector.bodyName };
1077
+ }
1078
+ return null;
1079
+ }
1080
+ function isMountedCameraFrameCaptureSource(source) {
1081
+ return source.kind === "mujoco-camera" || source.kind === "mujoco-site" || source.kind === "mujoco-body";
1082
+ }
1083
+ function getCameraFrameCaptureSourceTarget(source) {
1084
+ if (source.kind === "mujoco-camera") return source.cameraName;
1085
+ if (source.kind === "mujoco-site") return source.siteName;
1086
+ if (source.kind === "mujoco-body") return source.bodyName;
1087
+ if (source.kind === "custom-camera") return "custom camera";
1088
+ if (source.kind === "explicit-pose") return "explicit pose";
1089
+ return "fallback camera";
1090
+ }
1091
+ function createMountedCameraFrameSourceSuggestion(key, selector, resourceName, match) {
1092
+ const source = getMountedCameraFrameCaptureSource(selector);
1093
+ if (!source) return null;
1094
+ return {
1095
+ key,
1096
+ selector,
1097
+ source,
1098
+ resourceName,
1099
+ resourceKind: source.kind,
1100
+ match
1101
+ };
1102
+ }
1103
+ function addMountedCameraFrameSourceSuggestion(suggestions, seen, suggestion) {
1104
+ if (!suggestion) return;
1105
+ const selectorKey = getSelectorKey(suggestion.selector);
1106
+ if (!selectorKey || seen.has(selectorKey)) return;
1107
+ seen.add(selectorKey);
1108
+ suggestions.push(suggestion);
1109
+ }
1110
+ function getCameraFrameResourceMatch(key, resourceName) {
1111
+ if (resourceName === key) return MountedCameraFrameSourceSuggestionMatch.Direct;
1112
+ const normalizedResource = normalizeCameraSourceName(resourceName);
1113
+ if (!normalizedResource) return null;
1114
+ for (const variant of createCameraSourceKeyVariants(key)) {
1115
+ if (resourceName === variant) return MountedCameraFrameSourceSuggestionMatch.Direct;
1116
+ const normalizedKey = normalizeCameraSourceName(variant);
1117
+ if (!normalizedKey) continue;
1118
+ if (normalizedResource === normalizedKey) {
1119
+ return MountedCameraFrameSourceSuggestionMatch.Normalized;
1023
1120
  }
1024
- if (texId < 0) return null;
1025
- const cached = this.textureCache.get(texId);
1026
- if (cached) return cached;
1027
- const width = Number(mjModel.tex_width[texId]);
1028
- const height = Number(mjModel.tex_height[texId]);
1029
- const channels = Number(mjModel.tex_nchannel[texId]);
1030
- const offset = Number(mjModel.tex_adr[texId]);
1031
- if (width <= 0 || height <= 0 || channels <= 0 || offset < 0) return null;
1032
- const source = mjModel.tex_data.subarray(offset, offset + width * height * channels);
1033
- const rgba = new Uint8Array(width * height * 4);
1034
- for (let i = 0, j = 0; i < width * height; i += 1, j += channels) {
1035
- const r = source[j] ?? 255;
1036
- const g = channels > 1 ? source[j + 1] : r;
1037
- const b = channels > 2 ? source[j + 2] : r;
1038
- const a = channels > 3 ? source[j + 3] : 255;
1039
- const out = i * 4;
1040
- rgba[out] = r;
1041
- rgba[out + 1] = g;
1042
- rgba[out + 2] = b;
1043
- rgba[out + 3] = a;
1121
+ if (normalizedResource.startsWith(`${normalizedKey}_`)) {
1122
+ return MountedCameraFrameSourceSuggestionMatch.Prefix;
1123
+ }
1124
+ if (normalizedResource.endsWith(`_${normalizedKey}`)) {
1125
+ return MountedCameraFrameSourceSuggestionMatch.Suffix;
1126
+ }
1127
+ if (normalizedResource.includes(`_${normalizedKey}_`)) {
1128
+ return MountedCameraFrameSourceSuggestionMatch.Contains;
1044
1129
  }
1045
- const texture = new THREE12.DataTexture(rgba, width, height, THREE12.RGBAFormat);
1046
- texture.colorSpace = THREE12.LinearSRGBColorSpace;
1047
- texture.wrapS = THREE12.RepeatWrapping;
1048
- texture.wrapT = THREE12.RepeatWrapping;
1049
- texture.flipY = true;
1050
- const repeatOffset = matId * 2;
1051
- const repeatS = mjModel.mat_texrepeat?.[repeatOffset] ?? 1;
1052
- const repeatT = mjModel.mat_texrepeat?.[repeatOffset + 1] ?? 1;
1053
- texture.repeat.set(repeatS || 1, repeatT || 1);
1054
- texture.needsUpdate = true;
1055
- this.textureCache.set(texId, texture);
1056
- return texture;
1057
1130
  }
1058
- /**
1059
- * Creates a Three.js Object3D (usually a Mesh) for a specific geometry in the MuJoCo model.
1060
- * Returns null if the geometry shouldn't be rendered (e.g., invisible collision triggers).
1061
- */
1062
- create(mjModel, g) {
1063
- if (mjModel.geom_group[g] === 3) return null;
1064
- const type = mjModel.geom_type[g];
1065
- const size = mjModel.geom_size.subarray(g * 3, g * 3 + 3);
1066
- const pos = mjModel.geom_pos.subarray(g * 3, g * 3 + 3);
1067
- const quat = mjModel.geom_quat.subarray(g * 4, g * 4 + 4);
1068
- const matId = mjModel.geom_matid[g];
1069
- const color = new THREE12.Color(16777215);
1070
- const map = this.getMaterialTexture(mjModel, matId);
1071
- let opacity = 1;
1072
- if (matId >= 0) {
1073
- const rgba = mjModel.mat_rgba.subarray(matId * 4, matId * 4 + 4);
1074
- color.setRGB(rgba[0], rgba[1], rgba[2]);
1075
- opacity = rgba[3];
1076
- } else {
1077
- const rgba = mjModel.geom_rgba.subarray(g * 4, g * 4 + 4);
1078
- color.setRGB(rgba[0], rgba[1], rgba[2]);
1079
- opacity = rgba[3];
1131
+ return null;
1132
+ }
1133
+ function isSelectorMounted(selector, cameraNames, siteNames, bodyNames) {
1134
+ if (countMountedSelectors(selector) !== 1) return false;
1135
+ return (selector.cameraName ? cameraNames.has(selector.cameraName) : false) || (selector.siteName ? siteNames.has(selector.siteName) : false) || (selector.bodyName ? bodyNames.has(selector.bodyName) : false);
1136
+ }
1137
+ function createMountedCameraFrameSourceSuggestions(key, options) {
1138
+ const cameraNames = createNameSet(options.cameras);
1139
+ const siteNames = createNameSet(options.sites);
1140
+ const bodyNames = createNameSet(options.bodies);
1141
+ const suggestions = [];
1142
+ const seen = /* @__PURE__ */ new Set();
1143
+ for (const selector of normalizeAliasCandidates(options.aliases?.[key])) {
1144
+ if (!isSelectorMounted(selector, cameraNames, siteNames, bodyNames)) {
1145
+ continue;
1080
1146
  }
1081
- const MG = this.mujoco.mjtGeom;
1082
- let geo = null;
1083
- const getVal = (v) => v?.value ?? v;
1084
- if (type === getVal(MG.mjGEOM_PLANE)) {
1085
- geo = new THREE12.PlaneGeometry(size[0] * 2 || 5, size[1] * 2 || 5);
1086
- } else if (type === getVal(MG.mjGEOM_SPHERE)) {
1087
- geo = new THREE12.SphereGeometry(size[0], 24, 24);
1088
- } else if (type === getVal(MG.mjGEOM_CAPSULE)) {
1089
- geo = new CapsuleGeometry(size[0], size[1] * 2, 24, 12);
1090
- geo.rotateX(Math.PI / 2);
1091
- } else if (type === getVal(MG.mjGEOM_BOX)) {
1092
- geo = new THREE12.BoxGeometry(size[0] * 2, size[1] * 2, size[2] * 2);
1093
- } else if (type === getVal(MG.mjGEOM_CYLINDER)) {
1094
- geo = new THREE12.CylinderGeometry(size[0], size[0], size[1] * 2, 24);
1095
- geo.rotateX(Math.PI / 2);
1096
- } else if (type === getVal(MG.mjGEOM_MESH)) {
1097
- const mId = mjModel.geom_dataid[g];
1098
- const vAdr = mjModel.mesh_vertadr[mId];
1099
- const vNum = mjModel.mesh_vertnum[mId];
1100
- const fAdr = mjModel.mesh_faceadr[mId];
1101
- const fNum = mjModel.mesh_facenum[mId];
1102
- const tAdr = mjModel.mesh_texcoordadr?.[mId] ?? -1;
1103
- if (map && tAdr >= 0 && mjModel.mesh_texcoord && mjModel.mesh_facetexcoord) {
1104
- const corners = fNum * 3;
1105
- const positions = new Float32Array(corners * 3);
1106
- const uvs = new Float32Array(corners * 2);
1107
- for (let c = 0; c < corners; c++) {
1108
- const v = vAdr + mjModel.mesh_face[fAdr * 3 + c];
1109
- positions[c * 3] = mjModel.mesh_vert[v * 3];
1110
- positions[c * 3 + 1] = mjModel.mesh_vert[v * 3 + 1];
1111
- positions[c * 3 + 2] = mjModel.mesh_vert[v * 3 + 2];
1112
- const t = tAdr + mjModel.mesh_facetexcoord[fAdr * 3 + c];
1113
- uvs[c * 2] = mjModel.mesh_texcoord[t * 2];
1114
- uvs[c * 2 + 1] = 1 - mjModel.mesh_texcoord[t * 2 + 1];
1115
- }
1116
- geo = new THREE12.BufferGeometry();
1117
- geo.setAttribute("position", new THREE12.Float32BufferAttribute(positions, 3));
1118
- geo.setAttribute("uv", new THREE12.Float32BufferAttribute(uvs, 2));
1119
- geo.computeVertexNormals();
1120
- } else {
1121
- geo = new THREE12.BufferGeometry();
1122
- geo.setAttribute("position", new THREE12.Float32BufferAttribute(mjModel.mesh_vert.subarray(vAdr * 3, (vAdr + vNum) * 3), 3));
1123
- geo.setIndex(Array.from(mjModel.mesh_face.subarray(fAdr * 3, (fAdr + fNum) * 3)));
1124
- const smoothingTolerance = this.getMeshNormalSmoothingTolerance();
1125
- if (smoothingTolerance !== null) {
1126
- geo = mergeVertices(geo, smoothingTolerance);
1127
- }
1128
- geo.computeVertexNormals();
1129
- }
1130
- }
1131
- if (geo) {
1132
- const isPlane = type === getVal(MG.mjGEOM_PLANE);
1133
- const materialMap = isPlane && map ? map.clone() : map;
1134
- if (isPlane && materialMap) {
1135
- materialMap.repeat.multiplyScalar(2.5);
1136
- materialMap.needsUpdate = true;
1137
- }
1138
- const mesh = new THREE12.Mesh(geo, new THREE12.MeshStandardMaterial({
1139
- color,
1140
- map: materialMap,
1141
- transparent: opacity < 1,
1142
- opacity,
1143
- roughness: 0.6,
1144
- metalness: 0
1145
- }));
1146
- mesh.castShadow = type !== getVal(MG.mjGEOM_PLANE);
1147
- mesh.receiveShadow = true;
1148
- mesh.position.set(pos[0], pos[1], pos[2]);
1149
- mesh.quaternion.set(quat[1], quat[2], quat[3], quat[0]);
1150
- mesh.userData.bodyID = mjModel.geom_bodyid[g];
1151
- mesh.userData.geomID = g;
1152
- mesh.userData.geomGroup = mjModel.geom_group[g];
1153
- mesh.userData.geomName = getName(mjModel, mjModel.name_geomadr[g]);
1154
- return mesh;
1155
- }
1156
- return null;
1147
+ const source = getMountedCameraFrameCaptureSource(selector);
1148
+ if (!source) continue;
1149
+ addMountedCameraFrameSourceSuggestion(
1150
+ suggestions,
1151
+ seen,
1152
+ createMountedCameraFrameSourceSuggestion(
1153
+ key,
1154
+ selector,
1155
+ getCameraFrameCaptureSourceTarget(source),
1156
+ MountedCameraFrameSourceSuggestionMatch.Alias
1157
+ )
1158
+ );
1157
1159
  }
1158
- };
1159
- function getRenderOptionsKey(renderOptions) {
1160
- const smoothing = renderOptions?.meshNormalSmoothing;
1161
- if (!smoothing) return "default";
1162
- if (smoothing === true) return "meshNormalSmoothing:true";
1163
- return `meshNormalSmoothing:${smoothing.tolerance ?? "default"}`;
1160
+ for (const cameraName of createResourceNames(options.cameras)) {
1161
+ const match = getCameraFrameResourceMatch(key, cameraName);
1162
+ if (!match) continue;
1163
+ addMountedCameraFrameSourceSuggestion(
1164
+ suggestions,
1165
+ seen,
1166
+ createMountedCameraFrameSourceSuggestion(
1167
+ key,
1168
+ { cameraName },
1169
+ cameraName,
1170
+ match
1171
+ )
1172
+ );
1173
+ }
1174
+ for (const siteName of createResourceNames(options.sites)) {
1175
+ const match = getCameraFrameResourceMatch(key, siteName);
1176
+ if (!match) continue;
1177
+ addMountedCameraFrameSourceSuggestion(
1178
+ suggestions,
1179
+ seen,
1180
+ createMountedCameraFrameSourceSuggestion(
1181
+ key,
1182
+ { siteName },
1183
+ siteName,
1184
+ match
1185
+ )
1186
+ );
1187
+ }
1188
+ for (const bodyName of createResourceNames(options.bodies)) {
1189
+ const match = getCameraFrameResourceMatch(key, bodyName);
1190
+ if (!match) continue;
1191
+ addMountedCameraFrameSourceSuggestion(
1192
+ suggestions,
1193
+ seen,
1194
+ createMountedCameraFrameSourceSuggestion(
1195
+ key,
1196
+ { bodyName },
1197
+ bodyName,
1198
+ match
1199
+ )
1200
+ );
1201
+ }
1202
+ return suggestions;
1164
1203
  }
1165
- function SceneRenderer({ renderOptions, ...props }) {
1166
- const {
1167
- mjModelRef,
1168
- mjDataRef,
1169
- mujocoRef,
1170
- onSelectionRef,
1171
- hiddenBodiesRef,
1172
- hiddenBodiesVersion,
1173
- interpolateRef,
1174
- interpolationStateRef,
1175
- status
1176
- } = useMujocoContext();
1177
- const groupRef = useRef(null);
1178
- const bodyRefs = useRef([]);
1179
- const prevModelRef = useRef(null);
1180
- const prevRenderOptionsKeyRef = useRef(null);
1181
- const prevHiddenVersionRef = useRef(null);
1182
- const renderOptionsKey = getRenderOptionsKey(renderOptions);
1183
- const geomBuilder = useMemo(() => {
1184
- if (status !== "ready") return null;
1185
- return new GeomBuilder(mujocoRef.current, renderOptions);
1186
- }, [status, mujocoRef, renderOptionsKey]);
1187
- useEffect(() => {
1188
- if (status !== "ready" || !geomBuilder) return;
1189
- const model = mjModelRef.current;
1190
- const group = groupRef.current;
1191
- if (!model || !group) return;
1192
- if (prevModelRef.current === model && prevRenderOptionsKeyRef.current === renderOptionsKey && prevHiddenVersionRef.current === hiddenBodiesVersion) {
1193
- return;
1194
- }
1195
- prevModelRef.current = model;
1196
- prevRenderOptionsKeyRef.current = renderOptionsKey;
1197
- prevHiddenVersionRef.current = hiddenBodiesVersion;
1198
- while (group.children.length > 0) {
1199
- group.remove(group.children[0]);
1200
- }
1201
- const refs = [];
1202
- for (let i = 0; i < model.nbody; i++) {
1203
- const bodyGroup = new THREE12.Group();
1204
- bodyGroup.userData.bodyID = i;
1205
- const bodyName = getName(model, model.name_bodyadr[i]);
1206
- if (!hiddenBodiesRef.current.has(bodyName)) {
1207
- for (let g = 0; g < model.ngeom; g++) {
1208
- if (model.geom_bodyid[g] === i) {
1209
- const mesh = geomBuilder.create(model, g);
1210
- if (mesh) bodyGroup.add(mesh);
1211
- }
1212
- }
1213
- }
1214
- group.add(bodyGroup);
1215
- refs.push(bodyGroup);
1216
- }
1217
- bodyRefs.current = refs;
1218
- }, [status, geomBuilder, mjModelRef, renderOptionsKey, hiddenBodiesRef, hiddenBodiesVersion]);
1219
- const syncBodiesToData = useCallback(() => {
1220
- const data = mjDataRef.current;
1221
- if (!data) return;
1222
- const bodies = bodyRefs.current;
1223
- const interpolation = interpolationStateRef.current;
1224
- const useInterpolation = interpolateRef.current && interpolation.valid;
1225
- for (let i = 0; i < bodies.length; i++) {
1226
- const ref = bodies[i];
1227
- if (!ref) continue;
1228
- if (useInterpolation) {
1229
- const alpha = interpolation.alpha;
1230
- const i3 = i * 3;
1231
- ref.position.set(
1232
- THREE12.MathUtils.lerp(interpolation.previousXpos[i3], interpolation.currentXpos[i3], alpha),
1233
- THREE12.MathUtils.lerp(interpolation.previousXpos[i3 + 1], interpolation.currentXpos[i3 + 1], alpha),
1234
- THREE12.MathUtils.lerp(interpolation.previousXpos[i3 + 2], interpolation.currentXpos[i3 + 2], alpha)
1235
- );
1236
- const i4 = i * 4;
1237
- _previousQuat.set(
1238
- interpolation.previousXquat[i4 + 1],
1239
- interpolation.previousXquat[i4 + 2],
1240
- interpolation.previousXquat[i4 + 3],
1241
- interpolation.previousXquat[i4]
1242
- );
1243
- _currentQuat.set(
1244
- interpolation.currentXquat[i4 + 1],
1245
- interpolation.currentXquat[i4 + 2],
1246
- interpolation.currentXquat[i4 + 3],
1247
- interpolation.currentXquat[i4]
1248
- );
1249
- ref.quaternion.copy(_previousQuat).slerp(_currentQuat, alpha);
1250
- } else {
1251
- ref.position.set(
1252
- data.xpos[i * 3],
1253
- data.xpos[i * 3 + 1],
1254
- data.xpos[i * 3 + 2]
1255
- );
1256
- ref.quaternion.set(
1257
- data.xquat[i * 4 + 1],
1258
- data.xquat[i * 4 + 2],
1259
- data.xquat[i * 4 + 3],
1260
- data.xquat[i * 4]
1261
- );
1262
- }
1263
- }
1264
- }, [interpolateRef, interpolationStateRef, mjDataRef]);
1265
- useFrame(syncBodiesToData);
1266
- return /* @__PURE__ */ jsx(
1267
- "group",
1268
- {
1269
- ...props,
1270
- ref: groupRef,
1271
- userData: {
1272
- ...props.userData,
1273
- [CAMERA_FRAME_CAPTURE_PRE_RENDER_USER_DATA_KEY]: syncBodiesToData
1274
- },
1275
- onDoubleClick: (e) => {
1276
- if (typeof props.onDoubleClick === "function") props.onDoubleClick(e);
1277
- e.stopPropagation();
1278
- let obj = e.object;
1279
- while (obj && obj.userData.bodyID === void 0 && obj.parent) {
1280
- obj = obj.parent;
1281
- }
1282
- const bodyID = obj?.userData.bodyID;
1283
- if (typeof bodyID === "number" && bodyID > 0) {
1284
- const model = mjModelRef.current;
1285
- if (model && bodyID < model.nbody && onSelectionRef.current) {
1286
- const name = getName(model, model.name_bodyadr[bodyID]);
1287
- onSelectionRef.current({ bodyId: bodyID, name });
1288
- }
1289
- }
1290
- }
1204
+ function resolveMountedCameraFrameSource(key, options) {
1205
+ const cameraNames = createNameSet(options.cameras);
1206
+ const siteNames = createNameSet(options.sites);
1207
+ const bodyNames = createNameSet(options.bodies);
1208
+ const directCandidates = [
1209
+ { cameraName: key },
1210
+ { siteName: key },
1211
+ { bodyName: key }
1212
+ ];
1213
+ const aliasCandidates = normalizeAliasCandidates(options.aliases?.[key]);
1214
+ for (const selector of aliasCandidates) {
1215
+ if (!isSelectorMounted(selector, cameraNames, siteNames, bodyNames)) {
1216
+ continue;
1291
1217
  }
1292
- );
1293
- }
1294
- var _previousQuat = new THREE12.Quaternion();
1295
- var _currentQuat = new THREE12.Quaternion();
1296
- var CameraViewportRegistryContext = createContext(null);
1297
- var nextViewportId = 0;
1298
- var VIEWPORT_RENDER_PRIORITY = 1;
1299
- function CameraViewportRenderer({
1300
- viewportsRef
1301
- }) {
1302
- const gl = useThree((state) => state.gl);
1303
- const scene = useThree((state) => state.scene);
1304
- const mainCamera = useThree((state) => state.camera);
1305
- const mujoco = useMujoco();
1306
- useFrame(() => {
1307
- const drawWidth = gl.domElement.width;
1308
- const drawHeight = gl.domElement.height;
1309
- gl.setScissorTest(false);
1310
- gl.setViewport(0, 0, drawWidth, drawHeight);
1311
- gl.render(scene, mainCamera);
1312
- const api = mujoco.api;
1313
- if (!api || viewportsRef.current.size === 0) return;
1314
- const dpr = gl.getPixelRatio();
1315
- const canvasRect = gl.domElement.getBoundingClientRect();
1316
- for (const descriptor of viewportsRef.current.values()) {
1317
- const element = descriptor.getElement();
1318
- if (!element) continue;
1319
- const rect = element.getBoundingClientRect();
1320
- const isOffscreen = rect.bottom < canvasRect.top || rect.top > canvasRect.bottom || rect.right < canvasRect.left || rect.left > canvasRect.right;
1321
- if (isOffscreen) continue;
1322
- const width = Math.floor(rect.width * dpr);
1323
- const height = Math.floor(rect.height * dpr);
1324
- if (width <= 0 || height <= 0) continue;
1325
- const left = Math.floor((rect.left - canvasRect.left) * dpr);
1326
- const bottom = Math.floor((canvasRect.bottom - rect.bottom) * dpr);
1327
- let resolved;
1328
- try {
1329
- resolved = api.resolveCameraCaptureOptions(descriptor.getOptions());
1330
- } catch {
1331
- continue;
1332
- }
1333
- if (!descriptor.camera) {
1334
- descriptor.camera = createCaptureCamera(resolved, mainCamera, width, height);
1335
- } else {
1336
- prepareCaptureCamera(descriptor.camera, resolved, mainCamera, width, height);
1337
- }
1338
- gl.setViewport(left, bottom, width, height);
1339
- gl.setScissor(left, bottom, width, height);
1340
- gl.setScissorTest(true);
1341
- gl.render(scene, descriptor.camera);
1218
+ const source = getMountedCameraFrameCaptureSource(selector);
1219
+ if (!source) continue;
1220
+ return { key, selector, source };
1221
+ }
1222
+ const [suggestion] = createMountedCameraFrameSourceSuggestions(key, options);
1223
+ if (suggestion) {
1224
+ return {
1225
+ key,
1226
+ selector: suggestion.selector,
1227
+ source: suggestion.source
1228
+ };
1229
+ }
1230
+ if (options.allowAliasFallback) {
1231
+ for (const selector of aliasCandidates) {
1232
+ const source = getMountedCameraFrameCaptureSource(selector);
1233
+ if (!source) continue;
1234
+ return { key, selector, source };
1342
1235
  }
1343
- gl.setScissorTest(false);
1344
- gl.setViewport(0, 0, drawWidth, drawHeight);
1345
- }, VIEWPORT_RENDER_PRIORITY);
1236
+ }
1237
+ for (const selector of directCandidates) {
1238
+ if (!isSelectorMounted(selector, cameraNames, siteNames, bodyNames)) {
1239
+ continue;
1240
+ }
1241
+ const source = getMountedCameraFrameCaptureSource(selector);
1242
+ if (!source) continue;
1243
+ return { key, selector, source };
1244
+ }
1346
1245
  return null;
1347
1246
  }
1348
- function CameraViewportProvider({ children }) {
1349
- const viewportsRef = useRef(/* @__PURE__ */ new Map());
1350
- const [count, setCount] = useState(0);
1351
- const register = useCallback((descriptor) => {
1352
- const id = nextViewportId++;
1353
- viewportsRef.current.set(id, descriptor);
1354
- setCount((value2) => value2 + 1);
1355
- return () => {
1356
- viewportsRef.current.delete(id);
1357
- setCount((value2) => value2 - 1);
1358
- };
1359
- }, []);
1360
- const value = useMemo(() => ({ register }), [register]);
1361
- return /* @__PURE__ */ jsxs(CameraViewportRegistryContext.Provider, { value, children: [
1362
- children,
1363
- count > 0 && /* @__PURE__ */ jsx(CameraViewportRenderer, { viewportsRef })
1364
- ] });
1365
- }
1366
- function useCameraViewport(elementRef, options) {
1367
- const registry = useContext(CameraViewportRegistryContext);
1368
- if (!registry) {
1369
- throw new Error("useCameraViewport must be used inside <MujocoCanvas>.");
1247
+ function createMountedCameraFrameSequencePlan(cameraKeys, options) {
1248
+ const cameras = [];
1249
+ const resolved = {};
1250
+ const missingKeys = [];
1251
+ for (const key of cameraKeys) {
1252
+ const mountedSource = resolveMountedCameraFrameSource(key, options);
1253
+ if (!mountedSource) {
1254
+ missingKeys.push(key);
1255
+ continue;
1256
+ }
1257
+ resolved[key] = mountedSource;
1258
+ cameras.push({
1259
+ key,
1260
+ ...options.defaults,
1261
+ ...options.cameraOptions?.[key],
1262
+ ...mountedSource.selector,
1263
+ source: mountedSource.source
1264
+ });
1370
1265
  }
1371
- const optionsRef = useRef(options);
1372
- optionsRef.current = options;
1373
- useEffect(() => {
1374
- const descriptor = {
1375
- getElement: () => elementRef.current,
1376
- getOptions: () => optionsRef.current,
1377
- camera: null
1378
- };
1379
- return registry.register(descriptor);
1380
- }, [registry, elementRef]);
1266
+ if (options.requireAll && missingKeys.length > 0) {
1267
+ throw new Error(
1268
+ `Unable to resolve mounted MuJoCo camera source${missingKeys.length === 1 ? "" : "s"} for ${missingKeys.join(", ")}.`
1269
+ );
1270
+ }
1271
+ return {
1272
+ cameraKeys: [...cameraKeys],
1273
+ cameras,
1274
+ resolved,
1275
+ missingKeys
1276
+ };
1381
1277
  }
1382
- function CameraView({ className, style, ...options }) {
1383
- const gl = useThree((state) => state.gl);
1384
- const elementRef = useRef(null);
1385
- if (!elementRef.current && typeof document !== "undefined") {
1386
- elementRef.current = document.createElement("div");
1278
+ function createMountedCameraFrameSequenceReadiness(plan) {
1279
+ const cameras = {};
1280
+ const resolvedKeys = plan.cameraKeys.filter((key) => Boolean(plan.resolved[key]));
1281
+ for (const key of plan.cameraKeys) {
1282
+ const resolved = plan.resolved[key];
1283
+ cameras[key] = resolved ? {
1284
+ key,
1285
+ ready: true,
1286
+ selector: resolved.selector,
1287
+ source: resolved.source,
1288
+ message: `Camera stream "${key}" resolves to ${resolved.source.kind}:${getCameraFrameCaptureSourceTarget(resolved.source)}.`
1289
+ } : {
1290
+ key,
1291
+ ready: false,
1292
+ message: `Camera stream "${key}" does not resolve to a mounted MuJoCo camera, site, or body.`
1293
+ };
1387
1294
  }
1388
- useEffect(() => {
1389
- const element = elementRef.current;
1390
- const parent = gl.domElement.parentElement;
1391
- if (!element || !parent) return;
1392
- element.style.position = "absolute";
1393
- element.style.overflow = "hidden";
1394
- parent.appendChild(element);
1395
- return () => {
1396
- parent.removeChild(element);
1295
+ const missingKeys = [...plan.missingKeys];
1296
+ const ready = missingKeys.length === 0;
1297
+ const status = ready ? MountedCameraFrameSequenceReadinessStatus.Ready : resolvedKeys.length > 0 ? MountedCameraFrameSequenceReadinessStatus.Partial : MountedCameraFrameSequenceReadinessStatus.Missing;
1298
+ return {
1299
+ ready,
1300
+ status,
1301
+ cameraKeys: [...plan.cameraKeys],
1302
+ resolvedKeys,
1303
+ missingKeys,
1304
+ cameras,
1305
+ message: ready ? `All ${plan.cameraKeys.length} requested camera stream${plan.cameraKeys.length === 1 ? "" : "s"} resolve to mounted MuJoCo sources.` : `Missing mounted MuJoCo source${missingKeys.length === 1 ? "" : "s"} for ${missingKeys.join(", ")}.`
1306
+ };
1307
+ }
1308
+ function normalizeFrameCount(frameCount) {
1309
+ return Number.isFinite(frameCount) && frameCount !== void 0 ? Math.max(0, Math.floor(frameCount)) : 0;
1310
+ }
1311
+ function createMountedCameraFrameSequenceManifest(result, options = {}) {
1312
+ const cameraKeys = [
1313
+ ...options.cameraKeys ?? result.readiness.cameraKeys ?? result.plan.cameraKeys ?? result.cameraKeys
1314
+ ];
1315
+ const expectedFrameCount = normalizeFrameCount(
1316
+ options.expectedFrameCount ?? result.frameCount
1317
+ );
1318
+ const recordedFrameCount = normalizeFrameCount(result.frameCount);
1319
+ const streamSummaries = {};
1320
+ const streams = [];
1321
+ let missingFrameCount = 0;
1322
+ let completeStreamCount = 0;
1323
+ let resolvedOrRecordedStreamCount = 0;
1324
+ for (const key of cameraKeys) {
1325
+ const summary = result.cameraSummaries[key];
1326
+ const readiness = result.readiness.cameras[key];
1327
+ const source = summary?.source ?? readiness?.source;
1328
+ const ready = readiness?.ready ?? Boolean(summary);
1329
+ const recorded = normalizeFrameCount(summary?.frameCount);
1330
+ const missing = Math.max(expectedFrameCount - recorded, 0);
1331
+ const complete2 = ready && missing === 0;
1332
+ const status2 = complete2 ? MountedCameraFrameSequenceManifestStatus.Complete : ready || recorded > 0 ? MountedCameraFrameSequenceManifestStatus.Partial : MountedCameraFrameSequenceManifestStatus.Missing;
1333
+ const target = source ? getCameraFrameCaptureSourceTarget(source) : readiness?.message ? void 0 : "missing MuJoCo camera";
1334
+ const message = complete2 ? `Camera stream "${key}" recorded ${recorded} of ${expectedFrameCount} frame${expectedFrameCount === 1 ? "" : "s"}.` : ready || recorded > 0 ? `Camera stream "${key}" recorded ${recorded} of ${expectedFrameCount} frame${expectedFrameCount === 1 ? "" : "s"}.` : readiness?.message ?? `Camera stream "${key}" did not record any frames.`;
1335
+ const stream = {
1336
+ key,
1337
+ ready,
1338
+ complete: complete2,
1339
+ status: status2,
1340
+ source,
1341
+ selector: readiness?.selector,
1342
+ target,
1343
+ width: summary?.width,
1344
+ height: summary?.height,
1345
+ expectedFrameCount,
1346
+ recordedFrameCount: recorded,
1347
+ missingFrameCount: missing,
1348
+ firstFrameIndex: summary?.firstFrameIndex ?? null,
1349
+ lastFrameIndex: summary?.lastFrameIndex ?? null,
1350
+ firstTimestamp: summary?.firstTimestamp ?? null,
1351
+ lastTimestamp: summary?.lastTimestamp ?? null,
1352
+ message
1397
1353
  };
1398
- }, [gl]);
1399
- useEffect(() => {
1400
- const element = elementRef.current;
1401
- if (!element) return;
1402
- element.className = className ?? "";
1403
- if (style) Object.assign(element.style, style);
1404
- }, [className, style]);
1405
- useCameraViewport(elementRef, options);
1406
- return null;
1354
+ streamSummaries[key] = stream;
1355
+ streams.push(stream);
1356
+ missingFrameCount += missing;
1357
+ if (complete2) completeStreamCount += 1;
1358
+ if (ready || recorded > 0) resolvedOrRecordedStreamCount += 1;
1359
+ }
1360
+ const complete = result.readiness.ready && streams.length === completeStreamCount && missingFrameCount === 0;
1361
+ const status = complete ? MountedCameraFrameSequenceManifestStatus.Complete : resolvedOrRecordedStreamCount > 0 ? MountedCameraFrameSequenceManifestStatus.Partial : MountedCameraFrameSequenceManifestStatus.Missing;
1362
+ return {
1363
+ schema: "mujoco-react/mounted-camera-frame-sequence-manifest@1",
1364
+ ready: result.readiness.ready,
1365
+ complete,
1366
+ status,
1367
+ cameraKeys,
1368
+ resolvedKeys: [...result.readiness.resolvedKeys],
1369
+ missingKeys: [...result.readiness.missingKeys],
1370
+ expectedFrameCount,
1371
+ recordedFrameCount,
1372
+ missingFrameCount,
1373
+ streamSummaries,
1374
+ streams,
1375
+ readiness: result.readiness,
1376
+ message: complete ? `All ${cameraKeys.length} camera stream${cameraKeys.length === 1 ? "" : "s"} recorded ${expectedFrameCount} frame${expectedFrameCount === 1 ? "" : "s"}.` : `Mounted camera sequence coverage is ${status}.`
1377
+ };
1407
1378
  }
1408
- function isTargetRef(target) {
1409
- return Boolean(target && typeof target === "object" && "current" in target);
1379
+ function createMountedCameraFrameSequencePlanFromApi(api, cameraKeys, options = {}) {
1380
+ return createMountedCameraFrameSequencePlan(cameraKeys, {
1381
+ ...options,
1382
+ cameras: api.getCameras(),
1383
+ sites: api.getSites(),
1384
+ bodies: api.getBodies()
1385
+ });
1410
1386
  }
1411
- function resolveCanvasTarget(target) {
1412
- const resolvedTarget = isTargetRef(target) ? target.current : target;
1413
- if (!resolvedTarget) {
1414
- throw new Error("No frame capture target is available.");
1387
+ async function recordMountedCameraFrameSequence(api, options) {
1388
+ const { cameraKeys, ...restOptions } = options;
1389
+ const requireAll = restOptions.requireAll ?? restOptions.requireMountedSources ?? true;
1390
+ const plan = createMountedCameraFrameSequencePlanFromApi(
1391
+ api,
1392
+ cameraKeys,
1393
+ { ...restOptions, requireAll }
1394
+ );
1395
+ const readiness = createMountedCameraFrameSequenceReadiness(plan);
1396
+ const result = await api.recordCameraSequence({
1397
+ ...restOptions,
1398
+ cameras: plan.cameras,
1399
+ requireMountedSources: restOptions.requireMountedSources ?? true
1400
+ });
1401
+ return {
1402
+ ...result,
1403
+ plan,
1404
+ readiness
1405
+ };
1406
+ }
1407
+
1408
+ // src/policyCameraFrames.ts
1409
+ function addPolicyImageAliases(images, stream, frame, includeObservationImageAliases) {
1410
+ const keys = /* @__PURE__ */ new Set();
1411
+ keys.add(stream.key);
1412
+ for (const alias of stream.aliases ?? []) keys.add(alias);
1413
+ if (includeObservationImageAliases) {
1414
+ keys.add(`observation.images.${stream.key}`);
1415
+ for (const alias of stream.aliases ?? []) {
1416
+ keys.add(`observation.images.${alias}`);
1417
+ }
1415
1418
  }
1416
- if (resolvedTarget instanceof HTMLCanvasElement) {
1417
- return resolvedTarget;
1419
+ for (const key of keys) {
1420
+ images[key] = frame.dataUrl;
1421
+ }
1422
+ }
1423
+ function describeFrameSource(key, frame) {
1424
+ return `${key}:${frame.source.kind}`;
1425
+ }
1426
+ function hasExplicitPolicyCameraSource(options) {
1427
+ return Boolean(
1428
+ options?.camera || options?.position || options?.quaternion || options?.source
1429
+ );
1430
+ }
1431
+ function createPolicyCameraFrameCapturePlan(options) {
1432
+ const {
1433
+ cameraKeys,
1434
+ defaults,
1435
+ streamOptions,
1436
+ includeObservationImageAliases,
1437
+ requireAll,
1438
+ ...sourceOptions
1439
+ } = options;
1440
+ const mountedPlan = createMountedCameraFrameSequencePlan(cameraKeys, {
1441
+ ...sourceOptions,
1442
+ defaults,
1443
+ cameraOptions: streamOptions
1444
+ });
1445
+ const streams = [];
1446
+ const missingKeys = new Set(mountedPlan.missingKeys);
1447
+ for (const key of cameraKeys) {
1448
+ const perStreamOptions = streamOptions?.[key];
1449
+ if (hasExplicitPolicyCameraSource(perStreamOptions)) {
1450
+ missingKeys.delete(key);
1451
+ streams.push({
1452
+ ...defaults,
1453
+ ...perStreamOptions,
1454
+ key,
1455
+ aliases: perStreamOptions?.aliases
1456
+ });
1457
+ continue;
1458
+ }
1459
+ const mountedCamera = mountedPlan.cameras.find((camera) => camera.key === key);
1460
+ if (!mountedCamera) continue;
1461
+ const { key: _mountedKey, ...captureOptions } = mountedCamera;
1462
+ streams.push({
1463
+ ...captureOptions,
1464
+ key,
1465
+ aliases: perStreamOptions?.aliases
1466
+ });
1418
1467
  }
1419
- const canvas = resolvedTarget.querySelector("canvas");
1420
- if (!canvas) {
1421
- throw new Error("Frame capture target does not contain a canvas.");
1468
+ const result = {
1469
+ cameraKeys: [...cameraKeys],
1470
+ streams,
1471
+ includeObservationImageAliases,
1472
+ mountedPlan,
1473
+ missingKeys: [...missingKeys]
1474
+ };
1475
+ if (requireAll && result.missingKeys.length > 0) {
1476
+ throw new Error(
1477
+ `Unable to resolve policy camera stream${result.missingKeys.length === 1 ? "" : "s"} for ${result.missingKeys.join(", ")}.`
1478
+ );
1422
1479
  }
1423
- return canvas;
1480
+ return result;
1424
1481
  }
1425
- function waitForNextAnimationFrame() {
1426
- return new Promise((resolve) => {
1427
- requestAnimationFrame(() => resolve());
1482
+ function createPolicyCameraFrameCapturePlanFromApi(api, options) {
1483
+ return createPolicyCameraFrameCapturePlan({
1484
+ ...options,
1485
+ cameras: api.getCameras(),
1486
+ sites: api.getSites(),
1487
+ bodies: api.getBodies()
1428
1488
  });
1429
1489
  }
1430
- async function captureFrame(options) {
1431
- const type = options.type ?? "image/png";
1432
- const canvas = resolveCanvasTarget(options.target);
1433
- if (options.waitForAnimationFrame ?? true) {
1434
- await waitForNextAnimationFrame();
1490
+ async function capturePolicyCameraFrames(target, options) {
1491
+ const includeObservationImageAliases = options.includeObservationImageAliases ?? true;
1492
+ const entries = await Promise.all(
1493
+ options.streams.map(async ({ key, aliases, ...captureOptions }) => {
1494
+ const frame = await target.captureCameraFrame(captureOptions);
1495
+ return [key, { frame, aliases }];
1496
+ })
1497
+ );
1498
+ const frames = {};
1499
+ const images = {};
1500
+ const sourceParts = [];
1501
+ for (const [key, { frame, aliases }] of entries) {
1502
+ const stream = { key, aliases };
1503
+ frames[key] = frame;
1504
+ addPolicyImageAliases(
1505
+ images,
1506
+ stream,
1507
+ frame,
1508
+ includeObservationImageAliases
1509
+ );
1510
+ sourceParts.push(describeFrameSource(key, frame));
1435
1511
  }
1436
1512
  return {
1437
- canvas,
1438
- dataUrl: canvas.toDataURL(type, options.quality),
1439
- type
1513
+ frames,
1514
+ images,
1515
+ sourceSummary: sourceParts.length > 0 ? sourceParts.join(" + ") : "not used by policy",
1516
+ capturedAt: Date.now()
1440
1517
  };
1441
1518
  }
1442
- async function captureFrameBlob(options) {
1443
- const type = options.type ?? "image/png";
1444
- const canvas = resolveCanvasTarget(options.target);
1445
- if (options.waitForAnimationFrame ?? true) {
1446
- await waitForNextAnimationFrame();
1447
- }
1448
- const blob = await new Promise((resolve, reject) => {
1449
- canvas.toBlob(
1450
- (nextBlob) => {
1451
- if (nextBlob) {
1452
- resolve(nextBlob);
1453
- } else {
1454
- reject(new Error("Canvas frame capture did not produce a Blob."));
1455
- }
1456
- },
1457
- type,
1458
- options.quality
1459
- );
1460
- });
1461
- return { canvas, blob, type };
1519
+ async function capturePolicyCameraFramesFromMountedStreams(target, options) {
1520
+ const plan = createPolicyCameraFrameCapturePlanFromApi(target, options);
1521
+ const result = await capturePolicyCameraFrames(target, plan);
1522
+ return { ...result, plan };
1462
1523
  }
1463
- function useFrameCapture(defaultOptions = {}) {
1464
- const [status, setStatus] = useState("idle");
1465
- const [error, setError] = useState(null);
1466
- const reset = useCallback(() => {
1467
- setStatus("idle");
1468
- setError(null);
1469
- }, []);
1470
- const capture = useCallback(
1471
- async (options = {}) => {
1472
- setStatus("capturing");
1473
- setError(null);
1524
+
1525
+ // src/runtime/client.ts
1526
+ var owners = /* @__PURE__ */ new WeakMap();
1527
+ var isWorkerOwned = (model) => Boolean(model && owners.has(model));
1528
+ function closeModelRuntime(model) {
1529
+ if (model) owners.get(model)?.close();
1530
+ }
1531
+ function runtimeContacts(model) {
1532
+ return owners.get(model)?.contacts();
1533
+ }
1534
+ function assertMainThreadOwnership(model) {
1535
+ if (isWorkerOwned(model)) throw new Error("Worker runtime owns this simulation. Use runtime operations or dispose it before using viewer physics operations.");
1536
+ }
1537
+ var abortError = () => new DOMException("Simulation operation cancelled", "AbortError");
1538
+ var RuntimeClient = class {
1539
+ constructor(worker, apply, release, capture) {
1540
+ this.worker = worker;
1541
+ this.apply = apply;
1542
+ this.release = release;
1543
+ this.capture = capture;
1544
+ worker.onmessage = (event) => {
1545
+ const response = event.data;
1546
+ if (this.closed || response.id !== this.pending?.id) return;
1547
+ if (response.display) {
1548
+ try {
1549
+ this.apply(response.display);
1550
+ const f = response.display.fields;
1551
+ this.latest = {
1552
+ time: response.display.time,
1553
+ qpos: f.qpos,
1554
+ qvel: f.qvel,
1555
+ ctrl: f.ctrl,
1556
+ sensors: f.sensordata,
1557
+ contacts: response.done ? response.display.contacts : this.latest.contacts
1558
+ };
1559
+ } catch (error) {
1560
+ this.close(error instanceof Error ? error : new Error(String(error)));
1561
+ return;
1562
+ }
1563
+ }
1564
+ if (!response.done) this.worker.postMessage({ kind: "ack", id: response.id });
1565
+ if (response.done) {
1566
+ const pending = this.pending;
1567
+ this.pending = null;
1568
+ if (response.error) pending.reject(new Error(response.error));
1569
+ else pending.resolve(response);
1570
+ }
1571
+ };
1572
+ worker.onerror = (event) => this.close(new Error(event.message || "Simulation worker failed"));
1573
+ worker.onmessageerror = () => this.close(new Error("Cannot read simulation worker response"));
1574
+ }
1575
+ pending = null;
1576
+ nextId = 0;
1577
+ closed = false;
1578
+ busy = false;
1579
+ flight = Promise.resolve();
1580
+ generation = 0;
1581
+ latest = { time: 0, qpos: new Float64Array(), qvel: new Float64Array(), ctrl: new Float64Array(), sensors: new Float64Array(), contacts: [] };
1582
+ statistics = { steps: 0, physicsMs: 0, elapsedMs: 0, realTimeFactor: 0, captureMs: 0, busy: false };
1583
+ modelId = crypto.randomUUID();
1584
+ get state() {
1585
+ return {
1586
+ ...this.latest,
1587
+ qpos: this.latest.qpos.slice(),
1588
+ qvel: this.latest.qvel.slice(),
1589
+ ctrl: this.latest.ctrl.slice(),
1590
+ sensors: this.latest.sensors.slice(),
1591
+ contacts: this.latest.contacts.map((c) => ({ ...c, pos: [...c.pos] }))
1592
+ };
1593
+ }
1594
+ get metrics() {
1595
+ return { ...this.statistics, busy: this.busy };
1596
+ }
1597
+ request(request) {
1598
+ if (this.closed) return Promise.reject(new Error("Simulation runtime is disposed"));
1599
+ return new Promise((resolve, reject) => {
1600
+ const id = ++this.nextId;
1601
+ this.pending = { id, resolve, reject };
1474
1602
  try {
1475
- const result = await captureFrame({ ...defaultOptions, ...options });
1476
- setStatus("captured");
1477
- return result;
1478
- } catch (nextError) {
1479
- const error2 = nextError instanceof Error ? nextError : new Error("Unable to capture the current canvas frame.");
1480
- setError(error2);
1481
- setStatus("error");
1482
- throw error2;
1603
+ this.worker.postMessage({ ...request, id });
1604
+ } catch (error) {
1605
+ this.pending = null;
1606
+ reject(error instanceof Error ? error : new Error(String(error)));
1483
1607
  }
1484
- },
1485
- [defaultOptions]
1486
- );
1487
- const captureBlob = useCallback(
1488
- async (options = {}) => {
1489
- setStatus("capturing");
1490
- setError(null);
1608
+ });
1609
+ }
1610
+ exclusive(operation) {
1611
+ if (this.closed) return Promise.reject(new Error("Simulation runtime is disposed"));
1612
+ if (this.busy) return Promise.reject(new Error("Runtime is busy; await the current operation (work is not queued)"));
1613
+ this.busy = true;
1614
+ const result = operation().finally(() => {
1615
+ this.busy = false;
1616
+ });
1617
+ this.flight = result.catch(() => {
1618
+ });
1619
+ return result;
1620
+ }
1621
+ initialize(initial) {
1622
+ return this.exclusive(async () => {
1623
+ await this.request({ kind: "initialize", initial });
1624
+ });
1625
+ }
1626
+ advance(options) {
1627
+ return this.exclusive(async () => {
1628
+ if (options.signal?.aborted) throw abortError();
1629
+ const started = performance.now(), simStart = this.latest.time;
1630
+ const abort = () => {
1631
+ this.worker.postMessage({ kind: "cancel", id: ++this.nextId });
1632
+ };
1633
+ options.signal?.addEventListener("abort", abort, { once: true });
1491
1634
  try {
1492
- const result = await captureFrameBlob({
1493
- ...defaultOptions,
1494
- ...options
1635
+ const result = await this.request({
1636
+ kind: "advance",
1637
+ steps: options.steps,
1638
+ controls: options.controls ? Float64Array.from(options.controls) : void 0
1495
1639
  });
1496
- setStatus("captured");
1497
- return result;
1498
- } catch (nextError) {
1499
- const error2 = nextError instanceof Error ? nextError : new Error("Unable to capture the current canvas frame.");
1500
- setError(error2);
1501
- setStatus("error");
1502
- throw error2;
1640
+ const elapsedMs = performance.now() - started;
1641
+ this.statistics = {
1642
+ ...this.statistics,
1643
+ steps: result.steps ?? 0,
1644
+ physicsMs: result.physicsMs ?? 0,
1645
+ elapsedMs,
1646
+ realTimeFactor: (this.latest.time - simStart) * 1e3 / Math.max(elapsedMs, 1e-3)
1647
+ };
1648
+ if (result.cancelled || options.signal?.aborted) throw abortError();
1649
+ return this.state;
1650
+ } finally {
1651
+ options.signal?.removeEventListener("abort", abort);
1503
1652
  }
1504
- },
1505
- [defaultOptions]
1506
- );
1507
- return {
1508
- status,
1509
- error,
1510
- isCapturing: status === "capturing",
1511
- capture,
1512
- captureBlob,
1513
- reset
1514
- };
1515
- }
1516
-
1517
- // src/rendering/cameraFrameSource.ts
1518
- var MountedCameraFrameSourceSuggestionMatch = {
1519
- Direct: "direct",
1520
- Alias: "alias",
1521
- Normalized: "normalized",
1522
- Prefix: "prefix",
1523
- Suffix: "suffix",
1524
- Contains: "contains"
1525
- };
1526
- var MountedCameraFrameSequenceReadinessStatus = {
1527
- Ready: "ready",
1528
- Partial: "partial",
1529
- Missing: "missing"
1530
- };
1531
- var MountedCameraFrameSequenceManifestStatus = {
1532
- Complete: "complete",
1533
- Partial: "partial",
1534
- Missing: "missing"
1535
- };
1536
- function getResourceName(resource) {
1537
- if (!resource) return null;
1538
- return typeof resource === "string" ? resource : resource.name ?? null;
1539
- }
1540
- function createNameSet(resources) {
1541
- return new Set(
1542
- (resources ?? []).map((resource) => getResourceName(resource)).filter((name) => Boolean(name))
1543
- );
1544
- }
1545
- function createResourceNames(resources) {
1546
- return (resources ?? []).map((resource) => getResourceName(resource)).filter((name) => Boolean(name));
1547
- }
1548
- function normalizeAliasCandidates(value) {
1549
- if (!value) return [];
1550
- return Array.isArray(value) ? value : [value];
1551
- }
1552
- function countMountedSelectors(selector) {
1553
- return Number(Boolean(selector.cameraName)) + Number(Boolean(selector.siteName)) + Number(Boolean(selector.bodyName));
1554
- }
1555
- function normalizeCameraSourceName(value) {
1556
- return value.trim().toLowerCase().replace(/[^a-z0-9]+/g, "_").replace(/^_+|_+$/g, "");
1557
- }
1558
- function createCameraSourceKeyVariants(key) {
1559
- const candidates = [
1560
- key,
1561
- key.startsWith("observation.images.") ? key.slice("observation.images.".length) : "",
1562
- key.includes(".") ? key.split(".").at(-1) ?? "" : "",
1563
- key.includes("/") ? key.split("/").at(-1) ?? "" : ""
1564
- ];
1565
- return candidates.map((candidate) => candidate.trim()).filter((candidate, index, items) => candidate && items.indexOf(candidate) === index);
1566
- }
1567
- function getSelectorKey(selector) {
1568
- if (selector.cameraName) return `camera:${selector.cameraName}`;
1569
- if (selector.siteName) return `site:${selector.siteName}`;
1570
- if (selector.bodyName) return `body:${selector.bodyName}`;
1571
- return null;
1572
- }
1573
- function getMountedCameraFrameCaptureSource(selector) {
1574
- if (countMountedSelectors(selector) !== 1) return null;
1575
- if (selector.cameraName) {
1576
- return { kind: "mujoco-camera", cameraName: selector.cameraName };
1653
+ });
1654
+ }
1655
+ observe(cameras) {
1656
+ return this.exclusive(async () => {
1657
+ const generation = this.generation;
1658
+ await this.request({ kind: "snapshot" });
1659
+ const state = this.state, started = performance.now();
1660
+ const frames = cameras ? await this.capture?.(cameras) : void 0;
1661
+ if (cameras && !frames) throw new Error("Camera observations require an attached canvas");
1662
+ this.statistics.captureMs = performance.now() - started;
1663
+ if (generation !== this.generation || this.closed) throw abortError();
1664
+ return { ...state, ...frames ? { cameras: frames } : {} };
1665
+ });
1666
+ }
1667
+ saveState() {
1668
+ return this.exclusive(async () => {
1669
+ const result = await this.request({ kind: "snapshot" });
1670
+ const state = result.display.integration;
1671
+ return { modelId: this.modelId, signature: state.signature, values: state.values.slice() };
1672
+ });
1673
+ }
1674
+ restoreState(snapshot) {
1675
+ return this.exclusive(async () => {
1676
+ if (snapshot.modelId !== this.modelId) throw new Error("Snapshot belongs to a different runtime");
1677
+ await this.request({ kind: "restore", state: { signature: snapshot.signature, values: snapshot.values } });
1678
+ return this.state;
1679
+ });
1680
+ }
1681
+ async cancel() {
1682
+ if (this.closed) return;
1683
+ this.generation++;
1684
+ this.worker.postMessage({ kind: "cancel", id: ++this.nextId });
1685
+ await this.flight;
1686
+ }
1687
+ async reset() {
1688
+ await this.cancel();
1689
+ return this.exclusive(async () => {
1690
+ await this.request({ kind: "reset" });
1691
+ return this.state;
1692
+ });
1693
+ }
1694
+ async dispose() {
1695
+ await this.cancel();
1696
+ this.close();
1577
1697
  }
1578
- if (selector.siteName) {
1579
- return { kind: "mujoco-site", siteName: selector.siteName };
1698
+ close(error = new Error("Simulation runtime disposed"), restoreViewer = true) {
1699
+ if (this.closed) return;
1700
+ this.closed = true;
1701
+ this.generation++;
1702
+ this.worker.terminate();
1703
+ this.pending?.reject(error);
1704
+ this.pending = null;
1705
+ this.release(restoreViewer);
1580
1706
  }
1581
- if (selector.bodyName) {
1582
- return { kind: "mujoco-body", bodyName: selector.bodyName };
1707
+ };
1708
+ async function attachSimulationRuntime(module, api, waitForDisplay, invalidate, options = {}) {
1709
+ const model = api.mjModelRef.current, data = api.mjDataRef.current;
1710
+ if (!model || !data) throw new Error("Scene must be ready before creating a runtime");
1711
+ assertMainThreadOwnership(model);
1712
+ api.setPaused(true);
1713
+ const state = api.saveState().integration;
1714
+ const initialState = { signature: state.signature, values: state.values };
1715
+ const mj = module;
1716
+ const path = `/runtime-${crypto.randomUUID()}.mjb`;
1717
+ let binary;
1718
+ try {
1719
+ mj.mj_saveModel(model, path, null);
1720
+ binary = mj.FS.readFile(path);
1721
+ } finally {
1722
+ try {
1723
+ mj.FS.unlink(path);
1724
+ } catch {
1725
+ }
1583
1726
  }
1584
- return null;
1585
- }
1586
- function isMountedCameraFrameCaptureSource(source) {
1587
- return source.kind === "mujoco-camera" || source.kind === "mujoco-site" || source.kind === "mujoco-body";
1588
- }
1589
- function getCameraFrameCaptureSourceTarget(source) {
1590
- if (source.kind === "mujoco-camera") return source.cameraName;
1591
- if (source.kind === "mujoco-site") return source.siteName;
1592
- if (source.kind === "mujoco-body") return source.bodyName;
1593
- if (source.kind === "custom-camera") return "custom camera";
1594
- if (source.kind === "explicit-pose") return "explicit pose";
1595
- return "fallback camera";
1596
- }
1597
- function createMountedCameraFrameSourceSuggestion(key, selector, resourceName, match) {
1598
- const source = getMountedCameraFrameCaptureSource(selector);
1599
- if (!source) return null;
1600
- return {
1601
- key,
1602
- selector,
1603
- source,
1604
- resourceName,
1605
- resourceKind: source.kind,
1606
- match
1727
+ const worker = options.createWorker?.() ?? new Worker(new URL("./physics.worker.js", import.meta.url), { type: "module" });
1728
+ let complete;
1729
+ const apply = (display) => {
1730
+ if (api.mjModelRef.current !== model) throw new Error("Simulation model changed");
1731
+ if (display.integration) {
1732
+ module.mj_setState(model, data, Array.from(display.integration.values), display.integration.signature);
1733
+ complete = display;
1734
+ }
1735
+ for (const field of displayFields) {
1736
+ const target = data[field];
1737
+ if (!(target instanceof Float64Array || target instanceof Int32Array) || target.length !== display.fields[field].length) throw new Error(`Invalid display field ${field}`);
1738
+ target.set(display.fields[field]);
1739
+ }
1740
+ data.time = display.time;
1741
+ invalidate();
1607
1742
  };
1743
+ const client = new RuntimeClient(worker, apply, (restoreViewer) => {
1744
+ try {
1745
+ if (api.mjModelRef.current === model && complete) {
1746
+ apply(complete);
1747
+ if (restoreViewer) {
1748
+ module.mj_forward(model, data);
1749
+ const state2 = complete.integration;
1750
+ module.mj_setState(model, data, Array.from(state2.values), state2.signature);
1751
+ }
1752
+ }
1753
+ } finally {
1754
+ owners.delete(model);
1755
+ }
1756
+ }, async (cameras) => {
1757
+ await waitForDisplay();
1758
+ return capturePolicyCameraFrames(api, cameras);
1759
+ });
1760
+ owners.set(model, { close: () => client.close(void 0, false), contacts: () => client.state.contacts });
1761
+ try {
1762
+ await client.initialize({ model: binary, state: initialState });
1763
+ return client;
1764
+ } catch (error) {
1765
+ client.close();
1766
+ throw error;
1767
+ }
1608
1768
  }
1609
- function addMountedCameraFrameSourceSuggestion(suggestions, seen, suggestion) {
1610
- if (!suggestion) return;
1611
- const selectorKey = getSelectorKey(suggestion.selector);
1612
- if (!selectorKey || seen.has(selectorKey)) return;
1613
- seen.add(selectorKey);
1614
- suggestions.push(suggestion);
1769
+ var CapsuleGeometry = class extends THREE12.BufferGeometry {
1770
+ parameters;
1771
+ constructor(radius = 1, length = 1, capSegments = 4, radialSegments = 8) {
1772
+ super();
1773
+ this.type = "CapsuleGeometry";
1774
+ this.parameters = { radius, length, capSegments, radialSegments };
1775
+ const path = new THREE12.Path();
1776
+ path.absarc(0, -length / 2, radius, Math.PI * 1.5, 0, false);
1777
+ path.absarc(0, length / 2, radius, 0, Math.PI * 0.5, false);
1778
+ const latheGeometry = new THREE12.LatheGeometry(path.getPoints(capSegments), radialSegments);
1779
+ const self = this;
1780
+ self.setIndex(latheGeometry.getIndex());
1781
+ self.setAttribute("position", latheGeometry.getAttribute("position"));
1782
+ self.setAttribute("normal", latheGeometry.getAttribute("normal"));
1783
+ self.setAttribute("uv", latheGeometry.getAttribute("uv"));
1784
+ }
1785
+ };
1786
+
1787
+ // src/rendering/GeomBuilder.ts
1788
+ var DEFAULT_MESH_NORMAL_SMOOTHING_TOLERANCE = 1e-4;
1789
+ var GeomBuilder = class {
1790
+ mujoco;
1791
+ textureCache = /* @__PURE__ */ new Map();
1792
+ renderOptions;
1793
+ constructor(mujoco, renderOptions) {
1794
+ this.mujoco = mujoco;
1795
+ this.renderOptions = renderOptions;
1796
+ }
1797
+ getMeshNormalSmoothingTolerance() {
1798
+ const smoothing = this.renderOptions?.meshNormalSmoothing;
1799
+ if (!smoothing) return null;
1800
+ if (smoothing === true) return DEFAULT_MESH_NORMAL_SMOOTHING_TOLERANCE;
1801
+ return smoothing.tolerance ?? DEFAULT_MESH_NORMAL_SMOOTHING_TOLERANCE;
1802
+ }
1803
+ getMaterialTexture(mjModel, matId) {
1804
+ if (matId < 0 || !mjModel.mat_texid || !mjModel.tex_data) return null;
1805
+ const materialCount = Math.max(1, Math.floor(mjModel.mat_rgba.length / 4));
1806
+ const textureRoles = Math.max(1, Math.floor(mjModel.mat_texid.length / materialCount));
1807
+ let texId = -1;
1808
+ for (let role = 0; role < textureRoles; role += 1) {
1809
+ const candidate = mjModel.mat_texid[matId * textureRoles + role];
1810
+ if (candidate >= 0) {
1811
+ texId = candidate;
1812
+ break;
1813
+ }
1814
+ }
1815
+ if (texId < 0) return null;
1816
+ const cached = this.textureCache.get(texId);
1817
+ if (cached) return cached;
1818
+ const width = Number(mjModel.tex_width[texId]);
1819
+ const height = Number(mjModel.tex_height[texId]);
1820
+ const channels = Number(mjModel.tex_nchannel[texId]);
1821
+ const offset = Number(mjModel.tex_adr[texId]);
1822
+ if (width <= 0 || height <= 0 || channels <= 0 || offset < 0) return null;
1823
+ const source = mjModel.tex_data.subarray(offset, offset + width * height * channels);
1824
+ const rgba = new Uint8Array(width * height * 4);
1825
+ for (let i = 0, j = 0; i < width * height; i += 1, j += channels) {
1826
+ const r = source[j] ?? 255;
1827
+ const g = channels > 1 ? source[j + 1] : r;
1828
+ const b = channels > 2 ? source[j + 2] : r;
1829
+ const a = channels > 3 ? source[j + 3] : 255;
1830
+ const out = i * 4;
1831
+ rgba[out] = r;
1832
+ rgba[out + 1] = g;
1833
+ rgba[out + 2] = b;
1834
+ rgba[out + 3] = a;
1835
+ }
1836
+ const texture = new THREE12.DataTexture(rgba, width, height, THREE12.RGBAFormat);
1837
+ texture.colorSpace = THREE12.LinearSRGBColorSpace;
1838
+ texture.wrapS = THREE12.RepeatWrapping;
1839
+ texture.wrapT = THREE12.RepeatWrapping;
1840
+ texture.flipY = true;
1841
+ const repeatOffset = matId * 2;
1842
+ const repeatS = mjModel.mat_texrepeat?.[repeatOffset] ?? 1;
1843
+ const repeatT = mjModel.mat_texrepeat?.[repeatOffset + 1] ?? 1;
1844
+ texture.repeat.set(repeatS || 1, repeatT || 1);
1845
+ texture.needsUpdate = true;
1846
+ this.textureCache.set(texId, texture);
1847
+ return texture;
1848
+ }
1849
+ /**
1850
+ * Creates a Three.js Object3D (usually a Mesh) for a specific geometry in the MuJoCo model.
1851
+ * Returns null if the geometry shouldn't be rendered (e.g., invisible collision triggers).
1852
+ */
1853
+ create(mjModel, g) {
1854
+ if (mjModel.geom_group[g] === 3) return null;
1855
+ const type = mjModel.geom_type[g];
1856
+ const size = mjModel.geom_size.subarray(g * 3, g * 3 + 3);
1857
+ const pos = mjModel.geom_pos.subarray(g * 3, g * 3 + 3);
1858
+ const quat = mjModel.geom_quat.subarray(g * 4, g * 4 + 4);
1859
+ const matId = mjModel.geom_matid[g];
1860
+ const color = new THREE12.Color(16777215);
1861
+ const map = this.getMaterialTexture(mjModel, matId);
1862
+ let opacity = 1;
1863
+ if (matId >= 0) {
1864
+ const rgba = mjModel.mat_rgba.subarray(matId * 4, matId * 4 + 4);
1865
+ color.setRGB(rgba[0], rgba[1], rgba[2]);
1866
+ opacity = rgba[3];
1867
+ } else {
1868
+ const rgba = mjModel.geom_rgba.subarray(g * 4, g * 4 + 4);
1869
+ color.setRGB(rgba[0], rgba[1], rgba[2]);
1870
+ opacity = rgba[3];
1871
+ }
1872
+ const MG = this.mujoco.mjtGeom;
1873
+ let geo = null;
1874
+ const getVal = (v) => v?.value ?? v;
1875
+ if (type === getVal(MG.mjGEOM_PLANE)) {
1876
+ geo = new THREE12.PlaneGeometry(size[0] * 2 || 5, size[1] * 2 || 5);
1877
+ } else if (type === getVal(MG.mjGEOM_SPHERE)) {
1878
+ geo = new THREE12.SphereGeometry(size[0], 24, 24);
1879
+ } else if (type === getVal(MG.mjGEOM_CAPSULE)) {
1880
+ geo = new CapsuleGeometry(size[0], size[1] * 2, 24, 12);
1881
+ geo.rotateX(Math.PI / 2);
1882
+ } else if (type === getVal(MG.mjGEOM_BOX)) {
1883
+ geo = new THREE12.BoxGeometry(size[0] * 2, size[1] * 2, size[2] * 2);
1884
+ } else if (type === getVal(MG.mjGEOM_CYLINDER)) {
1885
+ geo = new THREE12.CylinderGeometry(size[0], size[0], size[1] * 2, 24);
1886
+ geo.rotateX(Math.PI / 2);
1887
+ } else if (type === getVal(MG.mjGEOM_MESH)) {
1888
+ const mId = mjModel.geom_dataid[g];
1889
+ const vAdr = mjModel.mesh_vertadr[mId];
1890
+ const vNum = mjModel.mesh_vertnum[mId];
1891
+ const fAdr = mjModel.mesh_faceadr[mId];
1892
+ const fNum = mjModel.mesh_facenum[mId];
1893
+ const tAdr = mjModel.mesh_texcoordadr?.[mId] ?? -1;
1894
+ if (map && tAdr >= 0 && mjModel.mesh_texcoord && mjModel.mesh_facetexcoord) {
1895
+ const corners = fNum * 3;
1896
+ const positions = new Float32Array(corners * 3);
1897
+ const uvs = new Float32Array(corners * 2);
1898
+ for (let c = 0; c < corners; c++) {
1899
+ const v = vAdr + mjModel.mesh_face[fAdr * 3 + c];
1900
+ positions[c * 3] = mjModel.mesh_vert[v * 3];
1901
+ positions[c * 3 + 1] = mjModel.mesh_vert[v * 3 + 1];
1902
+ positions[c * 3 + 2] = mjModel.mesh_vert[v * 3 + 2];
1903
+ const t = tAdr + mjModel.mesh_facetexcoord[fAdr * 3 + c];
1904
+ uvs[c * 2] = mjModel.mesh_texcoord[t * 2];
1905
+ uvs[c * 2 + 1] = 1 - mjModel.mesh_texcoord[t * 2 + 1];
1906
+ }
1907
+ geo = new THREE12.BufferGeometry();
1908
+ geo.setAttribute("position", new THREE12.Float32BufferAttribute(positions, 3));
1909
+ geo.setAttribute("uv", new THREE12.Float32BufferAttribute(uvs, 2));
1910
+ geo.computeVertexNormals();
1911
+ } else {
1912
+ geo = new THREE12.BufferGeometry();
1913
+ geo.setAttribute("position", new THREE12.Float32BufferAttribute(mjModel.mesh_vert.subarray(vAdr * 3, (vAdr + vNum) * 3), 3));
1914
+ geo.setIndex(Array.from(mjModel.mesh_face.subarray(fAdr * 3, (fAdr + fNum) * 3)));
1915
+ const smoothingTolerance = this.getMeshNormalSmoothingTolerance();
1916
+ if (smoothingTolerance !== null) {
1917
+ geo = mergeVertices(geo, smoothingTolerance);
1918
+ }
1919
+ geo.computeVertexNormals();
1920
+ }
1921
+ }
1922
+ if (geo) {
1923
+ const isPlane = type === getVal(MG.mjGEOM_PLANE);
1924
+ const materialMap = isPlane && map ? map.clone() : map;
1925
+ if (isPlane && materialMap) {
1926
+ materialMap.repeat.multiplyScalar(2.5);
1927
+ materialMap.needsUpdate = true;
1928
+ }
1929
+ const mesh = new THREE12.Mesh(geo, new THREE12.MeshStandardMaterial({
1930
+ color,
1931
+ map: materialMap,
1932
+ transparent: opacity < 1,
1933
+ opacity,
1934
+ roughness: 0.6,
1935
+ metalness: 0
1936
+ }));
1937
+ mesh.castShadow = type !== getVal(MG.mjGEOM_PLANE);
1938
+ mesh.receiveShadow = true;
1939
+ mesh.position.set(pos[0], pos[1], pos[2]);
1940
+ mesh.quaternion.set(quat[1], quat[2], quat[3], quat[0]);
1941
+ mesh.userData.bodyID = mjModel.geom_bodyid[g];
1942
+ mesh.userData.geomID = g;
1943
+ mesh.userData.geomGroup = mjModel.geom_group[g];
1944
+ mesh.userData.geomName = getName(mjModel, mjModel.name_geomadr[g]);
1945
+ return mesh;
1946
+ }
1947
+ return null;
1948
+ }
1949
+ };
1950
+ function getRenderOptionsKey(renderOptions) {
1951
+ const smoothing = renderOptions?.meshNormalSmoothing;
1952
+ if (!smoothing) return "default";
1953
+ if (smoothing === true) return "meshNormalSmoothing:true";
1954
+ return `meshNormalSmoothing:${smoothing.tolerance ?? "default"}`;
1615
1955
  }
1616
- function getCameraFrameResourceMatch(key, resourceName) {
1617
- if (resourceName === key) return MountedCameraFrameSourceSuggestionMatch.Direct;
1618
- const normalizedResource = normalizeCameraSourceName(resourceName);
1619
- if (!normalizedResource) return null;
1620
- for (const variant of createCameraSourceKeyVariants(key)) {
1621
- if (resourceName === variant) return MountedCameraFrameSourceSuggestionMatch.Direct;
1622
- const normalizedKey = normalizeCameraSourceName(variant);
1623
- if (!normalizedKey) continue;
1624
- if (normalizedResource === normalizedKey) {
1625
- return MountedCameraFrameSourceSuggestionMatch.Normalized;
1956
+ function SceneRenderer({ renderOptions, ...props }) {
1957
+ const {
1958
+ mjModelRef,
1959
+ mjDataRef,
1960
+ mujocoRef,
1961
+ onSelectionRef,
1962
+ hiddenBodiesRef,
1963
+ hiddenBodiesVersion,
1964
+ interpolateRef,
1965
+ interpolationStateRef,
1966
+ status
1967
+ } = useMujocoContext();
1968
+ const groupRef = useRef(null);
1969
+ const bodyRefs = useRef([]);
1970
+ const prevModelRef = useRef(null);
1971
+ const prevRenderOptionsKeyRef = useRef(null);
1972
+ const prevHiddenVersionRef = useRef(null);
1973
+ const renderOptionsKey = getRenderOptionsKey(renderOptions);
1974
+ const geomBuilder = useMemo(() => {
1975
+ if (status !== "ready") return null;
1976
+ return new GeomBuilder(mujocoRef.current, renderOptions);
1977
+ }, [status, mujocoRef, renderOptionsKey]);
1978
+ useEffect(() => {
1979
+ if (status !== "ready" || !geomBuilder) return;
1980
+ const model = mjModelRef.current;
1981
+ const group = groupRef.current;
1982
+ if (!model || !group) return;
1983
+ if (prevModelRef.current === model && prevRenderOptionsKeyRef.current === renderOptionsKey && prevHiddenVersionRef.current === hiddenBodiesVersion) {
1984
+ return;
1626
1985
  }
1627
- if (normalizedResource.startsWith(`${normalizedKey}_`)) {
1628
- return MountedCameraFrameSourceSuggestionMatch.Prefix;
1986
+ prevModelRef.current = model;
1987
+ prevRenderOptionsKeyRef.current = renderOptionsKey;
1988
+ prevHiddenVersionRef.current = hiddenBodiesVersion;
1989
+ while (group.children.length > 0) {
1990
+ group.remove(group.children[0]);
1629
1991
  }
1630
- if (normalizedResource.endsWith(`_${normalizedKey}`)) {
1631
- return MountedCameraFrameSourceSuggestionMatch.Suffix;
1992
+ const refs = [];
1993
+ for (let i = 0; i < model.nbody; i++) {
1994
+ const bodyGroup = new THREE12.Group();
1995
+ bodyGroup.userData.bodyID = i;
1996
+ const bodyName = getName(model, model.name_bodyadr[i]);
1997
+ if (!hiddenBodiesRef.current.has(bodyName)) {
1998
+ for (let g = 0; g < model.ngeom; g++) {
1999
+ if (model.geom_bodyid[g] === i) {
2000
+ const mesh = geomBuilder.create(model, g);
2001
+ if (mesh) bodyGroup.add(mesh);
2002
+ }
2003
+ }
2004
+ }
2005
+ group.add(bodyGroup);
2006
+ refs.push(bodyGroup);
1632
2007
  }
1633
- if (normalizedResource.includes(`_${normalizedKey}_`)) {
1634
- return MountedCameraFrameSourceSuggestionMatch.Contains;
2008
+ bodyRefs.current = refs;
2009
+ }, [status, geomBuilder, mjModelRef, renderOptionsKey, hiddenBodiesRef, hiddenBodiesVersion]);
2010
+ const syncBodiesToData = useCallback(() => {
2011
+ const data = mjDataRef.current;
2012
+ if (!data) return;
2013
+ const bodies = bodyRefs.current;
2014
+ const interpolation = interpolationStateRef.current;
2015
+ const useInterpolation = interpolateRef.current && interpolation.valid;
2016
+ for (let i = 0; i < bodies.length; i++) {
2017
+ const ref = bodies[i];
2018
+ if (!ref) continue;
2019
+ if (useInterpolation) {
2020
+ const alpha = interpolation.alpha;
2021
+ const i3 = i * 3;
2022
+ ref.position.set(
2023
+ THREE12.MathUtils.lerp(interpolation.previousXpos[i3], interpolation.currentXpos[i3], alpha),
2024
+ THREE12.MathUtils.lerp(interpolation.previousXpos[i3 + 1], interpolation.currentXpos[i3 + 1], alpha),
2025
+ THREE12.MathUtils.lerp(interpolation.previousXpos[i3 + 2], interpolation.currentXpos[i3 + 2], alpha)
2026
+ );
2027
+ const i4 = i * 4;
2028
+ _previousQuat.set(
2029
+ interpolation.previousXquat[i4 + 1],
2030
+ interpolation.previousXquat[i4 + 2],
2031
+ interpolation.previousXquat[i4 + 3],
2032
+ interpolation.previousXquat[i4]
2033
+ );
2034
+ _currentQuat.set(
2035
+ interpolation.currentXquat[i4 + 1],
2036
+ interpolation.currentXquat[i4 + 2],
2037
+ interpolation.currentXquat[i4 + 3],
2038
+ interpolation.currentXquat[i4]
2039
+ );
2040
+ ref.quaternion.copy(_previousQuat).slerp(_currentQuat, alpha);
2041
+ } else {
2042
+ ref.position.set(
2043
+ data.xpos[i * 3],
2044
+ data.xpos[i * 3 + 1],
2045
+ data.xpos[i * 3 + 2]
2046
+ );
2047
+ ref.quaternion.set(
2048
+ data.xquat[i * 4 + 1],
2049
+ data.xquat[i * 4 + 2],
2050
+ data.xquat[i * 4 + 3],
2051
+ data.xquat[i * 4]
2052
+ );
2053
+ }
1635
2054
  }
1636
- }
1637
- return null;
1638
- }
1639
- function isSelectorMounted(selector, cameraNames, siteNames, bodyNames) {
1640
- if (countMountedSelectors(selector) !== 1) return false;
1641
- return (selector.cameraName ? cameraNames.has(selector.cameraName) : false) || (selector.siteName ? siteNames.has(selector.siteName) : false) || (selector.bodyName ? bodyNames.has(selector.bodyName) : false);
2055
+ }, [interpolateRef, interpolationStateRef, mjDataRef]);
2056
+ useFrame(syncBodiesToData);
2057
+ return /* @__PURE__ */ jsx(
2058
+ "group",
2059
+ {
2060
+ ...props,
2061
+ ref: groupRef,
2062
+ userData: {
2063
+ ...props.userData,
2064
+ [CAMERA_FRAME_CAPTURE_PRE_RENDER_USER_DATA_KEY]: syncBodiesToData
2065
+ },
2066
+ onDoubleClick: (e) => {
2067
+ if (typeof props.onDoubleClick === "function") props.onDoubleClick(e);
2068
+ e.stopPropagation();
2069
+ let obj = e.object;
2070
+ while (obj && obj.userData.bodyID === void 0 && obj.parent) {
2071
+ obj = obj.parent;
2072
+ }
2073
+ const bodyID = obj?.userData.bodyID;
2074
+ if (typeof bodyID === "number" && bodyID > 0) {
2075
+ const model = mjModelRef.current;
2076
+ if (model && bodyID < model.nbody && onSelectionRef.current) {
2077
+ const name = getName(model, model.name_bodyadr[bodyID]);
2078
+ onSelectionRef.current({ bodyId: bodyID, name });
2079
+ }
2080
+ }
2081
+ }
2082
+ }
2083
+ );
1642
2084
  }
1643
- function createMountedCameraFrameSourceSuggestions(key, options) {
1644
- const cameraNames = createNameSet(options.cameras);
1645
- const siteNames = createNameSet(options.sites);
1646
- const bodyNames = createNameSet(options.bodies);
1647
- const suggestions = [];
1648
- const seen = /* @__PURE__ */ new Set();
1649
- for (const selector of normalizeAliasCandidates(options.aliases?.[key])) {
1650
- if (!isSelectorMounted(selector, cameraNames, siteNames, bodyNames)) {
1651
- continue;
2085
+ var _previousQuat = new THREE12.Quaternion();
2086
+ var _currentQuat = new THREE12.Quaternion();
2087
+ var CameraViewportRegistryContext = createContext(null);
2088
+ var nextViewportId = 0;
2089
+ var VIEWPORT_RENDER_PRIORITY = 1;
2090
+ function CameraViewportRenderer({
2091
+ viewportsRef
2092
+ }) {
2093
+ const gl = useThree((state) => state.gl);
2094
+ const scene = useThree((state) => state.scene);
2095
+ const mainCamera = useThree((state) => state.camera);
2096
+ const mujoco = useMujoco();
2097
+ useFrame(() => {
2098
+ const dpr = gl.getPixelRatio();
2099
+ const drawWidth = gl.domElement.width / dpr;
2100
+ const drawHeight = gl.domElement.height / dpr;
2101
+ gl.setScissorTest(false);
2102
+ gl.setViewport(0, 0, drawWidth, drawHeight);
2103
+ gl.render(scene, mainCamera);
2104
+ const api = mujoco.api;
2105
+ if (!api || viewportsRef.current.size === 0) return;
2106
+ const canvasRect = gl.domElement.getBoundingClientRect();
2107
+ for (const descriptor of viewportsRef.current.values()) {
2108
+ const element = descriptor.getElement();
2109
+ if (!element) continue;
2110
+ const rect = element.getBoundingClientRect();
2111
+ const isOffscreen = rect.bottom < canvasRect.top || rect.top > canvasRect.bottom || rect.right < canvasRect.left || rect.left > canvasRect.right;
2112
+ if (isOffscreen) continue;
2113
+ const width = rect.width;
2114
+ const height = rect.height;
2115
+ if (width <= 0 || height <= 0) continue;
2116
+ const left = rect.left - canvasRect.left;
2117
+ const bottom = canvasRect.bottom - rect.bottom;
2118
+ let resolved;
2119
+ try {
2120
+ resolved = api.resolveCameraCaptureOptions(descriptor.getOptions());
2121
+ } catch {
2122
+ continue;
2123
+ }
2124
+ if (!descriptor.camera) {
2125
+ descriptor.camera = createCaptureCamera(resolved, mainCamera, width, height);
2126
+ } else {
2127
+ prepareCaptureCamera(descriptor.camera, resolved, mainCamera, width, height);
2128
+ }
2129
+ gl.setViewport(left, bottom, width, height);
2130
+ gl.setScissor(left, bottom, width, height);
2131
+ gl.setScissorTest(true);
2132
+ gl.render(scene, descriptor.camera);
1652
2133
  }
1653
- const source = getMountedCameraFrameCaptureSource(selector);
1654
- if (!source) continue;
1655
- addMountedCameraFrameSourceSuggestion(
1656
- suggestions,
1657
- seen,
1658
- createMountedCameraFrameSourceSuggestion(
1659
- key,
1660
- selector,
1661
- getCameraFrameCaptureSourceTarget(source),
1662
- MountedCameraFrameSourceSuggestionMatch.Alias
1663
- )
1664
- );
1665
- }
1666
- for (const cameraName of createResourceNames(options.cameras)) {
1667
- const match = getCameraFrameResourceMatch(key, cameraName);
1668
- if (!match) continue;
1669
- addMountedCameraFrameSourceSuggestion(
1670
- suggestions,
1671
- seen,
1672
- createMountedCameraFrameSourceSuggestion(
1673
- key,
1674
- { cameraName },
1675
- cameraName,
1676
- match
1677
- )
1678
- );
1679
- }
1680
- for (const siteName of createResourceNames(options.sites)) {
1681
- const match = getCameraFrameResourceMatch(key, siteName);
1682
- if (!match) continue;
1683
- addMountedCameraFrameSourceSuggestion(
1684
- suggestions,
1685
- seen,
1686
- createMountedCameraFrameSourceSuggestion(
1687
- key,
1688
- { siteName },
1689
- siteName,
1690
- match
1691
- )
1692
- );
1693
- }
1694
- for (const bodyName of createResourceNames(options.bodies)) {
1695
- const match = getCameraFrameResourceMatch(key, bodyName);
1696
- if (!match) continue;
1697
- addMountedCameraFrameSourceSuggestion(
1698
- suggestions,
1699
- seen,
1700
- createMountedCameraFrameSourceSuggestion(
1701
- key,
1702
- { bodyName },
1703
- bodyName,
1704
- match
1705
- )
1706
- );
1707
- }
1708
- return suggestions;
2134
+ gl.setScissorTest(false);
2135
+ gl.setViewport(0, 0, drawWidth, drawHeight);
2136
+ }, VIEWPORT_RENDER_PRIORITY);
2137
+ return null;
2138
+ }
2139
+ function CameraViewportProvider({ children }) {
2140
+ const viewportsRef = useRef(/* @__PURE__ */ new Map());
2141
+ const [count, setCount] = useState(0);
2142
+ const register = useCallback((descriptor) => {
2143
+ const id = nextViewportId++;
2144
+ viewportsRef.current.set(id, descriptor);
2145
+ setCount((value2) => value2 + 1);
2146
+ return () => {
2147
+ viewportsRef.current.delete(id);
2148
+ setCount((value2) => value2 - 1);
2149
+ };
2150
+ }, []);
2151
+ const value = useMemo(() => ({ register }), [register]);
2152
+ return /* @__PURE__ */ jsxs(CameraViewportRegistryContext.Provider, { value, children: [
2153
+ children,
2154
+ count > 0 && /* @__PURE__ */ jsx(CameraViewportRenderer, { viewportsRef })
2155
+ ] });
1709
2156
  }
1710
- function resolveMountedCameraFrameSource(key, options) {
1711
- const cameraNames = createNameSet(options.cameras);
1712
- const siteNames = createNameSet(options.sites);
1713
- const bodyNames = createNameSet(options.bodies);
1714
- const directCandidates = [
1715
- { cameraName: key },
1716
- { siteName: key },
1717
- { bodyName: key }
1718
- ];
1719
- const aliasCandidates = normalizeAliasCandidates(options.aliases?.[key]);
1720
- for (const selector of aliasCandidates) {
1721
- if (!isSelectorMounted(selector, cameraNames, siteNames, bodyNames)) {
1722
- continue;
1723
- }
1724
- const source = getMountedCameraFrameCaptureSource(selector);
1725
- if (!source) continue;
1726
- return { key, selector, source };
2157
+ function useCameraViewport(elementRef, options) {
2158
+ const registry = useContext(CameraViewportRegistryContext);
2159
+ if (!registry) {
2160
+ throw new Error("useCameraViewport must be used inside <MujocoCanvas>.");
1727
2161
  }
1728
- const [suggestion] = createMountedCameraFrameSourceSuggestions(key, options);
1729
- if (suggestion) {
1730
- return {
1731
- key,
1732
- selector: suggestion.selector,
1733
- source: suggestion.source
2162
+ const optionsRef = useRef(options);
2163
+ optionsRef.current = options;
2164
+ useEffect(() => {
2165
+ const descriptor = {
2166
+ getElement: () => elementRef.current,
2167
+ getOptions: () => optionsRef.current,
2168
+ camera: null
1734
2169
  };
2170
+ return registry.register(descriptor);
2171
+ }, [registry, elementRef]);
2172
+ }
2173
+ function CameraView({ className, style, ...options }) {
2174
+ const gl = useThree((state) => state.gl);
2175
+ const elementRef = useRef(null);
2176
+ if (!elementRef.current && typeof document !== "undefined") {
2177
+ elementRef.current = document.createElement("div");
1735
2178
  }
1736
- if (options.allowAliasFallback) {
1737
- for (const selector of aliasCandidates) {
1738
- const source = getMountedCameraFrameCaptureSource(selector);
1739
- if (!source) continue;
1740
- return { key, selector, source };
1741
- }
1742
- }
1743
- for (const selector of directCandidates) {
1744
- if (!isSelectorMounted(selector, cameraNames, siteNames, bodyNames)) {
1745
- continue;
1746
- }
1747
- const source = getMountedCameraFrameCaptureSource(selector);
1748
- if (!source) continue;
1749
- return { key, selector, source };
1750
- }
2179
+ useEffect(() => {
2180
+ const element = elementRef.current;
2181
+ const parent = gl.domElement.parentElement;
2182
+ if (!element || !parent) return;
2183
+ element.style.position = "absolute";
2184
+ element.style.overflow = "hidden";
2185
+ parent.appendChild(element);
2186
+ return () => {
2187
+ parent.removeChild(element);
2188
+ };
2189
+ }, [gl]);
2190
+ useEffect(() => {
2191
+ const element = elementRef.current;
2192
+ if (!element) return;
2193
+ element.className = className ?? "";
2194
+ if (style) Object.assign(element.style, style);
2195
+ }, [className, style]);
2196
+ useCameraViewport(elementRef, options);
1751
2197
  return null;
1752
2198
  }
1753
- function createMountedCameraFrameSequencePlan(cameraKeys, options) {
1754
- const cameras = [];
1755
- const resolved = {};
1756
- const missingKeys = [];
1757
- for (const key of cameraKeys) {
1758
- const mountedSource = resolveMountedCameraFrameSource(key, options);
1759
- if (!mountedSource) {
1760
- missingKeys.push(key);
1761
- continue;
1762
- }
1763
- resolved[key] = mountedSource;
1764
- cameras.push({
1765
- key,
1766
- ...options.defaults,
1767
- ...options.cameraOptions?.[key],
1768
- ...mountedSource.selector,
1769
- source: mountedSource.source
1770
- });
2199
+ function isTargetRef(target) {
2200
+ return Boolean(target && typeof target === "object" && "current" in target);
2201
+ }
2202
+ function resolveCanvasTarget(target) {
2203
+ const resolvedTarget = isTargetRef(target) ? target.current : target;
2204
+ if (!resolvedTarget) {
2205
+ throw new Error("No frame capture target is available.");
1771
2206
  }
1772
- if (options.requireAll && missingKeys.length > 0) {
1773
- throw new Error(
1774
- `Unable to resolve mounted MuJoCo camera source${missingKeys.length === 1 ? "" : "s"} for ${missingKeys.join(", ")}.`
1775
- );
2207
+ if (resolvedTarget instanceof HTMLCanvasElement) {
2208
+ return resolvedTarget;
1776
2209
  }
1777
- return {
1778
- cameraKeys: [...cameraKeys],
1779
- cameras,
1780
- resolved,
1781
- missingKeys
1782
- };
1783
- }
1784
- function createMountedCameraFrameSequenceReadiness(plan) {
1785
- const cameras = {};
1786
- const resolvedKeys = plan.cameraKeys.filter((key) => Boolean(plan.resolved[key]));
1787
- for (const key of plan.cameraKeys) {
1788
- const resolved = plan.resolved[key];
1789
- cameras[key] = resolved ? {
1790
- key,
1791
- ready: true,
1792
- selector: resolved.selector,
1793
- source: resolved.source,
1794
- message: `Camera stream "${key}" resolves to ${resolved.source.kind}:${getCameraFrameCaptureSourceTarget(resolved.source)}.`
1795
- } : {
1796
- key,
1797
- ready: false,
1798
- message: `Camera stream "${key}" does not resolve to a mounted MuJoCo camera, site, or body.`
1799
- };
2210
+ const canvas = resolvedTarget.querySelector("canvas");
2211
+ if (!canvas) {
2212
+ throw new Error("Frame capture target does not contain a canvas.");
1800
2213
  }
1801
- const missingKeys = [...plan.missingKeys];
1802
- const ready = missingKeys.length === 0;
1803
- const status = ready ? MountedCameraFrameSequenceReadinessStatus.Ready : resolvedKeys.length > 0 ? MountedCameraFrameSequenceReadinessStatus.Partial : MountedCameraFrameSequenceReadinessStatus.Missing;
1804
- return {
1805
- ready,
1806
- status,
1807
- cameraKeys: [...plan.cameraKeys],
1808
- resolvedKeys,
1809
- missingKeys,
1810
- cameras,
1811
- message: ready ? `All ${plan.cameraKeys.length} requested camera stream${plan.cameraKeys.length === 1 ? "" : "s"} resolve to mounted MuJoCo sources.` : `Missing mounted MuJoCo source${missingKeys.length === 1 ? "" : "s"} for ${missingKeys.join(", ")}.`
1812
- };
2214
+ return canvas;
1813
2215
  }
1814
- function normalizeFrameCount(frameCount) {
1815
- return Number.isFinite(frameCount) && frameCount !== void 0 ? Math.max(0, Math.floor(frameCount)) : 0;
2216
+ function waitForNextAnimationFrame() {
2217
+ return new Promise((resolve) => {
2218
+ requestAnimationFrame(() => resolve());
2219
+ });
1816
2220
  }
1817
- function createMountedCameraFrameSequenceManifest(result, options = {}) {
1818
- const cameraKeys = [
1819
- ...options.cameraKeys ?? result.readiness.cameraKeys ?? result.plan.cameraKeys ?? result.cameraKeys
1820
- ];
1821
- const expectedFrameCount = normalizeFrameCount(
1822
- options.expectedFrameCount ?? result.frameCount
1823
- );
1824
- const recordedFrameCount = normalizeFrameCount(result.frameCount);
1825
- const streamSummaries = {};
1826
- const streams = [];
1827
- let missingFrameCount = 0;
1828
- let completeStreamCount = 0;
1829
- let resolvedOrRecordedStreamCount = 0;
1830
- for (const key of cameraKeys) {
1831
- const summary = result.cameraSummaries[key];
1832
- const readiness = result.readiness.cameras[key];
1833
- const source = summary?.source ?? readiness?.source;
1834
- const ready = readiness?.ready ?? Boolean(summary);
1835
- const recorded = normalizeFrameCount(summary?.frameCount);
1836
- const missing = Math.max(expectedFrameCount - recorded, 0);
1837
- const complete2 = ready && missing === 0;
1838
- const status2 = complete2 ? MountedCameraFrameSequenceManifestStatus.Complete : ready || recorded > 0 ? MountedCameraFrameSequenceManifestStatus.Partial : MountedCameraFrameSequenceManifestStatus.Missing;
1839
- const target = source ? getCameraFrameCaptureSourceTarget(source) : readiness?.message ? void 0 : "missing MuJoCo camera";
1840
- const message = complete2 ? `Camera stream "${key}" recorded ${recorded} of ${expectedFrameCount} frame${expectedFrameCount === 1 ? "" : "s"}.` : ready || recorded > 0 ? `Camera stream "${key}" recorded ${recorded} of ${expectedFrameCount} frame${expectedFrameCount === 1 ? "" : "s"}.` : readiness?.message ?? `Camera stream "${key}" did not record any frames.`;
1841
- const stream = {
1842
- key,
1843
- ready,
1844
- complete: complete2,
1845
- status: status2,
1846
- source,
1847
- selector: readiness?.selector,
1848
- target,
1849
- width: summary?.width,
1850
- height: summary?.height,
1851
- expectedFrameCount,
1852
- recordedFrameCount: recorded,
1853
- missingFrameCount: missing,
1854
- firstFrameIndex: summary?.firstFrameIndex ?? null,
1855
- lastFrameIndex: summary?.lastFrameIndex ?? null,
1856
- firstTimestamp: summary?.firstTimestamp ?? null,
1857
- lastTimestamp: summary?.lastTimestamp ?? null,
1858
- message
1859
- };
1860
- streamSummaries[key] = stream;
1861
- streams.push(stream);
1862
- missingFrameCount += missing;
1863
- if (complete2) completeStreamCount += 1;
1864
- if (ready || recorded > 0) resolvedOrRecordedStreamCount += 1;
2221
+ async function captureFrame(options) {
2222
+ const type = options.type ?? "image/png";
2223
+ const canvas = resolveCanvasTarget(options.target);
2224
+ if (options.waitForAnimationFrame ?? true) {
2225
+ await waitForNextAnimationFrame();
1865
2226
  }
1866
- const complete = result.readiness.ready && streams.length === completeStreamCount && missingFrameCount === 0;
1867
- const status = complete ? MountedCameraFrameSequenceManifestStatus.Complete : resolvedOrRecordedStreamCount > 0 ? MountedCameraFrameSequenceManifestStatus.Partial : MountedCameraFrameSequenceManifestStatus.Missing;
1868
2227
  return {
1869
- schema: "mujoco-react/mounted-camera-frame-sequence-manifest@1",
1870
- ready: result.readiness.ready,
1871
- complete,
1872
- status,
1873
- cameraKeys,
1874
- resolvedKeys: [...result.readiness.resolvedKeys],
1875
- missingKeys: [...result.readiness.missingKeys],
1876
- expectedFrameCount,
1877
- recordedFrameCount,
1878
- missingFrameCount,
1879
- streamSummaries,
1880
- streams,
1881
- readiness: result.readiness,
1882
- message: complete ? `All ${cameraKeys.length} camera stream${cameraKeys.length === 1 ? "" : "s"} recorded ${expectedFrameCount} frame${expectedFrameCount === 1 ? "" : "s"}.` : `Mounted camera sequence coverage is ${status}.`
2228
+ canvas,
2229
+ dataUrl: canvas.toDataURL(type, options.quality),
2230
+ type
1883
2231
  };
1884
2232
  }
1885
- function createMountedCameraFrameSequencePlanFromApi(api, cameraKeys, options = {}) {
1886
- return createMountedCameraFrameSequencePlan(cameraKeys, {
1887
- ...options,
1888
- cameras: api.getCameras(),
1889
- sites: api.getSites(),
1890
- bodies: api.getBodies()
1891
- });
1892
- }
1893
- async function recordMountedCameraFrameSequence(api, options) {
1894
- const { cameraKeys, ...restOptions } = options;
1895
- const requireAll = restOptions.requireAll ?? restOptions.requireMountedSources ?? true;
1896
- const plan = createMountedCameraFrameSequencePlanFromApi(
1897
- api,
1898
- cameraKeys,
1899
- { ...restOptions, requireAll }
1900
- );
1901
- const readiness = createMountedCameraFrameSequenceReadiness(plan);
1902
- const result = await api.recordCameraSequence({
1903
- ...restOptions,
1904
- cameras: plan.cameras,
1905
- requireMountedSources: restOptions.requireMountedSources ?? true
2233
+ async function captureFrameBlob(options) {
2234
+ const type = options.type ?? "image/png";
2235
+ const canvas = resolveCanvasTarget(options.target);
2236
+ if (options.waitForAnimationFrame ?? true) {
2237
+ await waitForNextAnimationFrame();
2238
+ }
2239
+ const blob = await new Promise((resolve, reject) => {
2240
+ canvas.toBlob(
2241
+ (nextBlob) => {
2242
+ if (nextBlob) {
2243
+ resolve(nextBlob);
2244
+ } else {
2245
+ reject(new Error("Canvas frame capture did not produce a Blob."));
2246
+ }
2247
+ },
2248
+ type,
2249
+ options.quality
2250
+ );
1906
2251
  });
2252
+ return { canvas, blob, type };
2253
+ }
2254
+ function useFrameCapture(defaultOptions = {}) {
2255
+ const [status, setStatus] = useState("idle");
2256
+ const [error, setError] = useState(null);
2257
+ const reset = useCallback(() => {
2258
+ setStatus("idle");
2259
+ setError(null);
2260
+ }, []);
2261
+ const capture = useCallback(
2262
+ async (options = {}) => {
2263
+ setStatus("capturing");
2264
+ setError(null);
2265
+ try {
2266
+ const result = await captureFrame({ ...defaultOptions, ...options });
2267
+ setStatus("captured");
2268
+ return result;
2269
+ } catch (nextError) {
2270
+ const error2 = nextError instanceof Error ? nextError : new Error("Unable to capture the current canvas frame.");
2271
+ setError(error2);
2272
+ setStatus("error");
2273
+ throw error2;
2274
+ }
2275
+ },
2276
+ [defaultOptions]
2277
+ );
2278
+ const captureBlob = useCallback(
2279
+ async (options = {}) => {
2280
+ setStatus("capturing");
2281
+ setError(null);
2282
+ try {
2283
+ const result = await captureFrameBlob({
2284
+ ...defaultOptions,
2285
+ ...options
2286
+ });
2287
+ setStatus("captured");
2288
+ return result;
2289
+ } catch (nextError) {
2290
+ const error2 = nextError instanceof Error ? nextError : new Error("Unable to capture the current canvas frame.");
2291
+ setError(error2);
2292
+ setStatus("error");
2293
+ throw error2;
2294
+ }
2295
+ },
2296
+ [defaultOptions]
2297
+ );
1907
2298
  return {
1908
- ...result,
1909
- plan,
1910
- readiness
2299
+ status,
2300
+ error,
2301
+ isCapturing: status === "capturing",
2302
+ capture,
2303
+ captureBlob,
2304
+ reset
1911
2305
  };
1912
2306
  }
1913
2307
  var _raycaster = new THREE12.Raycaster();
@@ -2358,10 +2752,11 @@ function useMujoco() {
2358
2752
  };
2359
2753
  }
2360
2754
  function useBeforePhysicsStep(callback) {
2361
- const { beforeStepCallbacks } = useMujocoContext();
2755
+ const { beforeStepCallbacks, mjModelRef } = useMujocoContext();
2362
2756
  const callbackRef = useRef(callback);
2363
2757
  callbackRef.current = callback;
2364
2758
  useEffect(() => {
2759
+ assertMainThreadOwnership(mjModelRef.current);
2365
2760
  const wrapped = (input) => callbackRef.current(input);
2366
2761
  beforeStepCallbacks.current.add(wrapped);
2367
2762
  return () => {
@@ -2370,10 +2765,11 @@ function useBeforePhysicsStep(callback) {
2370
2765
  }, [beforeStepCallbacks]);
2371
2766
  }
2372
2767
  function useAfterPhysicsStep(callback) {
2373
- const { afterStepCallbacks } = useMujocoContext();
2768
+ const { afterStepCallbacks, mjModelRef } = useMujocoContext();
2374
2769
  const callbackRef = useRef(callback);
2375
2770
  callbackRef.current = callback;
2376
2771
  useEffect(() => {
2772
+ assertMainThreadOwnership(mjModelRef.current);
2377
2773
  const wrapped = (input) => callbackRef.current(input);
2378
2774
  afterStepCallbacks.current.add(wrapped);
2379
2775
  return () => {
@@ -2393,6 +2789,7 @@ function MujocoSimProvider({
2393
2789
  timestep,
2394
2790
  integrator,
2395
2791
  substeps,
2792
+ physicsBudgetMs = 8,
2396
2793
  paused,
2397
2794
  speed,
2398
2795
  interpolate,
@@ -2400,7 +2797,32 @@ function MujocoSimProvider({
2400
2797
  hiddenBodies,
2401
2798
  children
2402
2799
  }) {
2403
- const { gl, camera, scene } = useThree();
2800
+ const { gl, camera, scene, invalidate } = useThree();
2801
+ const modelIdentity = sceneIdentity(config);
2802
+ const performanceRef = useRef({ steps: 0, physicsMs: 0, realTimeFactor: 0, backlogSteps: 0, budgetExceeded: false });
2803
+ const displayWaiters = useRef(/* @__PURE__ */ new Set());
2804
+ useFrame(() => {
2805
+ if (displayWaiters.current.size) {
2806
+ const waiters = [...displayWaiters.current];
2807
+ displayWaiters.current.clear();
2808
+ queueMicrotask(() => {
2809
+ scene.updateMatrixWorld(true);
2810
+ for (const ready of waiters) ready();
2811
+ });
2812
+ }
2813
+ }, 0);
2814
+ const waitForDisplay = () => new Promise((resolve, reject) => {
2815
+ const ready = () => {
2816
+ clearTimeout(timer);
2817
+ resolve();
2818
+ };
2819
+ const timer = setTimeout(() => {
2820
+ displayWaiters.current.delete(ready);
2821
+ reject(new Error("Canvas did not produce an observation frame within 5 seconds"));
2822
+ }, 5e3);
2823
+ displayWaiters.current.add(ready);
2824
+ invalidate();
2825
+ });
2404
2826
  const [status, setStatus] = useState("loading");
2405
2827
  const mjModelRef = useRef(null);
2406
2828
  const mjDataRef = useRef(null);
@@ -2436,18 +2858,23 @@ function MujocoSimProvider({
2436
2858
  const bodyReloadTimerRef = useRef(null);
2437
2859
  configRef.current = config;
2438
2860
  mujocoRef.current = mujoco;
2439
- pausedRef.current = paused ?? false;
2440
- speedRef.current = speed ?? 1;
2861
+ useEffect(() => {
2862
+ pausedRef.current = paused ?? false;
2863
+ }, [paused]);
2864
+ useEffect(() => {
2865
+ speedRef.current = speed ?? 1;
2866
+ }, [speed]);
2441
2867
  substepsRef.current = substeps ?? 1;
2442
2868
  interpolateRef.current = interpolate ?? false;
2443
2869
  useEffect(() => {
2444
2870
  if (!gravity) return;
2445
2871
  const model = mjModelRef.current;
2446
2872
  if (!model?.opt?.gravity) return;
2873
+ assertMainThreadOwnership(model);
2447
2874
  model.opt.gravity[0] = gravity[0];
2448
2875
  model.opt.gravity[1] = gravity[1];
2449
2876
  model.opt.gravity[2] = gravity[2];
2450
- }, [gravity]);
2877
+ }, [gravity?.[0], gravity?.[1], gravity?.[2]]);
2451
2878
  const integratorRef = useRef(integrator);
2452
2879
  integratorRef.current = integrator;
2453
2880
  const applyIntegrator = (model) => {
@@ -2455,12 +2882,16 @@ function MujocoSimProvider({
2455
2882
  if (value !== void 0) model.opt.integrator = { Euler: 0, RK4: 1, implicit: 2, implicitfast: 3, discrete: 4 }[value];
2456
2883
  };
2457
2884
  useEffect(() => {
2458
- if (mjModelRef.current) applyIntegrator(mjModelRef.current);
2885
+ if (mjModelRef.current) {
2886
+ assertMainThreadOwnership(mjModelRef.current);
2887
+ applyIntegrator(mjModelRef.current);
2888
+ }
2459
2889
  }, [integrator]);
2460
2890
  useEffect(() => {
2461
2891
  if (timestep === void 0) return;
2462
2892
  const model = mjModelRef.current;
2463
2893
  if (!model?.opt) return;
2894
+ assertMainThreadOwnership(model);
2464
2895
  model.opt.timestep = timestep;
2465
2896
  }, [timestep]);
2466
2897
  const hiddenBodiesKey = (hiddenBodies ?? []).join("\0");
@@ -2498,6 +2929,7 @@ function MujocoSimProvider({
2498
2929
  }
2499
2930
  useEffect(() => {
2500
2931
  let disposed = false;
2932
+ setStatus("loading");
2501
2933
  (async () => {
2502
2934
  try {
2503
2935
  const result = await loadScene(mujoco, buildMergedConfig(config));
@@ -2532,6 +2964,7 @@ function MujocoSimProvider({
2532
2964
  })();
2533
2965
  return () => {
2534
2966
  disposed = true;
2967
+ closeModelRuntime(mjModelRef.current);
2535
2968
  mjDataRef.current?.delete();
2536
2969
  mjModelRef.current?.delete();
2537
2970
  mjModelRef.current = null;
@@ -2543,7 +2976,7 @@ function MujocoSimProvider({
2543
2976
  } catch {
2544
2977
  }
2545
2978
  };
2546
- }, [mujoco, config]);
2979
+ }, [mujoco, modelIdentity]);
2547
2980
  useEffect(() => {
2548
2981
  if (status === "ready") {
2549
2982
  const api2 = apiRef.current;
@@ -2561,66 +2994,43 @@ function MujocoSimProvider({
2561
2994
  const model = mjModelRef.current;
2562
2995
  const data = mjDataRef.current;
2563
2996
  if (!model || !data) return;
2564
- const shouldStep = !pausedRef.current || stepsToRunRef.current > 0;
2565
- if (!shouldStep) return;
2566
- for (let i = 0; i < model.nv; i++) {
2567
- data.qfrc_applied[i] = 0;
2568
- }
2569
- for (const cb of beforeStepCallbacks.current) {
2570
- cb({ model, data });
2997
+ if (isWorkerOwned(model)) {
2998
+ interpolationStateRef.current.valid = false;
2999
+ return;
2571
3000
  }
2572
- const numSubsteps = substepsRef.current;
2573
- if (!interpolateRef.current) {
2574
- if (stepsToRunRef.current > 0) {
2575
- for (let s = 0; s < stepsToRunRef.current; s++) {
2576
- mujocoRef.current.mj_step(model, data);
2577
- }
2578
- stepsToRunRef.current = 0;
2579
- } else {
2580
- const startSimTime = data.time;
2581
- const clampedDelta = Math.min(delta, 1 / 15);
2582
- const frameTime = clampedDelta * speedRef.current;
2583
- while (data.time - startSimTime < frameTime) {
2584
- for (let s = 0; s < numSubsteps; s++) {
2585
- mujocoRef.current.mj_step(model, data);
2586
- }
2587
- }
2588
- }
2589
- } else if (stepsToRunRef.current > 0) {
3001
+ const queued = stepsToRunRef.current > 0;
3002
+ if (pausedRef.current && !queued) return;
3003
+ const startTime = data.time;
3004
+ const dt = model.opt.timestep;
3005
+ if (!Number.isFinite(dt) || dt <= 0) return;
3006
+ const substeps2 = Math.max(1, Math.floor(substepsRef.current));
3007
+ const frameTime = Math.max(0, Math.min(delta, 1 / 15) * speedRef.current);
3008
+ if (!queued && interpolateRef.current) physicsAccumulatorRef.current += frameTime;
3009
+ const desired = queued ? stepsToRunRef.current : interpolateRef.current ? Math.floor(physicsAccumulatorRef.current / (dt * substeps2)) * substeps2 : Math.ceil(frameTime / (dt * substeps2)) * substeps2;
3010
+ for (let i = 0; i < model.nv; i++) data.qfrc_applied[i] = 0;
3011
+ for (const cb of beforeStepCallbacks.current) cb({ model, data });
3012
+ if (interpolateRef.current) {
2590
3013
  ensureInterpolationBuffers(model);
2591
- copyBodyPose(data, interpolationStateRef.current.previousXpos, interpolationStateRef.current.previousXquat);
2592
- for (let s = 0; s < stepsToRunRef.current; s++) {
2593
- mujocoRef.current.mj_step(model, data);
2594
- }
3014
+ if (!interpolationStateRef.current.valid) copyBodyPose(data, interpolationStateRef.current.previousXpos, interpolationStateRef.current.previousXquat);
3015
+ }
3016
+ const measured = runStepBudget(desired, () => {
3017
+ if (interpolateRef.current) copyBodyPose(data, interpolationStateRef.current.previousXpos, interpolationStateRef.current.previousXquat);
3018
+ mujocoRef.current.mj_step(model, data);
3019
+ }, physicsBudgetMs);
3020
+ if (queued) stepsToRunRef.current -= measured.steps;
3021
+ performanceRef.current = {
3022
+ ...measured,
3023
+ backlogSteps: Math.max(0, desired - measured.steps),
3024
+ realTimeFactor: (data.time - startTime) / Math.max(delta, 1e-6)
3025
+ };
3026
+ if (interpolateRef.current) {
2595
3027
  copyBodyPose(data, interpolationStateRef.current.currentXpos, interpolationStateRef.current.currentXquat);
2596
- interpolationStateRef.current.alpha = 1;
2597
- interpolationStateRef.current.valid = true;
2598
- stepsToRunRef.current = 0;
2599
- } else {
2600
- ensureInterpolationBuffers(model);
2601
- const clampedDelta = Math.min(delta, 1 / 15);
2602
- physicsAccumulatorRef.current += clampedDelta * speedRef.current;
2603
- const stepDt = Math.max((model.opt?.timestep ?? 2e-3) * Math.max(1, numSubsteps), 1e-6);
2604
- let stepped = false;
2605
- while (physicsAccumulatorRef.current >= stepDt) {
2606
- copyBodyPose(data, interpolationStateRef.current.previousXpos, interpolationStateRef.current.previousXquat);
2607
- for (let s = 0; s < numSubsteps; s++) {
2608
- mujocoRef.current.mj_step(model, data);
2609
- }
2610
- copyBodyPose(data, interpolationStateRef.current.currentXpos, interpolationStateRef.current.currentXquat);
2611
- physicsAccumulatorRef.current -= stepDt;
2612
- stepped = true;
2613
- }
2614
- if (!interpolationStateRef.current.valid) {
2615
- copyBodyPose(data, interpolationStateRef.current.previousXpos, interpolationStateRef.current.previousXquat);
2616
- copyBodyPose(data, interpolationStateRef.current.currentXpos, interpolationStateRef.current.currentXquat);
2617
- }
2618
- interpolationStateRef.current.alpha = Math.min(Math.max(physicsAccumulatorRef.current / stepDt, 0), 1);
3028
+ if (!queued) {
3029
+ physicsAccumulatorRef.current -= measured.steps * dt;
3030
+ if (measured.budgetExceeded) physicsAccumulatorRef.current %= dt * substeps2;
3031
+ } else physicsAccumulatorRef.current = 0;
3032
+ interpolationStateRef.current.alpha = queued ? 1 : Math.min(1, physicsAccumulatorRef.current / dt);
2619
3033
  interpolationStateRef.current.valid = true;
2620
- if (!stepped) {
2621
- onStepRef.current?.({ time: data.time, model, data });
2622
- return;
2623
- }
2624
3034
  }
2625
3035
  for (const cb of afterStepCallbacks.current) {
2626
3036
  cb({ model, data });
@@ -2673,6 +3083,7 @@ function MujocoSimProvider({
2673
3083
  pausedRef.current = p;
2674
3084
  }, []);
2675
3085
  const step = useCallback((n = 1) => {
3086
+ if (!Number.isSafeInteger(n) || n <= 0) throw new Error("Step count must be a positive safe integer");
2676
3087
  stepsToRunRef.current = n;
2677
3088
  }, []);
2678
3089
  useCallback((steps = 1) => {
@@ -3091,6 +3502,7 @@ function MujocoSimProvider({
3091
3502
  const model = mjModelRef.current;
3092
3503
  const data = mjDataRef.current;
3093
3504
  if (!model || !data) return null;
3505
+ assertMainThreadOwnership(model);
3094
3506
  _rayPnt[0] = origin.x;
3095
3507
  _rayPnt[1] = origin.y;
3096
3508
  _rayPnt[2] = origin.z;
@@ -3162,6 +3574,7 @@ function MujocoSimProvider({
3162
3574
  const loadSceneApi = useCallback(async (newConfig) => {
3163
3575
  const gen = ++loadGenRef.current;
3164
3576
  try {
3577
+ closeModelRuntime(mjModelRef.current);
3165
3578
  mjDataRef.current?.delete();
3166
3579
  mjModelRef.current?.delete();
3167
3580
  mjModelRef.current = null;
@@ -3317,6 +3730,7 @@ function MujocoSimProvider({
3317
3730
  if (!model || !data) {
3318
3731
  throw new Error("MuJoCo scene is not ready for camera sequence stepping.");
3319
3732
  }
3733
+ let sliceStarted = performance.now();
3320
3734
  for (let stepIndex = 0; stepIndex < steps; stepIndex += 1) {
3321
3735
  for (let i = 0; i < model.nv; i += 1) {
3322
3736
  data.qfrc_applied[i] = 0;
@@ -3336,6 +3750,11 @@ function MujocoSimProvider({
3336
3750
  cb({ model, data });
3337
3751
  }
3338
3752
  onStepRef.current?.({ time: data.time, model, data });
3753
+ if (performance.now() - sliceStarted >= 8) {
3754
+ await new Promise((resolve) => setTimeout(resolve, 0));
3755
+ throwIfCameraSequenceAborted(options.signal);
3756
+ sliceStarted = performance.now();
3757
+ }
3339
3758
  await options.onAfterStep?.({
3340
3759
  frameIndex,
3341
3760
  stepIndex,
@@ -3543,7 +3962,14 @@ function MujocoSimProvider({
3543
3962
  model.geom_size[id * 3 + 2] = size[2];
3544
3963
  }, []);
3545
3964
  const api = useMemo(
3546
- () => ({
3965
+ () => guardRuntimeOwnership({
3966
+ createRuntime: (options) => {
3967
+ if (beforeStepCallbacks.current.size || afterStepCallbacks.current.size) {
3968
+ return Promise.reject(new Error("Unmount physics callbacks/controllers before creating a worker runtime; use runtime.advance({controls}) for policy actions."));
3969
+ }
3970
+ return attachSimulationRuntime(mujocoRef.current, apiRef.current, waitForDisplay, invalidate, options);
3971
+ },
3972
+ getPerformance: () => ({ ...performanceRef.current }),
3547
3973
  get status() {
3548
3974
  return status;
3549
3975
  },
@@ -3574,7 +4000,10 @@ function MujocoSimProvider({
3574
4000
  setExternalForce,
3575
4001
  applyGeneralizedForce,
3576
4002
  getSensorData,
3577
- getContacts,
4003
+ getContacts: () => {
4004
+ const model = mjModelRef.current;
4005
+ return model && isWorkerOwned(model) ? runtimeContacts(model) : getContacts();
4006
+ },
3578
4007
  getBodies,
3579
4008
  getJoints,
3580
4009
  getGeoms,
@@ -3610,7 +4039,7 @@ function MujocoSimProvider({
3610
4039
  setGeomSize,
3611
4040
  mjModelRef,
3612
4041
  mjDataRef
3613
- }),
4042
+ }, () => mjModelRef.current),
3614
4043
  [
3615
4044
  status,
3616
4045
  reset,
@@ -3705,6 +4134,45 @@ function MujocoSimProvider({
3705
4134
  /* @__PURE__ */ jsx(CameraViewportProvider, { children })
3706
4135
  ] });
3707
4136
  }
4137
+ var runtimeMutators = /* @__PURE__ */ new Set([
4138
+ "reset",
4139
+ "setSpeed",
4140
+ "togglePause",
4141
+ "setPaused",
4142
+ "step",
4143
+ "applyKeyframe",
4144
+ "saveState",
4145
+ "restoreState",
4146
+ "setQpos",
4147
+ "setQvel",
4148
+ "setCtrl",
4149
+ "applyForce",
4150
+ "applyTorque",
4151
+ "setExternalForce",
4152
+ "applyGeneralizedForce",
4153
+ "setGravity",
4154
+ "setTimestep",
4155
+ "setBodyMass",
4156
+ "setGeomFriction",
4157
+ "setGeomSize",
4158
+ "loadScene",
4159
+ "loadFromFiles",
4160
+ "addBody",
4161
+ "removeBody",
4162
+ "recompile",
4163
+ "recordCameraSequence"
4164
+ ]);
4165
+ function guardRuntimeOwnership(api, model) {
4166
+ for (const key of runtimeMutators) {
4167
+ const record = api;
4168
+ const method = record[key];
4169
+ record[key] = (...args) => {
4170
+ assertMainThreadOwnership(model());
4171
+ return method(...args);
4172
+ };
4173
+ }
4174
+ return api;
4175
+ }
3708
4176
  var MujocoCanvas = forwardRef(
3709
4177
  function MujocoCanvas2({
3710
4178
  config,
@@ -3717,6 +4185,7 @@ var MujocoCanvas = forwardRef(
3717
4185
  timestep,
3718
4186
  integrator,
3719
4187
  substeps,
4188
+ physicsBudgetMs,
3720
4189
  paused,
3721
4190
  speed,
3722
4191
  interpolate,
@@ -3754,6 +4223,7 @@ var MujocoCanvas = forwardRef(
3754
4223
  timestep,
3755
4224
  integrator,
3756
4225
  substeps,
4226
+ physicsBudgetMs,
3757
4227
  paused,
3758
4228
  speed,
3759
4229
  interpolate,
@@ -5248,10 +5718,10 @@ function useCameraStream(canvasRef, options) {
5248
5718
  if (!api || !canvasRef.current) return;
5249
5719
  const opts = optionsRef.current;
5250
5720
  if (opts.paused) return;
5251
- if (opts.fps && opts.fps > 0) {
5252
- elapsedRef.current += delta;
5253
- if (elapsedRef.current < 1 / opts.fps) return;
5254
- }
5721
+ const fps = opts.fps ?? 10;
5722
+ if (!Number.isFinite(fps) || fps <= 0) throw new Error("Camera stream fps must be positive and finite");
5723
+ elapsedRef.current += delta;
5724
+ if (elapsedRef.current < 1 / fps) return;
5255
5725
  if (inFlightRef.current) return;
5256
5726
  elapsedRef.current = 0;
5257
5727
  const signature = streamSignature(opts);
@@ -5272,7 +5742,10 @@ function useCameraStream(canvasRef, options) {
5272
5742
  canvas.height = frame.height;
5273
5743
  }
5274
5744
  ctx.drawImage(frame.canvas, 0, 0);
5275
- }).catch(() => {
5745
+ }).catch((error) => {
5746
+ const failure = error instanceof Error ? error : new Error(String(error));
5747
+ if (optionsRef.current.onError) optionsRef.current.onError(failure);
5748
+ else console.error("Camera stream capture failed", failure);
5276
5749
  }).finally(() => {
5277
5750
  inFlightRef.current = false;
5278
5751
  });
@@ -7407,7 +7880,9 @@ function isPolicyActionChunk(value) {
7407
7880
  function toPolicyActions(output) {
7408
7881
  return isPolicyActionChunk(output) ? [...output] : [output];
7409
7882
  }
7410
- function enqueuePolicyActions(queue, actions, observation, strategy) {
7883
+ function enqueuePolicyActions(queue, actions, observation, strategy, limit = 256) {
7884
+ if (!Number.isSafeInteger(limit) || limit < 1) throw new Error("maxQueuedActions must be a positive safe integer");
7885
+ if ((strategy === "replace" ? 0 : queue.length) + actions.length > limit) throw new Error(`Policy action queue exceeds maxQueuedActions (${limit})`);
7411
7886
  if (strategy === "replace") {
7412
7887
  queue.splice(0, queue.length);
7413
7888
  }
@@ -7428,7 +7903,6 @@ function usePolicy(config) {
7428
7903
  const clearQueue = useCallback(() => {
7429
7904
  epochRef.current += 1;
7430
7905
  actionQueueRef.current.splice(0, actionQueueRef.current.length);
7431
- inFlightRef.current = false;
7432
7906
  lastErrorRef.current = null;
7433
7907
  }, []);
7434
7908
  const reset = useCallback(() => {
@@ -7437,6 +7911,13 @@ function usePolicy(config) {
7437
7911
  lastObservationRef.current = null;
7438
7912
  lastActionRef.current = null;
7439
7913
  }, [clearQueue]);
7914
+ useEffect(() => {
7915
+ isRunningRef.current = configRef.current.enabled ?? true;
7916
+ return () => {
7917
+ epochRef.current++;
7918
+ isRunningRef.current = false;
7919
+ };
7920
+ }, []);
7440
7921
  useBeforePhysicsStep(({ model, data }) => {
7441
7922
  if (!isRunningRef.current) return;
7442
7923
  const cfg = configRef.current;
@@ -7469,7 +7950,8 @@ function usePolicy(config) {
7469
7950
  actionQueueRef.current,
7470
7951
  toPolicyActions(output),
7471
7952
  observation,
7472
- cfg.queueStrategy ?? "append"
7953
+ cfg.queueStrategy ?? "append",
7954
+ cfg.maxQueuedActions
7473
7955
  );
7474
7956
  lastErrorRef.current = null;
7475
7957
  }).catch((error) => {
@@ -7477,7 +7959,6 @@ function usePolicy(config) {
7477
7959
  lastErrorRef.current = error;
7478
7960
  cfg.onError?.(error);
7479
7961
  }).finally(() => {
7480
- if (epoch !== epochRef.current) return;
7481
7962
  inFlightRef.current = false;
7482
7963
  });
7483
7964
  } else {
@@ -7487,7 +7968,8 @@ function usePolicy(config) {
7487
7968
  actionQueueRef.current,
7488
7969
  actions,
7489
7970
  observation,
7490
- cfg.queueStrategy ?? "append"
7971
+ cfg.queueStrategy ?? "append",
7972
+ cfg.maxQueuedActions
7491
7973
  );
7492
7974
  } else {
7493
7975
  const [action, ...queuedActions2] = actions;
@@ -7496,7 +7978,8 @@ function usePolicy(config) {
7496
7978
  actionQueueRef.current,
7497
7979
  queuedActions2,
7498
7980
  observation,
7499
- cfg.queueStrategy ?? "append"
7981
+ cfg.queueStrategy ?? "append",
7982
+ cfg.maxQueuedActions
7500
7983
  );
7501
7984
  cfg.onAction({ action, observation, model, data });
7502
7985
  lastActionRef.current = action;
@@ -8197,125 +8680,6 @@ function useCameraFrameCapture(defaultOptions = {}) {
8197
8680
  reset
8198
8681
  };
8199
8682
  }
8200
-
8201
- // src/policyCameraFrames.ts
8202
- function addPolicyImageAliases(images, stream, frame, includeObservationImageAliases) {
8203
- const keys = /* @__PURE__ */ new Set();
8204
- keys.add(stream.key);
8205
- for (const alias of stream.aliases ?? []) keys.add(alias);
8206
- if (includeObservationImageAliases) {
8207
- keys.add(`observation.images.${stream.key}`);
8208
- for (const alias of stream.aliases ?? []) {
8209
- keys.add(`observation.images.${alias}`);
8210
- }
8211
- }
8212
- for (const key of keys) {
8213
- images[key] = frame.dataUrl;
8214
- }
8215
- }
8216
- function describeFrameSource(key, frame) {
8217
- return `${key}:${frame.source.kind}`;
8218
- }
8219
- function hasExplicitPolicyCameraSource(options) {
8220
- return Boolean(
8221
- options?.camera || options?.position || options?.quaternion || options?.source
8222
- );
8223
- }
8224
- function createPolicyCameraFrameCapturePlan(options) {
8225
- const {
8226
- cameraKeys,
8227
- defaults,
8228
- streamOptions,
8229
- includeObservationImageAliases,
8230
- requireAll,
8231
- ...sourceOptions
8232
- } = options;
8233
- const mountedPlan = createMountedCameraFrameSequencePlan(cameraKeys, {
8234
- ...sourceOptions,
8235
- defaults,
8236
- cameraOptions: streamOptions
8237
- });
8238
- const streams = [];
8239
- const missingKeys = new Set(mountedPlan.missingKeys);
8240
- for (const key of cameraKeys) {
8241
- const perStreamOptions = streamOptions?.[key];
8242
- if (hasExplicitPolicyCameraSource(perStreamOptions)) {
8243
- missingKeys.delete(key);
8244
- streams.push({
8245
- ...defaults,
8246
- ...perStreamOptions,
8247
- key,
8248
- aliases: perStreamOptions?.aliases
8249
- });
8250
- continue;
8251
- }
8252
- const mountedCamera = mountedPlan.cameras.find((camera) => camera.key === key);
8253
- if (!mountedCamera) continue;
8254
- const { key: _mountedKey, ...captureOptions } = mountedCamera;
8255
- streams.push({
8256
- ...captureOptions,
8257
- key,
8258
- aliases: perStreamOptions?.aliases
8259
- });
8260
- }
8261
- const result = {
8262
- cameraKeys: [...cameraKeys],
8263
- streams,
8264
- includeObservationImageAliases,
8265
- mountedPlan,
8266
- missingKeys: [...missingKeys]
8267
- };
8268
- if (requireAll && result.missingKeys.length > 0) {
8269
- throw new Error(
8270
- `Unable to resolve policy camera stream${result.missingKeys.length === 1 ? "" : "s"} for ${result.missingKeys.join(", ")}.`
8271
- );
8272
- }
8273
- return result;
8274
- }
8275
- function createPolicyCameraFrameCapturePlanFromApi(api, options) {
8276
- return createPolicyCameraFrameCapturePlan({
8277
- ...options,
8278
- cameras: api.getCameras(),
8279
- sites: api.getSites(),
8280
- bodies: api.getBodies()
8281
- });
8282
- }
8283
- async function capturePolicyCameraFrames(target, options) {
8284
- const includeObservationImageAliases = options.includeObservationImageAliases ?? true;
8285
- const entries = await Promise.all(
8286
- options.streams.map(async ({ key, aliases, ...captureOptions }) => {
8287
- const frame = await target.captureCameraFrame(captureOptions);
8288
- return [key, { frame, aliases }];
8289
- })
8290
- );
8291
- const frames = {};
8292
- const images = {};
8293
- const sourceParts = [];
8294
- for (const [key, { frame, aliases }] of entries) {
8295
- const stream = { key, aliases };
8296
- frames[key] = frame;
8297
- addPolicyImageAliases(
8298
- images,
8299
- stream,
8300
- frame,
8301
- includeObservationImageAliases
8302
- );
8303
- sourceParts.push(describeFrameSource(key, frame));
8304
- }
8305
- return {
8306
- frames,
8307
- images,
8308
- sourceSummary: sourceParts.length > 0 ? sourceParts.join(" + ") : "not used by policy",
8309
- capturedAt: Date.now()
8310
- };
8311
- }
8312
- async function capturePolicyCameraFramesFromMountedStreams(target, options) {
8313
- const plan = createPolicyCameraFrameCapturePlanFromApi(target, options);
8314
- const result = await capturePolicyCameraFrames(target, plan);
8315
- return { ...result, plan };
8316
- }
8317
-
8318
- // src/hooks/usePolicyCameraFrames.ts
8319
8683
  function mergePolicyCameraFrameCaptureOptions(defaultOptions, options) {
8320
8684
  return {
8321
8685
  ...defaultOptions,
@@ -8838,6 +9202,18 @@ function useCameraAnimation() {
8838
9202
  * @license
8839
9203
  * SPDX-License-Identifier: Apache-2.0
8840
9204
  */
9205
+ /**
9206
+ * @license
9207
+ * SPDX-License-Identifier: Apache-2.0
9208
+ *
9209
+ * Helpers for resolving dataset camera streams to mounted MuJoCo resources.
9210
+ */
9211
+ /**
9212
+ * @license
9213
+ * SPDX-License-Identifier: Apache-2.0
9214
+ *
9215
+ * Helpers for turning Three/MuJoCo camera captures into policy image payloads.
9216
+ */
8841
9217
  /**
8842
9218
  * @license
8843
9219
  * SPDX-License-Identifier: Apache-2.0
@@ -8861,12 +9237,6 @@ function useCameraAnimation() {
8861
9237
  *
8862
9238
  * useFrameCapture — still-frame capture for canvas-backed MuJoCo/R3F scenes.
8863
9239
  */
8864
- /**
8865
- * @license
8866
- * SPDX-License-Identifier: Apache-2.0
8867
- *
8868
- * Helpers for resolving dataset camera streams to mounted MuJoCo resources.
8869
- */
8870
9240
  /**
8871
9241
  * @license
8872
9242
  * SPDX-License-Identifier: Apache-2.0
@@ -8907,16 +9277,9 @@ function useCameraAnimation() {
8907
9277
  * @license
8908
9278
  * SPDX-License-Identifier: Apache-2.0
8909
9279
  *
8910
- * Stream a live MuJoCo camera into a DOM `<canvas>`. Each frame the scene is
8911
- * rendered offscreen from the selected camera and blitted into the canvas, so
8912
- * it composites normally in the DOM (works inside opaque panels) and does NOT
8913
- * take over the render loop. Prefer this over `useCameraViewport` for camera
8914
- * tiles embedded in HTML UI; use `useCameraViewport` for transparent overlays
8915
- * on a full-bleed canvas.
8916
- *
8917
- * Uses the async capture path so Gaussian-splat environments render through
8918
- * their dedicated capture renderer — streaming a splat scene at full rate does
8919
- * not disturb the main view's splat sort.
9280
+ * Readback stream for DOM compositing/encoding. For interactive camera previews,
9281
+ * use CameraView/useCameraViewport: they render on the GPU with no readback.
9282
+ * This path defaults to 10 FPS and permits only one capture in flight.
8920
9283
  */
8921
9284
  /**
8922
9285
  * @license
@@ -9071,12 +9434,6 @@ function useCameraAnimation() {
9071
9434
  *
9072
9435
  * React state wrapper around MuJoCo/R3F offscreen camera-frame capture.
9073
9436
  */
9074
- /**
9075
- * @license
9076
- * SPDX-License-Identifier: Apache-2.0
9077
- *
9078
- * Helpers for turning Three/MuJoCo camera captures into policy image payloads.
9079
- */
9080
9437
  /**
9081
9438
  * @license
9082
9439
  * SPDX-License-Identifier: Apache-2.0