mujoco-react 12.0.0 → 12.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -4
- package/dist/chunk-U2Y2BODT.js +137 -0
- package/dist/chunk-U2Y2BODT.js.map +1 -0
- package/dist/{chunk-7BCIHJMW.js → chunk-XD3AY7XZ.js} +47 -9
- package/dist/chunk-XD3AY7XZ.js.map +1 -0
- package/dist/index.d.ts +10 -15
- package/dist/index.js +1437 -1090
- package/dist/index.js.map +1 -1
- package/dist/onnx.d.ts +1 -1
- package/dist/physics.worker.d.ts +2 -0
- package/dist/physics.worker.js +78 -0
- package/dist/physics.worker.js.map +1 -0
- package/dist/spark.d.ts +1 -1
- package/dist/spark.js +1 -1
- package/dist/{types-BjHt_Urb.d.ts → types-CSi3fIVs.d.ts} +70 -1
- package/dist/vite.d.ts +5 -2
- package/dist/vite.js +6 -2
- package/dist/vite.js.map +1 -1
- package/package.json +5 -2
- package/src/components/CameraView.tsx +9 -7
- package/src/core/MujocoCanvas.tsx +2 -0
- package/src/core/MujocoPhysics.tsx +1 -0
- package/src/core/MujocoSimProvider.tsx +103 -81
- package/src/core/sceneIdentity.ts +15 -0
- package/src/core/stepBudget.ts +18 -0
- package/src/hooks/useCameraStream.ts +14 -17
- package/src/hooks/usePolicy.ts +16 -7
- package/src/index.ts +3 -0
- package/src/physics.worker.ts +13 -0
- package/src/rendering/cameraFrameCapture.ts +33 -7
- package/src/runtime/client.ts +186 -0
- package/src/runtime/engine.ts +93 -0
- package/src/runtime/protocol.ts +59 -0
- package/src/runtime/types.ts +51 -0
- package/src/types.ts +11 -0
- package/src/vite.ts +7 -2
- package/dist/chunk-7BCIHJMW.js.map +0 -1
package/dist/index.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
import {
|
|
2
|
-
|
|
1
|
+
import { displayFields } from './chunk-U2Y2BODT.js';
|
|
2
|
+
import { withContacts, getContact, captureCameraFrame, captureCameraFrameBlob, createCameraFrameCaptureSession, CAMERA_FRAME_CAPTURE_PRE_RENDER_USER_DATA_KEY, CAPTURE_EXCLUDE_KEY, createCaptureCamera, prepareCaptureCamera } from './chunk-XD3AY7XZ.js';
|
|
3
|
+
export { CAMERA_FRAME_CAPTURE_PRE_RENDER_USER_DATA_KEY, CAMERA_FRAME_CAPTURE_RENDER_USER_DATA_KEY, CAPTURE_EXCLUDE_KEY, ModelActuators, ModelBodies, ModelCameras, ModelGeoms, ModelJoints, ModelKeyframes, ModelResources, ModelSensors, ModelSites, ScenarioLighting, SplatEnvironment, SplatEnvironmentReadinessStatus, VisualScenarioEffects, captureCameraFrame, captureCameraFrameBlob, captureCameraFrameTensor, createCameraFrameCaptureSession, createPairedSplatEnvironment, createSparkSplatViewerUrl, createSplatEnvironmentUserData, createSplatSceneConfig, createVisualScenarioExecutionContext, dataUrlToPolicyImageTensor, getContact, getScenarioBackground, getScenarioCameraPosition, getSplatEnvironmentReadiness, imageDataToPolicyImageTensor, pixelsToPolicyImageTensor, registerModelResources, renderCameraFrameToCanvas, useSplatEnvironment, useSplatSceneConfig, useVisualScenarioEffects, useVisualScenarioExecutionContext, withSplatEnvironment } from './chunk-XD3AY7XZ.js';
|
|
3
4
|
import loadMujoco from '@mujoco/mujoco';
|
|
4
5
|
import defaultMujocoWasmUrl from '@mujoco/mujoco/mujoco.wasm?url';
|
|
5
6
|
import { createContext, forwardRef, useRef, useEffect, useContext, useState, useCallback, useMemo, useLayoutEffect, useId } from 'react';
|
|
@@ -975,939 +976,1322 @@ function collectDependencyPaths(xmlString, currentFile, parser) {
|
|
|
975
976
|
});
|
|
976
977
|
return paths;
|
|
977
978
|
}
|
|
978
|
-
var CapsuleGeometry = class extends THREE12.BufferGeometry {
|
|
979
|
-
parameters;
|
|
980
|
-
constructor(radius = 1, length = 1, capSegments = 4, radialSegments = 8) {
|
|
981
|
-
super();
|
|
982
|
-
this.type = "CapsuleGeometry";
|
|
983
|
-
this.parameters = { radius, length, capSegments, radialSegments };
|
|
984
|
-
const path = new THREE12.Path();
|
|
985
|
-
path.absarc(0, -length / 2, radius, Math.PI * 1.5, 0, false);
|
|
986
|
-
path.absarc(0, length / 2, radius, 0, Math.PI * 0.5, false);
|
|
987
|
-
const latheGeometry = new THREE12.LatheGeometry(path.getPoints(capSegments), radialSegments);
|
|
988
|
-
const self = this;
|
|
989
|
-
self.setIndex(latheGeometry.getIndex());
|
|
990
|
-
self.setAttribute("position", latheGeometry.getAttribute("position"));
|
|
991
|
-
self.setAttribute("normal", latheGeometry.getAttribute("normal"));
|
|
992
|
-
self.setAttribute("uv", latheGeometry.getAttribute("uv"));
|
|
993
|
-
}
|
|
994
|
-
};
|
|
995
979
|
|
|
996
|
-
// src/
|
|
997
|
-
var
|
|
998
|
-
var
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
980
|
+
// src/core/sceneIdentity.ts
|
|
981
|
+
var fileIds = /* @__PURE__ */ new WeakMap();
|
|
982
|
+
var nextFileId = 0;
|
|
983
|
+
function sceneIdentity(config) {
|
|
984
|
+
return JSON.stringify([
|
|
985
|
+
config.src,
|
|
986
|
+
config.sceneFile,
|
|
987
|
+
config.environmentFiles ?? [],
|
|
988
|
+
config.sceneObjects ?? [],
|
|
989
|
+
config.homeJoints ?? [],
|
|
990
|
+
config.xmlPatches ?? [],
|
|
991
|
+
(config.files ?? []).map((file) => {
|
|
992
|
+
if (!fileIds.has(file)) fileIds.set(file, ++nextFileId);
|
|
993
|
+
return fileIds.get(file);
|
|
994
|
+
})
|
|
995
|
+
], (_key, value) => value && typeof value === "object" && !Array.isArray(value) ? Object.fromEntries(Object.entries(value).sort(([a], [b]) => a.localeCompare(b))) : value);
|
|
996
|
+
}
|
|
997
|
+
|
|
998
|
+
// src/core/stepBudget.ts
|
|
999
|
+
function runStepBudget(requested, step, budgetMs = 8, now2 = () => performance.now()) {
|
|
1000
|
+
if (!Number.isFinite(budgetMs) || budgetMs <= 0) throw new Error("physicsBudgetMs must be positive and finite");
|
|
1001
|
+
const start = now2();
|
|
1002
|
+
let steps = 0;
|
|
1003
|
+
while (steps < requested && steps < 100) {
|
|
1004
|
+
step();
|
|
1005
|
+
steps++;
|
|
1006
|
+
if (now2() - start >= budgetMs) break;
|
|
1007
|
+
}
|
|
1008
|
+
return { steps, physicsMs: now2() - start, budgetExceeded: steps < requested };
|
|
1009
|
+
}
|
|
1010
|
+
|
|
1011
|
+
// src/rendering/cameraFrameSource.ts
|
|
1012
|
+
var MountedCameraFrameSourceSuggestionMatch = {
|
|
1013
|
+
Direct: "direct",
|
|
1014
|
+
Alias: "alias",
|
|
1015
|
+
Normalized: "normalized",
|
|
1016
|
+
Prefix: "prefix",
|
|
1017
|
+
Suffix: "suffix",
|
|
1018
|
+
Contains: "contains"
|
|
1019
|
+
};
|
|
1020
|
+
var MountedCameraFrameSequenceReadinessStatus = {
|
|
1021
|
+
Ready: "ready",
|
|
1022
|
+
Partial: "partial",
|
|
1023
|
+
Missing: "missing"
|
|
1024
|
+
};
|
|
1025
|
+
var MountedCameraFrameSequenceManifestStatus = {
|
|
1026
|
+
Complete: "complete",
|
|
1027
|
+
Partial: "partial",
|
|
1028
|
+
Missing: "missing"
|
|
1029
|
+
};
|
|
1030
|
+
function getResourceName(resource) {
|
|
1031
|
+
if (!resource) return null;
|
|
1032
|
+
return typeof resource === "string" ? resource : resource.name ?? null;
|
|
1033
|
+
}
|
|
1034
|
+
function createNameSet(resources) {
|
|
1035
|
+
return new Set(
|
|
1036
|
+
(resources ?? []).map((resource) => getResourceName(resource)).filter((name) => Boolean(name))
|
|
1037
|
+
);
|
|
1038
|
+
}
|
|
1039
|
+
function createResourceNames(resources) {
|
|
1040
|
+
return (resources ?? []).map((resource) => getResourceName(resource)).filter((name) => Boolean(name));
|
|
1041
|
+
}
|
|
1042
|
+
function normalizeAliasCandidates(value) {
|
|
1043
|
+
if (!value) return [];
|
|
1044
|
+
return Array.isArray(value) ? value : [value];
|
|
1045
|
+
}
|
|
1046
|
+
function countMountedSelectors(selector) {
|
|
1047
|
+
return Number(Boolean(selector.cameraName)) + Number(Boolean(selector.siteName)) + Number(Boolean(selector.bodyName));
|
|
1048
|
+
}
|
|
1049
|
+
function normalizeCameraSourceName(value) {
|
|
1050
|
+
return value.trim().toLowerCase().replace(/[^a-z0-9]+/g, "_").replace(/^_+|_+$/g, "");
|
|
1051
|
+
}
|
|
1052
|
+
function createCameraSourceKeyVariants(key) {
|
|
1053
|
+
const candidates = [
|
|
1054
|
+
key,
|
|
1055
|
+
key.startsWith("observation.images.") ? key.slice("observation.images.".length) : "",
|
|
1056
|
+
key.includes(".") ? key.split(".").at(-1) ?? "" : "",
|
|
1057
|
+
key.includes("/") ? key.split("/").at(-1) ?? "" : ""
|
|
1058
|
+
];
|
|
1059
|
+
return candidates.map((candidate) => candidate.trim()).filter((candidate, index, items) => candidate && items.indexOf(candidate) === index);
|
|
1060
|
+
}
|
|
1061
|
+
function getSelectorKey(selector) {
|
|
1062
|
+
if (selector.cameraName) return `camera:${selector.cameraName}`;
|
|
1063
|
+
if (selector.siteName) return `site:${selector.siteName}`;
|
|
1064
|
+
if (selector.bodyName) return `body:${selector.bodyName}`;
|
|
1065
|
+
return null;
|
|
1066
|
+
}
|
|
1067
|
+
function getMountedCameraFrameCaptureSource(selector) {
|
|
1068
|
+
if (countMountedSelectors(selector) !== 1) return null;
|
|
1069
|
+
if (selector.cameraName) {
|
|
1070
|
+
return { kind: "mujoco-camera", cameraName: selector.cameraName };
|
|
1005
1071
|
}
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
if (!smoothing) return null;
|
|
1009
|
-
if (smoothing === true) return DEFAULT_MESH_NORMAL_SMOOTHING_TOLERANCE;
|
|
1010
|
-
return smoothing.tolerance ?? DEFAULT_MESH_NORMAL_SMOOTHING_TOLERANCE;
|
|
1072
|
+
if (selector.siteName) {
|
|
1073
|
+
return { kind: "mujoco-site", siteName: selector.siteName };
|
|
1011
1074
|
}
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1075
|
+
if (selector.bodyName) {
|
|
1076
|
+
return { kind: "mujoco-body", bodyName: selector.bodyName };
|
|
1077
|
+
}
|
|
1078
|
+
return null;
|
|
1079
|
+
}
|
|
1080
|
+
function isMountedCameraFrameCaptureSource(source) {
|
|
1081
|
+
return source.kind === "mujoco-camera" || source.kind === "mujoco-site" || source.kind === "mujoco-body";
|
|
1082
|
+
}
|
|
1083
|
+
function getCameraFrameCaptureSourceTarget(source) {
|
|
1084
|
+
if (source.kind === "mujoco-camera") return source.cameraName;
|
|
1085
|
+
if (source.kind === "mujoco-site") return source.siteName;
|
|
1086
|
+
if (source.kind === "mujoco-body") return source.bodyName;
|
|
1087
|
+
if (source.kind === "custom-camera") return "custom camera";
|
|
1088
|
+
if (source.kind === "explicit-pose") return "explicit pose";
|
|
1089
|
+
return "fallback camera";
|
|
1090
|
+
}
|
|
1091
|
+
function createMountedCameraFrameSourceSuggestion(key, selector, resourceName, match) {
|
|
1092
|
+
const source = getMountedCameraFrameCaptureSource(selector);
|
|
1093
|
+
if (!source) return null;
|
|
1094
|
+
return {
|
|
1095
|
+
key,
|
|
1096
|
+
selector,
|
|
1097
|
+
source,
|
|
1098
|
+
resourceName,
|
|
1099
|
+
resourceKind: source.kind,
|
|
1100
|
+
match
|
|
1101
|
+
};
|
|
1102
|
+
}
|
|
1103
|
+
function addMountedCameraFrameSourceSuggestion(suggestions, seen, suggestion) {
|
|
1104
|
+
if (!suggestion) return;
|
|
1105
|
+
const selectorKey = getSelectorKey(suggestion.selector);
|
|
1106
|
+
if (!selectorKey || seen.has(selectorKey)) return;
|
|
1107
|
+
seen.add(selectorKey);
|
|
1108
|
+
suggestions.push(suggestion);
|
|
1109
|
+
}
|
|
1110
|
+
function getCameraFrameResourceMatch(key, resourceName) {
|
|
1111
|
+
if (resourceName === key) return MountedCameraFrameSourceSuggestionMatch.Direct;
|
|
1112
|
+
const normalizedResource = normalizeCameraSourceName(resourceName);
|
|
1113
|
+
if (!normalizedResource) return null;
|
|
1114
|
+
for (const variant of createCameraSourceKeyVariants(key)) {
|
|
1115
|
+
if (resourceName === variant) return MountedCameraFrameSourceSuggestionMatch.Direct;
|
|
1116
|
+
const normalizedKey = normalizeCameraSourceName(variant);
|
|
1117
|
+
if (!normalizedKey) continue;
|
|
1118
|
+
if (normalizedResource === normalizedKey) {
|
|
1119
|
+
return MountedCameraFrameSourceSuggestionMatch.Normalized;
|
|
1023
1120
|
}
|
|
1024
|
-
if (
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
const source = mjModel.tex_data.subarray(offset, offset + width * height * channels);
|
|
1033
|
-
const rgba = new Uint8Array(width * height * 4);
|
|
1034
|
-
for (let i = 0, j = 0; i < width * height; i += 1, j += channels) {
|
|
1035
|
-
const r = source[j] ?? 255;
|
|
1036
|
-
const g = channels > 1 ? source[j + 1] : r;
|
|
1037
|
-
const b = channels > 2 ? source[j + 2] : r;
|
|
1038
|
-
const a = channels > 3 ? source[j + 3] : 255;
|
|
1039
|
-
const out = i * 4;
|
|
1040
|
-
rgba[out] = r;
|
|
1041
|
-
rgba[out + 1] = g;
|
|
1042
|
-
rgba[out + 2] = b;
|
|
1043
|
-
rgba[out + 3] = a;
|
|
1121
|
+
if (normalizedResource.startsWith(`${normalizedKey}_`)) {
|
|
1122
|
+
return MountedCameraFrameSourceSuggestionMatch.Prefix;
|
|
1123
|
+
}
|
|
1124
|
+
if (normalizedResource.endsWith(`_${normalizedKey}`)) {
|
|
1125
|
+
return MountedCameraFrameSourceSuggestionMatch.Suffix;
|
|
1126
|
+
}
|
|
1127
|
+
if (normalizedResource.includes(`_${normalizedKey}_`)) {
|
|
1128
|
+
return MountedCameraFrameSourceSuggestionMatch.Contains;
|
|
1044
1129
|
}
|
|
1045
|
-
const texture = new THREE12.DataTexture(rgba, width, height, THREE12.RGBAFormat);
|
|
1046
|
-
texture.colorSpace = THREE12.LinearSRGBColorSpace;
|
|
1047
|
-
texture.wrapS = THREE12.RepeatWrapping;
|
|
1048
|
-
texture.wrapT = THREE12.RepeatWrapping;
|
|
1049
|
-
texture.flipY = true;
|
|
1050
|
-
const repeatOffset = matId * 2;
|
|
1051
|
-
const repeatS = mjModel.mat_texrepeat?.[repeatOffset] ?? 1;
|
|
1052
|
-
const repeatT = mjModel.mat_texrepeat?.[repeatOffset + 1] ?? 1;
|
|
1053
|
-
texture.repeat.set(repeatS || 1, repeatT || 1);
|
|
1054
|
-
texture.needsUpdate = true;
|
|
1055
|
-
this.textureCache.set(texId, texture);
|
|
1056
|
-
return texture;
|
|
1057
1130
|
}
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
const rgba = mjModel.mat_rgba.subarray(matId * 4, matId * 4 + 4);
|
|
1074
|
-
color.setRGB(rgba[0], rgba[1], rgba[2]);
|
|
1075
|
-
opacity = rgba[3];
|
|
1076
|
-
} else {
|
|
1077
|
-
const rgba = mjModel.geom_rgba.subarray(g * 4, g * 4 + 4);
|
|
1078
|
-
color.setRGB(rgba[0], rgba[1], rgba[2]);
|
|
1079
|
-
opacity = rgba[3];
|
|
1131
|
+
return null;
|
|
1132
|
+
}
|
|
1133
|
+
function isSelectorMounted(selector, cameraNames, siteNames, bodyNames) {
|
|
1134
|
+
if (countMountedSelectors(selector) !== 1) return false;
|
|
1135
|
+
return (selector.cameraName ? cameraNames.has(selector.cameraName) : false) || (selector.siteName ? siteNames.has(selector.siteName) : false) || (selector.bodyName ? bodyNames.has(selector.bodyName) : false);
|
|
1136
|
+
}
|
|
1137
|
+
function createMountedCameraFrameSourceSuggestions(key, options) {
|
|
1138
|
+
const cameraNames = createNameSet(options.cameras);
|
|
1139
|
+
const siteNames = createNameSet(options.sites);
|
|
1140
|
+
const bodyNames = createNameSet(options.bodies);
|
|
1141
|
+
const suggestions = [];
|
|
1142
|
+
const seen = /* @__PURE__ */ new Set();
|
|
1143
|
+
for (const selector of normalizeAliasCandidates(options.aliases?.[key])) {
|
|
1144
|
+
if (!isSelectorMounted(selector, cameraNames, siteNames, bodyNames)) {
|
|
1145
|
+
continue;
|
|
1080
1146
|
}
|
|
1081
|
-
const
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
} else if (type === getVal(MG.mjGEOM_CYLINDER)) {
|
|
1094
|
-
geo = new THREE12.CylinderGeometry(size[0], size[0], size[1] * 2, 24);
|
|
1095
|
-
geo.rotateX(Math.PI / 2);
|
|
1096
|
-
} else if (type === getVal(MG.mjGEOM_MESH)) {
|
|
1097
|
-
const mId = mjModel.geom_dataid[g];
|
|
1098
|
-
const vAdr = mjModel.mesh_vertadr[mId];
|
|
1099
|
-
const vNum = mjModel.mesh_vertnum[mId];
|
|
1100
|
-
const fAdr = mjModel.mesh_faceadr[mId];
|
|
1101
|
-
const fNum = mjModel.mesh_facenum[mId];
|
|
1102
|
-
const tAdr = mjModel.mesh_texcoordadr?.[mId] ?? -1;
|
|
1103
|
-
if (map && tAdr >= 0 && mjModel.mesh_texcoord && mjModel.mesh_facetexcoord) {
|
|
1104
|
-
const corners = fNum * 3;
|
|
1105
|
-
const positions = new Float32Array(corners * 3);
|
|
1106
|
-
const uvs = new Float32Array(corners * 2);
|
|
1107
|
-
for (let c = 0; c < corners; c++) {
|
|
1108
|
-
const v = vAdr + mjModel.mesh_face[fAdr * 3 + c];
|
|
1109
|
-
positions[c * 3] = mjModel.mesh_vert[v * 3];
|
|
1110
|
-
positions[c * 3 + 1] = mjModel.mesh_vert[v * 3 + 1];
|
|
1111
|
-
positions[c * 3 + 2] = mjModel.mesh_vert[v * 3 + 2];
|
|
1112
|
-
const t = tAdr + mjModel.mesh_facetexcoord[fAdr * 3 + c];
|
|
1113
|
-
uvs[c * 2] = mjModel.mesh_texcoord[t * 2];
|
|
1114
|
-
uvs[c * 2 + 1] = 1 - mjModel.mesh_texcoord[t * 2 + 1];
|
|
1115
|
-
}
|
|
1116
|
-
geo = new THREE12.BufferGeometry();
|
|
1117
|
-
geo.setAttribute("position", new THREE12.Float32BufferAttribute(positions, 3));
|
|
1118
|
-
geo.setAttribute("uv", new THREE12.Float32BufferAttribute(uvs, 2));
|
|
1119
|
-
geo.computeVertexNormals();
|
|
1120
|
-
} else {
|
|
1121
|
-
geo = new THREE12.BufferGeometry();
|
|
1122
|
-
geo.setAttribute("position", new THREE12.Float32BufferAttribute(mjModel.mesh_vert.subarray(vAdr * 3, (vAdr + vNum) * 3), 3));
|
|
1123
|
-
geo.setIndex(Array.from(mjModel.mesh_face.subarray(fAdr * 3, (fAdr + fNum) * 3)));
|
|
1124
|
-
const smoothingTolerance = this.getMeshNormalSmoothingTolerance();
|
|
1125
|
-
if (smoothingTolerance !== null) {
|
|
1126
|
-
geo = mergeVertices(geo, smoothingTolerance);
|
|
1127
|
-
}
|
|
1128
|
-
geo.computeVertexNormals();
|
|
1129
|
-
}
|
|
1130
|
-
}
|
|
1131
|
-
if (geo) {
|
|
1132
|
-
const isPlane = type === getVal(MG.mjGEOM_PLANE);
|
|
1133
|
-
const materialMap = isPlane && map ? map.clone() : map;
|
|
1134
|
-
if (isPlane && materialMap) {
|
|
1135
|
-
materialMap.repeat.multiplyScalar(2.5);
|
|
1136
|
-
materialMap.needsUpdate = true;
|
|
1137
|
-
}
|
|
1138
|
-
const mesh = new THREE12.Mesh(geo, new THREE12.MeshStandardMaterial({
|
|
1139
|
-
color,
|
|
1140
|
-
map: materialMap,
|
|
1141
|
-
transparent: opacity < 1,
|
|
1142
|
-
opacity,
|
|
1143
|
-
roughness: 0.6,
|
|
1144
|
-
metalness: 0
|
|
1145
|
-
}));
|
|
1146
|
-
mesh.castShadow = type !== getVal(MG.mjGEOM_PLANE);
|
|
1147
|
-
mesh.receiveShadow = true;
|
|
1148
|
-
mesh.position.set(pos[0], pos[1], pos[2]);
|
|
1149
|
-
mesh.quaternion.set(quat[1], quat[2], quat[3], quat[0]);
|
|
1150
|
-
mesh.userData.bodyID = mjModel.geom_bodyid[g];
|
|
1151
|
-
mesh.userData.geomID = g;
|
|
1152
|
-
mesh.userData.geomGroup = mjModel.geom_group[g];
|
|
1153
|
-
mesh.userData.geomName = getName(mjModel, mjModel.name_geomadr[g]);
|
|
1154
|
-
return mesh;
|
|
1155
|
-
}
|
|
1156
|
-
return null;
|
|
1147
|
+
const source = getMountedCameraFrameCaptureSource(selector);
|
|
1148
|
+
if (!source) continue;
|
|
1149
|
+
addMountedCameraFrameSourceSuggestion(
|
|
1150
|
+
suggestions,
|
|
1151
|
+
seen,
|
|
1152
|
+
createMountedCameraFrameSourceSuggestion(
|
|
1153
|
+
key,
|
|
1154
|
+
selector,
|
|
1155
|
+
getCameraFrameCaptureSourceTarget(source),
|
|
1156
|
+
MountedCameraFrameSourceSuggestionMatch.Alias
|
|
1157
|
+
)
|
|
1158
|
+
);
|
|
1157
1159
|
}
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
|
|
1162
|
-
|
|
1163
|
-
|
|
1160
|
+
for (const cameraName of createResourceNames(options.cameras)) {
|
|
1161
|
+
const match = getCameraFrameResourceMatch(key, cameraName);
|
|
1162
|
+
if (!match) continue;
|
|
1163
|
+
addMountedCameraFrameSourceSuggestion(
|
|
1164
|
+
suggestions,
|
|
1165
|
+
seen,
|
|
1166
|
+
createMountedCameraFrameSourceSuggestion(
|
|
1167
|
+
key,
|
|
1168
|
+
{ cameraName },
|
|
1169
|
+
cameraName,
|
|
1170
|
+
match
|
|
1171
|
+
)
|
|
1172
|
+
);
|
|
1173
|
+
}
|
|
1174
|
+
for (const siteName of createResourceNames(options.sites)) {
|
|
1175
|
+
const match = getCameraFrameResourceMatch(key, siteName);
|
|
1176
|
+
if (!match) continue;
|
|
1177
|
+
addMountedCameraFrameSourceSuggestion(
|
|
1178
|
+
suggestions,
|
|
1179
|
+
seen,
|
|
1180
|
+
createMountedCameraFrameSourceSuggestion(
|
|
1181
|
+
key,
|
|
1182
|
+
{ siteName },
|
|
1183
|
+
siteName,
|
|
1184
|
+
match
|
|
1185
|
+
)
|
|
1186
|
+
);
|
|
1187
|
+
}
|
|
1188
|
+
for (const bodyName of createResourceNames(options.bodies)) {
|
|
1189
|
+
const match = getCameraFrameResourceMatch(key, bodyName);
|
|
1190
|
+
if (!match) continue;
|
|
1191
|
+
addMountedCameraFrameSourceSuggestion(
|
|
1192
|
+
suggestions,
|
|
1193
|
+
seen,
|
|
1194
|
+
createMountedCameraFrameSourceSuggestion(
|
|
1195
|
+
key,
|
|
1196
|
+
{ bodyName },
|
|
1197
|
+
bodyName,
|
|
1198
|
+
match
|
|
1199
|
+
)
|
|
1200
|
+
);
|
|
1201
|
+
}
|
|
1202
|
+
return suggestions;
|
|
1164
1203
|
}
|
|
1165
|
-
function
|
|
1166
|
-
const
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
const bodyRefs = useRef([]);
|
|
1179
|
-
const prevModelRef = useRef(null);
|
|
1180
|
-
const prevRenderOptionsKeyRef = useRef(null);
|
|
1181
|
-
const prevHiddenVersionRef = useRef(null);
|
|
1182
|
-
const renderOptionsKey = getRenderOptionsKey(renderOptions);
|
|
1183
|
-
const geomBuilder = useMemo(() => {
|
|
1184
|
-
if (status !== "ready") return null;
|
|
1185
|
-
return new GeomBuilder(mujocoRef.current, renderOptions);
|
|
1186
|
-
}, [status, mujocoRef, renderOptionsKey]);
|
|
1187
|
-
useEffect(() => {
|
|
1188
|
-
if (status !== "ready" || !geomBuilder) return;
|
|
1189
|
-
const model = mjModelRef.current;
|
|
1190
|
-
const group = groupRef.current;
|
|
1191
|
-
if (!model || !group) return;
|
|
1192
|
-
if (prevModelRef.current === model && prevRenderOptionsKeyRef.current === renderOptionsKey && prevHiddenVersionRef.current === hiddenBodiesVersion) {
|
|
1193
|
-
return;
|
|
1194
|
-
}
|
|
1195
|
-
prevModelRef.current = model;
|
|
1196
|
-
prevRenderOptionsKeyRef.current = renderOptionsKey;
|
|
1197
|
-
prevHiddenVersionRef.current = hiddenBodiesVersion;
|
|
1198
|
-
while (group.children.length > 0) {
|
|
1199
|
-
group.remove(group.children[0]);
|
|
1200
|
-
}
|
|
1201
|
-
const refs = [];
|
|
1202
|
-
for (let i = 0; i < model.nbody; i++) {
|
|
1203
|
-
const bodyGroup = new THREE12.Group();
|
|
1204
|
-
bodyGroup.userData.bodyID = i;
|
|
1205
|
-
const bodyName = getName(model, model.name_bodyadr[i]);
|
|
1206
|
-
if (!hiddenBodiesRef.current.has(bodyName)) {
|
|
1207
|
-
for (let g = 0; g < model.ngeom; g++) {
|
|
1208
|
-
if (model.geom_bodyid[g] === i) {
|
|
1209
|
-
const mesh = geomBuilder.create(model, g);
|
|
1210
|
-
if (mesh) bodyGroup.add(mesh);
|
|
1211
|
-
}
|
|
1212
|
-
}
|
|
1213
|
-
}
|
|
1214
|
-
group.add(bodyGroup);
|
|
1215
|
-
refs.push(bodyGroup);
|
|
1216
|
-
}
|
|
1217
|
-
bodyRefs.current = refs;
|
|
1218
|
-
}, [status, geomBuilder, mjModelRef, renderOptionsKey, hiddenBodiesRef, hiddenBodiesVersion]);
|
|
1219
|
-
const syncBodiesToData = useCallback(() => {
|
|
1220
|
-
const data = mjDataRef.current;
|
|
1221
|
-
if (!data) return;
|
|
1222
|
-
const bodies = bodyRefs.current;
|
|
1223
|
-
const interpolation = interpolationStateRef.current;
|
|
1224
|
-
const useInterpolation = interpolateRef.current && interpolation.valid;
|
|
1225
|
-
for (let i = 0; i < bodies.length; i++) {
|
|
1226
|
-
const ref = bodies[i];
|
|
1227
|
-
if (!ref) continue;
|
|
1228
|
-
if (useInterpolation) {
|
|
1229
|
-
const alpha = interpolation.alpha;
|
|
1230
|
-
const i3 = i * 3;
|
|
1231
|
-
ref.position.set(
|
|
1232
|
-
THREE12.MathUtils.lerp(interpolation.previousXpos[i3], interpolation.currentXpos[i3], alpha),
|
|
1233
|
-
THREE12.MathUtils.lerp(interpolation.previousXpos[i3 + 1], interpolation.currentXpos[i3 + 1], alpha),
|
|
1234
|
-
THREE12.MathUtils.lerp(interpolation.previousXpos[i3 + 2], interpolation.currentXpos[i3 + 2], alpha)
|
|
1235
|
-
);
|
|
1236
|
-
const i4 = i * 4;
|
|
1237
|
-
_previousQuat.set(
|
|
1238
|
-
interpolation.previousXquat[i4 + 1],
|
|
1239
|
-
interpolation.previousXquat[i4 + 2],
|
|
1240
|
-
interpolation.previousXquat[i4 + 3],
|
|
1241
|
-
interpolation.previousXquat[i4]
|
|
1242
|
-
);
|
|
1243
|
-
_currentQuat.set(
|
|
1244
|
-
interpolation.currentXquat[i4 + 1],
|
|
1245
|
-
interpolation.currentXquat[i4 + 2],
|
|
1246
|
-
interpolation.currentXquat[i4 + 3],
|
|
1247
|
-
interpolation.currentXquat[i4]
|
|
1248
|
-
);
|
|
1249
|
-
ref.quaternion.copy(_previousQuat).slerp(_currentQuat, alpha);
|
|
1250
|
-
} else {
|
|
1251
|
-
ref.position.set(
|
|
1252
|
-
data.xpos[i * 3],
|
|
1253
|
-
data.xpos[i * 3 + 1],
|
|
1254
|
-
data.xpos[i * 3 + 2]
|
|
1255
|
-
);
|
|
1256
|
-
ref.quaternion.set(
|
|
1257
|
-
data.xquat[i * 4 + 1],
|
|
1258
|
-
data.xquat[i * 4 + 2],
|
|
1259
|
-
data.xquat[i * 4 + 3],
|
|
1260
|
-
data.xquat[i * 4]
|
|
1261
|
-
);
|
|
1262
|
-
}
|
|
1263
|
-
}
|
|
1264
|
-
}, [interpolateRef, interpolationStateRef, mjDataRef]);
|
|
1265
|
-
useFrame(syncBodiesToData);
|
|
1266
|
-
return /* @__PURE__ */ jsx(
|
|
1267
|
-
"group",
|
|
1268
|
-
{
|
|
1269
|
-
...props,
|
|
1270
|
-
ref: groupRef,
|
|
1271
|
-
userData: {
|
|
1272
|
-
...props.userData,
|
|
1273
|
-
[CAMERA_FRAME_CAPTURE_PRE_RENDER_USER_DATA_KEY]: syncBodiesToData
|
|
1274
|
-
},
|
|
1275
|
-
onDoubleClick: (e) => {
|
|
1276
|
-
if (typeof props.onDoubleClick === "function") props.onDoubleClick(e);
|
|
1277
|
-
e.stopPropagation();
|
|
1278
|
-
let obj = e.object;
|
|
1279
|
-
while (obj && obj.userData.bodyID === void 0 && obj.parent) {
|
|
1280
|
-
obj = obj.parent;
|
|
1281
|
-
}
|
|
1282
|
-
const bodyID = obj?.userData.bodyID;
|
|
1283
|
-
if (typeof bodyID === "number" && bodyID > 0) {
|
|
1284
|
-
const model = mjModelRef.current;
|
|
1285
|
-
if (model && bodyID < model.nbody && onSelectionRef.current) {
|
|
1286
|
-
const name = getName(model, model.name_bodyadr[bodyID]);
|
|
1287
|
-
onSelectionRef.current({ bodyId: bodyID, name });
|
|
1288
|
-
}
|
|
1289
|
-
}
|
|
1290
|
-
}
|
|
1204
|
+
function resolveMountedCameraFrameSource(key, options) {
|
|
1205
|
+
const cameraNames = createNameSet(options.cameras);
|
|
1206
|
+
const siteNames = createNameSet(options.sites);
|
|
1207
|
+
const bodyNames = createNameSet(options.bodies);
|
|
1208
|
+
const directCandidates = [
|
|
1209
|
+
{ cameraName: key },
|
|
1210
|
+
{ siteName: key },
|
|
1211
|
+
{ bodyName: key }
|
|
1212
|
+
];
|
|
1213
|
+
const aliasCandidates = normalizeAliasCandidates(options.aliases?.[key]);
|
|
1214
|
+
for (const selector of aliasCandidates) {
|
|
1215
|
+
if (!isSelectorMounted(selector, cameraNames, siteNames, bodyNames)) {
|
|
1216
|
+
continue;
|
|
1291
1217
|
}
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
gl.setScissorTest(false);
|
|
1310
|
-
gl.setViewport(0, 0, drawWidth, drawHeight);
|
|
1311
|
-
gl.render(scene, mainCamera);
|
|
1312
|
-
const api = mujoco.api;
|
|
1313
|
-
if (!api || viewportsRef.current.size === 0) return;
|
|
1314
|
-
const dpr = gl.getPixelRatio();
|
|
1315
|
-
const canvasRect = gl.domElement.getBoundingClientRect();
|
|
1316
|
-
for (const descriptor of viewportsRef.current.values()) {
|
|
1317
|
-
const element = descriptor.getElement();
|
|
1318
|
-
if (!element) continue;
|
|
1319
|
-
const rect = element.getBoundingClientRect();
|
|
1320
|
-
const isOffscreen = rect.bottom < canvasRect.top || rect.top > canvasRect.bottom || rect.right < canvasRect.left || rect.left > canvasRect.right;
|
|
1321
|
-
if (isOffscreen) continue;
|
|
1322
|
-
const width = Math.floor(rect.width * dpr);
|
|
1323
|
-
const height = Math.floor(rect.height * dpr);
|
|
1324
|
-
if (width <= 0 || height <= 0) continue;
|
|
1325
|
-
const left = Math.floor((rect.left - canvasRect.left) * dpr);
|
|
1326
|
-
const bottom = Math.floor((canvasRect.bottom - rect.bottom) * dpr);
|
|
1327
|
-
let resolved;
|
|
1328
|
-
try {
|
|
1329
|
-
resolved = api.resolveCameraCaptureOptions(descriptor.getOptions());
|
|
1330
|
-
} catch {
|
|
1331
|
-
continue;
|
|
1332
|
-
}
|
|
1333
|
-
if (!descriptor.camera) {
|
|
1334
|
-
descriptor.camera = createCaptureCamera(resolved, mainCamera, width, height);
|
|
1335
|
-
} else {
|
|
1336
|
-
prepareCaptureCamera(descriptor.camera, resolved, mainCamera, width, height);
|
|
1337
|
-
}
|
|
1338
|
-
gl.setViewport(left, bottom, width, height);
|
|
1339
|
-
gl.setScissor(left, bottom, width, height);
|
|
1340
|
-
gl.setScissorTest(true);
|
|
1341
|
-
gl.render(scene, descriptor.camera);
|
|
1218
|
+
const source = getMountedCameraFrameCaptureSource(selector);
|
|
1219
|
+
if (!source) continue;
|
|
1220
|
+
return { key, selector, source };
|
|
1221
|
+
}
|
|
1222
|
+
const [suggestion] = createMountedCameraFrameSourceSuggestions(key, options);
|
|
1223
|
+
if (suggestion) {
|
|
1224
|
+
return {
|
|
1225
|
+
key,
|
|
1226
|
+
selector: suggestion.selector,
|
|
1227
|
+
source: suggestion.source
|
|
1228
|
+
};
|
|
1229
|
+
}
|
|
1230
|
+
if (options.allowAliasFallback) {
|
|
1231
|
+
for (const selector of aliasCandidates) {
|
|
1232
|
+
const source = getMountedCameraFrameCaptureSource(selector);
|
|
1233
|
+
if (!source) continue;
|
|
1234
|
+
return { key, selector, source };
|
|
1342
1235
|
}
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
|
|
1236
|
+
}
|
|
1237
|
+
for (const selector of directCandidates) {
|
|
1238
|
+
if (!isSelectorMounted(selector, cameraNames, siteNames, bodyNames)) {
|
|
1239
|
+
continue;
|
|
1240
|
+
}
|
|
1241
|
+
const source = getMountedCameraFrameCaptureSource(selector);
|
|
1242
|
+
if (!source) continue;
|
|
1243
|
+
return { key, selector, source };
|
|
1244
|
+
}
|
|
1346
1245
|
return null;
|
|
1347
1246
|
}
|
|
1348
|
-
function
|
|
1349
|
-
const
|
|
1350
|
-
const
|
|
1351
|
-
const
|
|
1352
|
-
|
|
1353
|
-
|
|
1354
|
-
|
|
1355
|
-
|
|
1356
|
-
|
|
1357
|
-
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
}
|
|
1366
|
-
function useCameraViewport(elementRef, options) {
|
|
1367
|
-
const registry = useContext(CameraViewportRegistryContext);
|
|
1368
|
-
if (!registry) {
|
|
1369
|
-
throw new Error("useCameraViewport must be used inside <MujocoCanvas>.");
|
|
1247
|
+
function createMountedCameraFrameSequencePlan(cameraKeys, options) {
|
|
1248
|
+
const cameras = [];
|
|
1249
|
+
const resolved = {};
|
|
1250
|
+
const missingKeys = [];
|
|
1251
|
+
for (const key of cameraKeys) {
|
|
1252
|
+
const mountedSource = resolveMountedCameraFrameSource(key, options);
|
|
1253
|
+
if (!mountedSource) {
|
|
1254
|
+
missingKeys.push(key);
|
|
1255
|
+
continue;
|
|
1256
|
+
}
|
|
1257
|
+
resolved[key] = mountedSource;
|
|
1258
|
+
cameras.push({
|
|
1259
|
+
key,
|
|
1260
|
+
...options.defaults,
|
|
1261
|
+
...options.cameraOptions?.[key],
|
|
1262
|
+
...mountedSource.selector,
|
|
1263
|
+
source: mountedSource.source
|
|
1264
|
+
});
|
|
1370
1265
|
}
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
|
|
1266
|
+
if (options.requireAll && missingKeys.length > 0) {
|
|
1267
|
+
throw new Error(
|
|
1268
|
+
`Unable to resolve mounted MuJoCo camera source${missingKeys.length === 1 ? "" : "s"} for ${missingKeys.join(", ")}.`
|
|
1269
|
+
);
|
|
1270
|
+
}
|
|
1271
|
+
return {
|
|
1272
|
+
cameraKeys: [...cameraKeys],
|
|
1273
|
+
cameras,
|
|
1274
|
+
resolved,
|
|
1275
|
+
missingKeys
|
|
1276
|
+
};
|
|
1381
1277
|
}
|
|
1382
|
-
function
|
|
1383
|
-
const
|
|
1384
|
-
const
|
|
1385
|
-
|
|
1386
|
-
|
|
1278
|
+
function createMountedCameraFrameSequenceReadiness(plan) {
|
|
1279
|
+
const cameras = {};
|
|
1280
|
+
const resolvedKeys = plan.cameraKeys.filter((key) => Boolean(plan.resolved[key]));
|
|
1281
|
+
for (const key of plan.cameraKeys) {
|
|
1282
|
+
const resolved = plan.resolved[key];
|
|
1283
|
+
cameras[key] = resolved ? {
|
|
1284
|
+
key,
|
|
1285
|
+
ready: true,
|
|
1286
|
+
selector: resolved.selector,
|
|
1287
|
+
source: resolved.source,
|
|
1288
|
+
message: `Camera stream "${key}" resolves to ${resolved.source.kind}:${getCameraFrameCaptureSourceTarget(resolved.source)}.`
|
|
1289
|
+
} : {
|
|
1290
|
+
key,
|
|
1291
|
+
ready: false,
|
|
1292
|
+
message: `Camera stream "${key}" does not resolve to a mounted MuJoCo camera, site, or body.`
|
|
1293
|
+
};
|
|
1387
1294
|
}
|
|
1388
|
-
|
|
1389
|
-
|
|
1390
|
-
|
|
1391
|
-
|
|
1392
|
-
|
|
1393
|
-
|
|
1394
|
-
|
|
1395
|
-
|
|
1396
|
-
|
|
1295
|
+
const missingKeys = [...plan.missingKeys];
|
|
1296
|
+
const ready = missingKeys.length === 0;
|
|
1297
|
+
const status = ready ? MountedCameraFrameSequenceReadinessStatus.Ready : resolvedKeys.length > 0 ? MountedCameraFrameSequenceReadinessStatus.Partial : MountedCameraFrameSequenceReadinessStatus.Missing;
|
|
1298
|
+
return {
|
|
1299
|
+
ready,
|
|
1300
|
+
status,
|
|
1301
|
+
cameraKeys: [...plan.cameraKeys],
|
|
1302
|
+
resolvedKeys,
|
|
1303
|
+
missingKeys,
|
|
1304
|
+
cameras,
|
|
1305
|
+
message: ready ? `All ${plan.cameraKeys.length} requested camera stream${plan.cameraKeys.length === 1 ? "" : "s"} resolve to mounted MuJoCo sources.` : `Missing mounted MuJoCo source${missingKeys.length === 1 ? "" : "s"} for ${missingKeys.join(", ")}.`
|
|
1306
|
+
};
|
|
1307
|
+
}
|
|
1308
|
+
function normalizeFrameCount(frameCount) {
|
|
1309
|
+
return Number.isFinite(frameCount) && frameCount !== void 0 ? Math.max(0, Math.floor(frameCount)) : 0;
|
|
1310
|
+
}
|
|
1311
|
+
function createMountedCameraFrameSequenceManifest(result, options = {}) {
|
|
1312
|
+
const cameraKeys = [
|
|
1313
|
+
...options.cameraKeys ?? result.readiness.cameraKeys ?? result.plan.cameraKeys ?? result.cameraKeys
|
|
1314
|
+
];
|
|
1315
|
+
const expectedFrameCount = normalizeFrameCount(
|
|
1316
|
+
options.expectedFrameCount ?? result.frameCount
|
|
1317
|
+
);
|
|
1318
|
+
const recordedFrameCount = normalizeFrameCount(result.frameCount);
|
|
1319
|
+
const streamSummaries = {};
|
|
1320
|
+
const streams = [];
|
|
1321
|
+
let missingFrameCount = 0;
|
|
1322
|
+
let completeStreamCount = 0;
|
|
1323
|
+
let resolvedOrRecordedStreamCount = 0;
|
|
1324
|
+
for (const key of cameraKeys) {
|
|
1325
|
+
const summary = result.cameraSummaries[key];
|
|
1326
|
+
const readiness = result.readiness.cameras[key];
|
|
1327
|
+
const source = summary?.source ?? readiness?.source;
|
|
1328
|
+
const ready = readiness?.ready ?? Boolean(summary);
|
|
1329
|
+
const recorded = normalizeFrameCount(summary?.frameCount);
|
|
1330
|
+
const missing = Math.max(expectedFrameCount - recorded, 0);
|
|
1331
|
+
const complete2 = ready && missing === 0;
|
|
1332
|
+
const status2 = complete2 ? MountedCameraFrameSequenceManifestStatus.Complete : ready || recorded > 0 ? MountedCameraFrameSequenceManifestStatus.Partial : MountedCameraFrameSequenceManifestStatus.Missing;
|
|
1333
|
+
const target = source ? getCameraFrameCaptureSourceTarget(source) : readiness?.message ? void 0 : "missing MuJoCo camera";
|
|
1334
|
+
const message = complete2 ? `Camera stream "${key}" recorded ${recorded} of ${expectedFrameCount} frame${expectedFrameCount === 1 ? "" : "s"}.` : ready || recorded > 0 ? `Camera stream "${key}" recorded ${recorded} of ${expectedFrameCount} frame${expectedFrameCount === 1 ? "" : "s"}.` : readiness?.message ?? `Camera stream "${key}" did not record any frames.`;
|
|
1335
|
+
const stream = {
|
|
1336
|
+
key,
|
|
1337
|
+
ready,
|
|
1338
|
+
complete: complete2,
|
|
1339
|
+
status: status2,
|
|
1340
|
+
source,
|
|
1341
|
+
selector: readiness?.selector,
|
|
1342
|
+
target,
|
|
1343
|
+
width: summary?.width,
|
|
1344
|
+
height: summary?.height,
|
|
1345
|
+
expectedFrameCount,
|
|
1346
|
+
recordedFrameCount: recorded,
|
|
1347
|
+
missingFrameCount: missing,
|
|
1348
|
+
firstFrameIndex: summary?.firstFrameIndex ?? null,
|
|
1349
|
+
lastFrameIndex: summary?.lastFrameIndex ?? null,
|
|
1350
|
+
firstTimestamp: summary?.firstTimestamp ?? null,
|
|
1351
|
+
lastTimestamp: summary?.lastTimestamp ?? null,
|
|
1352
|
+
message
|
|
1397
1353
|
};
|
|
1398
|
-
|
|
1399
|
-
|
|
1400
|
-
|
|
1401
|
-
if (
|
|
1402
|
-
|
|
1403
|
-
|
|
1404
|
-
|
|
1405
|
-
|
|
1406
|
-
return
|
|
1354
|
+
streamSummaries[key] = stream;
|
|
1355
|
+
streams.push(stream);
|
|
1356
|
+
missingFrameCount += missing;
|
|
1357
|
+
if (complete2) completeStreamCount += 1;
|
|
1358
|
+
if (ready || recorded > 0) resolvedOrRecordedStreamCount += 1;
|
|
1359
|
+
}
|
|
1360
|
+
const complete = result.readiness.ready && streams.length === completeStreamCount && missingFrameCount === 0;
|
|
1361
|
+
const status = complete ? MountedCameraFrameSequenceManifestStatus.Complete : resolvedOrRecordedStreamCount > 0 ? MountedCameraFrameSequenceManifestStatus.Partial : MountedCameraFrameSequenceManifestStatus.Missing;
|
|
1362
|
+
return {
|
|
1363
|
+
schema: "mujoco-react/mounted-camera-frame-sequence-manifest@1",
|
|
1364
|
+
ready: result.readiness.ready,
|
|
1365
|
+
complete,
|
|
1366
|
+
status,
|
|
1367
|
+
cameraKeys,
|
|
1368
|
+
resolvedKeys: [...result.readiness.resolvedKeys],
|
|
1369
|
+
missingKeys: [...result.readiness.missingKeys],
|
|
1370
|
+
expectedFrameCount,
|
|
1371
|
+
recordedFrameCount,
|
|
1372
|
+
missingFrameCount,
|
|
1373
|
+
streamSummaries,
|
|
1374
|
+
streams,
|
|
1375
|
+
readiness: result.readiness,
|
|
1376
|
+
message: complete ? `All ${cameraKeys.length} camera stream${cameraKeys.length === 1 ? "" : "s"} recorded ${expectedFrameCount} frame${expectedFrameCount === 1 ? "" : "s"}.` : `Mounted camera sequence coverage is ${status}.`
|
|
1377
|
+
};
|
|
1407
1378
|
}
|
|
1408
|
-
function
|
|
1409
|
-
return
|
|
1379
|
+
function createMountedCameraFrameSequencePlanFromApi(api, cameraKeys, options = {}) {
|
|
1380
|
+
return createMountedCameraFrameSequencePlan(cameraKeys, {
|
|
1381
|
+
...options,
|
|
1382
|
+
cameras: api.getCameras(),
|
|
1383
|
+
sites: api.getSites(),
|
|
1384
|
+
bodies: api.getBodies()
|
|
1385
|
+
});
|
|
1410
1386
|
}
|
|
1411
|
-
function
|
|
1412
|
-
const
|
|
1413
|
-
|
|
1414
|
-
|
|
1387
|
+
async function recordMountedCameraFrameSequence(api, options) {
|
|
1388
|
+
const { cameraKeys, ...restOptions } = options;
|
|
1389
|
+
const requireAll = restOptions.requireAll ?? restOptions.requireMountedSources ?? true;
|
|
1390
|
+
const plan = createMountedCameraFrameSequencePlanFromApi(
|
|
1391
|
+
api,
|
|
1392
|
+
cameraKeys,
|
|
1393
|
+
{ ...restOptions, requireAll }
|
|
1394
|
+
);
|
|
1395
|
+
const readiness = createMountedCameraFrameSequenceReadiness(plan);
|
|
1396
|
+
const result = await api.recordCameraSequence({
|
|
1397
|
+
...restOptions,
|
|
1398
|
+
cameras: plan.cameras,
|
|
1399
|
+
requireMountedSources: restOptions.requireMountedSources ?? true
|
|
1400
|
+
});
|
|
1401
|
+
return {
|
|
1402
|
+
...result,
|
|
1403
|
+
plan,
|
|
1404
|
+
readiness
|
|
1405
|
+
};
|
|
1406
|
+
}
|
|
1407
|
+
|
|
1408
|
+
// src/policyCameraFrames.ts
|
|
1409
|
+
function addPolicyImageAliases(images, stream, frame, includeObservationImageAliases) {
|
|
1410
|
+
const keys = /* @__PURE__ */ new Set();
|
|
1411
|
+
keys.add(stream.key);
|
|
1412
|
+
for (const alias of stream.aliases ?? []) keys.add(alias);
|
|
1413
|
+
if (includeObservationImageAliases) {
|
|
1414
|
+
keys.add(`observation.images.${stream.key}`);
|
|
1415
|
+
for (const alias of stream.aliases ?? []) {
|
|
1416
|
+
keys.add(`observation.images.${alias}`);
|
|
1417
|
+
}
|
|
1415
1418
|
}
|
|
1416
|
-
|
|
1417
|
-
|
|
1419
|
+
for (const key of keys) {
|
|
1420
|
+
images[key] = frame.dataUrl;
|
|
1421
|
+
}
|
|
1422
|
+
}
|
|
1423
|
+
function describeFrameSource(key, frame) {
|
|
1424
|
+
return `${key}:${frame.source.kind}`;
|
|
1425
|
+
}
|
|
1426
|
+
function hasExplicitPolicyCameraSource(options) {
|
|
1427
|
+
return Boolean(
|
|
1428
|
+
options?.camera || options?.position || options?.quaternion || options?.source
|
|
1429
|
+
);
|
|
1430
|
+
}
|
|
1431
|
+
function createPolicyCameraFrameCapturePlan(options) {
|
|
1432
|
+
const {
|
|
1433
|
+
cameraKeys,
|
|
1434
|
+
defaults,
|
|
1435
|
+
streamOptions,
|
|
1436
|
+
includeObservationImageAliases,
|
|
1437
|
+
requireAll,
|
|
1438
|
+
...sourceOptions
|
|
1439
|
+
} = options;
|
|
1440
|
+
const mountedPlan = createMountedCameraFrameSequencePlan(cameraKeys, {
|
|
1441
|
+
...sourceOptions,
|
|
1442
|
+
defaults,
|
|
1443
|
+
cameraOptions: streamOptions
|
|
1444
|
+
});
|
|
1445
|
+
const streams = [];
|
|
1446
|
+
const missingKeys = new Set(mountedPlan.missingKeys);
|
|
1447
|
+
for (const key of cameraKeys) {
|
|
1448
|
+
const perStreamOptions = streamOptions?.[key];
|
|
1449
|
+
if (hasExplicitPolicyCameraSource(perStreamOptions)) {
|
|
1450
|
+
missingKeys.delete(key);
|
|
1451
|
+
streams.push({
|
|
1452
|
+
...defaults,
|
|
1453
|
+
...perStreamOptions,
|
|
1454
|
+
key,
|
|
1455
|
+
aliases: perStreamOptions?.aliases
|
|
1456
|
+
});
|
|
1457
|
+
continue;
|
|
1458
|
+
}
|
|
1459
|
+
const mountedCamera = mountedPlan.cameras.find((camera) => camera.key === key);
|
|
1460
|
+
if (!mountedCamera) continue;
|
|
1461
|
+
const { key: _mountedKey, ...captureOptions } = mountedCamera;
|
|
1462
|
+
streams.push({
|
|
1463
|
+
...captureOptions,
|
|
1464
|
+
key,
|
|
1465
|
+
aliases: perStreamOptions?.aliases
|
|
1466
|
+
});
|
|
1418
1467
|
}
|
|
1419
|
-
const
|
|
1420
|
-
|
|
1421
|
-
|
|
1468
|
+
const result = {
|
|
1469
|
+
cameraKeys: [...cameraKeys],
|
|
1470
|
+
streams,
|
|
1471
|
+
includeObservationImageAliases,
|
|
1472
|
+
mountedPlan,
|
|
1473
|
+
missingKeys: [...missingKeys]
|
|
1474
|
+
};
|
|
1475
|
+
if (requireAll && result.missingKeys.length > 0) {
|
|
1476
|
+
throw new Error(
|
|
1477
|
+
`Unable to resolve policy camera stream${result.missingKeys.length === 1 ? "" : "s"} for ${result.missingKeys.join(", ")}.`
|
|
1478
|
+
);
|
|
1422
1479
|
}
|
|
1423
|
-
return
|
|
1480
|
+
return result;
|
|
1424
1481
|
}
|
|
1425
|
-
function
|
|
1426
|
-
return
|
|
1427
|
-
|
|
1482
|
+
function createPolicyCameraFrameCapturePlanFromApi(api, options) {
|
|
1483
|
+
return createPolicyCameraFrameCapturePlan({
|
|
1484
|
+
...options,
|
|
1485
|
+
cameras: api.getCameras(),
|
|
1486
|
+
sites: api.getSites(),
|
|
1487
|
+
bodies: api.getBodies()
|
|
1428
1488
|
});
|
|
1429
1489
|
}
|
|
1430
|
-
async function
|
|
1431
|
-
const
|
|
1432
|
-
const
|
|
1433
|
-
|
|
1434
|
-
|
|
1490
|
+
async function capturePolicyCameraFrames(target, options) {
|
|
1491
|
+
const includeObservationImageAliases = options.includeObservationImageAliases ?? true;
|
|
1492
|
+
const entries = await Promise.all(
|
|
1493
|
+
options.streams.map(async ({ key, aliases, ...captureOptions }) => {
|
|
1494
|
+
const frame = await target.captureCameraFrame(captureOptions);
|
|
1495
|
+
return [key, { frame, aliases }];
|
|
1496
|
+
})
|
|
1497
|
+
);
|
|
1498
|
+
const frames = {};
|
|
1499
|
+
const images = {};
|
|
1500
|
+
const sourceParts = [];
|
|
1501
|
+
for (const [key, { frame, aliases }] of entries) {
|
|
1502
|
+
const stream = { key, aliases };
|
|
1503
|
+
frames[key] = frame;
|
|
1504
|
+
addPolicyImageAliases(
|
|
1505
|
+
images,
|
|
1506
|
+
stream,
|
|
1507
|
+
frame,
|
|
1508
|
+
includeObservationImageAliases
|
|
1509
|
+
);
|
|
1510
|
+
sourceParts.push(describeFrameSource(key, frame));
|
|
1435
1511
|
}
|
|
1436
1512
|
return {
|
|
1437
|
-
|
|
1438
|
-
|
|
1439
|
-
|
|
1513
|
+
frames,
|
|
1514
|
+
images,
|
|
1515
|
+
sourceSummary: sourceParts.length > 0 ? sourceParts.join(" + ") : "not used by policy",
|
|
1516
|
+
capturedAt: Date.now()
|
|
1440
1517
|
};
|
|
1441
1518
|
}
|
|
1442
|
-
async function
|
|
1443
|
-
const
|
|
1444
|
-
const
|
|
1445
|
-
|
|
1446
|
-
await waitForNextAnimationFrame();
|
|
1447
|
-
}
|
|
1448
|
-
const blob = await new Promise((resolve, reject) => {
|
|
1449
|
-
canvas.toBlob(
|
|
1450
|
-
(nextBlob) => {
|
|
1451
|
-
if (nextBlob) {
|
|
1452
|
-
resolve(nextBlob);
|
|
1453
|
-
} else {
|
|
1454
|
-
reject(new Error("Canvas frame capture did not produce a Blob."));
|
|
1455
|
-
}
|
|
1456
|
-
},
|
|
1457
|
-
type,
|
|
1458
|
-
options.quality
|
|
1459
|
-
);
|
|
1460
|
-
});
|
|
1461
|
-
return { canvas, blob, type };
|
|
1519
|
+
async function capturePolicyCameraFramesFromMountedStreams(target, options) {
|
|
1520
|
+
const plan = createPolicyCameraFrameCapturePlanFromApi(target, options);
|
|
1521
|
+
const result = await capturePolicyCameraFrames(target, plan);
|
|
1522
|
+
return { ...result, plan };
|
|
1462
1523
|
}
|
|
1463
|
-
|
|
1464
|
-
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
|
|
1468
|
-
|
|
1469
|
-
|
|
1470
|
-
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
|
|
1524
|
+
|
|
1525
|
+
// src/runtime/client.ts
|
|
1526
|
+
var owners = /* @__PURE__ */ new WeakMap();
|
|
1527
|
+
var isWorkerOwned = (model) => Boolean(model && owners.has(model));
|
|
1528
|
+
function closeModelRuntime(model) {
|
|
1529
|
+
if (model) owners.get(model)?.close();
|
|
1530
|
+
}
|
|
1531
|
+
function runtimeContacts(model) {
|
|
1532
|
+
return owners.get(model)?.contacts();
|
|
1533
|
+
}
|
|
1534
|
+
function assertMainThreadOwnership(model) {
|
|
1535
|
+
if (isWorkerOwned(model)) throw new Error("Worker runtime owns this simulation. Use runtime operations or dispose it before using viewer physics operations.");
|
|
1536
|
+
}
|
|
1537
|
+
var abortError = () => new DOMException("Simulation operation cancelled", "AbortError");
|
|
1538
|
+
var RuntimeClient = class {
|
|
1539
|
+
constructor(worker, apply, release, capture) {
|
|
1540
|
+
this.worker = worker;
|
|
1541
|
+
this.apply = apply;
|
|
1542
|
+
this.release = release;
|
|
1543
|
+
this.capture = capture;
|
|
1544
|
+
worker.onmessage = (event) => {
|
|
1545
|
+
const response = event.data;
|
|
1546
|
+
if (this.closed || response.id !== this.pending?.id) return;
|
|
1547
|
+
if (response.display) {
|
|
1548
|
+
try {
|
|
1549
|
+
this.apply(response.display);
|
|
1550
|
+
const f = response.display.fields;
|
|
1551
|
+
this.latest = {
|
|
1552
|
+
time: response.display.time,
|
|
1553
|
+
qpos: f.qpos,
|
|
1554
|
+
qvel: f.qvel,
|
|
1555
|
+
ctrl: f.ctrl,
|
|
1556
|
+
sensors: f.sensordata,
|
|
1557
|
+
contacts: response.done ? response.display.contacts : this.latest.contacts
|
|
1558
|
+
};
|
|
1559
|
+
} catch (error) {
|
|
1560
|
+
this.close(error instanceof Error ? error : new Error(String(error)));
|
|
1561
|
+
return;
|
|
1562
|
+
}
|
|
1563
|
+
}
|
|
1564
|
+
if (!response.done) this.worker.postMessage({ kind: "ack", id: response.id });
|
|
1565
|
+
if (response.done) {
|
|
1566
|
+
const pending = this.pending;
|
|
1567
|
+
this.pending = null;
|
|
1568
|
+
if (response.error) pending.reject(new Error(response.error));
|
|
1569
|
+
else pending.resolve(response);
|
|
1570
|
+
}
|
|
1571
|
+
};
|
|
1572
|
+
worker.onerror = (event) => this.close(new Error(event.message || "Simulation worker failed"));
|
|
1573
|
+
worker.onmessageerror = () => this.close(new Error("Cannot read simulation worker response"));
|
|
1574
|
+
}
|
|
1575
|
+
pending = null;
|
|
1576
|
+
nextId = 0;
|
|
1577
|
+
closed = false;
|
|
1578
|
+
busy = false;
|
|
1579
|
+
flight = Promise.resolve();
|
|
1580
|
+
generation = 0;
|
|
1581
|
+
latest = { time: 0, qpos: new Float64Array(), qvel: new Float64Array(), ctrl: new Float64Array(), sensors: new Float64Array(), contacts: [] };
|
|
1582
|
+
statistics = { steps: 0, physicsMs: 0, elapsedMs: 0, realTimeFactor: 0, captureMs: 0, busy: false };
|
|
1583
|
+
modelId = crypto.randomUUID();
|
|
1584
|
+
get state() {
|
|
1585
|
+
return {
|
|
1586
|
+
...this.latest,
|
|
1587
|
+
qpos: this.latest.qpos.slice(),
|
|
1588
|
+
qvel: this.latest.qvel.slice(),
|
|
1589
|
+
ctrl: this.latest.ctrl.slice(),
|
|
1590
|
+
sensors: this.latest.sensors.slice(),
|
|
1591
|
+
contacts: this.latest.contacts.map((c) => ({ ...c, pos: [...c.pos] }))
|
|
1592
|
+
};
|
|
1593
|
+
}
|
|
1594
|
+
get metrics() {
|
|
1595
|
+
return { ...this.statistics, busy: this.busy };
|
|
1596
|
+
}
|
|
1597
|
+
request(request) {
|
|
1598
|
+
if (this.closed) return Promise.reject(new Error("Simulation runtime is disposed"));
|
|
1599
|
+
return new Promise((resolve, reject) => {
|
|
1600
|
+
const id = ++this.nextId;
|
|
1601
|
+
this.pending = { id, resolve, reject };
|
|
1474
1602
|
try {
|
|
1475
|
-
|
|
1476
|
-
|
|
1477
|
-
|
|
1478
|
-
|
|
1479
|
-
const error2 = nextError instanceof Error ? nextError : new Error("Unable to capture the current canvas frame.");
|
|
1480
|
-
setError(error2);
|
|
1481
|
-
setStatus("error");
|
|
1482
|
-
throw error2;
|
|
1603
|
+
this.worker.postMessage({ ...request, id });
|
|
1604
|
+
} catch (error) {
|
|
1605
|
+
this.pending = null;
|
|
1606
|
+
reject(error instanceof Error ? error : new Error(String(error)));
|
|
1483
1607
|
}
|
|
1484
|
-
}
|
|
1485
|
-
|
|
1486
|
-
)
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1608
|
+
});
|
|
1609
|
+
}
|
|
1610
|
+
exclusive(operation) {
|
|
1611
|
+
if (this.closed) return Promise.reject(new Error("Simulation runtime is disposed"));
|
|
1612
|
+
if (this.busy) return Promise.reject(new Error("Runtime is busy; await the current operation (work is not queued)"));
|
|
1613
|
+
this.busy = true;
|
|
1614
|
+
const result = operation().finally(() => {
|
|
1615
|
+
this.busy = false;
|
|
1616
|
+
});
|
|
1617
|
+
this.flight = result.catch(() => {
|
|
1618
|
+
});
|
|
1619
|
+
return result;
|
|
1620
|
+
}
|
|
1621
|
+
initialize(initial) {
|
|
1622
|
+
return this.exclusive(async () => {
|
|
1623
|
+
await this.request({ kind: "initialize", initial });
|
|
1624
|
+
});
|
|
1625
|
+
}
|
|
1626
|
+
advance(options) {
|
|
1627
|
+
return this.exclusive(async () => {
|
|
1628
|
+
if (options.signal?.aborted) throw abortError();
|
|
1629
|
+
const started = performance.now(), simStart = this.latest.time;
|
|
1630
|
+
const abort = () => {
|
|
1631
|
+
this.worker.postMessage({ kind: "cancel", id: ++this.nextId });
|
|
1632
|
+
};
|
|
1633
|
+
options.signal?.addEventListener("abort", abort, { once: true });
|
|
1491
1634
|
try {
|
|
1492
|
-
const result = await
|
|
1493
|
-
|
|
1494
|
-
|
|
1635
|
+
const result = await this.request({
|
|
1636
|
+
kind: "advance",
|
|
1637
|
+
steps: options.steps,
|
|
1638
|
+
controls: options.controls ? Float64Array.from(options.controls) : void 0
|
|
1495
1639
|
});
|
|
1496
|
-
|
|
1497
|
-
|
|
1498
|
-
|
|
1499
|
-
|
|
1500
|
-
|
|
1501
|
-
|
|
1502
|
-
|
|
1640
|
+
const elapsedMs = performance.now() - started;
|
|
1641
|
+
this.statistics = {
|
|
1642
|
+
...this.statistics,
|
|
1643
|
+
steps: result.steps ?? 0,
|
|
1644
|
+
physicsMs: result.physicsMs ?? 0,
|
|
1645
|
+
elapsedMs,
|
|
1646
|
+
realTimeFactor: (this.latest.time - simStart) * 1e3 / Math.max(elapsedMs, 1e-3)
|
|
1647
|
+
};
|
|
1648
|
+
if (result.cancelled || options.signal?.aborted) throw abortError();
|
|
1649
|
+
return this.state;
|
|
1650
|
+
} finally {
|
|
1651
|
+
options.signal?.removeEventListener("abort", abort);
|
|
1503
1652
|
}
|
|
1504
|
-
}
|
|
1505
|
-
|
|
1506
|
-
)
|
|
1507
|
-
|
|
1508
|
-
|
|
1509
|
-
|
|
1510
|
-
|
|
1511
|
-
|
|
1512
|
-
|
|
1513
|
-
|
|
1514
|
-
|
|
1515
|
-
}
|
|
1516
|
-
|
|
1517
|
-
|
|
1518
|
-
|
|
1519
|
-
|
|
1520
|
-
|
|
1521
|
-
|
|
1522
|
-
|
|
1523
|
-
|
|
1524
|
-
Contains: "contains"
|
|
1525
|
-
};
|
|
1526
|
-
var MountedCameraFrameSequenceReadinessStatus = {
|
|
1527
|
-
Ready: "ready",
|
|
1528
|
-
Partial: "partial",
|
|
1529
|
-
Missing: "missing"
|
|
1530
|
-
};
|
|
1531
|
-
var MountedCameraFrameSequenceManifestStatus = {
|
|
1532
|
-
Complete: "complete",
|
|
1533
|
-
Partial: "partial",
|
|
1534
|
-
Missing: "missing"
|
|
1535
|
-
};
|
|
1536
|
-
function getResourceName(resource) {
|
|
1537
|
-
if (!resource) return null;
|
|
1538
|
-
return typeof resource === "string" ? resource : resource.name ?? null;
|
|
1539
|
-
}
|
|
1540
|
-
function createNameSet(resources) {
|
|
1541
|
-
return new Set(
|
|
1542
|
-
(resources ?? []).map((resource) => getResourceName(resource)).filter((name) => Boolean(name))
|
|
1543
|
-
);
|
|
1544
|
-
}
|
|
1545
|
-
function createResourceNames(resources) {
|
|
1546
|
-
return (resources ?? []).map((resource) => getResourceName(resource)).filter((name) => Boolean(name));
|
|
1547
|
-
}
|
|
1548
|
-
function normalizeAliasCandidates(value) {
|
|
1549
|
-
if (!value) return [];
|
|
1550
|
-
return Array.isArray(value) ? value : [value];
|
|
1551
|
-
}
|
|
1552
|
-
function countMountedSelectors(selector) {
|
|
1553
|
-
return Number(Boolean(selector.cameraName)) + Number(Boolean(selector.siteName)) + Number(Boolean(selector.bodyName));
|
|
1554
|
-
}
|
|
1555
|
-
function normalizeCameraSourceName(value) {
|
|
1556
|
-
return value.trim().toLowerCase().replace(/[^a-z0-9]+/g, "_").replace(/^_+|_+$/g, "");
|
|
1557
|
-
}
|
|
1558
|
-
function createCameraSourceKeyVariants(key) {
|
|
1559
|
-
const candidates = [
|
|
1560
|
-
key,
|
|
1561
|
-
key.startsWith("observation.images.") ? key.slice("observation.images.".length) : "",
|
|
1562
|
-
key.includes(".") ? key.split(".").at(-1) ?? "" : "",
|
|
1563
|
-
key.includes("/") ? key.split("/").at(-1) ?? "" : ""
|
|
1564
|
-
];
|
|
1565
|
-
return candidates.map((candidate) => candidate.trim()).filter((candidate, index, items) => candidate && items.indexOf(candidate) === index);
|
|
1566
|
-
}
|
|
1567
|
-
function getSelectorKey(selector) {
|
|
1568
|
-
if (selector.cameraName) return `camera:${selector.cameraName}`;
|
|
1569
|
-
if (selector.siteName) return `site:${selector.siteName}`;
|
|
1570
|
-
if (selector.bodyName) return `body:${selector.bodyName}`;
|
|
1571
|
-
return null;
|
|
1572
|
-
}
|
|
1573
|
-
function getMountedCameraFrameCaptureSource(selector) {
|
|
1574
|
-
if (countMountedSelectors(selector) !== 1) return null;
|
|
1575
|
-
if (selector.cameraName) {
|
|
1576
|
-
return { kind: "mujoco-camera", cameraName: selector.cameraName };
|
|
1653
|
+
});
|
|
1654
|
+
}
|
|
1655
|
+
observe(cameras) {
|
|
1656
|
+
return this.exclusive(async () => {
|
|
1657
|
+
const generation = this.generation;
|
|
1658
|
+
await this.request({ kind: "snapshot" });
|
|
1659
|
+
const state = this.state, started = performance.now();
|
|
1660
|
+
const frames = cameras ? await this.capture?.(cameras) : void 0;
|
|
1661
|
+
if (cameras && !frames) throw new Error("Camera observations require an attached canvas");
|
|
1662
|
+
this.statistics.captureMs = performance.now() - started;
|
|
1663
|
+
if (generation !== this.generation || this.closed) throw abortError();
|
|
1664
|
+
return { ...state, ...frames ? { cameras: frames } : {} };
|
|
1665
|
+
});
|
|
1666
|
+
}
|
|
1667
|
+
saveState() {
|
|
1668
|
+
return this.exclusive(async () => {
|
|
1669
|
+
const result = await this.request({ kind: "snapshot" });
|
|
1670
|
+
const state = result.display.integration;
|
|
1671
|
+
return { modelId: this.modelId, signature: state.signature, values: state.values.slice() };
|
|
1672
|
+
});
|
|
1577
1673
|
}
|
|
1578
|
-
|
|
1579
|
-
return
|
|
1674
|
+
restoreState(snapshot) {
|
|
1675
|
+
return this.exclusive(async () => {
|
|
1676
|
+
if (snapshot.modelId !== this.modelId) throw new Error("Snapshot belongs to a different runtime");
|
|
1677
|
+
await this.request({ kind: "restore", state: { signature: snapshot.signature, values: snapshot.values } });
|
|
1678
|
+
return this.state;
|
|
1679
|
+
});
|
|
1580
1680
|
}
|
|
1581
|
-
|
|
1582
|
-
|
|
1681
|
+
async cancel() {
|
|
1682
|
+
if (this.closed) return;
|
|
1683
|
+
this.generation++;
|
|
1684
|
+
this.worker.postMessage({ kind: "cancel", id: ++this.nextId });
|
|
1685
|
+
await this.flight;
|
|
1686
|
+
}
|
|
1687
|
+
async reset() {
|
|
1688
|
+
await this.cancel();
|
|
1689
|
+
return this.exclusive(async () => {
|
|
1690
|
+
await this.request({ kind: "reset" });
|
|
1691
|
+
return this.state;
|
|
1692
|
+
});
|
|
1583
1693
|
}
|
|
1584
|
-
|
|
1585
|
-
|
|
1586
|
-
|
|
1587
|
-
|
|
1588
|
-
|
|
1589
|
-
|
|
1590
|
-
|
|
1591
|
-
|
|
1592
|
-
|
|
1593
|
-
|
|
1594
|
-
|
|
1595
|
-
|
|
1596
|
-
}
|
|
1597
|
-
|
|
1598
|
-
|
|
1599
|
-
|
|
1600
|
-
|
|
1601
|
-
|
|
1602
|
-
|
|
1603
|
-
|
|
1604
|
-
|
|
1605
|
-
|
|
1606
|
-
|
|
1694
|
+
async dispose() {
|
|
1695
|
+
await this.cancel();
|
|
1696
|
+
this.close();
|
|
1697
|
+
}
|
|
1698
|
+
close(error = new Error("Simulation runtime disposed")) {
|
|
1699
|
+
if (this.closed) return;
|
|
1700
|
+
this.closed = true;
|
|
1701
|
+
this.generation++;
|
|
1702
|
+
this.worker.terminate();
|
|
1703
|
+
this.pending?.reject(error);
|
|
1704
|
+
this.pending = null;
|
|
1705
|
+
this.release();
|
|
1706
|
+
}
|
|
1707
|
+
};
|
|
1708
|
+
async function attachSimulationRuntime(module, api, waitForDisplay, invalidate, options = {}) {
|
|
1709
|
+
const model = api.mjModelRef.current, data = api.mjDataRef.current;
|
|
1710
|
+
if (!model || !data) throw new Error("Scene must be ready before creating a runtime");
|
|
1711
|
+
assertMainThreadOwnership(model);
|
|
1712
|
+
api.setPaused(true);
|
|
1713
|
+
const state = api.saveState().integration;
|
|
1714
|
+
const initialState = { signature: state.signature, values: state.values };
|
|
1715
|
+
const mj = module;
|
|
1716
|
+
const path = `/runtime-${crypto.randomUUID()}.mjb`;
|
|
1717
|
+
let binary;
|
|
1718
|
+
try {
|
|
1719
|
+
mj.mj_saveModel(model, path, null);
|
|
1720
|
+
binary = mj.FS.readFile(path);
|
|
1721
|
+
} finally {
|
|
1722
|
+
try {
|
|
1723
|
+
mj.FS.unlink(path);
|
|
1724
|
+
} catch {
|
|
1725
|
+
}
|
|
1726
|
+
}
|
|
1727
|
+
const worker = options.createWorker?.() ?? new Worker(new URL("./physics.worker.js", import.meta.url), { type: "module" });
|
|
1728
|
+
let complete;
|
|
1729
|
+
const apply = (display) => {
|
|
1730
|
+
if (api.mjModelRef.current !== model) throw new Error("Simulation model changed");
|
|
1731
|
+
if (display.integration) {
|
|
1732
|
+
module.mj_setState(model, data, Array.from(display.integration.values), display.integration.signature);
|
|
1733
|
+
complete = display;
|
|
1734
|
+
}
|
|
1735
|
+
for (const field of displayFields) {
|
|
1736
|
+
const target = data[field];
|
|
1737
|
+
if (!(target instanceof Float64Array || target instanceof Int32Array) || target.length !== display.fields[field].length) throw new Error(`Invalid display field ${field}`);
|
|
1738
|
+
target.set(display.fields[field]);
|
|
1739
|
+
}
|
|
1740
|
+
data.time = display.time;
|
|
1741
|
+
invalidate();
|
|
1607
1742
|
};
|
|
1743
|
+
const client = new RuntimeClient(worker, apply, () => {
|
|
1744
|
+
if (api.mjModelRef.current === model && complete) apply(complete);
|
|
1745
|
+
owners.delete(model);
|
|
1746
|
+
}, async (cameras) => {
|
|
1747
|
+
await waitForDisplay();
|
|
1748
|
+
return capturePolicyCameraFrames(api, cameras);
|
|
1749
|
+
});
|
|
1750
|
+
owners.set(model, { close: () => client.close(), contacts: () => client.state.contacts });
|
|
1751
|
+
try {
|
|
1752
|
+
await client.initialize({ model: binary, state: initialState });
|
|
1753
|
+
return client;
|
|
1754
|
+
} catch (error) {
|
|
1755
|
+
client.close();
|
|
1756
|
+
throw error;
|
|
1757
|
+
}
|
|
1608
1758
|
}
|
|
1609
|
-
|
|
1610
|
-
|
|
1611
|
-
|
|
1612
|
-
|
|
1613
|
-
|
|
1614
|
-
|
|
1759
|
+
var CapsuleGeometry = class extends THREE12.BufferGeometry {
|
|
1760
|
+
parameters;
|
|
1761
|
+
constructor(radius = 1, length = 1, capSegments = 4, radialSegments = 8) {
|
|
1762
|
+
super();
|
|
1763
|
+
this.type = "CapsuleGeometry";
|
|
1764
|
+
this.parameters = { radius, length, capSegments, radialSegments };
|
|
1765
|
+
const path = new THREE12.Path();
|
|
1766
|
+
path.absarc(0, -length / 2, radius, Math.PI * 1.5, 0, false);
|
|
1767
|
+
path.absarc(0, length / 2, radius, 0, Math.PI * 0.5, false);
|
|
1768
|
+
const latheGeometry = new THREE12.LatheGeometry(path.getPoints(capSegments), radialSegments);
|
|
1769
|
+
const self = this;
|
|
1770
|
+
self.setIndex(latheGeometry.getIndex());
|
|
1771
|
+
self.setAttribute("position", latheGeometry.getAttribute("position"));
|
|
1772
|
+
self.setAttribute("normal", latheGeometry.getAttribute("normal"));
|
|
1773
|
+
self.setAttribute("uv", latheGeometry.getAttribute("uv"));
|
|
1774
|
+
}
|
|
1775
|
+
};
|
|
1776
|
+
|
|
1777
|
+
// src/rendering/GeomBuilder.ts
|
|
1778
|
+
var DEFAULT_MESH_NORMAL_SMOOTHING_TOLERANCE = 1e-4;
|
|
1779
|
+
var GeomBuilder = class {
|
|
1780
|
+
mujoco;
|
|
1781
|
+
textureCache = /* @__PURE__ */ new Map();
|
|
1782
|
+
renderOptions;
|
|
1783
|
+
constructor(mujoco, renderOptions) {
|
|
1784
|
+
this.mujoco = mujoco;
|
|
1785
|
+
this.renderOptions = renderOptions;
|
|
1786
|
+
}
|
|
1787
|
+
getMeshNormalSmoothingTolerance() {
|
|
1788
|
+
const smoothing = this.renderOptions?.meshNormalSmoothing;
|
|
1789
|
+
if (!smoothing) return null;
|
|
1790
|
+
if (smoothing === true) return DEFAULT_MESH_NORMAL_SMOOTHING_TOLERANCE;
|
|
1791
|
+
return smoothing.tolerance ?? DEFAULT_MESH_NORMAL_SMOOTHING_TOLERANCE;
|
|
1792
|
+
}
|
|
1793
|
+
getMaterialTexture(mjModel, matId) {
|
|
1794
|
+
if (matId < 0 || !mjModel.mat_texid || !mjModel.tex_data) return null;
|
|
1795
|
+
const materialCount = Math.max(1, Math.floor(mjModel.mat_rgba.length / 4));
|
|
1796
|
+
const textureRoles = Math.max(1, Math.floor(mjModel.mat_texid.length / materialCount));
|
|
1797
|
+
let texId = -1;
|
|
1798
|
+
for (let role = 0; role < textureRoles; role += 1) {
|
|
1799
|
+
const candidate = mjModel.mat_texid[matId * textureRoles + role];
|
|
1800
|
+
if (candidate >= 0) {
|
|
1801
|
+
texId = candidate;
|
|
1802
|
+
break;
|
|
1803
|
+
}
|
|
1804
|
+
}
|
|
1805
|
+
if (texId < 0) return null;
|
|
1806
|
+
const cached = this.textureCache.get(texId);
|
|
1807
|
+
if (cached) return cached;
|
|
1808
|
+
const width = Number(mjModel.tex_width[texId]);
|
|
1809
|
+
const height = Number(mjModel.tex_height[texId]);
|
|
1810
|
+
const channels = Number(mjModel.tex_nchannel[texId]);
|
|
1811
|
+
const offset = Number(mjModel.tex_adr[texId]);
|
|
1812
|
+
if (width <= 0 || height <= 0 || channels <= 0 || offset < 0) return null;
|
|
1813
|
+
const source = mjModel.tex_data.subarray(offset, offset + width * height * channels);
|
|
1814
|
+
const rgba = new Uint8Array(width * height * 4);
|
|
1815
|
+
for (let i = 0, j = 0; i < width * height; i += 1, j += channels) {
|
|
1816
|
+
const r = source[j] ?? 255;
|
|
1817
|
+
const g = channels > 1 ? source[j + 1] : r;
|
|
1818
|
+
const b = channels > 2 ? source[j + 2] : r;
|
|
1819
|
+
const a = channels > 3 ? source[j + 3] : 255;
|
|
1820
|
+
const out = i * 4;
|
|
1821
|
+
rgba[out] = r;
|
|
1822
|
+
rgba[out + 1] = g;
|
|
1823
|
+
rgba[out + 2] = b;
|
|
1824
|
+
rgba[out + 3] = a;
|
|
1825
|
+
}
|
|
1826
|
+
const texture = new THREE12.DataTexture(rgba, width, height, THREE12.RGBAFormat);
|
|
1827
|
+
texture.colorSpace = THREE12.LinearSRGBColorSpace;
|
|
1828
|
+
texture.wrapS = THREE12.RepeatWrapping;
|
|
1829
|
+
texture.wrapT = THREE12.RepeatWrapping;
|
|
1830
|
+
texture.flipY = true;
|
|
1831
|
+
const repeatOffset = matId * 2;
|
|
1832
|
+
const repeatS = mjModel.mat_texrepeat?.[repeatOffset] ?? 1;
|
|
1833
|
+
const repeatT = mjModel.mat_texrepeat?.[repeatOffset + 1] ?? 1;
|
|
1834
|
+
texture.repeat.set(repeatS || 1, repeatT || 1);
|
|
1835
|
+
texture.needsUpdate = true;
|
|
1836
|
+
this.textureCache.set(texId, texture);
|
|
1837
|
+
return texture;
|
|
1838
|
+
}
|
|
1839
|
+
/**
|
|
1840
|
+
* Creates a Three.js Object3D (usually a Mesh) for a specific geometry in the MuJoCo model.
|
|
1841
|
+
* Returns null if the geometry shouldn't be rendered (e.g., invisible collision triggers).
|
|
1842
|
+
*/
|
|
1843
|
+
create(mjModel, g) {
|
|
1844
|
+
if (mjModel.geom_group[g] === 3) return null;
|
|
1845
|
+
const type = mjModel.geom_type[g];
|
|
1846
|
+
const size = mjModel.geom_size.subarray(g * 3, g * 3 + 3);
|
|
1847
|
+
const pos = mjModel.geom_pos.subarray(g * 3, g * 3 + 3);
|
|
1848
|
+
const quat = mjModel.geom_quat.subarray(g * 4, g * 4 + 4);
|
|
1849
|
+
const matId = mjModel.geom_matid[g];
|
|
1850
|
+
const color = new THREE12.Color(16777215);
|
|
1851
|
+
const map = this.getMaterialTexture(mjModel, matId);
|
|
1852
|
+
let opacity = 1;
|
|
1853
|
+
if (matId >= 0) {
|
|
1854
|
+
const rgba = mjModel.mat_rgba.subarray(matId * 4, matId * 4 + 4);
|
|
1855
|
+
color.setRGB(rgba[0], rgba[1], rgba[2]);
|
|
1856
|
+
opacity = rgba[3];
|
|
1857
|
+
} else {
|
|
1858
|
+
const rgba = mjModel.geom_rgba.subarray(g * 4, g * 4 + 4);
|
|
1859
|
+
color.setRGB(rgba[0], rgba[1], rgba[2]);
|
|
1860
|
+
opacity = rgba[3];
|
|
1861
|
+
}
|
|
1862
|
+
const MG = this.mujoco.mjtGeom;
|
|
1863
|
+
let geo = null;
|
|
1864
|
+
const getVal = (v) => v?.value ?? v;
|
|
1865
|
+
if (type === getVal(MG.mjGEOM_PLANE)) {
|
|
1866
|
+
geo = new THREE12.PlaneGeometry(size[0] * 2 || 5, size[1] * 2 || 5);
|
|
1867
|
+
} else if (type === getVal(MG.mjGEOM_SPHERE)) {
|
|
1868
|
+
geo = new THREE12.SphereGeometry(size[0], 24, 24);
|
|
1869
|
+
} else if (type === getVal(MG.mjGEOM_CAPSULE)) {
|
|
1870
|
+
geo = new CapsuleGeometry(size[0], size[1] * 2, 24, 12);
|
|
1871
|
+
geo.rotateX(Math.PI / 2);
|
|
1872
|
+
} else if (type === getVal(MG.mjGEOM_BOX)) {
|
|
1873
|
+
geo = new THREE12.BoxGeometry(size[0] * 2, size[1] * 2, size[2] * 2);
|
|
1874
|
+
} else if (type === getVal(MG.mjGEOM_CYLINDER)) {
|
|
1875
|
+
geo = new THREE12.CylinderGeometry(size[0], size[0], size[1] * 2, 24);
|
|
1876
|
+
geo.rotateX(Math.PI / 2);
|
|
1877
|
+
} else if (type === getVal(MG.mjGEOM_MESH)) {
|
|
1878
|
+
const mId = mjModel.geom_dataid[g];
|
|
1879
|
+
const vAdr = mjModel.mesh_vertadr[mId];
|
|
1880
|
+
const vNum = mjModel.mesh_vertnum[mId];
|
|
1881
|
+
const fAdr = mjModel.mesh_faceadr[mId];
|
|
1882
|
+
const fNum = mjModel.mesh_facenum[mId];
|
|
1883
|
+
const tAdr = mjModel.mesh_texcoordadr?.[mId] ?? -1;
|
|
1884
|
+
if (map && tAdr >= 0 && mjModel.mesh_texcoord && mjModel.mesh_facetexcoord) {
|
|
1885
|
+
const corners = fNum * 3;
|
|
1886
|
+
const positions = new Float32Array(corners * 3);
|
|
1887
|
+
const uvs = new Float32Array(corners * 2);
|
|
1888
|
+
for (let c = 0; c < corners; c++) {
|
|
1889
|
+
const v = vAdr + mjModel.mesh_face[fAdr * 3 + c];
|
|
1890
|
+
positions[c * 3] = mjModel.mesh_vert[v * 3];
|
|
1891
|
+
positions[c * 3 + 1] = mjModel.mesh_vert[v * 3 + 1];
|
|
1892
|
+
positions[c * 3 + 2] = mjModel.mesh_vert[v * 3 + 2];
|
|
1893
|
+
const t = tAdr + mjModel.mesh_facetexcoord[fAdr * 3 + c];
|
|
1894
|
+
uvs[c * 2] = mjModel.mesh_texcoord[t * 2];
|
|
1895
|
+
uvs[c * 2 + 1] = 1 - mjModel.mesh_texcoord[t * 2 + 1];
|
|
1896
|
+
}
|
|
1897
|
+
geo = new THREE12.BufferGeometry();
|
|
1898
|
+
geo.setAttribute("position", new THREE12.Float32BufferAttribute(positions, 3));
|
|
1899
|
+
geo.setAttribute("uv", new THREE12.Float32BufferAttribute(uvs, 2));
|
|
1900
|
+
geo.computeVertexNormals();
|
|
1901
|
+
} else {
|
|
1902
|
+
geo = new THREE12.BufferGeometry();
|
|
1903
|
+
geo.setAttribute("position", new THREE12.Float32BufferAttribute(mjModel.mesh_vert.subarray(vAdr * 3, (vAdr + vNum) * 3), 3));
|
|
1904
|
+
geo.setIndex(Array.from(mjModel.mesh_face.subarray(fAdr * 3, (fAdr + fNum) * 3)));
|
|
1905
|
+
const smoothingTolerance = this.getMeshNormalSmoothingTolerance();
|
|
1906
|
+
if (smoothingTolerance !== null) {
|
|
1907
|
+
geo = mergeVertices(geo, smoothingTolerance);
|
|
1908
|
+
}
|
|
1909
|
+
geo.computeVertexNormals();
|
|
1910
|
+
}
|
|
1911
|
+
}
|
|
1912
|
+
if (geo) {
|
|
1913
|
+
const isPlane = type === getVal(MG.mjGEOM_PLANE);
|
|
1914
|
+
const materialMap = isPlane && map ? map.clone() : map;
|
|
1915
|
+
if (isPlane && materialMap) {
|
|
1916
|
+
materialMap.repeat.multiplyScalar(2.5);
|
|
1917
|
+
materialMap.needsUpdate = true;
|
|
1918
|
+
}
|
|
1919
|
+
const mesh = new THREE12.Mesh(geo, new THREE12.MeshStandardMaterial({
|
|
1920
|
+
color,
|
|
1921
|
+
map: materialMap,
|
|
1922
|
+
transparent: opacity < 1,
|
|
1923
|
+
opacity,
|
|
1924
|
+
roughness: 0.6,
|
|
1925
|
+
metalness: 0
|
|
1926
|
+
}));
|
|
1927
|
+
mesh.castShadow = type !== getVal(MG.mjGEOM_PLANE);
|
|
1928
|
+
mesh.receiveShadow = true;
|
|
1929
|
+
mesh.position.set(pos[0], pos[1], pos[2]);
|
|
1930
|
+
mesh.quaternion.set(quat[1], quat[2], quat[3], quat[0]);
|
|
1931
|
+
mesh.userData.bodyID = mjModel.geom_bodyid[g];
|
|
1932
|
+
mesh.userData.geomID = g;
|
|
1933
|
+
mesh.userData.geomGroup = mjModel.geom_group[g];
|
|
1934
|
+
mesh.userData.geomName = getName(mjModel, mjModel.name_geomadr[g]);
|
|
1935
|
+
return mesh;
|
|
1936
|
+
}
|
|
1937
|
+
return null;
|
|
1938
|
+
}
|
|
1939
|
+
};
|
|
1940
|
+
function getRenderOptionsKey(renderOptions) {
|
|
1941
|
+
const smoothing = renderOptions?.meshNormalSmoothing;
|
|
1942
|
+
if (!smoothing) return "default";
|
|
1943
|
+
if (smoothing === true) return "meshNormalSmoothing:true";
|
|
1944
|
+
return `meshNormalSmoothing:${smoothing.tolerance ?? "default"}`;
|
|
1615
1945
|
}
|
|
1616
|
-
function
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
|
|
1620
|
-
|
|
1621
|
-
|
|
1622
|
-
|
|
1623
|
-
|
|
1624
|
-
|
|
1625
|
-
|
|
1946
|
+
function SceneRenderer({ renderOptions, ...props }) {
|
|
1947
|
+
const {
|
|
1948
|
+
mjModelRef,
|
|
1949
|
+
mjDataRef,
|
|
1950
|
+
mujocoRef,
|
|
1951
|
+
onSelectionRef,
|
|
1952
|
+
hiddenBodiesRef,
|
|
1953
|
+
hiddenBodiesVersion,
|
|
1954
|
+
interpolateRef,
|
|
1955
|
+
interpolationStateRef,
|
|
1956
|
+
status
|
|
1957
|
+
} = useMujocoContext();
|
|
1958
|
+
const groupRef = useRef(null);
|
|
1959
|
+
const bodyRefs = useRef([]);
|
|
1960
|
+
const prevModelRef = useRef(null);
|
|
1961
|
+
const prevRenderOptionsKeyRef = useRef(null);
|
|
1962
|
+
const prevHiddenVersionRef = useRef(null);
|
|
1963
|
+
const renderOptionsKey = getRenderOptionsKey(renderOptions);
|
|
1964
|
+
const geomBuilder = useMemo(() => {
|
|
1965
|
+
if (status !== "ready") return null;
|
|
1966
|
+
return new GeomBuilder(mujocoRef.current, renderOptions);
|
|
1967
|
+
}, [status, mujocoRef, renderOptionsKey]);
|
|
1968
|
+
useEffect(() => {
|
|
1969
|
+
if (status !== "ready" || !geomBuilder) return;
|
|
1970
|
+
const model = mjModelRef.current;
|
|
1971
|
+
const group = groupRef.current;
|
|
1972
|
+
if (!model || !group) return;
|
|
1973
|
+
if (prevModelRef.current === model && prevRenderOptionsKeyRef.current === renderOptionsKey && prevHiddenVersionRef.current === hiddenBodiesVersion) {
|
|
1974
|
+
return;
|
|
1626
1975
|
}
|
|
1627
|
-
|
|
1628
|
-
|
|
1976
|
+
prevModelRef.current = model;
|
|
1977
|
+
prevRenderOptionsKeyRef.current = renderOptionsKey;
|
|
1978
|
+
prevHiddenVersionRef.current = hiddenBodiesVersion;
|
|
1979
|
+
while (group.children.length > 0) {
|
|
1980
|
+
group.remove(group.children[0]);
|
|
1629
1981
|
}
|
|
1630
|
-
|
|
1631
|
-
|
|
1982
|
+
const refs = [];
|
|
1983
|
+
for (let i = 0; i < model.nbody; i++) {
|
|
1984
|
+
const bodyGroup = new THREE12.Group();
|
|
1985
|
+
bodyGroup.userData.bodyID = i;
|
|
1986
|
+
const bodyName = getName(model, model.name_bodyadr[i]);
|
|
1987
|
+
if (!hiddenBodiesRef.current.has(bodyName)) {
|
|
1988
|
+
for (let g = 0; g < model.ngeom; g++) {
|
|
1989
|
+
if (model.geom_bodyid[g] === i) {
|
|
1990
|
+
const mesh = geomBuilder.create(model, g);
|
|
1991
|
+
if (mesh) bodyGroup.add(mesh);
|
|
1992
|
+
}
|
|
1993
|
+
}
|
|
1994
|
+
}
|
|
1995
|
+
group.add(bodyGroup);
|
|
1996
|
+
refs.push(bodyGroup);
|
|
1632
1997
|
}
|
|
1633
|
-
|
|
1634
|
-
|
|
1998
|
+
bodyRefs.current = refs;
|
|
1999
|
+
}, [status, geomBuilder, mjModelRef, renderOptionsKey, hiddenBodiesRef, hiddenBodiesVersion]);
|
|
2000
|
+
const syncBodiesToData = useCallback(() => {
|
|
2001
|
+
const data = mjDataRef.current;
|
|
2002
|
+
if (!data) return;
|
|
2003
|
+
const bodies = bodyRefs.current;
|
|
2004
|
+
const interpolation = interpolationStateRef.current;
|
|
2005
|
+
const useInterpolation = interpolateRef.current && interpolation.valid;
|
|
2006
|
+
for (let i = 0; i < bodies.length; i++) {
|
|
2007
|
+
const ref = bodies[i];
|
|
2008
|
+
if (!ref) continue;
|
|
2009
|
+
if (useInterpolation) {
|
|
2010
|
+
const alpha = interpolation.alpha;
|
|
2011
|
+
const i3 = i * 3;
|
|
2012
|
+
ref.position.set(
|
|
2013
|
+
THREE12.MathUtils.lerp(interpolation.previousXpos[i3], interpolation.currentXpos[i3], alpha),
|
|
2014
|
+
THREE12.MathUtils.lerp(interpolation.previousXpos[i3 + 1], interpolation.currentXpos[i3 + 1], alpha),
|
|
2015
|
+
THREE12.MathUtils.lerp(interpolation.previousXpos[i3 + 2], interpolation.currentXpos[i3 + 2], alpha)
|
|
2016
|
+
);
|
|
2017
|
+
const i4 = i * 4;
|
|
2018
|
+
_previousQuat.set(
|
|
2019
|
+
interpolation.previousXquat[i4 + 1],
|
|
2020
|
+
interpolation.previousXquat[i4 + 2],
|
|
2021
|
+
interpolation.previousXquat[i4 + 3],
|
|
2022
|
+
interpolation.previousXquat[i4]
|
|
2023
|
+
);
|
|
2024
|
+
_currentQuat.set(
|
|
2025
|
+
interpolation.currentXquat[i4 + 1],
|
|
2026
|
+
interpolation.currentXquat[i4 + 2],
|
|
2027
|
+
interpolation.currentXquat[i4 + 3],
|
|
2028
|
+
interpolation.currentXquat[i4]
|
|
2029
|
+
);
|
|
2030
|
+
ref.quaternion.copy(_previousQuat).slerp(_currentQuat, alpha);
|
|
2031
|
+
} else {
|
|
2032
|
+
ref.position.set(
|
|
2033
|
+
data.xpos[i * 3],
|
|
2034
|
+
data.xpos[i * 3 + 1],
|
|
2035
|
+
data.xpos[i * 3 + 2]
|
|
2036
|
+
);
|
|
2037
|
+
ref.quaternion.set(
|
|
2038
|
+
data.xquat[i * 4 + 1],
|
|
2039
|
+
data.xquat[i * 4 + 2],
|
|
2040
|
+
data.xquat[i * 4 + 3],
|
|
2041
|
+
data.xquat[i * 4]
|
|
2042
|
+
);
|
|
2043
|
+
}
|
|
1635
2044
|
}
|
|
1636
|
-
}
|
|
1637
|
-
|
|
1638
|
-
|
|
1639
|
-
|
|
1640
|
-
|
|
1641
|
-
|
|
2045
|
+
}, [interpolateRef, interpolationStateRef, mjDataRef]);
|
|
2046
|
+
useFrame(syncBodiesToData);
|
|
2047
|
+
return /* @__PURE__ */ jsx(
|
|
2048
|
+
"group",
|
|
2049
|
+
{
|
|
2050
|
+
...props,
|
|
2051
|
+
ref: groupRef,
|
|
2052
|
+
userData: {
|
|
2053
|
+
...props.userData,
|
|
2054
|
+
[CAMERA_FRAME_CAPTURE_PRE_RENDER_USER_DATA_KEY]: syncBodiesToData
|
|
2055
|
+
},
|
|
2056
|
+
onDoubleClick: (e) => {
|
|
2057
|
+
if (typeof props.onDoubleClick === "function") props.onDoubleClick(e);
|
|
2058
|
+
e.stopPropagation();
|
|
2059
|
+
let obj = e.object;
|
|
2060
|
+
while (obj && obj.userData.bodyID === void 0 && obj.parent) {
|
|
2061
|
+
obj = obj.parent;
|
|
2062
|
+
}
|
|
2063
|
+
const bodyID = obj?.userData.bodyID;
|
|
2064
|
+
if (typeof bodyID === "number" && bodyID > 0) {
|
|
2065
|
+
const model = mjModelRef.current;
|
|
2066
|
+
if (model && bodyID < model.nbody && onSelectionRef.current) {
|
|
2067
|
+
const name = getName(model, model.name_bodyadr[bodyID]);
|
|
2068
|
+
onSelectionRef.current({ bodyId: bodyID, name });
|
|
2069
|
+
}
|
|
2070
|
+
}
|
|
2071
|
+
}
|
|
2072
|
+
}
|
|
2073
|
+
);
|
|
1642
2074
|
}
|
|
1643
|
-
|
|
1644
|
-
|
|
1645
|
-
|
|
1646
|
-
|
|
1647
|
-
|
|
1648
|
-
|
|
1649
|
-
|
|
1650
|
-
|
|
1651
|
-
|
|
2075
|
+
var _previousQuat = new THREE12.Quaternion();
|
|
2076
|
+
var _currentQuat = new THREE12.Quaternion();
|
|
2077
|
+
var CameraViewportRegistryContext = createContext(null);
|
|
2078
|
+
var nextViewportId = 0;
|
|
2079
|
+
var VIEWPORT_RENDER_PRIORITY = 1;
|
|
2080
|
+
function CameraViewportRenderer({
|
|
2081
|
+
viewportsRef
|
|
2082
|
+
}) {
|
|
2083
|
+
const gl = useThree((state) => state.gl);
|
|
2084
|
+
const scene = useThree((state) => state.scene);
|
|
2085
|
+
const mainCamera = useThree((state) => state.camera);
|
|
2086
|
+
const mujoco = useMujoco();
|
|
2087
|
+
useFrame(() => {
|
|
2088
|
+
const dpr = gl.getPixelRatio();
|
|
2089
|
+
const drawWidth = gl.domElement.width / dpr;
|
|
2090
|
+
const drawHeight = gl.domElement.height / dpr;
|
|
2091
|
+
gl.setScissorTest(false);
|
|
2092
|
+
gl.setViewport(0, 0, drawWidth, drawHeight);
|
|
2093
|
+
gl.render(scene, mainCamera);
|
|
2094
|
+
const api = mujoco.api;
|
|
2095
|
+
if (!api || viewportsRef.current.size === 0) return;
|
|
2096
|
+
const canvasRect = gl.domElement.getBoundingClientRect();
|
|
2097
|
+
for (const descriptor of viewportsRef.current.values()) {
|
|
2098
|
+
const element = descriptor.getElement();
|
|
2099
|
+
if (!element) continue;
|
|
2100
|
+
const rect = element.getBoundingClientRect();
|
|
2101
|
+
const isOffscreen = rect.bottom < canvasRect.top || rect.top > canvasRect.bottom || rect.right < canvasRect.left || rect.left > canvasRect.right;
|
|
2102
|
+
if (isOffscreen) continue;
|
|
2103
|
+
const width = rect.width;
|
|
2104
|
+
const height = rect.height;
|
|
2105
|
+
if (width <= 0 || height <= 0) continue;
|
|
2106
|
+
const left = rect.left - canvasRect.left;
|
|
2107
|
+
const bottom = canvasRect.bottom - rect.bottom;
|
|
2108
|
+
let resolved;
|
|
2109
|
+
try {
|
|
2110
|
+
resolved = api.resolveCameraCaptureOptions(descriptor.getOptions());
|
|
2111
|
+
} catch {
|
|
2112
|
+
continue;
|
|
2113
|
+
}
|
|
2114
|
+
if (!descriptor.camera) {
|
|
2115
|
+
descriptor.camera = createCaptureCamera(resolved, mainCamera, width, height);
|
|
2116
|
+
} else {
|
|
2117
|
+
prepareCaptureCamera(descriptor.camera, resolved, mainCamera, width, height);
|
|
2118
|
+
}
|
|
2119
|
+
gl.setViewport(left, bottom, width, height);
|
|
2120
|
+
gl.setScissor(left, bottom, width, height);
|
|
2121
|
+
gl.setScissorTest(true);
|
|
2122
|
+
gl.render(scene, descriptor.camera);
|
|
1652
2123
|
}
|
|
1653
|
-
|
|
1654
|
-
|
|
1655
|
-
|
|
1656
|
-
|
|
1657
|
-
|
|
1658
|
-
|
|
1659
|
-
|
|
1660
|
-
|
|
1661
|
-
|
|
1662
|
-
|
|
1663
|
-
|
|
1664
|
-
);
|
|
1665
|
-
|
|
1666
|
-
|
|
1667
|
-
|
|
1668
|
-
|
|
1669
|
-
|
|
1670
|
-
|
|
1671
|
-
|
|
1672
|
-
|
|
1673
|
-
|
|
1674
|
-
|
|
1675
|
-
cameraName,
|
|
1676
|
-
match
|
|
1677
|
-
)
|
|
1678
|
-
);
|
|
1679
|
-
}
|
|
1680
|
-
for (const siteName of createResourceNames(options.sites)) {
|
|
1681
|
-
const match = getCameraFrameResourceMatch(key, siteName);
|
|
1682
|
-
if (!match) continue;
|
|
1683
|
-
addMountedCameraFrameSourceSuggestion(
|
|
1684
|
-
suggestions,
|
|
1685
|
-
seen,
|
|
1686
|
-
createMountedCameraFrameSourceSuggestion(
|
|
1687
|
-
key,
|
|
1688
|
-
{ siteName },
|
|
1689
|
-
siteName,
|
|
1690
|
-
match
|
|
1691
|
-
)
|
|
1692
|
-
);
|
|
1693
|
-
}
|
|
1694
|
-
for (const bodyName of createResourceNames(options.bodies)) {
|
|
1695
|
-
const match = getCameraFrameResourceMatch(key, bodyName);
|
|
1696
|
-
if (!match) continue;
|
|
1697
|
-
addMountedCameraFrameSourceSuggestion(
|
|
1698
|
-
suggestions,
|
|
1699
|
-
seen,
|
|
1700
|
-
createMountedCameraFrameSourceSuggestion(
|
|
1701
|
-
key,
|
|
1702
|
-
{ bodyName },
|
|
1703
|
-
bodyName,
|
|
1704
|
-
match
|
|
1705
|
-
)
|
|
1706
|
-
);
|
|
1707
|
-
}
|
|
1708
|
-
return suggestions;
|
|
2124
|
+
gl.setScissorTest(false);
|
|
2125
|
+
gl.setViewport(0, 0, drawWidth, drawHeight);
|
|
2126
|
+
}, VIEWPORT_RENDER_PRIORITY);
|
|
2127
|
+
return null;
|
|
2128
|
+
}
|
|
2129
|
+
function CameraViewportProvider({ children }) {
|
|
2130
|
+
const viewportsRef = useRef(/* @__PURE__ */ new Map());
|
|
2131
|
+
const [count, setCount] = useState(0);
|
|
2132
|
+
const register = useCallback((descriptor) => {
|
|
2133
|
+
const id = nextViewportId++;
|
|
2134
|
+
viewportsRef.current.set(id, descriptor);
|
|
2135
|
+
setCount((value2) => value2 + 1);
|
|
2136
|
+
return () => {
|
|
2137
|
+
viewportsRef.current.delete(id);
|
|
2138
|
+
setCount((value2) => value2 - 1);
|
|
2139
|
+
};
|
|
2140
|
+
}, []);
|
|
2141
|
+
const value = useMemo(() => ({ register }), [register]);
|
|
2142
|
+
return /* @__PURE__ */ jsxs(CameraViewportRegistryContext.Provider, { value, children: [
|
|
2143
|
+
children,
|
|
2144
|
+
count > 0 && /* @__PURE__ */ jsx(CameraViewportRenderer, { viewportsRef })
|
|
2145
|
+
] });
|
|
1709
2146
|
}
|
|
1710
|
-
function
|
|
1711
|
-
const
|
|
1712
|
-
|
|
1713
|
-
|
|
1714
|
-
const directCandidates = [
|
|
1715
|
-
{ cameraName: key },
|
|
1716
|
-
{ siteName: key },
|
|
1717
|
-
{ bodyName: key }
|
|
1718
|
-
];
|
|
1719
|
-
const aliasCandidates = normalizeAliasCandidates(options.aliases?.[key]);
|
|
1720
|
-
for (const selector of aliasCandidates) {
|
|
1721
|
-
if (!isSelectorMounted(selector, cameraNames, siteNames, bodyNames)) {
|
|
1722
|
-
continue;
|
|
1723
|
-
}
|
|
1724
|
-
const source = getMountedCameraFrameCaptureSource(selector);
|
|
1725
|
-
if (!source) continue;
|
|
1726
|
-
return { key, selector, source };
|
|
2147
|
+
function useCameraViewport(elementRef, options) {
|
|
2148
|
+
const registry = useContext(CameraViewportRegistryContext);
|
|
2149
|
+
if (!registry) {
|
|
2150
|
+
throw new Error("useCameraViewport must be used inside <MujocoCanvas>.");
|
|
1727
2151
|
}
|
|
1728
|
-
const
|
|
1729
|
-
|
|
1730
|
-
|
|
1731
|
-
|
|
1732
|
-
|
|
1733
|
-
|
|
2152
|
+
const optionsRef = useRef(options);
|
|
2153
|
+
optionsRef.current = options;
|
|
2154
|
+
useEffect(() => {
|
|
2155
|
+
const descriptor = {
|
|
2156
|
+
getElement: () => elementRef.current,
|
|
2157
|
+
getOptions: () => optionsRef.current,
|
|
2158
|
+
camera: null
|
|
1734
2159
|
};
|
|
2160
|
+
return registry.register(descriptor);
|
|
2161
|
+
}, [registry, elementRef]);
|
|
2162
|
+
}
|
|
2163
|
+
function CameraView({ className, style, ...options }) {
|
|
2164
|
+
const gl = useThree((state) => state.gl);
|
|
2165
|
+
const elementRef = useRef(null);
|
|
2166
|
+
if (!elementRef.current && typeof document !== "undefined") {
|
|
2167
|
+
elementRef.current = document.createElement("div");
|
|
1735
2168
|
}
|
|
1736
|
-
|
|
1737
|
-
|
|
1738
|
-
|
|
1739
|
-
|
|
1740
|
-
|
|
1741
|
-
|
|
1742
|
-
|
|
1743
|
-
|
|
1744
|
-
|
|
1745
|
-
|
|
1746
|
-
|
|
1747
|
-
|
|
1748
|
-
|
|
1749
|
-
|
|
1750
|
-
|
|
2169
|
+
useEffect(() => {
|
|
2170
|
+
const element = elementRef.current;
|
|
2171
|
+
const parent = gl.domElement.parentElement;
|
|
2172
|
+
if (!element || !parent) return;
|
|
2173
|
+
element.style.position = "absolute";
|
|
2174
|
+
element.style.overflow = "hidden";
|
|
2175
|
+
parent.appendChild(element);
|
|
2176
|
+
return () => {
|
|
2177
|
+
parent.removeChild(element);
|
|
2178
|
+
};
|
|
2179
|
+
}, [gl]);
|
|
2180
|
+
useEffect(() => {
|
|
2181
|
+
const element = elementRef.current;
|
|
2182
|
+
if (!element) return;
|
|
2183
|
+
element.className = className ?? "";
|
|
2184
|
+
if (style) Object.assign(element.style, style);
|
|
2185
|
+
}, [className, style]);
|
|
2186
|
+
useCameraViewport(elementRef, options);
|
|
1751
2187
|
return null;
|
|
1752
2188
|
}
|
|
1753
|
-
function
|
|
1754
|
-
|
|
1755
|
-
|
|
1756
|
-
|
|
1757
|
-
|
|
1758
|
-
|
|
1759
|
-
|
|
1760
|
-
missingKeys.push(key);
|
|
1761
|
-
continue;
|
|
1762
|
-
}
|
|
1763
|
-
resolved[key] = mountedSource;
|
|
1764
|
-
cameras.push({
|
|
1765
|
-
key,
|
|
1766
|
-
...options.defaults,
|
|
1767
|
-
...options.cameraOptions?.[key],
|
|
1768
|
-
...mountedSource.selector,
|
|
1769
|
-
source: mountedSource.source
|
|
1770
|
-
});
|
|
2189
|
+
function isTargetRef(target) {
|
|
2190
|
+
return Boolean(target && typeof target === "object" && "current" in target);
|
|
2191
|
+
}
|
|
2192
|
+
function resolveCanvasTarget(target) {
|
|
2193
|
+
const resolvedTarget = isTargetRef(target) ? target.current : target;
|
|
2194
|
+
if (!resolvedTarget) {
|
|
2195
|
+
throw new Error("No frame capture target is available.");
|
|
1771
2196
|
}
|
|
1772
|
-
if (
|
|
1773
|
-
|
|
1774
|
-
`Unable to resolve mounted MuJoCo camera source${missingKeys.length === 1 ? "" : "s"} for ${missingKeys.join(", ")}.`
|
|
1775
|
-
);
|
|
2197
|
+
if (resolvedTarget instanceof HTMLCanvasElement) {
|
|
2198
|
+
return resolvedTarget;
|
|
1776
2199
|
}
|
|
1777
|
-
|
|
1778
|
-
|
|
1779
|
-
|
|
1780
|
-
resolved,
|
|
1781
|
-
missingKeys
|
|
1782
|
-
};
|
|
1783
|
-
}
|
|
1784
|
-
function createMountedCameraFrameSequenceReadiness(plan) {
|
|
1785
|
-
const cameras = {};
|
|
1786
|
-
const resolvedKeys = plan.cameraKeys.filter((key) => Boolean(plan.resolved[key]));
|
|
1787
|
-
for (const key of plan.cameraKeys) {
|
|
1788
|
-
const resolved = plan.resolved[key];
|
|
1789
|
-
cameras[key] = resolved ? {
|
|
1790
|
-
key,
|
|
1791
|
-
ready: true,
|
|
1792
|
-
selector: resolved.selector,
|
|
1793
|
-
source: resolved.source,
|
|
1794
|
-
message: `Camera stream "${key}" resolves to ${resolved.source.kind}:${getCameraFrameCaptureSourceTarget(resolved.source)}.`
|
|
1795
|
-
} : {
|
|
1796
|
-
key,
|
|
1797
|
-
ready: false,
|
|
1798
|
-
message: `Camera stream "${key}" does not resolve to a mounted MuJoCo camera, site, or body.`
|
|
1799
|
-
};
|
|
2200
|
+
const canvas = resolvedTarget.querySelector("canvas");
|
|
2201
|
+
if (!canvas) {
|
|
2202
|
+
throw new Error("Frame capture target does not contain a canvas.");
|
|
1800
2203
|
}
|
|
1801
|
-
|
|
1802
|
-
const ready = missingKeys.length === 0;
|
|
1803
|
-
const status = ready ? MountedCameraFrameSequenceReadinessStatus.Ready : resolvedKeys.length > 0 ? MountedCameraFrameSequenceReadinessStatus.Partial : MountedCameraFrameSequenceReadinessStatus.Missing;
|
|
1804
|
-
return {
|
|
1805
|
-
ready,
|
|
1806
|
-
status,
|
|
1807
|
-
cameraKeys: [...plan.cameraKeys],
|
|
1808
|
-
resolvedKeys,
|
|
1809
|
-
missingKeys,
|
|
1810
|
-
cameras,
|
|
1811
|
-
message: ready ? `All ${plan.cameraKeys.length} requested camera stream${plan.cameraKeys.length === 1 ? "" : "s"} resolve to mounted MuJoCo sources.` : `Missing mounted MuJoCo source${missingKeys.length === 1 ? "" : "s"} for ${missingKeys.join(", ")}.`
|
|
1812
|
-
};
|
|
2204
|
+
return canvas;
|
|
1813
2205
|
}
|
|
1814
|
-
function
|
|
1815
|
-
return
|
|
2206
|
+
function waitForNextAnimationFrame() {
|
|
2207
|
+
return new Promise((resolve) => {
|
|
2208
|
+
requestAnimationFrame(() => resolve());
|
|
2209
|
+
});
|
|
1816
2210
|
}
|
|
1817
|
-
function
|
|
1818
|
-
const
|
|
1819
|
-
|
|
1820
|
-
|
|
1821
|
-
|
|
1822
|
-
options.expectedFrameCount ?? result.frameCount
|
|
1823
|
-
);
|
|
1824
|
-
const recordedFrameCount = normalizeFrameCount(result.frameCount);
|
|
1825
|
-
const streamSummaries = {};
|
|
1826
|
-
const streams = [];
|
|
1827
|
-
let missingFrameCount = 0;
|
|
1828
|
-
let completeStreamCount = 0;
|
|
1829
|
-
let resolvedOrRecordedStreamCount = 0;
|
|
1830
|
-
for (const key of cameraKeys) {
|
|
1831
|
-
const summary = result.cameraSummaries[key];
|
|
1832
|
-
const readiness = result.readiness.cameras[key];
|
|
1833
|
-
const source = summary?.source ?? readiness?.source;
|
|
1834
|
-
const ready = readiness?.ready ?? Boolean(summary);
|
|
1835
|
-
const recorded = normalizeFrameCount(summary?.frameCount);
|
|
1836
|
-
const missing = Math.max(expectedFrameCount - recorded, 0);
|
|
1837
|
-
const complete2 = ready && missing === 0;
|
|
1838
|
-
const status2 = complete2 ? MountedCameraFrameSequenceManifestStatus.Complete : ready || recorded > 0 ? MountedCameraFrameSequenceManifestStatus.Partial : MountedCameraFrameSequenceManifestStatus.Missing;
|
|
1839
|
-
const target = source ? getCameraFrameCaptureSourceTarget(source) : readiness?.message ? void 0 : "missing MuJoCo camera";
|
|
1840
|
-
const message = complete2 ? `Camera stream "${key}" recorded ${recorded} of ${expectedFrameCount} frame${expectedFrameCount === 1 ? "" : "s"}.` : ready || recorded > 0 ? `Camera stream "${key}" recorded ${recorded} of ${expectedFrameCount} frame${expectedFrameCount === 1 ? "" : "s"}.` : readiness?.message ?? `Camera stream "${key}" did not record any frames.`;
|
|
1841
|
-
const stream = {
|
|
1842
|
-
key,
|
|
1843
|
-
ready,
|
|
1844
|
-
complete: complete2,
|
|
1845
|
-
status: status2,
|
|
1846
|
-
source,
|
|
1847
|
-
selector: readiness?.selector,
|
|
1848
|
-
target,
|
|
1849
|
-
width: summary?.width,
|
|
1850
|
-
height: summary?.height,
|
|
1851
|
-
expectedFrameCount,
|
|
1852
|
-
recordedFrameCount: recorded,
|
|
1853
|
-
missingFrameCount: missing,
|
|
1854
|
-
firstFrameIndex: summary?.firstFrameIndex ?? null,
|
|
1855
|
-
lastFrameIndex: summary?.lastFrameIndex ?? null,
|
|
1856
|
-
firstTimestamp: summary?.firstTimestamp ?? null,
|
|
1857
|
-
lastTimestamp: summary?.lastTimestamp ?? null,
|
|
1858
|
-
message
|
|
1859
|
-
};
|
|
1860
|
-
streamSummaries[key] = stream;
|
|
1861
|
-
streams.push(stream);
|
|
1862
|
-
missingFrameCount += missing;
|
|
1863
|
-
if (complete2) completeStreamCount += 1;
|
|
1864
|
-
if (ready || recorded > 0) resolvedOrRecordedStreamCount += 1;
|
|
2211
|
+
async function captureFrame(options) {
|
|
2212
|
+
const type = options.type ?? "image/png";
|
|
2213
|
+
const canvas = resolveCanvasTarget(options.target);
|
|
2214
|
+
if (options.waitForAnimationFrame ?? true) {
|
|
2215
|
+
await waitForNextAnimationFrame();
|
|
1865
2216
|
}
|
|
1866
|
-
const complete = result.readiness.ready && streams.length === completeStreamCount && missingFrameCount === 0;
|
|
1867
|
-
const status = complete ? MountedCameraFrameSequenceManifestStatus.Complete : resolvedOrRecordedStreamCount > 0 ? MountedCameraFrameSequenceManifestStatus.Partial : MountedCameraFrameSequenceManifestStatus.Missing;
|
|
1868
2217
|
return {
|
|
1869
|
-
|
|
1870
|
-
|
|
1871
|
-
|
|
1872
|
-
status,
|
|
1873
|
-
cameraKeys,
|
|
1874
|
-
resolvedKeys: [...result.readiness.resolvedKeys],
|
|
1875
|
-
missingKeys: [...result.readiness.missingKeys],
|
|
1876
|
-
expectedFrameCount,
|
|
1877
|
-
recordedFrameCount,
|
|
1878
|
-
missingFrameCount,
|
|
1879
|
-
streamSummaries,
|
|
1880
|
-
streams,
|
|
1881
|
-
readiness: result.readiness,
|
|
1882
|
-
message: complete ? `All ${cameraKeys.length} camera stream${cameraKeys.length === 1 ? "" : "s"} recorded ${expectedFrameCount} frame${expectedFrameCount === 1 ? "" : "s"}.` : `Mounted camera sequence coverage is ${status}.`
|
|
2218
|
+
canvas,
|
|
2219
|
+
dataUrl: canvas.toDataURL(type, options.quality),
|
|
2220
|
+
type
|
|
1883
2221
|
};
|
|
1884
2222
|
}
|
|
1885
|
-
function
|
|
1886
|
-
|
|
1887
|
-
|
|
1888
|
-
|
|
1889
|
-
|
|
1890
|
-
|
|
1891
|
-
|
|
1892
|
-
|
|
1893
|
-
|
|
1894
|
-
|
|
1895
|
-
|
|
1896
|
-
|
|
1897
|
-
|
|
1898
|
-
|
|
1899
|
-
|
|
1900
|
-
|
|
1901
|
-
|
|
1902
|
-
|
|
1903
|
-
...restOptions,
|
|
1904
|
-
cameras: plan.cameras,
|
|
1905
|
-
requireMountedSources: restOptions.requireMountedSources ?? true
|
|
2223
|
+
async function captureFrameBlob(options) {
|
|
2224
|
+
const type = options.type ?? "image/png";
|
|
2225
|
+
const canvas = resolveCanvasTarget(options.target);
|
|
2226
|
+
if (options.waitForAnimationFrame ?? true) {
|
|
2227
|
+
await waitForNextAnimationFrame();
|
|
2228
|
+
}
|
|
2229
|
+
const blob = await new Promise((resolve, reject) => {
|
|
2230
|
+
canvas.toBlob(
|
|
2231
|
+
(nextBlob) => {
|
|
2232
|
+
if (nextBlob) {
|
|
2233
|
+
resolve(nextBlob);
|
|
2234
|
+
} else {
|
|
2235
|
+
reject(new Error("Canvas frame capture did not produce a Blob."));
|
|
2236
|
+
}
|
|
2237
|
+
},
|
|
2238
|
+
type,
|
|
2239
|
+
options.quality
|
|
2240
|
+
);
|
|
1906
2241
|
});
|
|
2242
|
+
return { canvas, blob, type };
|
|
2243
|
+
}
|
|
2244
|
+
function useFrameCapture(defaultOptions = {}) {
|
|
2245
|
+
const [status, setStatus] = useState("idle");
|
|
2246
|
+
const [error, setError] = useState(null);
|
|
2247
|
+
const reset = useCallback(() => {
|
|
2248
|
+
setStatus("idle");
|
|
2249
|
+
setError(null);
|
|
2250
|
+
}, []);
|
|
2251
|
+
const capture = useCallback(
|
|
2252
|
+
async (options = {}) => {
|
|
2253
|
+
setStatus("capturing");
|
|
2254
|
+
setError(null);
|
|
2255
|
+
try {
|
|
2256
|
+
const result = await captureFrame({ ...defaultOptions, ...options });
|
|
2257
|
+
setStatus("captured");
|
|
2258
|
+
return result;
|
|
2259
|
+
} catch (nextError) {
|
|
2260
|
+
const error2 = nextError instanceof Error ? nextError : new Error("Unable to capture the current canvas frame.");
|
|
2261
|
+
setError(error2);
|
|
2262
|
+
setStatus("error");
|
|
2263
|
+
throw error2;
|
|
2264
|
+
}
|
|
2265
|
+
},
|
|
2266
|
+
[defaultOptions]
|
|
2267
|
+
);
|
|
2268
|
+
const captureBlob = useCallback(
|
|
2269
|
+
async (options = {}) => {
|
|
2270
|
+
setStatus("capturing");
|
|
2271
|
+
setError(null);
|
|
2272
|
+
try {
|
|
2273
|
+
const result = await captureFrameBlob({
|
|
2274
|
+
...defaultOptions,
|
|
2275
|
+
...options
|
|
2276
|
+
});
|
|
2277
|
+
setStatus("captured");
|
|
2278
|
+
return result;
|
|
2279
|
+
} catch (nextError) {
|
|
2280
|
+
const error2 = nextError instanceof Error ? nextError : new Error("Unable to capture the current canvas frame.");
|
|
2281
|
+
setError(error2);
|
|
2282
|
+
setStatus("error");
|
|
2283
|
+
throw error2;
|
|
2284
|
+
}
|
|
2285
|
+
},
|
|
2286
|
+
[defaultOptions]
|
|
2287
|
+
);
|
|
1907
2288
|
return {
|
|
1908
|
-
|
|
1909
|
-
|
|
1910
|
-
|
|
2289
|
+
status,
|
|
2290
|
+
error,
|
|
2291
|
+
isCapturing: status === "capturing",
|
|
2292
|
+
capture,
|
|
2293
|
+
captureBlob,
|
|
2294
|
+
reset
|
|
1911
2295
|
};
|
|
1912
2296
|
}
|
|
1913
2297
|
var _raycaster = new THREE12.Raycaster();
|
|
@@ -2358,10 +2742,11 @@ function useMujoco() {
|
|
|
2358
2742
|
};
|
|
2359
2743
|
}
|
|
2360
2744
|
function useBeforePhysicsStep(callback) {
|
|
2361
|
-
const { beforeStepCallbacks } = useMujocoContext();
|
|
2745
|
+
const { beforeStepCallbacks, mjModelRef } = useMujocoContext();
|
|
2362
2746
|
const callbackRef = useRef(callback);
|
|
2363
2747
|
callbackRef.current = callback;
|
|
2364
2748
|
useEffect(() => {
|
|
2749
|
+
assertMainThreadOwnership(mjModelRef.current);
|
|
2365
2750
|
const wrapped = (input) => callbackRef.current(input);
|
|
2366
2751
|
beforeStepCallbacks.current.add(wrapped);
|
|
2367
2752
|
return () => {
|
|
@@ -2370,10 +2755,11 @@ function useBeforePhysicsStep(callback) {
|
|
|
2370
2755
|
}, [beforeStepCallbacks]);
|
|
2371
2756
|
}
|
|
2372
2757
|
function useAfterPhysicsStep(callback) {
|
|
2373
|
-
const { afterStepCallbacks } = useMujocoContext();
|
|
2758
|
+
const { afterStepCallbacks, mjModelRef } = useMujocoContext();
|
|
2374
2759
|
const callbackRef = useRef(callback);
|
|
2375
2760
|
callbackRef.current = callback;
|
|
2376
2761
|
useEffect(() => {
|
|
2762
|
+
assertMainThreadOwnership(mjModelRef.current);
|
|
2377
2763
|
const wrapped = (input) => callbackRef.current(input);
|
|
2378
2764
|
afterStepCallbacks.current.add(wrapped);
|
|
2379
2765
|
return () => {
|
|
@@ -2393,6 +2779,7 @@ function MujocoSimProvider({
|
|
|
2393
2779
|
timestep,
|
|
2394
2780
|
integrator,
|
|
2395
2781
|
substeps,
|
|
2782
|
+
physicsBudgetMs = 8,
|
|
2396
2783
|
paused,
|
|
2397
2784
|
speed,
|
|
2398
2785
|
interpolate,
|
|
@@ -2400,7 +2787,32 @@ function MujocoSimProvider({
|
|
|
2400
2787
|
hiddenBodies,
|
|
2401
2788
|
children
|
|
2402
2789
|
}) {
|
|
2403
|
-
const { gl, camera, scene } = useThree();
|
|
2790
|
+
const { gl, camera, scene, invalidate } = useThree();
|
|
2791
|
+
const modelIdentity = sceneIdentity(config);
|
|
2792
|
+
const performanceRef = useRef({ steps: 0, physicsMs: 0, realTimeFactor: 0, backlogSteps: 0, budgetExceeded: false });
|
|
2793
|
+
const displayWaiters = useRef(/* @__PURE__ */ new Set());
|
|
2794
|
+
useFrame(() => {
|
|
2795
|
+
if (displayWaiters.current.size) {
|
|
2796
|
+
const waiters = [...displayWaiters.current];
|
|
2797
|
+
displayWaiters.current.clear();
|
|
2798
|
+
queueMicrotask(() => {
|
|
2799
|
+
scene.updateMatrixWorld(true);
|
|
2800
|
+
for (const ready of waiters) ready();
|
|
2801
|
+
});
|
|
2802
|
+
}
|
|
2803
|
+
}, 0);
|
|
2804
|
+
const waitForDisplay = () => new Promise((resolve, reject) => {
|
|
2805
|
+
const ready = () => {
|
|
2806
|
+
clearTimeout(timer);
|
|
2807
|
+
resolve();
|
|
2808
|
+
};
|
|
2809
|
+
const timer = setTimeout(() => {
|
|
2810
|
+
displayWaiters.current.delete(ready);
|
|
2811
|
+
reject(new Error("Canvas did not produce an observation frame within 5 seconds"));
|
|
2812
|
+
}, 5e3);
|
|
2813
|
+
displayWaiters.current.add(ready);
|
|
2814
|
+
invalidate();
|
|
2815
|
+
});
|
|
2404
2816
|
const [status, setStatus] = useState("loading");
|
|
2405
2817
|
const mjModelRef = useRef(null);
|
|
2406
2818
|
const mjDataRef = useRef(null);
|
|
@@ -2436,18 +2848,23 @@ function MujocoSimProvider({
|
|
|
2436
2848
|
const bodyReloadTimerRef = useRef(null);
|
|
2437
2849
|
configRef.current = config;
|
|
2438
2850
|
mujocoRef.current = mujoco;
|
|
2439
|
-
|
|
2440
|
-
|
|
2851
|
+
useEffect(() => {
|
|
2852
|
+
pausedRef.current = paused ?? false;
|
|
2853
|
+
}, [paused]);
|
|
2854
|
+
useEffect(() => {
|
|
2855
|
+
speedRef.current = speed ?? 1;
|
|
2856
|
+
}, [speed]);
|
|
2441
2857
|
substepsRef.current = substeps ?? 1;
|
|
2442
2858
|
interpolateRef.current = interpolate ?? false;
|
|
2443
2859
|
useEffect(() => {
|
|
2444
2860
|
if (!gravity) return;
|
|
2445
2861
|
const model = mjModelRef.current;
|
|
2446
2862
|
if (!model?.opt?.gravity) return;
|
|
2863
|
+
assertMainThreadOwnership(model);
|
|
2447
2864
|
model.opt.gravity[0] = gravity[0];
|
|
2448
2865
|
model.opt.gravity[1] = gravity[1];
|
|
2449
2866
|
model.opt.gravity[2] = gravity[2];
|
|
2450
|
-
}, [gravity]);
|
|
2867
|
+
}, [gravity?.[0], gravity?.[1], gravity?.[2]]);
|
|
2451
2868
|
const integratorRef = useRef(integrator);
|
|
2452
2869
|
integratorRef.current = integrator;
|
|
2453
2870
|
const applyIntegrator = (model) => {
|
|
@@ -2455,12 +2872,16 @@ function MujocoSimProvider({
|
|
|
2455
2872
|
if (value !== void 0) model.opt.integrator = { Euler: 0, RK4: 1, implicit: 2, implicitfast: 3, discrete: 4 }[value];
|
|
2456
2873
|
};
|
|
2457
2874
|
useEffect(() => {
|
|
2458
|
-
if (mjModelRef.current)
|
|
2875
|
+
if (mjModelRef.current) {
|
|
2876
|
+
assertMainThreadOwnership(mjModelRef.current);
|
|
2877
|
+
applyIntegrator(mjModelRef.current);
|
|
2878
|
+
}
|
|
2459
2879
|
}, [integrator]);
|
|
2460
2880
|
useEffect(() => {
|
|
2461
2881
|
if (timestep === void 0) return;
|
|
2462
2882
|
const model = mjModelRef.current;
|
|
2463
2883
|
if (!model?.opt) return;
|
|
2884
|
+
assertMainThreadOwnership(model);
|
|
2464
2885
|
model.opt.timestep = timestep;
|
|
2465
2886
|
}, [timestep]);
|
|
2466
2887
|
const hiddenBodiesKey = (hiddenBodies ?? []).join("\0");
|
|
@@ -2498,6 +2919,7 @@ function MujocoSimProvider({
|
|
|
2498
2919
|
}
|
|
2499
2920
|
useEffect(() => {
|
|
2500
2921
|
let disposed = false;
|
|
2922
|
+
setStatus("loading");
|
|
2501
2923
|
(async () => {
|
|
2502
2924
|
try {
|
|
2503
2925
|
const result = await loadScene(mujoco, buildMergedConfig(config));
|
|
@@ -2532,6 +2954,7 @@ function MujocoSimProvider({
|
|
|
2532
2954
|
})();
|
|
2533
2955
|
return () => {
|
|
2534
2956
|
disposed = true;
|
|
2957
|
+
closeModelRuntime(mjModelRef.current);
|
|
2535
2958
|
mjDataRef.current?.delete();
|
|
2536
2959
|
mjModelRef.current?.delete();
|
|
2537
2960
|
mjModelRef.current = null;
|
|
@@ -2543,7 +2966,7 @@ function MujocoSimProvider({
|
|
|
2543
2966
|
} catch {
|
|
2544
2967
|
}
|
|
2545
2968
|
};
|
|
2546
|
-
}, [mujoco,
|
|
2969
|
+
}, [mujoco, modelIdentity]);
|
|
2547
2970
|
useEffect(() => {
|
|
2548
2971
|
if (status === "ready") {
|
|
2549
2972
|
const api2 = apiRef.current;
|
|
@@ -2561,66 +2984,43 @@ function MujocoSimProvider({
|
|
|
2561
2984
|
const model = mjModelRef.current;
|
|
2562
2985
|
const data = mjDataRef.current;
|
|
2563
2986
|
if (!model || !data) return;
|
|
2564
|
-
|
|
2565
|
-
|
|
2566
|
-
|
|
2567
|
-
data.qfrc_applied[i] = 0;
|
|
2568
|
-
}
|
|
2569
|
-
for (const cb of beforeStepCallbacks.current) {
|
|
2570
|
-
cb({ model, data });
|
|
2987
|
+
if (isWorkerOwned(model)) {
|
|
2988
|
+
interpolationStateRef.current.valid = false;
|
|
2989
|
+
return;
|
|
2571
2990
|
}
|
|
2572
|
-
const
|
|
2573
|
-
if (
|
|
2574
|
-
|
|
2575
|
-
|
|
2576
|
-
|
|
2577
|
-
|
|
2578
|
-
|
|
2579
|
-
|
|
2580
|
-
|
|
2581
|
-
|
|
2582
|
-
|
|
2583
|
-
|
|
2584
|
-
for (let s = 0; s < numSubsteps; s++) {
|
|
2585
|
-
mujocoRef.current.mj_step(model, data);
|
|
2586
|
-
}
|
|
2587
|
-
}
|
|
2588
|
-
}
|
|
2589
|
-
} else if (stepsToRunRef.current > 0) {
|
|
2991
|
+
const queued = stepsToRunRef.current > 0;
|
|
2992
|
+
if (pausedRef.current && !queued) return;
|
|
2993
|
+
const startTime = data.time;
|
|
2994
|
+
const dt = model.opt.timestep;
|
|
2995
|
+
if (!Number.isFinite(dt) || dt <= 0) return;
|
|
2996
|
+
const substeps2 = Math.max(1, Math.floor(substepsRef.current));
|
|
2997
|
+
const frameTime = Math.max(0, Math.min(delta, 1 / 15) * speedRef.current);
|
|
2998
|
+
if (!queued && interpolateRef.current) physicsAccumulatorRef.current += frameTime;
|
|
2999
|
+
const desired = queued ? stepsToRunRef.current : interpolateRef.current ? Math.floor(physicsAccumulatorRef.current / (dt * substeps2)) * substeps2 : Math.ceil(frameTime / (dt * substeps2)) * substeps2;
|
|
3000
|
+
for (let i = 0; i < model.nv; i++) data.qfrc_applied[i] = 0;
|
|
3001
|
+
for (const cb of beforeStepCallbacks.current) cb({ model, data });
|
|
3002
|
+
if (interpolateRef.current) {
|
|
2590
3003
|
ensureInterpolationBuffers(model);
|
|
2591
|
-
copyBodyPose(data, interpolationStateRef.current.previousXpos, interpolationStateRef.current.previousXquat);
|
|
2592
|
-
|
|
2593
|
-
|
|
2594
|
-
|
|
3004
|
+
if (!interpolationStateRef.current.valid) copyBodyPose(data, interpolationStateRef.current.previousXpos, interpolationStateRef.current.previousXquat);
|
|
3005
|
+
}
|
|
3006
|
+
const measured = runStepBudget(desired, () => {
|
|
3007
|
+
if (interpolateRef.current) copyBodyPose(data, interpolationStateRef.current.previousXpos, interpolationStateRef.current.previousXquat);
|
|
3008
|
+
mujocoRef.current.mj_step(model, data);
|
|
3009
|
+
}, physicsBudgetMs);
|
|
3010
|
+
if (queued) stepsToRunRef.current -= measured.steps;
|
|
3011
|
+
performanceRef.current = {
|
|
3012
|
+
...measured,
|
|
3013
|
+
backlogSteps: Math.max(0, desired - measured.steps),
|
|
3014
|
+
realTimeFactor: (data.time - startTime) / Math.max(delta, 1e-6)
|
|
3015
|
+
};
|
|
3016
|
+
if (interpolateRef.current) {
|
|
2595
3017
|
copyBodyPose(data, interpolationStateRef.current.currentXpos, interpolationStateRef.current.currentXquat);
|
|
2596
|
-
|
|
2597
|
-
|
|
2598
|
-
|
|
2599
|
-
|
|
2600
|
-
|
|
2601
|
-
const clampedDelta = Math.min(delta, 1 / 15);
|
|
2602
|
-
physicsAccumulatorRef.current += clampedDelta * speedRef.current;
|
|
2603
|
-
const stepDt = Math.max((model.opt?.timestep ?? 2e-3) * Math.max(1, numSubsteps), 1e-6);
|
|
2604
|
-
let stepped = false;
|
|
2605
|
-
while (physicsAccumulatorRef.current >= stepDt) {
|
|
2606
|
-
copyBodyPose(data, interpolationStateRef.current.previousXpos, interpolationStateRef.current.previousXquat);
|
|
2607
|
-
for (let s = 0; s < numSubsteps; s++) {
|
|
2608
|
-
mujocoRef.current.mj_step(model, data);
|
|
2609
|
-
}
|
|
2610
|
-
copyBodyPose(data, interpolationStateRef.current.currentXpos, interpolationStateRef.current.currentXquat);
|
|
2611
|
-
physicsAccumulatorRef.current -= stepDt;
|
|
2612
|
-
stepped = true;
|
|
2613
|
-
}
|
|
2614
|
-
if (!interpolationStateRef.current.valid) {
|
|
2615
|
-
copyBodyPose(data, interpolationStateRef.current.previousXpos, interpolationStateRef.current.previousXquat);
|
|
2616
|
-
copyBodyPose(data, interpolationStateRef.current.currentXpos, interpolationStateRef.current.currentXquat);
|
|
2617
|
-
}
|
|
2618
|
-
interpolationStateRef.current.alpha = Math.min(Math.max(physicsAccumulatorRef.current / stepDt, 0), 1);
|
|
3018
|
+
if (!queued) {
|
|
3019
|
+
physicsAccumulatorRef.current -= measured.steps * dt;
|
|
3020
|
+
if (measured.budgetExceeded) physicsAccumulatorRef.current %= dt * substeps2;
|
|
3021
|
+
} else physicsAccumulatorRef.current = 0;
|
|
3022
|
+
interpolationStateRef.current.alpha = queued ? 1 : Math.min(1, physicsAccumulatorRef.current / dt);
|
|
2619
3023
|
interpolationStateRef.current.valid = true;
|
|
2620
|
-
if (!stepped) {
|
|
2621
|
-
onStepRef.current?.({ time: data.time, model, data });
|
|
2622
|
-
return;
|
|
2623
|
-
}
|
|
2624
3024
|
}
|
|
2625
3025
|
for (const cb of afterStepCallbacks.current) {
|
|
2626
3026
|
cb({ model, data });
|
|
@@ -2673,6 +3073,7 @@ function MujocoSimProvider({
|
|
|
2673
3073
|
pausedRef.current = p;
|
|
2674
3074
|
}, []);
|
|
2675
3075
|
const step = useCallback((n = 1) => {
|
|
3076
|
+
if (!Number.isSafeInteger(n) || n <= 0) throw new Error("Step count must be a positive safe integer");
|
|
2676
3077
|
stepsToRunRef.current = n;
|
|
2677
3078
|
}, []);
|
|
2678
3079
|
useCallback((steps = 1) => {
|
|
@@ -3091,6 +3492,7 @@ function MujocoSimProvider({
|
|
|
3091
3492
|
const model = mjModelRef.current;
|
|
3092
3493
|
const data = mjDataRef.current;
|
|
3093
3494
|
if (!model || !data) return null;
|
|
3495
|
+
assertMainThreadOwnership(model);
|
|
3094
3496
|
_rayPnt[0] = origin.x;
|
|
3095
3497
|
_rayPnt[1] = origin.y;
|
|
3096
3498
|
_rayPnt[2] = origin.z;
|
|
@@ -3162,6 +3564,7 @@ function MujocoSimProvider({
|
|
|
3162
3564
|
const loadSceneApi = useCallback(async (newConfig) => {
|
|
3163
3565
|
const gen = ++loadGenRef.current;
|
|
3164
3566
|
try {
|
|
3567
|
+
closeModelRuntime(mjModelRef.current);
|
|
3165
3568
|
mjDataRef.current?.delete();
|
|
3166
3569
|
mjModelRef.current?.delete();
|
|
3167
3570
|
mjModelRef.current = null;
|
|
@@ -3317,6 +3720,7 @@ function MujocoSimProvider({
|
|
|
3317
3720
|
if (!model || !data) {
|
|
3318
3721
|
throw new Error("MuJoCo scene is not ready for camera sequence stepping.");
|
|
3319
3722
|
}
|
|
3723
|
+
let sliceStarted = performance.now();
|
|
3320
3724
|
for (let stepIndex = 0; stepIndex < steps; stepIndex += 1) {
|
|
3321
3725
|
for (let i = 0; i < model.nv; i += 1) {
|
|
3322
3726
|
data.qfrc_applied[i] = 0;
|
|
@@ -3336,6 +3740,11 @@ function MujocoSimProvider({
|
|
|
3336
3740
|
cb({ model, data });
|
|
3337
3741
|
}
|
|
3338
3742
|
onStepRef.current?.({ time: data.time, model, data });
|
|
3743
|
+
if (performance.now() - sliceStarted >= 8) {
|
|
3744
|
+
await new Promise((resolve) => setTimeout(resolve, 0));
|
|
3745
|
+
throwIfCameraSequenceAborted(options.signal);
|
|
3746
|
+
sliceStarted = performance.now();
|
|
3747
|
+
}
|
|
3339
3748
|
await options.onAfterStep?.({
|
|
3340
3749
|
frameIndex,
|
|
3341
3750
|
stepIndex,
|
|
@@ -3543,7 +3952,14 @@ function MujocoSimProvider({
|
|
|
3543
3952
|
model.geom_size[id * 3 + 2] = size[2];
|
|
3544
3953
|
}, []);
|
|
3545
3954
|
const api = useMemo(
|
|
3546
|
-
() => ({
|
|
3955
|
+
() => guardRuntimeOwnership({
|
|
3956
|
+
createRuntime: (options) => {
|
|
3957
|
+
if (beforeStepCallbacks.current.size || afterStepCallbacks.current.size) {
|
|
3958
|
+
return Promise.reject(new Error("Unmount physics callbacks/controllers before creating a worker runtime; use runtime.advance({controls}) for policy actions."));
|
|
3959
|
+
}
|
|
3960
|
+
return attachSimulationRuntime(mujocoRef.current, apiRef.current, waitForDisplay, invalidate, options);
|
|
3961
|
+
},
|
|
3962
|
+
getPerformance: () => ({ ...performanceRef.current }),
|
|
3547
3963
|
get status() {
|
|
3548
3964
|
return status;
|
|
3549
3965
|
},
|
|
@@ -3574,7 +3990,10 @@ function MujocoSimProvider({
|
|
|
3574
3990
|
setExternalForce,
|
|
3575
3991
|
applyGeneralizedForce,
|
|
3576
3992
|
getSensorData,
|
|
3577
|
-
getContacts
|
|
3993
|
+
getContacts: () => {
|
|
3994
|
+
const model = mjModelRef.current;
|
|
3995
|
+
return model && isWorkerOwned(model) ? runtimeContacts(model) : getContacts();
|
|
3996
|
+
},
|
|
3578
3997
|
getBodies,
|
|
3579
3998
|
getJoints,
|
|
3580
3999
|
getGeoms,
|
|
@@ -3610,7 +4029,7 @@ function MujocoSimProvider({
|
|
|
3610
4029
|
setGeomSize,
|
|
3611
4030
|
mjModelRef,
|
|
3612
4031
|
mjDataRef
|
|
3613
|
-
}),
|
|
4032
|
+
}, () => mjModelRef.current),
|
|
3614
4033
|
[
|
|
3615
4034
|
status,
|
|
3616
4035
|
reset,
|
|
@@ -3705,6 +4124,45 @@ function MujocoSimProvider({
|
|
|
3705
4124
|
/* @__PURE__ */ jsx(CameraViewportProvider, { children })
|
|
3706
4125
|
] });
|
|
3707
4126
|
}
|
|
4127
|
+
var runtimeMutators = /* @__PURE__ */ new Set([
|
|
4128
|
+
"reset",
|
|
4129
|
+
"setSpeed",
|
|
4130
|
+
"togglePause",
|
|
4131
|
+
"setPaused",
|
|
4132
|
+
"step",
|
|
4133
|
+
"applyKeyframe",
|
|
4134
|
+
"saveState",
|
|
4135
|
+
"restoreState",
|
|
4136
|
+
"setQpos",
|
|
4137
|
+
"setQvel",
|
|
4138
|
+
"setCtrl",
|
|
4139
|
+
"applyForce",
|
|
4140
|
+
"applyTorque",
|
|
4141
|
+
"setExternalForce",
|
|
4142
|
+
"applyGeneralizedForce",
|
|
4143
|
+
"setGravity",
|
|
4144
|
+
"setTimestep",
|
|
4145
|
+
"setBodyMass",
|
|
4146
|
+
"setGeomFriction",
|
|
4147
|
+
"setGeomSize",
|
|
4148
|
+
"loadScene",
|
|
4149
|
+
"loadFromFiles",
|
|
4150
|
+
"addBody",
|
|
4151
|
+
"removeBody",
|
|
4152
|
+
"recompile",
|
|
4153
|
+
"recordCameraSequence"
|
|
4154
|
+
]);
|
|
4155
|
+
function guardRuntimeOwnership(api, model) {
|
|
4156
|
+
for (const key of runtimeMutators) {
|
|
4157
|
+
const record = api;
|
|
4158
|
+
const method = record[key];
|
|
4159
|
+
record[key] = (...args) => {
|
|
4160
|
+
assertMainThreadOwnership(model());
|
|
4161
|
+
return method(...args);
|
|
4162
|
+
};
|
|
4163
|
+
}
|
|
4164
|
+
return api;
|
|
4165
|
+
}
|
|
3708
4166
|
var MujocoCanvas = forwardRef(
|
|
3709
4167
|
function MujocoCanvas2({
|
|
3710
4168
|
config,
|
|
@@ -3717,6 +4175,7 @@ var MujocoCanvas = forwardRef(
|
|
|
3717
4175
|
timestep,
|
|
3718
4176
|
integrator,
|
|
3719
4177
|
substeps,
|
|
4178
|
+
physicsBudgetMs,
|
|
3720
4179
|
paused,
|
|
3721
4180
|
speed,
|
|
3722
4181
|
interpolate,
|
|
@@ -3754,6 +4213,7 @@ var MujocoCanvas = forwardRef(
|
|
|
3754
4213
|
timestep,
|
|
3755
4214
|
integrator,
|
|
3756
4215
|
substeps,
|
|
4216
|
+
physicsBudgetMs,
|
|
3757
4217
|
paused,
|
|
3758
4218
|
speed,
|
|
3759
4219
|
interpolate,
|
|
@@ -5248,10 +5708,10 @@ function useCameraStream(canvasRef, options) {
|
|
|
5248
5708
|
if (!api || !canvasRef.current) return;
|
|
5249
5709
|
const opts = optionsRef.current;
|
|
5250
5710
|
if (opts.paused) return;
|
|
5251
|
-
|
|
5252
|
-
|
|
5253
|
-
|
|
5254
|
-
|
|
5711
|
+
const fps = opts.fps ?? 10;
|
|
5712
|
+
if (!Number.isFinite(fps) || fps <= 0) throw new Error("Camera stream fps must be positive and finite");
|
|
5713
|
+
elapsedRef.current += delta;
|
|
5714
|
+
if (elapsedRef.current < 1 / fps) return;
|
|
5255
5715
|
if (inFlightRef.current) return;
|
|
5256
5716
|
elapsedRef.current = 0;
|
|
5257
5717
|
const signature = streamSignature(opts);
|
|
@@ -5272,7 +5732,10 @@ function useCameraStream(canvasRef, options) {
|
|
|
5272
5732
|
canvas.height = frame.height;
|
|
5273
5733
|
}
|
|
5274
5734
|
ctx.drawImage(frame.canvas, 0, 0);
|
|
5275
|
-
}).catch(() => {
|
|
5735
|
+
}).catch((error) => {
|
|
5736
|
+
const failure = error instanceof Error ? error : new Error(String(error));
|
|
5737
|
+
if (optionsRef.current.onError) optionsRef.current.onError(failure);
|
|
5738
|
+
else console.error("Camera stream capture failed", failure);
|
|
5276
5739
|
}).finally(() => {
|
|
5277
5740
|
inFlightRef.current = false;
|
|
5278
5741
|
});
|
|
@@ -7407,7 +7870,9 @@ function isPolicyActionChunk(value) {
|
|
|
7407
7870
|
function toPolicyActions(output) {
|
|
7408
7871
|
return isPolicyActionChunk(output) ? [...output] : [output];
|
|
7409
7872
|
}
|
|
7410
|
-
function enqueuePolicyActions(queue, actions, observation, strategy) {
|
|
7873
|
+
function enqueuePolicyActions(queue, actions, observation, strategy, limit = 256) {
|
|
7874
|
+
if (!Number.isSafeInteger(limit) || limit < 1) throw new Error("maxQueuedActions must be a positive safe integer");
|
|
7875
|
+
if ((strategy === "replace" ? 0 : queue.length) + actions.length > limit) throw new Error(`Policy action queue exceeds maxQueuedActions (${limit})`);
|
|
7411
7876
|
if (strategy === "replace") {
|
|
7412
7877
|
queue.splice(0, queue.length);
|
|
7413
7878
|
}
|
|
@@ -7428,7 +7893,6 @@ function usePolicy(config) {
|
|
|
7428
7893
|
const clearQueue = useCallback(() => {
|
|
7429
7894
|
epochRef.current += 1;
|
|
7430
7895
|
actionQueueRef.current.splice(0, actionQueueRef.current.length);
|
|
7431
|
-
inFlightRef.current = false;
|
|
7432
7896
|
lastErrorRef.current = null;
|
|
7433
7897
|
}, []);
|
|
7434
7898
|
const reset = useCallback(() => {
|
|
@@ -7437,6 +7901,13 @@ function usePolicy(config) {
|
|
|
7437
7901
|
lastObservationRef.current = null;
|
|
7438
7902
|
lastActionRef.current = null;
|
|
7439
7903
|
}, [clearQueue]);
|
|
7904
|
+
useEffect(() => {
|
|
7905
|
+
isRunningRef.current = configRef.current.enabled ?? true;
|
|
7906
|
+
return () => {
|
|
7907
|
+
epochRef.current++;
|
|
7908
|
+
isRunningRef.current = false;
|
|
7909
|
+
};
|
|
7910
|
+
}, []);
|
|
7440
7911
|
useBeforePhysicsStep(({ model, data }) => {
|
|
7441
7912
|
if (!isRunningRef.current) return;
|
|
7442
7913
|
const cfg = configRef.current;
|
|
@@ -7469,7 +7940,8 @@ function usePolicy(config) {
|
|
|
7469
7940
|
actionQueueRef.current,
|
|
7470
7941
|
toPolicyActions(output),
|
|
7471
7942
|
observation,
|
|
7472
|
-
cfg.queueStrategy ?? "append"
|
|
7943
|
+
cfg.queueStrategy ?? "append",
|
|
7944
|
+
cfg.maxQueuedActions
|
|
7473
7945
|
);
|
|
7474
7946
|
lastErrorRef.current = null;
|
|
7475
7947
|
}).catch((error) => {
|
|
@@ -7477,7 +7949,6 @@ function usePolicy(config) {
|
|
|
7477
7949
|
lastErrorRef.current = error;
|
|
7478
7950
|
cfg.onError?.(error);
|
|
7479
7951
|
}).finally(() => {
|
|
7480
|
-
if (epoch !== epochRef.current) return;
|
|
7481
7952
|
inFlightRef.current = false;
|
|
7482
7953
|
});
|
|
7483
7954
|
} else {
|
|
@@ -7487,7 +7958,8 @@ function usePolicy(config) {
|
|
|
7487
7958
|
actionQueueRef.current,
|
|
7488
7959
|
actions,
|
|
7489
7960
|
observation,
|
|
7490
|
-
cfg.queueStrategy ?? "append"
|
|
7961
|
+
cfg.queueStrategy ?? "append",
|
|
7962
|
+
cfg.maxQueuedActions
|
|
7491
7963
|
);
|
|
7492
7964
|
} else {
|
|
7493
7965
|
const [action, ...queuedActions2] = actions;
|
|
@@ -7496,7 +7968,8 @@ function usePolicy(config) {
|
|
|
7496
7968
|
actionQueueRef.current,
|
|
7497
7969
|
queuedActions2,
|
|
7498
7970
|
observation,
|
|
7499
|
-
cfg.queueStrategy ?? "append"
|
|
7971
|
+
cfg.queueStrategy ?? "append",
|
|
7972
|
+
cfg.maxQueuedActions
|
|
7500
7973
|
);
|
|
7501
7974
|
cfg.onAction({ action, observation, model, data });
|
|
7502
7975
|
lastActionRef.current = action;
|
|
@@ -8197,125 +8670,6 @@ function useCameraFrameCapture(defaultOptions = {}) {
|
|
|
8197
8670
|
reset
|
|
8198
8671
|
};
|
|
8199
8672
|
}
|
|
8200
|
-
|
|
8201
|
-
// src/policyCameraFrames.ts
|
|
8202
|
-
function addPolicyImageAliases(images, stream, frame, includeObservationImageAliases) {
|
|
8203
|
-
const keys = /* @__PURE__ */ new Set();
|
|
8204
|
-
keys.add(stream.key);
|
|
8205
|
-
for (const alias of stream.aliases ?? []) keys.add(alias);
|
|
8206
|
-
if (includeObservationImageAliases) {
|
|
8207
|
-
keys.add(`observation.images.${stream.key}`);
|
|
8208
|
-
for (const alias of stream.aliases ?? []) {
|
|
8209
|
-
keys.add(`observation.images.${alias}`);
|
|
8210
|
-
}
|
|
8211
|
-
}
|
|
8212
|
-
for (const key of keys) {
|
|
8213
|
-
images[key] = frame.dataUrl;
|
|
8214
|
-
}
|
|
8215
|
-
}
|
|
8216
|
-
function describeFrameSource(key, frame) {
|
|
8217
|
-
return `${key}:${frame.source.kind}`;
|
|
8218
|
-
}
|
|
8219
|
-
function hasExplicitPolicyCameraSource(options) {
|
|
8220
|
-
return Boolean(
|
|
8221
|
-
options?.camera || options?.position || options?.quaternion || options?.source
|
|
8222
|
-
);
|
|
8223
|
-
}
|
|
8224
|
-
function createPolicyCameraFrameCapturePlan(options) {
|
|
8225
|
-
const {
|
|
8226
|
-
cameraKeys,
|
|
8227
|
-
defaults,
|
|
8228
|
-
streamOptions,
|
|
8229
|
-
includeObservationImageAliases,
|
|
8230
|
-
requireAll,
|
|
8231
|
-
...sourceOptions
|
|
8232
|
-
} = options;
|
|
8233
|
-
const mountedPlan = createMountedCameraFrameSequencePlan(cameraKeys, {
|
|
8234
|
-
...sourceOptions,
|
|
8235
|
-
defaults,
|
|
8236
|
-
cameraOptions: streamOptions
|
|
8237
|
-
});
|
|
8238
|
-
const streams = [];
|
|
8239
|
-
const missingKeys = new Set(mountedPlan.missingKeys);
|
|
8240
|
-
for (const key of cameraKeys) {
|
|
8241
|
-
const perStreamOptions = streamOptions?.[key];
|
|
8242
|
-
if (hasExplicitPolicyCameraSource(perStreamOptions)) {
|
|
8243
|
-
missingKeys.delete(key);
|
|
8244
|
-
streams.push({
|
|
8245
|
-
...defaults,
|
|
8246
|
-
...perStreamOptions,
|
|
8247
|
-
key,
|
|
8248
|
-
aliases: perStreamOptions?.aliases
|
|
8249
|
-
});
|
|
8250
|
-
continue;
|
|
8251
|
-
}
|
|
8252
|
-
const mountedCamera = mountedPlan.cameras.find((camera) => camera.key === key);
|
|
8253
|
-
if (!mountedCamera) continue;
|
|
8254
|
-
const { key: _mountedKey, ...captureOptions } = mountedCamera;
|
|
8255
|
-
streams.push({
|
|
8256
|
-
...captureOptions,
|
|
8257
|
-
key,
|
|
8258
|
-
aliases: perStreamOptions?.aliases
|
|
8259
|
-
});
|
|
8260
|
-
}
|
|
8261
|
-
const result = {
|
|
8262
|
-
cameraKeys: [...cameraKeys],
|
|
8263
|
-
streams,
|
|
8264
|
-
includeObservationImageAliases,
|
|
8265
|
-
mountedPlan,
|
|
8266
|
-
missingKeys: [...missingKeys]
|
|
8267
|
-
};
|
|
8268
|
-
if (requireAll && result.missingKeys.length > 0) {
|
|
8269
|
-
throw new Error(
|
|
8270
|
-
`Unable to resolve policy camera stream${result.missingKeys.length === 1 ? "" : "s"} for ${result.missingKeys.join(", ")}.`
|
|
8271
|
-
);
|
|
8272
|
-
}
|
|
8273
|
-
return result;
|
|
8274
|
-
}
|
|
8275
|
-
function createPolicyCameraFrameCapturePlanFromApi(api, options) {
|
|
8276
|
-
return createPolicyCameraFrameCapturePlan({
|
|
8277
|
-
...options,
|
|
8278
|
-
cameras: api.getCameras(),
|
|
8279
|
-
sites: api.getSites(),
|
|
8280
|
-
bodies: api.getBodies()
|
|
8281
|
-
});
|
|
8282
|
-
}
|
|
8283
|
-
async function capturePolicyCameraFrames(target, options) {
|
|
8284
|
-
const includeObservationImageAliases = options.includeObservationImageAliases ?? true;
|
|
8285
|
-
const entries = await Promise.all(
|
|
8286
|
-
options.streams.map(async ({ key, aliases, ...captureOptions }) => {
|
|
8287
|
-
const frame = await target.captureCameraFrame(captureOptions);
|
|
8288
|
-
return [key, { frame, aliases }];
|
|
8289
|
-
})
|
|
8290
|
-
);
|
|
8291
|
-
const frames = {};
|
|
8292
|
-
const images = {};
|
|
8293
|
-
const sourceParts = [];
|
|
8294
|
-
for (const [key, { frame, aliases }] of entries) {
|
|
8295
|
-
const stream = { key, aliases };
|
|
8296
|
-
frames[key] = frame;
|
|
8297
|
-
addPolicyImageAliases(
|
|
8298
|
-
images,
|
|
8299
|
-
stream,
|
|
8300
|
-
frame,
|
|
8301
|
-
includeObservationImageAliases
|
|
8302
|
-
);
|
|
8303
|
-
sourceParts.push(describeFrameSource(key, frame));
|
|
8304
|
-
}
|
|
8305
|
-
return {
|
|
8306
|
-
frames,
|
|
8307
|
-
images,
|
|
8308
|
-
sourceSummary: sourceParts.length > 0 ? sourceParts.join(" + ") : "not used by policy",
|
|
8309
|
-
capturedAt: Date.now()
|
|
8310
|
-
};
|
|
8311
|
-
}
|
|
8312
|
-
async function capturePolicyCameraFramesFromMountedStreams(target, options) {
|
|
8313
|
-
const plan = createPolicyCameraFrameCapturePlanFromApi(target, options);
|
|
8314
|
-
const result = await capturePolicyCameraFrames(target, plan);
|
|
8315
|
-
return { ...result, plan };
|
|
8316
|
-
}
|
|
8317
|
-
|
|
8318
|
-
// src/hooks/usePolicyCameraFrames.ts
|
|
8319
8673
|
function mergePolicyCameraFrameCaptureOptions(defaultOptions, options) {
|
|
8320
8674
|
return {
|
|
8321
8675
|
...defaultOptions,
|
|
@@ -8838,6 +9192,18 @@ function useCameraAnimation() {
|
|
|
8838
9192
|
* @license
|
|
8839
9193
|
* SPDX-License-Identifier: Apache-2.0
|
|
8840
9194
|
*/
|
|
9195
|
+
/**
|
|
9196
|
+
* @license
|
|
9197
|
+
* SPDX-License-Identifier: Apache-2.0
|
|
9198
|
+
*
|
|
9199
|
+
* Helpers for resolving dataset camera streams to mounted MuJoCo resources.
|
|
9200
|
+
*/
|
|
9201
|
+
/**
|
|
9202
|
+
* @license
|
|
9203
|
+
* SPDX-License-Identifier: Apache-2.0
|
|
9204
|
+
*
|
|
9205
|
+
* Helpers for turning Three/MuJoCo camera captures into policy image payloads.
|
|
9206
|
+
*/
|
|
8841
9207
|
/**
|
|
8842
9208
|
* @license
|
|
8843
9209
|
* SPDX-License-Identifier: Apache-2.0
|
|
@@ -8861,12 +9227,6 @@ function useCameraAnimation() {
|
|
|
8861
9227
|
*
|
|
8862
9228
|
* useFrameCapture — still-frame capture for canvas-backed MuJoCo/R3F scenes.
|
|
8863
9229
|
*/
|
|
8864
|
-
/**
|
|
8865
|
-
* @license
|
|
8866
|
-
* SPDX-License-Identifier: Apache-2.0
|
|
8867
|
-
*
|
|
8868
|
-
* Helpers for resolving dataset camera streams to mounted MuJoCo resources.
|
|
8869
|
-
*/
|
|
8870
9230
|
/**
|
|
8871
9231
|
* @license
|
|
8872
9232
|
* SPDX-License-Identifier: Apache-2.0
|
|
@@ -8907,16 +9267,9 @@ function useCameraAnimation() {
|
|
|
8907
9267
|
* @license
|
|
8908
9268
|
* SPDX-License-Identifier: Apache-2.0
|
|
8909
9269
|
*
|
|
8910
|
-
*
|
|
8911
|
-
*
|
|
8912
|
-
*
|
|
8913
|
-
* take over the render loop. Prefer this over `useCameraViewport` for camera
|
|
8914
|
-
* tiles embedded in HTML UI; use `useCameraViewport` for transparent overlays
|
|
8915
|
-
* on a full-bleed canvas.
|
|
8916
|
-
*
|
|
8917
|
-
* Uses the async capture path so Gaussian-splat environments render through
|
|
8918
|
-
* their dedicated capture renderer — streaming a splat scene at full rate does
|
|
8919
|
-
* not disturb the main view's splat sort.
|
|
9270
|
+
* Readback stream for DOM compositing/encoding. For interactive camera previews,
|
|
9271
|
+
* use CameraView/useCameraViewport: they render on the GPU with no readback.
|
|
9272
|
+
* This path defaults to 10 FPS and permits only one capture in flight.
|
|
8920
9273
|
*/
|
|
8921
9274
|
/**
|
|
8922
9275
|
* @license
|
|
@@ -9071,12 +9424,6 @@ function useCameraAnimation() {
|
|
|
9071
9424
|
*
|
|
9072
9425
|
* React state wrapper around MuJoCo/R3F offscreen camera-frame capture.
|
|
9073
9426
|
*/
|
|
9074
|
-
/**
|
|
9075
|
-
* @license
|
|
9076
|
-
* SPDX-License-Identifier: Apache-2.0
|
|
9077
|
-
*
|
|
9078
|
-
* Helpers for turning Three/MuJoCo camera captures into policy image payloads.
|
|
9079
|
-
*/
|
|
9080
9427
|
/**
|
|
9081
9428
|
* @license
|
|
9082
9429
|
* SPDX-License-Identifier: Apache-2.0
|