mujoco-react 12.0.0 → 12.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -4
- package/dist/chunk-U2Y2BODT.js +137 -0
- package/dist/chunk-U2Y2BODT.js.map +1 -0
- package/dist/{chunk-7BCIHJMW.js → chunk-XD3AY7XZ.js} +47 -9
- package/dist/chunk-XD3AY7XZ.js.map +1 -0
- package/dist/index.d.ts +10 -15
- package/dist/index.js +1447 -1090
- package/dist/index.js.map +1 -1
- package/dist/onnx.d.ts +1 -1
- package/dist/physics.worker.d.ts +2 -0
- package/dist/physics.worker.js +78 -0
- package/dist/physics.worker.js.map +1 -0
- package/dist/spark.d.ts +1 -1
- package/dist/spark.js +1 -1
- package/dist/{types-BjHt_Urb.d.ts → types-CSi3fIVs.d.ts} +70 -1
- package/dist/vite.d.ts +5 -2
- package/dist/vite.js +6 -2
- package/dist/vite.js.map +1 -1
- package/package.json +5 -2
- package/src/components/CameraView.tsx +9 -7
- package/src/core/MujocoCanvas.tsx +2 -0
- package/src/core/MujocoPhysics.tsx +1 -0
- package/src/core/MujocoSimProvider.tsx +103 -81
- package/src/core/sceneIdentity.ts +15 -0
- package/src/core/stepBudget.ts +18 -0
- package/src/hooks/useCameraStream.ts +14 -17
- package/src/hooks/usePolicy.ts +16 -7
- package/src/index.ts +3 -0
- package/src/physics.worker.ts +13 -0
- package/src/rendering/cameraFrameCapture.ts +33 -7
- package/src/runtime/client.ts +196 -0
- package/src/runtime/engine.ts +93 -0
- package/src/runtime/protocol.ts +59 -0
- package/src/runtime/types.ts +51 -0
- package/src/types.ts +11 -0
- package/src/vite.ts +7 -2
- package/dist/chunk-7BCIHJMW.js.map +0 -1
package/dist/index.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
import {
|
|
2
|
-
|
|
1
|
+
import { displayFields } from './chunk-U2Y2BODT.js';
|
|
2
|
+
import { withContacts, getContact, captureCameraFrame, captureCameraFrameBlob, createCameraFrameCaptureSession, CAMERA_FRAME_CAPTURE_PRE_RENDER_USER_DATA_KEY, CAPTURE_EXCLUDE_KEY, createCaptureCamera, prepareCaptureCamera } from './chunk-XD3AY7XZ.js';
|
|
3
|
+
export { CAMERA_FRAME_CAPTURE_PRE_RENDER_USER_DATA_KEY, CAMERA_FRAME_CAPTURE_RENDER_USER_DATA_KEY, CAPTURE_EXCLUDE_KEY, ModelActuators, ModelBodies, ModelCameras, ModelGeoms, ModelJoints, ModelKeyframes, ModelResources, ModelSensors, ModelSites, ScenarioLighting, SplatEnvironment, SplatEnvironmentReadinessStatus, VisualScenarioEffects, captureCameraFrame, captureCameraFrameBlob, captureCameraFrameTensor, createCameraFrameCaptureSession, createPairedSplatEnvironment, createSparkSplatViewerUrl, createSplatEnvironmentUserData, createSplatSceneConfig, createVisualScenarioExecutionContext, dataUrlToPolicyImageTensor, getContact, getScenarioBackground, getScenarioCameraPosition, getSplatEnvironmentReadiness, imageDataToPolicyImageTensor, pixelsToPolicyImageTensor, registerModelResources, renderCameraFrameToCanvas, useSplatEnvironment, useSplatSceneConfig, useVisualScenarioEffects, useVisualScenarioExecutionContext, withSplatEnvironment } from './chunk-XD3AY7XZ.js';
|
|
3
4
|
import loadMujoco from '@mujoco/mujoco';
|
|
4
5
|
import defaultMujocoWasmUrl from '@mujoco/mujoco/mujoco.wasm?url';
|
|
5
6
|
import { createContext, forwardRef, useRef, useEffect, useContext, useState, useCallback, useMemo, useLayoutEffect, useId } from 'react';
|
|
@@ -975,939 +976,1332 @@ function collectDependencyPaths(xmlString, currentFile, parser) {
|
|
|
975
976
|
});
|
|
976
977
|
return paths;
|
|
977
978
|
}
|
|
978
|
-
var CapsuleGeometry = class extends THREE12.BufferGeometry {
|
|
979
|
-
parameters;
|
|
980
|
-
constructor(radius = 1, length = 1, capSegments = 4, radialSegments = 8) {
|
|
981
|
-
super();
|
|
982
|
-
this.type = "CapsuleGeometry";
|
|
983
|
-
this.parameters = { radius, length, capSegments, radialSegments };
|
|
984
|
-
const path = new THREE12.Path();
|
|
985
|
-
path.absarc(0, -length / 2, radius, Math.PI * 1.5, 0, false);
|
|
986
|
-
path.absarc(0, length / 2, radius, 0, Math.PI * 0.5, false);
|
|
987
|
-
const latheGeometry = new THREE12.LatheGeometry(path.getPoints(capSegments), radialSegments);
|
|
988
|
-
const self = this;
|
|
989
|
-
self.setIndex(latheGeometry.getIndex());
|
|
990
|
-
self.setAttribute("position", latheGeometry.getAttribute("position"));
|
|
991
|
-
self.setAttribute("normal", latheGeometry.getAttribute("normal"));
|
|
992
|
-
self.setAttribute("uv", latheGeometry.getAttribute("uv"));
|
|
993
|
-
}
|
|
994
|
-
};
|
|
995
979
|
|
|
996
|
-
// src/
|
|
997
|
-
var
|
|
998
|
-
var
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
980
|
+
// src/core/sceneIdentity.ts
|
|
981
|
+
var fileIds = /* @__PURE__ */ new WeakMap();
|
|
982
|
+
var nextFileId = 0;
|
|
983
|
+
function sceneIdentity(config) {
|
|
984
|
+
return JSON.stringify([
|
|
985
|
+
config.src,
|
|
986
|
+
config.sceneFile,
|
|
987
|
+
config.environmentFiles ?? [],
|
|
988
|
+
config.sceneObjects ?? [],
|
|
989
|
+
config.homeJoints ?? [],
|
|
990
|
+
config.xmlPatches ?? [],
|
|
991
|
+
(config.files ?? []).map((file) => {
|
|
992
|
+
if (!fileIds.has(file)) fileIds.set(file, ++nextFileId);
|
|
993
|
+
return fileIds.get(file);
|
|
994
|
+
})
|
|
995
|
+
], (_key, value) => value && typeof value === "object" && !Array.isArray(value) ? Object.fromEntries(Object.entries(value).sort(([a], [b]) => a.localeCompare(b))) : value);
|
|
996
|
+
}
|
|
997
|
+
|
|
998
|
+
// src/core/stepBudget.ts
|
|
999
|
+
function runStepBudget(requested, step, budgetMs = 8, now2 = () => performance.now()) {
|
|
1000
|
+
if (!Number.isFinite(budgetMs) || budgetMs <= 0) throw new Error("physicsBudgetMs must be positive and finite");
|
|
1001
|
+
const start = now2();
|
|
1002
|
+
let steps = 0;
|
|
1003
|
+
while (steps < requested && steps < 100) {
|
|
1004
|
+
step();
|
|
1005
|
+
steps++;
|
|
1006
|
+
if (now2() - start >= budgetMs) break;
|
|
1007
|
+
}
|
|
1008
|
+
return { steps, physicsMs: now2() - start, budgetExceeded: steps < requested };
|
|
1009
|
+
}
|
|
1010
|
+
|
|
1011
|
+
// src/rendering/cameraFrameSource.ts
|
|
1012
|
+
var MountedCameraFrameSourceSuggestionMatch = {
|
|
1013
|
+
Direct: "direct",
|
|
1014
|
+
Alias: "alias",
|
|
1015
|
+
Normalized: "normalized",
|
|
1016
|
+
Prefix: "prefix",
|
|
1017
|
+
Suffix: "suffix",
|
|
1018
|
+
Contains: "contains"
|
|
1019
|
+
};
|
|
1020
|
+
var MountedCameraFrameSequenceReadinessStatus = {
|
|
1021
|
+
Ready: "ready",
|
|
1022
|
+
Partial: "partial",
|
|
1023
|
+
Missing: "missing"
|
|
1024
|
+
};
|
|
1025
|
+
var MountedCameraFrameSequenceManifestStatus = {
|
|
1026
|
+
Complete: "complete",
|
|
1027
|
+
Partial: "partial",
|
|
1028
|
+
Missing: "missing"
|
|
1029
|
+
};
|
|
1030
|
+
function getResourceName(resource) {
|
|
1031
|
+
if (!resource) return null;
|
|
1032
|
+
return typeof resource === "string" ? resource : resource.name ?? null;
|
|
1033
|
+
}
|
|
1034
|
+
function createNameSet(resources) {
|
|
1035
|
+
return new Set(
|
|
1036
|
+
(resources ?? []).map((resource) => getResourceName(resource)).filter((name) => Boolean(name))
|
|
1037
|
+
);
|
|
1038
|
+
}
|
|
1039
|
+
function createResourceNames(resources) {
|
|
1040
|
+
return (resources ?? []).map((resource) => getResourceName(resource)).filter((name) => Boolean(name));
|
|
1041
|
+
}
|
|
1042
|
+
function normalizeAliasCandidates(value) {
|
|
1043
|
+
if (!value) return [];
|
|
1044
|
+
return Array.isArray(value) ? value : [value];
|
|
1045
|
+
}
|
|
1046
|
+
function countMountedSelectors(selector) {
|
|
1047
|
+
return Number(Boolean(selector.cameraName)) + Number(Boolean(selector.siteName)) + Number(Boolean(selector.bodyName));
|
|
1048
|
+
}
|
|
1049
|
+
function normalizeCameraSourceName(value) {
|
|
1050
|
+
return value.trim().toLowerCase().replace(/[^a-z0-9]+/g, "_").replace(/^_+|_+$/g, "");
|
|
1051
|
+
}
|
|
1052
|
+
function createCameraSourceKeyVariants(key) {
|
|
1053
|
+
const candidates = [
|
|
1054
|
+
key,
|
|
1055
|
+
key.startsWith("observation.images.") ? key.slice("observation.images.".length) : "",
|
|
1056
|
+
key.includes(".") ? key.split(".").at(-1) ?? "" : "",
|
|
1057
|
+
key.includes("/") ? key.split("/").at(-1) ?? "" : ""
|
|
1058
|
+
];
|
|
1059
|
+
return candidates.map((candidate) => candidate.trim()).filter((candidate, index, items) => candidate && items.indexOf(candidate) === index);
|
|
1060
|
+
}
|
|
1061
|
+
function getSelectorKey(selector) {
|
|
1062
|
+
if (selector.cameraName) return `camera:${selector.cameraName}`;
|
|
1063
|
+
if (selector.siteName) return `site:${selector.siteName}`;
|
|
1064
|
+
if (selector.bodyName) return `body:${selector.bodyName}`;
|
|
1065
|
+
return null;
|
|
1066
|
+
}
|
|
1067
|
+
function getMountedCameraFrameCaptureSource(selector) {
|
|
1068
|
+
if (countMountedSelectors(selector) !== 1) return null;
|
|
1069
|
+
if (selector.cameraName) {
|
|
1070
|
+
return { kind: "mujoco-camera", cameraName: selector.cameraName };
|
|
1005
1071
|
}
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
if (!smoothing) return null;
|
|
1009
|
-
if (smoothing === true) return DEFAULT_MESH_NORMAL_SMOOTHING_TOLERANCE;
|
|
1010
|
-
return smoothing.tolerance ?? DEFAULT_MESH_NORMAL_SMOOTHING_TOLERANCE;
|
|
1072
|
+
if (selector.siteName) {
|
|
1073
|
+
return { kind: "mujoco-site", siteName: selector.siteName };
|
|
1011
1074
|
}
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1075
|
+
if (selector.bodyName) {
|
|
1076
|
+
return { kind: "mujoco-body", bodyName: selector.bodyName };
|
|
1077
|
+
}
|
|
1078
|
+
return null;
|
|
1079
|
+
}
|
|
1080
|
+
function isMountedCameraFrameCaptureSource(source) {
|
|
1081
|
+
return source.kind === "mujoco-camera" || source.kind === "mujoco-site" || source.kind === "mujoco-body";
|
|
1082
|
+
}
|
|
1083
|
+
function getCameraFrameCaptureSourceTarget(source) {
|
|
1084
|
+
if (source.kind === "mujoco-camera") return source.cameraName;
|
|
1085
|
+
if (source.kind === "mujoco-site") return source.siteName;
|
|
1086
|
+
if (source.kind === "mujoco-body") return source.bodyName;
|
|
1087
|
+
if (source.kind === "custom-camera") return "custom camera";
|
|
1088
|
+
if (source.kind === "explicit-pose") return "explicit pose";
|
|
1089
|
+
return "fallback camera";
|
|
1090
|
+
}
|
|
1091
|
+
function createMountedCameraFrameSourceSuggestion(key, selector, resourceName, match) {
|
|
1092
|
+
const source = getMountedCameraFrameCaptureSource(selector);
|
|
1093
|
+
if (!source) return null;
|
|
1094
|
+
return {
|
|
1095
|
+
key,
|
|
1096
|
+
selector,
|
|
1097
|
+
source,
|
|
1098
|
+
resourceName,
|
|
1099
|
+
resourceKind: source.kind,
|
|
1100
|
+
match
|
|
1101
|
+
};
|
|
1102
|
+
}
|
|
1103
|
+
function addMountedCameraFrameSourceSuggestion(suggestions, seen, suggestion) {
|
|
1104
|
+
if (!suggestion) return;
|
|
1105
|
+
const selectorKey = getSelectorKey(suggestion.selector);
|
|
1106
|
+
if (!selectorKey || seen.has(selectorKey)) return;
|
|
1107
|
+
seen.add(selectorKey);
|
|
1108
|
+
suggestions.push(suggestion);
|
|
1109
|
+
}
|
|
1110
|
+
function getCameraFrameResourceMatch(key, resourceName) {
|
|
1111
|
+
if (resourceName === key) return MountedCameraFrameSourceSuggestionMatch.Direct;
|
|
1112
|
+
const normalizedResource = normalizeCameraSourceName(resourceName);
|
|
1113
|
+
if (!normalizedResource) return null;
|
|
1114
|
+
for (const variant of createCameraSourceKeyVariants(key)) {
|
|
1115
|
+
if (resourceName === variant) return MountedCameraFrameSourceSuggestionMatch.Direct;
|
|
1116
|
+
const normalizedKey = normalizeCameraSourceName(variant);
|
|
1117
|
+
if (!normalizedKey) continue;
|
|
1118
|
+
if (normalizedResource === normalizedKey) {
|
|
1119
|
+
return MountedCameraFrameSourceSuggestionMatch.Normalized;
|
|
1023
1120
|
}
|
|
1024
|
-
if (
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
const source = mjModel.tex_data.subarray(offset, offset + width * height * channels);
|
|
1033
|
-
const rgba = new Uint8Array(width * height * 4);
|
|
1034
|
-
for (let i = 0, j = 0; i < width * height; i += 1, j += channels) {
|
|
1035
|
-
const r = source[j] ?? 255;
|
|
1036
|
-
const g = channels > 1 ? source[j + 1] : r;
|
|
1037
|
-
const b = channels > 2 ? source[j + 2] : r;
|
|
1038
|
-
const a = channels > 3 ? source[j + 3] : 255;
|
|
1039
|
-
const out = i * 4;
|
|
1040
|
-
rgba[out] = r;
|
|
1041
|
-
rgba[out + 1] = g;
|
|
1042
|
-
rgba[out + 2] = b;
|
|
1043
|
-
rgba[out + 3] = a;
|
|
1121
|
+
if (normalizedResource.startsWith(`${normalizedKey}_`)) {
|
|
1122
|
+
return MountedCameraFrameSourceSuggestionMatch.Prefix;
|
|
1123
|
+
}
|
|
1124
|
+
if (normalizedResource.endsWith(`_${normalizedKey}`)) {
|
|
1125
|
+
return MountedCameraFrameSourceSuggestionMatch.Suffix;
|
|
1126
|
+
}
|
|
1127
|
+
if (normalizedResource.includes(`_${normalizedKey}_`)) {
|
|
1128
|
+
return MountedCameraFrameSourceSuggestionMatch.Contains;
|
|
1044
1129
|
}
|
|
1045
|
-
const texture = new THREE12.DataTexture(rgba, width, height, THREE12.RGBAFormat);
|
|
1046
|
-
texture.colorSpace = THREE12.LinearSRGBColorSpace;
|
|
1047
|
-
texture.wrapS = THREE12.RepeatWrapping;
|
|
1048
|
-
texture.wrapT = THREE12.RepeatWrapping;
|
|
1049
|
-
texture.flipY = true;
|
|
1050
|
-
const repeatOffset = matId * 2;
|
|
1051
|
-
const repeatS = mjModel.mat_texrepeat?.[repeatOffset] ?? 1;
|
|
1052
|
-
const repeatT = mjModel.mat_texrepeat?.[repeatOffset + 1] ?? 1;
|
|
1053
|
-
texture.repeat.set(repeatS || 1, repeatT || 1);
|
|
1054
|
-
texture.needsUpdate = true;
|
|
1055
|
-
this.textureCache.set(texId, texture);
|
|
1056
|
-
return texture;
|
|
1057
1130
|
}
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
const rgba = mjModel.mat_rgba.subarray(matId * 4, matId * 4 + 4);
|
|
1074
|
-
color.setRGB(rgba[0], rgba[1], rgba[2]);
|
|
1075
|
-
opacity = rgba[3];
|
|
1076
|
-
} else {
|
|
1077
|
-
const rgba = mjModel.geom_rgba.subarray(g * 4, g * 4 + 4);
|
|
1078
|
-
color.setRGB(rgba[0], rgba[1], rgba[2]);
|
|
1079
|
-
opacity = rgba[3];
|
|
1131
|
+
return null;
|
|
1132
|
+
}
|
|
1133
|
+
function isSelectorMounted(selector, cameraNames, siteNames, bodyNames) {
|
|
1134
|
+
if (countMountedSelectors(selector) !== 1) return false;
|
|
1135
|
+
return (selector.cameraName ? cameraNames.has(selector.cameraName) : false) || (selector.siteName ? siteNames.has(selector.siteName) : false) || (selector.bodyName ? bodyNames.has(selector.bodyName) : false);
|
|
1136
|
+
}
|
|
1137
|
+
function createMountedCameraFrameSourceSuggestions(key, options) {
|
|
1138
|
+
const cameraNames = createNameSet(options.cameras);
|
|
1139
|
+
const siteNames = createNameSet(options.sites);
|
|
1140
|
+
const bodyNames = createNameSet(options.bodies);
|
|
1141
|
+
const suggestions = [];
|
|
1142
|
+
const seen = /* @__PURE__ */ new Set();
|
|
1143
|
+
for (const selector of normalizeAliasCandidates(options.aliases?.[key])) {
|
|
1144
|
+
if (!isSelectorMounted(selector, cameraNames, siteNames, bodyNames)) {
|
|
1145
|
+
continue;
|
|
1080
1146
|
}
|
|
1081
|
-
const
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
} else if (type === getVal(MG.mjGEOM_CYLINDER)) {
|
|
1094
|
-
geo = new THREE12.CylinderGeometry(size[0], size[0], size[1] * 2, 24);
|
|
1095
|
-
geo.rotateX(Math.PI / 2);
|
|
1096
|
-
} else if (type === getVal(MG.mjGEOM_MESH)) {
|
|
1097
|
-
const mId = mjModel.geom_dataid[g];
|
|
1098
|
-
const vAdr = mjModel.mesh_vertadr[mId];
|
|
1099
|
-
const vNum = mjModel.mesh_vertnum[mId];
|
|
1100
|
-
const fAdr = mjModel.mesh_faceadr[mId];
|
|
1101
|
-
const fNum = mjModel.mesh_facenum[mId];
|
|
1102
|
-
const tAdr = mjModel.mesh_texcoordadr?.[mId] ?? -1;
|
|
1103
|
-
if (map && tAdr >= 0 && mjModel.mesh_texcoord && mjModel.mesh_facetexcoord) {
|
|
1104
|
-
const corners = fNum * 3;
|
|
1105
|
-
const positions = new Float32Array(corners * 3);
|
|
1106
|
-
const uvs = new Float32Array(corners * 2);
|
|
1107
|
-
for (let c = 0; c < corners; c++) {
|
|
1108
|
-
const v = vAdr + mjModel.mesh_face[fAdr * 3 + c];
|
|
1109
|
-
positions[c * 3] = mjModel.mesh_vert[v * 3];
|
|
1110
|
-
positions[c * 3 + 1] = mjModel.mesh_vert[v * 3 + 1];
|
|
1111
|
-
positions[c * 3 + 2] = mjModel.mesh_vert[v * 3 + 2];
|
|
1112
|
-
const t = tAdr + mjModel.mesh_facetexcoord[fAdr * 3 + c];
|
|
1113
|
-
uvs[c * 2] = mjModel.mesh_texcoord[t * 2];
|
|
1114
|
-
uvs[c * 2 + 1] = 1 - mjModel.mesh_texcoord[t * 2 + 1];
|
|
1115
|
-
}
|
|
1116
|
-
geo = new THREE12.BufferGeometry();
|
|
1117
|
-
geo.setAttribute("position", new THREE12.Float32BufferAttribute(positions, 3));
|
|
1118
|
-
geo.setAttribute("uv", new THREE12.Float32BufferAttribute(uvs, 2));
|
|
1119
|
-
geo.computeVertexNormals();
|
|
1120
|
-
} else {
|
|
1121
|
-
geo = new THREE12.BufferGeometry();
|
|
1122
|
-
geo.setAttribute("position", new THREE12.Float32BufferAttribute(mjModel.mesh_vert.subarray(vAdr * 3, (vAdr + vNum) * 3), 3));
|
|
1123
|
-
geo.setIndex(Array.from(mjModel.mesh_face.subarray(fAdr * 3, (fAdr + fNum) * 3)));
|
|
1124
|
-
const smoothingTolerance = this.getMeshNormalSmoothingTolerance();
|
|
1125
|
-
if (smoothingTolerance !== null) {
|
|
1126
|
-
geo = mergeVertices(geo, smoothingTolerance);
|
|
1127
|
-
}
|
|
1128
|
-
geo.computeVertexNormals();
|
|
1129
|
-
}
|
|
1130
|
-
}
|
|
1131
|
-
if (geo) {
|
|
1132
|
-
const isPlane = type === getVal(MG.mjGEOM_PLANE);
|
|
1133
|
-
const materialMap = isPlane && map ? map.clone() : map;
|
|
1134
|
-
if (isPlane && materialMap) {
|
|
1135
|
-
materialMap.repeat.multiplyScalar(2.5);
|
|
1136
|
-
materialMap.needsUpdate = true;
|
|
1137
|
-
}
|
|
1138
|
-
const mesh = new THREE12.Mesh(geo, new THREE12.MeshStandardMaterial({
|
|
1139
|
-
color,
|
|
1140
|
-
map: materialMap,
|
|
1141
|
-
transparent: opacity < 1,
|
|
1142
|
-
opacity,
|
|
1143
|
-
roughness: 0.6,
|
|
1144
|
-
metalness: 0
|
|
1145
|
-
}));
|
|
1146
|
-
mesh.castShadow = type !== getVal(MG.mjGEOM_PLANE);
|
|
1147
|
-
mesh.receiveShadow = true;
|
|
1148
|
-
mesh.position.set(pos[0], pos[1], pos[2]);
|
|
1149
|
-
mesh.quaternion.set(quat[1], quat[2], quat[3], quat[0]);
|
|
1150
|
-
mesh.userData.bodyID = mjModel.geom_bodyid[g];
|
|
1151
|
-
mesh.userData.geomID = g;
|
|
1152
|
-
mesh.userData.geomGroup = mjModel.geom_group[g];
|
|
1153
|
-
mesh.userData.geomName = getName(mjModel, mjModel.name_geomadr[g]);
|
|
1154
|
-
return mesh;
|
|
1155
|
-
}
|
|
1156
|
-
return null;
|
|
1147
|
+
const source = getMountedCameraFrameCaptureSource(selector);
|
|
1148
|
+
if (!source) continue;
|
|
1149
|
+
addMountedCameraFrameSourceSuggestion(
|
|
1150
|
+
suggestions,
|
|
1151
|
+
seen,
|
|
1152
|
+
createMountedCameraFrameSourceSuggestion(
|
|
1153
|
+
key,
|
|
1154
|
+
selector,
|
|
1155
|
+
getCameraFrameCaptureSourceTarget(source),
|
|
1156
|
+
MountedCameraFrameSourceSuggestionMatch.Alias
|
|
1157
|
+
)
|
|
1158
|
+
);
|
|
1157
1159
|
}
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
|
|
1162
|
-
|
|
1163
|
-
|
|
1160
|
+
for (const cameraName of createResourceNames(options.cameras)) {
|
|
1161
|
+
const match = getCameraFrameResourceMatch(key, cameraName);
|
|
1162
|
+
if (!match) continue;
|
|
1163
|
+
addMountedCameraFrameSourceSuggestion(
|
|
1164
|
+
suggestions,
|
|
1165
|
+
seen,
|
|
1166
|
+
createMountedCameraFrameSourceSuggestion(
|
|
1167
|
+
key,
|
|
1168
|
+
{ cameraName },
|
|
1169
|
+
cameraName,
|
|
1170
|
+
match
|
|
1171
|
+
)
|
|
1172
|
+
);
|
|
1173
|
+
}
|
|
1174
|
+
for (const siteName of createResourceNames(options.sites)) {
|
|
1175
|
+
const match = getCameraFrameResourceMatch(key, siteName);
|
|
1176
|
+
if (!match) continue;
|
|
1177
|
+
addMountedCameraFrameSourceSuggestion(
|
|
1178
|
+
suggestions,
|
|
1179
|
+
seen,
|
|
1180
|
+
createMountedCameraFrameSourceSuggestion(
|
|
1181
|
+
key,
|
|
1182
|
+
{ siteName },
|
|
1183
|
+
siteName,
|
|
1184
|
+
match
|
|
1185
|
+
)
|
|
1186
|
+
);
|
|
1187
|
+
}
|
|
1188
|
+
for (const bodyName of createResourceNames(options.bodies)) {
|
|
1189
|
+
const match = getCameraFrameResourceMatch(key, bodyName);
|
|
1190
|
+
if (!match) continue;
|
|
1191
|
+
addMountedCameraFrameSourceSuggestion(
|
|
1192
|
+
suggestions,
|
|
1193
|
+
seen,
|
|
1194
|
+
createMountedCameraFrameSourceSuggestion(
|
|
1195
|
+
key,
|
|
1196
|
+
{ bodyName },
|
|
1197
|
+
bodyName,
|
|
1198
|
+
match
|
|
1199
|
+
)
|
|
1200
|
+
);
|
|
1201
|
+
}
|
|
1202
|
+
return suggestions;
|
|
1164
1203
|
}
|
|
1165
|
-
function
|
|
1166
|
-
const
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
const bodyRefs = useRef([]);
|
|
1179
|
-
const prevModelRef = useRef(null);
|
|
1180
|
-
const prevRenderOptionsKeyRef = useRef(null);
|
|
1181
|
-
const prevHiddenVersionRef = useRef(null);
|
|
1182
|
-
const renderOptionsKey = getRenderOptionsKey(renderOptions);
|
|
1183
|
-
const geomBuilder = useMemo(() => {
|
|
1184
|
-
if (status !== "ready") return null;
|
|
1185
|
-
return new GeomBuilder(mujocoRef.current, renderOptions);
|
|
1186
|
-
}, [status, mujocoRef, renderOptionsKey]);
|
|
1187
|
-
useEffect(() => {
|
|
1188
|
-
if (status !== "ready" || !geomBuilder) return;
|
|
1189
|
-
const model = mjModelRef.current;
|
|
1190
|
-
const group = groupRef.current;
|
|
1191
|
-
if (!model || !group) return;
|
|
1192
|
-
if (prevModelRef.current === model && prevRenderOptionsKeyRef.current === renderOptionsKey && prevHiddenVersionRef.current === hiddenBodiesVersion) {
|
|
1193
|
-
return;
|
|
1194
|
-
}
|
|
1195
|
-
prevModelRef.current = model;
|
|
1196
|
-
prevRenderOptionsKeyRef.current = renderOptionsKey;
|
|
1197
|
-
prevHiddenVersionRef.current = hiddenBodiesVersion;
|
|
1198
|
-
while (group.children.length > 0) {
|
|
1199
|
-
group.remove(group.children[0]);
|
|
1200
|
-
}
|
|
1201
|
-
const refs = [];
|
|
1202
|
-
for (let i = 0; i < model.nbody; i++) {
|
|
1203
|
-
const bodyGroup = new THREE12.Group();
|
|
1204
|
-
bodyGroup.userData.bodyID = i;
|
|
1205
|
-
const bodyName = getName(model, model.name_bodyadr[i]);
|
|
1206
|
-
if (!hiddenBodiesRef.current.has(bodyName)) {
|
|
1207
|
-
for (let g = 0; g < model.ngeom; g++) {
|
|
1208
|
-
if (model.geom_bodyid[g] === i) {
|
|
1209
|
-
const mesh = geomBuilder.create(model, g);
|
|
1210
|
-
if (mesh) bodyGroup.add(mesh);
|
|
1211
|
-
}
|
|
1212
|
-
}
|
|
1213
|
-
}
|
|
1214
|
-
group.add(bodyGroup);
|
|
1215
|
-
refs.push(bodyGroup);
|
|
1216
|
-
}
|
|
1217
|
-
bodyRefs.current = refs;
|
|
1218
|
-
}, [status, geomBuilder, mjModelRef, renderOptionsKey, hiddenBodiesRef, hiddenBodiesVersion]);
|
|
1219
|
-
const syncBodiesToData = useCallback(() => {
|
|
1220
|
-
const data = mjDataRef.current;
|
|
1221
|
-
if (!data) return;
|
|
1222
|
-
const bodies = bodyRefs.current;
|
|
1223
|
-
const interpolation = interpolationStateRef.current;
|
|
1224
|
-
const useInterpolation = interpolateRef.current && interpolation.valid;
|
|
1225
|
-
for (let i = 0; i < bodies.length; i++) {
|
|
1226
|
-
const ref = bodies[i];
|
|
1227
|
-
if (!ref) continue;
|
|
1228
|
-
if (useInterpolation) {
|
|
1229
|
-
const alpha = interpolation.alpha;
|
|
1230
|
-
const i3 = i * 3;
|
|
1231
|
-
ref.position.set(
|
|
1232
|
-
THREE12.MathUtils.lerp(interpolation.previousXpos[i3], interpolation.currentXpos[i3], alpha),
|
|
1233
|
-
THREE12.MathUtils.lerp(interpolation.previousXpos[i3 + 1], interpolation.currentXpos[i3 + 1], alpha),
|
|
1234
|
-
THREE12.MathUtils.lerp(interpolation.previousXpos[i3 + 2], interpolation.currentXpos[i3 + 2], alpha)
|
|
1235
|
-
);
|
|
1236
|
-
const i4 = i * 4;
|
|
1237
|
-
_previousQuat.set(
|
|
1238
|
-
interpolation.previousXquat[i4 + 1],
|
|
1239
|
-
interpolation.previousXquat[i4 + 2],
|
|
1240
|
-
interpolation.previousXquat[i4 + 3],
|
|
1241
|
-
interpolation.previousXquat[i4]
|
|
1242
|
-
);
|
|
1243
|
-
_currentQuat.set(
|
|
1244
|
-
interpolation.currentXquat[i4 + 1],
|
|
1245
|
-
interpolation.currentXquat[i4 + 2],
|
|
1246
|
-
interpolation.currentXquat[i4 + 3],
|
|
1247
|
-
interpolation.currentXquat[i4]
|
|
1248
|
-
);
|
|
1249
|
-
ref.quaternion.copy(_previousQuat).slerp(_currentQuat, alpha);
|
|
1250
|
-
} else {
|
|
1251
|
-
ref.position.set(
|
|
1252
|
-
data.xpos[i * 3],
|
|
1253
|
-
data.xpos[i * 3 + 1],
|
|
1254
|
-
data.xpos[i * 3 + 2]
|
|
1255
|
-
);
|
|
1256
|
-
ref.quaternion.set(
|
|
1257
|
-
data.xquat[i * 4 + 1],
|
|
1258
|
-
data.xquat[i * 4 + 2],
|
|
1259
|
-
data.xquat[i * 4 + 3],
|
|
1260
|
-
data.xquat[i * 4]
|
|
1261
|
-
);
|
|
1262
|
-
}
|
|
1263
|
-
}
|
|
1264
|
-
}, [interpolateRef, interpolationStateRef, mjDataRef]);
|
|
1265
|
-
useFrame(syncBodiesToData);
|
|
1266
|
-
return /* @__PURE__ */ jsx(
|
|
1267
|
-
"group",
|
|
1268
|
-
{
|
|
1269
|
-
...props,
|
|
1270
|
-
ref: groupRef,
|
|
1271
|
-
userData: {
|
|
1272
|
-
...props.userData,
|
|
1273
|
-
[CAMERA_FRAME_CAPTURE_PRE_RENDER_USER_DATA_KEY]: syncBodiesToData
|
|
1274
|
-
},
|
|
1275
|
-
onDoubleClick: (e) => {
|
|
1276
|
-
if (typeof props.onDoubleClick === "function") props.onDoubleClick(e);
|
|
1277
|
-
e.stopPropagation();
|
|
1278
|
-
let obj = e.object;
|
|
1279
|
-
while (obj && obj.userData.bodyID === void 0 && obj.parent) {
|
|
1280
|
-
obj = obj.parent;
|
|
1281
|
-
}
|
|
1282
|
-
const bodyID = obj?.userData.bodyID;
|
|
1283
|
-
if (typeof bodyID === "number" && bodyID > 0) {
|
|
1284
|
-
const model = mjModelRef.current;
|
|
1285
|
-
if (model && bodyID < model.nbody && onSelectionRef.current) {
|
|
1286
|
-
const name = getName(model, model.name_bodyadr[bodyID]);
|
|
1287
|
-
onSelectionRef.current({ bodyId: bodyID, name });
|
|
1288
|
-
}
|
|
1289
|
-
}
|
|
1290
|
-
}
|
|
1204
|
+
function resolveMountedCameraFrameSource(key, options) {
|
|
1205
|
+
const cameraNames = createNameSet(options.cameras);
|
|
1206
|
+
const siteNames = createNameSet(options.sites);
|
|
1207
|
+
const bodyNames = createNameSet(options.bodies);
|
|
1208
|
+
const directCandidates = [
|
|
1209
|
+
{ cameraName: key },
|
|
1210
|
+
{ siteName: key },
|
|
1211
|
+
{ bodyName: key }
|
|
1212
|
+
];
|
|
1213
|
+
const aliasCandidates = normalizeAliasCandidates(options.aliases?.[key]);
|
|
1214
|
+
for (const selector of aliasCandidates) {
|
|
1215
|
+
if (!isSelectorMounted(selector, cameraNames, siteNames, bodyNames)) {
|
|
1216
|
+
continue;
|
|
1291
1217
|
}
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
gl.setScissorTest(false);
|
|
1310
|
-
gl.setViewport(0, 0, drawWidth, drawHeight);
|
|
1311
|
-
gl.render(scene, mainCamera);
|
|
1312
|
-
const api = mujoco.api;
|
|
1313
|
-
if (!api || viewportsRef.current.size === 0) return;
|
|
1314
|
-
const dpr = gl.getPixelRatio();
|
|
1315
|
-
const canvasRect = gl.domElement.getBoundingClientRect();
|
|
1316
|
-
for (const descriptor of viewportsRef.current.values()) {
|
|
1317
|
-
const element = descriptor.getElement();
|
|
1318
|
-
if (!element) continue;
|
|
1319
|
-
const rect = element.getBoundingClientRect();
|
|
1320
|
-
const isOffscreen = rect.bottom < canvasRect.top || rect.top > canvasRect.bottom || rect.right < canvasRect.left || rect.left > canvasRect.right;
|
|
1321
|
-
if (isOffscreen) continue;
|
|
1322
|
-
const width = Math.floor(rect.width * dpr);
|
|
1323
|
-
const height = Math.floor(rect.height * dpr);
|
|
1324
|
-
if (width <= 0 || height <= 0) continue;
|
|
1325
|
-
const left = Math.floor((rect.left - canvasRect.left) * dpr);
|
|
1326
|
-
const bottom = Math.floor((canvasRect.bottom - rect.bottom) * dpr);
|
|
1327
|
-
let resolved;
|
|
1328
|
-
try {
|
|
1329
|
-
resolved = api.resolveCameraCaptureOptions(descriptor.getOptions());
|
|
1330
|
-
} catch {
|
|
1331
|
-
continue;
|
|
1332
|
-
}
|
|
1333
|
-
if (!descriptor.camera) {
|
|
1334
|
-
descriptor.camera = createCaptureCamera(resolved, mainCamera, width, height);
|
|
1335
|
-
} else {
|
|
1336
|
-
prepareCaptureCamera(descriptor.camera, resolved, mainCamera, width, height);
|
|
1337
|
-
}
|
|
1338
|
-
gl.setViewport(left, bottom, width, height);
|
|
1339
|
-
gl.setScissor(left, bottom, width, height);
|
|
1340
|
-
gl.setScissorTest(true);
|
|
1341
|
-
gl.render(scene, descriptor.camera);
|
|
1218
|
+
const source = getMountedCameraFrameCaptureSource(selector);
|
|
1219
|
+
if (!source) continue;
|
|
1220
|
+
return { key, selector, source };
|
|
1221
|
+
}
|
|
1222
|
+
const [suggestion] = createMountedCameraFrameSourceSuggestions(key, options);
|
|
1223
|
+
if (suggestion) {
|
|
1224
|
+
return {
|
|
1225
|
+
key,
|
|
1226
|
+
selector: suggestion.selector,
|
|
1227
|
+
source: suggestion.source
|
|
1228
|
+
};
|
|
1229
|
+
}
|
|
1230
|
+
if (options.allowAliasFallback) {
|
|
1231
|
+
for (const selector of aliasCandidates) {
|
|
1232
|
+
const source = getMountedCameraFrameCaptureSource(selector);
|
|
1233
|
+
if (!source) continue;
|
|
1234
|
+
return { key, selector, source };
|
|
1342
1235
|
}
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
|
|
1236
|
+
}
|
|
1237
|
+
for (const selector of directCandidates) {
|
|
1238
|
+
if (!isSelectorMounted(selector, cameraNames, siteNames, bodyNames)) {
|
|
1239
|
+
continue;
|
|
1240
|
+
}
|
|
1241
|
+
const source = getMountedCameraFrameCaptureSource(selector);
|
|
1242
|
+
if (!source) continue;
|
|
1243
|
+
return { key, selector, source };
|
|
1244
|
+
}
|
|
1346
1245
|
return null;
|
|
1347
1246
|
}
|
|
1348
|
-
function
|
|
1349
|
-
const
|
|
1350
|
-
const
|
|
1351
|
-
const
|
|
1352
|
-
|
|
1353
|
-
|
|
1354
|
-
|
|
1355
|
-
|
|
1356
|
-
|
|
1357
|
-
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
}
|
|
1366
|
-
function useCameraViewport(elementRef, options) {
|
|
1367
|
-
const registry = useContext(CameraViewportRegistryContext);
|
|
1368
|
-
if (!registry) {
|
|
1369
|
-
throw new Error("useCameraViewport must be used inside <MujocoCanvas>.");
|
|
1247
|
+
function createMountedCameraFrameSequencePlan(cameraKeys, options) {
|
|
1248
|
+
const cameras = [];
|
|
1249
|
+
const resolved = {};
|
|
1250
|
+
const missingKeys = [];
|
|
1251
|
+
for (const key of cameraKeys) {
|
|
1252
|
+
const mountedSource = resolveMountedCameraFrameSource(key, options);
|
|
1253
|
+
if (!mountedSource) {
|
|
1254
|
+
missingKeys.push(key);
|
|
1255
|
+
continue;
|
|
1256
|
+
}
|
|
1257
|
+
resolved[key] = mountedSource;
|
|
1258
|
+
cameras.push({
|
|
1259
|
+
key,
|
|
1260
|
+
...options.defaults,
|
|
1261
|
+
...options.cameraOptions?.[key],
|
|
1262
|
+
...mountedSource.selector,
|
|
1263
|
+
source: mountedSource.source
|
|
1264
|
+
});
|
|
1370
1265
|
}
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
|
|
1266
|
+
if (options.requireAll && missingKeys.length > 0) {
|
|
1267
|
+
throw new Error(
|
|
1268
|
+
`Unable to resolve mounted MuJoCo camera source${missingKeys.length === 1 ? "" : "s"} for ${missingKeys.join(", ")}.`
|
|
1269
|
+
);
|
|
1270
|
+
}
|
|
1271
|
+
return {
|
|
1272
|
+
cameraKeys: [...cameraKeys],
|
|
1273
|
+
cameras,
|
|
1274
|
+
resolved,
|
|
1275
|
+
missingKeys
|
|
1276
|
+
};
|
|
1381
1277
|
}
|
|
1382
|
-
function
|
|
1383
|
-
const
|
|
1384
|
-
const
|
|
1385
|
-
|
|
1386
|
-
|
|
1278
|
+
function createMountedCameraFrameSequenceReadiness(plan) {
|
|
1279
|
+
const cameras = {};
|
|
1280
|
+
const resolvedKeys = plan.cameraKeys.filter((key) => Boolean(plan.resolved[key]));
|
|
1281
|
+
for (const key of plan.cameraKeys) {
|
|
1282
|
+
const resolved = plan.resolved[key];
|
|
1283
|
+
cameras[key] = resolved ? {
|
|
1284
|
+
key,
|
|
1285
|
+
ready: true,
|
|
1286
|
+
selector: resolved.selector,
|
|
1287
|
+
source: resolved.source,
|
|
1288
|
+
message: `Camera stream "${key}" resolves to ${resolved.source.kind}:${getCameraFrameCaptureSourceTarget(resolved.source)}.`
|
|
1289
|
+
} : {
|
|
1290
|
+
key,
|
|
1291
|
+
ready: false,
|
|
1292
|
+
message: `Camera stream "${key}" does not resolve to a mounted MuJoCo camera, site, or body.`
|
|
1293
|
+
};
|
|
1387
1294
|
}
|
|
1388
|
-
|
|
1389
|
-
|
|
1390
|
-
|
|
1391
|
-
|
|
1392
|
-
|
|
1393
|
-
|
|
1394
|
-
|
|
1395
|
-
|
|
1396
|
-
|
|
1295
|
+
const missingKeys = [...plan.missingKeys];
|
|
1296
|
+
const ready = missingKeys.length === 0;
|
|
1297
|
+
const status = ready ? MountedCameraFrameSequenceReadinessStatus.Ready : resolvedKeys.length > 0 ? MountedCameraFrameSequenceReadinessStatus.Partial : MountedCameraFrameSequenceReadinessStatus.Missing;
|
|
1298
|
+
return {
|
|
1299
|
+
ready,
|
|
1300
|
+
status,
|
|
1301
|
+
cameraKeys: [...plan.cameraKeys],
|
|
1302
|
+
resolvedKeys,
|
|
1303
|
+
missingKeys,
|
|
1304
|
+
cameras,
|
|
1305
|
+
message: ready ? `All ${plan.cameraKeys.length} requested camera stream${plan.cameraKeys.length === 1 ? "" : "s"} resolve to mounted MuJoCo sources.` : `Missing mounted MuJoCo source${missingKeys.length === 1 ? "" : "s"} for ${missingKeys.join(", ")}.`
|
|
1306
|
+
};
|
|
1307
|
+
}
|
|
1308
|
+
function normalizeFrameCount(frameCount) {
|
|
1309
|
+
return Number.isFinite(frameCount) && frameCount !== void 0 ? Math.max(0, Math.floor(frameCount)) : 0;
|
|
1310
|
+
}
|
|
1311
|
+
function createMountedCameraFrameSequenceManifest(result, options = {}) {
|
|
1312
|
+
const cameraKeys = [
|
|
1313
|
+
...options.cameraKeys ?? result.readiness.cameraKeys ?? result.plan.cameraKeys ?? result.cameraKeys
|
|
1314
|
+
];
|
|
1315
|
+
const expectedFrameCount = normalizeFrameCount(
|
|
1316
|
+
options.expectedFrameCount ?? result.frameCount
|
|
1317
|
+
);
|
|
1318
|
+
const recordedFrameCount = normalizeFrameCount(result.frameCount);
|
|
1319
|
+
const streamSummaries = {};
|
|
1320
|
+
const streams = [];
|
|
1321
|
+
let missingFrameCount = 0;
|
|
1322
|
+
let completeStreamCount = 0;
|
|
1323
|
+
let resolvedOrRecordedStreamCount = 0;
|
|
1324
|
+
for (const key of cameraKeys) {
|
|
1325
|
+
const summary = result.cameraSummaries[key];
|
|
1326
|
+
const readiness = result.readiness.cameras[key];
|
|
1327
|
+
const source = summary?.source ?? readiness?.source;
|
|
1328
|
+
const ready = readiness?.ready ?? Boolean(summary);
|
|
1329
|
+
const recorded = normalizeFrameCount(summary?.frameCount);
|
|
1330
|
+
const missing = Math.max(expectedFrameCount - recorded, 0);
|
|
1331
|
+
const complete2 = ready && missing === 0;
|
|
1332
|
+
const status2 = complete2 ? MountedCameraFrameSequenceManifestStatus.Complete : ready || recorded > 0 ? MountedCameraFrameSequenceManifestStatus.Partial : MountedCameraFrameSequenceManifestStatus.Missing;
|
|
1333
|
+
const target = source ? getCameraFrameCaptureSourceTarget(source) : readiness?.message ? void 0 : "missing MuJoCo camera";
|
|
1334
|
+
const message = complete2 ? `Camera stream "${key}" recorded ${recorded} of ${expectedFrameCount} frame${expectedFrameCount === 1 ? "" : "s"}.` : ready || recorded > 0 ? `Camera stream "${key}" recorded ${recorded} of ${expectedFrameCount} frame${expectedFrameCount === 1 ? "" : "s"}.` : readiness?.message ?? `Camera stream "${key}" did not record any frames.`;
|
|
1335
|
+
const stream = {
|
|
1336
|
+
key,
|
|
1337
|
+
ready,
|
|
1338
|
+
complete: complete2,
|
|
1339
|
+
status: status2,
|
|
1340
|
+
source,
|
|
1341
|
+
selector: readiness?.selector,
|
|
1342
|
+
target,
|
|
1343
|
+
width: summary?.width,
|
|
1344
|
+
height: summary?.height,
|
|
1345
|
+
expectedFrameCount,
|
|
1346
|
+
recordedFrameCount: recorded,
|
|
1347
|
+
missingFrameCount: missing,
|
|
1348
|
+
firstFrameIndex: summary?.firstFrameIndex ?? null,
|
|
1349
|
+
lastFrameIndex: summary?.lastFrameIndex ?? null,
|
|
1350
|
+
firstTimestamp: summary?.firstTimestamp ?? null,
|
|
1351
|
+
lastTimestamp: summary?.lastTimestamp ?? null,
|
|
1352
|
+
message
|
|
1397
1353
|
};
|
|
1398
|
-
|
|
1399
|
-
|
|
1400
|
-
|
|
1401
|
-
if (
|
|
1402
|
-
|
|
1403
|
-
|
|
1404
|
-
|
|
1405
|
-
|
|
1406
|
-
return
|
|
1354
|
+
streamSummaries[key] = stream;
|
|
1355
|
+
streams.push(stream);
|
|
1356
|
+
missingFrameCount += missing;
|
|
1357
|
+
if (complete2) completeStreamCount += 1;
|
|
1358
|
+
if (ready || recorded > 0) resolvedOrRecordedStreamCount += 1;
|
|
1359
|
+
}
|
|
1360
|
+
const complete = result.readiness.ready && streams.length === completeStreamCount && missingFrameCount === 0;
|
|
1361
|
+
const status = complete ? MountedCameraFrameSequenceManifestStatus.Complete : resolvedOrRecordedStreamCount > 0 ? MountedCameraFrameSequenceManifestStatus.Partial : MountedCameraFrameSequenceManifestStatus.Missing;
|
|
1362
|
+
return {
|
|
1363
|
+
schema: "mujoco-react/mounted-camera-frame-sequence-manifest@1",
|
|
1364
|
+
ready: result.readiness.ready,
|
|
1365
|
+
complete,
|
|
1366
|
+
status,
|
|
1367
|
+
cameraKeys,
|
|
1368
|
+
resolvedKeys: [...result.readiness.resolvedKeys],
|
|
1369
|
+
missingKeys: [...result.readiness.missingKeys],
|
|
1370
|
+
expectedFrameCount,
|
|
1371
|
+
recordedFrameCount,
|
|
1372
|
+
missingFrameCount,
|
|
1373
|
+
streamSummaries,
|
|
1374
|
+
streams,
|
|
1375
|
+
readiness: result.readiness,
|
|
1376
|
+
message: complete ? `All ${cameraKeys.length} camera stream${cameraKeys.length === 1 ? "" : "s"} recorded ${expectedFrameCount} frame${expectedFrameCount === 1 ? "" : "s"}.` : `Mounted camera sequence coverage is ${status}.`
|
|
1377
|
+
};
|
|
1407
1378
|
}
|
|
1408
|
-
function
|
|
1409
|
-
return
|
|
1379
|
+
function createMountedCameraFrameSequencePlanFromApi(api, cameraKeys, options = {}) {
|
|
1380
|
+
return createMountedCameraFrameSequencePlan(cameraKeys, {
|
|
1381
|
+
...options,
|
|
1382
|
+
cameras: api.getCameras(),
|
|
1383
|
+
sites: api.getSites(),
|
|
1384
|
+
bodies: api.getBodies()
|
|
1385
|
+
});
|
|
1410
1386
|
}
|
|
1411
|
-
function
|
|
1412
|
-
const
|
|
1413
|
-
|
|
1414
|
-
|
|
1387
|
+
async function recordMountedCameraFrameSequence(api, options) {
|
|
1388
|
+
const { cameraKeys, ...restOptions } = options;
|
|
1389
|
+
const requireAll = restOptions.requireAll ?? restOptions.requireMountedSources ?? true;
|
|
1390
|
+
const plan = createMountedCameraFrameSequencePlanFromApi(
|
|
1391
|
+
api,
|
|
1392
|
+
cameraKeys,
|
|
1393
|
+
{ ...restOptions, requireAll }
|
|
1394
|
+
);
|
|
1395
|
+
const readiness = createMountedCameraFrameSequenceReadiness(plan);
|
|
1396
|
+
const result = await api.recordCameraSequence({
|
|
1397
|
+
...restOptions,
|
|
1398
|
+
cameras: plan.cameras,
|
|
1399
|
+
requireMountedSources: restOptions.requireMountedSources ?? true
|
|
1400
|
+
});
|
|
1401
|
+
return {
|
|
1402
|
+
...result,
|
|
1403
|
+
plan,
|
|
1404
|
+
readiness
|
|
1405
|
+
};
|
|
1406
|
+
}
|
|
1407
|
+
|
|
1408
|
+
// src/policyCameraFrames.ts
|
|
1409
|
+
function addPolicyImageAliases(images, stream, frame, includeObservationImageAliases) {
|
|
1410
|
+
const keys = /* @__PURE__ */ new Set();
|
|
1411
|
+
keys.add(stream.key);
|
|
1412
|
+
for (const alias of stream.aliases ?? []) keys.add(alias);
|
|
1413
|
+
if (includeObservationImageAliases) {
|
|
1414
|
+
keys.add(`observation.images.${stream.key}`);
|
|
1415
|
+
for (const alias of stream.aliases ?? []) {
|
|
1416
|
+
keys.add(`observation.images.${alias}`);
|
|
1417
|
+
}
|
|
1415
1418
|
}
|
|
1416
|
-
|
|
1417
|
-
|
|
1419
|
+
for (const key of keys) {
|
|
1420
|
+
images[key] = frame.dataUrl;
|
|
1421
|
+
}
|
|
1422
|
+
}
|
|
1423
|
+
function describeFrameSource(key, frame) {
|
|
1424
|
+
return `${key}:${frame.source.kind}`;
|
|
1425
|
+
}
|
|
1426
|
+
function hasExplicitPolicyCameraSource(options) {
|
|
1427
|
+
return Boolean(
|
|
1428
|
+
options?.camera || options?.position || options?.quaternion || options?.source
|
|
1429
|
+
);
|
|
1430
|
+
}
|
|
1431
|
+
function createPolicyCameraFrameCapturePlan(options) {
|
|
1432
|
+
const {
|
|
1433
|
+
cameraKeys,
|
|
1434
|
+
defaults,
|
|
1435
|
+
streamOptions,
|
|
1436
|
+
includeObservationImageAliases,
|
|
1437
|
+
requireAll,
|
|
1438
|
+
...sourceOptions
|
|
1439
|
+
} = options;
|
|
1440
|
+
const mountedPlan = createMountedCameraFrameSequencePlan(cameraKeys, {
|
|
1441
|
+
...sourceOptions,
|
|
1442
|
+
defaults,
|
|
1443
|
+
cameraOptions: streamOptions
|
|
1444
|
+
});
|
|
1445
|
+
const streams = [];
|
|
1446
|
+
const missingKeys = new Set(mountedPlan.missingKeys);
|
|
1447
|
+
for (const key of cameraKeys) {
|
|
1448
|
+
const perStreamOptions = streamOptions?.[key];
|
|
1449
|
+
if (hasExplicitPolicyCameraSource(perStreamOptions)) {
|
|
1450
|
+
missingKeys.delete(key);
|
|
1451
|
+
streams.push({
|
|
1452
|
+
...defaults,
|
|
1453
|
+
...perStreamOptions,
|
|
1454
|
+
key,
|
|
1455
|
+
aliases: perStreamOptions?.aliases
|
|
1456
|
+
});
|
|
1457
|
+
continue;
|
|
1458
|
+
}
|
|
1459
|
+
const mountedCamera = mountedPlan.cameras.find((camera) => camera.key === key);
|
|
1460
|
+
if (!mountedCamera) continue;
|
|
1461
|
+
const { key: _mountedKey, ...captureOptions } = mountedCamera;
|
|
1462
|
+
streams.push({
|
|
1463
|
+
...captureOptions,
|
|
1464
|
+
key,
|
|
1465
|
+
aliases: perStreamOptions?.aliases
|
|
1466
|
+
});
|
|
1418
1467
|
}
|
|
1419
|
-
const
|
|
1420
|
-
|
|
1421
|
-
|
|
1468
|
+
const result = {
|
|
1469
|
+
cameraKeys: [...cameraKeys],
|
|
1470
|
+
streams,
|
|
1471
|
+
includeObservationImageAliases,
|
|
1472
|
+
mountedPlan,
|
|
1473
|
+
missingKeys: [...missingKeys]
|
|
1474
|
+
};
|
|
1475
|
+
if (requireAll && result.missingKeys.length > 0) {
|
|
1476
|
+
throw new Error(
|
|
1477
|
+
`Unable to resolve policy camera stream${result.missingKeys.length === 1 ? "" : "s"} for ${result.missingKeys.join(", ")}.`
|
|
1478
|
+
);
|
|
1422
1479
|
}
|
|
1423
|
-
return
|
|
1480
|
+
return result;
|
|
1424
1481
|
}
|
|
1425
|
-
function
|
|
1426
|
-
return
|
|
1427
|
-
|
|
1482
|
+
function createPolicyCameraFrameCapturePlanFromApi(api, options) {
|
|
1483
|
+
return createPolicyCameraFrameCapturePlan({
|
|
1484
|
+
...options,
|
|
1485
|
+
cameras: api.getCameras(),
|
|
1486
|
+
sites: api.getSites(),
|
|
1487
|
+
bodies: api.getBodies()
|
|
1428
1488
|
});
|
|
1429
1489
|
}
|
|
1430
|
-
async function
|
|
1431
|
-
const
|
|
1432
|
-
const
|
|
1433
|
-
|
|
1434
|
-
|
|
1490
|
+
async function capturePolicyCameraFrames(target, options) {
|
|
1491
|
+
const includeObservationImageAliases = options.includeObservationImageAliases ?? true;
|
|
1492
|
+
const entries = await Promise.all(
|
|
1493
|
+
options.streams.map(async ({ key, aliases, ...captureOptions }) => {
|
|
1494
|
+
const frame = await target.captureCameraFrame(captureOptions);
|
|
1495
|
+
return [key, { frame, aliases }];
|
|
1496
|
+
})
|
|
1497
|
+
);
|
|
1498
|
+
const frames = {};
|
|
1499
|
+
const images = {};
|
|
1500
|
+
const sourceParts = [];
|
|
1501
|
+
for (const [key, { frame, aliases }] of entries) {
|
|
1502
|
+
const stream = { key, aliases };
|
|
1503
|
+
frames[key] = frame;
|
|
1504
|
+
addPolicyImageAliases(
|
|
1505
|
+
images,
|
|
1506
|
+
stream,
|
|
1507
|
+
frame,
|
|
1508
|
+
includeObservationImageAliases
|
|
1509
|
+
);
|
|
1510
|
+
sourceParts.push(describeFrameSource(key, frame));
|
|
1435
1511
|
}
|
|
1436
1512
|
return {
|
|
1437
|
-
|
|
1438
|
-
|
|
1439
|
-
|
|
1513
|
+
frames,
|
|
1514
|
+
images,
|
|
1515
|
+
sourceSummary: sourceParts.length > 0 ? sourceParts.join(" + ") : "not used by policy",
|
|
1516
|
+
capturedAt: Date.now()
|
|
1440
1517
|
};
|
|
1441
1518
|
}
|
|
1442
|
-
async function
|
|
1443
|
-
const
|
|
1444
|
-
const
|
|
1445
|
-
|
|
1446
|
-
await waitForNextAnimationFrame();
|
|
1447
|
-
}
|
|
1448
|
-
const blob = await new Promise((resolve, reject) => {
|
|
1449
|
-
canvas.toBlob(
|
|
1450
|
-
(nextBlob) => {
|
|
1451
|
-
if (nextBlob) {
|
|
1452
|
-
resolve(nextBlob);
|
|
1453
|
-
} else {
|
|
1454
|
-
reject(new Error("Canvas frame capture did not produce a Blob."));
|
|
1455
|
-
}
|
|
1456
|
-
},
|
|
1457
|
-
type,
|
|
1458
|
-
options.quality
|
|
1459
|
-
);
|
|
1460
|
-
});
|
|
1461
|
-
return { canvas, blob, type };
|
|
1519
|
+
async function capturePolicyCameraFramesFromMountedStreams(target, options) {
|
|
1520
|
+
const plan = createPolicyCameraFrameCapturePlanFromApi(target, options);
|
|
1521
|
+
const result = await capturePolicyCameraFrames(target, plan);
|
|
1522
|
+
return { ...result, plan };
|
|
1462
1523
|
}
|
|
1463
|
-
|
|
1464
|
-
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
|
|
1468
|
-
|
|
1469
|
-
|
|
1470
|
-
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
|
|
1524
|
+
|
|
1525
|
+
// src/runtime/client.ts
|
|
1526
|
+
var owners = /* @__PURE__ */ new WeakMap();
|
|
1527
|
+
var isWorkerOwned = (model) => Boolean(model && owners.has(model));
|
|
1528
|
+
function closeModelRuntime(model) {
|
|
1529
|
+
if (model) owners.get(model)?.close();
|
|
1530
|
+
}
|
|
1531
|
+
function runtimeContacts(model) {
|
|
1532
|
+
return owners.get(model)?.contacts();
|
|
1533
|
+
}
|
|
1534
|
+
function assertMainThreadOwnership(model) {
|
|
1535
|
+
if (isWorkerOwned(model)) throw new Error("Worker runtime owns this simulation. Use runtime operations or dispose it before using viewer physics operations.");
|
|
1536
|
+
}
|
|
1537
|
+
var abortError = () => new DOMException("Simulation operation cancelled", "AbortError");
|
|
1538
|
+
var RuntimeClient = class {
|
|
1539
|
+
constructor(worker, apply, release, capture) {
|
|
1540
|
+
this.worker = worker;
|
|
1541
|
+
this.apply = apply;
|
|
1542
|
+
this.release = release;
|
|
1543
|
+
this.capture = capture;
|
|
1544
|
+
worker.onmessage = (event) => {
|
|
1545
|
+
const response = event.data;
|
|
1546
|
+
if (this.closed || response.id !== this.pending?.id) return;
|
|
1547
|
+
if (response.display) {
|
|
1548
|
+
try {
|
|
1549
|
+
this.apply(response.display);
|
|
1550
|
+
const f = response.display.fields;
|
|
1551
|
+
this.latest = {
|
|
1552
|
+
time: response.display.time,
|
|
1553
|
+
qpos: f.qpos,
|
|
1554
|
+
qvel: f.qvel,
|
|
1555
|
+
ctrl: f.ctrl,
|
|
1556
|
+
sensors: f.sensordata,
|
|
1557
|
+
contacts: response.done ? response.display.contacts : this.latest.contacts
|
|
1558
|
+
};
|
|
1559
|
+
} catch (error) {
|
|
1560
|
+
this.close(error instanceof Error ? error : new Error(String(error)));
|
|
1561
|
+
return;
|
|
1562
|
+
}
|
|
1563
|
+
}
|
|
1564
|
+
if (!response.done) this.worker.postMessage({ kind: "ack", id: response.id });
|
|
1565
|
+
if (response.done) {
|
|
1566
|
+
const pending = this.pending;
|
|
1567
|
+
this.pending = null;
|
|
1568
|
+
if (response.error) pending.reject(new Error(response.error));
|
|
1569
|
+
else pending.resolve(response);
|
|
1570
|
+
}
|
|
1571
|
+
};
|
|
1572
|
+
worker.onerror = (event) => this.close(new Error(event.message || "Simulation worker failed"));
|
|
1573
|
+
worker.onmessageerror = () => this.close(new Error("Cannot read simulation worker response"));
|
|
1574
|
+
}
|
|
1575
|
+
pending = null;
|
|
1576
|
+
nextId = 0;
|
|
1577
|
+
closed = false;
|
|
1578
|
+
busy = false;
|
|
1579
|
+
flight = Promise.resolve();
|
|
1580
|
+
generation = 0;
|
|
1581
|
+
latest = { time: 0, qpos: new Float64Array(), qvel: new Float64Array(), ctrl: new Float64Array(), sensors: new Float64Array(), contacts: [] };
|
|
1582
|
+
statistics = { steps: 0, physicsMs: 0, elapsedMs: 0, realTimeFactor: 0, captureMs: 0, busy: false };
|
|
1583
|
+
modelId = crypto.randomUUID();
|
|
1584
|
+
get state() {
|
|
1585
|
+
return {
|
|
1586
|
+
...this.latest,
|
|
1587
|
+
qpos: this.latest.qpos.slice(),
|
|
1588
|
+
qvel: this.latest.qvel.slice(),
|
|
1589
|
+
ctrl: this.latest.ctrl.slice(),
|
|
1590
|
+
sensors: this.latest.sensors.slice(),
|
|
1591
|
+
contacts: this.latest.contacts.map((c) => ({ ...c, pos: [...c.pos] }))
|
|
1592
|
+
};
|
|
1593
|
+
}
|
|
1594
|
+
get metrics() {
|
|
1595
|
+
return { ...this.statistics, busy: this.busy };
|
|
1596
|
+
}
|
|
1597
|
+
request(request) {
|
|
1598
|
+
if (this.closed) return Promise.reject(new Error("Simulation runtime is disposed"));
|
|
1599
|
+
return new Promise((resolve, reject) => {
|
|
1600
|
+
const id = ++this.nextId;
|
|
1601
|
+
this.pending = { id, resolve, reject };
|
|
1474
1602
|
try {
|
|
1475
|
-
|
|
1476
|
-
|
|
1477
|
-
|
|
1478
|
-
|
|
1479
|
-
const error2 = nextError instanceof Error ? nextError : new Error("Unable to capture the current canvas frame.");
|
|
1480
|
-
setError(error2);
|
|
1481
|
-
setStatus("error");
|
|
1482
|
-
throw error2;
|
|
1603
|
+
this.worker.postMessage({ ...request, id });
|
|
1604
|
+
} catch (error) {
|
|
1605
|
+
this.pending = null;
|
|
1606
|
+
reject(error instanceof Error ? error : new Error(String(error)));
|
|
1483
1607
|
}
|
|
1484
|
-
}
|
|
1485
|
-
|
|
1486
|
-
)
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1608
|
+
});
|
|
1609
|
+
}
|
|
1610
|
+
exclusive(operation) {
|
|
1611
|
+
if (this.closed) return Promise.reject(new Error("Simulation runtime is disposed"));
|
|
1612
|
+
if (this.busy) return Promise.reject(new Error("Runtime is busy; await the current operation (work is not queued)"));
|
|
1613
|
+
this.busy = true;
|
|
1614
|
+
const result = operation().finally(() => {
|
|
1615
|
+
this.busy = false;
|
|
1616
|
+
});
|
|
1617
|
+
this.flight = result.catch(() => {
|
|
1618
|
+
});
|
|
1619
|
+
return result;
|
|
1620
|
+
}
|
|
1621
|
+
initialize(initial) {
|
|
1622
|
+
return this.exclusive(async () => {
|
|
1623
|
+
await this.request({ kind: "initialize", initial });
|
|
1624
|
+
});
|
|
1625
|
+
}
|
|
1626
|
+
advance(options) {
|
|
1627
|
+
return this.exclusive(async () => {
|
|
1628
|
+
if (options.signal?.aborted) throw abortError();
|
|
1629
|
+
const started = performance.now(), simStart = this.latest.time;
|
|
1630
|
+
const abort = () => {
|
|
1631
|
+
this.worker.postMessage({ kind: "cancel", id: ++this.nextId });
|
|
1632
|
+
};
|
|
1633
|
+
options.signal?.addEventListener("abort", abort, { once: true });
|
|
1491
1634
|
try {
|
|
1492
|
-
const result = await
|
|
1493
|
-
|
|
1494
|
-
|
|
1635
|
+
const result = await this.request({
|
|
1636
|
+
kind: "advance",
|
|
1637
|
+
steps: options.steps,
|
|
1638
|
+
controls: options.controls ? Float64Array.from(options.controls) : void 0
|
|
1495
1639
|
});
|
|
1496
|
-
|
|
1497
|
-
|
|
1498
|
-
|
|
1499
|
-
|
|
1500
|
-
|
|
1501
|
-
|
|
1502
|
-
|
|
1640
|
+
const elapsedMs = performance.now() - started;
|
|
1641
|
+
this.statistics = {
|
|
1642
|
+
...this.statistics,
|
|
1643
|
+
steps: result.steps ?? 0,
|
|
1644
|
+
physicsMs: result.physicsMs ?? 0,
|
|
1645
|
+
elapsedMs,
|
|
1646
|
+
realTimeFactor: (this.latest.time - simStart) * 1e3 / Math.max(elapsedMs, 1e-3)
|
|
1647
|
+
};
|
|
1648
|
+
if (result.cancelled || options.signal?.aborted) throw abortError();
|
|
1649
|
+
return this.state;
|
|
1650
|
+
} finally {
|
|
1651
|
+
options.signal?.removeEventListener("abort", abort);
|
|
1503
1652
|
}
|
|
1504
|
-
}
|
|
1505
|
-
|
|
1506
|
-
)
|
|
1507
|
-
|
|
1508
|
-
|
|
1509
|
-
|
|
1510
|
-
|
|
1511
|
-
|
|
1512
|
-
|
|
1513
|
-
|
|
1514
|
-
|
|
1515
|
-
}
|
|
1516
|
-
|
|
1517
|
-
|
|
1518
|
-
|
|
1519
|
-
|
|
1520
|
-
|
|
1521
|
-
|
|
1522
|
-
|
|
1523
|
-
|
|
1524
|
-
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
|
|
1528
|
-
|
|
1529
|
-
|
|
1530
|
-
};
|
|
1531
|
-
|
|
1532
|
-
|
|
1533
|
-
|
|
1534
|
-
|
|
1535
|
-
};
|
|
1536
|
-
|
|
1537
|
-
|
|
1538
|
-
|
|
1539
|
-
|
|
1540
|
-
|
|
1541
|
-
|
|
1542
|
-
|
|
1543
|
-
|
|
1544
|
-
}
|
|
1545
|
-
|
|
1546
|
-
|
|
1547
|
-
|
|
1548
|
-
function normalizeAliasCandidates(value) {
|
|
1549
|
-
if (!value) return [];
|
|
1550
|
-
return Array.isArray(value) ? value : [value];
|
|
1551
|
-
}
|
|
1552
|
-
function countMountedSelectors(selector) {
|
|
1553
|
-
return Number(Boolean(selector.cameraName)) + Number(Boolean(selector.siteName)) + Number(Boolean(selector.bodyName));
|
|
1554
|
-
}
|
|
1555
|
-
function normalizeCameraSourceName(value) {
|
|
1556
|
-
return value.trim().toLowerCase().replace(/[^a-z0-9]+/g, "_").replace(/^_+|_+$/g, "");
|
|
1557
|
-
}
|
|
1558
|
-
function createCameraSourceKeyVariants(key) {
|
|
1559
|
-
const candidates = [
|
|
1560
|
-
key,
|
|
1561
|
-
key.startsWith("observation.images.") ? key.slice("observation.images.".length) : "",
|
|
1562
|
-
key.includes(".") ? key.split(".").at(-1) ?? "" : "",
|
|
1563
|
-
key.includes("/") ? key.split("/").at(-1) ?? "" : ""
|
|
1564
|
-
];
|
|
1565
|
-
return candidates.map((candidate) => candidate.trim()).filter((candidate, index, items) => candidate && items.indexOf(candidate) === index);
|
|
1566
|
-
}
|
|
1567
|
-
function getSelectorKey(selector) {
|
|
1568
|
-
if (selector.cameraName) return `camera:${selector.cameraName}`;
|
|
1569
|
-
if (selector.siteName) return `site:${selector.siteName}`;
|
|
1570
|
-
if (selector.bodyName) return `body:${selector.bodyName}`;
|
|
1571
|
-
return null;
|
|
1572
|
-
}
|
|
1573
|
-
function getMountedCameraFrameCaptureSource(selector) {
|
|
1574
|
-
if (countMountedSelectors(selector) !== 1) return null;
|
|
1575
|
-
if (selector.cameraName) {
|
|
1576
|
-
return { kind: "mujoco-camera", cameraName: selector.cameraName };
|
|
1653
|
+
});
|
|
1654
|
+
}
|
|
1655
|
+
observe(cameras) {
|
|
1656
|
+
return this.exclusive(async () => {
|
|
1657
|
+
const generation = this.generation;
|
|
1658
|
+
await this.request({ kind: "snapshot" });
|
|
1659
|
+
const state = this.state, started = performance.now();
|
|
1660
|
+
const frames = cameras ? await this.capture?.(cameras) : void 0;
|
|
1661
|
+
if (cameras && !frames) throw new Error("Camera observations require an attached canvas");
|
|
1662
|
+
this.statistics.captureMs = performance.now() - started;
|
|
1663
|
+
if (generation !== this.generation || this.closed) throw abortError();
|
|
1664
|
+
return { ...state, ...frames ? { cameras: frames } : {} };
|
|
1665
|
+
});
|
|
1666
|
+
}
|
|
1667
|
+
saveState() {
|
|
1668
|
+
return this.exclusive(async () => {
|
|
1669
|
+
const result = await this.request({ kind: "snapshot" });
|
|
1670
|
+
const state = result.display.integration;
|
|
1671
|
+
return { modelId: this.modelId, signature: state.signature, values: state.values.slice() };
|
|
1672
|
+
});
|
|
1673
|
+
}
|
|
1674
|
+
restoreState(snapshot) {
|
|
1675
|
+
return this.exclusive(async () => {
|
|
1676
|
+
if (snapshot.modelId !== this.modelId) throw new Error("Snapshot belongs to a different runtime");
|
|
1677
|
+
await this.request({ kind: "restore", state: { signature: snapshot.signature, values: snapshot.values } });
|
|
1678
|
+
return this.state;
|
|
1679
|
+
});
|
|
1680
|
+
}
|
|
1681
|
+
async cancel() {
|
|
1682
|
+
if (this.closed) return;
|
|
1683
|
+
this.generation++;
|
|
1684
|
+
this.worker.postMessage({ kind: "cancel", id: ++this.nextId });
|
|
1685
|
+
await this.flight;
|
|
1686
|
+
}
|
|
1687
|
+
async reset() {
|
|
1688
|
+
await this.cancel();
|
|
1689
|
+
return this.exclusive(async () => {
|
|
1690
|
+
await this.request({ kind: "reset" });
|
|
1691
|
+
return this.state;
|
|
1692
|
+
});
|
|
1693
|
+
}
|
|
1694
|
+
async dispose() {
|
|
1695
|
+
await this.cancel();
|
|
1696
|
+
this.close();
|
|
1577
1697
|
}
|
|
1578
|
-
|
|
1579
|
-
|
|
1698
|
+
close(error = new Error("Simulation runtime disposed"), restoreViewer = true) {
|
|
1699
|
+
if (this.closed) return;
|
|
1700
|
+
this.closed = true;
|
|
1701
|
+
this.generation++;
|
|
1702
|
+
this.worker.terminate();
|
|
1703
|
+
this.pending?.reject(error);
|
|
1704
|
+
this.pending = null;
|
|
1705
|
+
this.release(restoreViewer);
|
|
1580
1706
|
}
|
|
1581
|
-
|
|
1582
|
-
|
|
1707
|
+
};
|
|
1708
|
+
async function attachSimulationRuntime(module, api, waitForDisplay, invalidate, options = {}) {
|
|
1709
|
+
const model = api.mjModelRef.current, data = api.mjDataRef.current;
|
|
1710
|
+
if (!model || !data) throw new Error("Scene must be ready before creating a runtime");
|
|
1711
|
+
assertMainThreadOwnership(model);
|
|
1712
|
+
api.setPaused(true);
|
|
1713
|
+
const state = api.saveState().integration;
|
|
1714
|
+
const initialState = { signature: state.signature, values: state.values };
|
|
1715
|
+
const mj = module;
|
|
1716
|
+
const path = `/runtime-${crypto.randomUUID()}.mjb`;
|
|
1717
|
+
let binary;
|
|
1718
|
+
try {
|
|
1719
|
+
mj.mj_saveModel(model, path, null);
|
|
1720
|
+
binary = mj.FS.readFile(path);
|
|
1721
|
+
} finally {
|
|
1722
|
+
try {
|
|
1723
|
+
mj.FS.unlink(path);
|
|
1724
|
+
} catch {
|
|
1725
|
+
}
|
|
1583
1726
|
}
|
|
1584
|
-
|
|
1585
|
-
|
|
1586
|
-
|
|
1587
|
-
|
|
1588
|
-
|
|
1589
|
-
|
|
1590
|
-
|
|
1591
|
-
|
|
1592
|
-
|
|
1593
|
-
|
|
1594
|
-
|
|
1595
|
-
|
|
1596
|
-
}
|
|
1597
|
-
|
|
1598
|
-
|
|
1599
|
-
if (!source) return null;
|
|
1600
|
-
return {
|
|
1601
|
-
key,
|
|
1602
|
-
selector,
|
|
1603
|
-
source,
|
|
1604
|
-
resourceName,
|
|
1605
|
-
resourceKind: source.kind,
|
|
1606
|
-
match
|
|
1727
|
+
const worker = options.createWorker?.() ?? new Worker(new URL("./physics.worker.js", import.meta.url), { type: "module" });
|
|
1728
|
+
let complete;
|
|
1729
|
+
const apply = (display) => {
|
|
1730
|
+
if (api.mjModelRef.current !== model) throw new Error("Simulation model changed");
|
|
1731
|
+
if (display.integration) {
|
|
1732
|
+
module.mj_setState(model, data, Array.from(display.integration.values), display.integration.signature);
|
|
1733
|
+
complete = display;
|
|
1734
|
+
}
|
|
1735
|
+
for (const field of displayFields) {
|
|
1736
|
+
const target = data[field];
|
|
1737
|
+
if (!(target instanceof Float64Array || target instanceof Int32Array) || target.length !== display.fields[field].length) throw new Error(`Invalid display field ${field}`);
|
|
1738
|
+
target.set(display.fields[field]);
|
|
1739
|
+
}
|
|
1740
|
+
data.time = display.time;
|
|
1741
|
+
invalidate();
|
|
1607
1742
|
};
|
|
1743
|
+
const client = new RuntimeClient(worker, apply, (restoreViewer) => {
|
|
1744
|
+
try {
|
|
1745
|
+
if (api.mjModelRef.current === model && complete) {
|
|
1746
|
+
apply(complete);
|
|
1747
|
+
if (restoreViewer) {
|
|
1748
|
+
module.mj_forward(model, data);
|
|
1749
|
+
const state2 = complete.integration;
|
|
1750
|
+
module.mj_setState(model, data, Array.from(state2.values), state2.signature);
|
|
1751
|
+
}
|
|
1752
|
+
}
|
|
1753
|
+
} finally {
|
|
1754
|
+
owners.delete(model);
|
|
1755
|
+
}
|
|
1756
|
+
}, async (cameras) => {
|
|
1757
|
+
await waitForDisplay();
|
|
1758
|
+
return capturePolicyCameraFrames(api, cameras);
|
|
1759
|
+
});
|
|
1760
|
+
owners.set(model, { close: () => client.close(void 0, false), contacts: () => client.state.contacts });
|
|
1761
|
+
try {
|
|
1762
|
+
await client.initialize({ model: binary, state: initialState });
|
|
1763
|
+
return client;
|
|
1764
|
+
} catch (error) {
|
|
1765
|
+
client.close();
|
|
1766
|
+
throw error;
|
|
1767
|
+
}
|
|
1608
1768
|
}
|
|
1609
|
-
|
|
1610
|
-
|
|
1611
|
-
|
|
1612
|
-
|
|
1613
|
-
|
|
1614
|
-
|
|
1769
|
+
var CapsuleGeometry = class extends THREE12.BufferGeometry {
|
|
1770
|
+
parameters;
|
|
1771
|
+
constructor(radius = 1, length = 1, capSegments = 4, radialSegments = 8) {
|
|
1772
|
+
super();
|
|
1773
|
+
this.type = "CapsuleGeometry";
|
|
1774
|
+
this.parameters = { radius, length, capSegments, radialSegments };
|
|
1775
|
+
const path = new THREE12.Path();
|
|
1776
|
+
path.absarc(0, -length / 2, radius, Math.PI * 1.5, 0, false);
|
|
1777
|
+
path.absarc(0, length / 2, radius, 0, Math.PI * 0.5, false);
|
|
1778
|
+
const latheGeometry = new THREE12.LatheGeometry(path.getPoints(capSegments), radialSegments);
|
|
1779
|
+
const self = this;
|
|
1780
|
+
self.setIndex(latheGeometry.getIndex());
|
|
1781
|
+
self.setAttribute("position", latheGeometry.getAttribute("position"));
|
|
1782
|
+
self.setAttribute("normal", latheGeometry.getAttribute("normal"));
|
|
1783
|
+
self.setAttribute("uv", latheGeometry.getAttribute("uv"));
|
|
1784
|
+
}
|
|
1785
|
+
};
|
|
1786
|
+
|
|
1787
|
+
// src/rendering/GeomBuilder.ts
|
|
1788
|
+
var DEFAULT_MESH_NORMAL_SMOOTHING_TOLERANCE = 1e-4;
|
|
1789
|
+
var GeomBuilder = class {
|
|
1790
|
+
mujoco;
|
|
1791
|
+
textureCache = /* @__PURE__ */ new Map();
|
|
1792
|
+
renderOptions;
|
|
1793
|
+
constructor(mujoco, renderOptions) {
|
|
1794
|
+
this.mujoco = mujoco;
|
|
1795
|
+
this.renderOptions = renderOptions;
|
|
1796
|
+
}
|
|
1797
|
+
getMeshNormalSmoothingTolerance() {
|
|
1798
|
+
const smoothing = this.renderOptions?.meshNormalSmoothing;
|
|
1799
|
+
if (!smoothing) return null;
|
|
1800
|
+
if (smoothing === true) return DEFAULT_MESH_NORMAL_SMOOTHING_TOLERANCE;
|
|
1801
|
+
return smoothing.tolerance ?? DEFAULT_MESH_NORMAL_SMOOTHING_TOLERANCE;
|
|
1802
|
+
}
|
|
1803
|
+
getMaterialTexture(mjModel, matId) {
|
|
1804
|
+
if (matId < 0 || !mjModel.mat_texid || !mjModel.tex_data) return null;
|
|
1805
|
+
const materialCount = Math.max(1, Math.floor(mjModel.mat_rgba.length / 4));
|
|
1806
|
+
const textureRoles = Math.max(1, Math.floor(mjModel.mat_texid.length / materialCount));
|
|
1807
|
+
let texId = -1;
|
|
1808
|
+
for (let role = 0; role < textureRoles; role += 1) {
|
|
1809
|
+
const candidate = mjModel.mat_texid[matId * textureRoles + role];
|
|
1810
|
+
if (candidate >= 0) {
|
|
1811
|
+
texId = candidate;
|
|
1812
|
+
break;
|
|
1813
|
+
}
|
|
1814
|
+
}
|
|
1815
|
+
if (texId < 0) return null;
|
|
1816
|
+
const cached = this.textureCache.get(texId);
|
|
1817
|
+
if (cached) return cached;
|
|
1818
|
+
const width = Number(mjModel.tex_width[texId]);
|
|
1819
|
+
const height = Number(mjModel.tex_height[texId]);
|
|
1820
|
+
const channels = Number(mjModel.tex_nchannel[texId]);
|
|
1821
|
+
const offset = Number(mjModel.tex_adr[texId]);
|
|
1822
|
+
if (width <= 0 || height <= 0 || channels <= 0 || offset < 0) return null;
|
|
1823
|
+
const source = mjModel.tex_data.subarray(offset, offset + width * height * channels);
|
|
1824
|
+
const rgba = new Uint8Array(width * height * 4);
|
|
1825
|
+
for (let i = 0, j = 0; i < width * height; i += 1, j += channels) {
|
|
1826
|
+
const r = source[j] ?? 255;
|
|
1827
|
+
const g = channels > 1 ? source[j + 1] : r;
|
|
1828
|
+
const b = channels > 2 ? source[j + 2] : r;
|
|
1829
|
+
const a = channels > 3 ? source[j + 3] : 255;
|
|
1830
|
+
const out = i * 4;
|
|
1831
|
+
rgba[out] = r;
|
|
1832
|
+
rgba[out + 1] = g;
|
|
1833
|
+
rgba[out + 2] = b;
|
|
1834
|
+
rgba[out + 3] = a;
|
|
1835
|
+
}
|
|
1836
|
+
const texture = new THREE12.DataTexture(rgba, width, height, THREE12.RGBAFormat);
|
|
1837
|
+
texture.colorSpace = THREE12.LinearSRGBColorSpace;
|
|
1838
|
+
texture.wrapS = THREE12.RepeatWrapping;
|
|
1839
|
+
texture.wrapT = THREE12.RepeatWrapping;
|
|
1840
|
+
texture.flipY = true;
|
|
1841
|
+
const repeatOffset = matId * 2;
|
|
1842
|
+
const repeatS = mjModel.mat_texrepeat?.[repeatOffset] ?? 1;
|
|
1843
|
+
const repeatT = mjModel.mat_texrepeat?.[repeatOffset + 1] ?? 1;
|
|
1844
|
+
texture.repeat.set(repeatS || 1, repeatT || 1);
|
|
1845
|
+
texture.needsUpdate = true;
|
|
1846
|
+
this.textureCache.set(texId, texture);
|
|
1847
|
+
return texture;
|
|
1848
|
+
}
|
|
1849
|
+
/**
|
|
1850
|
+
* Creates a Three.js Object3D (usually a Mesh) for a specific geometry in the MuJoCo model.
|
|
1851
|
+
* Returns null if the geometry shouldn't be rendered (e.g., invisible collision triggers).
|
|
1852
|
+
*/
|
|
1853
|
+
create(mjModel, g) {
|
|
1854
|
+
if (mjModel.geom_group[g] === 3) return null;
|
|
1855
|
+
const type = mjModel.geom_type[g];
|
|
1856
|
+
const size = mjModel.geom_size.subarray(g * 3, g * 3 + 3);
|
|
1857
|
+
const pos = mjModel.geom_pos.subarray(g * 3, g * 3 + 3);
|
|
1858
|
+
const quat = mjModel.geom_quat.subarray(g * 4, g * 4 + 4);
|
|
1859
|
+
const matId = mjModel.geom_matid[g];
|
|
1860
|
+
const color = new THREE12.Color(16777215);
|
|
1861
|
+
const map = this.getMaterialTexture(mjModel, matId);
|
|
1862
|
+
let opacity = 1;
|
|
1863
|
+
if (matId >= 0) {
|
|
1864
|
+
const rgba = mjModel.mat_rgba.subarray(matId * 4, matId * 4 + 4);
|
|
1865
|
+
color.setRGB(rgba[0], rgba[1], rgba[2]);
|
|
1866
|
+
opacity = rgba[3];
|
|
1867
|
+
} else {
|
|
1868
|
+
const rgba = mjModel.geom_rgba.subarray(g * 4, g * 4 + 4);
|
|
1869
|
+
color.setRGB(rgba[0], rgba[1], rgba[2]);
|
|
1870
|
+
opacity = rgba[3];
|
|
1871
|
+
}
|
|
1872
|
+
const MG = this.mujoco.mjtGeom;
|
|
1873
|
+
let geo = null;
|
|
1874
|
+
const getVal = (v) => v?.value ?? v;
|
|
1875
|
+
if (type === getVal(MG.mjGEOM_PLANE)) {
|
|
1876
|
+
geo = new THREE12.PlaneGeometry(size[0] * 2 || 5, size[1] * 2 || 5);
|
|
1877
|
+
} else if (type === getVal(MG.mjGEOM_SPHERE)) {
|
|
1878
|
+
geo = new THREE12.SphereGeometry(size[0], 24, 24);
|
|
1879
|
+
} else if (type === getVal(MG.mjGEOM_CAPSULE)) {
|
|
1880
|
+
geo = new CapsuleGeometry(size[0], size[1] * 2, 24, 12);
|
|
1881
|
+
geo.rotateX(Math.PI / 2);
|
|
1882
|
+
} else if (type === getVal(MG.mjGEOM_BOX)) {
|
|
1883
|
+
geo = new THREE12.BoxGeometry(size[0] * 2, size[1] * 2, size[2] * 2);
|
|
1884
|
+
} else if (type === getVal(MG.mjGEOM_CYLINDER)) {
|
|
1885
|
+
geo = new THREE12.CylinderGeometry(size[0], size[0], size[1] * 2, 24);
|
|
1886
|
+
geo.rotateX(Math.PI / 2);
|
|
1887
|
+
} else if (type === getVal(MG.mjGEOM_MESH)) {
|
|
1888
|
+
const mId = mjModel.geom_dataid[g];
|
|
1889
|
+
const vAdr = mjModel.mesh_vertadr[mId];
|
|
1890
|
+
const vNum = mjModel.mesh_vertnum[mId];
|
|
1891
|
+
const fAdr = mjModel.mesh_faceadr[mId];
|
|
1892
|
+
const fNum = mjModel.mesh_facenum[mId];
|
|
1893
|
+
const tAdr = mjModel.mesh_texcoordadr?.[mId] ?? -1;
|
|
1894
|
+
if (map && tAdr >= 0 && mjModel.mesh_texcoord && mjModel.mesh_facetexcoord) {
|
|
1895
|
+
const corners = fNum * 3;
|
|
1896
|
+
const positions = new Float32Array(corners * 3);
|
|
1897
|
+
const uvs = new Float32Array(corners * 2);
|
|
1898
|
+
for (let c = 0; c < corners; c++) {
|
|
1899
|
+
const v = vAdr + mjModel.mesh_face[fAdr * 3 + c];
|
|
1900
|
+
positions[c * 3] = mjModel.mesh_vert[v * 3];
|
|
1901
|
+
positions[c * 3 + 1] = mjModel.mesh_vert[v * 3 + 1];
|
|
1902
|
+
positions[c * 3 + 2] = mjModel.mesh_vert[v * 3 + 2];
|
|
1903
|
+
const t = tAdr + mjModel.mesh_facetexcoord[fAdr * 3 + c];
|
|
1904
|
+
uvs[c * 2] = mjModel.mesh_texcoord[t * 2];
|
|
1905
|
+
uvs[c * 2 + 1] = 1 - mjModel.mesh_texcoord[t * 2 + 1];
|
|
1906
|
+
}
|
|
1907
|
+
geo = new THREE12.BufferGeometry();
|
|
1908
|
+
geo.setAttribute("position", new THREE12.Float32BufferAttribute(positions, 3));
|
|
1909
|
+
geo.setAttribute("uv", new THREE12.Float32BufferAttribute(uvs, 2));
|
|
1910
|
+
geo.computeVertexNormals();
|
|
1911
|
+
} else {
|
|
1912
|
+
geo = new THREE12.BufferGeometry();
|
|
1913
|
+
geo.setAttribute("position", new THREE12.Float32BufferAttribute(mjModel.mesh_vert.subarray(vAdr * 3, (vAdr + vNum) * 3), 3));
|
|
1914
|
+
geo.setIndex(Array.from(mjModel.mesh_face.subarray(fAdr * 3, (fAdr + fNum) * 3)));
|
|
1915
|
+
const smoothingTolerance = this.getMeshNormalSmoothingTolerance();
|
|
1916
|
+
if (smoothingTolerance !== null) {
|
|
1917
|
+
geo = mergeVertices(geo, smoothingTolerance);
|
|
1918
|
+
}
|
|
1919
|
+
geo.computeVertexNormals();
|
|
1920
|
+
}
|
|
1921
|
+
}
|
|
1922
|
+
if (geo) {
|
|
1923
|
+
const isPlane = type === getVal(MG.mjGEOM_PLANE);
|
|
1924
|
+
const materialMap = isPlane && map ? map.clone() : map;
|
|
1925
|
+
if (isPlane && materialMap) {
|
|
1926
|
+
materialMap.repeat.multiplyScalar(2.5);
|
|
1927
|
+
materialMap.needsUpdate = true;
|
|
1928
|
+
}
|
|
1929
|
+
const mesh = new THREE12.Mesh(geo, new THREE12.MeshStandardMaterial({
|
|
1930
|
+
color,
|
|
1931
|
+
map: materialMap,
|
|
1932
|
+
transparent: opacity < 1,
|
|
1933
|
+
opacity,
|
|
1934
|
+
roughness: 0.6,
|
|
1935
|
+
metalness: 0
|
|
1936
|
+
}));
|
|
1937
|
+
mesh.castShadow = type !== getVal(MG.mjGEOM_PLANE);
|
|
1938
|
+
mesh.receiveShadow = true;
|
|
1939
|
+
mesh.position.set(pos[0], pos[1], pos[2]);
|
|
1940
|
+
mesh.quaternion.set(quat[1], quat[2], quat[3], quat[0]);
|
|
1941
|
+
mesh.userData.bodyID = mjModel.geom_bodyid[g];
|
|
1942
|
+
mesh.userData.geomID = g;
|
|
1943
|
+
mesh.userData.geomGroup = mjModel.geom_group[g];
|
|
1944
|
+
mesh.userData.geomName = getName(mjModel, mjModel.name_geomadr[g]);
|
|
1945
|
+
return mesh;
|
|
1946
|
+
}
|
|
1947
|
+
return null;
|
|
1948
|
+
}
|
|
1949
|
+
};
|
|
1950
|
+
function getRenderOptionsKey(renderOptions) {
|
|
1951
|
+
const smoothing = renderOptions?.meshNormalSmoothing;
|
|
1952
|
+
if (!smoothing) return "default";
|
|
1953
|
+
if (smoothing === true) return "meshNormalSmoothing:true";
|
|
1954
|
+
return `meshNormalSmoothing:${smoothing.tolerance ?? "default"}`;
|
|
1615
1955
|
}
|
|
1616
|
-
function
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
|
|
1620
|
-
|
|
1621
|
-
|
|
1622
|
-
|
|
1623
|
-
|
|
1624
|
-
|
|
1625
|
-
|
|
1956
|
+
function SceneRenderer({ renderOptions, ...props }) {
|
|
1957
|
+
const {
|
|
1958
|
+
mjModelRef,
|
|
1959
|
+
mjDataRef,
|
|
1960
|
+
mujocoRef,
|
|
1961
|
+
onSelectionRef,
|
|
1962
|
+
hiddenBodiesRef,
|
|
1963
|
+
hiddenBodiesVersion,
|
|
1964
|
+
interpolateRef,
|
|
1965
|
+
interpolationStateRef,
|
|
1966
|
+
status
|
|
1967
|
+
} = useMujocoContext();
|
|
1968
|
+
const groupRef = useRef(null);
|
|
1969
|
+
const bodyRefs = useRef([]);
|
|
1970
|
+
const prevModelRef = useRef(null);
|
|
1971
|
+
const prevRenderOptionsKeyRef = useRef(null);
|
|
1972
|
+
const prevHiddenVersionRef = useRef(null);
|
|
1973
|
+
const renderOptionsKey = getRenderOptionsKey(renderOptions);
|
|
1974
|
+
const geomBuilder = useMemo(() => {
|
|
1975
|
+
if (status !== "ready") return null;
|
|
1976
|
+
return new GeomBuilder(mujocoRef.current, renderOptions);
|
|
1977
|
+
}, [status, mujocoRef, renderOptionsKey]);
|
|
1978
|
+
useEffect(() => {
|
|
1979
|
+
if (status !== "ready" || !geomBuilder) return;
|
|
1980
|
+
const model = mjModelRef.current;
|
|
1981
|
+
const group = groupRef.current;
|
|
1982
|
+
if (!model || !group) return;
|
|
1983
|
+
if (prevModelRef.current === model && prevRenderOptionsKeyRef.current === renderOptionsKey && prevHiddenVersionRef.current === hiddenBodiesVersion) {
|
|
1984
|
+
return;
|
|
1626
1985
|
}
|
|
1627
|
-
|
|
1628
|
-
|
|
1986
|
+
prevModelRef.current = model;
|
|
1987
|
+
prevRenderOptionsKeyRef.current = renderOptionsKey;
|
|
1988
|
+
prevHiddenVersionRef.current = hiddenBodiesVersion;
|
|
1989
|
+
while (group.children.length > 0) {
|
|
1990
|
+
group.remove(group.children[0]);
|
|
1629
1991
|
}
|
|
1630
|
-
|
|
1631
|
-
|
|
1992
|
+
const refs = [];
|
|
1993
|
+
for (let i = 0; i < model.nbody; i++) {
|
|
1994
|
+
const bodyGroup = new THREE12.Group();
|
|
1995
|
+
bodyGroup.userData.bodyID = i;
|
|
1996
|
+
const bodyName = getName(model, model.name_bodyadr[i]);
|
|
1997
|
+
if (!hiddenBodiesRef.current.has(bodyName)) {
|
|
1998
|
+
for (let g = 0; g < model.ngeom; g++) {
|
|
1999
|
+
if (model.geom_bodyid[g] === i) {
|
|
2000
|
+
const mesh = geomBuilder.create(model, g);
|
|
2001
|
+
if (mesh) bodyGroup.add(mesh);
|
|
2002
|
+
}
|
|
2003
|
+
}
|
|
2004
|
+
}
|
|
2005
|
+
group.add(bodyGroup);
|
|
2006
|
+
refs.push(bodyGroup);
|
|
1632
2007
|
}
|
|
1633
|
-
|
|
1634
|
-
|
|
2008
|
+
bodyRefs.current = refs;
|
|
2009
|
+
}, [status, geomBuilder, mjModelRef, renderOptionsKey, hiddenBodiesRef, hiddenBodiesVersion]);
|
|
2010
|
+
const syncBodiesToData = useCallback(() => {
|
|
2011
|
+
const data = mjDataRef.current;
|
|
2012
|
+
if (!data) return;
|
|
2013
|
+
const bodies = bodyRefs.current;
|
|
2014
|
+
const interpolation = interpolationStateRef.current;
|
|
2015
|
+
const useInterpolation = interpolateRef.current && interpolation.valid;
|
|
2016
|
+
for (let i = 0; i < bodies.length; i++) {
|
|
2017
|
+
const ref = bodies[i];
|
|
2018
|
+
if (!ref) continue;
|
|
2019
|
+
if (useInterpolation) {
|
|
2020
|
+
const alpha = interpolation.alpha;
|
|
2021
|
+
const i3 = i * 3;
|
|
2022
|
+
ref.position.set(
|
|
2023
|
+
THREE12.MathUtils.lerp(interpolation.previousXpos[i3], interpolation.currentXpos[i3], alpha),
|
|
2024
|
+
THREE12.MathUtils.lerp(interpolation.previousXpos[i3 + 1], interpolation.currentXpos[i3 + 1], alpha),
|
|
2025
|
+
THREE12.MathUtils.lerp(interpolation.previousXpos[i3 + 2], interpolation.currentXpos[i3 + 2], alpha)
|
|
2026
|
+
);
|
|
2027
|
+
const i4 = i * 4;
|
|
2028
|
+
_previousQuat.set(
|
|
2029
|
+
interpolation.previousXquat[i4 + 1],
|
|
2030
|
+
interpolation.previousXquat[i4 + 2],
|
|
2031
|
+
interpolation.previousXquat[i4 + 3],
|
|
2032
|
+
interpolation.previousXquat[i4]
|
|
2033
|
+
);
|
|
2034
|
+
_currentQuat.set(
|
|
2035
|
+
interpolation.currentXquat[i4 + 1],
|
|
2036
|
+
interpolation.currentXquat[i4 + 2],
|
|
2037
|
+
interpolation.currentXquat[i4 + 3],
|
|
2038
|
+
interpolation.currentXquat[i4]
|
|
2039
|
+
);
|
|
2040
|
+
ref.quaternion.copy(_previousQuat).slerp(_currentQuat, alpha);
|
|
2041
|
+
} else {
|
|
2042
|
+
ref.position.set(
|
|
2043
|
+
data.xpos[i * 3],
|
|
2044
|
+
data.xpos[i * 3 + 1],
|
|
2045
|
+
data.xpos[i * 3 + 2]
|
|
2046
|
+
);
|
|
2047
|
+
ref.quaternion.set(
|
|
2048
|
+
data.xquat[i * 4 + 1],
|
|
2049
|
+
data.xquat[i * 4 + 2],
|
|
2050
|
+
data.xquat[i * 4 + 3],
|
|
2051
|
+
data.xquat[i * 4]
|
|
2052
|
+
);
|
|
2053
|
+
}
|
|
1635
2054
|
}
|
|
1636
|
-
}
|
|
1637
|
-
|
|
1638
|
-
|
|
1639
|
-
|
|
1640
|
-
|
|
1641
|
-
|
|
2055
|
+
}, [interpolateRef, interpolationStateRef, mjDataRef]);
|
|
2056
|
+
useFrame(syncBodiesToData);
|
|
2057
|
+
return /* @__PURE__ */ jsx(
|
|
2058
|
+
"group",
|
|
2059
|
+
{
|
|
2060
|
+
...props,
|
|
2061
|
+
ref: groupRef,
|
|
2062
|
+
userData: {
|
|
2063
|
+
...props.userData,
|
|
2064
|
+
[CAMERA_FRAME_CAPTURE_PRE_RENDER_USER_DATA_KEY]: syncBodiesToData
|
|
2065
|
+
},
|
|
2066
|
+
onDoubleClick: (e) => {
|
|
2067
|
+
if (typeof props.onDoubleClick === "function") props.onDoubleClick(e);
|
|
2068
|
+
e.stopPropagation();
|
|
2069
|
+
let obj = e.object;
|
|
2070
|
+
while (obj && obj.userData.bodyID === void 0 && obj.parent) {
|
|
2071
|
+
obj = obj.parent;
|
|
2072
|
+
}
|
|
2073
|
+
const bodyID = obj?.userData.bodyID;
|
|
2074
|
+
if (typeof bodyID === "number" && bodyID > 0) {
|
|
2075
|
+
const model = mjModelRef.current;
|
|
2076
|
+
if (model && bodyID < model.nbody && onSelectionRef.current) {
|
|
2077
|
+
const name = getName(model, model.name_bodyadr[bodyID]);
|
|
2078
|
+
onSelectionRef.current({ bodyId: bodyID, name });
|
|
2079
|
+
}
|
|
2080
|
+
}
|
|
2081
|
+
}
|
|
2082
|
+
}
|
|
2083
|
+
);
|
|
1642
2084
|
}
|
|
1643
|
-
|
|
1644
|
-
|
|
1645
|
-
|
|
1646
|
-
|
|
1647
|
-
|
|
1648
|
-
|
|
1649
|
-
|
|
1650
|
-
|
|
1651
|
-
|
|
2085
|
+
var _previousQuat = new THREE12.Quaternion();
|
|
2086
|
+
var _currentQuat = new THREE12.Quaternion();
|
|
2087
|
+
var CameraViewportRegistryContext = createContext(null);
|
|
2088
|
+
var nextViewportId = 0;
|
|
2089
|
+
var VIEWPORT_RENDER_PRIORITY = 1;
|
|
2090
|
+
function CameraViewportRenderer({
|
|
2091
|
+
viewportsRef
|
|
2092
|
+
}) {
|
|
2093
|
+
const gl = useThree((state) => state.gl);
|
|
2094
|
+
const scene = useThree((state) => state.scene);
|
|
2095
|
+
const mainCamera = useThree((state) => state.camera);
|
|
2096
|
+
const mujoco = useMujoco();
|
|
2097
|
+
useFrame(() => {
|
|
2098
|
+
const dpr = gl.getPixelRatio();
|
|
2099
|
+
const drawWidth = gl.domElement.width / dpr;
|
|
2100
|
+
const drawHeight = gl.domElement.height / dpr;
|
|
2101
|
+
gl.setScissorTest(false);
|
|
2102
|
+
gl.setViewport(0, 0, drawWidth, drawHeight);
|
|
2103
|
+
gl.render(scene, mainCamera);
|
|
2104
|
+
const api = mujoco.api;
|
|
2105
|
+
if (!api || viewportsRef.current.size === 0) return;
|
|
2106
|
+
const canvasRect = gl.domElement.getBoundingClientRect();
|
|
2107
|
+
for (const descriptor of viewportsRef.current.values()) {
|
|
2108
|
+
const element = descriptor.getElement();
|
|
2109
|
+
if (!element) continue;
|
|
2110
|
+
const rect = element.getBoundingClientRect();
|
|
2111
|
+
const isOffscreen = rect.bottom < canvasRect.top || rect.top > canvasRect.bottom || rect.right < canvasRect.left || rect.left > canvasRect.right;
|
|
2112
|
+
if (isOffscreen) continue;
|
|
2113
|
+
const width = rect.width;
|
|
2114
|
+
const height = rect.height;
|
|
2115
|
+
if (width <= 0 || height <= 0) continue;
|
|
2116
|
+
const left = rect.left - canvasRect.left;
|
|
2117
|
+
const bottom = canvasRect.bottom - rect.bottom;
|
|
2118
|
+
let resolved;
|
|
2119
|
+
try {
|
|
2120
|
+
resolved = api.resolveCameraCaptureOptions(descriptor.getOptions());
|
|
2121
|
+
} catch {
|
|
2122
|
+
continue;
|
|
2123
|
+
}
|
|
2124
|
+
if (!descriptor.camera) {
|
|
2125
|
+
descriptor.camera = createCaptureCamera(resolved, mainCamera, width, height);
|
|
2126
|
+
} else {
|
|
2127
|
+
prepareCaptureCamera(descriptor.camera, resolved, mainCamera, width, height);
|
|
2128
|
+
}
|
|
2129
|
+
gl.setViewport(left, bottom, width, height);
|
|
2130
|
+
gl.setScissor(left, bottom, width, height);
|
|
2131
|
+
gl.setScissorTest(true);
|
|
2132
|
+
gl.render(scene, descriptor.camera);
|
|
1652
2133
|
}
|
|
1653
|
-
|
|
1654
|
-
|
|
1655
|
-
|
|
1656
|
-
|
|
1657
|
-
|
|
1658
|
-
|
|
1659
|
-
|
|
1660
|
-
|
|
1661
|
-
|
|
1662
|
-
|
|
1663
|
-
|
|
1664
|
-
);
|
|
1665
|
-
|
|
1666
|
-
|
|
1667
|
-
|
|
1668
|
-
|
|
1669
|
-
|
|
1670
|
-
|
|
1671
|
-
|
|
1672
|
-
|
|
1673
|
-
|
|
1674
|
-
|
|
1675
|
-
cameraName,
|
|
1676
|
-
match
|
|
1677
|
-
)
|
|
1678
|
-
);
|
|
1679
|
-
}
|
|
1680
|
-
for (const siteName of createResourceNames(options.sites)) {
|
|
1681
|
-
const match = getCameraFrameResourceMatch(key, siteName);
|
|
1682
|
-
if (!match) continue;
|
|
1683
|
-
addMountedCameraFrameSourceSuggestion(
|
|
1684
|
-
suggestions,
|
|
1685
|
-
seen,
|
|
1686
|
-
createMountedCameraFrameSourceSuggestion(
|
|
1687
|
-
key,
|
|
1688
|
-
{ siteName },
|
|
1689
|
-
siteName,
|
|
1690
|
-
match
|
|
1691
|
-
)
|
|
1692
|
-
);
|
|
1693
|
-
}
|
|
1694
|
-
for (const bodyName of createResourceNames(options.bodies)) {
|
|
1695
|
-
const match = getCameraFrameResourceMatch(key, bodyName);
|
|
1696
|
-
if (!match) continue;
|
|
1697
|
-
addMountedCameraFrameSourceSuggestion(
|
|
1698
|
-
suggestions,
|
|
1699
|
-
seen,
|
|
1700
|
-
createMountedCameraFrameSourceSuggestion(
|
|
1701
|
-
key,
|
|
1702
|
-
{ bodyName },
|
|
1703
|
-
bodyName,
|
|
1704
|
-
match
|
|
1705
|
-
)
|
|
1706
|
-
);
|
|
1707
|
-
}
|
|
1708
|
-
return suggestions;
|
|
2134
|
+
gl.setScissorTest(false);
|
|
2135
|
+
gl.setViewport(0, 0, drawWidth, drawHeight);
|
|
2136
|
+
}, VIEWPORT_RENDER_PRIORITY);
|
|
2137
|
+
return null;
|
|
2138
|
+
}
|
|
2139
|
+
function CameraViewportProvider({ children }) {
|
|
2140
|
+
const viewportsRef = useRef(/* @__PURE__ */ new Map());
|
|
2141
|
+
const [count, setCount] = useState(0);
|
|
2142
|
+
const register = useCallback((descriptor) => {
|
|
2143
|
+
const id = nextViewportId++;
|
|
2144
|
+
viewportsRef.current.set(id, descriptor);
|
|
2145
|
+
setCount((value2) => value2 + 1);
|
|
2146
|
+
return () => {
|
|
2147
|
+
viewportsRef.current.delete(id);
|
|
2148
|
+
setCount((value2) => value2 - 1);
|
|
2149
|
+
};
|
|
2150
|
+
}, []);
|
|
2151
|
+
const value = useMemo(() => ({ register }), [register]);
|
|
2152
|
+
return /* @__PURE__ */ jsxs(CameraViewportRegistryContext.Provider, { value, children: [
|
|
2153
|
+
children,
|
|
2154
|
+
count > 0 && /* @__PURE__ */ jsx(CameraViewportRenderer, { viewportsRef })
|
|
2155
|
+
] });
|
|
1709
2156
|
}
|
|
1710
|
-
function
|
|
1711
|
-
const
|
|
1712
|
-
|
|
1713
|
-
|
|
1714
|
-
const directCandidates = [
|
|
1715
|
-
{ cameraName: key },
|
|
1716
|
-
{ siteName: key },
|
|
1717
|
-
{ bodyName: key }
|
|
1718
|
-
];
|
|
1719
|
-
const aliasCandidates = normalizeAliasCandidates(options.aliases?.[key]);
|
|
1720
|
-
for (const selector of aliasCandidates) {
|
|
1721
|
-
if (!isSelectorMounted(selector, cameraNames, siteNames, bodyNames)) {
|
|
1722
|
-
continue;
|
|
1723
|
-
}
|
|
1724
|
-
const source = getMountedCameraFrameCaptureSource(selector);
|
|
1725
|
-
if (!source) continue;
|
|
1726
|
-
return { key, selector, source };
|
|
2157
|
+
function useCameraViewport(elementRef, options) {
|
|
2158
|
+
const registry = useContext(CameraViewportRegistryContext);
|
|
2159
|
+
if (!registry) {
|
|
2160
|
+
throw new Error("useCameraViewport must be used inside <MujocoCanvas>.");
|
|
1727
2161
|
}
|
|
1728
|
-
const
|
|
1729
|
-
|
|
1730
|
-
|
|
1731
|
-
|
|
1732
|
-
|
|
1733
|
-
|
|
2162
|
+
const optionsRef = useRef(options);
|
|
2163
|
+
optionsRef.current = options;
|
|
2164
|
+
useEffect(() => {
|
|
2165
|
+
const descriptor = {
|
|
2166
|
+
getElement: () => elementRef.current,
|
|
2167
|
+
getOptions: () => optionsRef.current,
|
|
2168
|
+
camera: null
|
|
1734
2169
|
};
|
|
2170
|
+
return registry.register(descriptor);
|
|
2171
|
+
}, [registry, elementRef]);
|
|
2172
|
+
}
|
|
2173
|
+
function CameraView({ className, style, ...options }) {
|
|
2174
|
+
const gl = useThree((state) => state.gl);
|
|
2175
|
+
const elementRef = useRef(null);
|
|
2176
|
+
if (!elementRef.current && typeof document !== "undefined") {
|
|
2177
|
+
elementRef.current = document.createElement("div");
|
|
1735
2178
|
}
|
|
1736
|
-
|
|
1737
|
-
|
|
1738
|
-
|
|
1739
|
-
|
|
1740
|
-
|
|
1741
|
-
|
|
1742
|
-
|
|
1743
|
-
|
|
1744
|
-
|
|
1745
|
-
|
|
1746
|
-
|
|
1747
|
-
|
|
1748
|
-
|
|
1749
|
-
|
|
1750
|
-
|
|
2179
|
+
useEffect(() => {
|
|
2180
|
+
const element = elementRef.current;
|
|
2181
|
+
const parent = gl.domElement.parentElement;
|
|
2182
|
+
if (!element || !parent) return;
|
|
2183
|
+
element.style.position = "absolute";
|
|
2184
|
+
element.style.overflow = "hidden";
|
|
2185
|
+
parent.appendChild(element);
|
|
2186
|
+
return () => {
|
|
2187
|
+
parent.removeChild(element);
|
|
2188
|
+
};
|
|
2189
|
+
}, [gl]);
|
|
2190
|
+
useEffect(() => {
|
|
2191
|
+
const element = elementRef.current;
|
|
2192
|
+
if (!element) return;
|
|
2193
|
+
element.className = className ?? "";
|
|
2194
|
+
if (style) Object.assign(element.style, style);
|
|
2195
|
+
}, [className, style]);
|
|
2196
|
+
useCameraViewport(elementRef, options);
|
|
1751
2197
|
return null;
|
|
1752
2198
|
}
|
|
1753
|
-
function
|
|
1754
|
-
|
|
1755
|
-
|
|
1756
|
-
|
|
1757
|
-
|
|
1758
|
-
|
|
1759
|
-
|
|
1760
|
-
missingKeys.push(key);
|
|
1761
|
-
continue;
|
|
1762
|
-
}
|
|
1763
|
-
resolved[key] = mountedSource;
|
|
1764
|
-
cameras.push({
|
|
1765
|
-
key,
|
|
1766
|
-
...options.defaults,
|
|
1767
|
-
...options.cameraOptions?.[key],
|
|
1768
|
-
...mountedSource.selector,
|
|
1769
|
-
source: mountedSource.source
|
|
1770
|
-
});
|
|
2199
|
+
function isTargetRef(target) {
|
|
2200
|
+
return Boolean(target && typeof target === "object" && "current" in target);
|
|
2201
|
+
}
|
|
2202
|
+
function resolveCanvasTarget(target) {
|
|
2203
|
+
const resolvedTarget = isTargetRef(target) ? target.current : target;
|
|
2204
|
+
if (!resolvedTarget) {
|
|
2205
|
+
throw new Error("No frame capture target is available.");
|
|
1771
2206
|
}
|
|
1772
|
-
if (
|
|
1773
|
-
|
|
1774
|
-
`Unable to resolve mounted MuJoCo camera source${missingKeys.length === 1 ? "" : "s"} for ${missingKeys.join(", ")}.`
|
|
1775
|
-
);
|
|
2207
|
+
if (resolvedTarget instanceof HTMLCanvasElement) {
|
|
2208
|
+
return resolvedTarget;
|
|
1776
2209
|
}
|
|
1777
|
-
|
|
1778
|
-
|
|
1779
|
-
|
|
1780
|
-
resolved,
|
|
1781
|
-
missingKeys
|
|
1782
|
-
};
|
|
1783
|
-
}
|
|
1784
|
-
function createMountedCameraFrameSequenceReadiness(plan) {
|
|
1785
|
-
const cameras = {};
|
|
1786
|
-
const resolvedKeys = plan.cameraKeys.filter((key) => Boolean(plan.resolved[key]));
|
|
1787
|
-
for (const key of plan.cameraKeys) {
|
|
1788
|
-
const resolved = plan.resolved[key];
|
|
1789
|
-
cameras[key] = resolved ? {
|
|
1790
|
-
key,
|
|
1791
|
-
ready: true,
|
|
1792
|
-
selector: resolved.selector,
|
|
1793
|
-
source: resolved.source,
|
|
1794
|
-
message: `Camera stream "${key}" resolves to ${resolved.source.kind}:${getCameraFrameCaptureSourceTarget(resolved.source)}.`
|
|
1795
|
-
} : {
|
|
1796
|
-
key,
|
|
1797
|
-
ready: false,
|
|
1798
|
-
message: `Camera stream "${key}" does not resolve to a mounted MuJoCo camera, site, or body.`
|
|
1799
|
-
};
|
|
2210
|
+
const canvas = resolvedTarget.querySelector("canvas");
|
|
2211
|
+
if (!canvas) {
|
|
2212
|
+
throw new Error("Frame capture target does not contain a canvas.");
|
|
1800
2213
|
}
|
|
1801
|
-
|
|
1802
|
-
const ready = missingKeys.length === 0;
|
|
1803
|
-
const status = ready ? MountedCameraFrameSequenceReadinessStatus.Ready : resolvedKeys.length > 0 ? MountedCameraFrameSequenceReadinessStatus.Partial : MountedCameraFrameSequenceReadinessStatus.Missing;
|
|
1804
|
-
return {
|
|
1805
|
-
ready,
|
|
1806
|
-
status,
|
|
1807
|
-
cameraKeys: [...plan.cameraKeys],
|
|
1808
|
-
resolvedKeys,
|
|
1809
|
-
missingKeys,
|
|
1810
|
-
cameras,
|
|
1811
|
-
message: ready ? `All ${plan.cameraKeys.length} requested camera stream${plan.cameraKeys.length === 1 ? "" : "s"} resolve to mounted MuJoCo sources.` : `Missing mounted MuJoCo source${missingKeys.length === 1 ? "" : "s"} for ${missingKeys.join(", ")}.`
|
|
1812
|
-
};
|
|
2214
|
+
return canvas;
|
|
1813
2215
|
}
|
|
1814
|
-
function
|
|
1815
|
-
return
|
|
2216
|
+
function waitForNextAnimationFrame() {
|
|
2217
|
+
return new Promise((resolve) => {
|
|
2218
|
+
requestAnimationFrame(() => resolve());
|
|
2219
|
+
});
|
|
1816
2220
|
}
|
|
1817
|
-
function
|
|
1818
|
-
const
|
|
1819
|
-
|
|
1820
|
-
|
|
1821
|
-
|
|
1822
|
-
options.expectedFrameCount ?? result.frameCount
|
|
1823
|
-
);
|
|
1824
|
-
const recordedFrameCount = normalizeFrameCount(result.frameCount);
|
|
1825
|
-
const streamSummaries = {};
|
|
1826
|
-
const streams = [];
|
|
1827
|
-
let missingFrameCount = 0;
|
|
1828
|
-
let completeStreamCount = 0;
|
|
1829
|
-
let resolvedOrRecordedStreamCount = 0;
|
|
1830
|
-
for (const key of cameraKeys) {
|
|
1831
|
-
const summary = result.cameraSummaries[key];
|
|
1832
|
-
const readiness = result.readiness.cameras[key];
|
|
1833
|
-
const source = summary?.source ?? readiness?.source;
|
|
1834
|
-
const ready = readiness?.ready ?? Boolean(summary);
|
|
1835
|
-
const recorded = normalizeFrameCount(summary?.frameCount);
|
|
1836
|
-
const missing = Math.max(expectedFrameCount - recorded, 0);
|
|
1837
|
-
const complete2 = ready && missing === 0;
|
|
1838
|
-
const status2 = complete2 ? MountedCameraFrameSequenceManifestStatus.Complete : ready || recorded > 0 ? MountedCameraFrameSequenceManifestStatus.Partial : MountedCameraFrameSequenceManifestStatus.Missing;
|
|
1839
|
-
const target = source ? getCameraFrameCaptureSourceTarget(source) : readiness?.message ? void 0 : "missing MuJoCo camera";
|
|
1840
|
-
const message = complete2 ? `Camera stream "${key}" recorded ${recorded} of ${expectedFrameCount} frame${expectedFrameCount === 1 ? "" : "s"}.` : ready || recorded > 0 ? `Camera stream "${key}" recorded ${recorded} of ${expectedFrameCount} frame${expectedFrameCount === 1 ? "" : "s"}.` : readiness?.message ?? `Camera stream "${key}" did not record any frames.`;
|
|
1841
|
-
const stream = {
|
|
1842
|
-
key,
|
|
1843
|
-
ready,
|
|
1844
|
-
complete: complete2,
|
|
1845
|
-
status: status2,
|
|
1846
|
-
source,
|
|
1847
|
-
selector: readiness?.selector,
|
|
1848
|
-
target,
|
|
1849
|
-
width: summary?.width,
|
|
1850
|
-
height: summary?.height,
|
|
1851
|
-
expectedFrameCount,
|
|
1852
|
-
recordedFrameCount: recorded,
|
|
1853
|
-
missingFrameCount: missing,
|
|
1854
|
-
firstFrameIndex: summary?.firstFrameIndex ?? null,
|
|
1855
|
-
lastFrameIndex: summary?.lastFrameIndex ?? null,
|
|
1856
|
-
firstTimestamp: summary?.firstTimestamp ?? null,
|
|
1857
|
-
lastTimestamp: summary?.lastTimestamp ?? null,
|
|
1858
|
-
message
|
|
1859
|
-
};
|
|
1860
|
-
streamSummaries[key] = stream;
|
|
1861
|
-
streams.push(stream);
|
|
1862
|
-
missingFrameCount += missing;
|
|
1863
|
-
if (complete2) completeStreamCount += 1;
|
|
1864
|
-
if (ready || recorded > 0) resolvedOrRecordedStreamCount += 1;
|
|
2221
|
+
async function captureFrame(options) {
|
|
2222
|
+
const type = options.type ?? "image/png";
|
|
2223
|
+
const canvas = resolveCanvasTarget(options.target);
|
|
2224
|
+
if (options.waitForAnimationFrame ?? true) {
|
|
2225
|
+
await waitForNextAnimationFrame();
|
|
1865
2226
|
}
|
|
1866
|
-
const complete = result.readiness.ready && streams.length === completeStreamCount && missingFrameCount === 0;
|
|
1867
|
-
const status = complete ? MountedCameraFrameSequenceManifestStatus.Complete : resolvedOrRecordedStreamCount > 0 ? MountedCameraFrameSequenceManifestStatus.Partial : MountedCameraFrameSequenceManifestStatus.Missing;
|
|
1868
2227
|
return {
|
|
1869
|
-
|
|
1870
|
-
|
|
1871
|
-
|
|
1872
|
-
status,
|
|
1873
|
-
cameraKeys,
|
|
1874
|
-
resolvedKeys: [...result.readiness.resolvedKeys],
|
|
1875
|
-
missingKeys: [...result.readiness.missingKeys],
|
|
1876
|
-
expectedFrameCount,
|
|
1877
|
-
recordedFrameCount,
|
|
1878
|
-
missingFrameCount,
|
|
1879
|
-
streamSummaries,
|
|
1880
|
-
streams,
|
|
1881
|
-
readiness: result.readiness,
|
|
1882
|
-
message: complete ? `All ${cameraKeys.length} camera stream${cameraKeys.length === 1 ? "" : "s"} recorded ${expectedFrameCount} frame${expectedFrameCount === 1 ? "" : "s"}.` : `Mounted camera sequence coverage is ${status}.`
|
|
2228
|
+
canvas,
|
|
2229
|
+
dataUrl: canvas.toDataURL(type, options.quality),
|
|
2230
|
+
type
|
|
1883
2231
|
};
|
|
1884
2232
|
}
|
|
1885
|
-
function
|
|
1886
|
-
|
|
1887
|
-
|
|
1888
|
-
|
|
1889
|
-
|
|
1890
|
-
|
|
1891
|
-
|
|
1892
|
-
|
|
1893
|
-
|
|
1894
|
-
|
|
1895
|
-
|
|
1896
|
-
|
|
1897
|
-
|
|
1898
|
-
|
|
1899
|
-
|
|
1900
|
-
|
|
1901
|
-
|
|
1902
|
-
|
|
1903
|
-
...restOptions,
|
|
1904
|
-
cameras: plan.cameras,
|
|
1905
|
-
requireMountedSources: restOptions.requireMountedSources ?? true
|
|
2233
|
+
async function captureFrameBlob(options) {
|
|
2234
|
+
const type = options.type ?? "image/png";
|
|
2235
|
+
const canvas = resolveCanvasTarget(options.target);
|
|
2236
|
+
if (options.waitForAnimationFrame ?? true) {
|
|
2237
|
+
await waitForNextAnimationFrame();
|
|
2238
|
+
}
|
|
2239
|
+
const blob = await new Promise((resolve, reject) => {
|
|
2240
|
+
canvas.toBlob(
|
|
2241
|
+
(nextBlob) => {
|
|
2242
|
+
if (nextBlob) {
|
|
2243
|
+
resolve(nextBlob);
|
|
2244
|
+
} else {
|
|
2245
|
+
reject(new Error("Canvas frame capture did not produce a Blob."));
|
|
2246
|
+
}
|
|
2247
|
+
},
|
|
2248
|
+
type,
|
|
2249
|
+
options.quality
|
|
2250
|
+
);
|
|
1906
2251
|
});
|
|
2252
|
+
return { canvas, blob, type };
|
|
2253
|
+
}
|
|
2254
|
+
function useFrameCapture(defaultOptions = {}) {
|
|
2255
|
+
const [status, setStatus] = useState("idle");
|
|
2256
|
+
const [error, setError] = useState(null);
|
|
2257
|
+
const reset = useCallback(() => {
|
|
2258
|
+
setStatus("idle");
|
|
2259
|
+
setError(null);
|
|
2260
|
+
}, []);
|
|
2261
|
+
const capture = useCallback(
|
|
2262
|
+
async (options = {}) => {
|
|
2263
|
+
setStatus("capturing");
|
|
2264
|
+
setError(null);
|
|
2265
|
+
try {
|
|
2266
|
+
const result = await captureFrame({ ...defaultOptions, ...options });
|
|
2267
|
+
setStatus("captured");
|
|
2268
|
+
return result;
|
|
2269
|
+
} catch (nextError) {
|
|
2270
|
+
const error2 = nextError instanceof Error ? nextError : new Error("Unable to capture the current canvas frame.");
|
|
2271
|
+
setError(error2);
|
|
2272
|
+
setStatus("error");
|
|
2273
|
+
throw error2;
|
|
2274
|
+
}
|
|
2275
|
+
},
|
|
2276
|
+
[defaultOptions]
|
|
2277
|
+
);
|
|
2278
|
+
const captureBlob = useCallback(
|
|
2279
|
+
async (options = {}) => {
|
|
2280
|
+
setStatus("capturing");
|
|
2281
|
+
setError(null);
|
|
2282
|
+
try {
|
|
2283
|
+
const result = await captureFrameBlob({
|
|
2284
|
+
...defaultOptions,
|
|
2285
|
+
...options
|
|
2286
|
+
});
|
|
2287
|
+
setStatus("captured");
|
|
2288
|
+
return result;
|
|
2289
|
+
} catch (nextError) {
|
|
2290
|
+
const error2 = nextError instanceof Error ? nextError : new Error("Unable to capture the current canvas frame.");
|
|
2291
|
+
setError(error2);
|
|
2292
|
+
setStatus("error");
|
|
2293
|
+
throw error2;
|
|
2294
|
+
}
|
|
2295
|
+
},
|
|
2296
|
+
[defaultOptions]
|
|
2297
|
+
);
|
|
1907
2298
|
return {
|
|
1908
|
-
|
|
1909
|
-
|
|
1910
|
-
|
|
2299
|
+
status,
|
|
2300
|
+
error,
|
|
2301
|
+
isCapturing: status === "capturing",
|
|
2302
|
+
capture,
|
|
2303
|
+
captureBlob,
|
|
2304
|
+
reset
|
|
1911
2305
|
};
|
|
1912
2306
|
}
|
|
1913
2307
|
var _raycaster = new THREE12.Raycaster();
|
|
@@ -2358,10 +2752,11 @@ function useMujoco() {
|
|
|
2358
2752
|
};
|
|
2359
2753
|
}
|
|
2360
2754
|
function useBeforePhysicsStep(callback) {
|
|
2361
|
-
const { beforeStepCallbacks } = useMujocoContext();
|
|
2755
|
+
const { beforeStepCallbacks, mjModelRef } = useMujocoContext();
|
|
2362
2756
|
const callbackRef = useRef(callback);
|
|
2363
2757
|
callbackRef.current = callback;
|
|
2364
2758
|
useEffect(() => {
|
|
2759
|
+
assertMainThreadOwnership(mjModelRef.current);
|
|
2365
2760
|
const wrapped = (input) => callbackRef.current(input);
|
|
2366
2761
|
beforeStepCallbacks.current.add(wrapped);
|
|
2367
2762
|
return () => {
|
|
@@ -2370,10 +2765,11 @@ function useBeforePhysicsStep(callback) {
|
|
|
2370
2765
|
}, [beforeStepCallbacks]);
|
|
2371
2766
|
}
|
|
2372
2767
|
function useAfterPhysicsStep(callback) {
|
|
2373
|
-
const { afterStepCallbacks } = useMujocoContext();
|
|
2768
|
+
const { afterStepCallbacks, mjModelRef } = useMujocoContext();
|
|
2374
2769
|
const callbackRef = useRef(callback);
|
|
2375
2770
|
callbackRef.current = callback;
|
|
2376
2771
|
useEffect(() => {
|
|
2772
|
+
assertMainThreadOwnership(mjModelRef.current);
|
|
2377
2773
|
const wrapped = (input) => callbackRef.current(input);
|
|
2378
2774
|
afterStepCallbacks.current.add(wrapped);
|
|
2379
2775
|
return () => {
|
|
@@ -2393,6 +2789,7 @@ function MujocoSimProvider({
|
|
|
2393
2789
|
timestep,
|
|
2394
2790
|
integrator,
|
|
2395
2791
|
substeps,
|
|
2792
|
+
physicsBudgetMs = 8,
|
|
2396
2793
|
paused,
|
|
2397
2794
|
speed,
|
|
2398
2795
|
interpolate,
|
|
@@ -2400,7 +2797,32 @@ function MujocoSimProvider({
|
|
|
2400
2797
|
hiddenBodies,
|
|
2401
2798
|
children
|
|
2402
2799
|
}) {
|
|
2403
|
-
const { gl, camera, scene } = useThree();
|
|
2800
|
+
const { gl, camera, scene, invalidate } = useThree();
|
|
2801
|
+
const modelIdentity = sceneIdentity(config);
|
|
2802
|
+
const performanceRef = useRef({ steps: 0, physicsMs: 0, realTimeFactor: 0, backlogSteps: 0, budgetExceeded: false });
|
|
2803
|
+
const displayWaiters = useRef(/* @__PURE__ */ new Set());
|
|
2804
|
+
useFrame(() => {
|
|
2805
|
+
if (displayWaiters.current.size) {
|
|
2806
|
+
const waiters = [...displayWaiters.current];
|
|
2807
|
+
displayWaiters.current.clear();
|
|
2808
|
+
queueMicrotask(() => {
|
|
2809
|
+
scene.updateMatrixWorld(true);
|
|
2810
|
+
for (const ready of waiters) ready();
|
|
2811
|
+
});
|
|
2812
|
+
}
|
|
2813
|
+
}, 0);
|
|
2814
|
+
const waitForDisplay = () => new Promise((resolve, reject) => {
|
|
2815
|
+
const ready = () => {
|
|
2816
|
+
clearTimeout(timer);
|
|
2817
|
+
resolve();
|
|
2818
|
+
};
|
|
2819
|
+
const timer = setTimeout(() => {
|
|
2820
|
+
displayWaiters.current.delete(ready);
|
|
2821
|
+
reject(new Error("Canvas did not produce an observation frame within 5 seconds"));
|
|
2822
|
+
}, 5e3);
|
|
2823
|
+
displayWaiters.current.add(ready);
|
|
2824
|
+
invalidate();
|
|
2825
|
+
});
|
|
2404
2826
|
const [status, setStatus] = useState("loading");
|
|
2405
2827
|
const mjModelRef = useRef(null);
|
|
2406
2828
|
const mjDataRef = useRef(null);
|
|
@@ -2436,18 +2858,23 @@ function MujocoSimProvider({
|
|
|
2436
2858
|
const bodyReloadTimerRef = useRef(null);
|
|
2437
2859
|
configRef.current = config;
|
|
2438
2860
|
mujocoRef.current = mujoco;
|
|
2439
|
-
|
|
2440
|
-
|
|
2861
|
+
useEffect(() => {
|
|
2862
|
+
pausedRef.current = paused ?? false;
|
|
2863
|
+
}, [paused]);
|
|
2864
|
+
useEffect(() => {
|
|
2865
|
+
speedRef.current = speed ?? 1;
|
|
2866
|
+
}, [speed]);
|
|
2441
2867
|
substepsRef.current = substeps ?? 1;
|
|
2442
2868
|
interpolateRef.current = interpolate ?? false;
|
|
2443
2869
|
useEffect(() => {
|
|
2444
2870
|
if (!gravity) return;
|
|
2445
2871
|
const model = mjModelRef.current;
|
|
2446
2872
|
if (!model?.opt?.gravity) return;
|
|
2873
|
+
assertMainThreadOwnership(model);
|
|
2447
2874
|
model.opt.gravity[0] = gravity[0];
|
|
2448
2875
|
model.opt.gravity[1] = gravity[1];
|
|
2449
2876
|
model.opt.gravity[2] = gravity[2];
|
|
2450
|
-
}, [gravity]);
|
|
2877
|
+
}, [gravity?.[0], gravity?.[1], gravity?.[2]]);
|
|
2451
2878
|
const integratorRef = useRef(integrator);
|
|
2452
2879
|
integratorRef.current = integrator;
|
|
2453
2880
|
const applyIntegrator = (model) => {
|
|
@@ -2455,12 +2882,16 @@ function MujocoSimProvider({
|
|
|
2455
2882
|
if (value !== void 0) model.opt.integrator = { Euler: 0, RK4: 1, implicit: 2, implicitfast: 3, discrete: 4 }[value];
|
|
2456
2883
|
};
|
|
2457
2884
|
useEffect(() => {
|
|
2458
|
-
if (mjModelRef.current)
|
|
2885
|
+
if (mjModelRef.current) {
|
|
2886
|
+
assertMainThreadOwnership(mjModelRef.current);
|
|
2887
|
+
applyIntegrator(mjModelRef.current);
|
|
2888
|
+
}
|
|
2459
2889
|
}, [integrator]);
|
|
2460
2890
|
useEffect(() => {
|
|
2461
2891
|
if (timestep === void 0) return;
|
|
2462
2892
|
const model = mjModelRef.current;
|
|
2463
2893
|
if (!model?.opt) return;
|
|
2894
|
+
assertMainThreadOwnership(model);
|
|
2464
2895
|
model.opt.timestep = timestep;
|
|
2465
2896
|
}, [timestep]);
|
|
2466
2897
|
const hiddenBodiesKey = (hiddenBodies ?? []).join("\0");
|
|
@@ -2498,6 +2929,7 @@ function MujocoSimProvider({
|
|
|
2498
2929
|
}
|
|
2499
2930
|
useEffect(() => {
|
|
2500
2931
|
let disposed = false;
|
|
2932
|
+
setStatus("loading");
|
|
2501
2933
|
(async () => {
|
|
2502
2934
|
try {
|
|
2503
2935
|
const result = await loadScene(mujoco, buildMergedConfig(config));
|
|
@@ -2532,6 +2964,7 @@ function MujocoSimProvider({
|
|
|
2532
2964
|
})();
|
|
2533
2965
|
return () => {
|
|
2534
2966
|
disposed = true;
|
|
2967
|
+
closeModelRuntime(mjModelRef.current);
|
|
2535
2968
|
mjDataRef.current?.delete();
|
|
2536
2969
|
mjModelRef.current?.delete();
|
|
2537
2970
|
mjModelRef.current = null;
|
|
@@ -2543,7 +2976,7 @@ function MujocoSimProvider({
|
|
|
2543
2976
|
} catch {
|
|
2544
2977
|
}
|
|
2545
2978
|
};
|
|
2546
|
-
}, [mujoco,
|
|
2979
|
+
}, [mujoco, modelIdentity]);
|
|
2547
2980
|
useEffect(() => {
|
|
2548
2981
|
if (status === "ready") {
|
|
2549
2982
|
const api2 = apiRef.current;
|
|
@@ -2561,66 +2994,43 @@ function MujocoSimProvider({
|
|
|
2561
2994
|
const model = mjModelRef.current;
|
|
2562
2995
|
const data = mjDataRef.current;
|
|
2563
2996
|
if (!model || !data) return;
|
|
2564
|
-
|
|
2565
|
-
|
|
2566
|
-
|
|
2567
|
-
data.qfrc_applied[i] = 0;
|
|
2568
|
-
}
|
|
2569
|
-
for (const cb of beforeStepCallbacks.current) {
|
|
2570
|
-
cb({ model, data });
|
|
2997
|
+
if (isWorkerOwned(model)) {
|
|
2998
|
+
interpolationStateRef.current.valid = false;
|
|
2999
|
+
return;
|
|
2571
3000
|
}
|
|
2572
|
-
const
|
|
2573
|
-
if (
|
|
2574
|
-
|
|
2575
|
-
|
|
2576
|
-
|
|
2577
|
-
|
|
2578
|
-
|
|
2579
|
-
|
|
2580
|
-
|
|
2581
|
-
|
|
2582
|
-
|
|
2583
|
-
|
|
2584
|
-
for (let s = 0; s < numSubsteps; s++) {
|
|
2585
|
-
mujocoRef.current.mj_step(model, data);
|
|
2586
|
-
}
|
|
2587
|
-
}
|
|
2588
|
-
}
|
|
2589
|
-
} else if (stepsToRunRef.current > 0) {
|
|
3001
|
+
const queued = stepsToRunRef.current > 0;
|
|
3002
|
+
if (pausedRef.current && !queued) return;
|
|
3003
|
+
const startTime = data.time;
|
|
3004
|
+
const dt = model.opt.timestep;
|
|
3005
|
+
if (!Number.isFinite(dt) || dt <= 0) return;
|
|
3006
|
+
const substeps2 = Math.max(1, Math.floor(substepsRef.current));
|
|
3007
|
+
const frameTime = Math.max(0, Math.min(delta, 1 / 15) * speedRef.current);
|
|
3008
|
+
if (!queued && interpolateRef.current) physicsAccumulatorRef.current += frameTime;
|
|
3009
|
+
const desired = queued ? stepsToRunRef.current : interpolateRef.current ? Math.floor(physicsAccumulatorRef.current / (dt * substeps2)) * substeps2 : Math.ceil(frameTime / (dt * substeps2)) * substeps2;
|
|
3010
|
+
for (let i = 0; i < model.nv; i++) data.qfrc_applied[i] = 0;
|
|
3011
|
+
for (const cb of beforeStepCallbacks.current) cb({ model, data });
|
|
3012
|
+
if (interpolateRef.current) {
|
|
2590
3013
|
ensureInterpolationBuffers(model);
|
|
2591
|
-
copyBodyPose(data, interpolationStateRef.current.previousXpos, interpolationStateRef.current.previousXquat);
|
|
2592
|
-
|
|
2593
|
-
|
|
2594
|
-
|
|
3014
|
+
if (!interpolationStateRef.current.valid) copyBodyPose(data, interpolationStateRef.current.previousXpos, interpolationStateRef.current.previousXquat);
|
|
3015
|
+
}
|
|
3016
|
+
const measured = runStepBudget(desired, () => {
|
|
3017
|
+
if (interpolateRef.current) copyBodyPose(data, interpolationStateRef.current.previousXpos, interpolationStateRef.current.previousXquat);
|
|
3018
|
+
mujocoRef.current.mj_step(model, data);
|
|
3019
|
+
}, physicsBudgetMs);
|
|
3020
|
+
if (queued) stepsToRunRef.current -= measured.steps;
|
|
3021
|
+
performanceRef.current = {
|
|
3022
|
+
...measured,
|
|
3023
|
+
backlogSteps: Math.max(0, desired - measured.steps),
|
|
3024
|
+
realTimeFactor: (data.time - startTime) / Math.max(delta, 1e-6)
|
|
3025
|
+
};
|
|
3026
|
+
if (interpolateRef.current) {
|
|
2595
3027
|
copyBodyPose(data, interpolationStateRef.current.currentXpos, interpolationStateRef.current.currentXquat);
|
|
2596
|
-
|
|
2597
|
-
|
|
2598
|
-
|
|
2599
|
-
|
|
2600
|
-
|
|
2601
|
-
const clampedDelta = Math.min(delta, 1 / 15);
|
|
2602
|
-
physicsAccumulatorRef.current += clampedDelta * speedRef.current;
|
|
2603
|
-
const stepDt = Math.max((model.opt?.timestep ?? 2e-3) * Math.max(1, numSubsteps), 1e-6);
|
|
2604
|
-
let stepped = false;
|
|
2605
|
-
while (physicsAccumulatorRef.current >= stepDt) {
|
|
2606
|
-
copyBodyPose(data, interpolationStateRef.current.previousXpos, interpolationStateRef.current.previousXquat);
|
|
2607
|
-
for (let s = 0; s < numSubsteps; s++) {
|
|
2608
|
-
mujocoRef.current.mj_step(model, data);
|
|
2609
|
-
}
|
|
2610
|
-
copyBodyPose(data, interpolationStateRef.current.currentXpos, interpolationStateRef.current.currentXquat);
|
|
2611
|
-
physicsAccumulatorRef.current -= stepDt;
|
|
2612
|
-
stepped = true;
|
|
2613
|
-
}
|
|
2614
|
-
if (!interpolationStateRef.current.valid) {
|
|
2615
|
-
copyBodyPose(data, interpolationStateRef.current.previousXpos, interpolationStateRef.current.previousXquat);
|
|
2616
|
-
copyBodyPose(data, interpolationStateRef.current.currentXpos, interpolationStateRef.current.currentXquat);
|
|
2617
|
-
}
|
|
2618
|
-
interpolationStateRef.current.alpha = Math.min(Math.max(physicsAccumulatorRef.current / stepDt, 0), 1);
|
|
3028
|
+
if (!queued) {
|
|
3029
|
+
physicsAccumulatorRef.current -= measured.steps * dt;
|
|
3030
|
+
if (measured.budgetExceeded) physicsAccumulatorRef.current %= dt * substeps2;
|
|
3031
|
+
} else physicsAccumulatorRef.current = 0;
|
|
3032
|
+
interpolationStateRef.current.alpha = queued ? 1 : Math.min(1, physicsAccumulatorRef.current / dt);
|
|
2619
3033
|
interpolationStateRef.current.valid = true;
|
|
2620
|
-
if (!stepped) {
|
|
2621
|
-
onStepRef.current?.({ time: data.time, model, data });
|
|
2622
|
-
return;
|
|
2623
|
-
}
|
|
2624
3034
|
}
|
|
2625
3035
|
for (const cb of afterStepCallbacks.current) {
|
|
2626
3036
|
cb({ model, data });
|
|
@@ -2673,6 +3083,7 @@ function MujocoSimProvider({
|
|
|
2673
3083
|
pausedRef.current = p;
|
|
2674
3084
|
}, []);
|
|
2675
3085
|
const step = useCallback((n = 1) => {
|
|
3086
|
+
if (!Number.isSafeInteger(n) || n <= 0) throw new Error("Step count must be a positive safe integer");
|
|
2676
3087
|
stepsToRunRef.current = n;
|
|
2677
3088
|
}, []);
|
|
2678
3089
|
useCallback((steps = 1) => {
|
|
@@ -3091,6 +3502,7 @@ function MujocoSimProvider({
|
|
|
3091
3502
|
const model = mjModelRef.current;
|
|
3092
3503
|
const data = mjDataRef.current;
|
|
3093
3504
|
if (!model || !data) return null;
|
|
3505
|
+
assertMainThreadOwnership(model);
|
|
3094
3506
|
_rayPnt[0] = origin.x;
|
|
3095
3507
|
_rayPnt[1] = origin.y;
|
|
3096
3508
|
_rayPnt[2] = origin.z;
|
|
@@ -3162,6 +3574,7 @@ function MujocoSimProvider({
|
|
|
3162
3574
|
const loadSceneApi = useCallback(async (newConfig) => {
|
|
3163
3575
|
const gen = ++loadGenRef.current;
|
|
3164
3576
|
try {
|
|
3577
|
+
closeModelRuntime(mjModelRef.current);
|
|
3165
3578
|
mjDataRef.current?.delete();
|
|
3166
3579
|
mjModelRef.current?.delete();
|
|
3167
3580
|
mjModelRef.current = null;
|
|
@@ -3317,6 +3730,7 @@ function MujocoSimProvider({
|
|
|
3317
3730
|
if (!model || !data) {
|
|
3318
3731
|
throw new Error("MuJoCo scene is not ready for camera sequence stepping.");
|
|
3319
3732
|
}
|
|
3733
|
+
let sliceStarted = performance.now();
|
|
3320
3734
|
for (let stepIndex = 0; stepIndex < steps; stepIndex += 1) {
|
|
3321
3735
|
for (let i = 0; i < model.nv; i += 1) {
|
|
3322
3736
|
data.qfrc_applied[i] = 0;
|
|
@@ -3336,6 +3750,11 @@ function MujocoSimProvider({
|
|
|
3336
3750
|
cb({ model, data });
|
|
3337
3751
|
}
|
|
3338
3752
|
onStepRef.current?.({ time: data.time, model, data });
|
|
3753
|
+
if (performance.now() - sliceStarted >= 8) {
|
|
3754
|
+
await new Promise((resolve) => setTimeout(resolve, 0));
|
|
3755
|
+
throwIfCameraSequenceAborted(options.signal);
|
|
3756
|
+
sliceStarted = performance.now();
|
|
3757
|
+
}
|
|
3339
3758
|
await options.onAfterStep?.({
|
|
3340
3759
|
frameIndex,
|
|
3341
3760
|
stepIndex,
|
|
@@ -3543,7 +3962,14 @@ function MujocoSimProvider({
|
|
|
3543
3962
|
model.geom_size[id * 3 + 2] = size[2];
|
|
3544
3963
|
}, []);
|
|
3545
3964
|
const api = useMemo(
|
|
3546
|
-
() => ({
|
|
3965
|
+
() => guardRuntimeOwnership({
|
|
3966
|
+
createRuntime: (options) => {
|
|
3967
|
+
if (beforeStepCallbacks.current.size || afterStepCallbacks.current.size) {
|
|
3968
|
+
return Promise.reject(new Error("Unmount physics callbacks/controllers before creating a worker runtime; use runtime.advance({controls}) for policy actions."));
|
|
3969
|
+
}
|
|
3970
|
+
return attachSimulationRuntime(mujocoRef.current, apiRef.current, waitForDisplay, invalidate, options);
|
|
3971
|
+
},
|
|
3972
|
+
getPerformance: () => ({ ...performanceRef.current }),
|
|
3547
3973
|
get status() {
|
|
3548
3974
|
return status;
|
|
3549
3975
|
},
|
|
@@ -3574,7 +4000,10 @@ function MujocoSimProvider({
|
|
|
3574
4000
|
setExternalForce,
|
|
3575
4001
|
applyGeneralizedForce,
|
|
3576
4002
|
getSensorData,
|
|
3577
|
-
getContacts
|
|
4003
|
+
getContacts: () => {
|
|
4004
|
+
const model = mjModelRef.current;
|
|
4005
|
+
return model && isWorkerOwned(model) ? runtimeContacts(model) : getContacts();
|
|
4006
|
+
},
|
|
3578
4007
|
getBodies,
|
|
3579
4008
|
getJoints,
|
|
3580
4009
|
getGeoms,
|
|
@@ -3610,7 +4039,7 @@ function MujocoSimProvider({
|
|
|
3610
4039
|
setGeomSize,
|
|
3611
4040
|
mjModelRef,
|
|
3612
4041
|
mjDataRef
|
|
3613
|
-
}),
|
|
4042
|
+
}, () => mjModelRef.current),
|
|
3614
4043
|
[
|
|
3615
4044
|
status,
|
|
3616
4045
|
reset,
|
|
@@ -3705,6 +4134,45 @@ function MujocoSimProvider({
|
|
|
3705
4134
|
/* @__PURE__ */ jsx(CameraViewportProvider, { children })
|
|
3706
4135
|
] });
|
|
3707
4136
|
}
|
|
4137
|
+
var runtimeMutators = /* @__PURE__ */ new Set([
|
|
4138
|
+
"reset",
|
|
4139
|
+
"setSpeed",
|
|
4140
|
+
"togglePause",
|
|
4141
|
+
"setPaused",
|
|
4142
|
+
"step",
|
|
4143
|
+
"applyKeyframe",
|
|
4144
|
+
"saveState",
|
|
4145
|
+
"restoreState",
|
|
4146
|
+
"setQpos",
|
|
4147
|
+
"setQvel",
|
|
4148
|
+
"setCtrl",
|
|
4149
|
+
"applyForce",
|
|
4150
|
+
"applyTorque",
|
|
4151
|
+
"setExternalForce",
|
|
4152
|
+
"applyGeneralizedForce",
|
|
4153
|
+
"setGravity",
|
|
4154
|
+
"setTimestep",
|
|
4155
|
+
"setBodyMass",
|
|
4156
|
+
"setGeomFriction",
|
|
4157
|
+
"setGeomSize",
|
|
4158
|
+
"loadScene",
|
|
4159
|
+
"loadFromFiles",
|
|
4160
|
+
"addBody",
|
|
4161
|
+
"removeBody",
|
|
4162
|
+
"recompile",
|
|
4163
|
+
"recordCameraSequence"
|
|
4164
|
+
]);
|
|
4165
|
+
function guardRuntimeOwnership(api, model) {
|
|
4166
|
+
for (const key of runtimeMutators) {
|
|
4167
|
+
const record = api;
|
|
4168
|
+
const method = record[key];
|
|
4169
|
+
record[key] = (...args) => {
|
|
4170
|
+
assertMainThreadOwnership(model());
|
|
4171
|
+
return method(...args);
|
|
4172
|
+
};
|
|
4173
|
+
}
|
|
4174
|
+
return api;
|
|
4175
|
+
}
|
|
3708
4176
|
var MujocoCanvas = forwardRef(
|
|
3709
4177
|
function MujocoCanvas2({
|
|
3710
4178
|
config,
|
|
@@ -3717,6 +4185,7 @@ var MujocoCanvas = forwardRef(
|
|
|
3717
4185
|
timestep,
|
|
3718
4186
|
integrator,
|
|
3719
4187
|
substeps,
|
|
4188
|
+
physicsBudgetMs,
|
|
3720
4189
|
paused,
|
|
3721
4190
|
speed,
|
|
3722
4191
|
interpolate,
|
|
@@ -3754,6 +4223,7 @@ var MujocoCanvas = forwardRef(
|
|
|
3754
4223
|
timestep,
|
|
3755
4224
|
integrator,
|
|
3756
4225
|
substeps,
|
|
4226
|
+
physicsBudgetMs,
|
|
3757
4227
|
paused,
|
|
3758
4228
|
speed,
|
|
3759
4229
|
interpolate,
|
|
@@ -5248,10 +5718,10 @@ function useCameraStream(canvasRef, options) {
|
|
|
5248
5718
|
if (!api || !canvasRef.current) return;
|
|
5249
5719
|
const opts = optionsRef.current;
|
|
5250
5720
|
if (opts.paused) return;
|
|
5251
|
-
|
|
5252
|
-
|
|
5253
|
-
|
|
5254
|
-
|
|
5721
|
+
const fps = opts.fps ?? 10;
|
|
5722
|
+
if (!Number.isFinite(fps) || fps <= 0) throw new Error("Camera stream fps must be positive and finite");
|
|
5723
|
+
elapsedRef.current += delta;
|
|
5724
|
+
if (elapsedRef.current < 1 / fps) return;
|
|
5255
5725
|
if (inFlightRef.current) return;
|
|
5256
5726
|
elapsedRef.current = 0;
|
|
5257
5727
|
const signature = streamSignature(opts);
|
|
@@ -5272,7 +5742,10 @@ function useCameraStream(canvasRef, options) {
|
|
|
5272
5742
|
canvas.height = frame.height;
|
|
5273
5743
|
}
|
|
5274
5744
|
ctx.drawImage(frame.canvas, 0, 0);
|
|
5275
|
-
}).catch(() => {
|
|
5745
|
+
}).catch((error) => {
|
|
5746
|
+
const failure = error instanceof Error ? error : new Error(String(error));
|
|
5747
|
+
if (optionsRef.current.onError) optionsRef.current.onError(failure);
|
|
5748
|
+
else console.error("Camera stream capture failed", failure);
|
|
5276
5749
|
}).finally(() => {
|
|
5277
5750
|
inFlightRef.current = false;
|
|
5278
5751
|
});
|
|
@@ -7407,7 +7880,9 @@ function isPolicyActionChunk(value) {
|
|
|
7407
7880
|
function toPolicyActions(output) {
|
|
7408
7881
|
return isPolicyActionChunk(output) ? [...output] : [output];
|
|
7409
7882
|
}
|
|
7410
|
-
function enqueuePolicyActions(queue, actions, observation, strategy) {
|
|
7883
|
+
function enqueuePolicyActions(queue, actions, observation, strategy, limit = 256) {
|
|
7884
|
+
if (!Number.isSafeInteger(limit) || limit < 1) throw new Error("maxQueuedActions must be a positive safe integer");
|
|
7885
|
+
if ((strategy === "replace" ? 0 : queue.length) + actions.length > limit) throw new Error(`Policy action queue exceeds maxQueuedActions (${limit})`);
|
|
7411
7886
|
if (strategy === "replace") {
|
|
7412
7887
|
queue.splice(0, queue.length);
|
|
7413
7888
|
}
|
|
@@ -7428,7 +7903,6 @@ function usePolicy(config) {
|
|
|
7428
7903
|
const clearQueue = useCallback(() => {
|
|
7429
7904
|
epochRef.current += 1;
|
|
7430
7905
|
actionQueueRef.current.splice(0, actionQueueRef.current.length);
|
|
7431
|
-
inFlightRef.current = false;
|
|
7432
7906
|
lastErrorRef.current = null;
|
|
7433
7907
|
}, []);
|
|
7434
7908
|
const reset = useCallback(() => {
|
|
@@ -7437,6 +7911,13 @@ function usePolicy(config) {
|
|
|
7437
7911
|
lastObservationRef.current = null;
|
|
7438
7912
|
lastActionRef.current = null;
|
|
7439
7913
|
}, [clearQueue]);
|
|
7914
|
+
useEffect(() => {
|
|
7915
|
+
isRunningRef.current = configRef.current.enabled ?? true;
|
|
7916
|
+
return () => {
|
|
7917
|
+
epochRef.current++;
|
|
7918
|
+
isRunningRef.current = false;
|
|
7919
|
+
};
|
|
7920
|
+
}, []);
|
|
7440
7921
|
useBeforePhysicsStep(({ model, data }) => {
|
|
7441
7922
|
if (!isRunningRef.current) return;
|
|
7442
7923
|
const cfg = configRef.current;
|
|
@@ -7469,7 +7950,8 @@ function usePolicy(config) {
|
|
|
7469
7950
|
actionQueueRef.current,
|
|
7470
7951
|
toPolicyActions(output),
|
|
7471
7952
|
observation,
|
|
7472
|
-
cfg.queueStrategy ?? "append"
|
|
7953
|
+
cfg.queueStrategy ?? "append",
|
|
7954
|
+
cfg.maxQueuedActions
|
|
7473
7955
|
);
|
|
7474
7956
|
lastErrorRef.current = null;
|
|
7475
7957
|
}).catch((error) => {
|
|
@@ -7477,7 +7959,6 @@ function usePolicy(config) {
|
|
|
7477
7959
|
lastErrorRef.current = error;
|
|
7478
7960
|
cfg.onError?.(error);
|
|
7479
7961
|
}).finally(() => {
|
|
7480
|
-
if (epoch !== epochRef.current) return;
|
|
7481
7962
|
inFlightRef.current = false;
|
|
7482
7963
|
});
|
|
7483
7964
|
} else {
|
|
@@ -7487,7 +7968,8 @@ function usePolicy(config) {
|
|
|
7487
7968
|
actionQueueRef.current,
|
|
7488
7969
|
actions,
|
|
7489
7970
|
observation,
|
|
7490
|
-
cfg.queueStrategy ?? "append"
|
|
7971
|
+
cfg.queueStrategy ?? "append",
|
|
7972
|
+
cfg.maxQueuedActions
|
|
7491
7973
|
);
|
|
7492
7974
|
} else {
|
|
7493
7975
|
const [action, ...queuedActions2] = actions;
|
|
@@ -7496,7 +7978,8 @@ function usePolicy(config) {
|
|
|
7496
7978
|
actionQueueRef.current,
|
|
7497
7979
|
queuedActions2,
|
|
7498
7980
|
observation,
|
|
7499
|
-
cfg.queueStrategy ?? "append"
|
|
7981
|
+
cfg.queueStrategy ?? "append",
|
|
7982
|
+
cfg.maxQueuedActions
|
|
7500
7983
|
);
|
|
7501
7984
|
cfg.onAction({ action, observation, model, data });
|
|
7502
7985
|
lastActionRef.current = action;
|
|
@@ -8197,125 +8680,6 @@ function useCameraFrameCapture(defaultOptions = {}) {
|
|
|
8197
8680
|
reset
|
|
8198
8681
|
};
|
|
8199
8682
|
}
|
|
8200
|
-
|
|
8201
|
-
// src/policyCameraFrames.ts
|
|
8202
|
-
function addPolicyImageAliases(images, stream, frame, includeObservationImageAliases) {
|
|
8203
|
-
const keys = /* @__PURE__ */ new Set();
|
|
8204
|
-
keys.add(stream.key);
|
|
8205
|
-
for (const alias of stream.aliases ?? []) keys.add(alias);
|
|
8206
|
-
if (includeObservationImageAliases) {
|
|
8207
|
-
keys.add(`observation.images.${stream.key}`);
|
|
8208
|
-
for (const alias of stream.aliases ?? []) {
|
|
8209
|
-
keys.add(`observation.images.${alias}`);
|
|
8210
|
-
}
|
|
8211
|
-
}
|
|
8212
|
-
for (const key of keys) {
|
|
8213
|
-
images[key] = frame.dataUrl;
|
|
8214
|
-
}
|
|
8215
|
-
}
|
|
8216
|
-
function describeFrameSource(key, frame) {
|
|
8217
|
-
return `${key}:${frame.source.kind}`;
|
|
8218
|
-
}
|
|
8219
|
-
function hasExplicitPolicyCameraSource(options) {
|
|
8220
|
-
return Boolean(
|
|
8221
|
-
options?.camera || options?.position || options?.quaternion || options?.source
|
|
8222
|
-
);
|
|
8223
|
-
}
|
|
8224
|
-
function createPolicyCameraFrameCapturePlan(options) {
|
|
8225
|
-
const {
|
|
8226
|
-
cameraKeys,
|
|
8227
|
-
defaults,
|
|
8228
|
-
streamOptions,
|
|
8229
|
-
includeObservationImageAliases,
|
|
8230
|
-
requireAll,
|
|
8231
|
-
...sourceOptions
|
|
8232
|
-
} = options;
|
|
8233
|
-
const mountedPlan = createMountedCameraFrameSequencePlan(cameraKeys, {
|
|
8234
|
-
...sourceOptions,
|
|
8235
|
-
defaults,
|
|
8236
|
-
cameraOptions: streamOptions
|
|
8237
|
-
});
|
|
8238
|
-
const streams = [];
|
|
8239
|
-
const missingKeys = new Set(mountedPlan.missingKeys);
|
|
8240
|
-
for (const key of cameraKeys) {
|
|
8241
|
-
const perStreamOptions = streamOptions?.[key];
|
|
8242
|
-
if (hasExplicitPolicyCameraSource(perStreamOptions)) {
|
|
8243
|
-
missingKeys.delete(key);
|
|
8244
|
-
streams.push({
|
|
8245
|
-
...defaults,
|
|
8246
|
-
...perStreamOptions,
|
|
8247
|
-
key,
|
|
8248
|
-
aliases: perStreamOptions?.aliases
|
|
8249
|
-
});
|
|
8250
|
-
continue;
|
|
8251
|
-
}
|
|
8252
|
-
const mountedCamera = mountedPlan.cameras.find((camera) => camera.key === key);
|
|
8253
|
-
if (!mountedCamera) continue;
|
|
8254
|
-
const { key: _mountedKey, ...captureOptions } = mountedCamera;
|
|
8255
|
-
streams.push({
|
|
8256
|
-
...captureOptions,
|
|
8257
|
-
key,
|
|
8258
|
-
aliases: perStreamOptions?.aliases
|
|
8259
|
-
});
|
|
8260
|
-
}
|
|
8261
|
-
const result = {
|
|
8262
|
-
cameraKeys: [...cameraKeys],
|
|
8263
|
-
streams,
|
|
8264
|
-
includeObservationImageAliases,
|
|
8265
|
-
mountedPlan,
|
|
8266
|
-
missingKeys: [...missingKeys]
|
|
8267
|
-
};
|
|
8268
|
-
if (requireAll && result.missingKeys.length > 0) {
|
|
8269
|
-
throw new Error(
|
|
8270
|
-
`Unable to resolve policy camera stream${result.missingKeys.length === 1 ? "" : "s"} for ${result.missingKeys.join(", ")}.`
|
|
8271
|
-
);
|
|
8272
|
-
}
|
|
8273
|
-
return result;
|
|
8274
|
-
}
|
|
8275
|
-
function createPolicyCameraFrameCapturePlanFromApi(api, options) {
|
|
8276
|
-
return createPolicyCameraFrameCapturePlan({
|
|
8277
|
-
...options,
|
|
8278
|
-
cameras: api.getCameras(),
|
|
8279
|
-
sites: api.getSites(),
|
|
8280
|
-
bodies: api.getBodies()
|
|
8281
|
-
});
|
|
8282
|
-
}
|
|
8283
|
-
async function capturePolicyCameraFrames(target, options) {
|
|
8284
|
-
const includeObservationImageAliases = options.includeObservationImageAliases ?? true;
|
|
8285
|
-
const entries = await Promise.all(
|
|
8286
|
-
options.streams.map(async ({ key, aliases, ...captureOptions }) => {
|
|
8287
|
-
const frame = await target.captureCameraFrame(captureOptions);
|
|
8288
|
-
return [key, { frame, aliases }];
|
|
8289
|
-
})
|
|
8290
|
-
);
|
|
8291
|
-
const frames = {};
|
|
8292
|
-
const images = {};
|
|
8293
|
-
const sourceParts = [];
|
|
8294
|
-
for (const [key, { frame, aliases }] of entries) {
|
|
8295
|
-
const stream = { key, aliases };
|
|
8296
|
-
frames[key] = frame;
|
|
8297
|
-
addPolicyImageAliases(
|
|
8298
|
-
images,
|
|
8299
|
-
stream,
|
|
8300
|
-
frame,
|
|
8301
|
-
includeObservationImageAliases
|
|
8302
|
-
);
|
|
8303
|
-
sourceParts.push(describeFrameSource(key, frame));
|
|
8304
|
-
}
|
|
8305
|
-
return {
|
|
8306
|
-
frames,
|
|
8307
|
-
images,
|
|
8308
|
-
sourceSummary: sourceParts.length > 0 ? sourceParts.join(" + ") : "not used by policy",
|
|
8309
|
-
capturedAt: Date.now()
|
|
8310
|
-
};
|
|
8311
|
-
}
|
|
8312
|
-
async function capturePolicyCameraFramesFromMountedStreams(target, options) {
|
|
8313
|
-
const plan = createPolicyCameraFrameCapturePlanFromApi(target, options);
|
|
8314
|
-
const result = await capturePolicyCameraFrames(target, plan);
|
|
8315
|
-
return { ...result, plan };
|
|
8316
|
-
}
|
|
8317
|
-
|
|
8318
|
-
// src/hooks/usePolicyCameraFrames.ts
|
|
8319
8683
|
function mergePolicyCameraFrameCaptureOptions(defaultOptions, options) {
|
|
8320
8684
|
return {
|
|
8321
8685
|
...defaultOptions,
|
|
@@ -8838,6 +9202,18 @@ function useCameraAnimation() {
|
|
|
8838
9202
|
* @license
|
|
8839
9203
|
* SPDX-License-Identifier: Apache-2.0
|
|
8840
9204
|
*/
|
|
9205
|
+
/**
|
|
9206
|
+
* @license
|
|
9207
|
+
* SPDX-License-Identifier: Apache-2.0
|
|
9208
|
+
*
|
|
9209
|
+
* Helpers for resolving dataset camera streams to mounted MuJoCo resources.
|
|
9210
|
+
*/
|
|
9211
|
+
/**
|
|
9212
|
+
* @license
|
|
9213
|
+
* SPDX-License-Identifier: Apache-2.0
|
|
9214
|
+
*
|
|
9215
|
+
* Helpers for turning Three/MuJoCo camera captures into policy image payloads.
|
|
9216
|
+
*/
|
|
8841
9217
|
/**
|
|
8842
9218
|
* @license
|
|
8843
9219
|
* SPDX-License-Identifier: Apache-2.0
|
|
@@ -8861,12 +9237,6 @@ function useCameraAnimation() {
|
|
|
8861
9237
|
*
|
|
8862
9238
|
* useFrameCapture — still-frame capture for canvas-backed MuJoCo/R3F scenes.
|
|
8863
9239
|
*/
|
|
8864
|
-
/**
|
|
8865
|
-
* @license
|
|
8866
|
-
* SPDX-License-Identifier: Apache-2.0
|
|
8867
|
-
*
|
|
8868
|
-
* Helpers for resolving dataset camera streams to mounted MuJoCo resources.
|
|
8869
|
-
*/
|
|
8870
9240
|
/**
|
|
8871
9241
|
* @license
|
|
8872
9242
|
* SPDX-License-Identifier: Apache-2.0
|
|
@@ -8907,16 +9277,9 @@ function useCameraAnimation() {
|
|
|
8907
9277
|
* @license
|
|
8908
9278
|
* SPDX-License-Identifier: Apache-2.0
|
|
8909
9279
|
*
|
|
8910
|
-
*
|
|
8911
|
-
*
|
|
8912
|
-
*
|
|
8913
|
-
* take over the render loop. Prefer this over `useCameraViewport` for camera
|
|
8914
|
-
* tiles embedded in HTML UI; use `useCameraViewport` for transparent overlays
|
|
8915
|
-
* on a full-bleed canvas.
|
|
8916
|
-
*
|
|
8917
|
-
* Uses the async capture path so Gaussian-splat environments render through
|
|
8918
|
-
* their dedicated capture renderer — streaming a splat scene at full rate does
|
|
8919
|
-
* not disturb the main view's splat sort.
|
|
9280
|
+
* Readback stream for DOM compositing/encoding. For interactive camera previews,
|
|
9281
|
+
* use CameraView/useCameraViewport: they render on the GPU with no readback.
|
|
9282
|
+
* This path defaults to 10 FPS and permits only one capture in flight.
|
|
8920
9283
|
*/
|
|
8921
9284
|
/**
|
|
8922
9285
|
* @license
|
|
@@ -9071,12 +9434,6 @@ function useCameraAnimation() {
|
|
|
9071
9434
|
*
|
|
9072
9435
|
* React state wrapper around MuJoCo/R3F offscreen camera-frame capture.
|
|
9073
9436
|
*/
|
|
9074
|
-
/**
|
|
9075
|
-
* @license
|
|
9076
|
-
* SPDX-License-Identifier: Apache-2.0
|
|
9077
|
-
*
|
|
9078
|
-
* Helpers for turning Three/MuJoCo camera captures into policy image payloads.
|
|
9079
|
-
*/
|
|
9080
9437
|
/**
|
|
9081
9438
|
* @license
|
|
9082
9439
|
* SPDX-License-Identifier: Apache-2.0
|