npm - mujoco-react - Versions diffs - 10.3.0 → 10.5.0 - Mend

mujoco-react 10.3.0 → 10.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (28) hide show

package/README.md +75 -137
package/dist/{chunk-6AZEFI6A.js → chunk-KHZ5U36J.js} +157 -16
package/dist/chunk-KHZ5U36J.js.map +1 -0
package/dist/index.d.ts +180 -49
package/dist/index.js +627 -19
package/dist/index.js.map +1 -1
package/dist/onnx.d.ts +65 -0
package/dist/onnx.js +58 -0
package/dist/onnx.js.map +1 -0
package/dist/spark.d.ts +1 -1
package/dist/spark.js +1 -1
package/dist/{types-BOhNDICK.d.ts → types-CViUme8D.d.ts} +157 -1
package/package.json +14 -3
package/src/components/CameraView.tsx +245 -0
package/src/components/Debug.tsx +174 -3
package/src/core/GenericIK.ts +16 -4
package/src/core/MujocoSimProvider.tsx +37 -1
package/src/core/SceneLoader.ts +3 -2
package/src/hooks/useCameraStream.ts +115 -0
package/src/hooks/useControlGroup.ts +0 -0
package/src/hooks/useIkController.ts +3 -0
package/src/hooks/usePolicyCameraTensors.ts +215 -0
package/src/index.ts +45 -0
package/src/onnx.ts +126 -0
package/src/policyImageTensors.ts +150 -0
package/src/rendering/cameraFrameCapture.ts +112 -15
package/src/types.ts +45 -0
package/dist/chunk-6AZEFI6A.js.map +0 -1

package/dist/index.js CHANGED Viewed

@@ -1,8 +1,8 @@
-import { withContacts, getContact, captureCameraFrame, captureCameraFrameBlob, createCameraFrameCaptureSession, CAMERA_FRAME_CAPTURE_PRE_RENDER_USER_DATA_KEY, CAPTURE_EXCLUDE_KEY } from './chunk-6AZEFI6A.js';
-export { CAMERA_FRAME_CAPTURE_PRE_RENDER_USER_DATA_KEY, CAMERA_FRAME_CAPTURE_RENDER_USER_DATA_KEY, CAPTURE_EXCLUDE_KEY, ModelActuators, ModelBodies, ModelCameras, ModelGeoms, ModelJoints, ModelKeyframes, ModelResources, ModelSensors, ModelSites, ScenarioLighting, SplatEnvironment, SplatEnvironmentReadinessStatus, VisualScenarioEffects, captureCameraFrame, captureCameraFrameBlob, createCameraFrameCaptureSession, createPairedSplatEnvironment, createSparkSplatViewerUrl, createSplatEnvironmentUserData, createSplatSceneConfig, createVisualScenarioExecutionContext, getContact, getScenarioBackground, getScenarioCameraPosition, getSplatEnvironmentReadiness, registerModelResources, renderCameraFrameToCanvas, useSplatEnvironment, useSplatSceneConfig, useVisualScenarioEffects, useVisualScenarioExecutionContext, withSplatEnvironment } from './chunk-6AZEFI6A.js';
+import { withContacts, getContact, captureCameraFrame, captureCameraFrameBlob, createCameraFrameCaptureSession, CAMERA_FRAME_CAPTURE_PRE_RENDER_USER_DATA_KEY, CAPTURE_EXCLUDE_KEY, createCaptureCamera, prepareCaptureCamera } from './chunk-KHZ5U36J.js';
+export { CAMERA_FRAME_CAPTURE_PRE_RENDER_USER_DATA_KEY, CAMERA_FRAME_CAPTURE_RENDER_USER_DATA_KEY, CAPTURE_EXCLUDE_KEY, ModelActuators, ModelBodies, ModelCameras, ModelGeoms, ModelJoints, ModelKeyframes, ModelResources, ModelSensors, ModelSites, ScenarioLighting, SplatEnvironment, SplatEnvironmentReadinessStatus, VisualScenarioEffects, captureCameraFrame, captureCameraFrameBlob, captureCameraFrameTensor, createCameraFrameCaptureSession, createPairedSplatEnvironment, createSparkSplatViewerUrl, createSplatEnvironmentUserData, createSplatSceneConfig, createVisualScenarioExecutionContext, dataUrlToPolicyImageTensor, getContact, getScenarioBackground, getScenarioCameraPosition, getSplatEnvironmentReadiness, imageDataToPolicyImageTensor, pixelsToPolicyImageTensor, registerModelResources, renderCameraFrameToCanvas, useSplatEnvironment, useSplatSceneConfig, useVisualScenarioEffects, useVisualScenarioExecutionContext, withSplatEnvironment } from './chunk-KHZ5U36J.js';
 import loadMujoco from '@mujoco/mujoco';
 import defaultMujocoWasmUrl from '@mujoco/mujoco/mujoco.wasm?url';
-import { createContext, forwardRef, useRef, useEffect, useContext, useState, useCallback, useMemo, useLayoutEffect } from 'react';
+import { createContext, forwardRef, useRef, useEffect, useContext, useState, useCallback, useMemo, useLayoutEffect, useId } from 'react';
 import { jsx, jsxs } from 'react/jsx-runtime';
 import { Canvas, useThree, useFrame } from '@react-three/fiber';
 import * as THREE11 from 'three';
@@ -492,8 +492,10 @@ function sceneObjectToXml(obj) {
   const solref = obj.solref ? ` solref="${obj.solref}"` : "";
   const solimp = obj.solimp ? ` solimp="${obj.solimp}"` : "";
   const condim = obj.condim ? ` condim="${obj.condim}"` : "";
+  const contype = obj.contype ?? 1;
+  const conaffinity = obj.conaffinity ?? 1;
   const group = obj.group !== void 0 ? ` group="${obj.group}"` : "";
-  return `<body name="${obj.name}" pos="${pos}">${joint}<geom name="${geomName}" type="${obj.type}" size="${size}" rgba="${rgba}" contype="1" conaffinity="1"${mass}${friction}${solref}${solimp}${condim}${group}/></body>`;
+  return `<body name="${obj.name}" pos="${pos}">${joint}<geom name="${geomName}" type="${obj.type}" size="${size}" rgba="${rgba}" contype="${contype}" conaffinity="${conaffinity}"${mass}${friction}${solref}${solimp}${condim}${group}/></body>`;
 }
 function ensureDir(mujoco, fname) {
   const dirParts = fname.split("/");
@@ -1170,6 +1172,118 @@ function SceneRenderer({ renderOptions, ...props }) {
 }
 var _previousQuat = new THREE11.Quaternion();
 var _currentQuat = new THREE11.Quaternion();
+var CameraViewportRegistryContext = createContext(null);
+var nextViewportId = 0;
+var VIEWPORT_RENDER_PRIORITY = 1;
+function CameraViewportRenderer({
+  viewportsRef
+}) {
+  const gl = useThree((state) => state.gl);
+  const scene = useThree((state) => state.scene);
+  const mainCamera = useThree((state) => state.camera);
+  const mujoco = useMujoco();
+  useFrame(() => {
+    const drawWidth = gl.domElement.width;
+    const drawHeight = gl.domElement.height;
+    gl.setScissorTest(false);
+    gl.setViewport(0, 0, drawWidth, drawHeight);
+    gl.render(scene, mainCamera);
+    const api = mujoco.api;
+    if (!api || viewportsRef.current.size === 0) return;
+    const dpr = gl.getPixelRatio();
+    const canvasRect = gl.domElement.getBoundingClientRect();
+    for (const descriptor of viewportsRef.current.values()) {
+      const element = descriptor.getElement();
+      if (!element) continue;
+      const rect = element.getBoundingClientRect();
+      const isOffscreen = rect.bottom < canvasRect.top || rect.top > canvasRect.bottom || rect.right < canvasRect.left || rect.left > canvasRect.right;
+      if (isOffscreen) continue;
+      const width = Math.floor(rect.width * dpr);
+      const height = Math.floor(rect.height * dpr);
+      if (width <= 0 || height <= 0) continue;
+      const left = Math.floor((rect.left - canvasRect.left) * dpr);
+      const bottom = Math.floor((canvasRect.bottom - rect.bottom) * dpr);
+      let resolved;
+      try {
+        resolved = api.resolveCameraCaptureOptions(descriptor.getOptions());
+      } catch {
+        continue;
+      }
+      if (!descriptor.camera) {
+        descriptor.camera = createCaptureCamera(resolved, mainCamera, width, height);
+      } else {
+        prepareCaptureCamera(descriptor.camera, resolved, mainCamera, width, height);
+      }
+      gl.setViewport(left, bottom, width, height);
+      gl.setScissor(left, bottom, width, height);
+      gl.setScissorTest(true);
+      gl.render(scene, descriptor.camera);
+    }
+    gl.setScissorTest(false);
+    gl.setViewport(0, 0, drawWidth, drawHeight);
+  }, VIEWPORT_RENDER_PRIORITY);
+  return null;
+}
+function CameraViewportProvider({ children }) {
+  const viewportsRef = useRef(/* @__PURE__ */ new Map());
+  const [count, setCount] = useState(0);
+  const register = useCallback((descriptor) => {
+    const id = nextViewportId++;
+    viewportsRef.current.set(id, descriptor);
+    setCount((value2) => value2 + 1);
+    return () => {
+      viewportsRef.current.delete(id);
+      setCount((value2) => value2 - 1);
+    };
+  }, []);
+  const value = useMemo(() => ({ register }), [register]);
+  return /* @__PURE__ */ jsxs(CameraViewportRegistryContext.Provider, { value, children: [
+    children,
+    count > 0 && /* @__PURE__ */ jsx(CameraViewportRenderer, { viewportsRef })
+  ] });
+}
+function useCameraViewport(elementRef, options) {
+  const registry = useContext(CameraViewportRegistryContext);
+  if (!registry) {
+    throw new Error("useCameraViewport must be used inside <MujocoCanvas>.");
+  }
+  const optionsRef = useRef(options);
+  optionsRef.current = options;
+  useEffect(() => {
+    const descriptor = {
+      getElement: () => elementRef.current,
+      getOptions: () => optionsRef.current,
+      camera: null
+    };
+    return registry.register(descriptor);
+  }, [registry, elementRef]);
+}
+function CameraView({ className, style, ...options }) {
+  const gl = useThree((state) => state.gl);
+  const elementRef = useRef(null);
+  if (!elementRef.current && typeof document !== "undefined") {
+    elementRef.current = document.createElement("div");
+  }
+  useEffect(() => {
+    const element = elementRef.current;
+    const parent = gl.domElement.parentElement;
+    if (!element || !parent) return;
+    element.style.position = "absolute";
+    element.style.overflow = "hidden";
+    parent.appendChild(element);
+    return () => {
+      parent.removeChild(element);
+    };
+  }, [gl]);
+  useEffect(() => {
+    const element = elementRef.current;
+    if (!element) return;
+    element.className = className ?? "";
+    if (style) Object.assign(element.style, style);
+  }, [className, style]);
+  useCameraViewport(elementRef, options);
+  return null;
+}
 function isTargetRef(target) {
   return Boolean(target && typeof target === "object" && "current" in target);
 }
@@ -3021,6 +3135,32 @@ function MujocoSimProvider({
     },
     [camera, gl, resolveCameraCaptureOptions, scene]
   );
+  const createCameraFrameCaptureSessionApi = useCallback(
+    (options = {}) => createCameraFrameCaptureSession(
+      gl,
+      scene,
+      camera,
+      resolveCameraCaptureOptions(options)
+    ),
+    [camera, gl, resolveCameraCaptureOptions, scene]
+  );
+  const captureCameraFrameTensorApi = useCallback(
+    (options = {}) => {
+      const resolved = {
+        ...resolveCameraCaptureOptions(options),
+        channels: options.channels,
+        layout: options.layout,
+        range: options.range
+      };
+      const session = createCameraFrameCaptureSession(gl, scene, camera, resolved);
+      try {
+        return session.captureTensor(resolved);
+      } finally {
+        session.dispose();
+      }
+    },
+    [camera, gl, resolveCameraCaptureOptions, scene]
+  );
   const recordCameraSequenceApi = useCallback(
     async (options) => {
       const frameCount = Math.max(0, Math.floor(options.frames));
@@ -3320,6 +3460,9 @@ function MujocoSimProvider({
       captureFrameBlob: captureFrameBlobApi,
       captureCameraFrame: captureCameraFrameApi,
       captureCameraFrameBlob: captureCameraFrameBlobApi,
+      captureCameraFrameTensor: captureCameraFrameTensorApi,
+      createCameraFrameCaptureSession: createCameraFrameCaptureSessionApi,
+      resolveCameraCaptureOptions,
       recordCameraSequence: recordCameraSequenceApi,
       project2DTo3D,
       projectImagePointTo3D: projectImagePointTo3D2,
@@ -3380,6 +3523,9 @@ function MujocoSimProvider({
       captureFrameBlobApi,
       captureCameraFrameApi,
       captureCameraFrameBlobApi,
+      captureCameraFrameTensorApi,
+      createCameraFrameCaptureSessionApi,
+      resolveCameraCaptureOptions,
       recordCameraSequenceApi,
       project2DTo3D,
       projectImagePointTo3D2,
@@ -3416,7 +3562,7 @@ function MujocoSimProvider({
   );
   return /* @__PURE__ */ jsxs(MujocoSimContext.Provider, { value: contextValue, children: [
     /* @__PURE__ */ jsx(SceneRenderer, { renderOptions }),
-    children
+    /* @__PURE__ */ jsx(CameraViewportProvider, { children })
   ] });
 }
 var MujocoCanvas = forwardRef(
@@ -3641,9 +3787,16 @@ function resolveOptions(opts) {
     tolerance: opts?.tolerance ?? DEFAULTS.tolerance,
     epsilon: opts?.epsilon ?? DEFAULTS.epsilon,
     posWeight: opts?.posWeight ?? DEFAULTS.posWeight,
-    rotWeight: opts?.rotWeight ?? DEFAULTS.rotWeight
+    rotWeight: opts?.rotWeight ?? DEFAULTS.rotWeight,
+    jointLimits: opts?.jointLimits
   };
 }
+function clampJoint(value, limit) {
+  if (!limit) return value;
+  const [min, max] = limit;
+  if (!Number.isFinite(min) || !Number.isFinite(max) || min >= max) return value;
+  return Math.max(min, Math.min(max, value));
+}
 var GenericIK = class {
   mujoco;
   constructor(mujoco) {
@@ -3668,7 +3821,7 @@ var GenericIK = class {
     savedQpos.set(data.qpos);
     const R_target = quatToMat3(targetQuat);
     const q = new Float64Array(n);
-    for (let i = 0; i < n; i++) q[i] = currentQ[i];
+    for (let i = 0; i < n; i++) q[i] = clampJoint(currentQ[i], o.jointLimits?.[i]);
     const J = new Float64Array(6 * n);
     const JJt = new Float64Array(36);
     const rhs = new Float64Array(6);
@@ -3745,7 +3898,7 @@ var GenericIK = class {
         }
         dq[j] = sum;
       }
-      for (let i = 0; i < n; i++) q[i] += dq[i];
+      for (let i = 0; i < n; i++) q[i] = clampJoint(q[i] + dq[i], o.jointLimits?.[i]);
     }
     data.qpos.set(savedQpos);
     this.mujoco.mj_forward(model, data);
@@ -3940,19 +4093,20 @@ var useIkController = createControllerHook(
         if (config.ikSolveFn) return config.ikSolveFn({ position, quaternion, currentQ, context });
         const model = mjModelRef.current;
         const data = mjDataRef.current;
-        const controlGroup = controlGroupRef.current;
-        if (!model || !data || !controlGroup || siteIdRef.current === -1) return null;
+        const controlGroup2 = controlGroupRef.current;
+        if (!model || !data || !controlGroup2 || siteIdRef.current === -1) return null;
         return genericIkRef.current.solve(
           model,
           data,
           siteIdRef.current,
-          controlGroup.qposAdr,
+          controlGroup2.qposAdr,
           position,
           quaternion,
           currentQ,
           {
             damping: config.damping,
             epsilon: config.epsilon,
+            jointLimits: config.jointLimits ?? controlGroup2.joints.map((joint) => joint.limited ? joint.range : joint.ctrlRange),
             maxIterations: config.maxIterations,
             posWeight: config.posWeight,
             rotWeight: config.rotWeight,
@@ -3992,9 +4146,9 @@ var useIkController = createControllerHook(
       const target = ikTargetRef.current;
       if (!target) return;
       ikCalculatingRef.current = true;
-      const controlGroup = controlGroupRef.current;
-      if (!controlGroup) return;
-      const currentQ = Array.from(controlGroup.readQpos(data));
+      const controlGroup2 = controlGroupRef.current;
+      if (!controlGroup2) return;
+      const currentQ = Array.from(controlGroup2.readQpos(data));
       const solution = config.ikSolveFn ? config.ikSolveFn({
         position: target.position,
         quaternion: target.quaternion,
@@ -4003,7 +4157,7 @@ var useIkController = createControllerHook(
           model,
           data,
           siteId: siteIdRef.current,
-          controlGroup
+          controlGroup: controlGroup2
         }
       }) : ikSolveFnRef.current({
         position: target.position,
@@ -4011,7 +4165,7 @@ var useIkController = createControllerHook(
         currentQ
       });
       if (solution) {
-        controlGroup.writeCtrl(data, solution);
+        controlGroup2.writeCtrl(data, solution);
       }
     });
     useEffect(() => {
@@ -4848,6 +5002,69 @@ function parseNumberList(value) {
 function addProxyVectors(a, b) {
   return [a[0] + b[0], a[1] + b[1], a[2] + b[2]];
 }
+function streamSignature(options) {
+  return JSON.stringify({
+    cameraName: options.cameraName,
+    siteName: options.siteName,
+    bodyName: options.bodyName,
+    width: options.width,
+    height: options.height,
+    renderIsolation: options.renderIsolation ?? false
+  });
+}
+function useCameraStream(canvasRef, options) {
+  const mujoco = useMujoco();
+  const sessionRef = useRef(null);
+  const signatureRef = useRef("");
+  const optionsRef = useRef(options);
+  optionsRef.current = options;
+  const elapsedRef = useRef(0);
+  const inFlightRef = useRef(false);
+  const mountedRef = useRef(true);
+  useEffect(() => {
+    mountedRef.current = true;
+    return () => {
+      mountedRef.current = false;
+      sessionRef.current?.dispose();
+      sessionRef.current = null;
+      signatureRef.current = "";
+    };
+  }, []);
+  useFrame((_state, delta) => {
+    const api = mujoco.api;
+    if (!api || !canvasRef.current) return;
+    const opts = optionsRef.current;
+    if (opts.paused) return;
+    if (opts.fps && opts.fps > 0) {
+      elapsedRef.current += delta;
+      if (elapsedRef.current < 1 / opts.fps) return;
+    }
+    if (inFlightRef.current) return;
+    elapsedRef.current = 0;
+    const signature = streamSignature(opts);
+    if (!sessionRef.current || signatureRef.current !== signature) {
+      sessionRef.current?.dispose();
+      sessionRef.current = api.createCameraFrameCaptureSession(opts);
+      signatureRef.current = signature;
+    }
+    const session = sessionRef.current;
+    inFlightRef.current = true;
+    session.captureAsync(api.resolveCameraCaptureOptions(opts)).then((frame) => {
+      const canvas = canvasRef.current;
+      if (!mountedRef.current || !canvas) return;
+      const ctx = canvas.getContext("2d");
+      if (!ctx) return;
+      if (canvas.width !== frame.width || canvas.height !== frame.height) {
+        canvas.width = frame.width;
+        canvas.height = frame.height;
+      }
+      ctx.drawImage(frame.canvas, 0, 0);
+    }).catch(() => {
+    }).finally(() => {
+      inFlightRef.current = false;
+    });
+  });
+}
 var JOINT_COLORS = {
   0: 16711680,
   // free - red
@@ -4867,11 +5084,161 @@ var _contactNormal = new THREE11.Vector3();
 var MAX_CONTACT_ARROWS = 50;
 var CAMERA_DEBUG_LENGTH = 0.12;
 var CAMERA_DEBUG_FRUSTUM_DEPTH = 0.08;
+function toVector32(value, fallback) {
+  if (!value) return fallback.clone();
+  return value instanceof THREE11.Vector3 ? value.clone() : new THREE11.Vector3(value[0], value[1], value[2]);
+}
+function createCameraLabel(text, color) {
+  const canvas = document.createElement("canvas");
+  canvas.width = 256;
+  canvas.height = 64;
+  const ctx = canvas.getContext("2d");
+  ctx.fillStyle = new THREE11.Color(color).getStyle();
+  ctx.font = "bold 32px monospace";
+  ctx.textAlign = "center";
+  ctx.fillText(text, 128, 42);
+  const texture = new THREE11.CanvasTexture(canvas);
+  const sprite = new THREE11.Sprite(
+    new THREE11.SpriteMaterial({
+      map: texture,
+      depthTest: false,
+      transparent: true
+    })
+  );
+  sprite.position.set(0, 0.014, 0.01);
+  sprite.scale.set(0.05, 0.012, 1);
+  sprite.renderOrder = 999;
+  return sprite;
+}
+function createVirtualCameraDebugObject(camera, index) {
+  const color = camera.color ?? "#ff3d71";
+  const aimColor = camera.aimColor ?? "#38bdf8";
+  const markerScale = camera.markerScale ?? 1;
+  const cameraPosition = toVector32(camera.position, new THREE11.Vector3());
+  const configuredUp = toVector32(camera.up, new THREE11.Vector3(0, 0, 1)).normalize();
+  const cameraQuaternion = new THREE11.Quaternion();
+  const forward = new THREE11.Vector3();
+  if (camera.quaternion) {
+    if (camera.quaternion instanceof THREE11.Quaternion) {
+      cameraQuaternion.copy(camera.quaternion);
+    } else {
+      cameraQuaternion.set(
+        camera.quaternion[0],
+        camera.quaternion[1],
+        camera.quaternion[2],
+        camera.quaternion[3]
+      );
+    }
+    forward.set(0, 0, -1).applyQuaternion(cameraQuaternion).normalize();
+  } else {
+    const target2 = toVector32(
+      camera.lookAt,
+      cameraPosition.clone().add(new THREE11.Vector3(0, 0, -1))
+    );
+    forward.copy(target2).sub(cameraPosition);
+    if (forward.lengthSq() < 1e-8) forward.set(0, 0, -1);
+    forward.normalize();
+    cameraQuaternion.setFromRotationMatrix(
+      new THREE11.Matrix4().lookAt(cameraPosition, target2, configuredUp)
+    );
+  }
+  const target = camera.lookAt ? toVector32(camera.lookAt, cameraPosition.clone().add(forward)) : cameraPosition.clone().addScaledVector(forward, 0.4);
+  const distanceToTarget = Math.max(target.distanceTo(cameraPosition), 1e-3);
+  const depth = camera.frustumDepth ?? Math.min(Math.max(distanceToTarget * 0.42, 0.16), 0.45);
+  const fov = camera.fov ?? 50;
+  const aspect = (camera.width ?? 640) / (camera.height ?? 480);
+  const right = forward.clone().cross(configuredUp);
+  if (right.lengthSq() < 1e-8) right.set(1, 0, 0);
+  right.normalize();
+  const orthogonalUp = right.clone().cross(forward).normalize();
+  const frustumHeight = 2 * Math.tan(THREE11.MathUtils.degToRad(fov) / 2) * depth;
+  const frustumWidth = frustumHeight * aspect;
+  const center = cameraPosition.clone().addScaledVector(forward, depth);
+  const halfRight = right.clone().multiplyScalar(frustumWidth / 2);
+  const halfUp = orthogonalUp.clone().multiplyScalar(frustumHeight / 2);
+  const topLeft = center.clone().sub(halfRight).add(halfUp);
+  const topRight = center.clone().add(halfRight).add(halfUp);
+  const bottomRight = center.clone().add(halfRight).sub(halfUp);
+  const bottomLeft = center.clone().sub(halfRight).sub(halfUp);
+  const frustumPoints = [
+    cameraPosition,
+    topLeft,
+    cameraPosition,
+    topRight,
+    cameraPosition,
+    bottomRight,
+    cameraPosition,
+    bottomLeft,
+    topLeft,
+    topRight,
+    topRight,
+    bottomRight,
+    bottomRight,
+    bottomLeft,
+    bottomLeft,
+    topLeft
+  ];
+  const group = new THREE11.Group();
+  group.name = camera.name ?? `virtual-camera-${index}`;
+  group.renderOrder = 999;
+  group.frustumCulled = false;
+  const frustum = new THREE11.LineSegments(
+    new THREE11.BufferGeometry().setFromPoints(frustumPoints),
+    new THREE11.LineBasicMaterial({
+      color,
+      transparent: true,
+      opacity: 0.9,
+      depthTest: false
+    })
+  );
+  frustum.renderOrder = 999;
+  frustum.frustumCulled = false;
+  group.add(frustum);
+  const aim = new THREE11.LineSegments(
+    new THREE11.BufferGeometry().setFromPoints([cameraPosition, target]),
+    new THREE11.LineBasicMaterial({
+      color: aimColor,
+      transparent: true,
+      opacity: 0.95,
+      depthTest: false
+    })
+  );
+  aim.renderOrder = 999;
+  aim.frustumCulled = false;
+  group.add(aim);
+  const markerGroup = new THREE11.Group();
+  markerGroup.position.copy(cameraPosition);
+  markerGroup.quaternion.copy(cameraQuaternion);
+  markerGroup.renderOrder = 999;
+  markerGroup.frustumCulled = false;
+  markerGroup.add(new THREE11.Mesh(
+    new THREE11.BoxGeometry(0.045 * markerScale, 0.028 * markerScale, 0.022 * markerScale),
+    new THREE11.MeshBasicMaterial({ color, depthTest: false })
+  ));
+  const lens = new THREE11.Mesh(
+    new THREE11.BoxGeometry(0.025 * markerScale, 0.018 * markerScale, 0.014 * markerScale),
+    new THREE11.MeshBasicMaterial({ color: aimColor, depthTest: false })
+  );
+  lens.position.set(0, 0, -0.021 * markerScale);
+  markerGroup.add(lens);
+  if (camera.name) markerGroup.add(createCameraLabel(camera.name, color));
+  group.add(markerGroup);
+  const targetMarker = new THREE11.Mesh(
+    new THREE11.SphereGeometry(0.018 * markerScale, 16, 10),
+    new THREE11.MeshBasicMaterial({ color: aimColor, depthTest: false })
+  );
+  targetMarker.position.copy(target);
+  targetMarker.renderOrder = 999;
+  targetMarker.frustumCulled = false;
+  group.add(targetMarker);
+  return group;
+}
 function Debug({
   showGeoms = false,
   showSites = false,
   showJoints = false,
   showCameras = false,
+  virtualCameras = [],
   showContacts = false,
   showCOM = false,
   showInertia = false,
@@ -4888,6 +5255,7 @@ function Debug({
     const sites = [];
     const joints = [];
     const cameras = [];
+    const virtualCameraObjects = [];
     const comMarkers = [];
     if (showGeoms) {
       for (let i = 0; i < model.ngeom; i++) {
@@ -5066,6 +5434,9 @@ function Debug({
         cameras.push(group);
       }
     }
+    for (let i = 0; i < virtualCameras.length; i += 1) {
+      virtualCameraObjects.push(createVirtualCameraDebugObject(virtualCameras[i], i));
+    }
     if (showCOM) {
       for (let i = 1; i < model.nbody; i++) {
         const geometry = new THREE11.SphereGeometry(5e-3, 6, 6);
@@ -5075,8 +5446,8 @@ function Debug({
         comMarkers.push(mesh);
       }
     }
-    return { geoms, sites, joints, cameras, comMarkers };
-  }, [status, mjModelRef, showGeoms, showSites, showJoints, showCameras, showCOM]);
+    return { geoms, sites, joints, cameras, virtualCameraObjects, comMarkers };
+  }, [status, mjModelRef, showGeoms, showSites, showJoints, showCameras, virtualCameras, showCOM]);
   useEffect(() => {
     const group = groupRef.current;
     if (!group || !debugGeometry) return;
@@ -5085,6 +5456,7 @@ function Debug({
       ...debugGeometry.sites,
       ...debugGeometry.joints,
       ...debugGeometry.cameras,
+      ...debugGeometry.virtualCameraObjects,
       ...debugGeometry.comMarkers
     ];
     for (const obj of allObjects) group.add(obj);
@@ -6371,6 +6743,73 @@ function useControlWriter(options) {
     release
   }), [canWrite, owner, read, release, write]);
 }
+// src/hooks/useControlGroup.ts
+function controlGroup(names) {
+  return { keys: names };
+}
+function useControlGroup(group, options = {}) {
+  const generatedOwner = useId();
+  const owner = options.owner ?? `control-group:${generatedOwner}`;
+  const { mjModelRef, mjDataRef, status } = useMujocoContext();
+  const names = group.keys;
+  const namesRef = useRef(names);
+  namesRef.current = names;
+  const namesKey = names.join("\0");
+  const selector = useMemo(
+    () => ({ actuators: [...names] }),
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+    [namesKey]
+  );
+  const { canWrite, release } = useControlWriter({ ...options, owner, selector });
+  const indicesRef = useRef([]);
+  useEffect(() => {
+    const model = mjModelRef.current;
+    if (!model || status !== "ready") {
+      indicesRef.current = [];
+      return;
+    }
+    indicesRef.current = namesRef.current.map((name) => {
+      const id = findActuatorByName(model, name);
+      if (id < 0) {
+        console.warn(`[mujoco-react] useControlGroup: actuator "${name}" was not found.`);
+      }
+      return id;
+    });
+  }, [mjModelRef, status, namesKey]);
+  const set = useCallback(
+    (values, setOptions = {}) => {
+      const data = mjDataRef.current;
+      if (!data) return false;
+      if (!setOptions.force && !canWrite()) return false;
+      const groupNames = namesRef.current;
+      const indices = indicesRef.current;
+      for (let i = 0; i < groupNames.length; i += 1) {
+        const value = values[groupNames[i]];
+        if (value === void 0) continue;
+        const adr = indices[i];
+        if (adr >= 0) data.ctrl[adr] = value;
+      }
+      return true;
+    },
+    [canWrite, mjDataRef]
+  );
+  const read = useCallback(() => {
+    const data = mjDataRef.current;
+    const groupNames = namesRef.current;
+    const indices = indicesRef.current;
+    const result = {};
+    for (let i = 0; i < groupNames.length; i += 1) {
+      const adr = indices[i];
+      result[groupNames[i]] = data && adr >= 0 ? data.ctrl[adr] ?? 0 : 0;
+    }
+    return result;
+  }, [mjDataRef]);
+  return useMemo(
+    () => ({ owner, set, read, canWrite, release }),
+    [owner, set, read, canWrite, release]
+  );
+}
 var geomNameCacheByModel2 = /* @__PURE__ */ new WeakMap();
 var bodyNameCacheByModel = /* @__PURE__ */ new WeakMap();
 function getCachedName(cacheByModel, model, id, address) {
@@ -7667,6 +8106,130 @@ function usePolicyCameraFramesFromMountedStreams(defaultOptions) {
     reset
   };
 }
+function sessionSignature(stream) {
+  return JSON.stringify({
+    width: stream.width,
+    height: stream.height,
+    channels: stream.channels,
+    renderIsolation: stream.renderIsolation ?? false,
+    cameraName: stream.cameraName,
+    siteName: stream.siteName,
+    bodyName: stream.bodyName
+  });
+}
+function addTensorAliases(tensors, stream, tensor, includeObservationImageAliases) {
+  const keys = /* @__PURE__ */ new Set([stream.key, ...stream.aliases ?? []]);
+  if (includeObservationImageAliases) {
+    for (const base of [stream.key, ...stream.aliases ?? []]) {
+      keys.add(`observation.images.${base}`);
+    }
+  }
+  for (const key of keys) tensors[key] = tensor;
+}
+function usePolicyCameraTensors(options) {
+  const mujoco = useMujoco();
+  const [status, setStatus] = useState("idle");
+  const [error, setError] = useState(null);
+  const sessionsRef = useRef(/* @__PURE__ */ new Map());
+  const disposeSessions = useCallback(() => {
+    for (const { session } of sessionsRef.current.values()) session.dispose();
+    sessionsRef.current.clear();
+  }, []);
+  useEffect(() => disposeSessions, [disposeSessions]);
+  const reset = useCallback(() => {
+    setStatus("idle");
+    setError(null);
+  }, []);
+  const capture = useCallback(() => {
+    const api = mujoco.api;
+    if (!api) {
+      throw new Error("MuJoCo scene is not ready for policy camera tensor capture.");
+    }
+    setStatus("capturing");
+    setError(null);
+    try {
+      const sessions = sessionsRef.current;
+      const seen = /* @__PURE__ */ new Set();
+      const tensors = {};
+      const sourceParts = [];
+      for (const stream of options.streams) {
+        seen.add(stream.key);
+        const resolved = {
+          ...api.resolveCameraCaptureOptions(stream),
+          channels: stream.channels,
+          layout: stream.layout,
+          range: stream.range
+        };
+        const signature = sessionSignature(stream);
+        let entry = sessions.get(stream.key);
+        if (!entry || entry.signature !== signature) {
+          entry?.session.dispose();
+          entry = {
+            session: api.createCameraFrameCaptureSession(resolved),
+            signature
+          };
+          sessions.set(stream.key, entry);
+        }
+        const tensor = entry.session.captureTensor(resolved);
+        addTensorAliases(
+          tensors,
+          stream,
+          tensor,
+          options.includeObservationImageAliases ?? false
+        );
+        sourceParts.push(`${stream.key}:${tensor.source.kind}`);
+      }
+      for (const key of [...sessions.keys()]) {
+        if (!seen.has(key)) {
+          sessions.get(key)?.session.dispose();
+          sessions.delete(key);
+        }
+      }
+      setStatus("captured");
+      return {
+        tensors,
+        sourceSummary: sourceParts.join(" + ") || "not used by policy",
+        capturedAt: Date.now()
+      };
+    } catch (nextError) {
+      const captureError = nextError instanceof Error ? nextError : new Error("Unable to capture policy camera tensors.");
+      setError(captureError);
+      setStatus("error");
+      throw captureError;
+    }
+  }, [mujoco.api, options.includeObservationImageAliases, options.streams]);
+  return {
+    status,
+    error,
+    isCapturing: status === "capturing",
+    capture,
+    reset
+  };
+}
+function usePolicyCameraTensorsFromMountedStreams(options) {
+  const mujoco = useMujoco();
+  const tensorOptions = options.tensor;
+  const mountedOptions = useMemo(() => {
+    const api = mujoco.api;
+    if (!api) {
+      return {
+        streams: [],
+        includeObservationImageAliases: options.includeObservationImageAliases ?? false
+      };
+    }
+    const plan = createPolicyCameraFrameCapturePlanFromApi(api, options);
+    return {
+      streams: plan.streams.map(({ key, aliases, ...stream }) => ({
+        ...stream,
+        ...tensorOptions,
+        key,
+        aliases
+      })),
+      includeObservationImageAliases: plan.includeObservationImageAliases ?? false
+    };
+  }, [mujoco.api, options, tensorOptions]);
+  return usePolicyCameraTensors(mountedOptions);
+}
 function useCameraSequenceRecorder() {
   const mujoco = useMujoco();
   const [status, setStatus] = useState("idle");
@@ -7971,6 +8534,19 @@ function useCameraAnimation() {
  * @license
  * SPDX-License-Identifier: Apache-2.0
 */
+/**
+ * @license
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Live on-screen camera viewports for MuJoCo scenes. Each view renders the
+ * shared scene from a named MuJoCo camera/site/body into a `gl.scissor` region
+ * tracking a DOM element — no GPU readback, no PNG encoding.
+ *
+ * While at least one viewport is mounted the canvas switches from R3F's
+ * automatic render to a managed render loop (main scene full-frame, then each
+ * viewport). This is incompatible with `EffectComposer`/postprocessing or other
+ * custom render loops; use the offscreen capture APIs in those setups instead.
+ */
 /**
  * @license
  * SPDX-License-Identifier: Apache-2.0
@@ -8019,6 +8595,21 @@ function useCameraAnimation() {
  * light_type: 0 = directional, 1 = spot (maps to mjLIGHT_DIRECTIONAL/mjLIGHT_SPOT)
  * Note: light_directional does NOT exist in WASM — use light_type instead.
  */
+/**
+ * @license
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Stream a live MuJoCo camera into a DOM `<canvas>`. Each frame the scene is
+ * rendered offscreen from the selected camera and blitted into the canvas, so
+ * it composites normally in the DOM (works inside opaque panels) and does NOT
+ * take over the render loop. Prefer this over `useCameraViewport` for camera
+ * tiles embedded in HTML UI; use `useCameraViewport` for transparent overlays
+ * on a full-bleed canvas.
+ *
+ * Uses the async capture path so Gaussian-splat environments render through
+ * their dedicated capture renderer — streaming a splat scene at full rate does
+ * not disturb the main view's splat sort.
+ */
 /**
  * @license
  * SPDX-License-Identifier: Apache-2.0
@@ -8104,6 +8695,14 @@ function useCameraAnimation() {
  *
  * Cooperative actuator/control ownership for policies, IK, teleop, and replay.
  */
+/**
+ * @license
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Typed, named control groups. Define the group once (like a zod schema or a
+ * tanstack route), then read/set actuators by name with inferred keys — no
+ * positional arrays, no `as const`, no owner bookkeeping.
+ */
 /**
  * @license
  * SPDX-License-Identifier: Apache-2.0
@@ -8176,6 +8775,15 @@ function useCameraAnimation() {
  *
  * React wrapper for capturing policy image payloads from Three/MuJoCo cameras.
  */
+/**
+ * @license
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Capture policy observation tensors directly from Three/MuJoCo cameras,
+ * skipping the data-URL/PNG round-trip. Sessions are created once per camera
+ * and reused every step, so live inference and dataset recording read straight
+ * from the GPU into Float32 tensors.
+ */
 /**
  * @license
  * SPDX-License-Identifier: Apache-2.0
@@ -8224,6 +8832,6 @@ function useCameraAnimation() {
  * useCameraAnimation — composable camera animation hook.
  */
-export { Body, ContactListener, ContactMarkers, Debug, DragInteraction, FlexRenderer, IkGizmo, InstancedGeomRenderer, MountedCameraFrameSequenceManifestStatus, MountedCameraFrameSequenceReadinessStatus, MountedCameraFrameSourceSuggestionMatch, MujocoCanvas, MujocoPhysics, MujocoProvider, MujocoSimProvider, SceneLights, SplatCollisionProxyPreview, TendonRenderer, TrajectoryPlayer, applyPolicyActionToControls, bodyPositionField, buildObservation, canFetchSplatCollisionProxyXml, captureFrame, captureFrameBlob, capturePolicyCameraFrames, capturePolicyCameraFramesFromMountedStreams, clampPolicyActionValue, createContiguousControlGroup, createController, createControllerHook, createMountedCameraFrameSequenceManifest, createMountedCameraFrameSequencePlan, createMountedCameraFrameSequencePlanFromApi, createMountedCameraFrameSequenceReadiness, createMountedCameraFrameSourceSuggestions, createNamedObservationBuilder, createPolicyCameraFrameCapturePlan, createPolicyCameraFrameCapturePlanFromApi, ctrlField, fetchSplatCollisionProxyXml, findActuatorByName, findBodyByName, findGeomByName, findJointByName, findKeyframeByName, findSensorByName, findSiteByName, findTendonByName, geomPositionField, getActuatedJoints, getCameraFrameCaptureSourceTarget, getControlMap, getMountedCameraFrameCaptureSource, getName, imagePointToNdc, isMountedCameraFrameCaptureSource, loadScene, parseSplatCollisionProxyGeoms, projectImagePointTo3D, qposField, qvelField, readNamedObservation, recordMountedCameraFrameSequence, resolveControlGroup, resolveMountedCameraFrameSource, sitePositionField, useActuators, useAfterPhysicsStep, useBeforePhysicsStep, useBodyMeshes, useBodyPose, useBodyState, useCameraAnimation, useCameraFrameCapture, useCameraSequenceRecorder, useContactEvents, useContactHistory, useContacts, useControlWriter, useCtrl, useCtrlNoise, useFrameCapture, useGamepad, useGeomPose, useGravityCompensation, useIkController, useJointState, useKeyboardIkTarget, useKeyboardTeleop, useMountedCameraSequenceRecorder, useMujoco, useMujocoWasm, useNamedObservation, useObservation, usePolicy, usePolicyCameraFrames, usePolicyCameraFramesFromMountedStreams, useRemotePolicy, useSceneLights, useSelectionHighlight, useSensor, useSensors, useSitePose, useSitePosition, useSplatCollisionProxyGeoms, useTrajectoryPlayer, useTrajectoryRecorder, useVideoRecorder };
+export { Body, CameraView, ContactListener, ContactMarkers, Debug, DragInteraction, FlexRenderer, GenericIK, IkGizmo, InstancedGeomRenderer, MountedCameraFrameSequenceManifestStatus, MountedCameraFrameSequenceReadinessStatus, MountedCameraFrameSourceSuggestionMatch, MujocoCanvas, MujocoPhysics, MujocoProvider, MujocoSimProvider, SceneLights, SplatCollisionProxyPreview, TendonRenderer, TrajectoryPlayer, applyPolicyActionToControls, bodyPositionField, buildObservation, canFetchSplatCollisionProxyXml, captureFrame, captureFrameBlob, capturePolicyCameraFrames, capturePolicyCameraFramesFromMountedStreams, clampPolicyActionValue, controlGroup, createContiguousControlGroup, createController, createControllerHook, createMountedCameraFrameSequenceManifest, createMountedCameraFrameSequencePlan, createMountedCameraFrameSequencePlanFromApi, createMountedCameraFrameSequenceReadiness, createMountedCameraFrameSourceSuggestions, createNamedObservationBuilder, createPolicyCameraFrameCapturePlan, createPolicyCameraFrameCapturePlanFromApi, ctrlField, fetchSplatCollisionProxyXml, findActuatorByName, findBodyByName, findGeomByName, findJointByName, findKeyframeByName, findSensorByName, findSiteByName, findTendonByName, geomPositionField, getActuatedJoints, getCameraFrameCaptureSourceTarget, getControlMap, getMountedCameraFrameCaptureSource, getName, imagePointToNdc, isMountedCameraFrameCaptureSource, loadScene, parseSplatCollisionProxyGeoms, projectImagePointTo3D, qposField, qvelField, readNamedObservation, recordMountedCameraFrameSequence, resolveControlGroup, resolveMountedCameraFrameSource, sitePositionField, useActuators, useAfterPhysicsStep, useBeforePhysicsStep, useBodyMeshes, useBodyPose, useBodyState, useCameraAnimation, useCameraFrameCapture, useCameraSequenceRecorder, useCameraStream, useCameraViewport, useContactEvents, useContactHistory, useContacts, useControlGroup, useControlWriter, useCtrl, useCtrlNoise, useFrameCapture, useGamepad, useGeomPose, useGravityCompensation, useIkController, useJointState, useKeyboardIkTarget, useKeyboardTeleop, useMountedCameraSequenceRecorder, useMujoco, useMujocoWasm, useNamedObservation, useObservation, usePolicy, usePolicyCameraFrames, usePolicyCameraFramesFromMountedStreams, usePolicyCameraTensors, usePolicyCameraTensorsFromMountedStreams, useRemotePolicy, useSceneLights, useSelectionHighlight, useSensor, useSensors, useSitePose, useSitePosition, useSplatCollisionProxyGeoms, useTrajectoryPlayer, useTrajectoryRecorder, useVideoRecorder };
 //# sourceMappingURL=index.js.map
 //# sourceMappingURL=index.js.map