npm - mujoco-react - Versions diffs - 10.4.0 → 10.6.0 - Mend

mujoco-react 10.4.0 → 10.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (27) hide show

package/README.md +73 -136
package/dist/{chunk-FBXXXPLQ.js → chunk-EN55TTGH.js} +157 -16
package/dist/chunk-EN55TTGH.js.map +1 -0
package/dist/index.d.ts +179 -48
package/dist/index.js +487 -20
package/dist/index.js.map +1 -1
package/dist/onnx.d.ts +65 -0
package/dist/onnx.js +58 -0
package/dist/onnx.js.map +1 -0
package/dist/spark.d.ts +1 -1
package/dist/spark.js +1 -1
package/dist/{types-CdFZCYmy.d.ts → types-Dvtm4I0o.d.ts} +155 -4
package/package.json +14 -3
package/src/components/CameraView.tsx +245 -0
package/src/core/GenericIK.ts +22 -3
package/src/core/MujocoSimProvider.tsx +37 -1
package/src/core/SceneLoader.ts +3 -2
package/src/hooks/useCameraStream.ts +115 -0
package/src/hooks/useControlGroup.ts +0 -0
package/src/hooks/useIkController.ts +36 -5
package/src/hooks/usePolicyCameraTensors.ts +215 -0
package/src/index.ts +44 -0
package/src/onnx.ts +126 -0
package/src/policyImageTensors.ts +150 -0
package/src/rendering/cameraFrameCapture.ts +112 -15
package/src/types.ts +50 -3
package/dist/chunk-FBXXXPLQ.js.map +0 -1

package/dist/index.js CHANGED Viewed

@@ -1,8 +1,8 @@
-import { withContacts, getContact, captureCameraFrame, captureCameraFrameBlob, createCameraFrameCaptureSession, CAMERA_FRAME_CAPTURE_PRE_RENDER_USER_DATA_KEY, CAPTURE_EXCLUDE_KEY } from './chunk-FBXXXPLQ.js';
-export { CAMERA_FRAME_CAPTURE_PRE_RENDER_USER_DATA_KEY, CAMERA_FRAME_CAPTURE_RENDER_USER_DATA_KEY, CAPTURE_EXCLUDE_KEY, ModelActuators, ModelBodies, ModelCameras, ModelGeoms, ModelJoints, ModelKeyframes, ModelResources, ModelSensors, ModelSites, ScenarioLighting, SplatEnvironment, SplatEnvironmentReadinessStatus, VisualScenarioEffects, captureCameraFrame, captureCameraFrameBlob, createCameraFrameCaptureSession, createPairedSplatEnvironment, createSparkSplatViewerUrl, createSplatEnvironmentUserData, createSplatSceneConfig, createVisualScenarioExecutionContext, getContact, getScenarioBackground, getScenarioCameraPosition, getSplatEnvironmentReadiness, registerModelResources, renderCameraFrameToCanvas, useSplatEnvironment, useSplatSceneConfig, useVisualScenarioEffects, useVisualScenarioExecutionContext, withSplatEnvironment } from './chunk-FBXXXPLQ.js';
+import { withContacts, getContact, captureCameraFrame, captureCameraFrameBlob, createCameraFrameCaptureSession, CAMERA_FRAME_CAPTURE_PRE_RENDER_USER_DATA_KEY, CAPTURE_EXCLUDE_KEY, createCaptureCamera, prepareCaptureCamera } from './chunk-EN55TTGH.js';
+export { CAMERA_FRAME_CAPTURE_PRE_RENDER_USER_DATA_KEY, CAMERA_FRAME_CAPTURE_RENDER_USER_DATA_KEY, CAPTURE_EXCLUDE_KEY, ModelActuators, ModelBodies, ModelCameras, ModelGeoms, ModelJoints, ModelKeyframes, ModelResources, ModelSensors, ModelSites, ScenarioLighting, SplatEnvironment, SplatEnvironmentReadinessStatus, VisualScenarioEffects, captureCameraFrame, captureCameraFrameBlob, captureCameraFrameTensor, createCameraFrameCaptureSession, createPairedSplatEnvironment, createSparkSplatViewerUrl, createSplatEnvironmentUserData, createSplatSceneConfig, createVisualScenarioExecutionContext, dataUrlToPolicyImageTensor, getContact, getScenarioBackground, getScenarioCameraPosition, getSplatEnvironmentReadiness, imageDataToPolicyImageTensor, pixelsToPolicyImageTensor, registerModelResources, renderCameraFrameToCanvas, useSplatEnvironment, useSplatSceneConfig, useVisualScenarioEffects, useVisualScenarioExecutionContext, withSplatEnvironment } from './chunk-EN55TTGH.js';
 import loadMujoco from '@mujoco/mujoco';
 import defaultMujocoWasmUrl from '@mujoco/mujoco/mujoco.wasm?url';
-import { createContext, forwardRef, useRef, useEffect, useContext, useState, useCallback, useMemo, useLayoutEffect } from 'react';
+import { createContext, forwardRef, useRef, useEffect, useContext, useState, useCallback, useMemo, useLayoutEffect, useId } from 'react';
 import { jsx, jsxs } from 'react/jsx-runtime';
 import { Canvas, useThree, useFrame } from '@react-three/fiber';
 import * as THREE11 from 'three';
@@ -492,8 +492,10 @@ function sceneObjectToXml(obj) {
   const solref = obj.solref ? ` solref="${obj.solref}"` : "";
   const solimp = obj.solimp ? ` solimp="${obj.solimp}"` : "";
   const condim = obj.condim ? ` condim="${obj.condim}"` : "";
+  const contype = obj.contype ?? 1;
+  const conaffinity = obj.conaffinity ?? 1;
   const group = obj.group !== void 0 ? ` group="${obj.group}"` : "";
-  return `<body name="${obj.name}" pos="${pos}">${joint}<geom name="${geomName}" type="${obj.type}" size="${size}" rgba="${rgba}" contype="1" conaffinity="1"${mass}${friction}${solref}${solimp}${condim}${group}/></body>`;
+  return `<body name="${obj.name}" pos="${pos}">${joint}<geom name="${geomName}" type="${obj.type}" size="${size}" rgba="${rgba}" contype="${contype}" conaffinity="${conaffinity}"${mass}${friction}${solref}${solimp}${condim}${group}/></body>`;
 }
 function ensureDir(mujoco, fname) {
   const dirParts = fname.split("/");
@@ -1170,6 +1172,118 @@ function SceneRenderer({ renderOptions, ...props }) {
 }
 var _previousQuat = new THREE11.Quaternion();
 var _currentQuat = new THREE11.Quaternion();
+var CameraViewportRegistryContext = createContext(null);
+var nextViewportId = 0;
+var VIEWPORT_RENDER_PRIORITY = 1;
+function CameraViewportRenderer({
+  viewportsRef
+}) {
+  const gl = useThree((state) => state.gl);
+  const scene = useThree((state) => state.scene);
+  const mainCamera = useThree((state) => state.camera);
+  const mujoco = useMujoco();
+  useFrame(() => {
+    const drawWidth = gl.domElement.width;
+    const drawHeight = gl.domElement.height;
+    gl.setScissorTest(false);
+    gl.setViewport(0, 0, drawWidth, drawHeight);
+    gl.render(scene, mainCamera);
+    const api = mujoco.api;
+    if (!api || viewportsRef.current.size === 0) return;
+    const dpr = gl.getPixelRatio();
+    const canvasRect = gl.domElement.getBoundingClientRect();
+    for (const descriptor of viewportsRef.current.values()) {
+      const element = descriptor.getElement();
+      if (!element) continue;
+      const rect = element.getBoundingClientRect();
+      const isOffscreen = rect.bottom < canvasRect.top || rect.top > canvasRect.bottom || rect.right < canvasRect.left || rect.left > canvasRect.right;
+      if (isOffscreen) continue;
+      const width = Math.floor(rect.width * dpr);
+      const height = Math.floor(rect.height * dpr);
+      if (width <= 0 || height <= 0) continue;
+      const left = Math.floor((rect.left - canvasRect.left) * dpr);
+      const bottom = Math.floor((canvasRect.bottom - rect.bottom) * dpr);
+      let resolved;
+      try {
+        resolved = api.resolveCameraCaptureOptions(descriptor.getOptions());
+      } catch {
+        continue;
+      }
+      if (!descriptor.camera) {
+        descriptor.camera = createCaptureCamera(resolved, mainCamera, width, height);
+      } else {
+        prepareCaptureCamera(descriptor.camera, resolved, mainCamera, width, height);
+      }
+      gl.setViewport(left, bottom, width, height);
+      gl.setScissor(left, bottom, width, height);
+      gl.setScissorTest(true);
+      gl.render(scene, descriptor.camera);
+    }
+    gl.setScissorTest(false);
+    gl.setViewport(0, 0, drawWidth, drawHeight);
+  }, VIEWPORT_RENDER_PRIORITY);
+  return null;
+}
+function CameraViewportProvider({ children }) {
+  const viewportsRef = useRef(/* @__PURE__ */ new Map());
+  const [count, setCount] = useState(0);
+  const register = useCallback((descriptor) => {
+    const id = nextViewportId++;
+    viewportsRef.current.set(id, descriptor);
+    setCount((value2) => value2 + 1);
+    return () => {
+      viewportsRef.current.delete(id);
+      setCount((value2) => value2 - 1);
+    };
+  }, []);
+  const value = useMemo(() => ({ register }), [register]);
+  return /* @__PURE__ */ jsxs(CameraViewportRegistryContext.Provider, { value, children: [
+    children,
+    count > 0 && /* @__PURE__ */ jsx(CameraViewportRenderer, { viewportsRef })
+  ] });
+}
+function useCameraViewport(elementRef, options) {
+  const registry = useContext(CameraViewportRegistryContext);
+  if (!registry) {
+    throw new Error("useCameraViewport must be used inside <MujocoCanvas>.");
+  }
+  const optionsRef = useRef(options);
+  optionsRef.current = options;
+  useEffect(() => {
+    const descriptor = {
+      getElement: () => elementRef.current,
+      getOptions: () => optionsRef.current,
+      camera: null
+    };
+    return registry.register(descriptor);
+  }, [registry, elementRef]);
+}
+function CameraView({ className, style, ...options }) {
+  const gl = useThree((state) => state.gl);
+  const elementRef = useRef(null);
+  if (!elementRef.current && typeof document !== "undefined") {
+    elementRef.current = document.createElement("div");
+  }
+  useEffect(() => {
+    const element = elementRef.current;
+    const parent = gl.domElement.parentElement;
+    if (!element || !parent) return;
+    element.style.position = "absolute";
+    element.style.overflow = "hidden";
+    parent.appendChild(element);
+    return () => {
+      parent.removeChild(element);
+    };
+  }, [gl]);
+  useEffect(() => {
+    const element = elementRef.current;
+    if (!element) return;
+    element.className = className ?? "";
+    if (style) Object.assign(element.style, style);
+  }, [className, style]);
+  useCameraViewport(elementRef, options);
+  return null;
+}
 function isTargetRef(target) {
   return Boolean(target && typeof target === "object" && "current" in target);
 }
@@ -3021,6 +3135,32 @@ function MujocoSimProvider({
     },
     [camera, gl, resolveCameraCaptureOptions, scene]
   );
+  const createCameraFrameCaptureSessionApi = useCallback(
+    (options = {}) => createCameraFrameCaptureSession(
+      gl,
+      scene,
+      camera,
+      resolveCameraCaptureOptions(options)
+    ),
+    [camera, gl, resolveCameraCaptureOptions, scene]
+  );
+  const captureCameraFrameTensorApi = useCallback(
+    (options = {}) => {
+      const resolved = {
+        ...resolveCameraCaptureOptions(options),
+        channels: options.channels,
+        layout: options.layout,
+        range: options.range
+      };
+      const session = createCameraFrameCaptureSession(gl, scene, camera, resolved);
+      try {
+        return session.captureTensor(resolved);
+      } finally {
+        session.dispose();
+      }
+    },
+    [camera, gl, resolveCameraCaptureOptions, scene]
+  );
   const recordCameraSequenceApi = useCallback(
     async (options) => {
       const frameCount = Math.max(0, Math.floor(options.frames));
@@ -3320,6 +3460,9 @@ function MujocoSimProvider({
       captureFrameBlob: captureFrameBlobApi,
       captureCameraFrame: captureCameraFrameApi,
       captureCameraFrameBlob: captureCameraFrameBlobApi,
+      captureCameraFrameTensor: captureCameraFrameTensorApi,
+      createCameraFrameCaptureSession: createCameraFrameCaptureSessionApi,
+      resolveCameraCaptureOptions,
       recordCameraSequence: recordCameraSequenceApi,
       project2DTo3D,
       projectImagePointTo3D: projectImagePointTo3D2,
@@ -3380,6 +3523,9 @@ function MujocoSimProvider({
       captureFrameBlobApi,
       captureCameraFrameApi,
       captureCameraFrameBlobApi,
+      captureCameraFrameTensorApi,
+      createCameraFrameCaptureSessionApi,
+      resolveCameraCaptureOptions,
       recordCameraSequenceApi,
       project2DTo3D,
       projectImagePointTo3D2,
@@ -3416,7 +3562,7 @@ function MujocoSimProvider({
   );
   return /* @__PURE__ */ jsxs(MujocoSimContext.Provider, { value: contextValue, children: [
     /* @__PURE__ */ jsx(SceneRenderer, { renderOptions }),
-    children
+    /* @__PURE__ */ jsx(CameraViewportProvider, { children })
   ] });
 }
 var MujocoCanvas = forwardRef(
@@ -3641,7 +3787,8 @@ function resolveOptions(opts) {
     tolerance: opts?.tolerance ?? DEFAULTS.tolerance,
     epsilon: opts?.epsilon ?? DEFAULTS.epsilon,
     posWeight: opts?.posWeight ?? DEFAULTS.posWeight,
-    rotWeight: opts?.rotWeight ?? DEFAULTS.rotWeight
+    rotWeight: opts?.rotWeight ?? DEFAULTS.rotWeight,
+    jointLimits: opts?.jointLimits
   };
 }
 var GenericIK = class {
@@ -3680,6 +3827,8 @@ var GenericIK = class {
     const pertSiteMat = new Float64Array(9);
     let bestQ = null;
     let bestErr = Infinity;
+    const patience = 4;
+    let noImprove = 0;
     if (n === 0) return null;
     for (let iter = 0; iter < o.maxIterations; iter++) {
       for (let i = 0; i < n; i++) data.qpos[qposAdr[i]] = q[i];
@@ -3705,11 +3854,15 @@ var GenericIK = class {
       const errNorm = Math.sqrt(
         error[0] * error[0] + error[1] * error[1] + error[2] * error[2] + error[3] * error[3] + error[4] * error[4] + error[5] * error[5]
       );
-      if (errNorm < bestErr) {
+      if (errNorm < bestErr - 1e-9) {
         bestErr = errNorm;
         bestQ = Array.from(q);
+        noImprove = 0;
+      } else {
+        noImprove++;
       }
       if (errNorm < o.tolerance) break;
+      if (noImprove >= patience) break;
       for (let j = 0; j < n; j++) {
         const adr = qposAdr[j];
         const saved = data.qpos[adr];
@@ -3865,6 +4018,12 @@ function solve6x6(A, b, x) {
 // src/hooks/useIkController.ts
 var _syncMat4 = new THREE11.Matrix4();
+function toIkVector3(value) {
+  return "x" in value ? new THREE11.Vector3(value.x, value.y, value.z) : new THREE11.Vector3(value[0], value[1], value[2]);
+}
+function toIkQuaternion(value) {
+  return "x" in value ? new THREE11.Quaternion(value.x, value.y, value.z, value.w) : new THREE11.Quaternion(value[0], value[1], value[2], value[3]);
+}
 function syncGizmoToSite(data, siteId, target) {
   if (siteId === -1) return;
   const sitePos = data.site_xpos.subarray(siteId * 3, siteId * 3 + 3);
@@ -3937,22 +4096,31 @@ var useIkController = createControllerHook(
     const ikSolveFn = useCallback(
       ({ position, quaternion, currentQ, context }) => {
         if (!config) return null;
-        if (config.ikSolveFn) return config.ikSolveFn({ position, quaternion, currentQ, context });
+        const targetPosition = toIkVector3(position);
+        const targetQuaternion = toIkQuaternion(quaternion);
+        const normalizedInput = {
+          position: targetPosition,
+          quaternion: targetQuaternion,
+          currentQ,
+          context
+        };
+        if (config.ikSolveFn) return config.ikSolveFn(normalizedInput);
         const model = mjModelRef.current;
         const data = mjDataRef.current;
-        const controlGroup = controlGroupRef.current;
-        if (!model || !data || !controlGroup || siteIdRef.current === -1) return null;
+        const controlGroup2 = controlGroupRef.current;
+        if (!model || !data || !controlGroup2 || siteIdRef.current === -1) return null;
         return genericIkRef.current.solve(
           model,
           data,
           siteIdRef.current,
-          controlGroup.qposAdr,
-          position,
-          quaternion,
+          controlGroup2.qposAdr,
+          targetPosition,
+          targetQuaternion,
           currentQ,
           {
             damping: config.damping,
             epsilon: config.epsilon,
+            jointLimits: config.jointLimits ?? controlGroup2.joints.map((joint) => joint.limited ? joint.range : joint.ctrlRange),
             maxIterations: config.maxIterations,
             posWeight: config.posWeight,
             rotWeight: config.rotWeight,
@@ -3992,9 +4160,9 @@ var useIkController = createControllerHook(
       const target = ikTargetRef.current;
       if (!target) return;
       ikCalculatingRef.current = true;
-      const controlGroup = controlGroupRef.current;
-      if (!controlGroup) return;
-      const currentQ = Array.from(controlGroup.readQpos(data));
+      const controlGroup2 = controlGroupRef.current;
+      if (!controlGroup2) return;
+      const currentQ = Array.from(controlGroup2.readQpos(data));
       const solution = config.ikSolveFn ? config.ikSolveFn({
         position: target.position,
         quaternion: target.quaternion,
@@ -4003,7 +4171,7 @@ var useIkController = createControllerHook(
           model,
           data,
           siteId: siteIdRef.current,
-          controlGroup
+          controlGroup: controlGroup2
         }
       }) : ikSolveFnRef.current({
         position: target.position,
@@ -4011,7 +4179,7 @@ var useIkController = createControllerHook(
         currentQ
       });
       if (solution) {
-        controlGroup.writeCtrl(data, solution);
+        controlGroup2.writeCtrl(data, solution);
       }
     });
     useEffect(() => {
@@ -4058,7 +4226,7 @@ var useIkController = createControllerHook(
         if (!ikEnabledRef.current) setIkEnabled(true);
         const target = ikTargetRef.current;
         if (!target) return;
-        const targetPos = pos.clone();
+        const targetPos = toIkVector3(pos);
         const targetRot = new THREE11.Quaternion().setFromEuler(
           new THREE11.Euler(Math.PI, 0, 0)
         );
@@ -4848,6 +5016,69 @@ function parseNumberList(value) {
 function addProxyVectors(a, b) {
   return [a[0] + b[0], a[1] + b[1], a[2] + b[2]];
 }
+function streamSignature(options) {
+  return JSON.stringify({
+    cameraName: options.cameraName,
+    siteName: options.siteName,
+    bodyName: options.bodyName,
+    width: options.width,
+    height: options.height,
+    renderIsolation: options.renderIsolation ?? false
+  });
+}
+function useCameraStream(canvasRef, options) {
+  const mujoco = useMujoco();
+  const sessionRef = useRef(null);
+  const signatureRef = useRef("");
+  const optionsRef = useRef(options);
+  optionsRef.current = options;
+  const elapsedRef = useRef(0);
+  const inFlightRef = useRef(false);
+  const mountedRef = useRef(true);
+  useEffect(() => {
+    mountedRef.current = true;
+    return () => {
+      mountedRef.current = false;
+      sessionRef.current?.dispose();
+      sessionRef.current = null;
+      signatureRef.current = "";
+    };
+  }, []);
+  useFrame((_state, delta) => {
+    const api = mujoco.api;
+    if (!api || !canvasRef.current) return;
+    const opts = optionsRef.current;
+    if (opts.paused) return;
+    if (opts.fps && opts.fps > 0) {
+      elapsedRef.current += delta;
+      if (elapsedRef.current < 1 / opts.fps) return;
+    }
+    if (inFlightRef.current) return;
+    elapsedRef.current = 0;
+    const signature = streamSignature(opts);
+    if (!sessionRef.current || signatureRef.current !== signature) {
+      sessionRef.current?.dispose();
+      sessionRef.current = api.createCameraFrameCaptureSession(opts);
+      signatureRef.current = signature;
+    }
+    const session = sessionRef.current;
+    inFlightRef.current = true;
+    session.captureAsync(api.resolveCameraCaptureOptions(opts)).then((frame) => {
+      const canvas = canvasRef.current;
+      if (!mountedRef.current || !canvas) return;
+      const ctx = canvas.getContext("2d");
+      if (!ctx) return;
+      if (canvas.width !== frame.width || canvas.height !== frame.height) {
+        canvas.width = frame.width;
+        canvas.height = frame.height;
+      }
+      ctx.drawImage(frame.canvas, 0, 0);
+    }).catch(() => {
+    }).finally(() => {
+      inFlightRef.current = false;
+    });
+  });
+}
 var JOINT_COLORS = {
   0: 16711680,
   // free - red
@@ -6526,6 +6757,73 @@ function useControlWriter(options) {
     release
   }), [canWrite, owner, read, release, write]);
 }
+// src/hooks/useControlGroup.ts
+function controlGroup(names) {
+  return { keys: names };
+}
+function useControlGroup(group, options = {}) {
+  const generatedOwner = useId();
+  const owner = options.owner ?? `control-group:${generatedOwner}`;
+  const { mjModelRef, mjDataRef, status } = useMujocoContext();
+  const names = group.keys;
+  const namesRef = useRef(names);
+  namesRef.current = names;
+  const namesKey = names.join("\0");
+  const selector = useMemo(
+    () => ({ actuators: [...names] }),
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+    [namesKey]
+  );
+  const { canWrite, release } = useControlWriter({ ...options, owner, selector });
+  const indicesRef = useRef([]);
+  useEffect(() => {
+    const model = mjModelRef.current;
+    if (!model || status !== "ready") {
+      indicesRef.current = [];
+      return;
+    }
+    indicesRef.current = namesRef.current.map((name) => {
+      const id = findActuatorByName(model, name);
+      if (id < 0) {
+        console.warn(`[mujoco-react] useControlGroup: actuator "${name}" was not found.`);
+      }
+      return id;
+    });
+  }, [mjModelRef, status, namesKey]);
+  const set = useCallback(
+    (values, setOptions = {}) => {
+      const data = mjDataRef.current;
+      if (!data) return false;
+      if (!setOptions.force && !canWrite()) return false;
+      const groupNames = namesRef.current;
+      const indices = indicesRef.current;
+      for (let i = 0; i < groupNames.length; i += 1) {
+        const value = values[groupNames[i]];
+        if (value === void 0) continue;
+        const adr = indices[i];
+        if (adr >= 0) data.ctrl[adr] = value;
+      }
+      return true;
+    },
+    [canWrite, mjDataRef]
+  );
+  const read = useCallback(() => {
+    const data = mjDataRef.current;
+    const groupNames = namesRef.current;
+    const indices = indicesRef.current;
+    const result = {};
+    for (let i = 0; i < groupNames.length; i += 1) {
+      const adr = indices[i];
+      result[groupNames[i]] = data && adr >= 0 ? data.ctrl[adr] ?? 0 : 0;
+    }
+    return result;
+  }, [mjDataRef]);
+  return useMemo(
+    () => ({ owner, set, read, canWrite, release }),
+    [owner, set, read, canWrite, release]
+  );
+}
 var geomNameCacheByModel2 = /* @__PURE__ */ new WeakMap();
 var bodyNameCacheByModel = /* @__PURE__ */ new WeakMap();
 function getCachedName(cacheByModel, model, id, address) {
@@ -7822,6 +8120,130 @@ function usePolicyCameraFramesFromMountedStreams(defaultOptions) {
     reset
   };
 }
+function sessionSignature(stream) {
+  return JSON.stringify({
+    width: stream.width,
+    height: stream.height,
+    channels: stream.channels,
+    renderIsolation: stream.renderIsolation ?? false,
+    cameraName: stream.cameraName,
+    siteName: stream.siteName,
+    bodyName: stream.bodyName
+  });
+}
+function addTensorAliases(tensors, stream, tensor, includeObservationImageAliases) {
+  const keys = /* @__PURE__ */ new Set([stream.key, ...stream.aliases ?? []]);
+  if (includeObservationImageAliases) {
+    for (const base of [stream.key, ...stream.aliases ?? []]) {
+      keys.add(`observation.images.${base}`);
+    }
+  }
+  for (const key of keys) tensors[key] = tensor;
+}
+function usePolicyCameraTensors(options) {
+  const mujoco = useMujoco();
+  const [status, setStatus] = useState("idle");
+  const [error, setError] = useState(null);
+  const sessionsRef = useRef(/* @__PURE__ */ new Map());
+  const disposeSessions = useCallback(() => {
+    for (const { session } of sessionsRef.current.values()) session.dispose();
+    sessionsRef.current.clear();
+  }, []);
+  useEffect(() => disposeSessions, [disposeSessions]);
+  const reset = useCallback(() => {
+    setStatus("idle");
+    setError(null);
+  }, []);
+  const capture = useCallback(() => {
+    const api = mujoco.api;
+    if (!api) {
+      throw new Error("MuJoCo scene is not ready for policy camera tensor capture.");
+    }
+    setStatus("capturing");
+    setError(null);
+    try {
+      const sessions = sessionsRef.current;
+      const seen = /* @__PURE__ */ new Set();
+      const tensors = {};
+      const sourceParts = [];
+      for (const stream of options.streams) {
+        seen.add(stream.key);
+        const resolved = {
+          ...api.resolveCameraCaptureOptions(stream),
+          channels: stream.channels,
+          layout: stream.layout,
+          range: stream.range
+        };
+        const signature = sessionSignature(stream);
+        let entry = sessions.get(stream.key);
+        if (!entry || entry.signature !== signature) {
+          entry?.session.dispose();
+          entry = {
+            session: api.createCameraFrameCaptureSession(resolved),
+            signature
+          };
+          sessions.set(stream.key, entry);
+        }
+        const tensor = entry.session.captureTensor(resolved);
+        addTensorAliases(
+          tensors,
+          stream,
+          tensor,
+          options.includeObservationImageAliases ?? false
+        );
+        sourceParts.push(`${stream.key}:${tensor.source.kind}`);
+      }
+      for (const key of [...sessions.keys()]) {
+        if (!seen.has(key)) {
+          sessions.get(key)?.session.dispose();
+          sessions.delete(key);
+        }
+      }
+      setStatus("captured");
+      return {
+        tensors,
+        sourceSummary: sourceParts.join(" + ") || "not used by policy",
+        capturedAt: Date.now()
+      };
+    } catch (nextError) {
+      const captureError = nextError instanceof Error ? nextError : new Error("Unable to capture policy camera tensors.");
+      setError(captureError);
+      setStatus("error");
+      throw captureError;
+    }
+  }, [mujoco.api, options.includeObservationImageAliases, options.streams]);
+  return {
+    status,
+    error,
+    isCapturing: status === "capturing",
+    capture,
+    reset
+  };
+}
+function usePolicyCameraTensorsFromMountedStreams(options) {
+  const mujoco = useMujoco();
+  const tensorOptions = options.tensor;
+  const mountedOptions = useMemo(() => {
+    const api = mujoco.api;
+    if (!api) {
+      return {
+        streams: [],
+        includeObservationImageAliases: options.includeObservationImageAliases ?? false
+      };
+    }
+    const plan = createPolicyCameraFrameCapturePlanFromApi(api, options);
+    return {
+      streams: plan.streams.map(({ key, aliases, ...stream }) => ({
+        ...stream,
+        ...tensorOptions,
+        key,
+        aliases
+      })),
+      includeObservationImageAliases: plan.includeObservationImageAliases ?? false
+    };
+  }, [mujoco.api, options, tensorOptions]);
+  return usePolicyCameraTensors(mountedOptions);
+}
 function useCameraSequenceRecorder() {
   const mujoco = useMujoco();
   const [status, setStatus] = useState("idle");
@@ -8126,6 +8548,19 @@ function useCameraAnimation() {
  * @license
  * SPDX-License-Identifier: Apache-2.0
 */
+/**
+ * @license
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Live on-screen camera viewports for MuJoCo scenes. Each view renders the
+ * shared scene from a named MuJoCo camera/site/body into a `gl.scissor` region
+ * tracking a DOM element — no GPU readback, no PNG encoding.
+ *
+ * While at least one viewport is mounted the canvas switches from R3F's
+ * automatic render to a managed render loop (main scene full-frame, then each
+ * viewport). This is incompatible with `EffectComposer`/postprocessing or other
+ * custom render loops; use the offscreen capture APIs in those setups instead.
+ */
 /**
  * @license
  * SPDX-License-Identifier: Apache-2.0
@@ -8174,6 +8609,21 @@ function useCameraAnimation() {
  * light_type: 0 = directional, 1 = spot (maps to mjLIGHT_DIRECTIONAL/mjLIGHT_SPOT)
  * Note: light_directional does NOT exist in WASM — use light_type instead.
  */
+/**
+ * @license
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Stream a live MuJoCo camera into a DOM `<canvas>`. Each frame the scene is
+ * rendered offscreen from the selected camera and blitted into the canvas, so
+ * it composites normally in the DOM (works inside opaque panels) and does NOT
+ * take over the render loop. Prefer this over `useCameraViewport` for camera
+ * tiles embedded in HTML UI; use `useCameraViewport` for transparent overlays
+ * on a full-bleed canvas.
+ *
+ * Uses the async capture path so Gaussian-splat environments render through
+ * their dedicated capture renderer — streaming a splat scene at full rate does
+ * not disturb the main view's splat sort.
+ */
 /**
  * @license
  * SPDX-License-Identifier: Apache-2.0
@@ -8259,6 +8709,14 @@ function useCameraAnimation() {
  *
  * Cooperative actuator/control ownership for policies, IK, teleop, and replay.
  */
+/**
+ * @license
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Typed, named control groups. Define the group once (like a zod schema or a
+ * tanstack route), then read/set actuators by name with inferred keys — no
+ * positional arrays, no `as const`, no owner bookkeeping.
+ */
 /**
  * @license
  * SPDX-License-Identifier: Apache-2.0
@@ -8331,6 +8789,15 @@ function useCameraAnimation() {
  *
  * React wrapper for capturing policy image payloads from Three/MuJoCo cameras.
  */
+/**
+ * @license
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Capture policy observation tensors directly from Three/MuJoCo cameras,
+ * skipping the data-URL/PNG round-trip. Sessions are created once per camera
+ * and reused every step, so live inference and dataset recording read straight
+ * from the GPU into Float32 tensors.
+ */
 /**
  * @license
  * SPDX-License-Identifier: Apache-2.0
@@ -8379,6 +8846,6 @@ function useCameraAnimation() {
  * useCameraAnimation — composable camera animation hook.
  */
-export { Body, ContactListener, ContactMarkers, Debug, DragInteraction, FlexRenderer, IkGizmo, InstancedGeomRenderer, MountedCameraFrameSequenceManifestStatus, MountedCameraFrameSequenceReadinessStatus, MountedCameraFrameSourceSuggestionMatch, MujocoCanvas, MujocoPhysics, MujocoProvider, MujocoSimProvider, SceneLights, SplatCollisionProxyPreview, TendonRenderer, TrajectoryPlayer, applyPolicyActionToControls, bodyPositionField, buildObservation, canFetchSplatCollisionProxyXml, captureFrame, captureFrameBlob, capturePolicyCameraFrames, capturePolicyCameraFramesFromMountedStreams, clampPolicyActionValue, createContiguousControlGroup, createController, createControllerHook, createMountedCameraFrameSequenceManifest, createMountedCameraFrameSequencePlan, createMountedCameraFrameSequencePlanFromApi, createMountedCameraFrameSequenceReadiness, createMountedCameraFrameSourceSuggestions, createNamedObservationBuilder, createPolicyCameraFrameCapturePlan, createPolicyCameraFrameCapturePlanFromApi, ctrlField, fetchSplatCollisionProxyXml, findActuatorByName, findBodyByName, findGeomByName, findJointByName, findKeyframeByName, findSensorByName, findSiteByName, findTendonByName, geomPositionField, getActuatedJoints, getCameraFrameCaptureSourceTarget, getControlMap, getMountedCameraFrameCaptureSource, getName, imagePointToNdc, isMountedCameraFrameCaptureSource, loadScene, parseSplatCollisionProxyGeoms, projectImagePointTo3D, qposField, qvelField, readNamedObservation, recordMountedCameraFrameSequence, resolveControlGroup, resolveMountedCameraFrameSource, sitePositionField, useActuators, useAfterPhysicsStep, useBeforePhysicsStep, useBodyMeshes, useBodyPose, useBodyState, useCameraAnimation, useCameraFrameCapture, useCameraSequenceRecorder, useContactEvents, useContactHistory, useContacts, useControlWriter, useCtrl, useCtrlNoise, useFrameCapture, useGamepad, useGeomPose, useGravityCompensation, useIkController, useJointState, useKeyboardIkTarget, useKeyboardTeleop, useMountedCameraSequenceRecorder, useMujoco, useMujocoWasm, useNamedObservation, useObservation, usePolicy, usePolicyCameraFrames, usePolicyCameraFramesFromMountedStreams, useRemotePolicy, useSceneLights, useSelectionHighlight, useSensor, useSensors, useSitePose, useSitePosition, useSplatCollisionProxyGeoms, useTrajectoryPlayer, useTrajectoryRecorder, useVideoRecorder };
+export { Body, CameraView, ContactListener, ContactMarkers, Debug, DragInteraction, FlexRenderer, GenericIK, IkGizmo, InstancedGeomRenderer, MountedCameraFrameSequenceManifestStatus, MountedCameraFrameSequenceReadinessStatus, MountedCameraFrameSourceSuggestionMatch, MujocoCanvas, MujocoPhysics, MujocoProvider, MujocoSimProvider, SceneLights, SplatCollisionProxyPreview, TendonRenderer, TrajectoryPlayer, applyPolicyActionToControls, bodyPositionField, buildObservation, canFetchSplatCollisionProxyXml, captureFrame, captureFrameBlob, capturePolicyCameraFrames, capturePolicyCameraFramesFromMountedStreams, clampPolicyActionValue, controlGroup, createContiguousControlGroup, createController, createControllerHook, createMountedCameraFrameSequenceManifest, createMountedCameraFrameSequencePlan, createMountedCameraFrameSequencePlanFromApi, createMountedCameraFrameSequenceReadiness, createMountedCameraFrameSourceSuggestions, createNamedObservationBuilder, createPolicyCameraFrameCapturePlan, createPolicyCameraFrameCapturePlanFromApi, ctrlField, fetchSplatCollisionProxyXml, findActuatorByName, findBodyByName, findGeomByName, findJointByName, findKeyframeByName, findSensorByName, findSiteByName, findTendonByName, geomPositionField, getActuatedJoints, getCameraFrameCaptureSourceTarget, getControlMap, getMountedCameraFrameCaptureSource, getName, imagePointToNdc, isMountedCameraFrameCaptureSource, loadScene, parseSplatCollisionProxyGeoms, projectImagePointTo3D, qposField, qvelField, readNamedObservation, recordMountedCameraFrameSequence, resolveControlGroup, resolveMountedCameraFrameSource, sitePositionField, useActuators, useAfterPhysicsStep, useBeforePhysicsStep, useBodyMeshes, useBodyPose, useBodyState, useCameraAnimation, useCameraFrameCapture, useCameraSequenceRecorder, useCameraStream, useCameraViewport, useContactEvents, useContactHistory, useContacts, useControlGroup, useControlWriter, useCtrl, useCtrlNoise, useFrameCapture, useGamepad, useGeomPose, useGravityCompensation, useIkController, useJointState, useKeyboardIkTarget, useKeyboardTeleop, useMountedCameraSequenceRecorder, useMujoco, useMujocoWasm, useNamedObservation, useObservation, usePolicy, usePolicyCameraFrames, usePolicyCameraFramesFromMountedStreams, usePolicyCameraTensors, usePolicyCameraTensorsFromMountedStreams, useRemotePolicy, useSceneLights, useSelectionHighlight, useSensor, useSensors, useSitePose, useSitePosition, useSplatCollisionProxyGeoms, useTrajectoryPlayer, useTrajectoryRecorder, useVideoRecorder };
 //# sourceMappingURL=index.js.map
 //# sourceMappingURL=index.js.map