mujoco-react 10.4.0 → 10.5.0

package/dist/onnx.d.ts

@@ -0,0 +1,65 @@
+import * as ort from 'onnxruntime-web';
+import { bq as PolicyActionChunk } from './types-CViUme8D.js';
+import 'react';
+import '@react-three/fiber';
+import 'three';
+
+/**
+ * @license
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Optional ONNX Runtime Web helpers for browser policy demos.
+ *
+ * This entry point is exported as `mujoco-react/onnx` so the main package does
+ * not import or bundle `onnxruntime-web`.
+ */
+
+type OnnxPolicyDtype = 'float32' | 'float64' | 'int32' | 'int64' | 'bool' | string;
+interface OnnxPolicyTensorSpec {
+    name: string;
+    shape: number[];
+    dtype: OnnxPolicyDtype;
+}
+interface OnnxPolicyImageSpec {
+    width: number;
+    height: number;
+    channels?: number;
+    layout?: 'CHW' | 'HWC' | string;
+    range?: readonly [number, number];
+}
+interface OnnxPolicyManifest {
+    model: string;
+    variants?: Record<string, string>;
+    fps?: number;
+    joints?: string[];
+    cameras?: string[];
+    image?: OnnxPolicyImageSpec;
+    chunk_size?: number;
+    n_action_steps?: number;
+    inputs: OnnxPolicyTensorSpec[];
+    output: OnnxPolicyTensorSpec & {
+        units?: string;
+    };
+    [key: string]: unknown;
+}
+interface LoadOnnxPolicyManifestResult<TManifest extends OnnxPolicyManifest = OnnxPolicyManifest> {
+    manifest: TManifest;
+    manifestUrl: URL;
+    modelUrl: URL;
+}
+interface CreateOnnxPolicySessionOptions<TManifest extends OnnxPolicyManifest = OnnxPolicyManifest> {
+    manifestUrl: string | URL;
+    variant?: string;
+    runtime: typeof ort;
+    sessionOptions?: ort.InferenceSession.SessionOptions;
+    fetcher?: typeof fetch;
+    readManifest?: (response: Response) => Promise<TManifest>;
+}
+interface OnnxPolicySession<TManifest extends OnnxPolicyManifest = OnnxPolicyManifest> extends LoadOnnxPolicyManifestResult<TManifest> {
+    session: ort.InferenceSession;
+}
+declare function loadOnnxPolicyManifest<TManifest extends OnnxPolicyManifest = OnnxPolicyManifest>(manifestUrlInput: string | URL, options?: Pick<CreateOnnxPolicySessionOptions<TManifest>, 'variant' | 'fetcher' | 'readManifest'>): Promise<LoadOnnxPolicyManifestResult<TManifest>>;
+declare function createOnnxPolicySession<TManifest extends OnnxPolicyManifest = OnnxPolicyManifest>(options: CreateOnnxPolicySessionOptions<TManifest>): Promise<OnnxPolicySession<TManifest>>;
+declare function onnxTensorToPolicyActionChunk(tensor: ort.Tensor, actionSize?: number, maxActions?: number): PolicyActionChunk;
+
+export { type CreateOnnxPolicySessionOptions, type LoadOnnxPolicyManifestResult, type OnnxPolicyDtype, type OnnxPolicyImageSpec, type OnnxPolicyManifest, type OnnxPolicySession, type OnnxPolicyTensorSpec, createOnnxPolicySession, loadOnnxPolicyManifest, onnxTensorToPolicyActionChunk };

package/dist/onnx.js

@@ -0,0 +1,58 @@
+// src/onnx.ts
+function asUrl(value, base = globalThis.location?.href) {
+  return value instanceof URL ? value : new URL(value, base);
+}
+function resolveModelPath(manifest, variant) {
+  if (variant && manifest.variants?.[variant]) return manifest.variants[variant];
+  return manifest.model;
+}
+async function loadOnnxPolicyManifest(manifestUrlInput, options = {}) {
+  const fetcher = options.fetcher ?? fetch;
+  const manifestUrl = asUrl(manifestUrlInput);
+  const response = await fetcher(manifestUrl);
+  if (!response.ok) {
+    throw new Error(`Unable to load ONNX policy manifest from ${manifestUrl.href} (${response.status}).`);
+  }
+  const manifest = options.readManifest ? await options.readManifest(response) : await response.json();
+  const modelPath = resolveModelPath(manifest, options.variant);
+  const modelUrl = asUrl(modelPath, manifestUrl.href);
+  return { manifest, manifestUrl, modelUrl };
+}
+async function createOnnxPolicySession(options) {
+  const fetcher = options.fetcher ?? fetch;
+  const resolved = await loadOnnxPolicyManifest(options.manifestUrl, options);
+  const response = await fetcher(resolved.modelUrl);
+  if (!response.ok) {
+    throw new Error(`Unable to load ONNX policy model from ${resolved.modelUrl.href} (${response.status}).`);
+  }
+  const modelBytes = await response.arrayBuffer();
+  const session = await options.runtime.InferenceSession.create(modelBytes, options.sessionOptions);
+  return {
+    ...resolved,
+    session
+  };
+}
+function onnxTensorToPolicyActionChunk(tensor, actionSize = tensor.dims.at(-1) ?? 1, maxActions) {
+  const rawData = Array.from(tensor.data, (value) => Number(value));
+  const actionCount = Math.floor(rawData.length / actionSize);
+  const cappedActionCount = maxActions === void 0 ? actionCount : Math.max(0, Math.min(actionCount, Math.floor(maxActions)));
+  const actions = [];
+  for (let actionIndex = 0; actionIndex < cappedActionCount; actionIndex += 1) {
+    const start = actionIndex * actionSize;
+    actions.push(rawData.slice(start, start + actionSize));
+  }
+  return actions;
+}
+/**
+ * @license
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Optional ONNX Runtime Web helpers for browser policy demos.
+ *
+ * This entry point is exported as `mujoco-react/onnx` so the main package does
+ * not import or bundle `onnxruntime-web`.
+ */
+
+export { createOnnxPolicySession, loadOnnxPolicyManifest, onnxTensorToPolicyActionChunk };
+//# sourceMappingURL=onnx.js.map
+//# sourceMappingURL=onnx.js.map

package/dist/onnx.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/onnx.ts"],"names":[],"mappings":";AAiEA,SAAS,KAAA,CAAM,KAAA,EAAqB,IAAA,GAAO,UAAA,CAAW,UAAU,IAAA,EAAM;AACpE,EAAA,OAAO,iBAAiB,GAAA,GAAM,KAAA,GAAQ,IAAI,GAAA,CAAI,OAAO,IAAI,CAAA;AAC3D;AAEA,SAAS,gBAAA,CAAiB,UAA8B,OAAA,EAA6B;AACnF,EAAA,IAAI,OAAA,IAAW,SAAS,QAAA,GAAW,OAAO,GAAG,OAAO,QAAA,CAAS,SAAS,OAAO,CAAA;AAC7E,EAAA,OAAO,QAAA,CAAS,KAAA;AAClB;AAEA,eAAsB,sBAAA,CACpB,gBAAA,EACA,OAAA,GAAmG,EAAC,EAClD;AAClD,EAAA,MAAM,OAAA,GAAU,QAAQ,OAAA,IAAW,KAAA;AACnC,EAAA,MAAM,WAAA,GAAc,MAAM,gBAAgB,CAAA;AAC1C,EAAA,MAAM,QAAA,GAAW,MAAM,OAAA,CAAQ,WAAW,CAAA;AAC1C,EAAA,IAAI,CAAC,SAAS,EAAA,EAAI;AAChB,IAAA,MAAM,IAAI,MAAM,CAAA,yCAAA,EAA4C,WAAA,CAAY,IAAI,CAAA,EAAA,EAAK,QAAA,CAAS,MAAM,CAAA,EAAA,CAAI,CAAA;AAAA,EACtG;AACA,EAAA,MAAM,QAAA,GAAW,OAAA,CAAQ,YAAA,GACrB,MAAM,OAAA,CAAQ,aAAa,QAAQ,CAAA,GACnC,MAAM,QAAA,CAAS,IAAA,EAAK;AACxB,EAAA,MAAM,SAAA,GAAY,gBAAA,CAAiB,QAAA,EAAU,OAAA,CAAQ,OAAO,CAAA;AAC5D,EAAA,MAAM,QAAA,GAAW,KAAA,CAAM,SAAA,EAAW,WAAA,CAAY,IAAI,CAAA;AAClD,EAAA,OAAO,EAAE,QAAA,EAAU,WAAA,EAAa,QAAA,EAAS;AAC3C;AAEA,eAAsB,wBACpB,OAAA,EACuC;AACvC,EAAA,MAAM,OAAA,GAAU,QAAQ,OAAA,IAAW,KAAA;AACnC,EAAA,MAAM,QAAA,GAAW,MAAM,sBAAA,CAAuB,OAAA,CAAQ,aAAa,OAAO,CAAA;AAC1E,EAAA,MAAM,QAAA,GAAW,MAAM,OAAA,CAAQ,QAAA,CAAS,QAAQ,CAAA;AAChD,EAAA,IAAI,CAAC,SAAS,EAAA,EAAI;AAChB,IAAA,MAAM,IAAI,MAAM,CAAA,sCAAA,EAAyC,QAAA,CAAS,SAAS,IAAI,CAAA,EAAA,EAAK,QAAA,CAAS,MAAM,CAAA,EAAA,CAAI,CAAA;AAAA,EACzG;AACA,EAAA,MAAM,UAAA,GAAa,MAAM,QAAA,CAAS,WAAA,EAAY;AAC9C,EAAA,MAAM,OAAA,GAAU,MAAM,OAAA,CAAQ,OAAA,CAAQ,iBAAiB,MAAA,CAAO,UAAA,EAAY,QAAQ,cAAc,CAAA;AAChG,EAAA,OAAO;AAAA,IACL,GAAG,QAAA;AAAA,IACH;AAAA,GACF;AACF;AAEO,SAAS,6BAAA,CACd,QACA,UAAA,GAAa,MAAA,CAAO,KAAK,EAAA,CAAG,EAAE,CAAA,IAAK,CAAA,EACnC,UAAA,EACmB;AACnB,EAAA,MAAM,OAAA,GAAU,MAAM,IAAA,CAAK,MAAA,CAAO,MAA2B,CAAC,KAAA,KAAU,MAAA,CAAO,KAAK,CAAC,CAAA;AACrF,EAAA,MAAM,WAAA,GAAc,IAAA,CAAK,KAAA,CAAM,OAAA,CAAQ,SAAS,UAAU,CAAA;AAC1D,EAAA,MAAM,iBAAA,GAAoB,UAAA,KAAe,MAAA,GACrC,WAAA,GACA,KAAK,GAAA,CAAI,CAAA,EAAG,IAAA,CAAK,GAAA,CAAI,WAAA,EAAa,IAAA,CAAK,KAAA,CAAM,UAAU,CAAC,CAAC,CAAA;AAC7D,EAAA,MAAM,UAAsB,EAAC;AAC7B,EAAA,KAAA,IAAS,WAAA,GAAc,CAAA,EAAG,WAAA,GAAc,iBAAA,EAAmB,eAAe,CAAA,EAAG;AAC3E,IAAA,MAAM,QAAQ,WAAA,GAAc,UAAA;AAC5B,IAAA,OAAA,CAAQ,KAAK,OAAA,CAAQ,KAAA,CAAM,KAAA,EAAO,KAAA,GAAQ,UAAU,CAAC,CAAA;AAAA,EACvD;AACA,EAAA,OAAO,OAAA;AACT","file":"onnx.js","sourcesContent":["/**\n * @license\n * SPDX-License-Identifier: Apache-2.0\n *\n * Optional ONNX Runtime Web helpers for browser policy demos.\n *\n * This entry point is exported as `mujoco-react/onnx` so the main package does\n * not import or bundle `onnxruntime-web`.\n */\n\nimport type * as ort from 'onnxruntime-web';\nimport type { PolicyActionChunk } from './types';\n\nexport type OnnxPolicyDtype = 'float32' | 'float64' | 'int32' | 'int64' | 'bool' | string;\n\nexport interface OnnxPolicyTensorSpec {\n  name: string;\n  shape: number[];\n  dtype: OnnxPolicyDtype;\n}\n\nexport interface OnnxPolicyImageSpec {\n  width: number;\n  height: number;\n  channels?: number;\n  layout?: 'CHW' | 'HWC' | string;\n  range?: readonly [number, number];\n}\n\nexport interface OnnxPolicyManifest {\n  model: string;\n  variants?: Record<string, string>;\n  fps?: number;\n  joints?: string[];\n  cameras?: string[];\n  image?: OnnxPolicyImageSpec;\n  chunk_size?: number;\n  n_action_steps?: number;\n  inputs: OnnxPolicyTensorSpec[];\n  output: OnnxPolicyTensorSpec & {\n    units?: string;\n  };\n  [key: string]: unknown;\n}\n\nexport interface LoadOnnxPolicyManifestResult<TManifest extends OnnxPolicyManifest = OnnxPolicyManifest> {\n  manifest: TManifest;\n  manifestUrl: URL;\n  modelUrl: URL;\n}\n\nexport interface CreateOnnxPolicySessionOptions<TManifest extends OnnxPolicyManifest = OnnxPolicyManifest> {\n  manifestUrl: string | URL;\n  variant?: string;\n  runtime: typeof ort;\n  sessionOptions?: ort.InferenceSession.SessionOptions;\n  fetcher?: typeof fetch;\n  readManifest?: (response: Response) => Promise<TManifest>;\n}\n\nexport interface OnnxPolicySession<TManifest extends OnnxPolicyManifest = OnnxPolicyManifest>\n  extends LoadOnnxPolicyManifestResult<TManifest> {\n  session: ort.InferenceSession;\n}\n\nfunction asUrl(value: string | URL, base = globalThis.location?.href) {\n  return value instanceof URL ? value : new URL(value, base);\n}\n\nfunction resolveModelPath(manifest: OnnxPolicyManifest, variant: string | undefined) {\n  if (variant && manifest.variants?.[variant]) return manifest.variants[variant];\n  return manifest.model;\n}\n\nexport async function loadOnnxPolicyManifest<TManifest extends OnnxPolicyManifest = OnnxPolicyManifest>(\n  manifestUrlInput: string | URL,\n  options: Pick<CreateOnnxPolicySessionOptions<TManifest>, 'variant' | 'fetcher' | 'readManifest'> = {}\n): Promise<LoadOnnxPolicyManifestResult<TManifest>> {\n  const fetcher = options.fetcher ?? fetch;\n  const manifestUrl = asUrl(manifestUrlInput);\n  const response = await fetcher(manifestUrl);\n  if (!response.ok) {\n    throw new Error(`Unable to load ONNX policy manifest from ${manifestUrl.href} (${response.status}).`);\n  }\n  const manifest = options.readManifest\n    ? await options.readManifest(response)\n    : await response.json() as TManifest;\n  const modelPath = resolveModelPath(manifest, options.variant);\n  const modelUrl = asUrl(modelPath, manifestUrl.href);\n  return { manifest, manifestUrl, modelUrl };\n}\n\nexport async function createOnnxPolicySession<TManifest extends OnnxPolicyManifest = OnnxPolicyManifest>(\n  options: CreateOnnxPolicySessionOptions<TManifest>\n): Promise<OnnxPolicySession<TManifest>> {\n  const fetcher = options.fetcher ?? fetch;\n  const resolved = await loadOnnxPolicyManifest(options.manifestUrl, options);\n  const response = await fetcher(resolved.modelUrl);\n  if (!response.ok) {\n    throw new Error(`Unable to load ONNX policy model from ${resolved.modelUrl.href} (${response.status}).`);\n  }\n  const modelBytes = await response.arrayBuffer();\n  const session = await options.runtime.InferenceSession.create(modelBytes, options.sessionOptions);\n  return {\n    ...resolved,\n    session,\n  };\n}\n\nexport function onnxTensorToPolicyActionChunk(\n  tensor: ort.Tensor,\n  actionSize = tensor.dims.at(-1) ?? 1,\n  maxActions?: number\n): PolicyActionChunk {\n  const rawData = Array.from(tensor.data as ArrayLike<number>, (value) => Number(value));\n  const actionCount = Math.floor(rawData.length / actionSize);\n  const cappedActionCount = maxActions === undefined\n    ? actionCount\n    : Math.max(0, Math.min(actionCount, Math.floor(maxActions)));\n  const actions: number[][] = [];\n  for (let actionIndex = 0; actionIndex < cappedActionCount; actionIndex += 1) {\n    const start = actionIndex * actionSize;\n    actions.push(rawData.slice(start, start + actionSize));\n  }\n  return actions;\n}\n"]}

package/dist/spark.d.ts

@@ -1,6 +1,6 @@
 import * as react_jsx_runtime from 'react/jsx-runtime';
 import * as _sparkjsdev_spark from '@sparkjsdev/spark';
-import { o as SplatEnvironmentProps, r as PairedSplatEnvironmentConfig, S as SceneConfig, u as SplatEnvironmentReadiness, p as VisualScenarioConfig } from './types-CdFZCYmy.js';
+import { o as SplatEnvironmentProps, r as PairedSplatEnvironmentConfig, S as SceneConfig, u as SplatEnvironmentReadiness, p as VisualScenarioConfig } from './types-CViUme8D.js';
 import 'react';
 import '@react-three/fiber';
 import 'three';

package/dist/spark.js

@@ -1,4 +1,4 @@
-import { useSplatSceneConfig, useSplatEnvironment, SplatEnvironment, CAMERA_FRAME_CAPTURE_RENDER_USER_DATA_KEY } from './chunk-FBXXXPLQ.js';
+import { useSplatSceneConfig, useSplatEnvironment, SplatEnvironment, CAMERA_FRAME_CAPTURE_RENDER_USER_DATA_KEY } from './chunk-KHZ5U36J.js';
 import { useThree } from '@react-three/fiber';
 import { useMemo, useState, useEffect, useCallback, useRef } from 'react';
 import * as THREE from 'three';

package/dist/{types-CdFZCYmy.d.ts → types-CViUme8D.d.ts}

@@ -2,6 +2,123 @@ import React__default, { ReactNode } from 'react';
 import { CanvasProps, ThreeElements } from '@react-three/fiber';
 import * as THREE from 'three';
 
+/**
+ * @license
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Helpers for turning browser camera captures into policy image tensors.
+ */
+type PolicyImageTensorLayout = 'CHW' | 'HWC';
+type PolicyImageTensorRange = readonly [number, number];
+/**
+ * Row order of a raw pixel buffer. WebGL `readRenderTargetPixels` returns rows
+ * bottom-to-top (`'bottom-left'`); `ImageData` is top-to-bottom (`'top-left'`).
+ */
+type PolicyImageTensorSourceOrigin = 'top-left' | 'bottom-left';
+interface PolicyImageTensorOptions {
+    width: number;
+    height: number;
+    channels?: 3 | 4;
+    layout?: PolicyImageTensorLayout;
+    range?: PolicyImageTensorRange;
+}
+interface PolicyImageTensorPixelOptions extends PolicyImageTensorOptions {
+    /** Row order of the source buffer. Defaults to `'top-left'`. */
+    sourceOrigin?: PolicyImageTensorSourceOrigin;
+    /** Mirror horizontally while reading. */
+    flipX?: boolean;
+}
+interface PolicyImageTensorResult {
+    data: Float32Array;
+    shape: [number, number, number];
+    width: number;
+    height: number;
+    channels: 3 | 4;
+    layout: PolicyImageTensorLayout;
+    range: PolicyImageTensorRange;
+}
+/**
+ * Convert a raw RGBA pixel buffer (4 bytes per pixel) directly into a policy
+ * image tensor. This is the fast path that skips canvas encoding entirely —
+ * feed it the `Uint8Array` returned by `readRenderTargetPixels` (which is
+ * bottom-left origin, so pass `sourceOrigin: 'bottom-left'`).
+ */
+declare function pixelsToPolicyImageTensor(pixels: Uint8Array | Uint8ClampedArray, options: PolicyImageTensorPixelOptions): PolicyImageTensorResult;
+declare function imageDataToPolicyImageTensor(imageData: ImageData, options: PolicyImageTensorOptions): PolicyImageTensorResult;
+declare function dataUrlToPolicyImageTensor(dataUrl: string, options: PolicyImageTensorOptions): Promise<PolicyImageTensorResult>;
+
+/**
+ * @license
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Offscreen camera-frame capture for R3F/MuJoCo scenes.
+ */
+
+/** Options for capturing a camera frame straight into a policy image tensor. */
+type CameraFrameCaptureTensorOptions = CameraFrameCaptureOptions & Pick<PolicyImageTensorOptions, 'channels' | 'layout' | 'range'>;
+interface CameraFramePixelsResult {
+    /** Raw RGBA pixels, bottom-left origin (reused buffer — consume before the next capture). */
+    pixels: Uint8Array;
+    camera: THREE.Camera;
+    width: number;
+    height: number;
+    source: CameraFrameCaptureSource;
+}
+interface CameraFrameTensorResult extends PolicyImageTensorResult {
+    camera: THREE.Camera;
+    source: CameraFrameCaptureSource;
+}
+interface CameraFrameCaptureSession {
+    readonly width: number;
+    readonly height: number;
+    capture(options?: CameraFrameCaptureOptions): {
+        canvas: HTMLCanvasElement;
+        camera: THREE.Camera;
+        width: number;
+        height: number;
+        source: CameraFrameCaptureSource;
+    };
+    captureAsync(options?: CameraFrameCaptureOptions): Promise<{
+        canvas: HTMLCanvasElement;
+        camera: THREE.Camera;
+        width: number;
+        height: number;
+        source: CameraFrameCaptureSource;
+    }>;
+    captureDataUrl(options?: CameraFrameCaptureOptions): CameraFrameCaptureResult;
+    captureDataUrlAsync(options?: CameraFrameCaptureOptions): Promise<CameraFrameCaptureResult>;
+    captureBlob(options?: CameraFrameCaptureOptions): Promise<CameraFrameCaptureBlobResult>;
+    /**
+     * Render and read raw RGBA pixels without any canvas/PNG round-trip. The
+     * returned buffer is reused between calls — copy or convert it before the
+     * next capture.
+     */
+    capturePixels(options?: CameraFrameCaptureOptions): CameraFramePixelsResult;
+    /** Render straight into a normalized policy image tensor (no canvas/PNG encode). */
+    captureTensor(options?: CameraFrameCaptureTensorOptions): CameraFrameTensorResult;
+    dispose(): void;
+}
+declare const CAMERA_FRAME_CAPTURE_RENDER_USER_DATA_KEY = "mujocoReactCameraFrameCaptureRender";
+declare const CAMERA_FRAME_CAPTURE_PRE_RENDER_USER_DATA_KEY = "mujocoReactCameraFrameCapturePreRender";
+declare const CAPTURE_EXCLUDE_KEY = "mujoco.capture.exclude";
+declare function createCameraFrameCaptureSession(renderer: THREE.WebGLRenderer, scene: THREE.Scene, fallbackCamera: THREE.Camera, options?: CameraFrameCaptureOptions): CameraFrameCaptureSession;
+declare function renderCameraFrameToCanvas(renderer: THREE.WebGLRenderer, scene: THREE.Scene, fallbackCamera: THREE.Camera, options?: CameraFrameCaptureOptions): {
+    canvas: HTMLCanvasElement;
+    camera: THREE.Camera;
+    width: number;
+    height: number;
+    source: CameraFrameCaptureSource;
+};
+declare function captureCameraFrame(renderer: THREE.WebGLRenderer, scene: THREE.Scene, fallbackCamera: THREE.Camera, options?: CameraFrameCaptureOptions): Promise<CameraFrameCaptureResult>;
+declare function captureCameraFrameBlob(renderer: THREE.WebGLRenderer, scene: THREE.Scene, fallbackCamera: THREE.Camera, options?: CameraFrameCaptureOptions): Promise<CameraFrameCaptureBlobResult>;
+/**
+ * One-shot camera frame capture straight into a policy image tensor, skipping
+ * the canvas/PNG round-trip. For repeated captures (live inference, recording),
+ * create a session once with {@link createCameraFrameCaptureSession} and call
+ * `session.captureTensor()` so the render target and buffers are reused.
+ */
+declare function captureCameraFrameTensor(renderer: THREE.WebGLRenderer, scene: THREE.Scene, fallbackCamera: THREE.Camera, options?: CameraFrameCaptureTensorOptions): CameraFrameTensorResult;
+
 /**
  * @license
  * SPDX-License-Identifier: Apache-2.0
@@ -336,6 +453,10 @@ interface SceneObject {
     solref?: string;
     solimp?: string;
     condim?: number;
+    /** MuJoCo geom contact type bitmask. Defaults to 1 for generated objects. */
+    contype?: number;
+    /** MuJoCo geom contact affinity bitmask. Defaults to 1 for generated objects. */
+    conaffinity?: number;
     /** MuJoCo geom group. Group 3 is conventionally used for collision-only helper geoms. */
     group?: number;
 }
@@ -392,6 +513,12 @@ interface IkConfig {
      * starting at index 0. Prefer inferred IK or `joints`/`actuators`.
      */
     numJoints?: number;
+    /**
+     * Optional solve-space joint limits in the same order as the resolved joints.
+     * Use this when MJCF limits are intentionally broad or when a setup/calibration
+     * tool should stay within a narrower envelope.
+     */
+    jointLimits?: ReadonlyArray<readonly [number, number] | null | undefined>;
     /** Custom IK solver. When omitted, uses built-in Damped Least-Squares solver. */
     ikSolveFn?: IKSolveFn;
     /** DLS damping. Default: 0.01. */
@@ -1141,6 +1268,19 @@ interface MujocoSimAPI {
     captureFrameBlob(options?: MujocoFrameCaptureOptions): Promise<FrameCaptureBlobResult>;
     captureCameraFrame(options?: CameraFrameCaptureOptions): Promise<CameraFrameCaptureResult>;
     captureCameraFrameBlob(options?: CameraFrameCaptureOptions): Promise<CameraFrameCaptureBlobResult>;
+    /** Capture a camera frame straight into a policy image tensor (no canvas/PNG encode). */
+    captureCameraFrameTensor(options?: CameraFrameCaptureTensorOptions): CameraFrameTensorResult;
+    /**
+     * Create a reusable offscreen capture session bound to this scene. Reuse it
+     * for live inference/recording so the render target and buffers persist
+     * across frames; call `session.captureTensor()` / `capturePixels()` each step.
+     */
+    createCameraFrameCaptureSession(options?: CameraFrameCaptureOptions): CameraFrameCaptureSession;
+    /**
+     * Resolve a named MuJoCo camera/site/body into concrete capture options with
+     * the current world pose. Useful for re-aiming a persistent session each step.
+     */
+    resolveCameraCaptureOptions(options?: CameraFrameCaptureOptions): CameraFrameCaptureOptions;
     recordCameraSequence(options: CameraFrameSequenceOptions): Promise<CameraFrameSequenceResult>;
     project2DTo3D(x: number, y: number, cameraPos: THREE.Vector3, lookAt: THREE.Vector3): {
         point: THREE.Vector3;
@@ -1498,4 +1638,4 @@ interface ArrayJointStateResult {
     velocity: React__default.RefObject<Float64Array>;
 }
 
-export { type ScalarJointStateResult as $, type ActuatedJointInfo as A, type BodyProps as B, type ControlGroupInfo as C, type DragInteractionProps as D, type VisualScenarioExecutionContext as E, type ScenarioLightingPreset as F, type SplatEnvironmentMetadataInput as G, type SplatEnvironmentMetadata as H, type IkConfig as I, type SplatSceneInput as J, type DebugProps as K, type GeomInfo as L, type MujocoContextValue as M, type ContactListenerProps as N, type ObservationConfig as O, type PhysicsStepCallback as P, type ActuatorInfo as Q, type ReadyCallbackInput as R, type SceneConfig as S, type TrajectoryPlayerProps as T, type Sites as U, type VisualScenarioEffectsProps as V, type SitePositionResult as W, type Sensors as X, type SensorHandle as Y, type SensorInfo as Z, type Joints as _, type MujocoCanvasProps as a, type KeyboardIkTargetBinding as a$, type ArrayJointStateResult as a0, type JointStateOptions as a1, type JointStateResult as a2, type Bodies as a3, type BodyStateResult as a4, type Geoms as a5, type Actuators as a6, type CtrlHandle as a7, type ContactInfo as a8, type KeyboardTeleopConfig as a9, type PolicyCameraFrameCaptureAPI as aA, type CameraFrameSequenceRecorderAPI as aB, type CameraFrameCaptureResult as aC, type CameraFrameCaptureBlobResult as aD, type ImagePointCoordinateSpace as aE, type ImagePointProjectionOptions as aF, type ImagePointProjectionResult as aG, type PolicyVector as aH, type BodyInfo as aI, type CameraFrameCaptureQuaternion as aJ, type CameraFrameCaptureVector3 as aK, type CameraFrameSequenceCameraSummary as aL, type CameraFrameSequenceFrame as aM, type CameraFrameSequenceSampleInput as aN, type CameraFrameSequenceStepInput as aO, type CameraInfo as aP, type ControlJointInfo as aQ, type DebugVirtualCamera as aR, type FrameCaptureTarget as aS, type FrameCaptureTargetRef as aT, type IKSolveFn as aU, type IkGizmoDragInput as aV, type IkSolveInput as aW, type JointInfo as aX, type JointStateKind as aY, type KeyBinding as aZ, type KeyboardIkTargetAction as a_, type KeyboardIkTargetConfig as aa, type PolicyConfig as ab, type PolicyAPI as ac, type RemotePolicyConfig as ad, type RemotePolicyAPI as ae, type ObservationHandle as af, type ObservationOutput as ag, type TrajectoryInput as ah, type TrajectoryStateChangeInput as ai, type PlaybackState as aj, type TrajectoryFrame as ak, type FrameCaptureOptions as al, type FrameCaptureResult as am, type FrameCaptureBlobResult as an, type FrameCaptureAPI as ao, type CameraFrameCaptureOptions as ap, type CameraFrameCaptureAPI as aq, type Cameras as ar, type CameraFrameSequenceCamera as as, type CameraFrameCaptureSource as at, type CameraFrameSequenceOptions as au, type CameraFrameSequenceResult as av, type PolicyCameraFrameStream as aw, type PolicyCameraFrameCaptureOptions as ax, type PolicyCameraFrameCaptureResult as ay, type FrameCaptureStatus as az, type MujocoSimAPI as b, type Keyframes as b0, ModelActuators as b1, ModelBodies as b2, ModelCameras as b3, ModelGeoms as b4, ModelJoints as b5, ModelKeyframes as b6, type ModelOptions as b7, type ModelResource as b8, ModelResources as b9, type ScenarioCameraConfig as bA, type ScenarioMaterialConfig as bB, type SceneMarker as bC, type SceneObject as bD, type SensorResult as bE, type SiteInfo as bF, type SplatAssetConfig as bG, type SplatScenarioConfig as bH, type StateSnapshot as bI, type TrajectoryData as bJ, type TrajectoryFrameCallbackInput as bK, type VisualScenarioMaterialFilterInput as bL, type XmlPatch as bM, getContact as bN, registerModelResources as bO, ModelSensors as ba, ModelSites as bb, type Models as bc, type MujocoContact as bd, type MujocoContactArray as be, type MujocoFrameCaptureOptions as bf, type ObservationLayoutItem as bg, type PhysicsConfig as bh, type PhysicsStepInput as bi, type PolicyActionChunk as bj, type PolicyActionInput as bk, type PolicyInferenceInput as bl, type PolicyInferenceOutput as bm, type PolicyInferenceResult as bn, type PolicyObservationInput as bo, type RayHit as bp, type Register as bq, type RegisteredModelMap as br, type RemotePolicyRequestInfo as bs, type RemotePolicyRequestInput as bt, type RemotePolicyResponseInfo as bu, type RemotePolicyStatus as bv, type ResetCallbackInput as bw, type ResolvedScenarioCameraConfig as bx, type ResolvedScenarioMaterialConfig as by, type ResourceSelector as bz, type StepCallbackInput as c, type SelectionCallbackInput as d, type MujocoModule as e, type MujocoRenderOptions as f, type MujocoModel as g, type MujocoData as h, type ControlGroupSelector as i, type ObservationResult as j, type IkContextValue as k, type IkGizmoProps as l, type SceneLightsProps as m, type ScenarioLightingProps as n, type SplatEnvironmentProps as o, type VisualScenarioConfig as p, type SplatRendererKind as q, type PairedSplatEnvironmentConfig as r, type SplatFormat as s, type SplatCollisionProxyConfig as t, type SplatEnvironmentReadiness as u, type SplatCollisionPrimitive as v, SplatEnvironmentReadinessStatus as w, type SplatSceneConfigInput as x, type SplatSceneConfigState as y, type VisualScenarioExecutionContextInput as z };
+export { type Joints as $, type ActuatedJointInfo as A, type BodyProps as B, type ControlGroupInfo as C, type DragInteractionProps as D, type VisualScenarioExecutionContext as E, type ScenarioLightingPreset as F, type SplatEnvironmentMetadataInput as G, type SplatEnvironmentMetadata as H, type IkConfig as I, type SplatSceneInput as J, type CameraFrameCaptureOptions as K, type DebugProps as L, type MujocoContextValue as M, type GeomInfo as N, type ObservationConfig as O, type PhysicsStepCallback as P, type ContactListenerProps as Q, type ReadyCallbackInput as R, type SceneConfig as S, type TrajectoryPlayerProps as T, type ActuatorInfo as U, type VisualScenarioEffectsProps as V, type Sites as W, type SitePositionResult as X, type Sensors as Y, type SensorHandle as Z, type SensorInfo as _, type MujocoCanvasProps as a, type IKSolveFn as a$, type ScalarJointStateResult as a0, type ArrayJointStateResult as a1, type JointStateOptions as a2, type JointStateResult as a3, type Bodies as a4, type BodyStateResult as a5, type Geoms as a6, type Actuators as a7, type CtrlHandle as a8, type ContactInfo as a9, type PolicyCameraFrameCaptureAPI as aA, type CameraFrameCaptureTensorOptions as aB, type CameraFrameTensorResult as aC, type CameraFrameSequenceRecorderAPI as aD, type ImagePointCoordinateSpace as aE, type ImagePointProjectionOptions as aF, type ImagePointProjectionResult as aG, type PolicyVector as aH, type BodyInfo as aI, CAMERA_FRAME_CAPTURE_PRE_RENDER_USER_DATA_KEY as aJ, CAMERA_FRAME_CAPTURE_RENDER_USER_DATA_KEY as aK, CAPTURE_EXCLUDE_KEY as aL, type CameraFrameCaptureBlobResult as aM, type CameraFrameCaptureQuaternion as aN, type CameraFrameCaptureResult as aO, type CameraFrameCaptureSession as aP, type CameraFrameCaptureVector3 as aQ, type CameraFramePixelsResult as aR, type CameraFrameSequenceCameraSummary as aS, type CameraFrameSequenceFrame as aT, type CameraFrameSequenceSampleInput as aU, type CameraFrameSequenceStepInput as aV, type CameraInfo as aW, type ControlJointInfo as aX, type DebugVirtualCamera as aY, type FrameCaptureTarget as aZ, type FrameCaptureTargetRef as a_, type KeyboardTeleopConfig as aa, type KeyboardIkTargetConfig as ab, type PolicyConfig as ac, type PolicyAPI as ad, type RemotePolicyConfig as ae, type RemotePolicyAPI as af, type ObservationHandle as ag, type ObservationOutput as ah, type TrajectoryInput as ai, type TrajectoryStateChangeInput as aj, type PlaybackState as ak, type TrajectoryFrame as al, type FrameCaptureOptions as am, type FrameCaptureResult as an, type FrameCaptureBlobResult as ao, type FrameCaptureAPI as ap, type CameraFrameCaptureAPI as aq, type Cameras as ar, type CameraFrameSequenceCamera as as, type CameraFrameCaptureSource as at, type CameraFrameSequenceOptions as au, type CameraFrameSequenceResult as av, type PolicyCameraFrameStream as aw, type PolicyCameraFrameCaptureOptions as ax, type PolicyCameraFrameCaptureResult as ay, type FrameCaptureStatus as az, type MujocoSimAPI as b, captureCameraFrameBlob as b$, type IkGizmoDragInput as b0, type IkSolveInput as b1, type JointInfo as b2, type JointStateKind as b3, type KeyBinding as b4, type KeyboardIkTargetAction as b5, type KeyboardIkTargetBinding as b6, type Keyframes as b7, ModelActuators as b8, ModelBodies as b9, type PolicyInferenceResult as bA, type PolicyObservationInput as bB, type RayHit as bC, type Register as bD, type RegisteredModelMap as bE, type RemotePolicyRequestInfo as bF, type RemotePolicyRequestInput as bG, type RemotePolicyResponseInfo as bH, type RemotePolicyStatus as bI, type ResetCallbackInput as bJ, type ResolvedScenarioCameraConfig as bK, type ResolvedScenarioMaterialConfig as bL, type ResourceSelector as bM, type ScenarioCameraConfig as bN, type ScenarioMaterialConfig as bO, type SceneMarker as bP, type SceneObject as bQ, type SensorResult as bR, type SiteInfo as bS, type SplatAssetConfig as bT, type SplatScenarioConfig as bU, type StateSnapshot as bV, type TrajectoryData as bW, type TrajectoryFrameCallbackInput as bX, type VisualScenarioMaterialFilterInput as bY, type XmlPatch as bZ, captureCameraFrame as b_, ModelCameras as ba, ModelGeoms as bb, ModelJoints as bc, ModelKeyframes as bd, type ModelOptions as be, type ModelResource as bf, ModelResources as bg, ModelSensors as bh, ModelSites as bi, type Models as bj, type MujocoContact as bk, type MujocoContactArray as bl, type MujocoFrameCaptureOptions as bm, type ObservationLayoutItem as bn, type PhysicsConfig as bo, type PhysicsStepInput as bp, type PolicyActionChunk as bq, type PolicyActionInput as br, type PolicyImageTensorLayout as bs, type PolicyImageTensorOptions as bt, type PolicyImageTensorPixelOptions as bu, type PolicyImageTensorRange as bv, type PolicyImageTensorResult as bw, type PolicyImageTensorSourceOrigin as bx, type PolicyInferenceInput as by, type PolicyInferenceOutput as bz, type StepCallbackInput as c, captureCameraFrameTensor as c0, createCameraFrameCaptureSession as c1, dataUrlToPolicyImageTensor as c2, getContact as c3, imageDataToPolicyImageTensor as c4, pixelsToPolicyImageTensor as c5, registerModelResources as c6, renderCameraFrameToCanvas as c7, type SelectionCallbackInput as d, type MujocoModule as e, type MujocoRenderOptions as f, type MujocoModel as g, type MujocoData as h, type ControlGroupSelector as i, type ObservationResult as j, type IkContextValue as k, type IkGizmoProps as l, type SceneLightsProps as m, type ScenarioLightingProps as n, type SplatEnvironmentProps as o, type VisualScenarioConfig as p, type SplatRendererKind as q, type PairedSplatEnvironmentConfig as r, type SplatFormat as s, type SplatCollisionProxyConfig as t, type SplatEnvironmentReadiness as u, type SplatCollisionPrimitive as v, SplatEnvironmentReadinessStatus as w, type SplatSceneConfigInput as x, type SplatSceneConfigState as y, type VisualScenarioExecutionContextInput as z };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mujoco-react",
-  "version": "10.4.0",
+  "version": "10.5.0",
   "description": "Composable React Three Fiber building blocks for MuJoCo WASM simulations",
   "type": "module",
   "main": "./dist/index.js",
@@ -25,7 +25,13 @@
         "default": "./dist/spark.js"
       }
     },
-    "./package.json": "./package.json"
+    "./package.json": "./package.json",
+    "./onnx": {
+      "import": {
+        "types": "./dist/onnx.d.ts",
+        "default": "./dist/onnx.js"
+      }
+    }
   },
   "bin": {
     "mujoco-react": "./bin/mujoco-react.mjs",
@@ -63,11 +69,15 @@
     "@react-three/fiber": ">=8",
     "@sparkjsdev/spark": ">=2.1.0",
     "react": ">=18",
-    "three": ">=0.180.0"
+    "three": ">=0.180.0",
+    "onnxruntime-web": ">=1.20.0"
   },
   "peerDependenciesMeta": {
     "@sparkjsdev/spark": {
       "optional": true
+    },
+    "onnxruntime-web": {
+      "optional": true
     }
   },
   "dependencies": {
@@ -82,6 +92,7 @@
     "@types/node": "^25.9.1",
     "@types/react": "^19.0.0",
     "@types/three": "^0.181.0",
+    "onnxruntime-web": "^1.27.0",
     "react": "^19.2.0",
     "semantic-release": "^25.0.3",
     "three": "^0.181.0",

package/src/components/CameraView.tsx

@@ -0,0 +1,245 @@
+/**
+ * @license
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Live on-screen camera viewports for MuJoCo scenes. Each view renders the
+ * shared scene from a named MuJoCo camera/site/body into a `gl.scissor` region
+ * tracking a DOM element — no GPU readback, no PNG encoding.
+ *
+ * While at least one viewport is mounted the canvas switches from R3F's
+ * automatic render to a managed render loop (main scene full-frame, then each
+ * viewport). This is incompatible with `EffectComposer`/postprocessing or other
+ * custom render loops; use the offscreen capture APIs in those setups instead.
+ */
+
+import {
+  createContext,
+  type CSSProperties,
+  type MutableRefObject,
+  type ReactNode,
+  type RefObject,
+  useCallback,
+  useContext,
+  useEffect,
+  useMemo,
+  useRef,
+  useState,
+} from 'react';
+import { useFrame, useThree } from '@react-three/fiber';
+import type * as THREE from 'three';
+import { useMujoco } from '../core/MujocoSimProvider';
+import {
+  createCaptureCamera,
+  prepareCaptureCamera,
+} from '../rendering/cameraFrameCapture';
+import type {
+  Bodies,
+  Cameras,
+  CameraFrameCaptureOptions,
+  Sites,
+} from '../types';
+
+/** Camera selection + pose options for a live viewport. */
+export type CameraViewportOptions = Pick<
+  CameraFrameCaptureOptions,
+  | 'camera'
+  | 'cameraName'
+  | 'siteName'
+  | 'bodyName'
+  | 'position'
+  | 'quaternion'
+  | 'lookAt'
+  | 'up'
+  | 'positionOffset'
+  | 'quaternionOffset'
+  | 'fov'
+  | 'near'
+  | 'far'
+  | 'projectionMatrix'
+  | 'mujocoCameraCompatibility'
+>;
+
+interface CameraViewportDescriptor {
+  getElement: () => HTMLElement | null;
+  getOptions: () => CameraViewportOptions;
+  camera: THREE.Camera | null;
+}
+
+interface CameraViewportRegistry {
+  register: (descriptor: CameraViewportDescriptor) => () => void;
+}
+
+const CameraViewportRegistryContext = createContext<CameraViewportRegistry | null>(null);
+
+let nextViewportId = 0;
+
+const VIEWPORT_RENDER_PRIORITY = 1;
+
+function CameraViewportRenderer({
+  viewportsRef,
+}: {
+  viewportsRef: MutableRefObject<Map<number, CameraViewportDescriptor>>;
+}) {
+  const gl = useThree((state) => state.gl);
+  const scene = useThree((state) => state.scene);
+  const mainCamera = useThree((state) => state.camera);
+  const mujoco = useMujoco();
+
+  useFrame(() => {
+    const drawWidth = gl.domElement.width;
+    const drawHeight = gl.domElement.height;
+
+    // We own the render now: draw the main scene full-frame first.
+    gl.setScissorTest(false);
+    gl.setViewport(0, 0, drawWidth, drawHeight);
+    gl.render(scene, mainCamera);
+
+    const api = mujoco.api;
+    if (!api || viewportsRef.current.size === 0) return;
+
+    const dpr = gl.getPixelRatio();
+    const canvasRect = gl.domElement.getBoundingClientRect();
+
+    for (const descriptor of viewportsRef.current.values()) {
+      const element = descriptor.getElement();
+      if (!element) continue;
+
+      const rect = element.getBoundingClientRect();
+      const isOffscreen =
+        rect.bottom < canvasRect.top ||
+        rect.top > canvasRect.bottom ||
+        rect.right < canvasRect.left ||
+        rect.left > canvasRect.right;
+      if (isOffscreen) continue;
+
+      const width = Math.floor(rect.width * dpr);
+      const height = Math.floor(rect.height * dpr);
+      if (width <= 0 || height <= 0) continue;
+
+      const left = Math.floor((rect.left - canvasRect.left) * dpr);
+      const bottom = Math.floor((canvasRect.bottom - rect.bottom) * dpr);
+
+      let resolved: CameraFrameCaptureOptions;
+      try {
+        resolved = api.resolveCameraCaptureOptions(descriptor.getOptions());
+      } catch {
+        continue;
+      }
+
+      if (!descriptor.camera) {
+        descriptor.camera = createCaptureCamera(resolved, mainCamera, width, height);
+      } else {
+        prepareCaptureCamera(descriptor.camera, resolved, mainCamera, width, height);
+      }
+
+      gl.setViewport(left, bottom, width, height);
+      gl.setScissor(left, bottom, width, height);
+      gl.setScissorTest(true);
+      gl.render(scene, descriptor.camera);
+    }
+
+    gl.setScissorTest(false);
+    gl.setViewport(0, 0, drawWidth, drawHeight);
+  }, VIEWPORT_RENDER_PRIORITY);
+
+  return null;
+}
+
+/**
+ * Provides the live-viewport registry and mounts the managed render loop only
+ * while at least one viewport is active. Mounted internally by the MuJoCo
+ * provider; you do not need to add it yourself.
+ */
+export function CameraViewportProvider({ children }: { children?: ReactNode }) {
+  const viewportsRef = useRef<Map<number, CameraViewportDescriptor>>(new Map());
+  const [count, setCount] = useState(0);
+
+  const register = useCallback((descriptor: CameraViewportDescriptor) => {
+    const id = nextViewportId++;
+    viewportsRef.current.set(id, descriptor);
+    setCount((value) => value + 1);
+    return () => {
+      viewportsRef.current.delete(id);
+      setCount((value) => value - 1);
+    };
+  }, []);
+
+  const value = useMemo<CameraViewportRegistry>(() => ({ register }), [register]);
+
+  return (
+    <CameraViewportRegistryContext.Provider value={value}>
+      {children}
+      {count > 0 && <CameraViewportRenderer viewportsRef={viewportsRef} />}
+    </CameraViewportRegistryContext.Provider>
+  );
+}
+
+/**
+ * Render the live MuJoCo scene from a named camera into the region covered by
+ * `elementRef`'s DOM element. Call this inside `<MujocoCanvas>` with a ref to a
+ * DOM element you position anywhere (it does not need to be in the R3F tree).
+ */
+export function useCameraViewport<T extends HTMLElement = HTMLElement>(
+  elementRef: RefObject<T | null>,
+  options: CameraViewportOptions
+) {
+  const registry = useContext(CameraViewportRegistryContext);
+  if (!registry) {
+    throw new Error('useCameraViewport must be used inside <MujocoCanvas>.');
+  }
+
+  const optionsRef = useRef(options);
+  optionsRef.current = options;
+
+  useEffect(() => {
+    const descriptor: CameraViewportDescriptor = {
+      getElement: () => elementRef.current,
+      getOptions: () => optionsRef.current,
+      camera: null,
+    };
+    return registry.register(descriptor);
+  }, [registry, elementRef]);
+}
+
+export interface CameraViewProps extends CameraViewportOptions {
+  className?: string;
+  style?: CSSProperties;
+}
+
+/**
+ * Drop-in live camera pane. Renders an absolutely-positioned overlay `<div>`
+ * over the canvas showing the selected MuJoCo camera. Position it with
+ * `style`/`className` (the canvas's parent should be positioned).
+ */
+export function CameraView({ className, style, ...options }: CameraViewProps) {
+  const gl = useThree((state) => state.gl);
+  const elementRef = useRef<HTMLDivElement | null>(null);
+  if (!elementRef.current && typeof document !== 'undefined') {
+    elementRef.current = document.createElement('div');
+  }
+
+  useEffect(() => {
+    const element = elementRef.current;
+    const parent = gl.domElement.parentElement;
+    if (!element || !parent) return;
+    element.style.position = 'absolute';
+    element.style.overflow = 'hidden';
+    parent.appendChild(element);
+    return () => {
+      parent.removeChild(element);
+    };
+  }, [gl]);
+
+  useEffect(() => {
+    const element = elementRef.current;
+    if (!element) return;
+    element.className = className ?? '';
+    if (style) Object.assign(element.style, style);
+  }, [className, style]);
+
+  useCameraViewport(elementRef, options);
+
+  return null;
+}
+
+export type { Bodies, Cameras, Sites };