@simarena/viewport 0.0.1-alpha.0 → 0.0.1-alpha.1

package/src/viewer/scene.ts

@@ -0,0 +1,253 @@
+import * as THREE from "three";
+import { LidarViz } from "../sensors/index.js";
+import { applyTransforms, buildBodyIdx } from "../sync.js";
+import type { SimScene as SimSceneMsg } from "../types.js";
+import { ContactViz } from "../sensors/contact.js";
+import type { SimCamera, VisualScene } from "./types.js";
+
+/** The subset of state that SimScene needs for per-frame visualization. */
+export interface VizState {
+  showContacts: boolean;
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  contacts: any[];
+  sensordata: Record<string, number[]> | null;
+}
+
+/**
+ * SimScene — Three.js renderer for simulation visualization.
+ *
+ * Lifecycle:
+ *   1. new SimScene(scene)
+ *   2. setBodies(map)               — register body objects
+ *   3. init(msg, visual)            — build bodyIdx, proxy meshes, lidars, thumb cameras
+ *   4. Per frame: applyFrame() → updateViz() → renderThumbnails()
+ *   5. dispose()
+ */
+export class SimScene {
+  static readonly THUMB_W = 160;
+  static readonly THUMB_H = 100;
+
+  #scene: THREE.Scene;
+  #bodies: Map<string, THREE.Object3D> = new Map();
+  #bodyIdx: Map<string, number> = new Map();
+  #proxies: THREE.Object3D[] = [];
+  #lidars: LidarViz[] = [];
+  #contactViz: ContactViz;
+  #thumbCams: {
+    name: string;
+    cam: THREE.PerspectiveCamera;
+    entityName?: string;
+    lookAtName?: string;
+  }[] = [];
+  #cameras: SimCamera[] = [];
+  #thumbTarget: THREE.WebGLRenderTarget | null = null;
+
+  /** Config list — updated on init(). */
+  thumbConfigs: { key: string; label: string }[] = [];
+
+  constructor(scene: THREE.Scene) {
+    this.#scene = scene;
+    this.#contactViz = new ContactViz(scene);
+  }
+
+  // ── Body management ────────────────────────────────────────────────────────
+
+  setBodies(bodies: Map<string, THREE.Object3D>): void {
+    this.#bodies = bodies;
+  }
+
+  get bodies(): Map<string, THREE.Object3D> {
+    return this.#bodies;
+  }
+
+  // ── Lifecycle ──────────────────────────────────────────────────────────────
+
+  init(msg: SimSceneMsg, visual: VisualScene): void {
+    this.#bodyIdx.clear();
+    buildBodyIdx(msg.bodies.map((b) => b.name)).forEach((v, k) => this.#bodyIdx.set(k, v));
+    this.#cameras = visual.cameras;
+    this.#initThumbCams(visual.cameras);
+    this.#createProxies(visual);
+    this.#createLidars(visual);
+  }
+
+  // ── Per-frame updates ──────────────────────────────────────────────────────
+
+  applyFrame(xpos: Float32Array, xquat: Float32Array): void {
+    applyTransforms(xpos, xquat, this.#bodyIdx, this.#bodies);
+  }
+
+  updateViz(state: VizState): void {
+    this.#syncThumbCams();
+    this.#contactViz.visible = state.showContacts;
+    this.#contactViz.update(state.contacts as Parameters<ContactViz["update"]>[0]);
+    for (const viz of this.#lidars) viz.update(state.sensordata?.[viz.name] ?? null);
+  }
+
+  // ── Thumbnail cameras ──────────────────────────────────────────────────────
+
+  renderThumbnails(
+    renderer: THREE.WebGLRenderer,
+    ctxs: (CanvasRenderingContext2D | null)[],
+    running: boolean,
+    time: number,
+  ): void {
+    if (!running || !this.#thumbCams.length) return;
+    if (Math.round(time * 50) % 3 !== 0) return;
+    const W = SimScene.THUMB_W;
+    const H = SimScene.THUMB_H;
+    const target = this.#getThumbTarget();
+    const prev = renderer.outputColorSpace;
+    renderer.outputColorSpace = THREE.SRGBColorSpace;
+    for (let i = 0; i < this.#thumbCams.length; i++) {
+      const ctx2d = ctxs[i];
+      if (!ctx2d) continue;
+      renderer.setRenderTarget(target);
+      renderer.render(this.#scene, this.#thumbCams[i]!.cam);
+      renderer.setRenderTarget(null);
+      const buf = new Uint8Array(W * H * 4);
+      renderer.readRenderTargetPixels(target, 0, 0, W, H, buf);
+      const img = ctx2d.createImageData(W, H);
+      for (let y = 0; y < H; y++) {
+        const src = (H - 1 - y) * W * 4;
+        img.data.set(buf.subarray(src, src + W * 4), y * W * 4);
+      }
+      ctx2d.putImageData(img, 0, 0);
+    }
+    renderer.outputColorSpace = prev;
+  }
+
+  getThumbCam(key: string): THREE.PerspectiveCamera | null {
+    return this.#thumbCams.find((c) => c.name === key)?.cam ?? null;
+  }
+
+  getThumbConfig(key: string): SimCamera | null {
+    return this.#cameras.find((c) => c.name === key) ?? null;
+  }
+
+  // ── Dispose ────────────────────────────────────────────────────────────────
+
+  dispose(): void {
+    for (const viz of this.#lidars) {
+      this.#scene.remove(viz.group);
+      viz.dispose();
+    }
+    this.#lidars.length = 0;
+
+    for (const obj of this.#proxies) {
+      this.#scene.remove(obj);
+      obj.traverse((c) => {
+        if (c instanceof THREE.Mesh) {
+          c.geometry.dispose();
+          (c.material as THREE.Material).dispose();
+        }
+      });
+    }
+    this.#proxies.length = 0;
+
+    this.#contactViz.dispose();
+    this.#thumbCams = [];
+    this.#cameras = [];
+    this.thumbConfigs = [];
+    this.#thumbTarget?.dispose();
+    this.#thumbTarget = null;
+    this.#bodyIdx.clear();
+  }
+
+  // ── Private ────────────────────────────────────────────────────────────────
+
+  #getThumbTarget(): THREE.WebGLRenderTarget {
+    if (!this.#thumbTarget) {
+      this.#thumbTarget = new THREE.WebGLRenderTarget(SimScene.THUMB_W, SimScene.THUMB_H, {
+        minFilter: THREE.LinearFilter,
+        magFilter: THREE.LinearFilter,
+        format: THREE.RGBAFormat,
+        type: THREE.UnsignedByteType,
+      });
+      this.#thumbTarget.texture.colorSpace = THREE.SRGBColorSpace;
+    }
+    return this.#thumbTarget;
+  }
+
+  #syncThumbCams(): void {
+    for (const tc of this.#thumbCams) {
+      if (!tc.entityName) continue;
+      const obj = this.#bodies.get(tc.entityName);
+      if (obj) {
+        tc.cam.position.copy(obj.position);
+        tc.cam.quaternion.copy(obj.quaternion);
+      }
+      if (tc.lookAtName) {
+        const target = this.#bodies.get(tc.lookAtName);
+        if (target) tc.cam.lookAt(target.position);
+      }
+    }
+  }
+
+  #initThumbCams(cameras: SimCamera[]): void {
+    const W = SimScene.THUMB_W;
+    const H = SimScene.THUMB_H;
+    this.#thumbCams = cameras.map((c) => {
+      const cam = new THREE.PerspectiveCamera(c.fov, W / H, 0.1, 100);
+      cam.position.set(...c.position);
+      cam.lookAt(...c.lookAt);
+      return { name: c.name, cam, entityName: c.entityName, lookAtName: c.lookAtName };
+    });
+    this.thumbConfigs = this.#thumbCams.map((c) => ({ key: c.name, label: c.name }));
+  }
+
+  #createProxies(visual: VisualScene): void {
+    for (const body of visual.bodies) {
+      if (body.kind !== "rigid") continue;
+      if (this.#bodies.has(body.name)) continue;
+      const grp = new THREE.Group();
+      grp.position.set(...body.pos);
+      if (body.quat) {
+        const [w, x, y, z] = body.quat;
+        grp.quaternion.set(x, y, z, w);
+      }
+      for (const geom of body.geoms) {
+        if (geom.type === "box") {
+          const [sx, sy, sz] = geom.size as [number, number, number];
+          const mesh = new THREE.Mesh(
+            new THREE.BoxGeometry(sx, sy, sz),
+            new THREE.MeshStandardMaterial({
+              color: 0x1f1f3e,
+              roughness: 0.7,
+              metalness: 0.2,
+              transparent: true,
+              opacity: 0.9,
+            }),
+          );
+          grp.add(mesh);
+        }
+        // TODO: add sphere, cylinder, capsule, mesh (GLB) handling
+      }
+      if (grp.children.length > 0) {
+        this.#scene.add(grp);
+        this.#bodies.set(body.name, grp);
+        this.#proxies.push(grp);
+      }
+    }
+  }
+
+  #createLidars(visual: VisualScene): void {
+    for (const s of visual.sensors) {
+      if (s.type !== "lidar") continue;
+      const body = this.#bodies.get(s.parentBody ?? s.name);
+      if (!body) continue;
+      const ch = s.channels ?? 1;
+      const viz = new LidarViz(s.name, body, {
+        pattern: ch > 1 ? "3d" : "2d",
+        channels: ch,
+        horizontalSamples: s.horizontalSamples ?? 120,
+        horizontalFov: s.horizontalFov ?? [-180, 180],
+        verticalFov: s.verticalFov as [number, number] | undefined,
+        range: s.range ?? [0.1, 10],
+        height: 0.3,
+      });
+      this.#scene.add(viz.group);
+      this.#lidars.push(viz);
+    }
+  }
+}

package/src/viewer/types.ts

@@ -0,0 +1,193 @@
+/**
+ * VisualScene — engine-agnostic visual description of a scene.
+ *
+ * Designed to be consumed by any renderer (Three.js, wgpu-rs headless, etc.).
+ * Produced by @simarena/format (SceneDoc → VisualScene) or any sim engine
+ * (MuJoCo, Genesis, Newton, USD, ...).
+ *
+ * Asset tables (materials, textures, meshes) deduplicate shared resources.
+ * GeomDesc references them by name; renderers build GPU resources once per
+ * unique material/mesh and batch draw calls accordingly.
+ */
+
+// ── Asset tables ──────────────────────────────────────────────────────────────
+
+/** PBR material — engine-agnostic, all values 0-1. */
+export interface MaterialDesc {
+  color?: [number, number, number, number]; // RGBA 0-1
+  roughness?: number;
+  metalness?: number;
+  texture?: string; // ref to VisualScene.textures
+}
+
+// ── Geometry ──────────────────────────────────────────────────────────────────
+
+/**
+ * Geometry primitive.
+ * size convention: box=[w,h,d] | sphere=[r] | cylinder=[r,h] | capsule=[r,h] | plane=[w,d] | mesh=[]
+ */
+export interface GeomDesc {
+  type: string; // 'box' | 'sphere' | 'capsule' | 'cylinder' | 'plane' | 'mesh'
+  size: number[];
+  pos?: [number, number, number];
+  rot?: [number, number, number]; // Euler degrees XYZ
+  quat?: [number, number, number, number]; // WXYZ
+  material?: string | MaterialDesc; // name ref OR inline MaterialDesc
+  mesh?: string; // name ref to VisualScene.meshes
+  meshUrl?: string; // direct URL (convenience; prefer mesh ref)
+}
+
+// ── Bodies ────────────────────────────────────────────────────────────────────
+
+/** Rigid body — updated per frame via transform (pos + quat). */
+export interface RigidBodyDesc {
+  kind: "rigid";
+  name: string;
+  pos: [number, number, number];
+  rot?: [number, number, number];
+  quat?: [number, number, number, number];
+  geoms: GeomDesc[];
+  parent?: string;
+}
+
+/** Deformable body — updated per frame via per-vertex positions. */
+export interface SoftBodyDesc {
+  kind: "soft";
+  name: string;
+  /** Total vertex count — pre-allocate GPU buffers. */
+  vertexCount: number;
+  /** Initial topology mesh (GLB/OBJ). Triangle indices stay fixed. */
+  mesh?: string; // ref to VisualScene.meshes
+  meshUrl?: string; // direct URL fallback
+  /** Inline triangle indices (alternative to mesh ref). */
+  indices?: number[];
+  material?: string | MaterialDesc;
+  doubleSided?: boolean;
+}
+
+/** Bone-driven mesh — GPU skeletal animation. One draw call; efficient for articulated robots. */
+export interface SkinnedBodyDesc {
+  kind: "skinned";
+  name: string;
+  /** GLB with embedded skeleton and skin weights. */
+  mesh?: string;
+  meshUrl?: string;
+  /** Number of bones — pre-allocate bone transform buffer. */
+  boneCount: number;
+  material?: string | MaterialDesc;
+}
+
+/** GPU-instanced copies of the same shape. One draw call for `count` copies. */
+export interface InstancedBodyDesc {
+  kind: "instanced";
+  name: string;
+  /** Number of instances. */
+  count: number;
+  /** Shared geometry for all instances. */
+  geom: GeomDesc;
+  /** Per-instance color overrides (optional). */
+  colors?: [number, number, number, number][];
+}
+
+export type BodyDesc = RigidBodyDesc | SoftBodyDesc | SkinnedBodyDesc | InstancedBodyDesc;
+
+// ── Lights ────────────────────────────────────────────────────────────────────
+
+export type LightType = "point" | "directional" | "spot" | "ambient" | "hemisphere";
+
+export interface LightDesc {
+  type: LightType;
+  name: string;
+  pos?: [number, number, number];
+  color?: [number, number, number]; // RGB 0-1
+  groundColor?: [number, number, number]; // hemisphere bottom color
+  intensity?: number;
+  castShadow?: boolean;
+  dir?: [number, number, number];
+  range?: number;
+  angle?: number;
+}
+
+// ── Cameras ───────────────────────────────────────────────────────────────────
+
+export interface SimCamera {
+  name: string;
+  fov: number;
+  position: [number, number, number];
+  lookAt: [number, number, number];
+  entityName?: string;
+  lookAtName?: string;
+}
+
+// ── Sensors ───────────────────────────────────────────────────────────────────
+
+export interface SensorDesc {
+  name: string;
+  type: string; // 'lidar' | 'camera' | ...
+  parentBody?: string;
+  channels?: number;
+  horizontalSamples?: number;
+  horizontalFov?: [number, number];
+  verticalFov?: [number, number];
+  range?: [number, number];
+}
+
+// ── Dynamic elements ──────────────────────────────────────────────────────────
+
+/** Point cloud — MPM particles, granular, fluid, debug markers. */
+export interface ParticleDesc {
+  name: string;
+  material?: string | MaterialDesc;
+  size?: number;
+  sizeAttenuation?: boolean;
+}
+
+// ── Gaussian Splats ───────────────────────────────────────────────────────────
+
+/**
+ * 3D Gaussian Splat scene — photorealistic environments from NeRF/3DGS reconstruction.
+ * Rendered via @sparkjsdev/spark (Three.js-integrated Gaussian rasterizer).
+ * Splat data is loaded from a .ply, .splat, or .spz file.
+ */
+export interface GaussianSplatDesc {
+  name: string;
+  /** URL to .ply / .splat / .spz file. */
+  url?: string;
+  /** Name ref to meshes table. */
+  splat?: string;
+  pos?: [number, number, number];
+  quat?: [number, number, number, number];
+  scale?: number;
+}
+
+// ── Environment ───────────────────────────────────────────────────────────────
+
+export interface EnvironmentDesc {
+  /** Background color [R,G,B] 0-1 or HDR/EXR URL. */
+  background?: [number, number, number] | string;
+  fog?: { color: [number, number, number]; near: number; far: number };
+}
+
+// ── Root ──────────────────────────────────────────────────────────────────────
+
+export interface VisualScene {
+  // ── Shared asset tables ──────────────────────────────────────────────────
+  /** name → PBR material. Geoms reference by name for GPU batching. */
+  materials?: Record<string, MaterialDesc>;
+  /** name → texture URL. Materials reference by name. */
+  textures?: Record<string, string>;
+  /** name → mesh URL (.stl/.obj/.glb). Geoms reference by name. */
+  meshes?: Record<string, string>;
+
+  // ── Scene graph ──────────────────────────────────────────────────────────
+  /** Rigid + soft bodies. Discriminate with body.kind. */
+  bodies: BodyDesc[];
+  lights: LightDesc[];
+  cameras: SimCamera[];
+  sensors: SensorDesc[];
+  /** Per-frame point cloud channels (particles, fluid, granular). */
+  particles?: ParticleDesc[];
+  /** Gaussian Splat environments — loaded via @sparkjsdev/spark. */
+  splats?: GaussianSplatDesc[];
+  environment?: EnvironmentDesc;
+}

package/src/viewport.ts

@@ -1,78 +1,91 @@
-import * as THREE from 'three';
-import { OrbitControls } from 'three/addons/controls/OrbitControls.js';
+import * as THREE from "three";
+import { OrbitControls } from "three/addons/controls/OrbitControls.js";
 
 export interface ViewportOpts {
-	fov?: number;
-	near?: number;
-	far?: number;
-	pos?: [number, number, number];
-	target?: [number, number, number];
-	grid?: number;
+  fov?: number;
+  near?: number;
+  far?: number;
+  pos?: [number, number, number];
+  target?: [number, number, number];
+  grid?: number;
 }
 
 export interface Viewport {
-	scene: THREE.Scene;
-	camera: THREE.PerspectiveCamera;
-	renderer: THREE.WebGLRenderer;
-	orbit: OrbitControls;
-	grid: THREE.GridHelper | null;
-	resize(): void;
-	dispose(): void;
+  scene: THREE.Scene;
+  camera: THREE.PerspectiveCamera;
+  renderer: THREE.WebGLRenderer;
+  orbit: OrbitControls;
+  grid: THREE.GridHelper | null;
+  resize(): void;
+  dispose(): void;
 }
 
 export function createViewport(
-	canvas: HTMLCanvasElement,
-	container: HTMLElement,
-	opts: ViewportOpts = {}
+  canvas: HTMLCanvasElement,
+  container: HTMLElement,
+  opts: ViewportOpts = {},
 ): Viewport {
-	const { fov = 60, near = 0.1, far = 100, pos = [1.5, -1.5, 1.5], target = [0, 0, 0.3], grid = 2 } = opts;
+  const {
+    fov = 60,
+    near = 0.1,
+    far = 100,
+    pos = [1.5, -1.5, 1.5],
+    target = [0, 0, 0.3],
+    grid = 2,
+  } = opts;
 
-	THREE.Object3D.DEFAULT_UP.set(0, 0, 1);
+  THREE.Object3D.DEFAULT_UP.set(0, 0, 1);
 
-	const scene = new THREE.Scene();
+  const scene = new THREE.Scene();
 
-	const camera = new THREE.PerspectiveCamera(fov, container.clientWidth / container.clientHeight, near, far);
-	camera.position.set(...pos);
-	camera.lookAt(...target);
+  const camera = new THREE.PerspectiveCamera(
+    fov,
+    container.clientWidth / container.clientHeight,
+    near,
+    far,
+  );
+  camera.position.set(...pos);
+  camera.lookAt(...target);
 
-	const gl = canvas.getContext('webgl2');
-	const renderer = new THREE.WebGLRenderer({ canvas, context: gl || undefined, antialias: true });
-	renderer.setSize(container.clientWidth, container.clientHeight);
-	renderer.setPixelRatio(Math.min(window.devicePixelRatio, 2));
-	renderer.shadowMap.enabled = true;
-	renderer.shadowMap.type = THREE.PCFSoftShadowMap;
-	renderer.toneMapping = THREE.ACESFilmicToneMapping;
-	renderer.toneMappingExposure = 1.0;
-	renderer.outputColorSpace = THREE.SRGBColorSpace;
+  const gl = canvas.getContext("webgl2");
+  const renderer = new THREE.WebGLRenderer({ canvas, context: gl || undefined, antialias: true });
+  renderer.setSize(container.clientWidth, container.clientHeight);
+  renderer.setPixelRatio(Math.min(window.devicePixelRatio, 2));
+  renderer.shadowMap.enabled = true;
+  renderer.shadowMap.type = THREE.PCFSoftShadowMap;
+  renderer.toneMapping = THREE.ACESFilmicToneMapping;
+  renderer.toneMappingExposure = 1.0;
+  renderer.outputColorSpace = THREE.SRGBColorSpace;
 
-	const orbit = new OrbitControls(camera, renderer.domElement);
-	orbit.enableDamping = true;
-	orbit.dampingFactor = 0.05;
-	orbit.target.set(...target);
-	orbit.update();
+  const orbit = new OrbitControls(camera, renderer.domElement);
+  orbit.enableDamping = true;
+  orbit.dampingFactor = 0.05;
+  orbit.target.set(...target);
+  orbit.update();
 
-	let gridHelper: THREE.GridHelper | null = null;
-	if (grid > 0) {
-		gridHelper = new THREE.GridHelper(grid * 10, 20, 0x444444, 0x222222);
-		gridHelper.rotation.x = Math.PI / 2;
-		scene.add(gridHelper);
-	}
+  let gridHelper: THREE.GridHelper | null = null;
+  if (grid > 0) {
+    gridHelper = new THREE.GridHelper(grid * 10, 20, 0x444444, 0x222222);
+    gridHelper.rotation.x = Math.PI / 2;
+    scene.add(gridHelper);
+  }
 
-	function resize() {
-		const w = container.clientWidth, h = container.clientHeight;
-		camera.aspect = w / h;
-		camera.updateProjectionMatrix();
-		renderer.setSize(w, h);
-	}
+  function resize() {
+    const w = container.clientWidth,
+      h = container.clientHeight;
+    camera.aspect = w / h;
+    camera.updateProjectionMatrix();
+    renderer.setSize(w, h);
+  }
 
-	const obs = new ResizeObserver(resize);
-	obs.observe(container);
+  const obs = new ResizeObserver(resize);
+  obs.observe(container);
 
-	function dispose() {
-		obs.disconnect();
-		renderer.dispose();
-		orbit.dispose();
-	}
+  function dispose() {
+    obs.disconnect();
+    renderer.dispose();
+    orbit.dispose();
+  }
 
-	return { scene, camera, renderer, orbit, grid: gridHelper, resize, dispose };
+  return { scene, camera, renderer, orbit, grid: gridHelper, resize, dispose };
 }