@miris-inc/core 0.0.8-08ebd50 → 0.0.8-0ce42f4

package/package.json

@@ -1,41 +1,35 @@
 {
   "name": "@miris-inc/core",
-  "version": "0.0.8-08ebd50",
+  "version": "0.0.8-0ce42f4",
   "description": "Core scene-graph runtime for the Miris 3D streaming SDK. WASM-backed Engine, Client, Scene, Stream, and LOD primitives shared by all higher-level packages.",
-  "license": "Apache-2.0",
-  "homepage": "https://miris.com",
-  "author": "Miris, Inc.",
-  "engines": {
-    "node": ">=18"
-  },
-  "publishConfig": {
-    "access": "public"
-  },
   "keywords": [
-    "miris",
     "3d",
-    "gaussian-splatting",
     "3dgs",
     "4dgs",
+    "gaussian-splatting",
+    "miris",
     "splats",
     "streaming",
     "wasm"
   ],
-  "type": "module",
-  "sideEffects": false,
+  "homepage": "https://miris.com",
+  "license": "Apache-2.0",
+  "author": "Miris, Inc.",
   "files": [
     "banner.svg",
+    "webshark.d.ts",
+    "webshark.js",
+    "webshark.wasm",
     "dist/*.js",
     "dist/*.d.ts",
     "dist/*.wasm",
-    "!dist/*.map",
-    "webshark.js",
-    "webshark.wasm",
-    "webshark.d.ts"
+    "!dist/**/*.map"
   ],
-  "types": "./dist/core.d.ts",
+  "type": "module",
+  "sideEffects": false,
   "main": "./dist/core.prod.js",
   "module": "./dist/core.prod.js",
+  "types": "./dist/core.d.ts",
   "unpkg": "./dist/core.standalone.prod.js",
   "jsdelivr": "./dist/core.standalone.prod.js",
   "exports": {
@@ -46,21 +40,27 @@
       "import": "./dist/core.prod.js",
       "default": "./dist/core.prod.js"
     },
+    "./webshark": {
+      "types": "./webshark.d.ts",
+      "default": "./webshark.js"
+    },
     "./standalone": {
       "types": "./dist/core.d.ts",
       "development": "./dist/core.standalone.js",
       "import": "./dist/core.standalone.prod.js",
       "default": "./dist/core.standalone.prod.js"
-    },
-    "./webshark": {
-      "types": "./webshark.d.ts",
-      "default": "./webshark.js"
     }
   },
+  "publishConfig": {
+    "access": "public"
+  },
   "scripts": {
     "d": "bun dev",
     "dev": "vite",
     "b": "cd ../.. && vite build --mode packages/core",
     "build": "cd ../.. && vite build --mode packages/core"
+  },
+  "engines": {
+    "node": ">=18"
   }
 }

package/webshark.d.ts

@@ -5,38 +5,11 @@ export interface Camera {
   eye: vec4;
   l: number;
   fov: number;
-}
-
-export interface Ellipsoid {
-  pos: vec4;
-  radii: vec4;
-  orientation: vec4;
-}
-
-export interface SphericalHarmonic {
-  col: vec4;
-  sh_0_1: vec4;
-  sh_0_2: vec4;
-  sh_0_3: vec4;
-}
-
-export interface AABB {
-  minExt: vec4;
-  maxExt: vec4;
-}
-
-export interface EmVector<T> {
-  push_back(value: T): void;
-  resize(count: number, value: T): void;
-  size(): number;
-  get(index: number): T | undefined;
-  set(index: number, value: T): boolean;
+  zNear: number;
+  zFar: number;
   delete(): void;
 }
 
-export type EllipsoidVector = EmVector<Ellipsoid>;
-export type SphericalHarmonicVector = EmVector<SphericalHarmonic>;
-
 export interface SortTypeValue {
   value: number;
 }
@@ -45,38 +18,27 @@ export interface TextureFormatValue {
   value: number;
 }
 
-export interface SharkComputeInfo {
-  dispatchCalls: number;
-  frameDispatchCalls: number;
-}
-
 export interface SharkRenderInfo {
-  totalRenderCalls: number;
-  frameRenderCalls: number;
   triangles: number;
-  splats: number;
 }
 
 export interface SharkInfo {
   renderCalls: number;
-  compute: SharkComputeInfo;
   render: SharkRenderInfo;
   reset(): void;
 }
 
 export interface RendererWGPU {
-  configureSurface(width: number, height: number): boolean;
   getProgramCount(): number;
   getTextureCount(): number;
   getEstimatedTextureSize(): number;
-  getBufferCount(): number;
   getTotalBufferSize(): number;
-  getBufferTypeSize(type: number): number;
-  surfaceFormat: TextureFormatValue;
   info: SharkInfo;
+  delete(): void;
 }
 
 export interface RenderState {
+  delete(): void;
   initRenderState(
     renderer: RendererWGPU,
     width: number,
@@ -91,16 +53,34 @@ export interface RenderState {
   initGraphicsState(
     width: number,
     height: number,
-    renderTextureFormat: TextureFormat,
+    renderTextureFormat: TextureFormatValue,
+    depthTextureFormat: TextureFormatValue,
   ): void;
   updateRenderDetails(
     width: number,
     height: number,
     splatTriangles: number,
   ): void;
-  uploadEllipsoids(ellipsoids: EllipsoidVector): void;
-  uploadSphericalHarmonics(sh: SphericalHarmonicVector): void;
-  uploadSphericalHarmonicsSOGs(sh: SphericalHarmonicVector): void;
+  // Decodes Radiance .hdr bytes into roughness-ladder level `level` (0 is the sharp mirror).
+  // hdrPtr must address THIS module's heap - _malloc here and copy the bytes in.
+  // Returns false for bytes that are not a usable Radiance image, or a level out of range.
+  uploadEnvironmentLadderLevelFromHdr(
+    level: number,
+    sigmaDeg: number,
+    hdrPtr: number,
+    hdrSizeBytes: number,
+  ): boolean;
+  // Republish the ladder's shape. Call after any level lands, or it is sampled at whatever shape
+  // the last uploadSplatSetData declared. Pass SplatSet.hasReflectionParams().
+  uploadReflectionLadderParams(reflectionsEnabled: boolean): void;
+  // Debug controls. Drop the reflection term, or replace the surfel colour with one of the layers
+  // it is shaded from, numbered as RenderViewModes.h's AOV_* defines. setRenderView below sets
+  // the AOV together with the rest of a named view.
+  setReflectionsEnabled(enabled: boolean): void;
+  // Render textured gaussians on pixels no surfel covers, resolved as fvdb's open gate does.
+  // Off by default: those pixels show the clear colour.
+  setSurfelLessTextureSplatsEnabled(enabled: boolean): void;
+  setDebugAovMode(mode: number): void;
 }
 
 export interface vec4 {
@@ -116,20 +96,22 @@ export interface SplatSet {
   // Number of splats actually drawn: ordering[0, activeSplats) holds the unmasked splat indices.
   // Set alongside orderingView() to restrict the draw to unmasked octant ranges.
   activeSplats: number;
-  setData(
-    splatCount: number,
-    ellipsoids: number,
-    sphericalHarmonics: number,
-  ): void;
+  // Selects which ellipsoid layout to allocate and what ellipsoidsView() exposes:
+  // packed (16 bytes/splat) or uncompressed (48). Must be set before the first resize.
+  usePackedEllipsoids: boolean;
   // direct-write upload: resize -> write into the views -> finalize.
   // views are valid only until the next resize.
   resize(splatCount: number): void;
   ellipsoidsView(): Uint32Array;
-  sphericalHarmonicsView(): Uint32Array;
+  colorsView(): Uint32Array;
   sh1View(): Uint32Array;
   sh2View(): Uint32Array;
   sh3aView(): Uint32Array;
   sh3bView(): Uint32Array;
+  // mod_depth is a half-float per primitive slot (dense, padded for non-surfels), so this is a
+  // Uint16Array unlike every other view here -- do not address it with the uint32 splat strides
+  // used for ellipsoids/colors/sh*.
+  modDepthView(): Uint16Array;
   // Per-chunk fade (opacity multiplier), one float per chunk, defaults to 1.0.
   // Also sizes the chunk slot to model root slot map below, since both are indexed by chunk slot
   resizeChunkFades(chunkCount: number): void;
@@ -149,88 +131,97 @@ export interface SplatSet {
   modelRootInverseTransformsView(): Float32Array;
   // Sized by resizeChunkFades. Defaults to slot 0 (identity) for every chunk
   chunkModelRootsView(): Int32Array;
+  // resizeModelRootSlots resets every slot to the absolute decode. False for a slot outside the table
+  setModelRootRootRelativeDecode(
+    slot: number,
+    originX: number,
+    originY: number,
+    originZ: number,
+    step: number,
+    lnScaleOffset: number,
+  ): boolean;
   // int32 render-order buffer (sized 2*splatCount after finalize). Valid only until the next resize.
   orderingView(): Int32Array;
   finalize(): void;
   // like finalize() but keeps splats in their delivered (octant-sorted) order — no kd-tree reorder —
   // so sparse-octant range masking can index child-octant ranges directly. RADIX_SORT only.
   finalizeBounds(): void;
+  // Whether the asset carries per-surfel reflectance records, which is what decides whether
+  // reflections are shaded at all. Pass to RenderState.uploadReflectionLadderParams.
+  hasReflectionParams(): boolean;
+  // Widens packed reflectance records - recordVersion 2 for 18-byte reflectanceDataV2, 1 for
+  // 26-byte reflectanceDataV1 - into the per-splat GPU records at bufferOffset. wirePtr must
+  // address THIS module's heap - _malloc here and copy the bytes in, since the attribute lives in
+  // the engine module's. Records cover a node's surfel range, which starts at its first splat.
+  // Returns the records written; 0 if the data was rejected.
+  decodeReflectionParams(
+    recordVersion: number,
+    wirePtr: number,
+    wireSizeBytes: number,
+    elementCount: number,
+    bufferOffset: number,
+    nodeSplatCount: number,
+    logId: number,
+  ): number;
+  // Zeroes records in [startSplat, endSplat). Needed wherever splats are re-packed at fresh
+  // offsets, or a reused slot shades its splat from the previous occupant's material.
+  clearReflectionParams(startSplat: number, endSplat: number): void;
+  // Splats in [0, count) holding a non-zero record. Diagnostic.
+  populatedReflectionParamCount(count: number): number;
+  delete(): void;
 }
 
-export interface WebGPUInternals {
-  jsObjects: unknown[];
-  bufferOnUnmaps: Record<number, unknown>;
-  futures: Promise<unknown>[];
-  jsObjectInsert(ptr: number, obj: unknown): void;
-  futureInsert(futureId: number, promise: Promise<unknown>): void;
+export interface GlassMeshRegistry {
+  // Decodes one octree node's glass payload and registers it, replacing any previous entry for
+  // that object. False for a malformed or empty payload, leaving the registry unchanged.
+  //
+  // The two slot arguments are different numbers here, unlike on the native path, and swapping
+  // them renders glass that does not follow its asset: modelRootSlot scopes the panel dedupe to
+  // one asset instance, while chunkModelRootIndex is stamped into every vertex and read back as
+  // chunkModelRoots[pos.w], so that one takes the chunk's fade slot.
+  //
+  // geometryPtr/panelPtr address THIS module's heap - _malloc here and copy the bytes in, since
+  // the attributes live in the engine module's. The AABB is the geometry attribute's own range.
+  registerObject(
+    objectId: number,
+    lodIndex: number,
+    modelRootSlot: number,
+    chunkModelRootIndex: number,
+    geometryPtr: number,
+    geometryByteCount: number,
+    panelPtr: number,
+    panelByteCount: number,
+    aabbMinX: number,
+    aabbMinY: number,
+    aabbMinZ: number,
+    aabbMaxX: number,
+    aabbMaxY: number,
+    aabbMaxZ: number,
+  ): boolean;
+  // Returns true if the object had glass, which is what says the buffers need rebuilding.
+  removeObject(objectId: number): boolean;
+  clear(): void;
+  isDirty(): boolean;
+  // Rebuilds the flattened buffers, keeping each panel from its deepest resident LOD. Stays dirty
+  // so uploadIfDirty still fires. Walks every registered object, so not per frame.
+  flattenIfDirty(): void;
+  uploadIfDirty(renderState: RenderState): void;
+  vertexCount(): number;
+  panelCount(): number;
+  delete(): void;
 }
 
 export interface WebGPUInterface {
-  Internals: WebGPUInternals;
-
   importJsAdapter(adapter: GPUAdapter, parentPtr?: number): number;
   importJsDevice(device: GPUDevice, parentPtr?: number): number;
-  importJsBindGroup(obj: GPUBindGroup, parentPtr?: number): number;
-  importJsBindGroupLayout(obj: GPUBindGroupLayout, parentPtr?: number): number;
-  importJsBuffer(buffer: GPUBuffer, parentPtr?: number): number;
-  importJsCommandBuffer(obj: GPUCommandBuffer, parentPtr?: number): number;
-  importJsCommandEncoder(obj: GPUCommandEncoder, parentPtr?: number): number;
-  importJsComputePassEncoder(
-    obj: GPUComputePassEncoder,
-    parentPtr?: number,
-  ): number;
-  importJsComputePipeline(obj: GPUComputePipeline, parentPtr?: number): number;
-  importJsPipelineLayout(obj: GPUPipelineLayout, parentPtr?: number): number;
-  importJsQuerySet(obj: GPUQuerySet, parentPtr?: number): number;
-  importJsQueue(obj: GPUQueue, parentPtr?: number): number;
-  importJsRenderBundle(obj: GPURenderBundle, parentPtr?: number): number;
-  importJsRenderBundleEncoder(
-    obj: GPURenderBundleEncoder,
-    parentPtr?: number,
-  ): number;
-  importJsRenderPassEncoder(
-    obj: GPURenderPassEncoder,
-    parentPtr?: number,
-  ): number;
-  importJsRenderPipeline(obj: GPURenderPipeline, parentPtr?: number): number;
-  importJsSampler(obj: GPUSampler, parentPtr?: number): number;
-  importJsShaderModule(obj: GPUShaderModule, parentPtr?: number): number;
   importJsTexture(obj: GPUTexture, parentPtr?: number): number;
-  importJsTextureView(obj: GPUTextureView, parentPtr?: number): number;
-
-  getJsObject(ptr: number): unknown;
-
-  makeStringFromStringView(stringViewPtr: number): string;
-  makeStringFromOptionalStringView(
-    stringViewPtr: number,
-  ): string | undefined | "";
-  setStringView(ptr: number, data: number, length: number): void;
-  makeColor(ptr: number): { r: number; g: number; b: number; a: number };
-  makeExtent3D(ptr: number): {
-    width: number;
-    height: number;
-    depthOrArrayLayers: number;
-  };
-  makeOrigin3D(ptr: number): { x: number; y: number; z: number };
-  fillLimitStruct(limits: GPUSupportedLimits, limitsOutPtr: number): void;
-  fillAdapterInfoStruct(adapterInfo: GPUAdapterInfo, infoStruct: number): void;
-  errorCallback(
-    callback: number,
-    type: number,
-    message: string,
-    userdata: number,
-  ): void;
 }
 
 export interface MainModule {
   Camera: new () => Camera;
   RenderState: new () => RenderState;
   SplatSet: new () => SplatSet;
-  AABB: new () => AABB;
-  Ellipsoid: new () => Ellipsoid;
-  SphericalHarmonic: new () => SphericalHarmonic;
-  EllipsoidVector: new () => EllipsoidVector;
-  SphericalHarmonicVector: new () => SphericalHarmonicVector;
+  GlassMeshRegistry: new () => GlassMeshRegistry;
   vec4: new (x: number, y: number, z: number, w: number) => vec4;
 
   SortType: {
@@ -243,6 +234,7 @@ export interface MainModule {
   TextureFormat: {
     RGBA32_Float: TextureFormatValue;
     RGBA16_Float: TextureFormatValue;
+    RG16_Float: TextureFormatValue;
     RGBA32_UInt: TextureFormatValue;
     RGBA32_SInt: TextureFormatValue;
     RGBA16_UInt: TextureFormatValue;
@@ -260,38 +252,19 @@ export interface MainModule {
     BC7RGBA_UNormSrgb: TextureFormatValue;
     BC3RGBA_UNorm: TextureFormatValue;
     BC3RGBA_UNormSrgb: TextureFormatValue;
+    Depth16Unorm: TextureFormatValue;
+    Depth24Plus: TextureFormatValue;
+    Depth32Float: TextureFormatValue;
   };
 
-  Usage_CopySrc: number;
-  Usage_CopyDest: number;
-  Usage_MapRead: number;
-  Usage_MapWrite: number;
-  Usage_Uniform: number;
-  Usage_Storage: number;
-  Usage_Index: number;
-  Usage_Vertex: number;
-  Usage_QueryResolve: number;
-
   WebGPU: WebGPUInterface;
 
-  _malloc(bytes: number): number;
+  // Emscripten runtime, exported by modules/shark/CMakeLists.txt. The pointer-taking bindings
+  // above address THIS heap, so bytes from the engine module have to be staged through it.
+  // HEAPU8 is replaced whenever the heap grows, so read it fresh rather than caching it.
+  _malloc(byteCount: number): number;
   _free(ptr: number): void;
-  HEAPU32: Uint32Array;
-
-  setupCamera(
-    minExt: vec4,
-    maxExt: vec4,
-    axis: number,
-    direction: number,
-    width: number,
-    height: number,
-    camera: Camera,
-  ): void;
-
-  createRendererWGPU(canvasSelector: string): RendererWGPU;
-  createRendererWGPUWithoutSurface(
-    colorTargetFormat: TextureFormatValue,
-  ): RendererWGPU;
+  readonly HEAPU8: Uint8Array;
 
   uploadSplatSetData(
     set: SplatSet,
@@ -316,33 +289,29 @@ export interface MainModule {
     rasterizeWithCompute: boolean,
     useSOGS: boolean,
     sortType: SortTypeValue,
+    skipSort: boolean,
   ): number;
-  renderSplatSet(
-    splats: number,
-    set: SplatSet,
-    renderState: RenderState,
-    width: number,
-    height: number,
-    triangles: number,
-    rasterizeWithCompute: boolean,
-    textureView: number,
-  ): void;
+  createRendererWGPUWithoutSurface(
+    colorTargetFormat: TextureFormatValue,
+  ): RendererWGPU;
+  /// Gives back the reference WebGPU.importJsTexture returned for a texture.
+  releaseImportedTexture(texture: number): void;
   renderSplatSetToSceneTextures(
     splatSet: SplatSet,
     renderState: RenderState,
     colorTarget: number,
     depthTarget: number,
+    renderGES: boolean,
   ): void;
-  mapToAABB(scale: number, minExt: vec4, maxExt: vec4, set: SplatSet): void;
-  loadPLYIntoSplatSet(filename: string, set: SplatSet): number;
-  getBoundsSplatSet(set: SplatSet, minExt: vec4, maxExt: vec4): void;
-  loadPLY(
-    filename: string,
-    count: number,
-    maxSplats: number,
-    ellipsoids: EllipsoidVector,
-    sphericalHarmonics: SphericalHarmonicVector,
-  ): number;
+  // Render view registry, indexed as RenderViews.h orders it. setRenderView ignores an index
+  // Shark cannot produce.
+  setRenderView(viewIndex: number): void;
+  renderView(): number;
+  renderViewCount(): number;
+  renderViewName(viewIndex: number): string;
+  renderViewSupported(viewIndex: number): boolean;
+  // The next supported view, forwards for direction 1 and backwards for -1.
+  stepRenderView(from: number, direction: number): number;
 }
 
 declare function WebShark(