@onerjs/core 8.31.9 → 8.32.0

package/Meshes/GaussianSplatting/gaussianSplattingMesh.js

@@ -245,6 +245,27 @@ export class GaussianSplattingMesh extends Mesh {
     get material() {
         return this._material;
     }
+    static _MakeSplatGeometryForMesh(mesh) {
+        const vertexData = new VertexData();
+        const originPositions = [-2, -2, 0, 2, -2, 0, 2, 2, 0, -2, 2, 0];
+        const originIndices = [0, 1, 2, 0, 2, 3];
+        const positions = [];
+        const indices = [];
+        for (let i = 0; i < GaussianSplattingMesh._BatchSize; i++) {
+            for (let j = 0; j < 12; j++) {
+                if (j == 2 || j == 5 || j == 8 || j == 11) {
+                    positions.push(i); // local splat index
+                }
+                else {
+                    positions.push(originPositions[j]);
+                }
+            }
+            indices.push(originIndices.map((v) => v + i * 4));
+        }
+        vertexData.positions = positions;
+        vertexData.indices = indices.flat();
+        vertexData.applyToMesh(mesh);
+    }
     /**
      * Creates a new gaussian splatting mesh
      * @param name defines the name of the mesh
@@ -256,7 +277,6 @@ export class GaussianSplattingMesh extends Mesh {
         super(name, scene);
         this._vertexCount = 0;
         this._worker = null;
-        this._frameIdLastUpdate = -1;
         this._modelViewMatrix = Matrix.Identity();
         this._canPostToWorker = true;
         this._readyToDisplay = false;
@@ -271,35 +291,15 @@ export class GaussianSplattingMesh extends Mesh {
         this._sh = null;
         this._keepInRam = false;
         this._delayedTextureUpdate = null;
-        this._oldDirection = new Vector3();
         this._useRGBACovariants = false;
         this._material = null;
         this._tmpCovariances = [0, 0, 0, 0, 0, 0];
         this._sortIsDirty = false;
         this._shDegree = 0;
         this._viewDirectionFactor = new Vector3(1, 1, -1);
-        const vertexData = new VertexData();
-        const originPositions = [-2, -2, 0, 2, -2, 0, 2, 2, 0, -2, 2, 0];
-        const originIndices = [0, 1, 2, 0, 2, 3];
-        const positions = [];
-        const indices = [];
-        const batchSize = 16; // 16 splats per instance
-        for (let i = 0; i < batchSize; i++) {
-            for (let j = 0; j < 12; j++) {
-                if (j == 2 || j == 5 || j == 8 || j == 11) {
-                    positions.push(i); // local splat index
-                }
-                else {
-                    positions.push(originPositions[j]);
-                }
-            }
-            indices.push(originIndices.map((v) => v + i * 4));
-        }
-        vertexData.positions = positions;
-        vertexData.indices = indices.flat();
-        vertexData.applyToMesh(this);
+        this._cameraViewInfos = new Map();
         this.subMeshes = [];
-        new SubMesh(0, 0, 4 * batchSize, 0, 6 * batchSize, this);
+        new SubMesh(0, 0, 4 * GaussianSplattingMesh._BatchSize, 0, 6 * GaussianSplattingMesh._BatchSize, this);
         this.setEnabled(false);
         // webGL2 and webGPU support for RG texture with float16 is fine. not webGL1
         this._useRGBACovariants = !this.getEngine().isWebGPU && this.getEngine().version === 1.0;
@@ -308,7 +308,19 @@ export class GaussianSplattingMesh extends Mesh {
             // eslint-disable-next-line @typescript-eslint/no-floating-promises
             this.loadFileAsync(url);
         }
-        this._material = new GaussianSplattingMaterial(this.name + "_material", this._scene);
+        const gaussianSplattingMaterial = new GaussianSplattingMaterial(this.name + "_material", this._scene);
+        gaussianSplattingMaterial.setSourceMesh(this);
+        this._material = gaussianSplattingMaterial;
+        // delete meshes created for cameras on camera removal
+        this._scene.onCameraRemovedObservable.add((camera) => {
+            const cameraId = camera.uniqueId;
+            // delete mesh for this camera
+            if (this._cameraViewInfos.has(cameraId)) {
+                const cameraViewInfos = this._cameraViewInfos.get(cameraId);
+                cameraViewInfos?.mesh.dispose();
+                this._cameraViewInfos.delete(cameraId);
+            }
+        });
     }
     /**
      * Returns the class name
@@ -340,25 +352,76 @@ export class GaussianSplattingMesh extends Mesh {
         }
         return true;
     }
+    _getCameraDirection(camera) {
+        const cameraMatrix = camera.getViewMatrix();
+        this.getWorldMatrix().multiplyToRef(cameraMatrix, this._modelViewMatrix);
+        cameraMatrix.invertToRef(TmpVectors.Matrix[0]);
+        this.getWorldMatrix().multiplyToRef(TmpVectors.Matrix[0], TmpVectors.Matrix[1]);
+        Vector3.TransformNormalToRef(Vector3.Forward(this._scene.useRightHandedSystem), TmpVectors.Matrix[1], TmpVectors.Vector3[2]);
+        TmpVectors.Vector3[2].normalize();
+        return TmpVectors.Vector3[2];
+    }
     /** @internal */
     _postToWorker(forced = false) {
-        const frameId = this.getScene().getFrameId();
-        if ((forced || frameId !== this._frameIdLastUpdate) && this._worker && this._scene.activeCamera && this._canPostToWorker) {
-            const cameraMatrix = this._scene.activeCamera.getViewMatrix();
-            this.getWorldMatrix().multiplyToRef(cameraMatrix, this._modelViewMatrix);
-            cameraMatrix.invertToRef(TmpVectors.Matrix[0]);
-            this.getWorldMatrix().multiplyToRef(TmpVectors.Matrix[0], TmpVectors.Matrix[1]);
-            Vector3.TransformNormalToRef(Vector3.Forward(this._scene.useRightHandedSystem), TmpVectors.Matrix[1], TmpVectors.Vector3[2]);
-            TmpVectors.Vector3[2].normalize();
-            const dot = Vector3.Dot(TmpVectors.Vector3[2], this._oldDirection);
-            if (forced || Math.abs(dot - 1) >= 0.01) {
-                this._oldDirection.copyFrom(TmpVectors.Vector3[2]);
-                this._frameIdLastUpdate = frameId;
-                this._canPostToWorker = false;
-                this._worker.postMessage({ view: this._modelViewMatrix.m, depthMix: this._depthMix, useRightHandedSystem: this._scene.useRightHandedSystem }, [
-                    this._depthMix.buffer,
-                ]);
+        const scene = this._scene;
+        const frameId = scene.getFrameId();
+        // force update or at least frame update for camera is outdated
+        let outdated = false;
+        this._cameraViewInfos.forEach((cameraViewInfos) => {
+            if (cameraViewInfos.frameIdLastUpdate !== frameId) {
+                outdated = true;
             }
+        });
+        if ((forced || outdated) && this._worker && (this._scene.activeCameras || this._scene.activeCamera) && this._canPostToWorker) {
+            // array of cameras used for rendering
+            const cameras = this._scene.activeCameras?.length ? this._scene.activeCameras : [this._scene.activeCamera];
+            // list view infos for active cameras
+            const activeViewInfos = [];
+            cameras.forEach((camera) => {
+                const cameraId = camera.uniqueId;
+                const cameraViewInfos = this._cameraViewInfos.get(cameraId);
+                if (cameraViewInfos) {
+                    activeViewInfos.push(cameraViewInfos);
+                }
+                else {
+                    // mesh doesn't exist yet for this camera
+                    const cameraMesh = new Mesh(this.name + "_cameraMesh_" + cameraId, this._scene);
+                    // not visible with inspector or the scene graph
+                    cameraMesh.reservedDataStore = { hidden: true };
+                    cameraMesh.setEnabled(false);
+                    cameraMesh.material = this.material;
+                    GaussianSplattingMesh._MakeSplatGeometryForMesh(cameraMesh);
+                    const newViewInfos = {
+                        camera: camera,
+                        cameraDirection: new Vector3(0, 0, 0),
+                        mesh: cameraMesh,
+                        frameIdLastUpdate: frameId,
+                        splatIndexBufferSet: false,
+                    };
+                    activeViewInfos.push(newViewInfos);
+                    this._cameraViewInfos.set(cameraId, newViewInfos);
+                }
+            });
+            // sort view infos by last updated frame id: first item is the least recently updated
+            activeViewInfos.sort((a, b) => a.frameIdLastUpdate - b.frameIdLastUpdate);
+            // view infos sorted by least recent updated frame id
+            activeViewInfos.forEach((cameraViewInfos) => {
+                const camera = cameraViewInfos.camera;
+                const cameraDirection = this._getCameraDirection(camera);
+                const previousCameraDirection = cameraViewInfos.cameraDirection;
+                const dot = Vector3.Dot(cameraDirection, previousCameraDirection);
+                if ((forced || cameraViewInfos.frameIdLastUpdate !== frameId || Math.abs(dot - 1) >= 0.01) && this._canPostToWorker) {
+                    cameraViewInfos.cameraDirection.copyFrom(cameraDirection);
+                    cameraViewInfos.frameIdLastUpdate = frameId;
+                    this._canPostToWorker = false;
+                    this._worker.postMessage({
+                        view: this._modelViewMatrix.m,
+                        depthMix: this._depthMix,
+                        useRightHandedSystem: this._scene.useRightHandedSystem,
+                        cameraId: camera.uniqueId,
+                    }, [this._depthMix.buffer]);
+                }
+            });
         }
     }
     /**
@@ -370,7 +433,18 @@ export class GaussianSplattingMesh extends Mesh {
      */
     render(subMesh, enableAlphaMode, effectiveMeshReplacement) {
         this._postToWorker();
-        return super.render(subMesh, enableAlphaMode, effectiveMeshReplacement);
+        // geometry used for shadows, bind the first found in the camera view infos
+        if (!this._geometry && this._cameraViewInfos.size) {
+            this._geometry = this._cameraViewInfos.values().next().value.mesh.geometry;
+        }
+        const cameraId = this._scene.activeCamera.uniqueId;
+        const cameraViewInfos = this._cameraViewInfos.get(cameraId);
+        if (!cameraViewInfos || !cameraViewInfos.splatIndexBufferSet) {
+            return this;
+        }
+        const mesh = cameraViewInfos.mesh;
+        mesh.getWorldMatrix().copyFrom(this.getWorldMatrix());
+        return mesh.render(subMesh, enableAlphaMode, effectiveMeshReplacement);
     }
     static _TypeNameToEnum(name) {
         switch (name) {
@@ -378,7 +452,6 @@ export class GaussianSplattingMesh extends Mesh {
                 return 0 /* PLYType.FLOAT */;
             case "int":
                 return 1 /* PLYType.INT */;
-                break;
             case "uint":
                 return 2 /* PLYType.UINT */;
             case "double":
@@ -1062,6 +1135,10 @@ export class GaussianSplattingMesh extends Mesh {
         this._shTextures = null;
         this._worker?.terminate();
         this._worker = null;
+        // delete meshes created for each camera
+        this._cameraViewInfos.forEach((cameraViewInfo) => {
+            cameraViewInfo.mesh.dispose();
+        });
         super.dispose(doNotRecurse, true);
     }
     _copyTextures(source) {
@@ -1300,7 +1377,10 @@ export class GaussianSplattingMesh extends Mesh {
         const paddedVertexCount = (vertexCount + 15) & ~0xf;
         if (!this._splatIndex || vertexCount > this._splatIndex.length) {
             this._splatIndex = new Float32Array(paddedVertexCount);
-            this.thinInstanceSetBuffer("splatIndex", this._splatIndex, 16, false);
+            // update meshes for knowns cameras
+            this._cameraViewInfos.forEach((cameraViewInfos) => {
+                cameraViewInfos.mesh.thinInstanceSetBuffer("splatIndex", this._splatIndex, 16, false);
+            });
         }
         this.forcedInstanceCount = paddedVertexCount >> 4;
     }
@@ -1344,6 +1424,7 @@ export class GaussianSplattingMesh extends Mesh {
         this._worker.postMessage({ positions, vertexCountPadded }, [positions.buffer]);
         this._worker.onmessage = (e) => {
             this._depthMix = e.data.depthMix;
+            const cameraId = e.data.cameraId;
             const indexMix = new Uint32Array(e.data.depthMix.buffer);
             if (this._splatIndex) {
                 for (let j = 0; j < vertexCountPadded; j++) {
@@ -1355,7 +1436,17 @@ export class GaussianSplattingMesh extends Mesh {
                 this._updateSubTextures(this._delayedTextureUpdate.centers, this._delayedTextureUpdate.covA, this._delayedTextureUpdate.covB, this._delayedTextureUpdate.colors, 0, textureSize.y, this._delayedTextureUpdate.sh);
                 this._delayedTextureUpdate = null;
             }
-            this.thinInstanceBufferUpdated("splatIndex");
+            // get mesh for camera and update its instance buffer
+            const cameraViewInfos = this._cameraViewInfos.get(cameraId);
+            if (cameraViewInfos) {
+                if (cameraViewInfos.splatIndexBufferSet) {
+                    cameraViewInfos.mesh.thinInstanceBufferUpdated("splatIndex");
+                }
+                else {
+                    cameraViewInfos.mesh.thinInstanceSetBuffer("splatIndex", this._splatIndex, 16, false);
+                    cameraViewInfos.splatIndexBufferSet = true;
+                }
+            }
             this._canPostToWorker = true;
             this._readyToDisplay = true;
             // sort is dirty when GS is visible for progressive update with a this message arriving but positions were partially filled
@@ -1392,6 +1483,7 @@ GaussianSplattingMesh._SH_C0 = 0.28209479177387814;
 GaussianSplattingMesh._SplatBatchSize = 327680;
 // batch size between 2 yield calls during the PLY to splat conversion.
 GaussianSplattingMesh._PlyConversionBatchSize = 32768;
+GaussianSplattingMesh._BatchSize = 16; // 16 splats per instance
 /**
  * Set the number of batch (a batch is 16384 splats) after which a display update is performed
  * A value of 0 (default) means display update will not happens before splat is ready.
@@ -1411,6 +1503,7 @@ GaussianSplattingMesh._CreateWorker = function (self) {
         }
         // udpate on view changed
         else {
+            const cameraId = e.data.cameraId;
             const viewProj = e.data.view;
             if (!positions || !viewProj) {
                 // Sanity check, it shouldn't happen!
@@ -1431,7 +1524,7 @@ GaussianSplattingMesh._CreateWorker = function (self) {
                 floatMix[2 * j + 1] = 10000 + (viewProj[2] * positions[4 * j + 0] + viewProj[6] * positions[4 * j + 1] + viewProj[10] * positions[4 * j + 2]) * depthFactor;
             }
             depthMix.sort();
-            self.postMessage({ depthMix }, [depthMix.buffer]);
+            self.postMessage({ depthMix, cameraId }, [depthMix.buffer]);
         }
     };
 };