@babylonjs/core 9.3.1 → 9.3.2

package/Meshes/GaussianSplatting/gaussianSplattingMeshBase.js

@@ -393,6 +393,8 @@ export class GaussianSplattingMeshBase extends Mesh {
         this._modelViewProjectionMatrix = Matrix.Identity();
         this._canPostToWorker = true;
         this._readyToDisplay = false;
+        this._sortRequestId = 0;
+        this._hasRenderedOnce = false;
         this._covariancesATexture = null;
         this._covariancesBTexture = null;
         this._centersTexture = null;
@@ -496,6 +498,66 @@ export class GaussianSplattingMeshBase extends Mesh {
             this._postToWorker(true);
             return false;
         }
+        // Before the first successful render, apply strict sort-state checks to ensure
+        // the first rendered frame uses correct splat ordering. Once the mesh has been
+        // rendered at least once, skip these checks — the render loop will continuously
+        // re-sort as the camera/world changes via _postToWorker() in render().
+        if (!this._hasRenderedOnce && !this._disableDepthSort) {
+            const cameras = this._scene.activeCameras?.length ? this._scene.activeCameras : [this._scene.activeCamera];
+            const worldMatrix = this.computeWorldMatrix(true);
+            let anyDirty = false;
+            for (const camera of cameras) {
+                if (!camera) {
+                    continue;
+                }
+                const cameraViewInfo = this._cameraViewInfos.get(camera.uniqueId);
+                if (!cameraViewInfo || !cameraViewInfo.splatIndexBufferSet) {
+                    anyDirty = true;
+                    continue;
+                }
+                // Wait for the most recently requested sort to be applied so that the splat indices
+                // match the latest world/camera state.
+                if (cameraViewInfo.sortAppliedId !== cameraViewInfo.sortRequestId) {
+                    anyDirty = true;
+                    continue;
+                }
+                // Also detect drift: if the world or camera state has changed since the last post,
+                // mark dirty so the next render does not silently queue a new sort that completes
+                // after isReady has reported true.
+                if (this._isSortStateDirty(cameraViewInfo, worldMatrix, camera)) {
+                    anyDirty = true;
+                }
+            }
+            if (anyDirty) {
+                // Try to post any pending sort so subsequent polling iterations make progress.
+                this._postToWorker(true);
+                return false;
+            }
+        }
+        // Attach the splat geometry to the GS top mesh so that the shadow generator (which renders
+        // shadow casters via the top mesh's subMeshes, NOT through this mesh's render() override)
+        // has valid geometry on the very first shadow pass. Without this, the first shadow render
+        // happens before render() is called and the GS produces no shadow caster output.
+        if (!this._geometry && this._cameraViewInfos.size) {
+            this._geometry = this._cameraViewInfos.values().next().value.mesh.geometry;
+        }
+        // If the material declares a shadow depth wrapper, make sure its effect is compiled for
+        // each subMesh against the scene's shadow generators. Otherwise the first shadow pass
+        // would be skipped (ShadowGenerator.isReady would return false) and we'd miss the shadow
+        // on a renderCount=1 capture.
+        if (this.material && this.material.shadowDepthWrapper) {
+            for (const light of this._scene.lights) {
+                const shadowGenerator = light.getShadowGenerator();
+                if (!shadowGenerator) {
+                    continue;
+                }
+                for (const subMesh of this.subMeshes) {
+                    if (!shadowGenerator.isReady(subMesh, true, false)) {
+                        return false;
+                    }
+                }
+            }
+        }
         return true;
     }
     _getCameraDirection(camera) {
@@ -503,7 +565,7 @@ export class GaussianSplattingMeshBase extends Mesh {
         const cameraProjectionMatrix = camera.getProjectionMatrix();
         const cameraViewProjectionMatrix = TmpVectors.Matrix[0];
         cameraViewMatrix.multiplyToRef(cameraProjectionMatrix, cameraViewProjectionMatrix);
-        const modelMatrix = this.getWorldMatrix();
+        const modelMatrix = this.computeWorldMatrix(true);
         const modelViewMatrix = TmpVectors.Matrix[1];
         modelMatrix.multiplyToRef(cameraViewMatrix, modelViewMatrix);
         modelMatrix.multiplyToRef(cameraViewProjectionMatrix, this._modelViewProjectionMatrix);
@@ -513,6 +575,25 @@ export class GaussianSplattingMeshBase extends Mesh {
         localDirection.normalize();
         return localDirection;
     }
+    _isSortStateDirty(cameraViewInfo, worldMatrix, camera) {
+        const world = worldMatrix.m;
+        const previousWorld = cameraViewInfo.sortWorldMatrix.m;
+        for (let i = 0; i < previousWorld.length; i++) {
+            if (!Scalar.WithinEpsilon(previousWorld[i], world[i], this.viewUpdateThreshold)) {
+                return true;
+            }
+        }
+        const cameraViewMatrix = camera.getViewMatrix();
+        if (!Scalar.WithinEpsilon(cameraViewInfo.sortCameraForward.x, cameraViewMatrix.m[2], this.viewUpdateThreshold) ||
+            !Scalar.WithinEpsilon(cameraViewInfo.sortCameraForward.y, cameraViewMatrix.m[6], this.viewUpdateThreshold) ||
+            !Scalar.WithinEpsilon(cameraViewInfo.sortCameraForward.z, cameraViewMatrix.m[10], this.viewUpdateThreshold)) {
+            return true;
+        }
+        const cameraPosition = camera.globalPosition;
+        return (!Scalar.WithinEpsilon(cameraViewInfo.sortCameraPosition.x, cameraPosition.x, this.viewUpdateThreshold) ||
+            !Scalar.WithinEpsilon(cameraViewInfo.sortCameraPosition.y, cameraPosition.y, this.viewUpdateThreshold) ||
+            !Scalar.WithinEpsilon(cameraViewInfo.sortCameraPosition.z, cameraPosition.z, this.viewUpdateThreshold));
+    }
     /** @internal */
     _postToWorker(forced = false) {
         const scene = this._scene;
@@ -559,6 +640,11 @@ export class GaussianSplattingMeshBase extends Mesh {
                 const newViewInfos = {
                     camera: camera,
                     cameraDirection: new Vector3(0, 0, 0),
+                    sortWorldMatrix: Matrix.Identity(),
+                    sortCameraForward: new Vector3(0, 0, 0),
+                    sortCameraPosition: new Vector3(0, 0, 0),
+                    sortRequestId: 0,
+                    sortAppliedId: 0,
                     mesh: cameraMesh,
                     frameIdLastUpdate: frameId,
                     splatIndexBufferSet: false,
@@ -567,28 +653,39 @@ export class GaussianSplattingMeshBase extends Mesh {
                 this._cameraViewInfos.set(cameraId, newViewInfos);
             }
         });
-        // sort view infos by last updated frame id: first item is the least recently updated
-        activeViewInfos.sort((a, b) => a.frameIdLastUpdate - b.frameIdLastUpdate);
+        // sort view infos: cameras without an initial splat-index buffer come first so they don't get starved
+        // by a `forced` re-sort of an already-initialized camera (which would consume `_canPostToWorker`).
+        // Among initialized cameras, the least recently updated comes first.
+        activeViewInfos.sort((a, b) => {
+            if (a.splatIndexBufferSet !== b.splatIndexBufferSet) {
+                return a.splatIndexBufferSet ? 1 : -1;
+            }
+            return a.frameIdLastUpdate - b.frameIdLastUpdate;
+        });
         const hasSortFunction = this._worker || Native?.sortSplats || this._disableDepthSort;
         if ((forced || outdated) && hasSortFunction && (this._scene.activeCameras?.length || this._scene.activeCamera) && this._canPostToWorker) {
+            const worldMatrix = this.computeWorldMatrix(true);
             // view infos sorted by least recent updated frame id
             activeViewInfos.forEach((cameraViewInfos) => {
                 const camera = cameraViewInfos.camera;
                 const cameraDirection = this._getCameraDirection(camera);
-                const previousCameraDirection = cameraViewInfos.cameraDirection;
-                const dot = Vector3.Dot(cameraDirection, previousCameraDirection);
-                if ((forced || Math.abs(dot - 1) >= this.viewUpdateThreshold) && this._canPostToWorker) {
+                if ((forced || this._isSortStateDirty(cameraViewInfos, worldMatrix, camera)) && this._canPostToWorker) {
+                    const cameraViewMatrix = camera.getViewMatrix();
                     cameraViewInfos.cameraDirection.copyFrom(cameraDirection);
+                    cameraViewInfos.sortWorldMatrix.copyFrom(worldMatrix);
+                    cameraViewInfos.sortCameraForward.set(cameraViewMatrix.m[2], cameraViewMatrix.m[6], cameraViewMatrix.m[10]);
+                    cameraViewInfos.sortCameraPosition.copyFrom(camera.globalPosition);
+                    cameraViewInfos.sortRequestId = ++this._sortRequestId;
                     cameraViewInfos.frameIdLastUpdate = frameId;
                     this._canPostToWorker = false;
                     if (this._worker) {
-                        const cameraViewMatrix = camera.getViewMatrix();
                         this._worker.postMessage({
-                            worldMatrix: this.getWorldMatrix().m,
+                            worldMatrix: worldMatrix.m,
                             cameraForward: [cameraViewMatrix.m[2], cameraViewMatrix.m[6], cameraViewMatrix.m[10]],
                             cameraPosition: [camera.globalPosition.x, camera.globalPosition.y, camera.globalPosition.z],
                             depthMix: this._depthMix,
                             cameraId: camera.uniqueId,
+                            sortRequestId: cameraViewInfos.sortRequestId,
                         }, [this._depthMix.buffer]);
                     }
                     else if (Native?.sortSplats) {
@@ -600,6 +697,7 @@ export class GaussianSplattingMeshBase extends Mesh {
                             cameraViewInfos.mesh.thinInstanceSetBuffer("splatIndex", this._splatIndex, 16, false);
                             cameraViewInfos.splatIndexBufferSet = true;
                         }
+                        cameraViewInfos.sortAppliedId = cameraViewInfos.sortRequestId;
                         this._canPostToWorker = true;
                         this._readyToDisplay = true;
                     }
@@ -650,6 +748,7 @@ export class GaussianSplattingMeshBase extends Mesh {
             mesh.setMaterialForRenderPass(renderPassId, renderPassMaterial);
         }
         const ret = mesh.render(subMesh, enableAlphaMode, effectiveMeshReplacement);
+        this._hasRenderedOnce = true;
         // Clean up the temporary override to avoid affecting other render passes
         if (renderPassMaterial) {
             mesh.setMaterialForRenderPass(renderPassId, undefined);
@@ -1459,6 +1558,7 @@ export class GaussianSplattingMeshBase extends Mesh {
         newGS._modelViewProjectionMatrix = Matrix.Identity();
         newGS._splatPositions = this._splatPositions;
         newGS._readyToDisplay = false;
+        newGS._hasRenderedOnce = false;
         newGS._disableDepthSort = this._disableDepthSort;
         newGS._instantiateWorker();
         const binfo = this.getBoundingInfo();
@@ -1976,6 +2076,7 @@ export class GaussianSplattingMeshBase extends Mesh {
             }
             this._depthMix = e.data.depthMix;
             const cameraId = e.data.cameraId;
+            const sortRequestId = e.data.sortRequestId;
             const indexMix = new Uint32Array(e.data.depthMix.buffer);
             if (this._splatIndex) {
                 for (let j = 0; j < vertexCountPadded; j++) {
@@ -1997,6 +2098,7 @@ export class GaussianSplattingMeshBase extends Mesh {
                     cameraViewInfos.mesh.thinInstanceSetBuffer("splatIndex", this._splatIndex, 16, false);
                     cameraViewInfos.splatIndexBufferSet = true;
                 }
+                cameraViewInfos.sortAppliedId = sortRequestId;
             }
             this._canPostToWorker = true;
             this._readyToDisplay = true;
@@ -2184,13 +2286,14 @@ GaussianSplattingMeshBase._CreateWorker = function (self) {
         // update on view changed
         else {
             const cameraId = e.data.cameraId;
+            const sortRequestId = e.data.sortRequestId;
             const globalWorldMatrix = e.data.worldMatrix;
             const cameraForward = e.data.cameraForward;
             const cameraPosition = e.data.cameraPosition;
             depthMix = e.data.depthMix;
             if (!positions || !cameraForward) {
                 // Sort request arrived before positions were initialized — return the buffer unchanged so the main thread can unlock _canPostToWorker.
-                self.postMessage({ depthMix, cameraId }, [depthMix.buffer]);
+                self.postMessage({ depthMix, cameraId, sortRequestId }, [depthMix.buffer]);
                 return;
             }
             const vertexCountPadded = (positions.length / 4 + 15) & ~0xf;
@@ -2243,7 +2346,7 @@ GaussianSplattingMeshBase._CreateWorker = function (self) {
                 // eslint-disable-next-line no-console
                 console.error("Gaussian splat sort worker encountered an error (will retry next frame):", sortError);
             }
-            self.postMessage({ depthMix, cameraId }, [depthMix.buffer]);
+            self.postMessage({ depthMix, cameraId, sortRequestId }, [depthMix.buffer]);
         }
     };
 };