@babylonjs/core 7.36.0 → 7.37.1

package/Meshes/GaussianSplatting/gaussianSplattingMesh.js

@@ -109,6 +109,12 @@ var PLYValue;
  * Class used to render a gaussian splatting mesh
  */
 export class GaussianSplattingMesh extends Mesh {
+    /**
+     * SH degree. 0 = no sh (default). 1 = 3 parameters. 2 = 8 parameters. 3 = 15 parameters.
+     */
+    get shDegree() {
+        return this._shDegree;
+    }
     /**
      * Gets the covariancesA texture
      */
@@ -133,6 +139,12 @@ export class GaussianSplattingMesh extends Mesh {
     get colorsTexture() {
         return this._colorsTexture;
     }
+    /**
+     * Gets the SH textures
+     */
+    get shTextures() {
+        return this._shTextures;
+    }
     /**
      * set rendering material
      */
@@ -169,12 +181,14 @@ export class GaussianSplattingMesh extends Mesh {
         this._colorsTexture = null;
         this._splatPositions = null;
         this._splatIndex = null;
+        this._shTextures = null;
         //@ts-expect-error
         this._covariancesA = null;
         //@ts-expect-error
         this._covariancesB = null;
         //@ts-expect-error
         this._colors = null;
+        this._sh = null;
         this._keepInRam = false;
         this._delayedTextureUpdate = null;
         this._oldDirection = new Vector3();
@@ -182,6 +196,7 @@ export class GaussianSplattingMesh extends Mesh {
         this._material = null;
         this._tmpCovariances = [0, 0, 0, 0, 0, 0];
         this._sortIsDirty = false;
+        this._shDegree = 0;
         const vertexData = new VertexData();
         // Use an intanced quad or triangle. Triangle might be a bit faster because of less shader invocation but I didn't see any difference.
         // Keeping both and use triangle for now.
@@ -234,6 +249,7 @@ export class GaussianSplattingMesh extends Mesh {
         }
         return true;
     }
+    /** @internal */
     _postToWorker(forced = false) {
         const frameId = this.getScene().getFrameId();
         if ((forced || frameId !== this._frameIdLastUpdate) && this._worker && this._scene.activeCamera && this._canPostToWorker) {
@@ -693,10 +709,16 @@ export class GaussianSplattingMesh extends Mesh {
         this._covariancesBTexture?.dispose();
         this._centersTexture?.dispose();
         this._colorsTexture?.dispose();
+        if (this._shTextures) {
+            this._shTextures.forEach((shTexture) => {
+                shTexture.dispose();
+            });
+        }
         this._covariancesATexture = null;
         this._covariancesBTexture = null;
         this._centersTexture = null;
         this._colorsTexture = null;
+        this._shTextures = null;
         this._worker?.terminate();
         this._worker = null;
         super.dispose(doNotRecurse, true);
@@ -706,6 +728,12 @@ export class GaussianSplattingMesh extends Mesh {
         this._covariancesBTexture = source.covariancesBTexture?.clone();
         this._centersTexture = source.centersTexture?.clone();
         this._colorsTexture = source.colorsTexture?.clone();
+        if (source._shTextures) {
+            this._shTextures = [];
+            this._shTextures.forEach((shTexture) => {
+                this._shTextures?.push(shTexture.clone());
+            });
+        }
     }
     /**
      * Returns a new Mesh object generated from the current mesh properties.
@@ -771,7 +799,7 @@ export class GaussianSplattingMesh extends Mesh {
         colorArray[destinationIndex * 4 + 2] = uBuffer[32 * sourceIndex + 24 + 2];
         colorArray[destinationIndex * 4 + 3] = uBuffer[32 * sourceIndex + 24 + 3];
     }
-    _updateTextures(covA, covB, colorArray) {
+    _updateTextures(covA, covB, colorArray, sh) {
         const textureSize = this._getTextureSize(this._vertexCount);
         // Update the textures
         const createTextureFromData = (data, width, height, format) => {
@@ -780,6 +808,9 @@ export class GaussianSplattingMesh extends Mesh {
         const createTextureFromDataU8 = (data, width, height, format) => {
             return new RawTexture(data, width, height, format, this._scene, false, false, 2, 0);
         };
+        const createTextureFromDataU32 = (data, width, height, format) => {
+            return new RawTexture(data, width, height, format, this._scene, false, false, 1, 7);
+        };
         const createTextureFromDataF16 = (data, width, height, format) => {
             return new RawTexture(data, width, height, format, this._scene, false, false, 2, 2);
         };
@@ -787,9 +818,12 @@ export class GaussianSplattingMesh extends Mesh {
             this._covariancesA = covA;
             this._covariancesB = covB;
             this._colors = colorArray;
+            if (sh) {
+                this._sh = sh;
+            }
         }
         if (this._covariancesATexture) {
-            this._delayedTextureUpdate = { covA: covA, covB: covB, colors: colorArray, centers: this._splatPositions };
+            this._delayedTextureUpdate = { covA: covA, covB: covB, colors: colorArray, centers: this._splatPositions, sh: sh };
             const positions = Float32Array.from(this._splatPositions);
             const vertexCount = this._vertexCount;
             this._worker.postMessage({ positions, vertexCount }, [positions.buffer]);
@@ -800,10 +834,20 @@ export class GaussianSplattingMesh extends Mesh {
             this._covariancesBTexture = createTextureFromDataF16(covB, textureSize.x, textureSize.y, this._useRGBACovariants ? 5 : 7);
             this._centersTexture = createTextureFromData(this._splatPositions, textureSize.x, textureSize.y, 5);
             this._colorsTexture = createTextureFromDataU8(colorArray, textureSize.x, textureSize.y, 5);
+            if (sh) {
+                this._shTextures = [];
+                sh.forEach((shData) => {
+                    const buffer = new Uint32Array(shData.buffer);
+                    const shTexture = createTextureFromDataU32(buffer, textureSize.x, textureSize.y, 11);
+                    shTexture.wrapU = 0;
+                    shTexture.wrapV = 0;
+                    this._shTextures.push(shTexture);
+                });
+            }
             this._instanciateWorker();
         }
     }
-    *_updateData(data, isAsync) {
+    *_updateData(data, isAsync, sh) {
         // if a covariance texture is present, then it's not a creation but an update
         if (!this._covariancesATexture) {
             this._readyToDisplay = false;
@@ -816,6 +860,8 @@ export class GaussianSplattingMesh extends Mesh {
             this._updateSplatIndexBuffer(vertexCount);
         }
         this._vertexCount = vertexCount;
+        // degree == 1 for 1 texture (3 terms), 2 for 2 textures(8 terms) and 3 for 3 textures (15 terms)
+        this._shDegree = sh ? sh.length : 0;
         const textureSize = this._getTextureSize(vertexCount);
         const textureLength = textureSize.x * textureSize.y;
         const lineCountUpdate = GaussianSplattingMesh.ProgressiveUpdateAmount ?? textureSize.y;
@@ -828,7 +874,7 @@ export class GaussianSplattingMesh extends Mesh {
         const maximum = new Vector3(-Number.MAX_VALUE, -Number.MAX_VALUE, -Number.MAX_VALUE);
         if (GaussianSplattingMesh.ProgressiveUpdateAmount) {
             // create textures with not filled-yet array, then update directly portions of it
-            this._updateTextures(covA, covB, colorArray);
+            this._updateTextures(covA, covB, colorArray, sh);
             this.setEnabled(true);
             const partCount = Math.ceil(textureSize.y / lineCountUpdate);
             for (let partIndex = 0; partIndex < partCount; partIndex++) {
@@ -858,7 +904,7 @@ export class GaussianSplattingMesh extends Mesh {
                 }
             }
             // textures
-            this._updateTextures(covA, covB, colorArray);
+            this._updateTextures(covA, covB, colorArray, sh);
             // Update the binfo
             this.getBoundingInfo().reConstruct(minimum, maximum, this.getWorldMatrix());
             this.setEnabled(true);
@@ -868,18 +914,20 @@ export class GaussianSplattingMesh extends Mesh {
     /**
      * Update asynchronously the buffer
      * @param data array buffer containing center, color, orientation and scale of splats
+     * @param sh optional array of uint8 array for SH data
      * @returns a promise
      */
-    async updateDataAsync(data) {
-        return runCoroutineAsync(this._updateData(data, true), createYieldingScheduler());
+    async updateDataAsync(data, sh) {
+        return runCoroutineAsync(this._updateData(data, true, sh), createYieldingScheduler());
     }
     /**
      * @experimental
      * Update data from GS (position, orientation, color, scaling)
      * @param data array that contain all the datas
+     * @param sh optional array of uint8 array for SH data
      */
-    updateData(data) {
-        runCoroutineSync(this._updateData(data, false));
+    updateData(data, sh) {
+        runCoroutineSync(this._updateData(data, false, sh));
     }
     // in case size is different
     _updateSplatIndexBuffer(vertexCount) {
@@ -889,16 +937,10 @@ export class GaussianSplattingMesh extends Mesh {
         }
         this.forcedInstanceCount = vertexCount;
     }
-    _updateSubTextures(centers, covA, covB, colors, lineStart, lineCount) {
+    _updateSubTextures(centers, covA, covB, colors, lineStart, lineCount, sh) {
         const updateTextureFromData = (texture, data, width, lineStart, lineCount) => {
             this.getEngine().updateTextureData(texture.getInternalTexture(), data, 0, lineStart, width, lineCount, 0, 0, false);
         };
-        const updateTextureFromDataU8 = (texture, data, width, lineStart, lineCount) => {
-            this.getEngine().updateTextureData(texture.getInternalTexture(), data, 0, lineStart, width, lineCount, 0, 0, false);
-        };
-        const updateTextureFromDataF16 = (texture, data, width, lineStart, lineCount) => {
-            this.getEngine().updateTextureData(texture.getInternalTexture(), data, 0, lineStart, width, lineCount, 0, 0, false);
-        };
         const textureSize = this._getTextureSize(this._vertexCount);
         const covBSItemSize = this._useRGBACovariants ? 4 : 2;
         const texelStart = lineStart * textureSize.x;
@@ -907,10 +949,17 @@ export class GaussianSplattingMesh extends Mesh {
         const covBView = new Uint16Array(covB.buffer, texelStart * covBSItemSize * Uint16Array.BYTES_PER_ELEMENT, texelCount * covBSItemSize);
         const colorsView = new Uint8Array(colors.buffer, texelStart * 4, texelCount * 4);
         const centersView = new Float32Array(centers.buffer, texelStart * 4 * Float32Array.BYTES_PER_ELEMENT, texelCount * 4);
-        updateTextureFromDataF16(this._covariancesATexture, covAView, textureSize.x, lineStart, lineCount);
-        updateTextureFromDataF16(this._covariancesBTexture, covBView, textureSize.x, lineStart, lineCount);
+        updateTextureFromData(this._covariancesATexture, covAView, textureSize.x, lineStart, lineCount);
+        updateTextureFromData(this._covariancesBTexture, covBView, textureSize.x, lineStart, lineCount);
         updateTextureFromData(this._centersTexture, centersView, textureSize.x, lineStart, lineCount);
-        updateTextureFromDataU8(this._colorsTexture, colorsView, textureSize.x, lineStart, lineCount);
+        updateTextureFromData(this._colorsTexture, colorsView, textureSize.x, lineStart, lineCount);
+        if (sh) {
+            for (let i = 0; i < sh.length; i++) {
+                const componentCount = 4;
+                const shView = new Uint8Array(this._sh[i].buffer, texelStart * componentCount, texelCount * componentCount);
+                updateTextureFromData(this._shTextures[i], shView, textureSize.x, lineStart, lineCount);
+            }
+        }
     }
     _instanciateWorker() {
         if (!this._vertexCount) {
@@ -936,7 +985,7 @@ export class GaussianSplattingMesh extends Mesh {
             }
             if (this._delayedTextureUpdate) {
                 const textureSize = this._getTextureSize(vertexCount);
-                this._updateSubTextures(this._delayedTextureUpdate.centers, this._delayedTextureUpdate.covA, this._delayedTextureUpdate.covB, this._delayedTextureUpdate.colors, 0, textureSize.y);
+                this._updateSubTextures(this._delayedTextureUpdate.centers, this._delayedTextureUpdate.covA, this._delayedTextureUpdate.covB, this._delayedTextureUpdate.colors, 0, textureSize.y, this._delayedTextureUpdate.sh);
                 this._delayedTextureUpdate = null;
             }
             this.thinInstanceBufferUpdated("splatIndex");