@onerjs/core 8.36.8 → 8.37.1

package/Meshes/GaussianSplatting/gaussianSplattingMesh.js

@@ -2,6 +2,7 @@ import { SubMesh } from "../subMesh.js";
 import { Mesh } from "../mesh.js";
 import { VertexData } from "../mesh.vertexData.js";
 import { Matrix, TmpVectors, Vector2, Vector3 } from "../../Maths/math.vector.js";
+import { Quaternion } from "../../Maths/math.vector.js";
 import { Logger } from "../../Misc/logger.js";
 import { GaussianSplattingMaterial } from "../../Materials/GaussianSplatting/gaussianSplattingMaterial.js";
 import { RawTexture } from "../../Materials/Textures/rawTexture.js";
@@ -12,6 +13,8 @@ import { Scalar } from "../../Maths/math.scalar.js";
 import { runCoroutineSync, runCoroutineAsync, createYieldingScheduler } from "../../Misc/coroutine.js";
 import { EngineStore } from "../../Engines/engineStore.js";
 import { ImportMeshAsync } from "../../Loading/sceneLoader.js";
+const IsNative = typeof _native !== "undefined";
+const Native = IsNative ? _native : null;
 // @internal
 const UnpackUnorm = (value, bits) => {
     const t = (1 << bits) - 1;
@@ -204,6 +207,30 @@ export class GaussianSplattingMesh extends Mesh {
     get splatsData() {
         return this._splatsData;
     }
+    /**
+     * returns the SH data arrays
+     */
+    get shData() {
+        return this._shData;
+    }
+    /**
+     * True when this mesh is a compound that regroups multiple Gaussian splatting parts.
+     */
+    get isCompound() {
+        return this._partMatrices.length > 0;
+    }
+    /**
+     * returns the part indices array
+     */
+    get partIndices() {
+        return this._partIndices;
+    }
+    /**
+     * Gets the part indices texture, if the mesh is a compound
+     */
+    get partIndicesTexture() {
+        return this._partIndicesTexture;
+    }
     /**
      * Gets the covariancesA texture
      */
@@ -297,6 +324,7 @@ export class GaussianSplattingMesh extends Mesh {
         this._vertexCount = 0;
         this._worker = null;
         this._modelViewProjectionMatrix = Matrix.Identity();
+        this._viewProjectionMatrix = Matrix.Identity();
         this._canPostToWorker = true;
         this._readyToDisplay = false;
         this._covariancesATexture = null;
@@ -307,6 +335,11 @@ export class GaussianSplattingMesh extends Mesh {
         this._splatIndex = null;
         this._shTextures = null;
         this._splatsData = null;
+        this._shData = null;
+        this._partIndicesTexture = null;
+        this._partIndices = null;
+        this._partMatrices = [];
+        this._textureSize = new Vector2(0, 0);
         this._keepInRam = false;
         this._delayedTextureUpdate = null;
         this._useRGBACovariants = false;
@@ -386,10 +419,13 @@ export class GaussianSplattingMesh extends Mesh {
         const cameraProjectionMatrix = camera.getProjectionMatrix();
         const cameraViewProjectionMatrix = TmpVectors.Matrix[0];
         cameraViewMatrix.multiplyToRef(cameraProjectionMatrix, cameraViewProjectionMatrix);
-        this.getWorldMatrix().multiplyToRef(cameraViewProjectionMatrix, this._modelViewProjectionMatrix);
+        this._viewProjectionMatrix.copyFrom(cameraViewProjectionMatrix);
+        const modelViewMatrix = TmpVectors.Matrix[1];
+        this.getWorldMatrix().multiplyToRef(cameraViewMatrix, modelViewMatrix);
+        modelViewMatrix.multiplyToRef(cameraProjectionMatrix, this._modelViewProjectionMatrix);
         // return vector used to compute distance to camera
         const localDirection = TmpVectors.Vector3[1];
-        localDirection.set(this._modelViewProjectionMatrix.m[8], this._modelViewProjectionMatrix.m[9], this._modelViewProjectionMatrix.m[10]);
+        localDirection.set(modelViewMatrix.m[2], modelViewMatrix.m[6], modelViewMatrix.m[10]);
         localDirection.normalize();
         return localDirection;
     }
@@ -438,7 +474,7 @@ export class GaussianSplattingMesh extends Mesh {
         });
         // sort view infos by last updated frame id: first item is the least recently updated
         activeViewInfos.sort((a, b) => a.frameIdLastUpdate - b.frameIdLastUpdate);
-        const hasSortFunction = this._worker || (_native && _native.sortSplats) || this._disableDepthSort;
+        const hasSortFunction = this._worker || Native?.sortSplats || this._disableDepthSort;
         if ((forced || outdated) && hasSortFunction && (this._scene.activeCameras?.length || this._scene.activeCamera) && this._canPostToWorker) {
             // view infos sorted by least recent updated frame id
             activeViewInfos.forEach((cameraViewInfos) => {
@@ -453,12 +489,13 @@ export class GaussianSplattingMesh extends Mesh {
                     if (this._worker) {
                         this._worker.postMessage({
                             modelViewProjection: this._modelViewProjectionMatrix.m,
+                            viewProjection: this._viewProjectionMatrix.m,
                             depthMix: this._depthMix,
                             cameraId: camera.uniqueId,
                         }, [this._depthMix.buffer]);
                     }
-                    else if (_native && _native.sortSplats) {
-                        _native.sortSplats(this._modelViewProjectionMatrix, this._splatPositions, this._splatIndex, this._scene.useRightHandedSystem);
+                    else if (Native?.sortSplats) {
+                        Native.sortSplats(this._modelViewProjectionMatrix, this._splatPositions, this._splatIndex, this._scene.useRightHandedSystem);
                         if (cameraViewInfos.splatIndexBufferSet) {
                             cameraViewInfos.mesh.thinInstanceBufferUpdated("splatIndex");
                         }
@@ -1190,11 +1227,16 @@ export class GaussianSplattingMesh extends Mesh {
                 shTexture.dispose();
             }
         }
+        if (this._partIndicesTexture) {
+            this._partIndicesTexture.dispose();
+        }
         this._covariancesATexture = null;
         this._covariancesBTexture = null;
         this._centersTexture = null;
         this._colorsTexture = null;
         this._shTextures = null;
+        this._partIndicesTexture = null;
+        this._partMatrices = [];
         this._worker?.terminate();
         this._worker = null;
         // delete meshes created for each camera
@@ -1208,9 +1250,10 @@ export class GaussianSplattingMesh extends Mesh {
         this._covariancesBTexture = source.covariancesBTexture?.clone();
         this._centersTexture = source.centersTexture?.clone();
         this._colorsTexture = source.colorsTexture?.clone();
+        this._partIndicesTexture = source._partIndicesTexture?.clone();
         if (source._shTextures) {
             this._shTextures = [];
-            for (const shTexture of this._shTextures) {
+            for (const shTexture of source._shTextures) {
                 this._shTextures?.push(shTexture.clone());
             }
         }
@@ -1227,9 +1270,11 @@ export class GaussianSplattingMesh extends Mesh {
         newGS._vertexCount = this._vertexCount;
         newGS._copyTextures(this);
         newGS._modelViewProjectionMatrix = Matrix.Identity();
+        newGS._viewProjectionMatrix = Matrix.Identity();
         newGS._splatPositions = this._splatPositions;
         newGS._readyToDisplay = false;
         newGS._disableDepthSort = this._disableDepthSort;
+        newGS._partMatrices = this._partMatrices.map((m) => m.clone());
         newGS._instanciateWorker();
         const binfo = this.getBoundingInfo();
         newGS.getBoundingInfo().reConstruct(binfo.minimum, binfo.maximum, this.getWorldMatrix());
@@ -1294,7 +1339,8 @@ export class GaussianSplattingMesh extends Mesh {
         colorArray[index * 4 + 2] = uBuffer[32 * index + 24 + 2];
         colorArray[index * 4 + 3] = uBuffer[32 * index + 24 + 3];
     }
-    _updateTextures(covA, covB, colorArray, sh) {
+    // NB: partIndices is assumed to be padded to a round texture size
+    _updateTextures(covA, covB, colorArray, sh, partIndices) {
         const textureSize = this._getTextureSize(this._vertexCount);
         // Update the textures
         const createTextureFromData = (data, width, height, format) => {
@@ -1309,16 +1355,40 @@ export class GaussianSplattingMesh extends Mesh {
         const createTextureFromDataF16 = (data, width, height, format) => {
             return new RawTexture(data, width, height, format, this._scene, false, false, 2, 2);
         };
-        if (this._covariancesATexture) {
-            this._delayedTextureUpdate = { covA: covA, covB: covB, colors: colorArray, centers: this._splatPositions, sh: sh };
+        const firstTime = this._covariancesATexture === null;
+        const textureSizeChanged = this._textureSize.y < textureSize.y;
+        if (!firstTime && !textureSizeChanged) {
+            this._delayedTextureUpdate = { covA, covB, colors: colorArray, centers: this._splatPositions, sh, partIndices };
             const positions = Float32Array.from(this._splatPositions);
             const vertexCount = this._vertexCount;
             if (this._worker) {
                 this._worker.postMessage({ positions, vertexCount }, [positions.buffer]);
             }
+            // Handle SH textures in update path - create if they don't exist
+            if (sh && !this._shTextures) {
+                this._shTextures = [];
+                for (const shData of sh) {
+                    const buffer = new Uint32Array(shData.buffer);
+                    const shTexture = createTextureFromDataU32(buffer, textureSize.x, textureSize.y, 11);
+                    shTexture.wrapU = 0;
+                    shTexture.wrapV = 0;
+                    this._shTextures.push(shTexture);
+                }
+            }
+            // Handle compound data, if any
+            if (partIndices && !this._partIndicesTexture) {
+                const buffer = new Uint8Array(partIndices);
+                this._partIndicesTexture = createTextureFromDataU8(buffer, textureSize.x, textureSize.y, 6);
+                this._partIndicesTexture.wrapU = 0;
+                this._partIndicesTexture.wrapV = 0;
+            }
+            if (this._worker) {
+                this._worker.postMessage({ partIndices: partIndices ?? null });
+            }
             this._postToWorker(true);
         }
         else {
+            this._textureSize = textureSize;
             this._covariancesATexture = createTextureFromDataF16(covA, textureSize.x, textureSize.y, 5);
             this._covariancesBTexture = createTextureFromDataF16(covB, textureSize.x, textureSize.y, this._useRGBACovariants ? 5 : 7);
             this._centersTexture = createTextureFromData(this._splatPositions, textureSize.x, textureSize.y, 5);
@@ -1333,10 +1403,27 @@ export class GaussianSplattingMesh extends Mesh {
                     this._shTextures.push(shTexture);
                 }
             }
-            this._instanciateWorker();
+            if (partIndices) {
+                const buffer = new Uint8Array(partIndices);
+                this._partIndicesTexture = createTextureFromDataU8(buffer, textureSize.x, textureSize.y, 6);
+                this._partIndicesTexture.wrapU = 0;
+                this._partIndicesTexture.wrapV = 0;
+            }
+            if (firstTime) {
+                this._instanciateWorker();
+            }
+            else {
+                if (this._worker) {
+                    const positions = Float32Array.from(this._splatPositions);
+                    const vertexCount = this._vertexCount;
+                    this._worker.postMessage({ positions, vertexCount }, [positions.buffer]);
+                    this._worker.postMessage({ partIndices: partIndices ?? null });
+                }
+                this._postToWorker(true);
+            }
         }
     }
-    *_updateData(data, isAsync, sh, options = { flipY: false }) {
+    *_updateData(data, isAsync, sh, partIndices, options = { flipY: false }) {
         // if a covariance texture is present, then it's not a creation but an update
         if (!this._covariancesATexture) {
             this._readyToDisplay = false;
@@ -1346,7 +1433,7 @@ export class GaussianSplattingMesh extends Mesh {
         const fBuffer = new Float32Array(uBuffer.buffer);
         if (this._keepInRam) {
             this._splatsData = data;
-            // keep sh in ram too ?
+            this._shData = sh ? sh.map((arr) => new Uint8Array(arr)) : null;
         }
         const vertexCount = uBuffer.length / GaussianSplattingMesh._RowOutputLength;
         if (vertexCount != this._vertexCount) {
@@ -1363,11 +1450,22 @@ export class GaussianSplattingMesh extends Mesh {
         const covA = new Uint16Array(textureLength * 4);
         const covB = new Uint16Array((this._useRGBACovariants ? 4 : 2) * textureLength);
         const colorArray = new Uint8Array(textureLength * 4);
+        // Ensure that partMatrices.length is at least the maximum part index + 1
+        if (partIndices) {
+            // We always keep part indices in RAM because they are needed for sorting
+            this._partIndices = new Uint8Array(textureLength);
+            this._partIndices.set(partIndices);
+            let maxPartIndex = -1;
+            for (let i = 0; i < partIndices.length; i++) {
+                maxPartIndex = Math.max(maxPartIndex, partIndices[i]);
+            }
+            this._ensureMinimumPartMatricesLength(maxPartIndex + 1);
+        }
         const minimum = new Vector3(Number.MAX_VALUE, Number.MAX_VALUE, Number.MAX_VALUE);
         const maximum = new Vector3(-Number.MAX_VALUE, -Number.MAX_VALUE, -Number.MAX_VALUE);
         if (GaussianSplattingMesh.ProgressiveUpdateAmount) {
             // create textures with not filled-yet array, then update directly portions of it
-            this._updateTextures(covA, covB, colorArray, sh);
+            this._updateTextures(covA, covB, colorArray, sh, this._partIndices ? this._partIndices : undefined);
             this.setEnabled(true);
             const partCount = Math.ceil(textureSize.y / lineCountUpdate);
             for (let partIndex = 0; partIndex < partCount; partIndex++) {
@@ -1388,6 +1486,7 @@ export class GaussianSplattingMesh extends Mesh {
             const vertexCount = this._vertexCount;
             if (this._worker) {
                 this._worker.postMessage({ positions, vertexCount }, [positions.buffer]);
+                this._worker.postMessage({ partIndices });
             }
             this._sortIsDirty = true;
         }
@@ -1404,7 +1503,7 @@ export class GaussianSplattingMesh extends Mesh {
                 this._makeEmptySplat(i, covA, covB, colorArray);
             }
             // textures
-            this._updateTextures(covA, covB, colorArray, sh);
+            this._updateTextures(covA, covB, colorArray, sh, this._partIndices ? this._partIndices : undefined);
             // Update the binfo
             this.getBoundingInfo().reConstruct(minimum, maximum, this.getWorldMatrix());
             this.setEnabled(true);
@@ -1416,20 +1515,22 @@ export class GaussianSplattingMesh extends Mesh {
      * Update asynchronously the buffer
      * @param data array buffer containing center, color, orientation and scale of splats
      * @param sh optional array of uint8 array for SH data
+     * @param partIndices optional array of uint8 for rig node indices
      * @returns a promise
      */
-    async updateDataAsync(data, sh) {
-        return await runCoroutineAsync(this._updateData(data, true, sh), createYieldingScheduler());
+    async updateDataAsync(data, sh, partIndices) {
+        return await runCoroutineAsync(this._updateData(data, true, sh, partIndices), createYieldingScheduler());
     }
     /**
      * @experimental
      * Update data from GS (position, orientation, color, scaling)
      * @param data array that contain all the datas
      * @param sh optional array of uint8 array for SH data
-     * @param options optional informations on how to treat data
+     * @param options optional informations on how to treat data (needs to be 3rd for backward compatibility)
+     * @param partIndices optional array of uint8 for rig node indices
      */
-    updateData(data, sh, options = { flipY: true }) {
-        runCoroutineSync(this._updateData(data, false, sh, options));
+    updateData(data, sh, options = { flipY: true }, partIndices) {
+        runCoroutineSync(this._updateData(data, false, sh, partIndices, options));
     }
     /**
      * Refreshes the bounding info, taking into account all the thin instances defined
@@ -1452,9 +1553,13 @@ export class GaussianSplattingMesh extends Mesh {
                 cameraViewInfos.mesh.thinInstanceSetBuffer("splatIndex", this._splatIndex, 16, false);
             });
         }
+        // Update depthMix
+        if ((!this._depthMix || vertexCount > this._depthMix.length) && !IsNative) {
+            this._depthMix = new BigInt64Array(paddedVertexCount);
+        }
         this.forcedInstanceCount = paddedVertexCount >> 4;
     }
-    _updateSubTextures(centers, covA, covB, colors, lineStart, lineCount, sh) {
+    _updateSubTextures(centers, covA, covB, colors, lineStart, lineCount, sh, partIndices) {
         const updateTextureFromData = (texture, data, width, lineStart, lineCount) => {
             this.getEngine().updateTextureData(texture.getInternalTexture(), data, 0, lineStart, width, lineCount, 0, 0, false);
         };
@@ -1477,6 +1582,10 @@ export class GaussianSplattingMesh extends Mesh {
                 updateTextureFromData(this._shTextures[i], shView, textureSize.x, lineStart, lineCount);
             }
         }
+        if (partIndices && this._partIndicesTexture) {
+            const partIndicesView = new Uint8Array(partIndices.buffer, texelStart, texelCount);
+            updateTextureFromData(this._partIndicesTexture, partIndicesView, textureSize.x, lineStart, lineCount);
+        }
     }
     _instanciateWorker() {
         if (!this._vertexCount) {
@@ -1487,7 +1596,7 @@ export class GaussianSplattingMesh extends Mesh {
         }
         this._updateSplatIndexBuffer(this._vertexCount);
         // no worker in native
-        if (_native) {
+        if (IsNative) {
             return;
         }
         // Start the worker thread
@@ -1495,11 +1604,22 @@ export class GaussianSplattingMesh extends Mesh {
         this._worker = new Worker(URL.createObjectURL(new Blob(["(", GaussianSplattingMesh._CreateWorker.toString(), ")(self)"], {
             type: "application/javascript",
         })));
-        const vertexCountPadded = (this._vertexCount + 15) & ~0xf;
-        this._depthMix = new BigInt64Array(vertexCountPadded);
         const positions = Float32Array.from(this._splatPositions);
+        const partIndices = this._partIndices ? new Uint8Array(this._partIndices) : null;
+        const partMatrices = this._partMatrices.map((matrix) => new Float32Array(matrix.m));
         this._worker.postMessage({ positions }, [positions.buffer]);
+        this._worker.postMessage({ partIndices });
+        this._worker.postMessage({ partMatrices });
         this._worker.onmessage = (e) => {
+            // Recompute vertexCountPadded in case _vertexCount has changed since the last update
+            const vertexCountPadded = (this._vertexCount + 15) & ~0xf;
+            // If the vertex count changed, we discard this result and trigger a new sort
+            if (e.data.depthMix.length != vertexCountPadded) {
+                this._canPostToWorker = true;
+                this._postToWorker(true);
+                this._sortIsDirty = false;
+                return;
+            }
             this._depthMix = e.data.depthMix;
             const cameraId = e.data.cameraId;
             const indexMix = new Uint32Array(e.data.depthMix.buffer);
@@ -1510,7 +1630,7 @@ export class GaussianSplattingMesh extends Mesh {
             }
             if (this._delayedTextureUpdate) {
                 const textureSize = this._getTextureSize(vertexCountPadded);
-                this._updateSubTextures(this._delayedTextureUpdate.centers, this._delayedTextureUpdate.covA, this._delayedTextureUpdate.covB, this._delayedTextureUpdate.colors, 0, textureSize.y, this._delayedTextureUpdate.sh);
+                this._updateSubTextures(this._delayedTextureUpdate.centers, this._delayedTextureUpdate.covA, this._delayedTextureUpdate.covB, this._delayedTextureUpdate.colors, 0, textureSize.y, this._delayedTextureUpdate.sh, this._delayedTextureUpdate.partIndices);
                 this._delayedTextureUpdate = null;
             }
             // get mesh for camera and update its instance buffer
@@ -1552,6 +1672,163 @@ export class GaussianSplattingMesh extends Mesh {
         }
         return new Vector2(width, height);
     }
+    /**
+     * Gets the number of parts in the compound
+     * @returns the number of parts in the compound, or 0 if the mesh is not a compound
+     */
+    get partCount() {
+        return this._partMatrices.length;
+    }
+    /**
+     * Sets the world matrix for a specific part of the compound (if this mesh is a compound).
+     * This will trigger a re-sort of the mesh.
+     * @param partIndex index of the part, that must be between 0 and partCount - 1
+     * @param worldMatrix the world matrix to set
+     */
+    setWorldMatrixForPart(partIndex, worldMatrix) {
+        this._partMatrices[partIndex].copyFrom(worldMatrix);
+        if (this._worker) {
+            this._worker.postMessage({ partMatrices: this._partMatrices.map((matrix) => new Float32Array(matrix.m)) });
+        }
+        this._postToWorker(true);
+    }
+    /**
+     * Gets the world matrix for a specific part of the compound (if this mesh is a compound).
+     * @param partIndex index of the part, that must be between 0 and partCount - 1
+     * @returns the world matrix for the part, or the current world matrix of the mesh if the mesh is not a compound
+     */
+    getWorldMatrixForPart(partIndex) {
+        return this._partMatrices[partIndex] ?? this.getWorldMatrix();
+    }
+    /**
+     * Ensure that the part world matrix array is at least the given length.
+     * NB: This length is used as reference for the number of parts in the compound.
+     * Newly inserted parts are initialized with the current world matrix of the mesh.
+     * @param length - The minimum length to ensure
+     */
+    _ensureMinimumPartMatricesLength(length) {
+        if (this._partMatrices.length < length) {
+            this._resizePartMatrices(length);
+        }
+    }
+    /**
+     * This sets the number of parts in the compound.
+     * Warning: This must be consistent with the indices used in the partIndices texture.
+     * Newly inserted parts are initialized with the current world matrix of the mesh.
+     * @param length - The length to resize to
+     */
+    _resizePartMatrices(length) {
+        if (this._partMatrices.length == length) {
+            return;
+        }
+        else if (this._partMatrices.length > length) {
+            this._partMatrices = this._partMatrices.slice(0, length);
+        }
+        else {
+            this.computeWorldMatrix(true);
+            const defaultMatrix = this.getWorldMatrix();
+            while (this._partMatrices.length < length) {
+                this._partMatrices.push(defaultMatrix.clone());
+            }
+        }
+        if (this._worker) {
+            this._worker.postMessage({ partMatrices: this._partMatrices.map((matrix) => new Float32Array(matrix.m)) });
+        }
+        this._postToWorker(true);
+    }
+    /**
+     * Add another mesh to this mesh, as a new part. This makes the current mesh a compound, if not already.
+     * NB: The current mesh needs to be loaded with keepInRam: true.
+     * @param other - The other mesh to add. This must be loaded with keepInRam: true.
+     * @param disposeOther - Whether to dispose the other mesh after adding it to the current mesh.
+     * @returns a placeholder mesh that can be used to manipulate the part transform
+     */
+    addPart(other, disposeOther = true) {
+        const splatCountA = this._vertexCount;
+        const splatsDataA = splatCountA == 0 ? new ArrayBuffer(0) : this.splatsData;
+        const shDataA = this.shData;
+        const splatCountB = other._vertexCount;
+        const splatsDataB = other.splatsData;
+        const shDataB = other.shData;
+        const mergedShDataLength = Math.max(shDataA?.length || 0, shDataB?.length || 0);
+        const hasMergedShData = shDataA !== null && shDataB !== null;
+        // Sanity checks
+        if (!splatsDataA) {
+            throw new Error(`To call addPart(), the current mesh must be loaded with keepInRam: true`);
+        }
+        const expectedSplatsDataSizeA = splatCountA * GaussianSplattingMesh._RowOutputLength;
+        if (splatsDataA.byteLength !== expectedSplatsDataSizeA) {
+            throw new Error(`splatsDataA size (${splatsDataA.byteLength}) does not match expected size (${expectedSplatsDataSizeA})`);
+        }
+        if (!splatsDataB) {
+            throw new Error(`To call addPart(), the other mesh must be loaded with keepInRam: true`);
+        }
+        const expectedSplatsDataSizeB = splatCountB * GaussianSplattingMesh._RowOutputLength;
+        if (splatsDataB.byteLength !== expectedSplatsDataSizeB) {
+            throw new Error(`splatsDataB size (${splatsDataB.byteLength}) does not match expected size (${expectedSplatsDataSizeB})`);
+        }
+        if (other.partIndices) {
+            throw new Error(`To call addPart(), the other mesh must not be a compound`);
+        }
+        // Concatenate splatsData (ArrayBuffer)
+        const mergedSplatsData = new Uint8Array(splatsDataA.byteLength + splatsDataB.byteLength);
+        mergedSplatsData.set(new Uint8Array(splatsDataA), 0);
+        mergedSplatsData.set(new Uint8Array(splatsDataB), splatsDataA.byteLength);
+        let mergedShData = undefined;
+        if (hasMergedShData) {
+            // Note: We need to calculate the texture size and pad accordingly
+            // Each SH texture texel stores 16 bytes (4 RGBA uint32 components)
+            const bytesPerTexel = 16;
+            const totalSplatCount = splatCountA + splatCountB;
+            mergedShData = [];
+            for (let i = 0; i < mergedShDataLength; i++) {
+                const mergedShDataItem = new Uint8Array(totalSplatCount * bytesPerTexel);
+                if (i < (shDataA?.length ?? 0)) {
+                    mergedShDataItem.set(shDataA[i], 0);
+                }
+                if (i < (shDataB?.length ?? 0)) {
+                    const byteOffset = bytesPerTexel * splatCountA;
+                    mergedShDataItem.set(shDataB[i], byteOffset);
+                }
+                mergedShData.push(mergedShDataItem);
+            }
+        }
+        // Concatenate partIndices (Uint8Array)
+        let newPartIndex = this.partCount;
+        let partIndicesA = this.partIndices;
+        if (!partIndicesA) {
+            partIndicesA = new Uint8Array(splatCountA);
+            newPartIndex = splatCountA > 0 ? 1 : 0;
+            //newPartIndex = 1;
+        }
+        if (partIndicesA.length < splatCountA) {
+            throw new Error(`partIndices length (${partIndicesA.length}) should be at least vertexCount (${splatCountA}) in the current mesh`);
+        }
+        const partIndicesB = new Uint8Array(splatCountB).fill(newPartIndex);
+        const mergedPartIndices = new Uint8Array(splatCountA + splatCountB);
+        mergedPartIndices.set(partIndicesA.slice(0, splatCountA), 0);
+        mergedPartIndices.set(partIndicesB, splatCountA);
+        this.updateData(mergedSplatsData.buffer, mergedShData, { flipY: false }, mergedPartIndices);
+        // Merge part matrices (TODO)
+        const partWorldMatrix = other.getWorldMatrix();
+        this.setWorldMatrixForPart(newPartIndex, partWorldMatrix);
+        // Create a placeholder mesh to manipulate the part transform
+        // Remove splats from the original mesh
+        if (disposeOther) {
+            other.dispose();
+        }
+        const placeholderMesh = new Mesh(other.name, this.getScene());
+        placeholderMesh.onAfterWorldMatrixUpdateObservable.add(() => {
+            this.setWorldMatrixForPart(newPartIndex, placeholderMesh.getWorldMatrix());
+        });
+        // Directly set the world matrix using freezeWorldMatrix
+        const quaternion = new Quaternion();
+        partWorldMatrix.decompose(placeholderMesh.scaling, quaternion, placeholderMesh.position);
+        placeholderMesh.rotationQuaternion = quaternion;
+        placeholderMesh.computeWorldMatrix(true);
+        placeholderMesh.metadata = { partIndex: newPartIndex };
+        return placeholderMesh;
+    }
 }
 GaussianSplattingMesh._RowOutputLength = 3 * 4 + 3 * 4 + 4 + 4; // Vector3 position, Vector3 scale, 1 u8 quaternion, 1 color with alpha
 GaussianSplattingMesh._SH_C0 = 0.28209479177387814;
@@ -1572,17 +1849,39 @@ GaussianSplattingMesh._CreateWorker = function (self) {
     let depthMix;
     let indices;
     let floatMix;
+    let partIndices;
+    let partMatrices;
+    function multiplyMatrices(matrix1, matrix2) {
+        const result = new Float32Array(16);
+        for (let i = 0; i < 4; i++) {
+            for (let j = 0; j < 4; j++) {
+                for (let k = 0; k < 4; k++) {
+                    result[j * 4 + i] += matrix1[k * 4 + i] * matrix2[j * 4 + k];
+                }
+            }
+        }
+        return result;
+    }
     self.onmessage = (e) => {
         // updated on init
         if (e.data.positions) {
             positions = e.data.positions;
         }
-        // udpate on view changed
+        // update on rig node changed
+        else if (e.data.partMatrices) {
+            partMatrices = e.data.partMatrices;
+        }
+        // update on rig node indices changed
+        else if (e.data.partIndices !== undefined) {
+            partIndices = e.data.partIndices;
+        }
+        // update on view changed
         else {
             const cameraId = e.data.cameraId;
-            const modelViewProjection = e.data.modelViewProjection;
+            const globalModelViewProjection = e.data.modelViewProjection;
+            const viewProjection = e.data.viewProjection;
             const vertexCountPadded = (positions.length / 4 + 15) & ~0xf;
-            if (!positions || !modelViewProjection) {
+            if (!positions || !globalModelViewProjection) {
                 // Sanity check, it shouldn't happen!
                 throw new Error("positions or modelViewProjection matrix is not defined!");
             }
@@ -1593,9 +1892,29 @@ GaussianSplattingMesh._CreateWorker = function (self) {
             for (let j = 0; j < vertexCountPadded; j++) {
                 indices[2 * j] = j;
             }
-            for (let j = 0; j < vertexCountPadded; j++) {
-                floatMix[2 * j + 1] =
-                    10000 - (modelViewProjection[2] * positions[4 * j + 0] + modelViewProjection[6] * positions[4 * j + 1] + modelViewProjection[10] * positions[4 * j + 2]);
+            let depthFactor = -1;
+            if (e.data.useRightHandedSystem) {
+                depthFactor = 1;
+            }
+            if (partMatrices && partIndices) {
+                // If there are rig node matrices, we use them instead of the global model view proj
+                // Precompute modelViewProj for each rig node
+                const modelViewProjs = partMatrices.map((model) => multiplyMatrices(viewProjection, model));
+                // NB: For performance reasons, we assume that part indices are valid
+                const length = partIndices.length;
+                for (let j = 0; j < vertexCountPadded; j++) {
+                    // NB: We need this 'min' because vertex array is padded, not partIndices
+                    const partIndex = partIndices[Math.min(j, length - 1)];
+                    const mvp = modelViewProjs[partIndex];
+                    floatMix[2 * j + 1] = 10000 + (mvp[2] * positions[4 * j + 0] + mvp[6] * positions[4 * j + 1] + mvp[10] * positions[4 * j + 2] + mvp[14]) * depthFactor;
+                }
+            }
+            else {
+                // If there are no rig node matrices, we use the global model view proj
+                const mvp = globalModelViewProjection;
+                for (let j = 0; j < vertexCountPadded; j++) {
+                    floatMix[2 * j + 1] = 10000 + (mvp[2] * positions[4 * j + 0] + mvp[6] * positions[4 * j + 1] + mvp[10] * positions[4 * j + 2] + mvp[14]) * depthFactor;
+                }
             }
             depthMix.sort();
             self.postMessage({ depthMix, cameraId }, [depthMix.buffer]);