@cesium/engine 18.3.1-ion.1 → 20.0.0

package/Source/Scene/GaussianSplat3DTileContent.js

@@ -7,16 +7,16 @@ import GaussianSplatPrimitive from "./GaussianSplatPrimitive.js";
 import destroyObject from "../Core/destroyObject.js";
 import ModelUtility from "./Model/ModelUtility.js";
 import VertexAttributeSemantic from "./VertexAttributeSemantic.js";
+import deprecationWarning from "../Core/deprecationWarning.js";
 
 /**
- * Represents the contents of a glTF or glb using the {@link https://github.com/CesiumGS/glTF/tree/draft-spz-splat-compression/extensions/2.0/Khronos/KHR_spz_gaussian_splats_compression|KHR_spz_gaussian_splats_compression} extension.
+ * Represents the contents of a glTF or glb using the {@link https://github.com/CesiumGS/glTF/tree/draft-splat-spz/extensions/2.0/Khronos/KHR_gaussian_splatting | KHR_gaussian_splatting} and {@link https://github.com/CesiumGS/glTF/tree/draft-splat-spz/extensions/2.0/Khronos/KHR_gaussian_splatting_compression_spz_2 | KHR_gaussian_splatting_compression_spz_2} extensions.
  * <p>
  * Implements the {@link Cesium3DTileContent} interface.
  * </p>
  *
  * @alias GaussianSplat3DTileContent
  * @constructor
- * @private
  */
 function GaussianSplat3DTileContent(loader, tileset, tile, resource) {
   this._tileset = tileset;
@@ -46,7 +46,7 @@ function GaussianSplat3DTileContent(loader, tileset, tile, resource) {
    * @type {undefined|Primitive}
    * @private
    */
-  this.splatPrimitive = undefined;
+  this.gltfPrimitive = undefined;
 
   /**
    * Transform matrix to turn model coordinates into world coordinates.
@@ -78,8 +78,66 @@ function GaussianSplat3DTileContent(loader, tileset, tile, resource) {
    * @private
    */
   this._transformed = false;
+
+  /**
+   * The degree of the spherical harmonics used for the Gaussian splats.
+   * @type {number}
+   * @private
+   */
+  this._sphericalHarmonicsDegree = 0;
+
+  /**
+   * The number of spherical harmonic coefficients used for the Gaussian splats.
+   * @type {number}
+   * @private
+   */
+  this._sphericalHarmonicsCoefficientCount = 0;
+
+  /**
+   * Spherical Harmonic data that has been packed for use in a texture or shader.
+   * @type {undefined|Uint32Array}
+   * @private
+   */
+  this._packedSphericalHarmonicsData = undefined;
 }
 
+/**
+ * Performs checks to ensure that the provided tileset has the Gaussian Splatting extensions.
+ *
+ * @param {Cesium3DTileset} tileset The tileset to check for the extensions.
+ * @returns {boolean} Returns <code>true</code> if the necessary extensions are included in the tileset.
+ * @static
+ */
+GaussianSplat3DTileContent.tilesetRequiresGaussianSplattingExt = function (
+  tileset,
+) {
+  let hasGaussianSplatExtension = false;
+  let hasLegacyGaussianSplatExtension = false;
+  if (tileset.isGltfExtensionRequired instanceof Function) {
+    hasGaussianSplatExtension =
+      tileset.isGltfExtensionRequired("KHR_gaussian_splatting") &&
+      tileset.isGltfExtensionRequired(
+        "KHR_gaussian_splatting_compression_spz_2",
+      );
+
+    hasLegacyGaussianSplatExtension = tileset.isGltfExtensionRequired(
+      "KHR_spz_gaussian_splats_compression",
+    );
+  }
+
+  if (hasLegacyGaussianSplatExtension) {
+    deprecationWarning(
+      "KHR_spz_gaussian_splats_compression",
+      "Support for the original KHR_spz_gaussian_splats_compression extension has been deprecated in favor " +
+        "of the up to date KHR_gaussian_splatting and KHR_gaussian_splatting_compression_spz_2 extensions and will be " +
+        "removed in CesiumJS 1.135.\n\nPlease retile your tileset with the KHR_gaussian_splatting and " +
+        "KHR_gaussian_splatting_compression_spz_2 extensions.",
+    );
+  }
+
+  return hasGaussianSplatExtension || hasLegacyGaussianSplatExtension;
+};
+
 Object.defineProperties(GaussianSplat3DTileContent.prototype, {
   /**
    * Gets the number of features in the tile. Currently this is always zero.
@@ -106,7 +164,7 @@ Object.defineProperties(GaussianSplat3DTileContent.prototype, {
    */
   pointsLength: {
     get: function () {
-      return this.splatPrimitive.attributes[0].count;
+      return this.gltfPrimitive.attributes[0].count;
     },
   },
   /**
@@ -131,7 +189,7 @@ Object.defineProperties(GaussianSplat3DTileContent.prototype, {
    */
   geometryByteLength: {
     get: function () {
-      return this.splatPrimitive.attributes.reduce((totalLength, attribute) => {
+      return this.gltfPrimitive.attributes.reduce((totalLength, attribute) => {
         return totalLength + attribute.byteLength;
       }, 0);
     },
@@ -309,8 +367,188 @@ Object.defineProperties(GaussianSplat3DTileContent.prototype, {
       this._group = value;
     },
   },
+
+  /**
+   * The number of spherical harmonic coefficients used for the Gaussian splats.
+   * @type {number}
+   * @private
+   * @experimental This feature is using part of the 3D Tiles spec that is not final and is subject to change without Cesium's standard deprecation policy.
+   */
+  sphericalHarmonicsCoefficientCount: {
+    get: function () {
+      return this._sphericalHarmonicsCoefficientCount;
+    },
+  },
+  /**
+   * The degree of the spherical harmonics used for the Gaussian splats.
+   * @type {number}
+   * @private
+   * @experimental This feature is using part of the 3D Tiles spec that is not final and is subject to change without Cesium's standard deprecation policy.
+   */
+  sphericalHarmonicsDegree: {
+    get: function () {
+      return this._sphericalHarmonicsDegree;
+    },
+  },
+
+  /**
+   * The packed spherical harmonic data for the Gaussian splats for use a shader or texture.
+   * @type {number}
+   * @private
+   * @experimental This feature is using part of the 3D Tiles spec that is not final and is subject to change without Cesium's standard deprecation policy.
+   */
+  packedSphericalHarmonicsData: {
+    get: function () {
+      return this._packedSphericalHarmonicsData;
+    },
+  },
 });
 
+function getShAttributePrefix(attribute) {
+  const prefix = attribute.startsWith("KHR_gaussian_splatting:")
+    ? "KHR_gaussian_splatting:"
+    : "_";
+  return `${prefix}SH_DEGREE_`;
+}
+
+/**
+ * Determine Spherical Harmonics degree and coefficient count from attributes
+ * @param {Attribute[]} attributes - The list of glTF attributes.
+ * @returns {object} An object containing the degree (l) and coefficient (n).
+ * @private
+ */
+function degreeAndCoefFromAttributes(attributes) {
+  const shAttributes = attributes.filter((attr) =>
+    attr.name.includes("SH_DEGREE_"),
+  );
+
+  switch (shAttributes.length) {
+    default:
+    case 0:
+      return { l: 0, n: 0 };
+    case 3:
+      return { l: 1, n: 9 };
+    case 8:
+      return { l: 2, n: 24 };
+    case 15:
+      return { l: 3, n: 45 };
+  }
+}
+
+/**
+ * Converts a 32-bit floating point number to a 16-bit floating point number.
+ * @param {number} float32 input
+ * @returns {number} Half precision float
+ * @private
+ */
+const buffer = new ArrayBuffer(4);
+const floatView = new Float32Array(buffer);
+const intView = new Uint32Array(buffer);
+
+function float32ToFloat16(float32) {
+  floatView[0] = float32;
+  const bits = intView[0];
+
+  const sign = (bits >> 31) & 0x1;
+  const exponent = (bits >> 23) & 0xff;
+  const mantissa = bits & 0x7fffff;
+
+  let half;
+
+  if (exponent === 0xff) {
+    half = (sign << 15) | (0x1f << 10) | (mantissa ? 0x200 : 0);
+  } else if (exponent === 0) {
+    half = sign << 15;
+  } else {
+    const newExponent = exponent - 127 + 15;
+    if (newExponent >= 31) {
+      half = (sign << 15) | (0x1f << 10);
+    } else if (newExponent <= 0) {
+      half = sign << 15;
+    } else {
+      half = (sign << 15) | (newExponent << 10) | (mantissa >>> 13);
+    }
+  }
+
+  return half;
+}
+
+/**
+ * Extracts the spherical harmonic degree and coefficient from the attribute name.
+ * @param {string} attribute - The attribute name.
+ * @returns {object} An object containing the degree (l) and coefficient (n).
+ * @private
+ */
+function extractSHDegreeAndCoef(attribute) {
+  const prefix = getShAttributePrefix(attribute);
+  const separator = "_COEF_";
+
+  const lStart = prefix.length;
+  const coefIndex = attribute.indexOf(separator, lStart);
+
+  const l = parseInt(attribute.slice(lStart, coefIndex), 10);
+  const n = parseInt(attribute.slice(coefIndex + separator.length), 10);
+
+  return { l, n };
+}
+
+/**
+ * Packs spherical harmonic data into half-precision floats.
+ * @param {GaussianSplat3DTileContent} tileContent - The tile content containing the spherical harmonic data.
+ * @returns {Uint32Array} - The Float16 packed spherical harmonic data.
+ * @private
+ */
+function packSphericalHarmonicsData(tileContent) {
+  const degree = tileContent.sphericalHarmonicsDegree;
+  const coefs = tileContent.sphericalHarmonicsCoefficientCount;
+  const totalLength = tileContent.pointsLength * (coefs * (2 / 3)); //3 packs into 2
+  const packedData = new Uint32Array(totalLength);
+
+  const shAttributes = tileContent.gltfPrimitive.attributes.filter((attr) =>
+    attr.name.includes("SH_DEGREE_"),
+  );
+  let stride = 0;
+  const base = [0, 9, 24];
+  switch (degree) {
+    case 1:
+      stride = 9;
+      break;
+    case 2:
+      stride = 24;
+      break;
+    case 3:
+      stride = 45;
+      break;
+  }
+  shAttributes.sort((a, b) => {
+    if (a.name < b.name) {
+      return -1;
+    }
+    if (a.name > b.name) {
+      return 1;
+    }
+
+    return 0;
+  });
+  const packedStride = stride * (2 / 3);
+  for (let i = 0; i < shAttributes.length; i++) {
+    const { l, n } = extractSHDegreeAndCoef(shAttributes[i].name);
+    for (let j = 0; j < tileContent.pointsLength; j++) {
+      //interleave the data
+      const packedBase = (base[l - 1] * 2) / 3;
+      const idx = j * packedStride + packedBase + n * 2;
+      const src = j * 3;
+      packedData[idx] =
+        float32ToFloat16(shAttributes[i].typedArray[src]) |
+        (float32ToFloat16(shAttributes[i].typedArray[src + 1]) << 16);
+      packedData[idx + 1] = float32ToFloat16(
+        shAttributes[i].typedArray[src + 2],
+      );
+    }
+  }
+  return packedData;
+}
+
 /**
  * Creates a new instance of {@link GaussianSplat3DTileContent} from a glTF or glb resource.
  *
@@ -387,31 +625,37 @@ GaussianSplat3DTileContent.prototype.update = function (primitive, frameState) {
   }
 
   if (this._resourcesLoaded) {
-    this.splatPrimitive = loader.components.scene.nodes[0].primitives[0];
+    this.gltfPrimitive = loader.components.scene.nodes[0].primitives[0];
     this.worldTransform = loader.components.scene.nodes[0].matrix;
     this._ready = true;
 
     this._originalPositions = new Float32Array(
       ModelUtility.getAttributeBySemantic(
-        this.splatPrimitive,
+        this.gltfPrimitive,
         VertexAttributeSemantic.POSITION,
       ).typedArray,
     );
 
     this._originalRotations = new Float32Array(
       ModelUtility.getAttributeBySemantic(
-        this.splatPrimitive,
+        this.gltfPrimitive,
         VertexAttributeSemantic.ROTATION,
       ).typedArray,
     );
 
     this._originalScales = new Float32Array(
       ModelUtility.getAttributeBySemantic(
-        this.splatPrimitive,
+        this.gltfPrimitive,
         VertexAttributeSemantic.SCALE,
       ).typedArray,
     );
 
+    const { l, n } = degreeAndCoefFromAttributes(this.gltfPrimitive.attributes);
+    this._sphericalHarmonicsDegree = l;
+    this._sphericalHarmonicsCoefficientCount = n;
+
+    this._packedSphericalHarmonicsData = packSphericalHarmonicsData(this);
+
     return;
   }
 

package/Source/Scene/GaussianSplatPrimitive.js

@@ -31,10 +31,12 @@ import Cartesian3 from "../Core/Cartesian3.js";
 import Quaternion from "../Core/Quaternion.js";
 import SplitDirection from "./SplitDirection.js";
 import destroyObject from "../Core/destroyObject.js";
+import ContextLimits from "../Renderer/ContextLimits.js";
 
 const scratchMatrix4A = new Matrix4();
 const scratchMatrix4B = new Matrix4();
 const scratchMatrix4C = new Matrix4();
+const scratchMatrix4D = new Matrix4();
 
 const GaussianSplatSortingState = {
   IDLE: 0,
@@ -44,6 +46,25 @@ const GaussianSplatSortingState = {
   ERROR: 4,
 };
 
+function createSphericalHarmonicsTexture(context, shData) {
+  const texture = new Texture({
+    context: context,
+    source: {
+      width: shData.width,
+      height: shData.height,
+      arrayBufferView: shData.data,
+    },
+    preMultiplyAlpha: false,
+    skipColorSpaceConversion: true,
+    pixelFormat: PixelFormat.RG_INTEGER,
+    pixelDatatype: PixelDatatype.UNSIGNED_INT,
+    flipY: false,
+    sampler: Sampler.NEAREST,
+  });
+
+  return texture;
+}
+
 function createGaussianSplatTexture(context, splatTextureData) {
   return new Texture({
     context: context,
@@ -64,7 +85,7 @@ function createGaussianSplatTexture(context, splatTextureData) {
 /** A primitive that renders Gaussian splats.
  * <p>
  * This primitive is used to render Gaussian splats in a 3D Tileset.
- * It is designed to work with the KHR_spz_gaussian_splats_compression extension.
+ * It is designed to work with the KHR_gaussian_splatting and KHR_gaussian_splatting_compression_spz_2 extensions.
  * </p>
  * @alias GaussianSplatPrimitive
  * @constructor
@@ -148,6 +169,14 @@ function GaussianSplatPrimitive(options) {
    * @see {@link GaussianSplatTextureGenerator}
    */
   this.gaussianSplatTexture = undefined;
+
+  /**
+   * The texture used to store the spherical harmonics coefficients for the Gaussian splats.
+   * @type {undefined|Texture}
+   * @private
+   */
+  this.sphericalHarmonicsTexture = undefined;
+
   /**
    * The last width of the Gaussian splat texture.
    * This is used to track changes in the texture size and update the primitive accordingly.
@@ -395,7 +424,7 @@ GaussianSplatPrimitive.prototype.onTileVisible = function (tile) {};
  */
 GaussianSplatPrimitive.transformTile = function (tile) {
   const computedTransform = tile.computedTransform;
-  const splatPrimitive = tile.content.splatPrimitive;
+  const gltfPrimitive = tile.content.gltfPrimitive;
   const gaussianSplatPrimitive = tile.tileset.gaussianSplatPrimitive;
 
   const computedModelMatrix = Matrix4.multiplyTransformation(
@@ -425,17 +454,17 @@ GaussianSplatPrimitive.transformTile = function (tile) {
   const rotations = tile.content._originalRotations;
   const scales = tile.content._originalScales;
   const attributePositions = ModelUtility.getAttributeBySemantic(
-    splatPrimitive,
+    gltfPrimitive,
     VertexAttributeSemantic.POSITION,
   ).typedArray;
 
   const attributeRotations = ModelUtility.getAttributeBySemantic(
-    splatPrimitive,
+    gltfPrimitive,
     VertexAttributeSemantic.ROTATION,
   ).typedArray;
 
   const attributeScales = ModelUtility.getAttributeBySemantic(
-    splatPrimitive,
+    gltfPrimitive,
     VertexAttributeSemantic.SCALE,
   ).typedArray;
 
@@ -601,12 +630,69 @@ GaussianSplatPrimitive.buildGSplatDrawCommand = function (
     ShaderDestination.VERTEX,
   );
 
+  shaderBuilder.addUniform(
+    "float",
+    "u_sphericalHarmonicsDegree",
+    ShaderDestination.VERTEX,
+  );
+
+  shaderBuilder.addUniform("float", "u_splatScale", ShaderDestination.VERTEX);
+
+  shaderBuilder.addUniform(
+    "vec3",
+    "u_cameraPositionWC",
+    ShaderDestination.VERTEX,
+  );
+
+  shaderBuilder.addUniform(
+    "mat3",
+    "u_inverseModelRotation",
+    ShaderDestination.VERTEX,
+  );
+
   const uniformMap = renderResources.uniformMap;
 
   uniformMap.u_splatAttributeTexture = function () {
     return primitive.gaussianSplatTexture;
   };
 
+  if (primitive._sphericalHarmonicsDegree > 0) {
+    shaderBuilder.addDefine(
+      "HAS_SPHERICAL_HARMONICS",
+      "1",
+      ShaderDestination.VERTEX,
+    );
+    shaderBuilder.addUniform(
+      "highp usampler2D",
+      "u_sphericalHarmonicsTexture",
+      ShaderDestination.VERTEX,
+    );
+    uniformMap.u_sphericalHarmonicsTexture = function () {
+      return primitive.sphericalHarmonicsTexture;
+    };
+  }
+  uniformMap.u_sphericalHarmonicsDegree = function () {
+    return primitive._sphericalHarmonicsDegree;
+  };
+
+  uniformMap.u_cameraPositionWC = function () {
+    return Cartesian3.clone(frameState.camera.positionWC);
+  };
+
+  uniformMap.u_inverseModelRotation = function () {
+    const tileset = primitive._tileset;
+    const modelMatrix = Matrix4.multiply(
+      tileset.modelMatrix,
+      Matrix4.fromArray(tileset.root.transform),
+      scratchMatrix4A,
+    );
+    const inverseModelRotation = Matrix4.getRotation(
+      Matrix4.inverse(modelMatrix, scratchMatrix4C),
+      scratchMatrix4D,
+    );
+    return inverseModelRotation;
+  };
+
   uniformMap.u_splitDirection = function () {
     return primitive.splitDirection;
   };
@@ -720,7 +806,11 @@ GaussianSplatPrimitive.prototype.update = function (frameState) {
     this._rootTransform = tileset.root.computedTransform;
   }
 
-  if (this._drawCommand && tileset.show) {
+  if (!tileset.show || tileset._selectedTiles.length === 0) {
+    return;
+  }
+
+  if (this._drawCommand) {
     frameState.commandList.push(this._drawCommand);
   }
 
@@ -756,6 +846,7 @@ GaussianSplatPrimitive.prototype.update = function (frameState) {
       this._scales = undefined;
       this._colors = undefined;
       this._indexes = undefined;
+      this._shData = undefined;
       this._needsGaussianSplatTexture = true;
       this._gaussianSplatTexturePending = false;
 
@@ -771,7 +862,7 @@ GaussianSplatPrimitive.prototype.update = function (frameState) {
         let aggregate;
         let offset = 0;
         for (const tile of tiles) {
-          const primitive = tile.content.splatPrimitive;
+          const primitive = tile.content.gltfPrimitive;
           const attribute = getAttributeCallback(primitive);
           if (!defined(aggregate)) {
             aggregate = ComponentDatatype.createTypedArray(
@@ -786,42 +877,71 @@ GaussianSplatPrimitive.prototype.update = function (frameState) {
         return aggregate;
       };
 
+      const aggregateShData = () => {
+        let offset = 0;
+        for (const tile of tiles) {
+          const shData = tile.content.packedSphericalHarmonicsData;
+          if (tile.content.sphericalHarmonicsDegree > 0) {
+            if (!defined(this._shData)) {
+              let coefs;
+              switch (tile.content.sphericalHarmonicsDegree) {
+                case 1:
+                  coefs = 9;
+                  break;
+                case 2:
+                  coefs = 24;
+                  break;
+                case 3:
+                  coefs = 45;
+              }
+              this._shData = new Uint32Array(totalElements * (coefs * (2 / 3)));
+            }
+            this._shData.set(shData, offset);
+            offset += shData.length;
+          }
+        }
+      };
+
       this._positions = aggregateAttributeValues(
         ComponentDatatype.FLOAT,
-        (splatPrimitive) =>
+        (gltfPrimitive) =>
           ModelUtility.getAttributeBySemantic(
-            splatPrimitive,
+            gltfPrimitive,
             VertexAttributeSemantic.POSITION,
           ),
       );
 
       this._scales = aggregateAttributeValues(
         ComponentDatatype.FLOAT,
-        (splatPrimitive) =>
+        (gltfPrimitive) =>
           ModelUtility.getAttributeBySemantic(
-            splatPrimitive,
+            gltfPrimitive,
             VertexAttributeSemantic.SCALE,
           ),
       );
 
       this._rotations = aggregateAttributeValues(
         ComponentDatatype.FLOAT,
-        (splatPrimitive) =>
+        (gltfPrimitive) =>
           ModelUtility.getAttributeBySemantic(
-            splatPrimitive,
+            gltfPrimitive,
             VertexAttributeSemantic.ROTATION,
           ),
       );
 
       this._colors = aggregateAttributeValues(
         ComponentDatatype.UNSIGNED_BYTE,
-        (splatPrimitive) =>
+        (gltfPrimitive) =>
           ModelUtility.getAttributeBySemantic(
-            splatPrimitive,
+            gltfPrimitive,
             VertexAttributeSemantic.COLOR,
           ),
       );
 
+      aggregateShData();
+      this._sphericalHarmonicsDegree =
+        tiles[0].content.sphericalHarmonicsDegree;
+
       this._numSplats = totalElements;
       this.selectedTileLength = tileset._selectedTiles.length;
     }
@@ -833,6 +953,38 @@ GaussianSplatPrimitive.prototype.update = function (frameState) {
     if (this._needsGaussianSplatTexture) {
       if (!this._gaussianSplatTexturePending) {
         GaussianSplatPrimitive.generateSplatTexture(this, frameState);
+        if (defined(this._shData)) {
+          const oldTex = this.sphericalHarmonicsTexture;
+          const width = ContextLimits.maximumTextureSize;
+          const dims =
+            tileset._selectedTiles[0].content
+              .sphericalHarmonicsCoefficientCount / 3;
+          const splatsPerRow = Math.floor(width / dims);
+          const floatsPerRow = splatsPerRow * (dims * 2);
+          const texBuf = new Uint32Array(
+            width * Math.ceil(this._numSplats / splatsPerRow) * 2,
+          );
+
+          let dataIndex = 0;
+          for (let i = 0; dataIndex < this._shData.length; i += width * 2) {
+            texBuf.set(
+              this._shData.subarray(dataIndex, dataIndex + floatsPerRow),
+              i,
+            );
+            dataIndex += floatsPerRow;
+          }
+          this.sphericalHarmonicsTexture = createSphericalHarmonicsTexture(
+            frameState.context,
+            {
+              data: texBuf,
+              width: width,
+              height: Math.ceil(this._numSplats / splatsPerRow),
+            },
+          );
+          if (defined(oldTex)) {
+            oldTex.destroy();
+          }
+        }
       }
       return;
     }