@cesium/engine 18.0.0 → 18.0.1-splats.0

package/Source/Scene/GaussianSplatRenderResources.js

@@ -0,0 +1,81 @@
+import ShaderBuilder from "../Renderer/ShaderBuilder.js";
+import RenderState from "../Renderer/RenderState.js";
+import DepthFunction from "../Scene/DepthFunction.js";
+import ModelAlphaOptions from "./Model/ModelAlphaOptions.js";
+import ShaderDestination from "../Renderer/ShaderDestination.js";
+
+function GaussianSplatRenderResources(primitive) {
+  const shaderBuilder = new ShaderBuilder();
+  /**
+   * An object used to build a shader incrementally. Each pipeline stage
+   * may add lines of shader code to this object.
+   *
+   * @type {ShaderBuilder}
+   * @readonly
+   *
+   * @private
+   */
+  this.shaderBuilder = shaderBuilder;
+
+  /**
+   * A dictionary mapping uniform name to functions that return the uniform
+   * values.
+   *
+   * @type {Object<string, Function>}
+   * @readonly
+   *
+   * @private
+   */
+  this.uniformMap = {};
+
+  /**
+   * An object storing options for creating a {@link RenderState}.
+   * The pipeline stages simply set the options, the render state is created
+   * when the {@link DrawCommand} is constructed.
+   *
+   * @type {object}
+   * @readonly
+   *
+   * @private
+   */
+  this.renderStateOptions = RenderState.getState(
+    RenderState.fromCache({
+      depthTest: {
+        enabled: true,
+        func: DepthFunction.LESS_OR_EQUAL,
+      },
+    }),
+  );
+
+  /**
+   * Options for configuring the alpha stage such as pass and alpha cutoff.
+   *
+   * @type {ModelAlphaOptions}
+   * @readonly
+   *
+   * @private
+   */
+  this.alphaOptions = new ModelAlphaOptions();
+
+  /**
+   * Whether the model is part of a tileset that uses the skipLevelOfDetail
+   * optimization. This value indicates what draw commands are needed and
+   * is set by TilesetPipelineStage.
+   *
+   * @type {boolean}
+   * @default false
+   *
+   * @private
+   */
+  this.hasSkipLevelOfDetail = false;
+
+  if (primitive._useLogDepth) {
+    shaderBuilder.addDefine(
+      "LOG_DEPTH_READ_ONLY",
+      undefined,
+      ShaderDestination.FRAGMENT,
+    );
+  }
+}
+
+export default GaussianSplatRenderResources;

package/Source/Scene/GaussianSplatSorter.js

@@ -0,0 +1,73 @@
+import defined from "../Core/defined.js";
+import FeatureDetection from "../Core/FeatureDetection.js";
+import RuntimeError from "../Core/RuntimeError.js";
+import TaskProcessor from "../Core/TaskProcessor.js";
+
+/** * A sorter for Gaussian splats that uses a task processor to handle sorting in parallel.
+ * This class is responsible for initializing the task processor and scheduling sorting tasks.
+ * * @constructor
+ * @private
+ */
+function GaussianSplatSorter() {}
+
+GaussianSplatSorter._maxSortingConcurrency = Math.max(
+  FeatureDetection.hardwareConcurrency - 1,
+  1,
+);
+
+GaussianSplatSorter._sorterTaskProcessor = undefined;
+GaussianSplatSorter._taskProcessorReady = false;
+GaussianSplatSorter._error = undefined;
+GaussianSplatSorter._getSorterTaskProcessor = function () {
+  if (!defined(GaussianSplatSorter._sorterTaskProcessor)) {
+    const processor = new TaskProcessor(
+      "gaussianSplatSorter",
+      GaussianSplatSorter._maxSortingConcurrency,
+    );
+    processor
+      .initWebAssemblyModule({
+        wasmBinaryFile: "ThirdParty/wasm_splats_bg.wasm",
+      })
+      .then(function (result) {
+        if (result) {
+          GaussianSplatSorter._taskProcessorReady = true;
+        } else {
+          GaussianSplatSorter._error = new RuntimeError(
+            "Gaussian splat sorter could not be initialized.",
+          );
+        }
+      })
+      .catch((error) => {
+        GaussianSplatSorter._error = error;
+      });
+    GaussianSplatSorter._sorterTaskProcessor = processor;
+  }
+
+  return GaussianSplatSorter._sorterTaskProcessor;
+};
+/**
+ * Sorts Gaussian splats using a radix sort algorithm. Sorted by distance from the camera.
+ * A new list of indexes is returned, which can be used to render the splats in the correct order.
+ *
+ * @param {Object} parameters - The parameters for sorting Gaussian splat indexes.
+ * @param {Object} parameters.primitive - The primitive containing positions and modelView matrices.
+ * @returns {Promise|undefined} A promise that resolves to the sorted indexes or undefined if the task cannot be scheduled.
+ * @exception {RuntimeError} Sorter could not be initialized.
+ * @private
+ */
+GaussianSplatSorter.radixSortIndexes = function (parameters) {
+  const sorterTaskProcessor = GaussianSplatSorter._getSorterTaskProcessor();
+  if (defined(GaussianSplatSorter._error)) {
+    throw GaussianSplatSorter._error;
+  }
+
+  if (!GaussianSplatSorter._taskProcessorReady) {
+    return;
+  }
+
+  return sorterTaskProcessor.scheduleTask(parameters, [
+    parameters.primitive.positions.buffer,
+  ]);
+};
+
+export default GaussianSplatSorter;

package/Source/Scene/GaussianSplatTextureGenerator.js

@@ -0,0 +1,64 @@
+import defined from "../Core/defined.js";
+import FeatureDetection from "../Core/FeatureDetection.js";
+import RuntimeError from "../Core/RuntimeError.js";
+import TaskProcessor from "../Core/TaskProcessor.js";
+
+function GaussianSplatTextureGenerator() {}
+
+GaussianSplatTextureGenerator._maxSortingConcurrency = Math.max(
+  FeatureDetection.hardwareConcurrency - 1,
+  1,
+);
+
+GaussianSplatTextureGenerator._textureTaskProcessor = undefined;
+GaussianSplatTextureGenerator._taskProcessorReady = false;
+GaussianSplatTextureGenerator._error = undefined;
+GaussianSplatTextureGenerator._getTextureTaskProcessor = function () {
+  if (!defined(GaussianSplatTextureGenerator._textureTaskProcessor)) {
+    const processor = new TaskProcessor(
+      "gaussianSplatTextureGenerator",
+      GaussianSplatTextureGenerator._maxSortingConcurrency,
+    );
+    processor
+      .initWebAssemblyModule({
+        wasmBinaryFile: "ThirdParty/wasm_splats_bg.wasm",
+      })
+      .then(function (result) {
+        if (result) {
+          GaussianSplatTextureGenerator._taskProcessorReady = true;
+        } else {
+          GaussianSplatTextureGenerator._error = new RuntimeError(
+            "Gaussian splat sorter could not be initialized.",
+          );
+        }
+      })
+      .catch((error) => {
+        GaussianSplatTextureGenerator._error = error;
+      });
+    GaussianSplatTextureGenerator._textureTaskProcessor = processor;
+  }
+
+  return GaussianSplatTextureGenerator._textureTaskProcessor;
+};
+
+GaussianSplatTextureGenerator.generateFromAttributes = function (parameters) {
+  const textureTaskProcessor =
+    GaussianSplatTextureGenerator._getTextureTaskProcessor();
+  if (defined(GaussianSplatTextureGenerator._error)) {
+    throw GaussianSplatTextureGenerator._error;
+  }
+
+  if (!GaussianSplatTextureGenerator._taskProcessorReady) {
+    return;
+  }
+
+  const { attributes } = parameters;
+  return textureTaskProcessor.scheduleTask(parameters, [
+    attributes.positions.buffer,
+    attributes.scales.buffer,
+    attributes.rotations.buffer,
+    attributes.colors.buffer,
+  ]);
+};
+
+export default GaussianSplatTextureGenerator;

package/Source/Scene/GltfLoader.js

@@ -507,8 +507,7 @@ function postProcessGeometry(loader, context) {
   for (let i = 0; i < loadPlans.length; i++) {
     const loadPlan = loadPlans[i];
     loadPlan.postProcess(context);
-
-    if (loadPlan.needsOutlines) {
+    if (loadPlan.needsOutlines || loadPlan.needsGaussianSplats) {
       // The glTF loader takes ownership of any buffers generated in the
       // post-process stage since they were created after the geometry loaders
       // finished. This way they can be destroyed when the loader is destroyed.
@@ -687,6 +686,7 @@ function getVertexBufferLoader(
   semantic,
   primitive,
   draco,
+  spzExtension,
   loadBuffer,
   loadTypedArray,
   frameState,
@@ -703,6 +703,7 @@ function getVertexBufferLoader(
     bufferViewId: bufferViewId,
     primitive: primitive,
     draco: draco,
+    spz: spzExtension,
     attributeSemantic: semantic,
     accessorId: accessorId,
     asynchronous: loader._asynchronous,
@@ -1032,7 +1033,10 @@ function createAttribute(gltf, accessorId, name, semantic, setIndex) {
     attribute.semantic === VertexAttributeSemantic.POSITION ||
     attribute.semantic === VertexAttributeSemantic.NORMAL ||
     attribute.semantic === VertexAttributeSemantic.TANGENT ||
-    attribute.semantic === VertexAttributeSemantic.TEXCOORD;
+    attribute.semantic === VertexAttributeSemantic.TEXCOORD ||
+    attribute.semantic === VertexAttributeSemantic.FEATURE_ID ||
+    attribute.semantic === VertexAttributeSemantic.SCALE ||
+    attribute.semantic === VertexAttributeSemantic.ROTATION;
 
   // In the glTF 2.0 spec, min and max are not affected by the normalized flag.
   // However, for KHR_mesh_quantization, min and max must be dequantized for
@@ -1124,6 +1128,58 @@ function finalizeDracoAttribute(
   }
 }
 
+function finalizeSpzAttribute(
+  attribute,
+  vertexBufferLoader,
+  loadBuffer,
+  loadTypedArray,
+) {
+  attribute.byteOffset = 0;
+  attribute.byteStride = undefined;
+
+  if (loadBuffer) {
+    attribute.buffer = vertexBufferLoader.buffer;
+  }
+
+  if (loadTypedArray && defined(vertexBufferLoader.typedArray)) {
+    attribute.typedArray = ComponentDatatype.createArrayBufferView(
+      attribute.componentDatatype,
+      vertexBufferLoader.typedArray.buffer,
+    );
+  }
+
+  if (attribute.semantic === VertexAttributeSemantic.POSITION) {
+    const findMinMaxXY = (flatArray) => {
+      let minX = Infinity;
+      let maxX = -Infinity;
+      let minY = Infinity;
+      let maxY = -Infinity;
+      let minZ = Infinity;
+      let maxZ = -Infinity;
+
+      for (let i = 0; i < flatArray.length; i += 3) {
+        const x = flatArray[i];
+        const y = flatArray[i + 1];
+        const z = flatArray[i + 2];
+
+        minX = Math.min(minX, x);
+        maxX = Math.max(maxX, x);
+        minY = Math.min(minY, y);
+        maxY = Math.max(maxY, y);
+        minZ = Math.min(minZ, z);
+        maxZ = Math.max(maxZ, z);
+      }
+
+      return [
+        new Cartesian3(minX, minY, minZ),
+        new Cartesian3(maxX, maxY, maxZ),
+      ];
+    };
+    const buffer = attribute.typedArray;
+    [attribute.min, attribute.max] = findMinMaxXY(buffer);
+  }
+}
+
 function finalizeAttribute(
   gltf,
   accessor,
@@ -1160,6 +1216,7 @@ function loadAttribute(
   semanticInfo,
   primitive,
   draco,
+  spz,
   loadBuffer,
   loadTypedArray,
   frameState,
@@ -1185,7 +1242,7 @@ function loadAttribute(
     setIndex,
   );
 
-  if (!defined(draco) && !defined(bufferViewId)) {
+  if (!defined(draco) && !defined(bufferViewId) && !defined(spz)) {
     return attribute;
   }
 
@@ -1195,6 +1252,7 @@ function loadAttribute(
     gltfSemantic,
     primitive,
     draco,
+    spz,
     loadBuffer,
     loadTypedArray,
     frameState,
@@ -1204,6 +1262,7 @@ function loadAttribute(
   loader._geometryLoaders.push(vertexBufferLoader);
   const promise = vertexBufferLoader.load();
   loader._loaderPromises.push(promise);
+
   // This can only execute once vertexBufferLoader.process() has run and returns true
   // Save this finish callback by the loader index so it can be called
   // in process().
@@ -1219,6 +1278,13 @@ function loadAttribute(
         loadBuffer,
         loadTypedArray,
       );
+    } else if (defined(spz)) {
+      finalizeSpzAttribute(
+        attribute,
+        vertexBufferLoader,
+        loadBuffer,
+        loadTypedArray,
+      );
     } else {
       finalizeAttribute(
         gltf,
@@ -1240,6 +1306,7 @@ function loadVertexAttribute(
   semanticInfo,
   primitive,
   draco,
+  spz,
   hasInstances,
   needsPostProcessing,
   frameState,
@@ -1287,6 +1354,7 @@ function loadVertexAttribute(
     semanticInfo,
     primitive,
     draco,
+    spz,
     loadBuffer,
     loadTypedArray,
     frameState,
@@ -1360,6 +1428,7 @@ function loadInstancedAttribute(
     semanticInfo,
     undefined,
     undefined,
+    undefined,
     loadBuffer,
     loadTypedArray,
     frameState,
@@ -1876,6 +1945,7 @@ function loadMorphTarget(
   // Don't pass in primitive or draco object since morph targets can't be draco compressed
   const primitive = undefined;
   const draco = undefined;
+  const spz = undefined;
   const hasInstances = false;
 
   for (const semantic in target) {
@@ -1896,6 +1966,7 @@ function loadMorphTarget(
       semanticInfo,
       primitive,
       draco,
+      spz,
       hasInstances,
       needsPostProcessing,
       frameState,
@@ -1947,6 +2018,12 @@ function loadPrimitive(loader, gltfPrimitive, hasInstances, frameState) {
     );
   }
 
+  const spzExtension = extensions.KHR_spz_gaussian_splats_compression;
+  if (defined(spzExtension)) {
+    needsPostProcessing = true;
+    primitivePlan.needsGaussianSplats = true;
+  }
+
   const loadForClassification = loader._loadForClassification;
   const draco = extensions.KHR_draco_mesh_compression;
 
@@ -1979,6 +2056,7 @@ function loadPrimitive(loader, gltfPrimitive, hasInstances, frameState) {
         semanticInfo,
         gltfPrimitive,
         draco,
+        spzExtension,
         hasInstances,
         needsPostProcessing,
         frameState,

package/Source/Scene/GltfSpzLoader.js

@@ -0,0 +1,222 @@
+import Check from "../Core/Check.js";
+import Frozen from "../Core/Frozen.js";
+import defined from "../Core/defined.js";
+import ResourceLoader from "./ResourceLoader.js";
+import ResourceLoaderState from "./ResourceLoaderState.js";
+import { loadSpz } from "@spz-loader/core";
+
+/**
+ * Load a SPZ buffer from a glTF.
+ * <p>
+ * Implements the {@link ResourceLoader} interface.
+ * </p>
+ * @alias GltfSpzLoader
+ * @constructor
+ * @augments ResourceLoader
+ * @param {object} options Object with the following properties:
+ * @param {ResourceCache} options.resourceCache The {@link ResourceCache} (to avoid circular dependencies).
+ * @param {object} options.gltf The glTF JSON.
+ * @param {object} options.primitive The primitive containing the SPZ extension.
+ * @param {object} options.spz The SPZ extension object.
+ * @param {Resource} options.gltfResource The {@link Resource} containing the glTF.
+ * @param {Resource} options.baseResource The {@link Resource} that paths in the glTF JSON are relative to.
+ * @param {string} [options.cacheKey] The cache key of the resource.
+ *
+ * @private
+ */
+function GltfSpzLoader(options) {
+  options = options ?? Frozen.EMPTY_OBJECT;
+  const resourceCache = options.resourceCache;
+  const gltf = options.gltf;
+  const primitive = options.primitive;
+  const spz = options.spz;
+  const gltfResource = options.gltfResource;
+  const baseResource = options.baseResource;
+  const cacheKey = options.cacheKey;
+
+  //>>includeStart('debug', pragmas.debug);
+  Check.typeOf.func("options.resourceCache", resourceCache);
+  Check.typeOf.object("options.gltf", gltf);
+  Check.typeOf.object("options.primitive", primitive);
+  Check.typeOf.object("options.spz", spz);
+  Check.typeOf.object("options.gltfResource", gltfResource);
+  Check.typeOf.object("options.baseResource", baseResource);
+  //>>includeEnd('debug');
+
+  this._resourceCache = resourceCache;
+  this._gltfResource = gltfResource;
+  this._baseResource = baseResource;
+  this._gltf = gltf;
+  this._primitive = primitive;
+  this._spz = spz;
+  this._cacheKey = cacheKey;
+  this._bufferViewLoader = undefined;
+  this._bufferViewTypedArray = undefined;
+  this._decodePromise = undefined;
+  this._decodedData = undefined;
+  this._state = ResourceLoaderState.UNLOADED;
+  this._promise = undefined;
+  this._spzError = undefined;
+}
+
+if (defined(Object.create)) {
+  GltfSpzLoader.prototype = Object.create(ResourceLoader.prototype);
+  GltfSpzLoader.prototype.constructor = GltfSpzLoader;
+}
+
+Object.defineProperties(GltfSpzLoader.prototype, {
+  /**
+   * The cache key of the resource.
+   * @memberof GltfSpzLoader.prototype
+   * @type {string}
+   * @readonly
+   * @private
+   */
+  cacheKey: {
+    get: function () {
+      return this._cacheKey;
+    },
+  },
+  /**
+   * The decoded SPZ data.
+   * @memberof GltfSpzLoader.prototype
+   * @type {object}
+   * @readonly
+   * @private
+   */
+  decodedData: {
+    get: function () {
+      return this._decodedData;
+    },
+  },
+});
+
+async function loadResources(loader) {
+  const resourceCache = loader._resourceCache;
+  try {
+    const bufferViewLoader = resourceCache.getBufferViewLoader({
+      gltf: loader._gltf,
+      bufferViewId: 0,
+      gltfResource: loader._gltfResource,
+      baseResource: loader._baseResource,
+    });
+    loader._bufferViewLoader = bufferViewLoader;
+    await bufferViewLoader.load();
+
+    if (loader.isDestroyed()) {
+      return;
+    }
+
+    loader._bufferViewTypedArray = bufferViewLoader.typedArray;
+    loader._state = ResourceLoaderState.PROCESSING;
+    return loader;
+  } catch (error) {
+    if (loader.isDestroyed()) {
+      return;
+    }
+
+    handleError(loader, error);
+  }
+}
+
+/**
+ * Loads the SPZ resource.
+ * @returns {Promise<Resource>} A promise that resolves to the resource when the SPZ is loaded.
+ * @private
+ */
+GltfSpzLoader.prototype.load = async function () {
+  if (defined(this._promise)) {
+    return this._promise;
+  }
+
+  this._state = ResourceLoaderState.LOADING;
+  this._promise = loadResources(this);
+  return this._promise;
+};
+
+function handleError(spzLoader, error) {
+  spzLoader.unload();
+  spzLoader._state = ResourceLoaderState.FAILED;
+  const errorMessage = "Failed to load SPZ";
+  throw spzLoader.getError(errorMessage, error);
+}
+
+async function processDecode(loader, decodePromise) {
+  try {
+    const gcloud = await decodePromise;
+    if (loader.isDestroyed()) {
+      return;
+    }
+
+    loader.unload();
+
+    loader._decodedData = {
+      gcloud: gcloud,
+    };
+    loader._state = ResourceLoaderState.READY;
+    return loader._baseResource;
+  } catch (error) {
+    if (loader.isDestroyed()) {
+      return;
+    }
+
+    loader._spzError = error;
+  }
+}
+
+/**
+ * Processes the SPZ resource.
+ * @param {FrameState} frameState The frame state.
+ * @private
+ */
+GltfSpzLoader.prototype.process = function (frameState) {
+  //>>includeStart('debug', pragmas.debug);
+  Check.typeOf.object("frameState", frameState);
+  //>>includeEnd('debug');
+
+  if (this._state === ResourceLoaderState.READY) {
+    return true;
+  }
+
+  if (this._state !== ResourceLoaderState.PROCESSING) {
+    return false;
+  }
+
+  if (defined(this._spzError)) {
+    handleError(this, this._spzError);
+  }
+
+  if (!defined(this._bufferViewTypedArray)) {
+    return false;
+  }
+
+  if (defined(this._decodePromise)) {
+    return false;
+  }
+
+  const decodePromise = loadSpz(this._bufferViewTypedArray);
+
+  if (!defined(decodePromise)) {
+    return false;
+  }
+
+  this._decodePromise = processDecode(this, decodePromise);
+};
+
+/**
+ * Unloads the SPZ resource and frees associated resources.
+ * @private
+ */
+GltfSpzLoader.prototype.unload = function () {
+  if (defined(this._bufferViewLoader)) {
+    this._resourceCache.unload(this._bufferViewLoader);
+  }
+
+  this._bufferViewLoader = undefined;
+  this._bufferViewTypedArray = undefined;
+  this._decodedData = undefined;
+  this._gltf = undefined;
+  this._primitive = undefined;
+};
+
+export default GltfSpzLoader;