@babylonjs/core 7.30.0 → 7.31.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (116) hide show
  1. package/Engines/WebGPU/webgpuCacheBindGroups.js +13 -2
  2. package/Engines/WebGPU/webgpuCacheBindGroups.js.map +1 -1
  3. package/Engines/WebGPU/webgpuCacheSampler.js +2 -3
  4. package/Engines/WebGPU/webgpuCacheSampler.js.map +1 -1
  5. package/Engines/abstractEngine.js +2 -2
  6. package/Engines/abstractEngine.js.map +1 -1
  7. package/Materials/Background/backgroundMaterial.js +1 -11
  8. package/Materials/Background/backgroundMaterial.js.map +1 -1
  9. package/Materials/GaussianSplatting/gaussianSplattingMaterial.d.ts +8 -0
  10. package/Materials/GaussianSplatting/gaussianSplattingMaterial.js +43 -34
  11. package/Materials/GaussianSplatting/gaussianSplattingMaterial.js.map +1 -1
  12. package/Materials/Node/Blocks/GaussianSplatting/gaussianBlock.d.ts +32 -0
  13. package/Materials/Node/Blocks/GaussianSplatting/gaussianBlock.js +64 -0
  14. package/Materials/Node/Blocks/GaussianSplatting/gaussianBlock.js.map +1 -0
  15. package/Materials/Node/Blocks/GaussianSplatting/gaussianSplattingBlock.d.ts +48 -0
  16. package/Materials/Node/Blocks/GaussianSplatting/gaussianSplattingBlock.js +102 -0
  17. package/Materials/Node/Blocks/GaussianSplatting/gaussianSplattingBlock.js.map +1 -0
  18. package/Materials/Node/Blocks/GaussianSplatting/index.d.ts +3 -0
  19. package/Materials/Node/Blocks/GaussianSplatting/index.js +4 -0
  20. package/Materials/Node/Blocks/GaussianSplatting/index.js.map +1 -0
  21. package/Materials/Node/Blocks/GaussianSplatting/splatReaderBlock.d.ts +40 -0
  22. package/Materials/Node/Blocks/GaussianSplatting/splatReaderBlock.js +96 -0
  23. package/Materials/Node/Blocks/GaussianSplatting/splatReaderBlock.js.map +1 -0
  24. package/Materials/Node/Blocks/Input/inputBlock.js +6 -0
  25. package/Materials/Node/Blocks/Input/inputBlock.js.map +1 -1
  26. package/Materials/Node/Blocks/index.d.ts +1 -0
  27. package/Materials/Node/Blocks/index.js +1 -0
  28. package/Materials/Node/Blocks/index.js.map +1 -1
  29. package/Materials/Node/Enums/nodeMaterialModes.d.ts +3 -1
  30. package/Materials/Node/Enums/nodeMaterialModes.js +2 -0
  31. package/Materials/Node/Enums/nodeMaterialModes.js.map +1 -1
  32. package/Materials/Node/index.d.ts +1 -0
  33. package/Materials/Node/index.js +1 -0
  34. package/Materials/Node/index.js.map +1 -1
  35. package/Materials/Node/nodeMaterialDefault.d.ts +6 -0
  36. package/Materials/Node/nodeMaterialDefault.js +53 -0
  37. package/Materials/Node/nodeMaterialDefault.js.map +1 -0
  38. package/Materials/PBR/pbrBaseMaterial.js +1 -11
  39. package/Materials/PBR/pbrBaseMaterial.js.map +1 -1
  40. package/Materials/Textures/texture.js +7 -3
  41. package/Materials/Textures/texture.js.map +1 -1
  42. package/Materials/Textures/textureSampler.d.ts +1 -1
  43. package/Materials/Textures/textureSampler.js.map +1 -1
  44. package/Materials/material.d.ts +5 -1
  45. package/Materials/material.js +19 -1
  46. package/Materials/material.js.map +1 -1
  47. package/Materials/materialPluginManager.js +2 -2
  48. package/Materials/materialPluginManager.js.map +1 -1
  49. package/Materials/pushMaterial.d.ts +1 -1
  50. package/Materials/pushMaterial.js +2 -2
  51. package/Materials/pushMaterial.js.map +1 -1
  52. package/Materials/standardMaterial.js +1 -11
  53. package/Materials/standardMaterial.js.map +1 -1
  54. package/Meshes/GaussianSplatting/gaussianSplattingMesh.d.ts +11 -0
  55. package/Meshes/GaussianSplatting/gaussianSplattingMesh.js +264 -61
  56. package/Meshes/GaussianSplatting/gaussianSplattingMesh.js.map +1 -1
  57. package/Meshes/Node/Blocks/Set/aggregatorBlock.d.ts +76 -0
  58. package/Meshes/Node/Blocks/Set/aggregatorBlock.js +210 -0
  59. package/Meshes/Node/Blocks/Set/aggregatorBlock.js.map +1 -0
  60. package/Meshes/Node/Blocks/booleanGeometryBlock.d.ts +6 -0
  61. package/Meshes/Node/Blocks/booleanGeometryBlock.js +16 -4
  62. package/Meshes/Node/Blocks/booleanGeometryBlock.js.map +1 -1
  63. package/Meshes/Node/Blocks/geometryTransformBlock.d.ts +6 -0
  64. package/Meshes/Node/Blocks/geometryTransformBlock.js +14 -2
  65. package/Meshes/Node/Blocks/geometryTransformBlock.js.map +1 -1
  66. package/Meshes/Node/index.d.ts +1 -0
  67. package/Meshes/Node/index.js +1 -0
  68. package/Meshes/Node/index.js.map +1 -1
  69. package/Meshes/Node/nodeGeometry.d.ts +1 -1
  70. package/Meshes/Node/nodeGeometry.js +14 -1
  71. package/Meshes/Node/nodeGeometry.js.map +1 -1
  72. package/Meshes/Node/nodeGeometryBlock.d.ts +4 -0
  73. package/Meshes/Node/nodeGeometryBlock.js +6 -0
  74. package/Meshes/Node/nodeGeometryBlock.js.map +1 -1
  75. package/Meshes/csg.d.ts +1 -0
  76. package/Meshes/csg.js +1 -0
  77. package/Meshes/csg.js.map +1 -1
  78. package/Meshes/csg2.d.ts +131 -0
  79. package/Meshes/csg2.js +355 -0
  80. package/Meshes/csg2.js.map +1 -0
  81. package/Meshes/index.d.ts +1 -0
  82. package/Meshes/index.js +1 -0
  83. package/Meshes/index.js.map +1 -1
  84. package/Meshes/mesh.js +3 -1
  85. package/Meshes/mesh.js.map +1 -1
  86. package/Meshes/mesh.vertexData.d.ts +66 -1
  87. package/Meshes/mesh.vertexData.js.map +1 -1
  88. package/Misc/dumpTools.js +1 -0
  89. package/Misc/dumpTools.js.map +1 -1
  90. package/Misc/tools.d.ts +2 -1
  91. package/Misc/tools.internals.d.ts +13 -0
  92. package/Misc/tools.internals.js +48 -0
  93. package/Misc/tools.internals.js.map +1 -0
  94. package/Misc/tools.js +13 -4
  95. package/Misc/tools.js.map +1 -1
  96. package/Morph/morphTarget.js +1 -0
  97. package/Morph/morphTarget.js.map +1 -1
  98. package/Shaders/ShadersInclude/gaussianSplatting.d.ts +5 -0
  99. package/Shaders/ShadersInclude/gaussianSplatting.js +33 -0
  100. package/Shaders/ShadersInclude/gaussianSplatting.js.map +1 -0
  101. package/Shaders/ShadersInclude/gaussianSplattingFragmentDeclaration.d.ts +7 -0
  102. package/Shaders/ShadersInclude/gaussianSplattingFragmentDeclaration.js +19 -0
  103. package/Shaders/ShadersInclude/gaussianSplattingFragmentDeclaration.js.map +1 -0
  104. package/Shaders/ShadersInclude/gaussianSplattingUboDeclaration.js +1 -1
  105. package/Shaders/ShadersInclude/gaussianSplattingUboDeclaration.js.map +1 -1
  106. package/Shaders/ShadersInclude/gaussianSplattingVertexDeclaration.js +1 -2
  107. package/Shaders/ShadersInclude/gaussianSplattingVertexDeclaration.js.map +1 -1
  108. package/Shaders/gaussianSplatting.fragment.d.ts +1 -2
  109. package/Shaders/gaussianSplatting.fragment.js +5 -10
  110. package/Shaders/gaussianSplatting.fragment.js.map +1 -1
  111. package/Shaders/gaussianSplatting.vertex.d.ts +1 -0
  112. package/Shaders/gaussianSplatting.vertex.js +4 -21
  113. package/Shaders/gaussianSplatting.vertex.js.map +1 -1
  114. package/Shaders/lod.fragment.js +1 -1
  115. package/Shaders/lod.fragment.js.map +1 -1
  116. package/package.json +1 -1
@@ -5,6 +5,7 @@ import { SubMesh } from "../subMesh";
5
5
  import type { AbstractMesh } from "../abstractMesh";
6
6
  import { Mesh } from "../mesh";
7
7
  import "../thinInstanceMesh.js";
8
+ import type { Material } from "../../Materials/material.js";
8
9
  /**
9
10
  * Class used to render a gaussian splatting mesh
10
11
  */
@@ -28,6 +29,8 @@ export declare class GaussianSplattingMesh extends Mesh {
28
29
  private readonly _keepInRam;
29
30
  private _delayedTextureUpdate;
30
31
  private _oldDirection;
32
+ private _useRGBACovariants;
33
+ private _material;
31
34
  /**
32
35
  * Gets the covariancesA texture
33
36
  */
@@ -44,6 +47,14 @@ export declare class GaussianSplattingMesh extends Mesh {
44
47
  * Gets the colors texture
45
48
  */
46
49
  get colorsTexture(): Nullable<BaseTexture>;
50
+ /**
51
+ * set rendering material
52
+ */
53
+ set material(value: Material);
54
+ /**
55
+ * get rendering material
56
+ */
57
+ get material(): Nullable<Material>;
47
58
  /**
48
59
  * Creates a new gaussian splatting mesh
49
60
  * @param name defines the name of the mesh
@@ -8,6 +8,8 @@ import { RawTexture } from "../../Materials/Textures/rawTexture.js";
8
8
 
9
9
  import { Tools } from "../../Misc/tools.js";
10
10
  import "../thinInstanceMesh.js";
11
+ import { ToHalfFloat } from "../../Misc/textureTools.js";
12
+ import { Scalar } from "../../Maths/math.scalar.js";
11
13
  /**
12
14
  * Class used to render a gaussian splatting mesh
13
15
  */
@@ -36,6 +38,20 @@ export class GaussianSplattingMesh extends Mesh {
36
38
  get colorsTexture() {
37
39
  return this._colorsTexture;
38
40
  }
41
+ /**
42
+ * set rendering material
43
+ */
44
+ set material(value) {
45
+ this._material = value;
46
+ this._material.backFaceCulling = true;
47
+ this._material.cullBackFaces = false;
48
+ }
49
+ /**
50
+ * get rendering material
51
+ */
52
+ get material() {
53
+ return this._material;
54
+ }
39
55
  /**
40
56
  * Creates a new gaussian splatting mesh
41
57
  * @param name defines the name of the mesh
@@ -66,6 +82,8 @@ export class GaussianSplattingMesh extends Mesh {
66
82
  this._keepInRam = false;
67
83
  this._delayedTextureUpdate = null;
68
84
  this._oldDirection = new Vector3();
85
+ this._useRGBACovariants = false;
86
+ this._material = null;
69
87
  const vertexData = new VertexData();
70
88
  vertexData.positions = [-2, -2, 0, 2, -2, 0, 2, 2, 0, -2, 2, 0];
71
89
  vertexData.indices = [0, 1, 2, 0, 2, 3];
@@ -73,11 +91,13 @@ export class GaussianSplattingMesh extends Mesh {
73
91
  this.subMeshes = [];
74
92
  new SubMesh(0, 0, 4, 0, 6, this);
75
93
  this.setEnabled(false);
94
+ // webGL2 and webGPU support for RG texture with float16 is fine. not webGL1
95
+ this._useRGBACovariants = !this.getEngine().isWebGPU && this.getEngine().version === 1.0;
76
96
  this._keepInRam = keepInRam;
77
97
  if (url) {
78
98
  this.loadFileAsync(url);
79
99
  }
80
- this.material = new GaussianSplattingMaterial(this.name + "_material", this._scene);
100
+ this._material = new GaussianSplattingMaterial(this.name + "_material", this._scene);
81
101
  }
82
102
  /**
83
103
  * Returns the class name
@@ -154,10 +174,17 @@ export class GaussianSplattingMesh extends Mesh {
154
174
  const headerEnd = "end_header\n";
155
175
  const headerEndIndex = header.indexOf(headerEnd);
156
176
  if (headerEndIndex < 0 || !header) {
177
+ // standard splat
157
178
  return data;
158
179
  }
159
180
  const vertexCount = parseInt(/element vertex (\d+)\n/.exec(header)[1]);
160
- let rowOffset = 0;
181
+ const chunkElement = /element chunk (\d+)\n/.exec(header);
182
+ let chunkCount = 0;
183
+ if (chunkElement) {
184
+ chunkCount = parseInt(chunkElement[1]);
185
+ }
186
+ let rowVertexOffset = 0;
187
+ let rowChunkOffset = 0;
161
188
  const offsets = {
162
189
  double: 8,
163
190
  int: 4,
@@ -166,51 +193,207 @@ export class GaussianSplattingMesh extends Mesh {
166
193
  short: 2,
167
194
  ushort: 2,
168
195
  uchar: 1,
196
+ list: 0,
169
197
  };
170
- const properties = [];
171
- const filtered = header
172
- .slice(0, headerEndIndex)
173
- .split("\n")
174
- .filter((k) => k.startsWith("property "));
198
+ let ElementMode;
199
+ (function (ElementMode) {
200
+ ElementMode[ElementMode["Vertex"] = 0] = "Vertex";
201
+ ElementMode[ElementMode["Chunk"] = 1] = "Chunk";
202
+ })(ElementMode || (ElementMode = {}));
203
+ let chunkMode = 1 /* ElementMode.Chunk */;
204
+ const vertexProperties = [];
205
+ const chunkProperties = [];
206
+ const filtered = header.slice(0, headerEndIndex).split("\n");
175
207
  for (const prop of filtered) {
176
- const [, type, name] = prop.split(" ");
177
- properties.push({ name, type, offset: rowOffset });
178
- if (offsets[type]) {
179
- rowOffset += offsets[type];
208
+ if (prop.startsWith("property ")) {
209
+ const [, type, name] = prop.split(" ");
210
+ if (chunkMode == 1 /* ElementMode.Chunk */) {
211
+ chunkProperties.push({ name, type, offset: rowChunkOffset });
212
+ rowChunkOffset += offsets[type];
213
+ }
214
+ else if (chunkMode == 0 /* ElementMode.Vertex */) {
215
+ vertexProperties.push({ name, type, offset: rowVertexOffset });
216
+ rowVertexOffset += offsets[type];
217
+ }
218
+ if (!offsets[type]) {
219
+ Logger.Warn(`Unsupported property type: ${type}.`);
220
+ }
180
221
  }
181
- else {
182
- Logger.Error(`Unsupported property type: ${type}. Are you sure it's a valid Gaussian Splatting file?`);
183
- return new ArrayBuffer(0);
222
+ else if (prop.startsWith("element ")) {
223
+ const [, type] = prop.split(" ");
224
+ if (type == "chunk") {
225
+ chunkMode = 1 /* ElementMode.Chunk */;
226
+ }
227
+ else if (type == "vertex") {
228
+ chunkMode = 0 /* ElementMode.Vertex */;
229
+ }
184
230
  }
185
231
  }
186
- const rowLength = 3 * 4 + 3 * 4 + 4 + 4;
232
+ const rowVertexLength = rowVertexOffset;
233
+ const rowChunkLength = rowChunkOffset;
234
+ const rowOutputLength = 3 * 4 + 3 * 4 + 4 + 4; // Vector3 position, Vector3 scale, 1 u8 quaternion, 1 color with alpha
187
235
  const SH_C0 = 0.28209479177387814;
236
+ let offset = 0;
188
237
  const dataView = new DataView(data, headerEndIndex + headerEnd.length);
189
- const buffer = new ArrayBuffer(rowLength * vertexCount);
238
+ const buffer = new ArrayBuffer(rowOutputLength * vertexCount);
190
239
  const q = new Quaternion();
240
+ const temp3 = TmpVectors.Vector3[0];
241
+ const unpackUnorm = (value, bits) => {
242
+ const t = (1 << bits) - 1;
243
+ return (value & t) / t;
244
+ };
245
+ const unpack111011 = (value, result) => {
246
+ result.x = unpackUnorm(value >>> 21, 11);
247
+ result.y = unpackUnorm(value >>> 11, 10);
248
+ result.z = unpackUnorm(value, 11);
249
+ };
250
+ const unpack8888 = (value, result) => {
251
+ result[0] = unpackUnorm(value >>> 24, 8) * 255;
252
+ result[1] = unpackUnorm(value >>> 16, 8) * 255;
253
+ result[2] = unpackUnorm(value >>> 8, 8) * 255;
254
+ result[3] = unpackUnorm(value, 8) * 255;
255
+ };
256
+ // unpack quaternion with 2,10,10,10 format (largest element, 3x10bit element)
257
+ const unpackRot = (value, result) => {
258
+ const norm = 1.0 / (Math.sqrt(2) * 0.5);
259
+ const a = (unpackUnorm(value >>> 20, 10) - 0.5) * norm;
260
+ const b = (unpackUnorm(value >>> 10, 10) - 0.5) * norm;
261
+ const c = (unpackUnorm(value, 10) - 0.5) * norm;
262
+ const m = Math.sqrt(1.0 - (a * a + b * b + c * c));
263
+ switch (value >>> 30) {
264
+ case 0:
265
+ result.set(m, a, b, c);
266
+ break;
267
+ case 1:
268
+ result.set(a, m, b, c);
269
+ break;
270
+ case 2:
271
+ result.set(a, b, m, c);
272
+ break;
273
+ case 3:
274
+ result.set(a, b, c, m);
275
+ break;
276
+ }
277
+ };
278
+ const compressedChunks = new Array(chunkCount);
279
+ for (let i = 0; i < chunkCount; i++) {
280
+ const currentChunk = { min: new Vector3(), max: new Vector3(), minScale: new Vector3(), maxScale: new Vector3() };
281
+ compressedChunks[i] = currentChunk;
282
+ for (let propertyIndex = 0; propertyIndex < chunkProperties.length; propertyIndex++) {
283
+ const property = chunkProperties[propertyIndex];
284
+ let value;
285
+ switch (property.type) {
286
+ case "float":
287
+ value = dataView.getFloat32(property.offset + offset, true);
288
+ break;
289
+ default:
290
+ continue;
291
+ }
292
+ switch (property.name) {
293
+ case "min_x":
294
+ currentChunk.min.x = value;
295
+ break;
296
+ case "min_y":
297
+ currentChunk.min.y = value;
298
+ break;
299
+ case "min_z":
300
+ currentChunk.min.z = value;
301
+ break;
302
+ case "max_x":
303
+ currentChunk.max.x = value;
304
+ break;
305
+ case "max_y":
306
+ currentChunk.max.y = value;
307
+ break;
308
+ case "max_z":
309
+ currentChunk.max.z = value;
310
+ break;
311
+ case "min_scale_x":
312
+ currentChunk.minScale.x = value;
313
+ break;
314
+ case "min_scale_y":
315
+ currentChunk.minScale.y = value;
316
+ break;
317
+ case "min_scale_z":
318
+ currentChunk.minScale.z = value;
319
+ break;
320
+ case "max_scale_x":
321
+ currentChunk.maxScale.x = value;
322
+ break;
323
+ case "max_scale_y":
324
+ currentChunk.maxScale.y = value;
325
+ break;
326
+ case "max_scale_z":
327
+ currentChunk.maxScale.z = value;
328
+ break;
329
+ }
330
+ }
331
+ offset += rowChunkLength;
332
+ }
191
333
  for (let i = 0; i < vertexCount; i++) {
192
- const position = new Float32Array(buffer, i * rowLength, 3);
193
- const scale = new Float32Array(buffer, i * rowLength + 12, 3);
194
- const rgba = new Uint8ClampedArray(buffer, i * rowLength + 24, 4);
195
- const rot = new Uint8ClampedArray(buffer, i * rowLength + 28, 4);
334
+ const position = new Float32Array(buffer, i * rowOutputLength, 3);
335
+ const scale = new Float32Array(buffer, i * rowOutputLength + 12, 3);
336
+ const rgba = new Uint8ClampedArray(buffer, i * rowOutputLength + 24, 4);
337
+ const rot = new Uint8ClampedArray(buffer, i * rowOutputLength + 28, 4);
338
+ const chunkIndex = i >> 8;
196
339
  let r0 = 255;
197
340
  let r1 = 0;
198
341
  let r2 = 0;
199
342
  let r3 = 0;
200
- for (let propertyIndex = 0; propertyIndex < properties.length; propertyIndex++) {
201
- const property = properties[propertyIndex];
343
+ for (let propertyIndex = 0; propertyIndex < vertexProperties.length; propertyIndex++) {
344
+ const property = vertexProperties[propertyIndex];
202
345
  let value;
203
346
  switch (property.type) {
204
347
  case "float":
205
- value = dataView.getFloat32(property.offset + i * rowOffset, true);
348
+ value = dataView.getFloat32(offset + property.offset, true);
206
349
  break;
207
350
  case "int":
208
- value = dataView.getInt32(property.offset + i * rowOffset, true);
351
+ value = dataView.getInt32(offset + property.offset, true);
352
+ break;
353
+ case "uint":
354
+ value = dataView.getUint32(offset + property.offset, true);
355
+ break;
356
+ case "double":
357
+ value = dataView.getFloat64(offset + property.offset, true);
358
+ break;
359
+ case "uchar":
360
+ value = dataView.getUint8(offset + property.offset);
209
361
  break;
210
362
  default:
211
- throw new Error(`Unsupported property type: ${property.type}`);
363
+ //throw new Error(`Unsupported property type: ${property.type}`);
364
+ continue;
212
365
  }
213
366
  switch (property.name) {
367
+ case "packed_position":
368
+ {
369
+ const compressedChunk = compressedChunks[chunkIndex];
370
+ unpack111011(value, temp3);
371
+ position[0] = Scalar.Lerp(compressedChunk.min.x, compressedChunk.max.x, temp3.x);
372
+ position[1] = -Scalar.Lerp(compressedChunk.min.y, compressedChunk.max.y, temp3.y);
373
+ position[2] = Scalar.Lerp(compressedChunk.min.z, compressedChunk.max.z, temp3.z);
374
+ }
375
+ break;
376
+ case "packed_rotation":
377
+ {
378
+ unpackRot(value, q);
379
+ r0 = q.w;
380
+ r1 = q.z;
381
+ r2 = q.y;
382
+ r3 = q.x;
383
+ }
384
+ break;
385
+ case "packed_scale":
386
+ {
387
+ const compressedChunk = compressedChunks[chunkIndex];
388
+ unpack111011(value, temp3);
389
+ scale[0] = Math.exp(Scalar.Lerp(compressedChunk.minScale.x, compressedChunk.maxScale.x, temp3.x));
390
+ scale[1] = Math.exp(Scalar.Lerp(compressedChunk.minScale.y, compressedChunk.maxScale.y, temp3.y));
391
+ scale[2] = Math.exp(Scalar.Lerp(compressedChunk.minScale.z, compressedChunk.maxScale.z, temp3.z));
392
+ }
393
+ break;
394
+ case "packed_color":
395
+ unpack8888(value, rgba);
396
+ break;
214
397
  case "x":
215
398
  position[0] = value;
216
399
  break;
@@ -229,12 +412,15 @@ export class GaussianSplattingMesh extends Mesh {
229
412
  case "scale_2":
230
413
  scale[2] = Math.exp(value);
231
414
  break;
415
+ case "diffuse_red":
232
416
  case "red":
233
417
  rgba[0] = value;
234
418
  break;
419
+ case "diffuse_green":
235
420
  case "green":
236
421
  rgba[1] = value;
237
422
  break;
423
+ case "diffuse_blue":
238
424
  case "blue":
239
425
  rgba[2] = value;
240
426
  break;
@@ -273,6 +459,7 @@ export class GaussianSplattingMesh extends Mesh {
273
459
  rot[1] = q.x * 128 + 128;
274
460
  rot[2] = q.y * 128 + 128;
275
461
  rot[3] = q.z * 128 + 128;
462
+ offset += rowVertexLength;
276
463
  }
277
464
  return buffer;
278
465
  }
@@ -363,33 +550,48 @@ export class GaussianSplattingMesh extends Mesh {
363
550
  this._vertexCount = vertexCount;
364
551
  const textureSize = this._getTextureSize(vertexCount);
365
552
  const textureLength = textureSize.x * textureSize.y;
366
- this._splatPositions = new Float32Array(3 * textureLength);
367
- const covA = new Float32Array(3 * textureLength);
368
- const covB = new Float32Array(3 * textureLength);
553
+ this._splatPositions = new Float32Array(4 * textureLength);
554
+ const covA = new Uint16Array(4 * textureLength);
555
+ const covB = new Uint16Array((this._useRGBACovariants ? 4 : 2) * textureLength);
369
556
  const matrixRotation = TmpVectors.Matrix[0];
370
557
  const matrixScale = TmpVectors.Matrix[1];
371
558
  const quaternion = TmpVectors.Quaternion[0];
372
559
  const minimum = new Vector3(Number.MAX_VALUE, Number.MAX_VALUE, Number.MAX_VALUE);
373
560
  const maximum = new Vector3(-Number.MAX_VALUE, -Number.MAX_VALUE, -Number.MAX_VALUE);
561
+ const covariances = [0, 0, 0, 0, 0, 0];
562
+ const covBSplatSize = this._useRGBACovariants ? 4 : 2;
374
563
  for (let i = 0; i < vertexCount; i++) {
375
564
  const x = fBuffer[8 * i + 0];
376
565
  const y = -fBuffer[8 * i + 1];
377
566
  const z = fBuffer[8 * i + 2];
378
- this._splatPositions[3 * i + 0] = x;
379
- this._splatPositions[3 * i + 1] = y;
380
- this._splatPositions[3 * i + 2] = z;
567
+ this._splatPositions[4 * i + 0] = x;
568
+ this._splatPositions[4 * i + 1] = y;
569
+ this._splatPositions[4 * i + 2] = z;
381
570
  minimum.minimizeInPlaceFromFloats(x, y, z);
382
571
  maximum.maximizeInPlaceFromFloats(x, y, z);
383
572
  quaternion.set((uBuffer[32 * i + 28 + 1] - 128) / 128, (uBuffer[32 * i + 28 + 2] - 128) / 128, (uBuffer[32 * i + 28 + 3] - 128) / 128, -(uBuffer[32 * i + 28 + 0] - 128) / 128);
384
573
  quaternion.toRotationMatrix(matrixRotation);
385
574
  Matrix.ScalingToRef(fBuffer[8 * i + 3 + 0] * 2, fBuffer[8 * i + 3 + 1] * 2, fBuffer[8 * i + 3 + 2] * 2, matrixScale);
386
575
  const M = matrixRotation.multiplyToRef(matrixScale, TmpVectors.Matrix[0]).m;
387
- covA[i * 3 + 0] = M[0] * M[0] + M[1] * M[1] + M[2] * M[2];
388
- covA[i * 3 + 1] = M[0] * M[4] + M[1] * M[5] + M[2] * M[6];
389
- covA[i * 3 + 2] = M[0] * M[8] + M[1] * M[9] + M[2] * M[10];
390
- covB[i * 3 + 0] = M[4] * M[4] + M[5] * M[5] + M[6] * M[6];
391
- covB[i * 3 + 1] = M[4] * M[8] + M[5] * M[9] + M[6] * M[10];
392
- covB[i * 3 + 2] = M[8] * M[8] + M[9] * M[9] + M[10] * M[10];
576
+ covariances[0] = M[0] * M[0] + M[1] * M[1] + M[2] * M[2];
577
+ covariances[1] = M[0] * M[4] + M[1] * M[5] + M[2] * M[6];
578
+ covariances[2] = M[0] * M[8] + M[1] * M[9] + M[2] * M[10];
579
+ covariances[3] = M[4] * M[4] + M[5] * M[5] + M[6] * M[6];
580
+ covariances[4] = M[4] * M[8] + M[5] * M[9] + M[6] * M[10];
581
+ covariances[5] = M[8] * M[8] + M[9] * M[9] + M[10] * M[10];
582
+ // normalize covA, covB
583
+ let factor = -10000;
584
+ for (let covIndex = 0; covIndex < 6; covIndex++) {
585
+ factor = Math.max(factor, Math.abs(covariances[covIndex]));
586
+ }
587
+ this._splatPositions[4 * i + 3] = factor;
588
+ const transform = factor;
589
+ covA[i * 4 + 0] = ToHalfFloat(covariances[0] / transform);
590
+ covA[i * 4 + 1] = ToHalfFloat(covariances[1] / transform);
591
+ covA[i * 4 + 2] = ToHalfFloat(covariances[2] / transform);
592
+ covA[i * 4 + 3] = ToHalfFloat(covariances[3] / transform);
593
+ covB[i * covBSplatSize + 0] = ToHalfFloat(covariances[4] / transform);
594
+ covB[i * covBSplatSize + 1] = ToHalfFloat(covariances[5] / transform);
393
595
  }
394
596
  // Update the mesh
395
597
  const binfo = this.getBoundingInfo();
@@ -399,23 +601,18 @@ export class GaussianSplattingMesh extends Mesh {
399
601
  const createTextureFromData = (data, width, height, format) => {
400
602
  return new RawTexture(data, width, height, format, this._scene, false, false, 2, 1);
401
603
  };
402
- const convertRgbToRgba = (rgb) => {
403
- const count = rgb.length / 3;
404
- const rgba = new Float32Array(count * 4);
405
- for (let i = 0; i < count; ++i) {
406
- rgba[i * 4 + 0] = rgb[i * 3 + 0];
407
- rgba[i * 4 + 1] = rgb[i * 3 + 1];
408
- rgba[i * 4 + 2] = rgb[i * 3 + 2];
409
- rgba[i * 4 + 3] = 1.0;
410
- }
411
- return rgba;
604
+ const createTextureFromDataU8 = (data, width, height, format) => {
605
+ return new RawTexture(data, width, height, format, this._scene, false, false, 2, 0);
606
+ };
607
+ const createTextureFromDataF16 = (data, width, height, format) => {
608
+ return new RawTexture(data, width, height, format, this._scene, false, false, 2, 2);
412
609
  };
413
- const colorArray = new Float32Array(textureSize.x * textureSize.y * 4);
610
+ const colorArray = new Uint8Array(textureSize.x * textureSize.y * 4);
414
611
  for (let i = 0; i < this._vertexCount; ++i) {
415
- colorArray[i * 4 + 0] = uBuffer[32 * i + 24 + 0] / 255;
416
- colorArray[i * 4 + 1] = uBuffer[32 * i + 24 + 1] / 255;
417
- colorArray[i * 4 + 2] = uBuffer[32 * i + 24 + 2] / 255;
418
- colorArray[i * 4 + 3] = uBuffer[32 * i + 24 + 3] / 255;
612
+ colorArray[i * 4 + 0] = uBuffer[32 * i + 24 + 0];
613
+ colorArray[i * 4 + 1] = uBuffer[32 * i + 24 + 1];
614
+ colorArray[i * 4 + 2] = uBuffer[32 * i + 24 + 2];
615
+ colorArray[i * 4 + 3] = uBuffer[32 * i + 24 + 3];
419
616
  }
420
617
  if (this._keepInRam) {
421
618
  this._covariancesA = covA;
@@ -423,17 +620,17 @@ export class GaussianSplattingMesh extends Mesh {
423
620
  this._colors = colorArray;
424
621
  }
425
622
  if (this._covariancesATexture) {
426
- this._delayedTextureUpdate = { covA: convertRgbToRgba(covA), covB: convertRgbToRgba(covB), colors: colorArray, centers: convertRgbToRgba(this._splatPositions) };
623
+ this._delayedTextureUpdate = { covA: covA, covB: covB, colors: colorArray, centers: this._splatPositions };
427
624
  const positions = Float32Array.from(this._splatPositions);
428
625
  const vertexCount = this._vertexCount;
429
626
  this._worker.postMessage({ positions, vertexCount }, [positions.buffer]);
430
627
  this._postToWorker(true);
431
628
  }
432
629
  else {
433
- this._covariancesATexture = createTextureFromData(convertRgbToRgba(covA), textureSize.x, textureSize.y, 5);
434
- this._covariancesBTexture = createTextureFromData(convertRgbToRgba(covB), textureSize.x, textureSize.y, 5);
435
- this._centersTexture = createTextureFromData(convertRgbToRgba(this._splatPositions), textureSize.x, textureSize.y, 5);
436
- this._colorsTexture = createTextureFromData(colorArray, textureSize.x, textureSize.y, 5);
630
+ this._covariancesATexture = createTextureFromDataF16(covA, textureSize.x, textureSize.y, 5);
631
+ this._covariancesBTexture = createTextureFromDataF16(covB, textureSize.x, textureSize.y, this._useRGBACovariants ? 5 : 7);
632
+ this._centersTexture = createTextureFromData(this._splatPositions, textureSize.x, textureSize.y, 5);
633
+ this._colorsTexture = createTextureFromDataU8(colorArray, textureSize.x, textureSize.y, 5);
437
634
  this._instanciateWorker();
438
635
  }
439
636
  }
@@ -477,11 +674,17 @@ export class GaussianSplattingMesh extends Mesh {
477
674
  const updateTextureFromData = (texture, data, width, height) => {
478
675
  this.getEngine().updateTextureData(texture.getInternalTexture(), data, 0, 0, width, height, 0, 0, false);
479
676
  };
677
+ const updateTextureFromDataU8 = (texture, data, width, height) => {
678
+ this.getEngine().updateTextureData(texture.getInternalTexture(), data, 0, 0, width, height, 0, 0, false);
679
+ };
680
+ const updateTextureFromDataF16 = (texture, data, width, height) => {
681
+ this.getEngine().updateTextureData(texture.getInternalTexture(), data, 0, 0, width, height, 0, 0, false);
682
+ };
480
683
  const textureSize = this._getTextureSize(vertexCount);
481
- updateTextureFromData(this._covariancesATexture, this._delayedTextureUpdate.covA, textureSize.x, textureSize.y);
482
- updateTextureFromData(this._covariancesBTexture, this._delayedTextureUpdate.covB, textureSize.x, textureSize.y);
684
+ updateTextureFromDataF16(this._covariancesATexture, this._delayedTextureUpdate.covA, textureSize.x, textureSize.y);
685
+ updateTextureFromDataF16(this._covariancesBTexture, this._delayedTextureUpdate.covB, textureSize.x, textureSize.y);
483
686
  updateTextureFromData(this._centersTexture, this._delayedTextureUpdate.centers, textureSize.x, textureSize.y);
484
- updateTextureFromData(this._colorsTexture, this._delayedTextureUpdate.colors, textureSize.x, textureSize.y);
687
+ updateTextureFromDataU8(this._colorsTexture, this._delayedTextureUpdate.colors, textureSize.x, textureSize.y);
485
688
  this._delayedTextureUpdate = null;
486
689
  }
487
690
  this.thinInstanceBufferUpdated("splatIndex");
@@ -539,7 +742,7 @@ GaussianSplattingMesh._CreateWorker = function (self) {
539
742
  depthFactor = 1;
540
743
  }
541
744
  for (let j = 0; j < vertexCount; j++) {
542
- floatMix[2 * j + 1] = 10000 + (viewProj[2] * positions[3 * j + 0] + viewProj[6] * positions[3 * j + 1] + viewProj[10] * positions[3 * j + 2]) * depthFactor;
745
+ floatMix[2 * j + 1] = 10000 + (viewProj[2] * positions[4 * j + 0] + viewProj[6] * positions[4 * j + 1] + viewProj[10] * positions[4 * j + 2]) * depthFactor;
543
746
  }
544
747
  depthMix.sort();
545
748
  self.postMessage({ depthMix }, [depthMix.buffer]);