@solidrt/3d 0.0.63 → 0.0.64
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +198 -17
- package/package.json +3 -2
- package/src/camera.ts +6 -2
- package/src/components/directional-light.tsx +6 -2
- package/src/components/hemisphere-light.tsx +12 -4
- package/src/components/index.ts +2 -0
- package/src/components/instanced-mesh.tsx +6 -2
- package/src/components/mesh.tsx +12 -1
- package/src/components/point-light.tsx +6 -2
- package/src/components/record-mesh.tsx +14 -2
- package/src/components/scene.tsx +8 -1
- package/src/components/splat-mesh.tsx +50 -0
- package/src/components/spot-light.tsx +7 -3
- package/src/environment.ts +55 -0
- package/src/glsl.ts +68 -14
- package/src/index.ts +9 -4
- package/src/light.ts +56 -4
- package/src/material.ts +26 -1
- package/src/mesh.ts +361 -6
- package/src/node.ts +3 -0
- package/src/scene-shadows.ts +8 -7
- package/src/scene.ts +225 -104
- package/src/splat-data.ts +617 -0
- package/src/splat.ts +401 -0
- package/tools/splat.ts +70 -0
package/src/splat.ts
ADDED
|
@@ -0,0 +1,401 @@
|
|
|
1
|
+
// The splat RUNTIME side: the stock splat material and createSplatMesh
|
|
2
|
+
// over a baked SplatData (parse and bake live in ./splat-data.ts, the
|
|
3
|
+
// runtime-free `@solidrt/3d/splat` entry; `srt tool 3d/splat` writes the
|
|
4
|
+
// .srts this loads). A splat cloud here is an ordinary record mesh in the
|
|
5
|
+
// INDEXED form - the records live in an rgba32ui data texture the vertex
|
|
6
|
+
// stage fetches by id, SPLAT_GROUP splats share one drawn instance (a
|
|
7
|
+
// merged-quad geometry; the instancing frontend was the measured wall),
|
|
8
|
+
// and the instance buffer is the id stream the core sorts back to front,
|
|
9
|
+
// the scene feeding the view direction: zero per-frame JS, a re-sort
|
|
10
|
+
// uploads 4 bytes per splat, a parked camera uploads nothing. Depth-tested
|
|
11
|
+
// against opaque geometry, placed by the transparent entry sort, picked by
|
|
12
|
+
// its bounds, transformed by its node. The spherical-harmonic bands a bake
|
|
13
|
+
// carries (`--sh`) ride a second data texture, evaluated per corner from
|
|
14
|
+
// the view direction.
|
|
15
|
+
|
|
16
|
+
import { file } from "flux:fs"
|
|
17
|
+
import { createTexture, limits } from "@solidrt/core/gpu"
|
|
18
|
+
import type { TextureBindings, TextureId, VertexAttribute } from "@solidrt/core/gpu"
|
|
19
|
+
import { mergeGeometries, plane } from "./geometry.ts"
|
|
20
|
+
import type { Geometry } from "./geometry.ts"
|
|
21
|
+
import { shaderMaterialClass } from "./material.ts"
|
|
22
|
+
import type { Material, ShaderMaterialClass } from "./material.ts"
|
|
23
|
+
import { createRecordMesh } from "./mesh.ts"
|
|
24
|
+
import type { RecordMesh } from "./mesh.ts"
|
|
25
|
+
import { decodeSplat, SPLAT_ATTRIBUTES, SPLAT_RECORD_TEXELS, SPLAT_SH_TEXELS, SPLAT_TEXEL_BYTES } from "./splat-data.ts"
|
|
26
|
+
import type { SplatData } from "./splat-data.ts"
|
|
27
|
+
import { SRGB } from "./glsl.ts"
|
|
28
|
+
|
|
29
|
+
// core/gpu's glsl tag, aliased locally like ./glsl.ts does.
|
|
30
|
+
let glsl = String.raw
|
|
31
|
+
|
|
32
|
+
/** Splats per drawn instance: the stock geometry is this many quads
|
|
33
|
+
* merged, and an id-stream record is this many u32 ids. Measured on the
|
|
34
|
+
* Pixel 7 at 1M splats: 4, 8 and 16 cost the same and all three collapse
|
|
35
|
+
* the instancing frontend the one-quad form paid, so the fewest instances
|
|
36
|
+
* it is (okf/done/gaussian-splats.md, stage D). */
|
|
37
|
+
export const SPLAT_GROUP = 16
|
|
38
|
+
|
|
39
|
+
/** The id stream's record: SPLAT_GROUP u32 splat ids as uvec4 attributes
|
|
40
|
+
* (iIds0..), what the stock material's one instance buffer declares and
|
|
41
|
+
* the core fills with the sorted record indices. */
|
|
42
|
+
export const SPLAT_ID_ATTRIBUTES: VertexAttribute[] = Array.from({ length: SPLAT_GROUP / 4 }, (_, j) => ({ name: `iIds${j}`, format: "uint32x4" }))
|
|
43
|
+
|
|
44
|
+
// The data textures' row width in texels, capped by the device: a texel
|
|
45
|
+
// index addresses (t & (width - 1), t >> log2 width). 4096 keeps a 1M
|
|
46
|
+
// cloud's records at 502 rows and its SH3 block at 1504.
|
|
47
|
+
const SPLAT_TEXTURE_WIDTH_MAX = 4096
|
|
48
|
+
// The id past the sorted population (the core's INDEX_NONE): culled
|
|
49
|
+
// before any fetch.
|
|
50
|
+
const SPLAT_NONE = 0xffffffff
|
|
51
|
+
|
|
52
|
+
// The data texture width this device gets, as a power of two with its
|
|
53
|
+
// log2 (compiled into the vertex source as constants).
|
|
54
|
+
let textureWidth = (): { width: number; log2: number } => {
|
|
55
|
+
let log2 = Math.floor(Math.log2(Math.min(SPLAT_TEXTURE_WIDTH_MAX, limits.maxTextureSize)))
|
|
56
|
+
return { width: 1 << log2, log2 }
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
const SPLAT_CONSTANTS = glsl`
|
|
60
|
+
// Quad half-extent in standard deviations: the gaussian is cut at
|
|
61
|
+
// exp(-0.5 * 8) = e^-4, where the reference viewers cut it.
|
|
62
|
+
const float SPLAT_EXTENT = 2.8284271;
|
|
63
|
+
`
|
|
64
|
+
|
|
65
|
+
// The 3DGS spherical-harmonic basis constants, bands 1..3.
|
|
66
|
+
const SH_GLSL = glsl`
|
|
67
|
+
const float SH_C1 = 0.4886025119029199;
|
|
68
|
+
const float SH_C2_0 = 1.0925484305920792;
|
|
69
|
+
const float SH_C2_1 = -1.0925484305920792;
|
|
70
|
+
const float SH_C2_2 = 0.31539156525252005;
|
|
71
|
+
const float SH_C2_3 = -1.0925484305920792;
|
|
72
|
+
const float SH_C2_4 = 0.5462742152960396;
|
|
73
|
+
const float SH_C3_0 = -0.5900435899266435;
|
|
74
|
+
const float SH_C3_1 = 2.890611442640554;
|
|
75
|
+
const float SH_C3_2 = -0.4570457994644658;
|
|
76
|
+
const float SH_C3_3 = 0.3731763325901154;
|
|
77
|
+
const float SH_C3_4 = -0.4570457994644658;
|
|
78
|
+
const float SH_C3_5 = 1.445305721320277;
|
|
79
|
+
const float SH_C3_6 = -0.5900435899266435;
|
|
80
|
+
`
|
|
81
|
+
|
|
82
|
+
// The SH evaluation for `degree`: the view direction in the model frame
|
|
83
|
+
// (uCamPos through the transposed model rotation - exact under rotation
|
|
84
|
+
// and uniform scale), the coefficients unpacked from the fetched texels
|
|
85
|
+
// at constant indices (coefficient-major, rgb interleaved, halves packed
|
|
86
|
+
// pairwise), the basis polynomials per band added to the base color.
|
|
87
|
+
let shBlock = (degree: number): string => {
|
|
88
|
+
let texels = SPLAT_SH_TEXELS[degree]!
|
|
89
|
+
let fetch = (j: number) => ` uvec4 t${j} = texelFetch(uSplatSh, splatTexel(s + ${j}u), 0);`
|
|
90
|
+
let half = (i: number) => `unpackHalf2x16(t${i >> 3}[${(i >> 1) & 3}]).${i & 1 ? "y" : "x"}`
|
|
91
|
+
let c = (k: number) => `vec3(${half(3 * k)}, ${half(3 * k + 1)}, ${half(3 * k + 2)})`
|
|
92
|
+
let lines = [
|
|
93
|
+
` vec3 worldCenter = (uModel * vec4(center, 1.0)).xyz;`,
|
|
94
|
+
` vec3 d = normalize(transpose(mat3(uModel)) * (worldCenter - uCamPos));`,
|
|
95
|
+
` uint s = id * ${texels}u;`,
|
|
96
|
+
...Array.from({ length: texels }, (_, j) => fetch(j)),
|
|
97
|
+
` float x = d.x, y = d.y, z = d.z;`,
|
|
98
|
+
` vec3 sh = -SH_C1 * y * ${c(0)} + SH_C1 * z * ${c(1)} - SH_C1 * x * ${c(2)};`,
|
|
99
|
+
]
|
|
100
|
+
if (degree > 1) {
|
|
101
|
+
lines.push(
|
|
102
|
+
` float xx = x * x, yy = y * y, zz = z * z, xy = x * y, yz = y * z, xz = x * z;`,
|
|
103
|
+
` sh += SH_C2_0 * xy * ${c(3)} + SH_C2_1 * yz * ${c(4)} + SH_C2_2 * (2.0 * zz - xx - yy) * ${c(5)} + SH_C2_3 * xz * ${c(6)} + SH_C2_4 * (xx - yy) * ${c(7)};`,
|
|
104
|
+
)
|
|
105
|
+
}
|
|
106
|
+
if (degree > 2) {
|
|
107
|
+
lines.push(
|
|
108
|
+
` sh += SH_C3_0 * y * (3.0 * xx - yy) * ${c(8)} + SH_C3_1 * xy * z * ${c(9)} + SH_C3_2 * y * (4.0 * zz - xx - yy) * ${c(10)} + SH_C3_3 * z * (2.0 * zz - 3.0 * xx - 3.0 * yy) * ${c(11)} + SH_C3_4 * x * (4.0 * zz - xx - yy) * ${c(12)} + SH_C3_5 * z * (xx - yy) * ${c(13)} + SH_C3_6 * x * (xx - 3.0 * yy) * ${c(14)};`,
|
|
109
|
+
)
|
|
110
|
+
}
|
|
111
|
+
// The reference rasterizer clamps below only; the blend saturates above.
|
|
112
|
+
lines.push(` color.rgb = max(color.rgb + sh, 0.0);`)
|
|
113
|
+
return lines.join("\n") + "\n"
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
// The splat id of this vertex: quad q of the instance reads id q of the
|
|
117
|
+
// instance's SPLAT_GROUP ids.
|
|
118
|
+
let idSelect = (): string => {
|
|
119
|
+
let lines = [` uvec4 ids = iIds0;`]
|
|
120
|
+
for (let j = 1; j < SPLAT_GROUP / 4; j++) lines.push(` if (quad >= ${4 * j}u) ids = iIds${j};`)
|
|
121
|
+
lines.push(` uint id = ids[quad & 3u];`)
|
|
122
|
+
return lines.join("\n") + "\n"
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
let checkDegree = (degree: number, site: string): void => {
|
|
126
|
+
if (!(Number.isInteger(degree) && degree >= 0 && degree <= 3)) throw new Error(site + ": shDegree must be an integer 0..3, got " + degree)
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* The stock splat vertex stage for a cloud baked at `shDegree`: the id
|
|
131
|
+
* from the instance's id record (quad `gl_VertexID / 4` of the merged
|
|
132
|
+
* geometry), the record's center, packed color and precomputed 3D
|
|
133
|
+
* covariance fetched from `uSplatRecords` (two rgba32ui texels), the SH
|
|
134
|
+
* bands from `uSplatSh` (evaluated per corner from uCamPos), the
|
|
135
|
+
* covariance projected to a screen ellipse through the Jacobian of
|
|
136
|
+
* `uViewProj * uModel`, the quad's corners placed along its axes. The
|
|
137
|
+
* texture width is this device's, compiled in. For a forked class that
|
|
138
|
+
* keeps the projection and swaps the shading: fork this source over
|
|
139
|
+
* SPLAT_ID_ATTRIBUTES, and pass the class's instance as the mesh
|
|
140
|
+
* material (the mesh binds the textures).
|
|
141
|
+
*/
|
|
142
|
+
export function splatVertexSource(shDegree = 0): string {
|
|
143
|
+
checkDegree(shDegree, "splatVertexSource")
|
|
144
|
+
let { width, log2 } = textureWidth()
|
|
145
|
+
let attributes = SPLAT_ID_ATTRIBUTES.map(a => ` in uvec4 ${a.name};\n`).join("")
|
|
146
|
+
return glsl`
|
|
147
|
+
in vec3 aPos;
|
|
148
|
+
${attributes} uniform mat4 uModel;
|
|
149
|
+
uniform mat4 uViewProj;
|
|
150
|
+
uniform vec2 uViewport;
|
|
151
|
+
uniform float uBlendSpace;
|
|
152
|
+
uniform highp usampler2D uSplatRecords;
|
|
153
|
+
${shDegree > 0 ? ` uniform highp usampler2D uSplatSh;\n uniform vec3 uCamPos;\n${SH_GLSL}` : ""} out vec4 vColor;
|
|
154
|
+
out vec2 vOffset;
|
|
155
|
+
${SPLAT_CONSTANTS}
|
|
156
|
+
${SRGB}
|
|
157
|
+
// The 3DGS anti-aliasing dilation: a variance in pixels squared added to
|
|
158
|
+
// the projected footprint, so no splat is thinner than about a pixel.
|
|
159
|
+
const float LOW_PASS = 0.3;
|
|
160
|
+
// The largest on-screen standard deviation in pixels, capping the quad
|
|
161
|
+
// of a splat the camera sits inside (the reference viewers' cap).
|
|
162
|
+
const float MAX_SIGMA_PX = 1024.0;
|
|
163
|
+
// Centers this far past the clip edge (in units of w) are culled; a
|
|
164
|
+
// margin keeps big splats centered just off screen.
|
|
165
|
+
const float FRUSTUM_MARGIN = 1.2;
|
|
166
|
+
// Where a culled splat goes: outside the clip volume, so it rasterizes
|
|
167
|
+
// nothing.
|
|
168
|
+
const vec4 CULLED = vec4(0.0, 0.0, 2.0, 1.0);
|
|
169
|
+
// The id past the sorted population (the core's sentinel).
|
|
170
|
+
const uint SPLAT_NONE = ${SPLAT_NONE}u;
|
|
171
|
+
// Texels per record, and the data textures' row addressing.
|
|
172
|
+
const uint SPLAT_RECORD_TEXELS = ${SPLAT_RECORD_TEXELS}u;
|
|
173
|
+
const uint SPLAT_TEXEL_MASK = ${width - 1}u;
|
|
174
|
+
const uint SPLAT_TEXEL_SHIFT = ${log2}u;
|
|
175
|
+
|
|
176
|
+
ivec2 splatTexel(uint t) {
|
|
177
|
+
return ivec2(int(t & SPLAT_TEXEL_MASK), int(t >> SPLAT_TEXEL_SHIFT));
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
void main() {
|
|
181
|
+
uint quad = uint(gl_VertexID) / 4u;
|
|
182
|
+
${idSelect()} if (id == SPLAT_NONE) {
|
|
183
|
+
gl_Position = CULLED;
|
|
184
|
+
return;
|
|
185
|
+
}
|
|
186
|
+
uint t = id * SPLAT_RECORD_TEXELS;
|
|
187
|
+
uvec4 recA = texelFetch(uSplatRecords, splatTexel(t), 0);
|
|
188
|
+
uvec4 recB = texelFetch(uSplatRecords, splatTexel(t + 1u), 0);
|
|
189
|
+
vec3 center = uintBitsToFloat(recA.xyz);
|
|
190
|
+
// The packed rgba8 (r in the low byte); GLSL ES 3.00 has no
|
|
191
|
+
// unpackUnorm4x8 (that is 3.10), so by hand.
|
|
192
|
+
vec4 color = vec4(float(recA.w & 255u), float((recA.w >> 8u) & 255u), float((recA.w >> 16u) & 255u), float(recA.w >> 24u)) / 255.0;
|
|
193
|
+
vec2 c01 = unpackHalf2x16(recB.x);
|
|
194
|
+
vec2 c23 = unpackHalf2x16(recB.y);
|
|
195
|
+
vec2 c45 = unpackHalf2x16(recB.z);
|
|
196
|
+
${shDegree > 0 ? shBlock(shDegree) : ""} mat4 mvp = uViewProj * uModel;
|
|
197
|
+
vec4 clip = mvp * vec4(center, 1.0);
|
|
198
|
+
if (clip.w <= 0.0 || clip.z < -clip.w || any(greaterThan(abs(clip.xy), vec2(clip.w * FRUSTUM_MARGIN)))) {
|
|
199
|
+
gl_Position = CULLED;
|
|
200
|
+
return;
|
|
201
|
+
}
|
|
202
|
+
// The baked covariance, upper triangle back to the symmetric matrix.
|
|
203
|
+
mat3 sigma = mat3(
|
|
204
|
+
c01.x, c01.y, c23.x,
|
|
205
|
+
c01.y, c23.y, c45.x,
|
|
206
|
+
c23.x, c45.x, c45.y);
|
|
207
|
+
// Jacobian of the pixel position with respect to the local position,
|
|
208
|
+
// straight from the clip transform: d(clip.xy / clip.w) / dp, scaled
|
|
209
|
+
// to pixels. Any clip flip in uViewProj flips center and footprint
|
|
210
|
+
// alike, and any node transform in uModel shapes the footprint with
|
|
211
|
+
// the same matrix that moves the center.
|
|
212
|
+
vec3 row0 = vec3(mvp[0][0], mvp[1][0], mvp[2][0]);
|
|
213
|
+
vec3 row1 = vec3(mvp[0][1], mvp[1][1], mvp[2][1]);
|
|
214
|
+
vec3 row3 = vec3(mvp[0][3], mvp[1][3], mvp[2][3]);
|
|
215
|
+
float w2 = clip.w * clip.w;
|
|
216
|
+
vec3 jx = (row0 * clip.w - row3 * clip.x) / w2 * (0.5 * uViewport.x);
|
|
217
|
+
vec3 jy = (row1 * clip.w - row3 * clip.y) / w2 * (0.5 * uViewport.y);
|
|
218
|
+
vec3 sx = sigma * jx;
|
|
219
|
+
vec3 sy = sigma * jy;
|
|
220
|
+
float a = dot(jx, sx) + LOW_PASS;
|
|
221
|
+
float b = dot(jx, sy);
|
|
222
|
+
float c = dot(jy, sy) + LOW_PASS;
|
|
223
|
+
// Eigen decomposition of the 2x2 screen covariance [[a, b], [b, c]].
|
|
224
|
+
float mid = 0.5 * (a + c);
|
|
225
|
+
float radius = length(vec2(0.5 * (a - c), b));
|
|
226
|
+
float major = mid + radius;
|
|
227
|
+
float minor = mid - radius;
|
|
228
|
+
if (minor <= 0.0) {
|
|
229
|
+
gl_Position = CULLED;
|
|
230
|
+
return;
|
|
231
|
+
}
|
|
232
|
+
// The major eigenvector from whichever form is better conditioned.
|
|
233
|
+
vec2 v1 = vec2(b, major - a);
|
|
234
|
+
vec2 v2 = vec2(major - c, b);
|
|
235
|
+
vec2 v = dot(v1, v1) > dot(v2, v2) ? v1 : v2;
|
|
236
|
+
vec2 e1 = dot(v, v) > 0.0 ? normalize(v) : vec2(1.0, 0.0);
|
|
237
|
+
vec2 e2 = vec2(-e1.y, e1.x);
|
|
238
|
+
vec2 axis1 = e1 * min(sqrt(major), MAX_SIGMA_PX);
|
|
239
|
+
vec2 axis2 = e2 * min(sqrt(minor), MAX_SIGMA_PX);
|
|
240
|
+
vOffset = aPos.xy * SPLAT_EXTENT;
|
|
241
|
+
vec2 pixels = vOffset.x * axis1 + vOffset.y * axis2;
|
|
242
|
+
vec2 ndc = clip.xy / clip.w + pixels * 2.0 / uViewport;
|
|
243
|
+
gl_Position = vec4(ndc * clip.w, clip.z, clip.w);
|
|
244
|
+
// A display-space scene (SceneOptions.blendSpace) blends the
|
|
245
|
+
// record's sRGB color as encoded - what the capture was trained
|
|
246
|
+
// against; a linear scene decodes it like every color input.
|
|
247
|
+
vColor = vec4(uBlendSpace > 0.5 ? color.rgb : srgbToLinear(color.rgb), color.a);
|
|
248
|
+
}
|
|
249
|
+
`
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
/** The stock splat fragment: the gaussian falloff times opacity, cut at
|
|
253
|
+
* the extent and below one 8-bit step, premultiplied out. */
|
|
254
|
+
export const SPLAT_FRAGMENT = glsl`
|
|
255
|
+
in vec4 vColor;
|
|
256
|
+
in vec2 vOffset;
|
|
257
|
+
${SPLAT_CONSTANTS}
|
|
258
|
+
// Below this coverage a fragment would not change an 8-bit pixel.
|
|
259
|
+
const float MIN_ALPHA = 1.0 / 255.0;
|
|
260
|
+
|
|
261
|
+
void main() {
|
|
262
|
+
float r2 = dot(vOffset, vOffset);
|
|
263
|
+
if (r2 > SPLAT_EXTENT * SPLAT_EXTENT) discard;
|
|
264
|
+
float alpha = vColor.a * exp(-0.5 * r2);
|
|
265
|
+
if (alpha < MIN_ALPHA) discard;
|
|
266
|
+
fragColor = vec4(vColor.rgb * alpha, alpha);
|
|
267
|
+
}
|
|
268
|
+
`
|
|
269
|
+
|
|
270
|
+
// One class and one shared instance per SH degree, created on first use
|
|
271
|
+
// (the source compiles the device's texture width in, so the class waits
|
|
272
|
+
// for the runtime). The textures bind per MESH, so every cloud of a
|
|
273
|
+
// degree shares the instance.
|
|
274
|
+
let classes = new Map<number, ShaderMaterialClass>()
|
|
275
|
+
let stockMaterials = new Map<number, Material>()
|
|
276
|
+
let stockQuads: Geometry | null = null
|
|
277
|
+
|
|
278
|
+
/**
|
|
279
|
+
* The stock splat material class for clouds baked at `shDegree` (default
|
|
280
|
+
* 0): splatVertexSource(shDegree) and SPLAT_FRAGMENT over the id-stream
|
|
281
|
+
* layout (SPLAT_ID_ATTRIBUTES), transparent, both faces. Cached per
|
|
282
|
+
* degree. A custom look forks the sources into its own shaderMaterialClass
|
|
283
|
+
* over the same layout; a mesh over it still wants createSplatMesh's
|
|
284
|
+
* geometry, order, textures and bounds, so pass its instance as the
|
|
285
|
+
* option there rather than building the mesh by hand.
|
|
286
|
+
*/
|
|
287
|
+
export function splatMaterialClass(shDegree = 0): ShaderMaterialClass {
|
|
288
|
+
checkDegree(shDegree, "splatMaterialClass")
|
|
289
|
+
let cls = classes.get(shDegree)
|
|
290
|
+
if (cls === undefined) {
|
|
291
|
+
cls = shaderMaterialClass({
|
|
292
|
+
vertex: splatVertexSource(shDegree),
|
|
293
|
+
fragment: SPLAT_FRAGMENT,
|
|
294
|
+
instanceBuffers: [{ attributes: SPLAT_ID_ATTRIBUTES }],
|
|
295
|
+
transparent: true,
|
|
296
|
+
cull: "none",
|
|
297
|
+
label: `splat-sh${shDegree}`,
|
|
298
|
+
})
|
|
299
|
+
classes.set(shDegree, cls)
|
|
300
|
+
}
|
|
301
|
+
return cls
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
/** A splat mesh is a record mesh in the indexed form; everything there
|
|
305
|
+
* applies (setVisible, setTransform, setRecordCount for the draw count -
|
|
306
|
+
* rounded up to whole groups of SPLAT_GROUP - disposeInstances, which
|
|
307
|
+
* frees the id buffer and the data textures). */
|
|
308
|
+
export type SplatMesh = RecordMesh
|
|
309
|
+
|
|
310
|
+
export type SplatMeshOptions = {
|
|
311
|
+
/** How many splats draw (default all). Records are importance-sorted
|
|
312
|
+
* at bake, so the first n are the scene at n; dial later with
|
|
313
|
+
* setRecordCount (the engine re-sorts and republishes the prefix of
|
|
314
|
+
* its own copy). */
|
|
315
|
+
count?: number
|
|
316
|
+
/** A custom splat material: a fork of splatVertexSource(data.shDegree)
|
|
317
|
+
* and SPLAT_FRAGMENT over SPLAT_ID_ATTRIBUTES; default the stock
|
|
318
|
+
* class's shared instance for the cloud's degree. The data textures
|
|
319
|
+
* bind per mesh under the stock names (uSplatRecords, uSplatSh), so a
|
|
320
|
+
* fork keeps them. */
|
|
321
|
+
material?: Material
|
|
322
|
+
/** Debug label for the id buffer and the data textures (default
|
|
323
|
+
* "splat"). */
|
|
324
|
+
label?: string
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
// A data texture from a texel block (SPLAT_TEXEL_BYTES per texel, `count`
|
|
328
|
+
// x `texels` of them): the block padded to whole rows of the device
|
|
329
|
+
// width, uploaded as rgba32ui. The pad copy is the one per-cloud copy
|
|
330
|
+
// loading makes (native, tens of ms at 1M), and it is transient.
|
|
331
|
+
function splatTexture(block: Uint8Array, texels: number, count: number, label: string): TextureId {
|
|
332
|
+
let { width } = textureWidth()
|
|
333
|
+
let rows = Math.max(1, Math.ceil((count * texels) / width))
|
|
334
|
+
if (rows > limits.maxTextureSize) {
|
|
335
|
+
throw new Error("createSplatMesh: " + count + " splats need a " + width + "x" + rows + " " + label + " texture; this device allows " + limits.maxTextureSize + " rows")
|
|
336
|
+
}
|
|
337
|
+
let padded = new Uint32Array(rows * width * (SPLAT_TEXEL_BYTES / 4))
|
|
338
|
+
new Uint8Array(padded.buffer).set(block.subarray(0, count * texels * SPLAT_TEXEL_BYTES))
|
|
339
|
+
return createTexture(padded, width, rows, { format: "rgba32ui", autoFree: false, label })
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
/**
|
|
343
|
+
* A mesh drawing a baked splat cloud (loadSplat, or parseSplat at bake
|
|
344
|
+
* scale): a record mesh in the indexed form over the stock splat material
|
|
345
|
+
* for the cloud's SH degree - the records handed to the engine as the
|
|
346
|
+
* depth-sort key source AND uploaded once as the rgba32ui record texture
|
|
347
|
+
* the vertex stage fetches by id, the SH block (when baked) as a second
|
|
348
|
+
* texture, SPLAT_GROUP splats per drawn instance of the merged-quad
|
|
349
|
+
* geometry, the cloud's header bounds as the picking and cull box, and
|
|
350
|
+
* the core-side back-to-front order on the splat centers materialized as
|
|
351
|
+
* the id stream (a camera turn re-sorts and republishes 4 bytes per splat
|
|
352
|
+
* core-side with no publish from JS; an order-preserving move uploads
|
|
353
|
+
* nothing). The cloud is write-once: treat `data` as consumed and let it
|
|
354
|
+
* go (the engine holds its own copy; the fetched bytes free with it), and
|
|
355
|
+
* swap clouds by a new mesh. disposeInstances frees the id buffer, the
|
|
356
|
+
* engine copy and both textures. One caveat carried from the capture
|
|
357
|
+
* side: captures are TRAINED against sRGB blending, so show a capture in
|
|
358
|
+
* a display-space scene (`blendSpace: "display"`) for the trained
|
|
359
|
+
* appearance - a linear scene blends it in linear light, which lays a
|
|
360
|
+
* milky veil over stacked translucent splats
|
|
361
|
+
* (okf/done/gaussian-splats.md).
|
|
362
|
+
*/
|
|
363
|
+
export function createSplatMesh(data: SplatData, opts?: SplatMeshOptions): SplatMesh {
|
|
364
|
+
checkDegree(data.shDegree, "createSplatMesh")
|
|
365
|
+
if (data.shDegree > 0 && (data.sh === null || data.sh.byteLength < data.count * SPLAT_SH_TEXELS[data.shDegree]! * SPLAT_TEXEL_BYTES)) {
|
|
366
|
+
throw new Error("createSplatMesh: the cloud claims SH degree " + data.shDegree + " but carries no SH block of that size")
|
|
367
|
+
}
|
|
368
|
+
if (stockQuads === null) {
|
|
369
|
+
let one = plane({ width: 2, height: 2, label: "splat-quad" })
|
|
370
|
+
stockQuads = mergeGeometries(Array.from({ length: SPLAT_GROUP }, () => one), "splat-quads")
|
|
371
|
+
}
|
|
372
|
+
let material = opts?.material
|
|
373
|
+
if (material === undefined) {
|
|
374
|
+
material = stockMaterials.get(data.shDegree)
|
|
375
|
+
if (material === undefined) {
|
|
376
|
+
material = splatMaterialClass(data.shDegree).instance()
|
|
377
|
+
stockMaterials.set(data.shDegree, material)
|
|
378
|
+
}
|
|
379
|
+
}
|
|
380
|
+
let label = opts?.label ?? "splat"
|
|
381
|
+
let mesh = createRecordMesh(stockQuads, material, data.records, opts?.count, {
|
|
382
|
+
bounds: data.bounds,
|
|
383
|
+
instanceOrder: { position: "iCenter", records: SPLAT_ATTRIBUTES, descending: true },
|
|
384
|
+
label,
|
|
385
|
+
})
|
|
386
|
+
let textures: TextureBindings = { uSplatRecords: splatTexture(data.records, SPLAT_RECORD_TEXELS, data.count, label + "-records") }
|
|
387
|
+
mesh._instances.textures.push(textures.uSplatRecords as TextureId)
|
|
388
|
+
if (data.shDegree > 0) {
|
|
389
|
+
textures.uSplatSh = splatTexture(data.sh!, SPLAT_SH_TEXELS[data.shDegree]!, data.count, label + "-sh")
|
|
390
|
+
mesh._instances.textures.push(textures.uSplatSh as TextureId)
|
|
391
|
+
}
|
|
392
|
+
mesh._textures = textures
|
|
393
|
+
return mesh
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
/** Read a baked .srts splat cloud (`srt tool 3d/splat`): no parsing, the
|
|
397
|
+
* records and the SH block view the file's bytes and upload as-is at
|
|
398
|
+
* createSplatMesh. */
|
|
399
|
+
export async function loadSplat(path: string): Promise<SplatData> {
|
|
400
|
+
return decodeSplat(await file(path).bytes())
|
|
401
|
+
}
|
package/tools/splat.ts
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
#!/usr/bin/env bun
|
|
2
|
+
/// <reference path="./node-env.d.ts" />
|
|
3
|
+
|
|
4
|
+
// srt tool 3d/splat: bake a captured splat cloud into a .srts file - the
|
|
5
|
+
// parse, covariance bake and SH packing (src/splat-data.ts) run once here
|
|
6
|
+
// under bun, the result written in the exact texel layout createSplatMesh
|
|
7
|
+
// uploads and the engine sorts, without any per-splat work at runtime.
|
|
8
|
+
// Put the output under assets/ so it ships with the app.
|
|
9
|
+
// Everything here comes from the runtime-free `@solidrt/3d/splat` entry,
|
|
10
|
+
// the same surface an app's own bake script uses.
|
|
11
|
+
//
|
|
12
|
+
// srt tool 3d/splat <in.ply|in.splat|in.spz> [-o <out.srts>] [--sh <0..3>] [--keep-orientation]
|
|
13
|
+
//
|
|
14
|
+
// Inputs: a trainer's .ply, the de-facto .splat interchange, or Niantic's
|
|
15
|
+
// .spz (gunzipped here; version 2 or 3). A y-down capture (.ply, .splat)
|
|
16
|
+
// is stood up to y-up unless --keep-orientation; .spz is y-up already.
|
|
17
|
+
// --sh bakes that many spherical-harmonic bands beyond the base color
|
|
18
|
+
// (capped at the capture's; default 0 - degree 1/2/3 adds 32/48/96 bytes
|
|
19
|
+
// per splat of texture).
|
|
20
|
+
|
|
21
|
+
import { readFileSync, writeFileSync } from "node:fs"
|
|
22
|
+
import { gunzipSync } from "node:zlib"
|
|
23
|
+
import { basename, extname } from "node:path"
|
|
24
|
+
import { encodeSplat, parseSplat } from "../src/splat-data.ts"
|
|
25
|
+
|
|
26
|
+
function usage(error?: string): never {
|
|
27
|
+
if (error) console.error(error)
|
|
28
|
+
console.log("Usage: srt tool 3d/splat <in.ply|in.splat|in.spz> [-o <out.srts>] [--sh <0..3>] [--keep-orientation]")
|
|
29
|
+
process.exit(error ? 1 : 0)
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
let input: string | undefined
|
|
33
|
+
let output: string | undefined
|
|
34
|
+
let keepOrientation = false
|
|
35
|
+
let shDegree = 0
|
|
36
|
+
let args = process.argv.slice(2)
|
|
37
|
+
for (let i = 0; i < args.length; i++) {
|
|
38
|
+
let arg = args[i]!
|
|
39
|
+
if (arg === "--help" || arg === "-h") usage()
|
|
40
|
+
else if (arg === "-o" || arg === "--output") {
|
|
41
|
+
output = args[++i]
|
|
42
|
+
if (output === undefined) usage("Missing value for " + arg)
|
|
43
|
+
} else if (arg === "--sh") {
|
|
44
|
+
let value = args[++i]
|
|
45
|
+
if (value === undefined) usage("Missing value for " + arg)
|
|
46
|
+
shDegree = Number(value)
|
|
47
|
+
if (!(Number.isInteger(shDegree) && shDegree >= 0 && shDegree <= 3)) usage("--sh takes a degree 0..3, got " + value)
|
|
48
|
+
} else if (arg === "--keep-orientation") keepOrientation = true
|
|
49
|
+
else if (arg.startsWith("-")) usage("Unknown option " + arg)
|
|
50
|
+
else if (input === undefined) input = arg
|
|
51
|
+
else usage("Unexpected argument " + arg)
|
|
52
|
+
}
|
|
53
|
+
if (input === undefined) usage("Missing input file")
|
|
54
|
+
if (output === undefined) output = basename(input, extname(input)) + ".srts"
|
|
55
|
+
|
|
56
|
+
let bytes = new Uint8Array(readFileSync(input))
|
|
57
|
+
// .spz is gzip-compressed; the parser takes the inflated bytes.
|
|
58
|
+
if (bytes.length >= 2 && bytes[0] === 0x1f && bytes[1] === 0x8b) bytes = new Uint8Array(gunzipSync(bytes))
|
|
59
|
+
let started = performance.now()
|
|
60
|
+
let data = parseSplat(bytes, { keepOrientation, shDegree })
|
|
61
|
+
let parsed = performance.now() - started
|
|
62
|
+
let encoded = encodeSplat(data)
|
|
63
|
+
writeFileSync(output, encoded)
|
|
64
|
+
|
|
65
|
+
let b = data.bounds
|
|
66
|
+
let extent = (i: number) => (b[i + 3]! - b[i]!).toFixed(1)
|
|
67
|
+
console.log(
|
|
68
|
+
`${output}: ${data.count} splats, bounds ${extent(0)} x ${extent(1)} x ${extent(2)}, SH degree ${data.shDegree}, ` +
|
|
69
|
+
`${(encoded.byteLength / (1024 * 1024)).toFixed(1)} MiB (baked in ${parsed.toFixed(0)} ms)`,
|
|
70
|
+
)
|