@driftengine/splats 3.61.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/NOTICE +9 -0
- package/README.md +56 -0
- package/dist/half.d.ts +32 -0
- package/dist/half.js +88 -0
- package/dist/index.d.ts +32 -0
- package/dist/index.js +38 -0
- package/dist/shaders/generated/splat.wgsl.d.ts +89 -0
- package/dist/shaders/generated/splat.wgsl.js +95 -0
- package/dist/shaders/splat.d.ts +25 -0
- package/dist/shaders/splat.js +337 -0
- package/dist/splat.d.ts +26 -0
- package/dist/splat.js +63 -0
- package/dist/splatBudget.d.ts +40 -0
- package/dist/splatBudget.js +45 -0
- package/dist/splatCapture.d.ts +76 -0
- package/dist/splatCapture.js +108 -0
- package/dist/splatCull.d.ts +25 -0
- package/dist/splatCull.js +80 -0
- package/dist/splatData.d.ts +177 -0
- package/dist/splatData.js +223 -0
- package/dist/splatGl.d.ts +49 -0
- package/dist/splatGl.js +176 -0
- package/dist/splatGpu.d.ts +50 -0
- package/dist/splatGpu.js +180 -0
- package/dist/splatLayout.d.ts +52 -0
- package/dist/splatLayout.js +75 -0
- package/dist/splatMatrix.d.ts +29 -0
- package/dist/splatMatrix.js +68 -0
- package/dist/splatPass.d.ts +83 -0
- package/dist/splatPass.js +206 -0
- package/dist/splatPly.d.ts +14 -0
- package/dist/splatPly.js +242 -0
- package/dist/splatSog.d.ts +110 -0
- package/dist/splatSog.js +285 -0
- package/dist/splatSogDecoder.d.ts +26 -0
- package/dist/splatSogDecoder.js +29 -0
- package/dist/splatSort.d.ts +137 -0
- package/dist/splatSort.js +199 -0
- package/dist/splatSortWorker.d.ts +14 -0
- package/dist/splatSortWorker.js +137 -0
- package/dist/splatSorter.d.ts +112 -0
- package/dist/splatSorter.js +231 -0
- package/dist/splatView.d.ts +52 -0
- package/dist/splatView.js +115 -0
- package/package.json +56 -0
- package/src/fixtures/README.md +36 -0
- package/src/fixtures/cloud.sog +0 -0
- package/src/fixtures/cloud.texels.json +27 -0
- package/src/fixtures/cloud.truth.json +582 -0
- package/src/half.ts +92 -0
- package/src/index.ts +55 -0
- package/src/shaders/generated/splat.wgsl.ts +98 -0
- package/src/shaders/splat.ts +344 -0
- package/src/splat.ts +75 -0
- package/src/splatBudget.ts +48 -0
- package/src/splatCapture.ts +154 -0
- package/src/splatCull.ts +91 -0
- package/src/splatData.ts +398 -0
- package/src/splatGl.ts +262 -0
- package/src/splatGpu.ts +259 -0
- package/src/splatLayout.ts +86 -0
- package/src/splatMatrix.ts +81 -0
- package/src/splatPass.ts +324 -0
- package/src/splatPly.ts +283 -0
- package/src/splatSog.ts +375 -0
- package/src/splatSogDecoder.ts +33 -0
- package/src/splatSort.ts +296 -0
- package/src/splatSortWorker.ts +155 -0
- package/src/splatSorter.ts +285 -0
- package/src/splatView.ts +147 -0
package/src/splatPass.ts
ADDED
|
@@ -0,0 +1,324 @@
|
|
|
1
|
+
/** A splat capture as a pass a consumer registers, drawn by whichever backend it is handed. */
|
|
2
|
+
|
|
3
|
+
import { SPLAT_BINDINGS } from './shaders/generated/splat.wgsl.ts';
|
|
4
|
+
import { SPLAT_STRIDE, splatRows, splatTexels } from './splatLayout.ts';
|
|
5
|
+
import { cameraInCaptureSpace, multiplyMat4 } from './splatMatrix.ts';
|
|
6
|
+
import { splatBoundsVisible } from './splatCull.ts';
|
|
7
|
+
import {
|
|
8
|
+
createWebgl2Splats,
|
|
9
|
+
disposeWebgl2Splats,
|
|
10
|
+
drawWebgl2Splats,
|
|
11
|
+
uploadWebgl2Order,
|
|
12
|
+
uploadWebgl2SplatRange,
|
|
13
|
+
} from './splatGl.ts';
|
|
14
|
+
import type { Webgl2Splats } from './splatGl.ts';
|
|
15
|
+
import {
|
|
16
|
+
createGpuSplats,
|
|
17
|
+
disposeGpuSplats,
|
|
18
|
+
uploadGpuOrder,
|
|
19
|
+
uploadGpuSplatRange,
|
|
20
|
+
} from './splatGpu.ts';
|
|
21
|
+
import type { GpuSplats } from './splatGpu.ts';
|
|
22
|
+
import type { SplatData } from './splatData.ts';
|
|
23
|
+
import type { PassContext, PassDefinition, PassDevice } from '@driftengine/core';
|
|
24
|
+
|
|
25
|
+
const VERT = SPLAT_BINDINGS.SPLAT_VERT;
|
|
26
|
+
const FRAG = SPLAT_BINDINGS.SPLAT_FRAG;
|
|
27
|
+
|
|
28
|
+
const IDENTITY = new Float32Array([1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1]);
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* What a consumer hands the pass each frame.
|
|
32
|
+
*
|
|
33
|
+
* **The camera comes from the caller rather than from the renderer**, because `PassContext`
|
|
34
|
+
* carries what the *frame* is and not what is being looked at — and a contributed pass has no
|
|
35
|
+
* business reaching into the renderer for a matrix the caller already has. It is the same shape
|
|
36
|
+
* every draw verb takes: the caller says where it is looking.
|
|
37
|
+
*/
|
|
38
|
+
export interface SplatView {
|
|
39
|
+
/**
|
|
40
|
+
* Column-major, as `Camera` publishes them.
|
|
41
|
+
*
|
|
42
|
+
* `ArrayLike<number>` rather than `Float32Array` because that is what `gl-matrix`'s `mat4` is,
|
|
43
|
+
* and a consumer handing over `camera.view` should not have to cast a matrix the engine gave it.
|
|
44
|
+
*/
|
|
45
|
+
readonly view: ArrayLike<number>;
|
|
46
|
+
readonly projection: ArrayLike<number>;
|
|
47
|
+
/** In pixels, because the ellipse is built in pixel space. */
|
|
48
|
+
readonly widthPx: number;
|
|
49
|
+
readonly heightPx: number;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
export interface SplatPass extends PassDefinition {
|
|
53
|
+
/** Called once a frame, before `drawPass`. Allocates nothing. */
|
|
54
|
+
setView(view: SplatView): void;
|
|
55
|
+
/**
|
|
56
|
+
* A new draw order, far to near. Four bytes a splat, into storage allocated once.
|
|
57
|
+
*
|
|
58
|
+
* Until one is set the pass draws nothing: an unsorted capture composited back to front is
|
|
59
|
+
* wrong at every silhouette, and drawing it anyway would look like a working feature.
|
|
60
|
+
*/
|
|
61
|
+
setOrder(order: Uint32Array, count: number): void;
|
|
62
|
+
/**
|
|
63
|
+
* The capture's own transform, so two captures compose in one scene.
|
|
64
|
+
*
|
|
65
|
+
* **Two batches are two orders and there is no order between them**, and that is a real limit
|
|
66
|
+
* rather than an omission. Each sorter ranks its own splats along the view direction expressed
|
|
67
|
+
* in that batch's space, so the two are each internally correct and the renderer draws one
|
|
68
|
+
* batch's whole cloud before the other's. Where they occupy different volumes — a statue and the
|
|
69
|
+
* room behind it — nothing shows. Where they interpenetrate, the seam is visible as a plane at
|
|
70
|
+
* which one capture starts winning every blend.
|
|
71
|
+
*
|
|
72
|
+
* What that buys is that a batch's positions never move: one transform on the camera instead of
|
|
73
|
+
* a million on the splats, every frame. What would make it wrong is a scene built from
|
|
74
|
+
* overlapping parts, where the answer is one batch with one order rather than a merge — merging
|
|
75
|
+
* two sorted orders is cheap, but the splats would still be drawn from two textures with two
|
|
76
|
+
* draw calls, so the merge has nowhere to go.
|
|
77
|
+
*/
|
|
78
|
+
setModel(model: ArrayLike<number>): void;
|
|
79
|
+
/**
|
|
80
|
+
* Push a run of splats that has just arrived into the data texture.
|
|
81
|
+
*
|
|
82
|
+
* **For a capture that is still streaming**, where the textures were sized for the final count
|
|
83
|
+
* at registration and are filled block by block — see `SplatCapture`. A caller that packed its
|
|
84
|
+
* whole capture before registering the pass never needs this: `init` uploads everything.
|
|
85
|
+
*
|
|
86
|
+
* Whole rows are re-sent, so calling this with overlapping ranges is correct and merely costs
|
|
87
|
+
* a few kilobytes; calling it every frame with the whole capture is not, and is the per-frame
|
|
88
|
+
* upload the scheduler exists to avoid.
|
|
89
|
+
*/
|
|
90
|
+
uploadSplats(from: number, count: number): void;
|
|
91
|
+
/**
|
|
92
|
+
* Whether the capture reaches the frame at all, as of the last `setView`.
|
|
93
|
+
*
|
|
94
|
+
* **Read this before asking a sorter for a new order.** A capture out of frame costs one
|
|
95
|
+
* discarded draw call and a whole linear sort over every splat it has, and the sort is the
|
|
96
|
+
* expensive half — so the caller's frame should skip `SplatSorter.frame` when this is false.
|
|
97
|
+
* `draw` checks it too, but by then the sort has already happened.
|
|
98
|
+
*/
|
|
99
|
+
readonly visible: boolean;
|
|
100
|
+
readonly count: number;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* Build a pass for one capture.
|
|
105
|
+
*
|
|
106
|
+
* **The consumer decides where it lands in the frame**, by calling `drawPass` at that point. The
|
|
107
|
+
* documented slot is **after opaque and translucent meshes, before particles and light volumes**:
|
|
108
|
+
* splats are scene content that particles and beams are drawn *through*, and the depth test that
|
|
109
|
+
* makes the composition work only sees geometry drawn before it.
|
|
110
|
+
*/
|
|
111
|
+
export function createSplatPass(splats: SplatData, label = 'splats'): SplatPass {
|
|
112
|
+
let gl: Webgl2Splats | null = null;
|
|
113
|
+
let gpu: GpuSplats | null = null;
|
|
114
|
+
let device: GPUDevice | null = null;
|
|
115
|
+
|
|
116
|
+
const rows = splatRows(splats.count);
|
|
117
|
+
/* One padded row buffer, reused by every order upload so a re-sort allocates nothing. */
|
|
118
|
+
const paddedOrder = new Uint32Array(SPLAT_STRIDE * rows);
|
|
119
|
+
const model = new Float32Array(IDENTITY);
|
|
120
|
+
let drawCount = 0;
|
|
121
|
+
let pendingOrder: Uint32Array | null = null;
|
|
122
|
+
/** A span of newly arrived splats waiting for a WebGL2 context. See `uploadSplats`. */
|
|
123
|
+
let pendingRange: { from: number; to: number } | null = null;
|
|
124
|
+
|
|
125
|
+
const view = new Float32Array(16);
|
|
126
|
+
/** The caller's projection, pre-multiplied by the backend's clip correction. */
|
|
127
|
+
const projection = new Float32Array(16);
|
|
128
|
+
/**
|
|
129
|
+
* Where the camera is in the capture's own space, which is where the harmonics were trained.
|
|
130
|
+
*
|
|
131
|
+
* Recomputed whenever the view or the model changes rather than every frame, because both are
|
|
132
|
+
* setters and neither is called more than once a frame. Zero for a capture with no harmonics,
|
|
133
|
+
* where nothing reads it.
|
|
134
|
+
*/
|
|
135
|
+
const cameraLocal = new Float32Array(3);
|
|
136
|
+
const shDegree = splats.shDegree;
|
|
137
|
+
const texels = splatTexels(splats.wordsPerSplat);
|
|
138
|
+
/* Identity until `init`, so a `setView` before registration is not silently zeroed. */
|
|
139
|
+
let clipCorrection: ArrayLike<number> = IDENTITY;
|
|
140
|
+
let viewportX = 1;
|
|
141
|
+
let viewportY = 1;
|
|
142
|
+
/* True until the first `setView`, so a caller that never sets one still draws. */
|
|
143
|
+
let visible = true;
|
|
144
|
+
|
|
145
|
+
const writeVertexUniforms = (target: GpuSplats): void => {
|
|
146
|
+
const f = target.vertexFloats;
|
|
147
|
+
const i = target.vertexInts;
|
|
148
|
+
i[VERT.fields.uSplatCount.offset / 4] = drawCount;
|
|
149
|
+
i[VERT.fields.uSplatStride.offset / 4] = SPLAT_STRIDE;
|
|
150
|
+
i[VERT.fields.uSplatTexels.offset / 4] = texels;
|
|
151
|
+
i[VERT.fields.uSplatShDegree.offset / 4] = shDegree;
|
|
152
|
+
f.set(cameraLocal, VERT.fields.uSplatCameraLocal.offset / 4);
|
|
153
|
+
f.set(view, VERT.fields.uView.offset / 4);
|
|
154
|
+
f.set(projection, VERT.fields.uProjection.offset / 4);
|
|
155
|
+
f[VERT.fields.uViewport.offset / 4] = viewportX;
|
|
156
|
+
f[VERT.fields.uViewport.offset / 4 + 1] = viewportY;
|
|
157
|
+
f.set(model, VERT.fields.uModel.offset / 4);
|
|
158
|
+
};
|
|
159
|
+
|
|
160
|
+
return {
|
|
161
|
+
label,
|
|
162
|
+
|
|
163
|
+
init(passDevice: PassDevice): void {
|
|
164
|
+
/*
|
|
165
|
+
* **The clip correction the renderer's own verbs use, not a copy of it.** WebGPU's
|
|
166
|
+
* framebuffer origin is the top-left and OpenGL's is the bottom-left, and the engine settles
|
|
167
|
+
* that in the matrix rather than in the shaders. This pass takes its camera from its caller,
|
|
168
|
+
* so it never sees the corrected matrix — without this it drew the world upside down on
|
|
169
|
+
* WebGPU, which is exactly the first frame that backend ever produced.
|
|
170
|
+
*/
|
|
171
|
+
clipCorrection = passDevice.clipCorrection;
|
|
172
|
+
if (passDevice.backend === 'webgl2') {
|
|
173
|
+
gl = createWebgl2Splats(passDevice.gl, splats, label);
|
|
174
|
+
return;
|
|
175
|
+
}
|
|
176
|
+
device = passDevice.device;
|
|
177
|
+
gpu = createGpuSplats(
|
|
178
|
+
passDevice.device,
|
|
179
|
+
passDevice.format,
|
|
180
|
+
passDevice.depthFormat,
|
|
181
|
+
passDevice.samples,
|
|
182
|
+
splats,
|
|
183
|
+
label,
|
|
184
|
+
);
|
|
185
|
+
},
|
|
186
|
+
|
|
187
|
+
setView(next: SplatView): void {
|
|
188
|
+
view.set(next.view);
|
|
189
|
+
multiplyMat4(projection, clipCorrection, next.projection);
|
|
190
|
+
viewportX = next.widthPx;
|
|
191
|
+
viewportY = next.heightPx;
|
|
192
|
+
/*
|
|
193
|
+
* Culled against the caller's **uncorrected** projection, because the frustum planes come
|
|
194
|
+
* out of it in the convention `mat4.perspective` writes; the corrected one has moved z into
|
|
195
|
+
* [0, 1] and flipped y, and its near plane would be a different plane.
|
|
196
|
+
*/
|
|
197
|
+
visible = splatBoundsVisible(
|
|
198
|
+
next.view,
|
|
199
|
+
next.projection,
|
|
200
|
+
model,
|
|
201
|
+
splats.boundsMin,
|
|
202
|
+
splats.boundsMax,
|
|
203
|
+
);
|
|
204
|
+
if (shDegree > 0) cameraInCaptureSpace(cameraLocal, view, model);
|
|
205
|
+
},
|
|
206
|
+
|
|
207
|
+
setModel(next: ArrayLike<number>): void {
|
|
208
|
+
model.set(next);
|
|
209
|
+
/* The camera's place in the capture's space moves when either matrix does, and a model set
|
|
210
|
+
after a view would otherwise evaluate the harmonics against the previous placement. */
|
|
211
|
+
if (shDegree > 0) cameraInCaptureSpace(cameraLocal, view, model);
|
|
212
|
+
},
|
|
213
|
+
|
|
214
|
+
uploadSplats(from: number, count: number): void {
|
|
215
|
+
const first = Math.max(0, Math.min(from, splats.count));
|
|
216
|
+
const howMany = Math.max(0, Math.min(count, splats.count - first));
|
|
217
|
+
if (howMany <= 0) return;
|
|
218
|
+
if (gpu !== null && device !== null) {
|
|
219
|
+
uploadGpuSplatRange(device, gpu, splats.packed, first, howMany);
|
|
220
|
+
return;
|
|
221
|
+
}
|
|
222
|
+
/*
|
|
223
|
+
* WebGL2 has no context outside `draw`, so the range is remembered and pushed there — the
|
|
224
|
+
* same reason `setOrder` holds a pending order. Ranges are merged rather than queued
|
|
225
|
+
* because they arrive in file order and a merged span re-sends whole rows anyway.
|
|
226
|
+
*/
|
|
227
|
+
if (pendingRange === null) pendingRange = { from: first, to: first + howMany };
|
|
228
|
+
else {
|
|
229
|
+
pendingRange.from = Math.min(pendingRange.from, first);
|
|
230
|
+
pendingRange.to = Math.max(pendingRange.to, first + howMany);
|
|
231
|
+
}
|
|
232
|
+
},
|
|
233
|
+
|
|
234
|
+
setOrder(order: Uint32Array, count: number): void {
|
|
235
|
+
drawCount = Math.max(0, Math.min(count, splats.count));
|
|
236
|
+
/*
|
|
237
|
+
* Held rather than uploaded here, because a consumer may sort before the pass has a device:
|
|
238
|
+
* `registerPass` runs `init` at registration, but a sorter finishing first is ordinary and
|
|
239
|
+
* dropping its result would leave the capture blank until the view turned again.
|
|
240
|
+
*/
|
|
241
|
+
pendingOrder = order;
|
|
242
|
+
if (gl !== null) {
|
|
243
|
+
// The upload needs a context, which only `draw` is handed on this backend.
|
|
244
|
+
return;
|
|
245
|
+
}
|
|
246
|
+
if (gpu !== null && device !== null) {
|
|
247
|
+
uploadGpuOrder(device, gpu, order, paddedOrder);
|
|
248
|
+
pendingOrder = null;
|
|
249
|
+
}
|
|
250
|
+
},
|
|
251
|
+
|
|
252
|
+
draw(ctx: PassContext): void {
|
|
253
|
+
if (drawCount <= 0 || !visible) return;
|
|
254
|
+
|
|
255
|
+
if (ctx.backend === 'webgl2') {
|
|
256
|
+
if (gl === null) return;
|
|
257
|
+
if (pendingRange !== null) {
|
|
258
|
+
uploadWebgl2SplatRange(
|
|
259
|
+
ctx.gl,
|
|
260
|
+
gl,
|
|
261
|
+
splats.packed,
|
|
262
|
+
pendingRange.from,
|
|
263
|
+
pendingRange.to - pendingRange.from,
|
|
264
|
+
);
|
|
265
|
+
pendingRange = null;
|
|
266
|
+
}
|
|
267
|
+
if (pendingOrder !== null) {
|
|
268
|
+
uploadWebgl2Order(ctx.gl, gl, pendingOrder, paddedOrder);
|
|
269
|
+
pendingOrder = null;
|
|
270
|
+
}
|
|
271
|
+
const { uniforms } = gl;
|
|
272
|
+
ctx.gl.useProgram(gl.program);
|
|
273
|
+
ctx.gl.uniform1i(uniforms['uSplatCount'] ?? null, drawCount);
|
|
274
|
+
ctx.gl.uniform1i(uniforms['uSplatStride'] ?? null, SPLAT_STRIDE);
|
|
275
|
+
ctx.gl.uniform1i(uniforms['uSplatTexels'] ?? null, texels);
|
|
276
|
+
ctx.gl.uniform1i(uniforms['uSplatShDegree'] ?? null, shDegree);
|
|
277
|
+
ctx.gl.uniform3fv(uniforms['uSplatCameraLocal'] ?? null, cameraLocal);
|
|
278
|
+
ctx.gl.uniformMatrix4fv(uniforms['uView'] ?? null, false, view);
|
|
279
|
+
ctx.gl.uniformMatrix4fv(uniforms['uProjection'] ?? null, false, projection);
|
|
280
|
+
ctx.gl.uniform2f(uniforms['uViewport'] ?? null, viewportX, viewportY);
|
|
281
|
+
ctx.gl.uniformMatrix4fv(uniforms['uModel'] ?? null, false, model);
|
|
282
|
+
/* The frame's own grade, per `PassContext`: this is a forward pass and may be last. */
|
|
283
|
+
ctx.gl.uniform1i(uniforms['uOutputTransform'] ?? null, ctx.outputTransform);
|
|
284
|
+
ctx.gl.uniform1f(uniforms['uOutputExposure'] ?? null, ctx.outputExposure);
|
|
285
|
+
drawWebgl2Splats(ctx.gl, gl, drawCount);
|
|
286
|
+
return;
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
if (gpu === null || device === null) return;
|
|
290
|
+
if (pendingOrder !== null) {
|
|
291
|
+
uploadGpuOrder(device, gpu, pendingOrder, paddedOrder);
|
|
292
|
+
pendingOrder = null;
|
|
293
|
+
}
|
|
294
|
+
writeVertexUniforms(gpu);
|
|
295
|
+
device.queue.writeBuffer(gpu.vertexUniforms, 0, gpu.vertexScratch);
|
|
296
|
+
gpu.fragmentInts[FRAG.fields.uOutputTransform.offset / 4] = ctx.outputTransform;
|
|
297
|
+
gpu.fragmentFloats[FRAG.fields.uOutputExposure.offset / 4] = ctx.outputExposure;
|
|
298
|
+
device.queue.writeBuffer(gpu.fragmentUniforms, 0, gpu.fragmentScratch);
|
|
299
|
+
|
|
300
|
+
ctx.pass.setPipeline(gpu.pipeline);
|
|
301
|
+
ctx.pass.setBindGroup(0, gpu.bindGroup);
|
|
302
|
+
ctx.pass.draw(drawCount * 6);
|
|
303
|
+
},
|
|
304
|
+
|
|
305
|
+
dispose(passDevice: PassDevice): void {
|
|
306
|
+
if (passDevice.backend === 'webgl2') {
|
|
307
|
+
if (gl !== null) disposeWebgl2Splats(passDevice.gl, gl);
|
|
308
|
+
gl = null;
|
|
309
|
+
return;
|
|
310
|
+
}
|
|
311
|
+
if (gpu !== null) disposeGpuSplats(gpu);
|
|
312
|
+
gpu = null;
|
|
313
|
+
device = null;
|
|
314
|
+
},
|
|
315
|
+
|
|
316
|
+
get visible(): boolean {
|
|
317
|
+
return visible;
|
|
318
|
+
},
|
|
319
|
+
|
|
320
|
+
get count(): number {
|
|
321
|
+
return splats.count;
|
|
322
|
+
},
|
|
323
|
+
};
|
|
324
|
+
}
|
package/src/splatPly.ts
ADDED
|
@@ -0,0 +1,283 @@
|
|
|
1
|
+
/** The `.ply` reader: an ASCII header describing a binary body, as a training run writes it. */
|
|
2
|
+
|
|
3
|
+
import { SPLAT_SH1_COEFFICIENTS, packSplats } from './splatData.ts';
|
|
4
|
+
import type { SplatData } from './splatData.ts';
|
|
5
|
+
|
|
6
|
+
/** Coefficients per channel in the l=1 band: the three basis functions Y(1,−1), Y(1,0), Y(1,1). */
|
|
7
|
+
const SH1_PER_CHANNEL = 3;
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* The spherical-harmonic band-0 constant, `0.5 * sqrt(1 / pi)`.
|
|
11
|
+
*
|
|
12
|
+
* A capture stores colour as the DC term of a spherical-harmonic expansion, which is a *radiance
|
|
13
|
+
* coefficient* and not a colour: it is signed, unbounded, and centred on zero rather than on a half.
|
|
14
|
+
* `0.5 + C0 * f_dc` is the conversion every viewer of this format uses, and getting it wrong is not
|
|
15
|
+
* subtle — dropping the 0.5 leaves half the capture negative and clamped to black.
|
|
16
|
+
*/
|
|
17
|
+
const SH_C0 = 0.28209479177387814;
|
|
18
|
+
|
|
19
|
+
/** How many bytes each PLY scalar type occupies, and how to read one. */
|
|
20
|
+
const SCALARS: Readonly<
|
|
21
|
+
Record<string, { readonly bytes: number; readonly read: (view: DataView, at: number) => number }>
|
|
22
|
+
> = {
|
|
23
|
+
char: { bytes: 1, read: (v, at) => v.getInt8(at) },
|
|
24
|
+
int8: { bytes: 1, read: (v, at) => v.getInt8(at) },
|
|
25
|
+
uchar: { bytes: 1, read: (v, at) => v.getUint8(at) },
|
|
26
|
+
uint8: { bytes: 1, read: (v, at) => v.getUint8(at) },
|
|
27
|
+
short: { bytes: 2, read: (v, at) => v.getInt16(at, true) },
|
|
28
|
+
int16: { bytes: 2, read: (v, at) => v.getInt16(at, true) },
|
|
29
|
+
ushort: { bytes: 2, read: (v, at) => v.getUint16(at, true) },
|
|
30
|
+
uint16: { bytes: 2, read: (v, at) => v.getUint16(at, true) },
|
|
31
|
+
int: { bytes: 4, read: (v, at) => v.getInt32(at, true) },
|
|
32
|
+
int32: { bytes: 4, read: (v, at) => v.getInt32(at, true) },
|
|
33
|
+
uint: { bytes: 4, read: (v, at) => v.getUint32(at, true) },
|
|
34
|
+
uint32: { bytes: 4, read: (v, at) => v.getUint32(at, true) },
|
|
35
|
+
float: { bytes: 4, read: (v, at) => v.getFloat32(at, true) },
|
|
36
|
+
float32: { bytes: 4, read: (v, at) => v.getFloat32(at, true) },
|
|
37
|
+
double: { bytes: 8, read: (v, at) => v.getFloat64(at, true) },
|
|
38
|
+
float64: { bytes: 8, read: (v, at) => v.getFloat64(at, true) },
|
|
39
|
+
};
|
|
40
|
+
|
|
41
|
+
const HEADER_END = 'end_header';
|
|
42
|
+
|
|
43
|
+
interface Property {
|
|
44
|
+
readonly name: string;
|
|
45
|
+
readonly offset: number;
|
|
46
|
+
readonly read: (view: DataView, at: number) => number;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
interface Header {
|
|
50
|
+
readonly count: number;
|
|
51
|
+
readonly stride: number;
|
|
52
|
+
readonly properties: ReadonlyMap<string, Property>;
|
|
53
|
+
readonly bodyAt: number;
|
|
54
|
+
readonly sphericalHarmonics: number;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* Parse the ASCII header and build a property offset table from the **declared** order.
|
|
59
|
+
*
|
|
60
|
+
* **Declared, never assumed.** Training runs emit roughly
|
|
61
|
+
* `x y z nx ny nz f_dc_0..2 f_rest_0..44 opacity scale_0..2 rot_0..3`, and the normals are unused
|
|
62
|
+
* and sometimes absent — so a reader that hard-codes offsets reads a file that is one property
|
|
63
|
+
* short as garbage, silently, at every field after the gap.
|
|
64
|
+
*/
|
|
65
|
+
function parseHeader(bytes: Uint8Array): Header {
|
|
66
|
+
/* Latin-1 rather than UTF-8: the header is ASCII and the body is not text, so decoding the whole
|
|
67
|
+
buffer as UTF-8 can throw on a byte sequence that is simply a float. */
|
|
68
|
+
const text = new TextDecoder('latin1').decode(bytes.subarray(0, Math.min(bytes.length, 65536)));
|
|
69
|
+
const endsAt = text.indexOf(HEADER_END);
|
|
70
|
+
if (endsAt < 0) {
|
|
71
|
+
throw new Error(
|
|
72
|
+
`no \`${HEADER_END}\` in the first ${Math.min(bytes.length, 65536)} bytes, so this is not a ` +
|
|
73
|
+
'PLY this reader understands.',
|
|
74
|
+
);
|
|
75
|
+
}
|
|
76
|
+
/* Past the marker and its line ending, which is \n or \r\n depending on the writer. */
|
|
77
|
+
const afterMarker = endsAt + HEADER_END.length;
|
|
78
|
+
let bodyAt = afterMarker;
|
|
79
|
+
if (text[bodyAt] === '\r') bodyAt++;
|
|
80
|
+
if (text[bodyAt] === '\n') bodyAt++;
|
|
81
|
+
|
|
82
|
+
const lines = text
|
|
83
|
+
.slice(0, endsAt)
|
|
84
|
+
.split(/\r?\n/)
|
|
85
|
+
.map((line) => line.trim());
|
|
86
|
+
if ((lines[0] ?? '') !== 'ply')
|
|
87
|
+
throw new Error('the first line of a PLY is `ply`; this one is not.');
|
|
88
|
+
|
|
89
|
+
const format = lines.find((line) => line.startsWith('format '));
|
|
90
|
+
if (format === undefined) throw new Error('the header declares no `format` line.');
|
|
91
|
+
if (format !== 'format binary_little_endian 1.0') {
|
|
92
|
+
throw new Error(
|
|
93
|
+
`this reader takes \`format binary_little_endian 1.0\` and the file says \`${format}\`. ` +
|
|
94
|
+
'An ASCII or big-endian body is refused rather than misread, because a silent misread of a ' +
|
|
95
|
+
'two-hundred-megabyte capture is worse than a stop.',
|
|
96
|
+
);
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/*
|
|
100
|
+
* Only the `vertex` element carries splats. Other elements are skipped entirely rather than
|
|
101
|
+
* parsed: a capture that also declares faces is still a capture, and this reader has no use for
|
|
102
|
+
* them — but their properties must not land in the vertex stride.
|
|
103
|
+
*/
|
|
104
|
+
const properties = new Map<string, Property>();
|
|
105
|
+
let count = -1;
|
|
106
|
+
let stride = 0;
|
|
107
|
+
let inVertex = false;
|
|
108
|
+
let sphericalHarmonics = 0;
|
|
109
|
+
for (const line of lines) {
|
|
110
|
+
if (line.startsWith('element ')) {
|
|
111
|
+
const [, name, howMany] = line.split(/\s+/);
|
|
112
|
+
inVertex = name === 'vertex';
|
|
113
|
+
if (inVertex) count = Number(howMany);
|
|
114
|
+
continue;
|
|
115
|
+
}
|
|
116
|
+
if (!inVertex || !line.startsWith('property ')) continue;
|
|
117
|
+
const [, type, name] = line.split(/\s+/);
|
|
118
|
+
if (type === 'list') {
|
|
119
|
+
throw new Error(
|
|
120
|
+
'a `property list` in the vertex element gives every splat a different size, which this ' +
|
|
121
|
+
'reader does not handle. A Gaussian capture has none.',
|
|
122
|
+
);
|
|
123
|
+
}
|
|
124
|
+
const scalar = SCALARS[type ?? ''];
|
|
125
|
+
if (scalar === undefined) throw new Error(`unknown PLY scalar type \`${type}\`.`);
|
|
126
|
+
if (name === undefined) throw new Error(`a \`property ${type}\` line names no property.`);
|
|
127
|
+
properties.set(name, { name, offset: stride, read: scalar.read });
|
|
128
|
+
/* Located and counted, never read: view-dependent colour is out of this row's scope, and
|
|
129
|
+
knowing a capture carries the coefficients is what makes adding it a reader change. */
|
|
130
|
+
if (name.startsWith('f_rest_')) sphericalHarmonics++;
|
|
131
|
+
stride += scalar.bytes;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
if (count < 0) throw new Error('the header declares no `vertex` element.');
|
|
135
|
+
return { count, stride, properties, bodyAt, sphericalHarmonics };
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
function require_(header: Header, name: string): Property {
|
|
139
|
+
const found = header.properties.get(name);
|
|
140
|
+
if (found === undefined) {
|
|
141
|
+
throw new Error(
|
|
142
|
+
`the vertex element declares no \`${name}\`, which a Gaussian capture must carry. ` +
|
|
143
|
+
`It declares: ${[...header.properties.keys()].slice(0, 12).join(', ')}…`,
|
|
144
|
+
);
|
|
145
|
+
}
|
|
146
|
+
return found;
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
/**
|
|
150
|
+
* Read a `.ply` Gaussian capture.
|
|
151
|
+
*
|
|
152
|
+
* **The encodings are the whole of this function's risk.** Scale is stored as its *logarithm* and
|
|
153
|
+
* opacity as its *logit*, so both are undone here — and here only, because `packSplats` documents
|
|
154
|
+
* that it takes linear values. Getting the boundary wrong produces a capture that is either
|
|
155
|
+
* invisible or a solid block, and both have been reported against other viewers as rendering bugs.
|
|
156
|
+
*
|
|
157
|
+
* **The file stores rotation as wxyz** — `rot_0` is the real part — and `SplatSource` takes xyzw,
|
|
158
|
+
* so the reorder happens here for the same reason it happens in the `.splat` reader.
|
|
159
|
+
*/
|
|
160
|
+
export function readSplatPly(buffer: ArrayBuffer): SplatData {
|
|
161
|
+
const bytes = new Uint8Array(buffer);
|
|
162
|
+
const header = parseHeader(bytes);
|
|
163
|
+
const { count, stride, bodyAt } = header;
|
|
164
|
+
|
|
165
|
+
const needed = bodyAt + count * stride;
|
|
166
|
+
if (buffer.byteLength < needed) {
|
|
167
|
+
throw new Error(
|
|
168
|
+
`the header declares ${count} vertices of ${stride} bytes after a ${bodyAt}-byte header, ` +
|
|
169
|
+
`which needs ${needed} bytes; the file is ${buffer.byteLength}. It is truncated.`,
|
|
170
|
+
);
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
const x = require_(header, 'x');
|
|
174
|
+
const y = require_(header, 'y');
|
|
175
|
+
const z = require_(header, 'z');
|
|
176
|
+
const dc0 = require_(header, 'f_dc_0');
|
|
177
|
+
const dc1 = require_(header, 'f_dc_1');
|
|
178
|
+
const dc2 = require_(header, 'f_dc_2');
|
|
179
|
+
const alpha = require_(header, 'opacity');
|
|
180
|
+
const s0 = require_(header, 'scale_0');
|
|
181
|
+
const s1 = require_(header, 'scale_1');
|
|
182
|
+
const s2 = require_(header, 'scale_2');
|
|
183
|
+
const r0 = require_(header, 'rot_0');
|
|
184
|
+
const r1 = require_(header, 'rot_1');
|
|
185
|
+
const r2 = require_(header, 'rot_2');
|
|
186
|
+
const r3 = require_(header, 'rot_3');
|
|
187
|
+
|
|
188
|
+
/*
|
|
189
|
+
* The l=1 band, if the file has one, and **the transpose that finding it needs**.
|
|
190
|
+
*
|
|
191
|
+
* A training run writes `f_rest_*` **channel-major**: all of red's coefficients, then all of
|
|
192
|
+
* green's, then all of blue's. So the per-channel stride is the total over three, and the l=1
|
|
193
|
+
* band is the first three of each channel — `f_rest_0..2`, then `f_rest_{n}..{n+2}`, then
|
|
194
|
+
* `f_rest_{2n}..{2n+2}` — where `n` is 3 at degree 1, 8 at degree 2 and 15 at degree 3.
|
|
195
|
+
*
|
|
196
|
+
* **Reading them as if they were interleaved is the failure to expect**, because it produces a
|
|
197
|
+
* capture that is plausible from a distance: the nine values are all real coefficients of
|
|
198
|
+
* *something*, so the cloud still has a sheen, and it is the wrong sheen in the wrong channel.
|
|
199
|
+
* The count is checked against a multiple of three rather than trusted, and a file whose
|
|
200
|
+
* `f_rest_*` count is not divisible by three is read as having no harmonics at all.
|
|
201
|
+
*/
|
|
202
|
+
const perChannel = Math.floor(header.sphericalHarmonics / 3);
|
|
203
|
+
const wantsSh =
|
|
204
|
+
header.sphericalHarmonics > 0 &&
|
|
205
|
+
header.sphericalHarmonics % 3 === 0 &&
|
|
206
|
+
perChannel >= SH1_PER_CHANNEL;
|
|
207
|
+
const restBands: Property[] = [];
|
|
208
|
+
if (wantsSh) {
|
|
209
|
+
for (let channel = 0; channel < 3; channel++) {
|
|
210
|
+
for (let band = 0; band < SH1_PER_CHANNEL; band++) {
|
|
211
|
+
const property = header.properties.get(`f_rest_${channel * perChannel + band}`);
|
|
212
|
+
if (property === undefined) {
|
|
213
|
+
restBands.length = 0;
|
|
214
|
+
break;
|
|
215
|
+
}
|
|
216
|
+
restBands.push(property);
|
|
217
|
+
}
|
|
218
|
+
if (restBands.length === 0) break;
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
const hasSh = restBands.length === SH1_PER_CHANNEL * 3;
|
|
222
|
+
const sh1 = hasSh ? new Float32Array(count * SPLAT_SH1_COEFFICIENTS) : undefined;
|
|
223
|
+
|
|
224
|
+
const view = new DataView(buffer);
|
|
225
|
+
const positions = new Float32Array(count * 3);
|
|
226
|
+
const scales = new Float32Array(count * 3);
|
|
227
|
+
const rotations = new Float32Array(count * 4);
|
|
228
|
+
const colors = new Float32Array(count * 3);
|
|
229
|
+
const opacities = new Float32Array(count);
|
|
230
|
+
|
|
231
|
+
const clamp01 = (value: number): number => (value < 0 ? 0 : value > 1 ? 1 : value);
|
|
232
|
+
|
|
233
|
+
for (let index = 0; index < count; index++) {
|
|
234
|
+
const at = bodyAt + index * stride;
|
|
235
|
+
const p = index * 3;
|
|
236
|
+
positions[p] = x.read(view, at + x.offset);
|
|
237
|
+
positions[p + 1] = y.read(view, at + y.offset);
|
|
238
|
+
positions[p + 2] = z.read(view, at + z.offset);
|
|
239
|
+
|
|
240
|
+
/* Stored as a logarithm, so the standard deviation is its exponential. */
|
|
241
|
+
scales[p] = Math.exp(s0.read(view, at + s0.offset));
|
|
242
|
+
scales[p + 1] = Math.exp(s1.read(view, at + s1.offset));
|
|
243
|
+
scales[p + 2] = Math.exp(s2.read(view, at + s2.offset));
|
|
244
|
+
|
|
245
|
+
colors[p] = clamp01(0.5 + SH_C0 * dc0.read(view, at + dc0.offset));
|
|
246
|
+
colors[p + 1] = clamp01(0.5 + SH_C0 * dc1.read(view, at + dc1.offset));
|
|
247
|
+
colors[p + 2] = clamp01(0.5 + SH_C0 * dc2.read(view, at + dc2.offset));
|
|
248
|
+
/* Stored as a logit, so the opacity is its logistic. */
|
|
249
|
+
opacities[index] = 1 / (1 + Math.exp(-alpha.read(view, at + alpha.offset)));
|
|
250
|
+
|
|
251
|
+
const r = index * 4;
|
|
252
|
+
rotations[r] = r1.read(view, at + r1.offset);
|
|
253
|
+
rotations[r + 1] = r2.read(view, at + r2.offset);
|
|
254
|
+
rotations[r + 2] = r3.read(view, at + r3.offset);
|
|
255
|
+
rotations[r + 3] = r0.read(view, at + r0.offset);
|
|
256
|
+
|
|
257
|
+
/*
|
|
258
|
+
* Channel-major in the file, basis-major in `SplatSource` — the transpose the block above
|
|
259
|
+
* describes, in the one loop that touches the file's own order.
|
|
260
|
+
*/
|
|
261
|
+
if (sh1 !== undefined) {
|
|
262
|
+
const to = index * SPLAT_SH1_COEFFICIENTS;
|
|
263
|
+
for (let band = 0; band < SH1_PER_CHANNEL; band++) {
|
|
264
|
+
for (let channel = 0; channel < 3; channel++) {
|
|
265
|
+
const property = restBands[channel * SH1_PER_CHANNEL + band];
|
|
266
|
+
sh1[to + band * 3 + channel] =
|
|
267
|
+
property === undefined ? 0 : property.read(view, at + property.offset);
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
const data = packSplats({
|
|
274
|
+
count,
|
|
275
|
+
positions,
|
|
276
|
+
scales,
|
|
277
|
+
rotations,
|
|
278
|
+
colors,
|
|
279
|
+
opacities,
|
|
280
|
+
...(sh1 === undefined ? {} : { sh1 }),
|
|
281
|
+
});
|
|
282
|
+
return { ...data, sphericalHarmonics: header.sphericalHarmonics };
|
|
283
|
+
}
|