@driftengine/splats 3.61.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/NOTICE +9 -0
- package/README.md +56 -0
- package/dist/half.d.ts +32 -0
- package/dist/half.js +88 -0
- package/dist/index.d.ts +32 -0
- package/dist/index.js +38 -0
- package/dist/shaders/generated/splat.wgsl.d.ts +89 -0
- package/dist/shaders/generated/splat.wgsl.js +95 -0
- package/dist/shaders/splat.d.ts +25 -0
- package/dist/shaders/splat.js +337 -0
- package/dist/splat.d.ts +26 -0
- package/dist/splat.js +63 -0
- package/dist/splatBudget.d.ts +40 -0
- package/dist/splatBudget.js +45 -0
- package/dist/splatCapture.d.ts +76 -0
- package/dist/splatCapture.js +108 -0
- package/dist/splatCull.d.ts +25 -0
- package/dist/splatCull.js +80 -0
- package/dist/splatData.d.ts +177 -0
- package/dist/splatData.js +223 -0
- package/dist/splatGl.d.ts +49 -0
- package/dist/splatGl.js +176 -0
- package/dist/splatGpu.d.ts +50 -0
- package/dist/splatGpu.js +180 -0
- package/dist/splatLayout.d.ts +52 -0
- package/dist/splatLayout.js +75 -0
- package/dist/splatMatrix.d.ts +29 -0
- package/dist/splatMatrix.js +68 -0
- package/dist/splatPass.d.ts +83 -0
- package/dist/splatPass.js +206 -0
- package/dist/splatPly.d.ts +14 -0
- package/dist/splatPly.js +242 -0
- package/dist/splatSog.d.ts +110 -0
- package/dist/splatSog.js +285 -0
- package/dist/splatSogDecoder.d.ts +26 -0
- package/dist/splatSogDecoder.js +29 -0
- package/dist/splatSort.d.ts +137 -0
- package/dist/splatSort.js +199 -0
- package/dist/splatSortWorker.d.ts +14 -0
- package/dist/splatSortWorker.js +137 -0
- package/dist/splatSorter.d.ts +112 -0
- package/dist/splatSorter.js +231 -0
- package/dist/splatView.d.ts +52 -0
- package/dist/splatView.js +115 -0
- package/package.json +56 -0
- package/src/fixtures/README.md +36 -0
- package/src/fixtures/cloud.sog +0 -0
- package/src/fixtures/cloud.texels.json +27 -0
- package/src/fixtures/cloud.truth.json +582 -0
- package/src/half.ts +92 -0
- package/src/index.ts +55 -0
- package/src/shaders/generated/splat.wgsl.ts +98 -0
- package/src/shaders/splat.ts +344 -0
- package/src/splat.ts +75 -0
- package/src/splatBudget.ts +48 -0
- package/src/splatCapture.ts +154 -0
- package/src/splatCull.ts +91 -0
- package/src/splatData.ts +398 -0
- package/src/splatGl.ts +262 -0
- package/src/splatGpu.ts +259 -0
- package/src/splatLayout.ts +86 -0
- package/src/splatMatrix.ts +81 -0
- package/src/splatPass.ts +324 -0
- package/src/splatPly.ts +283 -0
- package/src/splatSog.ts +375 -0
- package/src/splatSogDecoder.ts +33 -0
- package/src/splatSort.ts +296 -0
- package/src/splatSortWorker.ts +155 -0
- package/src/splatSorter.ts +285 -0
- package/src/splatView.ts +147 -0
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
/** A capture that fills up as its blocks arrive, sized once from the count the first one names. */
|
|
2
|
+
|
|
3
|
+
import { SPLAT_WORDS, SPLAT_WORDS_SH1 } from './splatData.ts';
|
|
4
|
+
import type { SplatData } from './splatData.ts';
|
|
5
|
+
|
|
6
|
+
export interface SplatCaptureOptions {
|
|
7
|
+
/** Splats in the whole capture. Every array is allocated for this and never grows. */
|
|
8
|
+
readonly total: number;
|
|
9
|
+
/** The whole capture's extent, which a `.drft` block carries in its own header. */
|
|
10
|
+
readonly boundsMin: ArrayLike<number>;
|
|
11
|
+
readonly boundsMax: ArrayLike<number>;
|
|
12
|
+
/** Coefficients the source carried, of which the shader reads the first band. Recorded. */
|
|
13
|
+
readonly sphericalHarmonics?: number;
|
|
14
|
+
/**
|
|
15
|
+
* Words in one record, from the block's own header.
|
|
16
|
+
*
|
|
17
|
+
* **Taken from the file rather than assumed**, which is what `FORMAT.md` §4.7 designed the field
|
|
18
|
+
* for: a capture with degree-1 view-dependent colour writes twelve where one without writes
|
|
19
|
+
* eight, and a streaming reader that hard-coded the width would read the second block's records
|
|
20
|
+
* at the wrong offset and draw a cloud of noise. Defaults to `SPLAT_WORDS` so a caller with no
|
|
21
|
+
* container behind it is unaffected.
|
|
22
|
+
*/
|
|
23
|
+
readonly wordsPerSplat?: number;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/** Where a block landed, so a caller uploads what arrived rather than the whole texture. */
|
|
27
|
+
export interface SplatAppend {
|
|
28
|
+
readonly from: number;
|
|
29
|
+
readonly count: number;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* A capture assembled from blocks, one `.drft` chunk at a time.
|
|
34
|
+
*
|
|
35
|
+
* **Allocated once at the final count and filled progressively**, which is what makes a streaming
|
|
36
|
+
* load cost nothing extra: the GPU textures are sized for the whole capture at registration, each
|
|
37
|
+
* block is a sub-upload into storage that already exists, and the drawn count rises. Growing an
|
|
38
|
+
* array as blocks land would mean reallocating and re-uploading a capture's worth of texels
|
|
39
|
+
* several times during a load, which is more work than the load itself.
|
|
40
|
+
*
|
|
41
|
+
* **Every block is a sparse version of the whole capture**, because the container writes them
|
|
42
|
+
* that way — see `coarseFirstOrder`. So a prefix of the arrived splats is a recognisable place at
|
|
43
|
+
* lower density rather than one finished corner, and the load opens on something worth looking
|
|
44
|
+
* at. That property lives in the writer; what lives here is that nothing is re-ordered or
|
|
45
|
+
* re-allocated on the way in.
|
|
46
|
+
*
|
|
47
|
+
* The unarrived tail is zeroed, which puts it at the origin with no size. Nothing draws it: the
|
|
48
|
+
* caller sorts and draws `ready` splats and the sorter is told the same number.
|
|
49
|
+
*/
|
|
50
|
+
export class SplatCapture {
|
|
51
|
+
readonly data: SplatData;
|
|
52
|
+
private arrived = 0;
|
|
53
|
+
private readonly words: number;
|
|
54
|
+
|
|
55
|
+
constructor(options: SplatCaptureOptions) {
|
|
56
|
+
const total = Math.max(0, options.total);
|
|
57
|
+
this.words = options.wordsPerSplat ?? SPLAT_WORDS;
|
|
58
|
+
this.data = {
|
|
59
|
+
count: total,
|
|
60
|
+
positions: new Float32Array(total * 3),
|
|
61
|
+
packed: new Uint32Array(total * this.words),
|
|
62
|
+
wordsPerSplat: this.words,
|
|
63
|
+
shDegree: this.words >= SPLAT_WORDS_SH1 ? 1 : 0,
|
|
64
|
+
extents: new Float32Array(total),
|
|
65
|
+
boundsMin: new Float32Array([
|
|
66
|
+
options.boundsMin[0] ?? 0,
|
|
67
|
+
options.boundsMin[1] ?? 0,
|
|
68
|
+
options.boundsMin[2] ?? 0,
|
|
69
|
+
]),
|
|
70
|
+
boundsMax: new Float32Array([
|
|
71
|
+
options.boundsMax[0] ?? 0,
|
|
72
|
+
options.boundsMax[1] ?? 0,
|
|
73
|
+
options.boundsMax[2] ?? 0,
|
|
74
|
+
]),
|
|
75
|
+
sphericalHarmonics: options.sphericalHarmonics ?? 0,
|
|
76
|
+
};
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/** How many splats have landed. The count to sort, to draw and to upload up to. */
|
|
80
|
+
get ready(): number {
|
|
81
|
+
return this.arrived;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* Copy one block's records in, deriving what the CPU needs from what the file carried.
|
|
86
|
+
*
|
|
87
|
+
* **Positions and extents come back out of the records rather than being stored beside them**,
|
|
88
|
+
* which is the whole reason a `.drft` splat block is thirty-two bytes a splat rather than
|
|
89
|
+
* forty-eight: the first three words are the position's own float bits and the eighth is the
|
|
90
|
+
* extent's. See `packSplats`, which is what puts them there.
|
|
91
|
+
*/
|
|
92
|
+
append(records: Uint32Array, count: number): SplatAppend {
|
|
93
|
+
if (count < 0 || this.arrived + count > this.data.count) {
|
|
94
|
+
throw new Error(
|
|
95
|
+
`splats: a block of ${count} splats lands at ${this.arrived} in a capture of ` +
|
|
96
|
+
`${this.data.count} splats, which does not fit`,
|
|
97
|
+
);
|
|
98
|
+
}
|
|
99
|
+
if (records.length < count * this.words) {
|
|
100
|
+
throw new Error(
|
|
101
|
+
`splats: ${count} splats need ${count * this.words} entries and the block has ` +
|
|
102
|
+
`${records.length}`,
|
|
103
|
+
);
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
const from = this.arrived;
|
|
107
|
+
this.data.packed.set(records.subarray(0, count * this.words), from * this.words);
|
|
108
|
+
/*
|
|
109
|
+
* Reinterpreting, not converting — the same inverse the shader applies. Going through a
|
|
110
|
+
* `Float32Array` view of the destination rather than of the source, because the source may be
|
|
111
|
+
* a view over the file at any four-byte offset and the destination is this capture's own
|
|
112
|
+
* buffer at a known one.
|
|
113
|
+
*/
|
|
114
|
+
const asFloat = new Float32Array(this.data.packed.buffer);
|
|
115
|
+
for (let slot = 0; slot < count; slot++) {
|
|
116
|
+
const splat = from + slot;
|
|
117
|
+
const at = splat * this.words;
|
|
118
|
+
this.data.positions[splat * 3] = asFloat[at] ?? 0;
|
|
119
|
+
this.data.positions[splat * 3 + 1] = asFloat[at + 1] ?? 0;
|
|
120
|
+
this.data.positions[splat * 3 + 2] = asFloat[at + 2] ?? 0;
|
|
121
|
+
this.data.extents[splat] = asFloat[at + 7] ?? 0;
|
|
122
|
+
}
|
|
123
|
+
this.arrived += count;
|
|
124
|
+
return { from, count };
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/**
|
|
129
|
+
* A whole capture in one go, for a caller that read the file rather than streamed it.
|
|
130
|
+
*
|
|
131
|
+
* The same assembly `SplatCapture` does per block, without the object — `readDrft` hands back
|
|
132
|
+
* every block already joined, so there is nothing left to append to.
|
|
133
|
+
*/
|
|
134
|
+
export function splatsFromRecords(source: {
|
|
135
|
+
readonly count: number;
|
|
136
|
+
readonly records: Uint32Array;
|
|
137
|
+
readonly boundsMin: ArrayLike<number>;
|
|
138
|
+
readonly boundsMax: ArrayLike<number>;
|
|
139
|
+
readonly sphericalHarmonics?: number;
|
|
140
|
+
/** The block's own record width. See `SplatCaptureOptions.wordsPerSplat`. */
|
|
141
|
+
readonly wordsPerSplat?: number;
|
|
142
|
+
}): SplatData {
|
|
143
|
+
const capture = new SplatCapture({
|
|
144
|
+
total: source.count,
|
|
145
|
+
boundsMin: source.boundsMin,
|
|
146
|
+
boundsMax: source.boundsMax,
|
|
147
|
+
...(source.wordsPerSplat === undefined ? {} : { wordsPerSplat: source.wordsPerSplat }),
|
|
148
|
+
...(source.sphericalHarmonics === undefined
|
|
149
|
+
? {}
|
|
150
|
+
: { sphericalHarmonics: source.sphericalHarmonics }),
|
|
151
|
+
});
|
|
152
|
+
capture.append(source.records, source.count);
|
|
153
|
+
return capture.data;
|
|
154
|
+
}
|
package/src/splatCull.ts
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
/** Whether a capture's own bounding box reaches the frame at all, decided before anything sorts. */
|
|
2
|
+
|
|
3
|
+
import { multiplyMat4 } from './splatMatrix.ts';
|
|
4
|
+
|
|
5
|
+
/* Scratch at module scope, so a per-frame visibility test allocates nothing. Safe to share
|
|
6
|
+
between two batches because this is synchronous: no second call runs inside one. */
|
|
7
|
+
const viewModel = new Float32Array(16);
|
|
8
|
+
const clip = new Float32Array(16);
|
|
9
|
+
/** Six planes, four numbers each: a, b, c, d with `a x + b y + c z + d >= 0` inside. */
|
|
10
|
+
const planes = new Float32Array(24);
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Is any part of the box `[boundsMin, boundsMax]` inside the frustum?
|
|
14
|
+
*
|
|
15
|
+
* The box is in the **capture's own space**, which is why this takes the model matrix rather than
|
|
16
|
+
* a world-space box: `SplatData` computes its bounds while the packing loop is open, in whatever
|
|
17
|
+
* frame the capture was authored in, and moving the box every frame would be work to reach the
|
|
18
|
+
* same answer as moving the planes.
|
|
19
|
+
*
|
|
20
|
+
* **Answered before the sort, not after, and that is where the saving is.** Drawing a capture that
|
|
21
|
+
* is out of frame costs one draw call the rasteriser throws away; *sorting* it costs a linear pass
|
|
22
|
+
* over every splat it has, in a worker, for a picture nobody sees. Culling after the sort would
|
|
23
|
+
* save the cheap half.
|
|
24
|
+
*
|
|
25
|
+
* **Conservative: a box that touches the frustum is visible.** The test is whether some plane has
|
|
26
|
+
* the whole box behind it, which is exact for rejection and admits a few boxes near a corner that
|
|
27
|
+
* are outside every plane pairwise but inside none singly. What that costs is a sort for a capture
|
|
28
|
+
* just off the corner of the frame; what would make it wrong is the opposite error, which loses a
|
|
29
|
+
* picture — so the asymmetry is deliberate.
|
|
30
|
+
*
|
|
31
|
+
* `projection` is the caller's own, **before** the backend's clip correction: the planes below are
|
|
32
|
+
* the OpenGL convention that `mat4.perspective` produces, and a corrected matrix has moved z into
|
|
33
|
+
* [0, 1] and flipped y, so the near plane extracted from one would be wrong.
|
|
34
|
+
*/
|
|
35
|
+
export function splatBoundsVisible(
|
|
36
|
+
view: ArrayLike<number>,
|
|
37
|
+
projection: ArrayLike<number>,
|
|
38
|
+
model: ArrayLike<number>,
|
|
39
|
+
boundsMin: ArrayLike<number>,
|
|
40
|
+
boundsMax: ArrayLike<number>,
|
|
41
|
+
): boolean {
|
|
42
|
+
multiplyMat4(viewModel, view, model);
|
|
43
|
+
multiplyMat4(clip, projection, viewModel);
|
|
44
|
+
|
|
45
|
+
/*
|
|
46
|
+
* Gribb and Hartmann: the clip-space inequalities `-w <= x <= w` and so on are linear in the
|
|
47
|
+
* object-space point, so each is a plane whose coefficients are a sum or difference of two rows
|
|
48
|
+
* of the matrix. A row of a column-major matrix is every fourth entry, which is why the strides
|
|
49
|
+
* below look transposed.
|
|
50
|
+
*/
|
|
51
|
+
for (let axis = 0; axis < 3; axis++) {
|
|
52
|
+
for (let component = 0; component < 4; component++) {
|
|
53
|
+
const w = clip[component * 4 + 3] ?? 0;
|
|
54
|
+
const a = clip[component * 4 + axis] ?? 0;
|
|
55
|
+
/* Two planes per axis: w + a is the low side, w - a the high one. */
|
|
56
|
+
planes[axis * 8 + component] = w + a;
|
|
57
|
+
planes[axis * 8 + 4 + component] = w - a;
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
const minX = boundsMin[0] ?? 0;
|
|
62
|
+
const minY = boundsMin[1] ?? 0;
|
|
63
|
+
const minZ = boundsMin[2] ?? 0;
|
|
64
|
+
const maxX = boundsMax[0] ?? 0;
|
|
65
|
+
const maxY = boundsMax[1] ?? 0;
|
|
66
|
+
const maxZ = boundsMax[2] ?? 0;
|
|
67
|
+
|
|
68
|
+
for (let plane = 0; plane < 6; plane++) {
|
|
69
|
+
const at = plane * 4;
|
|
70
|
+
const a = planes[at] ?? 0;
|
|
71
|
+
const b = planes[at + 1] ?? 0;
|
|
72
|
+
const c = planes[at + 2] ?? 0;
|
|
73
|
+
const d = planes[at + 3] ?? 0;
|
|
74
|
+
/*
|
|
75
|
+
* The box's corner **furthest along this plane's normal**, picked component by component.
|
|
76
|
+
* Testing that one decides the whole box: if even the corner most in front of the plane is
|
|
77
|
+
* behind it, all eight are — six dot products rather than forty-eight.
|
|
78
|
+
*
|
|
79
|
+
* **The opposite corner is the trap, and it looks equally plausible.** Taking the corner
|
|
80
|
+
* furthest *behind* the plane rejects every box that is partly behind one, which throws away
|
|
81
|
+
* any capture straddling the near plane and any capture the camera is standing inside — the
|
|
82
|
+
* two cases a room-scale capture is made of. Both are asserted in `splatCull.test.ts` for
|
|
83
|
+
* that reason, and both failed on the first writing of this line.
|
|
84
|
+
*/
|
|
85
|
+
const x = a >= 0 ? maxX : minX;
|
|
86
|
+
const y = b >= 0 ? maxY : minY;
|
|
87
|
+
const z = c >= 0 ? maxZ : minZ;
|
|
88
|
+
if (a * x + b * y + c * z + d < 0) return false;
|
|
89
|
+
}
|
|
90
|
+
return true;
|
|
91
|
+
}
|
package/src/splatData.ts
ADDED
|
@@ -0,0 +1,398 @@
|
|
|
1
|
+
/** The shape every splat reader produces, and the packing that turns it into two GPU texels. */
|
|
2
|
+
|
|
3
|
+
import { packHalf2x16 } from './half.ts';
|
|
4
|
+
|
|
5
|
+
/** How many `uint32`s a splat occupies with no view-dependent colour: two `RGBA32UI` texels. */
|
|
6
|
+
export const SPLAT_WORDS = 8;
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* The same with degree-1 view-dependent colour: **one more texel**, and no more than one.
|
|
10
|
+
*
|
|
11
|
+
* The costing in `sphericalHarmonics` below is what picked degree 1 and it is worth reading before
|
|
12
|
+
* anybody raises this. Nine coefficients fit a texel as bytes with a per-splat scale beside them,
|
|
13
|
+
* which is +50% on the record and on the read; degree 2 is three more texels and degree 3 is six,
|
|
14
|
+
* and degree 3 alone would put 307 MB a frame through a device already carrying 389.
|
|
15
|
+
*/
|
|
16
|
+
export const SPLAT_WORDS_SH1 = 12;
|
|
17
|
+
|
|
18
|
+
/** Three basis functions at l=1, three channels each. */
|
|
19
|
+
export const SPLAT_SH1_COEFFICIENTS = 9;
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* A capture as a reader hands it over: linear values, one array per attribute.
|
|
23
|
+
*
|
|
24
|
+
* **Linear, and that boundary is load-bearing.** A `.ply` from a training run stores scale as its
|
|
25
|
+
* logarithm and opacity as its logit, and undoing both is the *reader's* job rather than this
|
|
26
|
+
* one's. Getting the boundary wrong produces a capture that is either invisible or a solid block,
|
|
27
|
+
* and both have been reported against other viewers — because both look like a rendering fault.
|
|
28
|
+
*/
|
|
29
|
+
export interface SplatSource {
|
|
30
|
+
readonly count: number;
|
|
31
|
+
/** Three per splat, in the capture's own space. */
|
|
32
|
+
readonly positions: Float32Array;
|
|
33
|
+
/** Three per splat: the Gaussian's standard deviation along each of its own axes, in metres. */
|
|
34
|
+
readonly scales: Float32Array;
|
|
35
|
+
/**
|
|
36
|
+
* Four per splat, **xyzw**, normalised here rather than trusted.
|
|
37
|
+
*
|
|
38
|
+
* **Both file formats store wxyz and this does not**, which is a deliberate seam: `xyzw` is what
|
|
39
|
+
* `gl-matrix` uses and what every other quaternion in this engine is, so a consumer building a
|
|
40
|
+
* source by hand is not asked to learn a second convention for one type. Each reader reorders on
|
|
41
|
+
* the way in, in one line, at the point where the file's order is already on screen.
|
|
42
|
+
*/
|
|
43
|
+
readonly rotations: Float32Array;
|
|
44
|
+
/**
|
|
45
|
+
* Three per splat, 0 to 1, **linear**.
|
|
46
|
+
*
|
|
47
|
+
* Not sRGB-decoded, which is what every reference viewer does and is also what is correct here:
|
|
48
|
+
* a capture's colour comes from a spherical-harmonic DC term, which is linear radiance, and this
|
|
49
|
+
* engine shades and composites in linear and grades once at the end. What would make it wrong is
|
|
50
|
+
* a capture authored by a tool that baked a display transform into the DC term — which produces
|
|
51
|
+
* a washed-out cloud, not a subtly wrong one.
|
|
52
|
+
*/
|
|
53
|
+
readonly colors: Float32Array;
|
|
54
|
+
/** One per splat, 0 to 1, already through the logistic if the format stored a logit. */
|
|
55
|
+
readonly opacities: Float32Array;
|
|
56
|
+
/**
|
|
57
|
+
* The l=1 spherical-harmonic band, nine per splat, or absent for a capture that has none.
|
|
58
|
+
*
|
|
59
|
+
* **Interleaved by basis function and then by channel** — `Y(1,−1).rgb`, `Y(1,0).rgb`,
|
|
60
|
+
* `Y(1,1).rgb` — which is *not* how a `.ply` stores them. That file is channel-major, all
|
|
61
|
+
* fifteen of red before all fifteen of green, and the reader transposes on the way in for the
|
|
62
|
+
* reason every other reordering in this package happens there: the file's own order is on
|
|
63
|
+
* screen at exactly one place, and a shader that had to know it would be a second copy of a
|
|
64
|
+
* convention nothing checks.
|
|
65
|
+
*
|
|
66
|
+
* Raw coefficients, signed, in the same units the training run wrote — the band constant is the
|
|
67
|
+
* shader's to apply, like `SH_C0` is the reader's for the DC term.
|
|
68
|
+
*/
|
|
69
|
+
readonly sh1?: Float32Array;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* A capture packed for the GPU, plus the two things the CPU still needs.
|
|
74
|
+
*
|
|
75
|
+
* **`positions` survives packing because the sorter reads it every time the view turns**, and it is
|
|
76
|
+
* the only per-splat attribute that does. Everything else is in `packed` and is never looked at
|
|
77
|
+
* again by JavaScript.
|
|
78
|
+
*
|
|
79
|
+
* **This takes ownership of the arrays handed to `packSplats`.** They are twelve and four bytes a
|
|
80
|
+
* splat, so copying a million-splat capture would be sixteen megabytes of duplicate to no purpose;
|
|
81
|
+
* the cost is that a caller must not go on mutating a source after packing it, which is stated
|
|
82
|
+
* here because nothing enforces it.
|
|
83
|
+
*/
|
|
84
|
+
export interface SplatData {
|
|
85
|
+
readonly count: number;
|
|
86
|
+
/** Three per splat. See above: the sorter's input, and the reason this is not packed away. */
|
|
87
|
+
readonly positions: Float32Array;
|
|
88
|
+
/**
|
|
89
|
+
* `wordsPerSplat` per splat — texel 0 is position and colour, texel 1 is the covariance, and
|
|
90
|
+
* texel 2 is the l=1 band where the capture carries one.
|
|
91
|
+
*/
|
|
92
|
+
readonly packed: Uint32Array;
|
|
93
|
+
/**
|
|
94
|
+
* `SPLAT_WORDS` or `SPLAT_WORDS_SH1`. Everything that indexes `packed` reads this rather than a
|
|
95
|
+
* constant, and `FORMAT.md` §4.7 anticipated exactly this: a `SPLT` block already carries its
|
|
96
|
+
* own `wordsPerSplat` and says "a capture that carries spherical-harmonic coefficients later
|
|
97
|
+
* fits by raising it and nothing else".
|
|
98
|
+
*/
|
|
99
|
+
readonly wordsPerSplat: number;
|
|
100
|
+
/** The capture's axis-aligned extent, for the frustum test that precedes a sort. */
|
|
101
|
+
readonly boundsMin: Float32Array;
|
|
102
|
+
readonly boundsMax: Float32Array;
|
|
103
|
+
/**
|
|
104
|
+
* One per splat: the largest of its three standard deviations, in the capture's own units.
|
|
105
|
+
*
|
|
106
|
+
* **What the budget ranks by, and the second attribute that survives packing.** A splat's
|
|
107
|
+
* contribution to the picture is roughly how many pixels it covers, which is its world extent
|
|
108
|
+
* over its distance to the camera; the distance is per-frame and this is the half that is not.
|
|
109
|
+
* The largest sigma bounds the projected radius from above for every orientation, so it ranks
|
|
110
|
+
* correctly without a per-splat projection.
|
|
111
|
+
*
|
|
112
|
+
* What it costs is **four bytes a splat** — four megabytes at a million, on top of the twelve
|
|
113
|
+
* `positions` already keeps — and it is paid whether or not a capture is ever budgeted, because
|
|
114
|
+
* packing cannot know. What would make it wrong is a capture of extremely flat splats seen
|
|
115
|
+
* edge-on, where the largest sigma is a poor estimate of the pixels covered and this
|
|
116
|
+
* over-ranks them; the honest fix there is the projected radius, which is per-frame work this
|
|
117
|
+
* deliberately avoids.
|
|
118
|
+
*/
|
|
119
|
+
readonly extents: Float32Array;
|
|
120
|
+
/**
|
|
121
|
+
* How many `f_rest_*` coefficients the source carried, of which this kept the first band.
|
|
122
|
+
*
|
|
123
|
+
* Zero for a format that has none. **Nine are read as of 2026-08-27 and the rest are not**, and
|
|
124
|
+
* the costing that decided which is below, kept because it is the argument against raising it.
|
|
125
|
+
*
|
|
126
|
+
* ## What view-dependent colour costs, costed 2026-08-27 and built the same day
|
|
127
|
+
*
|
|
128
|
+
* A splat with no harmonics is `SPLAT_WORDS` uint32s — two `RGBA32UI` texels, **32 bytes** — and
|
|
129
|
+
* the vertex stage runs **six times a splat**, because the quad is two unindexed triangles and
|
|
130
|
+
* `SPLAT_VERT` says why. Every invocation fetches every texel, so the data texture is read at
|
|
131
|
+
* **192 bytes a splat a frame** before any texture cache.
|
|
132
|
+
*
|
|
133
|
+
* A degree-`n` expansion carries `((n+1)^2 − 1) * 3` coefficients:
|
|
134
|
+
*
|
|
135
|
+
* | Degree | Coefficients | Extra texels | Bytes a splat | Read a splat a frame |
|
|
136
|
+
* |---|---|---|---|---|
|
|
137
|
+
* | 0 | 0 | 0 | 32 | 192 |
|
|
138
|
+
* | 1 **(built)** | 9 | 1 | 48 | 288 |
|
|
139
|
+
* | 2 | 24 | 3 | 80 | 480 |
|
|
140
|
+
* | 3 | 45 | 6 | 128 | 768 |
|
|
141
|
+
*
|
|
142
|
+
* **Against the mobile budget of `SPLAT_BUDGET_DEFAULT`, 400,000 splats**: 76.8 MB a frame at
|
|
143
|
+
* degree 0, 115.2 at degree 1, 192 at degree 2 and **307.2 at degree 3**. `docs/CAPABILITIES.md`
|
|
144
|
+
* records a mid-range phone already moving **388.7 MB a frame of attachment traffic**, so degree 3
|
|
145
|
+
* would very nearly double the memory traffic of a frame on the device that is already the
|
|
146
|
+
* constraint — and the texture itself would grow from 12.8 MB to 51 MB, which is the mistake
|
|
147
|
+
* `lightBudget.ts`'s shadow pool records making once already.
|
|
148
|
+
*
|
|
149
|
+
* **Degree 1 is the band that earns its texel.** The l=1 lobe is broad and directional — it is
|
|
150
|
+
* what makes glass, a wet surface and a polished floor read as themselves — where degrees 2 and 3
|
|
151
|
+
* are the sharp specular detail that costs four and six times as much. Nine coefficients fit one
|
|
152
|
+
* texel as **bytes with a per-splat scale beside them**, rather than halves: nine halves is 18
|
|
153
|
+
* bytes and would not fit, and a per-splat scale spends the sixteenth byte on the accuracy that
|
|
154
|
+
* quantising to a capture-wide range would have thrown away.
|
|
155
|
+
*
|
|
156
|
+
* **What is still unmeasured**, and it is the number that would refine this: how much of the 6x
|
|
157
|
+
* amplification a real texture cache absorbs, since six consecutive invocations read the same
|
|
158
|
+
* texels. Answering it needs a GPU timing of the splat pass against a capture, and this checkout
|
|
159
|
+
* has none — `demo/dev/public/` being uncommitted.
|
|
160
|
+
*/
|
|
161
|
+
readonly sphericalHarmonics: number;
|
|
162
|
+
/**
|
|
163
|
+
* How many bands of that expansion this capture actually carries: 0 or 1.
|
|
164
|
+
*
|
|
165
|
+
* Separate from `sphericalHarmonics`, which counts what the *source* had — a `.ply` written at
|
|
166
|
+
* degree 3 reports 45 there and 1 here, because forty-five coefficients arrived and nine were
|
|
167
|
+
* kept. The two disagreeing is the honest state rather than a discrepancy.
|
|
168
|
+
*/
|
|
169
|
+
readonly shDegree: 0 | 1;
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
/* Scratch for one splat's rotation matrix, so the packing loop allocates nothing. */
|
|
173
|
+
const rotation = new Float32Array(9);
|
|
174
|
+
|
|
175
|
+
/**
|
|
176
|
+
* Turn a source into two texels per splat, and compute the capture's bounds while the loop is open.
|
|
177
|
+
*
|
|
178
|
+
* Texel 0 is the three position floats reinterpreted as `uint32` plus the colour as `RGBA8`.
|
|
179
|
+
* Texel 1 is the six unique terms of the 3x3 covariance as three half pairs, with the fourth
|
|
180
|
+
* component reserved and written zero.
|
|
181
|
+
*
|
|
182
|
+
* **The covariance is computed here and never again.** It is fixed for the lifetime of a capture,
|
|
183
|
+
* so evaluating it per splat per frame would cost a quaternion-to-matrix and two 3x3 multiplies
|
|
184
|
+
* for a value that cannot change. What that gives up is six halves of precision against the
|
|
185
|
+
* source's floats; what would make it wrong is a capture whose splats are animated, which is not a
|
|
186
|
+
* thing this format can express.
|
|
187
|
+
*/
|
|
188
|
+
export function packSplats(source: SplatSource): SplatData {
|
|
189
|
+
const { count, positions, scales, rotations, colors, opacities, sh1 } = source;
|
|
190
|
+
expectLength('positions', positions, count * 3);
|
|
191
|
+
expectLength('scales', scales, count * 3);
|
|
192
|
+
expectLength('rotations', rotations, count * 4);
|
|
193
|
+
expectLength('colors', colors, count * 3);
|
|
194
|
+
expectLength('opacities', opacities, count);
|
|
195
|
+
if (sh1 !== undefined) expectLength('sh1', sh1, count * SPLAT_SH1_COEFFICIENTS);
|
|
196
|
+
|
|
197
|
+
const wordsPerSplat = sh1 === undefined ? SPLAT_WORDS : SPLAT_WORDS_SH1;
|
|
198
|
+
const packed = new Uint32Array(count * wordsPerSplat);
|
|
199
|
+
/* Reinterpreting rather than converting: the shader reads these back with the inverse. */
|
|
200
|
+
const asFloat = new Float32Array(packed.buffer);
|
|
201
|
+
|
|
202
|
+
const boundsMin = new Float32Array([Infinity, Infinity, Infinity]);
|
|
203
|
+
const boundsMax = new Float32Array([-Infinity, -Infinity, -Infinity]);
|
|
204
|
+
const extents = new Float32Array(count);
|
|
205
|
+
|
|
206
|
+
for (let index = 0; index < count; index++) {
|
|
207
|
+
const at = index * wordsPerSplat;
|
|
208
|
+
const p = index * 3;
|
|
209
|
+
const x = positions[p] ?? 0;
|
|
210
|
+
const y = positions[p + 1] ?? 0;
|
|
211
|
+
const z = positions[p + 2] ?? 0;
|
|
212
|
+
asFloat[at] = x;
|
|
213
|
+
asFloat[at + 1] = y;
|
|
214
|
+
asFloat[at + 2] = z;
|
|
215
|
+
if (x < (boundsMin[0] ?? 0)) boundsMin[0] = x;
|
|
216
|
+
if (y < (boundsMin[1] ?? 0)) boundsMin[1] = y;
|
|
217
|
+
if (z < (boundsMin[2] ?? 0)) boundsMin[2] = z;
|
|
218
|
+
if (x > (boundsMax[0] ?? 0)) boundsMax[0] = x;
|
|
219
|
+
if (y > (boundsMax[1] ?? 0)) boundsMax[1] = y;
|
|
220
|
+
if (z > (boundsMax[2] ?? 0)) boundsMax[2] = z;
|
|
221
|
+
|
|
222
|
+
/* `unpackUnorm4x8` reads the lowest byte as x, so red goes in the lowest byte. */
|
|
223
|
+
packed[at + 3] =
|
|
224
|
+
(byte(colors[p]) |
|
|
225
|
+
(byte(colors[p + 1]) << 8) |
|
|
226
|
+
(byte(colors[p + 2]) << 16) |
|
|
227
|
+
(byte(opacities[index]) << 24)) >>>
|
|
228
|
+
0;
|
|
229
|
+
|
|
230
|
+
quaternionToMatrix(rotations, index * 4, index);
|
|
231
|
+
|
|
232
|
+
/*
|
|
233
|
+
* Sigma = R diag(s^2) R^T, six unique terms. Written as a sum over the three columns rather
|
|
234
|
+
* than as two matrix products, which is the same arithmetic with nothing to allocate: column
|
|
235
|
+
* j of R scaled by s_j is column j of R*S, and Sigma[i][k] is the dot of rows i and k of that.
|
|
236
|
+
*/
|
|
237
|
+
const sx = scales[p] ?? 0;
|
|
238
|
+
const sy = scales[p + 1] ?? 0;
|
|
239
|
+
const sz = scales[p + 2] ?? 0;
|
|
240
|
+
extents[index] = Math.max(sx, sy, sz);
|
|
241
|
+
const w0 = sx * sx;
|
|
242
|
+
const w1 = sy * sy;
|
|
243
|
+
const w2 = sz * sz;
|
|
244
|
+
const r00 = rotation[0] ?? 0;
|
|
245
|
+
const r01 = rotation[1] ?? 0;
|
|
246
|
+
const r02 = rotation[2] ?? 0;
|
|
247
|
+
const r10 = rotation[3] ?? 0;
|
|
248
|
+
const r11 = rotation[4] ?? 0;
|
|
249
|
+
const r12 = rotation[5] ?? 0;
|
|
250
|
+
const r20 = rotation[6] ?? 0;
|
|
251
|
+
const r21 = rotation[7] ?? 0;
|
|
252
|
+
const r22 = rotation[8] ?? 0;
|
|
253
|
+
|
|
254
|
+
const xx = r00 * r00 * w0 + r01 * r01 * w1 + r02 * r02 * w2;
|
|
255
|
+
const xy = r00 * r10 * w0 + r01 * r11 * w1 + r02 * r12 * w2;
|
|
256
|
+
const xz = r00 * r20 * w0 + r01 * r21 * w1 + r02 * r22 * w2;
|
|
257
|
+
const yy = r10 * r10 * w0 + r11 * r11 * w1 + r12 * r12 * w2;
|
|
258
|
+
const yz = r10 * r20 * w0 + r11 * r21 * w1 + r12 * r22 * w2;
|
|
259
|
+
const zz = r20 * r20 * w0 + r21 * r21 * w1 + r22 * r22 * w2;
|
|
260
|
+
|
|
261
|
+
packed[at + 4] = packHalf2x16(xx, xy);
|
|
262
|
+
packed[at + 5] = packHalf2x16(xz, yy);
|
|
263
|
+
packed[at + 6] = packHalf2x16(yz, zz);
|
|
264
|
+
/*
|
|
265
|
+
* The extent, as float bits, in what texel one's fourth component would otherwise waste.
|
|
266
|
+
*
|
|
267
|
+
* **The GPU never reads it** — the vertex stage takes three of that texel's four components
|
|
268
|
+
* for the covariance — so this costs the draw nothing and it means a `.drft` splat block can
|
|
269
|
+
* be exactly the packed record. Carrying a parallel four bytes a splat instead would be
|
|
270
|
+
* twelve percent on the one payload this engine most wants small. `extents` still exists
|
|
271
|
+
* beside it because the sort reads it in a tight loop, where reinterpreting bits per access
|
|
272
|
+
* would not be free. What would make this wrong is a later shader wanting that component for
|
|
273
|
+
* something else, at which point the block grows a ninth word and says so in its header.
|
|
274
|
+
*/
|
|
275
|
+
asFloat[at + 7] = extents[index] ?? 0;
|
|
276
|
+
|
|
277
|
+
if (sh1 !== undefined) packSh1(sh1, index, packed, asFloat, at + 8);
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
/* A capture of nothing has no extent, and Infinity in a bound is a frustum test that never
|
|
281
|
+
answers. Collapse to the origin, which a zero-count batch never reaches anyway. */
|
|
282
|
+
if (count === 0) {
|
|
283
|
+
boundsMin.fill(0);
|
|
284
|
+
boundsMax.fill(0);
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
return {
|
|
288
|
+
count,
|
|
289
|
+
positions,
|
|
290
|
+
packed,
|
|
291
|
+
wordsPerSplat,
|
|
292
|
+
boundsMin,
|
|
293
|
+
boundsMax,
|
|
294
|
+
extents,
|
|
295
|
+
sphericalHarmonics: 0,
|
|
296
|
+
shDegree: sh1 === undefined ? 0 : 1,
|
|
297
|
+
};
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
/**
|
|
301
|
+
* One splat's l=1 band into one `RGBA32UI` texel: nine bytes and a scale.
|
|
302
|
+
*
|
|
303
|
+
* **Bytes with a per-splat scale, and both halves of that are the decision.** Nine halves would be
|
|
304
|
+
* eighteen bytes and would not fit a texel at all; nine bytes against a *capture-wide* range would
|
|
305
|
+
* fit and would spend most of the range on the few splats with the largest coefficients, leaving
|
|
306
|
+
* the ordinary ones quantised to a handful of levels. A per-splat scale costs the sixteenth byte —
|
|
307
|
+
* which is four bytes as a float, filling the texel exactly — and gives every splat the whole
|
|
308
|
+
* 8-bit range for its own coefficients.
|
|
309
|
+
*
|
|
310
|
+
* A splat whose band is all zero gets a scale of zero and nine mid-range bytes, which the shader
|
|
311
|
+
* multiplies back to exactly zero. That is the common case in a capture's flat regions and it
|
|
312
|
+
* costs nothing to say so.
|
|
313
|
+
*
|
|
314
|
+
* The three spare bytes of the third word are written zero rather than left as whatever the
|
|
315
|
+
* allocation held: a `.drft` block is compared byte for byte by `drft-diff`, and a record with
|
|
316
|
+
* uninitialised padding is a file that differs from itself.
|
|
317
|
+
*/
|
|
318
|
+
function packSh1(
|
|
319
|
+
sh1: Float32Array,
|
|
320
|
+
index: number,
|
|
321
|
+
packed: Uint32Array,
|
|
322
|
+
asFloat: Float32Array,
|
|
323
|
+
at: number,
|
|
324
|
+
): void {
|
|
325
|
+
const from = index * SPLAT_SH1_COEFFICIENTS;
|
|
326
|
+
let scale = 0;
|
|
327
|
+
for (let k = 0; k < SPLAT_SH1_COEFFICIENTS; k++) {
|
|
328
|
+
const magnitude = Math.abs(sh1[from + k] ?? 0);
|
|
329
|
+
if (magnitude > scale) scale = magnitude;
|
|
330
|
+
}
|
|
331
|
+
const inverse = scale > 0 ? 1 / scale : 0;
|
|
332
|
+
/* Centre of the range, which is exactly zero once the scale multiplies it back. */
|
|
333
|
+
const quantise = (value: number): number =>
|
|
334
|
+
Math.max(0, Math.min(255, Math.round((value ?? 0) * inverse * 127 + 128)));
|
|
335
|
+
|
|
336
|
+
packed[at] =
|
|
337
|
+
(quantise(sh1[from] ?? 0) |
|
|
338
|
+
(quantise(sh1[from + 1] ?? 0) << 8) |
|
|
339
|
+
(quantise(sh1[from + 2] ?? 0) << 16) |
|
|
340
|
+
(quantise(sh1[from + 3] ?? 0) << 24)) >>>
|
|
341
|
+
0;
|
|
342
|
+
packed[at + 1] =
|
|
343
|
+
(quantise(sh1[from + 4] ?? 0) |
|
|
344
|
+
(quantise(sh1[from + 5] ?? 0) << 8) |
|
|
345
|
+
(quantise(sh1[from + 6] ?? 0) << 16) |
|
|
346
|
+
(quantise(sh1[from + 7] ?? 0) << 24)) >>>
|
|
347
|
+
0;
|
|
348
|
+
packed[at + 2] = quantise(sh1[from + 8] ?? 0) >>> 0;
|
|
349
|
+
asFloat[at + 3] = scale;
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
/**
|
|
353
|
+
* Fill `rotation` from a quaternion at `offset`, normalising on the way.
|
|
354
|
+
*
|
|
355
|
+
* **A zero quaternion is a refusal naming the splat**, not a silent identity and not a `NaN` that
|
|
356
|
+
* reaches the GPU. A NaN covariance produces a splat whose ellipse fails every comparison, which
|
|
357
|
+
* on most drivers is an invisible splat and on some is one that covers the screen — and neither
|
|
358
|
+
* points at the file that caused it.
|
|
359
|
+
*/
|
|
360
|
+
function quaternionToMatrix(source: Float32Array, offset: number, index: number): void {
|
|
361
|
+
const qx = source[offset] ?? 0;
|
|
362
|
+
const qy = source[offset + 1] ?? 0;
|
|
363
|
+
const qz = source[offset + 2] ?? 0;
|
|
364
|
+
const qw = source[offset + 3] ?? 0;
|
|
365
|
+
const length = Math.hypot(qx, qy, qz, qw);
|
|
366
|
+
if (length === 0 || !Number.isFinite(length)) {
|
|
367
|
+
throw new Error(
|
|
368
|
+
`splat ${index} carries a rotation of length ${length}, which names no orientation. ` +
|
|
369
|
+
'A capture with a zero or non-finite quaternion is malformed at the source.',
|
|
370
|
+
);
|
|
371
|
+
}
|
|
372
|
+
const x = qx / length;
|
|
373
|
+
const y = qy / length;
|
|
374
|
+
const z = qz / length;
|
|
375
|
+
const w = qw / length;
|
|
376
|
+
|
|
377
|
+
rotation[0] = 1 - 2 * (y * y + z * z);
|
|
378
|
+
rotation[1] = 2 * (x * y - w * z);
|
|
379
|
+
rotation[2] = 2 * (x * z + w * y);
|
|
380
|
+
rotation[3] = 2 * (x * y + w * z);
|
|
381
|
+
rotation[4] = 1 - 2 * (x * x + z * z);
|
|
382
|
+
rotation[5] = 2 * (y * z - w * x);
|
|
383
|
+
rotation[6] = 2 * (x * z - w * y);
|
|
384
|
+
rotation[7] = 2 * (y * z + w * x);
|
|
385
|
+
rotation[8] = 1 - 2 * (x * x + y * y);
|
|
386
|
+
}
|
|
387
|
+
|
|
388
|
+
function byte(value: number | undefined): number {
|
|
389
|
+
const clamped = Math.max(0, Math.min(1, value ?? 0));
|
|
390
|
+
return Math.round(clamped * 255);
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
function expectLength(name: string, array: Float32Array, expected: number): void {
|
|
394
|
+
if (array.length === expected) return;
|
|
395
|
+
throw new Error(
|
|
396
|
+
`${name} has ${array.length} entries where a capture of this count needs ${expected}.`,
|
|
397
|
+
);
|
|
398
|
+
}
|