@driftengine/capture 4.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (199) hide show
  1. package/LICENSE +202 -0
  2. package/NOTICE +29 -0
  3. package/README.md +68 -0
  4. package/dist/browserFrames.d.ts +31 -0
  5. package/dist/browserFrames.js +105 -0
  6. package/dist/collide.d.ts +59 -0
  7. package/dist/collide.js +92 -0
  8. package/dist/decimate.d.ts +42 -0
  9. package/dist/decimate.js +439 -0
  10. package/dist/delight.d.ts +42 -0
  11. package/dist/delight.js +351 -0
  12. package/dist/delightScene.d.ts +37 -0
  13. package/dist/delightScene.js +183 -0
  14. package/dist/depth.d.ts +44 -0
  15. package/dist/depth.js +53 -0
  16. package/dist/features.d.ts +33 -0
  17. package/dist/features.js +260 -0
  18. package/dist/frames.d.ts +47 -0
  19. package/dist/frames.js +87 -0
  20. package/dist/fusion.d.ts +77 -0
  21. package/dist/fusion.js +152 -0
  22. package/dist/gaussians/adam.d.ts +57 -0
  23. package/dist/gaussians/adam.js +88 -0
  24. package/dist/gaussians/depth.d.ts +8 -0
  25. package/dist/gaussians/depth.js +73 -0
  26. package/dist/gaussians/gradients.d.ts +36 -0
  27. package/dist/gaussians/gradients.js +228 -0
  28. package/dist/gaussians/harmonics.d.ts +46 -0
  29. package/dist/gaussians/harmonics.js +99 -0
  30. package/dist/gaussians/optimise.d.ts +50 -0
  31. package/dist/gaussians/optimise.js +125 -0
  32. package/dist/gaussians/parameters.d.ts +35 -0
  33. package/dist/gaussians/parameters.js +165 -0
  34. package/dist/gaussians/project.d.ts +72 -0
  35. package/dist/gaussians/project.js +137 -0
  36. package/dist/gaussians/projectGradient.d.ts +15 -0
  37. package/dist/gaussians/projectGradient.js +213 -0
  38. package/dist/gaussians/rasterise.d.ts +13 -0
  39. package/dist/gaussians/rasterise.js +65 -0
  40. package/dist/gaussians/refine.d.ts +36 -0
  41. package/dist/gaussians/refine.js +136 -0
  42. package/dist/gaussians/similarity.d.ts +25 -0
  43. package/dist/gaussians/similarity.js +80 -0
  44. package/dist/index.d.ts +101 -0
  45. package/dist/index.js +96 -0
  46. package/dist/marching.d.ts +38 -0
  47. package/dist/marching.js +258 -0
  48. package/dist/masks.d.ts +59 -0
  49. package/dist/masks.js +165 -0
  50. package/dist/math/dense.d.ts +68 -0
  51. package/dist/math/dense.js +294 -0
  52. package/dist/math/schur.d.ts +23 -0
  53. package/dist/math/schur.js +140 -0
  54. package/dist/models/batchNorm.d.ts +17 -0
  55. package/dist/models/batchNorm.js +24 -0
  56. package/dist/models/clipTokenizer.d.ts +40 -0
  57. package/dist/models/clipTokenizer.js +122 -0
  58. package/dist/models/clipTower.d.ts +27 -0
  59. package/dist/models/clipTower.js +45 -0
  60. package/dist/models/depthAnything2.d.ts +38 -0
  61. package/dist/models/depthAnything2.js +163 -0
  62. package/dist/models/depthAnything3.d.ts +26 -0
  63. package/dist/models/depthAnything3.js +171 -0
  64. package/dist/models/dualDpt.d.ts +19 -0
  65. package/dist/models/dualDpt.js +138 -0
  66. package/dist/models/hiera.d.ts +49 -0
  67. package/dist/models/hiera.js +120 -0
  68. package/dist/models/hieraAttention.d.ts +16 -0
  69. package/dist/models/hieraAttention.js +68 -0
  70. package/dist/models/miniature.d.ts +36 -0
  71. package/dist/models/miniature.js +216 -0
  72. package/dist/models/mobileSam.d.ts +41 -0
  73. package/dist/models/mobileSam.js +88 -0
  74. package/dist/models/owlv2.d.ts +49 -0
  75. package/dist/models/owlv2.js +131 -0
  76. package/dist/models/owlv2Detect.d.ts +25 -0
  77. package/dist/models/owlv2Detect.js +80 -0
  78. package/dist/models/owlv2Miniature.d.ts +27 -0
  79. package/dist/models/owlv2Miniature.js +87 -0
  80. package/dist/models/rope.d.ts +23 -0
  81. package/dist/models/rope.js +84 -0
  82. package/dist/models/sam21.d.ts +45 -0
  83. package/dist/models/sam21.js +126 -0
  84. package/dist/models/sam21Miniature.d.ts +37 -0
  85. package/dist/models/sam21Miniature.js +194 -0
  86. package/dist/models/sam21Video.d.ts +30 -0
  87. package/dist/models/sam21Video.js +101 -0
  88. package/dist/models/sam2Choice.d.ts +19 -0
  89. package/dist/models/sam2Choice.js +55 -0
  90. package/dist/models/sam2Decoder.d.ts +36 -0
  91. package/dist/models/sam2Decoder.js +108 -0
  92. package/dist/models/sam2Memories.d.ts +54 -0
  93. package/dist/models/sam2Memories.js +93 -0
  94. package/dist/models/sam2Memory.d.ts +30 -0
  95. package/dist/models/sam2Memory.js +136 -0
  96. package/dist/models/sam2Positions.d.ts +24 -0
  97. package/dist/models/sam2Positions.js +94 -0
  98. package/dist/models/sam2Tracker.d.ts +69 -0
  99. package/dist/models/sam2Tracker.js +172 -0
  100. package/dist/models/samDecoder.d.ts +45 -0
  101. package/dist/models/samDecoder.js +96 -0
  102. package/dist/models/samMiniature.d.ts +43 -0
  103. package/dist/models/samMiniature.js +161 -0
  104. package/dist/models/samPrompt.d.ts +57 -0
  105. package/dist/models/samPrompt.js +160 -0
  106. package/dist/models/seeded.d.ts +16 -0
  107. package/dist/models/seeded.js +48 -0
  108. package/dist/models/tinyVit.d.ts +38 -0
  109. package/dist/models/tinyVit.js +82 -0
  110. package/dist/models/twoWay.d.ts +46 -0
  111. package/dist/models/twoWay.js +67 -0
  112. package/dist/models/vit.d.ts +45 -0
  113. package/dist/models/vit.js +157 -0
  114. package/dist/models/windowAttention.d.ts +25 -0
  115. package/dist/models/windowAttention.js +104 -0
  116. package/dist/poses.d.ts +31 -0
  117. package/dist/poses.js +750 -0
  118. package/dist/prepare.d.ts +30 -0
  119. package/dist/prepare.js +238 -0
  120. package/dist/propose.d.ts +36 -0
  121. package/dist/propose.js +63 -0
  122. package/dist/resample.d.ts +39 -0
  123. package/dist/resample.js +165 -0
  124. package/dist/run.d.ts +3 -0
  125. package/dist/run.js +1 -0
  126. package/dist/scene.d.ts +104 -0
  127. package/dist/scene.js +109 -0
  128. package/dist/segment.d.ts +40 -0
  129. package/dist/segment.js +184 -0
  130. package/dist/testScene.d.ts +55 -0
  131. package/dist/testScene.js +214 -0
  132. package/dist/twoView.d.ts +22 -0
  133. package/dist/twoView.js +435 -0
  134. package/package.json +63 -0
  135. package/src/browserFrames.ts +145 -0
  136. package/src/collide.ts +128 -0
  137. package/src/decimate.ts +510 -0
  138. package/src/delight.ts +427 -0
  139. package/src/delightScene.ts +249 -0
  140. package/src/depth.ts +100 -0
  141. package/src/features.ts +302 -0
  142. package/src/frames.ts +111 -0
  143. package/src/fusion.ts +229 -0
  144. package/src/gaussians/adam.ts +131 -0
  145. package/src/gaussians/depth.ts +87 -0
  146. package/src/gaussians/gradients.ts +285 -0
  147. package/src/gaussians/harmonics.ts +122 -0
  148. package/src/gaussians/optimise.ts +191 -0
  149. package/src/gaussians/parameters.ts +186 -0
  150. package/src/gaussians/project.ts +207 -0
  151. package/src/gaussians/projectGradient.ts +234 -0
  152. package/src/gaussians/rasterise.ts +81 -0
  153. package/src/gaussians/refine.ts +176 -0
  154. package/src/gaussians/similarity.ts +87 -0
  155. package/src/index.ts +225 -0
  156. package/src/marching.ts +358 -0
  157. package/src/masks.ts +201 -0
  158. package/src/math/dense.ts +317 -0
  159. package/src/math/schur.ts +167 -0
  160. package/src/models/batchNorm.ts +57 -0
  161. package/src/models/clipTokenizer.ts +129 -0
  162. package/src/models/clipTower.ts +87 -0
  163. package/src/models/depthAnything2.ts +276 -0
  164. package/src/models/depthAnything3.ts +229 -0
  165. package/src/models/dualDpt.ts +233 -0
  166. package/src/models/hiera.ts +231 -0
  167. package/src/models/hieraAttention.ts +121 -0
  168. package/src/models/miniature.ts +241 -0
  169. package/src/models/mobileSam.ts +135 -0
  170. package/src/models/owlv2.ts +190 -0
  171. package/src/models/owlv2Detect.ts +116 -0
  172. package/src/models/owlv2Miniature.ts +108 -0
  173. package/src/models/rope.ts +111 -0
  174. package/src/models/sam21.ts +182 -0
  175. package/src/models/sam21Miniature.ts +225 -0
  176. package/src/models/sam21Video.ts +154 -0
  177. package/src/models/sam2Choice.ts +62 -0
  178. package/src/models/sam2Decoder.ts +218 -0
  179. package/src/models/sam2Memories.ts +165 -0
  180. package/src/models/sam2Memory.ts +247 -0
  181. package/src/models/sam2Positions.ts +106 -0
  182. package/src/models/sam2Tracker.ts +300 -0
  183. package/src/models/samDecoder.ts +225 -0
  184. package/src/models/samMiniature.ts +190 -0
  185. package/src/models/samPrompt.ts +227 -0
  186. package/src/models/seeded.ts +50 -0
  187. package/src/models/tinyVit.ts +178 -0
  188. package/src/models/twoWay.ts +122 -0
  189. package/src/models/vit.ts +265 -0
  190. package/src/models/windowAttention.ts +155 -0
  191. package/src/poses.ts +894 -0
  192. package/src/prepare.ts +306 -0
  193. package/src/propose.ts +96 -0
  194. package/src/resample.ts +197 -0
  195. package/src/run.ts +7 -0
  196. package/src/scene.ts +141 -0
  197. package/src/segment.ts +220 -0
  198. package/src/testScene.ts +283 -0
  199. package/src/twoView.ts +536 -0
@@ -0,0 +1,285 @@
1
+ /**
2
+ * What a Gaussian would have to change to make the picture right: the derivatives of the rendered
3
+ * frame with respect to every parameter of every Gaussian.
4
+ *
5
+ * **Analytic, and checked against a central difference.** Every term below is a chain rule through
6
+ * the same projection `rasterise.ts` draws with — the view transform, the perspective Jacobian, the
7
+ * screen covariance, its inverse, the exponent, the alpha, and the compositing — and
8
+ * `gradients.test.ts` compares each against the rasteriser it differentiates, nudged. A gradient
9
+ * that is subtly wrong does not fail: it trains to a slightly wrong cloud, which is why the
10
+ * comparison is the test rather than a measure of how well it fits.
11
+ *
12
+ * **The compositing is walked twice.** Forward, nearest first, to know what each Gaussian
13
+ * contributed and what light was left after it; then backward, furthest first, carrying the colour
14
+ * accumulated behind — because what a Gaussian's alpha is worth depends on everything in front of
15
+ * it and everything behind it at once.
16
+ */
17
+ import { exactExp } from '@driftengine/core';
18
+
19
+ import {
20
+ cameraCentre,
21
+ sh1Basis,
22
+ sh1DirectionGradient,
23
+ splatColour,
24
+ SH1_COEFFICIENTS,
25
+ viewDirection,
26
+ } from './harmonics.ts';
27
+ import {
28
+ visibleGaussians,
29
+ type GaussianSet,
30
+ type Projected,
31
+ type RasterCamera,
32
+ } from './project.ts';
33
+ import { screenToParameters } from './projectGradient.ts';
34
+ import type { Frame } from './rasterise.ts';
35
+
36
+ /** Where the gradients go: one entry per parameter of every Gaussian. */
37
+ export interface GaussianGradients {
38
+ /** Three per Gaussian. */
39
+ readonly positions: Float64Array;
40
+ readonly scales: Float64Array;
41
+ /** Four per Gaussian, xyzw. */
42
+ readonly rotations: Float64Array;
43
+ readonly colors: Float64Array;
44
+ /** Nine per Gaussian, and left at zero for a cloud with no band to fit. */
45
+ readonly sh1: Float64Array;
46
+ /**
47
+ * The gradients **as the screen sees them**, nine per Gaussian: the two of its centre in pixels,
48
+ * the three of its screen covariance, its opacity, and its resolved colour.
49
+ *
50
+ * Everything else here is a chain rule away from these. They are kept because the tiled device
51
+ * rasteriser computes exactly this much and nothing further — the projection and the band are
52
+ * per-splat work that is not worth a dispatch — so this is the surface
53
+ * `scripts/gpu-parity.mjs` holds the two rasterisers to.
54
+ */
55
+ readonly screen: Float64Array;
56
+ /** One per Gaussian. */
57
+ readonly opacities: Float64Array;
58
+ }
59
+
60
+ export function createGradients(count: number): GaussianGradients {
61
+ return {
62
+ positions: new Float64Array(count * 3),
63
+ scales: new Float64Array(count * 3),
64
+ rotations: new Float64Array(count * 4),
65
+ colors: new Float64Array(count * 3),
66
+ sh1: new Float64Array(count * SH1_COEFFICIENTS),
67
+ screen: new Float64Array(count * SCREEN_GRADIENTS),
68
+ opacities: new Float64Array(count),
69
+ };
70
+ }
71
+
72
+ const FAINT = 1 / 255 / 4;
73
+
74
+ /** Numbers a Gaussian's screen-space gradient occupies: mean, covariance, opacity, colour. */
75
+ export const SCREEN_GRADIENTS = 9;
76
+
77
+ /** The squared difference between a render and a frame, and the render's own gradient. */
78
+ export function imageLoss(rendered: Frame, target: Frame, gradient: Float64Array): number {
79
+ let sum = 0;
80
+ for (let at = 0; at < gradient.length; at += 1) {
81
+ const off = (rendered[at] as number) - (target[at] as number);
82
+ sum += off * off;
83
+ gradient[at] = 2 * off;
84
+ }
85
+ return sum;
86
+ }
87
+
88
+ /**
89
+ * The gradients of a render against `dPixels` — the loss's derivative by each rendered channel,
90
+ * four a pixel — added into `out`. The render is repeated here rather than taken as an argument,
91
+ * because the backward walk needs what each Gaussian contributed on the way.
92
+ */
93
+ export function accumulateGradients(
94
+ set: GaussianSet,
95
+ camera: RasterCamera,
96
+ dPixels: Float64Array,
97
+ out: GaussianGradients,
98
+ ): void {
99
+ const { width, height } = camera;
100
+ const { order, projected } = visibleGaussians(set, camera);
101
+ const pixels = width * height;
102
+
103
+ /* Forward: the light left at each pixel after every Gaussian in front of it. */
104
+ const transmittance = new Float64Array(pixels).fill(1);
105
+ for (const at of order) {
106
+ const one = projected[at] as Projected;
107
+ const determinant = one.a * one.d - one.b * one.b;
108
+ const inverse = [one.d / determinant, -one.b / determinant, one.a / determinant];
109
+ const box = boxOf(one, width, height);
110
+ for (let y = box.top; y <= box.bottom; y += 1) {
111
+ for (let x = box.left; x <= box.right; x += 1) {
112
+ const dx = x + 0.5 - one.x;
113
+ const dy = y + 0.5 - one.y;
114
+ const power =
115
+ -0.5 *
116
+ ((inverse[0] as number) * dx * dx +
117
+ 2 * (inverse[1] as number) * dx * dy +
118
+ (inverse[2] as number) * dy * dy);
119
+ const alpha = (set.opacities[at] as number) * exactExp(power);
120
+ if (alpha < FAINT) continue;
121
+ const pixel = y * width + x;
122
+ transmittance[pixel] = (transmittance[pixel] as number) * (1 - alpha);
123
+ }
124
+ }
125
+ }
126
+
127
+ /* Backward: furthest first, carrying what stands behind each Gaussian. */
128
+ const behind = new Float64Array(pixels * 3);
129
+ const behindAlpha = new Float64Array(pixels);
130
+ const light = Float64Array.from(transmittance);
131
+ const eye = new Float64Array(3);
132
+ cameraCentre(camera.worldToCamera, eye);
133
+ const direction = new Float64Array(3);
134
+ const basis = new Float64Array(3);
135
+ const colour = new Float64Array(3);
136
+ const dColour = new Float64Array(3);
137
+ const dDirection = new Float64Array(3);
138
+ for (let index = order.length - 1; index >= 0; index -= 1) {
139
+ const at = order[index] as number;
140
+ const one = projected[at] as Projected;
141
+ const determinant = one.a * one.d - one.b * one.b;
142
+ const inverse = [one.d / determinant, -one.b / determinant, one.a / determinant];
143
+ const box = boxOf(one, width, height);
144
+ const opacity = set.opacities[at] as number;
145
+ const distance = viewDirection(set.positions, at, eye, direction);
146
+ sh1Basis(direction, basis);
147
+ splatColour(set.colors, set.sh1, at, basis, colour);
148
+ dColour.fill(0);
149
+ /* The screen-space gradients this Gaussian gathers over its own pixels. */
150
+ let dMeanX = 0;
151
+ let dMeanY = 0;
152
+ let dA = 0;
153
+ let dB = 0;
154
+ let dD = 0;
155
+ let dOpacity = 0;
156
+ for (let y = box.top; y <= box.bottom; y += 1) {
157
+ for (let x = box.left; x <= box.right; x += 1) {
158
+ const dx = x + 0.5 - one.x;
159
+ const dy = y + 0.5 - one.y;
160
+ const power =
161
+ -0.5 *
162
+ ((inverse[0] as number) * dx * dx +
163
+ 2 * (inverse[1] as number) * dx * dy +
164
+ (inverse[2] as number) * dy * dy);
165
+ const weight = exactExp(power);
166
+ const alpha = opacity * weight;
167
+ if (alpha < FAINT) continue;
168
+ const pixel = y * width + x;
169
+ /* Undo this Gaussian's own share to recover the light that reached it. */
170
+ const after = light[pixel] as number;
171
+ const before = alpha === 1 ? after : after / (1 - alpha);
172
+ let dAlpha = 0;
173
+ for (let c = 0; c < 3; c += 1) {
174
+ const grad = dPixels[pixel * 4 + c] as number;
175
+ /* What this Gaussian puts on the pixel, and what its alpha takes from behind it. */
176
+ dColour[c] = (dColour[c] as number) + grad * alpha * before;
177
+ dAlpha += grad * before * ((colour[c] as number) - (behind[pixel * 3 + c] as number));
178
+ }
179
+ const gradAlpha = dPixels[pixel * 4 + 3] as number;
180
+ dAlpha += gradAlpha * before * (1 - (behindAlpha[pixel] as number));
181
+ dOpacity += dAlpha * weight;
182
+ /* Through the exponent to the mean and the covariance. */
183
+ const dPower = dAlpha * opacity * weight;
184
+ const gx = (inverse[0] as number) * dx + (inverse[1] as number) * dy;
185
+ const gy = (inverse[1] as number) * dx + (inverse[2] as number) * dy;
186
+ dMeanX += dPower * gx;
187
+ dMeanY += dPower * gy;
188
+ /*
189
+ * The exponent by the inverse covariance, **entry by entry rather than by three numbers**:
190
+ * the off-diagonal is one of a symmetric pair, so it takes half of what a single shared
191
+ * parameter would, and `dB` below is what puts the two halves back together. Written the
192
+ * other way it is a factor of two on every cross term, which is a wrong gradient that
193
+ * still points roughly the right way — so it fits, slightly wrongly, rather than failing.
194
+ */
195
+ const dInverseA = -0.5 * dPower * dx * dx;
196
+ const dInverseB = -0.5 * dPower * dx * dy;
197
+ const dInverseD = -0.5 * dPower * dy * dy;
198
+ /* d(Σ⁻¹)/dΣ for a symmetric 2 × 2, written out. */
199
+ const ia = inverse[0] as number;
200
+ const ib = inverse[1] as number;
201
+ const id = inverse[2] as number;
202
+ dA += -(
203
+ ia * dInverseA * ia +
204
+ ib * dInverseB * ia +
205
+ ia * dInverseB * ib +
206
+ ib * dInverseD * ib
207
+ );
208
+ dB +=
209
+ -(ia * dInverseA * ib + ib * dInverseB * ib + ia * dInverseB * id + ib * dInverseD * id) *
210
+ 2;
211
+ dD += -(
212
+ ib * dInverseA * ib +
213
+ id * dInverseB * ib +
214
+ ib * dInverseB * id +
215
+ id * dInverseD * id
216
+ );
217
+ /* And carry this Gaussian into what stands behind the next one. */
218
+ for (let c = 0; c < 3; c += 1) {
219
+ behind[pixel * 3 + c] =
220
+ alpha * (colour[c] as number) + (1 - alpha) * (behind[pixel * 3 + c] as number);
221
+ }
222
+ behindAlpha[pixel] = alpha + (1 - alpha) * (behindAlpha[pixel] as number);
223
+ light[pixel] = before;
224
+ }
225
+ }
226
+ out.opacities[at] = (out.opacities[at] as number) + dOpacity;
227
+ /* What a device's tiled pass answers, before any of the chain rules below it. */
228
+ const screen = at * SCREEN_GRADIENTS;
229
+ out.screen[screen] = (out.screen[screen] as number) + dMeanX;
230
+ out.screen[screen + 1] = (out.screen[screen + 1] as number) + dMeanY;
231
+ out.screen[screen + 2] = (out.screen[screen + 2] as number) + dA;
232
+ out.screen[screen + 3] = (out.screen[screen + 3] as number) + dB;
233
+ out.screen[screen + 4] = (out.screen[screen + 4] as number) + dD;
234
+ out.screen[screen + 5] = (out.screen[screen + 5] as number) + dOpacity;
235
+ for (let c = 0; c < 3; c += 1) {
236
+ out.screen[screen + 6 + c] = (out.screen[screen + 6 + c] as number) + (dColour[c] as number);
237
+ }
238
+ /*
239
+ * The colour this Gaussian showed, back to the terms it was made of. **A channel the shader
240
+ * clamped at zero took no gradient**: it is flat there, so a constant term already below zero
241
+ * is not pushed further down by a frame that wanted it darker still.
242
+ */
243
+ for (let c = 0; c < 3; c += 1) {
244
+ if (!((colour[c] as number) > 0)) continue;
245
+ out.colors[at * 3 + c] = (out.colors[at * 3 + c] as number) + (dColour[c] as number);
246
+ if (set.sh1 === undefined) continue;
247
+ for (let band = 0; band < 3; band += 1) {
248
+ const slot = at * SH1_COEFFICIENTS + band * 3 + c;
249
+ out.sh1[slot] =
250
+ (out.sh1[slot] as number) + (dColour[c] as number) * (basis[band] as number);
251
+ }
252
+ }
253
+ /*
254
+ * And the band is a **second way a position changes the picture**: moving a Gaussian turns the
255
+ * direction it is seen from, which reshades it. Small next to the term that moves it across the
256
+ * frame, and the difference finds it all the same.
257
+ */
258
+ if (set.sh1 !== undefined && distance > 0) {
259
+ sh1DirectionGradient(set.sh1, at, colour, dColour, dDirection);
260
+ let along = 0;
261
+ for (let axis = 0; axis < 3; axis += 1)
262
+ along += (dDirection[axis] as number) * (direction[axis] as number);
263
+ for (let axis = 0; axis < 3; axis += 1) {
264
+ out.positions[at * 3 + axis] =
265
+ (out.positions[at * 3 + axis] as number) +
266
+ ((dDirection[axis] as number) - along * (direction[axis] as number)) / distance;
267
+ }
268
+ }
269
+ /* Through the projection to the Gaussian's own parameters. */
270
+ screenToParameters(set, at, camera, dMeanX, dMeanY, dA, dB, dD, out);
271
+ }
272
+ }
273
+
274
+ function boxOf(
275
+ one: Projected,
276
+ width: number,
277
+ height: number,
278
+ ): { left: number; right: number; top: number; bottom: number } {
279
+ return {
280
+ left: Math.max(0, Math.floor(one.x - one.radius)),
281
+ right: Math.min(width - 1, Math.ceil(one.x + one.radius)),
282
+ top: Math.max(0, Math.floor(one.y - one.radius)),
283
+ bottom: Math.min(height - 1, Math.ceil(one.y + one.radius)),
284
+ };
285
+ }
@@ -0,0 +1,122 @@
1
+ /**
2
+ * The colour a Gaussian shows from a direction: its constant term plus the degree-1 band.
3
+ *
4
+ * **This is the splat shader's line, in JavaScript.** A capture fitted here is drawn by
5
+ * `@driftengine/splats`, so the two evaluations have to agree down to the sign of each basis —
6
+ * `SH_C1 * (-direction.y * c0 + direction.z * c1 - direction.x * c2)`, clamped at zero, shaded by
7
+ * the direction from the camera to the Gaussian in the capture's own world. A band evaluated with
8
+ * a different order here fits coefficients that look right in the fitter and wrong in the game,
9
+ * where it reads as a shader fault rather than as a capture that was never right.
10
+ *
11
+ * **Degree 1 and no further**, because that is what a record carries: nine coefficients and a
12
+ * scale in one more texel, and `SPLAT_WORDS_SH1` is what decided it.
13
+ */
14
+
15
+ /** The l=1 basis constant, as the shader's `SH_C1`. */
16
+ export const SH_C1 = 0.4886025119029199;
17
+
18
+ /** Nine coefficients a Gaussian, interleaved by basis and then by channel. */
19
+ export const SH1_COEFFICIENTS = 9;
20
+
21
+ /** The three degree-1 basis values for a direction, the band constant folded in, into `out`. */
22
+ export function sh1Basis(direction: ArrayLike<number>, out: Float64Array): void {
23
+ out[0] = -SH_C1 * (direction[1] as number);
24
+ out[1] = SH_C1 * (direction[2] as number);
25
+ out[2] = -SH_C1 * (direction[0] as number);
26
+ }
27
+
28
+ /** Where a camera stands in the world, from a 3 × 4 world-to-camera: −Rᵀ · t. */
29
+ export function cameraCentre(worldToCamera: ArrayLike<number>, out: Float64Array): void {
30
+ out[0] = 0;
31
+ out[1] = 0;
32
+ out[2] = 0;
33
+ for (let r = 0; r < 3; r += 1) {
34
+ for (let c = 0; c < 3; c += 1) {
35
+ out[c] =
36
+ (out[c] as number) -
37
+ (worldToCamera[r * 4 + c] as number) * (worldToCamera[r * 4 + 3] as number);
38
+ }
39
+ }
40
+ }
41
+
42
+ /**
43
+ * The direction the band is shaded by: from `centre` to the Gaussian at `at`, normalised.
44
+ *
45
+ * A Gaussian standing exactly where the camera does has no direction to give. It answers with the
46
+ * camera's own forward axis rather than a division by zero — which costs nothing, because a
47
+ * Gaussian at the camera is behind the near plane and is not drawn.
48
+ *
49
+ * Answers the distance as well, because the gradient below divides by it: a direction is a
50
+ * normalisation, and how much a step moves it depends on how far away the thing is.
51
+ */
52
+ export function viewDirection(
53
+ positions: ArrayLike<number>,
54
+ at: number,
55
+ centre: ArrayLike<number>,
56
+ out: Float64Array,
57
+ ): number {
58
+ const x = (positions[at * 3] as number) - (centre[0] as number);
59
+ const y = (positions[at * 3 + 1] as number) - (centre[1] as number);
60
+ const z = (positions[at * 3 + 2] as number) - (centre[2] as number);
61
+ const length = Math.sqrt(x * x + y * y + z * z);
62
+ if (!(length > 0)) {
63
+ out[0] = 0;
64
+ out[1] = 0;
65
+ out[2] = 1;
66
+ return 0;
67
+ }
68
+ out[0] = x / length;
69
+ out[1] = y / length;
70
+ out[2] = z / length;
71
+ return length;
72
+ }
73
+
74
+ /** A Gaussian's colour from a direction whose basis is already resolved, clamped at zero. */
75
+ export function splatColour(
76
+ colors: ArrayLike<number>,
77
+ sh1: ArrayLike<number> | undefined,
78
+ at: number,
79
+ basis: ArrayLike<number>,
80
+ out: Float64Array,
81
+ ): void {
82
+ for (let c = 0; c < 3; c += 1) {
83
+ let value = colors[at * 3 + c] as number;
84
+ if (sh1 !== undefined) {
85
+ for (let band = 0; band < 3; band += 1) {
86
+ value += (basis[band] as number) * (sh1[at * SH1_COEFFICIENTS + band * 3 + c] as number);
87
+ }
88
+ }
89
+ out[c] = value > 0 ? value : 0;
90
+ }
91
+ }
92
+
93
+ /**
94
+ * The band's own gradient by the **direction** it was shaded from, into `out`.
95
+ *
96
+ * The colour is `−C·u_y·c₀ + C·u_z·c₁ − C·u_x·c₂` a channel, so each axis of the direction gathers
97
+ * one basis's coefficients. A channel the clamp flattened contributes nothing, exactly as it
98
+ * contributes nothing to the constant term.
99
+ *
100
+ * What the caller still owes is the step from a direction to a position: a direction is a
101
+ * normalisation, so only the part of a move across the line of sight turns it, and the amount it
102
+ * turns by falls off with distance.
103
+ */
104
+ export function sh1DirectionGradient(
105
+ sh1: ArrayLike<number>,
106
+ at: number,
107
+ colour: ArrayLike<number>,
108
+ dColour: ArrayLike<number>,
109
+ out: Float64Array,
110
+ ): void {
111
+ out[0] = 0;
112
+ out[1] = 0;
113
+ out[2] = 0;
114
+ for (let c = 0; c < 3; c += 1) {
115
+ if (!((colour[c] as number) > 0)) continue;
116
+ const weight = (dColour[c] as number) * SH_C1;
117
+ const base = at * SH1_COEFFICIENTS;
118
+ out[0] = (out[0] as number) - weight * (sh1[base + 6 + c] as number);
119
+ out[1] = (out[1] as number) - weight * (sh1[base + c] as number);
120
+ out[2] = (out[2] as number) + weight * (sh1[base + 3 + c] as number);
121
+ }
122
+ }
@@ -0,0 +1,191 @@
1
+ /**
2
+ * A cloud of Gaussians fitted to a clip's frames, from a camera path and a sparse start.
3
+ *
4
+ * **The engine already knows how to draw the answer.** What comes out is `@driftengine/splats`'
5
+ * `SplatSource` — the same shape its `.ply` and `.sog` readers produce — so a fitted capture goes
6
+ * through `packSplats` and into a `.drft` beside anything else, and is drawn by the shader this
7
+ * package's rasteriser was written to match.
8
+ *
9
+ * **Two parameters are fitted through a transform, and it is what keeps the fit standing.** A
10
+ * scale is optimised as its logarithm and an opacity as its logit, so neither can be stepped past
11
+ * the end of its own range: a negative standard deviation is a covariance that is not a covariance,
12
+ * and an opacity above one is light arriving from a Gaussian that was never there. The gradients
13
+ * this receives are with respect to the real quantities, and the chain rule back to the transformed
14
+ * ones is one multiplication each.
15
+ *
16
+ * **The frames are linear and carry their own coverage.** A synthetic frame comes out of the
17
+ * rasteriser with both; a photograph has to be decoded from its display transform and given an
18
+ * alpha of one by whoever read it, because a photograph *is* fully covered and saying so is the
19
+ * caller's statement about their clip rather than this module's guess. A masked capture says
20
+ * something narrower with the same field, and the fit needs no further telling.
21
+ */
22
+ import type { SplatSource } from '@driftengine/splats';
23
+
24
+ import { adamStep, createAdamState } from './adam.ts';
25
+ import { accumulateGradients, createGradients, imageLoss } from './gradients.ts';
26
+ import {
27
+ chain,
28
+ clear,
29
+ createCloud,
30
+ harvest,
31
+ materialise,
32
+ start,
33
+ viewOf,
34
+ SPLIT_FRACTION,
35
+ } from './parameters.ts';
36
+ import type { RasterCamera } from './project.ts';
37
+ import { rasteriseGaussians, type Frame } from './rasterise.ts';
38
+ import { refineCloud } from './refine.ts';
39
+
40
+ export interface OptimiseOptions {
41
+ /** `fx`, `fy`, `cx`, `cy`, shared by every frame, as the path was solved with. */
42
+ readonly intrinsics: readonly [number, number, number, number];
43
+ readonly width: number;
44
+ readonly height: number;
45
+ /** Three per point: where the fit starts, usually the path's own sparse reconstruction. */
46
+ readonly seeds: Float64Array;
47
+ /** The most Gaussians the answer may carry. */
48
+ readonly budget: number;
49
+ readonly iterations: number;
50
+ /** The engine's seeded generator: the same one twice gives the same capture. */
51
+ readonly random: () => number;
52
+ /**
53
+ * The standard deviation every Gaussian starts at, in metres.
54
+ *
55
+ * Left out, it is the distance from each seed to its nearest neighbour — the spacing of the
56
+ * cloud, which is the size at which the seeds tile what they were sampled from. **A cloud of one
57
+ * seed has no spacing**, and is refused rather than given a number out of the air.
58
+ */
59
+ readonly initialScale?: number;
60
+ /** Fit the degree-1 band as well as the constant term. */
61
+ readonly viewDependent?: boolean;
62
+ /** How often the cloud is grown and pruned. */
63
+ readonly refineEvery?: number;
64
+ }
65
+
66
+ /** How often the cloud is grown and pruned, where the caller does not say. */
67
+ const DEFAULT_REFINE_EVERY = 50;
68
+
69
+ /** A clip fitted to Gaussians. Nothing here is the caller's array; everything is freshly cut. */
70
+ export function optimiseGaussians(
71
+ targets: readonly Frame[],
72
+ poses: readonly Float64Array[],
73
+ options: OptimiseOptions,
74
+ ): SplatSource {
75
+ const { intrinsics, width, height, seeds, budget, iterations, random } = options;
76
+ if (targets.length === 0 || targets.length !== poses.length) {
77
+ throw new RangeError(
78
+ `capture: ${targets.length} frames against ${poses.length} poses, and a fit needs one of each`,
79
+ );
80
+ }
81
+ const seeded = Math.floor(seeds.length / 3);
82
+ if (seeded === 0) throw new RangeError('capture: a fit needs at least one seed to start from');
83
+ if (seeded > budget) {
84
+ throw new RangeError(`capture: ${seeded} seeds do not fit a budget of ${budget} Gaussians`);
85
+ }
86
+ const cameras: RasterCamera[] = poses.map((worldToCamera) => ({
87
+ width,
88
+ height,
89
+ intrinsics,
90
+ worldToCamera,
91
+ }));
92
+ const extent = sceneExtent(seeds, seeded, cameras);
93
+ const scale = options.initialScale ?? seedSpacing(seeds, seeded);
94
+
95
+ const cloud = createCloud(budget, extent, options.viewDependent === true);
96
+ start(cloud, seeds, seeded, scale);
97
+
98
+ const pixels = width * height * 4;
99
+ const rendered = new Float64Array(pixels);
100
+ const dPixels = new Float64Array(pixels);
101
+ const gradients = createGradients(budget);
102
+ const adam = createAdamState();
103
+ const refineEvery = options.refineEvery ?? DEFAULT_REFINE_EVERY;
104
+ const set = viewOf(cloud, budget);
105
+
106
+ for (let iteration = 0; iteration < iterations; iteration += 1) {
107
+ materialise(cloud);
108
+ set.count = cloud.count;
109
+ const view = iteration % targets.length;
110
+ rasteriseGaussians(set, cameras[view] as RasterCamera, rendered);
111
+ imageLoss(rendered, targets[view] as Frame, dPixels);
112
+ clear(gradients, cloud.count);
113
+ accumulateGradients(set, cameras[view] as RasterCamera, dPixels, gradients);
114
+ chain(cloud, gradients);
115
+ adamStep(cloud.families, cloud.count, adam);
116
+ /*
117
+ * Grown and pruned between steps, never on the last one: a refinement leaves fresh Gaussians
118
+ * that have taken no step, and a cloud handed back in that state is one the caller is drawing
119
+ * before the fit has looked at it.
120
+ */
121
+ if (iteration + 1 < iterations && (iteration + 1) % refineEvery === 0) {
122
+ materialise(cloud);
123
+ refineCloud(cloud, { budget, splitAbove: SPLIT_FRACTION * extent, random });
124
+ }
125
+ }
126
+ materialise(cloud);
127
+ return harvest(cloud);
128
+ }
129
+
130
+ /**
131
+ * How big the capture is: the radius of the sphere holding the seeds and the cameras.
132
+ *
133
+ * **The cameras are in it deliberately.** A position's step is in metres, so it has to be a
134
+ * fraction of something real, and a seed cloud can be a single point — which has no size of its own
135
+ * and is still a scene, because the cameras looking at it are a distance away.
136
+ */
137
+ function sceneExtent(seeds: Float64Array, count: number, cameras: readonly RasterCamera[]): number {
138
+ const points: number[][] = [];
139
+ for (let at = 0; at < count; at += 1) {
140
+ points.push([
141
+ seeds[at * 3] as number,
142
+ seeds[at * 3 + 1] as number,
143
+ seeds[at * 3 + 2] as number,
144
+ ]);
145
+ }
146
+ for (const camera of cameras) {
147
+ const m = camera.worldToCamera;
148
+ const centre = [0, 0, 0];
149
+ for (let r = 0; r < 3; r += 1) {
150
+ for (let c = 0; c < 3; c += 1) {
151
+ centre[c] = (centre[c] as number) - (m[r * 4 + c] as number) * (m[r * 4 + 3] as number);
152
+ }
153
+ }
154
+ points.push(centre);
155
+ }
156
+ const middle = [0, 0, 0];
157
+ for (const point of points) {
158
+ for (let c = 0; c < 3; c += 1) middle[c] = (middle[c] as number) + (point[c] as number);
159
+ }
160
+ for (let c = 0; c < 3; c += 1) middle[c] = (middle[c] as number) / points.length;
161
+ let furthest = 0;
162
+ for (const point of points) {
163
+ let square = 0;
164
+ for (let c = 0; c < 3; c += 1) {
165
+ const off = (point[c] as number) - (middle[c] as number);
166
+ square += off * off;
167
+ }
168
+ furthest = square > furthest ? square : furthest;
169
+ }
170
+ return Math.sqrt(furthest);
171
+ }
172
+
173
+ /** The mean distance from a seed to its nearest neighbour: the spacing the cloud was sampled at. */
174
+ function seedSpacing(seeds: Float64Array, count: number): number {
175
+ if (count < 2) return 0;
176
+ let total = 0;
177
+ for (let at = 0; at < count; at += 1) {
178
+ let nearest = Infinity;
179
+ for (let other = 0; other < count; other += 1) {
180
+ if (other === at) continue;
181
+ let square = 0;
182
+ for (let c = 0; c < 3; c += 1) {
183
+ const off = (seeds[at * 3 + c] as number) - (seeds[other * 3 + c] as number);
184
+ square += off * off;
185
+ }
186
+ if (square < nearest) nearest = square;
187
+ }
188
+ total += Math.sqrt(nearest);
189
+ }
190
+ return total / count;
191
+ }