@driftengine/capture 4.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (199) hide show
  1. package/LICENSE +202 -0
  2. package/NOTICE +29 -0
  3. package/README.md +68 -0
  4. package/dist/browserFrames.d.ts +31 -0
  5. package/dist/browserFrames.js +105 -0
  6. package/dist/collide.d.ts +59 -0
  7. package/dist/collide.js +92 -0
  8. package/dist/decimate.d.ts +42 -0
  9. package/dist/decimate.js +439 -0
  10. package/dist/delight.d.ts +42 -0
  11. package/dist/delight.js +351 -0
  12. package/dist/delightScene.d.ts +37 -0
  13. package/dist/delightScene.js +183 -0
  14. package/dist/depth.d.ts +44 -0
  15. package/dist/depth.js +53 -0
  16. package/dist/features.d.ts +33 -0
  17. package/dist/features.js +260 -0
  18. package/dist/frames.d.ts +47 -0
  19. package/dist/frames.js +87 -0
  20. package/dist/fusion.d.ts +77 -0
  21. package/dist/fusion.js +152 -0
  22. package/dist/gaussians/adam.d.ts +57 -0
  23. package/dist/gaussians/adam.js +88 -0
  24. package/dist/gaussians/depth.d.ts +8 -0
  25. package/dist/gaussians/depth.js +73 -0
  26. package/dist/gaussians/gradients.d.ts +36 -0
  27. package/dist/gaussians/gradients.js +228 -0
  28. package/dist/gaussians/harmonics.d.ts +46 -0
  29. package/dist/gaussians/harmonics.js +99 -0
  30. package/dist/gaussians/optimise.d.ts +50 -0
  31. package/dist/gaussians/optimise.js +125 -0
  32. package/dist/gaussians/parameters.d.ts +35 -0
  33. package/dist/gaussians/parameters.js +165 -0
  34. package/dist/gaussians/project.d.ts +72 -0
  35. package/dist/gaussians/project.js +137 -0
  36. package/dist/gaussians/projectGradient.d.ts +15 -0
  37. package/dist/gaussians/projectGradient.js +213 -0
  38. package/dist/gaussians/rasterise.d.ts +13 -0
  39. package/dist/gaussians/rasterise.js +65 -0
  40. package/dist/gaussians/refine.d.ts +36 -0
  41. package/dist/gaussians/refine.js +136 -0
  42. package/dist/gaussians/similarity.d.ts +25 -0
  43. package/dist/gaussians/similarity.js +80 -0
  44. package/dist/index.d.ts +101 -0
  45. package/dist/index.js +96 -0
  46. package/dist/marching.d.ts +38 -0
  47. package/dist/marching.js +258 -0
  48. package/dist/masks.d.ts +59 -0
  49. package/dist/masks.js +165 -0
  50. package/dist/math/dense.d.ts +68 -0
  51. package/dist/math/dense.js +294 -0
  52. package/dist/math/schur.d.ts +23 -0
  53. package/dist/math/schur.js +140 -0
  54. package/dist/models/batchNorm.d.ts +17 -0
  55. package/dist/models/batchNorm.js +24 -0
  56. package/dist/models/clipTokenizer.d.ts +40 -0
  57. package/dist/models/clipTokenizer.js +122 -0
  58. package/dist/models/clipTower.d.ts +27 -0
  59. package/dist/models/clipTower.js +45 -0
  60. package/dist/models/depthAnything2.d.ts +38 -0
  61. package/dist/models/depthAnything2.js +163 -0
  62. package/dist/models/depthAnything3.d.ts +26 -0
  63. package/dist/models/depthAnything3.js +171 -0
  64. package/dist/models/dualDpt.d.ts +19 -0
  65. package/dist/models/dualDpt.js +138 -0
  66. package/dist/models/hiera.d.ts +49 -0
  67. package/dist/models/hiera.js +120 -0
  68. package/dist/models/hieraAttention.d.ts +16 -0
  69. package/dist/models/hieraAttention.js +68 -0
  70. package/dist/models/miniature.d.ts +36 -0
  71. package/dist/models/miniature.js +216 -0
  72. package/dist/models/mobileSam.d.ts +41 -0
  73. package/dist/models/mobileSam.js +88 -0
  74. package/dist/models/owlv2.d.ts +49 -0
  75. package/dist/models/owlv2.js +131 -0
  76. package/dist/models/owlv2Detect.d.ts +25 -0
  77. package/dist/models/owlv2Detect.js +80 -0
  78. package/dist/models/owlv2Miniature.d.ts +27 -0
  79. package/dist/models/owlv2Miniature.js +87 -0
  80. package/dist/models/rope.d.ts +23 -0
  81. package/dist/models/rope.js +84 -0
  82. package/dist/models/sam21.d.ts +45 -0
  83. package/dist/models/sam21.js +126 -0
  84. package/dist/models/sam21Miniature.d.ts +37 -0
  85. package/dist/models/sam21Miniature.js +194 -0
  86. package/dist/models/sam21Video.d.ts +30 -0
  87. package/dist/models/sam21Video.js +101 -0
  88. package/dist/models/sam2Choice.d.ts +19 -0
  89. package/dist/models/sam2Choice.js +55 -0
  90. package/dist/models/sam2Decoder.d.ts +36 -0
  91. package/dist/models/sam2Decoder.js +108 -0
  92. package/dist/models/sam2Memories.d.ts +54 -0
  93. package/dist/models/sam2Memories.js +93 -0
  94. package/dist/models/sam2Memory.d.ts +30 -0
  95. package/dist/models/sam2Memory.js +136 -0
  96. package/dist/models/sam2Positions.d.ts +24 -0
  97. package/dist/models/sam2Positions.js +94 -0
  98. package/dist/models/sam2Tracker.d.ts +69 -0
  99. package/dist/models/sam2Tracker.js +172 -0
  100. package/dist/models/samDecoder.d.ts +45 -0
  101. package/dist/models/samDecoder.js +96 -0
  102. package/dist/models/samMiniature.d.ts +43 -0
  103. package/dist/models/samMiniature.js +161 -0
  104. package/dist/models/samPrompt.d.ts +57 -0
  105. package/dist/models/samPrompt.js +160 -0
  106. package/dist/models/seeded.d.ts +16 -0
  107. package/dist/models/seeded.js +48 -0
  108. package/dist/models/tinyVit.d.ts +38 -0
  109. package/dist/models/tinyVit.js +82 -0
  110. package/dist/models/twoWay.d.ts +46 -0
  111. package/dist/models/twoWay.js +67 -0
  112. package/dist/models/vit.d.ts +45 -0
  113. package/dist/models/vit.js +157 -0
  114. package/dist/models/windowAttention.d.ts +25 -0
  115. package/dist/models/windowAttention.js +104 -0
  116. package/dist/poses.d.ts +31 -0
  117. package/dist/poses.js +750 -0
  118. package/dist/prepare.d.ts +30 -0
  119. package/dist/prepare.js +238 -0
  120. package/dist/propose.d.ts +36 -0
  121. package/dist/propose.js +63 -0
  122. package/dist/resample.d.ts +39 -0
  123. package/dist/resample.js +165 -0
  124. package/dist/run.d.ts +3 -0
  125. package/dist/run.js +1 -0
  126. package/dist/scene.d.ts +104 -0
  127. package/dist/scene.js +109 -0
  128. package/dist/segment.d.ts +40 -0
  129. package/dist/segment.js +184 -0
  130. package/dist/testScene.d.ts +55 -0
  131. package/dist/testScene.js +214 -0
  132. package/dist/twoView.d.ts +22 -0
  133. package/dist/twoView.js +435 -0
  134. package/package.json +63 -0
  135. package/src/browserFrames.ts +145 -0
  136. package/src/collide.ts +128 -0
  137. package/src/decimate.ts +510 -0
  138. package/src/delight.ts +427 -0
  139. package/src/delightScene.ts +249 -0
  140. package/src/depth.ts +100 -0
  141. package/src/features.ts +302 -0
  142. package/src/frames.ts +111 -0
  143. package/src/fusion.ts +229 -0
  144. package/src/gaussians/adam.ts +131 -0
  145. package/src/gaussians/depth.ts +87 -0
  146. package/src/gaussians/gradients.ts +285 -0
  147. package/src/gaussians/harmonics.ts +122 -0
  148. package/src/gaussians/optimise.ts +191 -0
  149. package/src/gaussians/parameters.ts +186 -0
  150. package/src/gaussians/project.ts +207 -0
  151. package/src/gaussians/projectGradient.ts +234 -0
  152. package/src/gaussians/rasterise.ts +81 -0
  153. package/src/gaussians/refine.ts +176 -0
  154. package/src/gaussians/similarity.ts +87 -0
  155. package/src/index.ts +225 -0
  156. package/src/marching.ts +358 -0
  157. package/src/masks.ts +201 -0
  158. package/src/math/dense.ts +317 -0
  159. package/src/math/schur.ts +167 -0
  160. package/src/models/batchNorm.ts +57 -0
  161. package/src/models/clipTokenizer.ts +129 -0
  162. package/src/models/clipTower.ts +87 -0
  163. package/src/models/depthAnything2.ts +276 -0
  164. package/src/models/depthAnything3.ts +229 -0
  165. package/src/models/dualDpt.ts +233 -0
  166. package/src/models/hiera.ts +231 -0
  167. package/src/models/hieraAttention.ts +121 -0
  168. package/src/models/miniature.ts +241 -0
  169. package/src/models/mobileSam.ts +135 -0
  170. package/src/models/owlv2.ts +190 -0
  171. package/src/models/owlv2Detect.ts +116 -0
  172. package/src/models/owlv2Miniature.ts +108 -0
  173. package/src/models/rope.ts +111 -0
  174. package/src/models/sam21.ts +182 -0
  175. package/src/models/sam21Miniature.ts +225 -0
  176. package/src/models/sam21Video.ts +154 -0
  177. package/src/models/sam2Choice.ts +62 -0
  178. package/src/models/sam2Decoder.ts +218 -0
  179. package/src/models/sam2Memories.ts +165 -0
  180. package/src/models/sam2Memory.ts +247 -0
  181. package/src/models/sam2Positions.ts +106 -0
  182. package/src/models/sam2Tracker.ts +300 -0
  183. package/src/models/samDecoder.ts +225 -0
  184. package/src/models/samMiniature.ts +190 -0
  185. package/src/models/samPrompt.ts +227 -0
  186. package/src/models/seeded.ts +50 -0
  187. package/src/models/tinyVit.ts +178 -0
  188. package/src/models/twoWay.ts +122 -0
  189. package/src/models/vit.ts +265 -0
  190. package/src/models/windowAttention.ts +155 -0
  191. package/src/poses.ts +894 -0
  192. package/src/prepare.ts +306 -0
  193. package/src/propose.ts +96 -0
  194. package/src/resample.ts +197 -0
  195. package/src/run.ts +7 -0
  196. package/src/scene.ts +141 -0
  197. package/src/segment.ts +220 -0
  198. package/src/testScene.ts +283 -0
  199. package/src/twoView.ts +536 -0
@@ -0,0 +1,81 @@
1
+ /**
2
+ * A cloud of Gaussians drawn on the CPU: `project.ts`'s ellipses, composited nearest first.
3
+ *
4
+ * **It exists to be differentiated**, which is what separates it from the shader that draws splats
5
+ * in a game. A reference rasteriser is how the gradients in `gradients.ts` are checked — a central
6
+ * difference of this against the analytic answer — and how a fitted cloud is compared with the
7
+ * frames it was fitted to. It is thousands of times slower than the shader, and per-frame rules
8
+ * about allocation do not reach it.
9
+ */
10
+ import { exactExp } from '@driftengine/core';
11
+
12
+ import { cameraCentre, sh1Basis, splatColour, viewDirection } from './harmonics.ts';
13
+ import {
14
+ visibleGaussians,
15
+ type GaussianSet,
16
+ type Projected,
17
+ type RasterCamera,
18
+ } from './project.ts';
19
+
20
+ /**
21
+ * A frame this rasteriser writes: **single precision where it stands in for a render target, double
22
+ * where the caller is differentiating through it.** The fitting loop accumulates its loss in double
23
+ * and a frame quantised to single is a floor under every difference taken through one — measured at
24
+ * about a part in five hundred of the slope, against a part in ten million when the frame is double.
25
+ */
26
+ export type Frame = Float32Array | Float64Array;
27
+
28
+ /** Below this the Gaussian contributes nothing worth compositing. */
29
+ const FAINT = 1 / 255 / 4;
30
+
31
+ /**
32
+ * `set` through `camera` into `out`, four floats a pixel: linear colour, premultiplied as it
33
+ * composites, and the alpha the cloud covered the pixel with.
34
+ */
35
+ export function rasteriseGaussians(set: GaussianSet, camera: RasterCamera, out: Frame): void {
36
+ const { width, height } = camera;
37
+ out.fill(0);
38
+ const transmittance = new Float64Array(width * height).fill(1);
39
+ const { order, projected } = visibleGaussians(set, camera);
40
+ /* A Gaussian's colour is a function of where it is seen from, so it is resolved once a frame. */
41
+ const eye = new Float64Array(3);
42
+ cameraCentre(camera.worldToCamera, eye);
43
+ const direction = new Float64Array(3);
44
+ const basis = new Float64Array(3);
45
+ const colour = new Float64Array(3);
46
+ for (const at of order) {
47
+ const one = projected[at] as Projected;
48
+ viewDirection(set.positions, at, eye, direction);
49
+ sh1Basis(direction, basis);
50
+ splatColour(set.colors, set.sh1, at, basis, colour);
51
+ const determinant = one.a * one.d - one.b * one.b;
52
+ const inverse = [one.d / determinant, -one.b / determinant, one.a / determinant];
53
+ const left = Math.max(0, Math.floor(one.x - one.radius));
54
+ const right = Math.min(width - 1, Math.ceil(one.x + one.radius));
55
+ const top = Math.max(0, Math.floor(one.y - one.radius));
56
+ const bottom = Math.min(height - 1, Math.ceil(one.y + one.radius));
57
+ const opacity = set.opacities[at] as number;
58
+ for (let y = top; y <= bottom; y += 1) {
59
+ for (let x = left; x <= right; x += 1) {
60
+ const dx = x + 0.5 - one.x;
61
+ const dy = y + 0.5 - one.y;
62
+ const power =
63
+ -0.5 *
64
+ ((inverse[0] as number) * dx * dx +
65
+ 2 * (inverse[1] as number) * dx * dy +
66
+ (inverse[2] as number) * dy * dy);
67
+ /* The inverse of a positive definite covariance is positive definite, so `power` is at
68
+ most zero and the exponential at most one: there is nothing to guard against here. */
69
+ const alpha = opacity * exactExp(power);
70
+ if (alpha < FAINT) continue;
71
+ const pixel = y * width + x;
72
+ const share = alpha * (transmittance[pixel] as number);
73
+ for (let c = 0; c < 3; c += 1) {
74
+ out[pixel * 4 + c] = (out[pixel * 4 + c] as number) + share * (colour[c] as number);
75
+ }
76
+ out[pixel * 4 + 3] = (out[pixel * 4 + 3] as number) + share;
77
+ transmittance[pixel] = (transmittance[pixel] as number) * (1 - alpha);
78
+ }
79
+ }
80
+ }
81
+ }
@@ -0,0 +1,176 @@
1
+ /**
2
+ * Where the cloud grows and where it is thrown away: densification and pruning, as one plan.
3
+ *
4
+ * **A fit that only moves the Gaussians it started with cannot see detail its seeds did not.** The
5
+ * seeds come from a sparse reconstruction — a few hundred corners — and a room has more surface
6
+ * than that. So the busiest Gaussians are made into two: a small one is **cloned**, which puts a
7
+ * second Gaussian where one was not enough, and a large one is **split**, which replaces a blur
8
+ * covering several surfaces with two that can each take one. The reference implementation decides
9
+ * between the two by the same test.
10
+ *
11
+ * **Busiest by rank rather than by a threshold.** The reference compares an accumulated screen-space
12
+ * gradient against 0.0002, a number in units of that implementation's own normalisation, and a
13
+ * number in units nobody can check is a number nobody can tune. This takes a fixed fraction of the
14
+ * cloud — the most-moved fifth — which says the same thing scale-free: *these are the Gaussians the
15
+ * frames keep pulling on.* What it gives up is the reference's property that a converged fit stops
16
+ * growing on its own; the budget is what stops it here, and the pruning below is what keeps the
17
+ * budget spent on Gaussians that are doing something.
18
+ *
19
+ * **Pruning is by opacity, and it happens first.** A Gaussian the fit has faded to nothing is
20
+ * carrying a slot and contributing less light than a pixel can hold.
21
+ */
22
+ import { exactExp, exactLog } from '@driftengine/core';
23
+
24
+ import { gather, type Family } from './adam.ts';
25
+ import { rotationFromQuaternion } from './project.ts';
26
+
27
+ /** Below this a Gaussian contributes less than a quarter of one value of an eight-bit channel. */
28
+ export const FADED = 1 / 255 / 4;
29
+
30
+ /** How much of the cloud is made into two, where there is room. */
31
+ const BUSIEST = 0.2;
32
+
33
+ /** A split halves what it divides, near enough: the reference's own divisor. */
34
+ const SPLIT_SHRINK = 1.6;
35
+
36
+ export interface RefineOptions {
37
+ /** The most Gaussians the cloud may hold. */
38
+ readonly budget: number;
39
+ /** A Gaussian wider than this is split rather than cloned, in metres. */
40
+ readonly splitAbove: number;
41
+ readonly random: () => number;
42
+ }
43
+
44
+ /** Everything the plan rewrites, in one place. */
45
+ export interface RefineCloud {
46
+ count: number;
47
+ readonly families: readonly Family[];
48
+ readonly positions: Family;
49
+ readonly logScales: Family;
50
+ readonly rotations: Family;
51
+ readonly opacities: Float64Array;
52
+ /** How hard the frames have pulled on each Gaussian since the last refinement. */
53
+ readonly motion: Float64Array;
54
+ readonly sources: Int32Array;
55
+ /** Where each of today's Gaussians ended up after a prune, so a split can find its own half. */
56
+ readonly reverse: Int32Array;
57
+ readonly fresh: Uint8Array;
58
+ readonly scratch: Float64Array;
59
+ }
60
+
61
+ /**
62
+ * Prune, then grow, then rewrite every family; answers the new count.
63
+ *
64
+ * The cloud's own arrays are rewritten in place through its scratch, so nothing is reallocated and
65
+ * the caller's view of it stays valid. **The materialised scales and opacities are not**: a slot
66
+ * that has just been split holds the logarithm it was given and the metres of whatever stood there
67
+ * before, so the caller materialises again before drawing. The fit does that at the top of every
68
+ * step, which is why this does not do it twice.
69
+ */
70
+ export function refineCloud(cloud: RefineCloud, options: RefineOptions): number {
71
+ const { budget, splitAbove, random } = options;
72
+ const { sources, fresh, motion } = cloud;
73
+
74
+ /* Everything still worth drawing, in order, with its history intact. */
75
+ const { reverse } = cloud;
76
+ let kept = 0;
77
+ for (let at = 0; at < cloud.count; at += 1) {
78
+ reverse[at] = -1;
79
+ if ((cloud.opacities[at] as number) < FADED) continue;
80
+ reverse[at] = kept;
81
+ sources[kept] = at;
82
+ fresh[kept] = 0;
83
+ kept += 1;
84
+ }
85
+ /* A prune that emptied the cloud would leave the fit nothing to descend, so it is refused. */
86
+ if (kept === 0) return cloud.count;
87
+
88
+ /* The busiest fifth of what is left, most-pulled first, as far as the budget allows. */
89
+ const order = Array.from({ length: kept }, (_, slot) => slot);
90
+ order.sort((a, b) => {
91
+ const first = motion[sources[a] as number] as number;
92
+ const second = motion[sources[b] as number] as number;
93
+ return first === second ? (sources[a] as number) - (sources[b] as number) : second - first;
94
+ });
95
+ /*
96
+ * At least one where there is room and something is pulling, because a fifth of a small cloud
97
+ * rounds to none — and a cloud of four that can never grow however long it is fitted is a cliff
98
+ * nobody would look for.
99
+ */
100
+ const room = budget - kept;
101
+ const share = Math.floor(kept * BUSIEST);
102
+ const wanted = Math.min(room, share > 1 ? share : 1);
103
+ let total = kept;
104
+ const rotation = new Float64Array(9);
105
+ for (let taken = 0; taken < wanted; taken += 1) {
106
+ const slot = order[taken] as number;
107
+ const from = sources[slot] as number;
108
+ if (!((motion[from] as number) > 0)) break;
109
+ sources[total] = from;
110
+ fresh[total] = 1;
111
+ total += 1;
112
+ }
113
+
114
+ gather(cloud.families, sources, fresh, total, cloud.scratch);
115
+
116
+ /*
117
+ * Now the two halves are told apart. A clone is left where it was — its own gradient is what
118
+ * separates it, and its Adam state is empty where its parent's is not. A split is shrunk and both
119
+ * halves are moved, because two Gaussians of the parent's size in the parent's place are one
120
+ * blur drawn twice.
121
+ */
122
+ for (let slot = kept; slot < total; slot += 1) {
123
+ const parent = sources[slot] as number;
124
+ let widest = 0;
125
+ for (let c = 0; c < 3; c += 1) {
126
+ const scale = exactExp(cloud.logScales.values[slot * 3 + c] as number);
127
+ widest = scale > widest ? scale : widest;
128
+ }
129
+ if (widest <= splitAbove) continue;
130
+ rotationFromQuaternion(cloud.rotations.values, slot, rotation);
131
+ shrinkAndDisplace(cloud, slot, rotation, random);
132
+ shrinkAndDisplace(cloud, reverse[parent] as number, rotation, random);
133
+ }
134
+ cloud.count = total;
135
+ motion.fill(0);
136
+ return total;
137
+ }
138
+
139
+ /** One half of a split: smaller than what it came from, and somewhere inside it. */
140
+ function shrinkAndDisplace(
141
+ cloud: RefineCloud,
142
+ slot: number,
143
+ rotation: Float64Array,
144
+ random: () => number,
145
+ ): void {
146
+ if (slot < 0) return;
147
+ displace(cloud, slot, rotation, random);
148
+ for (let c = 0; c < 3; c += 1) {
149
+ cloud.logScales.values[slot * 3 + c] =
150
+ (cloud.logScales.values[slot * 3 + c] as number) - exactLog(SPLIT_SHRINK);
151
+ }
152
+ }
153
+
154
+ /** One sample from a Gaussian's own shape, added to where it stands. */
155
+ function displace(
156
+ cloud: RefineCloud,
157
+ slot: number,
158
+ rotation: Float64Array,
159
+ random: () => number,
160
+ ): void {
161
+ /*
162
+ * A sum of three uniforms either side of zero rather than a normal deviate: the fit only needs
163
+ * two halves to land somewhere inside the shape they came from, and a Box–Muller transform would
164
+ * want a logarithm and a cosine for no gain the picture can show.
165
+ */
166
+ const local = [0, 0, 0];
167
+ for (let c = 0; c < 3; c += 1) {
168
+ const spread = random() + random() + random() - 1.5;
169
+ local[c] = spread * exactExp(cloud.logScales.values[slot * 3 + c] as number);
170
+ }
171
+ for (let r = 0; r < 3; r += 1) {
172
+ let sum = 0;
173
+ for (let c = 0; c < 3; c += 1) sum += (rotation[r * 3 + c] as number) * (local[c] as number);
174
+ cloud.positions.values[slot * 3 + r] = (cloud.positions.values[slot * 3 + r] as number) + sum;
175
+ }
176
+ }
@@ -0,0 +1,87 @@
1
+ /**
2
+ * How alike two pictures are, structure included.
3
+ *
4
+ * **A squared difference is not a likeness.** It is what the fit descends, because it has a
5
+ * gradient, and it is the wrong thing to judge the result by: a cloud that is uniformly a little
6
+ * dark scores badly on it and looks right, and one that is right on average with the structure
7
+ * smeared away scores well and looks wrong. Structural similarity compares local means, variances
8
+ * and covariance, so it answers the question a reader actually asks of a capture.
9
+ *
10
+ * **Uniform windows rather than a Gaussian one.** The original weights each window by an 11 × 11
11
+ * Gaussian; this walks 8 × 8 blocks with equal weight, which is the same three terms over a
12
+ * different support and is what every fast implementation does. What it gives up is comparability
13
+ * with a number quoted from the paper — so a floor here is measured against this implementation
14
+ * and not lifted from one written elsewhere.
15
+ */
16
+
17
+ /** The side of the block the three statistics are taken over. */
18
+ export const SIMILARITY_WINDOW = 8;
19
+
20
+ /*
21
+ * The stabilisers, for values on 0 to 1. They keep a window where both pictures are flat from
22
+ * dividing zero by zero, and they are the reason two dark patches are not declared identical on
23
+ * the strength of a rounding difference.
24
+ */
25
+ const C1 = 0.01 * 0.01;
26
+ const C2 = 0.03 * 0.03;
27
+
28
+ /**
29
+ * The mean structural similarity of `a` and `b`, one value a pixel, over every window.
30
+ *
31
+ * A picture narrower or shorter than a window is compared as one window rather than refused: it
32
+ * has a mean and a variance like any other, and the alternative is a measure that cannot answer
33
+ * for a thumbnail.
34
+ */
35
+ export function structuralSimilarity(
36
+ a: ArrayLike<number>,
37
+ b: ArrayLike<number>,
38
+ width: number,
39
+ height: number,
40
+ ): number {
41
+ let total = 0;
42
+ let windows = 0;
43
+ for (let top = 0; top < height; top += SIMILARITY_WINDOW) {
44
+ for (let left = 0; left < width; left += SIMILARITY_WINDOW) {
45
+ const right = Math.min(width, left + SIMILARITY_WINDOW);
46
+ const bottom = Math.min(height, top + SIMILARITY_WINDOW);
47
+ const pixels = (right - left) * (bottom - top);
48
+ if (pixels === 0) continue;
49
+ let sumA = 0;
50
+ let sumB = 0;
51
+ for (let y = top; y < bottom; y += 1) {
52
+ for (let x = left; x < right; x += 1) {
53
+ sumA += a[y * width + x] as number;
54
+ sumB += b[y * width + x] as number;
55
+ }
56
+ }
57
+ const meanA = sumA / pixels;
58
+ const meanB = sumB / pixels;
59
+ let varianceA = 0;
60
+ let varianceB = 0;
61
+ let covariance = 0;
62
+ for (let y = top; y < bottom; y += 1) {
63
+ for (let x = left; x < right; x += 1) {
64
+ const offA = (a[y * width + x] as number) - meanA;
65
+ const offB = (b[y * width + x] as number) - meanB;
66
+ varianceA += offA * offA;
67
+ varianceB += offB * offB;
68
+ covariance += offA * offB;
69
+ }
70
+ }
71
+ /*
72
+ * Divided by n − 1 where there is more than one pixel, which is the unbiased estimate the
73
+ * original uses. A single-pixel window has no spread to estimate and is compared by its
74
+ * means alone.
75
+ */
76
+ const spread = pixels > 1 ? pixels - 1 : 1;
77
+ varianceA /= spread;
78
+ varianceB /= spread;
79
+ covariance /= spread;
80
+ const luminance = (2 * meanA * meanB + C1) / (meanA * meanA + meanB * meanB + C1);
81
+ const structure = (2 * covariance + C2) / (varianceA + varianceB + C2);
82
+ total += luminance * structure;
83
+ windows += 1;
84
+ }
85
+ }
86
+ return windows === 0 ? 1 : total / windows;
87
+ }
package/src/index.ts ADDED
@@ -0,0 +1,225 @@
1
+ /*! DriftEngine | Copyright 2026 Drift Technologies | Apache-2.0 | https://github.com/drftrun/driftengine */
2
+ /**
3
+ * `@driftengine/capture` — a video to a playable scene, on the player's own device.
4
+ *
5
+ * **What is here so far is the models a capture runs**, each written as a definition the engine's
6
+ * one neural runtime builds a graph from: `@driftengine/texture` holds the operators and the rule
7
+ * that every weight is read or set aside by name, and this package says which weights a model reads
8
+ * and how. A definition is run over a checkpoint once, when it is converted, and over the converted
9
+ * file afterwards whenever a clip needs the graph at a new size — every device kernel bakes its
10
+ * shapes, and a clip's aspect ratio decides the patch grid.
11
+ *
12
+ * **Depth Anything 3**: depth, its confidence and a camera per view, from one view or several at
13
+ * once. **Depth Anything V2 Small**, the lighter fallback: relative depth from one view.
14
+ * **MobileSAM**: masks from a point or a box, an encoder run once an image and a decoder once a
15
+ * prompt. **SAM 2.1**: the same, and objects tracked through a video by a memory of the frames before.
16
+ * **OWLv2**: boxes for things named in words, an image graph and a text graph joined on the host,
17
+ * with CLIP's tokenizer. Each answers as its upstream's own code does on a seeded miniature of its
18
+ * layout, the fixtures shipped beside them.
19
+ */
20
+ export { eachFrame, selectFrames } from './frames.ts';
21
+ export type { FrameChoice, FrameSize, FrameSource } from './frames.ts';
22
+ export {
23
+ cholesky,
24
+ choleskySolve,
25
+ levenbergMarquardt,
26
+ svd3,
27
+ svdN,
28
+ symmetricEigen,
29
+ } from './math/dense.ts';
30
+ export type { FitOptions, LeastSquares } from './math/dense.ts';
31
+ export { schurSolve } from './math/schur.ts';
32
+ export type { BlockNormals } from './math/schur.ts';
33
+ export {
34
+ DESCRIPTOR_BYTES,
35
+ createFeatureSet,
36
+ describeFeatures,
37
+ detectFeatures,
38
+ matchFeatures,
39
+ } from './features.ts';
40
+ export type { Descriptors, FeatureSet } from './features.ts';
41
+ export { estimatePoses } from './poses.ts';
42
+ export type { CaptureFrame, PoseOptions, PoseResult } from './poses.ts';
43
+ export { relativePose, triangulate } from './twoView.ts';
44
+ export type { RelativePoseOut, RelativePoseResult } from './twoView.ts';
45
+ export { lookAt, renderTestScene } from './testScene.ts';
46
+ export type { TestBox, TestCamera, TestPlane, TestScene } from './testScene.ts';
47
+ export { createDepthEstimator } from './depth.ts';
48
+ export type { DepthEstimate, DepthEstimator, DepthView, RawFrame } from './depth.ts';
49
+ export type { GraphRun } from './run.ts';
50
+ export {
51
+ DEPTH_PIXEL_MEAN,
52
+ DEPTH_PIXEL_STD,
53
+ longestSideSize,
54
+ prepareDepthFrame,
55
+ prepareOwlv2Frame,
56
+ prepareSam2Frame,
57
+ prepareSamFrame,
58
+ } from './prepare.ts';
59
+ export type { PreparedFrame } from './prepare.ts';
60
+ export { areaResize, triangleResize, PILLOW_PRECISION, TORCHVISION_PRECISION } from './resample.ts';
61
+ export { browserFrameSource } from './browserFrames.ts';
62
+ export type { BrowserClip } from './browserFrames.ts';
63
+ export {
64
+ DEPTH_ANYTHING_3,
65
+ decodeCamera,
66
+ decodeDepth,
67
+ depthAnything3,
68
+ } from './models/depthAnything3.ts';
69
+ export type { DepthAnything3Config } from './models/depthAnything3.ts';
70
+ export { DEPTH_ANYTHING_2_SMALL, depthAnything2 } from './models/depthAnything2.ts';
71
+ export type { DepthAnything2Config } from './models/depthAnything2.ts';
72
+ export type { HeadConfig } from './models/dualDpt.ts';
73
+ export type { VitConfig } from './models/vit.ts';
74
+ export {
75
+ MINIATURE_CASES,
76
+ MINIATURE_DEPTH_ANYTHING_2,
77
+ MINIATURE_DEPTH_ANYTHING_3,
78
+ MINIATURE_SEED,
79
+ miniatureCheckpoint,
80
+ miniatureCheckpoint2,
81
+ miniatureImages,
82
+ } from './models/miniature.ts';
83
+ export {
84
+ MOBILE_SAM,
85
+ SAM_PIXEL_MEAN,
86
+ SAM_PIXEL_STD,
87
+ mobileSamDecoder,
88
+ mobileSamEncoder,
89
+ samMasksToImage,
90
+ } from './models/mobileSam.ts';
91
+ export type { MobileSamConfig } from './models/mobileSam.ts';
92
+ export type { SamDecoderConfig } from './models/samDecoder.ts';
93
+ export {
94
+ SAM2_PROMPT,
95
+ SAM2_VIDEO_PROMPT,
96
+ SAM_PROMPT,
97
+ samGridPositions,
98
+ samInputSize,
99
+ samPromptTokens,
100
+ samTokenCount,
101
+ stabilityScore,
102
+ } from './models/samPrompt.ts';
103
+ export type { SamPrompt, SamPromptLayout } from './models/samPrompt.ts';
104
+ export {
105
+ MINIATURE_MOBILE_SAM,
106
+ MINIATURE_SAM_IMAGE,
107
+ MINIATURE_SAM_PROMPTS,
108
+ miniatureSamCheckpoint,
109
+ } from './models/samMiniature.ts';
110
+ export type { TinyVitConfig } from './models/tinyVit.ts';
111
+ export { SAM_21_TINY, sam21Decoder, sam21Encoder } from './models/sam21.ts';
112
+ export type { Sam21Config, Sam21MemoryConfig } from './models/sam21.ts';
113
+ export {
114
+ sam21MasksToImage,
115
+ sam21MemoryAttention,
116
+ sam21MemoryEncoder,
117
+ sam21Upscale,
118
+ } from './models/sam21Video.ts';
119
+ export { Sam21Tracker } from './models/sam2Tracker.ts';
120
+ export type { Sam21Frame, Sam21Video, Sam21Weights } from './models/sam2Tracker.ts';
121
+ export type { HieraConfig } from './models/hiera.ts';
122
+ export {
123
+ MINIATURE_SAM_21,
124
+ MINIATURE_SAM_21_IMAGE,
125
+ MINIATURE_SAM_21_PROMPTS,
126
+ miniatureSam21Checkpoint,
127
+ } from './models/sam21Miniature.ts';
128
+ export {
129
+ OWLV2_BASE,
130
+ OWLV2_PIXEL_MEAN,
131
+ OWLV2_PIXEL_STD,
132
+ owlv2BoxBias,
133
+ owlv2Image,
134
+ owlv2Text,
135
+ } from './models/owlv2.ts';
136
+ export type { Owlv2Config } from './models/owlv2.ts';
137
+ export type { ClipTowerConfig } from './models/clipTower.ts';
138
+ export { owlv2Detections, owlv2Logits } from './models/owlv2Detect.ts';
139
+ export type { Owlv2Detection, Owlv2Patches } from './models/owlv2Detect.ts';
140
+ export { clipTokenizer, owlv2Tokens } from './models/clipTokenizer.ts';
141
+ export type { ClipTokenizer } from './models/clipTokenizer.ts';
142
+ export {
143
+ MINIATURE_OWLV2,
144
+ MINIATURE_OWLV2_IMAGE,
145
+ MINIATURE_OWLV2_MERGES,
146
+ MINIATURE_OWLV2_QUERIES,
147
+ miniatureOwlv2Checkpoint,
148
+ } from './models/owlv2Miniature.ts';
149
+ /*
150
+ * **Gaussians.** A reference rasteriser that projects the way `@driftengine/splats`' shader does,
151
+ * its analytic gradients, and the fit that descends them into a `SplatSource` the engine draws.
152
+ * `optimiseGaussians` is what a caller wants; the rest is exported because a caller measuring or
153
+ * extending a fit needs the same pieces the fit is built from.
154
+ */
155
+ export { optimiseGaussians } from './gaussians/optimise.ts';
156
+ export type { OptimiseOptions } from './gaussians/optimise.ts';
157
+ export { projectGaussian, visibleGaussians, LOW_PASS, REACH } from './gaussians/project.ts';
158
+ export type { GaussianSet, Projected, RasterCamera } from './gaussians/project.ts';
159
+ export { rasteriseGaussians } from './gaussians/rasterise.ts';
160
+ export type { Frame } from './gaussians/rasterise.ts';
161
+ export { accumulateGradients, createGradients, imageLoss } from './gaussians/gradients.ts';
162
+ export type { GaussianGradients } from './gaussians/gradients.ts';
163
+ export { sh1Basis, splatColour, SH1_COEFFICIENTS, SH_C1 } from './gaussians/harmonics.ts';
164
+ export { structuralSimilarity, SIMILARITY_WINDOW } from './gaussians/similarity.ts';
165
+ /*
166
+ * **Splats to a surface.** A cloud's depth, many views of it fused into one volume, that volume's
167
+ * crossing as a mesh facing outwards, and the mesh brought down to a budget — which is what a game
168
+ * stands on, rather than the cloud, which only draws.
169
+ */
170
+ export { renderDepth } from './gaussians/depth.ts';
171
+ export { createVolume, fuseDepth } from './fusion.ts';
172
+ export type { FuseOptions, SurfaceView, Volume } from './fusion.ts';
173
+ export { marchVolume } from './marching.ts';
174
+ export type { MarchOptions } from './marching.ts';
175
+ export { decimate } from './decimate.ts';
176
+ export type { DecimateOptions } from './decimate.ts';
177
+ /*
178
+ * **A surface to stand on.** The drawn mesh cleaned for collision, and a prop's convex hulls inside
179
+ * the container's own caps — which is where a capture stops being a picture.
180
+ */
181
+ export { collisionMesh, propHulls } from './collide.ts';
182
+ export type { CollisionOptions, CollisionSource, PropHulls } from './collide.ts';
183
+ /*
184
+ * **A surface's material, separated from the light it was photographed under.** Multi-view inverse
185
+ * rendering with a classical separation: what stays the same across views is the material, what
186
+ * moves is the light, and a chromaticity that holds across an intensity jump is a shadow. Every
187
+ * surface gets a confidence, and `delight.ts` states plainly what that confidence does and does not
188
+ * grade. `renderLit` is the lit fixture the measurements are made against.
189
+ */
190
+ export { createDelightOut, delight } from './delight.ts';
191
+ export type { DelightOptions, DelightOut, DelightView } from './delight.ts';
192
+ export { interpolate, nearestHit, renderLit } from './delightScene.ts';
193
+ export type { Light, LitScene, MeshHit } from './delightScene.ts';
194
+ /*
195
+ * **A mesh cut into regions, and each region proposed as something a scene could hold.** Stage four
196
+ * and labelled as such: with a model loaded the masks are voted onto the mesh, and with none the
197
+ * geometry answers on its own. Nothing here decides what an object *is* — a region nobody labelled
198
+ * comes back as scenery, which is drawn, solid, and doing nothing.
199
+ */
200
+ export { segmentGeometry } from './segment.ts';
201
+ export type { Region, SegmentOptions } from './segment.ts';
202
+ export { labelMasks, liftMasks, promptGrid } from './masks.ts';
203
+ export type { MaskView } from './masks.ts';
204
+ export {
205
+ MOVABLE_COMPONENTS,
206
+ SCENERY_COMPONENTS,
207
+ WALKABLE_COMPONENTS,
208
+ proposeEntities,
209
+ } from './propose.ts';
210
+ export type { EntityProposal, ProposeOptions } from './propose.ts';
211
+ /*
212
+ * **Every stage in one file.** A capture that arrives as six files is a capture somebody has to
213
+ * reassemble; this writes the mesh, its materials and decode program, the cloud, a prop's hulls,
214
+ * the way across the scene and the proposals into one `.drft`. The proposals travel under a
215
+ * component this package declares — a capture proposes in its own words, and what a scene ends up
216
+ * holding is the consumer's decision.
217
+ */
218
+ export {
219
+ PROPOSAL_COMPONENT,
220
+ PROPOSAL_SCHEMA,
221
+ captureFile,
222
+ proposalScene,
223
+ readProposals,
224
+ } from './scene.ts';
225
+ export type { CaptureScene } from './scene.ts';