@driftengine/capture 4.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (199) hide show
  1. package/LICENSE +202 -0
  2. package/NOTICE +29 -0
  3. package/README.md +68 -0
  4. package/dist/browserFrames.d.ts +31 -0
  5. package/dist/browserFrames.js +105 -0
  6. package/dist/collide.d.ts +59 -0
  7. package/dist/collide.js +92 -0
  8. package/dist/decimate.d.ts +42 -0
  9. package/dist/decimate.js +439 -0
  10. package/dist/delight.d.ts +42 -0
  11. package/dist/delight.js +351 -0
  12. package/dist/delightScene.d.ts +37 -0
  13. package/dist/delightScene.js +183 -0
  14. package/dist/depth.d.ts +44 -0
  15. package/dist/depth.js +53 -0
  16. package/dist/features.d.ts +33 -0
  17. package/dist/features.js +260 -0
  18. package/dist/frames.d.ts +47 -0
  19. package/dist/frames.js +87 -0
  20. package/dist/fusion.d.ts +77 -0
  21. package/dist/fusion.js +152 -0
  22. package/dist/gaussians/adam.d.ts +57 -0
  23. package/dist/gaussians/adam.js +88 -0
  24. package/dist/gaussians/depth.d.ts +8 -0
  25. package/dist/gaussians/depth.js +73 -0
  26. package/dist/gaussians/gradients.d.ts +36 -0
  27. package/dist/gaussians/gradients.js +228 -0
  28. package/dist/gaussians/harmonics.d.ts +46 -0
  29. package/dist/gaussians/harmonics.js +99 -0
  30. package/dist/gaussians/optimise.d.ts +50 -0
  31. package/dist/gaussians/optimise.js +125 -0
  32. package/dist/gaussians/parameters.d.ts +35 -0
  33. package/dist/gaussians/parameters.js +165 -0
  34. package/dist/gaussians/project.d.ts +72 -0
  35. package/dist/gaussians/project.js +137 -0
  36. package/dist/gaussians/projectGradient.d.ts +15 -0
  37. package/dist/gaussians/projectGradient.js +213 -0
  38. package/dist/gaussians/rasterise.d.ts +13 -0
  39. package/dist/gaussians/rasterise.js +65 -0
  40. package/dist/gaussians/refine.d.ts +36 -0
  41. package/dist/gaussians/refine.js +136 -0
  42. package/dist/gaussians/similarity.d.ts +25 -0
  43. package/dist/gaussians/similarity.js +80 -0
  44. package/dist/index.d.ts +101 -0
  45. package/dist/index.js +96 -0
  46. package/dist/marching.d.ts +38 -0
  47. package/dist/marching.js +258 -0
  48. package/dist/masks.d.ts +59 -0
  49. package/dist/masks.js +165 -0
  50. package/dist/math/dense.d.ts +68 -0
  51. package/dist/math/dense.js +294 -0
  52. package/dist/math/schur.d.ts +23 -0
  53. package/dist/math/schur.js +140 -0
  54. package/dist/models/batchNorm.d.ts +17 -0
  55. package/dist/models/batchNorm.js +24 -0
  56. package/dist/models/clipTokenizer.d.ts +40 -0
  57. package/dist/models/clipTokenizer.js +122 -0
  58. package/dist/models/clipTower.d.ts +27 -0
  59. package/dist/models/clipTower.js +45 -0
  60. package/dist/models/depthAnything2.d.ts +38 -0
  61. package/dist/models/depthAnything2.js +163 -0
  62. package/dist/models/depthAnything3.d.ts +26 -0
  63. package/dist/models/depthAnything3.js +171 -0
  64. package/dist/models/dualDpt.d.ts +19 -0
  65. package/dist/models/dualDpt.js +138 -0
  66. package/dist/models/hiera.d.ts +49 -0
  67. package/dist/models/hiera.js +120 -0
  68. package/dist/models/hieraAttention.d.ts +16 -0
  69. package/dist/models/hieraAttention.js +68 -0
  70. package/dist/models/miniature.d.ts +36 -0
  71. package/dist/models/miniature.js +216 -0
  72. package/dist/models/mobileSam.d.ts +41 -0
  73. package/dist/models/mobileSam.js +88 -0
  74. package/dist/models/owlv2.d.ts +49 -0
  75. package/dist/models/owlv2.js +131 -0
  76. package/dist/models/owlv2Detect.d.ts +25 -0
  77. package/dist/models/owlv2Detect.js +80 -0
  78. package/dist/models/owlv2Miniature.d.ts +27 -0
  79. package/dist/models/owlv2Miniature.js +87 -0
  80. package/dist/models/rope.d.ts +23 -0
  81. package/dist/models/rope.js +84 -0
  82. package/dist/models/sam21.d.ts +45 -0
  83. package/dist/models/sam21.js +126 -0
  84. package/dist/models/sam21Miniature.d.ts +37 -0
  85. package/dist/models/sam21Miniature.js +194 -0
  86. package/dist/models/sam21Video.d.ts +30 -0
  87. package/dist/models/sam21Video.js +101 -0
  88. package/dist/models/sam2Choice.d.ts +19 -0
  89. package/dist/models/sam2Choice.js +55 -0
  90. package/dist/models/sam2Decoder.d.ts +36 -0
  91. package/dist/models/sam2Decoder.js +108 -0
  92. package/dist/models/sam2Memories.d.ts +54 -0
  93. package/dist/models/sam2Memories.js +93 -0
  94. package/dist/models/sam2Memory.d.ts +30 -0
  95. package/dist/models/sam2Memory.js +136 -0
  96. package/dist/models/sam2Positions.d.ts +24 -0
  97. package/dist/models/sam2Positions.js +94 -0
  98. package/dist/models/sam2Tracker.d.ts +69 -0
  99. package/dist/models/sam2Tracker.js +172 -0
  100. package/dist/models/samDecoder.d.ts +45 -0
  101. package/dist/models/samDecoder.js +96 -0
  102. package/dist/models/samMiniature.d.ts +43 -0
  103. package/dist/models/samMiniature.js +161 -0
  104. package/dist/models/samPrompt.d.ts +57 -0
  105. package/dist/models/samPrompt.js +160 -0
  106. package/dist/models/seeded.d.ts +16 -0
  107. package/dist/models/seeded.js +48 -0
  108. package/dist/models/tinyVit.d.ts +38 -0
  109. package/dist/models/tinyVit.js +82 -0
  110. package/dist/models/twoWay.d.ts +46 -0
  111. package/dist/models/twoWay.js +67 -0
  112. package/dist/models/vit.d.ts +45 -0
  113. package/dist/models/vit.js +157 -0
  114. package/dist/models/windowAttention.d.ts +25 -0
  115. package/dist/models/windowAttention.js +104 -0
  116. package/dist/poses.d.ts +31 -0
  117. package/dist/poses.js +750 -0
  118. package/dist/prepare.d.ts +30 -0
  119. package/dist/prepare.js +238 -0
  120. package/dist/propose.d.ts +36 -0
  121. package/dist/propose.js +63 -0
  122. package/dist/resample.d.ts +39 -0
  123. package/dist/resample.js +165 -0
  124. package/dist/run.d.ts +3 -0
  125. package/dist/run.js +1 -0
  126. package/dist/scene.d.ts +104 -0
  127. package/dist/scene.js +109 -0
  128. package/dist/segment.d.ts +40 -0
  129. package/dist/segment.js +184 -0
  130. package/dist/testScene.d.ts +55 -0
  131. package/dist/testScene.js +214 -0
  132. package/dist/twoView.d.ts +22 -0
  133. package/dist/twoView.js +435 -0
  134. package/package.json +63 -0
  135. package/src/browserFrames.ts +145 -0
  136. package/src/collide.ts +128 -0
  137. package/src/decimate.ts +510 -0
  138. package/src/delight.ts +427 -0
  139. package/src/delightScene.ts +249 -0
  140. package/src/depth.ts +100 -0
  141. package/src/features.ts +302 -0
  142. package/src/frames.ts +111 -0
  143. package/src/fusion.ts +229 -0
  144. package/src/gaussians/adam.ts +131 -0
  145. package/src/gaussians/depth.ts +87 -0
  146. package/src/gaussians/gradients.ts +285 -0
  147. package/src/gaussians/harmonics.ts +122 -0
  148. package/src/gaussians/optimise.ts +191 -0
  149. package/src/gaussians/parameters.ts +186 -0
  150. package/src/gaussians/project.ts +207 -0
  151. package/src/gaussians/projectGradient.ts +234 -0
  152. package/src/gaussians/rasterise.ts +81 -0
  153. package/src/gaussians/refine.ts +176 -0
  154. package/src/gaussians/similarity.ts +87 -0
  155. package/src/index.ts +225 -0
  156. package/src/marching.ts +358 -0
  157. package/src/masks.ts +201 -0
  158. package/src/math/dense.ts +317 -0
  159. package/src/math/schur.ts +167 -0
  160. package/src/models/batchNorm.ts +57 -0
  161. package/src/models/clipTokenizer.ts +129 -0
  162. package/src/models/clipTower.ts +87 -0
  163. package/src/models/depthAnything2.ts +276 -0
  164. package/src/models/depthAnything3.ts +229 -0
  165. package/src/models/dualDpt.ts +233 -0
  166. package/src/models/hiera.ts +231 -0
  167. package/src/models/hieraAttention.ts +121 -0
  168. package/src/models/miniature.ts +241 -0
  169. package/src/models/mobileSam.ts +135 -0
  170. package/src/models/owlv2.ts +190 -0
  171. package/src/models/owlv2Detect.ts +116 -0
  172. package/src/models/owlv2Miniature.ts +108 -0
  173. package/src/models/rope.ts +111 -0
  174. package/src/models/sam21.ts +182 -0
  175. package/src/models/sam21Miniature.ts +225 -0
  176. package/src/models/sam21Video.ts +154 -0
  177. package/src/models/sam2Choice.ts +62 -0
  178. package/src/models/sam2Decoder.ts +218 -0
  179. package/src/models/sam2Memories.ts +165 -0
  180. package/src/models/sam2Memory.ts +247 -0
  181. package/src/models/sam2Positions.ts +106 -0
  182. package/src/models/sam2Tracker.ts +300 -0
  183. package/src/models/samDecoder.ts +225 -0
  184. package/src/models/samMiniature.ts +190 -0
  185. package/src/models/samPrompt.ts +227 -0
  186. package/src/models/seeded.ts +50 -0
  187. package/src/models/tinyVit.ts +178 -0
  188. package/src/models/twoWay.ts +122 -0
  189. package/src/models/vit.ts +265 -0
  190. package/src/models/windowAttention.ts +155 -0
  191. package/src/poses.ts +894 -0
  192. package/src/prepare.ts +306 -0
  193. package/src/propose.ts +96 -0
  194. package/src/resample.ts +197 -0
  195. package/src/run.ts +7 -0
  196. package/src/scene.ts +141 -0
  197. package/src/segment.ts +220 -0
  198. package/src/testScene.ts +283 -0
  199. package/src/twoView.ts +536 -0
@@ -0,0 +1,30 @@
1
+ export declare const DEPTH_PIXEL_MEAN: readonly [0.485, 0.456, 0.406];
2
+ export declare const DEPTH_PIXEL_STD: readonly [0.229, 0.224, 0.225];
3
+ export interface PreparedFrame {
4
+ /** `[3, height, width]`, normalised. */
5
+ readonly pixels: Float32Array;
6
+ readonly width: number;
7
+ readonly height: number;
8
+ }
9
+ /**
10
+ * `rgba`, `width × height` pixels of four bytes, as Depth Anything 3 takes it at `size`, in whole
11
+ * patches of `patch`.
12
+ */
13
+ export declare function prepareDepthFrame(rgba: Uint8Array, width: number, height: number, size?: number, patch?: number): PreparedFrame;
14
+ /** The size a frame is resized to, its longest side at `size`, rounded up at a half. */
15
+ export declare function longestSideSize(width: number, height: number, size: number): {
16
+ readonly width: number;
17
+ readonly height: number;
18
+ };
19
+ /**
20
+ * `rgba` as MobileSAM's predictor prepares it: `[3, square, square]`, the frame itself in the top
21
+ * left at the size this answers, and zeros after it.
22
+ */
23
+ export declare function prepareSamFrame(rgba: Uint8Array, width: number, height: number, mean: readonly number[], deviation: readonly number[], square?: number): PreparedFrame;
24
+ /**
25
+ * `rgba` as SAM 2's processor prepares it: `[3, square, square]`, each axis scaled on its own, and
26
+ * normalised by `mean` and `deviation` in 0–1.
27
+ */
28
+ export declare function prepareSam2Frame(rgba: Uint8Array, width: number, height: number, mean?: readonly number[], deviation?: readonly number[], square?: number): PreparedFrame;
29
+ /** `rgba` as OWLv2's processor prepares it: `[3, square, square]` at the model's size. */
30
+ export declare function prepareOwlv2Frame(rgba: Uint8Array, width: number, height: number, mean: readonly number[], deviation: readonly number[], square?: number): PreparedFrame;
@@ -0,0 +1,238 @@
1
+ /**
2
+ * A frame as a model takes it: the preparation each upstream does to its own inputs, done here so a
3
+ * host hands the engine pixels and nothing else.
4
+ *
5
+ * **Depth Anything 3**, as its `InputProcessor` prepares a frame: the longest side brought to 504
6
+ * by an area resize, each side then rounded to the nearest whole patch of 14 by another, the bytes
7
+ * divided by 255 and normalised by ImageNet's mean and deviation, and laid out channel by row. A
8
+ * clip's aspect ratio decides the grid, which is why the graph is rebuilt per size.
9
+ *
10
+ * **MobileSAM**, as its predictor prepares one: the longest side to 1,024 by Pillow's bilinear,
11
+ * normalised by the upstream's pixel mean and deviation in 0–255, and padded with zeros after its
12
+ * last row and column to the square the encoder takes. The prepared frame's own size is answered
13
+ * too, since a prompt's coordinates are in it and a mask comes back cropped to it.
14
+ *
15
+ * **SAM 2.1**, as its processor prepares one: the frame to a 1,024 square whatever its shape —
16
+ * each axis scaled by its own factor, which is why a prompt's coordinates scale by their own too —
17
+ * through torchvision's antialiased bilinear, divided by 255 and normalised by ImageNet's.
18
+ *
19
+ * **OWLv2**, as its processor prepares one, which is the odd one: the bytes divided by 255 first,
20
+ * the frame padded with zeros after its last row and column to a square — so a box's coordinates
21
+ * scale by the longer side — blurred by a gaussian wide enough for the reduction, resized to 960
22
+ * by a plain bilinear with no antialiasing of its own, and normalised by CLIP's statistics. The
23
+ * blur is what the antialiasing is here, which is why the resize after it has none.
24
+ *
25
+ * **A frame smaller than the model's size is refused rather than guessed at.** The upstream scales
26
+ * such a frame up with a cubic resize, which this does not have; a wrong preparation is a wrong
27
+ * depth that looks like a right one, and a refusal names what is missing. What would lift it is
28
+ * OpenCV's `INTER_CUBIC`, written and held to the same parity as the area resize.
29
+ */
30
+ import { exactExp } from '@driftengine/core';
31
+ import { resize } from '@driftengine/texture';
32
+ import { areaResize, triangleResize, TORCHVISION_PRECISION } from './resample.js';
33
+ export const DEPTH_PIXEL_MEAN = [0.485, 0.456, 0.406];
34
+ export const DEPTH_PIXEL_STD = [0.229, 0.224, 0.225];
35
+ /** Python's `round`: halves to even, which is what the upstream's sizes are rounded by. */
36
+ function halfToEven(value) {
37
+ const down = Math.floor(value);
38
+ const rest = value - down;
39
+ if (rest > 0.5)
40
+ return down + 1;
41
+ if (rest < 0.5)
42
+ return down;
43
+ return down % 2 === 0 ? down : down + 1;
44
+ }
45
+ /** The nearest multiple of `patch`, the larger one on a tie, as the upstream rounds it. */
46
+ function nearestMultiple(value, patch) {
47
+ const down = Math.floor(value / patch) * patch;
48
+ const up = down + patch;
49
+ return up - value <= value - down ? up : down;
50
+ }
51
+ /**
52
+ * `rgba`, `width × height` pixels of four bytes, as Depth Anything 3 takes it at `size`, in whole
53
+ * patches of `patch`.
54
+ */
55
+ export function prepareDepthFrame(rgba, width, height, size = 504, patch = 14) {
56
+ const longest = Math.max(width, height);
57
+ if (longest < size) {
58
+ throw new RangeError(`a frame of ${width}×${height} is smaller than the model's ${size} and would have to be ` +
59
+ 'scaled up, which needs the cubic resize this does not have');
60
+ }
61
+ const scale = size / longest;
62
+ let held = rgba;
63
+ let at = { width, height };
64
+ if (longest !== size) {
65
+ at = {
66
+ width: Math.max(1, halfToEven(width * scale)),
67
+ height: Math.max(1, halfToEven(height * scale)),
68
+ };
69
+ held = new Uint8Array(at.width * at.height * 4);
70
+ areaResize(rgba, width, height, 4, held, at.width, at.height);
71
+ }
72
+ const whole = {
73
+ width: Math.max(1, nearestMultiple(at.width, patch)),
74
+ height: Math.max(1, nearestMultiple(at.height, patch)),
75
+ };
76
+ if (whole.width > at.width || whole.height > at.height) {
77
+ throw new RangeError(`a frame of ${at.width}×${at.height} rounds up to ${whole.width}×${whole.height} whole ` +
78
+ 'patches, which needs the cubic resize this does not have');
79
+ }
80
+ if (whole.width !== at.width || whole.height !== at.height) {
81
+ const rounded = new Uint8Array(whole.width * whole.height * 4);
82
+ areaResize(held, at.width, at.height, 4, rounded, whole.width, whole.height);
83
+ held = rounded;
84
+ at = whole;
85
+ }
86
+ const pixels = new Float32Array(3 * at.width * at.height);
87
+ const cells = at.width * at.height;
88
+ for (let channel = 0; channel < 3; channel += 1) {
89
+ const mean = DEPTH_PIXEL_MEAN[channel];
90
+ const deviation = DEPTH_PIXEL_STD[channel];
91
+ for (let cell = 0; cell < cells; cell += 1) {
92
+ const value = held[cell * 4 + channel] / 255;
93
+ pixels[channel * cells + cell] = Math.fround((Math.fround(value) - mean) / deviation);
94
+ }
95
+ }
96
+ return { pixels, width: at.width, height: at.height };
97
+ }
98
+ /** The size a frame is resized to, its longest side at `size`, rounded up at a half. */
99
+ export function longestSideSize(width, height, size) {
100
+ const scale = size / Math.max(width, height);
101
+ return {
102
+ width: Math.trunc(width * scale + 0.5),
103
+ height: Math.trunc(height * scale + 0.5),
104
+ };
105
+ }
106
+ /**
107
+ * `rgba` as MobileSAM's predictor prepares it: `[3, square, square]`, the frame itself in the top
108
+ * left at the size this answers, and zeros after it.
109
+ */
110
+ export function prepareSamFrame(rgba, width, height, mean, deviation, square = 1024) {
111
+ const at = longestSideSize(width, height, square);
112
+ let held = rgba;
113
+ if (at.width !== width || at.height !== height) {
114
+ held = new Uint8Array(at.width * at.height * 4);
115
+ triangleResize(rgba, width, height, 4, held, at.width, at.height);
116
+ }
117
+ const pixels = new Float32Array(3 * square * square);
118
+ for (let channel = 0; channel < 3; channel += 1) {
119
+ const centre = mean[channel];
120
+ const spread = deviation[channel];
121
+ for (let y = 0; y < at.height; y += 1) {
122
+ for (let x = 0; x < at.width; x += 1) {
123
+ const value = held[(y * at.width + x) * 4 + channel];
124
+ pixels[channel * square * square + y * square + x] = Math.fround((value - centre) / spread);
125
+ }
126
+ }
127
+ }
128
+ return { pixels, width: at.width, height: at.height };
129
+ }
130
+ /**
131
+ * `rgba` as SAM 2's processor prepares it: `[3, square, square]`, each axis scaled on its own, and
132
+ * normalised by `mean` and `deviation` in 0–1.
133
+ */
134
+ export function prepareSam2Frame(rgba, width, height, mean = DEPTH_PIXEL_MEAN, deviation = DEPTH_PIXEL_STD, square = 1024) {
135
+ let held = rgba;
136
+ if (width !== square || height !== square) {
137
+ held = new Uint8Array(square * square * 4);
138
+ triangleResize(rgba, width, height, 4, held, square, square, TORCHVISION_PRECISION);
139
+ }
140
+ const pixels = new Float32Array(3 * square * square);
141
+ const cells = square * square;
142
+ for (let channel = 0; channel < 3; channel += 1) {
143
+ const centre = mean[channel];
144
+ const spread = deviation[channel];
145
+ for (let cell = 0; cell < cells; cell += 1) {
146
+ const value = Math.fround(held[cell * 4 + channel] / 255);
147
+ pixels[channel * cells + cell] = Math.fround((value - centre) / spread);
148
+ }
149
+ }
150
+ return { pixels, width: square, height: square };
151
+ }
152
+ /* torchvision's 1-D gaussian: samples about the centre, normalised to sum to one. */
153
+ function gaussianKernel(size, sigma) {
154
+ const half = (size - 1) * 0.5;
155
+ const kernel = new Float32Array(size);
156
+ let total = 0;
157
+ for (let i = 0; i < size; i += 1) {
158
+ /* `linspace` from −half to half, which is how torchvision places the samples. */
159
+ const at = size === 1 ? 0 : -half + (2 * half * i) / (size - 1);
160
+ const value = Math.fround(exactExp(-0.5 * (at / sigma) * (at / sigma)));
161
+ kernel[i] = value;
162
+ total += value;
163
+ }
164
+ for (let i = 0; i < size; i += 1)
165
+ kernel[i] = Math.fround(kernel[i] / total);
166
+ return kernel;
167
+ }
168
+ /* A separable blur with the edges reflected, as torchvision's `gaussian_blur` pads them. */
169
+ function blur(values, channels, height, width, down, across) {
170
+ const reflect = (at, length) => {
171
+ let index = at;
172
+ while (index < 0 || index >= length)
173
+ index = index < 0 ? -index : 2 * length - index - 2;
174
+ return index;
175
+ };
176
+ const middle = new Float32Array(values.length);
177
+ const half = (across.length - 1) / 2;
178
+ for (let c = 0; c < channels; c += 1) {
179
+ for (let y = 0; y < height; y += 1) {
180
+ for (let x = 0; x < width; x += 1) {
181
+ let sum = 0;
182
+ for (let k = 0; k < across.length; k += 1) {
183
+ sum = Math.fround(sum +
184
+ Math.fround(values[(c * height + y) * width + reflect(x + k - half, width)] *
185
+ across[k]));
186
+ }
187
+ middle[(c * height + y) * width + x] = sum;
188
+ }
189
+ }
190
+ }
191
+ const out = new Float32Array(values.length);
192
+ const tall = (down.length - 1) / 2;
193
+ for (let c = 0; c < channels; c += 1) {
194
+ for (let y = 0; y < height; y += 1) {
195
+ for (let x = 0; x < width; x += 1) {
196
+ let sum = 0;
197
+ for (let k = 0; k < down.length; k += 1) {
198
+ sum = Math.fround(sum +
199
+ Math.fround(middle[(c * height + reflect(y + k - tall, height)) * width + x] *
200
+ down[k]));
201
+ }
202
+ out[(c * height + y) * width + x] = sum;
203
+ }
204
+ }
205
+ }
206
+ return out;
207
+ }
208
+ /** `rgba` as OWLv2's processor prepares it: `[3, square, square]` at the model's size. */
209
+ export function prepareOwlv2Frame(rgba, width, height, mean, deviation, square = 960) {
210
+ const side = Math.max(width, height);
211
+ const padded = new Float32Array(3 * side * side);
212
+ for (let channel = 0; channel < 3; channel += 1) {
213
+ for (let y = 0; y < height; y += 1) {
214
+ for (let x = 0; x < width; x += 1) {
215
+ padded[(channel * side + y) * side + x] = Math.fround(rgba[(y * width + x) * 4 + channel] / 255);
216
+ }
217
+ }
218
+ }
219
+ const factor = side / square;
220
+ const sigma = Math.max(0, (factor - 1) / 2);
221
+ let blurred = padded;
222
+ if (sigma > 0) {
223
+ const size = 2 * Math.ceil(3 * sigma) + 1;
224
+ const kernel = gaussianKernel(size, sigma);
225
+ blurred = blur(padded, 3, side, side, kernel, kernel);
226
+ }
227
+ const resized = new Float32Array(3 * square * square);
228
+ resize(resized, blurred, 3, side, side, square, square, 'bilinear', false);
229
+ const cells = square * square;
230
+ for (let channel = 0; channel < 3; channel += 1) {
231
+ const centre = mean[channel];
232
+ const spread = deviation[channel];
233
+ for (let cell = 0; cell < cells; cell += 1) {
234
+ resized[channel * cells + cell] = Math.fround((resized[channel * cells + cell] - centre) / spread);
235
+ }
236
+ }
237
+ return { pixels: resized, width: square, height: square };
238
+ }
@@ -0,0 +1,36 @@
1
+ import type { Region } from './segment.ts';
2
+ export interface ProposeOptions {
3
+ /** The steepest surface a character could stand on, in degrees. */
4
+ readonly slopeDegrees?: number;
5
+ }
6
+ /** What a scene could make of a region. */
7
+ export interface EntityProposal {
8
+ /** Its index in the regions it came from. */
9
+ readonly region: number;
10
+ readonly bounds: Float64Array;
11
+ readonly label: string | null;
12
+ /**
13
+ * Components a scene might give it, by the engine's own names.
14
+ *
15
+ * A *suggestion*, and the word is load-bearing: nothing here knows what the thing is, only what
16
+ * shape it is and whether anybody named it.
17
+ */
18
+ readonly components: readonly string[];
19
+ /** Whether a character could stand on it. */
20
+ readonly walkable: boolean;
21
+ }
22
+ /** Drawn and solid and nothing else, which is what an unlabelled region gets. */
23
+ export declare const SCENERY_COMPONENTS: readonly string[];
24
+ /** The same, and a navigation surface on top of it. */
25
+ export declare const WALKABLE_COMPONENTS: readonly string[];
26
+ /** Something small enough and named: a scene may want to move it. */
27
+ export declare const MOVABLE_COMPONENTS: readonly string[];
28
+ /**
29
+ * Each region as a proposal.
30
+ *
31
+ * **An unlabelled region is scenery**, whatever its shape: drawn, solid, and doing nothing. A named
32
+ * region small enough to pick up is offered as movable, and the floor — the largest surface a
33
+ * character could stand on — is offered with a navigation surface. Nothing here is decided; a
34
+ * consumer reads `components` as a starting point and overrides whatever it likes.
35
+ */
36
+ export declare function proposeEntities(regions: readonly Region[], options?: ProposeOptions): EntityProposal[];
@@ -0,0 +1,63 @@
1
+ /**
2
+ * Each region as something a scene could hold — a *proposal*, and the word is load-bearing.
3
+ *
4
+ * **This is stage four and it is labelled as such.** A capture that guesses what an object is and
5
+ * acts on the guess is a capture that puts a door where a cupboard was, and the cost is paid by
6
+ * somebody who cannot see why. So a region arrives with what is known about it and nothing more,
7
+ * and **a region nobody labelled becomes static scenery** — the safe default in the strict sense:
8
+ * drawn, solid, and doing nothing. Being wrong about that costs nothing at all.
9
+ *
10
+ * The component names are the engine's own, as data. Nothing here imports an entity model, because
11
+ * a proposal a consumer has not accepted yet should not have built anything.
12
+ */
13
+ import { exactCos } from '@driftengine/core';
14
+ const DEFAULT_SLOPE = 45;
15
+ /** Drawn and solid and nothing else, which is what an unlabelled region gets. */
16
+ export const SCENERY_COMPONENTS = ['transform', 'mesh', 'collider'];
17
+ /** The same, and a navigation surface on top of it. */
18
+ export const WALKABLE_COMPONENTS = [
19
+ 'transform',
20
+ 'mesh',
21
+ 'collider',
22
+ 'navigation',
23
+ ];
24
+ /** Something small enough and named: a scene may want to move it. */
25
+ export const MOVABLE_COMPONENTS = ['transform', 'mesh', 'collider', 'body'];
26
+ /**
27
+ * Each region as a proposal.
28
+ *
29
+ * **An unlabelled region is scenery**, whatever its shape: drawn, solid, and doing nothing. A named
30
+ * region small enough to pick up is offered as movable, and the floor — the largest surface a
31
+ * character could stand on — is offered with a navigation surface. Nothing here is decided; a
32
+ * consumer reads `components` as a starting point and overrides whatever it likes.
33
+ */
34
+ export function proposeEntities(regions, options = {}) {
35
+ const slope = exactCos(((options.slopeDegrees ?? DEFAULT_SLOPE) * Math.PI) / 180);
36
+ let floor = -1;
37
+ let widest = 0;
38
+ regions.forEach((region, at) => {
39
+ if (region.normal[1] < slope)
40
+ return;
41
+ if (region.area <= widest)
42
+ return;
43
+ widest = region.area;
44
+ floor = at;
45
+ });
46
+ return regions.map((region, at) => {
47
+ const walkable = at === floor;
48
+ const span = Math.max(region.bounds[3] - region.bounds[0], region.bounds[4] - region.bounds[1], region.bounds[5] - region.bounds[2]);
49
+ /* Named and small enough that a person could shift it: the only case that is not scenery. */
50
+ const movable = region.label !== null && span < 1.2;
51
+ return {
52
+ region: at,
53
+ bounds: region.bounds,
54
+ label: region.label,
55
+ components: walkable
56
+ ? WALKABLE_COMPONENTS
57
+ : movable
58
+ ? MOVABLE_COMPONENTS
59
+ : SCENERY_COMPONENTS,
60
+ walkable,
61
+ };
62
+ });
63
+ }
@@ -0,0 +1,39 @@
1
+ /**
2
+ * The resampling a model's own preparation does, as the library its upstream uses does it.
3
+ *
4
+ * **Area resizing is OpenCV's `INTER_AREA`**, which Depth Anything 3's `InputProcessor` resizes a
5
+ * frame down with: an output pixel is the average of the source rectangle it covers, with the
6
+ * pixels at either end weighted by the fraction of them inside it. It is the right filter for
7
+ * shrinking a photograph — every source pixel reaches the output exactly once, so nothing aliases
8
+ * and nothing is dropped — and it is what the model was trained on, which is the reason that
9
+ * matters here.
10
+ *
11
+ * **The antialiased bilinear is the other one**, which MobileSAM's predictor and SAM 2's processor
12
+ * each resize a frame with: a triangle filter whose support grows with the reduction, so shrinking
13
+ * averages rather than samples. One axis at a time, through an eight-bit intermediate, with
14
+ * coefficients normalised to sum to one and held in fixed point — **and how many fraction bits they
15
+ * keep is the whole difference between the two libraries**: Pillow keeps 22 and torchvision's
16
+ * eight-bit path keeps 15, and at each the engine answers what that library answers, measured on
17
+ * the pinned frames at 0 of 3,145,728 values apart. A coefficient rounded differently is a pixel
18
+ * rounded differently, which is why this is a number rather than an approximation.
19
+ *
20
+ * **The area resize is summed the way OpenCV sums it**: one weight per source pixel of the rectangle, the row's
21
+ * times the column's, accumulated in single precision, and rounded by halves to even as `cvRound`
22
+ * does. Measured against OpenCV on the two pinned frames, that leaves **24 values of 846,720 a
23
+ * single level of 255 apart** — a sum landing either side of a half — where summing in double and
24
+ * rounding halves away from zero leaves 46. What would close the rest is OpenCV's own fixed-point
25
+ * table, which is worth writing when a measurement says a level of 255 matters.
26
+ */
27
+ /**
28
+ * `src`, `width × height` pixels of `channels` bytes, into `out` at `outWidth × outHeight`. Only
29
+ * shrinking is defined, which is what a frame's preparation asks for.
30
+ */
31
+ export declare function areaResize(src: Uint8Array, width: number, height: number, channels: number, out: Uint8Array, outWidth: number, outHeight: number): void;
32
+ /** Pillow's fraction bits, and torchvision's eight-bit path's. */
33
+ export declare const PILLOW_PRECISION = 22;
34
+ export declare const TORCHVISION_PRECISION = 15;
35
+ /**
36
+ * `src` into `out` by the antialiased bilinear: across first, then down, through an eight-bit
37
+ * intermediate, which is the order both libraries' passes have. `precision` says whose rounding.
38
+ */
39
+ export declare function triangleResize(src: Uint8Array, width: number, height: number, channels: number, out: Uint8Array, outWidth: number, outHeight: number, precision?: number): void;
@@ -0,0 +1,165 @@
1
+ /**
2
+ * The resampling a model's own preparation does, as the library its upstream uses does it.
3
+ *
4
+ * **Area resizing is OpenCV's `INTER_AREA`**, which Depth Anything 3's `InputProcessor` resizes a
5
+ * frame down with: an output pixel is the average of the source rectangle it covers, with the
6
+ * pixels at either end weighted by the fraction of them inside it. It is the right filter for
7
+ * shrinking a photograph — every source pixel reaches the output exactly once, so nothing aliases
8
+ * and nothing is dropped — and it is what the model was trained on, which is the reason that
9
+ * matters here.
10
+ *
11
+ * **The antialiased bilinear is the other one**, which MobileSAM's predictor and SAM 2's processor
12
+ * each resize a frame with: a triangle filter whose support grows with the reduction, so shrinking
13
+ * averages rather than samples. One axis at a time, through an eight-bit intermediate, with
14
+ * coefficients normalised to sum to one and held in fixed point — **and how many fraction bits they
15
+ * keep is the whole difference between the two libraries**: Pillow keeps 22 and torchvision's
16
+ * eight-bit path keeps 15, and at each the engine answers what that library answers, measured on
17
+ * the pinned frames at 0 of 3,145,728 values apart. A coefficient rounded differently is a pixel
18
+ * rounded differently, which is why this is a number rather than an approximation.
19
+ *
20
+ * **The area resize is summed the way OpenCV sums it**: one weight per source pixel of the rectangle, the row's
21
+ * times the column's, accumulated in single precision, and rounded by halves to even as `cvRound`
22
+ * does. Measured against OpenCV on the two pinned frames, that leaves **24 values of 846,720 a
23
+ * single level of 255 apart** — a sum landing either side of a half — where summing in double and
24
+ * rounding halves away from zero leaves 46. What would close the rest is OpenCV's own fixed-point
25
+ * table, which is worth writing when a measurement says a level of 255 matters.
26
+ */
27
+ const f = Math.fround;
28
+ /** `cvRound`: halves to even, which is what OpenCV's saturating cast does. */
29
+ function toByte(value) {
30
+ const down = Math.floor(value);
31
+ const rest = value - down;
32
+ const rounded = rest > 0.5 ? down + 1 : rest < 0.5 ? down : down % 2 === 0 ? down : down + 1;
33
+ return Math.min(255, Math.max(0, rounded));
34
+ }
35
+ function spans(from, to) {
36
+ const scale = from / to;
37
+ const out = [];
38
+ for (let at = 0; at < to; at += 1) {
39
+ const start = at * scale;
40
+ const end = start + scale;
41
+ const first = Math.ceil(start);
42
+ const last = Math.floor(end);
43
+ /* The cell is the part of it inside the image, as OpenCV's `resizeArea` measures it. */
44
+ const cell = Math.min(scale, from - start);
45
+ const weights = [];
46
+ if (first > start)
47
+ weights.push((first - start) / cell);
48
+ for (let source = first; source < last; source += 1)
49
+ weights.push(1 / cell);
50
+ if (end > last && last < from)
51
+ weights.push((end - last) / cell);
52
+ out.push({
53
+ first: Math.min(first > start ? first - 1 : first, from - 1),
54
+ weights: Float64Array.from(weights),
55
+ });
56
+ }
57
+ return out;
58
+ }
59
+ /**
60
+ * `src`, `width × height` pixels of `channels` bytes, into `out` at `outWidth × outHeight`. Only
61
+ * shrinking is defined, which is what a frame's preparation asks for.
62
+ */
63
+ export function areaResize(src, width, height, channels, out, outWidth, outHeight) {
64
+ const across = spans(width, outWidth);
65
+ const down = spans(height, outHeight);
66
+ for (let y = 0; y < outHeight; y += 1) {
67
+ const rows = down[y];
68
+ for (let x = 0; x < outWidth; x += 1) {
69
+ const columns = across[x];
70
+ for (let c = 0; c < channels; c += 1) {
71
+ let sum = 0;
72
+ for (let dy = 0; dy < rows.weights.length; dy += 1) {
73
+ const row = (rows.first + dy) * width;
74
+ for (let dx = 0; dx < columns.weights.length; dx += 1) {
75
+ const weight = f(f(rows.weights[dy]) * f(columns.weights[dx]));
76
+ sum = f(sum + f(src[(row + columns.first + dx) * channels + c] * weight));
77
+ }
78
+ }
79
+ out[(y * outWidth + x) * channels + c] = toByte(sum);
80
+ }
81
+ }
82
+ }
83
+ }
84
+ /** Pillow's fraction bits, and torchvision's eight-bit path's. */
85
+ export const PILLOW_PRECISION = 22;
86
+ export const TORCHVISION_PRECISION = 15;
87
+ /** One output pixel's source pixels and their fixed-point weights. */
88
+ function triangleCoefficients(from, to, precision) {
89
+ const scale = from / to;
90
+ const filterScale = Math.max(1, scale);
91
+ const support = filterScale;
92
+ const out = [];
93
+ for (let at = 0; at < to; at += 1) {
94
+ const centre = (at + 0.5) * scale;
95
+ const first = Math.max(0, Math.trunc(centre - support + 0.5));
96
+ const last = Math.min(from, Math.trunc(centre + support + 0.5));
97
+ const weights = [];
98
+ let total = 0;
99
+ for (let source = first; source < last; source += 1) {
100
+ /* The triangle, over a support the reduction widens. */
101
+ const distance = Math.abs((source - centre + 0.5) / filterScale);
102
+ const weight = distance < 1 ? 1 - distance : 0;
103
+ weights.push(weight);
104
+ total += weight;
105
+ }
106
+ const fixed = new Int32Array(weights.length);
107
+ for (let i = 0; i < weights.length; i += 1) {
108
+ const share = total === 0 ? 0 : weights[i] / total;
109
+ /* A coefficient rounds away from zero at this precision. */
110
+ fixed[i] =
111
+ share < 0
112
+ ? -Math.floor(-share * (1 << precision) + 0.5)
113
+ : Math.floor(share * (1 << precision) + 0.5);
114
+ }
115
+ out.push({ first, weights: fixed });
116
+ }
117
+ return out;
118
+ }
119
+ /** The clamp of a fixed-point sum back to a byte. */
120
+ function clip8(sum, precision) {
121
+ const value = Math.floor(sum / (1 << precision));
122
+ return value < 0 ? 0 : value > 255 ? 255 : value;
123
+ }
124
+ /**
125
+ * `src` into `out` by the antialiased bilinear: across first, then down, through an eight-bit
126
+ * intermediate, which is the order both libraries' passes have. `precision` says whose rounding.
127
+ */
128
+ export function triangleResize(src, width, height, channels, out, outWidth, outHeight, precision = PILLOW_PRECISION) {
129
+ const rounding = 1 << (precision - 1);
130
+ const across = triangleCoefficients(width, outWidth, precision);
131
+ const middle = width === outWidth ? src : new Uint8Array(outWidth * height * channels);
132
+ if (middle !== src) {
133
+ for (let y = 0; y < height; y += 1) {
134
+ for (let x = 0; x < outWidth; x += 1) {
135
+ const { first, weights } = across[x];
136
+ for (let c = 0; c < channels; c += 1) {
137
+ let sum = rounding;
138
+ for (let i = 0; i < weights.length; i += 1) {
139
+ sum += src[(y * width + first + i) * channels + c] * weights[i];
140
+ }
141
+ middle[(y * outWidth + x) * channels + c] = clip8(sum, precision);
142
+ }
143
+ }
144
+ }
145
+ }
146
+ const down = triangleCoefficients(height, outHeight, precision);
147
+ if (height === outHeight) {
148
+ out.set(middle.subarray(0, outWidth * outHeight * channels));
149
+ return;
150
+ }
151
+ for (let y = 0; y < outHeight; y += 1) {
152
+ const { first, weights } = down[y];
153
+ for (let x = 0; x < outWidth; x += 1) {
154
+ for (let c = 0; c < channels; c += 1) {
155
+ let sum = rounding;
156
+ for (let i = 0; i < weights.length; i += 1) {
157
+ sum +=
158
+ middle[((first + i) * outWidth + x) * channels + c] *
159
+ weights[i];
160
+ }
161
+ out[(y * outWidth + x) * channels + c] = clip8(sum, precision);
162
+ }
163
+ }
164
+ }
165
+ }
package/dist/run.d.ts ADDED
@@ -0,0 +1,3 @@
1
+ /** How a capture runs a graph: the device's runner, or the reference evaluator. */
2
+ import type { NetworkGraph } from '@driftengine/texture';
3
+ export type GraphRun = (graph: NetworkGraph, inputs: ReadonlyMap<string, Float32Array>) => Promise<ReadonlyMap<string, Float32Array>>;
package/dist/run.js ADDED
@@ -0,0 +1 @@
1
+ export {};