@srsergio/taptapp-ar 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (207) hide show
  1. package/README.md +62 -0
  2. package/dist/compiler/aframe.d.ts +1 -0
  3. package/dist/compiler/aframe.js +275 -0
  4. package/dist/compiler/compiler-base.d.ts +12 -0
  5. package/dist/compiler/compiler-base.js +165 -0
  6. package/dist/compiler/compiler.d.ts +9 -0
  7. package/dist/compiler/compiler.js +24 -0
  8. package/dist/compiler/compiler.worker.d.ts +1 -0
  9. package/dist/compiler/compiler.worker.js +28 -0
  10. package/dist/compiler/controller.d.ts +101 -0
  11. package/dist/compiler/controller.js +400 -0
  12. package/dist/compiler/controller.worker.d.ts +1 -0
  13. package/dist/compiler/controller.worker.js +61 -0
  14. package/dist/compiler/detector/crop-detector.d.ts +65 -0
  15. package/dist/compiler/detector/crop-detector.js +59 -0
  16. package/dist/compiler/detector/detector.d.ts +98 -0
  17. package/dist/compiler/detector/detector.js +1049 -0
  18. package/dist/compiler/detector/freak.d.ts +1 -0
  19. package/dist/compiler/detector/freak.js +89 -0
  20. package/dist/compiler/detector/kernels/cpu/binomialFilter.d.ts +6 -0
  21. package/dist/compiler/detector/kernels/cpu/binomialFilter.js +51 -0
  22. package/dist/compiler/detector/kernels/cpu/buildExtremas.d.ts +6 -0
  23. package/dist/compiler/detector/kernels/cpu/buildExtremas.js +89 -0
  24. package/dist/compiler/detector/kernels/cpu/computeExtremaAngles.d.ts +7 -0
  25. package/dist/compiler/detector/kernels/cpu/computeExtremaAngles.js +79 -0
  26. package/dist/compiler/detector/kernels/cpu/computeExtremaFreak.d.ts +6 -0
  27. package/dist/compiler/detector/kernels/cpu/computeExtremaFreak.js +68 -0
  28. package/dist/compiler/detector/kernels/cpu/computeFreakDescriptors.d.ts +6 -0
  29. package/dist/compiler/detector/kernels/cpu/computeFreakDescriptors.js +57 -0
  30. package/dist/compiler/detector/kernels/cpu/computeLocalization.d.ts +6 -0
  31. package/dist/compiler/detector/kernels/cpu/computeLocalization.js +54 -0
  32. package/dist/compiler/detector/kernels/cpu/computeOrientationHistograms.d.ts +6 -0
  33. package/dist/compiler/detector/kernels/cpu/computeOrientationHistograms.js +118 -0
  34. package/dist/compiler/detector/kernels/cpu/downsampleBilinear.d.ts +6 -0
  35. package/dist/compiler/detector/kernels/cpu/downsampleBilinear.js +29 -0
  36. package/dist/compiler/detector/kernels/cpu/extremaReduction.d.ts +6 -0
  37. package/dist/compiler/detector/kernels/cpu/extremaReduction.js +50 -0
  38. package/dist/compiler/detector/kernels/cpu/fakeShader.d.ts +20 -0
  39. package/dist/compiler/detector/kernels/cpu/fakeShader.js +80 -0
  40. package/dist/compiler/detector/kernels/cpu/index.d.ts +1 -0
  41. package/dist/compiler/detector/kernels/cpu/index.js +25 -0
  42. package/dist/compiler/detector/kernels/cpu/prune.d.ts +1 -0
  43. package/dist/compiler/detector/kernels/cpu/prune.js +103 -0
  44. package/dist/compiler/detector/kernels/cpu/smoothHistograms.d.ts +6 -0
  45. package/dist/compiler/detector/kernels/cpu/smoothHistograms.js +47 -0
  46. package/dist/compiler/detector/kernels/cpu/upsampleBilinear.d.ts +6 -0
  47. package/dist/compiler/detector/kernels/cpu/upsampleBilinear.js +43 -0
  48. package/dist/compiler/detector/kernels/index.d.ts +1 -0
  49. package/dist/compiler/detector/kernels/index.js +2 -0
  50. package/dist/compiler/detector/kernels/webgl/binomialFilter.d.ts +6 -0
  51. package/dist/compiler/detector/kernels/webgl/binomialFilter.js +67 -0
  52. package/dist/compiler/detector/kernels/webgl/buildExtremas.d.ts +6 -0
  53. package/dist/compiler/detector/kernels/webgl/buildExtremas.js +101 -0
  54. package/dist/compiler/detector/kernels/webgl/computeExtremaAngles.d.ts +6 -0
  55. package/dist/compiler/detector/kernels/webgl/computeExtremaAngles.js +78 -0
  56. package/dist/compiler/detector/kernels/webgl/computeExtremaFreak.d.ts +6 -0
  57. package/dist/compiler/detector/kernels/webgl/computeExtremaFreak.js +86 -0
  58. package/dist/compiler/detector/kernels/webgl/computeFreakDescriptors.d.ts +6 -0
  59. package/dist/compiler/detector/kernels/webgl/computeFreakDescriptors.js +52 -0
  60. package/dist/compiler/detector/kernels/webgl/computeLocalization.d.ts +6 -0
  61. package/dist/compiler/detector/kernels/webgl/computeLocalization.js +58 -0
  62. package/dist/compiler/detector/kernels/webgl/computeOrientationHistograms.d.ts +6 -0
  63. package/dist/compiler/detector/kernels/webgl/computeOrientationHistograms.js +116 -0
  64. package/dist/compiler/detector/kernels/webgl/downsampleBilinear.d.ts +6 -0
  65. package/dist/compiler/detector/kernels/webgl/downsampleBilinear.js +46 -0
  66. package/dist/compiler/detector/kernels/webgl/extremaReduction.d.ts +6 -0
  67. package/dist/compiler/detector/kernels/webgl/extremaReduction.js +48 -0
  68. package/dist/compiler/detector/kernels/webgl/index.d.ts +1 -0
  69. package/dist/compiler/detector/kernels/webgl/index.js +25 -0
  70. package/dist/compiler/detector/kernels/webgl/smoothHistograms.d.ts +6 -0
  71. package/dist/compiler/detector/kernels/webgl/smoothHistograms.js +49 -0
  72. package/dist/compiler/detector/kernels/webgl/upsampleBilinear.d.ts +6 -0
  73. package/dist/compiler/detector/kernels/webgl/upsampleBilinear.js +56 -0
  74. package/dist/compiler/estimation/esimate-experiment.d.ts +5 -0
  75. package/dist/compiler/estimation/esimate-experiment.js +267 -0
  76. package/dist/compiler/estimation/estimate.d.ts +5 -0
  77. package/dist/compiler/estimation/estimate.js +51 -0
  78. package/dist/compiler/estimation/estimator.d.ts +13 -0
  79. package/dist/compiler/estimation/estimator.js +30 -0
  80. package/dist/compiler/estimation/refine-estimate-experiment.d.ts +6 -0
  81. package/dist/compiler/estimation/refine-estimate-experiment.js +429 -0
  82. package/dist/compiler/estimation/refine-estimate.d.ts +6 -0
  83. package/dist/compiler/estimation/refine-estimate.js +299 -0
  84. package/dist/compiler/estimation/utils.d.ts +10 -0
  85. package/dist/compiler/estimation/utils.js +80 -0
  86. package/dist/compiler/image-list.d.ts +13 -0
  87. package/dist/compiler/image-list.js +52 -0
  88. package/dist/compiler/index.d.ts +3 -0
  89. package/dist/compiler/index.js +10 -0
  90. package/dist/compiler/input-loader.d.ts +23 -0
  91. package/dist/compiler/input-loader.js +88 -0
  92. package/dist/compiler/matching/hamming-distance.d.ts +1 -0
  93. package/dist/compiler/matching/hamming-distance.js +20 -0
  94. package/dist/compiler/matching/hierarchical-clustering.d.ts +7 -0
  95. package/dist/compiler/matching/hierarchical-clustering.js +109 -0
  96. package/dist/compiler/matching/hough.d.ts +1 -0
  97. package/dist/compiler/matching/hough.js +169 -0
  98. package/dist/compiler/matching/matcher.d.ts +28 -0
  99. package/dist/compiler/matching/matcher.js +48 -0
  100. package/dist/compiler/matching/matching.d.ts +41 -0
  101. package/dist/compiler/matching/matching.js +197 -0
  102. package/dist/compiler/matching/ransacHomography.d.ts +1 -0
  103. package/dist/compiler/matching/ransacHomography.js +136 -0
  104. package/dist/compiler/offline-compiler.d.ts +10 -0
  105. package/dist/compiler/offline-compiler.js +450 -0
  106. package/dist/compiler/tensorflow-setup.d.ts +7 -0
  107. package/dist/compiler/tensorflow-setup.js +73 -0
  108. package/dist/compiler/three.d.ts +66 -0
  109. package/dist/compiler/three.js +310 -0
  110. package/dist/compiler/tracker/extract-utils.d.ts +1 -0
  111. package/dist/compiler/tracker/extract-utils.js +29 -0
  112. package/dist/compiler/tracker/extract.d.ts +4 -0
  113. package/dist/compiler/tracker/extract.js +349 -0
  114. package/dist/compiler/tracker/tracker.d.ts +38 -0
  115. package/dist/compiler/tracker/tracker.js +327 -0
  116. package/dist/compiler/utils/cumsum.d.ts +5 -0
  117. package/dist/compiler/utils/cumsum.js +39 -0
  118. package/dist/compiler/utils/geometry.d.ts +8 -0
  119. package/dist/compiler/utils/geometry.js +101 -0
  120. package/dist/compiler/utils/homography.d.ts +1 -0
  121. package/dist/compiler/utils/homography.js +138 -0
  122. package/dist/compiler/utils/images.d.ts +24 -0
  123. package/dist/compiler/utils/images.js +99 -0
  124. package/dist/compiler/utils/randomizer.d.ts +5 -0
  125. package/dist/compiler/utils/randomizer.js +25 -0
  126. package/dist/index.d.ts +6 -0
  127. package/dist/index.js +7 -0
  128. package/dist/react/AREditor.d.ts +5 -0
  129. package/dist/react/AREditor.js +159 -0
  130. package/dist/react/ProgressDialog.d.ts +13 -0
  131. package/dist/react/ProgressDialog.js +57 -0
  132. package/dist/react/types.d.ts +22 -0
  133. package/dist/react/types.js +14 -0
  134. package/package.json +53 -0
  135. package/src/astro/ARScene.astro +59 -0
  136. package/src/astro/ARVideoTrigger.astro +73 -0
  137. package/src/astro/overlays/ErrorOverlay.astro +40 -0
  138. package/src/astro/overlays/LoadingOverlay.astro +28 -0
  139. package/src/astro/overlays/ScanningOverlay.astro +119 -0
  140. package/src/astro/scripts/ARScripts.astro +118 -0
  141. package/src/astro/styles/ARStyles.astro +147 -0
  142. package/src/compiler/aframe.js +343 -0
  143. package/src/compiler/compiler-base.js +195 -0
  144. package/src/compiler/compiler.js +25 -0
  145. package/src/compiler/compiler.worker.js +30 -0
  146. package/src/compiler/controller.js +473 -0
  147. package/src/compiler/controller.worker.js +77 -0
  148. package/src/compiler/detector/crop-detector.js +68 -0
  149. package/src/compiler/detector/detector.js +1130 -0
  150. package/src/compiler/detector/freak.js +91 -0
  151. package/src/compiler/detector/kernels/cpu/binomialFilter.js +59 -0
  152. package/src/compiler/detector/kernels/cpu/buildExtremas.js +108 -0
  153. package/src/compiler/detector/kernels/cpu/computeExtremaAngles.js +91 -0
  154. package/src/compiler/detector/kernels/cpu/computeExtremaFreak.js +92 -0
  155. package/src/compiler/detector/kernels/cpu/computeFreakDescriptors.js +68 -0
  156. package/src/compiler/detector/kernels/cpu/computeLocalization.js +71 -0
  157. package/src/compiler/detector/kernels/cpu/computeOrientationHistograms.js +141 -0
  158. package/src/compiler/detector/kernels/cpu/downsampleBilinear.js +33 -0
  159. package/src/compiler/detector/kernels/cpu/extremaReduction.js +53 -0
  160. package/src/compiler/detector/kernels/cpu/fakeShader.js +88 -0
  161. package/src/compiler/detector/kernels/cpu/index.js +26 -0
  162. package/src/compiler/detector/kernels/cpu/prune.js +114 -0
  163. package/src/compiler/detector/kernels/cpu/smoothHistograms.js +57 -0
  164. package/src/compiler/detector/kernels/cpu/upsampleBilinear.js +51 -0
  165. package/src/compiler/detector/kernels/index.js +2 -0
  166. package/src/compiler/detector/kernels/webgl/binomialFilter.js +72 -0
  167. package/src/compiler/detector/kernels/webgl/buildExtremas.js +109 -0
  168. package/src/compiler/detector/kernels/webgl/computeExtremaAngles.js +82 -0
  169. package/src/compiler/detector/kernels/webgl/computeExtremaFreak.js +105 -0
  170. package/src/compiler/detector/kernels/webgl/computeFreakDescriptors.js +56 -0
  171. package/src/compiler/detector/kernels/webgl/computeLocalization.js +70 -0
  172. package/src/compiler/detector/kernels/webgl/computeOrientationHistograms.js +129 -0
  173. package/src/compiler/detector/kernels/webgl/downsampleBilinear.js +50 -0
  174. package/src/compiler/detector/kernels/webgl/extremaReduction.js +50 -0
  175. package/src/compiler/detector/kernels/webgl/index.js +26 -0
  176. package/src/compiler/detector/kernels/webgl/smoothHistograms.js +53 -0
  177. package/src/compiler/detector/kernels/webgl/upsampleBilinear.js +62 -0
  178. package/src/compiler/estimation/esimate-experiment.js +316 -0
  179. package/src/compiler/estimation/estimate.js +67 -0
  180. package/src/compiler/estimation/estimator.js +34 -0
  181. package/src/compiler/estimation/refine-estimate-experiment.js +512 -0
  182. package/src/compiler/estimation/refine-estimate.js +365 -0
  183. package/src/compiler/estimation/utils.js +97 -0
  184. package/src/compiler/image-list.js +62 -0
  185. package/src/compiler/index.js +13 -0
  186. package/src/compiler/input-loader.js +107 -0
  187. package/src/compiler/matching/hamming-distance.js +23 -0
  188. package/src/compiler/matching/hierarchical-clustering.js +131 -0
  189. package/src/compiler/matching/hough.js +206 -0
  190. package/src/compiler/matching/matcher.js +59 -0
  191. package/src/compiler/matching/matching.js +237 -0
  192. package/src/compiler/matching/ransacHomography.js +192 -0
  193. package/src/compiler/offline-compiler.js +553 -0
  194. package/src/compiler/tensorflow-setup.js +88 -0
  195. package/src/compiler/three.js +368 -0
  196. package/src/compiler/tracker/extract-utils.js +34 -0
  197. package/src/compiler/tracker/extract.js +419 -0
  198. package/src/compiler/tracker/tracker.js +397 -0
  199. package/src/compiler/utils/cumsum.js +40 -0
  200. package/src/compiler/utils/geometry.js +114 -0
  201. package/src/compiler/utils/homography.js +150 -0
  202. package/src/compiler/utils/images.js +111 -0
  203. package/src/compiler/utils/randomizer.js +29 -0
  204. package/src/index.ts +8 -0
  205. package/src/react/AREditor.tsx +394 -0
  206. package/src/react/ProgressDialog.tsx +185 -0
  207. package/src/react/types.ts +35 -0
@@ -0,0 +1,109 @@
1
+ import { engine } from "@tensorflow/tfjs";
2
+
3
+ const FREAK_EXPANSION_FACTOR = 7.0;
4
+
5
+ const LAPLACIAN_THRESHOLD = 3.0;
6
+ const LAPLACIAN_SQR_THRESHOLD = LAPLACIAN_THRESHOLD * LAPLACIAN_THRESHOLD;
7
+
8
+ const EDGE_THRESHOLD = 4.0;
9
+ const EDGE_HESSIAN_THRESHOLD = ((EDGE_THRESHOLD + 1) * (EDGE_THRESHOLD + 1)) / EDGE_THRESHOLD;
10
+
11
+ const cache = {};
12
+ function GetProgram(image) {
13
+ const imageWidth = image.shape[1];
14
+ const imageHeight = image.shape[0];
15
+ const kernelKey = "w" + imageWidth + "h" + imageHeight;
16
+ if (!cache.hasOwnProperty(kernelKey)) {
17
+ const kernel = {
18
+ variableNames: ["image0", "image1", "image2"],
19
+ outputShape: [imageHeight, imageWidth],
20
+ userCode: `
21
+ void main() {
22
+ ivec2 coords = getOutputCoords();
23
+
24
+ int y = coords[0];
25
+ int x = coords[1];
26
+
27
+ float value = getImage1(y, x);
28
+
29
+ // Step 1: find local maxima/minima
30
+ if (value * value < ${LAPLACIAN_SQR_THRESHOLD}.) {
31
+ setOutput(0.);
32
+ return;
33
+ }
34
+ if (y < ${FREAK_EXPANSION_FACTOR} || y > ${imageHeight - 1 - FREAK_EXPANSION_FACTOR}) {
35
+ setOutput(0.);
36
+ return;
37
+ }
38
+ if (x < ${FREAK_EXPANSION_FACTOR} || x > ${imageWidth - 1 - FREAK_EXPANSION_FACTOR}) {
39
+ setOutput(0.);
40
+ return;
41
+ }
42
+
43
+ bool isMax = true;
44
+ bool isMin = true;
45
+ for (int dy = -1; dy <= 1; dy++) {
46
+ for (int dx = -1; dx <= 1; dx++) {
47
+ float value0 = getImage0(y+dy, x+dx);
48
+ float value1 = getImage1(y+dy, x+dx);
49
+ float value2 = getImage2(y+dy, x+dx);
50
+
51
+ if (value < value0 || value < value1 || value < value2) {
52
+ isMax = false;
53
+ }
54
+ if (value > value0 || value > value1 || value > value2) {
55
+ isMin = false;
56
+ }
57
+ }
58
+ }
59
+
60
+ if (!isMax && !isMin) {
61
+ setOutput(0.);
62
+ return;
63
+ }
64
+
65
+ // compute edge score and reject based on threshold
66
+ float dxx = getImage1(y, x+1) + getImage1(y, x-1) - 2. * getImage1(y, x);
67
+ float dyy = getImage1(y+1, x) + getImage1(y-1, x) - 2. * getImage1(y, x);
68
+ float dxy = 0.25 * (getImage1(y-1,x-1) + getImage1(y+1,x+1) - getImage1(y-1,x+1) - getImage1(y+1,x-1));
69
+
70
+ float det = (dxx * dyy) - (dxy * dxy);
71
+
72
+ if (abs(det) < 0.0001) { // determinant undefined. no solution
73
+ setOutput(0.);
74
+ return;
75
+ }
76
+
77
+ float edgeScore = (dxx + dyy) * (dxx + dyy) / det;
78
+
79
+ if (abs(edgeScore) >= ${EDGE_HESSIAN_THRESHOLD} ) {
80
+ setOutput(0.);
81
+ return;
82
+ }
83
+ setOutput(getImage1(y,x));
84
+ }
85
+ `,
86
+ };
87
+ cache[kernelKey] = kernel;
88
+ }
89
+ return cache[kernelKey];
90
+ }
91
+
92
+ export const buildExtremas = (args) => {
93
+ let { image0, image1, image2 } = args.inputs;
94
+ /** @type {MathBackendWebGL} */
95
+ const backend = args.backend;
96
+
97
+ const program = GetProgram(image1);
98
+
99
+ image0 = engine().runKernel("DownsampleBilinear", { image: image0 });
100
+ image2 = engine().runKernel("UpsampleBilinear", { image: image2, targetImage: image1 });
101
+ return backend.runWebGLProgram(program, [image0, image1, image2], image1.dtype);
102
+ };
103
+
104
+ export const buildExtremasConfig = {
105
+ //: KernelConfig
106
+ kernelName: "BuildExtremas",
107
+ backendName: "webgl",
108
+ kernelFunc: buildExtremas, // as {} as KernelFunc,
109
+ };
@@ -0,0 +1,82 @@
1
+ const ORIENTATION_NUM_BINS = 36;
2
+
3
+ const cache = {};
4
+ function GetProgram(histograms) {
5
+ const key = histograms.shape[0];
6
+ if (!cache.hasOwnProperty(key)) {
7
+ const kernel = {
8
+ variableNames: ["histogram"],
9
+ outputShape: [histograms.shape[0]],
10
+ userCode: `
11
+ void main() {
12
+ int featureIndex = getOutputCoords();
13
+
14
+ int maxIndex = 0;
15
+ for (int i = 1; i < ${ORIENTATION_NUM_BINS}; i++) {
16
+ if (getHistogram(featureIndex, i) > getHistogram(featureIndex, maxIndex)) {
17
+ maxIndex = i;
18
+ }
19
+ }
20
+
21
+ int prev = imod(maxIndex - 1 + ${ORIENTATION_NUM_BINS}, ${ORIENTATION_NUM_BINS});
22
+ int next = imod(maxIndex + 1, ${ORIENTATION_NUM_BINS});
23
+
24
+ /**
25
+ * Fit a quatratic to 3 points. The system of equations is:
26
+ *
27
+ * y0 = A*x0^2 + B*x0 + C
28
+ * y1 = A*x1^2 + B*x1 + C
29
+ * y2 = A*x2^2 + B*x2 + C
30
+ *
31
+ * This system of equations is solved for A,B,C.
32
+ */
33
+ float p10 = float(maxIndex - 1);
34
+ float p11 = getHistogram(featureIndex, prev);
35
+ float p20 = float(maxIndex);
36
+ float p21 = getHistogram(featureIndex, maxIndex);
37
+ float p30 = float(maxIndex + 1);
38
+ float p31 = getHistogram(featureIndex, next);
39
+
40
+ float d1 = (p30-p20)*(p30-p10);
41
+ float d2 = (p10-p20)*(p30-p10);
42
+ float d3 = p10-p20;
43
+
44
+ // If any of the denominators are zero then, just use maxIndex.
45
+ float fbin = float(maxIndex);
46
+ if ( abs(d1) > 0.00001 && abs(d2) > 0.00001 && abs(d3) > 0.00001) {
47
+ float a = p10*p10;
48
+ float b = p20*p20;
49
+
50
+ // Solve for the coefficients A,B,C
51
+ float A = ((p31-p21)/d1)-((p11-p21)/d2);
52
+ float B = ((p11-p21)+(A*(b-a)))/d3;
53
+ float C = p11-(A*a)-(B*p10);
54
+ fbin = -B / (2. * A);
55
+ }
56
+
57
+ float an = 2.0 *${Math.PI} * (fbin + 0.5) / ${ORIENTATION_NUM_BINS}. - ${Math.PI};
58
+ setOutput(an);
59
+ }
60
+ `,
61
+ };
62
+ cache[key] = kernel;
63
+ }
64
+ return cache[key];
65
+ }
66
+
67
+ export const computeExtremaAngles = (args) => {
68
+ /** @type {import('@tensorflow/tfjs').TensorInfo} */
69
+ const { histograms } = args.inputs;
70
+ /** @type {MathBackendWebGL} */
71
+ const backend = args.backend;
72
+
73
+ const program = GetProgram(histograms);
74
+ return backend.runWebGLProgram(program, [histograms], histograms.dtype);
75
+ };
76
+
77
+ export const computeExtremaAnglesConfig = {
78
+ //: KernelConfig
79
+ kernelName: "ComputeExtremaAngles",
80
+ backendName: "webgl",
81
+ kernelFunc: computeExtremaAngles, // as {} as KernelFunc,
82
+ };
@@ -0,0 +1,105 @@
1
+ import { FREAKPOINTS } from "../../freak.js";
2
+
3
+ const FREAK_EXPANSION_FACTOR = 7.0;
4
+ const cache = {};
5
+ function GetProgram(imageCount, prunedExtremas) {
6
+ const key = `${imageCount}|${prunedExtremas.shape[0]}`;
7
+ if (!cache.hasOwnProperty(key)) {
8
+ const imageVariableNames = [];
9
+ for (let i = 1; i < imageCount; i++) {
10
+ imageVariableNames.push("image" + i);
11
+ }
12
+
13
+ let pixelsSubCodes = `float getPixel(int octave, int y, int x) {`;
14
+ for (let i = 1; i < imageCount; i++) {
15
+ pixelsSubCodes += `
16
+ if (octave == ${i}) {
17
+ return getImage${i}(y, x);
18
+ }
19
+ `;
20
+ }
21
+ pixelsSubCodes += `}`;
22
+
23
+ const kernel = {
24
+ variableNames: [...imageVariableNames, "extrema", "angles", "freakPoints"],
25
+ outputShape: [prunedExtremas.shape[0], FREAKPOINTS.length],
26
+ userCode: `
27
+ ${pixelsSubCodes}
28
+ void main() {
29
+ ivec2 coords = getOutputCoords();
30
+ int featureIndex = coords[0];
31
+ int freakIndex = coords[1];
32
+
33
+ float freakSigma = getFreakPoints(freakIndex, 0);
34
+ float freakX = getFreakPoints(freakIndex, 1);
35
+ float freakY = getFreakPoints(freakIndex, 2);
36
+
37
+ int octave = int(getExtrema(featureIndex, 1));
38
+ float inputY = getExtrema(featureIndex, 2);
39
+ float inputX = getExtrema(featureIndex, 3);
40
+ float inputAngle = getAngles(featureIndex);
41
+ float cos = ${FREAK_EXPANSION_FACTOR}. * cos(inputAngle);
42
+ float sin = ${FREAK_EXPANSION_FACTOR}. * sin(inputAngle);
43
+
44
+ float yp = inputY + freakX * sin + freakY * cos;
45
+ float xp = inputX + freakX * cos + freakY * -sin;
46
+
47
+ int x0 = int(floor(xp));
48
+ int x1 = x0 + 1;
49
+ int y0 = int(floor(yp));
50
+ int y1 = y0 + 1;
51
+
52
+ float f1 = getPixel(octave, y0, x0);
53
+ float f2 = getPixel(octave, y0, x1);
54
+ float f3 = getPixel(octave, y1, x0);
55
+ float f4 = getPixel(octave, y1, x1);
56
+
57
+ float x1f = float(x1);
58
+ float y1f = float(y1);
59
+ float x0f = float(x0);
60
+ float y0f = float(y0);
61
+
62
+ // ratio for interpolation between four neighbouring points
63
+ float value = (x1f - xp) * (y1f - yp) * f1
64
+ + (xp - x0f) * (y1f - yp) * f2
65
+ + (x1f - xp) * (yp - y0f) * f3
66
+ + (xp - x0f) * (yp - y0f) * f4;
67
+
68
+ setOutput(value);
69
+ }
70
+ `,
71
+ };
72
+
73
+ cache[key] = kernel;
74
+ }
75
+
76
+ return cache[key];
77
+ }
78
+
79
+ export const computeExtremaFreak = (args) => {
80
+ /** @type {import('@tensorflow/tfjs').TensorInfo} */
81
+ const {
82
+ gaussianImagesT,
83
+ prunedExtremas,
84
+ prunedExtremasAngles,
85
+ freakPointsT,
86
+ pyramidImagesLength,
87
+ } = args.inputs;
88
+ /** @type {MathBackendWebGL} */
89
+ const backend = args.backend;
90
+
91
+ const program = GetProgram(pyramidImagesLength, prunedExtremas);
92
+
93
+ return backend.runWebGLProgram(
94
+ program,
95
+ [...gaussianImagesT, prunedExtremas, prunedExtremasAngles, freakPointsT],
96
+ "float32",
97
+ );
98
+ };
99
+
100
+ export const computeExtremaFreakConfig = {
101
+ //: KernelConfig
102
+ kernelName: "ComputeExtremaFreak",
103
+ backendName: "webgl",
104
+ kernelFunc: computeExtremaFreak, // as {} as KernelFunc,
105
+ };
@@ -0,0 +1,56 @@
1
+ import { FREAKPOINTS } from "../../freak.js";
2
+
3
+ const FREAK_CONPARISON_COUNT = ((FREAKPOINTS.length - 1) * FREAKPOINTS.length) / 2;
4
+ const descriptorCount = Math.ceil(FREAK_CONPARISON_COUNT / 8);
5
+
6
+ const cache = {};
7
+ function GetProgram(extremaFreaks) {
8
+ const key = `${extremaFreaks.shape[0]}`;
9
+ if (!cache.hasOwnProperty(key)) {
10
+ const kernel = {
11
+ variableNames: ["freak", "p"],
12
+ outputShape: [extremaFreaks.shape[0], descriptorCount],
13
+ userCode: `
14
+ void main() {
15
+ ivec2 coords = getOutputCoords();
16
+ int featureIndex = coords[0];
17
+ int descIndex = coords[1] * 8;
18
+
19
+ int sum = 0;
20
+ for (int i = 0; i < 8; i++) {
21
+ if (descIndex + i >= ${FREAK_CONPARISON_COUNT}) {
22
+ continue;
23
+ }
24
+
25
+ int p1 = int(getP(descIndex + i, 0));
26
+ int p2 = int(getP(descIndex + i, 1));
27
+
28
+ float v1 = getFreak(featureIndex, p1);
29
+ float v2 = getFreak(featureIndex, p2);
30
+
31
+ if (v1 < v2 + 0.01) {
32
+ sum += int(pow(2.0, float(7 - i)));
33
+ }
34
+ }
35
+ setOutput(float(sum));
36
+ }
37
+ `,
38
+ };
39
+ cache[key] = kernel;
40
+ }
41
+ return cache[key];
42
+ }
43
+
44
+ export const computeFreakDescriptor = (args) => {
45
+ const { extremaFreaks, positionT } = args.inputs;
46
+ const { backend } = args;
47
+ const program = GetProgram(extremaFreaks);
48
+ return backend.runWebGLProgram(program, [extremaFreaks, positionT], "int32");
49
+ };
50
+
51
+ export const computeFreakDescriptorConfig = {
52
+ //: KernelConfig
53
+ kernelName: "ComputeFreakDescriptors",
54
+ backendName: "webgl",
55
+ kernelFunc: computeFreakDescriptor, // as {} as KernelFunc,
56
+ };
@@ -0,0 +1,70 @@
1
+ import { tensor } from "@tensorflow/tfjs";
2
+
3
+ const cache = {};
4
+ function GetProgram(numDogPyramidImages, extremasListLength) {
5
+ const kernelKey = `${numDogPyramidImages}|${extremasListLength}`;
6
+ if (!cache.hasOwnProperty(kernelKey)) {
7
+ const dogVariableNames = [];
8
+ let dogSubCodes = `float getPixel(int octave, int y, int x) {`;
9
+ for (let i = 1; i < numDogPyramidImages; i++) {
10
+ // extrema starts from second octave
11
+ dogVariableNames.push("image" + i);
12
+ dogSubCodes += `
13
+ if (octave == ${i}) {
14
+ return getImage${i}(y, x);
15
+ }
16
+ `;
17
+ }
18
+ dogSubCodes += `}`;
19
+
20
+ cache[kernelKey] = {
21
+ variableNames: [...dogVariableNames, "extrema"],
22
+ outputShape: [extremasListLength, 3, 3], // 3x3 pixels around the extrema
23
+ userCode: `
24
+ ${dogSubCodes}
25
+
26
+ void main() {
27
+ ivec3 coords = getOutputCoords();
28
+ int featureIndex = coords[0];
29
+ float score = getExtrema(featureIndex, 0);
30
+ if (score == 0.0) {
31
+ return;
32
+ }
33
+
34
+ int dy = coords[1]-1;
35
+ int dx = coords[2]-1;
36
+ int octave = int(getExtrema(featureIndex, 1));
37
+ int y = int(getExtrema(featureIndex, 2));
38
+ int x = int(getExtrema(featureIndex, 3));
39
+ setOutput(getPixel(octave, y+dy, x+dx));
40
+ }
41
+ `,
42
+ };
43
+ }
44
+ return cache[kernelKey];
45
+ }
46
+
47
+ export const computeLocalization = (args) => {
48
+ /** @type {import('@tensorflow/tfjs').TensorInfo} */
49
+ const { prunedExtremasList, dogPyramidImagesT } = args.inputs;
50
+ /** @type {MathBackendWebGL} */
51
+ const backend = args.backend;
52
+ const program = GetProgram(dogPyramidImagesT.length, prunedExtremasList.length);
53
+ const prunedExtremasT = tensor(
54
+ prunedExtremasList,
55
+ [prunedExtremasList.length, prunedExtremasList[0].length],
56
+ "int32",
57
+ );
58
+ return backend.runWebGLProgram(
59
+ program,
60
+ [...dogPyramidImagesT.slice(1), prunedExtremasT],
61
+ dogPyramidImagesT[0].dtype,
62
+ );
63
+ };
64
+
65
+ export const computeLocalizationConfig = {
66
+ //: KernelConfig
67
+ kernelName: "ComputeLocalization",
68
+ backendName: "webgl",
69
+ kernelFunc: computeLocalization, // as {} as KernelFunc,
70
+ };
@@ -0,0 +1,129 @@
1
+ const oneOver2PI = 0.159154943091895;
2
+ const ORIENTATION_NUM_BINS = 36;
3
+
4
+ const cache = {};
5
+
6
+ function GetPrograms(prunedExtremasT, radialPropertiesT, pyramidImagesLength) {
7
+ const key = `${pyramidImagesLength}|${prunedExtremasT.shape[0]}|${radialPropertiesT.shape[0]}`;
8
+ if (!cache.hasOwnProperty(key)) {
9
+ const imageVariableNames = [];
10
+ for (let i = 1; i < pyramidImagesLength; i++) {
11
+ imageVariableNames.push("image" + i);
12
+ }
13
+
14
+ let kernel1SubCodes = `float getPixel(int octave, int y, int x) {`;
15
+ for (let i = 1; i < pyramidImagesLength; i++) {
16
+ kernel1SubCodes += `
17
+ if (octave == ${i}) {
18
+ return getImage${i}(y, x);
19
+ }
20
+ `;
21
+ }
22
+ kernel1SubCodes += `}`;
23
+
24
+ const kernel1 = {
25
+ variableNames: [...imageVariableNames, "extrema", "radial"],
26
+ outputShape: [prunedExtremasT.shape[0], radialPropertiesT.shape[0], 2], // last dimension: [fbin, magnitude]
27
+ userCode: `
28
+ ${kernel1SubCodes}
29
+
30
+ void main() {
31
+ ivec3 coords = getOutputCoords();
32
+ int featureIndex = coords[0];
33
+ int radialIndex = coords[1];
34
+ int propertyIndex = coords[2];
35
+
36
+ int radialY = int(getRadial(radialIndex, 0));
37
+ int radialX = int(getRadial(radialIndex, 1));
38
+ float radialW = getRadial(radialIndex, 2);
39
+
40
+ int octave = int(getExtrema(featureIndex, 1));
41
+ int y = int(getExtrema(featureIndex, 2));
42
+ int x = int(getExtrema(featureIndex, 3));
43
+
44
+ int xp = x + radialX;
45
+ int yp = y + radialY;
46
+
47
+ float dy = getPixel(octave, yp+1, xp) - getPixel(octave, yp-1, xp);
48
+ float dx = getPixel(octave, yp, xp+1) - getPixel(octave, yp, xp-1);
49
+
50
+ if (propertyIndex == 0) {
51
+ // be careful that atan(0, 0) gives 1.57 instead of 0 (different from js), but doesn't matter here, coz magnitude is 0
52
+
53
+ float angle = atan(dy, dx) + ${Math.PI};
54
+ float fbin = angle * ${ORIENTATION_NUM_BINS}. * ${oneOver2PI};
55
+ setOutput(fbin);
56
+ return;
57
+ }
58
+
59
+ if (propertyIndex == 1) {
60
+ float mag = sqrt(dx * dx + dy * dy);
61
+ float magnitude = radialW * mag;
62
+ setOutput(magnitude);
63
+ return;
64
+ }
65
+ }
66
+
67
+ `,
68
+ };
69
+
70
+ const kernel2 = {
71
+ variableNames: ["fbinMag"],
72
+ outputShape: [prunedExtremasT.shape[0], ORIENTATION_NUM_BINS],
73
+ userCode: `
74
+ void main() {
75
+ ivec2 coords = getOutputCoords();
76
+ int featureIndex = coords[0];
77
+ int binIndex = coords[1];
78
+
79
+ float sum = 0.;
80
+ for (int i = 0; i < ${radialPropertiesT.shape[0]}; i++) {
81
+ float fbin = getFbinMag(featureIndex, i, 0);
82
+ int bin = int(floor(fbin - 0.5));
83
+ int b1 = imod(bin + ${ORIENTATION_NUM_BINS}, ${ORIENTATION_NUM_BINS});
84
+ int b2 = imod(bin + 1 + ${ORIENTATION_NUM_BINS}, ${ORIENTATION_NUM_BINS});
85
+
86
+ if (b1 == binIndex || b2 == binIndex) {
87
+ float magnitude = getFbinMag(featureIndex, i, 1);
88
+ float w2 = fbin - float(bin) - 0.5;
89
+ float w1 = w2 * -1. + 1.;
90
+
91
+ if (b1 == binIndex) {
92
+ sum += w1 * magnitude;
93
+ }
94
+ if (b2 == binIndex) {
95
+ sum += w2 * magnitude;
96
+ }
97
+ }
98
+ }
99
+ setOutput(sum);
100
+ }
101
+ `,
102
+ };
103
+
104
+ cache[key] = [kernel1, kernel2];
105
+ }
106
+ return cache[key];
107
+ }
108
+
109
+ export const computeOrientationHistograms = (args) => {
110
+ const { gaussianImagesT, prunedExtremasT, radialPropertiesT, pyramidImagesLength } = args.inputs;
111
+ /** @type {MathBackendWebGL} */
112
+ const backend = args.backend;
113
+ const [program1, program2] = GetPrograms(prunedExtremasT, radialPropertiesT, pyramidImagesLength);
114
+
115
+ const result1 = backend.runWebGLProgram(
116
+ program1,
117
+ [...gaussianImagesT, prunedExtremasT, radialPropertiesT],
118
+ radialPropertiesT.dtype,
119
+ );
120
+ const result2 = backend.runWebGLProgram(program2, [result1], radialPropertiesT.dtype);
121
+ backend.disposeIntermediateTensorInfo(result1);
122
+ return result2;
123
+ };
124
+
125
+ export const computeOrientationHistogramsConfig = {
126
+ kernelName: "ComputeOrientationHistograms",
127
+ backendName: "webgl",
128
+ kernelFunc: computeOrientationHistograms, // as {} as KernelFunc,
129
+ };
@@ -0,0 +1,50 @@
1
+ const cache = {};
2
+ /**
3
+ *
4
+ * @param {TensorInfo} image
5
+ * @returns {GPGPUProgram}
6
+ */
7
+ function GetProgram(image) {
8
+ const imageWidth = image.shape[1];
9
+ const imageHeight = image.shape[0];
10
+ const kernelKey = "w" + imageWidth + "h" + imageHeight;
11
+ if (!cache.hasOwnProperty(kernelKey)) {
12
+ const kernel = {
13
+ variableNames: ["p"],
14
+ outputShape: [Math.floor(imageHeight / 2), Math.floor(imageWidth / 2)],
15
+ userCode: `
16
+ void main() {
17
+ ivec2 coords = getOutputCoords();
18
+ int y = coords[0] * 2;
19
+ int x = coords[1] * 2;
20
+
21
+ float sum = getP(y, x) * 0.25;
22
+ sum += getP(y+1,x) * 0.25;
23
+ sum += getP(y, x+1) * 0.25;
24
+ sum += getP(y+1,x+1) * 0.25;
25
+ setOutput(sum);
26
+ }
27
+ `,
28
+ };
29
+ cache[kernelKey] = kernel;
30
+ }
31
+ return cache[kernelKey];
32
+ }
33
+
34
+ export const downsampleBilinear = (args) => {
35
+ /** @type {import('@tensorflow/tfjs').TensorInfo} */
36
+ const image = args.inputs.image;
37
+ /** @type {MathBackendWebGL} */
38
+ const backend = args.backend;
39
+
40
+ const program = GetProgram(image);
41
+
42
+ return backend.runWebGLProgram(program, [image], image.dtype);
43
+ };
44
+
45
+ export const downsampleBilinearConfig = {
46
+ //: KernelConfig
47
+ kernelName: "DownsampleBilinear",
48
+ backendName: "webgl",
49
+ kernelFunc: downsampleBilinear, // as {} as KernelFunc,
50
+ };
@@ -0,0 +1,50 @@
1
+ export const extremaReduction = (args) => {
2
+ /** @type {import('@tensorflow/tfjs').TensorInfo[]} */
3
+ const { extremasResultT } = args.inputs;
4
+ /** @type {MathBackendWebGL} */
5
+ const backend = args.backend;
6
+ const extremaHeight = extremasResultT.shape[0];
7
+ const extremaWidth = extremasResultT.shape[1];
8
+ const kernel = {
9
+ variableNames: ["extrema"],
10
+ outputShape: [Math.floor(extremaHeight / 2), Math.floor(extremaWidth / 2)],
11
+ userCode: `
12
+ void main() {
13
+ ivec2 coords = getOutputCoords();
14
+ int y = coords[0] * 2;
15
+ int x = coords[1] * 2;
16
+
17
+ float location = 0.0;
18
+ float values = getExtrema(y, x);
19
+
20
+ if (getExtrema(y+1, x) != 0.0) {
21
+ location = 1.0;
22
+ values = getExtrema(y+1, x);
23
+ }
24
+ else if (getExtrema(y, x+1) != 0.0) {
25
+ location = 2.0;
26
+ values = getExtrema(y, x+1);
27
+ }
28
+ else if (getExtrema(y+1, x+1) != 0.0) {
29
+ location = 3.0;
30
+ values = getExtrema(y+1, x+1);
31
+ }
32
+
33
+ if (values < 0.0) {
34
+ setOutput(location * -1000.0 + values);
35
+ } else {
36
+ setOutput(location * 1000.0 + values);
37
+ }
38
+ }
39
+ `,
40
+ };
41
+
42
+ return backend.runWebGLProgram(kernel, [extremasResultT], extremasResultT.dtype);
43
+ };
44
+
45
+ export const extremaReductionConfig = {
46
+ //: KernelConfig
47
+ kernelName: "ExtremaReduction",
48
+ backendName: "webgl",
49
+ kernelFunc: extremaReduction, // as {} as KernelFunc,
50
+ };
@@ -0,0 +1,26 @@
1
+ import { registerKernel } from "@tensorflow/tfjs";
2
+ import { binomialFilterConfig } from "./binomialFilter.js";
3
+ import { buildExtremasConfig } from "./buildExtremas.js";
4
+ import { computeExtremaAnglesConfig } from "./computeExtremaAngles.js";
5
+ import { computeExtremaFreakConfig } from "./computeExtremaFreak.js";
6
+ import { computeFreakDescriptorConfig } from "./computeFreakDescriptors.js";
7
+ import { computeLocalizationConfig } from "./computeLocalization.js";
8
+ import { computeOrientationHistogramsConfig } from "./computeOrientationHistograms.js";
9
+ import { downsampleBilinearConfig } from "./downsampleBilinear.js";
10
+ import { extremaReductionConfig } from "./extremaReduction.js";
11
+ import { smoothHistogramsConfig } from "./smoothHistograms.js";
12
+ import { upsampleBilinearConfig } from "./upsampleBilinear.js";
13
+
14
+ //export function Register(){
15
+ registerKernel(binomialFilterConfig);
16
+ registerKernel(buildExtremasConfig);
17
+ registerKernel(computeExtremaAnglesConfig);
18
+ registerKernel(computeExtremaFreakConfig);
19
+ registerKernel(computeFreakDescriptorConfig);
20
+ registerKernel(computeLocalizationConfig);
21
+ registerKernel(computeOrientationHistogramsConfig);
22
+ registerKernel(downsampleBilinearConfig);
23
+ registerKernel(extremaReductionConfig);
24
+ registerKernel(smoothHistogramsConfig);
25
+ registerKernel(upsampleBilinearConfig);
26
+ //}