@srsergio/taptapp-ar 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (207) hide show
  1. package/README.md +62 -0
  2. package/dist/compiler/aframe.d.ts +1 -0
  3. package/dist/compiler/aframe.js +275 -0
  4. package/dist/compiler/compiler-base.d.ts +12 -0
  5. package/dist/compiler/compiler-base.js +165 -0
  6. package/dist/compiler/compiler.d.ts +9 -0
  7. package/dist/compiler/compiler.js +24 -0
  8. package/dist/compiler/compiler.worker.d.ts +1 -0
  9. package/dist/compiler/compiler.worker.js +28 -0
  10. package/dist/compiler/controller.d.ts +101 -0
  11. package/dist/compiler/controller.js +400 -0
  12. package/dist/compiler/controller.worker.d.ts +1 -0
  13. package/dist/compiler/controller.worker.js +61 -0
  14. package/dist/compiler/detector/crop-detector.d.ts +65 -0
  15. package/dist/compiler/detector/crop-detector.js +59 -0
  16. package/dist/compiler/detector/detector.d.ts +98 -0
  17. package/dist/compiler/detector/detector.js +1049 -0
  18. package/dist/compiler/detector/freak.d.ts +1 -0
  19. package/dist/compiler/detector/freak.js +89 -0
  20. package/dist/compiler/detector/kernels/cpu/binomialFilter.d.ts +6 -0
  21. package/dist/compiler/detector/kernels/cpu/binomialFilter.js +51 -0
  22. package/dist/compiler/detector/kernels/cpu/buildExtremas.d.ts +6 -0
  23. package/dist/compiler/detector/kernels/cpu/buildExtremas.js +89 -0
  24. package/dist/compiler/detector/kernels/cpu/computeExtremaAngles.d.ts +7 -0
  25. package/dist/compiler/detector/kernels/cpu/computeExtremaAngles.js +79 -0
  26. package/dist/compiler/detector/kernels/cpu/computeExtremaFreak.d.ts +6 -0
  27. package/dist/compiler/detector/kernels/cpu/computeExtremaFreak.js +68 -0
  28. package/dist/compiler/detector/kernels/cpu/computeFreakDescriptors.d.ts +6 -0
  29. package/dist/compiler/detector/kernels/cpu/computeFreakDescriptors.js +57 -0
  30. package/dist/compiler/detector/kernels/cpu/computeLocalization.d.ts +6 -0
  31. package/dist/compiler/detector/kernels/cpu/computeLocalization.js +54 -0
  32. package/dist/compiler/detector/kernels/cpu/computeOrientationHistograms.d.ts +6 -0
  33. package/dist/compiler/detector/kernels/cpu/computeOrientationHistograms.js +118 -0
  34. package/dist/compiler/detector/kernels/cpu/downsampleBilinear.d.ts +6 -0
  35. package/dist/compiler/detector/kernels/cpu/downsampleBilinear.js +29 -0
  36. package/dist/compiler/detector/kernels/cpu/extremaReduction.d.ts +6 -0
  37. package/dist/compiler/detector/kernels/cpu/extremaReduction.js +50 -0
  38. package/dist/compiler/detector/kernels/cpu/fakeShader.d.ts +20 -0
  39. package/dist/compiler/detector/kernels/cpu/fakeShader.js +80 -0
  40. package/dist/compiler/detector/kernels/cpu/index.d.ts +1 -0
  41. package/dist/compiler/detector/kernels/cpu/index.js +25 -0
  42. package/dist/compiler/detector/kernels/cpu/prune.d.ts +1 -0
  43. package/dist/compiler/detector/kernels/cpu/prune.js +103 -0
  44. package/dist/compiler/detector/kernels/cpu/smoothHistograms.d.ts +6 -0
  45. package/dist/compiler/detector/kernels/cpu/smoothHistograms.js +47 -0
  46. package/dist/compiler/detector/kernels/cpu/upsampleBilinear.d.ts +6 -0
  47. package/dist/compiler/detector/kernels/cpu/upsampleBilinear.js +43 -0
  48. package/dist/compiler/detector/kernels/index.d.ts +1 -0
  49. package/dist/compiler/detector/kernels/index.js +2 -0
  50. package/dist/compiler/detector/kernels/webgl/binomialFilter.d.ts +6 -0
  51. package/dist/compiler/detector/kernels/webgl/binomialFilter.js +67 -0
  52. package/dist/compiler/detector/kernels/webgl/buildExtremas.d.ts +6 -0
  53. package/dist/compiler/detector/kernels/webgl/buildExtremas.js +101 -0
  54. package/dist/compiler/detector/kernels/webgl/computeExtremaAngles.d.ts +6 -0
  55. package/dist/compiler/detector/kernels/webgl/computeExtremaAngles.js +78 -0
  56. package/dist/compiler/detector/kernels/webgl/computeExtremaFreak.d.ts +6 -0
  57. package/dist/compiler/detector/kernels/webgl/computeExtremaFreak.js +86 -0
  58. package/dist/compiler/detector/kernels/webgl/computeFreakDescriptors.d.ts +6 -0
  59. package/dist/compiler/detector/kernels/webgl/computeFreakDescriptors.js +52 -0
  60. package/dist/compiler/detector/kernels/webgl/computeLocalization.d.ts +6 -0
  61. package/dist/compiler/detector/kernels/webgl/computeLocalization.js +58 -0
  62. package/dist/compiler/detector/kernels/webgl/computeOrientationHistograms.d.ts +6 -0
  63. package/dist/compiler/detector/kernels/webgl/computeOrientationHistograms.js +116 -0
  64. package/dist/compiler/detector/kernels/webgl/downsampleBilinear.d.ts +6 -0
  65. package/dist/compiler/detector/kernels/webgl/downsampleBilinear.js +46 -0
  66. package/dist/compiler/detector/kernels/webgl/extremaReduction.d.ts +6 -0
  67. package/dist/compiler/detector/kernels/webgl/extremaReduction.js +48 -0
  68. package/dist/compiler/detector/kernels/webgl/index.d.ts +1 -0
  69. package/dist/compiler/detector/kernels/webgl/index.js +25 -0
  70. package/dist/compiler/detector/kernels/webgl/smoothHistograms.d.ts +6 -0
  71. package/dist/compiler/detector/kernels/webgl/smoothHistograms.js +49 -0
  72. package/dist/compiler/detector/kernels/webgl/upsampleBilinear.d.ts +6 -0
  73. package/dist/compiler/detector/kernels/webgl/upsampleBilinear.js +56 -0
  74. package/dist/compiler/estimation/esimate-experiment.d.ts +5 -0
  75. package/dist/compiler/estimation/esimate-experiment.js +267 -0
  76. package/dist/compiler/estimation/estimate.d.ts +5 -0
  77. package/dist/compiler/estimation/estimate.js +51 -0
  78. package/dist/compiler/estimation/estimator.d.ts +13 -0
  79. package/dist/compiler/estimation/estimator.js +30 -0
  80. package/dist/compiler/estimation/refine-estimate-experiment.d.ts +6 -0
  81. package/dist/compiler/estimation/refine-estimate-experiment.js +429 -0
  82. package/dist/compiler/estimation/refine-estimate.d.ts +6 -0
  83. package/dist/compiler/estimation/refine-estimate.js +299 -0
  84. package/dist/compiler/estimation/utils.d.ts +10 -0
  85. package/dist/compiler/estimation/utils.js +80 -0
  86. package/dist/compiler/image-list.d.ts +13 -0
  87. package/dist/compiler/image-list.js +52 -0
  88. package/dist/compiler/index.d.ts +3 -0
  89. package/dist/compiler/index.js +10 -0
  90. package/dist/compiler/input-loader.d.ts +23 -0
  91. package/dist/compiler/input-loader.js +88 -0
  92. package/dist/compiler/matching/hamming-distance.d.ts +1 -0
  93. package/dist/compiler/matching/hamming-distance.js +20 -0
  94. package/dist/compiler/matching/hierarchical-clustering.d.ts +7 -0
  95. package/dist/compiler/matching/hierarchical-clustering.js +109 -0
  96. package/dist/compiler/matching/hough.d.ts +1 -0
  97. package/dist/compiler/matching/hough.js +169 -0
  98. package/dist/compiler/matching/matcher.d.ts +28 -0
  99. package/dist/compiler/matching/matcher.js +48 -0
  100. package/dist/compiler/matching/matching.d.ts +41 -0
  101. package/dist/compiler/matching/matching.js +197 -0
  102. package/dist/compiler/matching/ransacHomography.d.ts +1 -0
  103. package/dist/compiler/matching/ransacHomography.js +136 -0
  104. package/dist/compiler/offline-compiler.d.ts +10 -0
  105. package/dist/compiler/offline-compiler.js +450 -0
  106. package/dist/compiler/tensorflow-setup.d.ts +7 -0
  107. package/dist/compiler/tensorflow-setup.js +73 -0
  108. package/dist/compiler/three.d.ts +66 -0
  109. package/dist/compiler/three.js +310 -0
  110. package/dist/compiler/tracker/extract-utils.d.ts +1 -0
  111. package/dist/compiler/tracker/extract-utils.js +29 -0
  112. package/dist/compiler/tracker/extract.d.ts +4 -0
  113. package/dist/compiler/tracker/extract.js +349 -0
  114. package/dist/compiler/tracker/tracker.d.ts +38 -0
  115. package/dist/compiler/tracker/tracker.js +327 -0
  116. package/dist/compiler/utils/cumsum.d.ts +5 -0
  117. package/dist/compiler/utils/cumsum.js +39 -0
  118. package/dist/compiler/utils/geometry.d.ts +8 -0
  119. package/dist/compiler/utils/geometry.js +101 -0
  120. package/dist/compiler/utils/homography.d.ts +1 -0
  121. package/dist/compiler/utils/homography.js +138 -0
  122. package/dist/compiler/utils/images.d.ts +24 -0
  123. package/dist/compiler/utils/images.js +99 -0
  124. package/dist/compiler/utils/randomizer.d.ts +5 -0
  125. package/dist/compiler/utils/randomizer.js +25 -0
  126. package/dist/index.d.ts +6 -0
  127. package/dist/index.js +7 -0
  128. package/dist/react/AREditor.d.ts +5 -0
  129. package/dist/react/AREditor.js +159 -0
  130. package/dist/react/ProgressDialog.d.ts +13 -0
  131. package/dist/react/ProgressDialog.js +57 -0
  132. package/dist/react/types.d.ts +22 -0
  133. package/dist/react/types.js +14 -0
  134. package/package.json +53 -0
  135. package/src/astro/ARScene.astro +59 -0
  136. package/src/astro/ARVideoTrigger.astro +73 -0
  137. package/src/astro/overlays/ErrorOverlay.astro +40 -0
  138. package/src/astro/overlays/LoadingOverlay.astro +28 -0
  139. package/src/astro/overlays/ScanningOverlay.astro +119 -0
  140. package/src/astro/scripts/ARScripts.astro +118 -0
  141. package/src/astro/styles/ARStyles.astro +147 -0
  142. package/src/compiler/aframe.js +343 -0
  143. package/src/compiler/compiler-base.js +195 -0
  144. package/src/compiler/compiler.js +25 -0
  145. package/src/compiler/compiler.worker.js +30 -0
  146. package/src/compiler/controller.js +473 -0
  147. package/src/compiler/controller.worker.js +77 -0
  148. package/src/compiler/detector/crop-detector.js +68 -0
  149. package/src/compiler/detector/detector.js +1130 -0
  150. package/src/compiler/detector/freak.js +91 -0
  151. package/src/compiler/detector/kernels/cpu/binomialFilter.js +59 -0
  152. package/src/compiler/detector/kernels/cpu/buildExtremas.js +108 -0
  153. package/src/compiler/detector/kernels/cpu/computeExtremaAngles.js +91 -0
  154. package/src/compiler/detector/kernels/cpu/computeExtremaFreak.js +92 -0
  155. package/src/compiler/detector/kernels/cpu/computeFreakDescriptors.js +68 -0
  156. package/src/compiler/detector/kernels/cpu/computeLocalization.js +71 -0
  157. package/src/compiler/detector/kernels/cpu/computeOrientationHistograms.js +141 -0
  158. package/src/compiler/detector/kernels/cpu/downsampleBilinear.js +33 -0
  159. package/src/compiler/detector/kernels/cpu/extremaReduction.js +53 -0
  160. package/src/compiler/detector/kernels/cpu/fakeShader.js +88 -0
  161. package/src/compiler/detector/kernels/cpu/index.js +26 -0
  162. package/src/compiler/detector/kernels/cpu/prune.js +114 -0
  163. package/src/compiler/detector/kernels/cpu/smoothHistograms.js +57 -0
  164. package/src/compiler/detector/kernels/cpu/upsampleBilinear.js +51 -0
  165. package/src/compiler/detector/kernels/index.js +2 -0
  166. package/src/compiler/detector/kernels/webgl/binomialFilter.js +72 -0
  167. package/src/compiler/detector/kernels/webgl/buildExtremas.js +109 -0
  168. package/src/compiler/detector/kernels/webgl/computeExtremaAngles.js +82 -0
  169. package/src/compiler/detector/kernels/webgl/computeExtremaFreak.js +105 -0
  170. package/src/compiler/detector/kernels/webgl/computeFreakDescriptors.js +56 -0
  171. package/src/compiler/detector/kernels/webgl/computeLocalization.js +70 -0
  172. package/src/compiler/detector/kernels/webgl/computeOrientationHistograms.js +129 -0
  173. package/src/compiler/detector/kernels/webgl/downsampleBilinear.js +50 -0
  174. package/src/compiler/detector/kernels/webgl/extremaReduction.js +50 -0
  175. package/src/compiler/detector/kernels/webgl/index.js +26 -0
  176. package/src/compiler/detector/kernels/webgl/smoothHistograms.js +53 -0
  177. package/src/compiler/detector/kernels/webgl/upsampleBilinear.js +62 -0
  178. package/src/compiler/estimation/esimate-experiment.js +316 -0
  179. package/src/compiler/estimation/estimate.js +67 -0
  180. package/src/compiler/estimation/estimator.js +34 -0
  181. package/src/compiler/estimation/refine-estimate-experiment.js +512 -0
  182. package/src/compiler/estimation/refine-estimate.js +365 -0
  183. package/src/compiler/estimation/utils.js +97 -0
  184. package/src/compiler/image-list.js +62 -0
  185. package/src/compiler/index.js +13 -0
  186. package/src/compiler/input-loader.js +107 -0
  187. package/src/compiler/matching/hamming-distance.js +23 -0
  188. package/src/compiler/matching/hierarchical-clustering.js +131 -0
  189. package/src/compiler/matching/hough.js +206 -0
  190. package/src/compiler/matching/matcher.js +59 -0
  191. package/src/compiler/matching/matching.js +237 -0
  192. package/src/compiler/matching/ransacHomography.js +192 -0
  193. package/src/compiler/offline-compiler.js +553 -0
  194. package/src/compiler/tensorflow-setup.js +88 -0
  195. package/src/compiler/three.js +368 -0
  196. package/src/compiler/tracker/extract-utils.js +34 -0
  197. package/src/compiler/tracker/extract.js +419 -0
  198. package/src/compiler/tracker/tracker.js +397 -0
  199. package/src/compiler/utils/cumsum.js +40 -0
  200. package/src/compiler/utils/geometry.js +114 -0
  201. package/src/compiler/utils/homography.js +150 -0
  202. package/src/compiler/utils/images.js +111 -0
  203. package/src/compiler/utils/randomizer.js +29 -0
  204. package/src/index.ts +8 -0
  205. package/src/react/AREditor.tsx +394 -0
  206. package/src/react/ProgressDialog.tsx +185 -0
  207. package/src/react/types.ts +35 -0
@@ -0,0 +1,78 @@
1
+ const ORIENTATION_NUM_BINS = 36;
2
+ const cache = {};
3
+ function GetProgram(histograms) {
4
+ const key = histograms.shape[0];
5
+ if (!cache.hasOwnProperty(key)) {
6
+ const kernel = {
7
+ variableNames: ["histogram"],
8
+ outputShape: [histograms.shape[0]],
9
+ userCode: `
10
+ void main() {
11
+ int featureIndex = getOutputCoords();
12
+
13
+ int maxIndex = 0;
14
+ for (int i = 1; i < ${ORIENTATION_NUM_BINS}; i++) {
15
+ if (getHistogram(featureIndex, i) > getHistogram(featureIndex, maxIndex)) {
16
+ maxIndex = i;
17
+ }
18
+ }
19
+
20
+ int prev = imod(maxIndex - 1 + ${ORIENTATION_NUM_BINS}, ${ORIENTATION_NUM_BINS});
21
+ int next = imod(maxIndex + 1, ${ORIENTATION_NUM_BINS});
22
+
23
+ /**
24
+ * Fit a quatratic to 3 points. The system of equations is:
25
+ *
26
+ * y0 = A*x0^2 + B*x0 + C
27
+ * y1 = A*x1^2 + B*x1 + C
28
+ * y2 = A*x2^2 + B*x2 + C
29
+ *
30
+ * This system of equations is solved for A,B,C.
31
+ */
32
+ float p10 = float(maxIndex - 1);
33
+ float p11 = getHistogram(featureIndex, prev);
34
+ float p20 = float(maxIndex);
35
+ float p21 = getHistogram(featureIndex, maxIndex);
36
+ float p30 = float(maxIndex + 1);
37
+ float p31 = getHistogram(featureIndex, next);
38
+
39
+ float d1 = (p30-p20)*(p30-p10);
40
+ float d2 = (p10-p20)*(p30-p10);
41
+ float d3 = p10-p20;
42
+
43
+ // If any of the denominators are zero then, just use maxIndex.
44
+ float fbin = float(maxIndex);
45
+ if ( abs(d1) > 0.00001 && abs(d2) > 0.00001 && abs(d3) > 0.00001) {
46
+ float a = p10*p10;
47
+ float b = p20*p20;
48
+
49
+ // Solve for the coefficients A,B,C
50
+ float A = ((p31-p21)/d1)-((p11-p21)/d2);
51
+ float B = ((p11-p21)+(A*(b-a)))/d3;
52
+ float C = p11-(A*a)-(B*p10);
53
+ fbin = -B / (2. * A);
54
+ }
55
+
56
+ float an = 2.0 *${Math.PI} * (fbin + 0.5) / ${ORIENTATION_NUM_BINS}. - ${Math.PI};
57
+ setOutput(an);
58
+ }
59
+ `,
60
+ };
61
+ cache[key] = kernel;
62
+ }
63
+ return cache[key];
64
+ }
65
+ export const computeExtremaAngles = (args) => {
66
+ /** @type {import('@tensorflow/tfjs').TensorInfo} */
67
+ const { histograms } = args.inputs;
68
+ /** @type {MathBackendWebGL} */
69
+ const backend = args.backend;
70
+ const program = GetProgram(histograms);
71
+ return backend.runWebGLProgram(program, [histograms], histograms.dtype);
72
+ };
73
+ export const computeExtremaAnglesConfig = {
74
+ //: KernelConfig
75
+ kernelName: "ComputeExtremaAngles",
76
+ backendName: "webgl",
77
+ kernelFunc: computeExtremaAngles, // as {} as KernelFunc,
78
+ };
@@ -0,0 +1,6 @@
1
+ export function computeExtremaFreak(args: any): any;
2
+ export namespace computeExtremaFreakConfig {
3
+ export let kernelName: string;
4
+ export let backendName: string;
5
+ export { computeExtremaFreak as kernelFunc };
6
+ }
@@ -0,0 +1,86 @@
1
+ import { FREAKPOINTS } from "../../freak.js";
2
+ const FREAK_EXPANSION_FACTOR = 7.0;
3
+ const cache = {};
4
+ function GetProgram(imageCount, prunedExtremas) {
5
+ const key = `${imageCount}|${prunedExtremas.shape[0]}`;
6
+ if (!cache.hasOwnProperty(key)) {
7
+ const imageVariableNames = [];
8
+ for (let i = 1; i < imageCount; i++) {
9
+ imageVariableNames.push("image" + i);
10
+ }
11
+ let pixelsSubCodes = `float getPixel(int octave, int y, int x) {`;
12
+ for (let i = 1; i < imageCount; i++) {
13
+ pixelsSubCodes += `
14
+ if (octave == ${i}) {
15
+ return getImage${i}(y, x);
16
+ }
17
+ `;
18
+ }
19
+ pixelsSubCodes += `}`;
20
+ const kernel = {
21
+ variableNames: [...imageVariableNames, "extrema", "angles", "freakPoints"],
22
+ outputShape: [prunedExtremas.shape[0], FREAKPOINTS.length],
23
+ userCode: `
24
+ ${pixelsSubCodes}
25
+ void main() {
26
+ ivec2 coords = getOutputCoords();
27
+ int featureIndex = coords[0];
28
+ int freakIndex = coords[1];
29
+
30
+ float freakSigma = getFreakPoints(freakIndex, 0);
31
+ float freakX = getFreakPoints(freakIndex, 1);
32
+ float freakY = getFreakPoints(freakIndex, 2);
33
+
34
+ int octave = int(getExtrema(featureIndex, 1));
35
+ float inputY = getExtrema(featureIndex, 2);
36
+ float inputX = getExtrema(featureIndex, 3);
37
+ float inputAngle = getAngles(featureIndex);
38
+ float cos = ${FREAK_EXPANSION_FACTOR}. * cos(inputAngle);
39
+ float sin = ${FREAK_EXPANSION_FACTOR}. * sin(inputAngle);
40
+
41
+ float yp = inputY + freakX * sin + freakY * cos;
42
+ float xp = inputX + freakX * cos + freakY * -sin;
43
+
44
+ int x0 = int(floor(xp));
45
+ int x1 = x0 + 1;
46
+ int y0 = int(floor(yp));
47
+ int y1 = y0 + 1;
48
+
49
+ float f1 = getPixel(octave, y0, x0);
50
+ float f2 = getPixel(octave, y0, x1);
51
+ float f3 = getPixel(octave, y1, x0);
52
+ float f4 = getPixel(octave, y1, x1);
53
+
54
+ float x1f = float(x1);
55
+ float y1f = float(y1);
56
+ float x0f = float(x0);
57
+ float y0f = float(y0);
58
+
59
+ // ratio for interpolation between four neighbouring points
60
+ float value = (x1f - xp) * (y1f - yp) * f1
61
+ + (xp - x0f) * (y1f - yp) * f2
62
+ + (x1f - xp) * (yp - y0f) * f3
63
+ + (xp - x0f) * (yp - y0f) * f4;
64
+
65
+ setOutput(value);
66
+ }
67
+ `,
68
+ };
69
+ cache[key] = kernel;
70
+ }
71
+ return cache[key];
72
+ }
73
+ export const computeExtremaFreak = (args) => {
74
+ /** @type {import('@tensorflow/tfjs').TensorInfo} */
75
+ const { gaussianImagesT, prunedExtremas, prunedExtremasAngles, freakPointsT, pyramidImagesLength, } = args.inputs;
76
+ /** @type {MathBackendWebGL} */
77
+ const backend = args.backend;
78
+ const program = GetProgram(pyramidImagesLength, prunedExtremas);
79
+ return backend.runWebGLProgram(program, [...gaussianImagesT, prunedExtremas, prunedExtremasAngles, freakPointsT], "float32");
80
+ };
81
+ export const computeExtremaFreakConfig = {
82
+ //: KernelConfig
83
+ kernelName: "ComputeExtremaFreak",
84
+ backendName: "webgl",
85
+ kernelFunc: computeExtremaFreak, // as {} as KernelFunc,
86
+ };
@@ -0,0 +1,6 @@
1
+ export function computeFreakDescriptor(args: any): any;
2
+ export namespace computeFreakDescriptorConfig {
3
+ export let kernelName: string;
4
+ export let backendName: string;
5
+ export { computeFreakDescriptor as kernelFunc };
6
+ }
@@ -0,0 +1,52 @@
1
+ import { FREAKPOINTS } from "../../freak.js";
2
+ const FREAK_CONPARISON_COUNT = ((FREAKPOINTS.length - 1) * FREAKPOINTS.length) / 2;
3
+ const descriptorCount = Math.ceil(FREAK_CONPARISON_COUNT / 8);
4
+ const cache = {};
5
+ function GetProgram(extremaFreaks) {
6
+ const key = `${extremaFreaks.shape[0]}`;
7
+ if (!cache.hasOwnProperty(key)) {
8
+ const kernel = {
9
+ variableNames: ["freak", "p"],
10
+ outputShape: [extremaFreaks.shape[0], descriptorCount],
11
+ userCode: `
12
+ void main() {
13
+ ivec2 coords = getOutputCoords();
14
+ int featureIndex = coords[0];
15
+ int descIndex = coords[1] * 8;
16
+
17
+ int sum = 0;
18
+ for (int i = 0; i < 8; i++) {
19
+ if (descIndex + i >= ${FREAK_CONPARISON_COUNT}) {
20
+ continue;
21
+ }
22
+
23
+ int p1 = int(getP(descIndex + i, 0));
24
+ int p2 = int(getP(descIndex + i, 1));
25
+
26
+ float v1 = getFreak(featureIndex, p1);
27
+ float v2 = getFreak(featureIndex, p2);
28
+
29
+ if (v1 < v2 + 0.01) {
30
+ sum += int(pow(2.0, float(7 - i)));
31
+ }
32
+ }
33
+ setOutput(float(sum));
34
+ }
35
+ `,
36
+ };
37
+ cache[key] = kernel;
38
+ }
39
+ return cache[key];
40
+ }
41
+ export const computeFreakDescriptor = (args) => {
42
+ const { extremaFreaks, positionT } = args.inputs;
43
+ const { backend } = args;
44
+ const program = GetProgram(extremaFreaks);
45
+ return backend.runWebGLProgram(program, [extremaFreaks, positionT], "int32");
46
+ };
47
+ export const computeFreakDescriptorConfig = {
48
+ //: KernelConfig
49
+ kernelName: "ComputeFreakDescriptors",
50
+ backendName: "webgl",
51
+ kernelFunc: computeFreakDescriptor, // as {} as KernelFunc,
52
+ };
@@ -0,0 +1,6 @@
1
+ export function computeLocalization(args: any): any;
2
+ export namespace computeLocalizationConfig {
3
+ export let kernelName: string;
4
+ export let backendName: string;
5
+ export { computeLocalization as kernelFunc };
6
+ }
@@ -0,0 +1,58 @@
1
+ import { tensor } from "@tensorflow/tfjs";
2
+ const cache = {};
3
+ function GetProgram(numDogPyramidImages, extremasListLength) {
4
+ const kernelKey = `${numDogPyramidImages}|${extremasListLength}`;
5
+ if (!cache.hasOwnProperty(kernelKey)) {
6
+ const dogVariableNames = [];
7
+ let dogSubCodes = `float getPixel(int octave, int y, int x) {`;
8
+ for (let i = 1; i < numDogPyramidImages; i++) {
9
+ // extrema starts from second octave
10
+ dogVariableNames.push("image" + i);
11
+ dogSubCodes += `
12
+ if (octave == ${i}) {
13
+ return getImage${i}(y, x);
14
+ }
15
+ `;
16
+ }
17
+ dogSubCodes += `}`;
18
+ cache[kernelKey] = {
19
+ variableNames: [...dogVariableNames, "extrema"],
20
+ outputShape: [extremasListLength, 3, 3], // 3x3 pixels around the extrema
21
+ userCode: `
22
+ ${dogSubCodes}
23
+
24
+ void main() {
25
+ ivec3 coords = getOutputCoords();
26
+ int featureIndex = coords[0];
27
+ float score = getExtrema(featureIndex, 0);
28
+ if (score == 0.0) {
29
+ return;
30
+ }
31
+
32
+ int dy = coords[1]-1;
33
+ int dx = coords[2]-1;
34
+ int octave = int(getExtrema(featureIndex, 1));
35
+ int y = int(getExtrema(featureIndex, 2));
36
+ int x = int(getExtrema(featureIndex, 3));
37
+ setOutput(getPixel(octave, y+dy, x+dx));
38
+ }
39
+ `,
40
+ };
41
+ }
42
+ return cache[kernelKey];
43
+ }
44
+ export const computeLocalization = (args) => {
45
+ /** @type {import('@tensorflow/tfjs').TensorInfo} */
46
+ const { prunedExtremasList, dogPyramidImagesT } = args.inputs;
47
+ /** @type {MathBackendWebGL} */
48
+ const backend = args.backend;
49
+ const program = GetProgram(dogPyramidImagesT.length, prunedExtremasList.length);
50
+ const prunedExtremasT = tensor(prunedExtremasList, [prunedExtremasList.length, prunedExtremasList[0].length], "int32");
51
+ return backend.runWebGLProgram(program, [...dogPyramidImagesT.slice(1), prunedExtremasT], dogPyramidImagesT[0].dtype);
52
+ };
53
+ export const computeLocalizationConfig = {
54
+ //: KernelConfig
55
+ kernelName: "ComputeLocalization",
56
+ backendName: "webgl",
57
+ kernelFunc: computeLocalization, // as {} as KernelFunc,
58
+ };
@@ -0,0 +1,6 @@
1
+ export function computeOrientationHistograms(args: any): any;
2
+ export namespace computeOrientationHistogramsConfig {
3
+ export let kernelName: string;
4
+ export let backendName: string;
5
+ export { computeOrientationHistograms as kernelFunc };
6
+ }
@@ -0,0 +1,116 @@
1
+ const oneOver2PI = 0.159154943091895;
2
+ const ORIENTATION_NUM_BINS = 36;
3
+ const cache = {};
4
+ function GetPrograms(prunedExtremasT, radialPropertiesT, pyramidImagesLength) {
5
+ const key = `${pyramidImagesLength}|${prunedExtremasT.shape[0]}|${radialPropertiesT.shape[0]}`;
6
+ if (!cache.hasOwnProperty(key)) {
7
+ const imageVariableNames = [];
8
+ for (let i = 1; i < pyramidImagesLength; i++) {
9
+ imageVariableNames.push("image" + i);
10
+ }
11
+ let kernel1SubCodes = `float getPixel(int octave, int y, int x) {`;
12
+ for (let i = 1; i < pyramidImagesLength; i++) {
13
+ kernel1SubCodes += `
14
+ if (octave == ${i}) {
15
+ return getImage${i}(y, x);
16
+ }
17
+ `;
18
+ }
19
+ kernel1SubCodes += `}`;
20
+ const kernel1 = {
21
+ variableNames: [...imageVariableNames, "extrema", "radial"],
22
+ outputShape: [prunedExtremasT.shape[0], radialPropertiesT.shape[0], 2], // last dimension: [fbin, magnitude]
23
+ userCode: `
24
+ ${kernel1SubCodes}
25
+
26
+ void main() {
27
+ ivec3 coords = getOutputCoords();
28
+ int featureIndex = coords[0];
29
+ int radialIndex = coords[1];
30
+ int propertyIndex = coords[2];
31
+
32
+ int radialY = int(getRadial(radialIndex, 0));
33
+ int radialX = int(getRadial(radialIndex, 1));
34
+ float radialW = getRadial(radialIndex, 2);
35
+
36
+ int octave = int(getExtrema(featureIndex, 1));
37
+ int y = int(getExtrema(featureIndex, 2));
38
+ int x = int(getExtrema(featureIndex, 3));
39
+
40
+ int xp = x + radialX;
41
+ int yp = y + radialY;
42
+
43
+ float dy = getPixel(octave, yp+1, xp) - getPixel(octave, yp-1, xp);
44
+ float dx = getPixel(octave, yp, xp+1) - getPixel(octave, yp, xp-1);
45
+
46
+ if (propertyIndex == 0) {
47
+ // be careful that atan(0, 0) gives 1.57 instead of 0 (different from js), but doesn't matter here, coz magnitude is 0
48
+
49
+ float angle = atan(dy, dx) + ${Math.PI};
50
+ float fbin = angle * ${ORIENTATION_NUM_BINS}. * ${oneOver2PI};
51
+ setOutput(fbin);
52
+ return;
53
+ }
54
+
55
+ if (propertyIndex == 1) {
56
+ float mag = sqrt(dx * dx + dy * dy);
57
+ float magnitude = radialW * mag;
58
+ setOutput(magnitude);
59
+ return;
60
+ }
61
+ }
62
+
63
+ `,
64
+ };
65
+ const kernel2 = {
66
+ variableNames: ["fbinMag"],
67
+ outputShape: [prunedExtremasT.shape[0], ORIENTATION_NUM_BINS],
68
+ userCode: `
69
+ void main() {
70
+ ivec2 coords = getOutputCoords();
71
+ int featureIndex = coords[0];
72
+ int binIndex = coords[1];
73
+
74
+ float sum = 0.;
75
+ for (int i = 0; i < ${radialPropertiesT.shape[0]}; i++) {
76
+ float fbin = getFbinMag(featureIndex, i, 0);
77
+ int bin = int(floor(fbin - 0.5));
78
+ int b1 = imod(bin + ${ORIENTATION_NUM_BINS}, ${ORIENTATION_NUM_BINS});
79
+ int b2 = imod(bin + 1 + ${ORIENTATION_NUM_BINS}, ${ORIENTATION_NUM_BINS});
80
+
81
+ if (b1 == binIndex || b2 == binIndex) {
82
+ float magnitude = getFbinMag(featureIndex, i, 1);
83
+ float w2 = fbin - float(bin) - 0.5;
84
+ float w1 = w2 * -1. + 1.;
85
+
86
+ if (b1 == binIndex) {
87
+ sum += w1 * magnitude;
88
+ }
89
+ if (b2 == binIndex) {
90
+ sum += w2 * magnitude;
91
+ }
92
+ }
93
+ }
94
+ setOutput(sum);
95
+ }
96
+ `,
97
+ };
98
+ cache[key] = [kernel1, kernel2];
99
+ }
100
+ return cache[key];
101
+ }
102
+ export const computeOrientationHistograms = (args) => {
103
+ const { gaussianImagesT, prunedExtremasT, radialPropertiesT, pyramidImagesLength } = args.inputs;
104
+ /** @type {MathBackendWebGL} */
105
+ const backend = args.backend;
106
+ const [program1, program2] = GetPrograms(prunedExtremasT, radialPropertiesT, pyramidImagesLength);
107
+ const result1 = backend.runWebGLProgram(program1, [...gaussianImagesT, prunedExtremasT, radialPropertiesT], radialPropertiesT.dtype);
108
+ const result2 = backend.runWebGLProgram(program2, [result1], radialPropertiesT.dtype);
109
+ backend.disposeIntermediateTensorInfo(result1);
110
+ return result2;
111
+ };
112
+ export const computeOrientationHistogramsConfig = {
113
+ kernelName: "ComputeOrientationHistograms",
114
+ backendName: "webgl",
115
+ kernelFunc: computeOrientationHistograms, // as {} as KernelFunc,
116
+ };
@@ -0,0 +1,6 @@
1
+ export function downsampleBilinear(args: any): any;
2
+ export namespace downsampleBilinearConfig {
3
+ export let kernelName: string;
4
+ export let backendName: string;
5
+ export { downsampleBilinear as kernelFunc };
6
+ }
@@ -0,0 +1,46 @@
1
+ const cache = {};
2
+ /**
3
+ *
4
+ * @param {TensorInfo} image
5
+ * @returns {GPGPUProgram}
6
+ */
7
+ function GetProgram(image) {
8
+ const imageWidth = image.shape[1];
9
+ const imageHeight = image.shape[0];
10
+ const kernelKey = "w" + imageWidth + "h" + imageHeight;
11
+ if (!cache.hasOwnProperty(kernelKey)) {
12
+ const kernel = {
13
+ variableNames: ["p"],
14
+ outputShape: [Math.floor(imageHeight / 2), Math.floor(imageWidth / 2)],
15
+ userCode: `
16
+ void main() {
17
+ ivec2 coords = getOutputCoords();
18
+ int y = coords[0] * 2;
19
+ int x = coords[1] * 2;
20
+
21
+ float sum = getP(y, x) * 0.25;
22
+ sum += getP(y+1,x) * 0.25;
23
+ sum += getP(y, x+1) * 0.25;
24
+ sum += getP(y+1,x+1) * 0.25;
25
+ setOutput(sum);
26
+ }
27
+ `,
28
+ };
29
+ cache[kernelKey] = kernel;
30
+ }
31
+ return cache[kernelKey];
32
+ }
33
+ export const downsampleBilinear = (args) => {
34
+ /** @type {import('@tensorflow/tfjs').TensorInfo} */
35
+ const image = args.inputs.image;
36
+ /** @type {MathBackendWebGL} */
37
+ const backend = args.backend;
38
+ const program = GetProgram(image);
39
+ return backend.runWebGLProgram(program, [image], image.dtype);
40
+ };
41
+ export const downsampleBilinearConfig = {
42
+ //: KernelConfig
43
+ kernelName: "DownsampleBilinear",
44
+ backendName: "webgl",
45
+ kernelFunc: downsampleBilinear, // as {} as KernelFunc,
46
+ };
@@ -0,0 +1,6 @@
1
+ export function extremaReduction(args: any): any;
2
+ export namespace extremaReductionConfig {
3
+ export let kernelName: string;
4
+ export let backendName: string;
5
+ export { extremaReduction as kernelFunc };
6
+ }
@@ -0,0 +1,48 @@
1
+ export const extremaReduction = (args) => {
2
+ /** @type {import('@tensorflow/tfjs').TensorInfo[]} */
3
+ const { extremasResultT } = args.inputs;
4
+ /** @type {MathBackendWebGL} */
5
+ const backend = args.backend;
6
+ const extremaHeight = extremasResultT.shape[0];
7
+ const extremaWidth = extremasResultT.shape[1];
8
+ const kernel = {
9
+ variableNames: ["extrema"],
10
+ outputShape: [Math.floor(extremaHeight / 2), Math.floor(extremaWidth / 2)],
11
+ userCode: `
12
+ void main() {
13
+ ivec2 coords = getOutputCoords();
14
+ int y = coords[0] * 2;
15
+ int x = coords[1] * 2;
16
+
17
+ float location = 0.0;
18
+ float values = getExtrema(y, x);
19
+
20
+ if (getExtrema(y+1, x) != 0.0) {
21
+ location = 1.0;
22
+ values = getExtrema(y+1, x);
23
+ }
24
+ else if (getExtrema(y, x+1) != 0.0) {
25
+ location = 2.0;
26
+ values = getExtrema(y, x+1);
27
+ }
28
+ else if (getExtrema(y+1, x+1) != 0.0) {
29
+ location = 3.0;
30
+ values = getExtrema(y+1, x+1);
31
+ }
32
+
33
+ if (values < 0.0) {
34
+ setOutput(location * -1000.0 + values);
35
+ } else {
36
+ setOutput(location * 1000.0 + values);
37
+ }
38
+ }
39
+ `,
40
+ };
41
+ return backend.runWebGLProgram(kernel, [extremasResultT], extremasResultT.dtype);
42
+ };
43
+ export const extremaReductionConfig = {
44
+ //: KernelConfig
45
+ kernelName: "ExtremaReduction",
46
+ backendName: "webgl",
47
+ kernelFunc: extremaReduction, // as {} as KernelFunc,
48
+ };
@@ -0,0 +1 @@
1
+ export {};
@@ -0,0 +1,25 @@
1
+ import { registerKernel } from "@tensorflow/tfjs";
2
+ import { binomialFilterConfig } from "./binomialFilter.js";
3
+ import { buildExtremasConfig } from "./buildExtremas.js";
4
+ import { computeExtremaAnglesConfig } from "./computeExtremaAngles.js";
5
+ import { computeExtremaFreakConfig } from "./computeExtremaFreak.js";
6
+ import { computeFreakDescriptorConfig } from "./computeFreakDescriptors.js";
7
+ import { computeLocalizationConfig } from "./computeLocalization.js";
8
+ import { computeOrientationHistogramsConfig } from "./computeOrientationHistograms.js";
9
+ import { downsampleBilinearConfig } from "./downsampleBilinear.js";
10
+ import { extremaReductionConfig } from "./extremaReduction.js";
11
+ import { smoothHistogramsConfig } from "./smoothHistograms.js";
12
+ import { upsampleBilinearConfig } from "./upsampleBilinear.js";
13
+ //export function Register(){
14
+ registerKernel(binomialFilterConfig);
15
+ registerKernel(buildExtremasConfig);
16
+ registerKernel(computeExtremaAnglesConfig);
17
+ registerKernel(computeExtremaFreakConfig);
18
+ registerKernel(computeFreakDescriptorConfig);
19
+ registerKernel(computeLocalizationConfig);
20
+ registerKernel(computeOrientationHistogramsConfig);
21
+ registerKernel(downsampleBilinearConfig);
22
+ registerKernel(extremaReductionConfig);
23
+ registerKernel(smoothHistogramsConfig);
24
+ registerKernel(upsampleBilinearConfig);
25
+ //}
@@ -0,0 +1,6 @@
1
+ export function smoothHistograms(args: any): any;
2
+ export namespace smoothHistogramsConfig {
3
+ export let kernelName: string;
4
+ export let backendName: string;
5
+ export { smoothHistograms as kernelFunc };
6
+ }
@@ -0,0 +1,49 @@
1
+ const ORIENTATION_NUM_BINS = 36;
2
+ const ORIENTATION_SMOOTHING_ITERATIONS = 5;
3
+ const cache = {};
4
+ function GetProgram(histograms) {
5
+ const kernelKey = `h${histograms.shape[0]}`;
6
+ if (!cache.hasOwnProperty(kernelKey)) {
7
+ const kernel = {
8
+ variableNames: ["histogram"],
9
+ outputShape: [histograms.shape[0], ORIENTATION_NUM_BINS],
10
+ userCode: `
11
+ void main() {
12
+ ivec2 coords = getOutputCoords();
13
+
14
+ int featureIndex = coords[0];
15
+ int binIndex = coords[1];
16
+
17
+ int prevBin = imod(binIndex - 1 + ${ORIENTATION_NUM_BINS}, ${ORIENTATION_NUM_BINS});
18
+ int nextBin = imod(binIndex + 1, ${ORIENTATION_NUM_BINS});
19
+ float result = 0.274068619061197 * getHistogram(featureIndex, prevBin) + 0.451862761877606 * getHistogram(featureIndex, binIndex) + 0.274068619061197 * getHistogram(featureIndex, nextBin);
20
+
21
+ setOutput(result);
22
+ }
23
+ `,
24
+ };
25
+ cache[kernelKey] = kernel;
26
+ }
27
+ return cache[kernelKey];
28
+ }
29
+ export const smoothHistograms = (args) => {
30
+ /** @type {import('@tensorflow/tfjs').TensorInfo} */
31
+ let { histograms } = args.inputs;
32
+ /** @type {MathBackendWebGL} */
33
+ const backend = args.backend;
34
+ const program = GetProgram(histograms);
35
+ for (let i = 0; i < ORIENTATION_SMOOTHING_ITERATIONS; i++) {
36
+ const _histograms = histograms;
37
+ histograms = backend.runWebGLProgram(program, [histograms], histograms.dtype); //this._compileAndRun(program, [histograms]);
38
+ if (i > 0) {
39
+ backend.disposeIntermediateTensorInfo(_histograms);
40
+ }
41
+ }
42
+ return histograms;
43
+ };
44
+ export const smoothHistogramsConfig = {
45
+ //: KernelConfig
46
+ kernelName: "SmoothHistograms",
47
+ backendName: "webgl",
48
+ kernelFunc: smoothHistograms, // as {} as KernelFunc,
49
+ };
@@ -0,0 +1,6 @@
1
+ export function upsampleBilinear(args: any): import("@tensorflow/tfjs-core").TensorInfo;
2
+ export namespace upsampleBilinearConfig {
3
+ export let kernelName: string;
4
+ export let backendName: string;
5
+ export { upsampleBilinear as kernelFunc };
6
+ }