@srsergio/taptapp-ar 1.0.0

package/src/compiler/detector/kernels/cpu/computeOrientationHistograms.js

@@ -0,0 +1,141 @@
+import * as FakeShader from "./fakeShader.js";
+const oneOver2PI = 0.159154943091895;
+const ORIENTATION_NUM_BINS = 36;
+
+const cache = {};
+function GetPrograms(prunedExtremasT, radialPropertiesT, pyramidImagesLength) {
+  const key = `${pyramidImagesLength}|${prunedExtremasT.shape[0]}|${radialPropertiesT.shape[0]}`;
+  //if (!cache.hasOwnProperty(key)) {
+  const imageVariableNames = [];
+  for (let i = 1; i < pyramidImagesLength; i++) {
+    imageVariableNames.push("image" + i);
+  }
+
+  /*  let kernel1SubCodes = `float getPixel(int octave, int y, int x) {`;
+    for (let i = 1; i < pyramidImagesLength; i++) {
+        kernel1SubCodes += `
+              if (octave == ${i}) {
+                  return getImage${i}(y, x);
+              }
+              `;
+    }
+    kernel1SubCodes += `}`; */
+
+  const kernel1 = {
+    variableNames: [...imageVariableNames, "extrema", "radial"],
+    outputShape: [prunedExtremasT.shape[0], radialPropertiesT.shape[0], 2], // last dimension: [fbin, magnitude]
+    userCode: function () {
+      //${ kernel1SubCodes }
+      const getPixel = (octave, y, x) => {
+        const k = "getImage" + octave;
+        if (!this.hasOwnProperty(k)) {
+          //console.error(`ComputeOrientationHistograms:: ${k} does not exist! y:${y} x:${x}`);
+          return 0.0;
+        }
+        return this[k](y, x);
+      };
+      /** replicated undefined behavior like you have on OpenGL */
+      function atan(x, y) {
+        if (x == 0 && y == 0) return 1.57;
+        return Math.atan2(x, y);
+      }
+      //void main() {
+      const coords = this.getOutputCoords();
+      const featureIndex = coords[0];
+      const radialIndex = coords[1];
+      const propertyIndex = coords[2];
+
+      const radialY = this.int(this.getRadial(radialIndex, 0));
+      const radialX = this.int(this.getRadial(radialIndex, 1));
+      const radialW = this.getRadial(radialIndex, 2);
+
+      const octave = this.int(this.getExtrema(featureIndex, 1));
+      const y = this.int(this.getExtrema(featureIndex, 2));
+      const x = this.int(this.getExtrema(featureIndex, 3));
+
+      const xp = x + radialX;
+      const yp = y + radialY;
+
+      const dy = getPixel(octave, yp + 1, xp) - getPixel(octave, yp - 1, xp);
+      const dx = getPixel(octave, yp, xp + 1) - getPixel(octave, yp, xp - 1);
+
+      if (propertyIndex == 0) {
+        // be careful that atan(0, 0) gives 1.57 instead of 0 (different from js), but doesn't matter here, coz magnitude is 0
+
+        const angle = this.atan(dy, dx) + Math.PI;
+        const fbin = angle * ORIENTATION_NUM_BINS * oneOver2PI;
+        this.setOutput(fbin);
+        return;
+      }
+
+      if (propertyIndex == 1) {
+        const mag = Math.sqrt(dx * dx + dy * dy);
+        const magnitude = radialW * mag;
+        this.setOutput(magnitude);
+        return;
+      }
+      // }
+    },
+  };
+
+  const kernel2 = {
+    variableNames: ["fbinMag"],
+    outputShape: [prunedExtremasT.shape[0], ORIENTATION_NUM_BINS],
+    userCode: function () {
+      function imod(x, y) {
+        return Math.trunc(x - y * Math.floor(x / y));
+      }
+
+      const coords = this.getOutputCoords();
+      const featureIndex = coords[0];
+      const binIndex = coords[1];
+
+      let sum = 0;
+      for (let i = 0; i < radialPropertiesT.shape[0]; i++) {
+        const fbin = this.getFbinMag(featureIndex, i, 0);
+        const bin = Math.trunc(Math.floor(fbin - 0.5));
+        const b1 = imod(bin + ORIENTATION_NUM_BINS, ORIENTATION_NUM_BINS);
+        const b2 = imod(bin + 1 + ORIENTATION_NUM_BINS, ORIENTATION_NUM_BINS);
+
+        if (b1 == binIndex || b2 == binIndex) {
+          const magnitude = this.getFbinMag(featureIndex, i, 1);
+          const w2 = fbin - bin - 0.5;
+          const w1 = w2 * -1 + 1;
+
+          if (b1 == binIndex) {
+            sum += w1 * magnitude;
+          }
+          if (b2 == binIndex) {
+            sum += w2 * magnitude;
+          }
+        }
+      }
+      this.setOutput(sum);
+    },
+  };
+
+  return [kernel1, kernel2];
+  //}
+  //return cache[key];
+}
+
+export const computeOrientationHistograms = (args) => {
+  const { gaussianImagesT, prunedExtremasT, radialPropertiesT, pyramidImagesLength } = args.inputs;
+  /** @type {MathBackendCPU} */
+  const backend = args.backend;
+  const [program1, program2] = GetPrograms(prunedExtremasT, radialPropertiesT, pyramidImagesLength);
+
+  const result1 = FakeShader.runCode(
+    backend,
+    program1,
+    [...gaussianImagesT, prunedExtremasT, radialPropertiesT],
+    radialPropertiesT.dtype,
+  );
+  return FakeShader.runCode(backend, program2, [result1], radialPropertiesT.dtype);
+};
+
+export const computeOrientationHistogramsConfig = {
+  kernelName: "ComputeOrientationHistograms",
+  backendName: "cpu",
+  kernelFunc: computeOrientationHistograms, // as {} as KernelFunc,
+};

package/src/compiler/detector/kernels/cpu/downsampleBilinear.js

@@ -0,0 +1,33 @@
+import * as FakeShader from "./fakeShader.js";
+
+export const downsampleBilinear = (args) => {
+  /** @type {import('@tensorflow/tfjs').TensorInfo} */
+  const x = args.inputs.image;
+  /** @type {MathBackendCPU} */
+  const backend = args.backend;
+
+  const kernel = {
+    variableNames: ["p"],
+    outputShape: [Math.floor(x.shape[0] / 2), Math.floor(x.shape[1] / 2)],
+    userCode: function () {
+      const coords = this.getOutputCoords();
+      const y = coords[0] * 2;
+      const x = coords[1] * 2;
+      let sum = new Float32Array(1);
+      sum[0] = Math.fround(this.getP(y, x) * 0.25);
+      sum[0] += Math.fround(this.getP(y + 1, x) * 0.25);
+      sum[0] += Math.fround(this.getP(y, x + 1) * 0.25);
+      sum[0] += Math.fround(this.getP(y + 1, x + 1) * 0.25);
+
+      this.setOutput(sum[0]);
+    },
+  };
+  return FakeShader.runCode(backend, kernel, [x], x.dtype);
+};
+
+export const downsampleBilinearConfig = {
+  //: KernelConfig
+  kernelName: "DownsampleBilinear",
+  backendName: "cpu",
+  kernelFunc: downsampleBilinear, // as {} as KernelFunc,
+};

package/src/compiler/detector/kernels/cpu/extremaReduction.js

@@ -0,0 +1,53 @@
+import * as FakeShader from "./fakeShader.js";
+function GetProgram(outHeight, outWidth) {
+  const kernel = {
+    variableNames: ["extrema"],
+    outputShape: [outHeight, outWidth],
+    userCode: function () {
+      const coords = this.getOutputCoords();
+      const y = coords[0] * 2;
+      const x = coords[1] * 2;
+
+      let location = 0.0;
+      let values = this.getExtrema(y, x);
+
+      if (this.getExtrema(y + 1, x) != 0.0) {
+        location = 1.0;
+        values = this.getExtrema(y + 1, x);
+      } else if (this.getExtrema(y, x + 1) != 0.0) {
+        location = 2.0;
+        values = this.getExtrema(y, x + 1);
+      } else if (this.getExtrema(y + 1, x + 1) != 0.0) {
+        location = 3.0;
+        values = this.getExtrema(y + 1, x + 1);
+      }
+
+      if (values < 0.0) {
+        this.setOutput(location * -1000.0 + values);
+      } else {
+        this.setOutput(location * 1000.0 + values);
+      }
+    },
+  };
+  return kernel;
+}
+
+export const extremaReduction = (args) => {
+  const { extremasResultT } = args.inputs;
+  /** @type {MathBackendCPU} */
+  const backend = args.backend;
+  const extremaHeight = extremasResultT.shape[0];
+  const extremaWidth = extremasResultT.shape[1];
+  const outHeight = Math.floor(extremaHeight / 2.0);
+  const outWidth = Math.floor(extremaWidth / 2.0);
+  const program = GetProgram(outHeight, outWidth);
+
+  return FakeShader.runCode(backend, program, [extremasResultT], extremasResultT.dtype);
+};
+
+export const extremaReductionConfig = {
+  //: KernelConfig
+  kernelName: "ExtremaReduction",
+  backendName: "cpu",
+  kernelFunc: extremaReduction, // as {} as KernelFunc,
+};

package/src/compiler/detector/kernels/cpu/fakeShader.js

@@ -0,0 +1,88 @@
+import { zeros, map, flatten as mathjsflatten } from "mathjs";
+/**
+ * @typedef {Object} Kernel
+ * @property {string[]} variableNames
+ * @property {number[]} outputShape
+ * @property {Function} userCode
+ */
+
+/**
+ *
+ * @param {MathBackendCPU} backend
+ * @param {Kernel} kernel
+ * @param {Array<.TensorInfo>} inputs
+ * @param {DataType} dtype
+ * @returns {Tensor}
+ */
+function runCode(backend, kernel, inputs, dtype) {
+  const inputData = inputs.map((value) => {
+    return backend.data.get(value.dataId).values;
+  });
+
+  //create getter functions for every variable name, clamping the input.
+  const tempData = {};
+  kernel.variableNames.forEach((name, index) => {
+    const funName = `get${capFirstLetter(name)}`;
+    //console.log("Making function:",funName,inputs[index].shape);
+    tempData[funName] = function (...args) {
+      const inputIndex = index;
+      for (let i = 0; i < args.length; i++) {
+        args[i] = clampInt(args[i], 0, inputs[inputIndex].shape[i]);
+      }
+      return inputData[index][flatten(args, inputs[inputIndex].shape)];
+    };
+  });
+  tempData.int = Math.trunc;
+  tempData.atan = Math.atan2;
+  //create an empty matrix to map the output size, because i'm lazy and want to use Matrix.map(...)
+  //const temp = new Matrix();
+  //console.log("Creating output shape:",kernel.outputShape);
+  const temp = zeros(kernel.outputShape); //reshape([0,0,0],kernel.outputShape);
+  const output = map(temp, (value, index, matrix) => {
+    tempData.getOutputCoords = () => {
+      return index;
+    };
+    let out;
+
+    tempData.setOutput = (newValue) => {
+      out = Number.isNaN(newValue) ? 0 : Math.fround(newValue);
+    };
+    //bind the method calls and run the code
+    kernel.userCode.bind(tempData)();
+    return out;
+  });
+
+  //output.flat()
+  //convert the output from a matrix into a tensor
+
+  return backend.makeOutput(mathjsflatten(output), kernel.outputShape, dtype);
+}
+
+/**
+ *
+ * @param {string} word
+ * @returns
+ */
+function capFirstLetter(word) {
+  return word[0].toUpperCase() + word.substring(1);
+}
+
+function clampInt(n, min, max) {
+  return Math.min(Math.max(n, min), max - 1);
+}
+/**
+ *
+ * @param {number[]} input
+ * @param {number[]} max
+ * @returns
+ */
+function flatten(input, max) {
+  return input.reduce((prev, current, index) => {
+    for (let i = index + 1; i < max.length; i++) {
+      current *= max[i];
+    }
+    return prev + current;
+  }, 0);
+}
+
+export { runCode };

package/src/compiler/detector/kernels/cpu/index.js

@@ -0,0 +1,26 @@
+import { registerKernel } from "@tensorflow/tfjs";
+import { binomialFilterConfig } from "./binomialFilter.js";
+import { buildExtremasConfig } from "./buildExtremas.js";
+import { computeExtremaAnglesConfig } from "./computeExtremaAngles.js";
+import { computeExtremaFreakConfig } from "./computeExtremaFreak.js";
+import { computeFreakDescriptorConfig } from "./computeFreakDescriptors.js";
+import { computeLocalizationConfig } from "./computeLocalization.js";
+import { computeOrientationHistogramsConfig } from "./computeOrientationHistograms.js";
+import { downsampleBilinearConfig } from "./downsampleBilinear.js";
+import { extremaReductionConfig } from "./extremaReduction.js";
+import { smoothHistogramsConfig } from "./smoothHistograms.js";
+import { upsampleBilinearConfig } from "./upsampleBilinear.js";
+
+//export function Register(){
+registerKernel(binomialFilterConfig);
+registerKernel(buildExtremasConfig);
+registerKernel(computeExtremaAnglesConfig);
+registerKernel(computeExtremaFreakConfig);
+registerKernel(computeFreakDescriptorConfig);
+registerKernel(computeLocalizationConfig);
+registerKernel(computeOrientationHistogramsConfig);
+registerKernel(downsampleBilinearConfig);
+registerKernel(extremaReductionConfig);
+registerKernel(smoothHistogramsConfig);
+registerKernel(upsampleBilinearConfig);
+//}

package/src/compiler/detector/kernels/cpu/prune.js

@@ -0,0 +1,114 @@
+import * as FakeShader from "./fakeShader.js";
+/*
+const kernel = {
+	  variableNames: ['extrema'],
+	  outputShape: [Math.floor(extremaHeight/2), Math.floor(extremaWidth/2)],
+	  userCode: `
+	    void main() {
+	      ivec2 coords = getOutputCoords();
+	      int y = coords[0] * 2;
+	      int x = coords[1] * 2;
+
+	      float location = 0.0;
+	      float values = getExtrema(y, x);
+
+	      if (getExtrema(y+1, x) != 0.0) {
+	        location = 1.0;
+		values = getExtrema(y+1, x);
+	      }
+	      else if (getExtrema(y, x+1) != 0.0) {
+	        location = 2.0;
+		values = getExtrema(y, x+1);
+	      }
+	      else if (getExtrema(y+1, x+1) != 0.0) {
+	        location = 3.0;
+		values = getExtrema(y+1, x+1);
+	      }
+
+	      if (values < 0.0) {
+	        setOutput(location * -1000.0 + values);
+	      } else {
+	        setOutput(location * 1000.0 + values);
+	      }
+	    }
+	  `
+	}
+
+*/
+function clamp(n, min, max) {
+  return Math.min(Math.max(min, n), max - 1);
+}
+
+const pruneImpl = (vals, width, height) => {
+  const w = Math.floor(width / 2);
+  const h = Math.floor(height / 2);
+  const resultValues = new Float32Array(w * h);
+  function getExtrema(x, y) {
+    x = clamp(x, 0, width);
+    y = clamp(y, 0, height);
+    return vals[y * width + x];
+  }
+  function setOutput(x, y, o) {
+    resultValues[y * width + x] = o;
+  }
+
+  for (let x = 0; x < w; x++) {
+    for (let y = 0; y < h; y++) {
+      const x2 = x * 2;
+      const y2 = y * 2;
+
+      let location = 0.0;
+      let values = getExtrema(x2, y2);
+
+      if (getExtrema(x2, y2 + 1) != 0.0) {
+        location = 1.0;
+        values = getExtrema(x2, y2 + 1);
+      } else if (getExtrema(x2 + 1, y2) != 0.0) {
+        location = 2.0;
+        values = getExtrema(x2 + 1, y2);
+      } else if (getExtrema(x2 + 1, y2 + 1) != 0.0) {
+        location = 3.0;
+        values = getExtrema(x2 + 1, y2 + 1);
+      }
+
+      if (values < 0.0) {
+        setOutput(x, y, location * -1000.0 + values);
+      } else {
+        setOutput(x, y, location * 1000.0 + values);
+      }
+    }
+  }
+  return resultValues;
+};
+
+const prune = (args) => {
+  /** @type {import('@tensorflow/tfjs').TensorInfo} */
+  const x = args.inputs.x;
+  /** @type {MathBackendCPU} */
+  const cpuBackend = args.backend;
+  const imageHeight = x.shape[0];
+  const imageWidth = x.shape[1];
+  /** @type {TypedArray} */
+  const values = cpuBackend.data.get(x.dataId).values;
+
+  const resultValues = pruneImpl(values, imageWidth, imageHeight);
+
+  return cpuBackend.makeOutput(
+    resultValues,
+    [Math.floor(imageHeight / 2), Math.floor(imageWidth / 2)],
+    "float32",
+  );
+};
+
+const pruneConfig = {
+  //: KernelConfig
+  kernelName: "Prune",
+  backendName: "cpu",
+  kernelFunc: prune, // as {} as KernelFunc,
+};
+
+module.exports = {
+  pruneConfig,
+  prune,
+  pruneImpl,
+};

package/src/compiler/detector/kernels/cpu/smoothHistograms.js

@@ -0,0 +1,57 @@
+const ORIENTATION_NUM_BINS = 36;
+const ORIENTATION_SMOOTHING_ITERATIONS = 5;
+
+function smoothHistogramsImpl(histograms) {
+  const resultValues = new Float32Array(histograms.height * ORIENTATION_NUM_BINS);
+  function getHistogram(y, x) {
+    return histograms.values[y * histograms.width + x];
+  }
+  function setOutput(y, x, o) {
+    resultValues[y * ORIENTATION_NUM_BINS + x] = o;
+  }
+  function imod(x, y) {
+    return Math.trunc(x - y * Math.floor(x / y));
+  }
+  for (let featureIndex = 0; featureIndex < histograms.height; featureIndex++) {
+    for (let binIndex = 0; binIndex < ORIENTATION_NUM_BINS; binIndex++) {
+      const prevBin = imod(binIndex - 1 + ORIENTATION_NUM_BINS, ORIENTATION_NUM_BINS);
+      const nextBin = imod(binIndex + 1, ORIENTATION_NUM_BINS);
+      const result =
+        0.274068619061197 * getHistogram(featureIndex, prevBin) +
+        0.451862761877606 * getHistogram(featureIndex, binIndex) +
+        0.274068619061197 * getHistogram(featureIndex, nextBin);
+
+      setOutput(featureIndex, binIndex, result);
+    }
+  }
+
+  return resultValues;
+}
+
+export const smoothHistograms = (args) => {
+  /** @type {import('@tensorflow/tfjs').TensorInfo} */
+  const { histograms } = args.inputs;
+  /** @type {MathBackendCPU} */
+  const backend = args.backend;
+  const histogramsData = {
+    values: backend.data.get(histograms.dataId).values,
+    height: histograms.shape[0],
+    width: histograms.shape[1],
+  };
+  //const program = GetProgram(histograms);
+  for (let i = 0; i < ORIENTATION_SMOOTHING_ITERATIONS; i++) {
+    histogramsData.values = smoothHistogramsImpl(histogramsData); //backend.runWebGLProgram(program, [histograms], histograms.dtype);//this._compileAndRun(program, [histograms]);
+  }
+  return backend.makeOutput(
+    histogramsData.values,
+    [histograms.shape[0], ORIENTATION_NUM_BINS],
+    histograms.dtype,
+  );
+};
+
+export const smoothHistogramsConfig = {
+  //: KernelConfig
+  kernelName: "SmoothHistograms",
+  backendName: "cpu",
+  kernelFunc: smoothHistograms, // as {} as KernelFunc,
+};

package/src/compiler/detector/kernels/cpu/upsampleBilinear.js

@@ -0,0 +1,51 @@
+import * as FakeShader from "./fakeShader.js";
+function getProgram(targetImage) {
+  const kernel = {
+    variableNames: ["p"],
+    outputShape: [targetImage.shape[0], targetImage.shape[1]],
+    userCode: function () {
+      const coords = this.getOutputCoords();
+      const j = coords[0];
+      const i = coords[1];
+
+      const sj = Math.fround(0.5 * j) - 0.25;
+      const si = Math.fround(0.5 * i) - 0.25;
+
+      const sj0 = Math.floor(sj);
+      const sj1 = Math.ceil(sj);
+      const si0 = Math.floor(si);
+      const si1 = Math.ceil(si);
+
+      const sj0I = this.int(sj0);
+      const sj1I = this.int(sj1);
+      const si0I = this.int(si0);
+      const si1I = this.int(si1);
+
+      let sum = 0.0;
+      sum += this.getP(sj0I, si0I) * Math.fround((si1 - si) * (sj1 - sj));
+      sum += this.getP(sj1I, si0I) * Math.fround((si1 - si) * (sj - sj0));
+      sum += this.getP(sj0I, si1I) * Math.fround((si - si0) * (sj1 - sj));
+      sum += this.getP(sj1I, si1I) * Math.fround((si - si0) * (sj - sj0));
+      this.setOutput(sum);
+    },
+  };
+  return kernel;
+}
+
+export const upsampleBilinear = (args) => {
+  /** @type {import('@tensorflow/tfjs').TensorInfo} */
+  const { image, targetImage } = args.inputs;
+
+  /** @type {MathBackendCPU} */
+  const cpuBackend = args.backend;
+
+  const program = getProgram(targetImage);
+  return FakeShader.runCode(cpuBackend, program, [image], image.dtype);
+};
+
+export const upsampleBilinearConfig = {
+  //: KernelConfig
+  kernelName: "UpsampleBilinear",
+  backendName: "cpu",
+  kernelFunc: upsampleBilinear, // as {} as KernelFunc,
+};

package/src/compiler/detector/kernels/index.js

@@ -0,0 +1,2 @@
+import "./webgl/index.js";
+import "./cpu/index.js";

package/src/compiler/detector/kernels/webgl/binomialFilter.js

@@ -0,0 +1,72 @@
+const cache = {};
+
+/**
+ *
+ * @param {string} key
+ * @param {TensorInfo} image
+ * @returns {[GPGPUProgram,GPGPUProgram]}
+ */
+function GetKernels(image) {
+  const imageWidth = image.shape[1];
+  const imageHeight = image.shape[0];
+  const key = "w" + imageWidth + "h" + imageHeight;
+  if (!cache.hasOwnProperty(key)) {
+    const kernel1 = {
+      variableNames: ["p"],
+      outputShape: [imageHeight, imageWidth],
+      userCode: `
+        void main() {
+          ivec2 coords = getOutputCoords();
+
+          float sum = getP(coords[0], coords[1]-2);
+          sum += getP(coords[0], coords[1]-1) * 4.;
+          sum += getP(coords[0], coords[1]) * 6.;
+          sum += getP(coords[0], coords[1]+1) * 4.;
+          sum += getP(coords[0], coords[1]+2);
+          setOutput(sum);
+        }
+      `,
+    };
+    const kernel2 = {
+      variableNames: ["p"],
+      outputShape: [imageHeight, imageWidth],
+      userCode: `
+        void main() {
+          ivec2 coords = getOutputCoords();
+
+          float sum = getP(coords[0]-2, coords[1]);
+          sum += getP(coords[0]-1, coords[1]) * 4.;
+          sum += getP(coords[0], coords[1]) * 6.;
+          sum += getP(coords[0]+1, coords[1]) * 4.;
+          sum += getP(coords[0]+2, coords[1]);
+          sum /= 256.;
+          setOutput(sum);
+        }
+      `,
+    };
+    cache[key] = [kernel1, kernel2];
+  }
+  return cache[key];
+}
+
+export const binomialFilter = (args) => {
+  //{inputs: UnaryInputs, backend: MathBackendCPU}
+  /** @type {import('@tensorflow/tfjs').TensorInfo} */
+  const image = args.inputs.image;
+  /** @type {MathBackendWebGL} */
+  const backend = args.backend;
+
+  const [kernel1, kernel2] = GetKernels(image);
+
+  const result1 = backend.runWebGLProgram(kernel1, [image], image.dtype);
+  const result2 = backend.runWebGLProgram(kernel2, [result1], image.dtype);
+  backend.disposeIntermediateTensorInfo(result1);
+  return result2;
+};
+
+export const binomialFilterConfig = {
+  //: KernelConfig
+  kernelName: "BinomialFilter",
+  backendName: "webgl",
+  kernelFunc: binomialFilter, // as {} as KernelFunc,
+};