@srsergio/taptapp-ar 1.0.0

package/src/compiler/detector/freak.js

@@ -0,0 +1,91 @@
+// 37 points = 6 rings x 6 points per ring + 1 center
+const FREAK_RINGS = [
+  // ring 5
+  {
+    sigma: 0.55,
+    points: [
+      [-1.0, 0.0],
+      [-0.5, -0.866025],
+      [0.5, -0.866025],
+      [1.0, -0.0],
+      [0.5, 0.866025],
+      [-0.5, 0.866025],
+    ],
+  },
+  // ring 4
+  {
+    sigma: 0.475,
+    points: [
+      [0.0, 0.930969],
+      [-0.806243, 0.465485],
+      [-0.806243, -0.465485],
+      [-0.0, -0.930969],
+      [0.806243, -0.465485],
+      [0.806243, 0.465485],
+    ],
+  },
+  // ring 3
+  {
+    sigma: 0.4,
+    points: [
+      [0.847306, -0.0],
+      [0.423653, 0.733789],
+      [-0.423653, 0.733789],
+      [-0.847306, 0.0],
+      [-0.423653, -0.733789],
+      [0.423653, -0.733789],
+    ],
+  },
+  // ring 2
+  {
+    sigma: 0.325,
+    points: [
+      [-0.0, -0.741094],
+      [0.641806, -0.370547],
+      [0.641806, 0.370547],
+      [0.0, 0.741094],
+      [-0.641806, 0.370547],
+      [-0.641806, -0.370547],
+    ],
+  },
+  // ring 1
+  {
+    sigma: 0.25,
+    points: [
+      [-0.595502, 0.0],
+      [-0.297751, -0.51572],
+      [0.297751, -0.51572],
+      [0.595502, -0.0],
+      [0.297751, 0.51572],
+      [-0.297751, 0.51572],
+    ],
+  },
+  // ring 0
+  {
+    sigma: 0.175,
+    points: [
+      [0.0, 0.362783],
+      [-0.314179, 0.181391],
+      [-0.314179, -0.181391],
+      [-0.0, -0.362783],
+      [0.314179, -0.181391],
+      [0.314179, 0.181391],
+    ],
+  },
+  // center
+  {
+    sigma: 0.1,
+    points: [[0, 0]],
+  },
+];
+
+const FREAKPOINTS = [];
+for (let r = 0; r < FREAK_RINGS.length; r++) {
+  const sigma = FREAK_RINGS[r].sigma;
+  for (let i = 0; i < FREAK_RINGS[r].points.length; i++) {
+    const point = FREAK_RINGS[r].points[i];
+    FREAKPOINTS.push([sigma, point[0], point[1]]);
+  }
+}
+
+export { FREAKPOINTS };

package/src/compiler/detector/kernels/cpu/binomialFilter.js

@@ -0,0 +1,59 @@
+import * as FakeShader from "./fakeShader.js";
+
+function GetKernels(image) {
+  const imageWidth = image.shape[1];
+  const key = "w" + imageWidth;
+
+  const imageHeight = image.shape[0];
+  const kernel1 = {
+    variableNames: ["p"],
+    outputShape: [imageHeight, imageWidth],
+    userCode: function () {
+      const coords = this.getOutputCoords();
+
+      let sum = this.getP(coords[0], coords[1] - 2);
+      sum += this.getP(coords[0], coords[1] - 1) * 4;
+      sum += this.getP(coords[0], coords[1]) * 6;
+      sum += this.getP(coords[0], coords[1] + 1) * 4;
+      sum += this.getP(coords[0], coords[1] + 2);
+      this.setOutput(sum);
+    },
+  };
+  const kernel2 = {
+    variableNames: ["p"],
+    outputShape: [imageHeight, imageWidth],
+    userCode: function () {
+      const coords = this.getOutputCoords();
+
+      let sum = this.getP(coords[0] - 2, coords[1]);
+      sum += this.getP(coords[0] - 1, coords[1]) * 4;
+      sum += this.getP(coords[0], coords[1]) * 6;
+      sum += this.getP(coords[0] + 1, coords[1]) * 4;
+      sum += this.getP(coords[0] + 2, coords[1]);
+      sum /= 256;
+      this.setOutput(sum);
+    },
+  };
+  return [kernel1, kernel2];
+  //}
+}
+
+export const binomialFilter = (args) => {
+  //{inputs: UnaryInputs, backend: MathBackendCPU}
+  /** @type {import('@tensorflow/tfjs').TensorInfo} */
+  const image = args.inputs.image;
+  /** @type {MathBackendCPU} */
+  const backend = args.backend;
+
+  const [kernel1, kernel2] = GetKernels(image);
+
+  const result1 = FakeShader.runCode(backend, kernel1, [image], image.dtype);
+  return FakeShader.runCode(backend, kernel2, [result1], image.dtype);
+};
+
+export const binomialFilterConfig = {
+  //: KernelConfig
+  kernelName: "BinomialFilter",
+  backendName: "cpu",
+  kernelFunc: binomialFilter, // as {} as KernelFunc,
+};

package/src/compiler/detector/kernels/cpu/buildExtremas.js

@@ -0,0 +1,108 @@
+import * as FakeShader from "./fakeShader.js";
+import { engine } from "@tensorflow/tfjs";
+const FREAK_EXPANSION_FACTOR = 7.0;
+
+const LAPLACIAN_THRESHOLD = 3.0;
+const LAPLACIAN_SQR_THRESHOLD = LAPLACIAN_THRESHOLD * LAPLACIAN_THRESHOLD;
+
+const EDGE_THRESHOLD = 4.0;
+const EDGE_HESSIAN_THRESHOLD = ((EDGE_THRESHOLD + 1) * (EDGE_THRESHOLD + 1)) / EDGE_THRESHOLD;
+
+function GetProgram(image) {
+  const imageWidth = image.shape[1];
+  const imageHeight = image.shape[0];
+  const kernel = {
+    variableNames: ["image0", "image1", "image2"],
+    outputShape: [imageHeight, imageWidth],
+    userCode: function () {
+      const coords = this.getOutputCoords();
+
+      const y = coords[0];
+      const x = coords[1];
+
+      const value = this.getImage1(y, x);
+
+      // Step 1: find local maxima/minima
+      if (value * value < LAPLACIAN_SQR_THRESHOLD) {
+        this.setOutput(0.0);
+        return;
+      }
+      if (y < FREAK_EXPANSION_FACTOR || y > imageHeight - 1 - FREAK_EXPANSION_FACTOR) {
+        this.setOutput(0.0);
+        return;
+      }
+      if (x < FREAK_EXPANSION_FACTOR || x > imageWidth - 1 - FREAK_EXPANSION_FACTOR) {
+        this.setOutput(0.0);
+        return;
+      }
+
+      let isMax = true;
+      let isMin = true;
+      for (let dy = -1; dy <= 1; dy++) {
+        for (let dx = -1; dx <= 1; dx++) {
+          const value0 = this.getImage0(y + dy, x + dx);
+          const value1 = this.getImage1(y + dy, x + dx);
+          const value2 = this.getImage2(y + dy, x + dx);
+
+          if (value < value0 || value < value1 || value < value2) {
+            isMax = false;
+          }
+          if (value > value0 || value > value1 || value > value2) {
+            isMin = false;
+          }
+        }
+      }
+
+      if (!isMax && !isMin) {
+        this.setOutput(0.0);
+        return;
+      }
+
+      // compute edge score and reject based on threshold
+      const dxx = this.getImage1(y, x + 1) + this.getImage1(y, x - 1) - 2 * this.getImage1(y, x);
+      const dyy = this.getImage1(y + 1, x) + this.getImage1(y - 1, x) - 2 * this.getImage1(y, x);
+      const dxy =
+        0.25 *
+        (this.getImage1(y - 1, x - 1) +
+          this.getImage1(y + 1, x + 1) -
+          this.getImage1(y - 1, x + 1) -
+          this.getImage1(y + 1, x - 1));
+
+      const det = dxx * dyy - dxy * dxy;
+
+      if (Math.abs(det) < 0.0001) {
+        // determinant undefined. no solution
+        this.setOutput(0.0);
+        return;
+      }
+
+      const edgeScore = ((dxx + dyy) * (dxx + dyy)) / det;
+
+      if (Math.abs(edgeScore) >= EDGE_HESSIAN_THRESHOLD) {
+        this.setOutput(0.0);
+        return;
+      }
+      this.setOutput(this.getImage1(y, x));
+    },
+  };
+
+  return kernel;
+}
+
+export const buildExtremas = (args) => {
+  let { image0, image1, image2 } = args.inputs;
+  /** @type {MathBackendCPU} */
+  const backend = args.backend;
+
+  image0 = engine().runKernel("DownsampleBilinear", { image: image0 });
+  image2 = engine().runKernel("UpsampleBilinear", { image: image2, targetImage: image1 });
+  const program = GetProgram(image1);
+  return FakeShader.runCode(backend, program, [image0, image1, image2], image1.dtype);
+};
+
+export const buildExtremasConfig = {
+  //: KernelConfig
+  kernelName: "BuildExtremas",
+  backendName: "cpu",
+  kernelFunc: buildExtremas, // as {} as KernelFunc,
+};

package/src/compiler/detector/kernels/cpu/computeExtremaAngles.js

@@ -0,0 +1,91 @@
+const ORIENTATION_NUM_BINS = 36;
+
+export function computeExtremaAnglesImpl(histogram) {
+  const resultValues = new Float32Array(histogram.height);
+
+  function getHistogram(featureIndex, prev) {
+    return histogram.values[featureIndex * histogram.width + prev];
+  }
+  function setOutput(featureIndex, an) {
+    resultValues[featureIndex] = an;
+  }
+
+  function imod(x, y) {
+    return Math.trunc(x - y * Math.floor(x / y));
+  }
+
+  for (let featureIndex = 0; featureIndex < histogram.height; featureIndex++) {
+    let maxIndex = 0;
+    for (let i = 1; i < ORIENTATION_NUM_BINS; i++) {
+      if (getHistogram(featureIndex, i) > getHistogram(featureIndex, maxIndex)) {
+        maxIndex = i;
+      }
+    }
+    let prev = imod(maxIndex - 1 + ORIENTATION_NUM_BINS, ORIENTATION_NUM_BINS);
+    let next = imod(maxIndex + 1, ORIENTATION_NUM_BINS);
+    /**
+     * Fit a quatratic to 3 points. The system of equations is:
+     *
+     * y0 = A*x0^2 + B*x0 + C
+     * y1 = A*x1^2 + B*x1 + C
+     * y2 = A*x2^2 + B*x2 + C
+     *
+     * This system of equations is solved for A,B,C.
+     */
+    const p10 = maxIndex - 1;
+    const p11 = getHistogram(featureIndex, prev);
+    const p20 = maxIndex;
+    const p21 = getHistogram(featureIndex, maxIndex);
+    const p30 = maxIndex + 1;
+    const p31 = getHistogram(featureIndex, next);
+
+    const d1 = (p30 - p20) * (p30 - p10);
+    const d2 = (p10 - p20) * (p30 - p10);
+    const d3 = p10 - p20;
+
+    // If any of the denominators are zero then, just use maxIndex.
+    let fbin = maxIndex;
+    if (Math.abs(d1) > 0.00001 && Math.abs(d2) > 0.00001 && Math.abs(d3) > 0.00001) {
+      const a = p10 * p10;
+      const b = p20 * p20;
+
+      // Solve for the coefficients A,B,C
+      let A = (p31 - p21) / d1 - (p11 - p21) / d2;
+      if (Number.isNaN(A)) A = 0;
+      const B = (p11 - p21 + A * (b - a)) / d3;
+      //const C = p11 - (A * a) - (B * p10);
+      fbin = -B / (2.0 * A);
+      if (Number.isNaN(fbin)) fbin = 0; //console.warn(`computeExtremaAngles::NaN! fbin=${fbin} maxIndex=${maxIndex} A=${A} B=${B} p31=${p31} p21=${p21} p11=${p11}`);
+    }
+
+    const an = (2.0 * Math.PI * (fbin + 0.5)) / ORIENTATION_NUM_BINS - Math.PI;
+
+    setOutput(featureIndex, an);
+  }
+  return resultValues;
+}
+
+export const computeExtremaAngles = (args) => {
+  /** @type {import('@tensorflow/tfjs').TensorInfo} */
+  const { histograms } = args.inputs;
+  /** @type {import('@tensorflow/tfjs-backend-cpu').MathBackendCPU} */
+  const backend = args.backend;
+
+  /** @type {TypedArray} */
+  const histogramValues = {
+    values: backend.data.get(histograms.dataId).values,
+    width: histograms.shape[1],
+    height: histograms.shape[0],
+  };
+
+  const resultValues = computeExtremaAnglesImpl(histogramValues);
+
+  return backend.makeOutput(resultValues, [histograms.shape[0]], histograms.dtype);
+};
+
+export const computeExtremaAnglesConfig = {
+  //: KernelConfig
+  kernelName: "ComputeExtremaAngles",
+  backendName: "cpu",
+  kernelFunc: computeExtremaAngles, // as {} as KernelFunc,
+};

package/src/compiler/detector/kernels/cpu/computeExtremaFreak.js

@@ -0,0 +1,92 @@
+import { FREAKPOINTS } from "../../freak.js";
+import * as FakeShader from "./fakeShader.js";
+const FREAK_EXPANSION_FACTOR = 7.0;
+
+function GetProgram(prunedExtremasHeight, pyramidImagesLength) {
+  const imageVariableNames = [];
+  for (let i = 1; i < pyramidImagesLength; i++) {
+    imageVariableNames.push("image" + i);
+  }
+
+  const kernel = {
+    variableNames: [...imageVariableNames, "extrema", "angles", "freakPoints"],
+    outputShape: [prunedExtremasHeight, FREAKPOINTS.length],
+    userCode: function () {
+      const getPixel = (octave, y, x) => {
+        const key = "getImage" + octave;
+        if (octave < 1 || octave >= pyramidImagesLength) return 0.0;
+        return this[key](y, x);
+      };
+      const coords = this.getOutputCoords();
+      const featureIndex = coords[0];
+      const freakIndex = coords[1];
+
+      //const freakSigma = this.getFreakPoints(freakIndex, 0);
+      const freakX = this.getFreakPoints(freakIndex, 1);
+      const freakY = this.getFreakPoints(freakIndex, 2);
+
+      const octave = this.int(this.getExtrema(featureIndex, 1));
+      const inputY = this.getExtrema(featureIndex, 2);
+      const inputX = this.getExtrema(featureIndex, 3);
+      const inputAngle = this.getAngles(featureIndex);
+      const cos = FREAK_EXPANSION_FACTOR * Math.cos(inputAngle);
+      const sin = FREAK_EXPANSION_FACTOR * Math.sin(inputAngle);
+
+      const yp = inputY + freakX * sin + freakY * cos;
+      const xp = inputX + freakX * cos + freakY * -sin;
+
+      const x0 = this.int(Math.floor(xp));
+      const x1 = x0 + 1;
+      const y0 = this.int(Math.floor(yp));
+      const y1 = y0 + 1;
+
+      const f1 = getPixel(octave, y0, x0);
+      const f2 = getPixel(octave, y0, x1);
+      const f3 = getPixel(octave, y1, x0);
+      const f4 = getPixel(octave, y1, x1);
+
+      /* const x1f = float(x1);
+            const y1f = float(y1);
+            const x0f = float(x0);
+            const y0f = float(y0); */
+
+      // ratio for interpolation between four neighbouring points
+      const value =
+        (x1 - xp) * (y1 - yp) * f1 +
+        (xp - x0) * (y1 - yp) * f2 +
+        (x1 - xp) * (yp - y0) * f3 +
+        (xp - x0) * (yp - y0) * f4;
+
+      this.setOutput(value);
+    },
+  };
+
+  return kernel;
+}
+
+export const computeExtremaFreak = (args) => {
+  /** @type {import('@tensorflow/tfjs').TensorInfo} */
+  const {
+    gaussianImagesT,
+    prunedExtremas,
+    prunedExtremasAngles,
+    freakPointsT,
+    pyramidImagesLength,
+  } = args.inputs;
+  /** @type {MathBackendCPU} */
+  const backend = args.backend;
+  const prog = GetProgram(prunedExtremas.shape[0], pyramidImagesLength);
+  return FakeShader.runCode(
+    backend,
+    prog,
+    [...gaussianImagesT, prunedExtremas, prunedExtremasAngles, freakPointsT],
+    "float32",
+  );
+};
+
+export const computeExtremaFreakConfig = {
+  //: KernelConfig
+  kernelName: "ComputeExtremaFreak",
+  backendName: "cpu",
+  kernelFunc: computeExtremaFreak, // as {} as KernelFunc,
+};

package/src/compiler/detector/kernels/cpu/computeFreakDescriptors.js

@@ -0,0 +1,68 @@
+import { FREAKPOINTS } from "../../freak.js";
+
+const FREAK_CONPARISON_COUNT = ((FREAKPOINTS.length - 1) * FREAKPOINTS.length) / 2;
+const descriptorCount = Math.ceil(FREAK_CONPARISON_COUNT / 8);
+
+function computeFreakDescriptorImpl(extremaFreaks, positionT) {
+  const resultValues = new Float32Array(extremaFreaks.height * descriptorCount);
+  function getP(y, x) {
+    return positionT.values[y * positionT.width + x];
+  }
+  function getFreak(y, x) {
+    return extremaFreaks.values[y * extremaFreaks.width + x];
+  }
+  function setOutput(y, x, o) {
+    resultValues[y * descriptorCount + x] = o;
+  }
+
+  for (let featureIndex = 0; featureIndex < extremaFreaks.height; featureIndex++) {
+    for (let _descIndex = 0; _descIndex < descriptorCount; _descIndex++) {
+      const descIndex = _descIndex * 8;
+      let sum = 0;
+      for (let i = 0; i < 8; i++) {
+        if (descIndex + i >= FREAK_CONPARISON_COUNT) {
+          continue;
+        }
+
+        const p1 = Math.trunc(getP(descIndex + i, 0));
+        const p2 = Math.trunc(getP(descIndex + i, 1));
+
+        const v1 = getFreak(featureIndex, p1);
+        const v2 = getFreak(featureIndex, p2);
+
+        if (v1 < v2 + 0.01) {
+          sum += Math.trunc(Math.pow(2.0, 7.0 - i));
+        }
+      }
+      setOutput(featureIndex, _descIndex, sum);
+    }
+  }
+
+  return resultValues;
+}
+
+export const computeFreakDescriptor = (args) => {
+  const { extremaFreaks, positionT } = args.inputs;
+  const { backend } = args;
+
+  const freaksData = {
+    values: backend.data.get(extremaFreaks.dataId).values,
+    height: extremaFreaks.shape[0],
+    width: extremaFreaks.shape[1],
+  };
+  const positionData = {
+    values: backend.data.get(positionT.dataId).values,
+    width: positionT.shape[1],
+  };
+  //backend.runWebGLProgram(program,[extremaFreaks, positionT],'int32');
+
+  const resultValues = computeFreakDescriptorImpl(freaksData, positionData);
+  return backend.makeOutput(resultValues, [extremaFreaks.shape[0], descriptorCount], "int32");
+};
+
+export const computeFreakDescriptorConfig = {
+  //: KernelConfig
+  kernelName: "ComputeFreakDescriptors",
+  backendName: "cpu",
+  kernelFunc: computeFreakDescriptor, // as {} as KernelFunc,
+};

package/src/compiler/detector/kernels/cpu/computeLocalization.js

@@ -0,0 +1,71 @@
+import { tensor } from "@tensorflow/tfjs";
+import * as FakeShader from "./fakeShader.js";
+
+function GetProgram(numDogPyramidImages, extremasListLength) {
+  const dogVariableNames = [];
+
+  for (let i = 1; i < numDogPyramidImages; i++) {
+    dogVariableNames.push("image" + i);
+  }
+
+  const program = {
+    variableNames: [...dogVariableNames, "extrema"],
+    outputShape: [extremasListLength, 3, 3], // 3x3 pixels around the extrema
+    userCode: function () {
+      const getPixel = (octave, y, x) => {
+        const k = "getImage" + octave;
+        if (!this.hasOwnProperty(k)) {
+          throw new Error(`ComputeLocalization:: ${k} does not exist`);
+        }
+        return this[k](y, x);
+      };
+      const coords = this.getOutputCoords();
+      const featureIndex = coords[0];
+      const score = this.getExtrema(featureIndex, 0);
+      if (score == 0.0) {
+        return;
+      }
+
+      const dy = coords[1] - 1;
+      const dx = coords[2] - 1;
+      const octave = this.int(this.getExtrema(featureIndex, 1));
+      const y = this.int(this.getExtrema(featureIndex, 2));
+      const x = this.int(this.getExtrema(featureIndex, 3));
+      this.setOutput(getPixel(octave, y + dy, x + dx));
+    },
+  };
+  //}
+  return program;
+}
+
+const int = Math.trunc;
+function clamp(n, min, max) {
+  return Math.min(Math.max(min, n), max - 1);
+}
+
+export const computeLocalization = (args) => {
+  /** @type {import('@tensorflow/tfjs').TensorInfo} */
+  const { prunedExtremasList, dogPyramidImagesT } = args.inputs;
+  /** @type {MathBackendCPU} */
+  const backend = args.backend;
+
+  const program = GetProgram(dogPyramidImagesT.length, prunedExtremasList.length);
+  const prunedExtremasT = tensor(
+    prunedExtremasList,
+    [prunedExtremasList.length, prunedExtremasList[0].length],
+    "int32",
+  );
+  return FakeShader.runCode(
+    backend,
+    program,
+    [...dogPyramidImagesT.slice(1), prunedExtremasT],
+    dogPyramidImagesT[0].dtype,
+  );
+};
+
+export const computeLocalizationConfig = {
+  //: KernelConfig
+  kernelName: "ComputeLocalization",
+  backendName: "cpu",
+  kernelFunc: computeLocalization, // as {} as KernelFunc,
+};