@srsergio/taptapp-ar 1.0.0

package/src/compiler/matching/hierarchical-clustering.js

@@ -0,0 +1,131 @@
+import { compute as hammingCompute } from "./hamming-distance.js";
+import { createRandomizer } from "../utils/randomizer.js";
+
+const MIN_FEATURE_PER_NODE = 16;
+const NUM_ASSIGNMENT_HYPOTHESES = 128;
+const NUM_CENTERS = 8;
+
+const _computeKMedoids = (options) => {
+  const { points, pointIndexes, randomizer } = options;
+
+  const randomPointIndexes = [];
+  for (let i = 0; i < pointIndexes.length; i++) {
+    randomPointIndexes.push(i);
+  }
+
+  let bestSumD = Number.MAX_SAFE_INTEGER;
+  let bestAssignmentIndex = -1;
+
+  const assignments = [];
+  for (let i = 0; i < NUM_ASSIGNMENT_HYPOTHESES; i++) {
+    randomizer.arrayShuffle({ arr: randomPointIndexes, sampleSize: NUM_CENTERS });
+
+    let sumD = 0;
+    const assignment = [];
+    for (let j = 0; j < pointIndexes.length; j++) {
+      let bestD = Number.MAX_SAFE_INTEGER;
+      for (let k = 0; k < NUM_CENTERS; k++) {
+        const centerIndex = pointIndexes[randomPointIndexes[k]];
+        const d = hammingCompute({
+          v1: points[pointIndexes[j]].descriptors,
+          v2: points[centerIndex].descriptors,
+        });
+        if (d < bestD) {
+          assignment[j] = randomPointIndexes[k];
+          bestD = d;
+        }
+      }
+      sumD += bestD;
+    }
+    assignments.push(assignment);
+
+    if (sumD < bestSumD) {
+      bestSumD = sumD;
+      bestAssignmentIndex = i;
+    }
+  }
+  return assignments[bestAssignmentIndex];
+};
+
+// kmedoids clustering of points, with hamming distance of FREAK descriptor
+//
+// node = {
+//   isLeaf: bool,
+//   children: [], list of children node
+//   pointIndexes: [], list of int, point indexes
+//   centerPointIndex: int
+// }
+const build = ({ points }) => {
+  const pointIndexes = [];
+  for (let i = 0; i < points.length; i++) {
+    pointIndexes.push(i);
+  }
+
+  const randomizer = createRandomizer();
+
+  const rootNode = _build({
+    points: points,
+    pointIndexes: pointIndexes,
+    centerPointIndex: null,
+    randomizer,
+  });
+  return { rootNode };
+};
+
+// recursive build hierarchy clusters
+const _build = (options) => {
+  const { points, pointIndexes, centerPointIndex, randomizer } = options;
+
+  let isLeaf = false;
+
+  if (pointIndexes.length <= NUM_CENTERS || pointIndexes.length <= MIN_FEATURE_PER_NODE) {
+    isLeaf = true;
+  }
+
+  const clusters = {};
+  if (!isLeaf) {
+    // compute clusters
+    const assignment = _computeKMedoids({ points, pointIndexes, randomizer });
+
+    for (let i = 0; i < assignment.length; i++) {
+      if (clusters[pointIndexes[assignment[i]]] === undefined) {
+        clusters[pointIndexes[assignment[i]]] = [];
+      }
+      clusters[pointIndexes[assignment[i]]].push(pointIndexes[i]);
+    }
+  }
+  if (Object.keys(clusters).length === 1) {
+    isLeaf = true;
+  }
+
+  const node = {
+    centerPointIndex: centerPointIndex,
+  };
+
+  if (isLeaf) {
+    node.leaf = true;
+    node.pointIndexes = [];
+    for (let i = 0; i < pointIndexes.length; i++) {
+      node.pointIndexes.push(pointIndexes[i]);
+    }
+    return node;
+  }
+
+  // recursive build children
+  node.leaf = false;
+  node.children = [];
+
+  Object.keys(clusters).forEach((centerIndex) => {
+    node.children.push(
+      _build({
+        points: points,
+        pointIndexes: clusters[centerIndex],
+        centerPointIndex: centerIndex,
+        randomizer,
+      }),
+    );
+  });
+  return node;
+};
+
+export { build };

package/src/compiler/matching/hough.js

@@ -0,0 +1,206 @@
+const kHoughBinDelta = 1;
+
+// mathces [querypointIndex:x, keypointIndex: x]
+const computeHoughMatches = (options) => {
+  const { keywidth, keyheight, querywidth, queryheight, matches } = options;
+
+  const maxX = querywidth * 1.2;
+  const minX = -maxX;
+  const maxY = queryheight * 1.2;
+  const minY = -maxY;
+  const numAngleBins = 12;
+  const numScaleBins = 10;
+  const minScale = -1;
+  const maxScale = 1;
+  const scaleK = 10.0;
+  const scaleOneOverLogK = 1.0 / Math.log(scaleK);
+  const maxDim = Math.max(keywidth, keyheight);
+  const keycenterX = Math.floor(keywidth / 2);
+  const keycenterY = Math.floor(keyheight / 2);
+
+  // compute numXBins and numYBins based on matches
+  const projectedDims = [];
+  for (let i = 0; i < matches.length; i++) {
+    const queryscale = matches[i].querypoint.scale;
+    const keyscale = matches[i].keypoint.scale;
+    if (keyscale == 0) console.log("ERROR divide zero");
+    const scale = queryscale / keyscale;
+    projectedDims.push(scale * maxDim);
+  }
+
+  // TODO optimize median
+  //   weird. median should be [Math.floor(projectedDims.length/2) - 1] ?
+  projectedDims.sort((a1, a2) => {
+    return a1 - a2;
+  });
+  const medianProjectedDim =
+    projectedDims[
+      Math.floor(projectedDims.length / 2) - (projectedDims.length % 2 == 0 ? 1 : 0) - 1
+    ];
+
+  const binSize = 0.25 * medianProjectedDim;
+  const numXBins = Math.max(5, Math.ceil((maxX - minX) / binSize));
+  const numYBins = Math.max(5, Math.ceil((maxY - minY) / binSize));
+
+  const numXYBins = numXBins * numYBins;
+  const numXYAngleBins = numXYBins * numAngleBins;
+
+  // do voting
+  const querypointValids = [];
+  const querypointBinLocations = [];
+  const votes = {};
+  for (let i = 0; i < matches.length; i++) {
+    const querypoint = matches[i].querypoint;
+    const keypoint = matches[i].keypoint;
+
+    const { x, y, scale, angle } = _mapCorrespondence({
+      querypoint,
+      keypoint,
+      keycenterX,
+      keycenterY,
+      scaleOneOverLogK,
+    });
+
+    // Check that the vote is within range
+    if (
+      x < minX ||
+      x >= maxX ||
+      y < minY ||
+      y >= maxY ||
+      angle <= -Math.PI ||
+      angle > Math.PI ||
+      scale < minScale ||
+      scale >= maxScale
+    ) {
+      querypointValids[i] = false;
+      continue;
+    }
+
+    // map properties to bins
+    let fbinX = (numXBins * (x - minX)) / (maxX - minX);
+    let fbinY = (numYBins * (y - minY)) / (maxY - minY);
+    let fbinAngle = (numAngleBins * (angle + Math.PI)) / (2.0 * Math.PI);
+    let fbinScale = (numScaleBins * (scale - minScale)) / (maxScale - minScale);
+
+    querypointBinLocations[i] = {
+      binX: fbinX,
+      binY: fbinY,
+      binAngle: fbinAngle,
+      binScale: fbinScale,
+    };
+
+    let binX = Math.floor(fbinX - 0.5);
+    let binY = Math.floor(fbinY - 0.5);
+    let binScale = Math.floor(fbinScale - 0.5);
+    let binAngle = (Math.floor(fbinAngle - 0.5) + numAngleBins) % numAngleBins;
+
+    // check can vote all 16 bins
+    if (
+      binX < 0 ||
+      binX + 1 >= numXBins ||
+      binY < 0 ||
+      binY + 1 >= numYBins ||
+      binScale < 0 ||
+      binScale + 1 >= numScaleBins
+    ) {
+      querypointValids[i] = false;
+      continue;
+    }
+
+    for (let dx = 0; dx < 2; dx++) {
+      let binX2 = binX + dx;
+
+      for (let dy = 0; dy < 2; dy++) {
+        let binY2 = binY + dy;
+
+        for (let dangle = 0; dangle < 2; dangle++) {
+          let binAngle2 = (binAngle + dangle) % numAngleBins;
+
+          for (let dscale = 0; dscale < 2; dscale++) {
+            let binScale2 = binScale + dscale;
+
+            const binIndex =
+              binX2 + binY2 * numXBins + binAngle2 * numXYBins + binScale2 * numXYAngleBins;
+
+            if (votes[binIndex] === undefined) votes[binIndex] = 0;
+            votes[binIndex] += 1;
+          }
+        }
+      }
+    }
+    querypointValids[i] = true;
+  }
+
+  let maxVotes = 0;
+  let maxVoteIndex = -1;
+  Object.keys(votes).forEach((index) => {
+    if (votes[index] > maxVotes) {
+      maxVotes = votes[index];
+      maxVoteIndex = index;
+    }
+  });
+
+  if (maxVotes < 3) return [];
+
+  // get back bins from vote index
+  const binX = Math.floor(((maxVoteIndex % numXYAngleBins) % numXYBins) % numXBins);
+  const binY = Math.floor((((maxVoteIndex - binX) % numXYAngleBins) % numXYBins) / numXBins);
+  const binAngle = Math.floor(
+    ((maxVoteIndex - binX - binY * numXBins) % numXYAngleBins) / numXYBins,
+  );
+  const binScale = Math.floor(
+    (maxVoteIndex - binX - binY * numXBins - binAngle * numXYBins) / numXYAngleBins,
+  );
+
+  //console.log("hough voted: ", {binX, binY, binAngle, binScale, maxVoteIndex});
+
+  const houghMatches = [];
+  for (let i = 0; i < matches.length; i++) {
+    if (!querypointValids[i]) continue;
+
+    const queryBins = querypointBinLocations[i];
+    // compute bin difference
+    const distBinX = Math.abs(queryBins.binX - (binX + 0.5));
+    if (distBinX >= kHoughBinDelta) continue;
+
+    const distBinY = Math.abs(queryBins.binY - (binY + 0.5));
+    if (distBinY >= kHoughBinDelta) continue;
+
+    const distBinScale = Math.abs(queryBins.binScale - (binScale + 0.5));
+    if (distBinScale >= kHoughBinDelta) continue;
+
+    const temp = Math.abs(queryBins.binAngle - (binAngle + 0.5));
+    const distBinAngle = Math.min(temp, numAngleBins - temp);
+    if (distBinAngle >= kHoughBinDelta) continue;
+
+    houghMatches.push(matches[i]);
+  }
+  return houghMatches;
+};
+
+const _mapCorrespondence = ({ querypoint, keypoint, keycenterX, keycenterY, scaleOneOverLogK }) => {
+  // map angle to (-pi, pi]
+  let angle = querypoint.angle - keypoint.angle;
+  if (angle <= -Math.PI) angle += 2 * Math.PI;
+  else if (angle > Math.PI) angle -= 2 * Math.PI;
+
+  const scale = querypoint.scale / keypoint.scale;
+
+  // 2x2 similarity
+  const cos = scale * Math.cos(angle);
+  const sin = scale * Math.sin(angle);
+  const S = [cos, -sin, sin, cos];
+
+  const tp = [S[0] * keypoint.x + S[1] * keypoint.y, S[2] * keypoint.x + S[3] * keypoint.y];
+  const tx = querypoint.x - tp[0];
+  const ty = querypoint.y - tp[1];
+
+  return {
+    x: S[0] * keycenterX + S[1] * keycenterY + tx,
+    y: S[2] * keycenterX + S[3] * keycenterY + ty,
+    angle: angle,
+    scale: Math.log(scale) * scaleOneOverLogK,
+  };
+};
+
+export { computeHoughMatches };

package/src/compiler/matching/matcher.js

@@ -0,0 +1,59 @@
+import { match } from "./matching.js";
+
+class Matcher {
+  constructor(queryWidth, queryHeight, debugMode = false) {
+    this.queryWidth = queryWidth;
+    this.queryHeight = queryHeight;
+    this.debugMode = debugMode;
+  }
+
+  matchDetection(keyframes, featurePoints) {
+    let debugExtra = { frames: [] };
+
+    let bestResult = null;
+    for (let i = 0; i < keyframes.length; i++) {
+      const {
+        H,
+        matches,
+        debugExtra: frameDebugExtra,
+      } = match({
+        keyframe: keyframes[i],
+        querypoints: featurePoints,
+        querywidth: this.queryWidth,
+        queryheight: this.queryHeight,
+        debugMode: this.debugMode,
+      });
+      debugExtra.frames.push(frameDebugExtra);
+
+      if (H) {
+        if (bestResult === null || bestResult.matches.length < matches.length) {
+          bestResult = { keyframeIndex: i, H, matches };
+        }
+      }
+    }
+
+    if (bestResult === null) {
+      return { keyframeIndex: -1, debugExtra };
+    }
+
+    const screenCoords = [];
+    const worldCoords = [];
+    const keyframe = keyframes[bestResult.keyframeIndex];
+    for (let i = 0; i < bestResult.matches.length; i++) {
+      const querypoint = bestResult.matches[i].querypoint;
+      const keypoint = bestResult.matches[i].keypoint;
+      screenCoords.push({
+        x: querypoint.x,
+        y: querypoint.y,
+      });
+      worldCoords.push({
+        x: (keypoint.x + 0.5) / keyframe.scale,
+        y: (keypoint.y + 0.5) / keyframe.scale,
+        z: 0,
+      });
+    }
+    return { screenCoords, worldCoords, keyframeIndex: bestResult.keyframeIndex, debugExtra };
+  }
+}
+
+export { Matcher };

package/src/compiler/matching/matching.js

@@ -0,0 +1,237 @@
+import TinyQueue from "tinyqueue";
+import { compute as hammingCompute } from "./hamming-distance.js";
+import { computeHoughMatches } from "./hough.js";
+import { computeHomography } from "./ransacHomography.js";
+import { multiplyPointHomographyInhomogenous, matrixInverse33 } from "../utils/geometry.js";
+
+const INLIER_THRESHOLD = 3;
+//const MIN_NUM_INLIERS = 8;  //default
+const MIN_NUM_INLIERS = 6;
+const CLUSTER_MAX_POP = 8;
+const HAMMING_THRESHOLD = 0.7;
+
+// match list of querpoints against pre-built list of keyframes
+const match = ({ keyframe, querypoints, querywidth, queryheight, debugMode }) => {
+  let debugExtra = {};
+
+  const matches = [];
+  for (let j = 0; j < querypoints.length; j++) {
+    const querypoint = querypoints[j];
+    const keypoints = querypoint.maxima ? keyframe.maximaPoints : keyframe.minimaPoints;
+    if (keypoints.length === 0) continue;
+
+    const rootNode = querypoint.maxima
+      ? keyframe.maximaPointsCluster.rootNode
+      : keyframe.minimaPointsCluster.rootNode;
+
+    const keypointIndexes = [];
+    const queue = new TinyQueue([], (a1, a2) => {
+      return a1.d - a2.d;
+    });
+
+    // query all potential keypoints
+    _query({ node: rootNode, keypoints, querypoint, queue, keypointIndexes, numPop: 0 });
+
+    let bestIndex = -1;
+    let bestD1 = Number.MAX_SAFE_INTEGER;
+    let bestD2 = Number.MAX_SAFE_INTEGER;
+
+    for (let k = 0; k < keypointIndexes.length; k++) {
+      const keypoint = keypoints[keypointIndexes[k]];
+
+      const d = hammingCompute({ v1: keypoint.descriptors, v2: querypoint.descriptors });
+      if (d < bestD1) {
+        bestD2 = bestD1;
+        bestD1 = d;
+        bestIndex = keypointIndexes[k];
+      } else if (d < bestD2) {
+        bestD2 = d;
+      }
+    }
+    if (
+      bestIndex !== -1 &&
+      (bestD2 === Number.MAX_SAFE_INTEGER || (1.0 * bestD1) / bestD2 < HAMMING_THRESHOLD)
+    ) {
+      matches.push({ querypoint, keypoint: keypoints[bestIndex] });
+    }
+  }
+
+  if (debugMode) {
+    debugExtra.matches = matches;
+  }
+
+  if (matches.length < MIN_NUM_INLIERS) return { debugExtra };
+
+  const houghMatches = computeHoughMatches({
+    keywidth: keyframe.width,
+    keyheight: keyframe.height,
+    querywidth,
+    queryheight,
+    matches,
+  });
+
+  if (debugMode) {
+    debugExtra.houghMatches = houghMatches;
+  }
+
+  const H = computeHomography({
+    srcPoints: houghMatches.map((m) => [m.keypoint.x, m.keypoint.y]),
+    dstPoints: houghMatches.map((m) => [m.querypoint.x, m.querypoint.y]),
+    keyframe,
+  });
+
+  if (H === null) return { debugExtra };
+
+  const inlierMatches = _findInlierMatches({
+    H,
+    matches: houghMatches,
+    threshold: INLIER_THRESHOLD,
+  });
+
+  if (debugMode) {
+    debugExtra.inlierMatches = inlierMatches;
+  }
+
+  if (inlierMatches.length < MIN_NUM_INLIERS) return { debugExtra };
+
+  // do another loop of match using the homography
+  const HInv = matrixInverse33(H, 0.00001);
+  const dThreshold2 = 10 * 10;
+  const matches2 = [];
+  for (let j = 0; j < querypoints.length; j++) {
+    const querypoint = querypoints[j];
+    const mapquerypoint = multiplyPointHomographyInhomogenous([querypoint.x, querypoint.y], HInv);
+
+    let bestIndex = -1;
+    let bestD1 = Number.MAX_SAFE_INTEGER;
+    let bestD2 = Number.MAX_SAFE_INTEGER;
+
+    const keypoints = querypoint.maxima ? keyframe.maximaPoints : keyframe.minimaPoints;
+
+    for (let k = 0; k < keypoints.length; k++) {
+      const keypoint = keypoints[k];
+
+      // check distance threshold
+      const d2 =
+        (keypoint.x - mapquerypoint[0]) * (keypoint.x - mapquerypoint[0]) +
+        (keypoint.y - mapquerypoint[1]) * (keypoint.y - mapquerypoint[1]);
+      if (d2 > dThreshold2) continue;
+
+      const d = hammingCompute({ v1: keypoint.descriptors, v2: querypoint.descriptors });
+      if (d < bestD1) {
+        bestD2 = bestD1;
+        bestD1 = d;
+        bestIndex = k;
+      } else if (d < bestD2) {
+        bestD2 = d;
+      }
+    }
+
+    if (
+      bestIndex !== -1 &&
+      (bestD2 === Number.MAX_SAFE_INTEGER || (1.0 * bestD1) / bestD2 < HAMMING_THRESHOLD)
+    ) {
+      matches2.push({ querypoint, keypoint: keypoints[bestIndex] });
+    }
+  }
+
+  if (debugMode) {
+    debugExtra.matches2 = matches2;
+  }
+
+  const houghMatches2 = computeHoughMatches({
+    keywidth: keyframe.width,
+    keyheight: keyframe.height,
+    querywidth,
+    queryheight,
+    matches: matches2,
+  });
+
+  if (debugMode) {
+    debugExtra.houghMatches2 = houghMatches2;
+  }
+
+  const H2 = computeHomography({
+    srcPoints: houghMatches2.map((m) => [m.keypoint.x, m.keypoint.y]),
+    dstPoints: houghMatches2.map((m) => [m.querypoint.x, m.querypoint.y]),
+    keyframe,
+  });
+
+  if (H2 === null) return { debugExtra };
+
+  const inlierMatches2 = _findInlierMatches({
+    H: H2,
+    matches: houghMatches2,
+    threshold: INLIER_THRESHOLD,
+  });
+
+  if (debugMode) {
+    debugExtra.inlierMatches2 = inlierMatches2;
+  }
+
+  return { H: H2, matches: inlierMatches2, debugExtra };
+};
+
+const _query = ({ node, keypoints, querypoint, queue, keypointIndexes, numPop }) => {
+  if (node.leaf) {
+    for (let i = 0; i < node.pointIndexes.length; i++) {
+      keypointIndexes.push(node.pointIndexes[i]);
+    }
+    return;
+  }
+
+  const distances = [];
+  for (let i = 0; i < node.children.length; i++) {
+    const childNode = node.children[i];
+    const centerPointIndex = childNode.centerPointIndex;
+    const d = hammingCompute({
+      v1: keypoints[centerPointIndex].descriptors,
+      v2: querypoint.descriptors,
+    });
+    distances.push(d);
+  }
+
+  let minD = Number.MAX_SAFE_INTEGER;
+  for (let i = 0; i < node.children.length; i++) {
+    minD = Math.min(minD, distances[i]);
+  }
+
+  for (let i = 0; i < node.children.length; i++) {
+    if (distances[i] !== minD) {
+      queue.push({ node: node.children[i], d: distances[i] });
+    }
+  }
+  for (let i = 0; i < node.children.length; i++) {
+    if (distances[i] === minD) {
+      _query({ node: node.children[i], keypoints, querypoint, queue, keypointIndexes, numPop });
+    }
+  }
+
+  if (numPop < CLUSTER_MAX_POP && queue.length > 0) {
+    const { node, d } = queue.pop();
+    numPop += 1;
+    _query({ node, keypoints, querypoint, queue, keypointIndexes, numPop });
+  }
+};
+
+const _findInlierMatches = (options) => {
+  const { H, matches, threshold } = options;
+
+  const threshold2 = threshold * threshold;
+
+  const goodMatches = [];
+  for (let i = 0; i < matches.length; i++) {
+    const querypoint = matches[i].querypoint;
+    const keypoint = matches[i].keypoint;
+    const mp = multiplyPointHomographyInhomogenous([keypoint.x, keypoint.y], H);
+    const d2 =
+      (mp[0] - querypoint.x) * (mp[0] - querypoint.x) +
+      (mp[1] - querypoint.y) * (mp[1] - querypoint.y);
+    if (d2 <= threshold2) {
+      goodMatches.push(matches[i]);
+    }
+  }
+  return goodMatches;
+};
+
+export { match };