@srsergio/taptapp-ar 1.0.0

package/dist/compiler/matching/hierarchical-clustering.js

@@ -0,0 +1,109 @@
+import { compute as hammingCompute } from "./hamming-distance.js";
+import { createRandomizer } from "../utils/randomizer.js";
+const MIN_FEATURE_PER_NODE = 16;
+const NUM_ASSIGNMENT_HYPOTHESES = 128;
+const NUM_CENTERS = 8;
+const _computeKMedoids = (options) => {
+    const { points, pointIndexes, randomizer } = options;
+    const randomPointIndexes = [];
+    for (let i = 0; i < pointIndexes.length; i++) {
+        randomPointIndexes.push(i);
+    }
+    let bestSumD = Number.MAX_SAFE_INTEGER;
+    let bestAssignmentIndex = -1;
+    const assignments = [];
+    for (let i = 0; i < NUM_ASSIGNMENT_HYPOTHESES; i++) {
+        randomizer.arrayShuffle({ arr: randomPointIndexes, sampleSize: NUM_CENTERS });
+        let sumD = 0;
+        const assignment = [];
+        for (let j = 0; j < pointIndexes.length; j++) {
+            let bestD = Number.MAX_SAFE_INTEGER;
+            for (let k = 0; k < NUM_CENTERS; k++) {
+                const centerIndex = pointIndexes[randomPointIndexes[k]];
+                const d = hammingCompute({
+                    v1: points[pointIndexes[j]].descriptors,
+                    v2: points[centerIndex].descriptors,
+                });
+                if (d < bestD) {
+                    assignment[j] = randomPointIndexes[k];
+                    bestD = d;
+                }
+            }
+            sumD += bestD;
+        }
+        assignments.push(assignment);
+        if (sumD < bestSumD) {
+            bestSumD = sumD;
+            bestAssignmentIndex = i;
+        }
+    }
+    return assignments[bestAssignmentIndex];
+};
+// kmedoids clustering of points, with hamming distance of FREAK descriptor
+//
+// node = {
+//   isLeaf: bool,
+//   children: [], list of children node
+//   pointIndexes: [], list of int, point indexes
+//   centerPointIndex: int
+// }
+const build = ({ points }) => {
+    const pointIndexes = [];
+    for (let i = 0; i < points.length; i++) {
+        pointIndexes.push(i);
+    }
+    const randomizer = createRandomizer();
+    const rootNode = _build({
+        points: points,
+        pointIndexes: pointIndexes,
+        centerPointIndex: null,
+        randomizer,
+    });
+    return { rootNode };
+};
+// recursive build hierarchy clusters
+const _build = (options) => {
+    const { points, pointIndexes, centerPointIndex, randomizer } = options;
+    let isLeaf = false;
+    if (pointIndexes.length <= NUM_CENTERS || pointIndexes.length <= MIN_FEATURE_PER_NODE) {
+        isLeaf = true;
+    }
+    const clusters = {};
+    if (!isLeaf) {
+        // compute clusters
+        const assignment = _computeKMedoids({ points, pointIndexes, randomizer });
+        for (let i = 0; i < assignment.length; i++) {
+            if (clusters[pointIndexes[assignment[i]]] === undefined) {
+                clusters[pointIndexes[assignment[i]]] = [];
+            }
+            clusters[pointIndexes[assignment[i]]].push(pointIndexes[i]);
+        }
+    }
+    if (Object.keys(clusters).length === 1) {
+        isLeaf = true;
+    }
+    const node = {
+        centerPointIndex: centerPointIndex,
+    };
+    if (isLeaf) {
+        node.leaf = true;
+        node.pointIndexes = [];
+        for (let i = 0; i < pointIndexes.length; i++) {
+            node.pointIndexes.push(pointIndexes[i]);
+        }
+        return node;
+    }
+    // recursive build children
+    node.leaf = false;
+    node.children = [];
+    Object.keys(clusters).forEach((centerIndex) => {
+        node.children.push(_build({
+            points: points,
+            pointIndexes: clusters[centerIndex],
+            centerPointIndex: centerIndex,
+            randomizer,
+        }));
+    });
+    return node;
+};
+export { build };

package/dist/compiler/matching/hough.d.ts

@@ -0,0 +1 @@
+export function computeHoughMatches(options: any): any[];

package/dist/compiler/matching/hough.js

@@ -0,0 +1,169 @@
+const kHoughBinDelta = 1;
+// mathces [querypointIndex:x, keypointIndex: x]
+const computeHoughMatches = (options) => {
+    const { keywidth, keyheight, querywidth, queryheight, matches } = options;
+    const maxX = querywidth * 1.2;
+    const minX = -maxX;
+    const maxY = queryheight * 1.2;
+    const minY = -maxY;
+    const numAngleBins = 12;
+    const numScaleBins = 10;
+    const minScale = -1;
+    const maxScale = 1;
+    const scaleK = 10.0;
+    const scaleOneOverLogK = 1.0 / Math.log(scaleK);
+    const maxDim = Math.max(keywidth, keyheight);
+    const keycenterX = Math.floor(keywidth / 2);
+    const keycenterY = Math.floor(keyheight / 2);
+    // compute numXBins and numYBins based on matches
+    const projectedDims = [];
+    for (let i = 0; i < matches.length; i++) {
+        const queryscale = matches[i].querypoint.scale;
+        const keyscale = matches[i].keypoint.scale;
+        if (keyscale == 0)
+            console.log("ERROR divide zero");
+        const scale = queryscale / keyscale;
+        projectedDims.push(scale * maxDim);
+    }
+    // TODO optimize median
+    //   weird. median should be [Math.floor(projectedDims.length/2) - 1] ?
+    projectedDims.sort((a1, a2) => {
+        return a1 - a2;
+    });
+    const medianProjectedDim = projectedDims[Math.floor(projectedDims.length / 2) - (projectedDims.length % 2 == 0 ? 1 : 0) - 1];
+    const binSize = 0.25 * medianProjectedDim;
+    const numXBins = Math.max(5, Math.ceil((maxX - minX) / binSize));
+    const numYBins = Math.max(5, Math.ceil((maxY - minY) / binSize));
+    const numXYBins = numXBins * numYBins;
+    const numXYAngleBins = numXYBins * numAngleBins;
+    // do voting
+    const querypointValids = [];
+    const querypointBinLocations = [];
+    const votes = {};
+    for (let i = 0; i < matches.length; i++) {
+        const querypoint = matches[i].querypoint;
+        const keypoint = matches[i].keypoint;
+        const { x, y, scale, angle } = _mapCorrespondence({
+            querypoint,
+            keypoint,
+            keycenterX,
+            keycenterY,
+            scaleOneOverLogK,
+        });
+        // Check that the vote is within range
+        if (x < minX ||
+            x >= maxX ||
+            y < minY ||
+            y >= maxY ||
+            angle <= -Math.PI ||
+            angle > Math.PI ||
+            scale < minScale ||
+            scale >= maxScale) {
+            querypointValids[i] = false;
+            continue;
+        }
+        // map properties to bins
+        let fbinX = (numXBins * (x - minX)) / (maxX - minX);
+        let fbinY = (numYBins * (y - minY)) / (maxY - minY);
+        let fbinAngle = (numAngleBins * (angle + Math.PI)) / (2.0 * Math.PI);
+        let fbinScale = (numScaleBins * (scale - minScale)) / (maxScale - minScale);
+        querypointBinLocations[i] = {
+            binX: fbinX,
+            binY: fbinY,
+            binAngle: fbinAngle,
+            binScale: fbinScale,
+        };
+        let binX = Math.floor(fbinX - 0.5);
+        let binY = Math.floor(fbinY - 0.5);
+        let binScale = Math.floor(fbinScale - 0.5);
+        let binAngle = (Math.floor(fbinAngle - 0.5) + numAngleBins) % numAngleBins;
+        // check can vote all 16 bins
+        if (binX < 0 ||
+            binX + 1 >= numXBins ||
+            binY < 0 ||
+            binY + 1 >= numYBins ||
+            binScale < 0 ||
+            binScale + 1 >= numScaleBins) {
+            querypointValids[i] = false;
+            continue;
+        }
+        for (let dx = 0; dx < 2; dx++) {
+            let binX2 = binX + dx;
+            for (let dy = 0; dy < 2; dy++) {
+                let binY2 = binY + dy;
+                for (let dangle = 0; dangle < 2; dangle++) {
+                    let binAngle2 = (binAngle + dangle) % numAngleBins;
+                    for (let dscale = 0; dscale < 2; dscale++) {
+                        let binScale2 = binScale + dscale;
+                        const binIndex = binX2 + binY2 * numXBins + binAngle2 * numXYBins + binScale2 * numXYAngleBins;
+                        if (votes[binIndex] === undefined)
+                            votes[binIndex] = 0;
+                        votes[binIndex] += 1;
+                    }
+                }
+            }
+        }
+        querypointValids[i] = true;
+    }
+    let maxVotes = 0;
+    let maxVoteIndex = -1;
+    Object.keys(votes).forEach((index) => {
+        if (votes[index] > maxVotes) {
+            maxVotes = votes[index];
+            maxVoteIndex = index;
+        }
+    });
+    if (maxVotes < 3)
+        return [];
+    // get back bins from vote index
+    const binX = Math.floor(((maxVoteIndex % numXYAngleBins) % numXYBins) % numXBins);
+    const binY = Math.floor((((maxVoteIndex - binX) % numXYAngleBins) % numXYBins) / numXBins);
+    const binAngle = Math.floor(((maxVoteIndex - binX - binY * numXBins) % numXYAngleBins) / numXYBins);
+    const binScale = Math.floor((maxVoteIndex - binX - binY * numXBins - binAngle * numXYBins) / numXYAngleBins);
+    //console.log("hough voted: ", {binX, binY, binAngle, binScale, maxVoteIndex});
+    const houghMatches = [];
+    for (let i = 0; i < matches.length; i++) {
+        if (!querypointValids[i])
+            continue;
+        const queryBins = querypointBinLocations[i];
+        // compute bin difference
+        const distBinX = Math.abs(queryBins.binX - (binX + 0.5));
+        if (distBinX >= kHoughBinDelta)
+            continue;
+        const distBinY = Math.abs(queryBins.binY - (binY + 0.5));
+        if (distBinY >= kHoughBinDelta)
+            continue;
+        const distBinScale = Math.abs(queryBins.binScale - (binScale + 0.5));
+        if (distBinScale >= kHoughBinDelta)
+            continue;
+        const temp = Math.abs(queryBins.binAngle - (binAngle + 0.5));
+        const distBinAngle = Math.min(temp, numAngleBins - temp);
+        if (distBinAngle >= kHoughBinDelta)
+            continue;
+        houghMatches.push(matches[i]);
+    }
+    return houghMatches;
+};
+const _mapCorrespondence = ({ querypoint, keypoint, keycenterX, keycenterY, scaleOneOverLogK }) => {
+    // map angle to (-pi, pi]
+    let angle = querypoint.angle - keypoint.angle;
+    if (angle <= -Math.PI)
+        angle += 2 * Math.PI;
+    else if (angle > Math.PI)
+        angle -= 2 * Math.PI;
+    const scale = querypoint.scale / keypoint.scale;
+    // 2x2 similarity
+    const cos = scale * Math.cos(angle);
+    const sin = scale * Math.sin(angle);
+    const S = [cos, -sin, sin, cos];
+    const tp = [S[0] * keypoint.x + S[1] * keypoint.y, S[2] * keypoint.x + S[3] * keypoint.y];
+    const tx = querypoint.x - tp[0];
+    const ty = querypoint.y - tp[1];
+    return {
+        x: S[0] * keycenterX + S[1] * keycenterY + tx,
+        y: S[2] * keycenterX + S[3] * keycenterY + ty,
+        angle: angle,
+        scale: Math.log(scale) * scaleOneOverLogK,
+    };
+};
+export { computeHoughMatches };

package/dist/compiler/matching/matcher.d.ts

@@ -0,0 +1,28 @@
+export class Matcher {
+    constructor(queryWidth: any, queryHeight: any, debugMode?: boolean);
+    queryWidth: any;
+    queryHeight: any;
+    debugMode: boolean;
+    matchDetection(keyframes: any, featurePoints: any): {
+        keyframeIndex: number;
+        debugExtra: {
+            frames: never[];
+        };
+        screenCoords?: undefined;
+        worldCoords?: undefined;
+    } | {
+        screenCoords: {
+            x: any;
+            y: any;
+        }[];
+        worldCoords: {
+            x: number;
+            y: number;
+            z: number;
+        }[];
+        keyframeIndex: number;
+        debugExtra: {
+            frames: never[];
+        };
+    };
+}

package/dist/compiler/matching/matcher.js

@@ -0,0 +1,48 @@
+import { match } from "./matching.js";
+class Matcher {
+    constructor(queryWidth, queryHeight, debugMode = false) {
+        this.queryWidth = queryWidth;
+        this.queryHeight = queryHeight;
+        this.debugMode = debugMode;
+    }
+    matchDetection(keyframes, featurePoints) {
+        let debugExtra = { frames: [] };
+        let bestResult = null;
+        for (let i = 0; i < keyframes.length; i++) {
+            const { H, matches, debugExtra: frameDebugExtra, } = match({
+                keyframe: keyframes[i],
+                querypoints: featurePoints,
+                querywidth: this.queryWidth,
+                queryheight: this.queryHeight,
+                debugMode: this.debugMode,
+            });
+            debugExtra.frames.push(frameDebugExtra);
+            if (H) {
+                if (bestResult === null || bestResult.matches.length < matches.length) {
+                    bestResult = { keyframeIndex: i, H, matches };
+                }
+            }
+        }
+        if (bestResult === null) {
+            return { keyframeIndex: -1, debugExtra };
+        }
+        const screenCoords = [];
+        const worldCoords = [];
+        const keyframe = keyframes[bestResult.keyframeIndex];
+        for (let i = 0; i < bestResult.matches.length; i++) {
+            const querypoint = bestResult.matches[i].querypoint;
+            const keypoint = bestResult.matches[i].keypoint;
+            screenCoords.push({
+                x: querypoint.x,
+                y: querypoint.y,
+            });
+            worldCoords.push({
+                x: (keypoint.x + 0.5) / keyframe.scale,
+                y: (keypoint.y + 0.5) / keyframe.scale,
+                z: 0,
+            });
+        }
+        return { screenCoords, worldCoords, keyframeIndex: bestResult.keyframeIndex, debugExtra };
+    }
+}
+export { Matcher };

package/dist/compiler/matching/matching.d.ts

@@ -0,0 +1,41 @@
+export function match({ keyframe, querypoints, querywidth, queryheight, debugMode }: {
+    keyframe: any;
+    querypoints: any;
+    querywidth: any;
+    queryheight: any;
+    debugMode: any;
+}): {
+    debugExtra: {
+        matches: {
+            querypoint: any;
+            keypoint: any;
+        }[];
+        houghMatches: any[];
+        inlierMatches: any[];
+        matches2: {
+            querypoint: any;
+            keypoint: any;
+        }[];
+        houghMatches2: any[];
+        inlierMatches2: any[];
+    };
+    H?: undefined;
+    matches?: undefined;
+} | {
+    H: number[];
+    matches: any[];
+    debugExtra: {
+        matches: {
+            querypoint: any;
+            keypoint: any;
+        }[];
+        houghMatches: any[];
+        inlierMatches: any[];
+        matches2: {
+            querypoint: any;
+            keypoint: any;
+        }[];
+        houghMatches2: any[];
+        inlierMatches2: any[];
+    };
+};

package/dist/compiler/matching/matching.js

@@ -0,0 +1,197 @@
+import TinyQueue from "tinyqueue";
+import { compute as hammingCompute } from "./hamming-distance.js";
+import { computeHoughMatches } from "./hough.js";
+import { computeHomography } from "./ransacHomography.js";
+import { multiplyPointHomographyInhomogenous, matrixInverse33 } from "../utils/geometry.js";
+const INLIER_THRESHOLD = 3;
+//const MIN_NUM_INLIERS = 8;  //default
+const MIN_NUM_INLIERS = 6;
+const CLUSTER_MAX_POP = 8;
+const HAMMING_THRESHOLD = 0.7;
+// match list of querpoints against pre-built list of keyframes
+const match = ({ keyframe, querypoints, querywidth, queryheight, debugMode }) => {
+    let debugExtra = {};
+    const matches = [];
+    for (let j = 0; j < querypoints.length; j++) {
+        const querypoint = querypoints[j];
+        const keypoints = querypoint.maxima ? keyframe.maximaPoints : keyframe.minimaPoints;
+        if (keypoints.length === 0)
+            continue;
+        const rootNode = querypoint.maxima
+            ? keyframe.maximaPointsCluster.rootNode
+            : keyframe.minimaPointsCluster.rootNode;
+        const keypointIndexes = [];
+        const queue = new TinyQueue([], (a1, a2) => {
+            return a1.d - a2.d;
+        });
+        // query all potential keypoints
+        _query({ node: rootNode, keypoints, querypoint, queue, keypointIndexes, numPop: 0 });
+        let bestIndex = -1;
+        let bestD1 = Number.MAX_SAFE_INTEGER;
+        let bestD2 = Number.MAX_SAFE_INTEGER;
+        for (let k = 0; k < keypointIndexes.length; k++) {
+            const keypoint = keypoints[keypointIndexes[k]];
+            const d = hammingCompute({ v1: keypoint.descriptors, v2: querypoint.descriptors });
+            if (d < bestD1) {
+                bestD2 = bestD1;
+                bestD1 = d;
+                bestIndex = keypointIndexes[k];
+            }
+            else if (d < bestD2) {
+                bestD2 = d;
+            }
+        }
+        if (bestIndex !== -1 &&
+            (bestD2 === Number.MAX_SAFE_INTEGER || (1.0 * bestD1) / bestD2 < HAMMING_THRESHOLD)) {
+            matches.push({ querypoint, keypoint: keypoints[bestIndex] });
+        }
+    }
+    if (debugMode) {
+        debugExtra.matches = matches;
+    }
+    if (matches.length < MIN_NUM_INLIERS)
+        return { debugExtra };
+    const houghMatches = computeHoughMatches({
+        keywidth: keyframe.width,
+        keyheight: keyframe.height,
+        querywidth,
+        queryheight,
+        matches,
+    });
+    if (debugMode) {
+        debugExtra.houghMatches = houghMatches;
+    }
+    const H = computeHomography({
+        srcPoints: houghMatches.map((m) => [m.keypoint.x, m.keypoint.y]),
+        dstPoints: houghMatches.map((m) => [m.querypoint.x, m.querypoint.y]),
+        keyframe,
+    });
+    if (H === null)
+        return { debugExtra };
+    const inlierMatches = _findInlierMatches({
+        H,
+        matches: houghMatches,
+        threshold: INLIER_THRESHOLD,
+    });
+    if (debugMode) {
+        debugExtra.inlierMatches = inlierMatches;
+    }
+    if (inlierMatches.length < MIN_NUM_INLIERS)
+        return { debugExtra };
+    // do another loop of match using the homography
+    const HInv = matrixInverse33(H, 0.00001);
+    const dThreshold2 = 10 * 10;
+    const matches2 = [];
+    for (let j = 0; j < querypoints.length; j++) {
+        const querypoint = querypoints[j];
+        const mapquerypoint = multiplyPointHomographyInhomogenous([querypoint.x, querypoint.y], HInv);
+        let bestIndex = -1;
+        let bestD1 = Number.MAX_SAFE_INTEGER;
+        let bestD2 = Number.MAX_SAFE_INTEGER;
+        const keypoints = querypoint.maxima ? keyframe.maximaPoints : keyframe.minimaPoints;
+        for (let k = 0; k < keypoints.length; k++) {
+            const keypoint = keypoints[k];
+            // check distance threshold
+            const d2 = (keypoint.x - mapquerypoint[0]) * (keypoint.x - mapquerypoint[0]) +
+                (keypoint.y - mapquerypoint[1]) * (keypoint.y - mapquerypoint[1]);
+            if (d2 > dThreshold2)
+                continue;
+            const d = hammingCompute({ v1: keypoint.descriptors, v2: querypoint.descriptors });
+            if (d < bestD1) {
+                bestD2 = bestD1;
+                bestD1 = d;
+                bestIndex = k;
+            }
+            else if (d < bestD2) {
+                bestD2 = d;
+            }
+        }
+        if (bestIndex !== -1 &&
+            (bestD2 === Number.MAX_SAFE_INTEGER || (1.0 * bestD1) / bestD2 < HAMMING_THRESHOLD)) {
+            matches2.push({ querypoint, keypoint: keypoints[bestIndex] });
+        }
+    }
+    if (debugMode) {
+        debugExtra.matches2 = matches2;
+    }
+    const houghMatches2 = computeHoughMatches({
+        keywidth: keyframe.width,
+        keyheight: keyframe.height,
+        querywidth,
+        queryheight,
+        matches: matches2,
+    });
+    if (debugMode) {
+        debugExtra.houghMatches2 = houghMatches2;
+    }
+    const H2 = computeHomography({
+        srcPoints: houghMatches2.map((m) => [m.keypoint.x, m.keypoint.y]),
+        dstPoints: houghMatches2.map((m) => [m.querypoint.x, m.querypoint.y]),
+        keyframe,
+    });
+    if (H2 === null)
+        return { debugExtra };
+    const inlierMatches2 = _findInlierMatches({
+        H: H2,
+        matches: houghMatches2,
+        threshold: INLIER_THRESHOLD,
+    });
+    if (debugMode) {
+        debugExtra.inlierMatches2 = inlierMatches2;
+    }
+    return { H: H2, matches: inlierMatches2, debugExtra };
+};
+const _query = ({ node, keypoints, querypoint, queue, keypointIndexes, numPop }) => {
+    if (node.leaf) {
+        for (let i = 0; i < node.pointIndexes.length; i++) {
+            keypointIndexes.push(node.pointIndexes[i]);
+        }
+        return;
+    }
+    const distances = [];
+    for (let i = 0; i < node.children.length; i++) {
+        const childNode = node.children[i];
+        const centerPointIndex = childNode.centerPointIndex;
+        const d = hammingCompute({
+            v1: keypoints[centerPointIndex].descriptors,
+            v2: querypoint.descriptors,
+        });
+        distances.push(d);
+    }
+    let minD = Number.MAX_SAFE_INTEGER;
+    for (let i = 0; i < node.children.length; i++) {
+        minD = Math.min(minD, distances[i]);
+    }
+    for (let i = 0; i < node.children.length; i++) {
+        if (distances[i] !== minD) {
+            queue.push({ node: node.children[i], d: distances[i] });
+        }
+    }
+    for (let i = 0; i < node.children.length; i++) {
+        if (distances[i] === minD) {
+            _query({ node: node.children[i], keypoints, querypoint, queue, keypointIndexes, numPop });
+        }
+    }
+    if (numPop < CLUSTER_MAX_POP && queue.length > 0) {
+        const { node, d } = queue.pop();
+        numPop += 1;
+        _query({ node, keypoints, querypoint, queue, keypointIndexes, numPop });
+    }
+};
+const _findInlierMatches = (options) => {
+    const { H, matches, threshold } = options;
+    const threshold2 = threshold * threshold;
+    const goodMatches = [];
+    for (let i = 0; i < matches.length; i++) {
+        const querypoint = matches[i].querypoint;
+        const keypoint = matches[i].keypoint;
+        const mp = multiplyPointHomographyInhomogenous([keypoint.x, keypoint.y], H);
+        const d2 = (mp[0] - querypoint.x) * (mp[0] - querypoint.x) +
+            (mp[1] - querypoint.y) * (mp[1] - querypoint.y);
+        if (d2 <= threshold2) {
+            goodMatches.push(matches[i]);
+        }
+    }
+    return goodMatches;
+};
+export { match };

package/dist/compiler/matching/ransacHomography.d.ts

@@ -0,0 +1 @@
+export function computeHomography(options: any): number[] | null;