@srsergio/taptapp-ar 1.0.0

package/src/compiler/estimation/refine-estimate.js

@@ -0,0 +1,365 @@
+import { Matrix, inverse } from "ml-matrix";
+import {
+  applyModelViewProjectionTransform,
+  buildModelViewProjectionTransform,
+  computeScreenCoordiate,
+} from "./utils.js";
+
+const TRACKING_THRESH = 5.0; // default
+const K2_FACTOR = 4.0; // Question: should it be relative to the size of the screen instead of hardcoded?
+const ICP_MAX_LOOP = 10;
+const ICP_BREAK_LOOP_ERROR_THRESH = 0.1;
+const ICP_BREAK_LOOP_ERROR_RATIO_THRESH = 0.99;
+const ICP_BREAK_LOOP_ERROR_THRESH2 = 4.0;
+
+// some temporary/intermediate variables used later. Declare them beforehand to reduce new object allocations
+let mat = [[], [], []];
+let J_U_Xc = [[], []]; // 2x3
+let J_Xc_S = [[], [], []]; // 3x6
+
+const refineEstimate = ({
+  initialModelViewTransform,
+  projectionTransform,
+  worldCoords,
+  screenCoords,
+}) => {
+  // Question: shall we normlize the screen coords as well?
+  // Question: do we need to normlize the scale as well, i.e. make coords from -1 to 1
+  //
+  // normalize world coords - reposition them to center of mass
+  //   assume z coordinate is always zero (in our case, the image target is planar with z = 0
+  let dx = 0;
+  let dy = 0;
+  for (let i = 0; i < worldCoords.length; i++) {
+    dx += worldCoords[i].x;
+    dy += worldCoords[i].y;
+  }
+  dx /= worldCoords.length;
+  dy /= worldCoords.length;
+
+  const normalizedWorldCoords = [];
+  for (let i = 0; i < worldCoords.length; i++) {
+    normalizedWorldCoords.push({
+      x: worldCoords[i].x - dx,
+      y: worldCoords[i].y - dy,
+      z: worldCoords[i].z,
+    });
+  }
+
+  const diffModelViewTransform = [[], [], []];
+  for (let j = 0; j < 3; j++) {
+    for (let i = 0; i < 3; i++) {
+      diffModelViewTransform[j][i] = initialModelViewTransform[j][i];
+    }
+  }
+  diffModelViewTransform[0][3] =
+    initialModelViewTransform[0][0] * dx +
+    initialModelViewTransform[0][1] * dy +
+    initialModelViewTransform[0][3];
+  diffModelViewTransform[1][3] =
+    initialModelViewTransform[1][0] * dx +
+    initialModelViewTransform[1][1] * dy +
+    initialModelViewTransform[1][3];
+  diffModelViewTransform[2][3] =
+    initialModelViewTransform[2][0] * dx +
+    initialModelViewTransform[2][1] * dy +
+    initialModelViewTransform[2][3];
+
+  // use iterative closest point algorithm to refine the modelViewTransform
+  const inlierProbs = [1.0, 0.8, 0.6, 0.4, 0.0];
+  let updatedModelViewTransform = diffModelViewTransform; // iteratively update this transform
+  let finalModelViewTransform = null;
+  for (let i = 0; i < inlierProbs.length; i++) {
+    const ret = _doICP({
+      initialModelViewTransform: updatedModelViewTransform,
+      projectionTransform,
+      worldCoords: normalizedWorldCoords,
+      screenCoords,
+      inlierProb: inlierProbs[i],
+    });
+
+    updatedModelViewTransform = ret.modelViewTransform;
+
+    //console.log("err", ret.err);
+
+    if (ret.err < TRACKING_THRESH) {
+      finalModelViewTransform = updatedModelViewTransform;
+      break;
+    }
+  }
+
+  if (finalModelViewTransform === null) return null;
+
+  // de-normalize
+  finalModelViewTransform[0][3] =
+    finalModelViewTransform[0][3] -
+    finalModelViewTransform[0][0] * dx -
+    finalModelViewTransform[0][1] * dy;
+  finalModelViewTransform[1][3] =
+    finalModelViewTransform[1][3] -
+    finalModelViewTransform[1][0] * dx -
+    finalModelViewTransform[1][1] * dy;
+  finalModelViewTransform[2][3] =
+    finalModelViewTransform[2][3] -
+    finalModelViewTransform[2][0] * dx -
+    finalModelViewTransform[2][1] * dy;
+
+  return finalModelViewTransform;
+};
+
+// ICP iteration
+// Question: can someone provide theoretical reference / mathematical proof for the following computations?
+const _doICP = ({
+  initialModelViewTransform,
+  projectionTransform,
+  worldCoords,
+  screenCoords,
+  inlierProb,
+}) => {
+  const isRobustMode = inlierProb < 1;
+
+  let modelViewTransform = initialModelViewTransform;
+
+  let err0 = 0.0;
+  let err1 = 0.0;
+
+  let E = new Array(worldCoords.length);
+  let E2 = new Array(worldCoords.length);
+  let dxs = new Array(worldCoords.length);
+  let dys = new Array(worldCoords.length);
+
+  for (let l = 0; l <= ICP_MAX_LOOP; l++) {
+    const modelViewProjectionTransform = buildModelViewProjectionTransform(
+      projectionTransform,
+      modelViewTransform,
+    );
+
+    for (let n = 0; n < worldCoords.length; n++) {
+      const u = computeScreenCoordiate(
+        modelViewProjectionTransform,
+        worldCoords[n].x,
+        worldCoords[n].y,
+        worldCoords[n].z,
+      );
+      const dx = screenCoords[n].x - u.x;
+      const dy = screenCoords[n].y - u.y;
+
+      dxs[n] = dx;
+      dys[n] = dy;
+      E[n] = dx * dx + dy * dy;
+    }
+
+    let K2; // robust mode only
+    err1 = 0.0;
+    if (isRobustMode) {
+      const inlierNum = Math.max(3, Math.floor(worldCoords.length * inlierProb) - 1);
+      for (let n = 0; n < worldCoords.length; n++) {
+        E2[n] = E[n];
+      }
+      E2.sort((a, b) => {
+        return a - b;
+      });
+
+      K2 = Math.max(E2[inlierNum] * K2_FACTOR, 16.0);
+      for (let n = 0; n < worldCoords.length; n++) {
+        if (E2[n] > K2) err1 += K2 / 6;
+        else
+          err1 += (K2 / 6.0) * (1.0 - (1.0 - E2[n] / K2) * (1.0 - E2[n] / K2) * (1.0 - E2[n] / K2));
+      }
+    } else {
+      for (let n = 0; n < worldCoords.length; n++) {
+        err1 += E[n];
+      }
+    }
+    err1 /= worldCoords.length;
+
+    //console.log("icp loop", inlierProb, l, err1);
+
+    if (err1 < ICP_BREAK_LOOP_ERROR_THRESH) break;
+    //if (l > 0 && err1 < ICP_BREAK_LOOP_ERROR_THRESH2 && err1/err0 > ICP_BREAK_LOOP_ERROR_RATIO_THRESH) break;
+    if (l > 0 && err1 / err0 > ICP_BREAK_LOOP_ERROR_RATIO_THRESH) break;
+    if (l === ICP_MAX_LOOP) break;
+
+    err0 = err1;
+
+    const dU = [];
+    const allJ_U_S = [];
+    for (let n = 0; n < worldCoords.length; n++) {
+      if (isRobustMode && E[n] > K2) {
+        continue;
+      }
+
+      const J_U_S = _getJ_U_S({
+        modelViewProjectionTransform,
+        modelViewTransform,
+        projectionTransform,
+        worldCoord: worldCoords[n],
+      });
+
+      if (isRobustMode) {
+        const W = (1.0 - E[n] / K2) * (1.0 - E[n] / K2);
+
+        for (let j = 0; j < 2; j++) {
+          for (let i = 0; i < 6; i++) {
+            J_U_S[j][i] *= W;
+          }
+        }
+        dU.push([dxs[n] * W]);
+        dU.push([dys[n] * W]);
+      } else {
+        dU.push([dxs[n]]);
+        dU.push([dys[n]]);
+      }
+
+      for (let i = 0; i < J_U_S.length; i++) {
+        allJ_U_S.push(J_U_S[i]);
+      }
+    }
+
+    const dS = _getDeltaS({ dU, J_U_S: allJ_U_S });
+    if (dS === null) break;
+
+    modelViewTransform = _updateModelViewTransform({ modelViewTransform, dS });
+  }
+  return { modelViewTransform, err: err1 };
+};
+
+const _updateModelViewTransform = ({ modelViewTransform, dS }) => {
+  /**
+   * dS has 6 paragrams, first half is rotation, second half is translation
+   * rotation is expressed in angle-axis,
+   *   [S[0], S[1] ,S[2]] is the axis of rotation, and the magnitude is the angle
+   */
+  let ra = dS[0] * dS[0] + dS[1] * dS[1] + dS[2] * dS[2];
+  let q0, q1, q2;
+  if (ra < 0.000001) {
+    q0 = 1.0;
+    q1 = 0.0;
+    q2 = 0.0;
+    ra = 0.0;
+  } else {
+    ra = Math.sqrt(ra);
+    q0 = dS[0] / ra;
+    q1 = dS[1] / ra;
+    q2 = dS[2] / ra;
+  }
+
+  const cra = Math.cos(ra);
+  const sra = Math.sin(ra);
+  const one_cra = 1.0 - cra;
+
+  // mat is [R|t], 3D rotation and translation
+  mat[0][0] = q0 * q0 * one_cra + cra;
+  mat[0][1] = q0 * q1 * one_cra - q2 * sra;
+  mat[0][2] = q0 * q2 * one_cra + q1 * sra;
+  mat[0][3] = dS[3];
+  mat[1][0] = q1 * q0 * one_cra + q2 * sra;
+  mat[1][1] = q1 * q1 * one_cra + cra;
+  mat[1][2] = q1 * q2 * one_cra - q0 * sra;
+  mat[1][3] = dS[4];
+  mat[2][0] = q2 * q0 * one_cra - q1 * sra;
+  mat[2][1] = q2 * q1 * one_cra + q0 * sra;
+  mat[2][2] = q2 * q2 * one_cra + cra;
+  mat[2][3] = dS[5];
+
+  // the updated transform is the original transform x delta transform
+  const mat2 = [[], [], []];
+  for (let j = 0; j < 3; j++) {
+    for (let i = 0; i < 4; i++) {
+      mat2[j][i] =
+        modelViewTransform[j][0] * mat[0][i] +
+        modelViewTransform[j][1] * mat[1][i] +
+        modelViewTransform[j][2] * mat[2][i];
+    }
+    mat2[j][3] += modelViewTransform[j][3];
+  }
+  return mat2;
+};
+
+const _getDeltaS = ({ dU, J_U_S }) => {
+  const J = new Matrix(J_U_S);
+  const U = new Matrix(dU);
+
+  const JT = J.transpose();
+  const JTJ = JT.mmul(J);
+  const JTU = JT.mmul(U);
+
+  let JTJInv;
+  try {
+    JTJInv = inverse(JTJ);
+  } catch (e) {
+    return null;
+  }
+
+  const S = JTJInv.mmul(JTU);
+  return S.to1DArray();
+};
+
+const _getJ_U_S = ({
+  modelViewProjectionTransform,
+  modelViewTransform,
+  projectionTransform,
+  worldCoord,
+}) => {
+  const T = modelViewTransform;
+  const { x, y, z } = worldCoord;
+
+  const u = applyModelViewProjectionTransform(modelViewProjectionTransform, x, y, z);
+
+  const z2 = u.z * u.z;
+  // Question: This is the most confusing matrix to me. I've no idea how to derive this.
+  //J_U_Xc[0][0] = (projectionTransform[0][0] * u.z - projectionTransform[2][0] * u.x) / z2;
+  //J_U_Xc[0][1] = (projectionTransform[0][1] * u.z - projectionTransform[2][1] * u.x) / z2;
+  //J_U_Xc[0][2] = (projectionTransform[0][2] * u.z - projectionTransform[2][2] * u.x) / z2;
+  //J_U_Xc[1][0] = (projectionTransform[1][0] * u.z - projectionTransform[2][0] * u.y) / z2;
+  //J_U_Xc[1][1] = (projectionTransform[1][1] * u.z - projectionTransform[2][1] * u.y) / z2;
+  //J_U_Xc[1][2] = (projectionTransform[1][2] * u.z - projectionTransform[2][2] * u.y) / z2;
+
+  // The above is the original implementation, but simplify to below becuase projetionTransform[2][0] and [2][1] are zero
+  J_U_Xc[0][0] = (projectionTransform[0][0] * u.z) / z2;
+  J_U_Xc[0][1] = (projectionTransform[0][1] * u.z) / z2;
+  J_U_Xc[0][2] = (projectionTransform[0][2] * u.z - projectionTransform[2][2] * u.x) / z2;
+  J_U_Xc[1][0] = (projectionTransform[1][0] * u.z) / z2;
+  J_U_Xc[1][1] = (projectionTransform[1][1] * u.z) / z2;
+  J_U_Xc[1][2] = (projectionTransform[1][2] * u.z - projectionTransform[2][2] * u.y) / z2;
+
+  /*
+    J_Xc_S should be like this, but z is zero, so we can simplify
+    [T[0][2] * y - T[0][1] * z, T[0][0] * z - T[0][2] * x, T[0][1] * x - T[0][0] * y, T[0][0], T[0][1], T[0][2]],
+    [T[1][2] * y - T[1][1] * z, T[1][0] * z - T[1][2] * x, T[1][1] * x - T[1][0] * y, T[1][0], T[1][1], T[1][2]],
+    [T[2][2] * y - T[2][1] * z, T[2][0] * z - T[2][2] * x, T[2][1] * x - T[2][0] * y, T[2][0], T[2][1], T[2][2]],
+  */
+  J_Xc_S[0][0] = T[0][2] * y;
+  J_Xc_S[0][1] = -T[0][2] * x;
+  J_Xc_S[0][2] = T[0][1] * x - T[0][0] * y;
+  J_Xc_S[0][3] = T[0][0];
+  J_Xc_S[0][4] = T[0][1];
+  J_Xc_S[0][5] = T[0][2];
+
+  J_Xc_S[1][0] = T[1][2] * y;
+  J_Xc_S[1][1] = -T[1][2] * x;
+  J_Xc_S[1][2] = T[1][1] * x - T[1][0] * y;
+  J_Xc_S[1][3] = T[1][0];
+  J_Xc_S[1][4] = T[1][1];
+  J_Xc_S[1][5] = T[1][2];
+
+  J_Xc_S[2][0] = T[2][2] * y;
+  J_Xc_S[2][1] = -T[2][2] * x;
+  J_Xc_S[2][2] = T[2][1] * x - T[2][0] * y;
+  J_Xc_S[2][3] = T[2][0];
+  J_Xc_S[2][4] = T[2][1];
+  J_Xc_S[2][5] = T[2][2];
+
+  const J_U_S = [[], []];
+  for (let j = 0; j < 2; j++) {
+    for (let i = 0; i < 6; i++) {
+      J_U_S[j][i] = 0.0;
+      for (let k = 0; k < 3; k++) {
+        J_U_S[j][i] += J_U_Xc[j][k] * J_Xc_S[k][i];
+      }
+    }
+  }
+  return J_U_S;
+};
+
+export { refineEstimate };

package/src/compiler/estimation/utils.js

@@ -0,0 +1,97 @@
+const buildModelViewProjectionTransform = (projectionTransform, modelViewTransform) => {
+  // assume the projectTransform has the following format:
+  // [[fx, 0, cx],
+  //  [0, fy, cy]
+  //  [0, 0, 1]]
+  const modelViewProjectionTransform = [
+    [
+      projectionTransform[0][0] * modelViewTransform[0][0] +
+        projectionTransform[0][2] * modelViewTransform[2][0],
+      projectionTransform[0][0] * modelViewTransform[0][1] +
+        projectionTransform[0][2] * modelViewTransform[2][1],
+      projectionTransform[0][0] * modelViewTransform[0][2] +
+        projectionTransform[0][2] * modelViewTransform[2][2],
+      projectionTransform[0][0] * modelViewTransform[0][3] +
+        projectionTransform[0][2] * modelViewTransform[2][3],
+    ],
+    [
+      projectionTransform[1][1] * modelViewTransform[1][0] +
+        projectionTransform[1][2] * modelViewTransform[2][0],
+      projectionTransform[1][1] * modelViewTransform[1][1] +
+        projectionTransform[1][2] * modelViewTransform[2][1],
+      projectionTransform[1][1] * modelViewTransform[1][2] +
+        projectionTransform[1][2] * modelViewTransform[2][2],
+      projectionTransform[1][1] * modelViewTransform[1][3] +
+        projectionTransform[1][2] * modelViewTransform[2][3],
+    ],
+    [
+      modelViewTransform[2][0],
+      modelViewTransform[2][1],
+      modelViewTransform[2][2],
+      modelViewTransform[2][3],
+    ],
+  ];
+  return modelViewProjectionTransform;
+
+  /*
+  // this is the full computation if the projectTransform does not look like the expected format, but more computations
+  //  
+  const modelViewProjectionTransform = [[],[],[]];
+  for (let j = 0; j < 3; j++ ) {
+    for (let i = 0; i < 4; i++) {
+      modelViewProjectionTransform[j][i] = projectionTransform[j][0] * modelViewTransform[0][i]
+                                         + projectionTransform[j][1] * modelViewTransform[1][i]
+                                         + projectionTransform[j][2] * modelViewTransform[2][i];
+    }
+  }
+  return modelViewProjectionTransform;
+  */
+};
+
+const applyModelViewProjectionTransform = (modelViewProjectionTransform, x, y, z) => {
+  // assume z is zero
+  const ux =
+    modelViewProjectionTransform[0][0] * x +
+    modelViewProjectionTransform[0][1] * y +
+    modelViewProjectionTransform[0][3];
+  const uy =
+    modelViewProjectionTransform[1][0] * x +
+    modelViewProjectionTransform[1][1] * y +
+    modelViewProjectionTransform[1][3];
+  const uz =
+    modelViewProjectionTransform[2][0] * x +
+    modelViewProjectionTransform[2][1] * y +
+    modelViewProjectionTransform[2][3];
+  return { x: ux, y: uy, z: uz };
+};
+
+const computeScreenCoordiate = (modelViewProjectionTransform, x, y, z) => {
+  const {
+    x: ux,
+    y: uy,
+    z: uz,
+  } = applyModelViewProjectionTransform(modelViewProjectionTransform, x, y, z);
+  //if( Math.abs(uz) < 0.000001 ) return null;
+  return { x: ux / uz, y: uy / uz };
+};
+
+const screenToMarkerCoordinate = (modelViewProjectionTransform, sx, sy) => {
+  const c11 = modelViewProjectionTransform[2][0] * sx - modelViewProjectionTransform[0][0];
+  const c12 = modelViewProjectionTransform[2][1] * sx - modelViewProjectionTransform[0][1];
+  const c21 = modelViewProjectionTransform[2][0] * sy - modelViewProjectionTransform[1][0];
+  const c22 = modelViewProjectionTransform[2][1] * sy - modelViewProjectionTransform[1][1];
+  const b1 = modelViewProjectionTransform[0][3] - modelViewProjectionTransform[2][3] * sx;
+  const b2 = modelViewProjectionTransform[1][3] - modelViewProjectionTransform[2][3] * sy;
+
+  const m = c11 * c22 - c12 * c21;
+  return {
+    x: (c22 * b1 - c12 * b2) / m,
+    y: (c11 * b2 - c21 * b1) / m,
+  };
+};
+
+export {
+  buildModelViewProjectionTransform,
+  applyModelViewProjectionTransform,
+  computeScreenCoordiate,
+};

package/src/compiler/image-list.js

@@ -0,0 +1,62 @@
+import { resize } from "./utils/images.js";
+
+/**
+ * Tamaño mínimo de píxeles para el procesamiento de imágenes
+ * Un valor más bajo permite detectar imágenes más pequeñas pero aumenta el tiempo de procesamiento
+ * @constant {number}
+ */
+const MIN_IMAGE_PIXEL_SIZE = 100;
+
+/**
+ * Construye una lista de imágenes con diferentes escalas para detección de características
+ * @param {Object} inputImage - Imagen de entrada con propiedades width, height y data
+ * @returns {Array<Object>} Lista de imágenes escaladas con propiedades data, width, height y scale
+ */
+const buildImageList = (inputImage) => {
+  const minScale = MIN_IMAGE_PIXEL_SIZE / Math.min(inputImage.width, inputImage.height);
+
+  const scaleList = [];
+  let c = minScale;
+  while (true) {
+    scaleList.push(c);
+    c *= Math.pow(2.0, 1.0 / 3.0);
+    if (c >= 0.95) {
+      c = 1;
+      break;
+    }
+  }
+  scaleList.push(c);
+  scaleList.reverse();
+
+  const imageList = [];
+  for (let i = 0; i < scaleList.length; i++) {
+    const w = inputImage.width * scaleList[i];
+    const h = inputImage.height * scaleList[i];
+    imageList.push(
+      Object.assign(resize({ image: inputImage, ratio: scaleList[i] }), { scale: scaleList[i] }),
+    );
+  }
+  return imageList;
+};
+
+/**
+ * Construye una lista optimizada de imágenes para tracking
+ * Genera dos versiones escaladas (256px y 128px) para tracking eficiente
+ * @param {Object} inputImage - Imagen de entrada con propiedades width, height y data
+ * @returns {Array<Object>} Lista de imágenes escaladas para tracking
+ */
+const buildTrackingImageList = (inputImage) => {
+  const minDimension = Math.min(inputImage.width, inputImage.height);
+  const scaleList = [];
+  const imageList = [];
+  scaleList.push(256.0 / minDimension);
+  scaleList.push(128.0 / minDimension);
+  for (let i = 0; i < scaleList.length; i++) {
+    imageList.push(
+      Object.assign(resize({ image: inputImage, ratio: scaleList[i] }), { scale: scaleList[i] }),
+    );
+  }
+  return imageList;
+};
+
+export { buildImageList, buildTrackingImageList };

package/src/compiler/index.js

@@ -0,0 +1,13 @@
+import { Controller } from "./controller.js";
+import { Compiler } from "./compiler.js";
+
+export { Controller, Compiler };
+
+if (!window.MINDAR) {
+  window.MINDAR = {};
+}
+
+window.MINDAR.IMAGE = {
+  Controller,
+  Compiler,
+};

package/src/compiler/input-loader.js

@@ -0,0 +1,107 @@
+import * as tf from "@tensorflow/tfjs";
+
+// More efficient implementation for tf.browser.fromPixels
+//   original implementation: /node_modules/@tensorflow/tfjs-backend-webgl/src/kernels/FromPixels.ts
+//
+// This implementation return grey scale instead of RGBA in the orignal implementation
+
+class InputLoader {
+  constructor(width, height) {
+    this.width = width;
+    this.height = height;
+    this.texShape = [height, width];
+
+    const context = document.createElement("canvas").getContext("2d");
+    context.canvas.width = width;
+    context.canvas.height = height;
+    this.context = context;
+
+    this.program = this.buildProgram(width, height);
+
+    const backend = tf.backend();
+    //this.tempPixelHandle = backend.makeTensorInfo(this.texShape, 'int32');
+    this.tempPixelHandle = backend.makeTensorInfo(this.texShape, "float32");
+    // warning!!!
+    // usage type should be TextureUsage.PIXELS, but tfjs didn't export this enum type, so we hard-coded 2 here
+    //   i.e. backend.texData.get(tempPixelHandle.dataId).usage = TextureUsage.PIXELS;
+    backend.texData.get(this.tempPixelHandle.dataId).usage = 2;
+  }
+
+  // old method
+  _loadInput(input) {
+    return tf.tidy(() => {
+      let inputImage = tf.browser.fromPixels(input);
+      inputImage = inputImage.mean(2);
+      return inputImage;
+    });
+  }
+
+  // input is instance of HTMLVideoElement or HTMLImageElement
+  loadInput(input) {
+    const context = this.context;
+    context.clearRect(0, 0, this.context.canvas.width, this.context.canvas.height);
+
+    const isInputRotated = input.width === this.height && input.height === this.width;
+    if (isInputRotated) {
+      // rotate 90 degree and draw
+      let x = this.context.canvas.width / 2;
+      let y = this.context.canvas.height / 2;
+      let angleInDegrees = 90;
+
+      context.save(); // save the current context state
+      context.translate(x, y); // move the context origin to the center of the image
+      context.rotate((angleInDegrees * Math.PI) / 180); // rotate the context
+
+      // draw the image with its center at the origin
+      context.drawImage(input, -input.width / 2, -input.height / 2);
+      context.restore(); // restore the context to its original state
+    } else {
+      this.context.drawImage(input, 0, 0, input.width, input.height);
+    }
+
+    const backend = tf.backend();
+    backend.gpgpu.uploadPixelDataToTexture(
+      backend.getTexture(this.tempPixelHandle.dataId),
+      this.context.canvas,
+    );
+
+    //const res = backend.compileAndRun(this.program, [this.tempPixelHandle]);
+    const res = this._compileAndRun(this.program, [this.tempPixelHandle]);
+    //const res = this._runWebGLProgram(this.program, [this.tempPixelHandle], 'float32');
+    //backend.disposeData(tempPixelHandle.dataId);
+    return res;
+  }
+
+  buildProgram(width, height) {
+    const textureMethod = tf.env().getNumber("WEBGL_VERSION") === 2 ? "texture" : "texture2D";
+
+    const program = {
+      variableNames: ["A"],
+      outputShape: this.texShape,
+      userCode: `
+	void main() {
+	  ivec2 coords = getOutputCoords();
+	  int texR = coords[0];
+	  int texC = coords[1];
+	  vec2 uv = (vec2(texC, texR) + halfCR) / vec2(${width}.0, ${height}.0);
+
+	  vec4 values = ${textureMethod}(A, uv);
+	  setOutput((0.299 * values.r + 0.587 * values.g + 0.114 * values.b) * 255.0);
+	}
+      `,
+    };
+    return program;
+  }
+
+  _compileAndRun(program, inputs) {
+    const outInfo = tf.backend().compileAndRun(program, inputs);
+    return tf.engine().makeTensorFromDataId(outInfo.dataId, outInfo.shape, outInfo.dtype);
+  }
+
+  _runWebGLProgram(program, inputs, outputType) {
+    const outInfo = tf.backend().runWebGLProgram(program, inputs, outputType);
+    return tf.engine().makeTensorFromDataId(outInfo.dataId, outInfo.shape, outInfo.dtype);
+  }
+}
+
+export { InputLoader };

package/src/compiler/matching/hamming-distance.js

@@ -0,0 +1,23 @@
+// Fast computation on number of bit sets
+// Ref: https://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel
+const compute = (options) => {
+  const { v1, v2 } = options;
+  let d = 0;
+
+  for (let i = 0; i < v1.length; i++) {
+    let x = (v1[i] ^ v2[i]) >>> 0;
+    d += bitCount(x);
+  }
+  return d;
+};
+
+const bitCount = (v) => {
+  var c = v - ((v >> 1) & 0x55555555);
+  c = ((c >> 2) & 0x33333333) + (c & 0x33333333);
+  c = ((c >> 4) + c) & 0x0f0f0f0f;
+  c = ((c >> 8) + c) & 0x00ff00ff;
+  c = ((c >> 16) + c) & 0x0000ffff;
+  return c;
+};
+
+export { compute };