@srsergio/taptapp-ar 1.0.9 → 1.0.10

package/src/compiler/controller.js

@@ -1,13 +1,20 @@
-import { memory, nextFrame } from "@tensorflow/tfjs";
-
-const tf = { memory, nextFrame };
-import ControllerWorker from "./controller.worker.js?worker&inline";
 import { Tracker } from "./tracker/tracker.js";
 import { CropDetector } from "./detector/crop-detector.js";
-import { Compiler } from "./compiler.js";
+import { OfflineCompiler as Compiler } from "./offline-compiler.js";
 import { InputLoader } from "./input-loader.js";
 import { OneEuroFilter } from "../libs/one-euro-filter.js";
 
+let ControllerWorker;
+
+// Conditional import for worker to avoid crash in non-vite environments
+try {
+  const workerModule = await import("./controller.worker.js?worker&inline");
+  ControllerWorker = workerModule.default;
+} catch (e) {
+  // Fallback for tests or other environments
+  ControllerWorker = null;
+}
+
 const DEFAULT_FILTER_CUTOFF = 0.001; // 1Hz. time period in milliseconds
 const DEFAULT_FILTER_BETA = 1000;
 const DEFAULT_WARMUP_TOLERANCE = 5;
@@ -24,6 +31,7 @@ class Controller {
     missTolerance = null,
     filterMinCF = null,
     filterBeta = null,
+    worker = null, // Allow custom worker injection
   }) {
     this.inputWidth = inputWidth;
     this.inputHeight = inputHeight;
@@ -40,14 +48,14 @@ class Controller {
     this.processingVideo = false;
     this.interestedTargetIndex = -1;
     this.trackingStates = [];
+    this.worker = worker;
+    if (this.worker) this._setupWorkerListener();
 
     const near = 10;
     const far = 100000;
-    const fovy = (45.0 * Math.PI) / 180; // 45 in radian. field of view vertical
+    const fovy = (45.0 * Math.PI) / 180;
     const f = this.inputHeight / 2 / Math.tan(fovy / 2);
-    //     [fx  s cx]
-    // K = [ 0 fx cy]
-    //     [ 0  0  1]
+
     this.projectionTransform = [
       [f, 0, this.inputWidth / 2],
       [0, f, this.inputHeight / 2],
@@ -61,10 +69,10 @@ class Controller {
       near: near,
       far: far,
     });
+  }
 
-    this.worker = new ControllerWorker(); //new Worker(new URL('./controller.worker.js', import.meta.url));
-    this.workerMatchDone = null;
-    this.workerTrackDone = null;
+  _setupWorkerListener() {
+    if (!this.worker) return;
     this.worker.onmessage = (e) => {
       if (e.data.type === "matchDone" && this.workerMatchDone !== null) {
         this.workerMatchDone(e.data);
@@ -75,9 +83,12 @@ class Controller {
     };
   }
 
-  showTFStats() {
-    console.log(tf.memory().numTensors);
-    console.table(tf.memory());
+  _ensureWorker() {
+    if (this.worker) return;
+    if (ControllerWorker) {
+      this.worker = new ControllerWorker();
+      this._setupWorkerListener();
+    }
   }
 
   addImageTargets(fileURL) {
@@ -97,9 +108,10 @@ class Controller {
     const matchingDataList = [];
     const dimensions = [];
     for (let i = 0; i < dataList.length; i++) {
-      matchingDataList.push(dataList[i].matchingData);
-      trackingDataList.push(dataList[i].trackingData);
-      dimensions.push([dataList[i].targetImage.width, dataList[i].targetImage.height]);
+      const item = dataList[i];
+      matchingDataList.push(item.matchingData);
+      trackingDataList.push(item.trackingData);
+      dimensions.push([item.targetImage.width, item.targetImage.height]);
     }
 
     this.tracker = new Tracker(
@@ -111,33 +123,34 @@ class Controller {
       this.debugMode,
     );
 
-    this.worker.postMessage({
-      type: "setup",
-      inputWidth: this.inputWidth,
-      inputHeight: this.inputHeight,
-      projectionTransform: this.projectionTransform,
-      debugMode: this.debugMode,
-      matchingDataList,
-    });
+    this._ensureWorker();
+    if (this.worker) {
+      this.worker.postMessage({
+        type: "setup",
+        inputWidth: this.inputWidth,
+        inputHeight: this.inputHeight,
+        projectionTransform: this.projectionTransform,
+        debugMode: this.debugMode,
+        matchingDataList,
+      });
+    }
 
     this.markerDimensions = dimensions;
-
-    return { dimensions: dimensions, matchingDataList, trackingDataList };
+    return { dimensions, matchingDataList, trackingDataList };
   }
 
   dispose() {
     this.stopProcessVideo();
-    this.worker.postMessage({
-      type: "dispose",
-    });
+    if (this.worker) {
+      this.worker.postMessage({ type: "dispose" });
+      this.worker = null;
+    }
   }
 
-  // warm up gpu - build kernels is slow
   dummyRun(input) {
-    const inputT = this.inputLoader.loadInput(input);
-    this.cropDetector.detect(inputT);
-    this.tracker.dummyRun(inputT);
-    inputT.dispose();
+    const inputData = this.inputLoader.loadInput(input);
+    this.cropDetector.detect(inputData);
+    this.tracker.dummyRun(inputData);
   }
 
   getProjectionMatrix() {
@@ -176,17 +189,17 @@ class Controller {
     return this._glModelViewMatrix(modelViewTransform, targetIndex);
   }
 
-  async _detectAndMatch(inputT, targetIndexes) {
-    const { featurePoints } = this.cropDetector.detectMoving(inputT);
+  async _detectAndMatch(inputData, targetIndexes) {
+    const { featurePoints } = this.cropDetector.detectMoving(inputData);
     const { targetIndex: matchedTargetIndex, modelViewTransform } = await this._workerMatch(
       featurePoints,
       targetIndexes,
     );
     return { targetIndex: matchedTargetIndex, modelViewTransform };
   }
-  async _trackAndUpdate(inputT, lastModelViewTransform, targetIndex) {
+  async _trackAndUpdate(inputData, lastModelViewTransform, targetIndex) {
     const { worldCoords, screenCoords } = this.tracker.track(
-      inputT,
+      inputData,
       lastModelViewTransform,
       targetIndex,
     );
@@ -213,14 +226,13 @@ class Controller {
         trackMiss: 0,
         filter: new OneEuroFilter({ minCutOff: this.filterMinCF, beta: this.filterBeta }),
       });
-      //console.log("filterMinCF", this.filterMinCF, this.filterBeta);
     }
 
     const startProcessing = async () => {
       while (true) {
         if (!this.processingVideo) break;
 
-        const inputT = this.inputLoader.loadInput(input);
+        const inputData = this.inputLoader.loadInput(input);
 
         const nTracking = this.trackingStates.reduce((acc, s) => {
           return acc + (!!s.isTracking ? 1 : 0);
@@ -238,7 +250,7 @@ class Controller {
           }
 
           const { targetIndex: matchedTargetIndex, modelViewTransform } =
-            await this._detectAndMatch(inputT, matchingIndexes);
+            await this._detectAndMatch(inputData, matchingIndexes);
 
           if (matchedTargetIndex !== -1) {
             this.trackingStates[matchedTargetIndex].isTracking = true;
@@ -252,7 +264,7 @@ class Controller {
 
           if (trackingState.isTracking) {
             let modelViewTransform = await this._trackAndUpdate(
-              inputT,
+              inputData,
               trackingState.currentModelViewTransform,
               i,
             );
@@ -314,9 +326,14 @@ class Controller {
           }
         }
 
-        inputT.dispose();
         this.onUpdate && this.onUpdate({ type: "processDone" });
-        await tf.nextFrame();
+
+        // Use requestAnimationFrame if available, otherwise just wait briefly
+        if (typeof requestAnimationFrame !== "undefined") {
+          await new Promise(requestAnimationFrame);
+        } else {
+          await new Promise(resolve => setTimeout(resolve, 16));
+        }
       }
     };
     startProcessing();
@@ -327,23 +344,21 @@ class Controller {
   }
 
   async detect(input) {
-    const inputT = this.inputLoader.loadInput(input);
-    const { featurePoints, debugExtra } = this.cropDetector.detect(inputT);
-    inputT.dispose();
+    const inputData = this.inputLoader.loadInput(input);
+    const { featurePoints, debugExtra } = this.cropDetector.detect(inputData);
     return { featurePoints, debugExtra };
   }
 
   async match(featurePoints, targetIndex) {
-    const { modelViewTransform, debugExtra } = await this._workerMatch(featurePoints, [
+    const { targetIndex: matchedTargetIndex, modelViewTransform, debugExtra } = await this._workerMatch(featurePoints, [
       targetIndex,
     ]);
-    return { modelViewTransform, debugExtra };
+    return { targetIndex: matchedTargetIndex, modelViewTransform, debugExtra };
   }
 
   async track(input, modelViewTransform, targetIndex) {
-    const inputT = this.inputLoader.loadInput(input);
-    const result = this.tracker.track(inputT, modelViewTransform, targetIndex);
-    inputT.dispose();
+    const inputData = this.inputLoader.loadInput(input);
+    const result = this.tracker.track(inputData, modelViewTransform, targetIndex);
     return result;
   }
 
@@ -362,7 +377,7 @@ class Controller {
           debugExtra: data.debugExtra,
         });
       };
-      this.worker.postMessage({ type: "match", featurePoints: featurePoints, targetIndexes });
+      this.worker && this.worker.postMessage({ type: "match", featurePoints: featurePoints, targetIndexes });
     });
   }
 
@@ -372,7 +387,7 @@ class Controller {
         resolve(data.modelViewTransform);
       };
       const { worldCoords, screenCoords } = trackingFeatures;
-      this.worker.postMessage({
+      this.worker && this.worker.postMessage({
         type: "trackUpdate",
         modelViewTransform,
         worldCoords,
@@ -384,41 +399,6 @@ class Controller {
   _glModelViewMatrix(modelViewTransform, targetIndex) {
     const height = this.markerDimensions[targetIndex][1];
 
-    // Question: can someone verify this interpreation is correct?
-    // I'm not very convinced, but more like trial and error and works......
-    //
-    // First, opengl has y coordinate system go from bottom to top, while the marker corrdinate goes from top to bottom,
-    //    since the modelViewTransform is estimated in marker coordinate, we need to apply this transform before modelViewTransform
-    //    I can see why y = h - y*, but why z = z* ? should we intepret it as rotate 90 deg along x-axis and then translate y by h?
-    //
-    //    [1  0  0  0]
-    //    [0 -1  0  h]
-    //    [0  0 -1  0]
-    //    [0  0  0  1]
-    //
-    //    This is tested that if we reverse marker coordinate from bottom to top and estimate the modelViewTransform,
-    //    then the above matrix is not necessary.
-    //
-    // Second, in opengl, positive z is away from camera, so we rotate 90 deg along x-axis after transform to fix the axis mismatch
-    //    [1  1  0  0]
-    //    [0 -1  0  0]
-    //    [0  0 -1  0]
-    //    [0  0  0  1]
-    //
-    // all together, the combined matrix is
-    //
-    //    [1  1  0  0]   [m00, m01, m02, m03]   [1  0  0  0]
-    //    [0 -1  0  0]   [m10, m11, m12, m13]   [0 -1  0  h]
-    //    [0  0 -1  0]   [m20, m21, m22, m23]   [0  0 -1  0]
-    //    [0  0  0  1]   [  0    0    0    1]   [0  0  0  1]
-    //
-    //    [ m00,  -m01,  -m02,  (m01 * h + m03) ]
-    //    [-m10,   m11,   m12, -(m11 * h + m13) ]
-    //  = [-m20,   m21,   m22, -(m21 * h + m23) ]
-    //    [   0,     0,     0,                1 ]
-    //
-    //
-    // Finally, in threejs, matrix is represented in col by row, so we transpose it, and get below:
     const openGLWorldMatrix = [
       modelViewTransform[0][0],
       -modelViewTransform[1][0],
@@ -440,8 +420,6 @@ class Controller {
     return openGLWorldMatrix;
   }
 
-  // build openGL projection matrix
-  // ref: https://strawlab.org/2011/11/05/augmented-reality-with-OpenGL/
   _glProjectionMatrix({ projectionTransform, width, height, near, far }) {
     const proj = [
       [

package/src/compiler/detector/crop-detector.js

@@ -1,4 +1,4 @@
-import { Detector } from "./detector.js";
+import { DetectorLite } from "./detector-lite.js";
 
 class CropDetector {
   constructor(width, height, debugMode = false) {
@@ -11,17 +11,18 @@ class CropDetector {
     let cropSize = Math.pow(2, Math.round(Math.log(minDimension) / Math.log(2)));
     this.cropSize = cropSize;
 
-    this.detector = new Detector(cropSize, cropSize, debugMode);
+    this.detector = new DetectorLite(cropSize, cropSize);
 
-    this.kernelCaches = {};
     this.lastRandomIndex = 4;
   }
 
-  detect(inputImageT) {
+  detect(input) {
+    const imageData = input;
+
     // crop center
     const startY = Math.floor(this.height / 2 - this.cropSize / 2);
     const startX = Math.floor(this.width / 2 - this.cropSize / 2);
-    const result = this._detect(inputImageT, startX, startY);
+    const result = this._detect(imageData, startX, startY);
 
     if (this.debugMode) {
       result.debugExtra.crop = { startX, startY, cropSize: this.cropSize };
@@ -29,7 +30,9 @@ class CropDetector {
     return result;
   }
 
-  detectMoving(inputImageT) {
+  detectMoving(input) {
+    const imageData = input;
+
     // loop a few locations around center
     const dx = this.lastRandomIndex % 3;
     const dy = Math.floor(this.lastRandomIndex / 3);
@@ -44,22 +47,30 @@ class CropDetector {
 
     this.lastRandomIndex = (this.lastRandomIndex + 1) % 9;
 
-    const result = this._detect(inputImageT, startX, startY);
+    const result = this._detect(imageData, startX, startY);
     return result;
   }
 
-  _detect(inputImageT, startX, startY) {
-    const cropInputImageT = inputImageT.slice([startY, startX], [this.cropSize, this.cropSize]);
-    const { featurePoints, debugExtra } = this.detector.detect(cropInputImageT);
+  _detect(imageData, startX, startY) {
+    // Crop manually since imageData is now a flat array (width * height)
+    const croppedData = new Float32Array(this.cropSize * this.cropSize);
+    for (let y = 0; y < this.cropSize; y++) {
+      for (let x = 0; x < this.cropSize; x++) {
+        croppedData[y * this.cropSize + x] = imageData[(startY + y) * this.width + (startX + x)];
+      }
+    }
+
+    const { featurePoints } = this.detector.detect(croppedData);
+
     featurePoints.forEach((p) => {
       p.x += startX;
       p.y += startY;
     });
-    if (this.debugMode) {
-      debugExtra.projectedImage = cropInputImageT.arraySync();
-    }
-    cropInputImageT.dispose();
-    return { featurePoints: featurePoints, debugExtra };
+
+    return {
+      featurePoints,
+      debugExtra: this.debugMode ? { projectedImage: Array.from(croppedData) } : {}
+    };
   }
 }
 

package/src/compiler/input-loader.js

@@ -1,106 +1,80 @@
-import * as tf from "@tensorflow/tfjs";
-
-// More efficient implementation for tf.browser.fromPixels
-//   original implementation: /node_modules/@tensorflow/tfjs-backend-webgl/src/kernels/FromPixels.ts
-//
-// This implementation return grey scale instead of RGBA in the orignal implementation
-
+/**
+ * InputLoader - Maneja la carga de imágenes y video sin TensorFlow
+ */
 class InputLoader {
   constructor(width, height) {
     this.width = width;
     this.height = height;
-    this.texShape = [height, width];
-
-    const context = document.createElement("canvas").getContext("2d");
-    context.canvas.width = width;
-    context.canvas.height = height;
-    this.context = context;
+    this.grayscaleBuffer = new Uint8Array(width * height);
 
-    this.program = this.buildProgram(width, height);
-
-    const backend = tf.backend();
-    //this.tempPixelHandle = backend.makeTensorInfo(this.texShape, 'int32');
-    this.tempPixelHandle = backend.makeTensorInfo(this.texShape, "float32");
-    // warning!!!
-    // usage type should be TextureUsage.PIXELS, but tfjs didn't export this enum type, so we hard-coded 2 here
-    //   i.e. backend.texData.get(tempPixelHandle.dataId).usage = TextureUsage.PIXELS;
-    backend.texData.get(this.tempPixelHandle.dataId).usage = 2;
-  }
-
-  // old method
-  _loadInput(input) {
-    return tf.tidy(() => {
-      let inputImage = tf.browser.fromPixels(input);
-      inputImage = inputImage.mean(2);
-      return inputImage;
-    });
+    if (typeof document !== "undefined") {
+      const canvas = document.createElement("canvas");
+      canvas.width = width;
+      canvas.height = height;
+      this.context = canvas.getContext("2d", { willReadFrequently: true, alpha: false });
+    }
   }
 
-  // input is instance of HTMLVideoElement or HTMLImageElement
+  /**
+   * Carga una imagen o video y devuelve los datos en escala de grises
+   * @param {HTMLVideoElement|HTMLImageElement|ImageData|Uint8Array} input - La fuente de entrada
+   * @returns {Uint8Array} Datos de imagen en escala de grises (width * height)
+   */
   loadInput(input) {
-    const context = this.context;
-    context.clearRect(0, 0, this.context.canvas.width, this.context.canvas.height);
-
-    const isInputRotated = input.width === this.height && input.height === this.width;
-    if (isInputRotated) {
-      // rotate 90 degree and draw
-      let x = this.context.canvas.width / 2;
-      let y = this.context.canvas.height / 2;
-      let angleInDegrees = 90;
-
-      context.save(); // save the current context state
-      context.translate(x, y); // move the context origin to the center of the image
-      context.rotate((angleInDegrees * Math.PI) / 180); // rotate the context
-
-      // draw the image with its center at the origin
-      context.drawImage(input, -input.width / 2, -input.height / 2);
-      context.restore(); // restore the context to its original state
-    } else {
-      this.context.drawImage(input, 0, 0, input.width, input.height);
+    // Si ya es un Uint8Array de escala de grises, lo devolvemos
+    if (input instanceof Uint8Array && input.length === this.width * this.height) {
+      return input;
     }
 
-    const backend = tf.backend();
-    backend.gpgpu.uploadPixelDataToTexture(
-      backend.getTexture(this.tempPixelHandle.dataId),
-      this.context.canvas,
-    );
-
-    //const res = backend.compileAndRun(this.program, [this.tempPixelHandle]);
-    const res = this._compileAndRun(this.program, [this.tempPixelHandle]);
-    //const res = this._runWebGLProgram(this.program, [this.tempPixelHandle], 'float32');
-    //backend.disposeData(tempPixelHandle.dataId);
-    return res;
-  }
-
-  buildProgram(width, height) {
-    const textureMethod = tf.env().getNumber("WEBGL_VERSION") === 2 ? "texture" : "texture2D";
+    // Si es ImageData, convertimos a escala de grises directamente
+    if (typeof ImageData !== "undefined" && input instanceof ImageData) {
+      this._convertToGrayscale(input.data, input.width, input.height);
+      return this.grayscaleBuffer;
+    }
 
-    const program = {
-      variableNames: ["A"],
-      outputShape: this.texShape,
-      userCode: `
-	void main() {
-	  ivec2 coords = getOutputCoords();
-	  int texR = coords[0];
-	  int texC = coords[1];
-	  vec2 uv = (vec2(texC, texR) + halfCR) / vec2(${width}.0, ${height}.0);
+    // En el navegador, usamos canvas para procesar video/imágenes
+    if (this.context) {
+      this.context.clearRect(0, 0, this.width, this.height);
+
+      const isInputRotated = input.width === this.height && input.height === this.width;
+
+      if (isInputRotated) {
+        this.context.save();
+        this.context.translate(this.width / 2, this.height / 2);
+        this.context.rotate(Math.PI / 2);
+        this.context.drawImage(input, -input.width / 2, -input.height / 2);
+        this.context.restore();
+      } else {
+        this.context.drawImage(input, 0, 0, this.width, this.height);
+      }
+
+      const imageData = this.context.getImageData(0, 0, this.width, this.height);
+      this._convertToGrayscale(imageData.data, this.width, this.height);
+      return this.grayscaleBuffer;
+    }
 
-	  vec4 values = ${textureMethod}(A, uv);
-	  setOutput((0.299 * values.r + 0.587 * values.g + 0.114 * values.b) * 255.0);
-	}
-      `,
-    };
-    return program;
-  }
+    // Fallback para Node.js o entornos sin DOM
+    if (input.data && input.data instanceof Uint8Array) {
+      this._convertToGrayscale(input.data, input.width || this.width, input.height || this.height);
+      return this.grayscaleBuffer;
+    }
 
-  _compileAndRun(program, inputs) {
-    const outInfo = tf.backend().compileAndRun(program, inputs);
-    return tf.engine().makeTensor(outInfo.dataId, outInfo.shape, outInfo.dtype);
+    throw new Error("Input no soportado o entorno sin Canvas");
   }
 
-  _runWebGLProgram(program, inputs, outputType) {
-    const outInfo = tf.backend().runWebGLProgram(program, inputs, outputType);
-    return tf.engine().makeTensor(outInfo.dataId, outInfo.shape, outInfo.dtype);
+  /**
+   * Convierte datos RGBA a escala de grises optimizada (reutilizando buffer)
+   */
+  _convertToGrayscale(rgbaData, width, height) {
+    const grayscale = this.grayscaleBuffer;
+    const len = (width * height);
+
+    // Optimized loop with bitwise ops
+    for (let i = 0; i < len; i++) {
+      const offset = i << 2;
+      // Formula de luminosidad estándar: 0.299R + 0.587G + 0.114B (scaled by 256)
+      grayscale[i] = (rgbaData[offset] * 77 + rgbaData[offset + 1] * 150 + rgbaData[offset + 2] * 29) >> 8;
+    }
   }
 }
 

package/src/compiler/matching/hamming-distance.js

@@ -7,11 +7,11 @@ for (let i = 0; i < 256; i++) {
 }
 
 const compute = (options) => {
-  const { v1, v2 } = options;
+  const { v1, v2, v1Offset = 0, v2Offset = 0 } = options;
   let d = 0;
-  const len = v1.length;
-  for (let i = 0; i < len; i++) {
-    d += BIT_COUNT_8[v1[i] ^ v2[i]];
+  // FREAK descriptors are 84 bytes
+  for (let i = 0; i < 84; i++) {
+    d += BIT_COUNT_8[v1[v1Offset + i] ^ v2[v2Offset + i]];
   }
   return d;
 };

package/src/compiler/matching/hough.js

@@ -35,7 +35,7 @@ const computeHoughMatches = (options) => {
   });
   const medianProjectedDim =
     projectedDims[
-      Math.floor(projectedDims.length / 2) - (projectedDims.length % 2 == 0 ? 1 : 0) - 1
+    Math.floor(projectedDims.length / 2) - (projectedDims.length % 2 == 0 ? 1 : 0) - 1
     ];
 
   const binSize = 0.25 * medianProjectedDim;

package/src/compiler/matching/matcher.js

@@ -47,8 +47,8 @@ class Matcher {
         y: querypoint.y,
       });
       worldCoords.push({
-        x: (keypoint.x + 0.5) / keyframe.scale,
-        y: (keypoint.y + 0.5) / keyframe.scale,
+        x: (keypoint.x + 0.5) / (keyframe.s || keyframe.scale || 1.0),
+        y: (keypoint.y + 0.5) / (keyframe.s || keyframe.scale || 1.0),
         z: 0,
       });
     }