@srsergio/taptapp-ar 1.0.73 → 1.0.75

package/dist/compiler/controller.js

@@ -18,7 +18,7 @@ const getControllerWorker = async () => {
     }
 };
 ControllerWorker = await getControllerWorker();
-const DEFAULT_FILTER_CUTOFF = 1.0;
+const DEFAULT_FILTER_CUTOFF = 0.5;
 const DEFAULT_FILTER_BETA = 0.1;
 const DEFAULT_WARMUP_TOLERANCE = 8;
 const DEFAULT_MISS_TOLERANCE = 2;

package/dist/compiler/detector/crop-detector.d.ts

@@ -22,6 +22,19 @@ export class CropDetector {
             projectedImage?: undefined;
         };
     };
+    /**
+     * Scans the ENTIRE frame by downsampling it to cropSize
+     */
+    _detectGlobal(imageData: any): {
+        featurePoints: any[];
+        debugExtra: {
+            projectedImage: number[];
+            isGlobal: boolean;
+        } | {
+            projectedImage?: undefined;
+            isGlobal?: undefined;
+        };
+    };
     _detect(imageData: any, startX: any, startY: any): {
         featurePoints: any[];
         debugExtra: {

package/dist/compiler/detector/crop-detector.js

@@ -24,27 +24,51 @@ class CropDetector {
     }
     detectMoving(input) {
         const imageData = input;
-        // Expanded to 5x5 grid (25 positions) for better coverage
+        // 🚀 MOONSHOT: Alternate between local crops and GLOBAL scan
+        // This solves the "not reading the whole screen" issue.
+        // Every 3 frames, we do a full screen downsampled scan.
+        if (this.lastRandomIndex % 3 === 0) {
+            this.lastRandomIndex = (this.lastRandomIndex + 1) % 25;
+            return this._detectGlobal(imageData);
+        }
+        // Original moving crop logic for high-detail local detection
         const gridSize = 5;
-        const dx = this.lastRandomIndex % gridSize;
-        const dy = Math.floor(this.lastRandomIndex / gridSize);
-        // Calculate offset from center, with overlap for better detection
+        const idx = (this.lastRandomIndex - 1) % (gridSize * gridSize);
+        const dx = idx % gridSize;
+        const dy = Math.floor(idx / gridSize);
         const stepX = this.cropSize / 3;
         const stepY = this.cropSize / 3;
         let startY = Math.floor(this.height / 2 - this.cropSize / 2 + (dy - 2) * stepY);
         let startX = Math.floor(this.width / 2 - this.cropSize / 2 + (dx - 2) * stepX);
-        // Clamp to valid bounds
-        if (startX < 0)
-            startX = 0;
-        if (startY < 0)
-            startY = 0;
-        if (startX >= this.width - this.cropSize)
-            startX = this.width - this.cropSize - 1;
-        if (startY >= this.height - this.cropSize)
-            startY = this.height - this.cropSize - 1;
-        this.lastRandomIndex = (this.lastRandomIndex + 1) % (gridSize * gridSize);
-        const result = this._detect(imageData, startX, startY);
-        return result;
+        startX = Math.max(0, Math.min(this.width - this.cropSize - 1, startX));
+        startY = Math.max(0, Math.min(this.height - this.cropSize - 1, startY));
+        this.lastRandomIndex = (this.lastRandomIndex + 1) % 25;
+        return this._detect(imageData, startX, startY);
+    }
+    /**
+     * Scans the ENTIRE frame by downsampling it to cropSize
+     */
+    _detectGlobal(imageData) {
+        const croppedData = new Float32Array(this.cropSize * this.cropSize);
+        const scaleX = this.width / this.cropSize;
+        const scaleY = this.height / this.cropSize;
+        // Fast downsample (nearest neighbor is enough for initial feature detection)
+        for (let y = 0; y < this.cropSize; y++) {
+            const srcY = Math.floor(y * scaleY) * this.width;
+            const dstY = y * this.cropSize;
+            for (let x = 0; x < this.cropSize; x++) {
+                croppedData[dstY + x] = imageData[srcY + Math.floor(x * scaleX)];
+            }
+        }
+        const { featurePoints } = this.detector.detect(croppedData);
+        featurePoints.forEach((p) => {
+            p.x *= scaleX;
+            p.y *= scaleY;
+        });
+        return {
+            featurePoints,
+            debugExtra: this.debugMode ? { projectedImage: Array.from(croppedData), isGlobal: true } : {}
+        };
     }
     _detect(imageData, startX, startY) {
         // Crop manually since imageData is now a flat array (width * height)

package/dist/compiler/simple-ar.js

@@ -123,7 +123,7 @@ class SimpleAR {
             }
             this.lastMatrix = worldMatrix;
             if (!this.filters[targetIndex]) {
-                this.filters[targetIndex] = new OneEuroFilter({ minCutOff: 1.0, beta: 0.1 });
+                this.filters[targetIndex] = new OneEuroFilter({ minCutOff: 0.8, beta: 0.2 });
             }
             const flatMVT = [
                 modelViewTransform[0][0], modelViewTransform[0][1], modelViewTransform[0][2], modelViewTransform[0][3],

package/dist/compiler/tracker/tracker.js

@@ -1,7 +1,7 @@
 import { buildModelViewProjectionTransform, computeScreenCoordiate } from "../estimation/utils.js";
 const AR2_DEFAULT_TS = 6;
 const AR2_DEFAULT_TS_GAP = 1;
-const AR2_SEARCH_SIZE = 18;
+const AR2_SEARCH_SIZE = 34; // Increased from 18 to 34 for much better fast-motion tracking
 const AR2_SEARCH_GAP = 1;
 const AR2_SIM_THRESH = 0.6;
 const TRACKING_KEYFRAME = 0; // 0: 128px (optimized)
@@ -165,18 +165,18 @@ class Tracker {
                 matchingPoints.push([bestX, bestY]);
                 continue;
             }
-            // Search in projected image
-            for (let sy = -searchOneSize; sy <= searchOneSize; sy += searchGap) {
+            // 🚀 MOONSHOT: Coarse-to-Fine Search for MAXIMUM FPS
+            // Step 1: Coarse search (Gap 4)
+            const coarseGap = 4;
+            for (let sy = -searchOneSize; sy <= searchOneSize; sy += coarseGap) {
                 const cy = sCenterY + sy;
                 if (cy < templateOneSize || cy >= markerHeight - templateOneSize)
                     continue;
-                for (let sx = -searchOneSize; sx <= searchOneSize; sx += searchGap) {
+                for (let sx = -searchOneSize; sx <= searchOneSize; sx += coarseGap) {
                     const cx = sCenterX + sx;
                     if (cx < templateOneSize || cx >= markerWidth - templateOneSize)
                         continue;
-                    let sumI = 0;
-                    let sumI2 = 0;
-                    let sumIT = 0;
+                    let sumI = 0, sumI2 = 0, sumIT = 0;
                     for (let ty = -templateOneSize; ty <= templateOneSize; ty++) {
                         const rowOffset = (cy + ty) * markerWidth;
                         const tRowOffset = (ty + templateOneSize) * templateSize;
@@ -199,6 +199,43 @@ class Tracker {
                     }
                 }
             }
+            // Step 2: Fine refinement (Gap 1) only around the best coarse match
+            if (bestSim > AR2_SIM_THRESH) {
+                const fineCenterX = (bestX * scale) | 0;
+                const fineCenterY = (bestY * scale) | 0;
+                const fineSearch = coarseGap;
+                for (let sy = -fineSearch; sy <= fineSearch; sy++) {
+                    const cy = fineCenterY + sy;
+                    if (cy < templateOneSize || cy >= markerHeight - templateOneSize)
+                        continue;
+                    for (let sx = -fineSearch; sx <= fineSearch; sx++) {
+                        const cx = fineCenterX + sx;
+                        if (cx < templateOneSize || cx >= markerWidth - templateOneSize)
+                            continue;
+                        let sumI = 0, sumI2 = 0, sumIT = 0;
+                        for (let ty = -templateOneSize; ty <= templateOneSize; ty++) {
+                            const rowOffset = (cy + ty) * markerWidth;
+                            const tRowOffset = (ty + templateOneSize) * templateSize;
+                            for (let tx = -templateOneSize; tx <= templateOneSize; tx++) {
+                                const valI = projectedImage[rowOffset + (cx + tx)];
+                                const valT = templateData[tRowOffset + (tx + templateOneSize)];
+                                sumI += valI;
+                                sumI2 += valI * valI;
+                                sumIT += valI * valT;
+                            }
+                        }
+                        const varI = Math.sqrt(Math.max(0, sumI2 - (sumI * sumI) * oneOverNPixels));
+                        if (varI < 0.0001)
+                            continue;
+                        const sim = (sumIT - (sumI * sumT) * oneOverNPixels) / (varI * varT);
+                        if (sim > bestSim) {
+                            bestSim = sim;
+                            bestX = cx / scale;
+                            bestY = cy / scale;
+                        }
+                    }
+                }
+            }
             sims[f] = bestSim;
             matchingPoints.push([bestX, bestY]);
         }

package/dist/react/TaptappAR.js

@@ -92,6 +92,9 @@ export const TaptappAR = ({ config, className = "", showScanningOverlay = true,
                     display: block;
                     width: 100%;
                     height: auto;
+                    will-change: transform;
+                    /* Visual interpolation to smooth out frame gaps */
+                    transition: transform 0.1s linear, opacity 0.3s ease;
                 }
             ` })] }));
 };

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@srsergio/taptapp-ar",
-    "version": "1.0.73",
+    "version": "1.0.75",
     "description": "AR Compiler for Node.js and Browser",
     "repository": {
         "type": "git",

package/src/compiler/controller.ts

@@ -19,7 +19,7 @@ const getControllerWorker = async () => {
 };
 ControllerWorker = await getControllerWorker();
 
-const DEFAULT_FILTER_CUTOFF = 1.0;
+const DEFAULT_FILTER_CUTOFF = 0.5;
 const DEFAULT_FILTER_BETA = 0.1;
 const DEFAULT_WARMUP_TOLERANCE = 8;
 const DEFAULT_MISS_TOLERANCE = 2;

package/src/compiler/detector/crop-detector.js

@@ -33,28 +33,62 @@ class CropDetector {
   detectMoving(input) {
     const imageData = input;
 
-    // Expanded to 5x5 grid (25 positions) for better coverage
+    // 🚀 MOONSHOT: Alternate between local crops and GLOBAL scan
+    // This solves the "not reading the whole screen" issue.
+    // Every 3 frames, we do a full screen downsampled scan.
+    if (this.lastRandomIndex % 3 === 0) {
+      this.lastRandomIndex = (this.lastRandomIndex + 1) % 25;
+      return this._detectGlobal(imageData);
+    }
+
+    // Original moving crop logic for high-detail local detection
     const gridSize = 5;
-    const dx = this.lastRandomIndex % gridSize;
-    const dy = Math.floor(this.lastRandomIndex / gridSize);
+    const idx = (this.lastRandomIndex - 1) % (gridSize * gridSize);
+    const dx = idx % gridSize;
+    const dy = Math.floor(idx / gridSize);
 
-    // Calculate offset from center, with overlap for better detection
     const stepX = this.cropSize / 3;
     const stepY = this.cropSize / 3;
 
     let startY = Math.floor(this.height / 2 - this.cropSize / 2 + (dy - 2) * stepY);
     let startX = Math.floor(this.width / 2 - this.cropSize / 2 + (dx - 2) * stepX);
 
-    // Clamp to valid bounds
-    if (startX < 0) startX = 0;
-    if (startY < 0) startY = 0;
-    if (startX >= this.width - this.cropSize) startX = this.width - this.cropSize - 1;
-    if (startY >= this.height - this.cropSize) startY = this.height - this.cropSize - 1;
+    startX = Math.max(0, Math.min(this.width - this.cropSize - 1, startX));
+    startY = Math.max(0, Math.min(this.height - this.cropSize - 1, startY));
 
-    this.lastRandomIndex = (this.lastRandomIndex + 1) % (gridSize * gridSize);
+    this.lastRandomIndex = (this.lastRandomIndex + 1) % 25;
 
-    const result = this._detect(imageData, startX, startY);
-    return result;
+    return this._detect(imageData, startX, startY);
+  }
+
+  /**
+   * Scans the ENTIRE frame by downsampling it to cropSize
+   */
+  _detectGlobal(imageData) {
+    const croppedData = new Float32Array(this.cropSize * this.cropSize);
+    const scaleX = this.width / this.cropSize;
+    const scaleY = this.height / this.cropSize;
+
+    // Fast downsample (nearest neighbor is enough for initial feature detection)
+    for (let y = 0; y < this.cropSize; y++) {
+      const srcY = Math.floor(y * scaleY) * this.width;
+      const dstY = y * this.cropSize;
+      for (let x = 0; x < this.cropSize; x++) {
+        croppedData[dstY + x] = imageData[srcY + Math.floor(x * scaleX)];
+      }
+    }
+
+    const { featurePoints } = this.detector.detect(croppedData);
+
+    featurePoints.forEach((p) => {
+      p.x *= scaleX;
+      p.y *= scaleY;
+    });
+
+    return {
+      featurePoints,
+      debugExtra: this.debugMode ? { projectedImage: Array.from(croppedData), isGlobal: true } : {}
+    };
   }
 
   _detect(imageData, startX, startY) {

package/src/compiler/simple-ar.ts

@@ -163,7 +163,7 @@ class SimpleAR {
             this.lastMatrix = worldMatrix;
 
             if (!this.filters[targetIndex]) {
-                this.filters[targetIndex] = new OneEuroFilter({ minCutOff: 1.0, beta: 0.1 });
+                this.filters[targetIndex] = new OneEuroFilter({ minCutOff: 0.8, beta: 0.2 });
             }
 
             const flatMVT = [

package/src/compiler/tracker/tracker.js

@@ -2,7 +2,7 @@ import { buildModelViewProjectionTransform, computeScreenCoordiate } from "../es
 
 const AR2_DEFAULT_TS = 6;
 const AR2_DEFAULT_TS_GAP = 1;
-const AR2_SEARCH_SIZE = 18;
+const AR2_SEARCH_SIZE = 34; // Increased from 18 to 34 for much better fast-motion tracking
 const AR2_SEARCH_GAP = 1;
 const AR2_SIM_THRESH = 0.6;
 
@@ -214,29 +214,25 @@ class Tracker {
         continue;
       }
 
-      // Search in projected image
-      for (let sy = -searchOneSize; sy <= searchOneSize; sy += searchGap) {
+      // 🚀 MOONSHOT: Coarse-to-Fine Search for MAXIMUM FPS
+      // Step 1: Coarse search (Gap 4)
+      const coarseGap = 4;
+      for (let sy = -searchOneSize; sy <= searchOneSize; sy += coarseGap) {
         const cy = sCenterY + sy;
         if (cy < templateOneSize || cy >= markerHeight - templateOneSize) continue;
 
-        for (let sx = -searchOneSize; sx <= searchOneSize; sx += searchGap) {
+        for (let sx = -searchOneSize; sx <= searchOneSize; sx += coarseGap) {
           const cx = sCenterX + sx;
           if (cx < templateOneSize || cx >= markerWidth - templateOneSize) continue;
 
-          let sumI = 0;
-          let sumI2 = 0;
-          let sumIT = 0;
-
+          let sumI = 0, sumI2 = 0, sumIT = 0;
           for (let ty = -templateOneSize; ty <= templateOneSize; ty++) {
             const rowOffset = (cy + ty) * markerWidth;
             const tRowOffset = (ty + templateOneSize) * templateSize;
             for (let tx = -templateOneSize; tx <= templateOneSize; tx++) {
               const valI = projectedImage[rowOffset + (cx + tx)];
               const valT = templateData[tRowOffset + (tx + templateOneSize)];
-
-              sumI += valI;
-              sumI2 += valI * valI;
-              sumIT += valI * valT;
+              sumI += valI; sumI2 += valI * valI; sumIT += valI * valT;
             }
           }
 
@@ -252,6 +248,41 @@ class Tracker {
         }
       }
 
+      // Step 2: Fine refinement (Gap 1) only around the best coarse match
+      if (bestSim > AR2_SIM_THRESH) {
+        const fineCenterX = (bestX * scale) | 0;
+        const fineCenterY = (bestY * scale) | 0;
+        const fineSearch = coarseGap;
+
+        for (let sy = -fineSearch; sy <= fineSearch; sy++) {
+          const cy = fineCenterY + sy;
+          if (cy < templateOneSize || cy >= markerHeight - templateOneSize) continue;
+          for (let sx = -fineSearch; sx <= fineSearch; sx++) {
+            const cx = fineCenterX + sx;
+            if (cx < templateOneSize || cx >= markerWidth - templateOneSize) continue;
+
+            let sumI = 0, sumI2 = 0, sumIT = 0;
+            for (let ty = -templateOneSize; ty <= templateOneSize; ty++) {
+              const rowOffset = (cy + ty) * markerWidth;
+              const tRowOffset = (ty + templateOneSize) * templateSize;
+              for (let tx = -templateOneSize; tx <= templateOneSize; tx++) {
+                const valI = projectedImage[rowOffset + (cx + tx)];
+                const valT = templateData[tRowOffset + (tx + templateOneSize)];
+                sumI += valI; sumI2 += valI * valI; sumIT += valI * valT;
+              }
+            }
+            const varI = Math.sqrt(Math.max(0, sumI2 - (sumI * sumI) * oneOverNPixels));
+            if (varI < 0.0001) continue;
+            const sim = (sumIT - (sumI * sumT) * oneOverNPixels) / (varI * varT);
+            if (sim > bestSim) {
+              bestSim = sim;
+              bestX = cx / scale;
+              bestY = cy / scale;
+            }
+          }
+        }
+      }
+
       sims[f] = bestSim;
       matchingPoints.push([bestX, bestY]);
     }

package/src/react/TaptappAR.tsx

@@ -169,6 +169,9 @@ export const TaptappAR: React.FC<TaptappARProps> = ({
                     display: block;
                     width: 100%;
                     height: auto;
+                    will-change: transform;
+                    /* Visual interpolation to smooth out frame gaps */
+                    transition: transform 0.1s linear, opacity 0.3s ease;
                 }
             `}</style>
         </div>