@srsergio/taptapp-ar 1.0.84 → 1.0.86

package/src/{compiler → runtime}/controller.ts

@@ -1,11 +1,11 @@
-import { Tracker } from "./tracker/tracker.js";
-import { OfflineCompiler as Compiler } from "./offline-compiler.js";
-import { InputLoader } from "./input-loader.js";
-import { FeatureManager } from "./features/feature-manager.js";
-import { OneEuroFilterFeature } from "./features/one-euro-filter-feature.js";
-import { TemporalFilterFeature } from "./features/temporal-filter-feature.js";
-import { AutoRotationFeature } from "./features/auto-rotation-feature.js";
-import { DetectorLite } from "./detector/detector-lite.js";
+import { Tracker } from "../core/tracker/tracker.js";
+import { InputLoader } from "../core/input-loader.js";
+import { FeatureManager } from "../core/features/feature-manager.js";
+import { OneEuroFilterFeature } from "../core/features/one-euro-filter-feature.js";
+import { TemporalFilterFeature } from "../core/features/temporal-filter-feature.js";
+import { AutoRotationFeature } from "../core/features/auto-rotation-feature.js";
+import { DetectorLite } from "../core/detector/detector-lite.js";
+import * as protocol from "../core/protocol.js";
 
 let ControllerWorker: any;
 
@@ -61,6 +61,7 @@ class Controller {
     matchingDataList: any;
     workerMatchDone: ((data: any) => void) | null = null;
     workerTrackDone: ((data: any) => void) | null = null;
+    workerFullTrackDone: ((data: any) => void) | null = null;
     mainThreadMatcher: any;
     mainThreadEstimator: any;
     featureManager: FeatureManager;
@@ -143,6 +144,9 @@ class Controller {
             if (e.data.type === "matchDone" && this.workerMatchDone !== null) {
                 this.workerMatchDone(e.data);
             }
+            if (e.data.type === "trackDone" && this.workerFullTrackDone !== null) {
+                this.workerFullTrackDone(e.data);
+            }
             if (e.data.type === "trackUpdateDone" && this.workerTrackDone !== null) {
                 this.workerTrackDone(e.data);
             }
@@ -174,9 +178,8 @@ class Controller {
         const allDimensions: any[] = [];
 
         for (const buffer of buffers) {
-            const compiler = new Compiler();
-            const result = compiler.importData(buffer);
-            const dataList = (result as any).dataList || [];
+            const result = protocol.decodeTaar(buffer);
+            const dataList = result.dataList || [];
 
             for (const item of dataList) {
                 allMatchingData.push(item.matchingData);
@@ -194,6 +197,7 @@ class Controller {
             this.debugMode,
         );
 
+
         this._ensureWorker();
         if (this.worker) {
             this.worker.postMessage({
@@ -203,6 +207,8 @@ class Controller {
                 projectionTransform: this.projectionTransform,
                 debugMode: this.debugMode,
                 matchingDataList: allMatchingData,
+                trackingDataList: allTrackingData,
+                markerDimensions: allDimensions
             });
         }
 
@@ -247,21 +253,25 @@ class Controller {
     }
 
     async _detectAndMatch(inputData: any, targetIndexes: number[]) {
-        const { featurePoints } = this.fullDetector!.detect(inputData);
-        const { targetIndex: matchedTargetIndex, modelViewTransform } = await this._workerMatch(
-            featurePoints,
+        const { targetIndex, modelViewTransform, screenCoords, worldCoords, featurePoints } = await this._workerMatch(
+            null, // No feature points, worker will detect from inputData
             targetIndexes,
+            inputData
         );
-        return { targetIndex: matchedTargetIndex, modelViewTransform, featurePoints };
+        return { targetIndex, modelViewTransform, screenCoords, worldCoords, featurePoints };
     }
 
     async _trackAndUpdate(inputData: any, lastModelViewTransform: number[][], targetIndex: number) {
-        const { worldCoords, screenCoords, reliabilities, indices = [], octaveIndex = 0 } = this.tracker!.track(
+        const { worldCoords, screenCoords, reliabilities, indices = [], octaveIndex = 0 } = await this._workerTrack(
             inputData,
             lastModelViewTransform,
             targetIndex,
         );
 
+        if (!worldCoords || worldCoords.length === 0) {
+            return { modelViewTransform: null, screenCoords: [], reliabilities: [], stabilities: [] };
+        }
+
         const state = this.trackingStates[targetIndex];
         if (!state.pointStabilities) state.pointStabilities = [];
         if (!state.lastScreenCoords) state.lastScreenCoords = [];
@@ -296,7 +306,11 @@ class Controller {
         for (let i = 0; i < stabilities.length; i++) {
             if (stabilities[i] > 0) {
                 const isCurrentlyTracked = indices.includes(i);
-                finalScreenCoords.push(lastCoords[i]);
+                finalScreenCoords.push({
+                    x: lastCoords[i].x,
+                    y: lastCoords[i].y,
+                    id: i // Unique index from tracker
+                });
                 finalStabilities.push(stabilities[i]);
 
                 if (isCurrentlyTracked) {
@@ -373,13 +387,17 @@ class Controller {
                         matchingIndexes.push(i);
                     }
 
-                    const { targetIndex: matchedTargetIndex, modelViewTransform } =
+                    const { targetIndex: matchedTargetIndex, modelViewTransform, featurePoints } =
                         await this._detectAndMatch(inputData, matchingIndexes);
 
                     if (matchedTargetIndex !== -1) {
                         this.trackingStates[matchedTargetIndex].isTracking = true;
                         this.trackingStates[matchedTargetIndex].currentModelViewTransform = modelViewTransform;
                     }
+
+                    // If we have feature points, we can store them in a special "lastSeenFeatures" 
+                    // or just pass them in processDone for general visualization
+                    this.onUpdate && this.onUpdate({ type: "featurePoints", featurePoints });
                 }
 
                 for (let i = 0; i < this.trackingStates.length; i++) {
@@ -491,10 +509,20 @@ class Controller {
         return this._workerTrackUpdate(modelViewTransform, trackFeatures);
     }
 
-    _workerMatch(featurePoints: any, targetIndexes: number[]): Promise<any> {
+    _workerMatch(featurePoints: any, targetIndexes: number[], inputData: any = null): Promise<any> {
         return new Promise((resolve) => {
             if (!this.worker) {
-                this._matchOnMainThread(featurePoints, targetIndexes).then(resolve).catch(() => resolve({ targetIndex: -1 }));
+                // If no feature points but we have input data, detect first
+                let fpPromise;
+                if (!featurePoints && inputData) {
+                    fpPromise = Promise.resolve(this.fullDetector!.detect(inputData).featurePoints);
+                } else {
+                    fpPromise = Promise.resolve(featurePoints);
+                }
+
+                fpPromise.then(fp => {
+                    this._matchOnMainThread(fp, targetIndexes).then(resolve);
+                }).catch(() => resolve({ targetIndex: -1 }));
                 return;
             }
 
@@ -511,17 +539,50 @@ class Controller {
                     modelViewTransform: data.modelViewTransform,
                     screenCoords: data.screenCoords,
                     worldCoords: data.worldCoords,
+                    featurePoints: data.featurePoints,
                     debugExtra: data.debugExtra,
                 });
             };
-            this.worker.postMessage({ type: "match", featurePoints: featurePoints, targetIndexes });
+
+            if (inputData) {
+                this.worker.postMessage({ type: "match", inputData, targetIndexes });
+            } else {
+                this.worker.postMessage({ type: "match", featurePoints: featurePoints, targetIndexes });
+            }
+        });
+    }
+
+    _workerTrack(inputData: any, lastModelViewTransform: number[][], targetIndex: number): Promise<any> {
+        return new Promise((resolve) => {
+            if (!this.worker) {
+                resolve(this.tracker!.track(inputData, lastModelViewTransform, targetIndex));
+                return;
+            }
+
+            const timeout = setTimeout(() => {
+                this.workerFullTrackDone = null;
+                resolve({ worldCoords: [], screenCoords: [], reliabilities: [] });
+            }, WORKER_TIMEOUT_MS);
+
+            this.workerFullTrackDone = (data: any) => {
+                clearTimeout(timeout);
+                this.workerFullTrackDone = null;
+                resolve(data);
+            };
+
+            this.worker.postMessage({
+                type: "track",
+                inputData,
+                lastModelViewTransform,
+                targetIndex
+            });
         });
     }
 
     async _matchOnMainThread(featurePoints: any, targetIndexes: number[]) {
         if (!this.mainThreadMatcher) {
-            const { Matcher } = await import("./matching/matcher.js");
-            const { Estimator } = await import("./estimation/estimator.js");
+            const { Matcher } = await import("../core/matching/matcher.js");
+            const { Estimator } = await import("../core/estimation/estimator.js");
             this.mainThreadMatcher = new Matcher(this.inputWidth, this.inputHeight, this.debugMode);
             this.mainThreadEstimator = new Estimator(this.projectionTransform);
         }
@@ -591,7 +652,7 @@ class Controller {
 
     async _trackUpdateOnMainThread(modelViewTransform: number[][], trackingFeatures: any) {
         if (!this.mainThreadEstimator) {
-            const { Estimator } = await import("./estimation/estimator.js");
+            const { Estimator } = await import("../core/estimation/estimator.js");
             this.mainThreadEstimator = new Estimator(this.projectionTransform);
         }
 

package/src/{compiler → runtime}/controller.worker.js

@@ -1,10 +1,14 @@
-import { Matcher } from "./matching/matcher.js";
-import { Estimator } from "./estimation/estimator.js";
+import { Matcher } from "../core/matching/matcher.js";
+import { Estimator } from "../core/estimation/estimator.js";
+import { Tracker } from "../core/tracker/tracker.js";
+import { DetectorLite } from "../core/detector/detector-lite.js";
 
 let matchingDataList = null;
 let debugMode = false;
 let matcher = null;
 let estimator = null;
+let tracker = null;
+let detector = null;
 
 onmessage = (msg) => {
   const { data } = msg;
@@ -15,6 +19,19 @@ onmessage = (msg) => {
       debugMode = data.debugMode;
       matcher = new Matcher(data.inputWidth, data.inputHeight, debugMode);
       estimator = new Estimator(data.projectionTransform);
+
+      if (data.trackingDataList && data.markerDimensions) {
+        tracker = new Tracker(
+          data.markerDimensions,
+          data.trackingDataList,
+          data.projectionTransform,
+          data.inputWidth,
+          data.inputHeight,
+          debugMode
+        );
+      }
+
+      detector = new DetectorLite(data.inputWidth, data.inputHeight, { useLSH: true });
       break;
 
     case "match":
@@ -22,14 +39,23 @@ onmessage = (msg) => {
 
       let matchedTargetIndex = -1;
       let matchedModelViewTransform = null;
+      let matchedScreenCoords = null;
+      let matchedWorldCoords = null;
       let matchedDebugExtra = null;
 
+      // New: If the worker received image data, run detector here too
+      let featurePoints = data.featurePoints;
+      if (data.inputData) {
+        const detectionResult = detector.detect(data.inputData);
+        featurePoints = detectionResult.featurePoints;
+      }
+
       for (let i = 0; i < interestedTargetIndexes.length; i++) {
         const matchingIndex = interestedTargetIndexes[i];
 
         const { keyframeIndex, screenCoords, worldCoords, debugExtra } = matcher.matchDetection(
           matchingDataList[matchingIndex],
-          data.featurePoints,
+          featurePoints,
         );
         matchedDebugExtra = debugExtra;
 
@@ -39,6 +65,8 @@ onmessage = (msg) => {
           if (modelViewTransform) {
             matchedTargetIndex = matchingIndex;
             matchedModelViewTransform = modelViewTransform;
+            matchedScreenCoords = screenCoords;
+            matchedWorldCoords = worldCoords;
           }
           break;
         }
@@ -48,10 +76,24 @@ onmessage = (msg) => {
         type: "matchDone",
         targetIndex: matchedTargetIndex,
         modelViewTransform: matchedModelViewTransform,
+        screenCoords: matchedScreenCoords,
+        worldCoords: matchedWorldCoords,
+        featurePoints: featurePoints,
         debugExtra: matchedDebugExtra,
       });
       break;
 
+    case "track":
+      const { inputData: trackInput, lastModelViewTransform, targetIndex } = data;
+      const trackResult = tracker.track(trackInput, lastModelViewTransform, targetIndex);
+
+      postMessage({
+        type: "trackDone",
+        targetIndex,
+        ...trackResult
+      });
+      break;
+
     case "trackUpdate":
       const { modelViewTransform, worldCoords, screenCoords, stabilities } = data;
       const finalModelViewTransform = estimator.refineEstimate({

package/src/runtime/index.ts

@@ -0,0 +1,4 @@
+export * from "./controller.js";
+export * from "./simple-ar.js";
+export * from "./three.js";
+export * from "./aframe.js";

package/src/{compiler → runtime}/simple-ar.ts

@@ -1,6 +1,6 @@
 import { Controller } from "./controller.js";
 import { OneEuroFilter } from "../libs/one-euro-filter.js";
-import { projectToScreen } from "./utils/projection.js";
+import { projectToScreen } from "../core/utils/projection.js";
 
 /**
  * 🍦 SimpleAR - Dead-simple vanilla AR for image overlays

package/dist/compiler/utils/fourier-encoder.d.ts

@@ -1,25 +0,0 @@
-/**
- * 🚀 Moonshot: Fourier Positional Encoding
- *
- * Maps 2D coordinates (x, y) to a high-dimensional frequency space.
- * Used in Transformer Positional Encoding, NeRFs, and modern Generative AI.
- *
- * Theory: gamma(p) = [sin(2^0 * pi * p), cos(2^0 * pi * p), ..., sin(2^L-1 * pi * p), cos(2^L-1 * pi * p)]
- */
-export declare class FourierEncoder {
-    private frequencies;
-    private L;
-    constructor(L?: number);
-    /**
-     * Encodes a normalized coordinate (0-1) into Fourier features
-     * @param x Normalized X
-     * @param y Normalized Y
-     * @returns Float32Array of size 4 * L
-     */
-    encode(x: number, y: number): Float32Array;
-    /**
-     * Fast dot product between two fourier encodings
-     * This measures "harmonic spatial similarity"
-     */
-    static similarity(v1: Float32Array, v2: Float32Array): number;
-}

package/dist/compiler/utils/fourier-encoder.js

@@ -1,47 +0,0 @@
-/**
- * 🚀 Moonshot: Fourier Positional Encoding
- *
- * Maps 2D coordinates (x, y) to a high-dimensional frequency space.
- * Used in Transformer Positional Encoding, NeRFs, and modern Generative AI.
- *
- * Theory: gamma(p) = [sin(2^0 * pi * p), cos(2^0 * pi * p), ..., sin(2^L-1 * pi * p), cos(2^L-1 * pi * p)]
- */
-export class FourierEncoder {
-    frequencies;
-    L;
-    constructor(L = 4) {
-        this.L = L;
-        this.frequencies = [];
-        for (let i = 0; i < L; i++) {
-            this.frequencies.push(Math.pow(2, i) * Math.PI);
-        }
-    }
-    /**
-     * Encodes a normalized coordinate (0-1) into Fourier features
-     * @param x Normalized X
-     * @param y Normalized Y
-     * @returns Float32Array of size 4 * L
-     */
-    encode(x, y) {
-        const result = new Float32Array(this.L * 4);
-        let idx = 0;
-        for (const freq of this.frequencies) {
-            result[idx++] = Math.sin(freq * x);
-            result[idx++] = Math.cos(freq * x);
-            result[idx++] = Math.sin(freq * y);
-            result[idx++] = Math.cos(freq * y);
-        }
-        return result;
-    }
-    /**
-     * Fast dot product between two fourier encodings
-     * This measures "harmonic spatial similarity"
-     */
-    static similarity(v1, v2) {
-        let dot = 0;
-        for (let i = 0; i < v1.length; i++) {
-            dot += v1[i] * v2[i];
-        }
-        return dot / (v1.length / 2); // Normalize by number of components
-    }
-}

package/src/compiler/utils/fourier-encoder.ts

@@ -1,53 +0,0 @@
-/**
- * 🚀 Moonshot: Fourier Positional Encoding
- * 
- * Maps 2D coordinates (x, y) to a high-dimensional frequency space.
- * Used in Transformer Positional Encoding, NeRFs, and modern Generative AI.
- * 
- * Theory: gamma(p) = [sin(2^0 * pi * p), cos(2^0 * pi * p), ..., sin(2^L-1 * pi * p), cos(2^L-1 * pi * p)]
- */
-
-export class FourierEncoder {
-    private frequencies: number[];
-    private L: number;
-
-    constructor(L: number = 4) {
-        this.L = L;
-        this.frequencies = [];
-        for (let i = 0; i < L; i++) {
-            this.frequencies.push(Math.pow(2, i) * Math.PI);
-        }
-    }
-
-    /**
-     * Encodes a normalized coordinate (0-1) into Fourier features
-     * @param x Normalized X
-     * @param y Normalized Y
-     * @returns Float32Array of size 4 * L
-     */
-    encode(x: number, y: number): Float32Array {
-        const result = new Float32Array(this.L * 4);
-        let idx = 0;
-
-        for (const freq of this.frequencies) {
-            result[idx++] = Math.sin(freq * x);
-            result[idx++] = Math.cos(freq * x);
-            result[idx++] = Math.sin(freq * y);
-            result[idx++] = Math.cos(freq * y);
-        }
-
-        return result;
-    }
-
-    /**
-     * Fast dot product between two fourier encodings
-     * This measures "harmonic spatial similarity"
-     */
-    static similarity(v1: Float32Array, v2: Float32Array): number {
-        let dot = 0;
-        for (let i = 0; i < v1.length; i++) {
-            dot += v1[i] * v2[i];
-        }
-        return dot / (v1.length / 2); // Normalize by number of components
-    }
-}