@srsergio/taptapp-ar 1.0.8 → 1.0.10

package/dist/compiler/controller.d.ts

@@ -1,5 +1,5 @@
 export class Controller {
-    constructor({ inputWidth, inputHeight, onUpdate, debugMode, maxTrack, warmupTolerance, missTolerance, filterMinCF, filterBeta, }: {
+    constructor({ inputWidth, inputHeight, onUpdate, debugMode, maxTrack, warmupTolerance, missTolerance, filterMinCF, filterBeta, worker, }: {
         inputWidth: any;
         inputHeight: any;
         onUpdate?: null | undefined;
@@ -9,6 +9,7 @@ export class Controller {
         missTolerance?: null | undefined;
         filterMinCF?: null | undefined;
         filterBeta?: null | undefined;
+        worker?: null | undefined;
     });
     inputWidth: any;
     inputHeight: any;
@@ -25,12 +26,11 @@ export class Controller {
     processingVideo: boolean;
     interestedTargetIndex: number;
     trackingStates: any[];
+    worker: any;
     projectionTransform: number[][];
     projectionMatrix: number[];
-    worker: any;
-    workerMatchDone: ((data: any) => void) | null;
-    workerTrackDone: ((data: any) => void) | null;
-    showTFStats(): void;
+    _setupWorkerListener(): void;
+    _ensureWorker(): void;
     addImageTargets(fileURL: any): Promise<any>;
     addImageTargetsFromBuffer(buffer: any): {
         dimensions: any[][];
@@ -43,32 +43,23 @@ export class Controller {
     getProjectionMatrix(): number[];
     getRotatedZ90Matrix(m: any): any[];
     getWorldMatrix(modelViewTransform: any, targetIndex: any): any[];
-    _detectAndMatch(inputT: any, targetIndexes: any): Promise<{
+    _detectAndMatch(inputData: any, targetIndexes: any): Promise<{
         targetIndex: any;
         modelViewTransform: any;
     }>;
-    _trackAndUpdate(inputT: any, lastModelViewTransform: any, targetIndex: any): Promise<any>;
+    _trackAndUpdate(inputData: any, lastModelViewTransform: any, targetIndex: any): Promise<any>;
     processVideo(input: any): void;
     stopProcessVideo(): void;
     detect(input: any): Promise<{
-        featurePoints: {
-            maxima: boolean;
-            x: number;
-            y: number;
-            scale: number;
-            angle: any;
-            descriptors: any[];
-        }[];
+        featurePoints: any[];
         debugExtra: {
-            pyramidImages: any[][];
-            dogPyramidImages: any[];
-            extremasResults: any[];
-            extremaAngles: any;
-            prunedExtremas: number[][];
-            localizedExtremas: any;
-        } | null;
+            projectedImage: number[];
+        } | {
+            projectedImage?: undefined;
+        };
     }>;
     match(featurePoints: any, targetIndex: any): Promise<{
+        targetIndex: any;
         modelViewTransform: any;
         debugExtra: any;
     }>;
@@ -86,7 +77,9 @@ export class Controller {
     }>;
     trackUpdate(modelViewTransform: any, trackFeatures: any): Promise<any>;
     _workerMatch(featurePoints: any, targetIndexes: any): Promise<any>;
+    workerMatchDone: ((data: any) => void) | undefined;
     _workerTrackUpdate(modelViewTransform: any, trackingFeatures: any): Promise<any>;
+    workerTrackDone: ((data: any) => void) | undefined;
     _glModelViewMatrix(modelViewTransform: any, targetIndex: any): any[];
     _glProjectionMatrix({ projectionTransform, width, height, near, far }: {
         projectionTransform: any;

package/dist/compiler/controller.js

@@ -1,17 +1,25 @@
-import { memory, nextFrame } from "@tensorflow/tfjs";
-const tf = { memory, nextFrame };
-import ControllerWorker from "./controller.worker.js?worker&inline";
 import { Tracker } from "./tracker/tracker.js";
 import { CropDetector } from "./detector/crop-detector.js";
-import { Compiler } from "./compiler.js";
+import { OfflineCompiler as Compiler } from "./offline-compiler.js";
 import { InputLoader } from "./input-loader.js";
 import { OneEuroFilter } from "../libs/one-euro-filter.js";
+let ControllerWorker;
+// Conditional import for worker to avoid crash in non-vite environments
+try {
+    const workerModule = await import("./controller.worker.js?worker&inline");
+    ControllerWorker = workerModule.default;
+}
+catch (e) {
+    // Fallback for tests or other environments
+    ControllerWorker = null;
+}
 const DEFAULT_FILTER_CUTOFF = 0.001; // 1Hz. time period in milliseconds
 const DEFAULT_FILTER_BETA = 1000;
 const DEFAULT_WARMUP_TOLERANCE = 5;
 const DEFAULT_MISS_TOLERANCE = 5;
 class Controller {
-    constructor({ inputWidth, inputHeight, onUpdate = null, debugMode = false, maxTrack = 1, warmupTolerance = null, missTolerance = null, filterMinCF = null, filterBeta = null, }) {
+    constructor({ inputWidth, inputHeight, onUpdate = null, debugMode = false, maxTrack = 1, warmupTolerance = null, missTolerance = null, filterMinCF = null, filterBeta = null, worker = null, // Allow custom worker injection
+     }) {
         this.inputWidth = inputWidth;
         this.inputHeight = inputHeight;
         this.maxTrack = maxTrack;
@@ -27,13 +35,13 @@ class Controller {
         this.processingVideo = false;
         this.interestedTargetIndex = -1;
         this.trackingStates = [];
+        this.worker = worker;
+        if (this.worker)
+            this._setupWorkerListener();
         const near = 10;
         const far = 100000;
-        const fovy = (45.0 * Math.PI) / 180; // 45 in radian. field of view vertical
+        const fovy = (45.0 * Math.PI) / 180;
         const f = this.inputHeight / 2 / Math.tan(fovy / 2);
-        //     [fx  s cx]
-        // K = [ 0 fx cy]
-        //     [ 0  0  1]
         this.projectionTransform = [
             [f, 0, this.inputWidth / 2],
             [0, f, this.inputHeight / 2],
@@ -46,9 +54,10 @@ class Controller {
             near: near,
             far: far,
         });
-        this.worker = new ControllerWorker(); //new Worker(new URL('./controller.worker.js', import.meta.url));
-        this.workerMatchDone = null;
-        this.workerTrackDone = null;
+    }
+    _setupWorkerListener() {
+        if (!this.worker)
+            return;
         this.worker.onmessage = (e) => {
             if (e.data.type === "matchDone" && this.workerMatchDone !== null) {
                 this.workerMatchDone(e.data);
@@ -58,9 +67,13 @@ class Controller {
             }
         };
     }
-    showTFStats() {
-        console.log(tf.memory().numTensors);
-        console.table(tf.memory());
+    _ensureWorker() {
+        if (this.worker)
+            return;
+        if (ControllerWorker) {
+            this.worker = new ControllerWorker();
+            this._setupWorkerListener();
+        }
     }
     addImageTargets(fileURL) {
         return new Promise(async (resolve) => {
@@ -77,34 +90,37 @@ class Controller {
         const matchingDataList = [];
         const dimensions = [];
         for (let i = 0; i < dataList.length; i++) {
-            matchingDataList.push(dataList[i].matchingData);
-            trackingDataList.push(dataList[i].trackingData);
-            dimensions.push([dataList[i].targetImage.width, dataList[i].targetImage.height]);
+            const item = dataList[i];
+            matchingDataList.push(item.matchingData);
+            trackingDataList.push(item.trackingData);
+            dimensions.push([item.targetImage.width, item.targetImage.height]);
         }
         this.tracker = new Tracker(dimensions, trackingDataList, this.projectionTransform, this.inputWidth, this.inputHeight, this.debugMode);
-        this.worker.postMessage({
-            type: "setup",
-            inputWidth: this.inputWidth,
-            inputHeight: this.inputHeight,
-            projectionTransform: this.projectionTransform,
-            debugMode: this.debugMode,
-            matchingDataList,
-        });
+        this._ensureWorker();
+        if (this.worker) {
+            this.worker.postMessage({
+                type: "setup",
+                inputWidth: this.inputWidth,
+                inputHeight: this.inputHeight,
+                projectionTransform: this.projectionTransform,
+                debugMode: this.debugMode,
+                matchingDataList,
+            });
+        }
         this.markerDimensions = dimensions;
-        return { dimensions: dimensions, matchingDataList, trackingDataList };
+        return { dimensions, matchingDataList, trackingDataList };
     }
     dispose() {
         this.stopProcessVideo();
-        this.worker.postMessage({
-            type: "dispose",
-        });
+        if (this.worker) {
+            this.worker.postMessage({ type: "dispose" });
+            this.worker = null;
+        }
     }
-    // warm up gpu - build kernels is slow
     dummyRun(input) {
-        const inputT = this.inputLoader.loadInput(input);
-        this.cropDetector.detect(inputT);
-        this.tracker.dummyRun(inputT);
-        inputT.dispose();
+        const inputData = this.inputLoader.loadInput(input);
+        this.cropDetector.detect(inputData);
+        this.tracker.dummyRun(inputData);
     }
     getProjectionMatrix() {
         return this.projectionMatrix;
@@ -139,13 +155,13 @@ class Controller {
     getWorldMatrix(modelViewTransform, targetIndex) {
         return this._glModelViewMatrix(modelViewTransform, targetIndex);
     }
-    async _detectAndMatch(inputT, targetIndexes) {
-        const { featurePoints } = this.cropDetector.detectMoving(inputT);
+    async _detectAndMatch(inputData, targetIndexes) {
+        const { featurePoints } = this.cropDetector.detectMoving(inputData);
         const { targetIndex: matchedTargetIndex, modelViewTransform } = await this._workerMatch(featurePoints, targetIndexes);
         return { targetIndex: matchedTargetIndex, modelViewTransform };
     }
-    async _trackAndUpdate(inputT, lastModelViewTransform, targetIndex) {
-        const { worldCoords, screenCoords } = this.tracker.track(inputT, lastModelViewTransform, targetIndex);
+    async _trackAndUpdate(inputData, lastModelViewTransform, targetIndex) {
+        const { worldCoords, screenCoords } = this.tracker.track(inputData, lastModelViewTransform, targetIndex);
         if (worldCoords.length < 4)
             return null;
         const modelViewTransform = await this._workerTrackUpdate(lastModelViewTransform, {
@@ -168,13 +184,12 @@ class Controller {
                 trackMiss: 0,
                 filter: new OneEuroFilter({ minCutOff: this.filterMinCF, beta: this.filterBeta }),
             });
-            //console.log("filterMinCF", this.filterMinCF, this.filterBeta);
         }
         const startProcessing = async () => {
             while (true) {
                 if (!this.processingVideo)
                     break;
-                const inputT = this.inputLoader.loadInput(input);
+                const inputData = this.inputLoader.loadInput(input);
                 const nTracking = this.trackingStates.reduce((acc, s) => {
                     return acc + (!!s.isTracking ? 1 : 0);
                 }, 0);
@@ -189,7 +204,7 @@ class Controller {
                             continue;
                         matchingIndexes.push(i);
                     }
-                    const { targetIndex: matchedTargetIndex, modelViewTransform } = await this._detectAndMatch(inputT, matchingIndexes);
+                    const { targetIndex: matchedTargetIndex, modelViewTransform } = await this._detectAndMatch(inputData, matchingIndexes);
                     if (matchedTargetIndex !== -1) {
                         this.trackingStates[matchedTargetIndex].isTracking = true;
                         this.trackingStates[matchedTargetIndex].currentModelViewTransform = modelViewTransform;
@@ -199,7 +214,7 @@ class Controller {
                 for (let i = 0; i < this.trackingStates.length; i++) {
                     const trackingState = this.trackingStates[i];
                     if (trackingState.isTracking) {
-                        let modelViewTransform = await this._trackAndUpdate(inputT, trackingState.currentModelViewTransform, i);
+                        let modelViewTransform = await this._trackAndUpdate(inputData, trackingState.currentModelViewTransform, i);
                         if (modelViewTransform === null) {
                             trackingState.isTracking = false;
                         }
@@ -251,9 +266,14 @@ class Controller {
                             this.onUpdate({ type: "updateMatrix", targetIndex: i, worldMatrix: clone });
                     }
                 }
-                inputT.dispose();
                 this.onUpdate && this.onUpdate({ type: "processDone" });
-                await tf.nextFrame();
+                // Use requestAnimationFrame if available, otherwise just wait briefly
+                if (typeof requestAnimationFrame !== "undefined") {
+                    await new Promise(requestAnimationFrame);
+                }
+                else {
+                    await new Promise(resolve => setTimeout(resolve, 16));
+                }
             }
         };
         startProcessing();
@@ -262,21 +282,19 @@ class Controller {
         this.processingVideo = false;
     }
     async detect(input) {
-        const inputT = this.inputLoader.loadInput(input);
-        const { featurePoints, debugExtra } = this.cropDetector.detect(inputT);
-        inputT.dispose();
+        const inputData = this.inputLoader.loadInput(input);
+        const { featurePoints, debugExtra } = this.cropDetector.detect(inputData);
         return { featurePoints, debugExtra };
     }
     async match(featurePoints, targetIndex) {
-        const { modelViewTransform, debugExtra } = await this._workerMatch(featurePoints, [
+        const { targetIndex: matchedTargetIndex, modelViewTransform, debugExtra } = await this._workerMatch(featurePoints, [
             targetIndex,
         ]);
-        return { modelViewTransform, debugExtra };
+        return { targetIndex: matchedTargetIndex, modelViewTransform, debugExtra };
     }
     async track(input, modelViewTransform, targetIndex) {
-        const inputT = this.inputLoader.loadInput(input);
-        const result = this.tracker.track(inputT, modelViewTransform, targetIndex);
-        inputT.dispose();
+        const inputData = this.inputLoader.loadInput(input);
+        const result = this.tracker.track(inputData, modelViewTransform, targetIndex);
         return result;
     }
     async trackUpdate(modelViewTransform, trackFeatures) {
@@ -294,7 +312,7 @@ class Controller {
                     debugExtra: data.debugExtra,
                 });
             };
-            this.worker.postMessage({ type: "match", featurePoints: featurePoints, targetIndexes });
+            this.worker && this.worker.postMessage({ type: "match", featurePoints: featurePoints, targetIndexes });
         });
     }
     _workerTrackUpdate(modelViewTransform, trackingFeatures) {
@@ -303,7 +321,7 @@ class Controller {
                 resolve(data.modelViewTransform);
             };
             const { worldCoords, screenCoords } = trackingFeatures;
-            this.worker.postMessage({
+            this.worker && this.worker.postMessage({
                 type: "trackUpdate",
                 modelViewTransform,
                 worldCoords,
@@ -313,41 +331,6 @@ class Controller {
     }
     _glModelViewMatrix(modelViewTransform, targetIndex) {
         const height = this.markerDimensions[targetIndex][1];
-        // Question: can someone verify this interpreation is correct?
-        // I'm not very convinced, but more like trial and error and works......
-        //
-        // First, opengl has y coordinate system go from bottom to top, while the marker corrdinate goes from top to bottom,
-        //    since the modelViewTransform is estimated in marker coordinate, we need to apply this transform before modelViewTransform
-        //    I can see why y = h - y*, but why z = z* ? should we intepret it as rotate 90 deg along x-axis and then translate y by h?
-        //
-        //    [1  0  0  0]
-        //    [0 -1  0  h]
-        //    [0  0 -1  0]
-        //    [0  0  0  1]
-        //
-        //    This is tested that if we reverse marker coordinate from bottom to top and estimate the modelViewTransform,
-        //    then the above matrix is not necessary.
-        //
-        // Second, in opengl, positive z is away from camera, so we rotate 90 deg along x-axis after transform to fix the axis mismatch
-        //    [1  1  0  0]
-        //    [0 -1  0  0]
-        //    [0  0 -1  0]
-        //    [0  0  0  1]
-        //
-        // all together, the combined matrix is
-        //
-        //    [1  1  0  0]   [m00, m01, m02, m03]   [1  0  0  0]
-        //    [0 -1  0  0]   [m10, m11, m12, m13]   [0 -1  0  h]
-        //    [0  0 -1  0]   [m20, m21, m22, m23]   [0  0 -1  0]
-        //    [0  0  0  1]   [  0    0    0    1]   [0  0  0  1]
-        //
-        //    [ m00,  -m01,  -m02,  (m01 * h + m03) ]
-        //    [-m10,   m11,   m12, -(m11 * h + m13) ]
-        //  = [-m20,   m21,   m22, -(m21 * h + m23) ]
-        //    [   0,     0,     0,                1 ]
-        //
-        //
-        // Finally, in threejs, matrix is represented in col by row, so we transpose it, and get below:
         const openGLWorldMatrix = [
             modelViewTransform[0][0],
             -modelViewTransform[1][0],
@@ -368,8 +351,6 @@ class Controller {
         ];
         return openGLWorldMatrix;
     }
-    // build openGL projection matrix
-    // ref: https://strawlab.org/2011/11/05/augmented-reality-with-OpenGL/
     _glProjectionMatrix({ projectionTransform, width, height, near, far }) {
         const proj = [
             [

package/dist/compiler/detector/crop-detector.d.ts

@@ -4,62 +4,31 @@ export class CropDetector {
     width: any;
     height: any;
     cropSize: number;
-    detector: Detector;
-    kernelCaches: {};
+    detector: DetectorLite;
     lastRandomIndex: number;
-    detect(inputImageT: any): {
-        featurePoints: {
-            maxima: boolean;
-            x: number;
-            y: number;
-            scale: number;
-            angle: any;
-            descriptors: any[];
-        }[];
+    detect(input: any): {
+        featurePoints: any[];
         debugExtra: {
-            pyramidImages: any[][];
-            dogPyramidImages: any[];
-            extremasResults: any[];
-            extremaAngles: any;
-            prunedExtremas: number[][];
-            localizedExtremas: any;
-        } | null;
+            projectedImage: number[];
+        } | {
+            projectedImage?: undefined;
+        };
     };
-    detectMoving(inputImageT: any): {
-        featurePoints: {
-            maxima: boolean;
-            x: number;
-            y: number;
-            scale: number;
-            angle: any;
-            descriptors: any[];
-        }[];
+    detectMoving(input: any): {
+        featurePoints: any[];
         debugExtra: {
-            pyramidImages: any[][];
-            dogPyramidImages: any[];
-            extremasResults: any[];
-            extremaAngles: any;
-            prunedExtremas: number[][];
-            localizedExtremas: any;
-        } | null;
+            projectedImage: number[];
+        } | {
+            projectedImage?: undefined;
+        };
     };
-    _detect(inputImageT: any, startX: any, startY: any): {
-        featurePoints: {
-            maxima: boolean;
-            x: number;
-            y: number;
-            scale: number;
-            angle: any;
-            descriptors: any[];
-        }[];
+    _detect(imageData: any, startX: any, startY: any): {
+        featurePoints: any[];
         debugExtra: {
-            pyramidImages: any[][];
-            dogPyramidImages: any[];
-            extremasResults: any[];
-            extremaAngles: any;
-            prunedExtremas: number[][];
-            localizedExtremas: any;
-        } | null;
+            projectedImage: number[];
+        } | {
+            projectedImage?: undefined;
+        };
     };
 }
-import { Detector } from "./detector.js";
+import { DetectorLite } from "./detector-lite.js";

package/dist/compiler/detector/crop-detector.js

@@ -1,4 +1,4 @@
-import { Detector } from "./detector.js";
+import { DetectorLite } from "./detector-lite.js";
 class CropDetector {
     constructor(width, height, debugMode = false) {
         this.debugMode = debugMode;
@@ -8,21 +8,22 @@ class CropDetector {
         let minDimension = Math.min(width, height) / 2;
         let cropSize = Math.pow(2, Math.round(Math.log(minDimension) / Math.log(2)));
         this.cropSize = cropSize;
-        this.detector = new Detector(cropSize, cropSize, debugMode);
-        this.kernelCaches = {};
+        this.detector = new DetectorLite(cropSize, cropSize);
         this.lastRandomIndex = 4;
     }
-    detect(inputImageT) {
+    detect(input) {
+        const imageData = input;
         // crop center
         const startY = Math.floor(this.height / 2 - this.cropSize / 2);
         const startX = Math.floor(this.width / 2 - this.cropSize / 2);
-        const result = this._detect(inputImageT, startX, startY);
+        const result = this._detect(imageData, startX, startY);
         if (this.debugMode) {
             result.debugExtra.crop = { startX, startY, cropSize: this.cropSize };
         }
         return result;
     }
-    detectMoving(inputImageT) {
+    detectMoving(input) {
+        const imageData = input;
         // loop a few locations around center
         const dx = this.lastRandomIndex % 3;
         const dy = Math.floor(this.lastRandomIndex / 3);
@@ -37,21 +38,26 @@ class CropDetector {
         if (startY >= this.height - this.cropSize)
             startY = this.height - this.cropSize - 1;
         this.lastRandomIndex = (this.lastRandomIndex + 1) % 9;
-        const result = this._detect(inputImageT, startX, startY);
+        const result = this._detect(imageData, startX, startY);
         return result;
     }
-    _detect(inputImageT, startX, startY) {
-        const cropInputImageT = inputImageT.slice([startY, startX], [this.cropSize, this.cropSize]);
-        const { featurePoints, debugExtra } = this.detector.detect(cropInputImageT);
+    _detect(imageData, startX, startY) {
+        // Crop manually since imageData is now a flat array (width * height)
+        const croppedData = new Float32Array(this.cropSize * this.cropSize);
+        for (let y = 0; y < this.cropSize; y++) {
+            for (let x = 0; x < this.cropSize; x++) {
+                croppedData[y * this.cropSize + x] = imageData[(startY + y) * this.width + (startX + x)];
+            }
+        }
+        const { featurePoints } = this.detector.detect(croppedData);
         featurePoints.forEach((p) => {
             p.x += startX;
             p.y += startY;
         });
-        if (this.debugMode) {
-            debugExtra.projectedImage = cropInputImageT.arraySync();
-        }
-        cropInputImageT.dispose();
-        return { featurePoints: featurePoints, debugExtra };
+        return {
+            featurePoints,
+            debugExtra: this.debugMode ? { projectedImage: Array.from(croppedData) } : {}
+        };
     }
 }
 export { CropDetector };

package/dist/compiler/input-loader.d.ts

@@ -1,22 +1,20 @@
+/**
+ * InputLoader - Maneja la carga de imágenes y video sin TensorFlow
+ */
 export class InputLoader {
     constructor(width: any, height: any);
     width: any;
     height: any;
-    texShape: any[];
-    context: CanvasRenderingContext2D | null;
-    program: {
-        variableNames: string[];
-        outputShape: any[];
-        userCode: string;
-    };
-    tempPixelHandle: any;
-    _loadInput(input: any): any;
-    loadInput(input: any): any;
-    buildProgram(width: any, height: any): {
-        variableNames: string[];
-        outputShape: any[];
-        userCode: string;
-    };
-    _compileAndRun(program: any, inputs: any): any;
-    _runWebGLProgram(program: any, inputs: any, outputType: any): any;
+    grayscaleBuffer: Uint8Array<ArrayBuffer>;
+    context: CanvasRenderingContext2D | null | undefined;
+    /**
+     * Carga una imagen o video y devuelve los datos en escala de grises
+     * @param {HTMLVideoElement|HTMLImageElement|ImageData|Uint8Array} input - La fuente de entrada
+     * @returns {Uint8Array} Datos de imagen en escala de grises (width * height)
+     */
+    loadInput(input: HTMLVideoElement | HTMLImageElement | ImageData | Uint8Array): Uint8Array;
+    /**
+     * Convierte datos RGBA a escala de grises optimizada (reutilizando buffer)
+     */
+    _convertToGrayscale(rgbaData: any, width: any, height: any): void;
 }