@srsergio/taptapp-ar 1.0.78 → 1.0.80

package/src/compiler/controller.ts

@@ -1,12 +1,11 @@
 import { Tracker } from "./tracker/tracker.js";
-import { CropDetector } from "./detector/crop-detector.js";
 import { OfflineCompiler as Compiler } from "./offline-compiler.js";
 import { InputLoader } from "./input-loader.js";
 import { FeatureManager } from "./features/feature-manager.js";
 import { OneEuroFilterFeature } from "./features/one-euro-filter-feature.js";
 import { TemporalFilterFeature } from "./features/temporal-filter-feature.js";
 import { AutoRotationFeature } from "./features/auto-rotation-feature.js";
-import { CropDetectionFeature } from "./features/crop-detection-feature.js";
+import { DetectorLite } from "./detector/detector-lite.js";
 
 let ControllerWorker: any;
 
@@ -26,7 +25,10 @@ ControllerWorker = await getControllerWorker();
 const DEFAULT_FILTER_CUTOFF = 0.5;
 const DEFAULT_FILTER_BETA = 0.1;
 const DEFAULT_WARMUP_TOLERANCE = 2; // Instant detection
-const DEFAULT_MISS_TOLERANCE = 5;   // More grace when partially hidden
+const DEFAULT_MISS_TOLERANCE = 1;   // Immediate response to tracking loss
+const WORKER_TIMEOUT_MS = 1000;    // Prevent worker hangs from killing the loop
+
+let loopIdCounter = 0;
 
 export interface ControllerOptions {
     inputWidth: number;
@@ -62,6 +64,7 @@ class Controller {
     mainThreadMatcher: any;
     mainThreadEstimator: any;
     featureManager: FeatureManager;
+    fullDetector: DetectorLite | null = null;
 
     constructor({
         inputWidth,
@@ -89,7 +92,7 @@ class Controller {
             missTolerance === null ? DEFAULT_MISS_TOLERANCE : missTolerance
         ));
         this.featureManager.addFeature(new AutoRotationFeature());
-        this.featureManager.addFeature(new CropDetectionFeature());
+        // User wants "sin recortes", so we don't add CropDetectionFeature
 
         this.inputLoader = new InputLoader(this.inputWidth, this.inputHeight);
         this.onUpdate = onUpdate;
@@ -97,6 +100,9 @@ class Controller {
         this.worker = worker;
         if (this.worker) this._setupWorkerListener();
 
+        // Moonshot: Full frame detector for better sensitivity
+        this.fullDetector = new DetectorLite(this.inputWidth, this.inputHeight, { useLSH: true });
+
         this.featureManager.init({
             inputWidth: this.inputWidth,
             inputHeight: this.inputHeight,
@@ -219,8 +225,7 @@ class Controller {
 
     dummyRun(input: any) {
         const inputData = this.inputLoader.loadInput(input);
-        const cropFeature = this.featureManager.getFeature<CropDetectionFeature>("crop-detection");
-        cropFeature?.detect(inputData, false);
+        this.fullDetector?.detect(inputData);
         this.tracker!.dummyRun(inputData);
     }
 
@@ -242,8 +247,7 @@ class Controller {
     }
 
     async _detectAndMatch(inputData: any, targetIndexes: number[]) {
-        const cropFeature = this.featureManager.getFeature<CropDetectionFeature>("crop-detection");
-        const { featurePoints } = cropFeature!.detect(inputData, true);
+        const { featurePoints } = this.fullDetector!.detect(inputData);
         const { targetIndex: matchedTargetIndex, modelViewTransform } = await this._workerMatch(
             featurePoints,
             targetIndexes,
@@ -252,22 +256,73 @@ class Controller {
     }
 
     async _trackAndUpdate(inputData: any, lastModelViewTransform: number[][], targetIndex: number) {
-        const { worldCoords, screenCoords } = this.tracker!.track(
+        const { worldCoords, screenCoords, reliabilities, indices = [], octaveIndex = 0 } = this.tracker!.track(
             inputData,
             lastModelViewTransform,
             targetIndex,
         );
-        if (worldCoords.length < 8) return null; // Resynced with Matcher (8 points) to allow initial detection
+
+        const state = this.trackingStates[targetIndex];
+        if (!state.pointStabilities) state.pointStabilities = [];
+        if (!state.pointStabilities[octaveIndex]) {
+            // Initialize stabilities for this octave if not exists
+            const numPoints = (this.tracker as any).prebuiltData[targetIndex][octaveIndex].px.length;
+            state.pointStabilities[octaveIndex] = new Float32Array(numPoints).fill(0.5); // Start at 0.5
+        }
+
+        const stabilities = state.pointStabilities[octaveIndex];
+        const currentStabilities: number[] = [];
+
+        // Update all points in this octave
+        for (let i = 0; i < stabilities.length; i++) {
+            const isTracked = indices.includes(i);
+            if (isTracked) {
+                stabilities[i] = Math.min(1.0, stabilities[i] + 0.35); // Fast recovery (approx 3 frames)
+            } else {
+                stabilities[i] = Math.max(0.0, stabilities[i] - 0.12); // Slightly more forgiving loss
+            }
+        }
+
+        // Collect stabilities and FILTER OUT excessive flickerers (Dead Zone)
+        const filteredWorldCoords = [];
+        const filteredScreenCoords = [];
+        const filteredStabilities = [];
+
+        for (let i = 0; i < indices.length; i++) {
+            const s = stabilities[indices[i]];
+            if (s > 0.3) { // Hard Cutoff: points with <30% stability are ignored
+                filteredWorldCoords.push(worldCoords[i]);
+                filteredScreenCoords.push(screenCoords[i]);
+                filteredStabilities.push(s);
+            }
+        }
+
+        // STRICT QUALITY CHECK: Prevent "sticky" tracking on background noise.
+        // We require a minimum number of high-confidence AND STABLE points.
+        const stableAndReliable = reliabilities.filter((r: number, idx: number) => r > 0.75 && stabilities[indices[idx]] > 0.5).length;
+
+        if (stableAndReliable < 6 || filteredWorldCoords.length < 8) {
+            return { modelViewTransform: null, screenCoords: [], reliabilities: [], stabilities: [] };
+        }
+
         const modelViewTransform = await this._workerTrackUpdate(lastModelViewTransform, {
-            worldCoords,
-            screenCoords,
+            worldCoords: filteredWorldCoords,
+            screenCoords: filteredScreenCoords,
+            stabilities: filteredStabilities
         });
-        return modelViewTransform;
+
+        return {
+            modelViewTransform,
+            screenCoords: filteredScreenCoords,
+            reliabilities: reliabilities.filter((_, idx) => stabilities[indices[idx]] > 0.3),
+            stabilities: filteredStabilities
+        };
     }
 
     processVideo(input: any) {
         if (this.processingVideo) return;
         this.processingVideo = true;
+        const currentLoopId = ++loopIdCounter; // Added for ghost loop prevention
 
         this.trackingStates = [];
         for (let i = 0; i < (this.markerDimensions?.length || 0); i++) {
@@ -282,7 +337,7 @@ class Controller {
 
         const startProcessing = async () => {
             while (true) {
-                if (!this.processingVideo) break;
+                if (!this.processingVideo || currentLoopId !== loopIdCounter) break;
 
                 const inputData = this.inputLoader.loadInput(input);
                 const nTracking = this.trackingStates.reduce((acc, s) => acc + (!!s.isTracking ? 1 : 0), 0);
@@ -309,15 +364,21 @@ class Controller {
                     const trackingState = this.trackingStates[i];
 
                     if (trackingState.isTracking) {
-                        let modelViewTransform = await this._trackAndUpdate(
+                        const result = await this._trackAndUpdate(
                             inputData,
                             trackingState.currentModelViewTransform,
                             i,
                         );
-                        if (modelViewTransform === null) {
+                        if (result === null || result.modelViewTransform === null) {
                             trackingState.isTracking = false;
+                            trackingState.screenCoords = [];
+                            trackingState.reliabilities = [];
+                            trackingState.stabilities = [];
                         } else {
-                            trackingState.currentModelViewTransform = modelViewTransform;
+                            trackingState.currentModelViewTransform = result.modelViewTransform;
+                            trackingState.screenCoords = result.screenCoords;
+                            trackingState.reliabilities = result.reliabilities;
+                            trackingState.stabilities = result.stabilities;
                         }
                     }
 
@@ -332,7 +393,14 @@ class Controller {
 
                     if (trackingState.showing) {
                         const worldMatrix = this._glModelViewMatrix(trackingState.currentModelViewTransform, i);
-                        const filteredMatrix = this.featureManager.applyWorldMatrixFilters(i, worldMatrix);
+
+                        // Calculate confidence score based on point stability
+                        const stabilities = trackingState.stabilities || [];
+                        const avgStability = stabilities.length > 0
+                            ? stabilities.reduce((a: number, b: number) => a + b, 0) / stabilities.length
+                            : 0;
+
+                        const filteredMatrix = this.featureManager.applyWorldMatrixFilters(i, worldMatrix, { stability: avgStability });
                         trackingState.trackingMatrix = filteredMatrix;
 
                         let finalMatrix = [...filteredMatrix];
@@ -349,7 +417,10 @@ class Controller {
                             type: "updateMatrix",
                             targetIndex: i,
                             worldMatrix: finalMatrix,
-                            modelViewTransform: trackingState.currentModelViewTransform
+                            modelViewTransform: trackingState.currentModelViewTransform,
+                            screenCoords: trackingState.screenCoords,
+                            reliabilities: trackingState.reliabilities,
+                            stabilities: trackingState.stabilities
                         });
                     }
                 }
@@ -372,9 +443,8 @@ class Controller {
 
     async detect(input: any) {
         const inputData = this.inputLoader.loadInput(input);
-        const cropFeature = this.featureManager.getFeature<CropDetectionFeature>("crop-detection");
-        const { featurePoints, debugExtra } = cropFeature!.detect(inputData, false);
-        return { featurePoints, debugExtra };
+        const { featurePoints } = this.fullDetector!.detect(inputData);
+        return { featurePoints, debugExtra: {} };
     }
 
     async match(featurePoints: any, targetIndex: number) {
@@ -397,11 +467,18 @@ class Controller {
     _workerMatch(featurePoints: any, targetIndexes: number[]): Promise<any> {
         return new Promise((resolve) => {
             if (!this.worker) {
-                this._matchOnMainThread(featurePoints, targetIndexes).then(resolve);
+                this._matchOnMainThread(featurePoints, targetIndexes).then(resolve).catch(() => resolve({ targetIndex: -1 }));
                 return;
             }
 
+            const timeout = setTimeout(() => {
+                this.workerMatchDone = null;
+                resolve({ targetIndex: -1 });
+            }, WORKER_TIMEOUT_MS);
+
             this.workerMatchDone = (data: any) => {
+                clearTimeout(timeout);
+                this.workerMatchDone = null;
                 resolve({
                     targetIndex: data.targetIndex,
                     modelViewTransform: data.modelViewTransform,
@@ -460,19 +537,27 @@ class Controller {
     _workerTrackUpdate(modelViewTransform: number[][], trackingFeatures: any): Promise<any> {
         return new Promise((resolve) => {
             if (!this.worker) {
-                this._trackUpdateOnMainThread(modelViewTransform, trackingFeatures).then(resolve);
+                this._trackUpdateOnMainThread(modelViewTransform, trackingFeatures).then(resolve).catch(() => resolve(null));
                 return;
             }
 
+            const timeout = setTimeout(() => {
+                this.workerTrackDone = null;
+                resolve(null);
+            }, WORKER_TIMEOUT_MS);
+
             this.workerTrackDone = (data: any) => {
+                clearTimeout(timeout);
+                this.workerTrackDone = null;
                 resolve(data.modelViewTransform);
             };
-            const { worldCoords, screenCoords } = trackingFeatures;
+            const { worldCoords, screenCoords, stabilities } = trackingFeatures;
             this.worker.postMessage({
                 type: "trackUpdate",
                 modelViewTransform,
                 worldCoords,
                 screenCoords,
+                stabilities
             });
         });
     }
@@ -483,11 +568,12 @@ class Controller {
             this.mainThreadEstimator = new Estimator(this.projectionTransform);
         }
 
-        const { worldCoords, screenCoords } = trackingFeatures;
+        const { worldCoords, screenCoords, stabilities } = trackingFeatures;
         return this.mainThreadEstimator.refineEstimate({
             initialModelViewTransform: modelViewTransform,
             worldCoords,
             screenCoords,
+            stabilities
         });
     }
 

package/src/compiler/controller.worker.js

@@ -53,11 +53,12 @@ onmessage = (msg) => {
       break;
 
     case "trackUpdate":
-      const { modelViewTransform, worldCoords, screenCoords } = data;
+      const { modelViewTransform, worldCoords, screenCoords, stabilities } = data;
       const finalModelViewTransform = estimator.refineEstimate({
         initialModelViewTransform: modelViewTransform,
         worldCoords,
         screenCoords,
+        stabilities, // Stability-based weights
       });
       postMessage({
         type: "trackUpdateDone",

package/src/compiler/estimation/refine-estimate.js

@@ -21,6 +21,7 @@ const refineEstimate = ({
   projectionTransform,
   worldCoords,
   screenCoords,
+  stabilities, // Stability-based weighting
 }) => {
   // Question: shall we normlize the screen coords as well?
   // Question: do we need to normlize the scale as well, i.e. make coords from -1 to 1
@@ -74,6 +75,7 @@ const refineEstimate = ({
       projectionTransform,
       worldCoords: normalizedWorldCoords,
       screenCoords,
+      stabilities, // Pass weights to ICP
       inlierProb: inlierProbs[i],
     });
 
@@ -113,6 +115,7 @@ const _doICP = ({
   projectionTransform,
   worldCoords,
   screenCoords,
+  stabilities,
   inlierProb,
 }) => {
   const isRobustMode = inlierProb < 1;
@@ -196,7 +199,13 @@ const _doICP = ({
       });
 
       if (isRobustMode) {
-        const W = (1.0 - E[n] / K2) * (1.0 - E[n] / K2);
+        const robustW = (1.0 - E[n] / K2) * (1.0 - E[n] / K2);
+
+        // Log-weighted stability: suppresses vibrators aggressively but allows recovery
+        const s = stabilities ? stabilities[n] : 1.0;
+        const stabilityW = s * Math.log10(9 * s + 1);
+
+        const W = robustW * stabilityW;
 
         for (let j = 0; j < 2; j++) {
           for (let i = 0; i < 6; i++) {
@@ -206,8 +215,16 @@ const _doICP = ({
         dU.push([dxs[n] * W]);
         dU.push([dys[n] * W]);
       } else {
-        dU.push([dxs[n]]);
-        dU.push([dys[n]]);
+        const s = stabilities ? stabilities[n] : 1.0;
+        const W = s * Math.log10(9 * s + 1);
+
+        for (let j = 0; j < 2; j++) {
+          for (let i = 0; i < 6; i++) {
+            J_U_S[j][i] *= W;
+          }
+        }
+        dU.push([dxs[n] * W]);
+        dU.push([dys[n] * W]);
       }
 
       for (let i = 0; i < J_U_S.length; i++) {

package/src/compiler/features/feature-base.ts

@@ -39,7 +39,7 @@ export interface ControllerFeature extends Feature {
     /**
      * Hook to filter or modify the final world matrix
      */
-    filterWorldMatrix?(targetIndex: number, worldMatrix: number[]): number[];
+    filterWorldMatrix?(targetIndex: number, worldMatrix: number[], context?: any): number[];
 
     /**
      * Hook to decide if a target should be shown

package/src/compiler/features/feature-manager.ts

@@ -27,11 +27,11 @@ export class FeatureManager {
         }
     }
 
-    applyWorldMatrixFilters(targetIndex: number, worldMatrix: number[]): number[] {
+    applyWorldMatrixFilters(targetIndex: number, worldMatrix: number[], context?: any): number[] {
         let result = worldMatrix;
         for (const feature of this.features) {
             if (feature.enabled && feature.filterWorldMatrix) {
-                result = feature.filterWorldMatrix(targetIndex, result);
+                result = feature.filterWorldMatrix(targetIndex, result, context);
             }
         }
         return result;

package/src/compiler/features/one-euro-filter-feature.ts

@@ -30,9 +30,19 @@ export class OneEuroFilterFeature implements ControllerFeature {
         return this.filters[targetIndex];
     }
 
-    filterWorldMatrix(targetIndex: number, worldMatrix: number[]): number[] {
+    filterWorldMatrix(targetIndex: number, worldMatrix: number[], context?: any): number[] {
         if (!this.enabled) return worldMatrix;
+
         const filter = this.getFilter(targetIndex);
+        const stability = context?.stability ?? 1.0;
+
+        // Dynamic Cutoff: If points are very stable (1.0), use higher cutoff (less responsiveness loss).
+        // If points are unstable (0.3), use much lower cutoff (heavy smoothing).
+        // We use a squared curve for even more aggressive suppression of jitter on unstable points.
+        const dynamicMinCutOff = this.minCutOff * (0.05 + Math.pow(stability, 2) * 0.95);
+        filter.minCutOff = dynamicMinCutOff;
+        filter.beta = this.beta;
+
         return filter.filter(Date.now(), worldMatrix);
     }
 

package/src/compiler/matching/matching.js

@@ -235,7 +235,7 @@ const _query = ({ node, descriptors, querypoint, queue, keypointIndexes, numPop
     if (dist !== minD) {
       queue.push({ node: childrenOrIndices[i], d: dist });
     } else {
-      _query({ node: childrenOrIndices[i], descriptors, querypoint, queue, keypointIndexes, numPop });
+      _query({ node: childrenOrIndices[i], descriptors, querypoint, queue, keypointIndexes, numPop: numPop + 1 });
     }
   }
 

package/src/compiler/offline-compiler.ts

@@ -11,7 +11,6 @@ import { extractTrackingFeatures } from "./tracker/extract-utils.js";
 import { DetectorLite } from "./detector/detector-lite.js";
 import { build as hierarchicalClusteringBuild } from "./matching/hierarchical-clustering.js";
 import * as msgpack from "@msgpack/msgpack";
-import { WorkerPool } from "./utils/worker-pool.js";
 
 // Detect environment
 const isNode = typeof process !== "undefined" &&
@@ -22,40 +21,9 @@ const CURRENT_VERSION = 7; // Protocol v7: Moonshot - 4-bit Packed Tracking Data
 
 export class OfflineCompiler {
     data: any = null;
-    workerPool: WorkerPool | null = null;
 
     constructor() {
-        // Workers only in Node.js
-        if (!isNode) {
-            console.log("🌐 OfflineCompiler: Browser mode (no workers)");
-        }
-    }
-
-    async _initNodeWorkers() {
-        try {
-            const pathModule = "path";
-            const urlModule = "url";
-            const osModule = "os";
-            const workerThreadsModule = "node:worker_threads";
-
-            const [path, url, os, { Worker }] = await Promise.all([
-                import(/* @vite-ignore */ pathModule),
-                import(/* @vite-ignore */ urlModule),
-                import(/* @vite-ignore */ osModule),
-                import(/* @vite-ignore */ workerThreadsModule)
-            ]);
-
-            const __filename = url.fileURLToPath(import.meta.url);
-            const __dirname = path.dirname(__filename);
-            const workerPath = path.join(__dirname, "node-worker.js");
-
-            // Limit workers to avoid freezing system
-            const numWorkers = Math.min(os.cpus().length, 4);
-
-            this.workerPool = new WorkerPool(workerPath, numWorkers, Worker);
-        } catch (e) {
-            console.log("⚡ OfflineCompiler: Running without workers (initialization failed)", e);
-        }
+        console.log("⚡ OfflineCompiler: Main thread mode (no workers)");
     }
 
     async compileImageTargets(images: any[], progressCallback: (p: number) => void) {
@@ -108,25 +76,7 @@ export class OfflineCompiler {
     }
 
     async _compileTarget(targetImages: any[], progressCallback: (p: number) => void) {
-        if (isNode) await this._initNodeWorkers();
-
-        if (this.workerPool) {
-            const progressMap = new Float32Array(targetImages.length);
-            const wrappedPromises = targetImages.map((targetImage: any, index: number) => {
-                return this.workerPool!.runTask({
-                    type: 'compile-all', // 🚀 MOONSHOT: Combined task
-                    targetImage,
-                    onProgress: (p: number) => {
-                        progressMap[index] = p;
-                        const sum = progressMap.reduce((a, b) => a + b, 0);
-                        progressCallback(sum / targetImages.length);
-                    }
-                });
-            });
-            return Promise.all(wrappedPromises);
-        }
-
-        // Fallback or non-worker implementation: run match and track sequentially
+        // Run match and track sequentially to match browser behavior exactly
         const matchingResults = await this._compileMatch(targetImages, (p) => progressCallback(p * 0.5));
         const trackingResults = await this._compileTrack(targetImages, (p) => progressCallback(50 + p * 0.5));
 
@@ -140,27 +90,6 @@ export class OfflineCompiler {
         const percentPerImage = 100 / targetImages.length;
         let currentPercent = 0;
 
-        if (isNode) await this._initNodeWorkers();
-        if (this.workerPool) {
-            const progressMap = new Float32Array(targetImages.length);
-
-            const wrappedPromises = targetImages.map((targetImage: any, index: number) => {
-                return this.workerPool!.runTask({
-                    type: 'match',
-                    targetImage,
-                    percentPerImage,
-                    basePercent: 0,
-                    onProgress: (p: number) => {
-                        progressMap[index] = p;
-                        const sum = progressMap.reduce((a, b) => a + b, 0);
-                        progressCallback(sum);
-                    }
-                });
-            });
-
-            return Promise.all(wrappedPromises);
-        }
-
         const results = [];
         for (let i = 0; i < targetImages.length; i++) {
             const targetImage = targetImages[i];
@@ -201,24 +130,6 @@ export class OfflineCompiler {
         const percentPerImage = 100 / targetImages.length;
         let currentPercent = 0;
 
-        if (this.workerPool) {
-            const progressMap = new Float32Array(targetImages.length);
-            const wrappedPromises = targetImages.map((targetImage: any, index: number) => {
-                return this.workerPool!.runTask({
-                    type: 'compile',
-                    targetImage,
-                    percentPerImage,
-                    basePercent: 0,
-                    onProgress: (p: number) => {
-                        progressMap[index] = p;
-                        const sum = progressMap.reduce((a, b) => a + b, 0);
-                        progressCallback(sum);
-                    }
-                });
-            });
-            return Promise.all(wrappedPromises);
-        }
-
         const results = [];
         for (let i = 0; i < targetImages.length; i++) {
             const targetImage = targetImages[i];
@@ -480,9 +391,7 @@ export class OfflineCompiler {
     }
 
     async destroy() {
-        if (this.workerPool) {
-            await this.workerPool.destroy();
-        }
+        // No workers to destroy
     }
 
 

package/src/compiler/simple-ar.ts

@@ -13,7 +13,13 @@ export interface SimpleAROptions {
     scale?: number;
     onFound?: ((data: { targetIndex: number }) => void | Promise<void>) | null;
     onLost?: ((data: { targetIndex: number }) => void | Promise<void>) | null;
-    onUpdate?: ((data: { targetIndex: number, worldMatrix: number[] }) => void) | null;
+    onUpdate?: ((data: {
+        targetIndex: number,
+        worldMatrix: number[],
+        screenCoords?: { x: number, y: number }[],
+        reliabilities?: number[],
+        stabilities?: number[]
+    }) => void) | null;
     cameraConfig?: MediaStreamConstraints['video'];
     debug?: boolean;
 }
@@ -25,7 +31,13 @@ class SimpleAR {
     scaleMultiplier: number;
     onFound: ((data: { targetIndex: number }) => void | Promise<void>) | null;
     onLost: ((data: { targetIndex: number }) => void | Promise<void>) | null;
-    onUpdateCallback: ((data: { targetIndex: number, worldMatrix: number[] }) => void) | null;
+    onUpdateCallback: ((data: {
+        targetIndex: number,
+        worldMatrix: number[],
+        screenCoords?: { x: number, y: number }[],
+        reliabilities?: number[],
+        stabilities?: number[]
+    }) => void) | null;
     cameraConfig: MediaStreamConstraints['video'];
     debug: boolean;
 
@@ -151,7 +163,7 @@ class SimpleAR {
             if (this.debug) this._updateDebugPanel(this.isTracking);
         }
 
-        const { targetIndex, worldMatrix, modelViewTransform } = data;
+        const { targetIndex, worldMatrix, modelViewTransform, screenCoords, reliabilities, stabilities } = data;
 
         if (worldMatrix) {
             if (!this.isTracking) {
@@ -162,31 +174,61 @@ class SimpleAR {
 
             this.lastMatrix = worldMatrix;
 
-            if (!this.filters[targetIndex]) {
-                this.filters[targetIndex] = new OneEuroFilter({ minCutOff: 0.8, beta: 0.2 });
+            // We use the matrix from the controller directly (it's already filtered there)
+            this._positionOverlay(modelViewTransform, targetIndex);
+
+            // Project points to screen coordinates
+            let projectedPoints = [];
+            if (screenCoords && screenCoords.length > 0) {
+                const containerRect = this.container.getBoundingClientRect();
+                const videoW = this.video!.videoWidth;
+                const videoH = this.video!.videoHeight;
+                const isPortrait = containerRect.height > containerRect.width;
+                const isVideoLandscape = videoW > videoH;
+                const needsRotation = isPortrait && isVideoLandscape;
+                const proj = this.controller!.projectionTransform;
+
+                const vW = needsRotation ? videoH : videoW;
+                const vH = needsRotation ? videoW : videoH;
+                const pScale = Math.max(containerRect.width / vW, containerRect.height / vH);
+                const dW = vW * pScale;
+                const dH = vH * pScale;
+                const oX = (containerRect.width - dW) / 2;
+                const oY = (containerRect.height - dH) / 2;
+
+                projectedPoints = screenCoords.map((p: any) => {
+                    let sx, sy;
+                    if (needsRotation) {
+                        sx = oX + (dW / 2) - (p.y - proj[1][2]) * pScale;
+                        sy = oY + (dH / 2) + (p.x - proj[0][2]) * pScale;
+                    } else {
+                        sx = oX + (dW / 2) + (p.x - proj[0][2]) * pScale;
+                        sy = oY + (dH / 2) + (p.y - proj[1][2]) * pScale;
+                    }
+                    return { x: sx, y: sy };
+                });
             }
 
-            const flatMVT = [
-                modelViewTransform[0][0], modelViewTransform[0][1], modelViewTransform[0][2], modelViewTransform[0][3],
-                modelViewTransform[1][0], modelViewTransform[1][1], modelViewTransform[1][2], modelViewTransform[1][3],
-                modelViewTransform[2][0], modelViewTransform[2][1], modelViewTransform[2][2], modelViewTransform[2][3]
-            ];
-            const smoothedFlat = this.filters[targetIndex].filter(Date.now(), flatMVT);
-            const smoothedMVT = [
-                [smoothedFlat[0], smoothedFlat[1], smoothedFlat[2], smoothedFlat[3]],
-                [smoothedFlat[4], smoothedFlat[5], smoothedFlat[6], smoothedFlat[7]],
-                [smoothedFlat[8], smoothedFlat[9], smoothedFlat[10], smoothedFlat[11]]
-            ];
-
-            this._positionOverlay(smoothedMVT, targetIndex);
-            this.onUpdateCallback && this.onUpdateCallback({ targetIndex, worldMatrix });
+            this.onUpdateCallback && this.onUpdateCallback({
+                targetIndex,
+                worldMatrix,
+                screenCoords: projectedPoints,
+                reliabilities,
+                stabilities
+            });
 
         } else {
             if (this.isTracking) {
                 this.isTracking = false;
-                if (this.filters[targetIndex]) this.filters[targetIndex].reset();
                 this.overlay && (this.overlay.style.opacity = '0');
                 this.onLost && this.onLost({ targetIndex });
+                this.onUpdateCallback && this.onUpdateCallback({
+                    targetIndex,
+                    worldMatrix: null as any,
+                    screenCoords: [],
+                    reliabilities: [],
+                    stabilities: []
+                });
             }
         }
     }