@srsergio/taptapp-ar 1.0.79 → 1.0.80

package/src/compiler/simple-ar.ts

@@ -13,7 +13,13 @@ export interface SimpleAROptions {
     scale?: number;
     onFound?: ((data: { targetIndex: number }) => void | Promise<void>) | null;
     onLost?: ((data: { targetIndex: number }) => void | Promise<void>) | null;
-    onUpdate?: ((data: { targetIndex: number, worldMatrix: number[] }) => void) | null;
+    onUpdate?: ((data: {
+        targetIndex: number,
+        worldMatrix: number[],
+        screenCoords?: { x: number, y: number }[],
+        reliabilities?: number[],
+        stabilities?: number[]
+    }) => void) | null;
     cameraConfig?: MediaStreamConstraints['video'];
     debug?: boolean;
 }
@@ -25,7 +31,13 @@ class SimpleAR {
     scaleMultiplier: number;
     onFound: ((data: { targetIndex: number }) => void | Promise<void>) | null;
     onLost: ((data: { targetIndex: number }) => void | Promise<void>) | null;
-    onUpdateCallback: ((data: { targetIndex: number, worldMatrix: number[] }) => void) | null;
+    onUpdateCallback: ((data: {
+        targetIndex: number,
+        worldMatrix: number[],
+        screenCoords?: { x: number, y: number }[],
+        reliabilities?: number[],
+        stabilities?: number[]
+    }) => void) | null;
     cameraConfig: MediaStreamConstraints['video'];
     debug: boolean;
 
@@ -151,7 +163,7 @@ class SimpleAR {
             if (this.debug) this._updateDebugPanel(this.isTracking);
         }
 
-        const { targetIndex, worldMatrix, modelViewTransform } = data;
+        const { targetIndex, worldMatrix, modelViewTransform, screenCoords, reliabilities, stabilities } = data;
 
         if (worldMatrix) {
             if (!this.isTracking) {
@@ -162,31 +174,61 @@ class SimpleAR {
 
             this.lastMatrix = worldMatrix;
 
-            if (!this.filters[targetIndex]) {
-                this.filters[targetIndex] = new OneEuroFilter({ minCutOff: 0.8, beta: 0.2 });
+            // We use the matrix from the controller directly (it's already filtered there)
+            this._positionOverlay(modelViewTransform, targetIndex);
+
+            // Project points to screen coordinates
+            let projectedPoints = [];
+            if (screenCoords && screenCoords.length > 0) {
+                const containerRect = this.container.getBoundingClientRect();
+                const videoW = this.video!.videoWidth;
+                const videoH = this.video!.videoHeight;
+                const isPortrait = containerRect.height > containerRect.width;
+                const isVideoLandscape = videoW > videoH;
+                const needsRotation = isPortrait && isVideoLandscape;
+                const proj = this.controller!.projectionTransform;
+
+                const vW = needsRotation ? videoH : videoW;
+                const vH = needsRotation ? videoW : videoH;
+                const pScale = Math.max(containerRect.width / vW, containerRect.height / vH);
+                const dW = vW * pScale;
+                const dH = vH * pScale;
+                const oX = (containerRect.width - dW) / 2;
+                const oY = (containerRect.height - dH) / 2;
+
+                projectedPoints = screenCoords.map((p: any) => {
+                    let sx, sy;
+                    if (needsRotation) {
+                        sx = oX + (dW / 2) - (p.y - proj[1][2]) * pScale;
+                        sy = oY + (dH / 2) + (p.x - proj[0][2]) * pScale;
+                    } else {
+                        sx = oX + (dW / 2) + (p.x - proj[0][2]) * pScale;
+                        sy = oY + (dH / 2) + (p.y - proj[1][2]) * pScale;
+                    }
+                    return { x: sx, y: sy };
+                });
             }
 
-            const flatMVT = [
-                modelViewTransform[0][0], modelViewTransform[0][1], modelViewTransform[0][2], modelViewTransform[0][3],
-                modelViewTransform[1][0], modelViewTransform[1][1], modelViewTransform[1][2], modelViewTransform[1][3],
-                modelViewTransform[2][0], modelViewTransform[2][1], modelViewTransform[2][2], modelViewTransform[2][3]
-            ];
-            const smoothedFlat = this.filters[targetIndex].filter(Date.now(), flatMVT);
-            const smoothedMVT = [
-                [smoothedFlat[0], smoothedFlat[1], smoothedFlat[2], smoothedFlat[3]],
-                [smoothedFlat[4], smoothedFlat[5], smoothedFlat[6], smoothedFlat[7]],
-                [smoothedFlat[8], smoothedFlat[9], smoothedFlat[10], smoothedFlat[11]]
-            ];
-
-            this._positionOverlay(smoothedMVT, targetIndex);
-            this.onUpdateCallback && this.onUpdateCallback({ targetIndex, worldMatrix });
+            this.onUpdateCallback && this.onUpdateCallback({
+                targetIndex,
+                worldMatrix,
+                screenCoords: projectedPoints,
+                reliabilities,
+                stabilities
+            });
 
         } else {
             if (this.isTracking) {
                 this.isTracking = false;
-                if (this.filters[targetIndex]) this.filters[targetIndex].reset();
                 this.overlay && (this.overlay.style.opacity = '0');
                 this.onLost && this.onLost({ targetIndex });
+                this.onUpdateCallback && this.onUpdateCallback({
+                    targetIndex,
+                    worldMatrix: null as any,
+                    screenCoords: [],
+                    reliabilities: [],
+                    stabilities: []
+                });
             }
         }
     }

package/src/compiler/tracker/tracker.js

@@ -2,9 +2,9 @@ import { buildModelViewProjectionTransform, computeScreenCoordiate } from "../es
 
 const AR2_DEFAULT_TS = 6;
 const AR2_DEFAULT_TS_GAP = 1;
-const AR2_SEARCH_SIZE = 34; // Increased from 18 to 34 for much better fast-motion tracking
+const AR2_SEARCH_SIZE = 25; // Reduced from 34 to 25 to prevent background latching
 const AR2_SEARCH_GAP = 1;
-const AR2_SIM_THRESH = 0.6;
+const AR2_SIM_THRESH = 0.65; // Increased from 0.6 to reduce false positives
 
 const TRACKING_KEYFRAME = 0; // 0: 128px (optimized)
 
@@ -119,6 +119,8 @@ class Tracker {
 
     const { px, py, s: scale } = trackingFrame;
 
+    const reliabilities = [];
+
     for (let i = 0; i < matchingPoints.length; i++) {
       if (sim[i] > AR2_SIM_THRESH && i < px.length) {
         goodTrack.push(i);
@@ -133,6 +135,7 @@ class Tracker {
           y: py[i] / scale,
           z: 0,
         });
+        reliabilities.push(sim[i]);
       }
     }
 
@@ -147,7 +150,7 @@ class Tracker {
 
       // If the points cover too little space compared to the screen size of the marker, it's a glitch
       if (detectedDiagonal < screenW * 0.15) {
-        return { worldCoords: [], screenCoords: [], debugExtra };
+        return { worldCoords: [], screenCoords: [], reliabilities: [], debugExtra };
       }
     }
     if (this.debugMode) {
@@ -160,7 +163,7 @@ class Tracker {
       };
     }
 
-    return { worldCoords, screenCoords, debugExtra };
+    return { worldCoords, screenCoords, reliabilities, indices: goodTrack, octaveIndex, debugExtra };
   }
 
   /**

package/src/react/TaptappAR.tsx

@@ -15,7 +15,7 @@ export const TaptappAR: React.FC<TaptappARProps> = ({
     showScanningOverlay = true,
     showErrorOverlay = true
 }) => {
-    const { containerRef, overlayRef, status, toggleVideo } = useAR(config);
+    const { containerRef, overlayRef, status, toggleVideo, trackedPoints } = useAR(config);
 
     // Simple heuristic to determine if it's a video or image
     // based on the presence of videoSrc and common extensions
@@ -89,6 +89,25 @@ export const TaptappAR: React.FC<TaptappARProps> = ({
                 )}
             </div>
 
+            {/* Tracking Points Layer */}
+            {status === "tracking" && (
+                <div className="taptapp-ar-points-overlay">
+                    {trackedPoints.filter(p => p.reliability > 0.7).map((point, i) => (
+                        <div
+                            key={i}
+                            className="tracking-point"
+                            style={{
+                                left: `${point.x}px`,
+                                top: `${point.y}px`,
+                                width: `${(2 + point.reliability * 6) * (0.4 + point.stability * 0.6)}px`,
+                                height: `${(2 + point.reliability * 6) * (0.4 + point.stability * 0.6)}px`,
+                                opacity: (0.3 + (point.reliability * 0.4)) * (0.2 + point.stability * 0.8)
+                            }}
+                        />
+                    ))}
+                </div>
+            )}
+
             <style>{`
                 .taptapp-ar-wrapper {
                     background: #000;
@@ -173,6 +192,24 @@ export const TaptappAR: React.FC<TaptappARProps> = ({
                     pointer-events: none;
                     transition: opacity 0.3s ease;
                 }
+                .taptapp-ar-points-overlay {
+                    position: absolute;
+                    top: 0;
+                    left: 0;
+                    width: 100%;
+                    height: 100%;
+                    pointer-events: none;
+                    z-index: 100; /* High z-index to be above overlay */
+                }
+                .tracking-point {
+                    position: absolute;
+                    background: black;
+                    border: 1px solid rgba(255,255,255,0.5); /* Better contrast */
+                    border-radius: 50%;
+                    transform: translate(-50%, -50%);
+                    box-shadow: 0 0 2px rgba(255, 255, 255, 0.8);
+                    pointer-events: none;
+                }
             `}</style>
         </div>
     );

package/src/react/use-ar.ts

@@ -4,12 +4,20 @@ import type { SimpleAR as SimpleARType } from "../compiler/simple-ar.js";
 
 export type ARStatus = "scanning" | "tracking" | "error";
 
+export interface TrackedPoint {
+    x: number;
+    y: number;
+    reliability: number;
+    stability: number;
+}
+
 export interface UseARReturn {
     containerRef: React.RefObject<HTMLDivElement>;
     overlayRef: React.RefObject<HTMLVideoElement | HTMLImageElement>;
     status: ARStatus;
     isPlaying: boolean;
     toggleVideo: () => Promise<void>;
+    trackedPoints: TrackedPoint[];
 }
 
 export const useAR = (config: ARConfig): UseARReturn => {
@@ -17,6 +25,7 @@ export const useAR = (config: ARConfig): UseARReturn => {
     const overlayRef = useRef<HTMLVideoElement | HTMLImageElement>(null);
     const [status, setStatus] = useState<ARStatus>("scanning");
     const [isPlaying, setIsPlaying] = useState(false);
+    const [trackedPoints, setTrackedPoints] = useState<TrackedPoint[]>([]);
     const arInstanceRef = useRef<SimpleARType | null>(null);
 
     const toggleVideo = useCallback(async () => {
@@ -53,6 +62,17 @@ export const useAR = (config: ARConfig): UseARReturn => {
                     overlay: overlayRef.current!,
                     scale: config.scale,
                     debug: false,
+                    onUpdate: ({ screenCoords, reliabilities, stabilities }) => {
+                        if (screenCoords && reliabilities && stabilities) {
+                            const points = screenCoords.map((p, i) => ({
+                                x: p.x,
+                                y: p.y,
+                                reliability: reliabilities[i],
+                                stability: stabilities[i]
+                            }));
+                            setTrackedPoints(points);
+                        }
+                    },
                     onFound: async ({ targetIndex }) => {
                         console.log(`🎯 Target ${targetIndex} detected!`);
                         if (!isMounted) return;
@@ -72,6 +92,7 @@ export const useAR = (config: ARConfig): UseARReturn => {
                         console.log(`👋 Target ${targetIndex} lost`);
                         if (!isMounted) return;
                         setStatus("scanning");
+                        setTrackedPoints([]);
 
                         const overlay = overlayRef.current;
                         if (overlay instanceof HTMLVideoElement) {
@@ -105,6 +126,7 @@ export const useAR = (config: ARConfig): UseARReturn => {
         overlayRef,
         status,
         isPlaying,
-        toggleVideo
+        toggleVideo,
+        trackedPoints
     };
 };