@srsergio/taptapp-ar 1.0.35 → 1.0.36

package/dist/compiler/offline-compiler.js

@@ -217,9 +217,9 @@ export class OfflineCompiler {
         }
         const dataList = this.data.map((item) => {
             const matchingData = item.matchingData.map((kf) => this._packKeyframe(kf));
-            const trackingData = [item.trackingData[0]].map((td) => {
+            const trackingData = item.trackingData.map((td) => {
                 const count = td.points.length;
-                // Step 1: Packed Coords - Normalize width/height to 16-bit
+                // Packed Coords - Float32 for now as in current import logic
                 const px = new Float32Array(count);
                 const py = new Float32Array(count);
                 for (let i = 0; i < count; i++) {

package/dist/compiler/simple-ar.js

@@ -192,60 +192,69 @@ class SimpleAR {
         const needsRotation = isPortrait && isVideoLandscape;
         // 3. Get intrinsic projection from controller
         const proj = this.controller.projectionTransform;
-        // 4. Position calculation via matrix3d (Support for 3D tilt/Z-rotation)
-        // We convert the OpenGL World Matrix to a CSS matrix3d.
-        // The OpenGL matrix is column-major. CSS matrix3d is also column-major.
-        const m = this.controller.getWorldMatrix(mVT, targetIndex);
-        // Map OpenGL coords to Screen Pixels using the projection logic
-        const vW = needsRotation ? videoH : videoW;
-        const vH = needsRotation ? videoW : videoH;
-        const perspectiveScale = Math.max(containerRect.width / vW, containerRect.height / vH);
-        const displayW = vW * perspectiveScale;
-        const displayH = vH * perspectiveScale;
-        const offsetX = (containerRect.width - displayW) / 2;
-        const offsetY = (containerRect.height - displayH) / 2;
-        // Adjust for centered marker and scaleMultiplier
-        const s = finalScale; // We still need the base scale factor for the pixel-to-marker mapping
-        // However, a cleaner way is to use the world matrix directly and map it.
-        // Actually, the simpler way to do 3D in CSS while keeping my projection logic is:
-        // Project the 4 corners and find the homography, OR
-        // Use the OpenGL matrix directly with a perspective mapping.
-        // Let's use the points projection to maintain the "needsRotation" logic compatibility
-        const pMid = projectToScreen(markerW / 2, markerH / 2, 0, mVT, proj, videoW, videoH, containerRect, needsRotation);
+        // 3. Project 4 corners to determine a full 3D perspective (homography)
         const pUL = projectToScreen(0, 0, 0, mVT, proj, videoW, videoH, containerRect, needsRotation);
         const pUR = projectToScreen(markerW, 0, 0, mVT, proj, videoW, videoH, containerRect, needsRotation);
         const pLL = projectToScreen(0, markerH, 0, mVT, proj, videoW, videoH, containerRect, needsRotation);
-        // Using these points we can calculate the 3D rotation and perspective
-        const dx = pUR.sx - pUL.sx;
-        const dy = pUR.sy - pUL.sy;
-        const dz = pUR.sx - pLL.sx; // Not really Z but used for slant
-        const angle = Math.atan2(dy, dx);
-        const scaleX = Math.sqrt(dx * dx + dy * dy) / markerW;
-        const scaleY = Math.sqrt((pLL.sx - pUL.sx) ** 2 + (pLL.sy - pUL.sy) ** 2) / markerH;
-        // For true 3D tilt, we'll use the projection of the axes
-        const screenX = pMid.sx;
-        const screenY = pMid.sy;
-        // Final Transform applying 3D perspective via matrix3d derived from projected points
-        // NOTE: For full 3D we'd need a homography solver, but for "tilt" we can use the 
-        // original modelViewTransform if we convert it carefully.
-        const openGLWorldMatrix = this.controller.getWorldMatrix(mVT, targetIndex);
-        // We need to apply the same scaling and offsets as projectToScreen to the matrix
+        const pLR = projectToScreen(markerW, markerH, 0, mVT, proj, videoW, videoH, containerRect, needsRotation);
+        // Helper to solve for 2D Homography (maps 0..1 square to pUL, pUR, pLL, pLR)
+        const solveHomography = (w, h, p1, p2, p3, p4) => {
+            const x1 = p1.sx, y1 = p1.sy;
+            const x2 = p2.sx, y2 = p2.sy;
+            const x3 = p3.sx, y3 = p3.sy;
+            const x4 = p4.sx, y4 = p4.sy;
+            const dx1 = x2 - x4, dx2 = x3 - x4, dx3 = x1 - x2 + x4 - x3;
+            const dy1 = y2 - y4, dy2 = y3 - y4, dy3 = y1 - y2 + y4 - y3;
+            let a, b, c, d, e, f, g, h_coeff;
+            if (dx3 === 0 && dy3 === 0) {
+                a = x2 - x1;
+                b = x3 - x1;
+                c = x1;
+                d = y2 - y1;
+                e = y3 - y1;
+                f = y1;
+                g = 0;
+                h_coeff = 0;
+            }
+            else {
+                const det = dx1 * dy2 - dx2 * dy1;
+                g = (dx3 * dy2 - dx2 * dy3) / det;
+                h_coeff = (dx1 * dy3 - dx3 * dy1) / det;
+                a = x2 - x1 + g * x2;
+                b = x3 - x1 + h_coeff * x3;
+                c = x1;
+                d = y2 - y1 + g * y2;
+                e = y3 - y1 + h_coeff * y3;
+                f = y1;
+            }
+            // This maps unit square (0..1) to the quadrilateral.
+            // We need to scale it by 1/w and 1/h to map (0..w, 0..h)
+            return [
+                a / w, d / w, 0, g / w,
+                b / h, e / h, 0, h_coeff / h,
+                0, 0, 1, 0,
+                c, f, 0, 1
+            ];
+        };
+        const matrix = solveHomography(markerW, markerH, pUL, pUR, pLL, pLR);
+        // Apply styles
         this.overlay.style.maxWidth = 'none';
         this.overlay.style.width = `${markerW}px`;
         this.overlay.style.height = `${markerH}px`;
         this.overlay.style.position = 'absolute';
-        this.overlay.style.transformOrigin = '0 0'; // Top-left based for simpler matrix mapping
+        this.overlay.style.transformOrigin = '0 0';
         this.overlay.style.left = '0';
         this.overlay.style.top = '0';
         this.overlay.style.display = 'block';
-        // Approximate 3D tilt using the projected corners to calculate a skew/scale/rotate combo
-        // This is more robust than a raw matrix3d if the projection isn't a perfect pinhole
+        // Apply 3D transform with matrix3d
+        // We also apply the user's custom scaleMultiplier AFTER the perspective transform
+        // but since we want to scale around the marker center, we apply it as a prefix/suffix
+        // Scale around top-left (0,0) is easy. Scale around center requires offset.
         this.overlay.style.transform = `
-            translate(${pUL.sx}px, ${pUL.sy}px)
-            matrix(${(pUR.sx - pUL.sx) / markerW}, ${(pUR.sy - pUL.sy) / markerW}, 
-                   ${(pLL.sx - pUL.sx) / markerH}, ${(pLL.sy - pUL.sy) / markerH}, 
-                   0, 0)
+            matrix3d(${matrix.join(',')})
+            translate(${markerW / 2}px, ${markerH / 2}px)
             scale(${this.scaleMultiplier})
+            translate(${-markerW / 2}px, ${-markerH / 2}px)
         `;
     }
     // Unified projection logic moved to ./utils/projection.js

package/dist/compiler/tracker/tracker.d.ts

@@ -7,15 +7,7 @@ export class Tracker {
     inputHeight: any;
     debugMode: boolean;
     trackingKeyframeList: any[];
-    prebuiltData: {
-        px: Float32Array<any>;
-        py: Float32Array<any>;
-        data: Uint8Array<any>;
-        width: any;
-        height: any;
-        scale: any;
-        projectedImage: Float32Array<ArrayBuffer>;
-    }[];
+    prebuiltData: any[];
     templateBuffer: Float32Array<ArrayBuffer>;
     dummyRun(inputData: any): void;
     track(inputData: any, lastModelViewTransform: any, targetIndex: any): {

package/dist/compiler/tracker/tracker.js

@@ -13,15 +13,12 @@ class Tracker {
         this.inputWidth = inputWidth;
         this.inputHeight = inputHeight;
         this.debugMode = debugMode;
-        this.trackingKeyframeList = [];
+        this.trackingKeyframeList = []; // All octaves for all targets: [targetIndex][octaveIndex]
+        this.prebuiltData = []; // [targetIndex][octaveIndex]
         for (let i = 0; i < trackingDataList.length; i++) {
-            this.trackingKeyframeList.push(trackingDataList[i][TRACKING_KEYFRAME]);
-        }
-        // Prebuild TypedArrays for features and pixels
-        this.prebuiltData = [];
-        for (let i = 0; i < this.trackingKeyframeList.length; i++) {
-            const keyframe = this.trackingKeyframeList[i];
-            this.prebuiltData[i] = {
+            const targetOctaves = trackingDataList[i];
+            this.trackingKeyframeList[i] = targetOctaves;
+            this.prebuiltData[i] = targetOctaves.map(keyframe => ({
                 px: new Float32Array(keyframe.px),
                 py: new Float32Array(keyframe.py),
                 data: new Uint8Array(keyframe.d),
@@ -30,7 +27,7 @@ class Tracker {
                 scale: keyframe.s,
                 // Recyclable projected image buffer
                 projectedImage: new Float32Array(keyframe.w * keyframe.h)
-            };
+            }));
         }
         // Pre-allocate template data buffer to avoid garbage collection
         const templateOneSize = AR2_DEFAULT_TS;
@@ -49,14 +46,31 @@ class Tracker {
     }
     track(inputData, lastModelViewTransform, targetIndex) {
         let debugExtra = {};
+        // Select the best octave based on current estimated distance/scale
+        // We want the octave where the marker size is closest to its projected size on screen
         const modelViewProjectionTransform = buildModelViewProjectionTransform(this.projectionTransform, lastModelViewTransform);
-        const prebuilt = this.prebuiltData[targetIndex];
+        // Estimate current marker width on screen
+        const [mW, mH] = this.markerDimensions[targetIndex];
+        const p0 = computeScreenCoordiate(modelViewProjectionTransform, 0, 0);
+        const p1 = computeScreenCoordiate(modelViewProjectionTransform, mW, 0);
+        const screenW = Math.sqrt((p1.x - p0.x) ** 2 + (p1.y - p0.y) ** 2);
+        // Select octave whose image width is closest to screenW
+        let octaveIndex = 0;
+        let minDiff = Infinity;
+        for (let i = 0; i < this.prebuiltData[targetIndex].length; i++) {
+            const diff = Math.abs(this.prebuiltData[targetIndex][i].width - screenW);
+            if (diff < minDiff) {
+                minDiff = diff;
+                octaveIndex = i;
+            }
+        }
+        const prebuilt = this.prebuiltData[targetIndex][octaveIndex];
         // 1. Compute Projection (Warping)
         this._computeProjection(modelViewProjectionTransform, inputData, prebuilt);
         const projectedImage = prebuilt.projectedImage;
         // 2. Compute Matching (NCC)
         const { matchingPoints, sim } = this._computeMatching(prebuilt, projectedImage);
-        const trackingFrame = this.trackingKeyframeList[targetIndex];
+        const trackingFrame = this.trackingKeyframeList[targetIndex][octaveIndex];
         const worldCoords = [];
         const screenCoords = [];
         const goodTrack = [];
@@ -75,6 +89,7 @@ class Tracker {
         }
         if (this.debugMode) {
             debugExtra = {
+                octaveIndex,
                 projectedImage: Array.from(projectedImage),
                 matchingPoints,
                 goodTrack,

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@srsergio/taptapp-ar",
-    "version": "1.0.35",
+    "version": "1.0.36",
     "description": "AR Compiler for Node.js and Browser",
     "repository": {
         "type": "git",

package/src/compiler/offline-compiler.js

@@ -260,9 +260,9 @@ export class OfflineCompiler {
     const dataList = this.data.map((item) => {
       const matchingData = item.matchingData.map((kf) => this._packKeyframe(kf));
 
-      const trackingData = [item.trackingData[0]].map((td) => {
+      const trackingData = item.trackingData.map((td) => {
         const count = td.points.length;
-        // Step 1: Packed Coords - Normalize width/height to 16-bit
+        // Packed Coords - Float32 for now as in current import logic
         const px = new Float32Array(count);
         const py = new Float32Array(count);
         for (let i = 0; i < count; i++) {

package/src/compiler/simple-ar.js

@@ -222,71 +222,70 @@ class SimpleAR {
         // 3. Get intrinsic projection from controller
         const proj = this.controller.projectionTransform;
 
-        // 4. Position calculation via matrix3d (Support for 3D tilt/Z-rotation)
-        // We convert the OpenGL World Matrix to a CSS matrix3d.
-        // The OpenGL matrix is column-major. CSS matrix3d is also column-major.
-        const m = this.controller.getWorldMatrix(mVT, targetIndex);
-
-        // Map OpenGL coords to Screen Pixels using the projection logic
-        const vW = needsRotation ? videoH : videoW;
-        const vH = needsRotation ? videoW : videoH;
-        const perspectiveScale = Math.max(containerRect.width / vW, containerRect.height / vH);
-        const displayW = vW * perspectiveScale;
-        const displayH = vH * perspectiveScale;
-        const offsetX = (containerRect.width - displayW) / 2;
-        const offsetY = (containerRect.height - displayH) / 2;
-
-        // Adjust for centered marker and scaleMultiplier
-        const s = finalScale; // We still need the base scale factor for the pixel-to-marker mapping
-        // However, a cleaner way is to use the world matrix directly and map it.
-
-        // Actually, the simpler way to do 3D in CSS while keeping my projection logic is:
-        // Project the 4 corners and find the homography, OR
-        // Use the OpenGL matrix directly with a perspective mapping.
-
-        // Let's use the points projection to maintain the "needsRotation" logic compatibility
-        const pMid = projectToScreen(markerW / 2, markerH / 2, 0, mVT, proj, videoW, videoH, containerRect, needsRotation);
+        // 3. Project 4 corners to determine a full 3D perspective (homography)
         const pUL = projectToScreen(0, 0, 0, mVT, proj, videoW, videoH, containerRect, needsRotation);
         const pUR = projectToScreen(markerW, 0, 0, mVT, proj, videoW, videoH, containerRect, needsRotation);
         const pLL = projectToScreen(0, markerH, 0, mVT, proj, videoW, videoH, containerRect, needsRotation);
+        const pLR = projectToScreen(markerW, markerH, 0, mVT, proj, videoW, videoH, containerRect, needsRotation);
+
+        // Helper to solve for 2D Homography (maps 0..1 square to pUL, pUR, pLL, pLR)
+        const solveHomography = (w, h, p1, p2, p3, p4) => {
+            const x1 = p1.sx, y1 = p1.sy;
+            const x2 = p2.sx, y2 = p2.sy;
+            const x3 = p3.sx, y3 = p3.sy;
+            const x4 = p4.sx, y4 = p4.sy;
+
+            const dx1 = x2 - x4, dx2 = x3 - x4, dx3 = x1 - x2 + x4 - x3;
+            const dy1 = y2 - y4, dy2 = y3 - y4, dy3 = y1 - y2 + y4 - y3;
+
+            let a, b, c, d, e, f, g, h_coeff;
+
+            if (dx3 === 0 && dy3 === 0) {
+                a = x2 - x1; b = x3 - x1; c = x1;
+                d = y2 - y1; e = y3 - y1; f = y1;
+                g = 0; h_coeff = 0;
+            } else {
+                const det = dx1 * dy2 - dx2 * dy1;
+                g = (dx3 * dy2 - dx2 * dy3) / det;
+                h_coeff = (dx1 * dy3 - dx3 * dy1) / det;
+                a = x2 - x1 + g * x2;
+                b = x3 - x1 + h_coeff * x3;
+                c = x1;
+                d = y2 - y1 + g * y2;
+                e = y3 - y1 + h_coeff * y3;
+                f = y1;
+            }
+            // This maps unit square (0..1) to the quadrilateral.
+            // We need to scale it by 1/w and 1/h to map (0..w, 0..h)
+            return [
+                a / w, d / w, 0, g / w,
+                b / h, e / h, 0, h_coeff / h,
+                0, 0, 1, 0,
+                c, f, 0, 1
+            ];
+        };
 
-        // Using these points we can calculate the 3D rotation and perspective
-        const dx = pUR.sx - pUL.sx;
-        const dy = pUR.sy - pUL.sy;
-        const dz = pUR.sx - pLL.sx; // Not really Z but used for slant
-
-        const angle = Math.atan2(dy, dx);
-        const scaleX = Math.sqrt(dx * dx + dy * dy) / markerW;
-        const scaleY = Math.sqrt((pLL.sx - pUL.sx) ** 2 + (pLL.sy - pUL.sy) ** 2) / markerH;
-
-        // For true 3D tilt, we'll use the projection of the axes
-        const screenX = pMid.sx;
-        const screenY = pMid.sy;
-
-        // Final Transform applying 3D perspective via matrix3d derived from projected points
-        // NOTE: For full 3D we'd need a homography solver, but for "tilt" we can use the 
-        // original modelViewTransform if we convert it carefully.
-
-        const openGLWorldMatrix = this.controller.getWorldMatrix(mVT, targetIndex);
-        // We need to apply the same scaling and offsets as projectToScreen to the matrix
+        const matrix = solveHomography(markerW, markerH, pUL, pUR, pLL, pLR);
 
+        // Apply styles
         this.overlay.style.maxWidth = 'none';
         this.overlay.style.width = `${markerW}px`;
         this.overlay.style.height = `${markerH}px`;
         this.overlay.style.position = 'absolute';
-        this.overlay.style.transformOrigin = '0 0'; // Top-left based for simpler matrix mapping
+        this.overlay.style.transformOrigin = '0 0';
         this.overlay.style.left = '0';
         this.overlay.style.top = '0';
         this.overlay.style.display = 'block';
 
-        // Approximate 3D tilt using the projected corners to calculate a skew/scale/rotate combo
-        // This is more robust than a raw matrix3d if the projection isn't a perfect pinhole
+        // Apply 3D transform with matrix3d
+        // We also apply the user's custom scaleMultiplier AFTER the perspective transform
+        // but since we want to scale around the marker center, we apply it as a prefix/suffix
+        // Scale around top-left (0,0) is easy. Scale around center requires offset.
         this.overlay.style.transform = `
-            translate(${pUL.sx}px, ${pUL.sy}px)
-            matrix(${(pUR.sx - pUL.sx) / markerW}, ${(pUR.sy - pUL.sy) / markerW}, 
-                   ${(pLL.sx - pUL.sx) / markerH}, ${(pLL.sy - pUL.sy) / markerH}, 
-                   0, 0)
+            matrix3d(${matrix.join(',')})
+            translate(${markerW / 2}px, ${markerH / 2}px)
             scale(${this.scaleMultiplier})
+            translate(${-markerW / 2}px, ${-markerH / 2}px)
         `;
     }
 

package/src/compiler/tracker/tracker.js

@@ -24,16 +24,13 @@ class Tracker {
     this.inputHeight = inputHeight;
     this.debugMode = debugMode;
 
-    this.trackingKeyframeList = [];
-    for (let i = 0; i < trackingDataList.length; i++) {
-      this.trackingKeyframeList.push(trackingDataList[i][TRACKING_KEYFRAME]);
-    }
+    this.trackingKeyframeList = []; // All octaves for all targets: [targetIndex][octaveIndex]
+    this.prebuiltData = []; // [targetIndex][octaveIndex]
 
-    // Prebuild TypedArrays for features and pixels
-    this.prebuiltData = [];
-    for (let i = 0; i < this.trackingKeyframeList.length; i++) {
-      const keyframe = this.trackingKeyframeList[i];
-      this.prebuiltData[i] = {
+    for (let i = 0; i < trackingDataList.length; i++) {
+      const targetOctaves = trackingDataList[i];
+      this.trackingKeyframeList[i] = targetOctaves;
+      this.prebuiltData[i] = targetOctaves.map(keyframe => ({
         px: new Float32Array(keyframe.px),
         py: new Float32Array(keyframe.py),
         data: new Uint8Array(keyframe.d),
@@ -42,7 +39,7 @@ class Tracker {
         scale: keyframe.s,
         // Recyclable projected image buffer
         projectedImage: new Float32Array(keyframe.w * keyframe.h)
-      };
+      }));
     }
 
     // Pre-allocate template data buffer to avoid garbage collection
@@ -65,12 +62,31 @@ class Tracker {
   track(inputData, lastModelViewTransform, targetIndex) {
     let debugExtra = {};
 
+    // Select the best octave based on current estimated distance/scale
+    // We want the octave where the marker size is closest to its projected size on screen
     const modelViewProjectionTransform = buildModelViewProjectionTransform(
       this.projectionTransform,
       lastModelViewTransform,
     );
 
-    const prebuilt = this.prebuiltData[targetIndex];
+    // Estimate current marker width on screen
+    const [mW, mH] = this.markerDimensions[targetIndex];
+    const p0 = computeScreenCoordiate(modelViewProjectionTransform, 0, 0);
+    const p1 = computeScreenCoordiate(modelViewProjectionTransform, mW, 0);
+    const screenW = Math.sqrt((p1.x - p0.x) ** 2 + (p1.y - p0.y) ** 2);
+
+    // Select octave whose image width is closest to screenW
+    let octaveIndex = 0;
+    let minDiff = Infinity;
+    for (let i = 0; i < this.prebuiltData[targetIndex].length; i++) {
+      const diff = Math.abs(this.prebuiltData[targetIndex][i].width - screenW);
+      if (diff < minDiff) {
+        minDiff = diff;
+        octaveIndex = i;
+      }
+    }
+
+    const prebuilt = this.prebuiltData[targetIndex][octaveIndex];
 
     // 1. Compute Projection (Warping)
     this._computeProjection(
@@ -87,7 +103,7 @@ class Tracker {
       projectedImage
     );
 
-    const trackingFrame = this.trackingKeyframeList[targetIndex];
+    const trackingFrame = this.trackingKeyframeList[targetIndex][octaveIndex];
     const worldCoords = [];
     const screenCoords = [];
     const goodTrack = [];
@@ -113,6 +129,7 @@ class Tracker {
 
     if (this.debugMode) {
       debugExtra = {
+        octaveIndex,
         projectedImage: Array.from(projectedImage),
         matchingPoints,
         goodTrack,