@srsergio/taptapp-ar 1.0.35 → 1.0.37

package/dist/compiler/controller.d.ts

@@ -70,7 +70,11 @@ export class Controller {
         targetIndex: any;
         modelViewTransform: any;
     }>;
-    _trackAndUpdate(inputData: any, lastModelViewTransform: any, targetIndex: any): Promise<any>;
+    _trackAndUpdate(inputData: any, lastModelViewTransform: any, targetIndex: any): Promise<{
+        modelViewTransform: any;
+        inliers: number;
+        octaveIndex: any;
+    } | null>;
     processVideo(input: any): void;
     stopProcessVideo(): void;
     detect(input: any): Promise<{

package/dist/compiler/controller.js

@@ -181,14 +181,18 @@ class Controller {
         return { targetIndex: matchedTargetIndex, modelViewTransform };
     }
     async _trackAndUpdate(inputData, lastModelViewTransform, targetIndex) {
-        const { worldCoords, screenCoords } = this.tracker.track(inputData, lastModelViewTransform, targetIndex);
+        const { worldCoords, screenCoords, debugExtra } = this.tracker.track(inputData, lastModelViewTransform, targetIndex);
         if (worldCoords.length < 6)
             return null; // Umbral de puntos mínimos para mantener el seguimiento
         const modelViewTransform = await this._workerTrackUpdate(lastModelViewTransform, {
             worldCoords,
             screenCoords,
         });
-        return modelViewTransform;
+        return {
+            modelViewTransform,
+            inliers: worldCoords.length,
+            octaveIndex: debugExtra.octaveIndex
+        };
     }
     processVideo(input) {
         if (this.processingVideo)
@@ -202,6 +206,7 @@ class Controller {
                 currentModelViewTransform: null,
                 trackCount: 0,
                 trackMiss: 0,
+                stabilityCount: 0, // Nuevo: Contador para Live Adaptation
                 filter: new OneEuroFilter({ minCutOff: this.filterMinCF, beta: this.filterBeta }),
             });
         }
@@ -234,12 +239,28 @@ class Controller {
                 for (let i = 0; i < this.trackingStates.length; i++) {
                     const trackingState = this.trackingStates[i];
                     if (trackingState.isTracking) {
-                        let modelViewTransform = await this._trackAndUpdate(inputData, trackingState.currentModelViewTransform, i);
-                        if (modelViewTransform === null) {
+                        let result = await this._trackAndUpdate(inputData, trackingState.currentModelViewTransform, i);
+                        if (result === null) {
                             trackingState.isTracking = false;
+                            trackingState.stabilityCount = 0;
                         }
                         else {
-                            trackingState.currentModelViewTransform = modelViewTransform;
+                            trackingState.currentModelViewTransform = result.modelViewTransform;
+                            // --- LIVE MODEL ADAPTATION LOGIC ---
+                            // Si el tracking es muy sólido (muchos inliers) y estable, refinamos el modelo
+                            if (result.inliers > 25) {
+                                trackingState.stabilityCount++;
+                                if (trackingState.stabilityCount > 20) { // 20 frames de estabilidad absoluta
+                                    this.tracker.applyLiveFeedback(i, result.octaveIndex, 0.1); // 10% de mezcla real
+                                    if (this.debugMode)
+                                        console.log(`✨ Live Reification: Target ${i} (Octave ${result.octaveIndex}) updated with real-world textures.`);
+                                    trackingState.stabilityCount = 0; // Reset para la siguiente actualización
+                                }
+                            }
+                            else {
+                                trackingState.stabilityCount = Math.max(0, trackingState.stabilityCount - 1);
+                            }
+                            // -----------------------------------
                         }
                     }
                     // if not showing, then show it once it reaches warmup number of frames

package/dist/compiler/offline-compiler.js

@@ -217,9 +217,9 @@ export class OfflineCompiler {
         }
         const dataList = this.data.map((item) => {
             const matchingData = item.matchingData.map((kf) => this._packKeyframe(kf));
-            const trackingData = [item.trackingData[0]].map((td) => {
+            const trackingData = item.trackingData.map((td) => {
                 const count = td.points.length;
-                // Step 1: Packed Coords - Normalize width/height to 16-bit
+                // Packed Coords - Float32 for now as in current import logic
                 const px = new Float32Array(count);
                 const py = new Float32Array(count);
                 for (let i = 0; i < count; i++) {

package/dist/compiler/simple-ar.js

@@ -192,60 +192,69 @@ class SimpleAR {
         const needsRotation = isPortrait && isVideoLandscape;
         // 3. Get intrinsic projection from controller
         const proj = this.controller.projectionTransform;
-        // 4. Position calculation via matrix3d (Support for 3D tilt/Z-rotation)
-        // We convert the OpenGL World Matrix to a CSS matrix3d.
-        // The OpenGL matrix is column-major. CSS matrix3d is also column-major.
-        const m = this.controller.getWorldMatrix(mVT, targetIndex);
-        // Map OpenGL coords to Screen Pixels using the projection logic
-        const vW = needsRotation ? videoH : videoW;
-        const vH = needsRotation ? videoW : videoH;
-        const perspectiveScale = Math.max(containerRect.width / vW, containerRect.height / vH);
-        const displayW = vW * perspectiveScale;
-        const displayH = vH * perspectiveScale;
-        const offsetX = (containerRect.width - displayW) / 2;
-        const offsetY = (containerRect.height - displayH) / 2;
-        // Adjust for centered marker and scaleMultiplier
-        const s = finalScale; // We still need the base scale factor for the pixel-to-marker mapping
-        // However, a cleaner way is to use the world matrix directly and map it.
-        // Actually, the simpler way to do 3D in CSS while keeping my projection logic is:
-        // Project the 4 corners and find the homography, OR
-        // Use the OpenGL matrix directly with a perspective mapping.
-        // Let's use the points projection to maintain the "needsRotation" logic compatibility
-        const pMid = projectToScreen(markerW / 2, markerH / 2, 0, mVT, proj, videoW, videoH, containerRect, needsRotation);
+        // 3. Project 4 corners to determine a full 3D perspective (homography)
         const pUL = projectToScreen(0, 0, 0, mVT, proj, videoW, videoH, containerRect, needsRotation);
         const pUR = projectToScreen(markerW, 0, 0, mVT, proj, videoW, videoH, containerRect, needsRotation);
         const pLL = projectToScreen(0, markerH, 0, mVT, proj, videoW, videoH, containerRect, needsRotation);
-        // Using these points we can calculate the 3D rotation and perspective
-        const dx = pUR.sx - pUL.sx;
-        const dy = pUR.sy - pUL.sy;
-        const dz = pUR.sx - pLL.sx; // Not really Z but used for slant
-        const angle = Math.atan2(dy, dx);
-        const scaleX = Math.sqrt(dx * dx + dy * dy) / markerW;
-        const scaleY = Math.sqrt((pLL.sx - pUL.sx) ** 2 + (pLL.sy - pUL.sy) ** 2) / markerH;
-        // For true 3D tilt, we'll use the projection of the axes
-        const screenX = pMid.sx;
-        const screenY = pMid.sy;
-        // Final Transform applying 3D perspective via matrix3d derived from projected points
-        // NOTE: For full 3D we'd need a homography solver, but for "tilt" we can use the 
-        // original modelViewTransform if we convert it carefully.
-        const openGLWorldMatrix = this.controller.getWorldMatrix(mVT, targetIndex);
-        // We need to apply the same scaling and offsets as projectToScreen to the matrix
+        const pLR = projectToScreen(markerW, markerH, 0, mVT, proj, videoW, videoH, containerRect, needsRotation);
+        // Helper to solve for 2D Homography (maps 0..1 square to pUL, pUR, pLL, pLR)
+        const solveHomography = (w, h, p1, p2, p3, p4) => {
+            const x1 = p1.sx, y1 = p1.sy;
+            const x2 = p2.sx, y2 = p2.sy;
+            const x3 = p3.sx, y3 = p3.sy;
+            const x4 = p4.sx, y4 = p4.sy;
+            const dx1 = x2 - x4, dx2 = x3 - x4, dx3 = x1 - x2 + x4 - x3;
+            const dy1 = y2 - y4, dy2 = y3 - y4, dy3 = y1 - y2 + y4 - y3;
+            let a, b, c, d, e, f, g, h_coeff;
+            if (dx3 === 0 && dy3 === 0) {
+                a = x2 - x1;
+                b = x3 - x1;
+                c = x1;
+                d = y2 - y1;
+                e = y3 - y1;
+                f = y1;
+                g = 0;
+                h_coeff = 0;
+            }
+            else {
+                const det = dx1 * dy2 - dx2 * dy1;
+                g = (dx3 * dy2 - dx2 * dy3) / det;
+                h_coeff = (dx1 * dy3 - dx3 * dy1) / det;
+                a = x2 - x1 + g * x2;
+                b = x3 - x1 + h_coeff * x3;
+                c = x1;
+                d = y2 - y1 + g * y2;
+                e = y3 - y1 + h_coeff * y3;
+                f = y1;
+            }
+            // This maps unit square (0..1) to the quadrilateral.
+            // We need to scale it by 1/w and 1/h to map (0..w, 0..h)
+            return [
+                a / w, d / w, 0, g / w,
+                b / h, e / h, 0, h_coeff / h,
+                0, 0, 1, 0,
+                c, f, 0, 1
+            ];
+        };
+        const matrix = solveHomography(markerW, markerH, pUL, pUR, pLL, pLR);
+        // Apply styles
         this.overlay.style.maxWidth = 'none';
         this.overlay.style.width = `${markerW}px`;
         this.overlay.style.height = `${markerH}px`;
         this.overlay.style.position = 'absolute';
-        this.overlay.style.transformOrigin = '0 0'; // Top-left based for simpler matrix mapping
+        this.overlay.style.transformOrigin = '0 0';
         this.overlay.style.left = '0';
         this.overlay.style.top = '0';
         this.overlay.style.display = 'block';
-        // Approximate 3D tilt using the projected corners to calculate a skew/scale/rotate combo
-        // This is more robust than a raw matrix3d if the projection isn't a perfect pinhole
+        // Apply 3D transform with matrix3d
+        // We also apply the user's custom scaleMultiplier AFTER the perspective transform
+        // but since we want to scale around the marker center, we apply it as a prefix/suffix
+        // Scale around top-left (0,0) is easy. Scale around center requires offset.
         this.overlay.style.transform = `
-            translate(${pUL.sx}px, ${pUL.sy}px)
-            matrix(${(pUR.sx - pUL.sx) / markerW}, ${(pUR.sy - pUL.sy) / markerW}, 
-                   ${(pLL.sx - pUL.sx) / markerH}, ${(pLL.sy - pUL.sy) / markerH}, 
-                   0, 0)
+            matrix3d(${matrix.join(',')})
+            translate(${markerW / 2}px, ${markerH / 2}px)
             scale(${this.scaleMultiplier})
+            translate(${-markerW / 2}px, ${-markerH / 2}px)
         `;
     }
     // Unified projection logic moved to ./utils/projection.js

package/dist/compiler/tracker/tracker.d.ts

@@ -7,15 +7,7 @@ export class Tracker {
     inputHeight: any;
     debugMode: boolean;
     trackingKeyframeList: any[];
-    prebuiltData: {
-        px: Float32Array<any>;
-        py: Float32Array<any>;
-        data: Uint8Array<any>;
-        width: any;
-        height: any;
-        scale: any;
-        projectedImage: Float32Array<ArrayBuffer>;
-    }[];
+    prebuiltData: any[];
     templateBuffer: Float32Array<ArrayBuffer>;
     dummyRun(inputData: any): void;
     track(inputData: any, lastModelViewTransform: any, targetIndex: any): {
@@ -41,4 +33,11 @@ export class Tracker {
      * Pure JS implementation of Bilinear Warping
      */
     _computeProjection(M: any, inputData: any, prebuilt: any): void;
+    /**
+     * Refines the target data (Living Mind Map) using actual camera feedback
+     * @param {number} targetIndex
+     * @param {number} octaveIndex
+     * @param {number} alpha - Blending factor (e.g. 0.1 for 10% new data)
+     */
+    applyLiveFeedback(targetIndex: number, octaveIndex: number, alpha: number): void;
 }

package/dist/compiler/tracker/tracker.js

@@ -13,15 +13,12 @@ class Tracker {
         this.inputWidth = inputWidth;
         this.inputHeight = inputHeight;
         this.debugMode = debugMode;
-        this.trackingKeyframeList = [];
+        this.trackingKeyframeList = []; // All octaves for all targets: [targetIndex][octaveIndex]
+        this.prebuiltData = []; // [targetIndex][octaveIndex]
         for (let i = 0; i < trackingDataList.length; i++) {
-            this.trackingKeyframeList.push(trackingDataList[i][TRACKING_KEYFRAME]);
-        }
-        // Prebuild TypedArrays for features and pixels
-        this.prebuiltData = [];
-        for (let i = 0; i < this.trackingKeyframeList.length; i++) {
-            const keyframe = this.trackingKeyframeList[i];
-            this.prebuiltData[i] = {
+            const targetOctaves = trackingDataList[i];
+            this.trackingKeyframeList[i] = targetOctaves;
+            this.prebuiltData[i] = targetOctaves.map(keyframe => ({
                 px: new Float32Array(keyframe.px),
                 py: new Float32Array(keyframe.py),
                 data: new Uint8Array(keyframe.d),
@@ -30,7 +27,7 @@ class Tracker {
                 scale: keyframe.s,
                 // Recyclable projected image buffer
                 projectedImage: new Float32Array(keyframe.w * keyframe.h)
-            };
+            }));
         }
         // Pre-allocate template data buffer to avoid garbage collection
         const templateOneSize = AR2_DEFAULT_TS;
@@ -49,14 +46,31 @@ class Tracker {
     }
     track(inputData, lastModelViewTransform, targetIndex) {
         let debugExtra = {};
+        // Select the best octave based on current estimated distance/scale
+        // We want the octave where the marker size is closest to its projected size on screen
         const modelViewProjectionTransform = buildModelViewProjectionTransform(this.projectionTransform, lastModelViewTransform);
-        const prebuilt = this.prebuiltData[targetIndex];
+        // Estimate current marker width on screen
+        const [mW, mH] = this.markerDimensions[targetIndex];
+        const p0 = computeScreenCoordiate(modelViewProjectionTransform, 0, 0);
+        const p1 = computeScreenCoordiate(modelViewProjectionTransform, mW, 0);
+        const screenW = Math.sqrt((p1.x - p0.x) ** 2 + (p1.y - p0.y) ** 2);
+        // Select octave whose image width is closest to screenW
+        let octaveIndex = 0;
+        let minDiff = Infinity;
+        for (let i = 0; i < this.prebuiltData[targetIndex].length; i++) {
+            const diff = Math.abs(this.prebuiltData[targetIndex][i].width - screenW);
+            if (diff < minDiff) {
+                minDiff = diff;
+                octaveIndex = i;
+            }
+        }
+        const prebuilt = this.prebuiltData[targetIndex][octaveIndex];
         // 1. Compute Projection (Warping)
         this._computeProjection(modelViewProjectionTransform, inputData, prebuilt);
         const projectedImage = prebuilt.projectedImage;
         // 2. Compute Matching (NCC)
         const { matchingPoints, sim } = this._computeMatching(prebuilt, projectedImage);
-        const trackingFrame = this.trackingKeyframeList[targetIndex];
+        const trackingFrame = this.trackingKeyframeList[targetIndex][octaveIndex];
         const worldCoords = [];
         const screenCoords = [];
         const goodTrack = [];
@@ -75,6 +89,7 @@ class Tracker {
         }
         if (this.debugMode) {
             debugExtra = {
+                octaveIndex,
                 projectedImage: Array.from(projectedImage),
                 matchingPoints,
                 goodTrack,
@@ -217,5 +232,25 @@ class Tracker {
             }
         }
     }
+    /**
+     * Refines the target data (Living Mind Map) using actual camera feedback
+     * @param {number} targetIndex
+     * @param {number} octaveIndex
+     * @param {number} alpha - Blending factor (e.g. 0.1 for 10% new data)
+     */
+    applyLiveFeedback(targetIndex, octaveIndex, alpha) {
+        const prebuilt = this.prebuiltData[targetIndex][octaveIndex];
+        if (!prebuilt || !prebuilt.projectedImage)
+            return;
+        const markerPixels = prebuilt.data;
+        const projectedPixels = prebuilt.projectedImage;
+        const count = markerPixels.length;
+        // Blend the projected (camera-sourced) pixels into the marker reference data
+        // This allows the NCC matching to adapt to real-world lighting and print quality
+        for (let i = 0; i < count; i++) {
+            // Simple linear blend
+            markerPixels[i] = (1 - alpha) * markerPixels[i] + alpha * projectedPixels[i];
+        }
+    }
 }
 export { Tracker };

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@srsergio/taptapp-ar",
-    "version": "1.0.35",
+    "version": "1.0.37",
     "description": "AR Compiler for Node.js and Browser",
     "repository": {
         "type": "git",

package/src/compiler/controller.js

@@ -226,7 +226,7 @@ class Controller {
     return { targetIndex: matchedTargetIndex, modelViewTransform };
   }
   async _trackAndUpdate(inputData, lastModelViewTransform, targetIndex) {
-    const { worldCoords, screenCoords } = this.tracker.track(
+    const { worldCoords, screenCoords, debugExtra } = this.tracker.track(
       inputData,
       lastModelViewTransform,
       targetIndex,
@@ -236,7 +236,11 @@ class Controller {
       worldCoords,
       screenCoords,
     });
-    return modelViewTransform;
+    return {
+      modelViewTransform,
+      inliers: worldCoords.length,
+      octaveIndex: debugExtra.octaveIndex
+    };
   }
 
   processVideo(input) {
@@ -252,6 +256,7 @@ class Controller {
         currentModelViewTransform: null,
         trackCount: 0,
         trackMiss: 0,
+        stabilityCount: 0, // Nuevo: Contador para Live Adaptation
         filter: new OneEuroFilter({ minCutOff: this.filterMinCF, beta: this.filterBeta }),
       });
     }
@@ -291,15 +296,30 @@ class Controller {
           const trackingState = this.trackingStates[i];
 
           if (trackingState.isTracking) {
-            let modelViewTransform = await this._trackAndUpdate(
+            let result = await this._trackAndUpdate(
               inputData,
               trackingState.currentModelViewTransform,
               i,
             );
-            if (modelViewTransform === null) {
+            if (result === null) {
               trackingState.isTracking = false;
+              trackingState.stabilityCount = 0;
             } else {
-              trackingState.currentModelViewTransform = modelViewTransform;
+              trackingState.currentModelViewTransform = result.modelViewTransform;
+
+              // --- LIVE MODEL ADAPTATION LOGIC ---
+              // Si el tracking es muy sólido (muchos inliers) y estable, refinamos el modelo
+              if (result.inliers > 25) {
+                trackingState.stabilityCount++;
+                if (trackingState.stabilityCount > 20) { // 20 frames de estabilidad absoluta
+                  this.tracker.applyLiveFeedback(i, result.octaveIndex, 0.1); // 10% de mezcla real
+                  if (this.debugMode) console.log(`✨ Live Reification: Target ${i} (Octave ${result.octaveIndex}) updated with real-world textures.`);
+                  trackingState.stabilityCount = 0; // Reset para la siguiente actualización
+                }
+              } else {
+                trackingState.stabilityCount = Math.max(0, trackingState.stabilityCount - 1);
+              }
+              // -----------------------------------
             }
           }
 

package/src/compiler/offline-compiler.js

@@ -260,9 +260,9 @@ export class OfflineCompiler {
     const dataList = this.data.map((item) => {
       const matchingData = item.matchingData.map((kf) => this._packKeyframe(kf));
 
-      const trackingData = [item.trackingData[0]].map((td) => {
+      const trackingData = item.trackingData.map((td) => {
         const count = td.points.length;
-        // Step 1: Packed Coords - Normalize width/height to 16-bit
+        // Packed Coords - Float32 for now as in current import logic
         const px = new Float32Array(count);
         const py = new Float32Array(count);
         for (let i = 0; i < count; i++) {

package/src/compiler/simple-ar.js

@@ -222,71 +222,70 @@ class SimpleAR {
         // 3. Get intrinsic projection from controller
         const proj = this.controller.projectionTransform;
 
-        // 4. Position calculation via matrix3d (Support for 3D tilt/Z-rotation)
-        // We convert the OpenGL World Matrix to a CSS matrix3d.
-        // The OpenGL matrix is column-major. CSS matrix3d is also column-major.
-        const m = this.controller.getWorldMatrix(mVT, targetIndex);
-
-        // Map OpenGL coords to Screen Pixels using the projection logic
-        const vW = needsRotation ? videoH : videoW;
-        const vH = needsRotation ? videoW : videoH;
-        const perspectiveScale = Math.max(containerRect.width / vW, containerRect.height / vH);
-        const displayW = vW * perspectiveScale;
-        const displayH = vH * perspectiveScale;
-        const offsetX = (containerRect.width - displayW) / 2;
-        const offsetY = (containerRect.height - displayH) / 2;
-
-        // Adjust for centered marker and scaleMultiplier
-        const s = finalScale; // We still need the base scale factor for the pixel-to-marker mapping
-        // However, a cleaner way is to use the world matrix directly and map it.
-
-        // Actually, the simpler way to do 3D in CSS while keeping my projection logic is:
-        // Project the 4 corners and find the homography, OR
-        // Use the OpenGL matrix directly with a perspective mapping.
-
-        // Let's use the points projection to maintain the "needsRotation" logic compatibility
-        const pMid = projectToScreen(markerW / 2, markerH / 2, 0, mVT, proj, videoW, videoH, containerRect, needsRotation);
+        // 3. Project 4 corners to determine a full 3D perspective (homography)
         const pUL = projectToScreen(0, 0, 0, mVT, proj, videoW, videoH, containerRect, needsRotation);
         const pUR = projectToScreen(markerW, 0, 0, mVT, proj, videoW, videoH, containerRect, needsRotation);
         const pLL = projectToScreen(0, markerH, 0, mVT, proj, videoW, videoH, containerRect, needsRotation);
+        const pLR = projectToScreen(markerW, markerH, 0, mVT, proj, videoW, videoH, containerRect, needsRotation);
+
+        // Helper to solve for 2D Homography (maps 0..1 square to pUL, pUR, pLL, pLR)
+        const solveHomography = (w, h, p1, p2, p3, p4) => {
+            const x1 = p1.sx, y1 = p1.sy;
+            const x2 = p2.sx, y2 = p2.sy;
+            const x3 = p3.sx, y3 = p3.sy;
+            const x4 = p4.sx, y4 = p4.sy;
+
+            const dx1 = x2 - x4, dx2 = x3 - x4, dx3 = x1 - x2 + x4 - x3;
+            const dy1 = y2 - y4, dy2 = y3 - y4, dy3 = y1 - y2 + y4 - y3;
+
+            let a, b, c, d, e, f, g, h_coeff;
+
+            if (dx3 === 0 && dy3 === 0) {
+                a = x2 - x1; b = x3 - x1; c = x1;
+                d = y2 - y1; e = y3 - y1; f = y1;
+                g = 0; h_coeff = 0;
+            } else {
+                const det = dx1 * dy2 - dx2 * dy1;
+                g = (dx3 * dy2 - dx2 * dy3) / det;
+                h_coeff = (dx1 * dy3 - dx3 * dy1) / det;
+                a = x2 - x1 + g * x2;
+                b = x3 - x1 + h_coeff * x3;
+                c = x1;
+                d = y2 - y1 + g * y2;
+                e = y3 - y1 + h_coeff * y3;
+                f = y1;
+            }
+            // This maps unit square (0..1) to the quadrilateral.
+            // We need to scale it by 1/w and 1/h to map (0..w, 0..h)
+            return [
+                a / w, d / w, 0, g / w,
+                b / h, e / h, 0, h_coeff / h,
+                0, 0, 1, 0,
+                c, f, 0, 1
+            ];
+        };
 
-        // Using these points we can calculate the 3D rotation and perspective
-        const dx = pUR.sx - pUL.sx;
-        const dy = pUR.sy - pUL.sy;
-        const dz = pUR.sx - pLL.sx; // Not really Z but used for slant
-
-        const angle = Math.atan2(dy, dx);
-        const scaleX = Math.sqrt(dx * dx + dy * dy) / markerW;
-        const scaleY = Math.sqrt((pLL.sx - pUL.sx) ** 2 + (pLL.sy - pUL.sy) ** 2) / markerH;
-
-        // For true 3D tilt, we'll use the projection of the axes
-        const screenX = pMid.sx;
-        const screenY = pMid.sy;
-
-        // Final Transform applying 3D perspective via matrix3d derived from projected points
-        // NOTE: For full 3D we'd need a homography solver, but for "tilt" we can use the 
-        // original modelViewTransform if we convert it carefully.
-
-        const openGLWorldMatrix = this.controller.getWorldMatrix(mVT, targetIndex);
-        // We need to apply the same scaling and offsets as projectToScreen to the matrix
+        const matrix = solveHomography(markerW, markerH, pUL, pUR, pLL, pLR);
 
+        // Apply styles
         this.overlay.style.maxWidth = 'none';
         this.overlay.style.width = `${markerW}px`;
         this.overlay.style.height = `${markerH}px`;
         this.overlay.style.position = 'absolute';
-        this.overlay.style.transformOrigin = '0 0'; // Top-left based for simpler matrix mapping
+        this.overlay.style.transformOrigin = '0 0';
         this.overlay.style.left = '0';
         this.overlay.style.top = '0';
         this.overlay.style.display = 'block';
 
-        // Approximate 3D tilt using the projected corners to calculate a skew/scale/rotate combo
-        // This is more robust than a raw matrix3d if the projection isn't a perfect pinhole
+        // Apply 3D transform with matrix3d
+        // We also apply the user's custom scaleMultiplier AFTER the perspective transform
+        // but since we want to scale around the marker center, we apply it as a prefix/suffix
+        // Scale around top-left (0,0) is easy. Scale around center requires offset.
         this.overlay.style.transform = `
-            translate(${pUL.sx}px, ${pUL.sy}px)
-            matrix(${(pUR.sx - pUL.sx) / markerW}, ${(pUR.sy - pUL.sy) / markerW}, 
-                   ${(pLL.sx - pUL.sx) / markerH}, ${(pLL.sy - pUL.sy) / markerH}, 
-                   0, 0)
+            matrix3d(${matrix.join(',')})
+            translate(${markerW / 2}px, ${markerH / 2}px)
             scale(${this.scaleMultiplier})
+            translate(${-markerW / 2}px, ${-markerH / 2}px)
         `;
     }
 

package/src/compiler/tracker/tracker.js

@@ -24,16 +24,13 @@ class Tracker {
     this.inputHeight = inputHeight;
     this.debugMode = debugMode;
 
-    this.trackingKeyframeList = [];
-    for (let i = 0; i < trackingDataList.length; i++) {
-      this.trackingKeyframeList.push(trackingDataList[i][TRACKING_KEYFRAME]);
-    }
+    this.trackingKeyframeList = []; // All octaves for all targets: [targetIndex][octaveIndex]
+    this.prebuiltData = []; // [targetIndex][octaveIndex]
 
-    // Prebuild TypedArrays for features and pixels
-    this.prebuiltData = [];
-    for (let i = 0; i < this.trackingKeyframeList.length; i++) {
-      const keyframe = this.trackingKeyframeList[i];
-      this.prebuiltData[i] = {
+    for (let i = 0; i < trackingDataList.length; i++) {
+      const targetOctaves = trackingDataList[i];
+      this.trackingKeyframeList[i] = targetOctaves;
+      this.prebuiltData[i] = targetOctaves.map(keyframe => ({
         px: new Float32Array(keyframe.px),
         py: new Float32Array(keyframe.py),
         data: new Uint8Array(keyframe.d),
@@ -42,7 +39,7 @@ class Tracker {
         scale: keyframe.s,
         // Recyclable projected image buffer
         projectedImage: new Float32Array(keyframe.w * keyframe.h)
-      };
+      }));
     }
 
     // Pre-allocate template data buffer to avoid garbage collection
@@ -65,12 +62,31 @@ class Tracker {
   track(inputData, lastModelViewTransform, targetIndex) {
     let debugExtra = {};
 
+    // Select the best octave based on current estimated distance/scale
+    // We want the octave where the marker size is closest to its projected size on screen
     const modelViewProjectionTransform = buildModelViewProjectionTransform(
       this.projectionTransform,
       lastModelViewTransform,
     );
 
-    const prebuilt = this.prebuiltData[targetIndex];
+    // Estimate current marker width on screen
+    const [mW, mH] = this.markerDimensions[targetIndex];
+    const p0 = computeScreenCoordiate(modelViewProjectionTransform, 0, 0);
+    const p1 = computeScreenCoordiate(modelViewProjectionTransform, mW, 0);
+    const screenW = Math.sqrt((p1.x - p0.x) ** 2 + (p1.y - p0.y) ** 2);
+
+    // Select octave whose image width is closest to screenW
+    let octaveIndex = 0;
+    let minDiff = Infinity;
+    for (let i = 0; i < this.prebuiltData[targetIndex].length; i++) {
+      const diff = Math.abs(this.prebuiltData[targetIndex][i].width - screenW);
+      if (diff < minDiff) {
+        minDiff = diff;
+        octaveIndex = i;
+      }
+    }
+
+    const prebuilt = this.prebuiltData[targetIndex][octaveIndex];
 
     // 1. Compute Projection (Warping)
     this._computeProjection(
@@ -87,7 +103,7 @@ class Tracker {
       projectedImage
     );
 
-    const trackingFrame = this.trackingKeyframeList[targetIndex];
+    const trackingFrame = this.trackingKeyframeList[targetIndex][octaveIndex];
     const worldCoords = [];
     const screenCoords = [];
     const goodTrack = [];
@@ -113,6 +129,7 @@ class Tracker {
 
     if (this.debugMode) {
       debugExtra = {
+        octaveIndex,
         projectedImage: Array.from(projectedImage),
         matchingPoints,
         goodTrack,
@@ -283,6 +300,28 @@ class Tracker {
       }
     }
   }
+
+  /**
+   * Refines the target data (Living Mind Map) using actual camera feedback
+   * @param {number} targetIndex 
+   * @param {number} octaveIndex 
+   * @param {number} alpha - Blending factor (e.g. 0.1 for 10% new data)
+   */
+  applyLiveFeedback(targetIndex, octaveIndex, alpha) {
+    const prebuilt = this.prebuiltData[targetIndex][octaveIndex];
+    if (!prebuilt || !prebuilt.projectedImage) return;
+
+    const markerPixels = prebuilt.data;
+    const projectedPixels = prebuilt.projectedImage;
+    const count = markerPixels.length;
+
+    // Blend the projected (camera-sourced) pixels into the marker reference data
+    // This allows the NCC matching to adapt to real-world lighting and print quality
+    for (let i = 0; i < count; i++) {
+      // Simple linear blend
+      markerPixels[i] = (1 - alpha) * markerPixels[i] + alpha * projectedPixels[i];
+    }
+  }
 }
 
 export { Tracker };