@srsergio/taptapp-ar 1.0.92 → 1.0.94

package/src/core/protocol.ts

@@ -1,6 +1,6 @@
 import * as msgpack from "@msgpack/msgpack";
 
-export const CURRENT_VERSION = 9; // Bumped for HDC support
+export const CURRENT_VERSION = 11; // Bumped for Nanite virtualized features support
 export const HDC_SEED = 0x1337BEEF; // Default system seed
 
 /**
@@ -104,6 +104,43 @@ export function columnarize(points: any[], tree: any, width: number, height: num
     };
 }
 
+/**
+ * Columnarizes point data with COMPACT 32-bit descriptors (XOR folding)
+ * Reduces descriptor storage by 50% with minimal accuracy loss
+ */
+export function columnarizeCompact(points: any[], tree: any, width: number, height: number) {
+    const count = points.length;
+    const x = new Uint16Array(count);
+    const y = new Uint16Array(count);
+    const angle = new Int16Array(count);
+    const scale = new Uint8Array(count);
+    const descriptors = new Uint32Array(count); // 32-bit compact descriptors
+
+    for (let i = 0; i < count; i++) {
+        x[i] = Math.round((points[i].x / width) * 65535);
+        y[i] = Math.round((points[i].y / height) * 65535);
+        angle[i] = Math.round((points[i].angle / Math.PI) * 32767);
+        scale[i] = Math.round(Math.log2(points[i].scale || 1));
+
+        if (points[i].descriptors && points[i].descriptors.length >= 2) {
+            // XOR folding: Combine two 32-bit values into one 32-bit value
+            // This preserves discriminative power while halving storage
+            descriptors[i] = (points[i].descriptors[0] ^ points[i].descriptors[1]) >>> 0;
+        }
+    }
+
+    return {
+        x,
+        y,
+        a: angle,
+        s: scale,
+        d: descriptors,
+        compact: 1, // Flag to indicate compact 32-bit descriptors
+        t: compactTree(tree.rootNode),
+    };
+}
+
+
 /**
  * Compacts hierarchical clustering tree into a minimal array structure
  */

package/src/runtime/bio-inspired-controller.ts

@@ -0,0 +1,448 @@
+/**
+ * Bio-Inspired Controller Adapter
+ * 
+ * Wraps the standard Controller with Bio-Inspired Perception capabilities.
+ * Provides significant performance improvements while maintaining API compatibility.
+ * 
+ * Key features:
+ * - Foveal attention: Processes only regions of interest at full resolution
+ * - Predictive coding: Skips processing when scene is static
+ * - Saccadic sampling: Strategic "glances" at high-saliency regions
+ * 
+ * Usage:
+ * ```javascript
+ * import { BioInspiredController } from './bio-inspired-controller.js';
+ * 
+ * const controller = new BioInspiredController({
+ *   inputWidth: 640,
+ *   inputHeight: 480,
+ *   onUpdate: (data) => console.log(data),
+ *   bioInspired: {
+ *     enabled: true,
+ *     aggressiveSkipping: true,
+ *   }
+ * });
+ * ```
+ */
+
+import { Controller, ControllerOptions } from './controller.js';
+import { BioInspiredEngine, BIO_CONFIG } from '../core/perception/index.js';
+import { AutoRotationFeature } from '../core/features/auto-rotation-feature.js';
+
+/**
+ * Result from BioInspiredEngine.process()
+ */
+interface BioResult {
+    skipped: boolean;
+    prediction?: {
+        worldMatrix?: Float32Array;
+    };
+    attentionRegions?: any[];
+    foveaCenter?: { x: number; y: number };
+    pixelsSaved?: number;
+    octavesToProcess?: number[]; // Added for scale orchestration
+}
+
+/**
+ * Extended options for Bio-Inspired Controller
+ */
+export interface BioInspiredControllerOptions extends ControllerOptions {
+    bioInspired?: {
+        enabled?: boolean;
+        aggressiveSkipping?: boolean;
+        foveaRadiusRatio?: number;
+        maxSaccades?: number;
+    };
+}
+
+/**
+ * Bio-Inspired Controller
+ * 
+ * Extends the standard Controller with bio-inspired perception capabilities.
+ */
+class BioInspiredController extends Controller {
+    private bioEngine: BioInspiredEngine | null = null;
+    private bioEnabled: boolean = true;
+    private bioMetricsInterval: number | null = null;
+    private lastBioResult: any = null;
+
+    constructor(options: BioInspiredControllerOptions) {
+        super(options);
+
+        const bioOptions = options.bioInspired || {};
+        this.bioEnabled = bioOptions.enabled !== false;
+
+        if (this.bioEnabled) {
+            // Initialize Bio-Inspired Engine
+            const bioConfig: any = {};
+
+            if (bioOptions.foveaRadiusRatio !== undefined) {
+                bioConfig.FOVEA_RADIUS_RATIO = bioOptions.foveaRadiusRatio;
+            }
+            if (bioOptions.maxSaccades !== undefined) {
+                bioConfig.MAX_SACCADES_PER_FRAME = bioOptions.maxSaccades;
+            }
+            if (bioOptions.aggressiveSkipping !== undefined) {
+                bioConfig.ENABLE_SKIP_FRAMES = bioOptions.aggressiveSkipping;
+                if (bioOptions.aggressiveSkipping) {
+                    bioConfig.CHANGE_THRESHOLD = 0.03; // More aggressive
+                }
+            }
+
+            this.bioEngine = new BioInspiredEngine(
+                options.inputWidth,
+                options.inputHeight,
+                bioConfig
+            );
+        }
+    }
+
+    /**
+     * Override processVideo to add bio-inspired perception
+     */
+    processVideo(input: any) {
+        if (!this.bioEnabled || !this.bioEngine) {
+            return super.processVideo(input);
+        }
+
+        if (this.processingVideo) return;
+        this.processingVideo = true;
+
+        // Reset tracking states
+        this.trackingStates = [];
+        for (let i = 0; i < (this.markerDimensions?.length || 0); i++) {
+            this.trackingStates.push({
+                showing: false,
+                isTracking: false,
+                currentModelViewTransform: null,
+                trackCount: 0,
+                trackMiss: 0,
+            });
+        }
+
+        const startProcessing = async () => {
+            while (this.processingVideo) {
+                const inputData = this.inputLoader.loadInput(input);
+
+                // Get current tracking state for bio engine
+                const activeTracking = this.trackingStates.find(s => s.isTracking);
+                const trackingState = activeTracking ? {
+                    isTracking: true,
+                    activeOctave: activeTracking.lastOctaveIndex, // Tracked octave index
+                    worldMatrix: activeTracking.currentModelViewTransform
+                        ? this._flattenMatrix(activeTracking.currentModelViewTransform)
+                        : null
+                } : null;
+
+                // Process through bio-inspired engine
+                const bioResult = this.bioEngine!.process(inputData, (trackingState as any) || undefined) as BioResult;
+                this.lastBioResult = bioResult;
+
+                // If bio engine says we can skip, use prediction
+                if (bioResult.skipped && activeTracking?.isTracking) {
+                    // Use predicted state
+                    this._handleSkippedFrame(activeTracking, bioResult);
+                } else {
+                    // Normal processing with attention regions
+                    await this._processWithAttention(input, inputData, bioResult);
+                }
+
+                // Wait for next frame
+                if (typeof requestAnimationFrame !== 'undefined') {
+                    await new Promise(requestAnimationFrame);
+                } else {
+                    await new Promise(resolve => setTimeout(resolve, 16));
+                }
+            }
+        };
+
+        startProcessing();
+    }
+
+    /**
+     * Handle a skipped frame using prediction
+     * @private
+     */
+    private _handleSkippedFrame(trackingState: any, bioResult: any) {
+        // Use predicted matrix
+        if (bioResult.prediction && bioResult.prediction.worldMatrix) {
+            trackingState.currentModelViewTransform = this._unflattenMatrix(bioResult.prediction.worldMatrix);
+        }
+
+        // Notify with skipped status
+        const worldMatrix = trackingState.currentModelViewTransform
+            ? this._glModelViewMatrix(trackingState.currentModelViewTransform, 0)
+            : null;
+
+        this.onUpdate?.({
+            type: 'updateMatrix',
+            targetIndex: 0,
+            worldMatrix: worldMatrix ? this.featureManager.applyWorldMatrixFilters(0, worldMatrix, { stability: 0.9 }) : null,
+            skipped: true,
+            bioMetrics: this.bioEngine?.getMetrics(),
+        });
+
+        this.onUpdate?.({ type: 'processDone' });
+    }
+
+    /**
+     * Process frame using bio-inspired attention regions
+     * @private
+     */
+    private async _processWithAttention(input: any, inputData: Uint8Array, bioResult: BioResult) {
+        const nTracking = this.trackingStates.reduce((acc, s) => acc + (s.isTracking ? 1 : 0), 0);
+
+        // Detection phase - use primary attention region for efficiency
+        if (nTracking < this.maxTrack) {
+            const matchingIndexes = this.trackingStates
+                .map((s, i) => ({ state: s, index: i }))
+                .filter(({ state, index }) =>
+                    !state.isTracking &&
+                    (this.interestedTargetIndex === -1 || this.interestedTargetIndex === index)
+                )
+                .map(({ index }) => index);
+
+            if (matchingIndexes.length > 0) {
+                // Use full input for detection (bio engine already optimized upstream processing)
+                const { targetIndex: matchedTargetIndex, modelViewTransform, featurePoints } =
+                    await this._detectAndMatch(inputData, matchingIndexes, bioResult.octavesToProcess || null);
+
+                if (matchedTargetIndex !== -1) {
+                    this.trackingStates[matchedTargetIndex].isTracking = true;
+                    this.trackingStates[matchedTargetIndex].currentModelViewTransform = modelViewTransform;
+
+                    // Update bio engine fovea to focus on detected target
+                    if (bioResult.attentionRegions?.[0]) {
+                        this.bioEngine?.reset();
+                    }
+                }
+
+                this.onUpdate?.({ type: 'featurePoints', featurePoints });
+            }
+        }
+
+        // Tracking phase
+        for (let i = 0; i < this.trackingStates.length; i++) {
+            const trackingState = this.trackingStates[i];
+
+            if (trackingState.isTracking) {
+                const result = await this._trackAndUpdate(
+                    inputData,
+                    trackingState.currentModelViewTransform,
+                    i
+                );
+
+                if (!result || !result.modelViewTransform) {
+                    trackingState.isTracking = false;
+                    trackingState.screenCoords = result?.screenCoords || [];
+                    trackingState.reliabilities = result?.reliabilities || [];
+                    trackingState.stabilities = result?.stabilities || [];
+                } else {
+                    trackingState.currentModelViewTransform = result.modelViewTransform;
+                    trackingState.screenCoords = result.screenCoords;
+                    trackingState.reliabilities = result.reliabilities;
+                    trackingState.stabilities = result.stabilities;
+                    (trackingState as any).deformedMesh = result.deformedMesh;
+                }
+            }
+
+            const wasShowing = trackingState.showing;
+            trackingState.showing = this.featureManager.shouldShow(i, trackingState.isTracking);
+
+            if (wasShowing && !trackingState.showing) {
+                trackingState.trackingMatrix = null;
+                this.featureManager.notifyUpdate({ type: 'reset', targetIndex: i });
+            }
+
+            // Emit update
+            if (trackingState.showing || trackingState.screenCoords?.length > 0 || (wasShowing && !trackingState.showing)) {
+                const worldMatrix = trackingState.showing
+                    ? this._glModelViewMatrix(trackingState.currentModelViewTransform, i)
+                    : null;
+
+                let finalMatrix = null;
+                if (worldMatrix) {
+                    const stabilities = trackingState.stabilities || [];
+                    const avgStability = stabilities.length > 0
+                        ? stabilities.reduce((a: number, b: number) => a + b, 0) / stabilities.length
+                        : 0;
+
+                    finalMatrix = this.featureManager.applyWorldMatrixFilters(i, worldMatrix, { stability: avgStability });
+                    trackingState.trackingMatrix = finalMatrix;
+
+                    const isInputRotated = input.width === this.inputHeight && input.height === this.inputWidth;
+                    if (isInputRotated) {
+                        const rotationFeature = this.featureManager.getFeature<AutoRotationFeature>('auto-rotation');
+                        if (rotationFeature) {
+                            finalMatrix = rotationFeature.rotate(finalMatrix);
+                        }
+                    }
+                }
+
+                this.onUpdate?.({
+                    type: 'updateMatrix',
+                    targetIndex: i,
+                    worldMatrix: finalMatrix,
+                    modelViewTransform: trackingState.currentModelViewTransform,
+                    screenCoords: trackingState.screenCoords,
+                    reliabilities: trackingState.reliabilities,
+                    stabilities: trackingState.stabilities,
+                    deformedMesh: (trackingState as any).deformedMesh,
+                    bioMetrics: this.bioEngine?.getMetrics(),
+                    foveaCenter: bioResult.foveaCenter,
+                    pixelsSaved: bioResult.pixelsSaved,
+                });
+            }
+        }
+
+        this.onUpdate?.({ type: 'processDone' });
+    }
+
+    /**
+     * Detect and match features, optionally limited to specific octaves
+     */
+    async _detectAndMatch(inputData: any, targetIndexes: number[], octavesToProcess: number[] | null = null) {
+        // 🚀 NANITE-STYLE: Estimate scale for filtered matching
+        let predictedScale: number | undefined = undefined;
+        for (const state of this.trackingStates) {
+            if (state.isTracking && state.currentModelViewTransform) {
+                const m = state.currentModelViewTransform;
+                predictedScale = Math.sqrt(m[0][0] ** 2 + m[1][0] ** 2 + m[2][0] ** 2);
+                break;
+            }
+        }
+
+        const { targetIndex, modelViewTransform, screenCoords, worldCoords, featurePoints } = await this._workerMatch(
+            null, // No feature points, worker will detect from inputData
+            targetIndexes,
+            inputData,
+            predictedScale,
+            octavesToProcess
+        );
+        return { targetIndex, modelViewTransform, screenCoords, worldCoords, featurePoints };
+    }
+
+    /**
+     * Communicate with worker for matching phase
+     */
+    _workerMatch(featurePoints: any, targetIndexes: number[], inputData: any = null, expectedScale?: number, octavesToProcess: number[] | null = null): Promise<any> {
+        return new Promise((resolve) => {
+            if (!this.worker) {
+                // If no feature points but we have input data, detect first
+                let fpPromise;
+                if (!featurePoints && inputData) {
+                    fpPromise = Promise.resolve(this.fullDetector!.detect(inputData, { octavesToProcess }).featurePoints);
+                } else {
+                    fpPromise = Promise.resolve(featurePoints);
+                }
+
+                fpPromise.then(fp => {
+                    this._matchOnMainThread(fp, targetIndexes, expectedScale).then(resolve);
+                }).catch(() => resolve({ targetIndex: -1 }));
+                return;
+            }
+
+            const timeout = setTimeout(() => {
+                (this as any).workerMatchDone = null;
+                resolve({ targetIndex: -1 });
+            }, 1000);
+
+            (this as any).workerMatchDone = (data: any) => {
+                clearTimeout(timeout);
+                (this as any).workerMatchDone = null;
+                resolve(data);
+            };
+
+            if (inputData) {
+                this.worker.postMessage({ type: "match", inputData, targetIndexes, octavesToProcess, expectedScale });
+            } else {
+                this.worker.postMessage({ type: "match", featurePoints: featurePoints, targetIndexes, expectedScale });
+            }
+        });
+    }
+
+    /**
+     * Override _trackAndUpdate to capture active octave for the next frame's orchestration
+     */
+    async _trackAndUpdate(inputData: any, lastModelViewTransform: number[][], targetIndex: number) {
+        const result = await super._trackAndUpdate(inputData, lastModelViewTransform, targetIndex);
+        if (result && (result as any).octaveIndex !== undefined) {
+            this.trackingStates[targetIndex].lastOctaveIndex = (result as any).octaveIndex;
+        }
+        return result;
+    }
+
+    /**
+     * Flatten a 3x4 matrix to Float32Array
+     * @private
+     */
+    private _flattenMatrix(matrix: number[][]): Float32Array {
+        const result = new Float32Array(16);
+        for (let i = 0; i < 3; i++) {
+            for (let j = 0; j < 4; j++) {
+                result[i * 4 + j] = matrix[i][j];
+            }
+        }
+        result[12] = 0;
+        result[13] = 0;
+        result[14] = 0;
+        result[15] = 1;
+        return result;
+    }
+
+    /**
+     * Unflatten Float32Array to 3x4 matrix
+     * @private
+     */
+    private _unflattenMatrix(flat: Float32Array): number[][] {
+        return [
+            [flat[0], flat[1], flat[2], flat[3]],
+            [flat[4], flat[5], flat[6], flat[7]],
+            [flat[8], flat[9], flat[10], flat[11]],
+        ];
+    }
+
+    /**
+     * Get bio-inspired engine metrics
+     */
+    getBioMetrics() {
+        return this.bioEngine?.getMetrics() || null;
+    }
+
+    /**
+     * Get last bio processing result
+     */
+    getLastBioResult() {
+        return this.lastBioResult;
+    }
+
+    /**
+     * Enable/disable bio-inspired processing dynamically
+     */
+    setBioEnabled(enabled: boolean) {
+        this.bioEnabled = enabled;
+        if (enabled && !this.bioEngine) {
+            this.bioEngine = new BioInspiredEngine(this.inputWidth, this.inputHeight);
+        }
+    }
+
+    /**
+     * Configure bio-inspired engine at runtime
+     */
+    configureBio(options: Partial<typeof BIO_CONFIG>) {
+        this.bioEngine?.configure(options);
+    }
+
+    /**
+     * Override dispose to clean up bio engine
+     */
+    dispose() {
+        super.dispose();
+        this.bioEngine = null;
+        if (this.bioMetricsInterval) {
+            clearInterval(this.bioMetricsInterval);
+        }
+    }
+}
+
+export { BioInspiredController };

package/src/runtime/controller.ts

@@ -257,10 +257,23 @@ class Controller {
     }
 
     async _detectAndMatch(inputData: any, targetIndexes: number[]) {
+        // 🚀 NANITE-STYLE: Estimate scale for filtered matching
+        // If we were already tracking a target, use its scale as a hint for faster matching
+        let predictedScale: number | undefined = undefined;
+        for (const state of this.trackingStates) {
+            if (state.isTracking && state.currentModelViewTransform) {
+                const m = state.currentModelViewTransform;
+                // Vector magnitude of the first column is a good approximation of the scale
+                predictedScale = Math.sqrt(m[0][0] ** 2 + m[1][0] ** 2 + m[2][0] ** 2);
+                break;
+            }
+        }
+
         const { targetIndex, modelViewTransform, screenCoords, worldCoords, featurePoints } = await this._workerMatch(
             null, // No feature points, worker will detect from inputData
             targetIndexes,
-            inputData
+            inputData,
+            predictedScale
         );
         return { targetIndex, modelViewTransform, screenCoords, worldCoords, featurePoints };
     }
@@ -361,7 +374,8 @@ class Controller {
             screenCoords: finalScreenCoords,
             reliabilities: finalReliabilities,
             stabilities: finalStabilities,
-            deformedMesh
+            deformedMesh,
+            octaveIndex // Pass this up for the orchestrator
         };
     }
 
@@ -521,7 +535,7 @@ class Controller {
         return this._workerTrackUpdate(modelViewTransform, trackFeatures);
     }
 
-    _workerMatch(featurePoints: any, targetIndexes: number[], inputData: any = null): Promise<any> {
+    _workerMatch(featurePoints: any, targetIndexes: number[], inputData: any = null, expectedScale?: number): Promise<any> {
         return new Promise((resolve) => {
             if (!this.worker) {
                 // If no feature points but we have input data, detect first
@@ -533,7 +547,7 @@ class Controller {
                 }
 
                 fpPromise.then(fp => {
-                    this._matchOnMainThread(fp, targetIndexes).then(resolve);
+                    this._matchOnMainThread(fp, targetIndexes, expectedScale).then(resolve);
                 }).catch(() => resolve({ targetIndex: -1 }));
                 return;
             }
@@ -557,9 +571,9 @@ class Controller {
             };
 
             if (inputData) {
-                this.worker.postMessage({ type: "match", inputData, targetIndexes });
+                this.worker.postMessage({ type: "match", inputData, targetIndexes, expectedScale });
             } else {
-                this.worker.postMessage({ type: "match", featurePoints: featurePoints, targetIndexes });
+                this.worker.postMessage({ type: "match", featurePoints: featurePoints, targetIndexes, expectedScale });
             }
         });
     }
@@ -591,7 +605,7 @@ class Controller {
         });
     }
 
-    async _matchOnMainThread(featurePoints: any, targetIndexes: number[]) {
+    async _matchOnMainThread(featurePoints: any, targetIndexes: number[], expectedScale?: number) {
         if (!this.mainThreadMatcher) {
             const { Matcher } = await import("../core/matching/matcher.js");
             const { Estimator } = await import("../core/estimation/estimator.js");
@@ -610,6 +624,7 @@ class Controller {
             const { keyframeIndex, screenCoords, worldCoords, debugExtra } = this.mainThreadMatcher.matchDetection(
                 this.matchingDataList[matchingIndex],
                 featurePoints,
+                expectedScale
             );
             matchedDebugExtra = debugExtra;
 

package/src/runtime/controller.worker.js

@@ -49,7 +49,7 @@ onmessage = (msg) => {
       // New: If the worker received image data, run detector here too
       let featurePoints = data.featurePoints;
       if (data.inputData) {
-        const detectionResult = detector.detect(data.inputData);
+        const detectionResult = detector.detect(data.inputData, { octavesToProcess: data.octavesToProcess });
         featurePoints = detectionResult.featurePoints;
       }
 
@@ -59,6 +59,7 @@ onmessage = (msg) => {
         const { keyframeIndex, screenCoords, worldCoords, debugExtra } = matcher.matchDetection(
           matchingDataList[matchingIndex],
           featurePoints,
+          data.expectedScale
         );
         matchedDebugExtra = debugExtra;