@srsergio/taptapp-ar 1.0.92 → 1.0.94

package/dist/core/perception/saliency-map.js

@@ -0,0 +1,254 @@
+/**
+ * Saliency Map Computation
+ *
+ * Computes visual saliency - regions that "pop out" and attract attention.
+ * Used to guide saccadic attention to visually important areas.
+ *
+ * Implements a simplified Itti-Koch saliency model:
+ * - Intensity contrast
+ * - Edge density
+ * - Local complexity
+ *
+ * For AR tracking, high-saliency regions often contain:
+ * - Corners and edges (good for feature detection)
+ * - High-contrast areas (robust to lighting changes)
+ * - Texture-rich regions (distinctive for matching)
+ */
+class SaliencyMap {
+    /**
+     * @param {number} width - Image width
+     * @param {number} height - Image height
+     */
+    constructor(width, height) {
+        this.width = width;
+        this.height = height;
+        // Downsampled dimensions for efficiency
+        this.scale = 8; // Process at 1/8 resolution
+        this.scaledW = Math.ceil(width / this.scale);
+        this.scaledH = Math.ceil(height / this.scale);
+        // Pre-allocate buffers
+        this.intensityMap = new Float32Array(this.scaledW * this.scaledH);
+        this.contrastMap = new Float32Array(this.scaledW * this.scaledH);
+        this.edgeMap = new Float32Array(this.scaledW * this.scaledH);
+        this.saliencyBuffer = new Float32Array(this.scaledW * this.scaledH);
+        // Peak detection parameters
+        this.maxPeaks = 5;
+        this.suppressionRadius = Math.max(this.scaledW, this.scaledH) * 0.15;
+    }
+    /**
+     * Compute saliency map for input image
+     *
+     * @param {Uint8Array} inputData - Grayscale input image
+     * @returns {Object} Saliency result with peaks
+     */
+    compute(inputData) {
+        // Step 1: Downsample and compute intensity
+        this._downsample(inputData);
+        // Step 2: Compute features
+        this._computeContrast();
+        this._computeEdges();
+        // Step 3: Combine into saliency map
+        this._combineSaliency();
+        // Step 4: Find peaks
+        const peaks = this._findPeaks();
+        return {
+            map: this.saliencyBuffer,
+            width: this.scaledW,
+            height: this.scaledH,
+            peaks,
+            maxSaliency: peaks.length > 0 ? peaks[0].value : 0,
+        };
+    }
+    /**
+     * Downsample input to working resolution
+     * @private
+     */
+    _downsample(inputData) {
+        const s = this.scale;
+        const w = this.width;
+        for (let sy = 0; sy < this.scaledH; sy++) {
+            const yStart = sy * s;
+            const yEnd = Math.min(yStart + s, this.height);
+            for (let sx = 0; sx < this.scaledW; sx++) {
+                const xStart = sx * s;
+                const xEnd = Math.min(xStart + s, this.width);
+                let sum = 0;
+                let count = 0;
+                for (let y = yStart; y < yEnd; y++) {
+                    const rowOffset = y * w;
+                    for (let x = xStart; x < xEnd; x++) {
+                        sum += inputData[rowOffset + x];
+                        count++;
+                    }
+                }
+                this.intensityMap[sy * this.scaledW + sx] = sum / count / 255;
+            }
+        }
+    }
+    /**
+     * Compute local contrast map
+     * @private
+     */
+    _computeContrast() {
+        const w = this.scaledW;
+        const h = this.scaledH;
+        const intensity = this.intensityMap;
+        const contrast = this.contrastMap;
+        // 3x3 local contrast using center-surround
+        for (let y = 1; y < h - 1; y++) {
+            for (let x = 1; x < w - 1; x++) {
+                const idx = y * w + x;
+                const center = intensity[idx];
+                // Compute average of 8 neighbors
+                let surround = 0;
+                surround += intensity[(y - 1) * w + (x - 1)];
+                surround += intensity[(y - 1) * w + x];
+                surround += intensity[(y - 1) * w + (x + 1)];
+                surround += intensity[y * w + (x - 1)];
+                surround += intensity[y * w + (x + 1)];
+                surround += intensity[(y + 1) * w + (x - 1)];
+                surround += intensity[(y + 1) * w + x];
+                surround += intensity[(y + 1) * w + (x + 1)];
+                surround /= 8;
+                // Contrast is absolute difference
+                contrast[idx] = Math.abs(center - surround);
+            }
+        }
+        // Handle borders
+        for (let y = 0; y < h; y++) {
+            contrast[y * w] = 0;
+            contrast[y * w + w - 1] = 0;
+        }
+        for (let x = 0; x < w; x++) {
+            contrast[x] = 0;
+            contrast[(h - 1) * w + x] = 0;
+        }
+    }
+    /**
+     * Compute edge density map using Sobel-like operator
+     * @private
+     */
+    _computeEdges() {
+        const w = this.scaledW;
+        const h = this.scaledH;
+        const intensity = this.intensityMap;
+        const edges = this.edgeMap;
+        for (let y = 1; y < h - 1; y++) {
+            for (let x = 1; x < w - 1; x++) {
+                // Simplified Sobel
+                const gx = -intensity[(y - 1) * w + (x - 1)] + intensity[(y - 1) * w + (x + 1)] +
+                    -2 * intensity[y * w + (x - 1)] + 2 * intensity[y * w + (x + 1)] +
+                    -intensity[(y + 1) * w + (x - 1)] + intensity[(y + 1) * w + (x + 1)];
+                const gy = -intensity[(y - 1) * w + (x - 1)] - 2 * intensity[(y - 1) * w + x] - intensity[(y - 1) * w + (x + 1)] +
+                    intensity[(y + 1) * w + (x - 1)] + 2 * intensity[(y + 1) * w + x] + intensity[(y + 1) * w + (x + 1)];
+                edges[y * w + x] = Math.sqrt(gx * gx + gy * gy) / 4; // Normalize
+            }
+        }
+        // Handle borders
+        for (let y = 0; y < h; y++) {
+            edges[y * w] = 0;
+            edges[y * w + w - 1] = 0;
+        }
+        for (let x = 0; x < w; x++) {
+            edges[x] = 0;
+            edges[(h - 1) * w + x] = 0;
+        }
+    }
+    /**
+     * Combine features into final saliency map
+     * @private
+     */
+    _combineSaliency() {
+        const n = this.saliencyBuffer.length;
+        const contrast = this.contrastMap;
+        const edges = this.edgeMap;
+        const saliency = this.saliencyBuffer;
+        // Weight: 60% contrast, 40% edges
+        for (let i = 0; i < n; i++) {
+            saliency[i] = contrast[i] * 0.6 + edges[i] * 0.4;
+        }
+        // Normalize to [0, 1]
+        let max = 0;
+        for (let i = 0; i < n; i++) {
+            max = Math.max(max, saliency[i]);
+        }
+        if (max > 0) {
+            for (let i = 0; i < n; i++) {
+                saliency[i] /= max;
+            }
+        }
+    }
+    /**
+     * Find peaks in saliency map using non-maximum suppression
+     * @private
+     */
+    _findPeaks() {
+        const w = this.scaledW;
+        const h = this.scaledH;
+        const saliency = this.saliencyBuffer;
+        const peaks = [];
+        const r = this.suppressionRadius;
+        const r2 = r * r;
+        // Find all local maxima
+        const candidates = [];
+        for (let y = 1; y < h - 1; y++) {
+            for (let x = 1; x < w - 1; x++) {
+                const idx = y * w + x;
+                const val = saliency[idx];
+                // Check if local maximum (8-connected)
+                if (val > saliency[(y - 1) * w + (x - 1)] &&
+                    val > saliency[(y - 1) * w + x] &&
+                    val > saliency[(y - 1) * w + (x + 1)] &&
+                    val > saliency[y * w + (x - 1)] &&
+                    val > saliency[y * w + (x + 1)] &&
+                    val > saliency[(y + 1) * w + (x - 1)] &&
+                    val > saliency[(y + 1) * w + x] &&
+                    val > saliency[(y + 1) * w + (x + 1)]) {
+                    candidates.push({ x, y, value: val });
+                }
+            }
+        }
+        // Sort by value descending
+        candidates.sort((a, b) => b.value - a.value);
+        // Non-maximum suppression
+        for (const cand of candidates) {
+            if (peaks.length >= this.maxPeaks)
+                break;
+            // Check if too close to existing peaks
+            let suppress = false;
+            for (const peak of peaks) {
+                const dx = cand.x - peak.x;
+                const dy = cand.y - peak.y;
+                if (dx * dx + dy * dy < r2) {
+                    suppress = true;
+                    break;
+                }
+            }
+            if (!suppress) {
+                // Convert to original image coordinates
+                peaks.push({
+                    x: (cand.x + 0.5) * this.scale,
+                    y: (cand.y + 0.5) * this.scale,
+                    value: cand.value,
+                });
+            }
+        }
+        return peaks;
+    }
+    /**
+     * Get saliency value at a specific location
+     *
+     * @param {number} x - X coordinate in original image
+     * @param {number} y - Y coordinate in original image
+     * @returns {number} Saliency value (0-1)
+     */
+    getSaliencyAt(x, y) {
+        const sx = Math.floor(x / this.scale);
+        const sy = Math.floor(y / this.scale);
+        if (sx < 0 || sx >= this.scaledW || sy < 0 || sy >= this.scaledH) {
+            return 0;
+        }
+        return this.saliencyBuffer[sy * this.scaledW + sx];
+    }
+}
+export { SaliencyMap };

package/dist/core/perception/scale-orchestrator.d.ts

@@ -0,0 +1,28 @@
+/**
+ * Scale Orchestrator
+ *
+ * Manages which octaves should be processed based on the current tracking state.
+ * Implements temporal consistency and interleave strategies to optimize performance.
+ */
+export class ScaleOrchestrator {
+    constructor(numOctaves: any, options?: {});
+    numOctaves: any;
+    options: {
+        interleaveInterval: number;
+        hysteresis: number;
+    };
+    frameCount: number;
+    lastActiveOctave: number;
+    interleaveOctave: number;
+    /**
+     * Determine which octaves should be processed in the current frame
+     *
+     * @param {Object} trackingState - Current state of tracking
+     * @returns {number[]} Array of octave indices to process
+     */
+    getOctavesToProcess(trackingState?: Object): number[];
+    /**
+     * Reset orchestrator state
+     */
+    reset(): void;
+}

package/dist/core/perception/scale-orchestrator.js

@@ -0,0 +1,68 @@
+/**
+ * Scale Orchestrator
+ *
+ * Manages which octaves should be processed based on the current tracking state.
+ * Implements temporal consistency and interleave strategies to optimize performance.
+ */
+export class ScaleOrchestrator {
+    constructor(numOctaves, options = {}) {
+        this.numOctaves = numOctaves;
+        this.options = {
+            interleaveInterval: 10,
+            hysteresis: 1, // Number of adjacent octaves to keep
+            ...options
+        };
+        this.frameCount = 0;
+        this.lastActiveOctave = -1;
+        this.interleaveOctave = 0;
+    }
+    /**
+     * Determine which octaves should be processed in the current frame
+     *
+     * @param {Object} trackingState - Current state of tracking
+     * @returns {number[]} Array of octave indices to process
+     */
+    getOctavesToProcess(trackingState = null) {
+        this.frameCount++;
+        // Case 1: No tracking or lost tracking -> Process all octaves
+        if (!trackingState || !trackingState.isTracking || trackingState.activeOctave === undefined) {
+            this.lastActiveOctave = -1;
+            return Array.from({ length: this.numOctaves }, (_, i) => i);
+        }
+        const activeScale = trackingState.activeOctave;
+        this.lastActiveOctave = activeScale;
+        // Case 2: Active tracking -> Focus on current scale and neighbors
+        const octaves = new Set();
+        // Add current and adjacent scales (Hysteresis)
+        for (let i = -this.options.hysteresis; i <= this.options.hysteresis; i++) {
+            const octave = activeScale + i;
+            if (octave >= 0 && octave < this.numOctaves) {
+                octaves.add(octave);
+            }
+        }
+        // Case 3: Interleave - Periodically check a distant octave to ensure we don't "drift"
+        if (this.frameCount % this.options.interleaveInterval === 0) {
+            this.interleaveOctave = (this.interleaveOctave + 1) % this.numOctaves;
+            // If the interleave octave is already being processed, pick the next one
+            if (octaves.has(this.interleaveOctave)) {
+                this.interleaveOctave = (this.interleaveOctave + 1) % this.numOctaves;
+            }
+            octaves.add(this.interleaveOctave);
+            if (this.options.debug) {
+                console.log(`[ScaleOrchestrator] Interleave check on octave ${this.interleaveOctave}`);
+            }
+        }
+        const result = Array.from(octaves).sort((a, b) => a - b);
+        if (this.options.debug) {
+            console.log(`[ScaleOrchestrator] Active: ${activeScale}, Processing: [${result.join(', ')}]`);
+        }
+        return result;
+    }
+    /**
+     * Reset orchestrator state
+     */
+    reset() {
+        this.frameCount = 0;
+        this.lastActiveOctave = -1;
+    }
+}

package/dist/core/protocol.d.ts

@@ -1,4 +1,4 @@
-export declare const CURRENT_VERSION = 9;
+export declare const CURRENT_VERSION = 11;
 export declare const HDC_SEED = 322420463;
 /**
  * Morton Order calculation for spatial sorting
@@ -24,6 +24,19 @@ export declare function columnarize(points: any[], tree: any, width: number, hei
     hdc: number;
     t: any;
 };
+/**
+ * Columnarizes point data with COMPACT 32-bit descriptors (XOR folding)
+ * Reduces descriptor storage by 50% with minimal accuracy loss
+ */
+export declare function columnarizeCompact(points: any[], tree: any, width: number, height: number): {
+    x: Uint16Array<ArrayBuffer>;
+    y: Uint16Array<ArrayBuffer>;
+    a: Int16Array<ArrayBuffer>;
+    s: Uint8Array<ArrayBuffer>;
+    d: Uint32Array<ArrayBuffer>;
+    compact: number;
+    t: any;
+};
 /**
  * Compacts hierarchical clustering tree into a minimal array structure
  */

package/dist/core/protocol.js

@@ -1,5 +1,5 @@
 import * as msgpack from "@msgpack/msgpack";
-export const CURRENT_VERSION = 9; // Bumped for HDC support
+export const CURRENT_VERSION = 11; // Bumped for Nanite virtualized features support
 export const HDC_SEED = 0x1337BEEF; // Default system seed
 /**
  * Morton Order calculation for spatial sorting
@@ -91,6 +91,38 @@ export function columnarize(points, tree, width, height, useHDC = false) {
         t: compactTree(tree.rootNode),
     };
 }
+/**
+ * Columnarizes point data with COMPACT 32-bit descriptors (XOR folding)
+ * Reduces descriptor storage by 50% with minimal accuracy loss
+ */
+export function columnarizeCompact(points, tree, width, height) {
+    const count = points.length;
+    const x = new Uint16Array(count);
+    const y = new Uint16Array(count);
+    const angle = new Int16Array(count);
+    const scale = new Uint8Array(count);
+    const descriptors = new Uint32Array(count); // 32-bit compact descriptors
+    for (let i = 0; i < count; i++) {
+        x[i] = Math.round((points[i].x / width) * 65535);
+        y[i] = Math.round((points[i].y / height) * 65535);
+        angle[i] = Math.round((points[i].angle / Math.PI) * 32767);
+        scale[i] = Math.round(Math.log2(points[i].scale || 1));
+        if (points[i].descriptors && points[i].descriptors.length >= 2) {
+            // XOR folding: Combine two 32-bit values into one 32-bit value
+            // This preserves discriminative power while halving storage
+            descriptors[i] = (points[i].descriptors[0] ^ points[i].descriptors[1]) >>> 0;
+        }
+    }
+    return {
+        x,
+        y,
+        a: angle,
+        s: scale,
+        d: descriptors,
+        compact: 1, // Flag to indicate compact 32-bit descriptors
+        t: compactTree(tree.rootNode),
+    };
+}
 /**
  * Compacts hierarchical clustering tree into a minimal array structure
  */

package/dist/runtime/bio-inspired-controller.d.ts

@@ -0,0 +1,135 @@
+/**
+ * Bio-Inspired Controller Adapter
+ *
+ * Wraps the standard Controller with Bio-Inspired Perception capabilities.
+ * Provides significant performance improvements while maintaining API compatibility.
+ *
+ * Key features:
+ * - Foveal attention: Processes only regions of interest at full resolution
+ * - Predictive coding: Skips processing when scene is static
+ * - Saccadic sampling: Strategic "glances" at high-saliency regions
+ *
+ * Usage:
+ * ```javascript
+ * import { BioInspiredController } from './bio-inspired-controller.js';
+ *
+ * const controller = new BioInspiredController({
+ *   inputWidth: 640,
+ *   inputHeight: 480,
+ *   onUpdate: (data) => console.log(data),
+ *   bioInspired: {
+ *     enabled: true,
+ *     aggressiveSkipping: true,
+ *   }
+ * });
+ * ```
+ */
+import { Controller, ControllerOptions } from './controller.js';
+import { BIO_CONFIG } from '../core/perception/index.js';
+/**
+ * Extended options for Bio-Inspired Controller
+ */
+export interface BioInspiredControllerOptions extends ControllerOptions {
+    bioInspired?: {
+        enabled?: boolean;
+        aggressiveSkipping?: boolean;
+        foveaRadiusRatio?: number;
+        maxSaccades?: number;
+    };
+}
+/**
+ * Bio-Inspired Controller
+ *
+ * Extends the standard Controller with bio-inspired perception capabilities.
+ */
+declare class BioInspiredController extends Controller {
+    private bioEngine;
+    private bioEnabled;
+    private bioMetricsInterval;
+    private lastBioResult;
+    constructor(options: BioInspiredControllerOptions);
+    /**
+     * Override processVideo to add bio-inspired perception
+     */
+    processVideo(input: any): void;
+    /**
+     * Handle a skipped frame using prediction
+     * @private
+     */
+    private _handleSkippedFrame;
+    /**
+     * Process frame using bio-inspired attention regions
+     * @private
+     */
+    private _processWithAttention;
+    /**
+     * Detect and match features, optionally limited to specific octaves
+     */
+    _detectAndMatch(inputData: any, targetIndexes: number[], octavesToProcess?: number[] | null): Promise<{
+        targetIndex: any;
+        modelViewTransform: any;
+        screenCoords: any;
+        worldCoords: any;
+        featurePoints: any;
+    }>;
+    /**
+     * Communicate with worker for matching phase
+     */
+    _workerMatch(featurePoints: any, targetIndexes: number[], inputData?: any, expectedScale?: number, octavesToProcess?: number[] | null): Promise<any>;
+    /**
+     * Override _trackAndUpdate to capture active octave for the next frame's orchestration
+     */
+    _trackAndUpdate(inputData: any, lastModelViewTransform: number[][], targetIndex: number): Promise<{
+        modelViewTransform: null;
+        screenCoords: never[];
+        reliabilities: never[];
+        stabilities: never[];
+        deformedMesh: null;
+        octaveIndex?: undefined;
+    } | {
+        modelViewTransform: null;
+        screenCoords: any[];
+        reliabilities: number[];
+        stabilities: number[];
+        deformedMesh?: undefined;
+        octaveIndex?: undefined;
+    } | {
+        modelViewTransform: any;
+        screenCoords: any[];
+        reliabilities: number[];
+        stabilities: number[];
+        deformedMesh: any;
+        octaveIndex: any;
+    }>;
+    /**
+     * Flatten a 3x4 matrix to Float32Array
+     * @private
+     */
+    private _flattenMatrix;
+    /**
+     * Unflatten Float32Array to 3x4 matrix
+     * @private
+     */
+    private _unflattenMatrix;
+    /**
+     * Get bio-inspired engine metrics
+     */
+    getBioMetrics(): Object | null;
+    /**
+     * Get last bio processing result
+     */
+    getLastBioResult(): any;
+    /**
+     * Enable/disable bio-inspired processing dynamically
+     */
+    setBioEnabled(enabled: boolean): void;
+    /**
+     * Configure bio-inspired engine at runtime
+     */
+    configureBio(options: Partial<typeof BIO_CONFIG>): void;
+    /**
+     * Override dispose to clean up bio engine
+     */
+    dispose(): void;
+}
+export { BioInspiredController };