@srsergio/taptapp-ar 1.0.92 → 1.0.94

package/src/core/perception/saccadic-controller.js

@@ -0,0 +1,303 @@
+/**
+ * Saccadic Controller
+ * 
+ * Mimics human eye saccades - rapid movements that redirect foveal attention
+ * to areas of interest. The human eye makes 3-4 saccades per second to
+ * build a complete picture of the visual scene.
+ * 
+ * Strategy:
+ * 1. Compute saliency map to find "interesting" regions
+ * 2. Use tracking state to predict where features should be
+ * 3. Generate priority-ordered list of "glance" targets
+ * 4. Limit saccades per frame to balance coverage vs. efficiency
+ */
+
+/**
+ * A saccade target representing where attention should be directed
+ * @typedef {Object} SaccadeTarget
+ * @property {number} x - X coordinate
+ * @property {number} y - Y coordinate
+ * @property {number} priority - Priority (0 = highest)
+ * @property {string} reason - Why this target was selected
+ * @property {number} saliency - Saliency score at this location
+ */
+
+class SaccadicController {
+    /**
+     * @param {number} width - Image width
+     * @param {number} height - Image height
+     * @param {Object} config - Configuration
+     */
+    constructor(width, height, config) {
+        this.width = width;
+        this.height = height;
+        this.config = config;
+
+        // Saccade history for inhibition of return
+        this.recentTargets = [];
+        this.inhibitionRadius = Math.min(width, height) * 0.1;
+
+        // Movement prediction
+        this.velocityHistory = [];
+        this.lastCenter = { x: width / 2, y: height / 2 };
+
+        // Grid for systematic coverage
+        this.gridCells = this._buildCoverageGrid(3, 3);
+        this.lastVisitedCell = 4; // Center
+
+        // State
+        this.lastSaccadeTime = 0;
+        this.saccadeCount = 0;
+    }
+
+    /**
+     * Build a grid for systematic coverage during tracking loss
+     * @private
+     */
+    _buildCoverageGrid(rows, cols) {
+        const cells = [];
+        const cellW = this.width / cols;
+        const cellH = this.height / rows;
+
+        for (let r = 0; r < rows; r++) {
+            for (let c = 0; c < cols; c++) {
+                cells.push({
+                    x: cellW * (c + 0.5),
+                    y: cellH * (r + 0.5),
+                    index: r * cols + c,
+                    lastVisit: 0,
+                });
+            }
+        }
+        return cells;
+    }
+
+    /**
+     * Compute saccade targets based on current state
+     * 
+     * @param {Object} saliency - Saliency map result
+     * @param {Object} currentFovea - Current fovea center {x, y}
+     * @param {Object} trackingState - Current tracking state (optional)
+     * @returns {SaccadeTarget[]} Priority-ordered list of targets
+     */
+    computeTargets(saliency, currentFovea, trackingState = null) {
+        const targets = [];
+        const maxTargets = this.config.MAX_SACCADES_PER_FRAME;
+
+        // Strategy 1: Follow tracking prediction (highest priority)
+        if (trackingState && trackingState.isTracking) {
+            const predicted = this._predictTrackingCenter(trackingState);
+            if (predicted) {
+                targets.push({
+                    x: predicted.x,
+                    y: predicted.y,
+                    priority: 0,
+                    reason: 'tracking_prediction',
+                    saliency: 1.0,
+                });
+            }
+        }
+
+        // Strategy 2: High saliency regions
+        if (saliency && saliency.peaks) {
+            for (const peak of saliency.peaks) {
+                if (targets.length >= maxTargets) break;
+
+                // Skip if too close to existing targets (inhibition of return)
+                if (this._isInhibited(peak.x, peak.y, targets)) continue;
+
+                if (peak.value > this.config.SALIENCY_THRESHOLD) {
+                    targets.push({
+                        x: peak.x,
+                        y: peak.y,
+                        priority: targets.length,
+                        reason: 'saliency_peak',
+                        saliency: peak.value,
+                    });
+                }
+            }
+        }
+
+        // Strategy 3: Systematic grid search (when not tracking)
+        if (!trackingState?.isTracking && targets.length < maxTargets) {
+            const gridTarget = this._getNextGridCell();
+            if (gridTarget && !this._isInhibited(gridTarget.x, gridTarget.y, targets)) {
+                targets.push({
+                    x: gridTarget.x,
+                    y: gridTarget.y,
+                    priority: targets.length,
+                    reason: 'grid_search',
+                    saliency: 0.5,
+                });
+            }
+        }
+
+        // Strategy 4: Stay at current center if no better options
+        if (targets.length === 0) {
+            targets.push({
+                x: currentFovea.x,
+                y: currentFovea.y,
+                priority: 0,
+                reason: 'maintain_position',
+                saliency: 0.3,
+            });
+        }
+
+        // Update history
+        this._updateHistory(targets);
+
+        return targets;
+    }
+
+    /**
+     * Predict center of tracking based on current state and velocity
+     * @private
+     */
+    _predictTrackingCenter(trackingState) {
+        if (!trackingState.worldMatrix) return null;
+
+        // Extract center from world matrix
+        const matrix = trackingState.worldMatrix;
+        const cx = matrix[12] || this.width / 2;
+        const cy = matrix[13] || this.height / 2;
+
+        // Apply velocity-based prediction
+        if (this.velocityHistory.length >= 2) {
+            const vx = this._computeAverageVelocity('x');
+            const vy = this._computeAverageVelocity('y');
+
+            // Predict 1 frame ahead
+            return {
+                x: Math.max(0, Math.min(this.width - 1, cx + vx)),
+                y: Math.max(0, Math.min(this.height - 1, cy + vy)),
+            };
+        }
+
+        return { x: cx, y: cy };
+    }
+
+    /**
+     * Compute average velocity from history
+     * @private
+     */
+    _computeAverageVelocity(axis) {
+        if (this.velocityHistory.length < 2) return 0;
+
+        let sum = 0;
+        for (let i = 1; i < this.velocityHistory.length; i++) {
+            sum += this.velocityHistory[i][axis] - this.velocityHistory[i - 1][axis];
+        }
+        return sum / (this.velocityHistory.length - 1);
+    }
+
+    /**
+     * Check if a location is inhibited (too close to recent targets)
+     * @private
+     */
+    _isInhibited(x, y, currentTargets) {
+        const r2 = this.inhibitionRadius ** 2;
+
+        // Check against current frame targets
+        for (const t of currentTargets) {
+            const dx = x - t.x;
+            const dy = y - t.y;
+            if (dx * dx + dy * dy < r2) return true;
+        }
+
+        // Check against recent history
+        for (const t of this.recentTargets) {
+            const dx = x - t.x;
+            const dy = y - t.y;
+            if (dx * dx + dy * dy < r2) return true;
+        }
+
+        return false;
+    }
+
+    /**
+     * Get next grid cell for systematic search
+     * @private
+     */
+    _getNextGridCell() {
+        // Find least recently visited cell
+        let oldest = this.gridCells[0];
+        let oldestTime = Infinity;
+
+        for (const cell of this.gridCells) {
+            if (cell.lastVisit < oldestTime) {
+                oldestTime = cell.lastVisit;
+                oldest = cell;
+            }
+        }
+
+        oldest.lastVisit = Date.now();
+        return oldest;
+    }
+
+    /**
+     * Update history with new targets
+     * @private
+     */
+    _updateHistory(targets) {
+        // Add to recent targets for inhibition of return
+        this.recentTargets.push(...targets);
+
+        // Keep only last N targets
+        const maxHistory = this.config.MOTION_HISTORY_FRAMES * this.config.MAX_SACCADES_PER_FRAME;
+        while (this.recentTargets.length > maxHistory) {
+            this.recentTargets.shift();
+        }
+
+        // Update velocity history
+        if (targets.length > 0) {
+            this.velocityHistory.push({ x: targets[0].x, y: targets[0].y });
+            while (this.velocityHistory.length > this.config.MOTION_HISTORY_FRAMES) {
+                this.velocityHistory.shift();
+            }
+            this.lastCenter = { x: targets[0].x, y: targets[0].y };
+        }
+
+        this.saccadeCount += targets.length;
+        this.lastSaccadeTime = Date.now();
+    }
+
+    /**
+     * Get the most likely location of interest based on history
+     * @returns {Object} {x, y} of predicted location
+     */
+    getPredictedLocation() {
+        if (this.velocityHistory.length >= 2) {
+            const vx = this._computeAverageVelocity('x');
+            const vy = this._computeAverageVelocity('y');
+            return {
+                x: Math.max(0, Math.min(this.width - 1, this.lastCenter.x + vx)),
+                y: Math.max(0, Math.min(this.height - 1, this.lastCenter.y + vy)),
+            };
+        }
+        return this.lastCenter;
+    }
+
+    /**
+     * Reset controller state
+     */
+    reset() {
+        this.recentTargets = [];
+        this.velocityHistory = [];
+        this.lastCenter = { x: this.width / 2, y: this.height / 2 };
+        this.saccadeCount = 0;
+
+        for (const cell of this.gridCells) {
+            cell.lastVisit = 0;
+        }
+    }
+
+    /**
+     * Update configuration
+     */
+    configure(config) {
+        this.config = { ...this.config, ...config };
+        this.inhibitionRadius = Math.min(this.width, this.height) * 0.1;
+    }
+}
+
+export { SaccadicController };

package/src/core/perception/saliency-map.js

@@ -0,0 +1,296 @@
+/**
+ * Saliency Map Computation
+ * 
+ * Computes visual saliency - regions that "pop out" and attract attention.
+ * Used to guide saccadic attention to visually important areas.
+ * 
+ * Implements a simplified Itti-Koch saliency model:
+ * - Intensity contrast
+ * - Edge density
+ * - Local complexity
+ * 
+ * For AR tracking, high-saliency regions often contain:
+ * - Corners and edges (good for feature detection)
+ * - High-contrast areas (robust to lighting changes)
+ * - Texture-rich regions (distinctive for matching)
+ */
+
+class SaliencyMap {
+    /**
+     * @param {number} width - Image width
+     * @param {number} height - Image height
+     */
+    constructor(width, height) {
+        this.width = width;
+        this.height = height;
+
+        // Downsampled dimensions for efficiency
+        this.scale = 8; // Process at 1/8 resolution
+        this.scaledW = Math.ceil(width / this.scale);
+        this.scaledH = Math.ceil(height / this.scale);
+
+        // Pre-allocate buffers
+        this.intensityMap = new Float32Array(this.scaledW * this.scaledH);
+        this.contrastMap = new Float32Array(this.scaledW * this.scaledH);
+        this.edgeMap = new Float32Array(this.scaledW * this.scaledH);
+        this.saliencyBuffer = new Float32Array(this.scaledW * this.scaledH);
+
+        // Peak detection parameters
+        this.maxPeaks = 5;
+        this.suppressionRadius = Math.max(this.scaledW, this.scaledH) * 0.15;
+    }
+
+    /**
+     * Compute saliency map for input image
+     * 
+     * @param {Uint8Array} inputData - Grayscale input image
+     * @returns {Object} Saliency result with peaks
+     */
+    compute(inputData) {
+        // Step 1: Downsample and compute intensity
+        this._downsample(inputData);
+
+        // Step 2: Compute features
+        this._computeContrast();
+        this._computeEdges();
+
+        // Step 3: Combine into saliency map
+        this._combineSaliency();
+
+        // Step 4: Find peaks
+        const peaks = this._findPeaks();
+
+        return {
+            map: this.saliencyBuffer,
+            width: this.scaledW,
+            height: this.scaledH,
+            peaks,
+            maxSaliency: peaks.length > 0 ? peaks[0].value : 0,
+        };
+    }
+
+    /**
+     * Downsample input to working resolution
+     * @private
+     */
+    _downsample(inputData) {
+        const s = this.scale;
+        const w = this.width;
+
+        for (let sy = 0; sy < this.scaledH; sy++) {
+            const yStart = sy * s;
+            const yEnd = Math.min(yStart + s, this.height);
+
+            for (let sx = 0; sx < this.scaledW; sx++) {
+                const xStart = sx * s;
+                const xEnd = Math.min(xStart + s, this.width);
+
+                let sum = 0;
+                let count = 0;
+
+                for (let y = yStart; y < yEnd; y++) {
+                    const rowOffset = y * w;
+                    for (let x = xStart; x < xEnd; x++) {
+                        sum += inputData[rowOffset + x];
+                        count++;
+                    }
+                }
+
+                this.intensityMap[sy * this.scaledW + sx] = sum / count / 255;
+            }
+        }
+    }
+
+    /**
+     * Compute local contrast map
+     * @private
+     */
+    _computeContrast() {
+        const w = this.scaledW;
+        const h = this.scaledH;
+        const intensity = this.intensityMap;
+        const contrast = this.contrastMap;
+
+        // 3x3 local contrast using center-surround
+        for (let y = 1; y < h - 1; y++) {
+            for (let x = 1; x < w - 1; x++) {
+                const idx = y * w + x;
+                const center = intensity[idx];
+
+                // Compute average of 8 neighbors
+                let surround = 0;
+                surround += intensity[(y - 1) * w + (x - 1)];
+                surround += intensity[(y - 1) * w + x];
+                surround += intensity[(y - 1) * w + (x + 1)];
+                surround += intensity[y * w + (x - 1)];
+                surround += intensity[y * w + (x + 1)];
+                surround += intensity[(y + 1) * w + (x - 1)];
+                surround += intensity[(y + 1) * w + x];
+                surround += intensity[(y + 1) * w + (x + 1)];
+                surround /= 8;
+
+                // Contrast is absolute difference
+                contrast[idx] = Math.abs(center - surround);
+            }
+        }
+
+        // Handle borders
+        for (let y = 0; y < h; y++) {
+            contrast[y * w] = 0;
+            contrast[y * w + w - 1] = 0;
+        }
+        for (let x = 0; x < w; x++) {
+            contrast[x] = 0;
+            contrast[(h - 1) * w + x] = 0;
+        }
+    }
+
+    /**
+     * Compute edge density map using Sobel-like operator
+     * @private
+     */
+    _computeEdges() {
+        const w = this.scaledW;
+        const h = this.scaledH;
+        const intensity = this.intensityMap;
+        const edges = this.edgeMap;
+
+        for (let y = 1; y < h - 1; y++) {
+            for (let x = 1; x < w - 1; x++) {
+                // Simplified Sobel
+                const gx =
+                    -intensity[(y - 1) * w + (x - 1)] + intensity[(y - 1) * w + (x + 1)] +
+                    -2 * intensity[y * w + (x - 1)] + 2 * intensity[y * w + (x + 1)] +
+                    -intensity[(y + 1) * w + (x - 1)] + intensity[(y + 1) * w + (x + 1)];
+
+                const gy =
+                    -intensity[(y - 1) * w + (x - 1)] - 2 * intensity[(y - 1) * w + x] - intensity[(y - 1) * w + (x + 1)] +
+                    intensity[(y + 1) * w + (x - 1)] + 2 * intensity[(y + 1) * w + x] + intensity[(y + 1) * w + (x + 1)];
+
+                edges[y * w + x] = Math.sqrt(gx * gx + gy * gy) / 4; // Normalize
+            }
+        }
+
+        // Handle borders
+        for (let y = 0; y < h; y++) {
+            edges[y * w] = 0;
+            edges[y * w + w - 1] = 0;
+        }
+        for (let x = 0; x < w; x++) {
+            edges[x] = 0;
+            edges[(h - 1) * w + x] = 0;
+        }
+    }
+
+    /**
+     * Combine features into final saliency map
+     * @private
+     */
+    _combineSaliency() {
+        const n = this.saliencyBuffer.length;
+        const contrast = this.contrastMap;
+        const edges = this.edgeMap;
+        const saliency = this.saliencyBuffer;
+
+        // Weight: 60% contrast, 40% edges
+        for (let i = 0; i < n; i++) {
+            saliency[i] = contrast[i] * 0.6 + edges[i] * 0.4;
+        }
+
+        // Normalize to [0, 1]
+        let max = 0;
+        for (let i = 0; i < n; i++) {
+            max = Math.max(max, saliency[i]);
+        }
+
+        if (max > 0) {
+            for (let i = 0; i < n; i++) {
+                saliency[i] /= max;
+            }
+        }
+    }
+
+    /**
+     * Find peaks in saliency map using non-maximum suppression
+     * @private
+     */
+    _findPeaks() {
+        const w = this.scaledW;
+        const h = this.scaledH;
+        const saliency = this.saliencyBuffer;
+        const peaks = [];
+        const r = this.suppressionRadius;
+        const r2 = r * r;
+
+        // Find all local maxima
+        const candidates = [];
+        for (let y = 1; y < h - 1; y++) {
+            for (let x = 1; x < w - 1; x++) {
+                const idx = y * w + x;
+                const val = saliency[idx];
+
+                // Check if local maximum (8-connected)
+                if (val > saliency[(y - 1) * w + (x - 1)] &&
+                    val > saliency[(y - 1) * w + x] &&
+                    val > saliency[(y - 1) * w + (x + 1)] &&
+                    val > saliency[y * w + (x - 1)] &&
+                    val > saliency[y * w + (x + 1)] &&
+                    val > saliency[(y + 1) * w + (x - 1)] &&
+                    val > saliency[(y + 1) * w + x] &&
+                    val > saliency[(y + 1) * w + (x + 1)]) {
+                    candidates.push({ x, y, value: val });
+                }
+            }
+        }
+
+        // Sort by value descending
+        candidates.sort((a, b) => b.value - a.value);
+
+        // Non-maximum suppression
+        for (const cand of candidates) {
+            if (peaks.length >= this.maxPeaks) break;
+
+            // Check if too close to existing peaks
+            let suppress = false;
+            for (const peak of peaks) {
+                const dx = cand.x - peak.x;
+                const dy = cand.y - peak.y;
+                if (dx * dx + dy * dy < r2) {
+                    suppress = true;
+                    break;
+                }
+            }
+
+            if (!suppress) {
+                // Convert to original image coordinates
+                peaks.push({
+                    x: (cand.x + 0.5) * this.scale,
+                    y: (cand.y + 0.5) * this.scale,
+                    value: cand.value,
+                });
+            }
+        }
+
+        return peaks;
+    }
+
+    /**
+     * Get saliency value at a specific location
+     * 
+     * @param {number} x - X coordinate in original image
+     * @param {number} y - Y coordinate in original image
+     * @returns {number} Saliency value (0-1)
+     */
+    getSaliencyAt(x, y) {
+        const sx = Math.floor(x / this.scale);
+        const sy = Math.floor(y / this.scale);
+
+        if (sx < 0 || sx >= this.scaledW || sy < 0 || sy >= this.scaledH) {
+            return 0;
+        }
+
+        return this.saliencyBuffer[sy * this.scaledW + sx];
+    }
+}
+
+export { SaliencyMap };

package/src/core/perception/scale-orchestrator.js

@@ -0,0 +1,80 @@
+/**
+ * Scale Orchestrator
+ * 
+ * Manages which octaves should be processed based on the current tracking state.
+ * Implements temporal consistency and interleave strategies to optimize performance.
+ */
+export class ScaleOrchestrator {
+    constructor(numOctaves, options = {}) {
+        this.numOctaves = numOctaves;
+        this.options = {
+            interleaveInterval: 10,
+            hysteresis: 1, // Number of adjacent octaves to keep
+            ...options
+        };
+
+        this.frameCount = 0;
+        this.lastActiveOctave = -1;
+        this.interleaveOctave = 0;
+    }
+
+    /**
+     * Determine which octaves should be processed in the current frame
+     * 
+     * @param {Object} trackingState - Current state of tracking
+     * @returns {number[]} Array of octave indices to process
+     */
+    getOctavesToProcess(trackingState = null) {
+        this.frameCount++;
+
+        // Case 1: No tracking or lost tracking -> Process all octaves
+        if (!trackingState || !trackingState.isTracking || trackingState.activeOctave === undefined) {
+            this.lastActiveOctave = -1;
+            return Array.from({ length: this.numOctaves }, (_, i) => i);
+        }
+
+        const activeScale = trackingState.activeOctave;
+        this.lastActiveOctave = activeScale;
+
+        // Case 2: Active tracking -> Focus on current scale and neighbors
+        const octaves = new Set();
+
+        // Add current and adjacent scales (Hysteresis)
+        for (let i = -this.options.hysteresis; i <= this.options.hysteresis; i++) {
+            const octave = activeScale + i;
+            if (octave >= 0 && octave < this.numOctaves) {
+                octaves.add(octave);
+            }
+        }
+
+        // Case 3: Interleave - Periodically check a distant octave to ensure we don't "drift"
+        if (this.frameCount % this.options.interleaveInterval === 0) {
+            this.interleaveOctave = (this.interleaveOctave + 1) % this.numOctaves;
+            // If the interleave octave is already being processed, pick the next one
+            if (octaves.has(this.interleaveOctave)) {
+                this.interleaveOctave = (this.interleaveOctave + 1) % this.numOctaves;
+            }
+            octaves.add(this.interleaveOctave);
+
+            if (this.options.debug) {
+                console.log(`[ScaleOrchestrator] Interleave check on octave ${this.interleaveOctave}`);
+            }
+        }
+
+        const result = Array.from(octaves).sort((a, b) => a - b);
+
+        if (this.options.debug) {
+            console.log(`[ScaleOrchestrator] Active: ${activeScale}, Processing: [${result.join(', ')}]`);
+        }
+
+        return result;
+    }
+
+    /**
+     * Reset orchestrator state
+     */
+    reset() {
+        this.frameCount = 0;
+        this.lastActiveOctave = -1;
+    }
+}