@srsergio/taptapp-ar 1.0.93 → 1.0.95

package/dist/core/perception/bio-inspired-engine.d.ts

@@ -0,0 +1,130 @@
+/**
+ * Main Bio-Inspired Perception Engine
+ * Integrates all bio-inspired components for efficient AR processing
+ */
+export class BioInspiredEngine {
+    /**
+     * @param {number} width - Input image width
+     * @param {number} height - Input image height
+     * @param {Object} options - Configuration options
+     */
+    constructor(width: number, height: number, options?: Object);
+    width: number;
+    height: number;
+    config: {
+        constructor: Function;
+        toString(): string;
+        toLocaleString(): string;
+        valueOf(): Object;
+        hasOwnProperty(v: PropertyKey): boolean;
+        isPrototypeOf(v: Object): boolean;
+        propertyIsEnumerable(v: PropertyKey): boolean;
+        FOVEA_RADIUS_RATIO: number;
+        PARAFOVEA_RADIUS_RATIO: number;
+        FOVEA_RESOLUTION: number;
+        PARAFOVEA_RESOLUTION: number;
+        PERIPHERY_RESOLUTION: number;
+        MAX_SACCADES_PER_FRAME: number;
+        SACCADE_COOLDOWN_MS: number;
+        SALIENCY_THRESHOLD: number;
+        CHANGE_THRESHOLD: number;
+        PREDICTION_CONFIDENCE: number;
+        MOTION_HISTORY_FRAMES: number;
+        ENABLE_SKIP_FRAMES: boolean;
+        MIN_PROCESSING_INTERVAL_MS: number;
+        NUM_OCTAVES: number;
+    };
+    fovealAttention: FovealAttention;
+    saccadicController: SaccadicController;
+    predictiveCoding: PredictiveCoding;
+    saliencyMap: SaliencyMap;
+    scaleOrchestrator: ScaleOrchestrator;
+    currentFoveaCenter: {
+        x: number;
+        y: number;
+    };
+    frameCount: number;
+    lastProcessTime: number;
+    skipCount: number;
+    metrics: {
+        totalFrames: number;
+        skippedFrames: number;
+        avgPixelsProcessed: number;
+        avgLatency: number;
+        saccadeCount: number;
+    };
+    /**
+     * Initialize pre-allocated buffers for efficient processing
+     * @private
+     */
+    private _initBuffers;
+    outputBuffer: {
+        fovea: Uint8Array<ArrayBuffer>;
+        parafovea: Uint8Array<ArrayBuffer>;
+        periphery: Uint8Array<ArrayBuffer>;
+    } | undefined;
+    changeBuffer: Float32Array<ArrayBuffer> | undefined;
+    /**
+     * Process an input frame using bio-inspired techniques
+     *
+     * @param {Uint8Array} inputData - Grayscale input image
+     * @param {Object} trackingState - Current tracking state (optional)
+     * @returns {Object} Processed result with attention regions
+     */
+    process(inputData: Uint8Array, trackingState?: Object): Object;
+    /**
+     * Get the primary attention region (highest resolution)
+     * This is the region that should be used for feature detection
+     *
+     * @param {Object} processResult - Result from process()
+     * @returns {Object} Primary attention region with data
+     */
+    getPrimaryRegion(processResult: Object): Object;
+    /**
+     * Suggest optimal processing based on change detection
+     *
+     * @param {Uint8Array} inputData - Current frame
+     * @returns {Object} Processing suggestion
+     */
+    suggestProcessing(inputData: Uint8Array): Object;
+    /**
+     * Update performance metrics
+     * @private
+     */
+    private _updateMetrics;
+    /**
+     * Get current performance metrics
+     * @returns {Object} Performance metrics
+     */
+    getMetrics(): Object;
+    /**
+     * Reset engine state (e.g., when target changes)
+     */
+    reset(): void;
+    /**
+     * Configure engine at runtime
+     * @param {Object} options - Configuration options to update
+     */
+    configure(options: Object): void;
+}
+export namespace BIO_CONFIG {
+    let FOVEA_RADIUS_RATIO: number;
+    let PARAFOVEA_RADIUS_RATIO: number;
+    let FOVEA_RESOLUTION: number;
+    let PARAFOVEA_RESOLUTION: number;
+    let PERIPHERY_RESOLUTION: number;
+    let MAX_SACCADES_PER_FRAME: number;
+    let SACCADE_COOLDOWN_MS: number;
+    let SALIENCY_THRESHOLD: number;
+    let CHANGE_THRESHOLD: number;
+    let PREDICTION_CONFIDENCE: number;
+    let MOTION_HISTORY_FRAMES: number;
+    let ENABLE_SKIP_FRAMES: boolean;
+    let MIN_PROCESSING_INTERVAL_MS: number;
+    let NUM_OCTAVES: number;
+}
+import { FovealAttention } from './foveal-attention.js';
+import { SaccadicController } from './saccadic-controller.js';
+import { PredictiveCoding } from './predictive-coding.js';
+import { SaliencyMap } from './saliency-map.js';
+import { ScaleOrchestrator } from './scale-orchestrator.js';

package/dist/core/perception/bio-inspired-engine.js

@@ -0,0 +1,232 @@
+/**
+ * Bio-Inspired Perception Engine
+ *
+ * Inspired by human visual system:
+ * - Foveal attention: High resolution in center, low in periphery
+ * - Saccadic sampling: Strategic "glances" at areas of interest
+ * - Predictive coding: Only process what's unexpected/changed
+ *
+ * Expected improvements:
+ * - ~75% reduction in pixels processed per frame
+ * - ~80% reduction in latency for static scenes
+ * - ~70% reduction in energy consumption
+ */
+import { FovealAttention } from './foveal-attention.js';
+import { SaccadicController } from './saccadic-controller.js';
+import { PredictiveCoding } from './predictive-coding.js';
+import { SaliencyMap } from './saliency-map.js';
+import { ScaleOrchestrator } from './scale-orchestrator.js';
+/**
+ * Configuration for Bio-Inspired Engine
+ */
+const BIO_CONFIG = {
+    // Foveal region (high resolution center)
+    FOVEA_RADIUS_RATIO: 0.15, // 15% of image dimension
+    PARAFOVEA_RADIUS_RATIO: 0.30, // 30% of image dimension
+    // Resolution multipliers
+    FOVEA_RESOLUTION: 1.0, // Full resolution
+    PARAFOVEA_RESOLUTION: 0.5, // Half resolution
+    PERIPHERY_RESOLUTION: 0.25, // Quarter resolution
+    // Saccadic behavior
+    MAX_SACCADES_PER_FRAME: 3, // Maximum "glances" per frame
+    SACCADE_COOLDOWN_MS: 50, // Minimum time between saccades
+    SALIENCY_THRESHOLD: 0.3, // Threshold for triggering saccade
+    // Predictive coding
+    CHANGE_THRESHOLD: 0.05, // 5% pixel difference to trigger processing
+    PREDICTION_CONFIDENCE: 0.8, // Confidence to skip processing
+    MOTION_HISTORY_FRAMES: 3, // Frames to consider for motion prediction
+    // Performance
+    ENABLE_SKIP_FRAMES: true, // Skip processing if nothing changed
+    MIN_PROCESSING_INTERVAL_MS: 8, // Minimum 8ms (~120fps cap)
+    NUM_OCTAVES: 5, // Default number of octaves
+};
+/**
+ * Main Bio-Inspired Perception Engine
+ * Integrates all bio-inspired components for efficient AR processing
+ */
+class BioInspiredEngine {
+    /**
+     * @param {number} width - Input image width
+     * @param {number} height - Input image height
+     * @param {Object} options - Configuration options
+     */
+    constructor(width, height, options = {}) {
+        this.width = width;
+        this.height = height;
+        this.config = { ...BIO_CONFIG, ...options };
+        // Initialize sub-components
+        this.fovealAttention = new FovealAttention(width, height, this.config);
+        this.saccadicController = new SaccadicController(width, height, this.config);
+        this.predictiveCoding = new PredictiveCoding(width, height, this.config);
+        this.saliencyMap = new SaliencyMap(width, height);
+        this.scaleOrchestrator = new ScaleOrchestrator(this.config.NUM_OCTAVES, {
+            debug: options.debugMode
+        });
+        // State tracking
+        this.currentFoveaCenter = { x: width / 2, y: height / 2 };
+        this.frameCount = 0;
+        this.lastProcessTime = 0;
+        this.skipCount = 0;
+        // Performance metrics
+        this.metrics = {
+            totalFrames: 0,
+            skippedFrames: 0,
+            avgPixelsProcessed: 0,
+            avgLatency: 0,
+            saccadeCount: 0,
+        };
+        // Pre-allocate buffers
+        this._initBuffers();
+    }
+    /**
+     * Initialize pre-allocated buffers for efficient processing
+     * @private
+     */
+    _initBuffers() {
+        const fullSize = this.width * this.height;
+        const foveaSize = Math.ceil(fullSize * this.config.FOVEA_RADIUS_RATIO ** 2 * Math.PI);
+        // Multi-resolution output buffer
+        this.outputBuffer = {
+            fovea: new Uint8Array(foveaSize),
+            parafovea: new Uint8Array(Math.ceil(foveaSize * 4)),
+            periphery: new Uint8Array(Math.ceil(fullSize * 0.25)),
+        };
+        // Change detection buffer
+        this.changeBuffer = new Float32Array(Math.ceil(fullSize / 64)); // 8x8 blocks
+    }
+    /**
+     * Process an input frame using bio-inspired techniques
+     *
+     * @param {Uint8Array} inputData - Grayscale input image
+     * @param {Object} trackingState - Current tracking state (optional)
+     * @returns {Object} Processed result with attention regions
+     */
+    process(inputData, trackingState = null) {
+        const startTime = performance.now();
+        this.frameCount++;
+        this.metrics.totalFrames++;
+        // Step 1: Predictive Coding - Check if we can skip processing
+        const prediction = this.predictiveCoding.predict(inputData, trackingState);
+        if (prediction.canSkip && this.config.ENABLE_SKIP_FRAMES) {
+            this.metrics.skippedFrames++;
+            this.skipCount++;
+            return {
+                skipped: true,
+                prediction: prediction.predictedState,
+                confidence: prediction.confidence,
+                pixelsProcessed: 0,
+                latency: performance.now() - startTime,
+            };
+        }
+        this.skipCount = 0;
+        // Step 2: Compute Saliency Map for attention guidance
+        const saliency = this.saliencyMap.compute(inputData);
+        // Step 3: Saccadic Controller - Decide where to "look"
+        const saccadeTargets = this.saccadicController.computeTargets(saliency, this.currentFoveaCenter, trackingState);
+        // Step 4: Extract foveal regions at different resolutions
+        const attentionRegions = [];
+        let totalPixelsProcessed = 0;
+        for (const target of saccadeTargets) {
+            const region = this.fovealAttention.extract(inputData, target.x, target.y, target.priority);
+            attentionRegions.push(region);
+            totalPixelsProcessed += region.pixelCount;
+            this.metrics.saccadeCount++;
+        }
+        // Step 5: Update fovea center based on highest priority target
+        if (saccadeTargets.length > 0) {
+            const primary = saccadeTargets[0];
+            this.currentFoveaCenter = { x: primary.x, y: primary.y };
+        }
+        // Step 6: Scale Orchestrator - Determine octaves to process
+        const octavesToProcess = this.scaleOrchestrator.getOctavesToProcess(trackingState);
+        // Step 7: Store frame for prediction
+        this.predictiveCoding.storeFrame(inputData, trackingState);
+        // Compute metrics
+        const latency = performance.now() - startTime;
+        this._updateMetrics(totalPixelsProcessed, latency);
+        return {
+            skipped: false,
+            attentionRegions,
+            foveaCenter: this.currentFoveaCenter,
+            saliencyPeaks: saliency.peaks,
+            octavesToProcess,
+            pixelsProcessed: totalPixelsProcessed,
+            pixelsSaved: this.width * this.height - totalPixelsProcessed,
+            savingsPercent: ((1 - totalPixelsProcessed / (this.width * this.height)) * 100).toFixed(1),
+            latency,
+        };
+    }
+    /**
+     * Get the primary attention region (highest resolution)
+     * This is the region that should be used for feature detection
+     *
+     * @param {Object} processResult - Result from process()
+     * @returns {Object} Primary attention region with data
+     */
+    getPrimaryRegion(processResult) {
+        if (processResult.skipped || !processResult.attentionRegions?.length) {
+            return null;
+        }
+        return processResult.attentionRegions[0];
+    }
+    /**
+     * Suggest optimal processing based on change detection
+     *
+     * @param {Uint8Array} inputData - Current frame
+     * @returns {Object} Processing suggestion
+     */
+    suggestProcessing(inputData) {
+        const changeLevel = this.predictiveCoding.getChangeLevel(inputData);
+        return {
+            shouldProcessFull: changeLevel > 0.3,
+            shouldProcessPartial: changeLevel > 0.05,
+            canSkip: changeLevel < 0.02,
+            changeLevel,
+            recommendedSaccades: Math.ceil(changeLevel * this.config.MAX_SACCADES_PER_FRAME),
+        };
+    }
+    /**
+     * Update performance metrics
+     * @private
+     */
+    _updateMetrics(pixelsProcessed, latency) {
+        const alpha = 0.1; // Exponential moving average factor
+        this.metrics.avgPixelsProcessed =
+            this.metrics.avgPixelsProcessed * (1 - alpha) + pixelsProcessed * alpha;
+        this.metrics.avgLatency =
+            this.metrics.avgLatency * (1 - alpha) + latency * alpha;
+    }
+    /**
+     * Get current performance metrics
+     * @returns {Object} Performance metrics
+     */
+    getMetrics() {
+        return {
+            ...this.metrics,
+            skipRate: ((this.metrics.skippedFrames / this.metrics.totalFrames) * 100).toFixed(1) + '%',
+            avgSavings: ((1 - this.metrics.avgPixelsProcessed / (this.width * this.height)) * 100).toFixed(1) + '%',
+            currentFovea: this.currentFoveaCenter,
+        };
+    }
+    /**
+     * Reset engine state (e.g., when target changes)
+     */
+    reset() {
+        this.currentFoveaCenter = { x: this.width / 2, y: this.height / 2 };
+        this.frameCount = 0;
+        this.skipCount = 0;
+        this.predictiveCoding.reset();
+        this.saccadicController.reset();
+    }
+    /**
+     * Configure engine at runtime
+     * @param {Object} options - Configuration options to update
+     */
+    configure(options) {
+        this.config = { ...this.config, ...options };
+        this.fovealAttention.configure(this.config);
+        this.saccadicController.configure(this.config);
+        this.predictiveCoding.configure(this.config);
+    }
+}
+export { BioInspiredEngine, BIO_CONFIG };

package/dist/core/perception/foveal-attention.d.ts

@@ -0,0 +1,142 @@
+/**
+ * A region extracted at a specific resolution
+ */
+export type AttentionRegion = {
+    /**
+     * - Center X coordinate in original image
+     */
+    x: number;
+    /**
+     * - Center Y coordinate in original image
+     */
+    y: number;
+    /**
+     * - Radius in original image pixels
+     */
+    radius: number;
+    /**
+     * - Resolution multiplier (1.0 = full)
+     */
+    resolution: number;
+    /**
+     * - Extracted pixel data
+     */
+    data: Uint8Array;
+    /**
+     * - Width of extracted region
+     */
+    width: number;
+    /**
+     * - Height of extracted region
+     */
+    height: number;
+    /**
+     * - Number of pixels in region
+     */
+    pixelCount: number;
+    /**
+     * - 'fovea' | 'parafovea' | 'periphery'
+     */
+    type: string;
+};
+/**
+ * Foveal Attention System
+ *
+ * Mimics the human eye's fovea-parafovea-periphery structure:
+ * - Fovea (center 5°): Maximum resolution, ~50% of visual processing power
+ * - Parafovea (5-10°): Medium resolution, pattern recognition
+ * - Periphery (>10°): Low resolution, motion detection
+ *
+ * This allows processing ~75% fewer pixels while maintaining
+ * high-quality tracking in the area of interest.
+ */
+/**
+ * A region extracted at a specific resolution
+ * @typedef {Object} AttentionRegion
+ * @property {number} x - Center X coordinate in original image
+ * @property {number} y - Center Y coordinate in original image
+ * @property {number} radius - Radius in original image pixels
+ * @property {number} resolution - Resolution multiplier (1.0 = full)
+ * @property {Uint8Array} data - Extracted pixel data
+ * @property {number} width - Width of extracted region
+ * @property {number} height - Height of extracted region
+ * @property {number} pixelCount - Number of pixels in region
+ * @property {string} type - 'fovea' | 'parafovea' | 'periphery'
+ */
+export class FovealAttention {
+    /**
+     * @param {number} width - Input image width
+     * @param {number} height - Input image height
+     * @param {Object} config - Configuration
+     */
+    constructor(width: number, height: number, config: Object);
+    width: number;
+    height: number;
+    config: Object;
+    minDim: number;
+    foveaRadius: number;
+    parafoveaRadius: number;
+    /**
+     * Initialize pre-allocated extraction buffers
+     * @private
+     */
+    private _initBuffers;
+    foveaBuffer: Uint8Array<ArrayBuffer> | undefined;
+    parafoveaBuffer: Uint8Array<ArrayBuffer> | undefined;
+    peripheryBuffer: Uint8Array<ArrayBuffer> | undefined;
+    peripheryDims: {
+        width: number;
+        height: number;
+    } | undefined;
+    /**
+     * Build a circular mask for foveal extraction
+     * @private
+     */
+    private _buildCircularMask;
+    circularMask: Uint8Array<ArrayBuffer> | undefined;
+    /**
+     * Extract attention region at specified center
+     *
+     * @param {Uint8Array} inputData - Grayscale input image
+     * @param {number} centerX - X coordinate of attention center
+     * @param {number} centerY - Y coordinate of attention center
+     * @param {number} priority - Priority level (0=highest)
+     * @returns {AttentionRegion} Extracted region
+     */
+    extract(inputData: Uint8Array, centerX: number, centerY: number, priority?: number): AttentionRegion;
+    /**
+     * Extract foveal region at full resolution
+     * @private
+     */
+    private _extractFovea;
+    /**
+     * Extract parafoveal region at half resolution
+     * @private
+     */
+    private _extractParafovea;
+    /**
+     * Extract periphery at quarter resolution (motion detection only)
+     * @private
+     */
+    private _extractPeriphery;
+    /**
+     * Get combined multi-resolution representation
+     * Uses fovea at center, parafovea around it, periphery for the rest
+     *
+     * @param {Uint8Array} inputData - Input image
+     * @param {number} cx - Fovea center X
+     * @param {number} cy - Fovea center Y
+     * @returns {Object} Multi-resolution representation
+     */
+    extractMultiResolution(inputData: Uint8Array, cx: number, cy: number): Object;
+    /**
+     * Compute total pixels in multi-resolution representation
+     * @private
+     */
+    private _computeTotalPixels;
+    /**
+     * Update configuration
+     * @param {Object} config - New configuration
+     */
+    configure(config: Object): void;
+}