@srsergio/taptapp-ar 1.1.1 → 1.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/compiler/node-worker.js +1 -197
- package/dist/compiler/offline-compiler.js +1 -207
- package/dist/core/constants.js +1 -38
- package/dist/core/detector/crop-detector.js +1 -88
- package/dist/core/detector/detector-lite.js +1 -455
- package/dist/core/detector/freak.js +1 -89
- package/dist/core/estimation/estimate.js +1 -16
- package/dist/core/estimation/estimator.js +1 -30
- package/dist/core/estimation/morph-refinement.js +1 -116
- package/dist/core/estimation/non-rigid-refine.js +1 -70
- package/dist/core/estimation/pnp-solver.js +1 -109
- package/dist/core/estimation/refine-estimate.js +1 -311
- package/dist/core/estimation/utils.js +1 -67
- package/dist/core/features/auto-rotation-feature.js +1 -30
- package/dist/core/features/crop-detection-feature.js +1 -26
- package/dist/core/features/feature-base.js +1 -1
- package/dist/core/features/feature-manager.js +1 -55
- package/dist/core/features/one-euro-filter-feature.js +1 -44
- package/dist/core/features/temporal-filter-feature.js +1 -57
- package/dist/core/image-list.js +1 -54
- package/dist/core/input-loader.js +1 -87
- package/dist/core/matching/hamming-distance.js +1 -66
- package/dist/core/matching/hdc.js +1 -102
- package/dist/core/matching/hierarchical-clustering.js +1 -130
- package/dist/core/matching/hough.js +1 -170
- package/dist/core/matching/matcher.js +1 -66
- package/dist/core/matching/matching.js +1 -401
- package/dist/core/matching/ransacHomography.js +1 -132
- package/dist/core/perception/bio-inspired-engine.js +1 -232
- package/dist/core/perception/foveal-attention.js +1 -280
- package/dist/core/perception/index.js +1 -17
- package/dist/core/perception/predictive-coding.js +1 -278
- package/dist/core/perception/saccadic-controller.js +1 -269
- package/dist/core/perception/saliency-map.js +1 -254
- package/dist/core/perception/scale-orchestrator.js +1 -68
- package/dist/core/protocol.js +1 -254
- package/dist/core/tracker/extract-utils.js +1 -29
- package/dist/core/tracker/extract.js +1 -306
- package/dist/core/tracker/tracker.js +1 -352
- package/dist/core/utils/cumsum.js +1 -37
- package/dist/core/utils/delaunay.js +1 -125
- package/dist/core/utils/geometry.js +1 -101
- package/dist/core/utils/gpu-compute.js +1 -231
- package/dist/core/utils/homography.js +1 -138
- package/dist/core/utils/images.js +1 -108
- package/dist/core/utils/lsh-binarizer.js +1 -37
- package/dist/core/utils/lsh-direct.js +1 -76
- package/dist/core/utils/projection.js +1 -51
- package/dist/core/utils/randomizer.js +1 -25
- package/dist/core/utils/worker-pool.js +1 -89
- package/dist/index.js +1 -7
- package/dist/libs/one-euro-filter.js +1 -70
- package/dist/react/TaptappAR.js +1 -151
- package/dist/react/types.js +1 -16
- package/dist/react/use-ar.js +1 -118
- package/dist/runtime/aframe.js +1 -272
- package/dist/runtime/bio-inspired-controller.js +1 -358
- package/dist/runtime/controller.js +1 -592
- package/dist/runtime/controller.worker.js +1 -93
- package/dist/runtime/index.js +1 -5
- package/dist/runtime/three.js +1 -304
- package/dist/runtime/track.js +1 -381
- package/package.json +10 -4
|
@@ -1,232 +1 @@
|
|
|
1
|
-
|
|
2
|
-
* Bio-Inspired Perception Engine
|
|
3
|
-
*
|
|
4
|
-
* Inspired by human visual system:
|
|
5
|
-
* - Foveal attention: High resolution in center, low in periphery
|
|
6
|
-
* - Saccadic sampling: Strategic "glances" at areas of interest
|
|
7
|
-
* - Predictive coding: Only process what's unexpected/changed
|
|
8
|
-
*
|
|
9
|
-
* Expected improvements:
|
|
10
|
-
* - ~75% reduction in pixels processed per frame
|
|
11
|
-
* - ~80% reduction in latency for static scenes
|
|
12
|
-
* - ~70% reduction in energy consumption
|
|
13
|
-
*/
|
|
14
|
-
import { FovealAttention } from './foveal-attention.js';
|
|
15
|
-
import { SaccadicController } from './saccadic-controller.js';
|
|
16
|
-
import { PredictiveCoding } from './predictive-coding.js';
|
|
17
|
-
import { SaliencyMap } from './saliency-map.js';
|
|
18
|
-
import { ScaleOrchestrator } from './scale-orchestrator.js';
|
|
19
|
-
/**
|
|
20
|
-
* Configuration for Bio-Inspired Engine
|
|
21
|
-
*/
|
|
22
|
-
const BIO_CONFIG = {
|
|
23
|
-
// Foveal region (high resolution center)
|
|
24
|
-
FOVEA_RADIUS_RATIO: 0.15, // 15% of image dimension
|
|
25
|
-
PARAFOVEA_RADIUS_RATIO: 0.30, // 30% of image dimension
|
|
26
|
-
// Resolution multipliers
|
|
27
|
-
FOVEA_RESOLUTION: 1.0, // Full resolution
|
|
28
|
-
PARAFOVEA_RESOLUTION: 0.5, // Half resolution
|
|
29
|
-
PERIPHERY_RESOLUTION: 0.25, // Quarter resolution
|
|
30
|
-
// Saccadic behavior
|
|
31
|
-
MAX_SACCADES_PER_FRAME: 3, // Maximum "glances" per frame
|
|
32
|
-
SACCADE_COOLDOWN_MS: 50, // Minimum time between saccades
|
|
33
|
-
SALIENCY_THRESHOLD: 0.3, // Threshold for triggering saccade
|
|
34
|
-
// Predictive coding
|
|
35
|
-
CHANGE_THRESHOLD: 0.05, // 5% pixel difference to trigger processing
|
|
36
|
-
PREDICTION_CONFIDENCE: 0.8, // Confidence to skip processing
|
|
37
|
-
MOTION_HISTORY_FRAMES: 3, // Frames to consider for motion prediction
|
|
38
|
-
// Performance
|
|
39
|
-
ENABLE_SKIP_FRAMES: true, // Skip processing if nothing changed
|
|
40
|
-
MIN_PROCESSING_INTERVAL_MS: 8, // Minimum 8ms (~120fps cap)
|
|
41
|
-
NUM_OCTAVES: 5, // Default number of octaves
|
|
42
|
-
};
|
|
43
|
-
/**
|
|
44
|
-
* Main Bio-Inspired Perception Engine
|
|
45
|
-
* Integrates all bio-inspired components for efficient AR processing
|
|
46
|
-
*/
|
|
47
|
-
class BioInspiredEngine {
|
|
48
|
-
/**
|
|
49
|
-
* @param {number} width - Input image width
|
|
50
|
-
* @param {number} height - Input image height
|
|
51
|
-
* @param {Object} options - Configuration options
|
|
52
|
-
*/
|
|
53
|
-
constructor(width, height, options = {}) {
|
|
54
|
-
this.width = width;
|
|
55
|
-
this.height = height;
|
|
56
|
-
this.config = { ...BIO_CONFIG, ...options };
|
|
57
|
-
// Initialize sub-components
|
|
58
|
-
this.fovealAttention = new FovealAttention(width, height, this.config);
|
|
59
|
-
this.saccadicController = new SaccadicController(width, height, this.config);
|
|
60
|
-
this.predictiveCoding = new PredictiveCoding(width, height, this.config);
|
|
61
|
-
this.saliencyMap = new SaliencyMap(width, height);
|
|
62
|
-
this.scaleOrchestrator = new ScaleOrchestrator(this.config.NUM_OCTAVES, {
|
|
63
|
-
debug: options.debugMode
|
|
64
|
-
});
|
|
65
|
-
// State tracking
|
|
66
|
-
this.currentFoveaCenter = { x: width / 2, y: height / 2 };
|
|
67
|
-
this.frameCount = 0;
|
|
68
|
-
this.lastProcessTime = 0;
|
|
69
|
-
this.skipCount = 0;
|
|
70
|
-
// Performance metrics
|
|
71
|
-
this.metrics = {
|
|
72
|
-
totalFrames: 0,
|
|
73
|
-
skippedFrames: 0,
|
|
74
|
-
avgPixelsProcessed: 0,
|
|
75
|
-
avgLatency: 0,
|
|
76
|
-
saccadeCount: 0,
|
|
77
|
-
};
|
|
78
|
-
// Pre-allocate buffers
|
|
79
|
-
this._initBuffers();
|
|
80
|
-
}
|
|
81
|
-
/**
|
|
82
|
-
* Initialize pre-allocated buffers for efficient processing
|
|
83
|
-
* @private
|
|
84
|
-
*/
|
|
85
|
-
_initBuffers() {
|
|
86
|
-
const fullSize = this.width * this.height;
|
|
87
|
-
const foveaSize = Math.ceil(fullSize * this.config.FOVEA_RADIUS_RATIO ** 2 * Math.PI);
|
|
88
|
-
// Multi-resolution output buffer
|
|
89
|
-
this.outputBuffer = {
|
|
90
|
-
fovea: new Uint8Array(foveaSize),
|
|
91
|
-
parafovea: new Uint8Array(Math.ceil(foveaSize * 4)),
|
|
92
|
-
periphery: new Uint8Array(Math.ceil(fullSize * 0.25)),
|
|
93
|
-
};
|
|
94
|
-
// Change detection buffer
|
|
95
|
-
this.changeBuffer = new Float32Array(Math.ceil(fullSize / 64)); // 8x8 blocks
|
|
96
|
-
}
|
|
97
|
-
/**
|
|
98
|
-
* Process an input frame using bio-inspired techniques
|
|
99
|
-
*
|
|
100
|
-
* @param {Uint8Array} inputData - Grayscale input image
|
|
101
|
-
* @param {Object} trackingState - Current tracking state (optional)
|
|
102
|
-
* @returns {Object} Processed result with attention regions
|
|
103
|
-
*/
|
|
104
|
-
process(inputData, trackingState = null) {
|
|
105
|
-
const startTime = performance.now();
|
|
106
|
-
this.frameCount++;
|
|
107
|
-
this.metrics.totalFrames++;
|
|
108
|
-
// Step 1: Predictive Coding - Check if we can skip processing
|
|
109
|
-
const prediction = this.predictiveCoding.predict(inputData, trackingState);
|
|
110
|
-
if (prediction.canSkip && this.config.ENABLE_SKIP_FRAMES) {
|
|
111
|
-
this.metrics.skippedFrames++;
|
|
112
|
-
this.skipCount++;
|
|
113
|
-
return {
|
|
114
|
-
skipped: true,
|
|
115
|
-
prediction: prediction.predictedState,
|
|
116
|
-
confidence: prediction.confidence,
|
|
117
|
-
pixelsProcessed: 0,
|
|
118
|
-
latency: performance.now() - startTime,
|
|
119
|
-
};
|
|
120
|
-
}
|
|
121
|
-
this.skipCount = 0;
|
|
122
|
-
// Step 2: Compute Saliency Map for attention guidance
|
|
123
|
-
const saliency = this.saliencyMap.compute(inputData);
|
|
124
|
-
// Step 3: Saccadic Controller - Decide where to "look"
|
|
125
|
-
const saccadeTargets = this.saccadicController.computeTargets(saliency, this.currentFoveaCenter, trackingState);
|
|
126
|
-
// Step 4: Extract foveal regions at different resolutions
|
|
127
|
-
const attentionRegions = [];
|
|
128
|
-
let totalPixelsProcessed = 0;
|
|
129
|
-
for (const target of saccadeTargets) {
|
|
130
|
-
const region = this.fovealAttention.extract(inputData, target.x, target.y, target.priority);
|
|
131
|
-
attentionRegions.push(region);
|
|
132
|
-
totalPixelsProcessed += region.pixelCount;
|
|
133
|
-
this.metrics.saccadeCount++;
|
|
134
|
-
}
|
|
135
|
-
// Step 5: Update fovea center based on highest priority target
|
|
136
|
-
if (saccadeTargets.length > 0) {
|
|
137
|
-
const primary = saccadeTargets[0];
|
|
138
|
-
this.currentFoveaCenter = { x: primary.x, y: primary.y };
|
|
139
|
-
}
|
|
140
|
-
// Step 6: Scale Orchestrator - Determine octaves to process
|
|
141
|
-
const octavesToProcess = this.scaleOrchestrator.getOctavesToProcess(trackingState);
|
|
142
|
-
// Step 7: Store frame for prediction
|
|
143
|
-
this.predictiveCoding.storeFrame(inputData, trackingState);
|
|
144
|
-
// Compute metrics
|
|
145
|
-
const latency = performance.now() - startTime;
|
|
146
|
-
this._updateMetrics(totalPixelsProcessed, latency);
|
|
147
|
-
return {
|
|
148
|
-
skipped: false,
|
|
149
|
-
attentionRegions,
|
|
150
|
-
foveaCenter: this.currentFoveaCenter,
|
|
151
|
-
saliencyPeaks: saliency.peaks,
|
|
152
|
-
octavesToProcess,
|
|
153
|
-
pixelsProcessed: totalPixelsProcessed,
|
|
154
|
-
pixelsSaved: this.width * this.height - totalPixelsProcessed,
|
|
155
|
-
savingsPercent: ((1 - totalPixelsProcessed / (this.width * this.height)) * 100).toFixed(1),
|
|
156
|
-
latency,
|
|
157
|
-
};
|
|
158
|
-
}
|
|
159
|
-
/**
|
|
160
|
-
* Get the primary attention region (highest resolution)
|
|
161
|
-
* This is the region that should be used for feature detection
|
|
162
|
-
*
|
|
163
|
-
* @param {Object} processResult - Result from process()
|
|
164
|
-
* @returns {Object} Primary attention region with data
|
|
165
|
-
*/
|
|
166
|
-
getPrimaryRegion(processResult) {
|
|
167
|
-
if (processResult.skipped || !processResult.attentionRegions?.length) {
|
|
168
|
-
return null;
|
|
169
|
-
}
|
|
170
|
-
return processResult.attentionRegions[0];
|
|
171
|
-
}
|
|
172
|
-
/**
|
|
173
|
-
* Suggest optimal processing based on change detection
|
|
174
|
-
*
|
|
175
|
-
* @param {Uint8Array} inputData - Current frame
|
|
176
|
-
* @returns {Object} Processing suggestion
|
|
177
|
-
*/
|
|
178
|
-
suggestProcessing(inputData) {
|
|
179
|
-
const changeLevel = this.predictiveCoding.getChangeLevel(inputData);
|
|
180
|
-
return {
|
|
181
|
-
shouldProcessFull: changeLevel > 0.3,
|
|
182
|
-
shouldProcessPartial: changeLevel > 0.05,
|
|
183
|
-
canSkip: changeLevel < 0.02,
|
|
184
|
-
changeLevel,
|
|
185
|
-
recommendedSaccades: Math.ceil(changeLevel * this.config.MAX_SACCADES_PER_FRAME),
|
|
186
|
-
};
|
|
187
|
-
}
|
|
188
|
-
/**
|
|
189
|
-
* Update performance metrics
|
|
190
|
-
* @private
|
|
191
|
-
*/
|
|
192
|
-
_updateMetrics(pixelsProcessed, latency) {
|
|
193
|
-
const alpha = 0.1; // Exponential moving average factor
|
|
194
|
-
this.metrics.avgPixelsProcessed =
|
|
195
|
-
this.metrics.avgPixelsProcessed * (1 - alpha) + pixelsProcessed * alpha;
|
|
196
|
-
this.metrics.avgLatency =
|
|
197
|
-
this.metrics.avgLatency * (1 - alpha) + latency * alpha;
|
|
198
|
-
}
|
|
199
|
-
/**
|
|
200
|
-
* Get current performance metrics
|
|
201
|
-
* @returns {Object} Performance metrics
|
|
202
|
-
*/
|
|
203
|
-
getMetrics() {
|
|
204
|
-
return {
|
|
205
|
-
...this.metrics,
|
|
206
|
-
skipRate: ((this.metrics.skippedFrames / this.metrics.totalFrames) * 100).toFixed(1) + '%',
|
|
207
|
-
avgSavings: ((1 - this.metrics.avgPixelsProcessed / (this.width * this.height)) * 100).toFixed(1) + '%',
|
|
208
|
-
currentFovea: this.currentFoveaCenter,
|
|
209
|
-
};
|
|
210
|
-
}
|
|
211
|
-
/**
|
|
212
|
-
* Reset engine state (e.g., when target changes)
|
|
213
|
-
*/
|
|
214
|
-
reset() {
|
|
215
|
-
this.currentFoveaCenter = { x: this.width / 2, y: this.height / 2 };
|
|
216
|
-
this.frameCount = 0;
|
|
217
|
-
this.skipCount = 0;
|
|
218
|
-
this.predictiveCoding.reset();
|
|
219
|
-
this.saccadicController.reset();
|
|
220
|
-
}
|
|
221
|
-
/**
|
|
222
|
-
* Configure engine at runtime
|
|
223
|
-
* @param {Object} options - Configuration options to update
|
|
224
|
-
*/
|
|
225
|
-
configure(options) {
|
|
226
|
-
this.config = { ...this.config, ...options };
|
|
227
|
-
this.fovealAttention.configure(this.config);
|
|
228
|
-
this.saccadicController.configure(this.config);
|
|
229
|
-
this.predictiveCoding.configure(this.config);
|
|
230
|
-
}
|
|
231
|
-
}
|
|
232
|
-
export { BioInspiredEngine, BIO_CONFIG };
|
|
1
|
+
import{FovealAttention as t}from"./foveal-attention.js";import{SaccadicController as e}from"./saccadic-controller.js";import{PredictiveCoding as i}from"./predictive-coding.js";import{SaliencyMap as s}from"./saliency-map.js";import{ScaleOrchestrator as r}from"./scale-orchestrator.js";const o={FOVEA_RADIUS_RATIO:.15,PARAFOVEA_RADIUS_RATIO:.3,FOVEA_RESOLUTION:1,PARAFOVEA_RESOLUTION:.5,PERIPHERY_RESOLUTION:.25,MAX_SACCADES_PER_FRAME:3,SACCADE_COOLDOWN_MS:50,SALIENCY_THRESHOLD:.3,CHANGE_THRESHOLD:.05,PREDICTION_CONFIDENCE:.8,MOTION_HISTORY_FRAMES:3,ENABLE_SKIP_FRAMES:!0,MIN_PROCESSING_INTERVAL_MS:8,NUM_OCTAVES:5};class c{constructor(c,n,a={}){this.width=c,this.height=n,this.config={...o,...a},this.fovealAttention=new t(c,n,this.config),this.saccadicController=new e(c,n,this.config),this.predictiveCoding=new i(c,n,this.config),this.saliencyMap=new s(c,n),this.scaleOrchestrator=new r(this.config.NUM_OCTAVES,{debug:a.debugMode}),this.currentFoveaCenter={x:c/2,y:n/2},this.frameCount=0,this.lastProcessTime=0,this.skipCount=0,this.metrics={totalFrames:0,skippedFrames:0,avgPixelsProcessed:0,avgLatency:0,saccadeCount:0},this._initBuffers()}_initBuffers(){const t=this.width*this.height,e=Math.ceil(t*this.config.FOVEA_RADIUS_RATIO**2*Math.PI);this.outputBuffer={fovea:new Uint8Array(e),parafovea:new Uint8Array(Math.ceil(4*e)),periphery:new Uint8Array(Math.ceil(.25*t))},this.changeBuffer=new Float32Array(Math.ceil(t/64))}process(t,e=null){const i=performance.now();this.frameCount++,this.metrics.totalFrames++;const s=this.predictiveCoding.predict(t,e);if(s.canSkip&&this.config.ENABLE_SKIP_FRAMES)return this.metrics.skippedFrames++,this.skipCount++,{skipped:!0,prediction:s.predictedState,confidence:s.confidence,pixelsProcessed:0,latency:performance.now()-i};this.skipCount=0;const r=this.saliencyMap.compute(t),o=this.saccadicController.computeTargets(r,this.currentFoveaCenter,e),c=[];let n=0;for(const e of o){const i=this.fovealAttention.extract(t,e.x,e.y,e.priority);c.push(i),n+=i.pixelCount,this.metrics.saccadeCount++}if(o.length>0){const t=o[0];this.currentFoveaCenter={x:t.x,y:t.y}}const a=this.scaleOrchestrator.getOctavesToProcess(e);this.predictiveCoding.storeFrame(t,e);const h=performance.now()-i;return this._updateMetrics(n,h),{skipped:!1,attentionRegions:c,foveaCenter:this.currentFoveaCenter,saliencyPeaks:r.peaks,octavesToProcess:a,pixelsProcessed:n,pixelsSaved:this.width*this.height-n,savingsPercent:(100*(1-n/(this.width*this.height))).toFixed(1),latency:h}}getPrimaryRegion(t){return t.skipped||!t.attentionRegions?.length?null:t.attentionRegions[0]}suggestProcessing(t){const e=this.predictiveCoding.getChangeLevel(t);return{shouldProcessFull:e>.3,shouldProcessPartial:e>.05,canSkip:e<.02,changeLevel:e,recommendedSaccades:Math.ceil(e*this.config.MAX_SACCADES_PER_FRAME)}}_updateMetrics(t,e){this.metrics.avgPixelsProcessed=.9*this.metrics.avgPixelsProcessed+.1*t,this.metrics.avgLatency=.9*this.metrics.avgLatency+.1*e}getMetrics(){return{...this.metrics,skipRate:(this.metrics.skippedFrames/this.metrics.totalFrames*100).toFixed(1)+"%",avgSavings:(100*(1-this.metrics.avgPixelsProcessed/(this.width*this.height))).toFixed(1)+"%",currentFovea:this.currentFoveaCenter}}reset(){this.currentFoveaCenter={x:this.width/2,y:this.height/2},this.frameCount=0,this.skipCount=0,this.predictiveCoding.reset(),this.saccadicController.reset()}configure(t){this.config={...this.config,...t},this.fovealAttention.configure(this.config),this.saccadicController.configure(this.config),this.predictiveCoding.configure(this.config)}}export{c as BioInspiredEngine,o as BIO_CONFIG};
|
|
@@ -1,280 +1 @@
|
|
|
1
|
-
|
|
2
|
-
* Foveal Attention System
|
|
3
|
-
*
|
|
4
|
-
* Mimics the human eye's fovea-parafovea-periphery structure:
|
|
5
|
-
* - Fovea (center 5°): Maximum resolution, ~50% of visual processing power
|
|
6
|
-
* - Parafovea (5-10°): Medium resolution, pattern recognition
|
|
7
|
-
* - Periphery (>10°): Low resolution, motion detection
|
|
8
|
-
*
|
|
9
|
-
* This allows processing ~75% fewer pixels while maintaining
|
|
10
|
-
* high-quality tracking in the area of interest.
|
|
11
|
-
*/
|
|
12
|
-
/**
|
|
13
|
-
* A region extracted at a specific resolution
|
|
14
|
-
* @typedef {Object} AttentionRegion
|
|
15
|
-
* @property {number} x - Center X coordinate in original image
|
|
16
|
-
* @property {number} y - Center Y coordinate in original image
|
|
17
|
-
* @property {number} radius - Radius in original image pixels
|
|
18
|
-
* @property {number} resolution - Resolution multiplier (1.0 = full)
|
|
19
|
-
* @property {Uint8Array} data - Extracted pixel data
|
|
20
|
-
* @property {number} width - Width of extracted region
|
|
21
|
-
* @property {number} height - Height of extracted region
|
|
22
|
-
* @property {number} pixelCount - Number of pixels in region
|
|
23
|
-
* @property {string} type - 'fovea' | 'parafovea' | 'periphery'
|
|
24
|
-
*/
|
|
25
|
-
class FovealAttention {
|
|
26
|
-
/**
|
|
27
|
-
* @param {number} width - Input image width
|
|
28
|
-
* @param {number} height - Input image height
|
|
29
|
-
* @param {Object} config - Configuration
|
|
30
|
-
*/
|
|
31
|
-
constructor(width, height, config) {
|
|
32
|
-
this.width = width;
|
|
33
|
-
this.height = height;
|
|
34
|
-
this.config = config;
|
|
35
|
-
// Calculate region sizes
|
|
36
|
-
this.minDim = Math.min(width, height);
|
|
37
|
-
this.foveaRadius = Math.floor(this.minDim * config.FOVEA_RADIUS_RATIO);
|
|
38
|
-
this.parafoveaRadius = Math.floor(this.minDim * config.PARAFOVEA_RADIUS_RATIO);
|
|
39
|
-
// Pre-allocate buffers for each region type
|
|
40
|
-
this._initBuffers();
|
|
41
|
-
}
|
|
42
|
-
/**
|
|
43
|
-
* Initialize pre-allocated extraction buffers
|
|
44
|
-
* @private
|
|
45
|
-
*/
|
|
46
|
-
_initBuffers() {
|
|
47
|
-
// Fovea buffer (full resolution, circular region)
|
|
48
|
-
const foveaDiam = this.foveaRadius * 2;
|
|
49
|
-
this.foveaBuffer = new Uint8Array(foveaDiam * foveaDiam);
|
|
50
|
-
// Parafovea buffer (half resolution)
|
|
51
|
-
const parafoveaDiam = this.parafoveaRadius * 2;
|
|
52
|
-
const parafoveaScaled = Math.ceil(parafoveaDiam * this.config.PARAFOVEA_RESOLUTION);
|
|
53
|
-
this.parafoveaBuffer = new Uint8Array(parafoveaScaled * parafoveaScaled);
|
|
54
|
-
// Periphery buffer (quarter resolution, full image)
|
|
55
|
-
const periphW = Math.ceil(this.width * this.config.PERIPHERY_RESOLUTION);
|
|
56
|
-
const periphH = Math.ceil(this.height * this.config.PERIPHERY_RESOLUTION);
|
|
57
|
-
this.peripheryBuffer = new Uint8Array(periphW * periphH);
|
|
58
|
-
this.peripheryDims = { width: periphW, height: periphH };
|
|
59
|
-
// Mask for circular extraction (reusable)
|
|
60
|
-
this._buildCircularMask();
|
|
61
|
-
}
|
|
62
|
-
/**
|
|
63
|
-
* Build a circular mask for foveal extraction
|
|
64
|
-
* @private
|
|
65
|
-
*/
|
|
66
|
-
_buildCircularMask() {
|
|
67
|
-
const r = this.foveaRadius;
|
|
68
|
-
const size = r * 2;
|
|
69
|
-
this.circularMask = new Uint8Array(size * size);
|
|
70
|
-
for (let y = 0; y < size; y++) {
|
|
71
|
-
for (let x = 0; x < size; x++) {
|
|
72
|
-
const dx = x - r;
|
|
73
|
-
const dy = y - r;
|
|
74
|
-
const dist = Math.sqrt(dx * dx + dy * dy);
|
|
75
|
-
this.circularMask[y * size + x] = dist <= r ? 1 : 0;
|
|
76
|
-
}
|
|
77
|
-
}
|
|
78
|
-
}
|
|
79
|
-
/**
|
|
80
|
-
* Extract attention region at specified center
|
|
81
|
-
*
|
|
82
|
-
* @param {Uint8Array} inputData - Grayscale input image
|
|
83
|
-
* @param {number} centerX - X coordinate of attention center
|
|
84
|
-
* @param {number} centerY - Y coordinate of attention center
|
|
85
|
-
* @param {number} priority - Priority level (0=highest)
|
|
86
|
-
* @returns {AttentionRegion} Extracted region
|
|
87
|
-
*/
|
|
88
|
-
extract(inputData, centerX, centerY, priority = 0) {
|
|
89
|
-
// Clamp center to valid range
|
|
90
|
-
centerX = Math.max(this.foveaRadius, Math.min(this.width - this.foveaRadius - 1, centerX));
|
|
91
|
-
centerY = Math.max(this.foveaRadius, Math.min(this.height - this.foveaRadius - 1, centerY));
|
|
92
|
-
// Priority 0 = full foveal extraction
|
|
93
|
-
// Priority 1 = parafoveal only
|
|
94
|
-
// Priority 2+ = periphery glimpse
|
|
95
|
-
if (priority === 0) {
|
|
96
|
-
return this._extractFovea(inputData, centerX, centerY);
|
|
97
|
-
}
|
|
98
|
-
else if (priority === 1) {
|
|
99
|
-
return this._extractParafovea(inputData, centerX, centerY);
|
|
100
|
-
}
|
|
101
|
-
else {
|
|
102
|
-
return this._extractPeriphery(inputData);
|
|
103
|
-
}
|
|
104
|
-
}
|
|
105
|
-
/**
|
|
106
|
-
* Extract foveal region at full resolution
|
|
107
|
-
* @private
|
|
108
|
-
*/
|
|
109
|
-
_extractFovea(inputData, cx, cy) {
|
|
110
|
-
const r = this.foveaRadius;
|
|
111
|
-
const diam = r * 2;
|
|
112
|
-
const buffer = this.foveaBuffer;
|
|
113
|
-
let idx = 0;
|
|
114
|
-
let validPixels = 0;
|
|
115
|
-
for (let dy = -r; dy < r; dy++) {
|
|
116
|
-
const y = cy + dy;
|
|
117
|
-
const rowStart = y * this.width;
|
|
118
|
-
for (let dx = -r; dx < r; dx++) {
|
|
119
|
-
const maskIdx = (dy + r) * diam + (dx + r);
|
|
120
|
-
if (this.circularMask[maskIdx]) {
|
|
121
|
-
const x = cx + dx;
|
|
122
|
-
buffer[idx] = inputData[rowStart + x];
|
|
123
|
-
validPixels++;
|
|
124
|
-
}
|
|
125
|
-
else {
|
|
126
|
-
buffer[idx] = 0;
|
|
127
|
-
}
|
|
128
|
-
idx++;
|
|
129
|
-
}
|
|
130
|
-
}
|
|
131
|
-
return {
|
|
132
|
-
x: cx,
|
|
133
|
-
y: cy,
|
|
134
|
-
radius: r,
|
|
135
|
-
resolution: this.config.FOVEA_RESOLUTION,
|
|
136
|
-
data: buffer,
|
|
137
|
-
width: diam,
|
|
138
|
-
height: diam,
|
|
139
|
-
pixelCount: validPixels,
|
|
140
|
-
type: 'fovea',
|
|
141
|
-
// Transform helpers
|
|
142
|
-
toOriginalCoord: (localX, localY) => ({
|
|
143
|
-
x: cx - r + localX,
|
|
144
|
-
y: cy - r + localY,
|
|
145
|
-
}),
|
|
146
|
-
toLocalCoord: (origX, origY) => ({
|
|
147
|
-
x: origX - (cx - r),
|
|
148
|
-
y: origY - (cy - r),
|
|
149
|
-
}),
|
|
150
|
-
};
|
|
151
|
-
}
|
|
152
|
-
/**
|
|
153
|
-
* Extract parafoveal region at half resolution
|
|
154
|
-
* @private
|
|
155
|
-
*/
|
|
156
|
-
_extractParafovea(inputData, cx, cy) {
|
|
157
|
-
const r = this.parafoveaRadius;
|
|
158
|
-
const res = this.config.PARAFOVEA_RESOLUTION;
|
|
159
|
-
const scaledR = Math.ceil(r * res);
|
|
160
|
-
const scaledDiam = scaledR * 2;
|
|
161
|
-
const buffer = this.parafoveaBuffer;
|
|
162
|
-
const step = Math.round(1 / res);
|
|
163
|
-
let idx = 0;
|
|
164
|
-
let validPixels = 0;
|
|
165
|
-
for (let sy = 0; sy < scaledDiam; sy++) {
|
|
166
|
-
const y = cy - r + Math.floor(sy / res);
|
|
167
|
-
if (y < 0 || y >= this.height)
|
|
168
|
-
continue;
|
|
169
|
-
const rowStart = y * this.width;
|
|
170
|
-
for (let sx = 0; sx < scaledDiam; sx++) {
|
|
171
|
-
const x = cx - r + Math.floor(sx / res);
|
|
172
|
-
if (x < 0 || x >= this.width) {
|
|
173
|
-
buffer[idx++] = 0;
|
|
174
|
-
continue;
|
|
175
|
-
}
|
|
176
|
-
// Sample with bilinear interpolation for smoother downscaling
|
|
177
|
-
buffer[idx++] = inputData[rowStart + x];
|
|
178
|
-
validPixels++;
|
|
179
|
-
}
|
|
180
|
-
}
|
|
181
|
-
return {
|
|
182
|
-
x: cx,
|
|
183
|
-
y: cy,
|
|
184
|
-
radius: r,
|
|
185
|
-
resolution: res,
|
|
186
|
-
data: buffer,
|
|
187
|
-
width: scaledDiam,
|
|
188
|
-
height: scaledDiam,
|
|
189
|
-
pixelCount: validPixels,
|
|
190
|
-
type: 'parafovea',
|
|
191
|
-
toOriginalCoord: (localX, localY) => ({
|
|
192
|
-
x: cx - r + localX / res,
|
|
193
|
-
y: cy - r + localY / res,
|
|
194
|
-
}),
|
|
195
|
-
toLocalCoord: (origX, origY) => ({
|
|
196
|
-
x: (origX - (cx - r)) * res,
|
|
197
|
-
y: (origY - (cy - r)) * res,
|
|
198
|
-
}),
|
|
199
|
-
};
|
|
200
|
-
}
|
|
201
|
-
/**
|
|
202
|
-
* Extract periphery at quarter resolution (motion detection only)
|
|
203
|
-
* @private
|
|
204
|
-
*/
|
|
205
|
-
_extractPeriphery(inputData) {
|
|
206
|
-
const res = this.config.PERIPHERY_RESOLUTION;
|
|
207
|
-
const outW = this.peripheryDims.width;
|
|
208
|
-
const outH = this.peripheryDims.height;
|
|
209
|
-
const buffer = this.peripheryBuffer;
|
|
210
|
-
const step = Math.round(1 / res);
|
|
211
|
-
let idx = 0;
|
|
212
|
-
for (let y = 0; y < this.height; y += step) {
|
|
213
|
-
const rowStart = y * this.width;
|
|
214
|
-
for (let x = 0; x < this.width; x += step) {
|
|
215
|
-
if (idx < buffer.length) {
|
|
216
|
-
buffer[idx++] = inputData[rowStart + x];
|
|
217
|
-
}
|
|
218
|
-
}
|
|
219
|
-
}
|
|
220
|
-
return {
|
|
221
|
-
x: this.width / 2,
|
|
222
|
-
y: this.height / 2,
|
|
223
|
-
radius: Math.max(this.width, this.height) / 2,
|
|
224
|
-
resolution: res,
|
|
225
|
-
data: buffer,
|
|
226
|
-
width: outW,
|
|
227
|
-
height: outH,
|
|
228
|
-
pixelCount: outW * outH,
|
|
229
|
-
type: 'periphery',
|
|
230
|
-
toOriginalCoord: (localX, localY) => ({
|
|
231
|
-
x: localX / res,
|
|
232
|
-
y: localY / res,
|
|
233
|
-
}),
|
|
234
|
-
toLocalCoord: (origX, origY) => ({
|
|
235
|
-
x: origX * res,
|
|
236
|
-
y: origY * res,
|
|
237
|
-
}),
|
|
238
|
-
};
|
|
239
|
-
}
|
|
240
|
-
/**
|
|
241
|
-
* Get combined multi-resolution representation
|
|
242
|
-
* Uses fovea at center, parafovea around it, periphery for the rest
|
|
243
|
-
*
|
|
244
|
-
* @param {Uint8Array} inputData - Input image
|
|
245
|
-
* @param {number} cx - Fovea center X
|
|
246
|
-
* @param {number} cy - Fovea center Y
|
|
247
|
-
* @returns {Object} Multi-resolution representation
|
|
248
|
-
*/
|
|
249
|
-
extractMultiResolution(inputData, cx, cy) {
|
|
250
|
-
return {
|
|
251
|
-
fovea: this._extractFovea(inputData, cx, cy),
|
|
252
|
-
parafovea: this._extractParafovea(inputData, cx, cy),
|
|
253
|
-
periphery: this._extractPeriphery(inputData),
|
|
254
|
-
center: { x: cx, y: cy },
|
|
255
|
-
totalPixels: this._computeTotalPixels(),
|
|
256
|
-
originalPixels: this.width * this.height,
|
|
257
|
-
};
|
|
258
|
-
}
|
|
259
|
-
/**
|
|
260
|
-
* Compute total pixels in multi-resolution representation
|
|
261
|
-
* @private
|
|
262
|
-
*/
|
|
263
|
-
_computeTotalPixels() {
|
|
264
|
-
const foveaPixels = Math.PI * this.foveaRadius ** 2;
|
|
265
|
-
const parafoveaPixels = Math.PI * this.parafoveaRadius ** 2 * this.config.PARAFOVEA_RESOLUTION ** 2;
|
|
266
|
-
const peripheryPixels = this.peripheryDims.width * this.peripheryDims.height;
|
|
267
|
-
return Math.ceil(foveaPixels + parafoveaPixels + peripheryPixels);
|
|
268
|
-
}
|
|
269
|
-
/**
|
|
270
|
-
* Update configuration
|
|
271
|
-
* @param {Object} config - New configuration
|
|
272
|
-
*/
|
|
273
|
-
configure(config) {
|
|
274
|
-
this.config = { ...this.config, ...config };
|
|
275
|
-
this.foveaRadius = Math.floor(this.minDim * config.FOVEA_RADIUS_RATIO);
|
|
276
|
-
this.parafoveaRadius = Math.floor(this.minDim * config.PARAFOVEA_RADIUS_RATIO);
|
|
277
|
-
this._initBuffers();
|
|
278
|
-
}
|
|
279
|
-
}
|
|
280
|
-
export { FovealAttention };
|
|
1
|
+
class t{constructor(t,i,h){this.width=t,this.height=i,this.config=h,this.minDim=Math.min(t,i),this.foveaRadius=Math.floor(this.minDim*h.FOVEA_RADIUS_RATIO),this.parafoveaRadius=Math.floor(this.minDim*h.PARAFOVEA_RADIUS_RATIO),this._initBuffers()}_initBuffers(){const t=2*this.foveaRadius;this.foveaBuffer=new Uint8Array(t*t);const i=2*this.parafoveaRadius,h=Math.ceil(i*this.config.PARAFOVEA_RESOLUTION);this.parafoveaBuffer=new Uint8Array(h*h);const a=Math.ceil(this.width*this.config.PERIPHERY_RESOLUTION),e=Math.ceil(this.height*this.config.PERIPHERY_RESOLUTION);this.peripheryBuffer=new Uint8Array(a*e),this.peripheryDims={width:a,height:e},this._buildCircularMask()}_buildCircularMask(){const t=this.foveaRadius,i=2*t;this.circularMask=new Uint8Array(i*i);for(let h=0;h<i;h++)for(let a=0;a<i;a++){const e=a-t,s=h-t,r=Math.sqrt(e*e+s*s);this.circularMask[h*i+a]=r<=t?1:0}}extract(t,i,h,a=0){return i=Math.max(this.foveaRadius,Math.min(this.width-this.foveaRadius-1,i)),h=Math.max(this.foveaRadius,Math.min(this.height-this.foveaRadius-1,h)),0===a?this._extractFovea(t,i,h):1===a?this._extractParafovea(t,i,h):this._extractPeriphery(t)}_extractFovea(t,i,h){const a=this.foveaRadius,e=2*a,s=this.foveaBuffer;let r=0,o=0;for(let f=-a;f<a;f++){const n=(h+f)*this.width;for(let h=-a;h<a;h++){const c=(f+a)*e+(h+a);if(this.circularMask[c]){const a=i+h;s[r]=t[n+a],o++}else s[r]=0;r++}}return{x:i,y:h,radius:a,resolution:this.config.FOVEA_RESOLUTION,data:s,width:e,height:e,pixelCount:o,type:"fovea",toOriginalCoord:(t,e)=>({x:i-a+t,y:h-a+e}),toLocalCoord:(t,e)=>({x:t-(i-a),y:e-(h-a)})}}_extractParafovea(t,i,h){const a=this.parafoveaRadius,e=this.config.PARAFOVEA_RESOLUTION,s=2*Math.ceil(a*e),r=this.parafoveaBuffer;Math.round(1/e);let o=0,f=0;for(let n=0;n<s;n++){const c=h-a+Math.floor(n/e);if(c<0||c>=this.height)continue;const u=c*this.width;for(let h=0;h<s;h++){const s=i-a+Math.floor(h/e);s<0||s>=this.width?r[o++]=0:(r[o++]=t[u+s],f++)}}return{x:i,y:h,radius:a,resolution:e,data:r,width:s,height:s,pixelCount:f,type:"parafovea",toOriginalCoord:(t,s)=>({x:i-a+t/e,y:h-a+s/e}),toLocalCoord:(t,s)=>({x:(t-(i-a))*e,y:(s-(h-a))*e})}}_extractPeriphery(t){const i=this.config.PERIPHERY_RESOLUTION,h=this.peripheryDims.width,a=this.peripheryDims.height,e=this.peripheryBuffer,s=Math.round(1/i);let r=0;for(let i=0;i<this.height;i+=s){const h=i*this.width;for(let i=0;i<this.width;i+=s)r<e.length&&(e[r++]=t[h+i])}return{x:this.width/2,y:this.height/2,radius:Math.max(this.width,this.height)/2,resolution:i,data:e,width:h,height:a,pixelCount:h*a,type:"periphery",toOriginalCoord:(t,h)=>({x:t/i,y:h/i}),toLocalCoord:(t,h)=>({x:t*i,y:h*i})}}extractMultiResolution(t,i,h){return{fovea:this._extractFovea(t,i,h),parafovea:this._extractParafovea(t,i,h),periphery:this._extractPeriphery(t),center:{x:i,y:h},totalPixels:this._computeTotalPixels(),originalPixels:this.width*this.height}}_computeTotalPixels(){const t=Math.PI*this.foveaRadius**2,i=Math.PI*this.parafoveaRadius**2*this.config.PARAFOVEA_RESOLUTION**2,h=this.peripheryDims.width*this.peripheryDims.height;return Math.ceil(t+i+h)}configure(t){this.config={...this.config,...t},this.foveaRadius=Math.floor(this.minDim*t.FOVEA_RADIUS_RATIO),this.parafoveaRadius=Math.floor(this.minDim*t.PARAFOVEA_RADIUS_RATIO),this._initBuffers()}}export{t as FovealAttention};
|
|
@@ -1,17 +1 @@
|
|
|
1
|
-
|
|
2
|
-
* Bio-Inspired Perception Module
|
|
3
|
-
*
|
|
4
|
-
* Human visual system-inspired components for efficient AR processing.
|
|
5
|
-
* Expected improvements over traditional frame-based processing:
|
|
6
|
-
*
|
|
7
|
-
* - ~75% reduction in pixels processed per frame (foveal attention)
|
|
8
|
-
* - ~80% reduction in latency for static scenes (predictive coding)
|
|
9
|
-
* - ~70% reduction in energy consumption
|
|
10
|
-
* - Maintains tracking accuracy through strategic attention allocation
|
|
11
|
-
*/
|
|
12
|
-
export { BioInspiredEngine, BIO_CONFIG } from './bio-inspired-engine.js';
|
|
13
|
-
export { FovealAttention } from './foveal-attention.js';
|
|
14
|
-
export { SaccadicController } from './saccadic-controller.js';
|
|
15
|
-
export { PredictiveCoding } from './predictive-coding.js';
|
|
16
|
-
export { SaliencyMap } from './saliency-map.js';
|
|
17
|
-
export { ScaleOrchestrator } from './scale-orchestrator.js';
|
|
1
|
+
export{BioInspiredEngine,BIO_CONFIG}from"./bio-inspired-engine.js";export{FovealAttention}from"./foveal-attention.js";export{SaccadicController}from"./saccadic-controller.js";export{PredictiveCoding}from"./predictive-coding.js";export{SaliencyMap}from"./saliency-map.js";export{ScaleOrchestrator}from"./scale-orchestrator.js";
|