@srsergio/taptapp-ar 1.0.92 → 1.0.94
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -14
- package/dist/compiler/offline-compiler.d.ts +3 -3
- package/dist/compiler/offline-compiler.js +50 -33
- package/dist/core/constants.d.ts +2 -0
- package/dist/core/constants.js +4 -1
- package/dist/core/detector/detector-lite.d.ts +6 -5
- package/dist/core/detector/detector-lite.js +46 -16
- package/dist/core/image-list.d.ts +24 -6
- package/dist/core/image-list.js +4 -4
- package/dist/core/matching/matcher.d.ts +1 -1
- package/dist/core/matching/matcher.js +7 -4
- package/dist/core/matching/matching.d.ts +2 -1
- package/dist/core/matching/matching.js +43 -11
- package/dist/core/perception/bio-inspired-engine.d.ts +130 -0
- package/dist/core/perception/bio-inspired-engine.js +232 -0
- package/dist/core/perception/foveal-attention.d.ts +142 -0
- package/dist/core/perception/foveal-attention.js +280 -0
- package/dist/core/perception/index.d.ts +6 -0
- package/dist/core/perception/index.js +17 -0
- package/dist/core/perception/predictive-coding.d.ts +92 -0
- package/dist/core/perception/predictive-coding.js +278 -0
- package/dist/core/perception/saccadic-controller.d.ts +126 -0
- package/dist/core/perception/saccadic-controller.js +269 -0
- package/dist/core/perception/saliency-map.d.ts +74 -0
- package/dist/core/perception/saliency-map.js +254 -0
- package/dist/core/perception/scale-orchestrator.d.ts +28 -0
- package/dist/core/perception/scale-orchestrator.js +68 -0
- package/dist/core/protocol.d.ts +14 -1
- package/dist/core/protocol.js +33 -1
- package/dist/runtime/bio-inspired-controller.d.ts +135 -0
- package/dist/runtime/bio-inspired-controller.js +358 -0
- package/dist/runtime/controller.d.ts +11 -2
- package/dist/runtime/controller.js +20 -8
- package/dist/runtime/controller.worker.js +2 -2
- package/package.json +1 -1
- package/src/compiler/offline-compiler.ts +56 -36
- package/src/core/constants.ts +5 -1
- package/src/core/detector/detector-lite.js +46 -16
- package/src/core/image-list.js +4 -4
- package/src/core/matching/matcher.js +8 -4
- package/src/core/matching/matching.js +51 -12
- package/src/core/perception/bio-inspired-engine.js +275 -0
- package/src/core/perception/foveal-attention.js +306 -0
- package/src/core/perception/index.js +18 -0
- package/src/core/perception/predictive-coding.js +327 -0
- package/src/core/perception/saccadic-controller.js +303 -0
- package/src/core/perception/saliency-map.js +296 -0
- package/src/core/perception/scale-orchestrator.js +80 -0
- package/src/core/protocol.ts +38 -1
- package/src/runtime/bio-inspired-controller.ts +448 -0
- package/src/runtime/controller.ts +22 -7
- package/src/runtime/controller.worker.js +2 -1
|
@@ -66,9 +66,11 @@ export class DetectorLite {
|
|
|
66
66
|
/**
|
|
67
67
|
* Detecta características en una imagen en escala de grises
|
|
68
68
|
* @param {Float32Array|Uint8Array} imageData - Datos de imagen (width * height)
|
|
69
|
+
* @param {Object} options - Opciones de detección (ej. octavesToProcess)
|
|
69
70
|
* @returns {{featurePoints: Array}} Puntos de características detectados
|
|
70
71
|
*/
|
|
71
|
-
detect(imageData) {
|
|
72
|
+
detect(imageData, options = {}) {
|
|
73
|
+
const octavesToProcess = options.octavesToProcess || Array.from({ length: this.numOctaves }, (_, i) => i);
|
|
72
74
|
// Normalizar a Float32Array si es necesario
|
|
73
75
|
let data;
|
|
74
76
|
if (imageData instanceof Float32Array) {
|
|
@@ -80,11 +82,11 @@ export class DetectorLite {
|
|
|
80
82
|
}
|
|
81
83
|
}
|
|
82
84
|
|
|
83
|
-
// 1. Construir pirámide gaussiana
|
|
84
|
-
const pyramidImages = this._buildGaussianPyramid(data, this.width, this.height);
|
|
85
|
+
// 1. Construir pirámide gaussiana (solo octavas solicitadas)
|
|
86
|
+
const pyramidImages = this._buildGaussianPyramid(data, this.width, this.height, octavesToProcess);
|
|
85
87
|
|
|
86
88
|
// 2. Construir pirámide DoG (Difference of Gaussians)
|
|
87
|
-
const dogPyramid = this._buildDogPyramid(pyramidImages);
|
|
89
|
+
const dogPyramid = this._buildDogPyramid(pyramidImages, octavesToProcess);
|
|
88
90
|
|
|
89
91
|
// 3. Encontrar extremos locales
|
|
90
92
|
const extremas = this._findExtremas(dogPyramid, pyramidImages);
|
|
@@ -121,7 +123,7 @@ export class DetectorLite {
|
|
|
121
123
|
/**
|
|
122
124
|
* Construye una pirámide gaussiana
|
|
123
125
|
*/
|
|
124
|
-
_buildGaussianPyramid(data, width, height) {
|
|
126
|
+
_buildGaussianPyramid(data, width, height, octavesToProcess = null) {
|
|
125
127
|
// Use GPU-accelerated pyramid if available
|
|
126
128
|
if (this.useGPU) {
|
|
127
129
|
try {
|
|
@@ -130,6 +132,10 @@ export class DetectorLite {
|
|
|
130
132
|
// Convert GPU pyramid format to expected format
|
|
131
133
|
const pyramid = [];
|
|
132
134
|
for (let i = 0; i < gpuPyramid.length && i < this.numOctaves; i++) {
|
|
135
|
+
if (octavesToProcess && !octavesToProcess.includes(i)) {
|
|
136
|
+
pyramid.push(null);
|
|
137
|
+
continue;
|
|
138
|
+
}
|
|
133
139
|
const level = gpuPyramid[i];
|
|
134
140
|
// Apply second blur for DoG computation
|
|
135
141
|
const img2 = this._applyGaussianFilter(level.data, level.width, level.height);
|
|
@@ -156,19 +162,37 @@ export class DetectorLite {
|
|
|
156
162
|
let currentHeight = height;
|
|
157
163
|
|
|
158
164
|
for (let i = 0; i < this.numOctaves; i++) {
|
|
159
|
-
const
|
|
160
|
-
|
|
165
|
+
const shouldProcess = !octavesToProcess || octavesToProcess.includes(i);
|
|
166
|
+
|
|
167
|
+
if (shouldProcess) {
|
|
168
|
+
const img1 = this._applyGaussianFilter(currentData, currentWidth, currentHeight);
|
|
169
|
+
const img2 = this._applyGaussianFilter(img1.data, currentWidth, currentHeight);
|
|
161
170
|
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
171
|
+
pyramid.push([
|
|
172
|
+
{ data: img1.data, width: currentWidth, height: currentHeight },
|
|
173
|
+
{ data: img2.data, width: currentWidth, height: currentHeight }
|
|
174
|
+
]);
|
|
175
|
+
} else {
|
|
176
|
+
pyramid.push(null);
|
|
177
|
+
}
|
|
166
178
|
|
|
167
179
|
if (i < this.numOctaves - 1) {
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
180
|
+
// For CPU downsampling, we STILL need to downsample even if we skip processing the current octave
|
|
181
|
+
// UNLESS the next octave is also skipped. But for simplicity and safety, we downsample if needed by ANY future octave.
|
|
182
|
+
const needsDownsample = !octavesToProcess || octavesToProcess.some(o => o > i);
|
|
183
|
+
|
|
184
|
+
if (needsDownsample) {
|
|
185
|
+
// If current octave was processed, we use img1.data (or original data if i=0 and not processed?).
|
|
186
|
+
// Wait, standard is to downsample from the blurred image of previous octave.
|
|
187
|
+
const sourceData = shouldProcess ? pyramid[i][0].data : currentData;
|
|
188
|
+
const downsampled = this._downsample(sourceData, currentWidth, currentHeight);
|
|
189
|
+
currentData = downsampled.data;
|
|
190
|
+
currentWidth = downsampled.width;
|
|
191
|
+
currentHeight = downsampled.height;
|
|
192
|
+
} else {
|
|
193
|
+
// Optimization: if no more octaves are needed, we can stop here
|
|
194
|
+
break;
|
|
195
|
+
}
|
|
172
196
|
}
|
|
173
197
|
}
|
|
174
198
|
|
|
@@ -251,10 +275,14 @@ export class DetectorLite {
|
|
|
251
275
|
/**
|
|
252
276
|
* Construye pirámide de diferencia de gaussianas
|
|
253
277
|
*/
|
|
254
|
-
_buildDogPyramid(pyramidImages) {
|
|
278
|
+
_buildDogPyramid(pyramidImages, octavesToProcess = null) {
|
|
255
279
|
const dogPyramid = [];
|
|
256
280
|
|
|
257
281
|
for (let i = 0; i < pyramidImages.length; i++) {
|
|
282
|
+
if (!pyramidImages[i]) {
|
|
283
|
+
dogPyramid.push(null);
|
|
284
|
+
continue;
|
|
285
|
+
}
|
|
258
286
|
const img1 = pyramidImages[i][0];
|
|
259
287
|
const img2 = pyramidImages[i][1];
|
|
260
288
|
const width = img1.width;
|
|
@@ -279,6 +307,8 @@ export class DetectorLite {
|
|
|
279
307
|
|
|
280
308
|
for (let octave = 0; octave < dogPyramid.length; octave++) {
|
|
281
309
|
const curr = dogPyramid[octave];
|
|
310
|
+
if (!curr) continue;
|
|
311
|
+
|
|
282
312
|
const prev = octave > 0 ? dogPyramid[octave - 1] : null;
|
|
283
313
|
const next = octave < dogPyramid.length - 1 ? dogPyramid[octave + 1] : null;
|
|
284
314
|
|
package/src/core/image-list.js
CHANGED
|
@@ -12,8 +12,8 @@ const MIN_IMAGE_PIXEL_SIZE = AR_CONFIG.MIN_IMAGE_PIXEL_SIZE;
|
|
|
12
12
|
|
|
13
13
|
/**
|
|
14
14
|
* Construye una lista de imágenes con diferentes escalas para detección de características
|
|
15
|
-
* @param {
|
|
16
|
-
* @returns {Array<
|
|
15
|
+
* @param {{width: number, height: number, data: any}} inputImage - Imagen de entrada con propiedades width, height y data
|
|
16
|
+
* @returns {Array<{data: Uint8Array, width: number, height: number, scale: number}>} Lista de imágenes escaladas con propiedades data, width, height y scale
|
|
17
17
|
*/
|
|
18
18
|
const buildImageList = (inputImage) => {
|
|
19
19
|
const minScale = MIN_IMAGE_PIXEL_SIZE / Math.min(inputImage.width, inputImage.height);
|
|
@@ -45,8 +45,8 @@ const buildImageList = (inputImage) => {
|
|
|
45
45
|
/**
|
|
46
46
|
* Construye una lista optimizada de imágenes para tracking
|
|
47
47
|
* Genera dos versiones escaladas (256px y 128px) para tracking eficiente
|
|
48
|
-
* @param {
|
|
49
|
-
* @returns {Array<
|
|
48
|
+
* @param {{width: number, height: number, data: any}} inputImage - Imagen de entrada con propiedades width, height y data
|
|
49
|
+
* @returns {Array<{data: Uint8Array, width: number, height: number, scale: number}>} Lista de imágenes escaladas para tracking
|
|
50
50
|
*/
|
|
51
51
|
const buildTrackingImageList = (inputImage) => {
|
|
52
52
|
const minDimension = Math.min(inputImage.width, inputImage.height);
|
|
@@ -7,7 +7,7 @@ class Matcher {
|
|
|
7
7
|
this.debugMode = debugMode;
|
|
8
8
|
}
|
|
9
9
|
|
|
10
|
-
matchDetection(keyframes, featurePoints) {
|
|
10
|
+
matchDetection(keyframes, featurePoints, expectedScale) {
|
|
11
11
|
let debugExtra = { frames: [] };
|
|
12
12
|
let bestResult = null;
|
|
13
13
|
|
|
@@ -27,6 +27,7 @@ class Matcher {
|
|
|
27
27
|
querywidth: this.queryWidth,
|
|
28
28
|
queryheight: this.queryHeight,
|
|
29
29
|
debugMode: this.debugMode,
|
|
30
|
+
expectedScale,
|
|
30
31
|
});
|
|
31
32
|
|
|
32
33
|
if (frameDebugExtra) {
|
|
@@ -48,18 +49,21 @@ class Matcher {
|
|
|
48
49
|
const screenCoords = [];
|
|
49
50
|
const worldCoords = [];
|
|
50
51
|
const keyframe = keyframes[bestResult.keyframeIndex];
|
|
51
|
-
const
|
|
52
|
+
const kfScale = keyframe.s || keyframe.scale || 1.0;
|
|
52
53
|
|
|
53
54
|
for (let i = 0; i < bestResult.matches.length; i++) {
|
|
54
55
|
const querypoint = bestResult.matches[i].querypoint;
|
|
55
56
|
const keypoint = bestResult.matches[i].keypoint;
|
|
57
|
+
// 🚀 NANITE-STYLE: Use per-keypoint scale (octave) for accurate world mapping
|
|
58
|
+
const pointScale = keypoint.scale || kfScale;
|
|
59
|
+
|
|
56
60
|
screenCoords.push({
|
|
57
61
|
x: querypoint.x,
|
|
58
62
|
y: querypoint.y,
|
|
59
63
|
});
|
|
60
64
|
worldCoords.push({
|
|
61
|
-
x: (keypoint.x + 0.5) /
|
|
62
|
-
y: (keypoint.y + 0.5) /
|
|
65
|
+
x: (keypoint.x + 0.5) / kfScale,
|
|
66
|
+
y: (keypoint.y + 0.5) / kfScale,
|
|
63
67
|
z: 0,
|
|
64
68
|
});
|
|
65
69
|
}
|
|
@@ -15,7 +15,7 @@ const HDC_RATIO_THRESHOLD = AR_CONFIG.HDC_RATIO_THRESHOLD;
|
|
|
15
15
|
const MAX_MATCH_QUERY_POINTS = AR_CONFIG.MAX_MATCH_QUERY_POINTS;
|
|
16
16
|
|
|
17
17
|
// match list of querpoints against pre-built list of keyframes
|
|
18
|
-
const match = ({ keyframe, querypoints: rawQuerypoints, querywidth, queryheight, debugMode }) => {
|
|
18
|
+
const match = ({ keyframe, querypoints: rawQuerypoints, querywidth, queryheight, debugMode, expectedScale }) => {
|
|
19
19
|
let debugExtra = {};
|
|
20
20
|
|
|
21
21
|
// 🎯 Performance Optimizer: Use only the most "salient" points (highest response)
|
|
@@ -27,12 +27,16 @@ const match = ({ keyframe, querypoints: rawQuerypoints, querywidth, queryheight,
|
|
|
27
27
|
const qlen = querypoints.length;
|
|
28
28
|
const kmax = keyframe.max;
|
|
29
29
|
const kmin = keyframe.min;
|
|
30
|
+
|
|
31
|
+
// Detect descriptor mode: HDC (32-bit signature), Compact (32-bit XOR folded), or Raw (64-bit)
|
|
30
32
|
const isHDC = keyframe.hdc === true || (kmax && kmax.hdc === 1);
|
|
31
|
-
const
|
|
33
|
+
const isCompact = (kmax && kmax.compact === 1) || (kmin && kmin.compact === 1);
|
|
34
|
+
const descSize = (isHDC || isCompact) ? 1 : 2; // Compact uses 32-bit like HDC
|
|
32
35
|
|
|
33
36
|
const currentRatioThreshold = isHDC ? HDC_RATIO_THRESHOLD : HAMMING_THRESHOLD;
|
|
34
37
|
|
|
35
38
|
for (let j = 0; j < qlen; j++) {
|
|
39
|
+
|
|
36
40
|
const querypoint = querypoints[j];
|
|
37
41
|
const col = querypoint.maxima ? kmax : kmin;
|
|
38
42
|
if (!col || col.x.length === 0) continue;
|
|
@@ -49,7 +53,8 @@ const match = ({ keyframe, querypoints: rawQuerypoints, querywidth, queryheight,
|
|
|
49
53
|
keypointIndexes,
|
|
50
54
|
numPop: 0,
|
|
51
55
|
isHDC,
|
|
52
|
-
descSize
|
|
56
|
+
descSize,
|
|
57
|
+
isCompact
|
|
53
58
|
});
|
|
54
59
|
|
|
55
60
|
let bestIndex = -1;
|
|
@@ -59,12 +64,31 @@ const match = ({ keyframe, querypoints: rawQuerypoints, querywidth, queryheight,
|
|
|
59
64
|
const qDesc = querypoint.descriptors;
|
|
60
65
|
const cDesc = col.d;
|
|
61
66
|
|
|
67
|
+
// For compact mode: pre-compute XOR folded query descriptor (64-bit → 32-bit)
|
|
68
|
+
const qDescCompact = isCompact && qDesc && qDesc.length >= 2
|
|
69
|
+
? (qDesc[0] ^ qDesc[1]) >>> 0
|
|
70
|
+
: 0;
|
|
71
|
+
|
|
62
72
|
for (let k = 0; k < keypointIndexes.length; k++) {
|
|
63
73
|
const idx = keypointIndexes[k];
|
|
64
74
|
|
|
75
|
+
// 🚀 NANITE-STYLE: Dynamic scale filtering
|
|
76
|
+
// If we have an expected scale, skip points that are outside the resolution range
|
|
77
|
+
if (expectedScale !== undefined && col.s) {
|
|
78
|
+
const featureScale = col.s[idx]; // Octave scale (1, 2, 4...)
|
|
79
|
+
const idealKeyScale = (querypoint.scale || 1.0) / expectedScale;
|
|
80
|
+
// allow ~1 octave of margin
|
|
81
|
+
if (featureScale < idealKeyScale * 0.4 || featureScale > idealKeyScale * 2.5) {
|
|
82
|
+
continue;
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
|
|
65
86
|
let d;
|
|
66
87
|
if (isHDC) {
|
|
67
88
|
d = popcount32(cDesc[idx] ^ querypoint.hdcSignature);
|
|
89
|
+
} else if (isCompact) {
|
|
90
|
+
// Compact mode: compare 32-bit XOR folded descriptors
|
|
91
|
+
d = popcount32(cDesc[idx] ^ qDescCompact);
|
|
68
92
|
} else {
|
|
69
93
|
d = hammingCompute({ v1: cDesc, v1Offset: idx * descSize, v2: qDesc });
|
|
70
94
|
}
|
|
@@ -78,6 +102,7 @@ const match = ({ keyframe, querypoints: rawQuerypoints, querywidth, queryheight,
|
|
|
78
102
|
}
|
|
79
103
|
}
|
|
80
104
|
|
|
105
|
+
|
|
81
106
|
if (bestIndex !== -1) {
|
|
82
107
|
if (bestD2 === Number.MAX_SAFE_INTEGER || (bestD1 / bestD2) < currentRatioThreshold) {
|
|
83
108
|
matches.push({
|
|
@@ -99,11 +124,6 @@ const match = ({ keyframe, querypoints: rawQuerypoints, querywidth, queryheight,
|
|
|
99
124
|
return { debugExtra };
|
|
100
125
|
}
|
|
101
126
|
|
|
102
|
-
// Debug: Log Hamming results
|
|
103
|
-
if (Math.random() < 0.1 && debugMode) {
|
|
104
|
-
console.log(`MATCH_DL: Hamming found ${matches.length} initial matches`);
|
|
105
|
-
}
|
|
106
|
-
|
|
107
127
|
// 🌌 Moonshot: Constellation matching disabled for performance calibration
|
|
108
128
|
const constellationMatches = matches;
|
|
109
129
|
if (debugMode) debugExtra.constellationMatches = constellationMatches;
|
|
@@ -116,7 +136,9 @@ const match = ({ keyframe, querypoints: rawQuerypoints, querywidth, queryheight,
|
|
|
116
136
|
matches: constellationMatches,
|
|
117
137
|
});
|
|
118
138
|
|
|
119
|
-
if (debugMode)
|
|
139
|
+
if (debugMode) {
|
|
140
|
+
debugExtra.houghMatches = houghMatches;
|
|
141
|
+
}
|
|
120
142
|
|
|
121
143
|
if (houghMatches.length < MIN_NUM_INLIERS) {
|
|
122
144
|
return { debugExtra };
|
|
@@ -175,6 +197,11 @@ const match = ({ keyframe, querypoints: rawQuerypoints, querywidth, queryheight,
|
|
|
175
197
|
const cx = col.x, cy = col.y, cd = col.d;
|
|
176
198
|
const qDesc = querypoint.descriptors;
|
|
177
199
|
|
|
200
|
+
// For compact mode: XOR fold query descriptor
|
|
201
|
+
const qDescCompact = isCompact && qDesc && qDesc.length >= 2
|
|
202
|
+
? (qDesc[0] ^ qDesc[1]) >>> 0
|
|
203
|
+
: 0;
|
|
204
|
+
|
|
178
205
|
for (let k = 0, clen = cx.length; k < clen; k++) {
|
|
179
206
|
const dx = cx[k] - mapX;
|
|
180
207
|
const dy = cy[k] - mapY;
|
|
@@ -185,6 +212,8 @@ const match = ({ keyframe, querypoints: rawQuerypoints, querywidth, queryheight,
|
|
|
185
212
|
let d;
|
|
186
213
|
if (isHDC) {
|
|
187
214
|
d = popcount32(cd[k] ^ querypoint.hdcSignature);
|
|
215
|
+
} else if (isCompact) {
|
|
216
|
+
d = popcount32(cd[k] ^ qDescCompact);
|
|
188
217
|
} else {
|
|
189
218
|
d = hammingCompute({ v1: cd, v1Offset: k * descSize, v2: qDesc });
|
|
190
219
|
}
|
|
@@ -198,6 +227,7 @@ const match = ({ keyframe, querypoints: rawQuerypoints, querywidth, queryheight,
|
|
|
198
227
|
}
|
|
199
228
|
}
|
|
200
229
|
|
|
230
|
+
|
|
201
231
|
if (
|
|
202
232
|
bestIndex !== -1 &&
|
|
203
233
|
(bestD2 === Number.MAX_SAFE_INTEGER || (bestD1 / bestD2) < currentRatioThreshold)
|
|
@@ -254,7 +284,7 @@ const match = ({ keyframe, querypoints: rawQuerypoints, querywidth, queryheight,
|
|
|
254
284
|
return { H: refinedH || H2, matches: inlierMatches2, debugExtra };
|
|
255
285
|
};
|
|
256
286
|
|
|
257
|
-
const _query = ({ node, descriptors, querypoint, queue, keypointIndexes, numPop, isHDC, descSize }) => {
|
|
287
|
+
const _query = ({ node, descriptors, querypoint, queue, keypointIndexes, numPop, isHDC, descSize, isCompact }) => {
|
|
258
288
|
const isLeaf = node[0] === 1;
|
|
259
289
|
const childrenOrIndices = node[2];
|
|
260
290
|
|
|
@@ -266,6 +296,12 @@ const _query = ({ node, descriptors, querypoint, queue, keypointIndexes, numPop,
|
|
|
266
296
|
}
|
|
267
297
|
|
|
268
298
|
const qDesc = querypoint.descriptors;
|
|
299
|
+
|
|
300
|
+
// For compact mode: XOR fold query descriptor
|
|
301
|
+
const qDescCompact = isCompact && qDesc && qDesc.length >= 2
|
|
302
|
+
? (qDesc[0] ^ qDesc[1]) >>> 0
|
|
303
|
+
: 0;
|
|
304
|
+
|
|
269
305
|
let minD = Number.MAX_SAFE_INTEGER;
|
|
270
306
|
const clen = childrenOrIndices.length;
|
|
271
307
|
const distances = new Int32Array(clen);
|
|
@@ -277,6 +313,8 @@ const _query = ({ node, descriptors, querypoint, queue, keypointIndexes, numPop,
|
|
|
277
313
|
let d;
|
|
278
314
|
if (isHDC) {
|
|
279
315
|
d = popcount32(descriptors[cIdx] ^ querypoint.hdcSignature);
|
|
316
|
+
} else if (isCompact) {
|
|
317
|
+
d = popcount32(descriptors[cIdx] ^ qDescCompact);
|
|
280
318
|
} else {
|
|
281
319
|
d = hammingCompute({
|
|
282
320
|
v1: descriptors,
|
|
@@ -291,7 +329,7 @@ const _query = ({ node, descriptors, querypoint, queue, keypointIndexes, numPop,
|
|
|
291
329
|
for (let i = 0; i < clen; i++) {
|
|
292
330
|
const dist = distances[i];
|
|
293
331
|
if (dist <= minD) {
|
|
294
|
-
_query({ node: childrenOrIndices[i], descriptors, querypoint, queue, keypointIndexes, numPop: numPop + 1, isHDC, descSize });
|
|
332
|
+
_query({ node: childrenOrIndices[i], descriptors, querypoint, queue, keypointIndexes, numPop: numPop + 1, isHDC, descSize, isCompact });
|
|
295
333
|
} else {
|
|
296
334
|
queue.push({ node: childrenOrIndices[i], d: dist });
|
|
297
335
|
}
|
|
@@ -299,10 +337,11 @@ const _query = ({ node, descriptors, querypoint, queue, keypointIndexes, numPop,
|
|
|
299
337
|
|
|
300
338
|
if (numPop < CLUSTER_MAX_POP && queue.length > 0) {
|
|
301
339
|
const { node } = queue.pop();
|
|
302
|
-
_query({ node, descriptors, querypoint, queue, keypointIndexes, numPop: numPop + 1, isHDC, descSize });
|
|
340
|
+
_query({ node, descriptors, querypoint, queue, keypointIndexes, numPop: numPop + 1, isHDC, descSize, isCompact });
|
|
303
341
|
}
|
|
304
342
|
};
|
|
305
343
|
|
|
344
|
+
|
|
306
345
|
const _findInlierMatches = (options) => {
|
|
307
346
|
const { H, matches, threshold } = options;
|
|
308
347
|
const threshold2 = threshold * threshold;
|
|
@@ -0,0 +1,275 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Bio-Inspired Perception Engine
|
|
3
|
+
*
|
|
4
|
+
* Inspired by human visual system:
|
|
5
|
+
* - Foveal attention: High resolution in center, low in periphery
|
|
6
|
+
* - Saccadic sampling: Strategic "glances" at areas of interest
|
|
7
|
+
* - Predictive coding: Only process what's unexpected/changed
|
|
8
|
+
*
|
|
9
|
+
* Expected improvements:
|
|
10
|
+
* - ~75% reduction in pixels processed per frame
|
|
11
|
+
* - ~80% reduction in latency for static scenes
|
|
12
|
+
* - ~70% reduction in energy consumption
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
import { FovealAttention } from './foveal-attention.js';
|
|
16
|
+
import { SaccadicController } from './saccadic-controller.js';
|
|
17
|
+
import { PredictiveCoding } from './predictive-coding.js';
|
|
18
|
+
import { SaliencyMap } from './saliency-map.js';
|
|
19
|
+
import { ScaleOrchestrator } from './scale-orchestrator.js';
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* Configuration for Bio-Inspired Engine
|
|
23
|
+
*/
|
|
24
|
+
const BIO_CONFIG = {
|
|
25
|
+
// Foveal region (high resolution center)
|
|
26
|
+
FOVEA_RADIUS_RATIO: 0.15, // 15% of image dimension
|
|
27
|
+
PARAFOVEA_RADIUS_RATIO: 0.30, // 30% of image dimension
|
|
28
|
+
|
|
29
|
+
// Resolution multipliers
|
|
30
|
+
FOVEA_RESOLUTION: 1.0, // Full resolution
|
|
31
|
+
PARAFOVEA_RESOLUTION: 0.5, // Half resolution
|
|
32
|
+
PERIPHERY_RESOLUTION: 0.25, // Quarter resolution
|
|
33
|
+
|
|
34
|
+
// Saccadic behavior
|
|
35
|
+
MAX_SACCADES_PER_FRAME: 3, // Maximum "glances" per frame
|
|
36
|
+
SACCADE_COOLDOWN_MS: 50, // Minimum time between saccades
|
|
37
|
+
SALIENCY_THRESHOLD: 0.3, // Threshold for triggering saccade
|
|
38
|
+
|
|
39
|
+
// Predictive coding
|
|
40
|
+
CHANGE_THRESHOLD: 0.05, // 5% pixel difference to trigger processing
|
|
41
|
+
PREDICTION_CONFIDENCE: 0.8, // Confidence to skip processing
|
|
42
|
+
MOTION_HISTORY_FRAMES: 3, // Frames to consider for motion prediction
|
|
43
|
+
|
|
44
|
+
// Performance
|
|
45
|
+
ENABLE_SKIP_FRAMES: true, // Skip processing if nothing changed
|
|
46
|
+
MIN_PROCESSING_INTERVAL_MS: 8, // Minimum 8ms (~120fps cap)
|
|
47
|
+
NUM_OCTAVES: 5, // Default number of octaves
|
|
48
|
+
};
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Main Bio-Inspired Perception Engine
|
|
52
|
+
* Integrates all bio-inspired components for efficient AR processing
|
|
53
|
+
*/
|
|
54
|
+
class BioInspiredEngine {
|
|
55
|
+
/**
|
|
56
|
+
* @param {number} width - Input image width
|
|
57
|
+
* @param {number} height - Input image height
|
|
58
|
+
* @param {Object} options - Configuration options
|
|
59
|
+
*/
|
|
60
|
+
constructor(width, height, options = {}) {
|
|
61
|
+
this.width = width;
|
|
62
|
+
this.height = height;
|
|
63
|
+
this.config = { ...BIO_CONFIG, ...options };
|
|
64
|
+
|
|
65
|
+
// Initialize sub-components
|
|
66
|
+
this.fovealAttention = new FovealAttention(width, height, this.config);
|
|
67
|
+
this.saccadicController = new SaccadicController(width, height, this.config);
|
|
68
|
+
this.predictiveCoding = new PredictiveCoding(width, height, this.config);
|
|
69
|
+
this.saliencyMap = new SaliencyMap(width, height);
|
|
70
|
+
this.scaleOrchestrator = new ScaleOrchestrator(this.config.NUM_OCTAVES, {
|
|
71
|
+
debug: options.debugMode
|
|
72
|
+
});
|
|
73
|
+
|
|
74
|
+
// State tracking
|
|
75
|
+
this.currentFoveaCenter = { x: width / 2, y: height / 2 };
|
|
76
|
+
this.frameCount = 0;
|
|
77
|
+
this.lastProcessTime = 0;
|
|
78
|
+
this.skipCount = 0;
|
|
79
|
+
|
|
80
|
+
// Performance metrics
|
|
81
|
+
this.metrics = {
|
|
82
|
+
totalFrames: 0,
|
|
83
|
+
skippedFrames: 0,
|
|
84
|
+
avgPixelsProcessed: 0,
|
|
85
|
+
avgLatency: 0,
|
|
86
|
+
saccadeCount: 0,
|
|
87
|
+
};
|
|
88
|
+
|
|
89
|
+
// Pre-allocate buffers
|
|
90
|
+
this._initBuffers();
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* Initialize pre-allocated buffers for efficient processing
|
|
95
|
+
* @private
|
|
96
|
+
*/
|
|
97
|
+
_initBuffers() {
|
|
98
|
+
const fullSize = this.width * this.height;
|
|
99
|
+
const foveaSize = Math.ceil(fullSize * this.config.FOVEA_RADIUS_RATIO ** 2 * Math.PI);
|
|
100
|
+
|
|
101
|
+
// Multi-resolution output buffer
|
|
102
|
+
this.outputBuffer = {
|
|
103
|
+
fovea: new Uint8Array(foveaSize),
|
|
104
|
+
parafovea: new Uint8Array(Math.ceil(foveaSize * 4)),
|
|
105
|
+
periphery: new Uint8Array(Math.ceil(fullSize * 0.25)),
|
|
106
|
+
};
|
|
107
|
+
|
|
108
|
+
// Change detection buffer
|
|
109
|
+
this.changeBuffer = new Float32Array(Math.ceil(fullSize / 64)); // 8x8 blocks
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* Process an input frame using bio-inspired techniques
|
|
114
|
+
*
|
|
115
|
+
* @param {Uint8Array} inputData - Grayscale input image
|
|
116
|
+
* @param {Object} trackingState - Current tracking state (optional)
|
|
117
|
+
* @returns {Object} Processed result with attention regions
|
|
118
|
+
*/
|
|
119
|
+
process(inputData, trackingState = null) {
|
|
120
|
+
const startTime = performance.now();
|
|
121
|
+
this.frameCount++;
|
|
122
|
+
this.metrics.totalFrames++;
|
|
123
|
+
|
|
124
|
+
// Step 1: Predictive Coding - Check if we can skip processing
|
|
125
|
+
const prediction = this.predictiveCoding.predict(inputData, trackingState);
|
|
126
|
+
|
|
127
|
+
if (prediction.canSkip && this.config.ENABLE_SKIP_FRAMES) {
|
|
128
|
+
this.metrics.skippedFrames++;
|
|
129
|
+
this.skipCount++;
|
|
130
|
+
return {
|
|
131
|
+
skipped: true,
|
|
132
|
+
prediction: prediction.predictedState,
|
|
133
|
+
confidence: prediction.confidence,
|
|
134
|
+
pixelsProcessed: 0,
|
|
135
|
+
latency: performance.now() - startTime,
|
|
136
|
+
};
|
|
137
|
+
}
|
|
138
|
+
this.skipCount = 0;
|
|
139
|
+
|
|
140
|
+
// Step 2: Compute Saliency Map for attention guidance
|
|
141
|
+
const saliency = this.saliencyMap.compute(inputData);
|
|
142
|
+
|
|
143
|
+
// Step 3: Saccadic Controller - Decide where to "look"
|
|
144
|
+
const saccadeTargets = this.saccadicController.computeTargets(
|
|
145
|
+
saliency,
|
|
146
|
+
this.currentFoveaCenter,
|
|
147
|
+
trackingState
|
|
148
|
+
);
|
|
149
|
+
|
|
150
|
+
// Step 4: Extract foveal regions at different resolutions
|
|
151
|
+
const attentionRegions = [];
|
|
152
|
+
let totalPixelsProcessed = 0;
|
|
153
|
+
|
|
154
|
+
for (const target of saccadeTargets) {
|
|
155
|
+
const region = this.fovealAttention.extract(
|
|
156
|
+
inputData,
|
|
157
|
+
target.x,
|
|
158
|
+
target.y,
|
|
159
|
+
target.priority
|
|
160
|
+
);
|
|
161
|
+
attentionRegions.push(region);
|
|
162
|
+
totalPixelsProcessed += region.pixelCount;
|
|
163
|
+
this.metrics.saccadeCount++;
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
// Step 5: Update fovea center based on highest priority target
|
|
167
|
+
if (saccadeTargets.length > 0) {
|
|
168
|
+
const primary = saccadeTargets[0];
|
|
169
|
+
this.currentFoveaCenter = { x: primary.x, y: primary.y };
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
// Step 6: Scale Orchestrator - Determine octaves to process
|
|
173
|
+
const octavesToProcess = this.scaleOrchestrator.getOctavesToProcess(trackingState);
|
|
174
|
+
|
|
175
|
+
// Step 7: Store frame for prediction
|
|
176
|
+
this.predictiveCoding.storeFrame(inputData, trackingState);
|
|
177
|
+
|
|
178
|
+
// Compute metrics
|
|
179
|
+
const latency = performance.now() - startTime;
|
|
180
|
+
this._updateMetrics(totalPixelsProcessed, latency);
|
|
181
|
+
|
|
182
|
+
return {
|
|
183
|
+
skipped: false,
|
|
184
|
+
attentionRegions,
|
|
185
|
+
foveaCenter: this.currentFoveaCenter,
|
|
186
|
+
saliencyPeaks: saliency.peaks,
|
|
187
|
+
octavesToProcess,
|
|
188
|
+
pixelsProcessed: totalPixelsProcessed,
|
|
189
|
+
pixelsSaved: this.width * this.height - totalPixelsProcessed,
|
|
190
|
+
savingsPercent: ((1 - totalPixelsProcessed / (this.width * this.height)) * 100).toFixed(1),
|
|
191
|
+
latency,
|
|
192
|
+
};
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
/**
|
|
196
|
+
* Get the primary attention region (highest resolution)
|
|
197
|
+
* This is the region that should be used for feature detection
|
|
198
|
+
*
|
|
199
|
+
* @param {Object} processResult - Result from process()
|
|
200
|
+
* @returns {Object} Primary attention region with data
|
|
201
|
+
*/
|
|
202
|
+
getPrimaryRegion(processResult) {
|
|
203
|
+
if (processResult.skipped || !processResult.attentionRegions?.length) {
|
|
204
|
+
return null;
|
|
205
|
+
}
|
|
206
|
+
return processResult.attentionRegions[0];
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
/**
|
|
210
|
+
* Suggest optimal processing based on change detection
|
|
211
|
+
*
|
|
212
|
+
* @param {Uint8Array} inputData - Current frame
|
|
213
|
+
* @returns {Object} Processing suggestion
|
|
214
|
+
*/
|
|
215
|
+
suggestProcessing(inputData) {
|
|
216
|
+
const changeLevel = this.predictiveCoding.getChangeLevel(inputData);
|
|
217
|
+
|
|
218
|
+
return {
|
|
219
|
+
shouldProcessFull: changeLevel > 0.3,
|
|
220
|
+
shouldProcessPartial: changeLevel > 0.05,
|
|
221
|
+
canSkip: changeLevel < 0.02,
|
|
222
|
+
changeLevel,
|
|
223
|
+
recommendedSaccades: Math.ceil(changeLevel * this.config.MAX_SACCADES_PER_FRAME),
|
|
224
|
+
};
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
/**
|
|
228
|
+
* Update performance metrics
|
|
229
|
+
* @private
|
|
230
|
+
*/
|
|
231
|
+
_updateMetrics(pixelsProcessed, latency) {
|
|
232
|
+
const alpha = 0.1; // Exponential moving average factor
|
|
233
|
+
this.metrics.avgPixelsProcessed =
|
|
234
|
+
this.metrics.avgPixelsProcessed * (1 - alpha) + pixelsProcessed * alpha;
|
|
235
|
+
this.metrics.avgLatency =
|
|
236
|
+
this.metrics.avgLatency * (1 - alpha) + latency * alpha;
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
/**
|
|
240
|
+
* Get current performance metrics
|
|
241
|
+
* @returns {Object} Performance metrics
|
|
242
|
+
*/
|
|
243
|
+
getMetrics() {
|
|
244
|
+
return {
|
|
245
|
+
...this.metrics,
|
|
246
|
+
skipRate: ((this.metrics.skippedFrames / this.metrics.totalFrames) * 100).toFixed(1) + '%',
|
|
247
|
+
avgSavings: ((1 - this.metrics.avgPixelsProcessed / (this.width * this.height)) * 100).toFixed(1) + '%',
|
|
248
|
+
currentFovea: this.currentFoveaCenter,
|
|
249
|
+
};
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
/**
|
|
253
|
+
* Reset engine state (e.g., when target changes)
|
|
254
|
+
*/
|
|
255
|
+
reset() {
|
|
256
|
+
this.currentFoveaCenter = { x: this.width / 2, y: this.height / 2 };
|
|
257
|
+
this.frameCount = 0;
|
|
258
|
+
this.skipCount = 0;
|
|
259
|
+
this.predictiveCoding.reset();
|
|
260
|
+
this.saccadicController.reset();
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
/**
|
|
264
|
+
* Configure engine at runtime
|
|
265
|
+
* @param {Object} options - Configuration options to update
|
|
266
|
+
*/
|
|
267
|
+
configure(options) {
|
|
268
|
+
this.config = { ...this.config, ...options };
|
|
269
|
+
this.fovealAttention.configure(this.config);
|
|
270
|
+
this.saccadicController.configure(this.config);
|
|
271
|
+
this.predictiveCoding.configure(this.config);
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
export { BioInspiredEngine, BIO_CONFIG };
|