@srsergio/taptapp-ar 1.0.93 โ†’ 1.0.95

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. package/README.md +16 -14
  2. package/dist/compiler/offline-compiler.d.ts +3 -3
  3. package/dist/compiler/offline-compiler.js +50 -33
  4. package/dist/core/constants.d.ts +2 -0
  5. package/dist/core/constants.js +4 -1
  6. package/dist/core/detector/detector-lite.d.ts +6 -5
  7. package/dist/core/detector/detector-lite.js +46 -16
  8. package/dist/core/matching/matcher.d.ts +1 -1
  9. package/dist/core/matching/matcher.js +7 -4
  10. package/dist/core/matching/matching.d.ts +2 -1
  11. package/dist/core/matching/matching.js +43 -11
  12. package/dist/core/perception/bio-inspired-engine.d.ts +130 -0
  13. package/dist/core/perception/bio-inspired-engine.js +232 -0
  14. package/dist/core/perception/foveal-attention.d.ts +142 -0
  15. package/dist/core/perception/foveal-attention.js +280 -0
  16. package/dist/core/perception/index.d.ts +6 -0
  17. package/dist/core/perception/index.js +17 -0
  18. package/dist/core/perception/predictive-coding.d.ts +92 -0
  19. package/dist/core/perception/predictive-coding.js +278 -0
  20. package/dist/core/perception/saccadic-controller.d.ts +126 -0
  21. package/dist/core/perception/saccadic-controller.js +269 -0
  22. package/dist/core/perception/saliency-map.d.ts +74 -0
  23. package/dist/core/perception/saliency-map.js +254 -0
  24. package/dist/core/perception/scale-orchestrator.d.ts +28 -0
  25. package/dist/core/perception/scale-orchestrator.js +68 -0
  26. package/dist/core/protocol.d.ts +14 -1
  27. package/dist/core/protocol.js +33 -1
  28. package/dist/runtime/bio-inspired-controller.d.ts +135 -0
  29. package/dist/runtime/bio-inspired-controller.js +358 -0
  30. package/dist/runtime/controller.d.ts +11 -2
  31. package/dist/runtime/controller.js +20 -8
  32. package/dist/runtime/controller.worker.js +2 -2
  33. package/dist/runtime/simple-ar.d.ts +24 -20
  34. package/dist/runtime/simple-ar.js +172 -156
  35. package/package.json +1 -1
  36. package/src/compiler/offline-compiler.ts +56 -36
  37. package/src/core/constants.ts +5 -1
  38. package/src/core/detector/detector-lite.js +46 -16
  39. package/src/core/matching/matcher.js +8 -4
  40. package/src/core/matching/matching.js +51 -12
  41. package/src/core/perception/bio-inspired-engine.js +275 -0
  42. package/src/core/perception/foveal-attention.js +306 -0
  43. package/src/core/perception/index.js +18 -0
  44. package/src/core/perception/predictive-coding.js +327 -0
  45. package/src/core/perception/saccadic-controller.js +303 -0
  46. package/src/core/perception/saliency-map.js +296 -0
  47. package/src/core/perception/scale-orchestrator.js +80 -0
  48. package/src/core/protocol.ts +38 -1
  49. package/src/runtime/bio-inspired-controller.ts +448 -0
  50. package/src/runtime/controller.ts +22 -7
  51. package/src/runtime/controller.worker.js +2 -1
  52. package/src/runtime/simple-ar.ts +197 -171
package/README.md CHANGED
@@ -30,19 +30,19 @@
30
30
  ## ๐ŸŒŸ Key Features
31
31
 
32
32
  - ๐ŸŽญ **Non-Rigid Surface Tracking**: Supports curved and deformable surfaces using **Delaunay Meshes** and **Mass-Spring Relaxation**.
33
- - ๐Ÿš€ **Hyper-Fast Compiler**: Pure JavaScript compiler that generates `.taar` files in **< 3s**.
33
+ - ๐Ÿš€ **Nanite-style Virtualized Features**: Single-pass multi-octave detection with stratified sampling.
34
34
  - โšก **No TensorFlow Dependency**: No TFJS at all. Works natively in any JS environment (Node, Browser, Workers).
35
35
  - ๐Ÿงฌ **Fourier Positional Encoding**: Uses high-frequency sine/cosine mappings (GPT-style) for neural-like spatial consistency.
36
- - ๐Ÿš€ **Protocol V7 (Moonshot)**:
37
- - **Delaunay Triangular Grid**: Adaptive mesh that tracks surface deformations.
38
- - **16-bit Fourier Signatures**: Spatial ADN embedded in every feature for harmonic matching.
36
+ - ๐Ÿš€ **Protocol V11 (Nanite)**:
37
+ - **Stratified Multi-Octave Sampling**: 300 reliable features per octave, capped at 1,200 total per target.
38
+ - **Dynamic Scale Filtering (LOD)**: Runtime matching engine skips irrelevant octaves based on estimated scale.
39
39
  - **4-bit Packed Tracking Data**: Grayscale images are compressed to 4-bit depth, slashing file size.
40
- - **64-bit LSH Descriptors**: Optimized Locality Sensitive Hashing for descriptors.
41
- - ๐Ÿงต **High-Precision Tracking**: Now using **Float32** coordinate precision with sub-pixel resolution and multi-octave verification (1%, 50%, 25%, 12.5% scales).
42
- - ๐Ÿ“ **Ultra-Wide Scaling**: Enhanced Hough Transform supporting a massive scale range from **1% (distant targets)** to **1000% (extreme close-up)**.
40
+ - **64-bit LSH Descriptors**: Optimized Locality Sensitive Hashing with XOR folding support.
41
+ - ๐Ÿงต **High-Precision Tracking**: Now using **Float32** coordinate precision with sub-pixel resolution and stratified scale coverage.
42
+ - ๐Ÿ“ **Virtualized Scale Range**: Stable tracking from **20% (distant targets)** to **1000% (close-up)** using a single high-res keyframe.
43
43
  - โšก **Immediate AR Detection**: Optimized "warm-up" period (15 frames) with relaxed inlier thresholds (6 pts) for instant tracking lock.
44
44
  - ๐Ÿ“ฆ **Framework Agnostic**: Includes wrappers for **A-Frame**, **Three.js**, and a raw **Controller** for custom engines.
45
- - ๐Ÿ“‰ **Ultra-Compact Files**: Output `.taar` files are **~50KB** (vs ~380KB+ previously).
45
+ - ๐Ÿ“‰ **Ultra-Compact Files**: Output `.taar` files are now **~100KB** (vs ~380KB+ previously).
46
46
 
47
47
  ---
48
48
 
@@ -56,11 +56,11 @@ npm install @srsergio/taptapp-ar
56
56
 
57
57
  ## ๐Ÿ“Š Industry-Leading Benchmarks (v7 Moonshot)
58
58
 
59
- | Metric | Official MindAR | TapTapp AR V7 | Improvement |
59
+ | Metric | Official MindAR | TapTapp AR V11 | Improvement |
60
60
  | :--- | :--- | :--- | :--- |
61
- | **Compilation Time** | ~23.50s | **~0.93s** | ๐Ÿš€ **~25x Faster** |
62
- | **Output Size (.taar)** | ~770 KB | **~338 KB** | ๐Ÿ“‰ **56% Smaller** |
63
- | **Descriptor Format** | 84-byte Float | **128-bit LSH** | ๐Ÿง  **Massive Data Saving** |
61
+ | **Compilation Time** | ~23.50s | **~1.15s** | ๐Ÿš€ **~20x Faster** |
62
+ | **Output Size (.taar)** | ~770 KB | **~103 KB** | ๐Ÿ“‰ **86% Smaller** |
63
+ | **Matching Logic** | Brute-force | **Nanite LOD (Scale-Filtered)** | ๐Ÿง  **Smart Extraction** |
64
64
  | **Tracking Data** | 8-bit Gray | **4-bit Packed** | ๐Ÿ“ฆ **50% Data Saving** |
65
65
  | **Dependency Size** | ~20MB (TFJS) | **< 100KB** | ๐Ÿ“ฆ **99% Smaller Bundle** |
66
66
 
@@ -259,9 +259,11 @@ ar.stop();
259
259
 
260
260
  ---
261
261
 
262
- ## ๐Ÿ—๏ธ Protocol V7 (Moonshot Packed Format)
263
- TapTapp AR uses a proprietary **Moonshot Vision Codec** that is significantly more efficient than standard AR formats.
262
+ ## ๐Ÿ—๏ธ Protocol V11 (Nanite Virtualized Format)
263
+ TapTapp AR uses a proprietary **Nanite-style Vision Codec** that is significantly more efficient than standard AR formats.
264
264
 
265
+ - **Virtualized Multi-Octave Features**: Instead of storing redundant images for each scale, V11 stores a single high-res keyframe with features stratified across 6 octaves.
266
+ - **Dynamic Scale Filtering**: The tracking engine estimates the target's current scale and dynamically filters the matching search space, reducing Hamming distance ops by up to 90%.
265
267
  - **Non-Rigid Surface Tracking**: Replaces the standard rigid homography with a dynamic **Delaunay Mesh**. This allows the tracker to follow the curvature of posters on cylinders, t-shirts, or slightly bent magazines.
266
268
  - **Mass-Spring Relaxation**: The tracking mesh is optimized using physical relaxation, minimizing L2 distance between predicted and tracked points while maintaining topological rigidity.
267
269
  - **Fourier Positional Encoding**: Maps 2D coordinates into a 16-dimensional frequency space. This creates a "Neural Consistency Check" that filters out noise and motion blur by checking for harmonic spatial agreement.
@@ -37,7 +37,7 @@ export declare class OfflineCompiler {
37
37
  };
38
38
  width: any;
39
39
  height: any;
40
- scale: any;
40
+ scale: number;
41
41
  }[];
42
42
  trackingData: Object[];
43
43
  }[]>;
@@ -68,7 +68,7 @@ export declare class OfflineCompiler {
68
68
  };
69
69
  width: any;
70
70
  height: any;
71
- scale: any;
71
+ scale: number;
72
72
  }[][]>;
73
73
  _compileTrack(targetImages: any[], progressCallback: (p: number) => void): Promise<Object[][]>;
74
74
  compileTrack({ progressCallback, targetImages, basePercent }: {
@@ -107,7 +107,7 @@ export declare class OfflineCompiler {
107
107
  };
108
108
  width: any;
109
109
  height: any;
110
- scale: any;
110
+ scale: number;
111
111
  }[][]>;
112
112
  exportData(): Uint8Array<ArrayBuffer>;
113
113
  importData(buffer: ArrayBuffer | Uint8Array): {
@@ -5,7 +5,7 @@
5
5
  * que NO depende de TensorFlow, eliminando todos los problemas de
6
6
  * inicializaciรณn, bloqueos y compatibilidad.
7
7
  */
8
- import { buildTrackingImageList, buildImageList } from "../core/image-list.js";
8
+ import { buildTrackingImageList } from "../core/image-list.js";
9
9
  import { extractTrackingFeatures } from "../core/tracker/extract-utils.js";
10
10
  import { DetectorLite } from "../core/detector/detector-lite.js";
11
11
  import { build as hierarchicalClusteringBuild } from "../core/matching/hierarchical-clustering.js";
@@ -73,31 +73,44 @@ export class OfflineCompiler {
73
73
  const results = [];
74
74
  for (let i = 0; i < targetImages.length; i++) {
75
75
  const targetImage = targetImages[i];
76
- const fullImageList = buildImageList(targetImage);
77
- // ๐Ÿš€ MOONSHOT: Keep many scales for better robustness
78
- const imageList = fullImageList;
79
- const percentPerImageScale = percentPerImage / imageList.length;
80
- const keyframes = [];
81
- for (const image of imageList) {
82
- const detector = new DetectorLite(image.width, image.height, { useLSH: AR_CONFIG.USE_LSH, maxFeaturesPerBucket: AR_CONFIG.MAX_FEATURES_PER_BUCKET });
83
- const { featurePoints: ps } = detector.detect(image.data);
84
- const maximaPoints = ps.filter((p) => p.maxima);
85
- const minimaPoints = ps.filter((p) => !p.maxima);
86
- const maximaPointsCluster = hierarchicalClusteringBuild({ points: maximaPoints });
87
- const minimaPointsCluster = hierarchicalClusteringBuild({ points: minimaPoints });
88
- keyframes.push({
89
- maximaPoints,
90
- minimaPoints,
91
- maximaPointsCluster,
92
- minimaPointsCluster,
93
- width: image.width,
94
- height: image.height,
95
- scale: image.scale,
96
- });
97
- currentPercent += percentPerImageScale;
98
- progressCallback(currentPercent);
76
+ // ๐Ÿš€ NANITE-STYLE: Only process the target at scale 1.0
77
+ // The DetectorLite already builds its own pyramid and finds features at all octaves (virtualized LOD)
78
+ const detector = new DetectorLite(targetImage.width, targetImage.height, {
79
+ useLSH: AR_CONFIG.USE_LSH,
80
+ maxFeaturesPerBucket: AR_CONFIG.MAX_FEATURES_PER_BUCKET
81
+ });
82
+ const { featurePoints: rawPs } = detector.detect(targetImage.data);
83
+ // ๐ŸŽฏ Stratified Sampling: Ensure we have features from ALL scales
84
+ // We take the top N features per octave to guarantee scale coverage (Nanite-style)
85
+ const octaves = [0, 1, 2, 3, 4, 5];
86
+ const ps = [];
87
+ const featuresPerOctave = 300;
88
+ for (const oct of octaves) {
89
+ const octScale = Math.pow(2, oct);
90
+ const octFeatures = rawPs
91
+ .filter(p => Math.abs(p.scale - octScale) < 0.1)
92
+ .sort((a, b) => (b.score || 0) - (a.score || 0))
93
+ .slice(0, featuresPerOctave);
94
+ ps.push(...octFeatures);
99
95
  }
100
- results.push(keyframes);
96
+ const maximaPoints = ps.filter((p) => p.maxima);
97
+ const minimaPoints = ps.filter((p) => !p.maxima);
98
+ const maximaPointsCluster = hierarchicalClusteringBuild({ points: maximaPoints });
99
+ const minimaPointsCluster = hierarchicalClusteringBuild({ points: minimaPoints });
100
+ const keyframe = {
101
+ maximaPoints,
102
+ minimaPoints,
103
+ maximaPointsCluster,
104
+ minimaPointsCluster,
105
+ width: targetImage.width,
106
+ height: targetImage.height,
107
+ scale: 1.0,
108
+ };
109
+ // Wrapped in array because the protocol expects matchingData to be an array of keyframes
110
+ // We provide only one keyframe containing features from all octaves
111
+ results.push([keyframe]);
112
+ currentPercent += percentPerImage;
113
+ progressCallback(currentPercent);
101
114
  }
102
115
  return results;
103
116
  }
@@ -167,14 +180,18 @@ export class OfflineCompiler {
167
180
  }
168
181
  };
169
182
  }),
170
- matchingData: item.matchingData.map((kf) => ({
171
- w: kf.width,
172
- h: kf.height,
173
- s: kf.scale,
174
- hdc: false,
175
- max: protocol.columnarize(kf.maximaPoints, kf.maximaPointsCluster, kf.width, kf.height, false),
176
- min: protocol.columnarize(kf.minimaPoints, kf.minimaPointsCluster, kf.width, kf.height, false),
177
- })),
183
+ matchingData: item.matchingData.map((kf) => {
184
+ const useCompact = AR_CONFIG.USE_COMPACT_DESCRIPTORS;
185
+ const columnarizeFn = useCompact ? protocol.columnarizeCompact : protocol.columnarize;
186
+ return {
187
+ w: kf.width,
188
+ h: kf.height,
189
+ s: kf.scale,
190
+ hdc: false,
191
+ max: columnarizeFn(kf.maximaPoints, kf.maximaPointsCluster, kf.width, kf.height),
192
+ min: columnarizeFn(kf.minimaPoints, kf.minimaPointsCluster, kf.width, kf.height),
193
+ };
194
+ }),
178
195
  };
179
196
  });
180
197
  return protocol.encodeTaar(dataList);
@@ -26,4 +26,6 @@ export declare const AR_CONFIG: {
26
26
  MISS_TOLERANCE: number;
27
27
  ONE_EURO_FILTER_CUTOFF: number;
28
28
  ONE_EURO_FILTER_BETA: number;
29
+ USE_COMPACT_DESCRIPTORS: boolean;
30
+ COMPACT_HAMMING_THRESHOLD: number;
29
31
  };
@@ -24,7 +24,7 @@ export const AR_CONFIG = {
24
24
  TRACKER_SIMILARITY_THRESHOLD: 0.65,
25
25
  // Image processing / Scale list
26
26
  MIN_IMAGE_PIXEL_SIZE: 32,
27
- SCALE_STEP_EXPONENT: 0.6,
27
+ SCALE_STEP_EXPONENT: 1.0, // Optimized: was 0.6, now 1.0 (reduces scales from ~7 to ~4)
28
28
  TRACKING_DOWNSCALE_LEVEL_1: 256.0,
29
29
  TRACKING_DOWNSCALE_LEVEL_2: 128.0,
30
30
  // Tracker settings
@@ -32,4 +32,7 @@ export const AR_CONFIG = {
32
32
  MISS_TOLERANCE: 1,
33
33
  ONE_EURO_FILTER_CUTOFF: 0.5,
34
34
  ONE_EURO_FILTER_BETA: 0.1,
35
+ // TAAR Size Optimization
36
+ USE_COMPACT_DESCRIPTORS: true, // 32-bit XOR folded descriptors vs 64-bit raw
37
+ COMPACT_HAMMING_THRESHOLD: 8, // Threshold for 32-bit descriptors (vs 15 for 64-bit)
35
38
  };
@@ -14,19 +14,20 @@ export class DetectorLite {
14
14
  /**
15
15
  * Detecta caracterรญsticas en una imagen en escala de grises
16
16
  * @param {Float32Array|Uint8Array} imageData - Datos de imagen (width * height)
17
+ * @param {Object} options - Opciones de detecciรณn (ej. octavesToProcess)
17
18
  * @returns {{featurePoints: Array}} Puntos de caracterรญsticas detectados
18
19
  */
19
- detect(imageData: Float32Array | Uint8Array): {
20
+ detect(imageData: Float32Array | Uint8Array, options?: Object): {
20
21
  featurePoints: any[];
21
22
  };
22
23
  /**
23
24
  * Construye una pirรกmide gaussiana
24
25
  */
25
- _buildGaussianPyramid(data: any, width: any, height: any): {
26
+ _buildGaussianPyramid(data: any, width: any, height: any, octavesToProcess?: null): ({
26
27
  data: Float32Array<ArrayBuffer>;
27
28
  width: any;
28
29
  height: any;
29
- }[][];
30
+ }[] | null)[];
30
31
  _pyramidBuffers: {
31
32
  width: any;
32
33
  height: any;
@@ -51,11 +52,11 @@ export class DetectorLite {
51
52
  /**
52
53
  * Construye pirรกmide de diferencia de gaussianas
53
54
  */
54
- _buildDogPyramid(pyramidImages: any): {
55
+ _buildDogPyramid(pyramidImages: any, octavesToProcess?: null): ({
55
56
  data: Float32Array<ArrayBuffer>;
56
57
  width: any;
57
58
  height: any;
58
- }[];
59
+ } | null)[];
59
60
  /**
60
61
  * Encuentra extremos locales en la pirรกmide DoG
61
62
  */
@@ -55,9 +55,11 @@ export class DetectorLite {
55
55
  /**
56
56
  * Detecta caracterรญsticas en una imagen en escala de grises
57
57
  * @param {Float32Array|Uint8Array} imageData - Datos de imagen (width * height)
58
+ * @param {Object} options - Opciones de detecciรณn (ej. octavesToProcess)
58
59
  * @returns {{featurePoints: Array}} Puntos de caracterรญsticas detectados
59
60
  */
60
- detect(imageData) {
61
+ detect(imageData, options = {}) {
62
+ const octavesToProcess = options.octavesToProcess || Array.from({ length: this.numOctaves }, (_, i) => i);
61
63
  // Normalizar a Float32Array si es necesario
62
64
  let data;
63
65
  if (imageData instanceof Float32Array) {
@@ -69,10 +71,10 @@ export class DetectorLite {
69
71
  data[i] = imageData[i];
70
72
  }
71
73
  }
72
- // 1. Construir pirรกmide gaussiana
73
- const pyramidImages = this._buildGaussianPyramid(data, this.width, this.height);
74
+ // 1. Construir pirรกmide gaussiana (solo octavas solicitadas)
75
+ const pyramidImages = this._buildGaussianPyramid(data, this.width, this.height, octavesToProcess);
74
76
  // 2. Construir pirรกmide DoG (Difference of Gaussians)
75
- const dogPyramid = this._buildDogPyramid(pyramidImages);
77
+ const dogPyramid = this._buildDogPyramid(pyramidImages, octavesToProcess);
76
78
  // 3. Encontrar extremos locales
77
79
  const extremas = this._findExtremas(dogPyramid, pyramidImages);
78
80
  // 4. Aplicar pruning por buckets
@@ -100,7 +102,7 @@ export class DetectorLite {
100
102
  /**
101
103
  * Construye una pirรกmide gaussiana
102
104
  */
103
- _buildGaussianPyramid(data, width, height) {
105
+ _buildGaussianPyramid(data, width, height, octavesToProcess = null) {
104
106
  // Use GPU-accelerated pyramid if available
105
107
  if (this.useGPU) {
106
108
  try {
@@ -108,6 +110,10 @@ export class DetectorLite {
108
110
  // Convert GPU pyramid format to expected format
109
111
  const pyramid = [];
110
112
  for (let i = 0; i < gpuPyramid.length && i < this.numOctaves; i++) {
113
+ if (octavesToProcess && !octavesToProcess.includes(i)) {
114
+ pyramid.push(null);
115
+ continue;
116
+ }
111
117
  const level = gpuPyramid[i];
112
118
  // Apply second blur for DoG computation
113
119
  const img2 = this._applyGaussianFilter(level.data, level.width, level.height);
@@ -132,17 +138,35 @@ export class DetectorLite {
132
138
  let currentWidth = width;
133
139
  let currentHeight = height;
134
140
  for (let i = 0; i < this.numOctaves; i++) {
135
- const img1 = this._applyGaussianFilter(currentData, currentWidth, currentHeight);
136
- const img2 = this._applyGaussianFilter(img1.data, currentWidth, currentHeight);
137
- pyramid.push([
138
- { data: img1.data, width: currentWidth, height: currentHeight },
139
- { data: img2.data, width: currentWidth, height: currentHeight }
140
- ]);
141
+ const shouldProcess = !octavesToProcess || octavesToProcess.includes(i);
142
+ if (shouldProcess) {
143
+ const img1 = this._applyGaussianFilter(currentData, currentWidth, currentHeight);
144
+ const img2 = this._applyGaussianFilter(img1.data, currentWidth, currentHeight);
145
+ pyramid.push([
146
+ { data: img1.data, width: currentWidth, height: currentHeight },
147
+ { data: img2.data, width: currentWidth, height: currentHeight }
148
+ ]);
149
+ }
150
+ else {
151
+ pyramid.push(null);
152
+ }
141
153
  if (i < this.numOctaves - 1) {
142
- const downsampled = this._downsample(img2.data, currentWidth, currentHeight);
143
- currentData = downsampled.data;
144
- currentWidth = downsampled.width;
145
- currentHeight = downsampled.height;
154
+ // For CPU downsampling, we STILL need to downsample even if we skip processing the current octave
155
+ // UNLESS the next octave is also skipped. But for simplicity and safety, we downsample if needed by ANY future octave.
156
+ const needsDownsample = !octavesToProcess || octavesToProcess.some(o => o > i);
157
+ if (needsDownsample) {
158
+ // If current octave was processed, we use img1.data (or original data if i=0 and not processed?).
159
+ // Wait, standard is to downsample from the blurred image of previous octave.
160
+ const sourceData = shouldProcess ? pyramid[i][0].data : currentData;
161
+ const downsampled = this._downsample(sourceData, currentWidth, currentHeight);
162
+ currentData = downsampled.data;
163
+ currentWidth = downsampled.width;
164
+ currentHeight = downsampled.height;
165
+ }
166
+ else {
167
+ // Optimization: if no more octaves are needed, we can stop here
168
+ break;
169
+ }
146
170
  }
147
171
  }
148
172
  return pyramid;
@@ -212,9 +236,13 @@ export class DetectorLite {
212
236
  /**
213
237
  * Construye pirรกmide de diferencia de gaussianas
214
238
  */
215
- _buildDogPyramid(pyramidImages) {
239
+ _buildDogPyramid(pyramidImages, octavesToProcess = null) {
216
240
  const dogPyramid = [];
217
241
  for (let i = 0; i < pyramidImages.length; i++) {
242
+ if (!pyramidImages[i]) {
243
+ dogPyramid.push(null);
244
+ continue;
245
+ }
218
246
  const img1 = pyramidImages[i][0];
219
247
  const img2 = pyramidImages[i][1];
220
248
  const width = img1.width;
@@ -234,6 +262,8 @@ export class DetectorLite {
234
262
  const extremas = [];
235
263
  for (let octave = 0; octave < dogPyramid.length; octave++) {
236
264
  const curr = dogPyramid[octave];
265
+ if (!curr)
266
+ continue;
237
267
  const prev = octave > 0 ? dogPyramid[octave - 1] : null;
238
268
  const next = octave < dogPyramid.length - 1 ? dogPyramid[octave + 1] : null;
239
269
  const width = curr.width;
@@ -3,7 +3,7 @@ export class Matcher {
3
3
  queryWidth: any;
4
4
  queryHeight: any;
5
5
  debugMode: boolean;
6
- matchDetection(keyframes: any, featurePoints: any): {
6
+ matchDetection(keyframes: any, featurePoints: any, expectedScale: any): {
7
7
  targetIndex: number;
8
8
  keyframeIndex: number;
9
9
  debugExtra: {
@@ -5,7 +5,7 @@ class Matcher {
5
5
  this.queryHeight = queryHeight;
6
6
  this.debugMode = debugMode;
7
7
  }
8
- matchDetection(keyframes, featurePoints) {
8
+ matchDetection(keyframes, featurePoints, expectedScale) {
9
9
  let debugExtra = { frames: [] };
10
10
  let bestResult = null;
11
11
  // keyframes is actually the matchingData array for a single target
@@ -19,6 +19,7 @@ class Matcher {
19
19
  querywidth: this.queryWidth,
20
20
  queryheight: this.queryHeight,
21
21
  debugMode: this.debugMode,
22
+ expectedScale,
22
23
  });
23
24
  if (frameDebugExtra) {
24
25
  frameDebugExtra.keyframeIndex = j;
@@ -36,17 +37,19 @@ class Matcher {
36
37
  const screenCoords = [];
37
38
  const worldCoords = [];
38
39
  const keyframe = keyframes[bestResult.keyframeIndex];
39
- const scale = keyframe.s || keyframe.scale || 1.0;
40
+ const kfScale = keyframe.s || keyframe.scale || 1.0;
40
41
  for (let i = 0; i < bestResult.matches.length; i++) {
41
42
  const querypoint = bestResult.matches[i].querypoint;
42
43
  const keypoint = bestResult.matches[i].keypoint;
44
+ // ๐Ÿš€ NANITE-STYLE: Use per-keypoint scale (octave) for accurate world mapping
45
+ const pointScale = keypoint.scale || kfScale;
43
46
  screenCoords.push({
44
47
  x: querypoint.x,
45
48
  y: querypoint.y,
46
49
  });
47
50
  worldCoords.push({
48
- x: (keypoint.x + 0.5) / scale,
49
- y: (keypoint.y + 0.5) / scale,
51
+ x: (keypoint.x + 0.5) / kfScale,
52
+ y: (keypoint.y + 0.5) / kfScale,
50
53
  z: 0,
51
54
  });
52
55
  }
@@ -1,9 +1,10 @@
1
- export function match({ keyframe, querypoints: rawQuerypoints, querywidth, queryheight, debugMode }: {
1
+ export function match({ keyframe, querypoints: rawQuerypoints, querywidth, queryheight, debugMode, expectedScale }: {
2
2
  keyframe: any;
3
3
  querypoints: any;
4
4
  querywidth: any;
5
5
  queryheight: any;
6
6
  debugMode: any;
7
+ expectedScale: any;
7
8
  }): {
8
9
  debugExtra: {
9
10
  constellationMatches: {
@@ -13,7 +13,7 @@ const HAMMING_THRESHOLD = AR_CONFIG.HAMMING_THRESHOLD;
13
13
  const HDC_RATIO_THRESHOLD = AR_CONFIG.HDC_RATIO_THRESHOLD;
14
14
  const MAX_MATCH_QUERY_POINTS = AR_CONFIG.MAX_MATCH_QUERY_POINTS;
15
15
  // match list of querpoints against pre-built list of keyframes
16
- const match = ({ keyframe, querypoints: rawQuerypoints, querywidth, queryheight, debugMode }) => {
16
+ const match = ({ keyframe, querypoints: rawQuerypoints, querywidth, queryheight, debugMode, expectedScale }) => {
17
17
  let debugExtra = {};
18
18
  // ๐ŸŽฏ Performance Optimizer: Use only the most "salient" points (highest response)
19
19
  const querypoints = rawQuerypoints.length > MAX_MATCH_QUERY_POINTS
@@ -23,8 +23,10 @@ const match = ({ keyframe, querypoints: rawQuerypoints, querywidth, queryheight,
23
23
  const qlen = querypoints.length;
24
24
  const kmax = keyframe.max;
25
25
  const kmin = keyframe.min;
26
+ // Detect descriptor mode: HDC (32-bit signature), Compact (32-bit XOR folded), or Raw (64-bit)
26
27
  const isHDC = keyframe.hdc === true || (kmax && kmax.hdc === 1);
27
- const descSize = isHDC ? 1 : 2;
28
+ const isCompact = (kmax && kmax.compact === 1) || (kmin && kmin.compact === 1);
29
+ const descSize = (isHDC || isCompact) ? 1 : 2; // Compact uses 32-bit like HDC
28
30
  const currentRatioThreshold = isHDC ? HDC_RATIO_THRESHOLD : HAMMING_THRESHOLD;
29
31
  for (let j = 0; j < qlen; j++) {
30
32
  const querypoint = querypoints[j];
@@ -42,19 +44,38 @@ const match = ({ keyframe, querypoints: rawQuerypoints, querywidth, queryheight,
42
44
  keypointIndexes,
43
45
  numPop: 0,
44
46
  isHDC,
45
- descSize
47
+ descSize,
48
+ isCompact
46
49
  });
47
50
  let bestIndex = -1;
48
51
  let bestD1 = Number.MAX_SAFE_INTEGER;
49
52
  let bestD2 = Number.MAX_SAFE_INTEGER;
50
53
  const qDesc = querypoint.descriptors;
51
54
  const cDesc = col.d;
55
+ // For compact mode: pre-compute XOR folded query descriptor (64-bit โ†’ 32-bit)
56
+ const qDescCompact = isCompact && qDesc && qDesc.length >= 2
57
+ ? (qDesc[0] ^ qDesc[1]) >>> 0
58
+ : 0;
52
59
  for (let k = 0; k < keypointIndexes.length; k++) {
53
60
  const idx = keypointIndexes[k];
61
+ // ๐Ÿš€ NANITE-STYLE: Dynamic scale filtering
62
+ // If we have an expected scale, skip points that are outside the resolution range
63
+ if (expectedScale !== undefined && col.s) {
64
+ const featureScale = col.s[idx]; // Octave scale (1, 2, 4...)
65
+ const idealKeyScale = (querypoint.scale || 1.0) / expectedScale;
66
+ // allow ~1 octave of margin
67
+ if (featureScale < idealKeyScale * 0.4 || featureScale > idealKeyScale * 2.5) {
68
+ continue;
69
+ }
70
+ }
54
71
  let d;
55
72
  if (isHDC) {
56
73
  d = popcount32(cDesc[idx] ^ querypoint.hdcSignature);
57
74
  }
75
+ else if (isCompact) {
76
+ // Compact mode: compare 32-bit XOR folded descriptors
77
+ d = popcount32(cDesc[idx] ^ qDescCompact);
78
+ }
58
79
  else {
59
80
  d = hammingCompute({ v1: cDesc, v1Offset: idx * descSize, v2: qDesc });
60
81
  }
@@ -85,10 +106,6 @@ const match = ({ keyframe, querypoints: rawQuerypoints, querywidth, queryheight,
85
106
  if (matches.length < MIN_NUM_INLIERS) {
86
107
  return { debugExtra };
87
108
  }
88
- // Debug: Log Hamming results
89
- if (Math.random() < 0.1 && debugMode) {
90
- console.log(`MATCH_DL: Hamming found ${matches.length} initial matches`);
91
- }
92
109
  // ๐ŸŒŒ Moonshot: Constellation matching disabled for performance calibration
93
110
  const constellationMatches = matches;
94
111
  if (debugMode)
@@ -100,8 +117,9 @@ const match = ({ keyframe, querypoints: rawQuerypoints, querywidth, queryheight,
100
117
  queryheight,
101
118
  matches: constellationMatches,
102
119
  });
103
- if (debugMode)
120
+ if (debugMode) {
104
121
  debugExtra.houghMatches = houghMatches;
122
+ }
105
123
  if (houghMatches.length < MIN_NUM_INLIERS) {
106
124
  return { debugExtra };
107
125
  }
@@ -147,6 +165,10 @@ const match = ({ keyframe, querypoints: rawQuerypoints, querywidth, queryheight,
147
165
  continue;
148
166
  const cx = col.x, cy = col.y, cd = col.d;
149
167
  const qDesc = querypoint.descriptors;
168
+ // For compact mode: XOR fold query descriptor
169
+ const qDescCompact = isCompact && qDesc && qDesc.length >= 2
170
+ ? (qDesc[0] ^ qDesc[1]) >>> 0
171
+ : 0;
150
172
  for (let k = 0, clen = cx.length; k < clen; k++) {
151
173
  const dx = cx[k] - mapX;
152
174
  const dy = cy[k] - mapY;
@@ -157,6 +179,9 @@ const match = ({ keyframe, querypoints: rawQuerypoints, querywidth, queryheight,
157
179
  if (isHDC) {
158
180
  d = popcount32(cd[k] ^ querypoint.hdcSignature);
159
181
  }
182
+ else if (isCompact) {
183
+ d = popcount32(cd[k] ^ qDescCompact);
184
+ }
160
185
  else {
161
186
  d = hammingCompute({ v1: cd, v1Offset: k * descSize, v2: qDesc });
162
187
  }
@@ -217,7 +242,7 @@ const match = ({ keyframe, querypoints: rawQuerypoints, querywidth, queryheight,
217
242
  });
218
243
  return { H: refinedH || H2, matches: inlierMatches2, debugExtra };
219
244
  };
220
- const _query = ({ node, descriptors, querypoint, queue, keypointIndexes, numPop, isHDC, descSize }) => {
245
+ const _query = ({ node, descriptors, querypoint, queue, keypointIndexes, numPop, isHDC, descSize, isCompact }) => {
221
246
  const isLeaf = node[0] === 1;
222
247
  const childrenOrIndices = node[2];
223
248
  if (isLeaf) {
@@ -227,6 +252,10 @@ const _query = ({ node, descriptors, querypoint, queue, keypointIndexes, numPop,
227
252
  return;
228
253
  }
229
254
  const qDesc = querypoint.descriptors;
255
+ // For compact mode: XOR fold query descriptor
256
+ const qDescCompact = isCompact && qDesc && qDesc.length >= 2
257
+ ? (qDesc[0] ^ qDesc[1]) >>> 0
258
+ : 0;
230
259
  let minD = Number.MAX_SAFE_INTEGER;
231
260
  const clen = childrenOrIndices.length;
232
261
  const distances = new Int32Array(clen);
@@ -237,6 +266,9 @@ const _query = ({ node, descriptors, querypoint, queue, keypointIndexes, numPop,
237
266
  if (isHDC) {
238
267
  d = popcount32(descriptors[cIdx] ^ querypoint.hdcSignature);
239
268
  }
269
+ else if (isCompact) {
270
+ d = popcount32(descriptors[cIdx] ^ qDescCompact);
271
+ }
240
272
  else {
241
273
  d = hammingCompute({
242
274
  v1: descriptors,
@@ -251,7 +283,7 @@ const _query = ({ node, descriptors, querypoint, queue, keypointIndexes, numPop,
251
283
  for (let i = 0; i < clen; i++) {
252
284
  const dist = distances[i];
253
285
  if (dist <= minD) {
254
- _query({ node: childrenOrIndices[i], descriptors, querypoint, queue, keypointIndexes, numPop: numPop + 1, isHDC, descSize });
286
+ _query({ node: childrenOrIndices[i], descriptors, querypoint, queue, keypointIndexes, numPop: numPop + 1, isHDC, descSize, isCompact });
255
287
  }
256
288
  else {
257
289
  queue.push({ node: childrenOrIndices[i], d: dist });
@@ -259,7 +291,7 @@ const _query = ({ node, descriptors, querypoint, queue, keypointIndexes, numPop,
259
291
  }
260
292
  if (numPop < CLUSTER_MAX_POP && queue.length > 0) {
261
293
  const { node } = queue.pop();
262
- _query({ node, descriptors, querypoint, queue, keypointIndexes, numPop: numPop + 1, isHDC, descSize });
294
+ _query({ node, descriptors, querypoint, queue, keypointIndexes, numPop: numPop + 1, isHDC, descSize, isCompact });
263
295
  }
264
296
  };
265
297
  const _findInlierMatches = (options) => {