@srsergio/taptapp-ar 1.0.88 → 1.0.90
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -10
- package/dist/compiler/offline-compiler.js +16 -4
- package/dist/core/detector/detector-lite.d.ts +1 -0
- package/dist/core/detector/detector-lite.js +31 -15
- package/dist/core/estimation/estimate.d.ts +7 -0
- package/dist/core/estimation/estimate.js +13 -48
- package/dist/core/estimation/morph-refinement.d.ts +8 -0
- package/dist/core/estimation/morph-refinement.js +116 -0
- package/dist/core/estimation/pnp-solver.d.ts +5 -0
- package/dist/core/estimation/pnp-solver.js +109 -0
- package/dist/core/input-loader.js +19 -2
- package/dist/core/matching/hdc.d.ts +27 -0
- package/dist/core/matching/hdc.js +102 -0
- package/dist/core/matching/hierarchical-clustering.d.ts +1 -3
- package/dist/core/matching/hierarchical-clustering.js +30 -29
- package/dist/core/matching/hough.js +12 -11
- package/dist/core/matching/matcher.d.ts +4 -0
- package/dist/core/matching/matcher.js +23 -8
- package/dist/core/matching/matching.d.ts +22 -2
- package/dist/core/matching/matching.js +169 -39
- package/dist/core/matching/ransacHomography.js +3 -6
- package/dist/core/protocol.d.ts +5 -3
- package/dist/core/protocol.js +28 -6
- package/dist/runtime/controller.js +19 -14
- package/dist/runtime/controller.worker.js +4 -1
- package/package.json +3 -2
- package/src/compiler/offline-compiler.ts +17 -4
- package/src/core/detector/detector-lite.js +32 -15
- package/src/core/estimation/estimate.js +14 -63
- package/src/core/estimation/morph-refinement.js +139 -0
- package/src/core/estimation/pnp-solver.js +131 -0
- package/src/core/input-loader.js +21 -2
- package/src/core/matching/hdc.ts +117 -0
- package/src/core/matching/hierarchical-clustering.js +30 -29
- package/src/core/matching/hough.js +12 -11
- package/src/core/matching/matcher.js +27 -9
- package/src/core/matching/matching.js +192 -39
- package/src/core/matching/ransacHomography.js +3 -6
- package/src/core/protocol.ts +26 -6
- package/src/runtime/controller.ts +20 -14
- package/src/runtime/controller.worker.js +4 -1
|
@@ -13,13 +13,15 @@
|
|
|
13
13
|
import { FREAKPOINTS } from "./freak.js";
|
|
14
14
|
import { gpuCompute } from "../utils/gpu-compute.js";
|
|
15
15
|
import { computeLSH64, computeFullFREAK, packLSHIntoDescriptor } from "../utils/lsh-direct.js";
|
|
16
|
+
import { generateBasis, projectDescriptor, compressToSignature } from "../matching/hdc.js";
|
|
17
|
+
import { HDC_SEED } from "../protocol.js";
|
|
16
18
|
|
|
17
19
|
const PYRAMID_MIN_SIZE = 4; // Restored to 4 for better small-scale detection
|
|
18
20
|
// PYRAMID_MAX_OCTAVE ya no es necesario, el límite lo da PYRAMID_MIN_SIZE
|
|
19
21
|
|
|
20
22
|
|
|
21
|
-
const NUM_BUCKETS_PER_DIMENSION = 10
|
|
22
|
-
const DEFAULT_MAX_FEATURES_PER_BUCKET = 8
|
|
23
|
+
const NUM_BUCKETS_PER_DIMENSION = 15; // Increased from 10 to 15 for better local detail
|
|
24
|
+
const DEFAULT_MAX_FEATURES_PER_BUCKET = 12; // Increased from 8 to 12
|
|
23
25
|
|
|
24
26
|
|
|
25
27
|
const ORIENTATION_NUM_BINS = 36;
|
|
@@ -46,6 +48,7 @@ export class DetectorLite {
|
|
|
46
48
|
this.useGPU = options.useGPU !== undefined ? options.useGPU : globalUseGPU;
|
|
47
49
|
// Protocol V6 (Moonshot): 64-bit LSH is the standard descriptor format
|
|
48
50
|
this.useLSH = options.useLSH !== undefined ? options.useLSH : true;
|
|
51
|
+
this.useHDC = options.useHDC !== undefined ? options.useHDC : true; // Enabled by default for Moonshot
|
|
49
52
|
this.maxFeaturesPerBucket = options.maxFeaturesPerBucket !== undefined ? options.maxFeaturesPerBucket : DEFAULT_MAX_FEATURES_PER_BUCKET;
|
|
50
53
|
|
|
51
54
|
let numOctaves = 0;
|
|
@@ -96,6 +99,17 @@ export class DetectorLite {
|
|
|
96
99
|
// 6. Calcular descriptores FREAK
|
|
97
100
|
this._computeFreakDescriptors(prunedExtremas, pyramidImages);
|
|
98
101
|
|
|
102
|
+
// 7. 🚀 MOONSHOT: HDC Hyper-projection
|
|
103
|
+
if (this.useHDC) {
|
|
104
|
+
const hdcBasis = generateBasis(HDC_SEED, 1024);
|
|
105
|
+
for (const ext of prunedExtremas) {
|
|
106
|
+
if (ext.lsh) {
|
|
107
|
+
const hv = projectDescriptor(ext.lsh, hdcBasis);
|
|
108
|
+
ext.hdcSignature = compressToSignature(hv);
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
99
113
|
// Convertir a formato de salida
|
|
100
114
|
const featurePoints = prunedExtremas.map(ext => {
|
|
101
115
|
const scale = Math.pow(2, ext.octave);
|
|
@@ -105,7 +119,9 @@ export class DetectorLite {
|
|
|
105
119
|
y: ext.y * scale + scale * 0.5 - 0.5,
|
|
106
120
|
scale: scale,
|
|
107
121
|
angle: ext.angle || 0,
|
|
108
|
-
descriptors: (this.useLSH && ext.lsh) ? ext.lsh : (ext.descriptors || [])
|
|
122
|
+
descriptors: (this.useLSH && ext.lsh) ? ext.lsh : (ext.descriptors || []),
|
|
123
|
+
hdcSignature: ext.hdcSignature || 0,
|
|
124
|
+
imageData: data // Pass source image for refinement
|
|
109
125
|
};
|
|
110
126
|
});
|
|
111
127
|
|
|
@@ -182,9 +198,10 @@ export class DetectorLite {
|
|
|
182
198
|
for (let y = 0; y < height; y++) {
|
|
183
199
|
const rowOffset = y * width;
|
|
184
200
|
|
|
185
|
-
// Left border
|
|
186
|
-
|
|
187
|
-
temp[rowOffset
|
|
201
|
+
// Left border (Normalized)
|
|
202
|
+
const sumL0 = k0 + k1 + k2 + k1 + k0; // Ideal sum
|
|
203
|
+
temp[rowOffset] = (data[rowOffset] * (k0 + k1 + k2) + data[rowOffset + 1] * k1 + data[rowOffset + 2] * k0) * (1.0 / (k0 + k1 + k2));
|
|
204
|
+
temp[rowOffset + 1] = (data[rowOffset] * k1 + data[rowOffset + 1] * k2 + data[rowOffset + 2] * k1 + data[rowOffset + 3] * k0) * (1.0 / (k1 + k2 + k1 + k0));
|
|
188
205
|
|
|
189
206
|
// Main loop - NO boundary checks
|
|
190
207
|
for (let x = 2; x < width - 2; x++) {
|
|
@@ -192,18 +209,18 @@ export class DetectorLite {
|
|
|
192
209
|
temp[pos] = data[pos - 2] * k0 + data[pos - 1] * k1 + data[pos] * k2 + data[pos + 1] * k1 + data[pos + 2] * k0;
|
|
193
210
|
}
|
|
194
211
|
|
|
195
|
-
// Right border
|
|
212
|
+
// Right border (Normalized)
|
|
196
213
|
const r2 = rowOffset + width - 2;
|
|
197
214
|
const r1 = rowOffset + width - 1;
|
|
198
|
-
temp[r2] = data[r2 - 2] * k0 + data[r2 - 1] * k1 + data[r2] * k2 + data[r1] * k1;
|
|
199
|
-
temp[r1] = data[r1 - 2] * k0 + data[r1 - 1] * k1 + data[r1] * (k2 + k1 + k0);
|
|
215
|
+
temp[r2] = (data[r2 - 2] * k0 + data[r2 - 1] * k1 + data[r2] * k2 + data[r1] * k1) * (1.0 / (k0 + k1 + k2 + k1));
|
|
216
|
+
temp[r1] = (data[r1 - 2] * k0 + data[r1 - 1] * k1 + data[r1] * (k2 + k1 + k0)) * (1.0 / (k0 + k1 + k2));
|
|
200
217
|
}
|
|
201
218
|
|
|
202
219
|
// Vertical pass - Speed optimized
|
|
203
220
|
for (let x = 0; x < width; x++) {
|
|
204
|
-
// Top border
|
|
205
|
-
output[x] = temp[x] * (k0 + k1 + k2) + temp[x + width] * k1 + temp[x + width * 2] * k0;
|
|
206
|
-
output[x + width] = temp[x] * k1 + temp[x + width] * k2 + temp[x + width * 2] * k1 + temp[x + width * 3] * k0;
|
|
221
|
+
// Top border (Normalized)
|
|
222
|
+
output[x] = (temp[x] * (k0 + k1 + k2) + temp[x + width] * k1 + temp[x + width * 2] * k0) * (1.0 / (k0 + k1 + k2));
|
|
223
|
+
output[x + width] = (temp[x] * k1 + temp[x + width] * k2 + temp[x + width * 2] * k1 + temp[x + width * 3] * k0) * (1.0 / (k1 + k2 + k1 + k0));
|
|
207
224
|
|
|
208
225
|
// Main loop - NO boundary checks
|
|
209
226
|
for (let y = 2; y < height - 2; y++) {
|
|
@@ -211,11 +228,11 @@ export class DetectorLite {
|
|
|
211
228
|
output[p] = temp[p - width * 2] * k0 + temp[p - width] * k1 + temp[p] * k2 + temp[p + width] * k1 + temp[p + width * 2] * k0;
|
|
212
229
|
}
|
|
213
230
|
|
|
214
|
-
// Bottom border
|
|
231
|
+
// Bottom border (Normalized)
|
|
215
232
|
const b2 = (height - 2) * width + x;
|
|
216
233
|
const b1 = (height - 1) * width + x;
|
|
217
|
-
output[b2] = temp[b2 - width * 2] * k0 + temp[b2 - width] * k1 + temp[b2] * k2 + temp[b1] * k1;
|
|
218
|
-
output[b1] = temp[b1 - width * 2] * k0 + temp[b1 - width] * k1 + temp[b1] * (k2 + k1 + k0);
|
|
234
|
+
output[b2] = (temp[b2 - width * 2] * k0 + temp[b2 - width] * k1 + temp[b2] * k2 + temp[b1] * k1) * (1.0 / (k0 + k1 + k2 + k1));
|
|
235
|
+
output[b1] = (temp[b1 - width * 2] * k0 + temp[b1 - width] * k1 + temp[b1] * (k2 + k1 + k0)) * (1.0 / (k0 + k1 + k2));
|
|
219
236
|
}
|
|
220
237
|
|
|
221
238
|
return { data: output, width, height };
|
|
@@ -1,67 +1,18 @@
|
|
|
1
|
-
import {
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
// Ref: can anyone provide reference?
|
|
1
|
+
import { solvePosePnP } from "./pnp-solver.js";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* 🚀 MOONSHOT: Direct PnP Solver for AR
|
|
5
|
+
*
|
|
6
|
+
* Instead of estimating a 2D Homography and decomposing it,
|
|
7
|
+
* we solve for the 3D Pose [R|t] directly using the
|
|
8
|
+
* Perspective-n-Point algorithm.
|
|
9
|
+
*/
|
|
11
10
|
const estimate = ({ screenCoords, worldCoords, projectionTransform }) => {
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
[Harray[0], Harray[1], Harray[2]],
|
|
18
|
-
[Harray[3], Harray[4], Harray[5]],
|
|
19
|
-
[Harray[6], Harray[7], Harray[8]],
|
|
20
|
-
]);
|
|
21
|
-
|
|
22
|
-
const K = new Matrix(projectionTransform);
|
|
23
|
-
const KInv = inverse(K);
|
|
24
|
-
|
|
25
|
-
const _KInvH = KInv.mmul(H);
|
|
26
|
-
const KInvH = _KInvH.to1DArray();
|
|
27
|
-
|
|
28
|
-
const norm1 = Math.sqrt(KInvH[0] * KInvH[0] + KInvH[3] * KInvH[3] + KInvH[6] * KInvH[6]);
|
|
29
|
-
const norm2 = Math.sqrt(KInvH[1] * KInvH[1] + KInvH[4] * KInvH[4] + KInvH[7] * KInvH[7]);
|
|
30
|
-
const tnorm = (norm1 + norm2) / 2;
|
|
31
|
-
|
|
32
|
-
const rotate = [];
|
|
33
|
-
rotate[0] = KInvH[0] / norm1;
|
|
34
|
-
rotate[3] = KInvH[3] / norm1;
|
|
35
|
-
rotate[6] = KInvH[6] / norm1;
|
|
36
|
-
|
|
37
|
-
rotate[1] = KInvH[1] / norm2;
|
|
38
|
-
rotate[4] = KInvH[4] / norm2;
|
|
39
|
-
rotate[7] = KInvH[7] / norm2;
|
|
40
|
-
|
|
41
|
-
rotate[2] = rotate[3] * rotate[7] - rotate[6] * rotate[4];
|
|
42
|
-
rotate[5] = rotate[6] * rotate[1] - rotate[0] * rotate[7];
|
|
43
|
-
rotate[8] = rotate[0] * rotate[4] - rotate[1] * rotate[3];
|
|
44
|
-
|
|
45
|
-
const norm3 = Math.sqrt(rotate[2] * rotate[2] + rotate[5] * rotate[5] + rotate[8] * rotate[8]);
|
|
46
|
-
rotate[2] /= norm3;
|
|
47
|
-
rotate[5] /= norm3;
|
|
48
|
-
rotate[8] /= norm3;
|
|
49
|
-
|
|
50
|
-
// TODO: artoolkit has check_rotation() that somehow switch the rotate vector. not sure what that does. Can anyone advice?
|
|
51
|
-
// https://github.com/artoolkitx/artoolkit5/blob/5bf0b671ff16ead527b9b892e6aeb1a2771f97be/lib/SRC/ARICP/icpUtil.c#L215
|
|
52
|
-
|
|
53
|
-
const tran = [];
|
|
54
|
-
tran[0] = KInvH[2] / tnorm;
|
|
55
|
-
tran[1] = KInvH[5] / tnorm;
|
|
56
|
-
tran[2] = KInvH[8] / tnorm;
|
|
57
|
-
|
|
58
|
-
let initialModelViewTransform = [
|
|
59
|
-
[rotate[0], rotate[1], rotate[2], tran[0]],
|
|
60
|
-
[rotate[3], rotate[4], rotate[5], tran[1]],
|
|
61
|
-
[rotate[6], rotate[7], rotate[8], tran[2]],
|
|
62
|
-
];
|
|
63
|
-
|
|
64
|
-
return initialModelViewTransform;
|
|
11
|
+
return solvePosePnP({
|
|
12
|
+
screenCoords,
|
|
13
|
+
worldCoords,
|
|
14
|
+
projectionTransform
|
|
15
|
+
});
|
|
65
16
|
};
|
|
66
17
|
|
|
67
18
|
export { estimate };
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Morphological Refinement - "Active Edge Alignment"
|
|
3
|
+
*
|
|
4
|
+
* This Moonshot algorithm snaps the projected target to the real image edges.
|
|
5
|
+
* It solves the "Small Box / Drift" problem by maximizing alignment with
|
|
6
|
+
* local image gradients using a Spring-Mass optimization system.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import { Matrix, SingularValueDecomposition } from "ml-matrix";
|
|
10
|
+
|
|
11
|
+
export function refineWithMorphology({
|
|
12
|
+
imageData,
|
|
13
|
+
width,
|
|
14
|
+
height,
|
|
15
|
+
targetData,
|
|
16
|
+
initialH,
|
|
17
|
+
iterations = 3
|
|
18
|
+
}) {
|
|
19
|
+
let currentH = [...initialH];
|
|
20
|
+
|
|
21
|
+
// 1. Boundary Points (The "Anchors" of our elastic malla)
|
|
22
|
+
const boundaryPoints = [];
|
|
23
|
+
const step = 0.05;
|
|
24
|
+
for (let i = 0; i <= 1.0; i += step) {
|
|
25
|
+
boundaryPoints.push({ x: i * targetData.w, y: 0 });
|
|
26
|
+
boundaryPoints.push({ x: i * targetData.w, y: targetData.h });
|
|
27
|
+
boundaryPoints.push({ x: 0, y: i * targetData.h });
|
|
28
|
+
boundaryPoints.push({ x: targetData.w, y: i * targetData.h });
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
for (let iter = 0; iter < iterations; iter++) {
|
|
32
|
+
const correspondences = [];
|
|
33
|
+
|
|
34
|
+
for (const pt of boundaryPoints) {
|
|
35
|
+
// Project
|
|
36
|
+
const w = currentH[6] * pt.x + currentH[7] * pt.y + currentH[8];
|
|
37
|
+
const sx = (currentH[0] * pt.x + currentH[1] * pt.y + currentH[2]) / w;
|
|
38
|
+
const sy = (currentH[3] * pt.x + currentH[4] * pt.y + currentH[5]) / w;
|
|
39
|
+
|
|
40
|
+
if (sx < 2 || sx >= width - 2 || sy < 2 || sy >= height - 2) continue;
|
|
41
|
+
|
|
42
|
+
// 2. Local Gradient Search (The "Pull" of the image)
|
|
43
|
+
const searchDist = 10;
|
|
44
|
+
let bestX = sx;
|
|
45
|
+
let bestY = sy;
|
|
46
|
+
let maxGrad = -1;
|
|
47
|
+
|
|
48
|
+
for (let dy = -searchDist; dy <= searchDist; dy += 2) {
|
|
49
|
+
for (let dx = -searchDist; dx <= searchDist; dx += 2) {
|
|
50
|
+
const nx = Math.floor(sx + dx);
|
|
51
|
+
const ny = Math.floor(sy + dy);
|
|
52
|
+
|
|
53
|
+
const idx = ny * width + nx;
|
|
54
|
+
// Sobel-like gradient magnitude
|
|
55
|
+
const gx = imageData[idx + 1] - imageData[idx - 1];
|
|
56
|
+
const gy = imageData[idx + width] - imageData[idx - width];
|
|
57
|
+
const grad = gx * gx + gy * gy;
|
|
58
|
+
|
|
59
|
+
if (grad > maxGrad) {
|
|
60
|
+
maxGrad = grad;
|
|
61
|
+
bestX = nx;
|
|
62
|
+
bestY = ny;
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
if (maxGrad > 500) {
|
|
68
|
+
correspondences.push({
|
|
69
|
+
src: pt,
|
|
70
|
+
dst: { x: bestX, y: bestY },
|
|
71
|
+
weight: Math.min(1.0, maxGrad / 15000)
|
|
72
|
+
});
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
if (correspondences.length < 10) break;
|
|
77
|
+
|
|
78
|
+
// 3. Solve for best Homography using SVD
|
|
79
|
+
const nextH = solveDLTWeight(correspondences);
|
|
80
|
+
if (nextH) {
|
|
81
|
+
// Soft-Update (Momentum)
|
|
82
|
+
for (let i = 0; i < 9; i++) {
|
|
83
|
+
currentH[i] = currentH[i] * 0.5 + nextH[i] * 0.5;
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
return currentH;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/**
|
|
92
|
+
* Direct Linear Transform with Weights
|
|
93
|
+
*/
|
|
94
|
+
function solveDLTWeight(pairs) {
|
|
95
|
+
const n = pairs.length;
|
|
96
|
+
const A = new Matrix(n * 2, 9);
|
|
97
|
+
|
|
98
|
+
for (let i = 0; i < n; i++) {
|
|
99
|
+
const { src, dst, weight: w } = pairs[i];
|
|
100
|
+
const x = src.x;
|
|
101
|
+
const y = src.y;
|
|
102
|
+
const xp = dst.x;
|
|
103
|
+
const yp = dst.y;
|
|
104
|
+
|
|
105
|
+
// Row 1
|
|
106
|
+
A.set(i * 2, 0, 0);
|
|
107
|
+
A.set(i * 2, 1, 0);
|
|
108
|
+
A.set(i * 2, 2, 0);
|
|
109
|
+
A.set(i * 2, 3, -x * w);
|
|
110
|
+
A.set(i * 2, 4, -y * w);
|
|
111
|
+
A.set(i * 2, 5, -w);
|
|
112
|
+
A.set(i * 2, 6, yp * x * w);
|
|
113
|
+
A.set(i * 2, 7, yp * y * w);
|
|
114
|
+
A.set(i * 2, 8, yp * w);
|
|
115
|
+
|
|
116
|
+
// Row 2
|
|
117
|
+
A.set(i * 2 + 1, 0, x * w);
|
|
118
|
+
A.set(i * 2 + 1, 1, y * w);
|
|
119
|
+
A.set(i * 2 + 1, 2, w);
|
|
120
|
+
A.set(i * 2 + 1, 3, 0);
|
|
121
|
+
A.set(i * 2 + 1, 4, 0);
|
|
122
|
+
A.set(i * 2 + 1, 5, 0);
|
|
123
|
+
A.set(i * 2 + 1, 6, -xp * x * w);
|
|
124
|
+
A.set(i * 2 + 1, 7, -xp * y * w);
|
|
125
|
+
A.set(i * 2 + 1, 8, -xp * w);
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
try {
|
|
129
|
+
const svd = new SingularValueDecomposition(A);
|
|
130
|
+
const V = svd.rightSingularVectors;
|
|
131
|
+
// Last column of V is the solution
|
|
132
|
+
const h = V.getColumn(8);
|
|
133
|
+
// Normalize H[8] to 1
|
|
134
|
+
const scale = 1.0 / h[8];
|
|
135
|
+
return h.map(v => v * scale);
|
|
136
|
+
} catch (e) {
|
|
137
|
+
return null;
|
|
138
|
+
}
|
|
139
|
+
}
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Direct PnP (Perspective-n-Point) Solver for Planar Targets
|
|
3
|
+
*
|
|
4
|
+
* This Moonshot algorithm ignores octave-relative scales and works
|
|
5
|
+
* purely in Physical World Units. It uses the Camera Matrix (K)
|
|
6
|
+
* to deduce the real-world distance (Z).
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import { Matrix, SingularValueDecomposition } from "ml-matrix";
|
|
10
|
+
|
|
11
|
+
export function solvePosePnP({
|
|
12
|
+
screenCoords,
|
|
13
|
+
worldCoords,
|
|
14
|
+
projectionTransform
|
|
15
|
+
}) {
|
|
16
|
+
const K = new Matrix(projectionTransform);
|
|
17
|
+
const n = screenCoords.length;
|
|
18
|
+
|
|
19
|
+
// 1. Build the DLT matrix for Pose (Directly estimating [R|t])
|
|
20
|
+
// We assume worldCoords are [X, Y, 0]
|
|
21
|
+
// Eq: x = K * [R|t] * X
|
|
22
|
+
// K^-1 * x = [r1 r2 t] * [X Y 1]^T
|
|
23
|
+
|
|
24
|
+
const KI = Inverse3x3(projectionTransform);
|
|
25
|
+
|
|
26
|
+
const A = new Matrix(n * 2, 9);
|
|
27
|
+
for (let i = 0; i < n; i++) {
|
|
28
|
+
const sci = screenCoords[i];
|
|
29
|
+
const wci = worldCoords[i];
|
|
30
|
+
|
|
31
|
+
// Normalized camera coordinates
|
|
32
|
+
const nx = KI[0] * sci.x + KI[1] * sci.y + KI[2];
|
|
33
|
+
const ny = KI[3] * sci.x + KI[4] * sci.y + KI[5];
|
|
34
|
+
const nz = KI[6] * sci.x + KI[7] * sci.y + KI[8];
|
|
35
|
+
const unx = nx / nz;
|
|
36
|
+
const uny = ny / nz;
|
|
37
|
+
|
|
38
|
+
// DLT equations for [r11 r12 r21 r22 r31 r32 t1 t2 t3]
|
|
39
|
+
const X = wci.x;
|
|
40
|
+
const Y = wci.y;
|
|
41
|
+
|
|
42
|
+
// Row 1: X*r11 + Y*r12 + t1 - unx*(X*r31 + Y*r32 + t3) = 0
|
|
43
|
+
A.set(i * 2, 0, X);
|
|
44
|
+
A.set(i * 2, 1, Y);
|
|
45
|
+
A.set(i * 2, 2, 1);
|
|
46
|
+
A.set(i * 2, 3, 0);
|
|
47
|
+
A.set(i * 2, 4, 0);
|
|
48
|
+
A.set(i * 2, 5, 0);
|
|
49
|
+
A.set(i * 2, 6, -unx * X);
|
|
50
|
+
A.set(i * 2, 7, -unx * Y);
|
|
51
|
+
A.set(i * 2, 8, -unx);
|
|
52
|
+
|
|
53
|
+
// Row 2: X*r21 + Y*r22 + t2 - uny*(X*r31 + Y*r32 + t3) = 0
|
|
54
|
+
A.set(i * 2 + 1, 0, 0);
|
|
55
|
+
A.set(i * 2 + 1, 1, 0);
|
|
56
|
+
A.set(i * 2 + 1, 2, 0);
|
|
57
|
+
A.set(i * 2 + 1, 3, X);
|
|
58
|
+
A.set(i * 2 + 1, 4, Y);
|
|
59
|
+
A.set(i * 2 + 1, 5, 1);
|
|
60
|
+
A.set(i * 2 + 1, 6, -uny * X);
|
|
61
|
+
A.set(i * 2 + 1, 7, -uny * Y);
|
|
62
|
+
A.set(i * 2 + 1, 8, -uny);
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
// Solve via SVD
|
|
66
|
+
const svd = new SingularValueDecomposition(A);
|
|
67
|
+
const V = svd.rightSingularVectors;
|
|
68
|
+
const sol = V.getColumn(8); // last column
|
|
69
|
+
|
|
70
|
+
// 3. Extract r1, r2 and t from the DLT solution
|
|
71
|
+
// Standard DLT has an overall sign ambiguity. We force sol[8] (t3) to be positive.
|
|
72
|
+
if (sol[8] < 0) {
|
|
73
|
+
for (let i = 0; i < 9; i++) sol[i] = -sol[i];
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
const r1_raw = [sol[0], sol[3], sol[6]];
|
|
77
|
+
const r2_raw = [sol[1], sol[4], sol[7]];
|
|
78
|
+
const t_raw = [sol[2], sol[5], sol[8]];
|
|
79
|
+
|
|
80
|
+
const scale1 = Math.sqrt(r1_raw[0] ** 2 + r1_raw[1] ** 2 + r1_raw[2] ** 2);
|
|
81
|
+
const scale2 = Math.sqrt(r2_raw[0] ** 2 + r2_raw[1] ** 2 + r2_raw[2] ** 2);
|
|
82
|
+
const scale = (scale1 + scale2) / 2;
|
|
83
|
+
|
|
84
|
+
// 4. Construct Rotation Matrix and orthogonalize via SVD
|
|
85
|
+
const R_approx = new Matrix([
|
|
86
|
+
[r1_raw[0] / scale1, r2_raw[0] / scale2, 0],
|
|
87
|
+
[r1_raw[1] / scale1, r2_raw[1] / scale2, 0],
|
|
88
|
+
[r1_raw[2] / scale1, r2_raw[2] / scale2, 0]
|
|
89
|
+
]);
|
|
90
|
+
|
|
91
|
+
// R3 = R1 x R2
|
|
92
|
+
R_approx.set(0, 2, R_approx.get(1, 0) * R_approx.get(2, 1) - R_approx.get(2, 0) * R_approx.get(1, 1));
|
|
93
|
+
R_approx.set(1, 2, R_approx.get(2, 0) * R_approx.get(0, 1) - R_approx.get(0, 0) * R_approx.get(2, 1));
|
|
94
|
+
R_approx.set(2, 2, R_approx.get(0, 0) * R_approx.get(1, 1) - R_approx.get(1, 0) * R_approx.get(0, 1));
|
|
95
|
+
|
|
96
|
+
const svdRot = new SingularValueDecomposition(R_approx);
|
|
97
|
+
const U = svdRot.leftSingularVectors;
|
|
98
|
+
const Vrot = svdRot.rightSingularVectors;
|
|
99
|
+
let R = U.mmul(Vrot.transpose());
|
|
100
|
+
|
|
101
|
+
const getDet3 = (m) => {
|
|
102
|
+
return m.get(0, 0) * (m.get(1, 1) * m.get(2, 2) - m.get(1, 2) * m.get(2, 1)) -
|
|
103
|
+
m.get(0, 1) * (m.get(1, 0) * m.get(2, 2) - m.get(1, 2) * m.get(2, 0)) +
|
|
104
|
+
m.get(0, 2) * (m.get(1, 0) * m.get(2, 1) - m.get(1, 1) * m.get(2, 0));
|
|
105
|
+
};
|
|
106
|
+
|
|
107
|
+
if (getDet3(R) < 0) {
|
|
108
|
+
const U_mat = U.clone();
|
|
109
|
+
for (let i = 0; i < 3; i++) U_mat.set(i, 2, -U_mat.get(i, 2));
|
|
110
|
+
R = U_mat.mmul(Vrot.transpose());
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
return [
|
|
114
|
+
[R.get(0, 0), R.get(0, 1), R.get(0, 2), t_raw[0] / scale],
|
|
115
|
+
[R.get(1, 0), R.get(1, 1), R.get(1, 2), t_raw[1] / scale],
|
|
116
|
+
[R.get(2, 0), R.get(2, 1), R.get(2, 2), t_raw[2] / scale]
|
|
117
|
+
];
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
function Inverse3x3(m) {
|
|
121
|
+
const k00 = m[0][0], k01 = m[0][1], k02 = m[0][2];
|
|
122
|
+
const k10 = m[1][0], k11 = m[1][1], k12 = m[1][2];
|
|
123
|
+
const k20 = m[2][0], k21 = m[2][1], k22 = m[2][2];
|
|
124
|
+
const det = k00 * (k11 * k22 - k21 * k12) - k01 * (k10 * k22 - k12 * k20) + k02 * (k10 * k21 - k11 * k20);
|
|
125
|
+
const invDet = 1.0 / det;
|
|
126
|
+
return [
|
|
127
|
+
(k11 * k22 - k12 * k21) * invDet, (k02 * k21 - k01 * k22) * invDet, (k01 * k12 - k02 * k11) * invDet,
|
|
128
|
+
(k12 * k20 - k10 * k22) * invDet, (k00 * k22 - k02 * k20) * invDet, (k10 * k02 - k00 * k12) * invDet,
|
|
129
|
+
(k10 * k21 - k11 * k20) * invDet, (k20 * k01 - k21 * k00) * invDet, (k00 * k11 - k10 * k01) * invDet
|
|
130
|
+
];
|
|
131
|
+
}
|
package/src/core/input-loader.js
CHANGED
|
@@ -38,14 +38,33 @@ class InputLoader {
|
|
|
38
38
|
|
|
39
39
|
const isInputRotated = input.width === this.height && input.height === this.width;
|
|
40
40
|
|
|
41
|
+
const inputW = isInputRotated ? input.height : input.width;
|
|
42
|
+
const inputH = isInputRotated ? input.width : input.height;
|
|
43
|
+
const inputAspect = inputW / inputH;
|
|
44
|
+
const canvasAspect = this.width / this.height;
|
|
45
|
+
|
|
46
|
+
let sx = 0, sy = 0, sw = inputW, sh = inputH;
|
|
47
|
+
|
|
48
|
+
if (inputAspect > canvasAspect) {
|
|
49
|
+
// Input is wider than canvas - crop sides
|
|
50
|
+
sw = inputH * canvasAspect;
|
|
51
|
+
sx = (inputW - sw) / 2;
|
|
52
|
+
} else if (inputAspect < canvasAspect) {
|
|
53
|
+
// Input is taller than canvas - crop top/bottom
|
|
54
|
+
sh = inputW / canvasAspect;
|
|
55
|
+
sy = (inputH - sh) / 2;
|
|
56
|
+
}
|
|
57
|
+
|
|
41
58
|
if (isInputRotated) {
|
|
42
59
|
this.context.save();
|
|
43
60
|
this.context.translate(this.width / 2, this.height / 2);
|
|
44
61
|
this.context.rotate(Math.PI / 2);
|
|
45
|
-
|
|
62
|
+
// Map source crop (relative to rotated input)
|
|
63
|
+
// Since input is already rotated, we crop based on the rotated dimensions
|
|
64
|
+
this.context.drawImage(input, sx, sy, sw, sh, -this.height / 2, -this.width / 2, this.height, this.width);
|
|
46
65
|
this.context.restore();
|
|
47
66
|
} else {
|
|
48
|
-
this.context.drawImage(input, 0, 0, this.width, this.height);
|
|
67
|
+
this.context.drawImage(input, sx, sy, sw, sh, 0, 0, this.width, this.height);
|
|
49
68
|
}
|
|
50
69
|
|
|
51
70
|
const imageData = this.context.getImageData(0, 0, this.width, this.height);
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Hyperdimensional Computing (HDC) Core for AR
|
|
3
|
+
*
|
|
4
|
+
* Provides ultra-fast, ultra-compressed feature matching using
|
|
5
|
+
* High-Dimensional Random Vectors.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
export const HDC_DIMENSION = 1024; // bits
|
|
9
|
+
export const HDC_WORDS = HDC_DIMENSION / 32;
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* Deterministic Random Number Generator (PCG-like)
|
|
13
|
+
*/
|
|
14
|
+
class PRNG {
|
|
15
|
+
state: number;
|
|
16
|
+
constructor(seed: number) {
|
|
17
|
+
this.state = seed;
|
|
18
|
+
}
|
|
19
|
+
next() {
|
|
20
|
+
this.state = (this.state * 1664525 + 1013904223) >>> 0;
|
|
21
|
+
return this.state / 0xFFFFFFFF;
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* Generates a deterministic basis of Hypervectors
|
|
27
|
+
*/
|
|
28
|
+
export function generateBasis(seed: number, count: number): Uint32Array[] {
|
|
29
|
+
const prng = new PRNG(seed);
|
|
30
|
+
const basis: Uint32Array[] = [];
|
|
31
|
+
for (let i = 0; i < count; i++) {
|
|
32
|
+
const hv = new Uint32Array(HDC_WORDS);
|
|
33
|
+
for (let j = 0; j < HDC_WORDS; j++) {
|
|
34
|
+
hv[j] = (prng.next() * 0xFFFFFFFF) >>> 0;
|
|
35
|
+
}
|
|
36
|
+
basis.push(hv);
|
|
37
|
+
}
|
|
38
|
+
return basis;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Projects a 64-bit descriptor into the Hyperdimensional Space
|
|
43
|
+
* Uses "Random Projection" logic (Locality Sensitive Hashing in HDC)
|
|
44
|
+
*/
|
|
45
|
+
export function projectDescriptor(desc: Uint32Array, basis: Uint32Array[]): Uint32Array {
|
|
46
|
+
const result = new Uint32Array(HDC_WORDS);
|
|
47
|
+
|
|
48
|
+
// For each bit in the HDC space
|
|
49
|
+
for (let i = 0; i < HDC_DIMENSION; i++) {
|
|
50
|
+
const wordIdx = i >>> 5;
|
|
51
|
+
const bitIdx = i & 31;
|
|
52
|
+
|
|
53
|
+
// This is a simplified random projection
|
|
54
|
+
// In a real HDC system, we'd use more complex binding
|
|
55
|
+
// But for Vanilla JS performance, we use bitwise voting
|
|
56
|
+
let sum = 0;
|
|
57
|
+
const b = basis[i % basis.length];
|
|
58
|
+
|
|
59
|
+
// Dot product between descriptor and basis vector (subset)
|
|
60
|
+
// desc[0] and desc[1] are the 64 bits
|
|
61
|
+
for (let j = 0; j < 2; j++) {
|
|
62
|
+
sum += popcount(desc[j] & b[j]);
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
if (sum > 16) { // Threshold for "firing"
|
|
66
|
+
result[wordIdx] |= (1 << bitIdx);
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
return result;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* Compresses an HDC vector into an "Ultra-Short Signature" (32 bits)
|
|
75
|
+
* This allows storing 1000 points in just 4KB of descriptors.
|
|
76
|
+
*/
|
|
77
|
+
export function compressToSignature(hv: Uint32Array): number {
|
|
78
|
+
// FNV-1a Hash for robust 32-bit compression
|
|
79
|
+
let h1 = 0x811c9dc5;
|
|
80
|
+
for (let i = 0; i < hv.length; i++) {
|
|
81
|
+
h1 ^= hv[i];
|
|
82
|
+
h1 = Math.imul(h1, 0x01000193);
|
|
83
|
+
}
|
|
84
|
+
return h1 >>> 0;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
function popcount(n: number): number {
|
|
88
|
+
n = n - ((n >> 1) & 0x55555555);
|
|
89
|
+
n = (n & 0x33333333) + ((n >> 2) & 0x33333333);
|
|
90
|
+
return (((n + (n >> 4)) & 0x0F0F0F0F) * 0x01010101) >> 24;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* Bundles multiple points into a single Global Hypervector (The "Image DNA")
|
|
95
|
+
* This allows checking if an image is present with ONE vector comparison.
|
|
96
|
+
*/
|
|
97
|
+
export function bundle(hvs: Uint32Array[]): Uint32Array {
|
|
98
|
+
const global = new Uint32Array(HDC_WORDS);
|
|
99
|
+
const threshold = hvs.length / 2;
|
|
100
|
+
const counters = new Uint16Array(HDC_DIMENSION);
|
|
101
|
+
|
|
102
|
+
for (const hv of hvs) {
|
|
103
|
+
for (let i = 0; i < HDC_DIMENSION; i++) {
|
|
104
|
+
if (hv[i >>> 5] & (1 << (i & 31))) {
|
|
105
|
+
counters[i]++;
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
for (let i = 0; i < HDC_DIMENSION; i++) {
|
|
111
|
+
if (counters[i] > threshold) {
|
|
112
|
+
global[i >>> 5] |= (1 << (i & 31));
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
return global;
|
|
117
|
+
}
|