@srsergio/taptapp-ar 1.0.32 → 1.0.33
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -10
- package/dist/compiler/controller.d.ts +11 -0
- package/dist/compiler/controller.js +12 -4
- package/dist/compiler/detector/crop-detector.js +1 -1
- package/dist/compiler/detector/detector-lite.d.ts +1 -0
- package/dist/compiler/detector/detector-lite.js +7 -1
- package/dist/compiler/matching/hamming-distance.js +33 -6
- package/dist/compiler/matching/matching.js +20 -16
- package/dist/compiler/node-worker.js +1 -1
- package/dist/compiler/offline-compiler.d.ts +28 -24
- package/dist/compiler/offline-compiler.js +112 -35
- package/dist/compiler/simple-ar.d.ts +0 -7
- package/dist/compiler/simple-ar.js +7 -37
- package/dist/compiler/tracker/tracker.js +1 -1
- package/dist/compiler/utils/lsh-binarizer.d.ts +18 -0
- package/dist/compiler/utils/lsh-binarizer.js +37 -0
- package/dist/compiler/utils/projection.d.ts +22 -0
- package/dist/compiler/utils/projection.js +51 -0
- package/package.json +1 -1
- package/src/compiler/controller.js +12 -4
- package/src/compiler/detector/crop-detector.js +1 -1
- package/src/compiler/detector/detector-lite.js +7 -1
- package/src/compiler/matching/hamming-distance.js +38 -6
- package/src/compiler/matching/matching.js +21 -18
- package/src/compiler/node-worker.js +1 -1
- package/src/compiler/offline-compiler.js +128 -35
- package/src/compiler/simple-ar.js +8 -41
- package/src/compiler/tracker/tracker.js +1 -1
- package/src/compiler/utils/lsh-binarizer.js +43 -0
- package/src/compiler/utils/projection.js +58 -0
|
@@ -24,7 +24,7 @@ import { WorkerPool } from "./utils/worker-pool.js";
|
|
|
24
24
|
const isNode = typeof process !== "undefined" &&
|
|
25
25
|
process.versions != null &&
|
|
26
26
|
process.versions.node != null;
|
|
27
|
-
const CURRENT_VERSION =
|
|
27
|
+
const CURRENT_VERSION = 6; // Protocol v6: Moonshot - LSH 64-bit
|
|
28
28
|
/**
|
|
29
29
|
* Compilador offline optimizado sin TensorFlow
|
|
30
30
|
*/
|
|
@@ -32,9 +32,9 @@ export class OfflineCompiler {
|
|
|
32
32
|
constructor() {
|
|
33
33
|
this.data = null;
|
|
34
34
|
this.workerPool = null;
|
|
35
|
-
// Workers
|
|
35
|
+
// Workers only in Node.js (no en browser)
|
|
36
36
|
if (isNode) {
|
|
37
|
-
|
|
37
|
+
// Lazy init workers only when needed
|
|
38
38
|
}
|
|
39
39
|
else {
|
|
40
40
|
console.log("🌐 OfflineCompiler: Browser mode (no workers)");
|
|
@@ -59,7 +59,6 @@ export class OfflineCompiler {
|
|
|
59
59
|
// Limit workers to avoid freezing system
|
|
60
60
|
const numWorkers = Math.min(os.cpus().length, 4);
|
|
61
61
|
this.workerPool = new WorkerPool(workerPath, numWorkers, Worker);
|
|
62
|
-
console.log(`🚀 OfflineCompiler: Node.js mode with ${numWorkers} workers`);
|
|
63
62
|
}
|
|
64
63
|
catch (e) {
|
|
65
64
|
console.log("⚡ OfflineCompiler: Running without workers (initialization failed)", e);
|
|
@@ -119,6 +118,8 @@ export class OfflineCompiler {
|
|
|
119
118
|
const percentPerImage = 100 / targetImages.length;
|
|
120
119
|
let currentPercent = 0;
|
|
121
120
|
// Use workers if available
|
|
121
|
+
if (isNode)
|
|
122
|
+
await this._initNodeWorkers();
|
|
122
123
|
if (this.workerPool) {
|
|
123
124
|
const progressMap = new Float32Array(targetImages.length);
|
|
124
125
|
const wrappedPromises = targetImages.map((targetImage, index) => {
|
|
@@ -144,7 +145,7 @@ export class OfflineCompiler {
|
|
|
144
145
|
const percentPerScale = percentPerImage / imageList.length;
|
|
145
146
|
const keyframes = [];
|
|
146
147
|
for (const image of imageList) {
|
|
147
|
-
const detector = new DetectorLite(image.width, image.height);
|
|
148
|
+
const detector = new DetectorLite(image.width, image.height, { useLSH: true });
|
|
148
149
|
const { featurePoints: ps } = detector.detect(image.data);
|
|
149
150
|
const maximaPoints = ps.filter((p) => p.maxima);
|
|
150
151
|
const minimaPoints = ps.filter((p) => !p.maxima);
|
|
@@ -216,8 +217,9 @@ export class OfflineCompiler {
|
|
|
216
217
|
}
|
|
217
218
|
const dataList = this.data.map((item) => {
|
|
218
219
|
const matchingData = item.matchingData.map((kf) => this._packKeyframe(kf));
|
|
219
|
-
const trackingData = item.trackingData.map((td) => {
|
|
220
|
+
const trackingData = [item.trackingData[0]].map((td) => {
|
|
220
221
|
const count = td.points.length;
|
|
222
|
+
// Step 1: Packed Coords - Normalize width/height to 16-bit
|
|
221
223
|
const px = new Float32Array(count);
|
|
222
224
|
const py = new Float32Array(count);
|
|
223
225
|
for (let i = 0; i < count; i++) {
|
|
@@ -247,28 +249,60 @@ export class OfflineCompiler {
|
|
|
247
249
|
dataList,
|
|
248
250
|
});
|
|
249
251
|
}
|
|
252
|
+
_getMorton(x, y) {
|
|
253
|
+
// Interleave bits of x and y
|
|
254
|
+
let x_int = x | 0;
|
|
255
|
+
let y_int = y | 0;
|
|
256
|
+
x_int = (x_int | (x_int << 8)) & 0x00FF00FF;
|
|
257
|
+
x_int = (x_int | (x_int << 4)) & 0x0F0F0F0F;
|
|
258
|
+
x_int = (x_int | (x_int << 2)) & 0x33333333;
|
|
259
|
+
x_int = (x_int | (x_int << 1)) & 0x55555555;
|
|
260
|
+
y_int = (y_int | (y_int << 8)) & 0x00FF00FF;
|
|
261
|
+
y_int = (y_int | (y_int << 4)) & 0x0F0F0F0F;
|
|
262
|
+
y_int = (y_int | (y_int << 2)) & 0x33333333;
|
|
263
|
+
y_int = (y_int | (y_int << 1)) & 0x55555555;
|
|
264
|
+
return x_int | (y_int << 1);
|
|
265
|
+
}
|
|
250
266
|
_packKeyframe(kf) {
|
|
267
|
+
// Step 2.1: Morton Sorting - Sort points spatially to improve Delta-Descriptor XOR
|
|
268
|
+
const sortPoints = (points) => {
|
|
269
|
+
return [...points].sort((a, b) => {
|
|
270
|
+
return this._getMorton(a.x, a.y) - this._getMorton(b.x, b.y);
|
|
271
|
+
});
|
|
272
|
+
};
|
|
273
|
+
const sortedMaxima = sortPoints(kf.maximaPoints);
|
|
274
|
+
const sortedMinima = sortPoints(kf.minimaPoints);
|
|
275
|
+
// Rebuild clusters with sorted indices
|
|
276
|
+
const sortedMaximaCluster = hierarchicalClusteringBuild({ points: sortedMaxima });
|
|
277
|
+
const sortedMinimaCluster = hierarchicalClusteringBuild({ points: sortedMinima });
|
|
251
278
|
return {
|
|
252
279
|
w: kf.width,
|
|
253
280
|
h: kf.height,
|
|
254
281
|
s: kf.scale,
|
|
255
|
-
max: this._columnarize(kf.
|
|
256
|
-
min: this._columnarize(kf.
|
|
282
|
+
max: this._columnarize(sortedMaxima, sortedMaximaCluster, kf.width, kf.height),
|
|
283
|
+
min: this._columnarize(sortedMinima, sortedMinimaCluster, kf.width, kf.height),
|
|
257
284
|
};
|
|
258
285
|
}
|
|
259
|
-
_columnarize(points, tree) {
|
|
286
|
+
_columnarize(points, tree, width, height) {
|
|
260
287
|
const count = points.length;
|
|
261
|
-
|
|
262
|
-
const
|
|
263
|
-
const
|
|
264
|
-
|
|
265
|
-
const
|
|
288
|
+
// Step 1: Packed Coords - Normalize to 16-bit
|
|
289
|
+
const x = new Uint16Array(count);
|
|
290
|
+
const y = new Uint16Array(count);
|
|
291
|
+
// Step 1.1: Angle Quantization - Int16
|
|
292
|
+
const angle = new Int16Array(count);
|
|
293
|
+
// Step 1.2: Scale Indexing - Uint8
|
|
294
|
+
const scale = new Uint8Array(count);
|
|
295
|
+
// Step 3: LSH 64-bit Descriptors - Uint32Array (2 elements per point)
|
|
296
|
+
const descriptors = new Uint32Array(count * 2);
|
|
266
297
|
for (let i = 0; i < count; i++) {
|
|
267
|
-
x[i] = points[i].x;
|
|
268
|
-
y[i] = points[i].y;
|
|
269
|
-
angle[i] = points[i].angle;
|
|
270
|
-
scale[i] = points[i].scale;
|
|
271
|
-
|
|
298
|
+
x[i] = Math.round((points[i].x / width) * 65535);
|
|
299
|
+
y[i] = Math.round((points[i].y / height) * 65535);
|
|
300
|
+
angle[i] = Math.round((points[i].angle / Math.PI) * 32767);
|
|
301
|
+
scale[i] = Math.round(Math.log2(points[i].scale || 1));
|
|
302
|
+
if (points[i].descriptors && points[i].descriptors.length >= 2) {
|
|
303
|
+
descriptors[i * 2] = points[i].descriptors[0];
|
|
304
|
+
descriptors[(i * 2) + 1] = points[i].descriptors[1];
|
|
305
|
+
}
|
|
272
306
|
}
|
|
273
307
|
return {
|
|
274
308
|
x,
|
|
@@ -287,55 +321,98 @@ export class OfflineCompiler {
|
|
|
287
321
|
}
|
|
288
322
|
importData(buffer) {
|
|
289
323
|
const content = msgpack.decode(new Uint8Array(buffer));
|
|
290
|
-
|
|
291
|
-
|
|
324
|
+
const version = content.v || 0;
|
|
325
|
+
if (version !== CURRENT_VERSION && version !== 5) {
|
|
326
|
+
console.error(`Incompatible .mind version: ${version}. This engine only supports Protocol V5/V6.`);
|
|
292
327
|
return [];
|
|
293
328
|
}
|
|
294
|
-
|
|
329
|
+
const descSize = version >= 6 ? 2 : 4;
|
|
330
|
+
// Restore TypedArrays from Uint8Arrays returned by msgpack
|
|
295
331
|
const dataList = content.dataList;
|
|
296
332
|
for (let i = 0; i < dataList.length; i++) {
|
|
297
333
|
const item = dataList[i];
|
|
334
|
+
// Unpack Tracking Data
|
|
335
|
+
for (const td of item.trackingData) {
|
|
336
|
+
let px = td.px;
|
|
337
|
+
let py = td.py;
|
|
338
|
+
if (px instanceof Uint8Array) {
|
|
339
|
+
px = new Float32Array(px.buffer.slice(px.byteOffset, px.byteOffset + px.byteLength));
|
|
340
|
+
}
|
|
341
|
+
if (py instanceof Uint8Array) {
|
|
342
|
+
py = new Float32Array(py.buffer.slice(py.byteOffset, py.byteOffset + py.byteLength));
|
|
343
|
+
}
|
|
344
|
+
td.px = px;
|
|
345
|
+
td.py = py;
|
|
346
|
+
}
|
|
347
|
+
// Unpack Matching Data
|
|
298
348
|
for (const kf of item.matchingData) {
|
|
299
349
|
for (const col of [kf.max, kf.min]) {
|
|
300
|
-
|
|
301
|
-
|
|
350
|
+
let xRaw = col.x;
|
|
351
|
+
let yRaw = col.y;
|
|
352
|
+
if (xRaw instanceof Uint8Array) {
|
|
353
|
+
xRaw = new Uint16Array(xRaw.buffer.slice(xRaw.byteOffset, xRaw.byteOffset + xRaw.byteLength));
|
|
354
|
+
}
|
|
355
|
+
if (yRaw instanceof Uint8Array) {
|
|
356
|
+
yRaw = new Uint16Array(yRaw.buffer.slice(yRaw.byteOffset, yRaw.byteOffset + yRaw.byteLength));
|
|
302
357
|
}
|
|
303
|
-
|
|
304
|
-
|
|
358
|
+
// Rescale for compatibility with Matcher
|
|
359
|
+
const count = xRaw.length;
|
|
360
|
+
const x = new Float32Array(count);
|
|
361
|
+
const y = new Float32Array(count);
|
|
362
|
+
for (let k = 0; k < count; k++) {
|
|
363
|
+
x[k] = (xRaw[k] / 65535) * kf.w;
|
|
364
|
+
y[k] = (yRaw[k] / 65535) * kf.h;
|
|
305
365
|
}
|
|
366
|
+
col.x = x;
|
|
367
|
+
col.y = y;
|
|
306
368
|
if (col.a instanceof Uint8Array) {
|
|
307
|
-
|
|
369
|
+
const aRaw = new Int16Array(col.a.buffer.slice(col.a.byteOffset, col.a.byteOffset + col.a.byteLength));
|
|
370
|
+
const a = new Float32Array(count);
|
|
371
|
+
for (let k = 0; k < count; k++) {
|
|
372
|
+
a[k] = (aRaw[k] / 32767) * Math.PI;
|
|
373
|
+
}
|
|
374
|
+
col.a = a;
|
|
308
375
|
}
|
|
309
376
|
if (col.s instanceof Uint8Array) {
|
|
310
|
-
|
|
377
|
+
const sRaw = col.s;
|
|
378
|
+
const s = new Float32Array(count);
|
|
379
|
+
for (let k = 0; k < count; k++) {
|
|
380
|
+
s[k] = Math.pow(2, sRaw[k]);
|
|
381
|
+
}
|
|
382
|
+
col.s = s;
|
|
383
|
+
}
|
|
384
|
+
// Restore LSH descriptors (Uint32Array)
|
|
385
|
+
if (col.d instanceof Uint8Array) {
|
|
386
|
+
col.d = new Uint32Array(col.d.buffer.slice(col.d.byteOffset, col.d.byteOffset + col.d.byteLength));
|
|
311
387
|
}
|
|
312
388
|
}
|
|
313
389
|
}
|
|
314
390
|
}
|
|
315
391
|
this.data = dataList;
|
|
316
|
-
return
|
|
392
|
+
return { version, dataList };
|
|
317
393
|
}
|
|
318
394
|
_unpackKeyframe(kf) {
|
|
319
395
|
return {
|
|
320
396
|
width: kf.w,
|
|
321
397
|
height: kf.h,
|
|
322
398
|
scale: kf.s,
|
|
323
|
-
maximaPoints: this._decolumnarize(kf.max),
|
|
324
|
-
minimaPoints: this._decolumnarize(kf.min),
|
|
399
|
+
maximaPoints: this._decolumnarize(kf.max, kf.w, kf.h),
|
|
400
|
+
minimaPoints: this._decolumnarize(kf.min, kf.w, kf.h),
|
|
325
401
|
maximaPointsCluster: { rootNode: this._expandTree(kf.max.t) },
|
|
326
402
|
minimaPointsCluster: { rootNode: this._expandTree(kf.min.t) },
|
|
327
403
|
};
|
|
328
404
|
}
|
|
329
|
-
_decolumnarize(col) {
|
|
405
|
+
_decolumnarize(col, width, height) {
|
|
330
406
|
const points = [];
|
|
331
407
|
const count = col.x.length;
|
|
408
|
+
const descSize = col.d.length / count;
|
|
332
409
|
for (let i = 0; i < count; i++) {
|
|
333
410
|
points.push({
|
|
334
|
-
x: col.x[i],
|
|
335
|
-
y: col.y[i],
|
|
411
|
+
x: (col.x[i] / 65535) * width,
|
|
412
|
+
y: (col.y[i] / 65535) * height,
|
|
336
413
|
angle: col.a[i],
|
|
337
414
|
scale: col.s ? col.s[i] : 1.0,
|
|
338
|
-
descriptors: col.d.slice(i *
|
|
415
|
+
descriptors: col.d.slice(i * descSize, (i + 1) * descSize),
|
|
339
416
|
});
|
|
340
417
|
}
|
|
341
418
|
return points;
|
|
@@ -77,12 +77,5 @@ export class SimpleAR {
|
|
|
77
77
|
_initController(): void;
|
|
78
78
|
_handleUpdate(data: any): void;
|
|
79
79
|
_positionOverlay(mVT: any, targetIndex: any): void;
|
|
80
|
-
/**
|
|
81
|
-
* Projects a 3D marker-space point all the way to 2D screen CSS pixels
|
|
82
|
-
*/
|
|
83
|
-
_projectToScreen(x: any, y: any, z: any, mVT: any, proj: any, videoW: any, videoH: any, containerRect: any, needsRotation: any): {
|
|
84
|
-
sx: number;
|
|
85
|
-
sy: number;
|
|
86
|
-
};
|
|
87
80
|
}
|
|
88
81
|
import { Controller } from "./controller.js";
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { Controller } from "./controller.js";
|
|
2
2
|
import { OneEuroFilter } from "../libs/one-euro-filter.js";
|
|
3
|
+
import { projectToScreen } from "./utils/projection.js";
|
|
3
4
|
/**
|
|
4
5
|
* 🍦 SimpleAR - Dead-simple vanilla AR for image overlays
|
|
5
6
|
*
|
|
@@ -132,11 +133,11 @@ class SimpleAR {
|
|
|
132
133
|
if (!this.filters[targetIndex]) {
|
|
133
134
|
this.filters[targetIndex] = new OneEuroFilter({ minCutOff: 0.1, beta: 0.01 });
|
|
134
135
|
}
|
|
135
|
-
// Flatten
|
|
136
|
+
// Flatten modelViewTransform for filtering (3x4 matrix = 12 values)
|
|
136
137
|
const flatMVT = [
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
138
|
+
modelViewTransform[0][0], modelViewTransform[0][1], modelViewTransform[0][2], modelViewTransform[0][3],
|
|
139
|
+
modelViewTransform[1][0], modelViewTransform[1][1], modelViewTransform[1][2], modelViewTransform[1][3],
|
|
140
|
+
modelViewTransform[2][0], modelViewTransform[2][1], modelViewTransform[2][2], modelViewTransform[2][3]
|
|
140
141
|
];
|
|
141
142
|
const smoothedFlat = this.filters[targetIndex].filter(Date.now(), flatMVT);
|
|
142
143
|
const smoothedMVT = [
|
|
@@ -173,8 +174,8 @@ class SimpleAR {
|
|
|
173
174
|
const proj = this.controller.projectionTransform;
|
|
174
175
|
// 3. Project 3 points to determine position, scale, and rotation
|
|
175
176
|
// Points in Marker Space: Center, Right-Edge, and Down-Edge
|
|
176
|
-
const pMid =
|
|
177
|
-
const pRight =
|
|
177
|
+
const pMid = projectToScreen(markerW / 2, markerH / 2, 0, mVT, proj, videoW, videoH, containerRect, needsRotation);
|
|
178
|
+
const pRight = projectToScreen(markerW / 2 + 100, markerH / 2, 0, mVT, proj, videoW, videoH, containerRect, needsRotation);
|
|
178
179
|
// 4. Calculate Screen Position
|
|
179
180
|
const screenX = pMid.sx;
|
|
180
181
|
const screenY = pMid.sy;
|
|
@@ -215,36 +216,5 @@ class SimpleAR {
|
|
|
215
216
|
scale(${finalScale})
|
|
216
217
|
`;
|
|
217
218
|
}
|
|
218
|
-
/**
|
|
219
|
-
* Projects a 3D marker-space point all the way to 2D screen CSS pixels
|
|
220
|
-
*/
|
|
221
|
-
_projectToScreen(x, y, z, mVT, proj, videoW, videoH, containerRect, needsRotation) {
|
|
222
|
-
// Marker -> Camera Space
|
|
223
|
-
const tx = mVT[0][0] * x + mVT[0][1] * y + mVT[0][2] * z + mVT[0][3];
|
|
224
|
-
const ty = mVT[1][0] * x + mVT[1][1] * y + mVT[1][2] * z + mVT[1][3];
|
|
225
|
-
const tz = mVT[2][0] * x + mVT[2][1] * y + mVT[2][2] * z + mVT[2][3];
|
|
226
|
-
// Camera -> Buffer Pixels (e.g. 1280x720)
|
|
227
|
-
const bx = (proj[0][0] * tx / tz) + proj[0][2];
|
|
228
|
-
const by = (proj[1][1] * ty / tz) + proj[1][2];
|
|
229
|
-
// Buffer -> Screen CSS Pixels
|
|
230
|
-
const vW = needsRotation ? videoH : videoW;
|
|
231
|
-
const vH = needsRotation ? videoW : videoH;
|
|
232
|
-
const perspectiveScale = Math.max(containerRect.width / vW, containerRect.height / vH);
|
|
233
|
-
const displayW = vW * perspectiveScale;
|
|
234
|
-
const displayH = vH * perspectiveScale;
|
|
235
|
-
const offsetX = (containerRect.width - displayW) / 2;
|
|
236
|
-
const offsetY = (containerRect.height - displayH) / 2;
|
|
237
|
-
let sx, sy;
|
|
238
|
-
if (needsRotation) {
|
|
239
|
-
// Mapping: Camera +X (Right) -> Screen +Y (Down), Camera +Y (Down) -> Screen -X (Left)
|
|
240
|
-
sx = offsetX + (displayW / 2) - (by - proj[1][2]) * perspectiveScale;
|
|
241
|
-
sy = offsetY + (displayH / 2) + (bx - proj[0][2]) * perspectiveScale;
|
|
242
|
-
}
|
|
243
|
-
else {
|
|
244
|
-
sx = offsetX + (displayW / 2) + (bx - proj[0][2]) * perspectiveScale;
|
|
245
|
-
sy = offsetY + (displayH / 2) + (by - proj[1][2]) * perspectiveScale;
|
|
246
|
-
}
|
|
247
|
-
return { sx, sy };
|
|
248
|
-
}
|
|
249
219
|
}
|
|
250
220
|
export { SimpleAR };
|
|
@@ -3,7 +3,7 @@ const AR2_DEFAULT_TS = 6;
|
|
|
3
3
|
const AR2_DEFAULT_TS_GAP = 1;
|
|
4
4
|
const AR2_SEARCH_SIZE = 10;
|
|
5
5
|
const AR2_SEARCH_GAP = 1;
|
|
6
|
-
const AR2_SIM_THRESH = 0.
|
|
6
|
+
const AR2_SIM_THRESH = 0.6;
|
|
7
7
|
const TRACKING_KEYFRAME = 0; // 0: 128px (optimized)
|
|
8
8
|
class Tracker {
|
|
9
9
|
constructor(markerDimensions, trackingDataList, projectionTransform, inputWidth, inputHeight, debugMode = false) {
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Converts an 84-byte FREAK descriptor into a Uint32Array of 2 elements (64 bits).
|
|
3
|
+
* @param {Uint8Array} descriptor - The 84-byte FREAK descriptor.
|
|
4
|
+
* @returns {Uint32Array} Array of two 32-bit integers.
|
|
5
|
+
*/
|
|
6
|
+
export function binarizeFREAK64(descriptor: Uint8Array): Uint32Array;
|
|
7
|
+
/**
|
|
8
|
+
* Converts an 84-byte FREAK descriptor into a Uint32Array of 2 elements (64 bits).
|
|
9
|
+
* @param {Uint8Array} descriptor - The 84-byte FREAK descriptor.
|
|
10
|
+
* @returns {Uint32Array} Array of two 32-bit integers.
|
|
11
|
+
*/
|
|
12
|
+
export function binarizeFREAK128(descriptor: Uint8Array): Uint32Array;
|
|
13
|
+
/**
|
|
14
|
+
* Converts an 84-byte FREAK descriptor into a Uint32Array of 2 elements (64 bits).
|
|
15
|
+
* @param {Uint8Array} descriptor - The 84-byte FREAK descriptor.
|
|
16
|
+
* @returns {Uint32Array} Array of two 32-bit integers.
|
|
17
|
+
*/
|
|
18
|
+
export function binarizeFREAK32(descriptor: Uint8Array): Uint32Array;
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* LSH Binarizer for FREAK descriptors.
|
|
3
|
+
*
|
|
4
|
+
* This utility implements Locality Sensitive Hashing (LSH) for binary descriptors.
|
|
5
|
+
* It uses simple Bit-Sampling to reduce the 672 bits (84 bytes) of a FREAK
|
|
6
|
+
* descriptor into a 64-bit (8-byte) fingerprint.
|
|
7
|
+
*
|
|
8
|
+
* Bit-sampling is chosen for maximum speed and zero memory overhead,
|
|
9
|
+
* which fits the Moonshot goal of an ultra-lightweight bundle.
|
|
10
|
+
*/
|
|
11
|
+
// For 64-bit LSH, we use a uniform sampling across the 672-bit descriptor.
|
|
12
|
+
const SAMPLING_INDICES = new Int32Array(64);
|
|
13
|
+
for (let i = 0; i < 64; i++) {
|
|
14
|
+
SAMPLING_INDICES[i] = Math.floor(i * (672 / 64));
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* Converts an 84-byte FREAK descriptor into a Uint32Array of 2 elements (64 bits).
|
|
18
|
+
* @param {Uint8Array} descriptor - The 84-byte FREAK descriptor.
|
|
19
|
+
* @returns {Uint32Array} Array of two 32-bit integers.
|
|
20
|
+
*/
|
|
21
|
+
export function binarizeFREAK64(descriptor) {
|
|
22
|
+
const result = new Uint32Array(2);
|
|
23
|
+
for (let i = 0; i < 64; i++) {
|
|
24
|
+
const bitIndex = SAMPLING_INDICES[i];
|
|
25
|
+
const byteIdx = bitIndex >> 3;
|
|
26
|
+
const bitIdx = 7 - (bitIndex & 7);
|
|
27
|
+
if ((descriptor[byteIdx] >> bitIdx) & 1) {
|
|
28
|
+
const uintIdx = i >> 5; // i / 32
|
|
29
|
+
const uintBitIdx = i & 31; // i % 32
|
|
30
|
+
result[uintIdx] |= (1 << uintBitIdx);
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
return result;
|
|
34
|
+
}
|
|
35
|
+
// Backward compatibility or for other uses
|
|
36
|
+
export const binarizeFREAK128 = binarizeFREAK64;
|
|
37
|
+
export const binarizeFREAK32 = binarizeFREAK64;
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 📐 AR Projection Utilities
|
|
3
|
+
* Common logic for projecting 3D marker-space points to 2D screen CSS pixels.
|
|
4
|
+
*/
|
|
5
|
+
/**
|
|
6
|
+
* Projects a 3D marker-space point (x, y, z) into 2D screen coordinates.
|
|
7
|
+
*
|
|
8
|
+
* @param {number} x - Marker X coordinate
|
|
9
|
+
* @param {number} y - Marker Y coordinate
|
|
10
|
+
* @param {number} z - Marker Z coordinate (height from surface)
|
|
11
|
+
* @param {number[][]} mVT - ModelViewTransform matrix (3x4)
|
|
12
|
+
* @param {number[][]} proj - Projection matrix (3x3)
|
|
13
|
+
* @param {number} videoW - Internal video width
|
|
14
|
+
* @param {number} videoH - Internal video height
|
|
15
|
+
* @param {Object} containerRect - {width, height} of the display container
|
|
16
|
+
* @param {boolean} needsRotation - Whether the feed needs 90deg rotation (e.g. portrait mobile)
|
|
17
|
+
* @returns {{sx: number, sy: number}} Screen coordinates [X, Y]
|
|
18
|
+
*/
|
|
19
|
+
export function projectToScreen(x: number, y: number, z: number, mVT: number[][], proj: number[][], videoW: number, videoH: number, containerRect: Object, needsRotation?: boolean): {
|
|
20
|
+
sx: number;
|
|
21
|
+
sy: number;
|
|
22
|
+
};
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 📐 AR Projection Utilities
|
|
3
|
+
* Common logic for projecting 3D marker-space points to 2D screen CSS pixels.
|
|
4
|
+
*/
|
|
5
|
+
/**
|
|
6
|
+
* Projects a 3D marker-space point (x, y, z) into 2D screen coordinates.
|
|
7
|
+
*
|
|
8
|
+
* @param {number} x - Marker X coordinate
|
|
9
|
+
* @param {number} y - Marker Y coordinate
|
|
10
|
+
* @param {number} z - Marker Z coordinate (height from surface)
|
|
11
|
+
* @param {number[][]} mVT - ModelViewTransform matrix (3x4)
|
|
12
|
+
* @param {number[][]} proj - Projection matrix (3x3)
|
|
13
|
+
* @param {number} videoW - Internal video width
|
|
14
|
+
* @param {number} videoH - Internal video height
|
|
15
|
+
* @param {Object} containerRect - {width, height} of the display container
|
|
16
|
+
* @param {boolean} needsRotation - Whether the feed needs 90deg rotation (e.g. portrait mobile)
|
|
17
|
+
* @returns {{sx: number, sy: number}} Screen coordinates [X, Y]
|
|
18
|
+
*/
|
|
19
|
+
export function projectToScreen(x, y, z, mVT, proj, videoW, videoH, containerRect, needsRotation = false) {
|
|
20
|
+
// 1. Marker Space -> Camera Space (3D)
|
|
21
|
+
const tx = mVT[0][0] * x + mVT[0][1] * y + mVT[0][2] * z + mVT[0][3];
|
|
22
|
+
const ty = mVT[1][0] * x + mVT[1][1] * y + mVT[1][2] * z + mVT[1][3];
|
|
23
|
+
const tz = mVT[2][0] * x + mVT[2][1] * y + mVT[2][2] * z + mVT[2][3];
|
|
24
|
+
// 2. Camera Space -> Buffer Pixels (2D)
|
|
25
|
+
// Using intrinsic projection from controller
|
|
26
|
+
const bx = (proj[0][0] * tx / tz) + proj[0][2];
|
|
27
|
+
const by = (proj[1][1] * ty / tz) + proj[1][2];
|
|
28
|
+
// 3. Buffer Pixels -> Screen CSS Pixels
|
|
29
|
+
const vW = needsRotation ? videoH : videoW;
|
|
30
|
+
const vH = needsRotation ? videoW : videoH;
|
|
31
|
+
// Calculate how the video is scaled to cover (object-fit: cover) the container
|
|
32
|
+
const perspectiveScale = Math.max(containerRect.width / vW, containerRect.height / vH);
|
|
33
|
+
const displayW = vW * perspectiveScale;
|
|
34
|
+
const displayH = vH * perspectiveScale;
|
|
35
|
+
// Centering offsets
|
|
36
|
+
const offsetX = (containerRect.width - displayW) / 2;
|
|
37
|
+
const offsetY = (containerRect.height - displayH) / 2;
|
|
38
|
+
let sx, sy;
|
|
39
|
+
if (needsRotation) {
|
|
40
|
+
// Rotation Mapping:
|
|
41
|
+
// Camera +X (Right) -> Screen +Y (Down)
|
|
42
|
+
// Camera +Y (Down) -> Screen -X (Left)
|
|
43
|
+
sx = offsetX + (displayW / 2) - (by - proj[1][2]) * perspectiveScale;
|
|
44
|
+
sy = offsetY + (displayH / 2) + (bx - proj[0][2]) * perspectiveScale;
|
|
45
|
+
}
|
|
46
|
+
else {
|
|
47
|
+
sx = offsetX + (displayW / 2) + (bx - proj[0][2]) * perspectiveScale;
|
|
48
|
+
sy = offsetY + (displayH / 2) + (by - proj[1][2]) * perspectiveScale;
|
|
49
|
+
}
|
|
50
|
+
return { sx, sy };
|
|
51
|
+
}
|
package/package.json
CHANGED
|
@@ -42,7 +42,7 @@ class Controller {
|
|
|
42
42
|
this.filterBeta = filterBeta === null ? DEFAULT_FILTER_BETA : filterBeta;
|
|
43
43
|
this.warmupTolerance = warmupTolerance === null ? DEFAULT_WARMUP_TOLERANCE : warmupTolerance;
|
|
44
44
|
this.missTolerance = missTolerance === null ? DEFAULT_MISS_TOLERANCE : missTolerance;
|
|
45
|
-
this.cropDetector = new CropDetector(this.inputWidth, this.inputHeight, debugMode);
|
|
45
|
+
this.cropDetector = new CropDetector(this.inputWidth, this.inputHeight, debugMode, true);
|
|
46
46
|
this.inputLoader = new InputLoader(this.inputWidth, this.inputHeight);
|
|
47
47
|
this.markerDimensions = null;
|
|
48
48
|
this.onUpdate = onUpdate;
|
|
@@ -124,7 +124,7 @@ class Controller {
|
|
|
124
124
|
|
|
125
125
|
for (const buffer of buffers) {
|
|
126
126
|
const compiler = new Compiler();
|
|
127
|
-
const dataList = compiler.importData(buffer);
|
|
127
|
+
const { dataList } = compiler.importData(buffer);
|
|
128
128
|
|
|
129
129
|
for (const item of dataList) {
|
|
130
130
|
allMatchingData.push(item.matchingData);
|
|
@@ -378,10 +378,10 @@ class Controller {
|
|
|
378
378
|
}
|
|
379
379
|
|
|
380
380
|
async match(featurePoints, targetIndex) {
|
|
381
|
-
const { targetIndex: matchedTargetIndex, modelViewTransform, debugExtra } = await this._workerMatch(featurePoints, [
|
|
381
|
+
const { targetIndex: matchedTargetIndex, modelViewTransform, screenCoords, worldCoords, debugExtra } = await this._workerMatch(featurePoints, [
|
|
382
382
|
targetIndex,
|
|
383
383
|
]);
|
|
384
|
-
return { targetIndex: matchedTargetIndex, modelViewTransform, debugExtra };
|
|
384
|
+
return { targetIndex: matchedTargetIndex, modelViewTransform, screenCoords, worldCoords, debugExtra };
|
|
385
385
|
}
|
|
386
386
|
|
|
387
387
|
async track(input, modelViewTransform, targetIndex) {
|
|
@@ -408,6 +408,8 @@ class Controller {
|
|
|
408
408
|
resolve({
|
|
409
409
|
targetIndex: data.targetIndex,
|
|
410
410
|
modelViewTransform: data.modelViewTransform,
|
|
411
|
+
screenCoords: data.screenCoords,
|
|
412
|
+
worldCoords: data.worldCoords,
|
|
411
413
|
debugExtra: data.debugExtra,
|
|
412
414
|
});
|
|
413
415
|
};
|
|
@@ -426,6 +428,8 @@ class Controller {
|
|
|
426
428
|
|
|
427
429
|
let matchedTargetIndex = -1;
|
|
428
430
|
let matchedModelViewTransform = null;
|
|
431
|
+
let matchedScreenCoords = null;
|
|
432
|
+
let matchedWorldCoords = null;
|
|
429
433
|
let matchedDebugExtra = null;
|
|
430
434
|
|
|
431
435
|
for (let i = 0; i < targetIndexes.length; i++) {
|
|
@@ -443,6 +447,8 @@ class Controller {
|
|
|
443
447
|
if (modelViewTransform) {
|
|
444
448
|
matchedTargetIndex = matchingIndex;
|
|
445
449
|
matchedModelViewTransform = modelViewTransform;
|
|
450
|
+
matchedScreenCoords = screenCoords;
|
|
451
|
+
matchedWorldCoords = worldCoords;
|
|
446
452
|
}
|
|
447
453
|
break;
|
|
448
454
|
}
|
|
@@ -451,6 +457,8 @@ class Controller {
|
|
|
451
457
|
return {
|
|
452
458
|
targetIndex: matchedTargetIndex,
|
|
453
459
|
modelViewTransform: matchedModelViewTransform,
|
|
460
|
+
screenCoords: matchedScreenCoords,
|
|
461
|
+
worldCoords: matchedWorldCoords,
|
|
454
462
|
debugExtra: matchedDebugExtra,
|
|
455
463
|
};
|
|
456
464
|
}
|
|
@@ -11,7 +11,7 @@ class CropDetector {
|
|
|
11
11
|
let cropSize = Math.pow(2, Math.round(Math.log(minDimension) / Math.log(2)));
|
|
12
12
|
this.cropSize = cropSize;
|
|
13
13
|
|
|
14
|
-
this.detector = new DetectorLite(cropSize, cropSize);
|
|
14
|
+
this.detector = new DetectorLite(cropSize, cropSize, { useLSH: true });
|
|
15
15
|
|
|
16
16
|
this.lastRandomIndex = 4;
|
|
17
17
|
}
|
|
@@ -12,6 +12,7 @@
|
|
|
12
12
|
|
|
13
13
|
import { FREAKPOINTS } from "./freak.js";
|
|
14
14
|
import { gpuCompute } from "../utils/gpu-compute.js";
|
|
15
|
+
import { binarizeFREAK32 } from "../utils/lsh-binarizer.js";
|
|
15
16
|
|
|
16
17
|
const PYRAMID_MIN_SIZE = 4; // Reducido de 8 a 4 para exprimir al máximo la resolución
|
|
17
18
|
// PYRAMID_MAX_OCTAVE ya no es necesario, el límite lo da PYRAMID_MIN_SIZE
|
|
@@ -43,6 +44,8 @@ export class DetectorLite {
|
|
|
43
44
|
this.width = width;
|
|
44
45
|
this.height = height;
|
|
45
46
|
this.useGPU = options.useGPU !== undefined ? options.useGPU : globalUseGPU;
|
|
47
|
+
// Protocol V6 (Moonshot): 64-bit LSH is the standard descriptor format
|
|
48
|
+
this.useLSH = options.useLSH !== undefined ? options.useLSH : true;
|
|
46
49
|
|
|
47
50
|
let numOctaves = 0;
|
|
48
51
|
let w = width, h = height;
|
|
@@ -99,7 +102,7 @@ export class DetectorLite {
|
|
|
99
102
|
y: ext.y * Math.pow(2, ext.octave) + Math.pow(2, ext.octave - 1) - 0.5,
|
|
100
103
|
scale: Math.pow(2, ext.octave),
|
|
101
104
|
angle: ext.angle || 0,
|
|
102
|
-
descriptors: ext.descriptors || []
|
|
105
|
+
descriptors: (this.useLSH && ext.lsh) ? ext.lsh : (ext.descriptors || [])
|
|
103
106
|
}));
|
|
104
107
|
|
|
105
108
|
return { featurePoints };
|
|
@@ -494,6 +497,9 @@ export class DetectorLite {
|
|
|
494
497
|
}
|
|
495
498
|
}
|
|
496
499
|
}
|
|
500
|
+
if (this.useLSH) {
|
|
501
|
+
ext.lsh = binarizeFREAK32(descriptor);
|
|
502
|
+
}
|
|
497
503
|
ext.descriptors = descriptor;
|
|
498
504
|
}
|
|
499
505
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
// Precomputed bit count lookup table for Uint8Array
|
|
1
|
+
// Precomputed bit count lookup table for Uint8Array
|
|
2
2
|
const BIT_COUNT_8 = new Uint8Array(256);
|
|
3
3
|
for (let i = 0; i < 256; i++) {
|
|
4
4
|
let c = 0, n = i;
|
|
@@ -6,14 +6,46 @@ for (let i = 0; i < 256; i++) {
|
|
|
6
6
|
BIT_COUNT_8[i] = c;
|
|
7
7
|
}
|
|
8
8
|
|
|
9
|
+
/**
|
|
10
|
+
* Optimized popcount for 32-bit integers
|
|
11
|
+
*/
|
|
12
|
+
function popcount32(n) {
|
|
13
|
+
n = n - ((n >> 1) & 0x55555555);
|
|
14
|
+
n = (n & 0x33333333) + ((n >> 2) & 0x33333333);
|
|
15
|
+
return (((n + (n >> 4)) & 0x0F0F0F0F) * 0x01010101) >> 24;
|
|
16
|
+
}
|
|
17
|
+
|
|
9
18
|
const compute = (options) => {
|
|
10
19
|
const { v1, v2, v1Offset = 0, v2Offset = 0 } = options;
|
|
11
|
-
|
|
12
|
-
//
|
|
13
|
-
|
|
14
|
-
|
|
20
|
+
|
|
21
|
+
// Protocol V5 Path: 64-bit LSH (two Uint32)
|
|
22
|
+
if (v1.length === v2.length && (v1.length / (v1.buffer.byteLength / v1.length)) === 2) {
|
|
23
|
+
// This is a bit hacky check, better if we know the version.
|
|
24
|
+
// Assuming if it's not 84 bytes, it's the new 8-byte format.
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
// If descriptors are 84 bytes (Protocol V4)
|
|
28
|
+
if (v1.length >= v1Offset + 84 && v2.length >= v2Offset + 84 && v1[v1Offset + 83] !== undefined) {
|
|
29
|
+
let d = 0;
|
|
30
|
+
for (let i = 0; i < 84; i++) {
|
|
31
|
+
d += BIT_COUNT_8[v1[v1Offset + i] ^ v2[v2Offset + i]];
|
|
32
|
+
}
|
|
33
|
+
return d;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
// Protocol V5.1 Path: LSH 128-bit (4 x 32-bit)
|
|
37
|
+
// We expect v1 and v2 to be slices or offsets of Uint32Array
|
|
38
|
+
if (v1.length >= v1Offset + 4 && v2.length >= v2Offset + 4 && v1[v1Offset + 3] !== undefined) {
|
|
39
|
+
return popcount32(v1[v1Offset] ^ v2[v2Offset]) +
|
|
40
|
+
popcount32(v1[v1Offset + 1] ^ v2[v2Offset + 1]) +
|
|
41
|
+
popcount32(v1[v1Offset + 2] ^ v2[v2Offset + 2]) +
|
|
42
|
+
popcount32(v1[v1Offset + 3] ^ v2[v2Offset + 3]);
|
|
15
43
|
}
|
|
16
|
-
|
|
44
|
+
|
|
45
|
+
// Protocol V5 Path: LSH 64-bit (2 x 32-bit)
|
|
46
|
+
// We expect v1 and v2 to be slices or offsets of Uint32Array
|
|
47
|
+
return popcount32(v1[v1Offset] ^ v2[v2Offset]) +
|
|
48
|
+
popcount32(v1[v1Offset + 1] ^ v2[v2Offset + 1]);
|
|
17
49
|
};
|
|
18
50
|
|
|
19
51
|
export { compute };
|