@srsergio/taptapp-ar 1.0.84 → 1.0.86
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/compiler/node-worker.js +5 -18
- package/dist/compiler/offline-compiler.d.ts +0 -48
- package/dist/compiler/offline-compiler.js +12 -208
- package/dist/{compiler → core}/detector/detector-lite.d.ts +6 -0
- package/dist/{compiler → core}/detector/detector-lite.js +9 -9
- package/dist/{compiler → core}/matching/matching.js +1 -1
- package/dist/core/protocol.d.ts +43 -0
- package/dist/core/protocol.js +195 -0
- package/dist/{compiler → core}/tracker/tracker.js +3 -2
- package/dist/index.d.ts +3 -2
- package/dist/index.js +3 -2
- package/dist/react/use-ar.js +1 -1
- package/dist/{compiler → runtime}/controller.d.ts +10 -6
- package/dist/{compiler → runtime}/controller.js +76 -22
- package/dist/{compiler → runtime}/controller.worker.js +33 -3
- package/dist/runtime/index.d.ts +4 -0
- package/dist/runtime/index.js +4 -0
- package/dist/{compiler → runtime}/simple-ar.js +1 -1
- package/package.json +1 -1
- package/src/compiler/node-worker.js +5 -19
- package/src/compiler/offline-compiler.ts +12 -241
- package/src/{compiler → core}/detector/detector-lite.js +10 -10
- package/src/{compiler → core}/matching/matching.js +1 -1
- package/src/core/protocol.ts +223 -0
- package/src/{compiler → core}/tracker/tracker.js +3 -2
- package/src/index.ts +3 -2
- package/src/react/use-ar.ts +2 -2
- package/src/{compiler → runtime}/controller.ts +85 -24
- package/src/{compiler → runtime}/controller.worker.js +45 -3
- package/src/runtime/index.ts +4 -0
- package/src/{compiler → runtime}/simple-ar.ts +1 -1
- package/dist/compiler/utils/fourier-encoder.d.ts +0 -25
- package/dist/compiler/utils/fourier-encoder.js +0 -47
- package/src/compiler/utils/fourier-encoder.ts +0 -53
- /package/dist/compiler/{index.d.ts → offline-compiler-browsertest.d.ts} +0 -0
- /package/dist/compiler/{index.js → offline-compiler-browsertest.js} +0 -0
- /package/dist/{compiler → core}/detector/crop-detector.d.ts +0 -0
- /package/dist/{compiler → core}/detector/crop-detector.js +0 -0
- /package/dist/{compiler → core}/detector/freak.d.ts +0 -0
- /package/dist/{compiler → core}/detector/freak.js +0 -0
- /package/dist/{compiler → core}/estimation/estimate.d.ts +0 -0
- /package/dist/{compiler → core}/estimation/estimate.js +0 -0
- /package/dist/{compiler → core}/estimation/estimator.d.ts +0 -0
- /package/dist/{compiler → core}/estimation/estimator.js +0 -0
- /package/dist/{compiler → core}/estimation/refine-estimate.d.ts +0 -0
- /package/dist/{compiler → core}/estimation/refine-estimate.js +0 -0
- /package/dist/{compiler → core}/estimation/utils.d.ts +0 -0
- /package/dist/{compiler → core}/estimation/utils.js +0 -0
- /package/dist/{compiler → core}/features/auto-rotation-feature.d.ts +0 -0
- /package/dist/{compiler → core}/features/auto-rotation-feature.js +0 -0
- /package/dist/{compiler → core}/features/crop-detection-feature.d.ts +0 -0
- /package/dist/{compiler → core}/features/crop-detection-feature.js +0 -0
- /package/dist/{compiler → core}/features/feature-base.d.ts +0 -0
- /package/dist/{compiler → core}/features/feature-base.js +0 -0
- /package/dist/{compiler → core}/features/feature-manager.d.ts +0 -0
- /package/dist/{compiler → core}/features/feature-manager.js +0 -0
- /package/dist/{compiler → core}/features/one-euro-filter-feature.d.ts +0 -0
- /package/dist/{compiler → core}/features/one-euro-filter-feature.js +0 -0
- /package/dist/{compiler → core}/features/temporal-filter-feature.d.ts +0 -0
- /package/dist/{compiler → core}/features/temporal-filter-feature.js +0 -0
- /package/dist/{compiler → core}/image-list.d.ts +0 -0
- /package/dist/{compiler → core}/image-list.js +0 -0
- /package/dist/{compiler → core}/input-loader.d.ts +0 -0
- /package/dist/{compiler → core}/input-loader.js +0 -0
- /package/dist/{compiler → core}/matching/hamming-distance.d.ts +0 -0
- /package/dist/{compiler → core}/matching/hamming-distance.js +0 -0
- /package/dist/{compiler → core}/matching/hierarchical-clustering.d.ts +0 -0
- /package/dist/{compiler → core}/matching/hierarchical-clustering.js +0 -0
- /package/dist/{compiler → core}/matching/hough.d.ts +0 -0
- /package/dist/{compiler → core}/matching/hough.js +0 -0
- /package/dist/{compiler → core}/matching/matcher.d.ts +0 -0
- /package/dist/{compiler → core}/matching/matcher.js +0 -0
- /package/dist/{compiler → core}/matching/matching.d.ts +0 -0
- /package/dist/{compiler → core}/matching/ransacHomography.d.ts +0 -0
- /package/dist/{compiler → core}/matching/ransacHomography.js +0 -0
- /package/dist/{compiler → core}/tracker/extract-utils.d.ts +0 -0
- /package/dist/{compiler → core}/tracker/extract-utils.js +0 -0
- /package/dist/{compiler → core}/tracker/extract.d.ts +0 -0
- /package/dist/{compiler → core}/tracker/extract.js +0 -0
- /package/dist/{compiler → core}/tracker/tracker.d.ts +0 -0
- /package/dist/{compiler → core}/utils/cumsum.d.ts +0 -0
- /package/dist/{compiler → core}/utils/cumsum.js +0 -0
- /package/dist/{compiler → core}/utils/geometry.d.ts +0 -0
- /package/dist/{compiler → core}/utils/geometry.js +0 -0
- /package/dist/{compiler → core}/utils/gpu-compute.d.ts +0 -0
- /package/dist/{compiler → core}/utils/gpu-compute.js +0 -0
- /package/dist/{compiler → core}/utils/homography.d.ts +0 -0
- /package/dist/{compiler → core}/utils/homography.js +0 -0
- /package/dist/{compiler → core}/utils/images.d.ts +0 -0
- /package/dist/{compiler → core}/utils/images.js +0 -0
- /package/dist/{compiler → core}/utils/lsh-binarizer.d.ts +0 -0
- /package/dist/{compiler → core}/utils/lsh-binarizer.js +0 -0
- /package/dist/{compiler → core}/utils/lsh-direct.d.ts +0 -0
- /package/dist/{compiler → core}/utils/lsh-direct.js +0 -0
- /package/dist/{compiler → core}/utils/projection.d.ts +0 -0
- /package/dist/{compiler → core}/utils/projection.js +0 -0
- /package/dist/{compiler → core}/utils/randomizer.d.ts +0 -0
- /package/dist/{compiler → core}/utils/randomizer.js +0 -0
- /package/dist/{compiler → core}/utils/worker-pool.d.ts +0 -0
- /package/dist/{compiler → core}/utils/worker-pool.js +0 -0
- /package/dist/{compiler → runtime}/aframe.d.ts +0 -0
- /package/dist/{compiler → runtime}/aframe.js +0 -0
- /package/dist/{compiler → runtime}/controller.worker.d.ts +0 -0
- /package/dist/{compiler → runtime}/simple-ar.d.ts +0 -0
- /package/dist/{compiler → runtime}/three.d.ts +0 -0
- /package/dist/{compiler → runtime}/three.js +0 -0
- /package/src/compiler/{index.js → offline-compiler-browsertest.js} +0 -0
- /package/src/{compiler → core}/detector/crop-detector.js +0 -0
- /package/src/{compiler → core}/detector/freak.js +0 -0
- /package/src/{compiler → core}/estimation/estimate.js +0 -0
- /package/src/{compiler → core}/estimation/estimator.js +0 -0
- /package/src/{compiler → core}/estimation/refine-estimate.js +0 -0
- /package/src/{compiler → core}/estimation/utils.js +0 -0
- /package/src/{compiler → core}/features/auto-rotation-feature.ts +0 -0
- /package/src/{compiler → core}/features/crop-detection-feature.ts +0 -0
- /package/src/{compiler → core}/features/feature-base.ts +0 -0
- /package/src/{compiler → core}/features/feature-manager.ts +0 -0
- /package/src/{compiler → core}/features/one-euro-filter-feature.ts +0 -0
- /package/src/{compiler → core}/features/temporal-filter-feature.ts +0 -0
- /package/src/{compiler → core}/image-list.js +0 -0
- /package/src/{compiler → core}/input-loader.js +0 -0
- /package/src/{compiler → core}/matching/hamming-distance.js +0 -0
- /package/src/{compiler → core}/matching/hierarchical-clustering.js +0 -0
- /package/src/{compiler → core}/matching/hough.js +0 -0
- /package/src/{compiler → core}/matching/matcher.js +0 -0
- /package/src/{compiler → core}/matching/ransacHomography.js +0 -0
- /package/src/{compiler → core}/tracker/extract-utils.js +0 -0
- /package/src/{compiler → core}/tracker/extract.js +0 -0
- /package/src/{compiler → core}/utils/cumsum.js +0 -0
- /package/src/{compiler → core}/utils/geometry.js +0 -0
- /package/src/{compiler → core}/utils/gpu-compute.js +0 -0
- /package/src/{compiler → core}/utils/homography.js +0 -0
- /package/src/{compiler → core}/utils/images.js +0 -0
- /package/src/{compiler → core}/utils/lsh-binarizer.js +0 -0
- /package/src/{compiler → core}/utils/lsh-direct.js +0 -0
- /package/src/{compiler → core}/utils/projection.js +0 -0
- /package/src/{compiler → core}/utils/randomizer.js +0 -0
- /package/src/{compiler → core}/utils/worker-pool.js +0 -0
- /package/src/{compiler → runtime}/aframe.js +0 -0
- /package/src/{compiler → runtime}/three.js +0 -0
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
import * as msgpack from "@msgpack/msgpack";
|
|
2
|
+
export const CURRENT_VERSION = 7;
|
|
3
|
+
/**
|
|
4
|
+
* Morton Order calculation for spatial sorting
|
|
5
|
+
*/
|
|
6
|
+
export function getMorton(x, y) {
|
|
7
|
+
let x_int = x | 0;
|
|
8
|
+
let y_int = y | 0;
|
|
9
|
+
x_int = (x_int | (x_int << 8)) & 0x00FF00FF;
|
|
10
|
+
x_int = (x_int | (x_int << 4)) & 0x0F0F0F0F;
|
|
11
|
+
x_int = (x_int | (x_int << 2)) & 0x33333333;
|
|
12
|
+
x_int = (x_int | (x_int << 1)) & 0x55555555;
|
|
13
|
+
y_int = (y_int | (y_int << 8)) & 0x00FF00FF;
|
|
14
|
+
y_int = (y_int | (y_int << 4)) & 0x0F0F0F0F;
|
|
15
|
+
y_int = (y_int | (y_int << 2)) & 0x33333333;
|
|
16
|
+
y_int = (y_int | (y_int << 1)) & 0x55555555;
|
|
17
|
+
return x_int | (y_int << 1);
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Packs 8-bit image data into 4-bit packed data
|
|
21
|
+
*/
|
|
22
|
+
export function pack4Bit(data) {
|
|
23
|
+
const length = data.length;
|
|
24
|
+
if (length % 2 !== 0)
|
|
25
|
+
return data;
|
|
26
|
+
const packed = new Uint8Array(length / 2);
|
|
27
|
+
for (let i = 0; i < length; i += 2) {
|
|
28
|
+
const p1 = (data[i] & 0xF0) >> 4;
|
|
29
|
+
const p2 = (data[i + 1] & 0xF0) >> 4;
|
|
30
|
+
packed[i / 2] = (p1 << 4) | p2;
|
|
31
|
+
}
|
|
32
|
+
return packed;
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Unpacks 4-bit data back to 8-bit image data
|
|
36
|
+
*/
|
|
37
|
+
export function unpack4Bit(packed, width, height) {
|
|
38
|
+
const length = width * height;
|
|
39
|
+
const data = new Uint8Array(length);
|
|
40
|
+
for (let i = 0; i < packed.length; i++) {
|
|
41
|
+
const byte = packed[i];
|
|
42
|
+
const p1 = (byte & 0xF0);
|
|
43
|
+
const p2 = (byte & 0x0F) << 4;
|
|
44
|
+
data[i * 2] = p1;
|
|
45
|
+
data[i * 2 + 1] = p2;
|
|
46
|
+
}
|
|
47
|
+
return data;
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Columnarizes point data for efficient storage and transfer
|
|
51
|
+
*/
|
|
52
|
+
export function columnarize(points, tree, width, height) {
|
|
53
|
+
const count = points.length;
|
|
54
|
+
const x = new Uint16Array(count);
|
|
55
|
+
const y = new Uint16Array(count);
|
|
56
|
+
const angle = new Int16Array(count);
|
|
57
|
+
const scale = new Uint8Array(count);
|
|
58
|
+
const descriptors = new Uint32Array(count * 2);
|
|
59
|
+
for (let i = 0; i < count; i++) {
|
|
60
|
+
x[i] = Math.round((points[i].x / width) * 65535);
|
|
61
|
+
y[i] = Math.round((points[i].y / height) * 65535);
|
|
62
|
+
angle[i] = Math.round((points[i].angle / Math.PI) * 32767);
|
|
63
|
+
scale[i] = Math.round(Math.log2(points[i].scale || 1));
|
|
64
|
+
if (points[i].descriptors && points[i].descriptors.length >= 2) {
|
|
65
|
+
descriptors[i * 2] = points[i].descriptors[0];
|
|
66
|
+
descriptors[(i * 2) + 1] = points[i].descriptors[1];
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
return {
|
|
70
|
+
x,
|
|
71
|
+
y,
|
|
72
|
+
a: angle,
|
|
73
|
+
s: scale,
|
|
74
|
+
d: descriptors,
|
|
75
|
+
t: compactTree(tree.rootNode),
|
|
76
|
+
};
|
|
77
|
+
}
|
|
78
|
+
/**
|
|
79
|
+
* Compacts hierarchical clustering tree into a minimal array structure
|
|
80
|
+
*/
|
|
81
|
+
export function compactTree(node) {
|
|
82
|
+
if (node.leaf) {
|
|
83
|
+
return [1, node.centerPointIndex || 0, node.pointIndexes];
|
|
84
|
+
}
|
|
85
|
+
return [0, node.centerPointIndex || 0, node.children.map((c) => compactTree(c))];
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* Expands a compacted tree back into an object structure
|
|
89
|
+
*/
|
|
90
|
+
export function expandTree(node) {
|
|
91
|
+
const isLeaf = node[0] === 1;
|
|
92
|
+
if (isLeaf) {
|
|
93
|
+
return {
|
|
94
|
+
leaf: true,
|
|
95
|
+
centerPointIndex: node[1],
|
|
96
|
+
pointIndexes: node[2],
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
return {
|
|
100
|
+
leaf: false,
|
|
101
|
+
centerPointIndex: node[1],
|
|
102
|
+
children: node[2].map((c) => expandTree(c)),
|
|
103
|
+
};
|
|
104
|
+
}
|
|
105
|
+
/**
|
|
106
|
+
* Deserializes and normalizes .taar data from a buffer
|
|
107
|
+
*/
|
|
108
|
+
export function decodeTaar(buffer) {
|
|
109
|
+
const content = msgpack.decode(new Uint8Array(buffer));
|
|
110
|
+
const version = content.v || 0;
|
|
111
|
+
// Support Protocol V5/V6/V7
|
|
112
|
+
if (version < 5 || version > CURRENT_VERSION) {
|
|
113
|
+
console.warn(`Potential incompatible .taar version: ${version}. Standard is ${CURRENT_VERSION}.`);
|
|
114
|
+
}
|
|
115
|
+
const dataList = content.dataList;
|
|
116
|
+
for (let i = 0; i < dataList.length; i++) {
|
|
117
|
+
const item = dataList[i];
|
|
118
|
+
// 1. Process Tracking Data
|
|
119
|
+
for (const td of item.trackingData) {
|
|
120
|
+
// Helper to ensure we have the right TypedArray if it was decoded as Uint8Array by msgpack
|
|
121
|
+
const normalizeBuffer = (arr, Type) => {
|
|
122
|
+
if (arr instanceof Uint8Array && Type !== Uint8Array) {
|
|
123
|
+
return new Type(arr.buffer.slice(arr.byteOffset, arr.byteOffset + arr.byteLength));
|
|
124
|
+
}
|
|
125
|
+
return arr;
|
|
126
|
+
};
|
|
127
|
+
td.px = normalizeBuffer(td.px, Float32Array);
|
|
128
|
+
td.py = normalizeBuffer(td.py, Float32Array);
|
|
129
|
+
// Backwards compatibility for fields named 'd' vs 'data'
|
|
130
|
+
const rawData = td.data || td.d;
|
|
131
|
+
const w = td.width || td.w;
|
|
132
|
+
const h = td.height || td.h;
|
|
133
|
+
if (rawData && rawData.length === (w * h) / 2) {
|
|
134
|
+
const unpacked = unpack4Bit(rawData, w, h);
|
|
135
|
+
if (td.data)
|
|
136
|
+
td.data = unpacked;
|
|
137
|
+
if (td.d)
|
|
138
|
+
td.d = unpacked;
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
// 2. Process Matching Data
|
|
142
|
+
for (const kf of item.matchingData) {
|
|
143
|
+
for (const col of [kf.max, kf.min]) {
|
|
144
|
+
if (!col)
|
|
145
|
+
continue;
|
|
146
|
+
let xRaw = col.x;
|
|
147
|
+
let yRaw = col.y;
|
|
148
|
+
if (xRaw instanceof Uint8Array) {
|
|
149
|
+
xRaw = new Uint16Array(xRaw.buffer.slice(xRaw.byteOffset, xRaw.byteOffset + xRaw.byteLength));
|
|
150
|
+
}
|
|
151
|
+
if (yRaw instanceof Uint8Array) {
|
|
152
|
+
yRaw = new Uint16Array(yRaw.buffer.slice(yRaw.byteOffset, yRaw.byteOffset + yRaw.byteLength));
|
|
153
|
+
}
|
|
154
|
+
const count = xRaw.length;
|
|
155
|
+
const x = new Float32Array(count);
|
|
156
|
+
const y = new Float32Array(count);
|
|
157
|
+
for (let k = 0; k < count; k++) {
|
|
158
|
+
x[k] = (xRaw[k] / 65535) * kf.w;
|
|
159
|
+
y[k] = (yRaw[k] / 65535) * kf.h;
|
|
160
|
+
}
|
|
161
|
+
col.x = x;
|
|
162
|
+
col.y = y;
|
|
163
|
+
if (col.a instanceof Uint8Array) {
|
|
164
|
+
const aRaw = new Int16Array(col.a.buffer.slice(col.a.byteOffset, col.a.byteOffset + col.a.byteLength));
|
|
165
|
+
const a = new Float32Array(count);
|
|
166
|
+
for (let k = 0; k < count; k++) {
|
|
167
|
+
a[k] = (aRaw[k] / 32767) * Math.PI;
|
|
168
|
+
}
|
|
169
|
+
col.a = a;
|
|
170
|
+
}
|
|
171
|
+
if (col.s instanceof Uint8Array) {
|
|
172
|
+
const sRaw = col.s;
|
|
173
|
+
const s = new Float32Array(count);
|
|
174
|
+
for (let k = 0; k < count; k++) {
|
|
175
|
+
s[k] = Math.pow(2, sRaw[k]);
|
|
176
|
+
}
|
|
177
|
+
col.s = s;
|
|
178
|
+
}
|
|
179
|
+
if (col.d instanceof Uint8Array) {
|
|
180
|
+
col.d = new Uint32Array(col.d.buffer.slice(col.d.byteOffset, col.d.byteOffset + col.d.byteLength));
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
return { version, dataList };
|
|
186
|
+
}
|
|
187
|
+
/**
|
|
188
|
+
* Serializes target data into a .taar binary buffer
|
|
189
|
+
*/
|
|
190
|
+
export function encodeTaar(dataList) {
|
|
191
|
+
return msgpack.encode({
|
|
192
|
+
v: CURRENT_VERSION,
|
|
193
|
+
dataList,
|
|
194
|
+
});
|
|
195
|
+
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { buildModelViewProjectionTransform, computeScreenCoordiate } from "../estimation/utils.js";
|
|
2
2
|
const AR2_DEFAULT_TS = 6;
|
|
3
3
|
const AR2_DEFAULT_TS_GAP = 1;
|
|
4
|
-
const AR2_SEARCH_SIZE =
|
|
4
|
+
const AR2_SEARCH_SIZE = 12; // Reduced from 25 to 12 for high-speed tracking (25 is overkill)
|
|
5
5
|
const AR2_SEARCH_GAP = 1;
|
|
6
6
|
const AR2_SIM_THRESH = 0.65; // Increased from 0.6 to reduce false positives
|
|
7
7
|
const TRACKING_KEYFRAME = 0; // 0: 128px (optimized)
|
|
@@ -118,7 +118,8 @@ class Tracker {
|
|
|
118
118
|
if (this.debugMode) {
|
|
119
119
|
debugExtra = {
|
|
120
120
|
octaveIndex,
|
|
121
|
-
|
|
121
|
+
// Remove Array.from to avoid massive GC pressure
|
|
122
|
+
projectedImage: projectedImage,
|
|
122
123
|
matchingPoints,
|
|
123
124
|
goodTrack,
|
|
124
125
|
trackedPoints: screenCoords,
|
package/dist/index.d.ts
CHANGED
|
@@ -2,5 +2,6 @@ export * from "./react/types.js";
|
|
|
2
2
|
export * from "./react/TaptappAR.js";
|
|
3
3
|
export * from "./react/use-ar.js";
|
|
4
4
|
export * from "./compiler/offline-compiler.js";
|
|
5
|
-
export { Controller } from "./
|
|
6
|
-
export { SimpleAR } from "./
|
|
5
|
+
export { Controller } from "./runtime/controller.js";
|
|
6
|
+
export { SimpleAR } from "./runtime/simple-ar.js";
|
|
7
|
+
export * as protocol from "./core/protocol.js";
|
package/dist/index.js
CHANGED
|
@@ -2,5 +2,6 @@ export * from "./react/types.js";
|
|
|
2
2
|
export * from "./react/TaptappAR.js";
|
|
3
3
|
export * from "./react/use-ar.js";
|
|
4
4
|
export * from "./compiler/offline-compiler.js";
|
|
5
|
-
export { Controller } from "./
|
|
6
|
-
export { SimpleAR } from "./
|
|
5
|
+
export { Controller } from "./runtime/controller.js";
|
|
6
|
+
export { SimpleAR } from "./runtime/simple-ar.js";
|
|
7
|
+
export * as protocol from "./core/protocol.js";
|
package/dist/react/use-ar.js
CHANGED
|
@@ -31,7 +31,7 @@ export const useAR = (config) => {
|
|
|
31
31
|
const initAR = async () => {
|
|
32
32
|
try {
|
|
33
33
|
// Safe hybrid import for SSR + Speed
|
|
34
|
-
const { SimpleAR } = await import("../
|
|
34
|
+
const { SimpleAR } = await import("../runtime/simple-ar.js");
|
|
35
35
|
if (!isMounted)
|
|
36
36
|
return;
|
|
37
37
|
const instance = new SimpleAR({
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { Tracker } from "
|
|
2
|
-
import { InputLoader } from "
|
|
3
|
-
import { FeatureManager } from "
|
|
4
|
-
import { DetectorLite } from "
|
|
1
|
+
import { Tracker } from "../core/tracker/tracker.js";
|
|
2
|
+
import { InputLoader } from "../core/input-loader.js";
|
|
3
|
+
import { FeatureManager } from "../core/features/feature-manager.js";
|
|
4
|
+
import { DetectorLite } from "../core/detector/detector-lite.js";
|
|
5
5
|
export interface ControllerOptions {
|
|
6
6
|
inputWidth: number;
|
|
7
7
|
inputHeight: number;
|
|
@@ -32,6 +32,7 @@ declare class Controller {
|
|
|
32
32
|
matchingDataList: any;
|
|
33
33
|
workerMatchDone: ((data: any) => void) | null;
|
|
34
34
|
workerTrackDone: ((data: any) => void) | null;
|
|
35
|
+
workerFullTrackDone: ((data: any) => void) | null;
|
|
35
36
|
mainThreadMatcher: any;
|
|
36
37
|
mainThreadEstimator: any;
|
|
37
38
|
featureManager: FeatureManager;
|
|
@@ -62,7 +63,9 @@ declare class Controller {
|
|
|
62
63
|
_detectAndMatch(inputData: any, targetIndexes: number[]): Promise<{
|
|
63
64
|
targetIndex: any;
|
|
64
65
|
modelViewTransform: any;
|
|
65
|
-
|
|
66
|
+
screenCoords: any;
|
|
67
|
+
worldCoords: any;
|
|
68
|
+
featurePoints: any;
|
|
66
69
|
}>;
|
|
67
70
|
_trackAndUpdate(inputData: any, lastModelViewTransform: number[][], targetIndex: number): Promise<{
|
|
68
71
|
modelViewTransform: any;
|
|
@@ -106,7 +109,8 @@ declare class Controller {
|
|
|
106
109
|
debugExtra: {};
|
|
107
110
|
}>;
|
|
108
111
|
trackUpdate(modelViewTransform: number[][], trackFeatures: any): Promise<any>;
|
|
109
|
-
_workerMatch(featurePoints: any, targetIndexes: number[]): Promise<any>;
|
|
112
|
+
_workerMatch(featurePoints: any, targetIndexes: number[], inputData?: any): Promise<any>;
|
|
113
|
+
_workerTrack(inputData: any, lastModelViewTransform: number[][], targetIndex: number): Promise<any>;
|
|
110
114
|
_matchOnMainThread(featurePoints: any, targetIndexes: number[]): Promise<{
|
|
111
115
|
targetIndex: number;
|
|
112
116
|
modelViewTransform: any;
|
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
import { Tracker } from "
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
8
|
-
import
|
|
1
|
+
import { Tracker } from "../core/tracker/tracker.js";
|
|
2
|
+
import { InputLoader } from "../core/input-loader.js";
|
|
3
|
+
import { FeatureManager } from "../core/features/feature-manager.js";
|
|
4
|
+
import { OneEuroFilterFeature } from "../core/features/one-euro-filter-feature.js";
|
|
5
|
+
import { TemporalFilterFeature } from "../core/features/temporal-filter-feature.js";
|
|
6
|
+
import { AutoRotationFeature } from "../core/features/auto-rotation-feature.js";
|
|
7
|
+
import { DetectorLite } from "../core/detector/detector-lite.js";
|
|
8
|
+
import * as protocol from "../core/protocol.js";
|
|
9
9
|
let ControllerWorker;
|
|
10
10
|
// Conditional import for worker to avoid crash in non-vite environments
|
|
11
11
|
const getControllerWorker = async () => {
|
|
@@ -45,6 +45,7 @@ class Controller {
|
|
|
45
45
|
matchingDataList;
|
|
46
46
|
workerMatchDone = null;
|
|
47
47
|
workerTrackDone = null;
|
|
48
|
+
workerFullTrackDone = null;
|
|
48
49
|
mainThreadMatcher;
|
|
49
50
|
mainThreadEstimator;
|
|
50
51
|
featureManager;
|
|
@@ -102,6 +103,9 @@ class Controller {
|
|
|
102
103
|
if (e.data.type === "matchDone" && this.workerMatchDone !== null) {
|
|
103
104
|
this.workerMatchDone(e.data);
|
|
104
105
|
}
|
|
106
|
+
if (e.data.type === "trackDone" && this.workerFullTrackDone !== null) {
|
|
107
|
+
this.workerFullTrackDone(e.data);
|
|
108
|
+
}
|
|
105
109
|
if (e.data.type === "trackUpdateDone" && this.workerTrackDone !== null) {
|
|
106
110
|
this.workerTrackDone(e.data);
|
|
107
111
|
}
|
|
@@ -128,8 +132,7 @@ class Controller {
|
|
|
128
132
|
const allMatchingData = [];
|
|
129
133
|
const allDimensions = [];
|
|
130
134
|
for (const buffer of buffers) {
|
|
131
|
-
const
|
|
132
|
-
const result = compiler.importData(buffer);
|
|
135
|
+
const result = protocol.decodeTaar(buffer);
|
|
133
136
|
const dataList = result.dataList || [];
|
|
134
137
|
for (const item of dataList) {
|
|
135
138
|
allMatchingData.push(item.matchingData);
|
|
@@ -147,6 +150,8 @@ class Controller {
|
|
|
147
150
|
projectionTransform: this.projectionTransform,
|
|
148
151
|
debugMode: this.debugMode,
|
|
149
152
|
matchingDataList: allMatchingData,
|
|
153
|
+
trackingDataList: allTrackingData,
|
|
154
|
+
markerDimensions: allDimensions
|
|
150
155
|
});
|
|
151
156
|
}
|
|
152
157
|
this.markerDimensions = allDimensions;
|
|
@@ -183,12 +188,15 @@ class Controller {
|
|
|
183
188
|
return this._glModelViewMatrix(modelViewTransform, targetIndex);
|
|
184
189
|
}
|
|
185
190
|
async _detectAndMatch(inputData, targetIndexes) {
|
|
186
|
-
const { featurePoints } = this.
|
|
187
|
-
|
|
188
|
-
return { targetIndex
|
|
191
|
+
const { targetIndex, modelViewTransform, screenCoords, worldCoords, featurePoints } = await this._workerMatch(null, // No feature points, worker will detect from inputData
|
|
192
|
+
targetIndexes, inputData);
|
|
193
|
+
return { targetIndex, modelViewTransform, screenCoords, worldCoords, featurePoints };
|
|
189
194
|
}
|
|
190
195
|
async _trackAndUpdate(inputData, lastModelViewTransform, targetIndex) {
|
|
191
|
-
const { worldCoords, screenCoords, reliabilities, indices = [], octaveIndex = 0 } = this.
|
|
196
|
+
const { worldCoords, screenCoords, reliabilities, indices = [], octaveIndex = 0 } = await this._workerTrack(inputData, lastModelViewTransform, targetIndex);
|
|
197
|
+
if (!worldCoords || worldCoords.length === 0) {
|
|
198
|
+
return { modelViewTransform: null, screenCoords: [], reliabilities: [], stabilities: [] };
|
|
199
|
+
}
|
|
192
200
|
const state = this.trackingStates[targetIndex];
|
|
193
201
|
if (!state.pointStabilities)
|
|
194
202
|
state.pointStabilities = [];
|
|
@@ -221,7 +229,11 @@ class Controller {
|
|
|
221
229
|
for (let i = 0; i < stabilities.length; i++) {
|
|
222
230
|
if (stabilities[i] > 0) {
|
|
223
231
|
const isCurrentlyTracked = indices.includes(i);
|
|
224
|
-
finalScreenCoords.push(
|
|
232
|
+
finalScreenCoords.push({
|
|
233
|
+
x: lastCoords[i].x,
|
|
234
|
+
y: lastCoords[i].y,
|
|
235
|
+
id: i // Unique index from tracker
|
|
236
|
+
});
|
|
225
237
|
finalStabilities.push(stabilities[i]);
|
|
226
238
|
if (isCurrentlyTracked) {
|
|
227
239
|
const idxInResult = indices.indexOf(i);
|
|
@@ -292,11 +304,14 @@ class Controller {
|
|
|
292
304
|
continue;
|
|
293
305
|
matchingIndexes.push(i);
|
|
294
306
|
}
|
|
295
|
-
const { targetIndex: matchedTargetIndex, modelViewTransform } = await this._detectAndMatch(inputData, matchingIndexes);
|
|
307
|
+
const { targetIndex: matchedTargetIndex, modelViewTransform, featurePoints } = await this._detectAndMatch(inputData, matchingIndexes);
|
|
296
308
|
if (matchedTargetIndex !== -1) {
|
|
297
309
|
this.trackingStates[matchedTargetIndex].isTracking = true;
|
|
298
310
|
this.trackingStates[matchedTargetIndex].currentModelViewTransform = modelViewTransform;
|
|
299
311
|
}
|
|
312
|
+
// If we have feature points, we can store them in a special "lastSeenFeatures"
|
|
313
|
+
// or just pass them in processDone for general visualization
|
|
314
|
+
this.onUpdate && this.onUpdate({ type: "featurePoints", featurePoints });
|
|
300
315
|
}
|
|
301
316
|
for (let i = 0; i < this.trackingStates.length; i++) {
|
|
302
317
|
const trackingState = this.trackingStates[i];
|
|
@@ -388,10 +403,20 @@ class Controller {
|
|
|
388
403
|
return null;
|
|
389
404
|
return this._workerTrackUpdate(modelViewTransform, trackFeatures);
|
|
390
405
|
}
|
|
391
|
-
_workerMatch(featurePoints, targetIndexes) {
|
|
406
|
+
_workerMatch(featurePoints, targetIndexes, inputData = null) {
|
|
392
407
|
return new Promise((resolve) => {
|
|
393
408
|
if (!this.worker) {
|
|
394
|
-
|
|
409
|
+
// If no feature points but we have input data, detect first
|
|
410
|
+
let fpPromise;
|
|
411
|
+
if (!featurePoints && inputData) {
|
|
412
|
+
fpPromise = Promise.resolve(this.fullDetector.detect(inputData).featurePoints);
|
|
413
|
+
}
|
|
414
|
+
else {
|
|
415
|
+
fpPromise = Promise.resolve(featurePoints);
|
|
416
|
+
}
|
|
417
|
+
fpPromise.then(fp => {
|
|
418
|
+
this._matchOnMainThread(fp, targetIndexes).then(resolve);
|
|
419
|
+
}).catch(() => resolve({ targetIndex: -1 }));
|
|
395
420
|
return;
|
|
396
421
|
}
|
|
397
422
|
const timeout = setTimeout(() => {
|
|
@@ -406,16 +431,45 @@ class Controller {
|
|
|
406
431
|
modelViewTransform: data.modelViewTransform,
|
|
407
432
|
screenCoords: data.screenCoords,
|
|
408
433
|
worldCoords: data.worldCoords,
|
|
434
|
+
featurePoints: data.featurePoints,
|
|
409
435
|
debugExtra: data.debugExtra,
|
|
410
436
|
});
|
|
411
437
|
};
|
|
412
|
-
|
|
438
|
+
if (inputData) {
|
|
439
|
+
this.worker.postMessage({ type: "match", inputData, targetIndexes });
|
|
440
|
+
}
|
|
441
|
+
else {
|
|
442
|
+
this.worker.postMessage({ type: "match", featurePoints: featurePoints, targetIndexes });
|
|
443
|
+
}
|
|
444
|
+
});
|
|
445
|
+
}
|
|
446
|
+
_workerTrack(inputData, lastModelViewTransform, targetIndex) {
|
|
447
|
+
return new Promise((resolve) => {
|
|
448
|
+
if (!this.worker) {
|
|
449
|
+
resolve(this.tracker.track(inputData, lastModelViewTransform, targetIndex));
|
|
450
|
+
return;
|
|
451
|
+
}
|
|
452
|
+
const timeout = setTimeout(() => {
|
|
453
|
+
this.workerFullTrackDone = null;
|
|
454
|
+
resolve({ worldCoords: [], screenCoords: [], reliabilities: [] });
|
|
455
|
+
}, WORKER_TIMEOUT_MS);
|
|
456
|
+
this.workerFullTrackDone = (data) => {
|
|
457
|
+
clearTimeout(timeout);
|
|
458
|
+
this.workerFullTrackDone = null;
|
|
459
|
+
resolve(data);
|
|
460
|
+
};
|
|
461
|
+
this.worker.postMessage({
|
|
462
|
+
type: "track",
|
|
463
|
+
inputData,
|
|
464
|
+
lastModelViewTransform,
|
|
465
|
+
targetIndex
|
|
466
|
+
});
|
|
413
467
|
});
|
|
414
468
|
}
|
|
415
469
|
async _matchOnMainThread(featurePoints, targetIndexes) {
|
|
416
470
|
if (!this.mainThreadMatcher) {
|
|
417
|
-
const { Matcher } = await import("
|
|
418
|
-
const { Estimator } = await import("
|
|
471
|
+
const { Matcher } = await import("../core/matching/matcher.js");
|
|
472
|
+
const { Estimator } = await import("../core/estimation/estimator.js");
|
|
419
473
|
this.mainThreadMatcher = new Matcher(this.inputWidth, this.inputHeight, this.debugMode);
|
|
420
474
|
this.mainThreadEstimator = new Estimator(this.projectionTransform);
|
|
421
475
|
}
|
|
@@ -474,7 +528,7 @@ class Controller {
|
|
|
474
528
|
}
|
|
475
529
|
async _trackUpdateOnMainThread(modelViewTransform, trackingFeatures) {
|
|
476
530
|
if (!this.mainThreadEstimator) {
|
|
477
|
-
const { Estimator } = await import("
|
|
531
|
+
const { Estimator } = await import("../core/estimation/estimator.js");
|
|
478
532
|
this.mainThreadEstimator = new Estimator(this.projectionTransform);
|
|
479
533
|
}
|
|
480
534
|
const { worldCoords, screenCoords, stabilities } = trackingFeatures;
|
|
@@ -1,9 +1,13 @@
|
|
|
1
|
-
import { Matcher } from "
|
|
2
|
-
import { Estimator } from "
|
|
1
|
+
import { Matcher } from "../core/matching/matcher.js";
|
|
2
|
+
import { Estimator } from "../core/estimation/estimator.js";
|
|
3
|
+
import { Tracker } from "../core/tracker/tracker.js";
|
|
4
|
+
import { DetectorLite } from "../core/detector/detector-lite.js";
|
|
3
5
|
let matchingDataList = null;
|
|
4
6
|
let debugMode = false;
|
|
5
7
|
let matcher = null;
|
|
6
8
|
let estimator = null;
|
|
9
|
+
let tracker = null;
|
|
10
|
+
let detector = null;
|
|
7
11
|
onmessage = (msg) => {
|
|
8
12
|
const { data } = msg;
|
|
9
13
|
switch (data.type) {
|
|
@@ -12,21 +16,35 @@ onmessage = (msg) => {
|
|
|
12
16
|
debugMode = data.debugMode;
|
|
13
17
|
matcher = new Matcher(data.inputWidth, data.inputHeight, debugMode);
|
|
14
18
|
estimator = new Estimator(data.projectionTransform);
|
|
19
|
+
if (data.trackingDataList && data.markerDimensions) {
|
|
20
|
+
tracker = new Tracker(data.markerDimensions, data.trackingDataList, data.projectionTransform, data.inputWidth, data.inputHeight, debugMode);
|
|
21
|
+
}
|
|
22
|
+
detector = new DetectorLite(data.inputWidth, data.inputHeight, { useLSH: true });
|
|
15
23
|
break;
|
|
16
24
|
case "match":
|
|
17
25
|
const interestedTargetIndexes = data.targetIndexes;
|
|
18
26
|
let matchedTargetIndex = -1;
|
|
19
27
|
let matchedModelViewTransform = null;
|
|
28
|
+
let matchedScreenCoords = null;
|
|
29
|
+
let matchedWorldCoords = null;
|
|
20
30
|
let matchedDebugExtra = null;
|
|
31
|
+
// New: If the worker received image data, run detector here too
|
|
32
|
+
let featurePoints = data.featurePoints;
|
|
33
|
+
if (data.inputData) {
|
|
34
|
+
const detectionResult = detector.detect(data.inputData);
|
|
35
|
+
featurePoints = detectionResult.featurePoints;
|
|
36
|
+
}
|
|
21
37
|
for (let i = 0; i < interestedTargetIndexes.length; i++) {
|
|
22
38
|
const matchingIndex = interestedTargetIndexes[i];
|
|
23
|
-
const { keyframeIndex, screenCoords, worldCoords, debugExtra } = matcher.matchDetection(matchingDataList[matchingIndex],
|
|
39
|
+
const { keyframeIndex, screenCoords, worldCoords, debugExtra } = matcher.matchDetection(matchingDataList[matchingIndex], featurePoints);
|
|
24
40
|
matchedDebugExtra = debugExtra;
|
|
25
41
|
if (keyframeIndex !== -1) {
|
|
26
42
|
const modelViewTransform = estimator.estimate({ screenCoords, worldCoords });
|
|
27
43
|
if (modelViewTransform) {
|
|
28
44
|
matchedTargetIndex = matchingIndex;
|
|
29
45
|
matchedModelViewTransform = modelViewTransform;
|
|
46
|
+
matchedScreenCoords = screenCoords;
|
|
47
|
+
matchedWorldCoords = worldCoords;
|
|
30
48
|
}
|
|
31
49
|
break;
|
|
32
50
|
}
|
|
@@ -35,9 +53,21 @@ onmessage = (msg) => {
|
|
|
35
53
|
type: "matchDone",
|
|
36
54
|
targetIndex: matchedTargetIndex,
|
|
37
55
|
modelViewTransform: matchedModelViewTransform,
|
|
56
|
+
screenCoords: matchedScreenCoords,
|
|
57
|
+
worldCoords: matchedWorldCoords,
|
|
58
|
+
featurePoints: featurePoints,
|
|
38
59
|
debugExtra: matchedDebugExtra,
|
|
39
60
|
});
|
|
40
61
|
break;
|
|
62
|
+
case "track":
|
|
63
|
+
const { inputData: trackInput, lastModelViewTransform, targetIndex } = data;
|
|
64
|
+
const trackResult = tracker.track(trackInput, lastModelViewTransform, targetIndex);
|
|
65
|
+
postMessage({
|
|
66
|
+
type: "trackDone",
|
|
67
|
+
targetIndex,
|
|
68
|
+
...trackResult
|
|
69
|
+
});
|
|
70
|
+
break;
|
|
41
71
|
case "trackUpdate":
|
|
42
72
|
const { modelViewTransform, worldCoords, screenCoords, stabilities } = data;
|
|
43
73
|
const finalModelViewTransform = estimator.refineEstimate({
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@srsergio/taptapp-ar",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.86",
|
|
4
4
|
"description": "Ultra-fast Augmented Reality (AR) SDK for Node.js and Browser. Image tracking with 100% pure JavaScript, zero-dependencies, and high-performance compilation.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"augmented reality",
|
|
@@ -5,30 +5,16 @@
|
|
|
5
5
|
* Usa JavaScript puro para máxima velocidad.
|
|
6
6
|
*/
|
|
7
7
|
import { parentPort } from 'node:worker_threads';
|
|
8
|
-
import { extractTrackingFeatures } from '
|
|
9
|
-
import { buildTrackingImageList } from '
|
|
10
|
-
import { DetectorLite } from '
|
|
11
|
-
import { build as hierarchicalClusteringBuild } from '
|
|
8
|
+
import { extractTrackingFeatures } from '../core/tracker/extract-utils.js';
|
|
9
|
+
import { buildTrackingImageList } from '../core/image-list.js';
|
|
10
|
+
import { DetectorLite } from '../core/detector/detector-lite.js';
|
|
11
|
+
import { build as hierarchicalClusteringBuild } from '../core/matching/hierarchical-clustering.js';
|
|
12
|
+
import { getMorton } from '../core/protocol.js';
|
|
12
13
|
|
|
13
14
|
if (!parentPort) {
|
|
14
15
|
throw new Error('This file must be run as a worker thread.');
|
|
15
16
|
}
|
|
16
17
|
|
|
17
|
-
// Helper for Morton Order sorting inside worker
|
|
18
|
-
function getMorton(x, y) {
|
|
19
|
-
let x_int = x | 0;
|
|
20
|
-
let y_int = y | 0;
|
|
21
|
-
x_int = (x_int | (x_int << 8)) & 0x00FF00FF;
|
|
22
|
-
x_int = (x_int | (x_int << 4)) & 0x0F0F0F0F;
|
|
23
|
-
x_int = (x_int | (x_int << 2)) & 0x33333333;
|
|
24
|
-
x_int = (x_int | (x_int << 1)) & 0x55555555;
|
|
25
|
-
y_int = (y_int | (y_int << 8)) & 0x00FF00FF;
|
|
26
|
-
y_int = (y_int | (y_int << 4)) & 0x0F0F0F0F;
|
|
27
|
-
y_int = (y_int | (y_int << 2)) & 0x33333333;
|
|
28
|
-
y_int = (y_int | (y_int << 1)) & 0x55555555;
|
|
29
|
-
return x_int | (y_int << 1);
|
|
30
|
-
}
|
|
31
|
-
|
|
32
18
|
const mortonCache = new Int32Array(2048); // Cache for sorting stability
|
|
33
19
|
|
|
34
20
|
function sortPoints(points) {
|