@srsergio/taptapp-ar 1.0.78 → 1.0.80
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +21 -0
- package/dist/compiler/controller.d.ts +22 -6
- package/dist/compiler/controller.js +99 -26
- package/dist/compiler/controller.worker.js +2 -1
- package/dist/compiler/estimation/refine-estimate.d.ts +2 -1
- package/dist/compiler/estimation/refine-estimate.js +18 -5
- package/dist/compiler/features/feature-base.d.ts +1 -1
- package/dist/compiler/features/feature-manager.d.ts +1 -1
- package/dist/compiler/features/feature-manager.js +2 -2
- package/dist/compiler/features/one-euro-filter-feature.d.ts +1 -1
- package/dist/compiler/features/one-euro-filter-feature.js +8 -1
- package/dist/compiler/matching/matching.js +1 -1
- package/dist/compiler/offline-compiler.d.ts +92 -8
- package/dist/compiler/offline-compiler.js +3 -86
- package/dist/compiler/simple-ar.d.ts +12 -0
- package/dist/compiler/simple-ar.js +46 -19
- package/dist/compiler/tracker/tracker.d.ts +10 -0
- package/dist/compiler/tracker/tracker.js +6 -4
- package/dist/react/TaptappAR.js +29 -2
- package/dist/react/use-ar.d.ts +7 -0
- package/dist/react/use-ar.js +16 -1
- package/package.json +24 -2
- package/src/compiler/controller.ts +112 -26
- package/src/compiler/controller.worker.js +2 -1
- package/src/compiler/estimation/refine-estimate.js +20 -3
- package/src/compiler/features/feature-base.ts +1 -1
- package/src/compiler/features/feature-manager.ts +2 -2
- package/src/compiler/features/one-euro-filter-feature.ts +11 -1
- package/src/compiler/matching/matching.js +1 -1
- package/src/compiler/offline-compiler.ts +3 -94
- package/src/compiler/simple-ar.ts +62 -20
- package/src/compiler/tracker/tracker.js +7 -4
- package/src/react/TaptappAR.tsx +41 -1
- package/src/react/use-ar.ts +24 -1
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Sergio Lázaro
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
CHANGED
|
@@ -1,11 +1,32 @@
|
|
|
1
1
|
# @srsergio/taptapp-ar
|
|
2
2
|
|
|
3
|
+
[](https://www.npmjs.com/package/@srsergio/taptapp-ar)
|
|
4
|
+
[](https://www.npmjs.com/package/@srsergio/taptapp-ar)
|
|
5
|
+
[](https://opensource.org/licenses/MIT)
|
|
6
|
+
[](https://bundlephobia.com/package/@srsergio/taptapp-ar)
|
|
7
|
+
|
|
3
8
|
🚀 **TapTapp AR** is a high-performance Augmented Reality (AR) toolkit for **Node.js** and **Browser** environments. It provides an ultra-fast offline compiler and a lightweight runtime for image tracking.
|
|
4
9
|
|
|
5
10
|
**100% Pure JavaScript**: This package is now completely independent of **TensorFlow.js** for both compilation and real-time tracking, resulting in massive performance gains and zero-latency initialization.
|
|
6
11
|
|
|
7
12
|
---
|
|
8
13
|
|
|
14
|
+
## 📖 Table of Contents
|
|
15
|
+
- [🌟 Key Features](#-key-features)
|
|
16
|
+
- [🛠 Installation](#-installation)
|
|
17
|
+
- [📊 Industry-Leading Benchmarks](#-industry-leading-benchmarks-v7-moonshot)
|
|
18
|
+
- [🛡️ Robustness & Stability](#️-robustness--stability-stress-tested)
|
|
19
|
+
- [🖼️ Compiler Usage](#️-compiler-usage-nodejs--web)
|
|
20
|
+
- [🎥 Runtime Usage](#-runtime-usage-ar-tracking)
|
|
21
|
+
- [A-Frame Integration](#1-simple-a-frame-integration)
|
|
22
|
+
- [Three.js Wrapper](#2-high-performance-threejs-wrapper)
|
|
23
|
+
- [Raw Controller](#3-raw-controller-advanced--custom-engines)
|
|
24
|
+
- [Vanilla JS (SimpleAR)](#4-vanilla-js-no-framework-)
|
|
25
|
+
- [🏗️ Protocol V7](#️-protocol-v7-moonshot-packed-format)
|
|
26
|
+
- [📄 License & Credits](#-license--credits)
|
|
27
|
+
|
|
28
|
+
---
|
|
29
|
+
|
|
9
30
|
## 🌟 Key Features
|
|
10
31
|
|
|
11
32
|
- 🖼️ **Hyper-Fast Compiler**: Pure JavaScript compiler that generates `.taar` files in **< 3s**.
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { Tracker } from "./tracker/tracker.js";
|
|
2
2
|
import { InputLoader } from "./input-loader.js";
|
|
3
3
|
import { FeatureManager } from "./features/feature-manager.js";
|
|
4
|
+
import { DetectorLite } from "./detector/detector-lite.js";
|
|
4
5
|
export interface ControllerOptions {
|
|
5
6
|
inputWidth: number;
|
|
6
7
|
inputHeight: number;
|
|
@@ -34,6 +35,7 @@ declare class Controller {
|
|
|
34
35
|
mainThreadMatcher: any;
|
|
35
36
|
mainThreadEstimator: any;
|
|
36
37
|
featureManager: FeatureManager;
|
|
38
|
+
fullDetector: DetectorLite | null;
|
|
37
39
|
constructor({ inputWidth, inputHeight, onUpdate, debugMode, maxTrack, warmupTolerance, missTolerance, filterMinCF, filterBeta, worker, }: ControllerOptions);
|
|
38
40
|
_setupWorkerListener(): void;
|
|
39
41
|
_ensureWorker(): void;
|
|
@@ -61,16 +63,20 @@ declare class Controller {
|
|
|
61
63
|
targetIndex: any;
|
|
62
64
|
modelViewTransform: any;
|
|
63
65
|
}>;
|
|
64
|
-
_trackAndUpdate(inputData: any, lastModelViewTransform: number[][], targetIndex: number): Promise<
|
|
66
|
+
_trackAndUpdate(inputData: any, lastModelViewTransform: number[][], targetIndex: number): Promise<{
|
|
67
|
+
modelViewTransform: any;
|
|
68
|
+
screenCoords: {
|
|
69
|
+
x: number;
|
|
70
|
+
y: number;
|
|
71
|
+
}[];
|
|
72
|
+
reliabilities: number[];
|
|
73
|
+
stabilities: any[];
|
|
74
|
+
}>;
|
|
65
75
|
processVideo(input: any): void;
|
|
66
76
|
stopProcessVideo(): void;
|
|
67
77
|
detect(input: any): Promise<{
|
|
68
78
|
featurePoints: any[];
|
|
69
|
-
debugExtra: {
|
|
70
|
-
projectedImage: number[];
|
|
71
|
-
} | {
|
|
72
|
-
projectedImage?: undefined;
|
|
73
|
-
};
|
|
79
|
+
debugExtra: {};
|
|
74
80
|
}>;
|
|
75
81
|
match(featurePoints: any, targetIndex: number): Promise<{
|
|
76
82
|
targetIndex: any;
|
|
@@ -80,6 +86,13 @@ declare class Controller {
|
|
|
80
86
|
debugExtra: any;
|
|
81
87
|
}>;
|
|
82
88
|
track(input: any, modelViewTransform: number[][], targetIndex: number): Promise<{
|
|
89
|
+
worldCoords: never[];
|
|
90
|
+
screenCoords: never[];
|
|
91
|
+
reliabilities: never[];
|
|
92
|
+
debugExtra: {};
|
|
93
|
+
indices?: undefined;
|
|
94
|
+
octaveIndex?: undefined;
|
|
95
|
+
} | {
|
|
83
96
|
worldCoords: {
|
|
84
97
|
x: number;
|
|
85
98
|
y: number;
|
|
@@ -89,6 +102,9 @@ declare class Controller {
|
|
|
89
102
|
x: number;
|
|
90
103
|
y: number;
|
|
91
104
|
}[];
|
|
105
|
+
reliabilities: number[];
|
|
106
|
+
indices: number[];
|
|
107
|
+
octaveIndex: any;
|
|
92
108
|
debugExtra: {};
|
|
93
109
|
}>;
|
|
94
110
|
trackUpdate(modelViewTransform: number[][], trackFeatures: any): Promise<any>;
|
|
@@ -5,7 +5,7 @@ import { FeatureManager } from "./features/feature-manager.js";
|
|
|
5
5
|
import { OneEuroFilterFeature } from "./features/one-euro-filter-feature.js";
|
|
6
6
|
import { TemporalFilterFeature } from "./features/temporal-filter-feature.js";
|
|
7
7
|
import { AutoRotationFeature } from "./features/auto-rotation-feature.js";
|
|
8
|
-
import {
|
|
8
|
+
import { DetectorLite } from "./detector/detector-lite.js";
|
|
9
9
|
let ControllerWorker;
|
|
10
10
|
// Conditional import for worker to avoid crash in non-vite environments
|
|
11
11
|
const getControllerWorker = async () => {
|
|
@@ -24,7 +24,9 @@ ControllerWorker = await getControllerWorker();
|
|
|
24
24
|
const DEFAULT_FILTER_CUTOFF = 0.5;
|
|
25
25
|
const DEFAULT_FILTER_BETA = 0.1;
|
|
26
26
|
const DEFAULT_WARMUP_TOLERANCE = 2; // Instant detection
|
|
27
|
-
const DEFAULT_MISS_TOLERANCE =
|
|
27
|
+
const DEFAULT_MISS_TOLERANCE = 1; // Immediate response to tracking loss
|
|
28
|
+
const WORKER_TIMEOUT_MS = 1000; // Prevent worker hangs from killing the loop
|
|
29
|
+
let loopIdCounter = 0;
|
|
28
30
|
class Controller {
|
|
29
31
|
inputWidth;
|
|
30
32
|
inputHeight;
|
|
@@ -46,6 +48,7 @@ class Controller {
|
|
|
46
48
|
mainThreadMatcher;
|
|
47
49
|
mainThreadEstimator;
|
|
48
50
|
featureManager;
|
|
51
|
+
fullDetector = null;
|
|
49
52
|
constructor({ inputWidth, inputHeight, onUpdate = null, debugMode = false, maxTrack = 1, warmupTolerance = null, missTolerance = null, filterMinCF = null, filterBeta = null, worker = null, }) {
|
|
50
53
|
this.inputWidth = inputWidth;
|
|
51
54
|
this.inputHeight = inputHeight;
|
|
@@ -54,13 +57,15 @@ class Controller {
|
|
|
54
57
|
this.featureManager.addFeature(new OneEuroFilterFeature(filterMinCF === null ? DEFAULT_FILTER_CUTOFF : filterMinCF, filterBeta === null ? DEFAULT_FILTER_BETA : filterBeta));
|
|
55
58
|
this.featureManager.addFeature(new TemporalFilterFeature(warmupTolerance === null ? DEFAULT_WARMUP_TOLERANCE : warmupTolerance, missTolerance === null ? DEFAULT_MISS_TOLERANCE : missTolerance));
|
|
56
59
|
this.featureManager.addFeature(new AutoRotationFeature());
|
|
57
|
-
|
|
60
|
+
// User wants "sin recortes", so we don't add CropDetectionFeature
|
|
58
61
|
this.inputLoader = new InputLoader(this.inputWidth, this.inputHeight);
|
|
59
62
|
this.onUpdate = onUpdate;
|
|
60
63
|
this.debugMode = debugMode;
|
|
61
64
|
this.worker = worker;
|
|
62
65
|
if (this.worker)
|
|
63
66
|
this._setupWorkerListener();
|
|
67
|
+
// Moonshot: Full frame detector for better sensitivity
|
|
68
|
+
this.fullDetector = new DetectorLite(this.inputWidth, this.inputHeight, { useLSH: true });
|
|
64
69
|
this.featureManager.init({
|
|
65
70
|
inputWidth: this.inputWidth,
|
|
66
71
|
inputHeight: this.inputHeight,
|
|
@@ -160,8 +165,7 @@ class Controller {
|
|
|
160
165
|
}
|
|
161
166
|
dummyRun(input) {
|
|
162
167
|
const inputData = this.inputLoader.loadInput(input);
|
|
163
|
-
|
|
164
|
-
cropFeature?.detect(inputData, false);
|
|
168
|
+
this.fullDetector?.detect(inputData);
|
|
165
169
|
this.tracker.dummyRun(inputData);
|
|
166
170
|
}
|
|
167
171
|
getProjectionMatrix() {
|
|
@@ -179,25 +183,67 @@ class Controller {
|
|
|
179
183
|
return this._glModelViewMatrix(modelViewTransform, targetIndex);
|
|
180
184
|
}
|
|
181
185
|
async _detectAndMatch(inputData, targetIndexes) {
|
|
182
|
-
const
|
|
183
|
-
const { featurePoints } = cropFeature.detect(inputData, true);
|
|
186
|
+
const { featurePoints } = this.fullDetector.detect(inputData);
|
|
184
187
|
const { targetIndex: matchedTargetIndex, modelViewTransform } = await this._workerMatch(featurePoints, targetIndexes);
|
|
185
188
|
return { targetIndex: matchedTargetIndex, modelViewTransform };
|
|
186
189
|
}
|
|
187
190
|
async _trackAndUpdate(inputData, lastModelViewTransform, targetIndex) {
|
|
188
|
-
const { worldCoords, screenCoords } = this.tracker.track(inputData, lastModelViewTransform, targetIndex);
|
|
189
|
-
|
|
190
|
-
|
|
191
|
+
const { worldCoords, screenCoords, reliabilities, indices = [], octaveIndex = 0 } = this.tracker.track(inputData, lastModelViewTransform, targetIndex);
|
|
192
|
+
const state = this.trackingStates[targetIndex];
|
|
193
|
+
if (!state.pointStabilities)
|
|
194
|
+
state.pointStabilities = [];
|
|
195
|
+
if (!state.pointStabilities[octaveIndex]) {
|
|
196
|
+
// Initialize stabilities for this octave if not exists
|
|
197
|
+
const numPoints = this.tracker.prebuiltData[targetIndex][octaveIndex].px.length;
|
|
198
|
+
state.pointStabilities[octaveIndex] = new Float32Array(numPoints).fill(0.5); // Start at 0.5
|
|
199
|
+
}
|
|
200
|
+
const stabilities = state.pointStabilities[octaveIndex];
|
|
201
|
+
const currentStabilities = [];
|
|
202
|
+
// Update all points in this octave
|
|
203
|
+
for (let i = 0; i < stabilities.length; i++) {
|
|
204
|
+
const isTracked = indices.includes(i);
|
|
205
|
+
if (isTracked) {
|
|
206
|
+
stabilities[i] = Math.min(1.0, stabilities[i] + 0.35); // Fast recovery (approx 3 frames)
|
|
207
|
+
}
|
|
208
|
+
else {
|
|
209
|
+
stabilities[i] = Math.max(0.0, stabilities[i] - 0.12); // Slightly more forgiving loss
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
// Collect stabilities and FILTER OUT excessive flickerers (Dead Zone)
|
|
213
|
+
const filteredWorldCoords = [];
|
|
214
|
+
const filteredScreenCoords = [];
|
|
215
|
+
const filteredStabilities = [];
|
|
216
|
+
for (let i = 0; i < indices.length; i++) {
|
|
217
|
+
const s = stabilities[indices[i]];
|
|
218
|
+
if (s > 0.3) { // Hard Cutoff: points with <30% stability are ignored
|
|
219
|
+
filteredWorldCoords.push(worldCoords[i]);
|
|
220
|
+
filteredScreenCoords.push(screenCoords[i]);
|
|
221
|
+
filteredStabilities.push(s);
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
// STRICT QUALITY CHECK: Prevent "sticky" tracking on background noise.
|
|
225
|
+
// We require a minimum number of high-confidence AND STABLE points.
|
|
226
|
+
const stableAndReliable = reliabilities.filter((r, idx) => r > 0.75 && stabilities[indices[idx]] > 0.5).length;
|
|
227
|
+
if (stableAndReliable < 6 || filteredWorldCoords.length < 8) {
|
|
228
|
+
return { modelViewTransform: null, screenCoords: [], reliabilities: [], stabilities: [] };
|
|
229
|
+
}
|
|
191
230
|
const modelViewTransform = await this._workerTrackUpdate(lastModelViewTransform, {
|
|
192
|
-
worldCoords,
|
|
193
|
-
screenCoords,
|
|
231
|
+
worldCoords: filteredWorldCoords,
|
|
232
|
+
screenCoords: filteredScreenCoords,
|
|
233
|
+
stabilities: filteredStabilities
|
|
194
234
|
});
|
|
195
|
-
return
|
|
235
|
+
return {
|
|
236
|
+
modelViewTransform,
|
|
237
|
+
screenCoords: filteredScreenCoords,
|
|
238
|
+
reliabilities: reliabilities.filter((_, idx) => stabilities[indices[idx]] > 0.3),
|
|
239
|
+
stabilities: filteredStabilities
|
|
240
|
+
};
|
|
196
241
|
}
|
|
197
242
|
processVideo(input) {
|
|
198
243
|
if (this.processingVideo)
|
|
199
244
|
return;
|
|
200
245
|
this.processingVideo = true;
|
|
246
|
+
const currentLoopId = ++loopIdCounter; // Added for ghost loop prevention
|
|
201
247
|
this.trackingStates = [];
|
|
202
248
|
for (let i = 0; i < (this.markerDimensions?.length || 0); i++) {
|
|
203
249
|
this.trackingStates.push({
|
|
@@ -210,7 +256,7 @@ class Controller {
|
|
|
210
256
|
}
|
|
211
257
|
const startProcessing = async () => {
|
|
212
258
|
while (true) {
|
|
213
|
-
if (!this.processingVideo)
|
|
259
|
+
if (!this.processingVideo || currentLoopId !== loopIdCounter)
|
|
214
260
|
break;
|
|
215
261
|
const inputData = this.inputLoader.loadInput(input);
|
|
216
262
|
const nTracking = this.trackingStates.reduce((acc, s) => acc + (!!s.isTracking ? 1 : 0), 0);
|
|
@@ -233,12 +279,18 @@ class Controller {
|
|
|
233
279
|
for (let i = 0; i < this.trackingStates.length; i++) {
|
|
234
280
|
const trackingState = this.trackingStates[i];
|
|
235
281
|
if (trackingState.isTracking) {
|
|
236
|
-
|
|
237
|
-
if (modelViewTransform === null) {
|
|
282
|
+
const result = await this._trackAndUpdate(inputData, trackingState.currentModelViewTransform, i);
|
|
283
|
+
if (result === null || result.modelViewTransform === null) {
|
|
238
284
|
trackingState.isTracking = false;
|
|
285
|
+
trackingState.screenCoords = [];
|
|
286
|
+
trackingState.reliabilities = [];
|
|
287
|
+
trackingState.stabilities = [];
|
|
239
288
|
}
|
|
240
289
|
else {
|
|
241
|
-
trackingState.currentModelViewTransform = modelViewTransform;
|
|
290
|
+
trackingState.currentModelViewTransform = result.modelViewTransform;
|
|
291
|
+
trackingState.screenCoords = result.screenCoords;
|
|
292
|
+
trackingState.reliabilities = result.reliabilities;
|
|
293
|
+
trackingState.stabilities = result.stabilities;
|
|
242
294
|
}
|
|
243
295
|
}
|
|
244
296
|
const wasShowing = trackingState.showing;
|
|
@@ -250,7 +302,12 @@ class Controller {
|
|
|
250
302
|
}
|
|
251
303
|
if (trackingState.showing) {
|
|
252
304
|
const worldMatrix = this._glModelViewMatrix(trackingState.currentModelViewTransform, i);
|
|
253
|
-
|
|
305
|
+
// Calculate confidence score based on point stability
|
|
306
|
+
const stabilities = trackingState.stabilities || [];
|
|
307
|
+
const avgStability = stabilities.length > 0
|
|
308
|
+
? stabilities.reduce((a, b) => a + b, 0) / stabilities.length
|
|
309
|
+
: 0;
|
|
310
|
+
const filteredMatrix = this.featureManager.applyWorldMatrixFilters(i, worldMatrix, { stability: avgStability });
|
|
254
311
|
trackingState.trackingMatrix = filteredMatrix;
|
|
255
312
|
let finalMatrix = [...filteredMatrix];
|
|
256
313
|
const isInputRotated = input.width === this.inputHeight && input.height === this.inputWidth;
|
|
@@ -264,7 +321,10 @@ class Controller {
|
|
|
264
321
|
type: "updateMatrix",
|
|
265
322
|
targetIndex: i,
|
|
266
323
|
worldMatrix: finalMatrix,
|
|
267
|
-
modelViewTransform: trackingState.currentModelViewTransform
|
|
324
|
+
modelViewTransform: trackingState.currentModelViewTransform,
|
|
325
|
+
screenCoords: trackingState.screenCoords,
|
|
326
|
+
reliabilities: trackingState.reliabilities,
|
|
327
|
+
stabilities: trackingState.stabilities
|
|
268
328
|
});
|
|
269
329
|
}
|
|
270
330
|
}
|
|
@@ -284,9 +344,8 @@ class Controller {
|
|
|
284
344
|
}
|
|
285
345
|
async detect(input) {
|
|
286
346
|
const inputData = this.inputLoader.loadInput(input);
|
|
287
|
-
const
|
|
288
|
-
|
|
289
|
-
return { featurePoints, debugExtra };
|
|
347
|
+
const { featurePoints } = this.fullDetector.detect(inputData);
|
|
348
|
+
return { featurePoints, debugExtra: {} };
|
|
290
349
|
}
|
|
291
350
|
async match(featurePoints, targetIndex) {
|
|
292
351
|
const { targetIndex: matchedTargetIndex, modelViewTransform, screenCoords, worldCoords, debugExtra } = await this._workerMatch(featurePoints, [
|
|
@@ -306,10 +365,16 @@ class Controller {
|
|
|
306
365
|
_workerMatch(featurePoints, targetIndexes) {
|
|
307
366
|
return new Promise((resolve) => {
|
|
308
367
|
if (!this.worker) {
|
|
309
|
-
this._matchOnMainThread(featurePoints, targetIndexes).then(resolve);
|
|
368
|
+
this._matchOnMainThread(featurePoints, targetIndexes).then(resolve).catch(() => resolve({ targetIndex: -1 }));
|
|
310
369
|
return;
|
|
311
370
|
}
|
|
371
|
+
const timeout = setTimeout(() => {
|
|
372
|
+
this.workerMatchDone = null;
|
|
373
|
+
resolve({ targetIndex: -1 });
|
|
374
|
+
}, WORKER_TIMEOUT_MS);
|
|
312
375
|
this.workerMatchDone = (data) => {
|
|
376
|
+
clearTimeout(timeout);
|
|
377
|
+
this.workerMatchDone = null;
|
|
313
378
|
resolve({
|
|
314
379
|
targetIndex: data.targetIndex,
|
|
315
380
|
modelViewTransform: data.modelViewTransform,
|
|
@@ -359,18 +424,25 @@ class Controller {
|
|
|
359
424
|
_workerTrackUpdate(modelViewTransform, trackingFeatures) {
|
|
360
425
|
return new Promise((resolve) => {
|
|
361
426
|
if (!this.worker) {
|
|
362
|
-
this._trackUpdateOnMainThread(modelViewTransform, trackingFeatures).then(resolve);
|
|
427
|
+
this._trackUpdateOnMainThread(modelViewTransform, trackingFeatures).then(resolve).catch(() => resolve(null));
|
|
363
428
|
return;
|
|
364
429
|
}
|
|
430
|
+
const timeout = setTimeout(() => {
|
|
431
|
+
this.workerTrackDone = null;
|
|
432
|
+
resolve(null);
|
|
433
|
+
}, WORKER_TIMEOUT_MS);
|
|
365
434
|
this.workerTrackDone = (data) => {
|
|
435
|
+
clearTimeout(timeout);
|
|
436
|
+
this.workerTrackDone = null;
|
|
366
437
|
resolve(data.modelViewTransform);
|
|
367
438
|
};
|
|
368
|
-
const { worldCoords, screenCoords } = trackingFeatures;
|
|
439
|
+
const { worldCoords, screenCoords, stabilities } = trackingFeatures;
|
|
369
440
|
this.worker.postMessage({
|
|
370
441
|
type: "trackUpdate",
|
|
371
442
|
modelViewTransform,
|
|
372
443
|
worldCoords,
|
|
373
444
|
screenCoords,
|
|
445
|
+
stabilities
|
|
374
446
|
});
|
|
375
447
|
});
|
|
376
448
|
}
|
|
@@ -379,11 +451,12 @@ class Controller {
|
|
|
379
451
|
const { Estimator } = await import("./estimation/estimator.js");
|
|
380
452
|
this.mainThreadEstimator = new Estimator(this.projectionTransform);
|
|
381
453
|
}
|
|
382
|
-
const { worldCoords, screenCoords } = trackingFeatures;
|
|
454
|
+
const { worldCoords, screenCoords, stabilities } = trackingFeatures;
|
|
383
455
|
return this.mainThreadEstimator.refineEstimate({
|
|
384
456
|
initialModelViewTransform: modelViewTransform,
|
|
385
457
|
worldCoords,
|
|
386
458
|
screenCoords,
|
|
459
|
+
stabilities
|
|
387
460
|
});
|
|
388
461
|
}
|
|
389
462
|
_glModelViewMatrix(modelViewTransform, targetIndex) {
|
|
@@ -39,11 +39,12 @@ onmessage = (msg) => {
|
|
|
39
39
|
});
|
|
40
40
|
break;
|
|
41
41
|
case "trackUpdate":
|
|
42
|
-
const { modelViewTransform, worldCoords, screenCoords } = data;
|
|
42
|
+
const { modelViewTransform, worldCoords, screenCoords, stabilities } = data;
|
|
43
43
|
const finalModelViewTransform = estimator.refineEstimate({
|
|
44
44
|
initialModelViewTransform: modelViewTransform,
|
|
45
45
|
worldCoords,
|
|
46
46
|
screenCoords,
|
|
47
|
+
stabilities, // Stability-based weights
|
|
47
48
|
});
|
|
48
49
|
postMessage({
|
|
49
50
|
type: "trackUpdateDone",
|
|
@@ -1,6 +1,7 @@
|
|
|
1
|
-
export function refineEstimate({ initialModelViewTransform, projectionTransform, worldCoords, screenCoords, }: {
|
|
1
|
+
export function refineEstimate({ initialModelViewTransform, projectionTransform, worldCoords, screenCoords, stabilities, }: {
|
|
2
2
|
initialModelViewTransform: any;
|
|
3
3
|
projectionTransform: any;
|
|
4
4
|
worldCoords: any;
|
|
5
5
|
screenCoords: any;
|
|
6
|
+
stabilities: any;
|
|
6
7
|
}): never[][] | null;
|
|
@@ -9,7 +9,8 @@ const ICP_BREAK_LOOP_ERROR_RATIO_THRESH = 0.99;
|
|
|
9
9
|
let mat = [[], [], []];
|
|
10
10
|
let J_U_Xc = [[], []]; // 2x3
|
|
11
11
|
let J_Xc_S = [[], [], []]; // 3x6
|
|
12
|
-
const refineEstimate = ({ initialModelViewTransform, projectionTransform, worldCoords, screenCoords,
|
|
12
|
+
const refineEstimate = ({ initialModelViewTransform, projectionTransform, worldCoords, screenCoords, stabilities, // Stability-based weighting
|
|
13
|
+
}) => {
|
|
13
14
|
// Question: shall we normlize the screen coords as well?
|
|
14
15
|
// Question: do we need to normlize the scale as well, i.e. make coords from -1 to 1
|
|
15
16
|
//
|
|
@@ -59,6 +60,7 @@ const refineEstimate = ({ initialModelViewTransform, projectionTransform, worldC
|
|
|
59
60
|
projectionTransform,
|
|
60
61
|
worldCoords: normalizedWorldCoords,
|
|
61
62
|
screenCoords,
|
|
63
|
+
stabilities, // Pass weights to ICP
|
|
62
64
|
inlierProb: inlierProbs[i],
|
|
63
65
|
});
|
|
64
66
|
updatedModelViewTransform = ret.modelViewTransform;
|
|
@@ -87,7 +89,7 @@ const refineEstimate = ({ initialModelViewTransform, projectionTransform, worldC
|
|
|
87
89
|
};
|
|
88
90
|
// ICP iteration
|
|
89
91
|
// Question: can someone provide theoretical reference / mathematical proof for the following computations?
|
|
90
|
-
const _doICP = ({ initialModelViewTransform, projectionTransform, worldCoords, screenCoords, inlierProb, }) => {
|
|
92
|
+
const _doICP = ({ initialModelViewTransform, projectionTransform, worldCoords, screenCoords, stabilities, inlierProb, }) => {
|
|
91
93
|
const isRobustMode = inlierProb < 1;
|
|
92
94
|
let modelViewTransform = initialModelViewTransform;
|
|
93
95
|
let err0 = 0.0;
|
|
@@ -152,7 +154,11 @@ const _doICP = ({ initialModelViewTransform, projectionTransform, worldCoords, s
|
|
|
152
154
|
worldCoord: worldCoords[n],
|
|
153
155
|
});
|
|
154
156
|
if (isRobustMode) {
|
|
155
|
-
const
|
|
157
|
+
const robustW = (1.0 - E[n] / K2) * (1.0 - E[n] / K2);
|
|
158
|
+
// Log-weighted stability: suppresses vibrators aggressively but allows recovery
|
|
159
|
+
const s = stabilities ? stabilities[n] : 1.0;
|
|
160
|
+
const stabilityW = s * Math.log10(9 * s + 1);
|
|
161
|
+
const W = robustW * stabilityW;
|
|
156
162
|
for (let j = 0; j < 2; j++) {
|
|
157
163
|
for (let i = 0; i < 6; i++) {
|
|
158
164
|
J_U_S[j][i] *= W;
|
|
@@ -162,8 +168,15 @@ const _doICP = ({ initialModelViewTransform, projectionTransform, worldCoords, s
|
|
|
162
168
|
dU.push([dys[n] * W]);
|
|
163
169
|
}
|
|
164
170
|
else {
|
|
165
|
-
|
|
166
|
-
|
|
171
|
+
const s = stabilities ? stabilities[n] : 1.0;
|
|
172
|
+
const W = s * Math.log10(9 * s + 1);
|
|
173
|
+
for (let j = 0; j < 2; j++) {
|
|
174
|
+
for (let i = 0; i < 6; i++) {
|
|
175
|
+
J_U_S[j][i] *= W;
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
dU.push([dxs[n] * W]);
|
|
179
|
+
dU.push([dys[n] * W]);
|
|
167
180
|
}
|
|
168
181
|
for (let i = 0; i < J_U_S.length; i++) {
|
|
169
182
|
allJ_U_S.push(J_U_S[i]);
|
|
@@ -38,7 +38,7 @@ export interface ControllerFeature extends Feature {
|
|
|
38
38
|
/**
|
|
39
39
|
* Hook to filter or modify the final world matrix
|
|
40
40
|
*/
|
|
41
|
-
filterWorldMatrix?(targetIndex: number, worldMatrix: number[]): number[];
|
|
41
|
+
filterWorldMatrix?(targetIndex: number, worldMatrix: number[], context?: any): number[];
|
|
42
42
|
/**
|
|
43
43
|
* Hook to decide if a target should be shown
|
|
44
44
|
*/
|
|
@@ -5,7 +5,7 @@ export declare class FeatureManager {
|
|
|
5
5
|
getFeature<T extends ControllerFeature>(id: string): T | undefined;
|
|
6
6
|
init(context: FeatureContext): void;
|
|
7
7
|
beforeProcess(inputData: any): void;
|
|
8
|
-
applyWorldMatrixFilters(targetIndex: number, worldMatrix: number[]): number[];
|
|
8
|
+
applyWorldMatrixFilters(targetIndex: number, worldMatrix: number[], context?: any): number[];
|
|
9
9
|
shouldShow(targetIndex: number, isTracking: boolean): boolean;
|
|
10
10
|
notifyUpdate(data: any): void;
|
|
11
11
|
dispose(): void;
|
|
@@ -20,11 +20,11 @@ export class FeatureManager {
|
|
|
20
20
|
}
|
|
21
21
|
}
|
|
22
22
|
}
|
|
23
|
-
applyWorldMatrixFilters(targetIndex, worldMatrix) {
|
|
23
|
+
applyWorldMatrixFilters(targetIndex, worldMatrix, context) {
|
|
24
24
|
let result = worldMatrix;
|
|
25
25
|
for (const feature of this.features) {
|
|
26
26
|
if (feature.enabled && feature.filterWorldMatrix) {
|
|
27
|
-
result = feature.filterWorldMatrix(targetIndex, result);
|
|
27
|
+
result = feature.filterWorldMatrix(targetIndex, result, context);
|
|
28
28
|
}
|
|
29
29
|
}
|
|
30
30
|
return result;
|
|
@@ -10,6 +10,6 @@ export declare class OneEuroFilterFeature implements ControllerFeature {
|
|
|
10
10
|
constructor(minCutOff?: number, beta?: number);
|
|
11
11
|
init(context: FeatureContext): void;
|
|
12
12
|
private getFilter;
|
|
13
|
-
filterWorldMatrix(targetIndex: number, worldMatrix: number[]): number[];
|
|
13
|
+
filterWorldMatrix(targetIndex: number, worldMatrix: number[], context?: any): number[];
|
|
14
14
|
onUpdate(data: any): void;
|
|
15
15
|
}
|
|
@@ -23,10 +23,17 @@ export class OneEuroFilterFeature {
|
|
|
23
23
|
}
|
|
24
24
|
return this.filters[targetIndex];
|
|
25
25
|
}
|
|
26
|
-
filterWorldMatrix(targetIndex, worldMatrix) {
|
|
26
|
+
filterWorldMatrix(targetIndex, worldMatrix, context) {
|
|
27
27
|
if (!this.enabled)
|
|
28
28
|
return worldMatrix;
|
|
29
29
|
const filter = this.getFilter(targetIndex);
|
|
30
|
+
const stability = context?.stability ?? 1.0;
|
|
31
|
+
// Dynamic Cutoff: If points are very stable (1.0), use higher cutoff (less responsiveness loss).
|
|
32
|
+
// If points are unstable (0.3), use much lower cutoff (heavy smoothing).
|
|
33
|
+
// We use a squared curve for even more aggressive suppression of jitter on unstable points.
|
|
34
|
+
const dynamicMinCutOff = this.minCutOff * (0.05 + Math.pow(stability, 2) * 0.95);
|
|
35
|
+
filter.minCutOff = dynamicMinCutOff;
|
|
36
|
+
filter.beta = this.beta;
|
|
30
37
|
return filter.filter(Date.now(), worldMatrix);
|
|
31
38
|
}
|
|
32
39
|
onUpdate(data) {
|
|
@@ -203,7 +203,7 @@ const _query = ({ node, descriptors, querypoint, queue, keypointIndexes, numPop
|
|
|
203
203
|
queue.push({ node: childrenOrIndices[i], d: dist });
|
|
204
204
|
}
|
|
205
205
|
else {
|
|
206
|
-
_query({ node: childrenOrIndices[i], descriptors, querypoint, queue, keypointIndexes, numPop });
|
|
206
|
+
_query({ node: childrenOrIndices[i], descriptors, querypoint, queue, keypointIndexes, numPop: numPop + 1 });
|
|
207
207
|
}
|
|
208
208
|
}
|
|
209
209
|
if (numPop < CLUSTER_MAX_POP && queue.length > 0) {
|
|
@@ -5,26 +5,110 @@
|
|
|
5
5
|
* que NO depende de TensorFlow, eliminando todos los problemas de
|
|
6
6
|
* inicialización, bloqueos y compatibilidad.
|
|
7
7
|
*/
|
|
8
|
-
import { WorkerPool } from "./utils/worker-pool.js";
|
|
9
8
|
export declare class OfflineCompiler {
|
|
10
9
|
data: any;
|
|
11
|
-
workerPool: WorkerPool | null;
|
|
12
10
|
constructor();
|
|
13
|
-
_initNodeWorkers(): Promise<void>;
|
|
14
11
|
compileImageTargets(images: any[], progressCallback: (p: number) => void): Promise<any>;
|
|
15
|
-
_compileTarget(targetImages: any[], progressCallback: (p: number) => void): Promise<
|
|
16
|
-
|
|
17
|
-
|
|
12
|
+
_compileTarget(targetImages: any[], progressCallback: (p: number) => void): Promise<{
|
|
13
|
+
matchingData: {
|
|
14
|
+
maximaPoints: any[];
|
|
15
|
+
minimaPoints: any[];
|
|
16
|
+
maximaPointsCluster: {
|
|
17
|
+
rootNode: {
|
|
18
|
+
leaf: boolean;
|
|
19
|
+
pointIndexes: never[];
|
|
20
|
+
centerPointIndex: null;
|
|
21
|
+
};
|
|
22
|
+
} | {
|
|
23
|
+
rootNode: {
|
|
24
|
+
centerPointIndex: any;
|
|
25
|
+
};
|
|
26
|
+
};
|
|
27
|
+
minimaPointsCluster: {
|
|
28
|
+
rootNode: {
|
|
29
|
+
leaf: boolean;
|
|
30
|
+
pointIndexes: never[];
|
|
31
|
+
centerPointIndex: null;
|
|
32
|
+
};
|
|
33
|
+
} | {
|
|
34
|
+
rootNode: {
|
|
35
|
+
centerPointIndex: any;
|
|
36
|
+
};
|
|
37
|
+
};
|
|
38
|
+
width: any;
|
|
39
|
+
height: any;
|
|
40
|
+
scale: any;
|
|
41
|
+
}[];
|
|
42
|
+
trackingData: Object[];
|
|
43
|
+
}[]>;
|
|
44
|
+
_compileMatch(targetImages: any[], progressCallback: (p: number) => void): Promise<{
|
|
45
|
+
maximaPoints: any[];
|
|
46
|
+
minimaPoints: any[];
|
|
47
|
+
maximaPointsCluster: {
|
|
48
|
+
rootNode: {
|
|
49
|
+
leaf: boolean;
|
|
50
|
+
pointIndexes: never[];
|
|
51
|
+
centerPointIndex: null;
|
|
52
|
+
};
|
|
53
|
+
} | {
|
|
54
|
+
rootNode: {
|
|
55
|
+
centerPointIndex: any;
|
|
56
|
+
};
|
|
57
|
+
};
|
|
58
|
+
minimaPointsCluster: {
|
|
59
|
+
rootNode: {
|
|
60
|
+
leaf: boolean;
|
|
61
|
+
pointIndexes: never[];
|
|
62
|
+
centerPointIndex: null;
|
|
63
|
+
};
|
|
64
|
+
} | {
|
|
65
|
+
rootNode: {
|
|
66
|
+
centerPointIndex: any;
|
|
67
|
+
};
|
|
68
|
+
};
|
|
69
|
+
width: any;
|
|
70
|
+
height: any;
|
|
71
|
+
scale: any;
|
|
72
|
+
}[][]>;
|
|
73
|
+
_compileTrack(targetImages: any[], progressCallback: (p: number) => void): Promise<Object[][]>;
|
|
18
74
|
compileTrack({ progressCallback, targetImages, basePercent }: {
|
|
19
75
|
progressCallback: (p: number) => void;
|
|
20
76
|
targetImages: any[];
|
|
21
77
|
basePercent?: number;
|
|
22
|
-
}): Promise<
|
|
78
|
+
}): Promise<Object[][]>;
|
|
23
79
|
compileMatch({ progressCallback, targetImages, basePercent }: {
|
|
24
80
|
progressCallback: (p: number) => void;
|
|
25
81
|
targetImages: any[];
|
|
26
82
|
basePercent?: number;
|
|
27
|
-
}): Promise<
|
|
83
|
+
}): Promise<{
|
|
84
|
+
maximaPoints: any[];
|
|
85
|
+
minimaPoints: any[];
|
|
86
|
+
maximaPointsCluster: {
|
|
87
|
+
rootNode: {
|
|
88
|
+
leaf: boolean;
|
|
89
|
+
pointIndexes: never[];
|
|
90
|
+
centerPointIndex: null;
|
|
91
|
+
};
|
|
92
|
+
} | {
|
|
93
|
+
rootNode: {
|
|
94
|
+
centerPointIndex: any;
|
|
95
|
+
};
|
|
96
|
+
};
|
|
97
|
+
minimaPointsCluster: {
|
|
98
|
+
rootNode: {
|
|
99
|
+
leaf: boolean;
|
|
100
|
+
pointIndexes: never[];
|
|
101
|
+
centerPointIndex: null;
|
|
102
|
+
};
|
|
103
|
+
} | {
|
|
104
|
+
rootNode: {
|
|
105
|
+
centerPointIndex: any;
|
|
106
|
+
};
|
|
107
|
+
};
|
|
108
|
+
width: any;
|
|
109
|
+
height: any;
|
|
110
|
+
scale: any;
|
|
111
|
+
}[][]>;
|
|
28
112
|
exportData(): Uint8Array<ArrayBuffer>;
|
|
29
113
|
_getMorton(x: number, y: number): number;
|
|
30
114
|
_columnarize(points: any[], tree: any, width: number, height: number): {
|