@srsergio/taptapp-ar 1.0.78 → 1.0.80

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +21 -0
  3. package/dist/compiler/controller.d.ts +22 -6
  4. package/dist/compiler/controller.js +99 -26
  5. package/dist/compiler/controller.worker.js +2 -1
  6. package/dist/compiler/estimation/refine-estimate.d.ts +2 -1
  7. package/dist/compiler/estimation/refine-estimate.js +18 -5
  8. package/dist/compiler/features/feature-base.d.ts +1 -1
  9. package/dist/compiler/features/feature-manager.d.ts +1 -1
  10. package/dist/compiler/features/feature-manager.js +2 -2
  11. package/dist/compiler/features/one-euro-filter-feature.d.ts +1 -1
  12. package/dist/compiler/features/one-euro-filter-feature.js +8 -1
  13. package/dist/compiler/matching/matching.js +1 -1
  14. package/dist/compiler/offline-compiler.d.ts +92 -8
  15. package/dist/compiler/offline-compiler.js +3 -86
  16. package/dist/compiler/simple-ar.d.ts +12 -0
  17. package/dist/compiler/simple-ar.js +46 -19
  18. package/dist/compiler/tracker/tracker.d.ts +10 -0
  19. package/dist/compiler/tracker/tracker.js +6 -4
  20. package/dist/react/TaptappAR.js +29 -2
  21. package/dist/react/use-ar.d.ts +7 -0
  22. package/dist/react/use-ar.js +16 -1
  23. package/package.json +24 -2
  24. package/src/compiler/controller.ts +112 -26
  25. package/src/compiler/controller.worker.js +2 -1
  26. package/src/compiler/estimation/refine-estimate.js +20 -3
  27. package/src/compiler/features/feature-base.ts +1 -1
  28. package/src/compiler/features/feature-manager.ts +2 -2
  29. package/src/compiler/features/one-euro-filter-feature.ts +11 -1
  30. package/src/compiler/matching/matching.js +1 -1
  31. package/src/compiler/offline-compiler.ts +3 -94
  32. package/src/compiler/simple-ar.ts +62 -20
  33. package/src/compiler/tracker/tracker.js +7 -4
  34. package/src/react/TaptappAR.tsx +41 -1
  35. package/src/react/use-ar.ts +24 -1
package/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Sergio Lázaro
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
package/README.md CHANGED
@@ -1,11 +1,32 @@
1
1
  # @srsergio/taptapp-ar
2
2
 
3
+ [![npm version](https://img.shields.io/npm/v/@srsergio/taptapp-ar.svg?style=flat-square)](https://www.npmjs.com/package/@srsergio/taptapp-ar)
4
+ [![npm downloads](https://img.shields.io/npm/dm/@srsergio/taptapp-ar.svg?style=flat-square)](https://www.npmjs.com/package/@srsergio/taptapp-ar)
5
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg?style=flat-square)](https://opensource.org/licenses/MIT)
6
+ [![Bundle Size](https://img.shields.io/bundlephobia/minzip/@srsergio/taptapp-ar?style=flat-square)](https://bundlephobia.com/package/@srsergio/taptapp-ar)
7
+
3
8
  🚀 **TapTapp AR** is a high-performance Augmented Reality (AR) toolkit for **Node.js** and **Browser** environments. It provides an ultra-fast offline compiler and a lightweight runtime for image tracking.
4
9
 
5
10
  **100% Pure JavaScript**: This package is now completely independent of **TensorFlow.js** for both compilation and real-time tracking, resulting in massive performance gains and zero-latency initialization.
6
11
 
7
12
  ---
8
13
 
14
+ ## 📖 Table of Contents
15
+ - [🌟 Key Features](#-key-features)
16
+ - [🛠 Installation](#-installation)
17
+ - [📊 Industry-Leading Benchmarks](#-industry-leading-benchmarks-v7-moonshot)
18
+ - [🛡️ Robustness & Stability](#️-robustness--stability-stress-tested)
19
+ - [🖼️ Compiler Usage](#️-compiler-usage-nodejs--web)
20
+ - [🎥 Runtime Usage](#-runtime-usage-ar-tracking)
21
+ - [A-Frame Integration](#1-simple-a-frame-integration)
22
+ - [Three.js Wrapper](#2-high-performance-threejs-wrapper)
23
+ - [Raw Controller](#3-raw-controller-advanced--custom-engines)
24
+ - [Vanilla JS (SimpleAR)](#4-vanilla-js-no-framework-)
25
+ - [🏗️ Protocol V7](#️-protocol-v7-moonshot-packed-format)
26
+ - [📄 License & Credits](#-license--credits)
27
+
28
+ ---
29
+
9
30
  ## 🌟 Key Features
10
31
 
11
32
  - 🖼️ **Hyper-Fast Compiler**: Pure JavaScript compiler that generates `.taar` files in **< 3s**.
@@ -1,6 +1,7 @@
1
1
  import { Tracker } from "./tracker/tracker.js";
2
2
  import { InputLoader } from "./input-loader.js";
3
3
  import { FeatureManager } from "./features/feature-manager.js";
4
+ import { DetectorLite } from "./detector/detector-lite.js";
4
5
  export interface ControllerOptions {
5
6
  inputWidth: number;
6
7
  inputHeight: number;
@@ -34,6 +35,7 @@ declare class Controller {
34
35
  mainThreadMatcher: any;
35
36
  mainThreadEstimator: any;
36
37
  featureManager: FeatureManager;
38
+ fullDetector: DetectorLite | null;
37
39
  constructor({ inputWidth, inputHeight, onUpdate, debugMode, maxTrack, warmupTolerance, missTolerance, filterMinCF, filterBeta, worker, }: ControllerOptions);
38
40
  _setupWorkerListener(): void;
39
41
  _ensureWorker(): void;
@@ -61,16 +63,20 @@ declare class Controller {
61
63
  targetIndex: any;
62
64
  modelViewTransform: any;
63
65
  }>;
64
- _trackAndUpdate(inputData: any, lastModelViewTransform: number[][], targetIndex: number): Promise<any>;
66
+ _trackAndUpdate(inputData: any, lastModelViewTransform: number[][], targetIndex: number): Promise<{
67
+ modelViewTransform: any;
68
+ screenCoords: {
69
+ x: number;
70
+ y: number;
71
+ }[];
72
+ reliabilities: number[];
73
+ stabilities: any[];
74
+ }>;
65
75
  processVideo(input: any): void;
66
76
  stopProcessVideo(): void;
67
77
  detect(input: any): Promise<{
68
78
  featurePoints: any[];
69
- debugExtra: {
70
- projectedImage: number[];
71
- } | {
72
- projectedImage?: undefined;
73
- };
79
+ debugExtra: {};
74
80
  }>;
75
81
  match(featurePoints: any, targetIndex: number): Promise<{
76
82
  targetIndex: any;
@@ -80,6 +86,13 @@ declare class Controller {
80
86
  debugExtra: any;
81
87
  }>;
82
88
  track(input: any, modelViewTransform: number[][], targetIndex: number): Promise<{
89
+ worldCoords: never[];
90
+ screenCoords: never[];
91
+ reliabilities: never[];
92
+ debugExtra: {};
93
+ indices?: undefined;
94
+ octaveIndex?: undefined;
95
+ } | {
83
96
  worldCoords: {
84
97
  x: number;
85
98
  y: number;
@@ -89,6 +102,9 @@ declare class Controller {
89
102
  x: number;
90
103
  y: number;
91
104
  }[];
105
+ reliabilities: number[];
106
+ indices: number[];
107
+ octaveIndex: any;
92
108
  debugExtra: {};
93
109
  }>;
94
110
  trackUpdate(modelViewTransform: number[][], trackFeatures: any): Promise<any>;
@@ -5,7 +5,7 @@ import { FeatureManager } from "./features/feature-manager.js";
5
5
  import { OneEuroFilterFeature } from "./features/one-euro-filter-feature.js";
6
6
  import { TemporalFilterFeature } from "./features/temporal-filter-feature.js";
7
7
  import { AutoRotationFeature } from "./features/auto-rotation-feature.js";
8
- import { CropDetectionFeature } from "./features/crop-detection-feature.js";
8
+ import { DetectorLite } from "./detector/detector-lite.js";
9
9
  let ControllerWorker;
10
10
  // Conditional import for worker to avoid crash in non-vite environments
11
11
  const getControllerWorker = async () => {
@@ -24,7 +24,9 @@ ControllerWorker = await getControllerWorker();
24
24
  const DEFAULT_FILTER_CUTOFF = 0.5;
25
25
  const DEFAULT_FILTER_BETA = 0.1;
26
26
  const DEFAULT_WARMUP_TOLERANCE = 2; // Instant detection
27
- const DEFAULT_MISS_TOLERANCE = 5; // More grace when partially hidden
27
+ const DEFAULT_MISS_TOLERANCE = 1; // Immediate response to tracking loss
28
+ const WORKER_TIMEOUT_MS = 1000; // Prevent worker hangs from killing the loop
29
+ let loopIdCounter = 0;
28
30
  class Controller {
29
31
  inputWidth;
30
32
  inputHeight;
@@ -46,6 +48,7 @@ class Controller {
46
48
  mainThreadMatcher;
47
49
  mainThreadEstimator;
48
50
  featureManager;
51
+ fullDetector = null;
49
52
  constructor({ inputWidth, inputHeight, onUpdate = null, debugMode = false, maxTrack = 1, warmupTolerance = null, missTolerance = null, filterMinCF = null, filterBeta = null, worker = null, }) {
50
53
  this.inputWidth = inputWidth;
51
54
  this.inputHeight = inputHeight;
@@ -54,13 +57,15 @@ class Controller {
54
57
  this.featureManager.addFeature(new OneEuroFilterFeature(filterMinCF === null ? DEFAULT_FILTER_CUTOFF : filterMinCF, filterBeta === null ? DEFAULT_FILTER_BETA : filterBeta));
55
58
  this.featureManager.addFeature(new TemporalFilterFeature(warmupTolerance === null ? DEFAULT_WARMUP_TOLERANCE : warmupTolerance, missTolerance === null ? DEFAULT_MISS_TOLERANCE : missTolerance));
56
59
  this.featureManager.addFeature(new AutoRotationFeature());
57
- this.featureManager.addFeature(new CropDetectionFeature());
60
+ // User wants "sin recortes", so we don't add CropDetectionFeature
58
61
  this.inputLoader = new InputLoader(this.inputWidth, this.inputHeight);
59
62
  this.onUpdate = onUpdate;
60
63
  this.debugMode = debugMode;
61
64
  this.worker = worker;
62
65
  if (this.worker)
63
66
  this._setupWorkerListener();
67
+ // Moonshot: Full frame detector for better sensitivity
68
+ this.fullDetector = new DetectorLite(this.inputWidth, this.inputHeight, { useLSH: true });
64
69
  this.featureManager.init({
65
70
  inputWidth: this.inputWidth,
66
71
  inputHeight: this.inputHeight,
@@ -160,8 +165,7 @@ class Controller {
160
165
  }
161
166
  dummyRun(input) {
162
167
  const inputData = this.inputLoader.loadInput(input);
163
- const cropFeature = this.featureManager.getFeature("crop-detection");
164
- cropFeature?.detect(inputData, false);
168
+ this.fullDetector?.detect(inputData);
165
169
  this.tracker.dummyRun(inputData);
166
170
  }
167
171
  getProjectionMatrix() {
@@ -179,25 +183,67 @@ class Controller {
179
183
  return this._glModelViewMatrix(modelViewTransform, targetIndex);
180
184
  }
181
185
  async _detectAndMatch(inputData, targetIndexes) {
182
- const cropFeature = this.featureManager.getFeature("crop-detection");
183
- const { featurePoints } = cropFeature.detect(inputData, true);
186
+ const { featurePoints } = this.fullDetector.detect(inputData);
184
187
  const { targetIndex: matchedTargetIndex, modelViewTransform } = await this._workerMatch(featurePoints, targetIndexes);
185
188
  return { targetIndex: matchedTargetIndex, modelViewTransform };
186
189
  }
187
190
  async _trackAndUpdate(inputData, lastModelViewTransform, targetIndex) {
188
- const { worldCoords, screenCoords } = this.tracker.track(inputData, lastModelViewTransform, targetIndex);
189
- if (worldCoords.length < 8)
190
- return null; // Resynced with Matcher (8 points) to allow initial detection
191
+ const { worldCoords, screenCoords, reliabilities, indices = [], octaveIndex = 0 } = this.tracker.track(inputData, lastModelViewTransform, targetIndex);
192
+ const state = this.trackingStates[targetIndex];
193
+ if (!state.pointStabilities)
194
+ state.pointStabilities = [];
195
+ if (!state.pointStabilities[octaveIndex]) {
196
+ // Initialize stabilities for this octave if not exists
197
+ const numPoints = this.tracker.prebuiltData[targetIndex][octaveIndex].px.length;
198
+ state.pointStabilities[octaveIndex] = new Float32Array(numPoints).fill(0.5); // Start at 0.5
199
+ }
200
+ const stabilities = state.pointStabilities[octaveIndex];
201
+ const currentStabilities = [];
202
+ // Update all points in this octave
203
+ for (let i = 0; i < stabilities.length; i++) {
204
+ const isTracked = indices.includes(i);
205
+ if (isTracked) {
206
+ stabilities[i] = Math.min(1.0, stabilities[i] + 0.35); // Fast recovery (approx 3 frames)
207
+ }
208
+ else {
209
+ stabilities[i] = Math.max(0.0, stabilities[i] - 0.12); // Slightly more forgiving loss
210
+ }
211
+ }
212
+ // Collect stabilities and FILTER OUT excessive flickerers (Dead Zone)
213
+ const filteredWorldCoords = [];
214
+ const filteredScreenCoords = [];
215
+ const filteredStabilities = [];
216
+ for (let i = 0; i < indices.length; i++) {
217
+ const s = stabilities[indices[i]];
218
+ if (s > 0.3) { // Hard Cutoff: points with <30% stability are ignored
219
+ filteredWorldCoords.push(worldCoords[i]);
220
+ filteredScreenCoords.push(screenCoords[i]);
221
+ filteredStabilities.push(s);
222
+ }
223
+ }
224
+ // STRICT QUALITY CHECK: Prevent "sticky" tracking on background noise.
225
+ // We require a minimum number of high-confidence AND STABLE points.
226
+ const stableAndReliable = reliabilities.filter((r, idx) => r > 0.75 && stabilities[indices[idx]] > 0.5).length;
227
+ if (stableAndReliable < 6 || filteredWorldCoords.length < 8) {
228
+ return { modelViewTransform: null, screenCoords: [], reliabilities: [], stabilities: [] };
229
+ }
191
230
  const modelViewTransform = await this._workerTrackUpdate(lastModelViewTransform, {
192
- worldCoords,
193
- screenCoords,
231
+ worldCoords: filteredWorldCoords,
232
+ screenCoords: filteredScreenCoords,
233
+ stabilities: filteredStabilities
194
234
  });
195
- return modelViewTransform;
235
+ return {
236
+ modelViewTransform,
237
+ screenCoords: filteredScreenCoords,
238
+ reliabilities: reliabilities.filter((_, idx) => stabilities[indices[idx]] > 0.3),
239
+ stabilities: filteredStabilities
240
+ };
196
241
  }
197
242
  processVideo(input) {
198
243
  if (this.processingVideo)
199
244
  return;
200
245
  this.processingVideo = true;
246
+ const currentLoopId = ++loopIdCounter; // Added for ghost loop prevention
201
247
  this.trackingStates = [];
202
248
  for (let i = 0; i < (this.markerDimensions?.length || 0); i++) {
203
249
  this.trackingStates.push({
@@ -210,7 +256,7 @@ class Controller {
210
256
  }
211
257
  const startProcessing = async () => {
212
258
  while (true) {
213
- if (!this.processingVideo)
259
+ if (!this.processingVideo || currentLoopId !== loopIdCounter)
214
260
  break;
215
261
  const inputData = this.inputLoader.loadInput(input);
216
262
  const nTracking = this.trackingStates.reduce((acc, s) => acc + (!!s.isTracking ? 1 : 0), 0);
@@ -233,12 +279,18 @@ class Controller {
233
279
  for (let i = 0; i < this.trackingStates.length; i++) {
234
280
  const trackingState = this.trackingStates[i];
235
281
  if (trackingState.isTracking) {
236
- let modelViewTransform = await this._trackAndUpdate(inputData, trackingState.currentModelViewTransform, i);
237
- if (modelViewTransform === null) {
282
+ const result = await this._trackAndUpdate(inputData, trackingState.currentModelViewTransform, i);
283
+ if (result === null || result.modelViewTransform === null) {
238
284
  trackingState.isTracking = false;
285
+ trackingState.screenCoords = [];
286
+ trackingState.reliabilities = [];
287
+ trackingState.stabilities = [];
239
288
  }
240
289
  else {
241
- trackingState.currentModelViewTransform = modelViewTransform;
290
+ trackingState.currentModelViewTransform = result.modelViewTransform;
291
+ trackingState.screenCoords = result.screenCoords;
292
+ trackingState.reliabilities = result.reliabilities;
293
+ trackingState.stabilities = result.stabilities;
242
294
  }
243
295
  }
244
296
  const wasShowing = trackingState.showing;
@@ -250,7 +302,12 @@ class Controller {
250
302
  }
251
303
  if (trackingState.showing) {
252
304
  const worldMatrix = this._glModelViewMatrix(trackingState.currentModelViewTransform, i);
253
- const filteredMatrix = this.featureManager.applyWorldMatrixFilters(i, worldMatrix);
305
+ // Calculate confidence score based on point stability
306
+ const stabilities = trackingState.stabilities || [];
307
+ const avgStability = stabilities.length > 0
308
+ ? stabilities.reduce((a, b) => a + b, 0) / stabilities.length
309
+ : 0;
310
+ const filteredMatrix = this.featureManager.applyWorldMatrixFilters(i, worldMatrix, { stability: avgStability });
254
311
  trackingState.trackingMatrix = filteredMatrix;
255
312
  let finalMatrix = [...filteredMatrix];
256
313
  const isInputRotated = input.width === this.inputHeight && input.height === this.inputWidth;
@@ -264,7 +321,10 @@ class Controller {
264
321
  type: "updateMatrix",
265
322
  targetIndex: i,
266
323
  worldMatrix: finalMatrix,
267
- modelViewTransform: trackingState.currentModelViewTransform
324
+ modelViewTransform: trackingState.currentModelViewTransform,
325
+ screenCoords: trackingState.screenCoords,
326
+ reliabilities: trackingState.reliabilities,
327
+ stabilities: trackingState.stabilities
268
328
  });
269
329
  }
270
330
  }
@@ -284,9 +344,8 @@ class Controller {
284
344
  }
285
345
  async detect(input) {
286
346
  const inputData = this.inputLoader.loadInput(input);
287
- const cropFeature = this.featureManager.getFeature("crop-detection");
288
- const { featurePoints, debugExtra } = cropFeature.detect(inputData, false);
289
- return { featurePoints, debugExtra };
347
+ const { featurePoints } = this.fullDetector.detect(inputData);
348
+ return { featurePoints, debugExtra: {} };
290
349
  }
291
350
  async match(featurePoints, targetIndex) {
292
351
  const { targetIndex: matchedTargetIndex, modelViewTransform, screenCoords, worldCoords, debugExtra } = await this._workerMatch(featurePoints, [
@@ -306,10 +365,16 @@ class Controller {
306
365
  _workerMatch(featurePoints, targetIndexes) {
307
366
  return new Promise((resolve) => {
308
367
  if (!this.worker) {
309
- this._matchOnMainThread(featurePoints, targetIndexes).then(resolve);
368
+ this._matchOnMainThread(featurePoints, targetIndexes).then(resolve).catch(() => resolve({ targetIndex: -1 }));
310
369
  return;
311
370
  }
371
+ const timeout = setTimeout(() => {
372
+ this.workerMatchDone = null;
373
+ resolve({ targetIndex: -1 });
374
+ }, WORKER_TIMEOUT_MS);
312
375
  this.workerMatchDone = (data) => {
376
+ clearTimeout(timeout);
377
+ this.workerMatchDone = null;
313
378
  resolve({
314
379
  targetIndex: data.targetIndex,
315
380
  modelViewTransform: data.modelViewTransform,
@@ -359,18 +424,25 @@ class Controller {
359
424
  _workerTrackUpdate(modelViewTransform, trackingFeatures) {
360
425
  return new Promise((resolve) => {
361
426
  if (!this.worker) {
362
- this._trackUpdateOnMainThread(modelViewTransform, trackingFeatures).then(resolve);
427
+ this._trackUpdateOnMainThread(modelViewTransform, trackingFeatures).then(resolve).catch(() => resolve(null));
363
428
  return;
364
429
  }
430
+ const timeout = setTimeout(() => {
431
+ this.workerTrackDone = null;
432
+ resolve(null);
433
+ }, WORKER_TIMEOUT_MS);
365
434
  this.workerTrackDone = (data) => {
435
+ clearTimeout(timeout);
436
+ this.workerTrackDone = null;
366
437
  resolve(data.modelViewTransform);
367
438
  };
368
- const { worldCoords, screenCoords } = trackingFeatures;
439
+ const { worldCoords, screenCoords, stabilities } = trackingFeatures;
369
440
  this.worker.postMessage({
370
441
  type: "trackUpdate",
371
442
  modelViewTransform,
372
443
  worldCoords,
373
444
  screenCoords,
445
+ stabilities
374
446
  });
375
447
  });
376
448
  }
@@ -379,11 +451,12 @@ class Controller {
379
451
  const { Estimator } = await import("./estimation/estimator.js");
380
452
  this.mainThreadEstimator = new Estimator(this.projectionTransform);
381
453
  }
382
- const { worldCoords, screenCoords } = trackingFeatures;
454
+ const { worldCoords, screenCoords, stabilities } = trackingFeatures;
383
455
  return this.mainThreadEstimator.refineEstimate({
384
456
  initialModelViewTransform: modelViewTransform,
385
457
  worldCoords,
386
458
  screenCoords,
459
+ stabilities
387
460
  });
388
461
  }
389
462
  _glModelViewMatrix(modelViewTransform, targetIndex) {
@@ -39,11 +39,12 @@ onmessage = (msg) => {
39
39
  });
40
40
  break;
41
41
  case "trackUpdate":
42
- const { modelViewTransform, worldCoords, screenCoords } = data;
42
+ const { modelViewTransform, worldCoords, screenCoords, stabilities } = data;
43
43
  const finalModelViewTransform = estimator.refineEstimate({
44
44
  initialModelViewTransform: modelViewTransform,
45
45
  worldCoords,
46
46
  screenCoords,
47
+ stabilities, // Stability-based weights
47
48
  });
48
49
  postMessage({
49
50
  type: "trackUpdateDone",
@@ -1,6 +1,7 @@
1
- export function refineEstimate({ initialModelViewTransform, projectionTransform, worldCoords, screenCoords, }: {
1
+ export function refineEstimate({ initialModelViewTransform, projectionTransform, worldCoords, screenCoords, stabilities, }: {
2
2
  initialModelViewTransform: any;
3
3
  projectionTransform: any;
4
4
  worldCoords: any;
5
5
  screenCoords: any;
6
+ stabilities: any;
6
7
  }): never[][] | null;
@@ -9,7 +9,8 @@ const ICP_BREAK_LOOP_ERROR_RATIO_THRESH = 0.99;
9
9
  let mat = [[], [], []];
10
10
  let J_U_Xc = [[], []]; // 2x3
11
11
  let J_Xc_S = [[], [], []]; // 3x6
12
- const refineEstimate = ({ initialModelViewTransform, projectionTransform, worldCoords, screenCoords, }) => {
12
+ const refineEstimate = ({ initialModelViewTransform, projectionTransform, worldCoords, screenCoords, stabilities, // Stability-based weighting
13
+ }) => {
13
14
  // Question: shall we normlize the screen coords as well?
14
15
  // Question: do we need to normlize the scale as well, i.e. make coords from -1 to 1
15
16
  //
@@ -59,6 +60,7 @@ const refineEstimate = ({ initialModelViewTransform, projectionTransform, worldC
59
60
  projectionTransform,
60
61
  worldCoords: normalizedWorldCoords,
61
62
  screenCoords,
63
+ stabilities, // Pass weights to ICP
62
64
  inlierProb: inlierProbs[i],
63
65
  });
64
66
  updatedModelViewTransform = ret.modelViewTransform;
@@ -87,7 +89,7 @@ const refineEstimate = ({ initialModelViewTransform, projectionTransform, worldC
87
89
  };
88
90
  // ICP iteration
89
91
  // Question: can someone provide theoretical reference / mathematical proof for the following computations?
90
- const _doICP = ({ initialModelViewTransform, projectionTransform, worldCoords, screenCoords, inlierProb, }) => {
92
+ const _doICP = ({ initialModelViewTransform, projectionTransform, worldCoords, screenCoords, stabilities, inlierProb, }) => {
91
93
  const isRobustMode = inlierProb < 1;
92
94
  let modelViewTransform = initialModelViewTransform;
93
95
  let err0 = 0.0;
@@ -152,7 +154,11 @@ const _doICP = ({ initialModelViewTransform, projectionTransform, worldCoords, s
152
154
  worldCoord: worldCoords[n],
153
155
  });
154
156
  if (isRobustMode) {
155
- const W = (1.0 - E[n] / K2) * (1.0 - E[n] / K2);
157
+ const robustW = (1.0 - E[n] / K2) * (1.0 - E[n] / K2);
158
+ // Log-weighted stability: suppresses vibrators aggressively but allows recovery
159
+ const s = stabilities ? stabilities[n] : 1.0;
160
+ const stabilityW = s * Math.log10(9 * s + 1);
161
+ const W = robustW * stabilityW;
156
162
  for (let j = 0; j < 2; j++) {
157
163
  for (let i = 0; i < 6; i++) {
158
164
  J_U_S[j][i] *= W;
@@ -162,8 +168,15 @@ const _doICP = ({ initialModelViewTransform, projectionTransform, worldCoords, s
162
168
  dU.push([dys[n] * W]);
163
169
  }
164
170
  else {
165
- dU.push([dxs[n]]);
166
- dU.push([dys[n]]);
171
+ const s = stabilities ? stabilities[n] : 1.0;
172
+ const W = s * Math.log10(9 * s + 1);
173
+ for (let j = 0; j < 2; j++) {
174
+ for (let i = 0; i < 6; i++) {
175
+ J_U_S[j][i] *= W;
176
+ }
177
+ }
178
+ dU.push([dxs[n] * W]);
179
+ dU.push([dys[n] * W]);
167
180
  }
168
181
  for (let i = 0; i < J_U_S.length; i++) {
169
182
  allJ_U_S.push(J_U_S[i]);
@@ -38,7 +38,7 @@ export interface ControllerFeature extends Feature {
38
38
  /**
39
39
  * Hook to filter or modify the final world matrix
40
40
  */
41
- filterWorldMatrix?(targetIndex: number, worldMatrix: number[]): number[];
41
+ filterWorldMatrix?(targetIndex: number, worldMatrix: number[], context?: any): number[];
42
42
  /**
43
43
  * Hook to decide if a target should be shown
44
44
  */
@@ -5,7 +5,7 @@ export declare class FeatureManager {
5
5
  getFeature<T extends ControllerFeature>(id: string): T | undefined;
6
6
  init(context: FeatureContext): void;
7
7
  beforeProcess(inputData: any): void;
8
- applyWorldMatrixFilters(targetIndex: number, worldMatrix: number[]): number[];
8
+ applyWorldMatrixFilters(targetIndex: number, worldMatrix: number[], context?: any): number[];
9
9
  shouldShow(targetIndex: number, isTracking: boolean): boolean;
10
10
  notifyUpdate(data: any): void;
11
11
  dispose(): void;
@@ -20,11 +20,11 @@ export class FeatureManager {
20
20
  }
21
21
  }
22
22
  }
23
- applyWorldMatrixFilters(targetIndex, worldMatrix) {
23
+ applyWorldMatrixFilters(targetIndex, worldMatrix, context) {
24
24
  let result = worldMatrix;
25
25
  for (const feature of this.features) {
26
26
  if (feature.enabled && feature.filterWorldMatrix) {
27
- result = feature.filterWorldMatrix(targetIndex, result);
27
+ result = feature.filterWorldMatrix(targetIndex, result, context);
28
28
  }
29
29
  }
30
30
  return result;
@@ -10,6 +10,6 @@ export declare class OneEuroFilterFeature implements ControllerFeature {
10
10
  constructor(minCutOff?: number, beta?: number);
11
11
  init(context: FeatureContext): void;
12
12
  private getFilter;
13
- filterWorldMatrix(targetIndex: number, worldMatrix: number[]): number[];
13
+ filterWorldMatrix(targetIndex: number, worldMatrix: number[], context?: any): number[];
14
14
  onUpdate(data: any): void;
15
15
  }
@@ -23,10 +23,17 @@ export class OneEuroFilterFeature {
23
23
  }
24
24
  return this.filters[targetIndex];
25
25
  }
26
- filterWorldMatrix(targetIndex, worldMatrix) {
26
+ filterWorldMatrix(targetIndex, worldMatrix, context) {
27
27
  if (!this.enabled)
28
28
  return worldMatrix;
29
29
  const filter = this.getFilter(targetIndex);
30
+ const stability = context?.stability ?? 1.0;
31
+ // Dynamic Cutoff: If points are very stable (1.0), use higher cutoff (less responsiveness loss).
32
+ // If points are unstable (0.3), use much lower cutoff (heavy smoothing).
33
+ // We use a squared curve for even more aggressive suppression of jitter on unstable points.
34
+ const dynamicMinCutOff = this.minCutOff * (0.05 + Math.pow(stability, 2) * 0.95);
35
+ filter.minCutOff = dynamicMinCutOff;
36
+ filter.beta = this.beta;
30
37
  return filter.filter(Date.now(), worldMatrix);
31
38
  }
32
39
  onUpdate(data) {
@@ -203,7 +203,7 @@ const _query = ({ node, descriptors, querypoint, queue, keypointIndexes, numPop
203
203
  queue.push({ node: childrenOrIndices[i], d: dist });
204
204
  }
205
205
  else {
206
- _query({ node: childrenOrIndices[i], descriptors, querypoint, queue, keypointIndexes, numPop });
206
+ _query({ node: childrenOrIndices[i], descriptors, querypoint, queue, keypointIndexes, numPop: numPop + 1 });
207
207
  }
208
208
  }
209
209
  if (numPop < CLUSTER_MAX_POP && queue.length > 0) {
@@ -5,26 +5,110 @@
5
5
  * que NO depende de TensorFlow, eliminando todos los problemas de
6
6
  * inicialización, bloqueos y compatibilidad.
7
7
  */
8
- import { WorkerPool } from "./utils/worker-pool.js";
9
8
  export declare class OfflineCompiler {
10
9
  data: any;
11
- workerPool: WorkerPool | null;
12
10
  constructor();
13
- _initNodeWorkers(): Promise<void>;
14
11
  compileImageTargets(images: any[], progressCallback: (p: number) => void): Promise<any>;
15
- _compileTarget(targetImages: any[], progressCallback: (p: number) => void): Promise<any[]>;
16
- _compileMatch(targetImages: any[], progressCallback: (p: number) => void): Promise<any[]>;
17
- _compileTrack(targetImages: any[], progressCallback: (p: number) => void): Promise<any[]>;
12
+ _compileTarget(targetImages: any[], progressCallback: (p: number) => void): Promise<{
13
+ matchingData: {
14
+ maximaPoints: any[];
15
+ minimaPoints: any[];
16
+ maximaPointsCluster: {
17
+ rootNode: {
18
+ leaf: boolean;
19
+ pointIndexes: never[];
20
+ centerPointIndex: null;
21
+ };
22
+ } | {
23
+ rootNode: {
24
+ centerPointIndex: any;
25
+ };
26
+ };
27
+ minimaPointsCluster: {
28
+ rootNode: {
29
+ leaf: boolean;
30
+ pointIndexes: never[];
31
+ centerPointIndex: null;
32
+ };
33
+ } | {
34
+ rootNode: {
35
+ centerPointIndex: any;
36
+ };
37
+ };
38
+ width: any;
39
+ height: any;
40
+ scale: any;
41
+ }[];
42
+ trackingData: Object[];
43
+ }[]>;
44
+ _compileMatch(targetImages: any[], progressCallback: (p: number) => void): Promise<{
45
+ maximaPoints: any[];
46
+ minimaPoints: any[];
47
+ maximaPointsCluster: {
48
+ rootNode: {
49
+ leaf: boolean;
50
+ pointIndexes: never[];
51
+ centerPointIndex: null;
52
+ };
53
+ } | {
54
+ rootNode: {
55
+ centerPointIndex: any;
56
+ };
57
+ };
58
+ minimaPointsCluster: {
59
+ rootNode: {
60
+ leaf: boolean;
61
+ pointIndexes: never[];
62
+ centerPointIndex: null;
63
+ };
64
+ } | {
65
+ rootNode: {
66
+ centerPointIndex: any;
67
+ };
68
+ };
69
+ width: any;
70
+ height: any;
71
+ scale: any;
72
+ }[][]>;
73
+ _compileTrack(targetImages: any[], progressCallback: (p: number) => void): Promise<Object[][]>;
18
74
  compileTrack({ progressCallback, targetImages, basePercent }: {
19
75
  progressCallback: (p: number) => void;
20
76
  targetImages: any[];
21
77
  basePercent?: number;
22
- }): Promise<any[]>;
78
+ }): Promise<Object[][]>;
23
79
  compileMatch({ progressCallback, targetImages, basePercent }: {
24
80
  progressCallback: (p: number) => void;
25
81
  targetImages: any[];
26
82
  basePercent?: number;
27
- }): Promise<any[]>;
83
+ }): Promise<{
84
+ maximaPoints: any[];
85
+ minimaPoints: any[];
86
+ maximaPointsCluster: {
87
+ rootNode: {
88
+ leaf: boolean;
89
+ pointIndexes: never[];
90
+ centerPointIndex: null;
91
+ };
92
+ } | {
93
+ rootNode: {
94
+ centerPointIndex: any;
95
+ };
96
+ };
97
+ minimaPointsCluster: {
98
+ rootNode: {
99
+ leaf: boolean;
100
+ pointIndexes: never[];
101
+ centerPointIndex: null;
102
+ };
103
+ } | {
104
+ rootNode: {
105
+ centerPointIndex: any;
106
+ };
107
+ };
108
+ width: any;
109
+ height: any;
110
+ scale: any;
111
+ }[][]>;
28
112
  exportData(): Uint8Array<ArrayBuffer>;
29
113
  _getMorton(x: number, y: number): number;
30
114
  _columnarize(points: any[], tree: any, width: number, height: number): {