@srsergio/taptapp-ar 1.1.1 → 1.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/compiler/node-worker.js +1 -197
- package/dist/compiler/offline-compiler.js +1 -207
- package/dist/core/constants.js +1 -38
- package/dist/core/detector/crop-detector.js +1 -88
- package/dist/core/detector/detector-lite.js +1 -455
- package/dist/core/detector/freak.js +1 -89
- package/dist/core/estimation/estimate.js +1 -16
- package/dist/core/estimation/estimator.js +1 -30
- package/dist/core/estimation/morph-refinement.js +1 -116
- package/dist/core/estimation/non-rigid-refine.js +1 -70
- package/dist/core/estimation/pnp-solver.js +1 -109
- package/dist/core/estimation/refine-estimate.js +1 -311
- package/dist/core/estimation/utils.js +1 -67
- package/dist/core/features/auto-rotation-feature.js +1 -30
- package/dist/core/features/crop-detection-feature.js +1 -26
- package/dist/core/features/feature-base.js +1 -1
- package/dist/core/features/feature-manager.js +1 -55
- package/dist/core/features/one-euro-filter-feature.js +1 -44
- package/dist/core/features/temporal-filter-feature.js +1 -57
- package/dist/core/image-list.js +1 -54
- package/dist/core/input-loader.js +1 -87
- package/dist/core/matching/hamming-distance.js +1 -66
- package/dist/core/matching/hdc.js +1 -102
- package/dist/core/matching/hierarchical-clustering.js +1 -130
- package/dist/core/matching/hough.js +1 -170
- package/dist/core/matching/matcher.js +1 -66
- package/dist/core/matching/matching.js +1 -401
- package/dist/core/matching/ransacHomography.js +1 -132
- package/dist/core/perception/bio-inspired-engine.js +1 -232
- package/dist/core/perception/foveal-attention.js +1 -280
- package/dist/core/perception/index.js +1 -17
- package/dist/core/perception/predictive-coding.js +1 -278
- package/dist/core/perception/saccadic-controller.js +1 -269
- package/dist/core/perception/saliency-map.js +1 -254
- package/dist/core/perception/scale-orchestrator.js +1 -68
- package/dist/core/protocol.js +1 -254
- package/dist/core/tracker/extract-utils.js +1 -29
- package/dist/core/tracker/extract.js +1 -306
- package/dist/core/tracker/tracker.js +1 -352
- package/dist/core/utils/cumsum.js +1 -37
- package/dist/core/utils/delaunay.js +1 -125
- package/dist/core/utils/geometry.js +1 -101
- package/dist/core/utils/gpu-compute.js +1 -231
- package/dist/core/utils/homography.js +1 -138
- package/dist/core/utils/images.js +1 -108
- package/dist/core/utils/lsh-binarizer.js +1 -37
- package/dist/core/utils/lsh-direct.js +1 -76
- package/dist/core/utils/projection.js +1 -51
- package/dist/core/utils/randomizer.js +1 -25
- package/dist/core/utils/worker-pool.js +1 -89
- package/dist/index.js +1 -7
- package/dist/libs/one-euro-filter.js +1 -70
- package/dist/react/TaptappAR.js +1 -151
- package/dist/react/types.js +1 -16
- package/dist/react/use-ar.js +1 -118
- package/dist/runtime/aframe.js +1 -272
- package/dist/runtime/bio-inspired-controller.js +1 -358
- package/dist/runtime/controller.js +1 -592
- package/dist/runtime/controller.worker.js +1 -93
- package/dist/runtime/index.js +1 -5
- package/dist/runtime/three.js +1 -304
- package/dist/runtime/track.js +1 -381
- package/package.json +10 -4
|
@@ -1,278 +1 @@
|
|
|
1
|
-
|
|
2
|
-
* Predictive Coding System
|
|
3
|
-
*
|
|
4
|
-
* Inspired by the brain's predictive processing theory:
|
|
5
|
-
* - The brain constantly predicts incoming sensory data
|
|
6
|
-
* - Only "prediction errors" (unexpected changes) are processed fully
|
|
7
|
-
* - If prediction matches reality, minimal processing is needed
|
|
8
|
-
*
|
|
9
|
-
* For AR tracking:
|
|
10
|
-
* - Predict next frame based on motion model
|
|
11
|
-
* - Compare prediction to actual frame
|
|
12
|
-
* - Skip or minimize processing if difference is below threshold
|
|
13
|
-
* - In static scenes, ~90% of frames can be skipped
|
|
14
|
-
*/
|
|
15
|
-
class PredictiveCoding {
|
|
16
|
-
/**
|
|
17
|
-
* @param {number} width - Image width
|
|
18
|
-
* @param {number} height - Image height
|
|
19
|
-
* @param {Object} config - Configuration
|
|
20
|
-
*/
|
|
21
|
-
constructor(width, height, config) {
|
|
22
|
-
this.width = width;
|
|
23
|
-
this.height = height;
|
|
24
|
-
this.config = config;
|
|
25
|
-
// Frame history for prediction
|
|
26
|
-
this.frameHistory = [];
|
|
27
|
-
this.stateHistory = [];
|
|
28
|
-
// Motion model parameters
|
|
29
|
-
this.motionModel = {
|
|
30
|
-
vx: 0, // Velocity X
|
|
31
|
-
vy: 0, // Velocity Y
|
|
32
|
-
vtheta: 0, // Angular velocity
|
|
33
|
-
vscale: 0, // Scale velocity
|
|
34
|
-
confidence: 0, // Model confidence
|
|
35
|
-
};
|
|
36
|
-
// Block-based change detection (8x8 blocks)
|
|
37
|
-
this.blockSize = 8;
|
|
38
|
-
this.blocksX = Math.ceil(width / this.blockSize);
|
|
39
|
-
this.blocksY = Math.ceil(height / this.blockSize);
|
|
40
|
-
this.blockMeans = new Float32Array(this.blocksX * this.blocksY);
|
|
41
|
-
this.prevBlockMeans = new Float32Array(this.blocksX * this.blocksY);
|
|
42
|
-
// Statistics
|
|
43
|
-
this.consecutiveSkips = 0;
|
|
44
|
-
this.maxConsecutiveSkips = 10; // Force processing every N frames
|
|
45
|
-
}
|
|
46
|
-
/**
|
|
47
|
-
* Predict whether current frame can be skipped
|
|
48
|
-
*
|
|
49
|
-
* @param {Uint8Array} inputData - Current frame grayscale data
|
|
50
|
-
* @param {Object} trackingState - Current tracking state
|
|
51
|
-
* @returns {Object} Prediction result
|
|
52
|
-
*/
|
|
53
|
-
predict(inputData, trackingState) {
|
|
54
|
-
// Always process first few frames
|
|
55
|
-
if (this.frameHistory.length < 2) {
|
|
56
|
-
return { canSkip: false, confidence: 0, reason: 'insufficient_history' };
|
|
57
|
-
}
|
|
58
|
-
// Force processing periodically
|
|
59
|
-
if (this.consecutiveSkips >= this.maxConsecutiveSkips) {
|
|
60
|
-
return { canSkip: false, confidence: 0, reason: 'forced_refresh' };
|
|
61
|
-
}
|
|
62
|
-
// Compute change level
|
|
63
|
-
const changeLevel = this.getChangeLevel(inputData);
|
|
64
|
-
// If not tracking, be more conservative
|
|
65
|
-
const threshold = trackingState?.isTracking
|
|
66
|
-
? this.config.CHANGE_THRESHOLD
|
|
67
|
-
: this.config.CHANGE_THRESHOLD * 0.5;
|
|
68
|
-
// Decision
|
|
69
|
-
const canSkip = changeLevel < threshold;
|
|
70
|
-
const confidence = canSkip
|
|
71
|
-
? Math.min(1, (threshold - changeLevel) / threshold)
|
|
72
|
-
: 0;
|
|
73
|
-
// Predict state if skipping
|
|
74
|
-
let predictedState = null;
|
|
75
|
-
if (canSkip && trackingState) {
|
|
76
|
-
predictedState = this._predictState(trackingState);
|
|
77
|
-
}
|
|
78
|
-
if (canSkip) {
|
|
79
|
-
this.consecutiveSkips++;
|
|
80
|
-
}
|
|
81
|
-
return {
|
|
82
|
-
canSkip,
|
|
83
|
-
confidence,
|
|
84
|
-
changeLevel,
|
|
85
|
-
predictedState,
|
|
86
|
-
reason: canSkip ? 'low_change' : 'significant_change',
|
|
87
|
-
};
|
|
88
|
-
}
|
|
89
|
-
/**
|
|
90
|
-
* Compute change level between current and previous frame
|
|
91
|
-
* Uses block-based comparison for efficiency
|
|
92
|
-
*
|
|
93
|
-
* @param {Uint8Array} inputData - Current frame
|
|
94
|
-
* @returns {number} Change level (0-1)
|
|
95
|
-
*/
|
|
96
|
-
getChangeLevel(inputData) {
|
|
97
|
-
if (this.frameHistory.length === 0) {
|
|
98
|
-
return 1.0; // Assume maximum change for first frame
|
|
99
|
-
}
|
|
100
|
-
// Compute block means for current frame
|
|
101
|
-
this._computeBlockMeans(inputData, this.blockMeans);
|
|
102
|
-
// Compare with previous block means
|
|
103
|
-
let totalDiff = 0;
|
|
104
|
-
let maxDiff = 0;
|
|
105
|
-
const numBlocks = this.blocksX * this.blocksY;
|
|
106
|
-
for (let i = 0; i < numBlocks; i++) {
|
|
107
|
-
const diff = Math.abs(this.blockMeans[i] - this.prevBlockMeans[i]) / 255;
|
|
108
|
-
totalDiff += diff;
|
|
109
|
-
maxDiff = Math.max(maxDiff, diff);
|
|
110
|
-
}
|
|
111
|
-
// Combine average and max differences
|
|
112
|
-
const avgDiff = totalDiff / numBlocks;
|
|
113
|
-
const changeLevel = avgDiff * 0.7 + maxDiff * 0.3;
|
|
114
|
-
return Math.min(1, changeLevel);
|
|
115
|
-
}
|
|
116
|
-
/**
|
|
117
|
-
* Compute mean intensity for each block
|
|
118
|
-
* @private
|
|
119
|
-
*/
|
|
120
|
-
_computeBlockMeans(data, output) {
|
|
121
|
-
const bs = this.blockSize;
|
|
122
|
-
const w = this.width;
|
|
123
|
-
for (let by = 0; by < this.blocksY; by++) {
|
|
124
|
-
const yStart = by * bs;
|
|
125
|
-
const yEnd = Math.min(yStart + bs, this.height);
|
|
126
|
-
for (let bx = 0; bx < this.blocksX; bx++) {
|
|
127
|
-
const xStart = bx * bs;
|
|
128
|
-
const xEnd = Math.min(xStart + bs, this.width);
|
|
129
|
-
let sum = 0;
|
|
130
|
-
let count = 0;
|
|
131
|
-
for (let y = yStart; y < yEnd; y++) {
|
|
132
|
-
const rowOffset = y * w;
|
|
133
|
-
for (let x = xStart; x < xEnd; x++) {
|
|
134
|
-
sum += data[rowOffset + x];
|
|
135
|
-
count++;
|
|
136
|
-
}
|
|
137
|
-
}
|
|
138
|
-
output[by * this.blocksX + bx] = sum / count;
|
|
139
|
-
}
|
|
140
|
-
}
|
|
141
|
-
}
|
|
142
|
-
/**
|
|
143
|
-
* Predict next tracking state based on motion model
|
|
144
|
-
* @private
|
|
145
|
-
*/
|
|
146
|
-
_predictState(currentState) {
|
|
147
|
-
if (!currentState.worldMatrix)
|
|
148
|
-
return null;
|
|
149
|
-
// Extract current parameters
|
|
150
|
-
const matrix = currentState.worldMatrix;
|
|
151
|
-
// Apply motion model
|
|
152
|
-
const predictedMatrix = new Float32Array(16);
|
|
153
|
-
for (let i = 0; i < 16; i++) {
|
|
154
|
-
predictedMatrix[i] = matrix[i];
|
|
155
|
-
}
|
|
156
|
-
// Add predicted motion
|
|
157
|
-
predictedMatrix[12] += this.motionModel.vx;
|
|
158
|
-
predictedMatrix[13] += this.motionModel.vy;
|
|
159
|
-
// Apply scale change (to diagonal elements)
|
|
160
|
-
const scaleFactor = 1 + this.motionModel.vscale;
|
|
161
|
-
predictedMatrix[0] *= scaleFactor;
|
|
162
|
-
predictedMatrix[5] *= scaleFactor;
|
|
163
|
-
predictedMatrix[10] *= scaleFactor;
|
|
164
|
-
return {
|
|
165
|
-
worldMatrix: predictedMatrix,
|
|
166
|
-
isTracking: true,
|
|
167
|
-
isPredicted: true,
|
|
168
|
-
predictionConfidence: this.motionModel.confidence,
|
|
169
|
-
};
|
|
170
|
-
}
|
|
171
|
-
/**
|
|
172
|
-
* Store frame for future prediction
|
|
173
|
-
*
|
|
174
|
-
* @param {Uint8Array} inputData - Frame data
|
|
175
|
-
* @param {Object} trackingState - Tracking state
|
|
176
|
-
*/
|
|
177
|
-
storeFrame(inputData, trackingState) {
|
|
178
|
-
// Copy current block means to previous before computing new
|
|
179
|
-
for (let i = 0; i < this.blockMeans.length; i++) {
|
|
180
|
-
this.prevBlockMeans[i] = this.blockMeans[i];
|
|
181
|
-
}
|
|
182
|
-
// Compute new block means
|
|
183
|
-
this._computeBlockMeans(inputData, this.blockMeans);
|
|
184
|
-
// Store state
|
|
185
|
-
if (trackingState?.worldMatrix) {
|
|
186
|
-
this.stateHistory.push({
|
|
187
|
-
timestamp: Date.now(),
|
|
188
|
-
matrix: new Float32Array(trackingState.worldMatrix),
|
|
189
|
-
});
|
|
190
|
-
// Update motion model
|
|
191
|
-
this._updateMotionModel();
|
|
192
|
-
// Keep history bounded
|
|
193
|
-
while (this.stateHistory.length > this.config.MOTION_HISTORY_FRAMES) {
|
|
194
|
-
this.stateHistory.shift();
|
|
195
|
-
}
|
|
196
|
-
}
|
|
197
|
-
// Reset skip counter
|
|
198
|
-
this.consecutiveSkips = 0;
|
|
199
|
-
// Keep frame count bounded
|
|
200
|
-
this.frameHistory.push(Date.now());
|
|
201
|
-
while (this.frameHistory.length > this.config.MOTION_HISTORY_FRAMES) {
|
|
202
|
-
this.frameHistory.shift();
|
|
203
|
-
}
|
|
204
|
-
}
|
|
205
|
-
/**
|
|
206
|
-
* Update motion model from state history
|
|
207
|
-
* @private
|
|
208
|
-
*/
|
|
209
|
-
_updateMotionModel() {
|
|
210
|
-
const history = this.stateHistory;
|
|
211
|
-
if (history.length < 2) {
|
|
212
|
-
this.motionModel.confidence = 0;
|
|
213
|
-
return;
|
|
214
|
-
}
|
|
215
|
-
// Compute velocity from recent frames
|
|
216
|
-
const n = history.length;
|
|
217
|
-
const latest = history[n - 1].matrix;
|
|
218
|
-
const prev = history[n - 2].matrix;
|
|
219
|
-
const dt = (history[n - 1].timestamp - history[n - 2].timestamp) / 1000;
|
|
220
|
-
if (dt > 0) {
|
|
221
|
-
// Position velocity
|
|
222
|
-
this.motionModel.vx = (latest[12] - prev[12]) / dt * 0.016; // Normalize to ~60fps
|
|
223
|
-
this.motionModel.vy = (latest[13] - prev[13]) / dt * 0.016;
|
|
224
|
-
// Scale velocity (from diagonal average)
|
|
225
|
-
const prevScale = (Math.abs(prev[0]) + Math.abs(prev[5])) / 2;
|
|
226
|
-
const currScale = (Math.abs(latest[0]) + Math.abs(latest[5])) / 2;
|
|
227
|
-
this.motionModel.vscale = (currScale - prevScale) / prevScale / dt * 0.016;
|
|
228
|
-
// Compute confidence based on consistency
|
|
229
|
-
if (history.length >= 3) {
|
|
230
|
-
const older = history[n - 3].matrix;
|
|
231
|
-
const expectedVx = (prev[12] - older[12]) / dt * 0.016;
|
|
232
|
-
const expectedVy = (prev[13] - older[13]) / dt * 0.016;
|
|
233
|
-
const errorX = Math.abs(this.motionModel.vx - expectedVx);
|
|
234
|
-
const errorY = Math.abs(this.motionModel.vy - expectedVy);
|
|
235
|
-
const error = Math.sqrt(errorX * errorX + errorY * errorY);
|
|
236
|
-
this.motionModel.confidence = Math.max(0, 1 - error / 10);
|
|
237
|
-
}
|
|
238
|
-
else {
|
|
239
|
-
this.motionModel.confidence = 0.5;
|
|
240
|
-
}
|
|
241
|
-
}
|
|
242
|
-
}
|
|
243
|
-
/**
|
|
244
|
-
* Check if we're in a static scene (good candidate for aggressive skipping)
|
|
245
|
-
* @returns {boolean} True if scene appears static
|
|
246
|
-
*/
|
|
247
|
-
isStaticScene() {
|
|
248
|
-
if (this.stateHistory.length < 3)
|
|
249
|
-
return false;
|
|
250
|
-
const velocity = Math.sqrt(this.motionModel.vx ** 2 +
|
|
251
|
-
this.motionModel.vy ** 2);
|
|
252
|
-
return velocity < 0.5 && Math.abs(this.motionModel.vscale) < 0.01;
|
|
253
|
-
}
|
|
254
|
-
/**
|
|
255
|
-
* Reset prediction state
|
|
256
|
-
*/
|
|
257
|
-
reset() {
|
|
258
|
-
this.frameHistory = [];
|
|
259
|
-
this.stateHistory = [];
|
|
260
|
-
this.consecutiveSkips = 0;
|
|
261
|
-
this.motionModel = {
|
|
262
|
-
vx: 0,
|
|
263
|
-
vy: 0,
|
|
264
|
-
vtheta: 0,
|
|
265
|
-
vscale: 0,
|
|
266
|
-
confidence: 0,
|
|
267
|
-
};
|
|
268
|
-
this.blockMeans.fill(0);
|
|
269
|
-
this.prevBlockMeans.fill(0);
|
|
270
|
-
}
|
|
271
|
-
/**
|
|
272
|
-
* Update configuration
|
|
273
|
-
*/
|
|
274
|
-
configure(config) {
|
|
275
|
-
this.config = { ...this.config, ...config };
|
|
276
|
-
}
|
|
277
|
-
}
|
|
278
|
-
export { PredictiveCoding };
|
|
1
|
+
class t{constructor(t,i,s){this.width=t,this.height=i,this.config=s,this.frameHistory=[],this.stateHistory=[],this.motionModel={vx:0,vy:0,vtheta:0,vscale:0,confidence:0},this.blockSize=8,this.blocksX=Math.ceil(t/this.blockSize),this.blocksY=Math.ceil(i/this.blockSize),this.blockMeans=new Float32Array(this.blocksX*this.blocksY),this.prevBlockMeans=new Float32Array(this.blocksX*this.blocksY),this.consecutiveSkips=0,this.maxConsecutiveSkips=10}predict(t,i){if(this.frameHistory.length<2)return{canSkip:!1,confidence:0,reason:"insufficient_history"};if(this.consecutiveSkips>=this.maxConsecutiveSkips)return{canSkip:!1,confidence:0,reason:"forced_refresh"};const s=this.getChangeLevel(t),e=i?.isTracking?this.config.CHANGE_THRESHOLD:.5*this.config.CHANGE_THRESHOLD,o=s<e,n=o?Math.min(1,(e-s)/e):0;let h=null;return o&&i&&(h=this._predictState(i)),o&&this.consecutiveSkips++,{canSkip:o,confidence:n,changeLevel:s,predictedState:h,reason:o?"low_change":"significant_change"}}getChangeLevel(t){if(0===this.frameHistory.length)return 1;this._computeBlockMeans(t,this.blockMeans);let i=0,s=0;const e=this.blocksX*this.blocksY;for(let t=0;t<e;t++){const e=Math.abs(this.blockMeans[t]-this.prevBlockMeans[t])/255;i+=e,s=Math.max(s,e)}const o=i/e*.7+.3*s;return Math.min(1,o)}_computeBlockMeans(t,i){const s=this.blockSize,e=this.width;for(let o=0;o<this.blocksY;o++){const n=o*s,h=Math.min(n+s,this.height);for(let c=0;c<this.blocksX;c++){const a=c*s,r=Math.min(a+s,this.width);let l=0,M=0;for(let i=n;i<h;i++){const s=i*e;for(let i=a;i<r;i++)l+=t[s+i],M++}i[o*this.blocksX+c]=l/M}}}_predictState(t){if(!t.worldMatrix)return null;const i=t.worldMatrix,s=new Float32Array(16);for(let t=0;t<16;t++)s[t]=i[t];s[12]+=this.motionModel.vx,s[13]+=this.motionModel.vy;const e=1+this.motionModel.vscale;return s[0]*=e,s[5]*=e,s[10]*=e,{worldMatrix:s,isTracking:!0,isPredicted:!0,predictionConfidence:this.motionModel.confidence}}storeFrame(t,i){for(let t=0;t<this.blockMeans.length;t++)this.prevBlockMeans[t]=this.blockMeans[t];if(this._computeBlockMeans(t,this.blockMeans),i?.worldMatrix)for(this.stateHistory.push({timestamp:Date.now(),matrix:new Float32Array(i.worldMatrix)}),this._updateMotionModel();this.stateHistory.length>this.config.MOTION_HISTORY_FRAMES;)this.stateHistory.shift();for(this.consecutiveSkips=0,this.frameHistory.push(Date.now());this.frameHistory.length>this.config.MOTION_HISTORY_FRAMES;)this.frameHistory.shift()}_updateMotionModel(){const t=this.stateHistory;if(t.length<2)return void(this.motionModel.confidence=0);const i=t.length,s=t[i-1].matrix,e=t[i-2].matrix,o=(t[i-1].timestamp-t[i-2].timestamp)/1e3;if(o>0){this.motionModel.vx=(s[12]-e[12])/o*.016,this.motionModel.vy=(s[13]-e[13])/o*.016;const n=(Math.abs(e[0])+Math.abs(e[5]))/2,h=(Math.abs(s[0])+Math.abs(s[5]))/2;if(this.motionModel.vscale=(h-n)/n/o*.016,t.length>=3){const s=t[i-3].matrix,n=(e[12]-s[12])/o*.016,h=(e[13]-s[13])/o*.016,c=Math.abs(this.motionModel.vx-n),a=Math.abs(this.motionModel.vy-h),r=Math.sqrt(c*c+a*a);this.motionModel.confidence=Math.max(0,1-r/10)}else this.motionModel.confidence=.5}}isStaticScene(){return!(this.stateHistory.length<3)&&(Math.sqrt(this.motionModel.vx**2+this.motionModel.vy**2)<.5&&Math.abs(this.motionModel.vscale)<.01)}reset(){this.frameHistory=[],this.stateHistory=[],this.consecutiveSkips=0,this.motionModel={vx:0,vy:0,vtheta:0,vscale:0,confidence:0},this.blockMeans.fill(0),this.prevBlockMeans.fill(0)}configure(t){this.config={...this.config,...t}}}export{t as PredictiveCoding};
|
|
@@ -1,269 +1 @@
|
|
|
1
|
-
|
|
2
|
-
* Saccadic Controller
|
|
3
|
-
*
|
|
4
|
-
* Mimics human eye saccades - rapid movements that redirect foveal attention
|
|
5
|
-
* to areas of interest. The human eye makes 3-4 saccades per second to
|
|
6
|
-
* build a complete picture of the visual scene.
|
|
7
|
-
*
|
|
8
|
-
* Strategy:
|
|
9
|
-
* 1. Compute saliency map to find "interesting" regions
|
|
10
|
-
* 2. Use tracking state to predict where features should be
|
|
11
|
-
* 3. Generate priority-ordered list of "glance" targets
|
|
12
|
-
* 4. Limit saccades per frame to balance coverage vs. efficiency
|
|
13
|
-
*/
|
|
14
|
-
/**
|
|
15
|
-
* A saccade target representing where attention should be directed
|
|
16
|
-
* @typedef {Object} SaccadeTarget
|
|
17
|
-
* @property {number} x - X coordinate
|
|
18
|
-
* @property {number} y - Y coordinate
|
|
19
|
-
* @property {number} priority - Priority (0 = highest)
|
|
20
|
-
* @property {string} reason - Why this target was selected
|
|
21
|
-
* @property {number} saliency - Saliency score at this location
|
|
22
|
-
*/
|
|
23
|
-
class SaccadicController {
|
|
24
|
-
/**
|
|
25
|
-
* @param {number} width - Image width
|
|
26
|
-
* @param {number} height - Image height
|
|
27
|
-
* @param {Object} config - Configuration
|
|
28
|
-
*/
|
|
29
|
-
constructor(width, height, config) {
|
|
30
|
-
this.width = width;
|
|
31
|
-
this.height = height;
|
|
32
|
-
this.config = config;
|
|
33
|
-
// Saccade history for inhibition of return
|
|
34
|
-
this.recentTargets = [];
|
|
35
|
-
this.inhibitionRadius = Math.min(width, height) * 0.1;
|
|
36
|
-
// Movement prediction
|
|
37
|
-
this.velocityHistory = [];
|
|
38
|
-
this.lastCenter = { x: width / 2, y: height / 2 };
|
|
39
|
-
// Grid for systematic coverage
|
|
40
|
-
this.gridCells = this._buildCoverageGrid(3, 3);
|
|
41
|
-
this.lastVisitedCell = 4; // Center
|
|
42
|
-
// State
|
|
43
|
-
this.lastSaccadeTime = 0;
|
|
44
|
-
this.saccadeCount = 0;
|
|
45
|
-
}
|
|
46
|
-
/**
|
|
47
|
-
* Build a grid for systematic coverage during tracking loss
|
|
48
|
-
* @private
|
|
49
|
-
*/
|
|
50
|
-
_buildCoverageGrid(rows, cols) {
|
|
51
|
-
const cells = [];
|
|
52
|
-
const cellW = this.width / cols;
|
|
53
|
-
const cellH = this.height / rows;
|
|
54
|
-
for (let r = 0; r < rows; r++) {
|
|
55
|
-
for (let c = 0; c < cols; c++) {
|
|
56
|
-
cells.push({
|
|
57
|
-
x: cellW * (c + 0.5),
|
|
58
|
-
y: cellH * (r + 0.5),
|
|
59
|
-
index: r * cols + c,
|
|
60
|
-
lastVisit: 0,
|
|
61
|
-
});
|
|
62
|
-
}
|
|
63
|
-
}
|
|
64
|
-
return cells;
|
|
65
|
-
}
|
|
66
|
-
/**
|
|
67
|
-
* Compute saccade targets based on current state
|
|
68
|
-
*
|
|
69
|
-
* @param {Object} saliency - Saliency map result
|
|
70
|
-
* @param {Object} currentFovea - Current fovea center {x, y}
|
|
71
|
-
* @param {Object} trackingState - Current tracking state (optional)
|
|
72
|
-
* @returns {SaccadeTarget[]} Priority-ordered list of targets
|
|
73
|
-
*/
|
|
74
|
-
computeTargets(saliency, currentFovea, trackingState = null) {
|
|
75
|
-
const targets = [];
|
|
76
|
-
const maxTargets = this.config.MAX_SACCADES_PER_FRAME;
|
|
77
|
-
// Strategy 1: Follow tracking prediction (highest priority)
|
|
78
|
-
if (trackingState && trackingState.isTracking) {
|
|
79
|
-
const predicted = this._predictTrackingCenter(trackingState);
|
|
80
|
-
if (predicted) {
|
|
81
|
-
targets.push({
|
|
82
|
-
x: predicted.x,
|
|
83
|
-
y: predicted.y,
|
|
84
|
-
priority: 0,
|
|
85
|
-
reason: 'tracking_prediction',
|
|
86
|
-
saliency: 1.0,
|
|
87
|
-
});
|
|
88
|
-
}
|
|
89
|
-
}
|
|
90
|
-
// Strategy 2: High saliency regions
|
|
91
|
-
if (saliency && saliency.peaks) {
|
|
92
|
-
for (const peak of saliency.peaks) {
|
|
93
|
-
if (targets.length >= maxTargets)
|
|
94
|
-
break;
|
|
95
|
-
// Skip if too close to existing targets (inhibition of return)
|
|
96
|
-
if (this._isInhibited(peak.x, peak.y, targets))
|
|
97
|
-
continue;
|
|
98
|
-
if (peak.value > this.config.SALIENCY_THRESHOLD) {
|
|
99
|
-
targets.push({
|
|
100
|
-
x: peak.x,
|
|
101
|
-
y: peak.y,
|
|
102
|
-
priority: targets.length,
|
|
103
|
-
reason: 'saliency_peak',
|
|
104
|
-
saliency: peak.value,
|
|
105
|
-
});
|
|
106
|
-
}
|
|
107
|
-
}
|
|
108
|
-
}
|
|
109
|
-
// Strategy 3: Systematic grid search (when not tracking)
|
|
110
|
-
if (!trackingState?.isTracking && targets.length < maxTargets) {
|
|
111
|
-
const gridTarget = this._getNextGridCell();
|
|
112
|
-
if (gridTarget && !this._isInhibited(gridTarget.x, gridTarget.y, targets)) {
|
|
113
|
-
targets.push({
|
|
114
|
-
x: gridTarget.x,
|
|
115
|
-
y: gridTarget.y,
|
|
116
|
-
priority: targets.length,
|
|
117
|
-
reason: 'grid_search',
|
|
118
|
-
saliency: 0.5,
|
|
119
|
-
});
|
|
120
|
-
}
|
|
121
|
-
}
|
|
122
|
-
// Strategy 4: Stay at current center if no better options
|
|
123
|
-
if (targets.length === 0) {
|
|
124
|
-
targets.push({
|
|
125
|
-
x: currentFovea.x,
|
|
126
|
-
y: currentFovea.y,
|
|
127
|
-
priority: 0,
|
|
128
|
-
reason: 'maintain_position',
|
|
129
|
-
saliency: 0.3,
|
|
130
|
-
});
|
|
131
|
-
}
|
|
132
|
-
// Update history
|
|
133
|
-
this._updateHistory(targets);
|
|
134
|
-
return targets;
|
|
135
|
-
}
|
|
136
|
-
/**
|
|
137
|
-
* Predict center of tracking based on current state and velocity
|
|
138
|
-
* @private
|
|
139
|
-
*/
|
|
140
|
-
_predictTrackingCenter(trackingState) {
|
|
141
|
-
if (!trackingState.worldMatrix)
|
|
142
|
-
return null;
|
|
143
|
-
// Extract center from world matrix
|
|
144
|
-
const matrix = trackingState.worldMatrix;
|
|
145
|
-
const cx = matrix[12] || this.width / 2;
|
|
146
|
-
const cy = matrix[13] || this.height / 2;
|
|
147
|
-
// Apply velocity-based prediction
|
|
148
|
-
if (this.velocityHistory.length >= 2) {
|
|
149
|
-
const vx = this._computeAverageVelocity('x');
|
|
150
|
-
const vy = this._computeAverageVelocity('y');
|
|
151
|
-
// Predict 1 frame ahead
|
|
152
|
-
return {
|
|
153
|
-
x: Math.max(0, Math.min(this.width - 1, cx + vx)),
|
|
154
|
-
y: Math.max(0, Math.min(this.height - 1, cy + vy)),
|
|
155
|
-
};
|
|
156
|
-
}
|
|
157
|
-
return { x: cx, y: cy };
|
|
158
|
-
}
|
|
159
|
-
/**
|
|
160
|
-
* Compute average velocity from history
|
|
161
|
-
* @private
|
|
162
|
-
*/
|
|
163
|
-
_computeAverageVelocity(axis) {
|
|
164
|
-
if (this.velocityHistory.length < 2)
|
|
165
|
-
return 0;
|
|
166
|
-
let sum = 0;
|
|
167
|
-
for (let i = 1; i < this.velocityHistory.length; i++) {
|
|
168
|
-
sum += this.velocityHistory[i][axis] - this.velocityHistory[i - 1][axis];
|
|
169
|
-
}
|
|
170
|
-
return sum / (this.velocityHistory.length - 1);
|
|
171
|
-
}
|
|
172
|
-
/**
|
|
173
|
-
* Check if a location is inhibited (too close to recent targets)
|
|
174
|
-
* @private
|
|
175
|
-
*/
|
|
176
|
-
_isInhibited(x, y, currentTargets) {
|
|
177
|
-
const r2 = this.inhibitionRadius ** 2;
|
|
178
|
-
// Check against current frame targets
|
|
179
|
-
for (const t of currentTargets) {
|
|
180
|
-
const dx = x - t.x;
|
|
181
|
-
const dy = y - t.y;
|
|
182
|
-
if (dx * dx + dy * dy < r2)
|
|
183
|
-
return true;
|
|
184
|
-
}
|
|
185
|
-
// Check against recent history
|
|
186
|
-
for (const t of this.recentTargets) {
|
|
187
|
-
const dx = x - t.x;
|
|
188
|
-
const dy = y - t.y;
|
|
189
|
-
if (dx * dx + dy * dy < r2)
|
|
190
|
-
return true;
|
|
191
|
-
}
|
|
192
|
-
return false;
|
|
193
|
-
}
|
|
194
|
-
/**
|
|
195
|
-
* Get next grid cell for systematic search
|
|
196
|
-
* @private
|
|
197
|
-
*/
|
|
198
|
-
_getNextGridCell() {
|
|
199
|
-
// Find least recently visited cell
|
|
200
|
-
let oldest = this.gridCells[0];
|
|
201
|
-
let oldestTime = Infinity;
|
|
202
|
-
for (const cell of this.gridCells) {
|
|
203
|
-
if (cell.lastVisit < oldestTime) {
|
|
204
|
-
oldestTime = cell.lastVisit;
|
|
205
|
-
oldest = cell;
|
|
206
|
-
}
|
|
207
|
-
}
|
|
208
|
-
oldest.lastVisit = Date.now();
|
|
209
|
-
return oldest;
|
|
210
|
-
}
|
|
211
|
-
/**
|
|
212
|
-
* Update history with new targets
|
|
213
|
-
* @private
|
|
214
|
-
*/
|
|
215
|
-
_updateHistory(targets) {
|
|
216
|
-
// Add to recent targets for inhibition of return
|
|
217
|
-
this.recentTargets.push(...targets);
|
|
218
|
-
// Keep only last N targets
|
|
219
|
-
const maxHistory = this.config.MOTION_HISTORY_FRAMES * this.config.MAX_SACCADES_PER_FRAME;
|
|
220
|
-
while (this.recentTargets.length > maxHistory) {
|
|
221
|
-
this.recentTargets.shift();
|
|
222
|
-
}
|
|
223
|
-
// Update velocity history
|
|
224
|
-
if (targets.length > 0) {
|
|
225
|
-
this.velocityHistory.push({ x: targets[0].x, y: targets[0].y });
|
|
226
|
-
while (this.velocityHistory.length > this.config.MOTION_HISTORY_FRAMES) {
|
|
227
|
-
this.velocityHistory.shift();
|
|
228
|
-
}
|
|
229
|
-
this.lastCenter = { x: targets[0].x, y: targets[0].y };
|
|
230
|
-
}
|
|
231
|
-
this.saccadeCount += targets.length;
|
|
232
|
-
this.lastSaccadeTime = Date.now();
|
|
233
|
-
}
|
|
234
|
-
/**
|
|
235
|
-
* Get the most likely location of interest based on history
|
|
236
|
-
* @returns {Object} {x, y} of predicted location
|
|
237
|
-
*/
|
|
238
|
-
getPredictedLocation() {
|
|
239
|
-
if (this.velocityHistory.length >= 2) {
|
|
240
|
-
const vx = this._computeAverageVelocity('x');
|
|
241
|
-
const vy = this._computeAverageVelocity('y');
|
|
242
|
-
return {
|
|
243
|
-
x: Math.max(0, Math.min(this.width - 1, this.lastCenter.x + vx)),
|
|
244
|
-
y: Math.max(0, Math.min(this.height - 1, this.lastCenter.y + vy)),
|
|
245
|
-
};
|
|
246
|
-
}
|
|
247
|
-
return this.lastCenter;
|
|
248
|
-
}
|
|
249
|
-
/**
|
|
250
|
-
* Reset controller state
|
|
251
|
-
*/
|
|
252
|
-
reset() {
|
|
253
|
-
this.recentTargets = [];
|
|
254
|
-
this.velocityHistory = [];
|
|
255
|
-
this.lastCenter = { x: this.width / 2, y: this.height / 2 };
|
|
256
|
-
this.saccadeCount = 0;
|
|
257
|
-
for (const cell of this.gridCells) {
|
|
258
|
-
cell.lastVisit = 0;
|
|
259
|
-
}
|
|
260
|
-
}
|
|
261
|
-
/**
|
|
262
|
-
* Update configuration
|
|
263
|
-
*/
|
|
264
|
-
configure(config) {
|
|
265
|
-
this.config = { ...this.config, ...config };
|
|
266
|
-
this.inhibitionRadius = Math.min(this.width, this.height) * 0.1;
|
|
267
|
-
}
|
|
268
|
-
}
|
|
269
|
-
export { SaccadicController };
|
|
1
|
+
class t{constructor(t,i,s){this.width=t,this.height=i,this.config=s,this.recentTargets=[],this.inhibitionRadius=.1*Math.min(t,i),this.velocityHistory=[],this.lastCenter={x:t/2,y:i/2},this.gridCells=this._buildCoverageGrid(3,3),this.lastVisitedCell=4,this.lastSaccadeTime=0,this.saccadeCount=0}_buildCoverageGrid(t,i){const s=[],e=this.width/i,h=this.height/t;for(let r=0;r<t;r++)for(let t=0;t<i;t++)s.push({x:e*(t+.5),y:h*(r+.5),index:r*i+t,lastVisit:0});return s}computeTargets(t,i,s=null){const e=[],h=this.config.MAX_SACCADES_PER_FRAME;if(s&&s.isTracking){const t=this._predictTrackingCenter(s);t&&e.push({x:t.x,y:t.y,priority:0,reason:"tracking_prediction",saliency:1})}if(t&&t.peaks)for(const i of t.peaks){if(e.length>=h)break;this._isInhibited(i.x,i.y,e)||i.value>this.config.SALIENCY_THRESHOLD&&e.push({x:i.x,y:i.y,priority:e.length,reason:"saliency_peak",saliency:i.value})}if(!s?.isTracking&&e.length<h){const t=this._getNextGridCell();t&&!this._isInhibited(t.x,t.y,e)&&e.push({x:t.x,y:t.y,priority:e.length,reason:"grid_search",saliency:.5})}return 0===e.length&&e.push({x:i.x,y:i.y,priority:0,reason:"maintain_position",saliency:.3}),this._updateHistory(e),e}_predictTrackingCenter(t){if(!t.worldMatrix)return null;const i=t.worldMatrix,s=i[12]||this.width/2,e=i[13]||this.height/2;if(this.velocityHistory.length>=2){const t=this._computeAverageVelocity("x"),i=this._computeAverageVelocity("y");return{x:Math.max(0,Math.min(this.width-1,s+t)),y:Math.max(0,Math.min(this.height-1,e+i))}}return{x:s,y:e}}_computeAverageVelocity(t){if(this.velocityHistory.length<2)return 0;let i=0;for(let s=1;s<this.velocityHistory.length;s++)i+=this.velocityHistory[s][t]-this.velocityHistory[s-1][t];return i/(this.velocityHistory.length-1)}_isInhibited(t,i,s){const e=this.inhibitionRadius**2;for(const h of s){const s=t-h.x,r=i-h.y;if(s*s+r*r<e)return!0}for(const s of this.recentTargets){const h=t-s.x,r=i-s.y;if(h*h+r*r<e)return!0}return!1}_getNextGridCell(){let t=this.gridCells[0],i=1/0;for(const s of this.gridCells)s.lastVisit<i&&(i=s.lastVisit,t=s);return t.lastVisit=Date.now(),t}_updateHistory(t){this.recentTargets.push(...t);const i=this.config.MOTION_HISTORY_FRAMES*this.config.MAX_SACCADES_PER_FRAME;for(;this.recentTargets.length>i;)this.recentTargets.shift();if(t.length>0){for(this.velocityHistory.push({x:t[0].x,y:t[0].y});this.velocityHistory.length>this.config.MOTION_HISTORY_FRAMES;)this.velocityHistory.shift();this.lastCenter={x:t[0].x,y:t[0].y}}this.saccadeCount+=t.length,this.lastSaccadeTime=Date.now()}getPredictedLocation(){if(this.velocityHistory.length>=2){const t=this._computeAverageVelocity("x"),i=this._computeAverageVelocity("y");return{x:Math.max(0,Math.min(this.width-1,this.lastCenter.x+t)),y:Math.max(0,Math.min(this.height-1,this.lastCenter.y+i))}}return this.lastCenter}reset(){this.recentTargets=[],this.velocityHistory=[],this.lastCenter={x:this.width/2,y:this.height/2},this.saccadeCount=0;for(const t of this.gridCells)t.lastVisit=0}configure(t){this.config={...this.config,...t},this.inhibitionRadius=.1*Math.min(this.width,this.height)}}export{t as SaccadicController};
|