@pioneer-platform/pioneer-cache 1.19.3 → 1.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +1 -1
- package/CHANGELOG.md +30 -0
- package/dist/core/cache-manager.d.ts +6 -0
- package/dist/core/cache-manager.js +85 -0
- package/dist/stores/balance-cache.d.ts +4 -1
- package/dist/stores/balance-cache.js +28 -13
- package/dist/workers/refresh-worker.d.ts +5 -0
- package/dist/workers/refresh-worker.js +75 -1
- package/dist/workers/stale-balance-scanner.d.ts +35 -0
- package/dist/workers/stale-balance-scanner.js +180 -0
- package/package.json +5 -3
- package/src/core/cache-manager.ts +96 -0
- package/src/stores/balance-cache.ts +32 -12
- package/src/workers/refresh-worker.ts +88 -1
- package/src/workers/stale-balance-scanner.ts +223 -0
- package/tsconfig.json +14 -5
package/.turbo/turbo-build.log
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
|
|
2
2
|
|
|
3
|
-
> @pioneer-platform/pioneer-cache@1.
|
|
3
|
+
> @pioneer-platform/pioneer-cache@1.21.0 build /Users/highlander/WebstormProjects/keepkey-stack/projects/pioneer/modules/pioneer/pioneer-cache
|
|
4
4
|
> tsc
|
|
5
5
|
|
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,35 @@
|
|
|
1
1
|
# @pioneer-platform/pioneer-cache
|
|
2
2
|
|
|
3
|
+
## 1.22.0
|
|
4
|
+
|
|
5
|
+
### Minor Changes
|
|
6
|
+
|
|
7
|
+
- chore: feat(e2e): add blue API support to swap roundtrip tests
|
|
8
|
+
|
|
9
|
+
### Patch Changes
|
|
10
|
+
|
|
11
|
+
- Updated dependencies
|
|
12
|
+
- @pioneer-platform/pioneer-discovery@8.44.0
|
|
13
|
+
|
|
14
|
+
## 1.21.1
|
|
15
|
+
|
|
16
|
+
### Patch Changes
|
|
17
|
+
|
|
18
|
+
- Updated dependencies
|
|
19
|
+
- @pioneer-platform/pioneer-discovery@8.43.0
|
|
20
|
+
|
|
21
|
+
## 1.21.0
|
|
22
|
+
|
|
23
|
+
### Minor Changes
|
|
24
|
+
|
|
25
|
+
- feat(monorepo): production release preparation with strict TypeScript and enhanced swap functionality
|
|
26
|
+
|
|
27
|
+
## 1.20.0
|
|
28
|
+
|
|
29
|
+
### Minor Changes
|
|
30
|
+
|
|
31
|
+
- feat(pioneer-client): add client-side host override for staging/blue testing
|
|
32
|
+
|
|
3
33
|
## 1.19.3
|
|
4
34
|
|
|
5
35
|
### Patch Changes
|
|
@@ -38,12 +38,18 @@ export declare class CacheManager {
|
|
|
38
38
|
private stakingCache?;
|
|
39
39
|
private zapperCache?;
|
|
40
40
|
private workers;
|
|
41
|
+
private staleScanner?;
|
|
41
42
|
constructor(config: CacheManagerConfig);
|
|
42
43
|
/**
|
|
43
44
|
* Purge all pending jobs from the cache refresh queue
|
|
44
45
|
* Used in local dev to clear stale jobs on startup
|
|
45
46
|
*/
|
|
46
47
|
purgeQueue(queueName?: string): Promise<void>;
|
|
48
|
+
/**
|
|
49
|
+
* Purge unsupported job types from the queue
|
|
50
|
+
* Scans the queue and removes jobs for cache types that aren't enabled
|
|
51
|
+
*/
|
|
52
|
+
purgeUnsupportedJobs(queueName?: string): Promise<void>;
|
|
47
53
|
/**
|
|
48
54
|
* Start background refresh workers for all caches
|
|
49
55
|
*/
|
|
@@ -14,6 +14,7 @@ const transaction_cache_1 = require("../stores/transaction-cache");
|
|
|
14
14
|
const staking_cache_1 = require("../stores/staking-cache");
|
|
15
15
|
const zapper_cache_1 = require("../stores/zapper-cache");
|
|
16
16
|
const refresh_worker_1 = require("../workers/refresh-worker");
|
|
17
|
+
const stale_balance_scanner_1 = require("../workers/stale-balance-scanner");
|
|
17
18
|
const log = require('@pioneer-platform/loggerdog')();
|
|
18
19
|
const TAG = ' | CacheManager | ';
|
|
19
20
|
/**
|
|
@@ -103,6 +104,73 @@ class CacheManager {
|
|
|
103
104
|
throw error;
|
|
104
105
|
}
|
|
105
106
|
}
|
|
107
|
+
/**
|
|
108
|
+
* Purge unsupported job types from the queue
|
|
109
|
+
* Scans the queue and removes jobs for cache types that aren't enabled
|
|
110
|
+
*/
|
|
111
|
+
async purgeUnsupportedJobs(queueName = 'cache-refresh') {
|
|
112
|
+
const tag = TAG + 'purgeUnsupportedJobs | ';
|
|
113
|
+
try {
|
|
114
|
+
const redisQueue = this.redisQueue;
|
|
115
|
+
let purgedCount = 0;
|
|
116
|
+
let scannedCount = 0;
|
|
117
|
+
log.info(tag, `Scanning queue for unsupported job types...`);
|
|
118
|
+
// Get enabled cache types
|
|
119
|
+
const enabledCaches = new Set();
|
|
120
|
+
if (this.balanceCache)
|
|
121
|
+
enabledCaches.add('balance');
|
|
122
|
+
if (this.priceCache)
|
|
123
|
+
enabledCaches.add('price');
|
|
124
|
+
if (this.portfolioCache)
|
|
125
|
+
enabledCaches.add('portfolio');
|
|
126
|
+
if (this.transactionCache)
|
|
127
|
+
enabledCaches.add('transaction');
|
|
128
|
+
if (this.stakingCache)
|
|
129
|
+
enabledCaches.add('staking');
|
|
130
|
+
if (this.zapperCache)
|
|
131
|
+
enabledCaches.add('zapper');
|
|
132
|
+
log.info(tag, `Enabled caches: ${Array.from(enabledCaches).join(', ') || 'NONE'}`);
|
|
133
|
+
// Get queue length
|
|
134
|
+
const queueLength = await redisQueue.count(queueName);
|
|
135
|
+
if (queueLength === 0) {
|
|
136
|
+
log.info(tag, '✅ Queue is empty, no jobs to purge');
|
|
137
|
+
return;
|
|
138
|
+
}
|
|
139
|
+
log.info(tag, `Found ${queueLength} jobs in queue`);
|
|
140
|
+
// Scan all jobs
|
|
141
|
+
const tempJobs = [];
|
|
142
|
+
for (let i = 0; i < queueLength; i++) {
|
|
143
|
+
const job = await redisQueue.getWork(queueName, 1);
|
|
144
|
+
if (!job)
|
|
145
|
+
break;
|
|
146
|
+
scannedCount++;
|
|
147
|
+
const jobType = job.type || '';
|
|
148
|
+
const cacheName = jobType.replace('REFRESH_', '').toLowerCase();
|
|
149
|
+
if (!enabledCaches.has(cacheName)) {
|
|
150
|
+
purgedCount++;
|
|
151
|
+
log.warn(tag, ` Purged unsupported job: ${jobType} (cache "${cacheName}" not enabled)`);
|
|
152
|
+
}
|
|
153
|
+
else {
|
|
154
|
+
// Keep supported jobs - we'll re-queue them
|
|
155
|
+
tempJobs.push(job);
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
// Re-queue supported jobs
|
|
159
|
+
for (const job of tempJobs) {
|
|
160
|
+
await redisQueue.createWork(queueName, job);
|
|
161
|
+
}
|
|
162
|
+
if (purgedCount > 0) {
|
|
163
|
+
log.info(tag, `✅ Purged ${purgedCount} unsupported jobs (scanned ${scannedCount})`);
|
|
164
|
+
}
|
|
165
|
+
else {
|
|
166
|
+
log.info(tag, `✅ No unsupported jobs found (scanned ${scannedCount})`);
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
catch (error) {
|
|
170
|
+
log.error(tag, '⚠️ Error during unsupported job purge (non-fatal):', error);
|
|
171
|
+
// Don't throw - purge failure shouldn't prevent system from starting
|
|
172
|
+
}
|
|
173
|
+
}
|
|
106
174
|
/**
|
|
107
175
|
* Start background refresh workers for all caches
|
|
108
176
|
*/
|
|
@@ -110,6 +178,8 @@ class CacheManager {
|
|
|
110
178
|
const tag = TAG + 'startWorkers | ';
|
|
111
179
|
try {
|
|
112
180
|
log.info(tag, 'Starting refresh workers...');
|
|
181
|
+
// FIX: Purge unsupported job types before starting workers
|
|
182
|
+
await this.purgeUnsupportedJobs('cache-refresh');
|
|
113
183
|
// Create cache registry for workers
|
|
114
184
|
const cacheRegistry = new Map();
|
|
115
185
|
if (this.balanceCache) {
|
|
@@ -139,6 +209,17 @@ class CacheManager {
|
|
|
139
209
|
this.workers.push(worker);
|
|
140
210
|
log.info(tag, '✅ Refresh worker started for all caches');
|
|
141
211
|
}
|
|
212
|
+
// Start stale balance scanner if balance cache is enabled
|
|
213
|
+
if (this.balanceCache) {
|
|
214
|
+
this.staleScanner = new stale_balance_scanner_1.StaleBalanceScanner(this.redis, this.balanceCache, {
|
|
215
|
+
staleThresholdMs: 60 * 60 * 1000, // 1 hour (more aggressive than the 5min reactive threshold)
|
|
216
|
+
scanIntervalMs: 10 * 60 * 1000, // Scan every 10 minutes
|
|
217
|
+
batchSize: 100,
|
|
218
|
+
maxRefreshPerScan: 50
|
|
219
|
+
});
|
|
220
|
+
await this.staleScanner.start();
|
|
221
|
+
log.info(tag, '✅ Stale balance scanner started');
|
|
222
|
+
}
|
|
142
223
|
}
|
|
143
224
|
catch (error) {
|
|
144
225
|
log.error(tag, '❌ Failed to start workers:', error);
|
|
@@ -155,6 +236,10 @@ class CacheManager {
|
|
|
155
236
|
for (const worker of this.workers) {
|
|
156
237
|
await worker.stop();
|
|
157
238
|
}
|
|
239
|
+
if (this.staleScanner) {
|
|
240
|
+
await this.staleScanner.stop();
|
|
241
|
+
this.staleScanner = undefined;
|
|
242
|
+
}
|
|
158
243
|
this.workers = [];
|
|
159
244
|
log.info(tag, '✅ All workers stopped');
|
|
160
245
|
}
|
|
@@ -58,9 +58,12 @@ export declare class BalanceCache extends BaseCache<BalanceData> {
|
|
|
58
58
|
*
|
|
59
59
|
* CRITICAL FIX: When waitForFresh=true (forceRefresh), bypass cache entirely
|
|
60
60
|
* and fetch fresh blockchain data for ALL items, not just cache misses
|
|
61
|
+
*
|
|
62
|
+
* PERFORMANCE FIX: When skipAsyncRefresh=true, disable background refresh jobs
|
|
63
|
+
* to prevent 50+ individual refresh jobs from being queued
|
|
61
64
|
*/
|
|
62
65
|
getBatchBalances(items: Array<{
|
|
63
66
|
caip: string;
|
|
64
67
|
pubkey: string;
|
|
65
|
-
}>, waitForFresh?: boolean): Promise<BalanceData[]>;
|
|
68
|
+
}>, waitForFresh?: boolean, skipAsyncRefresh?: boolean): Promise<BalanceData[]>;
|
|
66
69
|
}
|
|
@@ -12,6 +12,8 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
12
12
|
exports.BalanceCache = void 0;
|
|
13
13
|
const base_cache_1 = require("../core/base-cache");
|
|
14
14
|
const crypto_1 = __importDefault(require("crypto"));
|
|
15
|
+
const pioneer_discovery_1 = require("@pioneer-platform/pioneer-discovery");
|
|
16
|
+
const pioneer_caip_1 = require("@pioneer-platform/pioneer-caip");
|
|
15
17
|
const log = require('@pioneer-platform/loggerdog')();
|
|
16
18
|
/**
|
|
17
19
|
* Hash pubkey/xpub for privacy-protecting cache keys
|
|
@@ -110,11 +112,18 @@ class BalanceCache extends base_cache_1.BaseCache {
|
|
|
110
112
|
const owner = { pubkey };
|
|
111
113
|
const balanceInfo = await this.balanceModule.getBalance(asset, owner);
|
|
112
114
|
if (!balanceInfo || !balanceInfo.balance) {
|
|
113
|
-
|
|
115
|
+
// CRITICAL FIX: Distinguish between legitimate zero balance and failed API call
|
|
116
|
+
// Failed API calls should NOT be cached as valid "0" balance
|
|
117
|
+
const errorMsg = balanceInfo?.error || 'No balance data returned';
|
|
118
|
+
log.warn(tag, `No balance returned for ${caip}/${sanitizePubkey(pubkey)} - ${errorMsg}`);
|
|
119
|
+
// If balanceInfo explicitly indicates an error, throw it instead of caching "0"
|
|
120
|
+
if (balanceInfo?.error || balanceInfo === null || balanceInfo === undefined) {
|
|
121
|
+
throw new Error(`Failed to fetch balance for ${caip}: ${errorMsg}`);
|
|
122
|
+
}
|
|
123
|
+
// Otherwise, this is a legitimate zero balance (balanceInfo exists but balance is falsy)
|
|
114
124
|
const now = Date.now();
|
|
115
125
|
// Get asset metadata even for zero balances
|
|
116
|
-
const
|
|
117
|
-
const assetInfo = assetData[caip.toUpperCase()] || assetData[caip.toLowerCase()] || {};
|
|
126
|
+
const assetInfo = pioneer_discovery_1.assetData[caip.toUpperCase()] || pioneer_discovery_1.assetData[caip.toLowerCase()] || {};
|
|
118
127
|
return {
|
|
119
128
|
caip,
|
|
120
129
|
pubkey,
|
|
@@ -122,14 +131,13 @@ class BalanceCache extends base_cache_1.BaseCache {
|
|
|
122
131
|
decimals: assetInfo.decimals,
|
|
123
132
|
precision: assetInfo.decimals, // Set precision to decimals for compatibility
|
|
124
133
|
fetchedAt: now,
|
|
125
|
-
fetchedAtISO: new Date(now).toISOString()
|
|
134
|
+
fetchedAtISO: new Date(now).toISOString(),
|
|
135
|
+
dataSource: 'Zero balance (legitimate)'
|
|
126
136
|
};
|
|
127
137
|
}
|
|
128
138
|
// Get asset metadata
|
|
129
|
-
const
|
|
130
|
-
const
|
|
131
|
-
const assetInfo = assetData[caip.toUpperCase()] || assetData[caip.toLowerCase()] || {};
|
|
132
|
-
const networkId = caipToNetworkId(caip);
|
|
139
|
+
const assetInfo = pioneer_discovery_1.assetData[caip.toUpperCase()] || pioneer_discovery_1.assetData[caip.toLowerCase()] || {};
|
|
140
|
+
const networkId = (0, pioneer_caip_1.caipToNetworkId)(caip);
|
|
133
141
|
// Use actual node URL if available, otherwise construct generic description
|
|
134
142
|
let dataSource = 'Unknown';
|
|
135
143
|
if (balanceInfo.nodeUrl) {
|
|
@@ -187,8 +195,7 @@ class BalanceCache extends base_cache_1.BaseCache {
|
|
|
187
195
|
const tag = this.TAG + 'getLegacyCached | ';
|
|
188
196
|
try {
|
|
189
197
|
const { caip, pubkey } = params;
|
|
190
|
-
const
|
|
191
|
-
const networkId = caipToNetworkId(caip);
|
|
198
|
+
const networkId = (0, pioneer_caip_1.caipToNetworkId)(caip);
|
|
192
199
|
// Try legacy format: cache:balance:pubkey:asset
|
|
193
200
|
const legacyKey = `cache:balance:${pubkey}:${networkId}`;
|
|
194
201
|
const legacyData = await this.redis.get(legacyKey);
|
|
@@ -229,8 +236,11 @@ class BalanceCache extends base_cache_1.BaseCache {
|
|
|
229
236
|
*
|
|
230
237
|
* CRITICAL FIX: When waitForFresh=true (forceRefresh), bypass cache entirely
|
|
231
238
|
* and fetch fresh blockchain data for ALL items, not just cache misses
|
|
239
|
+
*
|
|
240
|
+
* PERFORMANCE FIX: When skipAsyncRefresh=true, disable background refresh jobs
|
|
241
|
+
* to prevent 50+ individual refresh jobs from being queued
|
|
232
242
|
*/
|
|
233
|
-
async getBatchBalances(items, waitForFresh) {
|
|
243
|
+
async getBatchBalances(items, waitForFresh, skipAsyncRefresh) {
|
|
234
244
|
const tag = this.TAG + 'getBatchBalances | ';
|
|
235
245
|
const startTime = Date.now();
|
|
236
246
|
try {
|
|
@@ -347,12 +357,17 @@ class BalanceCache extends base_cache_1.BaseCache {
|
|
|
347
357
|
await Promise.all(fetchPromises);
|
|
348
358
|
log.info(tag, `Fetched ${missedItems.length} misses in ${Date.now() - fetchStart}ms`);
|
|
349
359
|
}
|
|
350
|
-
else {
|
|
351
|
-
// Non-blocking: trigger background refresh for misses
|
|
360
|
+
else if (!skipAsyncRefresh) {
|
|
361
|
+
// Non-blocking: trigger background refresh for misses (unless disabled)
|
|
362
|
+
log.info(tag, `📋 Queueing ${missedItems.length} background refresh jobs for cache misses`);
|
|
352
363
|
missedItems.forEach(item => {
|
|
353
364
|
this.triggerAsyncRefresh({ caip: item.caip, pubkey: item.pubkey }, 'high');
|
|
354
365
|
});
|
|
355
366
|
}
|
|
367
|
+
else {
|
|
368
|
+
// Background refresh disabled for performance
|
|
369
|
+
log.info(tag, `⚡ PERFORMANCE MODE: Skipping ${missedItems.length} background refresh jobs (skipAsyncRefresh=true)`);
|
|
370
|
+
}
|
|
356
371
|
}
|
|
357
372
|
return results;
|
|
358
373
|
}
|
|
@@ -29,6 +29,11 @@ export declare class RefreshWorker {
|
|
|
29
29
|
* Start processing jobs from the queue
|
|
30
30
|
*/
|
|
31
31
|
start(): Promise<void>;
|
|
32
|
+
/**
|
|
33
|
+
* Purge stale jobs from queue on startup
|
|
34
|
+
* Scans queue and removes jobs older than 1 hour
|
|
35
|
+
*/
|
|
36
|
+
private purgeStaleJobsOnStartup;
|
|
32
37
|
/**
|
|
33
38
|
* Stop the worker gracefully
|
|
34
39
|
*/
|
|
@@ -49,10 +49,66 @@ class RefreshWorker {
|
|
|
49
49
|
}
|
|
50
50
|
log.info(tag, `🚀 Starting refresh worker for queue: ${this.config.queueName}`);
|
|
51
51
|
log.info(tag, `Registered caches: ${Array.from(this.cacheRegistry.keys()).join(', ')}`);
|
|
52
|
+
// FIX #3: Purge stale jobs at startup
|
|
53
|
+
await this.purgeStaleJobsOnStartup();
|
|
52
54
|
this.isRunning = true;
|
|
53
55
|
this.poll();
|
|
54
56
|
log.info(tag, '✅ Refresh worker started successfully');
|
|
55
57
|
}
|
|
58
|
+
/**
|
|
59
|
+
* Purge stale jobs from queue on startup
|
|
60
|
+
* Scans queue and removes jobs older than 1 hour
|
|
61
|
+
*/
|
|
62
|
+
async purgeStaleJobsOnStartup() {
|
|
63
|
+
const tag = TAG + 'purgeStaleJobsOnStartup | ';
|
|
64
|
+
try {
|
|
65
|
+
const MAX_JOB_AGE_MS = 60 * 60 * 1000; // 1 hour
|
|
66
|
+
const now = Date.now();
|
|
67
|
+
let purgedCount = 0;
|
|
68
|
+
let scannedCount = 0;
|
|
69
|
+
log.info(tag, `Scanning queue for stale jobs (age > ${MAX_JOB_AGE_MS / 1000}s)...`);
|
|
70
|
+
// Get queue length
|
|
71
|
+
const queueLength = await this.redisQueue.count(this.config.queueName);
|
|
72
|
+
if (queueLength === 0) {
|
|
73
|
+
log.info(tag, '✅ Queue is empty, no stale jobs to purge');
|
|
74
|
+
return;
|
|
75
|
+
}
|
|
76
|
+
log.info(tag, `Found ${queueLength} jobs in queue, checking for stale jobs...`);
|
|
77
|
+
// Scan up to 100 jobs to find stale ones
|
|
78
|
+
const MAX_SCAN = Math.min(queueLength, 100);
|
|
79
|
+
const tempJobs = [];
|
|
80
|
+
for (let i = 0; i < MAX_SCAN; i++) {
|
|
81
|
+
const job = await this.redisQueue.getWork(this.config.queueName, 1);
|
|
82
|
+
if (!job)
|
|
83
|
+
break;
|
|
84
|
+
scannedCount++;
|
|
85
|
+
const jobTimestamp = job.timestamp || 0;
|
|
86
|
+
const jobAge = jobTimestamp > 0 ? now - jobTimestamp : 0;
|
|
87
|
+
if (jobAge > MAX_JOB_AGE_MS) {
|
|
88
|
+
purgedCount++;
|
|
89
|
+
log.warn(tag, ` Purged stale job: ${job.type} (age: ${Math.round(jobAge / 1000)}s)`);
|
|
90
|
+
}
|
|
91
|
+
else {
|
|
92
|
+
// Keep fresh jobs - we'll re-queue them
|
|
93
|
+
tempJobs.push(job);
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
// Re-queue fresh jobs
|
|
97
|
+
for (const job of tempJobs) {
|
|
98
|
+
await this.redisQueue.createWork(this.config.queueName, job);
|
|
99
|
+
}
|
|
100
|
+
if (purgedCount > 0) {
|
|
101
|
+
log.info(tag, `✅ Purged ${purgedCount} stale jobs (scanned ${scannedCount})`);
|
|
102
|
+
}
|
|
103
|
+
else {
|
|
104
|
+
log.info(tag, `✅ No stale jobs found (scanned ${scannedCount})`);
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
catch (error) {
|
|
108
|
+
log.error(tag, '⚠️ Error during startup purge (non-fatal):', error);
|
|
109
|
+
// Don't throw - startup purge failure shouldn't prevent worker from starting
|
|
110
|
+
}
|
|
111
|
+
}
|
|
56
112
|
/**
|
|
57
113
|
* Stop the worker gracefully
|
|
58
114
|
*/
|
|
@@ -130,12 +186,30 @@ class RefreshWorker {
|
|
|
130
186
|
try {
|
|
131
187
|
const { type, key, params, retryCount = 0 } = job;
|
|
132
188
|
log.info(tag, `Processing ${type} for ${key} (retry: ${retryCount})`);
|
|
189
|
+
// FIX #1: Check job age and drop if too old (>1 hour)
|
|
190
|
+
const MAX_JOB_AGE_MS = 60 * 60 * 1000; // 1 hour
|
|
191
|
+
const jobAge = Date.now() - startTime;
|
|
192
|
+
const jobTimestamp = job.timestamp || 0;
|
|
193
|
+
if (jobTimestamp > 0) {
|
|
194
|
+
const actualJobAge = Date.now() - jobTimestamp;
|
|
195
|
+
if (actualJobAge > MAX_JOB_AGE_MS) {
|
|
196
|
+
log.error(tag, `⚠️ DROPPING STALE JOB: ${type} (age: ${Math.round(actualJobAge / 1000)}s, max: ${MAX_JOB_AGE_MS / 1000}s)`);
|
|
197
|
+
log.error(tag, ` Job key: ${key}`);
|
|
198
|
+
log.error(tag, ` Job has been removed from queue and will not be retried`);
|
|
199
|
+
return;
|
|
200
|
+
}
|
|
201
|
+
}
|
|
133
202
|
// Extract cache name from job type (e.g., "REFRESH_BALANCE" -> "balance")
|
|
134
203
|
const cacheName = type.replace('REFRESH_', '').toLowerCase();
|
|
135
204
|
// Get the appropriate cache instance
|
|
136
205
|
const cache = this.cacheRegistry.get(cacheName);
|
|
206
|
+
// FIX #2: Drop unsupported job types with clear error message
|
|
137
207
|
if (!cache) {
|
|
138
|
-
log.error(tag,
|
|
208
|
+
log.error(tag, `❌ DROPPING UNSUPPORTED JOB: ${type}`);
|
|
209
|
+
log.error(tag, ` Cache type "${cacheName}" is not registered in this worker`);
|
|
210
|
+
log.error(tag, ` Registered caches: ${Array.from(this.cacheRegistry.keys()).join(', ') || 'NONE'}`);
|
|
211
|
+
log.error(tag, ` Job key: ${key}`);
|
|
212
|
+
log.error(tag, ` Job has been removed from queue and will not be retried`);
|
|
139
213
|
return;
|
|
140
214
|
}
|
|
141
215
|
// Fetch fresh data using the cache's fetchFresh method
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import type { BalanceCache } from '../stores/balance-cache';
|
|
2
|
+
export interface StaleScannerConfig {
|
|
3
|
+
staleThresholdMs: number;
|
|
4
|
+
scanIntervalMs: number;
|
|
5
|
+
batchSize: number;
|
|
6
|
+
maxRefreshPerScan: number;
|
|
7
|
+
}
|
|
8
|
+
export declare class StaleBalanceScanner {
|
|
9
|
+
private redis;
|
|
10
|
+
private balanceCache;
|
|
11
|
+
private config;
|
|
12
|
+
private isRunning;
|
|
13
|
+
private scanTimer?;
|
|
14
|
+
constructor(redis: any, balanceCache: BalanceCache, config?: Partial<StaleScannerConfig>);
|
|
15
|
+
/**
|
|
16
|
+
* Start the scanner
|
|
17
|
+
*/
|
|
18
|
+
start(): Promise<void>;
|
|
19
|
+
/**
|
|
20
|
+
* Stop the scanner
|
|
21
|
+
*/
|
|
22
|
+
stop(): Promise<void>;
|
|
23
|
+
/**
|
|
24
|
+
* Schedule next scan
|
|
25
|
+
*/
|
|
26
|
+
private scheduleNextScan;
|
|
27
|
+
/**
|
|
28
|
+
* Scan for stale balances and queue refreshes
|
|
29
|
+
*/
|
|
30
|
+
private scan;
|
|
31
|
+
/**
|
|
32
|
+
* Get scanner statistics
|
|
33
|
+
*/
|
|
34
|
+
getStats(): any;
|
|
35
|
+
}
|
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/*
|
|
3
|
+
Stale Balance Scanner
|
|
4
|
+
|
|
5
|
+
Proactively scans for stale balance cache entries and queues them for refresh.
|
|
6
|
+
This ensures balances that aren't actively accessed still get refreshed.
|
|
7
|
+
|
|
8
|
+
Prevents the issue where stale balances sit for days/weeks without being updated.
|
|
9
|
+
*/
|
|
10
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
11
|
+
exports.StaleBalanceScanner = void 0;
|
|
12
|
+
const log = require('@pioneer-platform/loggerdog')();
|
|
13
|
+
const TAG = ' | StaleBalanceScanner | ';
|
|
14
|
+
class StaleBalanceScanner {
|
|
15
|
+
constructor(redis, balanceCache, config) {
|
|
16
|
+
this.isRunning = false;
|
|
17
|
+
this.redis = redis;
|
|
18
|
+
this.balanceCache = balanceCache;
|
|
19
|
+
this.config = {
|
|
20
|
+
staleThresholdMs: config?.staleThresholdMs || 60 * 60 * 1000, // 1 hour
|
|
21
|
+
scanIntervalMs: config?.scanIntervalMs || 5 * 60 * 1000, // 5 minutes
|
|
22
|
+
batchSize: config?.batchSize || 100,
|
|
23
|
+
maxRefreshPerScan: config?.maxRefreshPerScan || 50
|
|
24
|
+
};
|
|
25
|
+
log.info(TAG, 'Stale balance scanner initialized', {
|
|
26
|
+
staleThresholdHours: this.config.staleThresholdMs / (1000 * 60 * 60),
|
|
27
|
+
scanIntervalMinutes: this.config.scanIntervalMs / (1000 * 60)
|
|
28
|
+
});
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Start the scanner
|
|
32
|
+
*/
|
|
33
|
+
async start() {
|
|
34
|
+
const tag = TAG + 'start | ';
|
|
35
|
+
if (this.isRunning) {
|
|
36
|
+
log.warn(tag, 'Scanner already running');
|
|
37
|
+
return;
|
|
38
|
+
}
|
|
39
|
+
this.isRunning = true;
|
|
40
|
+
log.info(tag, '🔍 Starting stale balance scanner');
|
|
41
|
+
// Run first scan immediately, then schedule recurring
|
|
42
|
+
await this.scan();
|
|
43
|
+
this.scheduleNextScan();
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Stop the scanner
|
|
47
|
+
*/
|
|
48
|
+
async stop() {
|
|
49
|
+
const tag = TAG + 'stop | ';
|
|
50
|
+
log.info(tag, 'Stopping stale balance scanner');
|
|
51
|
+
this.isRunning = false;
|
|
52
|
+
if (this.scanTimer) {
|
|
53
|
+
clearTimeout(this.scanTimer);
|
|
54
|
+
this.scanTimer = undefined;
|
|
55
|
+
}
|
|
56
|
+
log.info(tag, '✅ Scanner stopped');
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* Schedule next scan
|
|
60
|
+
*/
|
|
61
|
+
scheduleNextScan() {
|
|
62
|
+
if (!this.isRunning)
|
|
63
|
+
return;
|
|
64
|
+
this.scanTimer = setTimeout(async () => {
|
|
65
|
+
await this.scan();
|
|
66
|
+
this.scheduleNextScan();
|
|
67
|
+
}, this.config.scanIntervalMs);
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* Scan for stale balances and queue refreshes
|
|
71
|
+
*/
|
|
72
|
+
async scan() {
|
|
73
|
+
const tag = TAG + 'scan | ';
|
|
74
|
+
const startTime = Date.now();
|
|
75
|
+
try {
|
|
76
|
+
log.info(tag, 'Starting stale balance scan...');
|
|
77
|
+
// Scan balance_v2:* keys in batches
|
|
78
|
+
const pattern = 'balance_v2:*';
|
|
79
|
+
const staleKeys = [];
|
|
80
|
+
const now = Date.now();
|
|
81
|
+
let cursor = '0';
|
|
82
|
+
let totalScanned = 0;
|
|
83
|
+
let refreshCount = 0;
|
|
84
|
+
do {
|
|
85
|
+
// SCAN with COUNT for batching
|
|
86
|
+
const [nextCursor, keys] = await this.redis.scan(cursor, 'MATCH', pattern, 'COUNT', this.config.batchSize);
|
|
87
|
+
cursor = nextCursor;
|
|
88
|
+
totalScanned += keys.length;
|
|
89
|
+
// Check each key for staleness
|
|
90
|
+
for (const key of keys) {
|
|
91
|
+
try {
|
|
92
|
+
const cached = await this.redis.get(key);
|
|
93
|
+
if (!cached)
|
|
94
|
+
continue;
|
|
95
|
+
const parsed = JSON.parse(cached);
|
|
96
|
+
const age = now - (parsed.timestamp || 0);
|
|
97
|
+
// Check if stale
|
|
98
|
+
if (age > this.config.staleThresholdMs) {
|
|
99
|
+
staleKeys.push(key);
|
|
100
|
+
// Stop if we've hit max refresh limit
|
|
101
|
+
if (staleKeys.length >= this.config.maxRefreshPerScan) {
|
|
102
|
+
log.info(tag, `Reached max refresh limit (${this.config.maxRefreshPerScan}), stopping scan`);
|
|
103
|
+
cursor = '0'; // Force exit
|
|
104
|
+
break;
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
catch (error) {
|
|
109
|
+
log.error(tag, `Error checking key ${key}:`, error);
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
} while (cursor !== '0' && this.isRunning);
|
|
113
|
+
// Queue refresh for stale balances
|
|
114
|
+
if (staleKeys.length > 0) {
|
|
115
|
+
log.info(tag, `Found ${staleKeys.length} stale balances out of ${totalScanned} scanned`);
|
|
116
|
+
for (const key of staleKeys) {
|
|
117
|
+
try {
|
|
118
|
+
// Extract CAIP and pubkey hash from key
|
|
119
|
+
// Key format: balance_v2:caip:hashedPubkey
|
|
120
|
+
// IMPORTANT: CAIP itself contains colons (e.g., ripple:4109c6f2045fc7eff4cde8f9905d19c2/slip44:144)
|
|
121
|
+
// So we must split from the right to get hashedPubkey, then everything else is the CAIP
|
|
122
|
+
const withoutPrefix = key.replace('balance_v2:', '');
|
|
123
|
+
const lastColonIndex = withoutPrefix.lastIndexOf(':');
|
|
124
|
+
if (lastColonIndex === -1)
|
|
125
|
+
continue;
|
|
126
|
+
const caip = withoutPrefix.substring(0, lastColonIndex);
|
|
127
|
+
const pubkeyHash = withoutPrefix.substring(lastColonIndex + 1);
|
|
128
|
+
// We need the original pubkey to refresh, but it's hashed in the key
|
|
129
|
+
// So we need to get it from the cached value
|
|
130
|
+
const cached = await this.redis.get(key);
|
|
131
|
+
if (!cached)
|
|
132
|
+
continue;
|
|
133
|
+
const parsed = JSON.parse(cached);
|
|
134
|
+
const pubkey = parsed.value?.pubkey;
|
|
135
|
+
if (!pubkey) {
|
|
136
|
+
log.warn(tag, `No pubkey found in cache for ${key}, skipping`);
|
|
137
|
+
continue;
|
|
138
|
+
}
|
|
139
|
+
// Trigger refresh by calling getBalance with waitForFresh=true
|
|
140
|
+
// This will queue a background refresh job
|
|
141
|
+
log.debug(tag, `Queueing refresh for ${caip} (age: ${Math.round((now - parsed.timestamp) / 1000 / 60)}min)`);
|
|
142
|
+
// Use setImmediate to avoid blocking the scan
|
|
143
|
+
setImmediate(async () => {
|
|
144
|
+
try {
|
|
145
|
+
await this.balanceCache.getBalance(caip, pubkey, true);
|
|
146
|
+
refreshCount++;
|
|
147
|
+
}
|
|
148
|
+
catch (error) {
|
|
149
|
+
log.error(tag, `Failed to refresh ${caip}/${pubkey}:`, error);
|
|
150
|
+
}
|
|
151
|
+
});
|
|
152
|
+
}
|
|
153
|
+
catch (error) {
|
|
154
|
+
log.error(tag, `Error queueing refresh for ${key}:`, error);
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
const scanTime = Date.now() - startTime;
|
|
159
|
+
log.info(tag, `✅ Scan complete: ${totalScanned} scanned, ${staleKeys.length} stale, ${refreshCount} queued in ${scanTime}ms`);
|
|
160
|
+
}
|
|
161
|
+
catch (error) {
|
|
162
|
+
log.error(tag, 'Error during scan:', error);
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
/**
|
|
166
|
+
* Get scanner statistics
|
|
167
|
+
*/
|
|
168
|
+
getStats() {
|
|
169
|
+
return {
|
|
170
|
+
isRunning: this.isRunning,
|
|
171
|
+
config: {
|
|
172
|
+
staleThresholdHours: this.config.staleThresholdMs / (1000 * 60 * 60),
|
|
173
|
+
scanIntervalMinutes: this.config.scanIntervalMs / (1000 * 60),
|
|
174
|
+
batchSize: this.config.batchSize,
|
|
175
|
+
maxRefreshPerScan: this.config.maxRefreshPerScan
|
|
176
|
+
}
|
|
177
|
+
};
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
exports.StaleBalanceScanner = StaleBalanceScanner;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@pioneer-platform/pioneer-cache",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.22.0",
|
|
4
4
|
"description": "Unified caching system for Pioneer platform with Redis backend",
|
|
5
5
|
"main": "./dist/index.js",
|
|
6
6
|
"types": "./dist/index.d.ts",
|
|
@@ -15,8 +15,10 @@
|
|
|
15
15
|
"license": "MIT",
|
|
16
16
|
"dependencies": {
|
|
17
17
|
"@pioneer-platform/loggerdog": "8.11.0",
|
|
18
|
-
"@pioneer-platform/
|
|
19
|
-
"@pioneer-platform/
|
|
18
|
+
"@pioneer-platform/pioneer-discovery": "8.44.0",
|
|
19
|
+
"@pioneer-platform/pioneer-caip": "9.21.0",
|
|
20
|
+
"@pioneer-platform/default-redis": "8.11.7",
|
|
21
|
+
"@pioneer-platform/redis-queue": "8.12.17"
|
|
20
22
|
},
|
|
21
23
|
"devDependencies": {
|
|
22
24
|
"@types/jest": "^29.5.0",
|
|
@@ -12,6 +12,7 @@ import { TransactionCache } from '../stores/transaction-cache';
|
|
|
12
12
|
import { StakingCache } from '../stores/staking-cache';
|
|
13
13
|
import { ZapperCache } from '../stores/zapper-cache';
|
|
14
14
|
import { RefreshWorker, startUnifiedWorker } from '../workers/refresh-worker';
|
|
15
|
+
import { StaleBalanceScanner } from '../workers/stale-balance-scanner';
|
|
15
16
|
import type { BaseCache } from './base-cache';
|
|
16
17
|
import type { HealthCheckResult } from '../types';
|
|
17
18
|
|
|
@@ -53,6 +54,7 @@ export class CacheManager {
|
|
|
53
54
|
private stakingCache?: StakingCache;
|
|
54
55
|
private zapperCache?: ZapperCache;
|
|
55
56
|
private workers: RefreshWorker[] = [];
|
|
57
|
+
private staleScanner?: StaleBalanceScanner;
|
|
56
58
|
|
|
57
59
|
constructor(config: CacheManagerConfig) {
|
|
58
60
|
this.redis = config.redis;
|
|
@@ -147,6 +149,79 @@ export class CacheManager {
|
|
|
147
149
|
}
|
|
148
150
|
}
|
|
149
151
|
|
|
152
|
+
/**
|
|
153
|
+
* Purge unsupported job types from the queue
|
|
154
|
+
* Scans the queue and removes jobs for cache types that aren't enabled
|
|
155
|
+
*/
|
|
156
|
+
async purgeUnsupportedJobs(queueName: string = 'cache-refresh'): Promise<void> {
|
|
157
|
+
const tag = TAG + 'purgeUnsupportedJobs | ';
|
|
158
|
+
|
|
159
|
+
try {
|
|
160
|
+
const redisQueue = this.redisQueue;
|
|
161
|
+
let purgedCount = 0;
|
|
162
|
+
let scannedCount = 0;
|
|
163
|
+
|
|
164
|
+
log.info(tag, `Scanning queue for unsupported job types...`);
|
|
165
|
+
|
|
166
|
+
// Get enabled cache types
|
|
167
|
+
const enabledCaches = new Set<string>();
|
|
168
|
+
if (this.balanceCache) enabledCaches.add('balance');
|
|
169
|
+
if (this.priceCache) enabledCaches.add('price');
|
|
170
|
+
if (this.portfolioCache) enabledCaches.add('portfolio');
|
|
171
|
+
if (this.transactionCache) enabledCaches.add('transaction');
|
|
172
|
+
if (this.stakingCache) enabledCaches.add('staking');
|
|
173
|
+
if (this.zapperCache) enabledCaches.add('zapper');
|
|
174
|
+
|
|
175
|
+
log.info(tag, `Enabled caches: ${Array.from(enabledCaches).join(', ') || 'NONE'}`);
|
|
176
|
+
|
|
177
|
+
// Get queue length
|
|
178
|
+
const queueLength = await redisQueue.count(queueName);
|
|
179
|
+
|
|
180
|
+
if (queueLength === 0) {
|
|
181
|
+
log.info(tag, '✅ Queue is empty, no jobs to purge');
|
|
182
|
+
return;
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
log.info(tag, `Found ${queueLength} jobs in queue`);
|
|
186
|
+
|
|
187
|
+
// Scan all jobs
|
|
188
|
+
const tempJobs: any[] = [];
|
|
189
|
+
|
|
190
|
+
for (let i = 0; i < queueLength; i++) {
|
|
191
|
+
const job = await redisQueue.getWork(queueName, 1);
|
|
192
|
+
|
|
193
|
+
if (!job) break;
|
|
194
|
+
|
|
195
|
+
scannedCount++;
|
|
196
|
+
const jobType = job.type || '';
|
|
197
|
+
const cacheName = jobType.replace('REFRESH_', '').toLowerCase();
|
|
198
|
+
|
|
199
|
+
if (!enabledCaches.has(cacheName)) {
|
|
200
|
+
purgedCount++;
|
|
201
|
+
log.warn(tag, ` Purged unsupported job: ${jobType} (cache "${cacheName}" not enabled)`);
|
|
202
|
+
} else {
|
|
203
|
+
// Keep supported jobs - we'll re-queue them
|
|
204
|
+
tempJobs.push(job);
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
// Re-queue supported jobs
|
|
209
|
+
for (const job of tempJobs) {
|
|
210
|
+
await redisQueue.createWork(queueName, job);
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
if (purgedCount > 0) {
|
|
214
|
+
log.info(tag, `✅ Purged ${purgedCount} unsupported jobs (scanned ${scannedCount})`);
|
|
215
|
+
} else {
|
|
216
|
+
log.info(tag, `✅ No unsupported jobs found (scanned ${scannedCount})`);
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
} catch (error) {
|
|
220
|
+
log.error(tag, '⚠️ Error during unsupported job purge (non-fatal):', error);
|
|
221
|
+
// Don't throw - purge failure shouldn't prevent system from starting
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
|
|
150
225
|
/**
|
|
151
226
|
* Start background refresh workers for all caches
|
|
152
227
|
*/
|
|
@@ -156,6 +231,9 @@ export class CacheManager {
|
|
|
156
231
|
try {
|
|
157
232
|
log.info(tag, 'Starting refresh workers...');
|
|
158
233
|
|
|
234
|
+
// FIX: Purge unsupported job types before starting workers
|
|
235
|
+
await this.purgeUnsupportedJobs('cache-refresh');
|
|
236
|
+
|
|
159
237
|
// Create cache registry for workers
|
|
160
238
|
const cacheRegistry = new Map<string, BaseCache<any>>();
|
|
161
239
|
|
|
@@ -196,6 +274,19 @@ export class CacheManager {
|
|
|
196
274
|
log.info(tag, '✅ Refresh worker started for all caches');
|
|
197
275
|
}
|
|
198
276
|
|
|
277
|
+
// Start stale balance scanner if balance cache is enabled
|
|
278
|
+
if (this.balanceCache) {
|
|
279
|
+
this.staleScanner = new StaleBalanceScanner(this.redis, this.balanceCache, {
|
|
280
|
+
staleThresholdMs: 60 * 60 * 1000, // 1 hour (more aggressive than the 5min reactive threshold)
|
|
281
|
+
scanIntervalMs: 10 * 60 * 1000, // Scan every 10 minutes
|
|
282
|
+
batchSize: 100,
|
|
283
|
+
maxRefreshPerScan: 50
|
|
284
|
+
});
|
|
285
|
+
|
|
286
|
+
await this.staleScanner.start();
|
|
287
|
+
log.info(tag, '✅ Stale balance scanner started');
|
|
288
|
+
}
|
|
289
|
+
|
|
199
290
|
} catch (error) {
|
|
200
291
|
log.error(tag, '❌ Failed to start workers:', error);
|
|
201
292
|
throw error;
|
|
@@ -215,6 +306,11 @@ export class CacheManager {
|
|
|
215
306
|
await worker.stop();
|
|
216
307
|
}
|
|
217
308
|
|
|
309
|
+
if (this.staleScanner) {
|
|
310
|
+
await this.staleScanner.stop();
|
|
311
|
+
this.staleScanner = undefined;
|
|
312
|
+
}
|
|
313
|
+
|
|
218
314
|
this.workers = [];
|
|
219
315
|
log.info(tag, '✅ All workers stopped');
|
|
220
316
|
|
|
@@ -8,6 +8,8 @@
|
|
|
8
8
|
import { BaseCache } from '../core/base-cache';
|
|
9
9
|
import type { CacheConfig } from '../types';
|
|
10
10
|
import crypto from 'crypto';
|
|
11
|
+
import { assetData } from '@pioneer-platform/pioneer-discovery';
|
|
12
|
+
import { caipToNetworkId } from '@pioneer-platform/pioneer-caip';
|
|
11
13
|
|
|
12
14
|
const log = require('@pioneer-platform/loggerdog')();
|
|
13
15
|
|
|
@@ -148,12 +150,22 @@ export class BalanceCache extends BaseCache<BalanceData> {
|
|
|
148
150
|
const balanceInfo = await this.balanceModule.getBalance(asset, owner);
|
|
149
151
|
|
|
150
152
|
if (!balanceInfo || !balanceInfo.balance) {
|
|
151
|
-
|
|
153
|
+
// CRITICAL FIX: Distinguish between legitimate zero balance and failed API call
|
|
154
|
+
// Failed API calls should NOT be cached as valid "0" balance
|
|
155
|
+
const errorMsg = balanceInfo?.error || 'No balance data returned';
|
|
156
|
+
|
|
157
|
+
log.warn(tag, `No balance returned for ${caip}/${sanitizePubkey(pubkey)} - ${errorMsg}`);
|
|
158
|
+
|
|
159
|
+
// If balanceInfo explicitly indicates an error, throw it instead of caching "0"
|
|
160
|
+
if (balanceInfo?.error || balanceInfo === null || balanceInfo === undefined) {
|
|
161
|
+
throw new Error(`Failed to fetch balance for ${caip}: ${errorMsg}`);
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
// Otherwise, this is a legitimate zero balance (balanceInfo exists but balance is falsy)
|
|
152
165
|
const now = Date.now();
|
|
153
166
|
|
|
154
167
|
// Get asset metadata even for zero balances
|
|
155
|
-
const
|
|
156
|
-
const assetInfo = assetData[caip.toUpperCase()] || assetData[caip.toLowerCase()] || {};
|
|
168
|
+
const assetInfo = (assetData as any)[caip.toUpperCase()] || (assetData as any)[caip.toLowerCase()] || {};
|
|
157
169
|
|
|
158
170
|
return {
|
|
159
171
|
caip,
|
|
@@ -162,15 +174,13 @@ export class BalanceCache extends BaseCache<BalanceData> {
|
|
|
162
174
|
decimals: assetInfo.decimals,
|
|
163
175
|
precision: assetInfo.decimals, // Set precision to decimals for compatibility
|
|
164
176
|
fetchedAt: now,
|
|
165
|
-
fetchedAtISO: new Date(now).toISOString()
|
|
177
|
+
fetchedAtISO: new Date(now).toISOString(),
|
|
178
|
+
dataSource: 'Zero balance (legitimate)'
|
|
166
179
|
};
|
|
167
180
|
}
|
|
168
181
|
|
|
169
182
|
// Get asset metadata
|
|
170
|
-
const
|
|
171
|
-
const { caipToNetworkId } = require('@pioneer-platform/pioneer-caip');
|
|
172
|
-
|
|
173
|
-
const assetInfo = assetData[caip.toUpperCase()] || assetData[caip.toLowerCase()] || {};
|
|
183
|
+
const assetInfo = (assetData as any)[caip.toUpperCase()] || (assetData as any)[caip.toLowerCase()] || {};
|
|
174
184
|
const networkId = caipToNetworkId(caip);
|
|
175
185
|
|
|
176
186
|
// Use actual node URL if available, otherwise construct generic description
|
|
@@ -229,7 +239,6 @@ export class BalanceCache extends BaseCache<BalanceData> {
|
|
|
229
239
|
|
|
230
240
|
try {
|
|
231
241
|
const { caip, pubkey } = params;
|
|
232
|
-
const { caipToNetworkId } = require('@pioneer-platform/pioneer-caip');
|
|
233
242
|
const networkId = caipToNetworkId(caip);
|
|
234
243
|
|
|
235
244
|
// Try legacy format: cache:balance:pubkey:asset
|
|
@@ -276,8 +285,15 @@ export class BalanceCache extends BaseCache<BalanceData> {
|
|
|
276
285
|
*
|
|
277
286
|
* CRITICAL FIX: When waitForFresh=true (forceRefresh), bypass cache entirely
|
|
278
287
|
* and fetch fresh blockchain data for ALL items, not just cache misses
|
|
288
|
+
*
|
|
289
|
+
* PERFORMANCE FIX: When skipAsyncRefresh=true, disable background refresh jobs
|
|
290
|
+
* to prevent 50+ individual refresh jobs from being queued
|
|
279
291
|
*/
|
|
280
|
-
async getBatchBalances(
|
|
292
|
+
async getBatchBalances(
|
|
293
|
+
items: Array<{ caip: string; pubkey: string }>,
|
|
294
|
+
waitForFresh?: boolean,
|
|
295
|
+
skipAsyncRefresh?: boolean
|
|
296
|
+
): Promise<BalanceData[]> {
|
|
281
297
|
const tag = this.TAG + 'getBatchBalances | ';
|
|
282
298
|
const startTime = Date.now();
|
|
283
299
|
|
|
@@ -409,11 +425,15 @@ export class BalanceCache extends BaseCache<BalanceData> {
|
|
|
409
425
|
|
|
410
426
|
await Promise.all(fetchPromises);
|
|
411
427
|
log.info(tag, `Fetched ${missedItems.length} misses in ${Date.now() - fetchStart}ms`);
|
|
412
|
-
} else {
|
|
413
|
-
// Non-blocking: trigger background refresh for misses
|
|
428
|
+
} else if (!skipAsyncRefresh) {
|
|
429
|
+
// Non-blocking: trigger background refresh for misses (unless disabled)
|
|
430
|
+
log.info(tag, `📋 Queueing ${missedItems.length} background refresh jobs for cache misses`);
|
|
414
431
|
missedItems.forEach(item => {
|
|
415
432
|
this.triggerAsyncRefresh({ caip: item.caip, pubkey: item.pubkey }, 'high');
|
|
416
433
|
});
|
|
434
|
+
} else {
|
|
435
|
+
// Background refresh disabled for performance
|
|
436
|
+
log.info(tag, `⚡ PERFORMANCE MODE: Skipping ${missedItems.length} background refresh jobs (skipAsyncRefresh=true)`);
|
|
417
437
|
}
|
|
418
438
|
}
|
|
419
439
|
|
|
@@ -69,12 +69,79 @@ export class RefreshWorker {
|
|
|
69
69
|
log.info(tag, `🚀 Starting refresh worker for queue: ${this.config.queueName}`);
|
|
70
70
|
log.info(tag, `Registered caches: ${Array.from(this.cacheRegistry.keys()).join(', ')}`);
|
|
71
71
|
|
|
72
|
+
// FIX #3: Purge stale jobs at startup
|
|
73
|
+
await this.purgeStaleJobsOnStartup();
|
|
74
|
+
|
|
72
75
|
this.isRunning = true;
|
|
73
76
|
this.poll();
|
|
74
77
|
|
|
75
78
|
log.info(tag, '✅ Refresh worker started successfully');
|
|
76
79
|
}
|
|
77
80
|
|
|
81
|
+
/**
|
|
82
|
+
* Purge stale jobs from queue on startup
|
|
83
|
+
* Scans queue and removes jobs older than 1 hour
|
|
84
|
+
*/
|
|
85
|
+
private async purgeStaleJobsOnStartup(): Promise<void> {
|
|
86
|
+
const tag = TAG + 'purgeStaleJobsOnStartup | ';
|
|
87
|
+
|
|
88
|
+
try {
|
|
89
|
+
const MAX_JOB_AGE_MS = 60 * 60 * 1000; // 1 hour
|
|
90
|
+
const now = Date.now();
|
|
91
|
+
let purgedCount = 0;
|
|
92
|
+
let scannedCount = 0;
|
|
93
|
+
|
|
94
|
+
log.info(tag, `Scanning queue for stale jobs (age > ${MAX_JOB_AGE_MS / 1000}s)...`);
|
|
95
|
+
|
|
96
|
+
// Get queue length
|
|
97
|
+
const queueLength = await this.redisQueue.count(this.config.queueName);
|
|
98
|
+
|
|
99
|
+
if (queueLength === 0) {
|
|
100
|
+
log.info(tag, '✅ Queue is empty, no stale jobs to purge');
|
|
101
|
+
return;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
log.info(tag, `Found ${queueLength} jobs in queue, checking for stale jobs...`);
|
|
105
|
+
|
|
106
|
+
// Scan up to 100 jobs to find stale ones
|
|
107
|
+
const MAX_SCAN = Math.min(queueLength, 100);
|
|
108
|
+
const tempJobs: RefreshJob[] = [];
|
|
109
|
+
|
|
110
|
+
for (let i = 0; i < MAX_SCAN; i++) {
|
|
111
|
+
const job = await this.redisQueue.getWork(this.config.queueName, 1);
|
|
112
|
+
|
|
113
|
+
if (!job) break;
|
|
114
|
+
|
|
115
|
+
scannedCount++;
|
|
116
|
+
const jobTimestamp = (job as any).timestamp || 0;
|
|
117
|
+
const jobAge = jobTimestamp > 0 ? now - jobTimestamp : 0;
|
|
118
|
+
|
|
119
|
+
if (jobAge > MAX_JOB_AGE_MS) {
|
|
120
|
+
purgedCount++;
|
|
121
|
+
log.warn(tag, ` Purged stale job: ${job.type} (age: ${Math.round(jobAge / 1000)}s)`);
|
|
122
|
+
} else {
|
|
123
|
+
// Keep fresh jobs - we'll re-queue them
|
|
124
|
+
tempJobs.push(job);
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
// Re-queue fresh jobs
|
|
129
|
+
for (const job of tempJobs) {
|
|
130
|
+
await this.redisQueue.createWork(this.config.queueName, job);
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
if (purgedCount > 0) {
|
|
134
|
+
log.info(tag, `✅ Purged ${purgedCount} stale jobs (scanned ${scannedCount})`);
|
|
135
|
+
} else {
|
|
136
|
+
log.info(tag, `✅ No stale jobs found (scanned ${scannedCount})`);
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
} catch (error) {
|
|
140
|
+
log.error(tag, '⚠️ Error during startup purge (non-fatal):', error);
|
|
141
|
+
// Don't throw - startup purge failure shouldn't prevent worker from starting
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
|
|
78
145
|
/**
|
|
79
146
|
* Stop the worker gracefully
|
|
80
147
|
*/
|
|
@@ -166,14 +233,34 @@ export class RefreshWorker {
|
|
|
166
233
|
|
|
167
234
|
log.info(tag, `Processing ${type} for ${key} (retry: ${retryCount})`);
|
|
168
235
|
|
|
236
|
+
// FIX #1: Check job age and drop if too old (>1 hour)
|
|
237
|
+
const MAX_JOB_AGE_MS = 60 * 60 * 1000; // 1 hour
|
|
238
|
+
const jobAge = Date.now() - startTime;
|
|
239
|
+
const jobTimestamp = (job as any).timestamp || 0;
|
|
240
|
+
|
|
241
|
+
if (jobTimestamp > 0) {
|
|
242
|
+
const actualJobAge = Date.now() - jobTimestamp;
|
|
243
|
+
if (actualJobAge > MAX_JOB_AGE_MS) {
|
|
244
|
+
log.error(tag, `⚠️ DROPPING STALE JOB: ${type} (age: ${Math.round(actualJobAge / 1000)}s, max: ${MAX_JOB_AGE_MS / 1000}s)`);
|
|
245
|
+
log.error(tag, ` Job key: ${key}`);
|
|
246
|
+
log.error(tag, ` Job has been removed from queue and will not be retried`);
|
|
247
|
+
return;
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
|
|
169
251
|
// Extract cache name from job type (e.g., "REFRESH_BALANCE" -> "balance")
|
|
170
252
|
const cacheName = type.replace('REFRESH_', '').toLowerCase();
|
|
171
253
|
|
|
172
254
|
// Get the appropriate cache instance
|
|
173
255
|
const cache = this.cacheRegistry.get(cacheName);
|
|
174
256
|
|
|
257
|
+
// FIX #2: Drop unsupported job types with clear error message
|
|
175
258
|
if (!cache) {
|
|
176
|
-
log.error(tag,
|
|
259
|
+
log.error(tag, `❌ DROPPING UNSUPPORTED JOB: ${type}`);
|
|
260
|
+
log.error(tag, ` Cache type "${cacheName}" is not registered in this worker`);
|
|
261
|
+
log.error(tag, ` Registered caches: ${Array.from(this.cacheRegistry.keys()).join(', ') || 'NONE'}`);
|
|
262
|
+
log.error(tag, ` Job key: ${key}`);
|
|
263
|
+
log.error(tag, ` Job has been removed from queue and will not be retried`);
|
|
177
264
|
return;
|
|
178
265
|
}
|
|
179
266
|
|
|
@@ -0,0 +1,223 @@
|
|
|
1
|
+
/*
|
|
2
|
+
Stale Balance Scanner
|
|
3
|
+
|
|
4
|
+
Proactively scans for stale balance cache entries and queues them for refresh.
|
|
5
|
+
This ensures balances that aren't actively accessed still get refreshed.
|
|
6
|
+
|
|
7
|
+
Prevents the issue where stale balances sit for days/weeks without being updated.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import type { BalanceCache } from '../stores/balance-cache';
|
|
11
|
+
|
|
12
|
+
const log = require('@pioneer-platform/loggerdog')();
|
|
13
|
+
const TAG = ' | StaleBalanceScanner | ';
|
|
14
|
+
|
|
15
|
+
export interface StaleScannerConfig {
|
|
16
|
+
staleThresholdMs: number; // Refresh balances older than this (default: 1 hour)
|
|
17
|
+
scanIntervalMs: number; // How often to scan (default: 5 minutes)
|
|
18
|
+
batchSize: number; // How many keys to scan per batch (default: 100)
|
|
19
|
+
maxRefreshPerScan: number; // Max balances to refresh per scan (default: 50)
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export class StaleBalanceScanner {
|
|
23
|
+
private redis: any;
|
|
24
|
+
private balanceCache: BalanceCache;
|
|
25
|
+
private config: StaleScannerConfig;
|
|
26
|
+
private isRunning: boolean = false;
|
|
27
|
+
private scanTimer?: NodeJS.Timeout;
|
|
28
|
+
|
|
29
|
+
constructor(redis: any, balanceCache: BalanceCache, config?: Partial<StaleScannerConfig>) {
|
|
30
|
+
this.redis = redis;
|
|
31
|
+
this.balanceCache = balanceCache;
|
|
32
|
+
|
|
33
|
+
this.config = {
|
|
34
|
+
staleThresholdMs: config?.staleThresholdMs || 60 * 60 * 1000, // 1 hour
|
|
35
|
+
scanIntervalMs: config?.scanIntervalMs || 5 * 60 * 1000, // 5 minutes
|
|
36
|
+
batchSize: config?.batchSize || 100,
|
|
37
|
+
maxRefreshPerScan: config?.maxRefreshPerScan || 50
|
|
38
|
+
};
|
|
39
|
+
|
|
40
|
+
log.info(TAG, 'Stale balance scanner initialized', {
|
|
41
|
+
staleThresholdHours: this.config.staleThresholdMs / (1000 * 60 * 60),
|
|
42
|
+
scanIntervalMinutes: this.config.scanIntervalMs / (1000 * 60)
|
|
43
|
+
});
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* Start the scanner
|
|
48
|
+
*/
|
|
49
|
+
async start(): Promise<void> {
|
|
50
|
+
const tag = TAG + 'start | ';
|
|
51
|
+
|
|
52
|
+
if (this.isRunning) {
|
|
53
|
+
log.warn(tag, 'Scanner already running');
|
|
54
|
+
return;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
this.isRunning = true;
|
|
58
|
+
log.info(tag, '🔍 Starting stale balance scanner');
|
|
59
|
+
|
|
60
|
+
// Run first scan immediately, then schedule recurring
|
|
61
|
+
await this.scan();
|
|
62
|
+
this.scheduleNextScan();
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* Stop the scanner
|
|
67
|
+
*/
|
|
68
|
+
async stop(): Promise<void> {
|
|
69
|
+
const tag = TAG + 'stop | ';
|
|
70
|
+
|
|
71
|
+
log.info(tag, 'Stopping stale balance scanner');
|
|
72
|
+
this.isRunning = false;
|
|
73
|
+
|
|
74
|
+
if (this.scanTimer) {
|
|
75
|
+
clearTimeout(this.scanTimer);
|
|
76
|
+
this.scanTimer = undefined;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
log.info(tag, '✅ Scanner stopped');
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* Schedule next scan
|
|
84
|
+
*/
|
|
85
|
+
private scheduleNextScan(): void {
|
|
86
|
+
if (!this.isRunning) return;
|
|
87
|
+
|
|
88
|
+
this.scanTimer = setTimeout(async () => {
|
|
89
|
+
await this.scan();
|
|
90
|
+
this.scheduleNextScan();
|
|
91
|
+
}, this.config.scanIntervalMs);
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/**
|
|
95
|
+
* Scan for stale balances and queue refreshes
|
|
96
|
+
*/
|
|
97
|
+
private async scan(): Promise<void> {
|
|
98
|
+
const tag = TAG + 'scan | ';
|
|
99
|
+
const startTime = Date.now();
|
|
100
|
+
|
|
101
|
+
try {
|
|
102
|
+
log.info(tag, 'Starting stale balance scan...');
|
|
103
|
+
|
|
104
|
+
// Scan balance_v2:* keys in batches
|
|
105
|
+
const pattern = 'balance_v2:*';
|
|
106
|
+
const staleKeys: string[] = [];
|
|
107
|
+
const now = Date.now();
|
|
108
|
+
|
|
109
|
+
let cursor = '0';
|
|
110
|
+
let totalScanned = 0;
|
|
111
|
+
let refreshCount = 0;
|
|
112
|
+
|
|
113
|
+
do {
|
|
114
|
+
// SCAN with COUNT for batching
|
|
115
|
+
const [nextCursor, keys] = await this.redis.scan(
|
|
116
|
+
cursor,
|
|
117
|
+
'MATCH', pattern,
|
|
118
|
+
'COUNT', this.config.batchSize
|
|
119
|
+
);
|
|
120
|
+
|
|
121
|
+
cursor = nextCursor;
|
|
122
|
+
totalScanned += keys.length;
|
|
123
|
+
|
|
124
|
+
// Check each key for staleness
|
|
125
|
+
for (const key of keys) {
|
|
126
|
+
try {
|
|
127
|
+
const cached = await this.redis.get(key);
|
|
128
|
+
if (!cached) continue;
|
|
129
|
+
|
|
130
|
+
const parsed = JSON.parse(cached);
|
|
131
|
+
const age = now - (parsed.timestamp || 0);
|
|
132
|
+
|
|
133
|
+
// Check if stale
|
|
134
|
+
if (age > this.config.staleThresholdMs) {
|
|
135
|
+
staleKeys.push(key);
|
|
136
|
+
|
|
137
|
+
// Stop if we've hit max refresh limit
|
|
138
|
+
if (staleKeys.length >= this.config.maxRefreshPerScan) {
|
|
139
|
+
log.info(tag, `Reached max refresh limit (${this.config.maxRefreshPerScan}), stopping scan`);
|
|
140
|
+
cursor = '0'; // Force exit
|
|
141
|
+
break;
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
} catch (error) {
|
|
145
|
+
log.error(tag, `Error checking key ${key}:`, error);
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
} while (cursor !== '0' && this.isRunning);
|
|
150
|
+
|
|
151
|
+
// Queue refresh for stale balances
|
|
152
|
+
if (staleKeys.length > 0) {
|
|
153
|
+
log.info(tag, `Found ${staleKeys.length} stale balances out of ${totalScanned} scanned`);
|
|
154
|
+
|
|
155
|
+
for (const key of staleKeys) {
|
|
156
|
+
try {
|
|
157
|
+
// Extract CAIP and pubkey hash from key
|
|
158
|
+
// Key format: balance_v2:caip:hashedPubkey
|
|
159
|
+
// IMPORTANT: CAIP itself contains colons (e.g., ripple:4109c6f2045fc7eff4cde8f9905d19c2/slip44:144)
|
|
160
|
+
// So we must split from the right to get hashedPubkey, then everything else is the CAIP
|
|
161
|
+
const withoutPrefix = key.replace('balance_v2:', '');
|
|
162
|
+
const lastColonIndex = withoutPrefix.lastIndexOf(':');
|
|
163
|
+
if (lastColonIndex === -1) continue;
|
|
164
|
+
|
|
165
|
+
const caip = withoutPrefix.substring(0, lastColonIndex);
|
|
166
|
+
const pubkeyHash = withoutPrefix.substring(lastColonIndex + 1);
|
|
167
|
+
|
|
168
|
+
// We need the original pubkey to refresh, but it's hashed in the key
|
|
169
|
+
// So we need to get it from the cached value
|
|
170
|
+
const cached = await this.redis.get(key);
|
|
171
|
+
if (!cached) continue;
|
|
172
|
+
|
|
173
|
+
const parsed = JSON.parse(cached);
|
|
174
|
+
const pubkey = parsed.value?.pubkey;
|
|
175
|
+
|
|
176
|
+
if (!pubkey) {
|
|
177
|
+
log.warn(tag, `No pubkey found in cache for ${key}, skipping`);
|
|
178
|
+
continue;
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
// Trigger refresh by calling getBalance with waitForFresh=true
|
|
182
|
+
// This will queue a background refresh job
|
|
183
|
+
log.debug(tag, `Queueing refresh for ${caip} (age: ${Math.round((now - parsed.timestamp) / 1000 / 60)}min)`);
|
|
184
|
+
|
|
185
|
+
// Use setImmediate to avoid blocking the scan
|
|
186
|
+
setImmediate(async () => {
|
|
187
|
+
try {
|
|
188
|
+
await this.balanceCache.getBalance(caip, pubkey, true);
|
|
189
|
+
refreshCount++;
|
|
190
|
+
} catch (error) {
|
|
191
|
+
log.error(tag, `Failed to refresh ${caip}/${pubkey}:`, error);
|
|
192
|
+
}
|
|
193
|
+
});
|
|
194
|
+
|
|
195
|
+
} catch (error) {
|
|
196
|
+
log.error(tag, `Error queueing refresh for ${key}:`, error);
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
const scanTime = Date.now() - startTime;
|
|
202
|
+
log.info(tag, `✅ Scan complete: ${totalScanned} scanned, ${staleKeys.length} stale, ${refreshCount} queued in ${scanTime}ms`);
|
|
203
|
+
|
|
204
|
+
} catch (error) {
|
|
205
|
+
log.error(tag, 'Error during scan:', error);
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
/**
|
|
210
|
+
* Get scanner statistics
|
|
211
|
+
*/
|
|
212
|
+
getStats(): any {
|
|
213
|
+
return {
|
|
214
|
+
isRunning: this.isRunning,
|
|
215
|
+
config: {
|
|
216
|
+
staleThresholdHours: this.config.staleThresholdMs / (1000 * 60 * 60),
|
|
217
|
+
scanIntervalMinutes: this.config.scanIntervalMs / (1000 * 60),
|
|
218
|
+
batchSize: this.config.batchSize,
|
|
219
|
+
maxRefreshPerScan: this.config.maxRefreshPerScan
|
|
220
|
+
}
|
|
221
|
+
};
|
|
222
|
+
}
|
|
223
|
+
}
|
package/tsconfig.json
CHANGED
|
@@ -2,17 +2,26 @@
|
|
|
2
2
|
"compilerOptions": {
|
|
3
3
|
"target": "ES2020",
|
|
4
4
|
"module": "commonjs",
|
|
5
|
-
"lib": [
|
|
5
|
+
"lib": [
|
|
6
|
+
"ES2020",
|
|
7
|
+
"DOM"
|
|
8
|
+
],
|
|
6
9
|
"declaration": true,
|
|
7
10
|
"outDir": "./dist",
|
|
8
11
|
"rootDir": "./src",
|
|
9
12
|
"strict": true,
|
|
10
13
|
"esModuleInterop": true,
|
|
11
|
-
"skipLibCheck": true,
|
|
12
14
|
"forceConsistentCasingInFileNames": true,
|
|
13
15
|
"resolveJsonModule": true,
|
|
14
|
-
"moduleResolution": "node"
|
|
16
|
+
"moduleResolution": "node",
|
|
17
|
+
"skipLibCheck": true
|
|
15
18
|
},
|
|
16
|
-
"include": [
|
|
17
|
-
|
|
19
|
+
"include": [
|
|
20
|
+
"src/**/*"
|
|
21
|
+
],
|
|
22
|
+
"exclude": [
|
|
23
|
+
"node_modules",
|
|
24
|
+
"dist",
|
|
25
|
+
"**/*.test.ts"
|
|
26
|
+
]
|
|
18
27
|
}
|