@pioneer-platform/pioneer-cache 1.21.0 → 1.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -0
- package/dist/core/cache-manager.d.ts +5 -0
- package/dist/core/cache-manager.js +69 -0
- package/dist/stores/balance-cache.d.ts +4 -1
- package/dist/stores/balance-cache.js +17 -11
- package/dist/workers/refresh-worker.d.ts +5 -0
- package/dist/workers/refresh-worker.js +75 -1
- package/package.json +3 -1
- package/src/core/cache-manager.ts +76 -0
- package/src/stores/balance-cache.ts +18 -10
- package/src/workers/refresh-worker.ts +88 -1
- package/tsconfig.json +14 -4
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,23 @@
|
|
|
1
1
|
# @pioneer-platform/pioneer-cache
|
|
2
2
|
|
|
3
|
+
## 1.22.0
|
|
4
|
+
|
|
5
|
+
### Minor Changes
|
|
6
|
+
|
|
7
|
+
- chore: feat(e2e): add blue API support to swap roundtrip tests
|
|
8
|
+
|
|
9
|
+
### Patch Changes
|
|
10
|
+
|
|
11
|
+
- Updated dependencies
|
|
12
|
+
- @pioneer-platform/pioneer-discovery@8.44.0
|
|
13
|
+
|
|
14
|
+
## 1.21.1
|
|
15
|
+
|
|
16
|
+
### Patch Changes
|
|
17
|
+
|
|
18
|
+
- Updated dependencies
|
|
19
|
+
- @pioneer-platform/pioneer-discovery@8.43.0
|
|
20
|
+
|
|
3
21
|
## 1.21.0
|
|
4
22
|
|
|
5
23
|
### Minor Changes
|
|
@@ -45,6 +45,11 @@ export declare class CacheManager {
|
|
|
45
45
|
* Used in local dev to clear stale jobs on startup
|
|
46
46
|
*/
|
|
47
47
|
purgeQueue(queueName?: string): Promise<void>;
|
|
48
|
+
/**
|
|
49
|
+
* Purge unsupported job types from the queue
|
|
50
|
+
* Scans the queue and removes jobs for cache types that aren't enabled
|
|
51
|
+
*/
|
|
52
|
+
purgeUnsupportedJobs(queueName?: string): Promise<void>;
|
|
48
53
|
/**
|
|
49
54
|
* Start background refresh workers for all caches
|
|
50
55
|
*/
|
|
@@ -104,6 +104,73 @@ class CacheManager {
|
|
|
104
104
|
throw error;
|
|
105
105
|
}
|
|
106
106
|
}
|
|
107
|
+
/**
|
|
108
|
+
* Purge unsupported job types from the queue
|
|
109
|
+
* Scans the queue and removes jobs for cache types that aren't enabled
|
|
110
|
+
*/
|
|
111
|
+
async purgeUnsupportedJobs(queueName = 'cache-refresh') {
|
|
112
|
+
const tag = TAG + 'purgeUnsupportedJobs | ';
|
|
113
|
+
try {
|
|
114
|
+
const redisQueue = this.redisQueue;
|
|
115
|
+
let purgedCount = 0;
|
|
116
|
+
let scannedCount = 0;
|
|
117
|
+
log.info(tag, `Scanning queue for unsupported job types...`);
|
|
118
|
+
// Get enabled cache types
|
|
119
|
+
const enabledCaches = new Set();
|
|
120
|
+
if (this.balanceCache)
|
|
121
|
+
enabledCaches.add('balance');
|
|
122
|
+
if (this.priceCache)
|
|
123
|
+
enabledCaches.add('price');
|
|
124
|
+
if (this.portfolioCache)
|
|
125
|
+
enabledCaches.add('portfolio');
|
|
126
|
+
if (this.transactionCache)
|
|
127
|
+
enabledCaches.add('transaction');
|
|
128
|
+
if (this.stakingCache)
|
|
129
|
+
enabledCaches.add('staking');
|
|
130
|
+
if (this.zapperCache)
|
|
131
|
+
enabledCaches.add('zapper');
|
|
132
|
+
log.info(tag, `Enabled caches: ${Array.from(enabledCaches).join(', ') || 'NONE'}`);
|
|
133
|
+
// Get queue length
|
|
134
|
+
const queueLength = await redisQueue.count(queueName);
|
|
135
|
+
if (queueLength === 0) {
|
|
136
|
+
log.info(tag, '✅ Queue is empty, no jobs to purge');
|
|
137
|
+
return;
|
|
138
|
+
}
|
|
139
|
+
log.info(tag, `Found ${queueLength} jobs in queue`);
|
|
140
|
+
// Scan all jobs
|
|
141
|
+
const tempJobs = [];
|
|
142
|
+
for (let i = 0; i < queueLength; i++) {
|
|
143
|
+
const job = await redisQueue.getWork(queueName, 1);
|
|
144
|
+
if (!job)
|
|
145
|
+
break;
|
|
146
|
+
scannedCount++;
|
|
147
|
+
const jobType = job.type || '';
|
|
148
|
+
const cacheName = jobType.replace('REFRESH_', '').toLowerCase();
|
|
149
|
+
if (!enabledCaches.has(cacheName)) {
|
|
150
|
+
purgedCount++;
|
|
151
|
+
log.warn(tag, ` Purged unsupported job: ${jobType} (cache "${cacheName}" not enabled)`);
|
|
152
|
+
}
|
|
153
|
+
else {
|
|
154
|
+
// Keep supported jobs - we'll re-queue them
|
|
155
|
+
tempJobs.push(job);
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
// Re-queue supported jobs
|
|
159
|
+
for (const job of tempJobs) {
|
|
160
|
+
await redisQueue.createWork(queueName, job);
|
|
161
|
+
}
|
|
162
|
+
if (purgedCount > 0) {
|
|
163
|
+
log.info(tag, `✅ Purged ${purgedCount} unsupported jobs (scanned ${scannedCount})`);
|
|
164
|
+
}
|
|
165
|
+
else {
|
|
166
|
+
log.info(tag, `✅ No unsupported jobs found (scanned ${scannedCount})`);
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
catch (error) {
|
|
170
|
+
log.error(tag, '⚠️ Error during unsupported job purge (non-fatal):', error);
|
|
171
|
+
// Don't throw - purge failure shouldn't prevent system from starting
|
|
172
|
+
}
|
|
173
|
+
}
|
|
107
174
|
/**
|
|
108
175
|
* Start background refresh workers for all caches
|
|
109
176
|
*/
|
|
@@ -111,6 +178,8 @@ class CacheManager {
|
|
|
111
178
|
const tag = TAG + 'startWorkers | ';
|
|
112
179
|
try {
|
|
113
180
|
log.info(tag, 'Starting refresh workers...');
|
|
181
|
+
// FIX: Purge unsupported job types before starting workers
|
|
182
|
+
await this.purgeUnsupportedJobs('cache-refresh');
|
|
114
183
|
// Create cache registry for workers
|
|
115
184
|
const cacheRegistry = new Map();
|
|
116
185
|
if (this.balanceCache) {
|
|
@@ -58,9 +58,12 @@ export declare class BalanceCache extends BaseCache<BalanceData> {
|
|
|
58
58
|
*
|
|
59
59
|
* CRITICAL FIX: When waitForFresh=true (forceRefresh), bypass cache entirely
|
|
60
60
|
* and fetch fresh blockchain data for ALL items, not just cache misses
|
|
61
|
+
*
|
|
62
|
+
* PERFORMANCE FIX: When skipAsyncRefresh=true, disable background refresh jobs
|
|
63
|
+
* to prevent 50+ individual refresh jobs from being queued
|
|
61
64
|
*/
|
|
62
65
|
getBatchBalances(items: Array<{
|
|
63
66
|
caip: string;
|
|
64
67
|
pubkey: string;
|
|
65
|
-
}>, waitForFresh?: boolean): Promise<BalanceData[]>;
|
|
68
|
+
}>, waitForFresh?: boolean, skipAsyncRefresh?: boolean): Promise<BalanceData[]>;
|
|
66
69
|
}
|
|
@@ -12,6 +12,8 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
12
12
|
exports.BalanceCache = void 0;
|
|
13
13
|
const base_cache_1 = require("../core/base-cache");
|
|
14
14
|
const crypto_1 = __importDefault(require("crypto"));
|
|
15
|
+
const pioneer_discovery_1 = require("@pioneer-platform/pioneer-discovery");
|
|
16
|
+
const pioneer_caip_1 = require("@pioneer-platform/pioneer-caip");
|
|
15
17
|
const log = require('@pioneer-platform/loggerdog')();
|
|
16
18
|
/**
|
|
17
19
|
* Hash pubkey/xpub for privacy-protecting cache keys
|
|
@@ -121,8 +123,7 @@ class BalanceCache extends base_cache_1.BaseCache {
|
|
|
121
123
|
// Otherwise, this is a legitimate zero balance (balanceInfo exists but balance is falsy)
|
|
122
124
|
const now = Date.now();
|
|
123
125
|
// Get asset metadata even for zero balances
|
|
124
|
-
const
|
|
125
|
-
const assetInfo = assetData[caip.toUpperCase()] || assetData[caip.toLowerCase()] || {};
|
|
126
|
+
const assetInfo = pioneer_discovery_1.assetData[caip.toUpperCase()] || pioneer_discovery_1.assetData[caip.toLowerCase()] || {};
|
|
126
127
|
return {
|
|
127
128
|
caip,
|
|
128
129
|
pubkey,
|
|
@@ -135,10 +136,8 @@ class BalanceCache extends base_cache_1.BaseCache {
|
|
|
135
136
|
};
|
|
136
137
|
}
|
|
137
138
|
// Get asset metadata
|
|
138
|
-
const
|
|
139
|
-
const
|
|
140
|
-
const assetInfo = assetData[caip.toUpperCase()] || assetData[caip.toLowerCase()] || {};
|
|
141
|
-
const networkId = caipToNetworkId(caip);
|
|
139
|
+
const assetInfo = pioneer_discovery_1.assetData[caip.toUpperCase()] || pioneer_discovery_1.assetData[caip.toLowerCase()] || {};
|
|
140
|
+
const networkId = (0, pioneer_caip_1.caipToNetworkId)(caip);
|
|
142
141
|
// Use actual node URL if available, otherwise construct generic description
|
|
143
142
|
let dataSource = 'Unknown';
|
|
144
143
|
if (balanceInfo.nodeUrl) {
|
|
@@ -196,8 +195,7 @@ class BalanceCache extends base_cache_1.BaseCache {
|
|
|
196
195
|
const tag = this.TAG + 'getLegacyCached | ';
|
|
197
196
|
try {
|
|
198
197
|
const { caip, pubkey } = params;
|
|
199
|
-
const
|
|
200
|
-
const networkId = caipToNetworkId(caip);
|
|
198
|
+
const networkId = (0, pioneer_caip_1.caipToNetworkId)(caip);
|
|
201
199
|
// Try legacy format: cache:balance:pubkey:asset
|
|
202
200
|
const legacyKey = `cache:balance:${pubkey}:${networkId}`;
|
|
203
201
|
const legacyData = await this.redis.get(legacyKey);
|
|
@@ -238,8 +236,11 @@ class BalanceCache extends base_cache_1.BaseCache {
|
|
|
238
236
|
*
|
|
239
237
|
* CRITICAL FIX: When waitForFresh=true (forceRefresh), bypass cache entirely
|
|
240
238
|
* and fetch fresh blockchain data for ALL items, not just cache misses
|
|
239
|
+
*
|
|
240
|
+
* PERFORMANCE FIX: When skipAsyncRefresh=true, disable background refresh jobs
|
|
241
|
+
* to prevent 50+ individual refresh jobs from being queued
|
|
241
242
|
*/
|
|
242
|
-
async getBatchBalances(items, waitForFresh) {
|
|
243
|
+
async getBatchBalances(items, waitForFresh, skipAsyncRefresh) {
|
|
243
244
|
const tag = this.TAG + 'getBatchBalances | ';
|
|
244
245
|
const startTime = Date.now();
|
|
245
246
|
try {
|
|
@@ -356,12 +357,17 @@ class BalanceCache extends base_cache_1.BaseCache {
|
|
|
356
357
|
await Promise.all(fetchPromises);
|
|
357
358
|
log.info(tag, `Fetched ${missedItems.length} misses in ${Date.now() - fetchStart}ms`);
|
|
358
359
|
}
|
|
359
|
-
else {
|
|
360
|
-
// Non-blocking: trigger background refresh for misses
|
|
360
|
+
else if (!skipAsyncRefresh) {
|
|
361
|
+
// Non-blocking: trigger background refresh for misses (unless disabled)
|
|
362
|
+
log.info(tag, `📋 Queueing ${missedItems.length} background refresh jobs for cache misses`);
|
|
361
363
|
missedItems.forEach(item => {
|
|
362
364
|
this.triggerAsyncRefresh({ caip: item.caip, pubkey: item.pubkey }, 'high');
|
|
363
365
|
});
|
|
364
366
|
}
|
|
367
|
+
else {
|
|
368
|
+
// Background refresh disabled for performance
|
|
369
|
+
log.info(tag, `⚡ PERFORMANCE MODE: Skipping ${missedItems.length} background refresh jobs (skipAsyncRefresh=true)`);
|
|
370
|
+
}
|
|
365
371
|
}
|
|
366
372
|
return results;
|
|
367
373
|
}
|
|
@@ -29,6 +29,11 @@ export declare class RefreshWorker {
|
|
|
29
29
|
* Start processing jobs from the queue
|
|
30
30
|
*/
|
|
31
31
|
start(): Promise<void>;
|
|
32
|
+
/**
|
|
33
|
+
* Purge stale jobs from queue on startup
|
|
34
|
+
* Scans queue and removes jobs older than 1 hour
|
|
35
|
+
*/
|
|
36
|
+
private purgeStaleJobsOnStartup;
|
|
32
37
|
/**
|
|
33
38
|
* Stop the worker gracefully
|
|
34
39
|
*/
|
|
@@ -49,10 +49,66 @@ class RefreshWorker {
|
|
|
49
49
|
}
|
|
50
50
|
log.info(tag, `🚀 Starting refresh worker for queue: ${this.config.queueName}`);
|
|
51
51
|
log.info(tag, `Registered caches: ${Array.from(this.cacheRegistry.keys()).join(', ')}`);
|
|
52
|
+
// FIX #3: Purge stale jobs at startup
|
|
53
|
+
await this.purgeStaleJobsOnStartup();
|
|
52
54
|
this.isRunning = true;
|
|
53
55
|
this.poll();
|
|
54
56
|
log.info(tag, '✅ Refresh worker started successfully');
|
|
55
57
|
}
|
|
58
|
+
/**
|
|
59
|
+
* Purge stale jobs from queue on startup
|
|
60
|
+
* Scans queue and removes jobs older than 1 hour
|
|
61
|
+
*/
|
|
62
|
+
async purgeStaleJobsOnStartup() {
|
|
63
|
+
const tag = TAG + 'purgeStaleJobsOnStartup | ';
|
|
64
|
+
try {
|
|
65
|
+
const MAX_JOB_AGE_MS = 60 * 60 * 1000; // 1 hour
|
|
66
|
+
const now = Date.now();
|
|
67
|
+
let purgedCount = 0;
|
|
68
|
+
let scannedCount = 0;
|
|
69
|
+
log.info(tag, `Scanning queue for stale jobs (age > ${MAX_JOB_AGE_MS / 1000}s)...`);
|
|
70
|
+
// Get queue length
|
|
71
|
+
const queueLength = await this.redisQueue.count(this.config.queueName);
|
|
72
|
+
if (queueLength === 0) {
|
|
73
|
+
log.info(tag, '✅ Queue is empty, no stale jobs to purge');
|
|
74
|
+
return;
|
|
75
|
+
}
|
|
76
|
+
log.info(tag, `Found ${queueLength} jobs in queue, checking for stale jobs...`);
|
|
77
|
+
// Scan up to 100 jobs to find stale ones
|
|
78
|
+
const MAX_SCAN = Math.min(queueLength, 100);
|
|
79
|
+
const tempJobs = [];
|
|
80
|
+
for (let i = 0; i < MAX_SCAN; i++) {
|
|
81
|
+
const job = await this.redisQueue.getWork(this.config.queueName, 1);
|
|
82
|
+
if (!job)
|
|
83
|
+
break;
|
|
84
|
+
scannedCount++;
|
|
85
|
+
const jobTimestamp = job.timestamp || 0;
|
|
86
|
+
const jobAge = jobTimestamp > 0 ? now - jobTimestamp : 0;
|
|
87
|
+
if (jobAge > MAX_JOB_AGE_MS) {
|
|
88
|
+
purgedCount++;
|
|
89
|
+
log.warn(tag, ` Purged stale job: ${job.type} (age: ${Math.round(jobAge / 1000)}s)`);
|
|
90
|
+
}
|
|
91
|
+
else {
|
|
92
|
+
// Keep fresh jobs - we'll re-queue them
|
|
93
|
+
tempJobs.push(job);
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
// Re-queue fresh jobs
|
|
97
|
+
for (const job of tempJobs) {
|
|
98
|
+
await this.redisQueue.createWork(this.config.queueName, job);
|
|
99
|
+
}
|
|
100
|
+
if (purgedCount > 0) {
|
|
101
|
+
log.info(tag, `✅ Purged ${purgedCount} stale jobs (scanned ${scannedCount})`);
|
|
102
|
+
}
|
|
103
|
+
else {
|
|
104
|
+
log.info(tag, `✅ No stale jobs found (scanned ${scannedCount})`);
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
catch (error) {
|
|
108
|
+
log.error(tag, '⚠️ Error during startup purge (non-fatal):', error);
|
|
109
|
+
// Don't throw - startup purge failure shouldn't prevent worker from starting
|
|
110
|
+
}
|
|
111
|
+
}
|
|
56
112
|
/**
|
|
57
113
|
* Stop the worker gracefully
|
|
58
114
|
*/
|
|
@@ -130,12 +186,30 @@ class RefreshWorker {
|
|
|
130
186
|
try {
|
|
131
187
|
const { type, key, params, retryCount = 0 } = job;
|
|
132
188
|
log.info(tag, `Processing ${type} for ${key} (retry: ${retryCount})`);
|
|
189
|
+
// FIX #1: Check job age and drop if too old (>1 hour)
|
|
190
|
+
const MAX_JOB_AGE_MS = 60 * 60 * 1000; // 1 hour
|
|
191
|
+
const jobAge = Date.now() - startTime;
|
|
192
|
+
const jobTimestamp = job.timestamp || 0;
|
|
193
|
+
if (jobTimestamp > 0) {
|
|
194
|
+
const actualJobAge = Date.now() - jobTimestamp;
|
|
195
|
+
if (actualJobAge > MAX_JOB_AGE_MS) {
|
|
196
|
+
log.error(tag, `⚠️ DROPPING STALE JOB: ${type} (age: ${Math.round(actualJobAge / 1000)}s, max: ${MAX_JOB_AGE_MS / 1000}s)`);
|
|
197
|
+
log.error(tag, ` Job key: ${key}`);
|
|
198
|
+
log.error(tag, ` Job has been removed from queue and will not be retried`);
|
|
199
|
+
return;
|
|
200
|
+
}
|
|
201
|
+
}
|
|
133
202
|
// Extract cache name from job type (e.g., "REFRESH_BALANCE" -> "balance")
|
|
134
203
|
const cacheName = type.replace('REFRESH_', '').toLowerCase();
|
|
135
204
|
// Get the appropriate cache instance
|
|
136
205
|
const cache = this.cacheRegistry.get(cacheName);
|
|
206
|
+
// FIX #2: Drop unsupported job types with clear error message
|
|
137
207
|
if (!cache) {
|
|
138
|
-
log.error(tag,
|
|
208
|
+
log.error(tag, `❌ DROPPING UNSUPPORTED JOB: ${type}`);
|
|
209
|
+
log.error(tag, ` Cache type "${cacheName}" is not registered in this worker`);
|
|
210
|
+
log.error(tag, ` Registered caches: ${Array.from(this.cacheRegistry.keys()).join(', ') || 'NONE'}`);
|
|
211
|
+
log.error(tag, ` Job key: ${key}`);
|
|
212
|
+
log.error(tag, ` Job has been removed from queue and will not be retried`);
|
|
139
213
|
return;
|
|
140
214
|
}
|
|
141
215
|
// Fetch fresh data using the cache's fetchFresh method
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@pioneer-platform/pioneer-cache",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.22.0",
|
|
4
4
|
"description": "Unified caching system for Pioneer platform with Redis backend",
|
|
5
5
|
"main": "./dist/index.js",
|
|
6
6
|
"types": "./dist/index.d.ts",
|
|
@@ -15,6 +15,8 @@
|
|
|
15
15
|
"license": "MIT",
|
|
16
16
|
"dependencies": {
|
|
17
17
|
"@pioneer-platform/loggerdog": "8.11.0",
|
|
18
|
+
"@pioneer-platform/pioneer-discovery": "8.44.0",
|
|
19
|
+
"@pioneer-platform/pioneer-caip": "9.21.0",
|
|
18
20
|
"@pioneer-platform/default-redis": "8.11.7",
|
|
19
21
|
"@pioneer-platform/redis-queue": "8.12.17"
|
|
20
22
|
},
|
|
@@ -149,6 +149,79 @@ export class CacheManager {
|
|
|
149
149
|
}
|
|
150
150
|
}
|
|
151
151
|
|
|
152
|
+
/**
|
|
153
|
+
* Purge unsupported job types from the queue
|
|
154
|
+
* Scans the queue and removes jobs for cache types that aren't enabled
|
|
155
|
+
*/
|
|
156
|
+
async purgeUnsupportedJobs(queueName: string = 'cache-refresh'): Promise<void> {
|
|
157
|
+
const tag = TAG + 'purgeUnsupportedJobs | ';
|
|
158
|
+
|
|
159
|
+
try {
|
|
160
|
+
const redisQueue = this.redisQueue;
|
|
161
|
+
let purgedCount = 0;
|
|
162
|
+
let scannedCount = 0;
|
|
163
|
+
|
|
164
|
+
log.info(tag, `Scanning queue for unsupported job types...`);
|
|
165
|
+
|
|
166
|
+
// Get enabled cache types
|
|
167
|
+
const enabledCaches = new Set<string>();
|
|
168
|
+
if (this.balanceCache) enabledCaches.add('balance');
|
|
169
|
+
if (this.priceCache) enabledCaches.add('price');
|
|
170
|
+
if (this.portfolioCache) enabledCaches.add('portfolio');
|
|
171
|
+
if (this.transactionCache) enabledCaches.add('transaction');
|
|
172
|
+
if (this.stakingCache) enabledCaches.add('staking');
|
|
173
|
+
if (this.zapperCache) enabledCaches.add('zapper');
|
|
174
|
+
|
|
175
|
+
log.info(tag, `Enabled caches: ${Array.from(enabledCaches).join(', ') || 'NONE'}`);
|
|
176
|
+
|
|
177
|
+
// Get queue length
|
|
178
|
+
const queueLength = await redisQueue.count(queueName);
|
|
179
|
+
|
|
180
|
+
if (queueLength === 0) {
|
|
181
|
+
log.info(tag, '✅ Queue is empty, no jobs to purge');
|
|
182
|
+
return;
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
log.info(tag, `Found ${queueLength} jobs in queue`);
|
|
186
|
+
|
|
187
|
+
// Scan all jobs
|
|
188
|
+
const tempJobs: any[] = [];
|
|
189
|
+
|
|
190
|
+
for (let i = 0; i < queueLength; i++) {
|
|
191
|
+
const job = await redisQueue.getWork(queueName, 1);
|
|
192
|
+
|
|
193
|
+
if (!job) break;
|
|
194
|
+
|
|
195
|
+
scannedCount++;
|
|
196
|
+
const jobType = job.type || '';
|
|
197
|
+
const cacheName = jobType.replace('REFRESH_', '').toLowerCase();
|
|
198
|
+
|
|
199
|
+
if (!enabledCaches.has(cacheName)) {
|
|
200
|
+
purgedCount++;
|
|
201
|
+
log.warn(tag, ` Purged unsupported job: ${jobType} (cache "${cacheName}" not enabled)`);
|
|
202
|
+
} else {
|
|
203
|
+
// Keep supported jobs - we'll re-queue them
|
|
204
|
+
tempJobs.push(job);
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
// Re-queue supported jobs
|
|
209
|
+
for (const job of tempJobs) {
|
|
210
|
+
await redisQueue.createWork(queueName, job);
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
if (purgedCount > 0) {
|
|
214
|
+
log.info(tag, `✅ Purged ${purgedCount} unsupported jobs (scanned ${scannedCount})`);
|
|
215
|
+
} else {
|
|
216
|
+
log.info(tag, `✅ No unsupported jobs found (scanned ${scannedCount})`);
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
} catch (error) {
|
|
220
|
+
log.error(tag, '⚠️ Error during unsupported job purge (non-fatal):', error);
|
|
221
|
+
// Don't throw - purge failure shouldn't prevent system from starting
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
|
|
152
225
|
/**
|
|
153
226
|
* Start background refresh workers for all caches
|
|
154
227
|
*/
|
|
@@ -158,6 +231,9 @@ export class CacheManager {
|
|
|
158
231
|
try {
|
|
159
232
|
log.info(tag, 'Starting refresh workers...');
|
|
160
233
|
|
|
234
|
+
// FIX: Purge unsupported job types before starting workers
|
|
235
|
+
await this.purgeUnsupportedJobs('cache-refresh');
|
|
236
|
+
|
|
161
237
|
// Create cache registry for workers
|
|
162
238
|
const cacheRegistry = new Map<string, BaseCache<any>>();
|
|
163
239
|
|
|
@@ -8,6 +8,8 @@
|
|
|
8
8
|
import { BaseCache } from '../core/base-cache';
|
|
9
9
|
import type { CacheConfig } from '../types';
|
|
10
10
|
import crypto from 'crypto';
|
|
11
|
+
import { assetData } from '@pioneer-platform/pioneer-discovery';
|
|
12
|
+
import { caipToNetworkId } from '@pioneer-platform/pioneer-caip';
|
|
11
13
|
|
|
12
14
|
const log = require('@pioneer-platform/loggerdog')();
|
|
13
15
|
|
|
@@ -163,8 +165,7 @@ export class BalanceCache extends BaseCache<BalanceData> {
|
|
|
163
165
|
const now = Date.now();
|
|
164
166
|
|
|
165
167
|
// Get asset metadata even for zero balances
|
|
166
|
-
const
|
|
167
|
-
const assetInfo = assetData[caip.toUpperCase()] || assetData[caip.toLowerCase()] || {};
|
|
168
|
+
const assetInfo = (assetData as any)[caip.toUpperCase()] || (assetData as any)[caip.toLowerCase()] || {};
|
|
168
169
|
|
|
169
170
|
return {
|
|
170
171
|
caip,
|
|
@@ -179,10 +180,7 @@ export class BalanceCache extends BaseCache<BalanceData> {
|
|
|
179
180
|
}
|
|
180
181
|
|
|
181
182
|
// Get asset metadata
|
|
182
|
-
const
|
|
183
|
-
const { caipToNetworkId } = require('@pioneer-platform/pioneer-caip');
|
|
184
|
-
|
|
185
|
-
const assetInfo = assetData[caip.toUpperCase()] || assetData[caip.toLowerCase()] || {};
|
|
183
|
+
const assetInfo = (assetData as any)[caip.toUpperCase()] || (assetData as any)[caip.toLowerCase()] || {};
|
|
186
184
|
const networkId = caipToNetworkId(caip);
|
|
187
185
|
|
|
188
186
|
// Use actual node URL if available, otherwise construct generic description
|
|
@@ -241,7 +239,6 @@ export class BalanceCache extends BaseCache<BalanceData> {
|
|
|
241
239
|
|
|
242
240
|
try {
|
|
243
241
|
const { caip, pubkey } = params;
|
|
244
|
-
const { caipToNetworkId } = require('@pioneer-platform/pioneer-caip');
|
|
245
242
|
const networkId = caipToNetworkId(caip);
|
|
246
243
|
|
|
247
244
|
// Try legacy format: cache:balance:pubkey:asset
|
|
@@ -288,8 +285,15 @@ export class BalanceCache extends BaseCache<BalanceData> {
|
|
|
288
285
|
*
|
|
289
286
|
* CRITICAL FIX: When waitForFresh=true (forceRefresh), bypass cache entirely
|
|
290
287
|
* and fetch fresh blockchain data for ALL items, not just cache misses
|
|
288
|
+
*
|
|
289
|
+
* PERFORMANCE FIX: When skipAsyncRefresh=true, disable background refresh jobs
|
|
290
|
+
* to prevent 50+ individual refresh jobs from being queued
|
|
291
291
|
*/
|
|
292
|
-
async getBatchBalances(
|
|
292
|
+
async getBatchBalances(
|
|
293
|
+
items: Array<{ caip: string; pubkey: string }>,
|
|
294
|
+
waitForFresh?: boolean,
|
|
295
|
+
skipAsyncRefresh?: boolean
|
|
296
|
+
): Promise<BalanceData[]> {
|
|
293
297
|
const tag = this.TAG + 'getBatchBalances | ';
|
|
294
298
|
const startTime = Date.now();
|
|
295
299
|
|
|
@@ -421,11 +425,15 @@ export class BalanceCache extends BaseCache<BalanceData> {
|
|
|
421
425
|
|
|
422
426
|
await Promise.all(fetchPromises);
|
|
423
427
|
log.info(tag, `Fetched ${missedItems.length} misses in ${Date.now() - fetchStart}ms`);
|
|
424
|
-
} else {
|
|
425
|
-
// Non-blocking: trigger background refresh for misses
|
|
428
|
+
} else if (!skipAsyncRefresh) {
|
|
429
|
+
// Non-blocking: trigger background refresh for misses (unless disabled)
|
|
430
|
+
log.info(tag, `📋 Queueing ${missedItems.length} background refresh jobs for cache misses`);
|
|
426
431
|
missedItems.forEach(item => {
|
|
427
432
|
this.triggerAsyncRefresh({ caip: item.caip, pubkey: item.pubkey }, 'high');
|
|
428
433
|
});
|
|
434
|
+
} else {
|
|
435
|
+
// Background refresh disabled for performance
|
|
436
|
+
log.info(tag, `⚡ PERFORMANCE MODE: Skipping ${missedItems.length} background refresh jobs (skipAsyncRefresh=true)`);
|
|
429
437
|
}
|
|
430
438
|
}
|
|
431
439
|
|
|
@@ -69,12 +69,79 @@ export class RefreshWorker {
|
|
|
69
69
|
log.info(tag, `🚀 Starting refresh worker for queue: ${this.config.queueName}`);
|
|
70
70
|
log.info(tag, `Registered caches: ${Array.from(this.cacheRegistry.keys()).join(', ')}`);
|
|
71
71
|
|
|
72
|
+
// FIX #3: Purge stale jobs at startup
|
|
73
|
+
await this.purgeStaleJobsOnStartup();
|
|
74
|
+
|
|
72
75
|
this.isRunning = true;
|
|
73
76
|
this.poll();
|
|
74
77
|
|
|
75
78
|
log.info(tag, '✅ Refresh worker started successfully');
|
|
76
79
|
}
|
|
77
80
|
|
|
81
|
+
/**
|
|
82
|
+
* Purge stale jobs from queue on startup
|
|
83
|
+
* Scans queue and removes jobs older than 1 hour
|
|
84
|
+
*/
|
|
85
|
+
private async purgeStaleJobsOnStartup(): Promise<void> {
|
|
86
|
+
const tag = TAG + 'purgeStaleJobsOnStartup | ';
|
|
87
|
+
|
|
88
|
+
try {
|
|
89
|
+
const MAX_JOB_AGE_MS = 60 * 60 * 1000; // 1 hour
|
|
90
|
+
const now = Date.now();
|
|
91
|
+
let purgedCount = 0;
|
|
92
|
+
let scannedCount = 0;
|
|
93
|
+
|
|
94
|
+
log.info(tag, `Scanning queue for stale jobs (age > ${MAX_JOB_AGE_MS / 1000}s)...`);
|
|
95
|
+
|
|
96
|
+
// Get queue length
|
|
97
|
+
const queueLength = await this.redisQueue.count(this.config.queueName);
|
|
98
|
+
|
|
99
|
+
if (queueLength === 0) {
|
|
100
|
+
log.info(tag, '✅ Queue is empty, no stale jobs to purge');
|
|
101
|
+
return;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
log.info(tag, `Found ${queueLength} jobs in queue, checking for stale jobs...`);
|
|
105
|
+
|
|
106
|
+
// Scan up to 100 jobs to find stale ones
|
|
107
|
+
const MAX_SCAN = Math.min(queueLength, 100);
|
|
108
|
+
const tempJobs: RefreshJob[] = [];
|
|
109
|
+
|
|
110
|
+
for (let i = 0; i < MAX_SCAN; i++) {
|
|
111
|
+
const job = await this.redisQueue.getWork(this.config.queueName, 1);
|
|
112
|
+
|
|
113
|
+
if (!job) break;
|
|
114
|
+
|
|
115
|
+
scannedCount++;
|
|
116
|
+
const jobTimestamp = (job as any).timestamp || 0;
|
|
117
|
+
const jobAge = jobTimestamp > 0 ? now - jobTimestamp : 0;
|
|
118
|
+
|
|
119
|
+
if (jobAge > MAX_JOB_AGE_MS) {
|
|
120
|
+
purgedCount++;
|
|
121
|
+
log.warn(tag, ` Purged stale job: ${job.type} (age: ${Math.round(jobAge / 1000)}s)`);
|
|
122
|
+
} else {
|
|
123
|
+
// Keep fresh jobs - we'll re-queue them
|
|
124
|
+
tempJobs.push(job);
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
// Re-queue fresh jobs
|
|
129
|
+
for (const job of tempJobs) {
|
|
130
|
+
await this.redisQueue.createWork(this.config.queueName, job);
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
if (purgedCount > 0) {
|
|
134
|
+
log.info(tag, `✅ Purged ${purgedCount} stale jobs (scanned ${scannedCount})`);
|
|
135
|
+
} else {
|
|
136
|
+
log.info(tag, `✅ No stale jobs found (scanned ${scannedCount})`);
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
} catch (error) {
|
|
140
|
+
log.error(tag, '⚠️ Error during startup purge (non-fatal):', error);
|
|
141
|
+
// Don't throw - startup purge failure shouldn't prevent worker from starting
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
|
|
78
145
|
/**
|
|
79
146
|
* Stop the worker gracefully
|
|
80
147
|
*/
|
|
@@ -166,14 +233,34 @@ export class RefreshWorker {
|
|
|
166
233
|
|
|
167
234
|
log.info(tag, `Processing ${type} for ${key} (retry: ${retryCount})`);
|
|
168
235
|
|
|
236
|
+
// FIX #1: Check job age and drop if too old (>1 hour)
|
|
237
|
+
const MAX_JOB_AGE_MS = 60 * 60 * 1000; // 1 hour
|
|
238
|
+
const jobAge = Date.now() - startTime;
|
|
239
|
+
const jobTimestamp = (job as any).timestamp || 0;
|
|
240
|
+
|
|
241
|
+
if (jobTimestamp > 0) {
|
|
242
|
+
const actualJobAge = Date.now() - jobTimestamp;
|
|
243
|
+
if (actualJobAge > MAX_JOB_AGE_MS) {
|
|
244
|
+
log.error(tag, `⚠️ DROPPING STALE JOB: ${type} (age: ${Math.round(actualJobAge / 1000)}s, max: ${MAX_JOB_AGE_MS / 1000}s)`);
|
|
245
|
+
log.error(tag, ` Job key: ${key}`);
|
|
246
|
+
log.error(tag, ` Job has been removed from queue and will not be retried`);
|
|
247
|
+
return;
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
|
|
169
251
|
// Extract cache name from job type (e.g., "REFRESH_BALANCE" -> "balance")
|
|
170
252
|
const cacheName = type.replace('REFRESH_', '').toLowerCase();
|
|
171
253
|
|
|
172
254
|
// Get the appropriate cache instance
|
|
173
255
|
const cache = this.cacheRegistry.get(cacheName);
|
|
174
256
|
|
|
257
|
+
// FIX #2: Drop unsupported job types with clear error message
|
|
175
258
|
if (!cache) {
|
|
176
|
-
log.error(tag,
|
|
259
|
+
log.error(tag, `❌ DROPPING UNSUPPORTED JOB: ${type}`);
|
|
260
|
+
log.error(tag, ` Cache type "${cacheName}" is not registered in this worker`);
|
|
261
|
+
log.error(tag, ` Registered caches: ${Array.from(this.cacheRegistry.keys()).join(', ') || 'NONE'}`);
|
|
262
|
+
log.error(tag, ` Job key: ${key}`);
|
|
263
|
+
log.error(tag, ` Job has been removed from queue and will not be retried`);
|
|
177
264
|
return;
|
|
178
265
|
}
|
|
179
266
|
|
package/tsconfig.json
CHANGED
|
@@ -2,7 +2,10 @@
|
|
|
2
2
|
"compilerOptions": {
|
|
3
3
|
"target": "ES2020",
|
|
4
4
|
"module": "commonjs",
|
|
5
|
-
"lib": [
|
|
5
|
+
"lib": [
|
|
6
|
+
"ES2020",
|
|
7
|
+
"DOM"
|
|
8
|
+
],
|
|
6
9
|
"declaration": true,
|
|
7
10
|
"outDir": "./dist",
|
|
8
11
|
"rootDir": "./src",
|
|
@@ -10,8 +13,15 @@
|
|
|
10
13
|
"esModuleInterop": true,
|
|
11
14
|
"forceConsistentCasingInFileNames": true,
|
|
12
15
|
"resolveJsonModule": true,
|
|
13
|
-
"moduleResolution": "node"
|
|
16
|
+
"moduleResolution": "node",
|
|
17
|
+
"skipLibCheck": true
|
|
14
18
|
},
|
|
15
|
-
"include": [
|
|
16
|
-
|
|
19
|
+
"include": [
|
|
20
|
+
"src/**/*"
|
|
21
|
+
],
|
|
22
|
+
"exclude": [
|
|
23
|
+
"node_modules",
|
|
24
|
+
"dist",
|
|
25
|
+
"**/*.test.ts"
|
|
26
|
+
]
|
|
17
27
|
}
|