@pioneer-platform/pioneer-cache 1.21.0 → 1.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -1,5 +1,23 @@
1
1
  # @pioneer-platform/pioneer-cache
2
2
 
3
+ ## 1.22.0
4
+
5
+ ### Minor Changes
6
+
7
+ - chore: feat(e2e): add blue API support to swap roundtrip tests
8
+
9
+ ### Patch Changes
10
+
11
+ - Updated dependencies
12
+ - @pioneer-platform/pioneer-discovery@8.44.0
13
+
14
+ ## 1.21.1
15
+
16
+ ### Patch Changes
17
+
18
+ - Updated dependencies
19
+ - @pioneer-platform/pioneer-discovery@8.43.0
20
+
3
21
  ## 1.21.0
4
22
 
5
23
  ### Minor Changes
@@ -45,6 +45,11 @@ export declare class CacheManager {
45
45
  * Used in local dev to clear stale jobs on startup
46
46
  */
47
47
  purgeQueue(queueName?: string): Promise<void>;
48
+ /**
49
+ * Purge unsupported job types from the queue
50
+ * Scans the queue and removes jobs for cache types that aren't enabled
51
+ */
52
+ purgeUnsupportedJobs(queueName?: string): Promise<void>;
48
53
  /**
49
54
  * Start background refresh workers for all caches
50
55
  */
@@ -104,6 +104,73 @@ class CacheManager {
104
104
  throw error;
105
105
  }
106
106
  }
107
+ /**
108
+ * Purge unsupported job types from the queue
109
+ * Scans the queue and removes jobs for cache types that aren't enabled
110
+ */
111
+ async purgeUnsupportedJobs(queueName = 'cache-refresh') {
112
+ const tag = TAG + 'purgeUnsupportedJobs | ';
113
+ try {
114
+ const redisQueue = this.redisQueue;
115
+ let purgedCount = 0;
116
+ let scannedCount = 0;
117
+ log.info(tag, `Scanning queue for unsupported job types...`);
118
+ // Get enabled cache types
119
+ const enabledCaches = new Set();
120
+ if (this.balanceCache)
121
+ enabledCaches.add('balance');
122
+ if (this.priceCache)
123
+ enabledCaches.add('price');
124
+ if (this.portfolioCache)
125
+ enabledCaches.add('portfolio');
126
+ if (this.transactionCache)
127
+ enabledCaches.add('transaction');
128
+ if (this.stakingCache)
129
+ enabledCaches.add('staking');
130
+ if (this.zapperCache)
131
+ enabledCaches.add('zapper');
132
+ log.info(tag, `Enabled caches: ${Array.from(enabledCaches).join(', ') || 'NONE'}`);
133
+ // Get queue length
134
+ const queueLength = await redisQueue.count(queueName);
135
+ if (queueLength === 0) {
136
+ log.info(tag, '✅ Queue is empty, no jobs to purge');
137
+ return;
138
+ }
139
+ log.info(tag, `Found ${queueLength} jobs in queue`);
140
+ // Scan all jobs
141
+ const tempJobs = [];
142
+ for (let i = 0; i < queueLength; i++) {
143
+ const job = await redisQueue.getWork(queueName, 1);
144
+ if (!job)
145
+ break;
146
+ scannedCount++;
147
+ const jobType = job.type || '';
148
+ const cacheName = jobType.replace('REFRESH_', '').toLowerCase();
149
+ if (!enabledCaches.has(cacheName)) {
150
+ purgedCount++;
151
+ log.warn(tag, ` Purged unsupported job: ${jobType} (cache "${cacheName}" not enabled)`);
152
+ }
153
+ else {
154
+ // Keep supported jobs - we'll re-queue them
155
+ tempJobs.push(job);
156
+ }
157
+ }
158
+ // Re-queue supported jobs
159
+ for (const job of tempJobs) {
160
+ await redisQueue.createWork(queueName, job);
161
+ }
162
+ if (purgedCount > 0) {
163
+ log.info(tag, `✅ Purged ${purgedCount} unsupported jobs (scanned ${scannedCount})`);
164
+ }
165
+ else {
166
+ log.info(tag, `✅ No unsupported jobs found (scanned ${scannedCount})`);
167
+ }
168
+ }
169
+ catch (error) {
170
+ log.error(tag, '⚠️ Error during unsupported job purge (non-fatal):', error);
171
+ // Don't throw - purge failure shouldn't prevent system from starting
172
+ }
173
+ }
107
174
  /**
108
175
  * Start background refresh workers for all caches
109
176
  */
@@ -111,6 +178,8 @@ class CacheManager {
111
178
  const tag = TAG + 'startWorkers | ';
112
179
  try {
113
180
  log.info(tag, 'Starting refresh workers...');
181
+ // FIX: Purge unsupported job types before starting workers
182
+ await this.purgeUnsupportedJobs('cache-refresh');
114
183
  // Create cache registry for workers
115
184
  const cacheRegistry = new Map();
116
185
  if (this.balanceCache) {
@@ -58,9 +58,12 @@ export declare class BalanceCache extends BaseCache<BalanceData> {
58
58
  *
59
59
  * CRITICAL FIX: When waitForFresh=true (forceRefresh), bypass cache entirely
60
60
  * and fetch fresh blockchain data for ALL items, not just cache misses
61
+ *
62
+ * PERFORMANCE FIX: When skipAsyncRefresh=true, disable background refresh jobs
63
+ * to prevent 50+ individual refresh jobs from being queued
61
64
  */
62
65
  getBatchBalances(items: Array<{
63
66
  caip: string;
64
67
  pubkey: string;
65
- }>, waitForFresh?: boolean): Promise<BalanceData[]>;
68
+ }>, waitForFresh?: boolean, skipAsyncRefresh?: boolean): Promise<BalanceData[]>;
66
69
  }
@@ -12,6 +12,8 @@ Object.defineProperty(exports, "__esModule", { value: true });
12
12
  exports.BalanceCache = void 0;
13
13
  const base_cache_1 = require("../core/base-cache");
14
14
  const crypto_1 = __importDefault(require("crypto"));
15
+ const pioneer_discovery_1 = require("@pioneer-platform/pioneer-discovery");
16
+ const pioneer_caip_1 = require("@pioneer-platform/pioneer-caip");
15
17
  const log = require('@pioneer-platform/loggerdog')();
16
18
  /**
17
19
  * Hash pubkey/xpub for privacy-protecting cache keys
@@ -121,8 +123,7 @@ class BalanceCache extends base_cache_1.BaseCache {
121
123
  // Otherwise, this is a legitimate zero balance (balanceInfo exists but balance is falsy)
122
124
  const now = Date.now();
123
125
  // Get asset metadata even for zero balances
124
- const assetData = require('@pioneer-platform/pioneer-discovery').assetData;
125
- const assetInfo = assetData[caip.toUpperCase()] || assetData[caip.toLowerCase()] || {};
126
+ const assetInfo = pioneer_discovery_1.assetData[caip.toUpperCase()] || pioneer_discovery_1.assetData[caip.toLowerCase()] || {};
126
127
  return {
127
128
  caip,
128
129
  pubkey,
@@ -135,10 +136,8 @@ class BalanceCache extends base_cache_1.BaseCache {
135
136
  };
136
137
  }
137
138
  // Get asset metadata
138
- const assetData = require('@pioneer-platform/pioneer-discovery').assetData;
139
- const { caipToNetworkId } = require('@pioneer-platform/pioneer-caip');
140
- const assetInfo = assetData[caip.toUpperCase()] || assetData[caip.toLowerCase()] || {};
141
- const networkId = caipToNetworkId(caip);
139
+ const assetInfo = pioneer_discovery_1.assetData[caip.toUpperCase()] || pioneer_discovery_1.assetData[caip.toLowerCase()] || {};
140
+ const networkId = (0, pioneer_caip_1.caipToNetworkId)(caip);
142
141
  // Use actual node URL if available, otherwise construct generic description
143
142
  let dataSource = 'Unknown';
144
143
  if (balanceInfo.nodeUrl) {
@@ -196,8 +195,7 @@ class BalanceCache extends base_cache_1.BaseCache {
196
195
  const tag = this.TAG + 'getLegacyCached | ';
197
196
  try {
198
197
  const { caip, pubkey } = params;
199
- const { caipToNetworkId } = require('@pioneer-platform/pioneer-caip');
200
- const networkId = caipToNetworkId(caip);
198
+ const networkId = (0, pioneer_caip_1.caipToNetworkId)(caip);
201
199
  // Try legacy format: cache:balance:pubkey:asset
202
200
  const legacyKey = `cache:balance:${pubkey}:${networkId}`;
203
201
  const legacyData = await this.redis.get(legacyKey);
@@ -238,8 +236,11 @@ class BalanceCache extends base_cache_1.BaseCache {
238
236
  *
239
237
  * CRITICAL FIX: When waitForFresh=true (forceRefresh), bypass cache entirely
240
238
  * and fetch fresh blockchain data for ALL items, not just cache misses
239
+ *
240
+ * PERFORMANCE FIX: When skipAsyncRefresh=true, disable background refresh jobs
241
+ * to prevent 50+ individual refresh jobs from being queued
241
242
  */
242
- async getBatchBalances(items, waitForFresh) {
243
+ async getBatchBalances(items, waitForFresh, skipAsyncRefresh) {
243
244
  const tag = this.TAG + 'getBatchBalances | ';
244
245
  const startTime = Date.now();
245
246
  try {
@@ -356,12 +357,17 @@ class BalanceCache extends base_cache_1.BaseCache {
356
357
  await Promise.all(fetchPromises);
357
358
  log.info(tag, `Fetched ${missedItems.length} misses in ${Date.now() - fetchStart}ms`);
358
359
  }
359
- else {
360
- // Non-blocking: trigger background refresh for misses
360
+ else if (!skipAsyncRefresh) {
361
+ // Non-blocking: trigger background refresh for misses (unless disabled)
362
+ log.info(tag, `📋 Queueing ${missedItems.length} background refresh jobs for cache misses`);
361
363
  missedItems.forEach(item => {
362
364
  this.triggerAsyncRefresh({ caip: item.caip, pubkey: item.pubkey }, 'high');
363
365
  });
364
366
  }
367
+ else {
368
+ // Background refresh disabled for performance
369
+ log.info(tag, `⚡ PERFORMANCE MODE: Skipping ${missedItems.length} background refresh jobs (skipAsyncRefresh=true)`);
370
+ }
365
371
  }
366
372
  return results;
367
373
  }
@@ -29,6 +29,11 @@ export declare class RefreshWorker {
29
29
  * Start processing jobs from the queue
30
30
  */
31
31
  start(): Promise<void>;
32
+ /**
33
+ * Purge stale jobs from queue on startup
34
+ * Scans queue and removes jobs older than 1 hour
35
+ */
36
+ private purgeStaleJobsOnStartup;
32
37
  /**
33
38
  * Stop the worker gracefully
34
39
  */
@@ -49,10 +49,66 @@ class RefreshWorker {
49
49
  }
50
50
  log.info(tag, `🚀 Starting refresh worker for queue: ${this.config.queueName}`);
51
51
  log.info(tag, `Registered caches: ${Array.from(this.cacheRegistry.keys()).join(', ')}`);
52
+ // FIX #3: Purge stale jobs at startup
53
+ await this.purgeStaleJobsOnStartup();
52
54
  this.isRunning = true;
53
55
  this.poll();
54
56
  log.info(tag, '✅ Refresh worker started successfully');
55
57
  }
58
+ /**
59
+ * Purge stale jobs from queue on startup
60
+ * Scans queue and removes jobs older than 1 hour
61
+ */
62
+ async purgeStaleJobsOnStartup() {
63
+ const tag = TAG + 'purgeStaleJobsOnStartup | ';
64
+ try {
65
+ const MAX_JOB_AGE_MS = 60 * 60 * 1000; // 1 hour
66
+ const now = Date.now();
67
+ let purgedCount = 0;
68
+ let scannedCount = 0;
69
+ log.info(tag, `Scanning queue for stale jobs (age > ${MAX_JOB_AGE_MS / 1000}s)...`);
70
+ // Get queue length
71
+ const queueLength = await this.redisQueue.count(this.config.queueName);
72
+ if (queueLength === 0) {
73
+ log.info(tag, '✅ Queue is empty, no stale jobs to purge');
74
+ return;
75
+ }
76
+ log.info(tag, `Found ${queueLength} jobs in queue, checking for stale jobs...`);
77
+ // Scan up to 100 jobs to find stale ones
78
+ const MAX_SCAN = Math.min(queueLength, 100);
79
+ const tempJobs = [];
80
+ for (let i = 0; i < MAX_SCAN; i++) {
81
+ const job = await this.redisQueue.getWork(this.config.queueName, 1);
82
+ if (!job)
83
+ break;
84
+ scannedCount++;
85
+ const jobTimestamp = job.timestamp || 0;
86
+ const jobAge = jobTimestamp > 0 ? now - jobTimestamp : 0;
87
+ if (jobAge > MAX_JOB_AGE_MS) {
88
+ purgedCount++;
89
+ log.warn(tag, ` Purged stale job: ${job.type} (age: ${Math.round(jobAge / 1000)}s)`);
90
+ }
91
+ else {
92
+ // Keep fresh jobs - we'll re-queue them
93
+ tempJobs.push(job);
94
+ }
95
+ }
96
+ // Re-queue fresh jobs
97
+ for (const job of tempJobs) {
98
+ await this.redisQueue.createWork(this.config.queueName, job);
99
+ }
100
+ if (purgedCount > 0) {
101
+ log.info(tag, `✅ Purged ${purgedCount} stale jobs (scanned ${scannedCount})`);
102
+ }
103
+ else {
104
+ log.info(tag, `✅ No stale jobs found (scanned ${scannedCount})`);
105
+ }
106
+ }
107
+ catch (error) {
108
+ log.error(tag, '⚠️ Error during startup purge (non-fatal):', error);
109
+ // Don't throw - startup purge failure shouldn't prevent worker from starting
110
+ }
111
+ }
56
112
  /**
57
113
  * Stop the worker gracefully
58
114
  */
@@ -130,12 +186,30 @@ class RefreshWorker {
130
186
  try {
131
187
  const { type, key, params, retryCount = 0 } = job;
132
188
  log.info(tag, `Processing ${type} for ${key} (retry: ${retryCount})`);
189
+ // FIX #1: Check job age and drop if too old (>1 hour)
190
+ const MAX_JOB_AGE_MS = 60 * 60 * 1000; // 1 hour
191
+ const jobAge = Date.now() - startTime;
192
+ const jobTimestamp = job.timestamp || 0;
193
+ if (jobTimestamp > 0) {
194
+ const actualJobAge = Date.now() - jobTimestamp;
195
+ if (actualJobAge > MAX_JOB_AGE_MS) {
196
+ log.error(tag, `⚠️ DROPPING STALE JOB: ${type} (age: ${Math.round(actualJobAge / 1000)}s, max: ${MAX_JOB_AGE_MS / 1000}s)`);
197
+ log.error(tag, ` Job key: ${key}`);
198
+ log.error(tag, ` Job has been removed from queue and will not be retried`);
199
+ return;
200
+ }
201
+ }
133
202
  // Extract cache name from job type (e.g., "REFRESH_BALANCE" -> "balance")
134
203
  const cacheName = type.replace('REFRESH_', '').toLowerCase();
135
204
  // Get the appropriate cache instance
136
205
  const cache = this.cacheRegistry.get(cacheName);
206
+ // FIX #2: Drop unsupported job types with clear error message
137
207
  if (!cache) {
138
- log.error(tag, `No cache registered for type: ${type}`);
208
+ log.error(tag, `❌ DROPPING UNSUPPORTED JOB: ${type}`);
209
+ log.error(tag, ` Cache type "${cacheName}" is not registered in this worker`);
210
+ log.error(tag, ` Registered caches: ${Array.from(this.cacheRegistry.keys()).join(', ') || 'NONE'}`);
211
+ log.error(tag, ` Job key: ${key}`);
212
+ log.error(tag, ` Job has been removed from queue and will not be retried`);
139
213
  return;
140
214
  }
141
215
  // Fetch fresh data using the cache's fetchFresh method
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@pioneer-platform/pioneer-cache",
3
- "version": "1.21.0",
3
+ "version": "1.22.0",
4
4
  "description": "Unified caching system for Pioneer platform with Redis backend",
5
5
  "main": "./dist/index.js",
6
6
  "types": "./dist/index.d.ts",
@@ -15,6 +15,8 @@
15
15
  "license": "MIT",
16
16
  "dependencies": {
17
17
  "@pioneer-platform/loggerdog": "8.11.0",
18
+ "@pioneer-platform/pioneer-discovery": "8.44.0",
19
+ "@pioneer-platform/pioneer-caip": "9.21.0",
18
20
  "@pioneer-platform/default-redis": "8.11.7",
19
21
  "@pioneer-platform/redis-queue": "8.12.17"
20
22
  },
@@ -149,6 +149,79 @@ export class CacheManager {
149
149
  }
150
150
  }
151
151
 
152
+ /**
153
+ * Purge unsupported job types from the queue
154
+ * Scans the queue and removes jobs for cache types that aren't enabled
155
+ */
156
+ async purgeUnsupportedJobs(queueName: string = 'cache-refresh'): Promise<void> {
157
+ const tag = TAG + 'purgeUnsupportedJobs | ';
158
+
159
+ try {
160
+ const redisQueue = this.redisQueue;
161
+ let purgedCount = 0;
162
+ let scannedCount = 0;
163
+
164
+ log.info(tag, `Scanning queue for unsupported job types...`);
165
+
166
+ // Get enabled cache types
167
+ const enabledCaches = new Set<string>();
168
+ if (this.balanceCache) enabledCaches.add('balance');
169
+ if (this.priceCache) enabledCaches.add('price');
170
+ if (this.portfolioCache) enabledCaches.add('portfolio');
171
+ if (this.transactionCache) enabledCaches.add('transaction');
172
+ if (this.stakingCache) enabledCaches.add('staking');
173
+ if (this.zapperCache) enabledCaches.add('zapper');
174
+
175
+ log.info(tag, `Enabled caches: ${Array.from(enabledCaches).join(', ') || 'NONE'}`);
176
+
177
+ // Get queue length
178
+ const queueLength = await redisQueue.count(queueName);
179
+
180
+ if (queueLength === 0) {
181
+ log.info(tag, '✅ Queue is empty, no jobs to purge');
182
+ return;
183
+ }
184
+
185
+ log.info(tag, `Found ${queueLength} jobs in queue`);
186
+
187
+ // Scan all jobs
188
+ const tempJobs: any[] = [];
189
+
190
+ for (let i = 0; i < queueLength; i++) {
191
+ const job = await redisQueue.getWork(queueName, 1);
192
+
193
+ if (!job) break;
194
+
195
+ scannedCount++;
196
+ const jobType = job.type || '';
197
+ const cacheName = jobType.replace('REFRESH_', '').toLowerCase();
198
+
199
+ if (!enabledCaches.has(cacheName)) {
200
+ purgedCount++;
201
+ log.warn(tag, ` Purged unsupported job: ${jobType} (cache "${cacheName}" not enabled)`);
202
+ } else {
203
+ // Keep supported jobs - we'll re-queue them
204
+ tempJobs.push(job);
205
+ }
206
+ }
207
+
208
+ // Re-queue supported jobs
209
+ for (const job of tempJobs) {
210
+ await redisQueue.createWork(queueName, job);
211
+ }
212
+
213
+ if (purgedCount > 0) {
214
+ log.info(tag, `✅ Purged ${purgedCount} unsupported jobs (scanned ${scannedCount})`);
215
+ } else {
216
+ log.info(tag, `✅ No unsupported jobs found (scanned ${scannedCount})`);
217
+ }
218
+
219
+ } catch (error) {
220
+ log.error(tag, '⚠️ Error during unsupported job purge (non-fatal):', error);
221
+ // Don't throw - purge failure shouldn't prevent system from starting
222
+ }
223
+ }
224
+
152
225
  /**
153
226
  * Start background refresh workers for all caches
154
227
  */
@@ -158,6 +231,9 @@ export class CacheManager {
158
231
  try {
159
232
  log.info(tag, 'Starting refresh workers...');
160
233
 
234
+ // FIX: Purge unsupported job types before starting workers
235
+ await this.purgeUnsupportedJobs('cache-refresh');
236
+
161
237
  // Create cache registry for workers
162
238
  const cacheRegistry = new Map<string, BaseCache<any>>();
163
239
 
@@ -8,6 +8,8 @@
8
8
  import { BaseCache } from '../core/base-cache';
9
9
  import type { CacheConfig } from '../types';
10
10
  import crypto from 'crypto';
11
+ import { assetData } from '@pioneer-platform/pioneer-discovery';
12
+ import { caipToNetworkId } from '@pioneer-platform/pioneer-caip';
11
13
 
12
14
  const log = require('@pioneer-platform/loggerdog')();
13
15
 
@@ -163,8 +165,7 @@ export class BalanceCache extends BaseCache<BalanceData> {
163
165
  const now = Date.now();
164
166
 
165
167
  // Get asset metadata even for zero balances
166
- const assetData = require('@pioneer-platform/pioneer-discovery').assetData;
167
- const assetInfo = assetData[caip.toUpperCase()] || assetData[caip.toLowerCase()] || {};
168
+ const assetInfo = (assetData as any)[caip.toUpperCase()] || (assetData as any)[caip.toLowerCase()] || {};
168
169
 
169
170
  return {
170
171
  caip,
@@ -179,10 +180,7 @@ export class BalanceCache extends BaseCache<BalanceData> {
179
180
  }
180
181
 
181
182
  // Get asset metadata
182
- const assetData = require('@pioneer-platform/pioneer-discovery').assetData;
183
- const { caipToNetworkId } = require('@pioneer-platform/pioneer-caip');
184
-
185
- const assetInfo = assetData[caip.toUpperCase()] || assetData[caip.toLowerCase()] || {};
183
+ const assetInfo = (assetData as any)[caip.toUpperCase()] || (assetData as any)[caip.toLowerCase()] || {};
186
184
  const networkId = caipToNetworkId(caip);
187
185
 
188
186
  // Use actual node URL if available, otherwise construct generic description
@@ -241,7 +239,6 @@ export class BalanceCache extends BaseCache<BalanceData> {
241
239
 
242
240
  try {
243
241
  const { caip, pubkey } = params;
244
- const { caipToNetworkId } = require('@pioneer-platform/pioneer-caip');
245
242
  const networkId = caipToNetworkId(caip);
246
243
 
247
244
  // Try legacy format: cache:balance:pubkey:asset
@@ -288,8 +285,15 @@ export class BalanceCache extends BaseCache<BalanceData> {
288
285
  *
289
286
  * CRITICAL FIX: When waitForFresh=true (forceRefresh), bypass cache entirely
290
287
  * and fetch fresh blockchain data for ALL items, not just cache misses
288
+ *
289
+ * PERFORMANCE FIX: When skipAsyncRefresh=true, disable background refresh jobs
290
+ * to prevent 50+ individual refresh jobs from being queued
291
291
  */
292
- async getBatchBalances(items: Array<{ caip: string; pubkey: string }>, waitForFresh?: boolean): Promise<BalanceData[]> {
292
+ async getBatchBalances(
293
+ items: Array<{ caip: string; pubkey: string }>,
294
+ waitForFresh?: boolean,
295
+ skipAsyncRefresh?: boolean
296
+ ): Promise<BalanceData[]> {
293
297
  const tag = this.TAG + 'getBatchBalances | ';
294
298
  const startTime = Date.now();
295
299
 
@@ -421,11 +425,15 @@ export class BalanceCache extends BaseCache<BalanceData> {
421
425
 
422
426
  await Promise.all(fetchPromises);
423
427
  log.info(tag, `Fetched ${missedItems.length} misses in ${Date.now() - fetchStart}ms`);
424
- } else {
425
- // Non-blocking: trigger background refresh for misses
428
+ } else if (!skipAsyncRefresh) {
429
+ // Non-blocking: trigger background refresh for misses (unless disabled)
430
+ log.info(tag, `📋 Queueing ${missedItems.length} background refresh jobs for cache misses`);
426
431
  missedItems.forEach(item => {
427
432
  this.triggerAsyncRefresh({ caip: item.caip, pubkey: item.pubkey }, 'high');
428
433
  });
434
+ } else {
435
+ // Background refresh disabled for performance
436
+ log.info(tag, `⚡ PERFORMANCE MODE: Skipping ${missedItems.length} background refresh jobs (skipAsyncRefresh=true)`);
429
437
  }
430
438
  }
431
439
 
@@ -69,12 +69,79 @@ export class RefreshWorker {
69
69
  log.info(tag, `🚀 Starting refresh worker for queue: ${this.config.queueName}`);
70
70
  log.info(tag, `Registered caches: ${Array.from(this.cacheRegistry.keys()).join(', ')}`);
71
71
 
72
+ // FIX #3: Purge stale jobs at startup
73
+ await this.purgeStaleJobsOnStartup();
74
+
72
75
  this.isRunning = true;
73
76
  this.poll();
74
77
 
75
78
  log.info(tag, '✅ Refresh worker started successfully');
76
79
  }
77
80
 
81
+ /**
82
+ * Purge stale jobs from queue on startup
83
+ * Scans queue and removes jobs older than 1 hour
84
+ */
85
+ private async purgeStaleJobsOnStartup(): Promise<void> {
86
+ const tag = TAG + 'purgeStaleJobsOnStartup | ';
87
+
88
+ try {
89
+ const MAX_JOB_AGE_MS = 60 * 60 * 1000; // 1 hour
90
+ const now = Date.now();
91
+ let purgedCount = 0;
92
+ let scannedCount = 0;
93
+
94
+ log.info(tag, `Scanning queue for stale jobs (age > ${MAX_JOB_AGE_MS / 1000}s)...`);
95
+
96
+ // Get queue length
97
+ const queueLength = await this.redisQueue.count(this.config.queueName);
98
+
99
+ if (queueLength === 0) {
100
+ log.info(tag, '✅ Queue is empty, no stale jobs to purge');
101
+ return;
102
+ }
103
+
104
+ log.info(tag, `Found ${queueLength} jobs in queue, checking for stale jobs...`);
105
+
106
+ // Scan up to 100 jobs to find stale ones
107
+ const MAX_SCAN = Math.min(queueLength, 100);
108
+ const tempJobs: RefreshJob[] = [];
109
+
110
+ for (let i = 0; i < MAX_SCAN; i++) {
111
+ const job = await this.redisQueue.getWork(this.config.queueName, 1);
112
+
113
+ if (!job) break;
114
+
115
+ scannedCount++;
116
+ const jobTimestamp = (job as any).timestamp || 0;
117
+ const jobAge = jobTimestamp > 0 ? now - jobTimestamp : 0;
118
+
119
+ if (jobAge > MAX_JOB_AGE_MS) {
120
+ purgedCount++;
121
+ log.warn(tag, ` Purged stale job: ${job.type} (age: ${Math.round(jobAge / 1000)}s)`);
122
+ } else {
123
+ // Keep fresh jobs - we'll re-queue them
124
+ tempJobs.push(job);
125
+ }
126
+ }
127
+
128
+ // Re-queue fresh jobs
129
+ for (const job of tempJobs) {
130
+ await this.redisQueue.createWork(this.config.queueName, job);
131
+ }
132
+
133
+ if (purgedCount > 0) {
134
+ log.info(tag, `✅ Purged ${purgedCount} stale jobs (scanned ${scannedCount})`);
135
+ } else {
136
+ log.info(tag, `✅ No stale jobs found (scanned ${scannedCount})`);
137
+ }
138
+
139
+ } catch (error) {
140
+ log.error(tag, '⚠️ Error during startup purge (non-fatal):', error);
141
+ // Don't throw - startup purge failure shouldn't prevent worker from starting
142
+ }
143
+ }
144
+
78
145
  /**
79
146
  * Stop the worker gracefully
80
147
  */
@@ -166,14 +233,34 @@ export class RefreshWorker {
166
233
 
167
234
  log.info(tag, `Processing ${type} for ${key} (retry: ${retryCount})`);
168
235
 
236
+ // FIX #1: Check job age and drop if too old (>1 hour)
237
+ const MAX_JOB_AGE_MS = 60 * 60 * 1000; // 1 hour
238
+ const jobAge = Date.now() - startTime;
239
+ const jobTimestamp = (job as any).timestamp || 0;
240
+
241
+ if (jobTimestamp > 0) {
242
+ const actualJobAge = Date.now() - jobTimestamp;
243
+ if (actualJobAge > MAX_JOB_AGE_MS) {
244
+ log.error(tag, `⚠️ DROPPING STALE JOB: ${type} (age: ${Math.round(actualJobAge / 1000)}s, max: ${MAX_JOB_AGE_MS / 1000}s)`);
245
+ log.error(tag, ` Job key: ${key}`);
246
+ log.error(tag, ` Job has been removed from queue and will not be retried`);
247
+ return;
248
+ }
249
+ }
250
+
169
251
  // Extract cache name from job type (e.g., "REFRESH_BALANCE" -> "balance")
170
252
  const cacheName = type.replace('REFRESH_', '').toLowerCase();
171
253
 
172
254
  // Get the appropriate cache instance
173
255
  const cache = this.cacheRegistry.get(cacheName);
174
256
 
257
+ // FIX #2: Drop unsupported job types with clear error message
175
258
  if (!cache) {
176
- log.error(tag, `No cache registered for type: ${type}`);
259
+ log.error(tag, `❌ DROPPING UNSUPPORTED JOB: ${type}`);
260
+ log.error(tag, ` Cache type "${cacheName}" is not registered in this worker`);
261
+ log.error(tag, ` Registered caches: ${Array.from(this.cacheRegistry.keys()).join(', ') || 'NONE'}`);
262
+ log.error(tag, ` Job key: ${key}`);
263
+ log.error(tag, ` Job has been removed from queue and will not be retried`);
177
264
  return;
178
265
  }
179
266
 
package/tsconfig.json CHANGED
@@ -2,7 +2,10 @@
2
2
  "compilerOptions": {
3
3
  "target": "ES2020",
4
4
  "module": "commonjs",
5
- "lib": ["ES2020", "DOM"],
5
+ "lib": [
6
+ "ES2020",
7
+ "DOM"
8
+ ],
6
9
  "declaration": true,
7
10
  "outDir": "./dist",
8
11
  "rootDir": "./src",
@@ -10,8 +13,15 @@
10
13
  "esModuleInterop": true,
11
14
  "forceConsistentCasingInFileNames": true,
12
15
  "resolveJsonModule": true,
13
- "moduleResolution": "node"
16
+ "moduleResolution": "node",
17
+ "skipLibCheck": true
14
18
  },
15
- "include": ["src/**/*"],
16
- "exclude": ["node_modules", "dist", "**/*.test.ts"]
19
+ "include": [
20
+ "src/**/*"
21
+ ],
22
+ "exclude": [
23
+ "node_modules",
24
+ "dist",
25
+ "**/*.test.ts"
26
+ ]
17
27
  }