@pioneer-platform/pioneer-cache 1.19.3 → 1.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,5 +1,5 @@
1
1
 
2
2
  
3
- > @pioneer-platform/pioneer-cache@1.19.3 build /Users/highlander/WebstormProjects/keepkey-stack/projects/pioneer/modules/pioneer/pioneer-cache
3
+ > @pioneer-platform/pioneer-cache@1.21.0 build /Users/highlander/WebstormProjects/keepkey-stack/projects/pioneer/modules/pioneer/pioneer-cache
4
4
  > tsc
5
5
 
package/CHANGELOG.md CHANGED
@@ -1,5 +1,35 @@
1
1
  # @pioneer-platform/pioneer-cache
2
2
 
3
+ ## 1.22.0
4
+
5
+ ### Minor Changes
6
+
7
+ - chore: feat(e2e): add blue API support to swap roundtrip tests
8
+
9
+ ### Patch Changes
10
+
11
+ - Updated dependencies
12
+ - @pioneer-platform/pioneer-discovery@8.44.0
13
+
14
+ ## 1.21.1
15
+
16
+ ### Patch Changes
17
+
18
+ - Updated dependencies
19
+ - @pioneer-platform/pioneer-discovery@8.43.0
20
+
21
+ ## 1.21.0
22
+
23
+ ### Minor Changes
24
+
25
+ - feat(monorepo): production release preparation with strict TypeScript and enhanced swap functionality
26
+
27
+ ## 1.20.0
28
+
29
+ ### Minor Changes
30
+
31
+ - feat(pioneer-client): add client-side host override for staging/blue testing
32
+
3
33
  ## 1.19.3
4
34
 
5
35
  ### Patch Changes
@@ -38,12 +38,18 @@ export declare class CacheManager {
38
38
  private stakingCache?;
39
39
  private zapperCache?;
40
40
  private workers;
41
+ private staleScanner?;
41
42
  constructor(config: CacheManagerConfig);
42
43
  /**
43
44
  * Purge all pending jobs from the cache refresh queue
44
45
  * Used in local dev to clear stale jobs on startup
45
46
  */
46
47
  purgeQueue(queueName?: string): Promise<void>;
48
+ /**
49
+ * Purge unsupported job types from the queue
50
+ * Scans the queue and removes jobs for cache types that aren't enabled
51
+ */
52
+ purgeUnsupportedJobs(queueName?: string): Promise<void>;
47
53
  /**
48
54
  * Start background refresh workers for all caches
49
55
  */
@@ -14,6 +14,7 @@ const transaction_cache_1 = require("../stores/transaction-cache");
14
14
  const staking_cache_1 = require("../stores/staking-cache");
15
15
  const zapper_cache_1 = require("../stores/zapper-cache");
16
16
  const refresh_worker_1 = require("../workers/refresh-worker");
17
+ const stale_balance_scanner_1 = require("../workers/stale-balance-scanner");
17
18
  const log = require('@pioneer-platform/loggerdog')();
18
19
  const TAG = ' | CacheManager | ';
19
20
  /**
@@ -103,6 +104,73 @@ class CacheManager {
103
104
  throw error;
104
105
  }
105
106
  }
107
+ /**
108
+ * Purge unsupported job types from the queue
109
+ * Scans the queue and removes jobs for cache types that aren't enabled
110
+ */
111
+ async purgeUnsupportedJobs(queueName = 'cache-refresh') {
112
+ const tag = TAG + 'purgeUnsupportedJobs | ';
113
+ try {
114
+ const redisQueue = this.redisQueue;
115
+ let purgedCount = 0;
116
+ let scannedCount = 0;
117
+ log.info(tag, `Scanning queue for unsupported job types...`);
118
+ // Get enabled cache types
119
+ const enabledCaches = new Set();
120
+ if (this.balanceCache)
121
+ enabledCaches.add('balance');
122
+ if (this.priceCache)
123
+ enabledCaches.add('price');
124
+ if (this.portfolioCache)
125
+ enabledCaches.add('portfolio');
126
+ if (this.transactionCache)
127
+ enabledCaches.add('transaction');
128
+ if (this.stakingCache)
129
+ enabledCaches.add('staking');
130
+ if (this.zapperCache)
131
+ enabledCaches.add('zapper');
132
+ log.info(tag, `Enabled caches: ${Array.from(enabledCaches).join(', ') || 'NONE'}`);
133
+ // Get queue length
134
+ const queueLength = await redisQueue.count(queueName);
135
+ if (queueLength === 0) {
136
+ log.info(tag, '✅ Queue is empty, no jobs to purge');
137
+ return;
138
+ }
139
+ log.info(tag, `Found ${queueLength} jobs in queue`);
140
+ // Scan all jobs
141
+ const tempJobs = [];
142
+ for (let i = 0; i < queueLength; i++) {
143
+ const job = await redisQueue.getWork(queueName, 1);
144
+ if (!job)
145
+ break;
146
+ scannedCount++;
147
+ const jobType = job.type || '';
148
+ const cacheName = jobType.replace('REFRESH_', '').toLowerCase();
149
+ if (!enabledCaches.has(cacheName)) {
150
+ purgedCount++;
151
+ log.warn(tag, ` Purged unsupported job: ${jobType} (cache "${cacheName}" not enabled)`);
152
+ }
153
+ else {
154
+ // Keep supported jobs - we'll re-queue them
155
+ tempJobs.push(job);
156
+ }
157
+ }
158
+ // Re-queue supported jobs
159
+ for (const job of tempJobs) {
160
+ await redisQueue.createWork(queueName, job);
161
+ }
162
+ if (purgedCount > 0) {
163
+ log.info(tag, `✅ Purged ${purgedCount} unsupported jobs (scanned ${scannedCount})`);
164
+ }
165
+ else {
166
+ log.info(tag, `✅ No unsupported jobs found (scanned ${scannedCount})`);
167
+ }
168
+ }
169
+ catch (error) {
170
+ log.error(tag, '⚠️ Error during unsupported job purge (non-fatal):', error);
171
+ // Don't throw - purge failure shouldn't prevent system from starting
172
+ }
173
+ }
106
174
  /**
107
175
  * Start background refresh workers for all caches
108
176
  */
@@ -110,6 +178,8 @@ class CacheManager {
110
178
  const tag = TAG + 'startWorkers | ';
111
179
  try {
112
180
  log.info(tag, 'Starting refresh workers...');
181
+ // FIX: Purge unsupported job types before starting workers
182
+ await this.purgeUnsupportedJobs('cache-refresh');
113
183
  // Create cache registry for workers
114
184
  const cacheRegistry = new Map();
115
185
  if (this.balanceCache) {
@@ -139,6 +209,17 @@ class CacheManager {
139
209
  this.workers.push(worker);
140
210
  log.info(tag, '✅ Refresh worker started for all caches');
141
211
  }
212
+ // Start stale balance scanner if balance cache is enabled
213
+ if (this.balanceCache) {
214
+ this.staleScanner = new stale_balance_scanner_1.StaleBalanceScanner(this.redis, this.balanceCache, {
215
+ staleThresholdMs: 60 * 60 * 1000, // 1 hour (more aggressive than the 5min reactive threshold)
216
+ scanIntervalMs: 10 * 60 * 1000, // Scan every 10 minutes
217
+ batchSize: 100,
218
+ maxRefreshPerScan: 50
219
+ });
220
+ await this.staleScanner.start();
221
+ log.info(tag, '✅ Stale balance scanner started');
222
+ }
142
223
  }
143
224
  catch (error) {
144
225
  log.error(tag, '❌ Failed to start workers:', error);
@@ -155,6 +236,10 @@ class CacheManager {
155
236
  for (const worker of this.workers) {
156
237
  await worker.stop();
157
238
  }
239
+ if (this.staleScanner) {
240
+ await this.staleScanner.stop();
241
+ this.staleScanner = undefined;
242
+ }
158
243
  this.workers = [];
159
244
  log.info(tag, '✅ All workers stopped');
160
245
  }
@@ -58,9 +58,12 @@ export declare class BalanceCache extends BaseCache<BalanceData> {
58
58
  *
59
59
  * CRITICAL FIX: When waitForFresh=true (forceRefresh), bypass cache entirely
60
60
  * and fetch fresh blockchain data for ALL items, not just cache misses
61
+ *
62
+ * PERFORMANCE FIX: When skipAsyncRefresh=true, disable background refresh jobs
63
+ * to prevent 50+ individual refresh jobs from being queued
61
64
  */
62
65
  getBatchBalances(items: Array<{
63
66
  caip: string;
64
67
  pubkey: string;
65
- }>, waitForFresh?: boolean): Promise<BalanceData[]>;
68
+ }>, waitForFresh?: boolean, skipAsyncRefresh?: boolean): Promise<BalanceData[]>;
66
69
  }
@@ -12,6 +12,8 @@ Object.defineProperty(exports, "__esModule", { value: true });
12
12
  exports.BalanceCache = void 0;
13
13
  const base_cache_1 = require("../core/base-cache");
14
14
  const crypto_1 = __importDefault(require("crypto"));
15
+ const pioneer_discovery_1 = require("@pioneer-platform/pioneer-discovery");
16
+ const pioneer_caip_1 = require("@pioneer-platform/pioneer-caip");
15
17
  const log = require('@pioneer-platform/loggerdog')();
16
18
  /**
17
19
  * Hash pubkey/xpub for privacy-protecting cache keys
@@ -110,11 +112,18 @@ class BalanceCache extends base_cache_1.BaseCache {
110
112
  const owner = { pubkey };
111
113
  const balanceInfo = await this.balanceModule.getBalance(asset, owner);
112
114
  if (!balanceInfo || !balanceInfo.balance) {
113
- log.warn(tag, `No balance returned for ${caip}/${sanitizePubkey(pubkey)}`);
115
+ // CRITICAL FIX: Distinguish between legitimate zero balance and failed API call
116
+ // Failed API calls should NOT be cached as valid "0" balance
117
+ const errorMsg = balanceInfo?.error || 'No balance data returned';
118
+ log.warn(tag, `No balance returned for ${caip}/${sanitizePubkey(pubkey)} - ${errorMsg}`);
119
+ // If balanceInfo explicitly indicates an error, throw it instead of caching "0"
120
+ if (balanceInfo?.error || balanceInfo === null || balanceInfo === undefined) {
121
+ throw new Error(`Failed to fetch balance for ${caip}: ${errorMsg}`);
122
+ }
123
+ // Otherwise, this is a legitimate zero balance (balanceInfo exists but balance is falsy)
114
124
  const now = Date.now();
115
125
  // Get asset metadata even for zero balances
116
- const assetData = require('@pioneer-platform/pioneer-discovery').assetData;
117
- const assetInfo = assetData[caip.toUpperCase()] || assetData[caip.toLowerCase()] || {};
126
+ const assetInfo = pioneer_discovery_1.assetData[caip.toUpperCase()] || pioneer_discovery_1.assetData[caip.toLowerCase()] || {};
118
127
  return {
119
128
  caip,
120
129
  pubkey,
@@ -122,14 +131,13 @@ class BalanceCache extends base_cache_1.BaseCache {
122
131
  decimals: assetInfo.decimals,
123
132
  precision: assetInfo.decimals, // Set precision to decimals for compatibility
124
133
  fetchedAt: now,
125
- fetchedAtISO: new Date(now).toISOString()
134
+ fetchedAtISO: new Date(now).toISOString(),
135
+ dataSource: 'Zero balance (legitimate)'
126
136
  };
127
137
  }
128
138
  // Get asset metadata
129
- const assetData = require('@pioneer-platform/pioneer-discovery').assetData;
130
- const { caipToNetworkId } = require('@pioneer-platform/pioneer-caip');
131
- const assetInfo = assetData[caip.toUpperCase()] || assetData[caip.toLowerCase()] || {};
132
- const networkId = caipToNetworkId(caip);
139
+ const assetInfo = pioneer_discovery_1.assetData[caip.toUpperCase()] || pioneer_discovery_1.assetData[caip.toLowerCase()] || {};
140
+ const networkId = (0, pioneer_caip_1.caipToNetworkId)(caip);
133
141
  // Use actual node URL if available, otherwise construct generic description
134
142
  let dataSource = 'Unknown';
135
143
  if (balanceInfo.nodeUrl) {
@@ -187,8 +195,7 @@ class BalanceCache extends base_cache_1.BaseCache {
187
195
  const tag = this.TAG + 'getLegacyCached | ';
188
196
  try {
189
197
  const { caip, pubkey } = params;
190
- const { caipToNetworkId } = require('@pioneer-platform/pioneer-caip');
191
- const networkId = caipToNetworkId(caip);
198
+ const networkId = (0, pioneer_caip_1.caipToNetworkId)(caip);
192
199
  // Try legacy format: cache:balance:pubkey:asset
193
200
  const legacyKey = `cache:balance:${pubkey}:${networkId}`;
194
201
  const legacyData = await this.redis.get(legacyKey);
@@ -229,8 +236,11 @@ class BalanceCache extends base_cache_1.BaseCache {
229
236
  *
230
237
  * CRITICAL FIX: When waitForFresh=true (forceRefresh), bypass cache entirely
231
238
  * and fetch fresh blockchain data for ALL items, not just cache misses
239
+ *
240
+ * PERFORMANCE FIX: When skipAsyncRefresh=true, disable background refresh jobs
241
+ * to prevent 50+ individual refresh jobs from being queued
232
242
  */
233
- async getBatchBalances(items, waitForFresh) {
243
+ async getBatchBalances(items, waitForFresh, skipAsyncRefresh) {
234
244
  const tag = this.TAG + 'getBatchBalances | ';
235
245
  const startTime = Date.now();
236
246
  try {
@@ -347,12 +357,17 @@ class BalanceCache extends base_cache_1.BaseCache {
347
357
  await Promise.all(fetchPromises);
348
358
  log.info(tag, `Fetched ${missedItems.length} misses in ${Date.now() - fetchStart}ms`);
349
359
  }
350
- else {
351
- // Non-blocking: trigger background refresh for misses
360
+ else if (!skipAsyncRefresh) {
361
+ // Non-blocking: trigger background refresh for misses (unless disabled)
362
+ log.info(tag, `📋 Queueing ${missedItems.length} background refresh jobs for cache misses`);
352
363
  missedItems.forEach(item => {
353
364
  this.triggerAsyncRefresh({ caip: item.caip, pubkey: item.pubkey }, 'high');
354
365
  });
355
366
  }
367
+ else {
368
+ // Background refresh disabled for performance
369
+ log.info(tag, `⚡ PERFORMANCE MODE: Skipping ${missedItems.length} background refresh jobs (skipAsyncRefresh=true)`);
370
+ }
356
371
  }
357
372
  return results;
358
373
  }
@@ -29,6 +29,11 @@ export declare class RefreshWorker {
29
29
  * Start processing jobs from the queue
30
30
  */
31
31
  start(): Promise<void>;
32
+ /**
33
+ * Purge stale jobs from queue on startup
34
+ * Scans queue and removes jobs older than 1 hour
35
+ */
36
+ private purgeStaleJobsOnStartup;
32
37
  /**
33
38
  * Stop the worker gracefully
34
39
  */
@@ -49,10 +49,66 @@ class RefreshWorker {
49
49
  }
50
50
  log.info(tag, `🚀 Starting refresh worker for queue: ${this.config.queueName}`);
51
51
  log.info(tag, `Registered caches: ${Array.from(this.cacheRegistry.keys()).join(', ')}`);
52
+ // FIX #3: Purge stale jobs at startup
53
+ await this.purgeStaleJobsOnStartup();
52
54
  this.isRunning = true;
53
55
  this.poll();
54
56
  log.info(tag, '✅ Refresh worker started successfully');
55
57
  }
58
+ /**
59
+ * Purge stale jobs from queue on startup
60
+ * Scans queue and removes jobs older than 1 hour
61
+ */
62
+ async purgeStaleJobsOnStartup() {
63
+ const tag = TAG + 'purgeStaleJobsOnStartup | ';
64
+ try {
65
+ const MAX_JOB_AGE_MS = 60 * 60 * 1000; // 1 hour
66
+ const now = Date.now();
67
+ let purgedCount = 0;
68
+ let scannedCount = 0;
69
+ log.info(tag, `Scanning queue for stale jobs (age > ${MAX_JOB_AGE_MS / 1000}s)...`);
70
+ // Get queue length
71
+ const queueLength = await this.redisQueue.count(this.config.queueName);
72
+ if (queueLength === 0) {
73
+ log.info(tag, '✅ Queue is empty, no stale jobs to purge');
74
+ return;
75
+ }
76
+ log.info(tag, `Found ${queueLength} jobs in queue, checking for stale jobs...`);
77
+ // Scan up to 100 jobs to find stale ones
78
+ const MAX_SCAN = Math.min(queueLength, 100);
79
+ const tempJobs = [];
80
+ for (let i = 0; i < MAX_SCAN; i++) {
81
+ const job = await this.redisQueue.getWork(this.config.queueName, 1);
82
+ if (!job)
83
+ break;
84
+ scannedCount++;
85
+ const jobTimestamp = job.timestamp || 0;
86
+ const jobAge = jobTimestamp > 0 ? now - jobTimestamp : 0;
87
+ if (jobAge > MAX_JOB_AGE_MS) {
88
+ purgedCount++;
89
+ log.warn(tag, ` Purged stale job: ${job.type} (age: ${Math.round(jobAge / 1000)}s)`);
90
+ }
91
+ else {
92
+ // Keep fresh jobs - we'll re-queue them
93
+ tempJobs.push(job);
94
+ }
95
+ }
96
+ // Re-queue fresh jobs
97
+ for (const job of tempJobs) {
98
+ await this.redisQueue.createWork(this.config.queueName, job);
99
+ }
100
+ if (purgedCount > 0) {
101
+ log.info(tag, `✅ Purged ${purgedCount} stale jobs (scanned ${scannedCount})`);
102
+ }
103
+ else {
104
+ log.info(tag, `✅ No stale jobs found (scanned ${scannedCount})`);
105
+ }
106
+ }
107
+ catch (error) {
108
+ log.error(tag, '⚠️ Error during startup purge (non-fatal):', error);
109
+ // Don't throw - startup purge failure shouldn't prevent worker from starting
110
+ }
111
+ }
56
112
  /**
57
113
  * Stop the worker gracefully
58
114
  */
@@ -130,12 +186,30 @@ class RefreshWorker {
130
186
  try {
131
187
  const { type, key, params, retryCount = 0 } = job;
132
188
  log.info(tag, `Processing ${type} for ${key} (retry: ${retryCount})`);
189
+ // FIX #1: Check job age and drop if too old (>1 hour)
190
+ const MAX_JOB_AGE_MS = 60 * 60 * 1000; // 1 hour
191
+ const jobAge = Date.now() - startTime;
192
+ const jobTimestamp = job.timestamp || 0;
193
+ if (jobTimestamp > 0) {
194
+ const actualJobAge = Date.now() - jobTimestamp;
195
+ if (actualJobAge > MAX_JOB_AGE_MS) {
196
+ log.error(tag, `⚠️ DROPPING STALE JOB: ${type} (age: ${Math.round(actualJobAge / 1000)}s, max: ${MAX_JOB_AGE_MS / 1000}s)`);
197
+ log.error(tag, ` Job key: ${key}`);
198
+ log.error(tag, ` Job has been removed from queue and will not be retried`);
199
+ return;
200
+ }
201
+ }
133
202
  // Extract cache name from job type (e.g., "REFRESH_BALANCE" -> "balance")
134
203
  const cacheName = type.replace('REFRESH_', '').toLowerCase();
135
204
  // Get the appropriate cache instance
136
205
  const cache = this.cacheRegistry.get(cacheName);
206
+ // FIX #2: Drop unsupported job types with clear error message
137
207
  if (!cache) {
138
- log.error(tag, `No cache registered for type: ${type}`);
208
+ log.error(tag, `❌ DROPPING UNSUPPORTED JOB: ${type}`);
209
+ log.error(tag, ` Cache type "${cacheName}" is not registered in this worker`);
210
+ log.error(tag, ` Registered caches: ${Array.from(this.cacheRegistry.keys()).join(', ') || 'NONE'}`);
211
+ log.error(tag, ` Job key: ${key}`);
212
+ log.error(tag, ` Job has been removed from queue and will not be retried`);
139
213
  return;
140
214
  }
141
215
  // Fetch fresh data using the cache's fetchFresh method
@@ -0,0 +1,35 @@
1
+ import type { BalanceCache } from '../stores/balance-cache';
2
+ export interface StaleScannerConfig {
3
+ staleThresholdMs: number;
4
+ scanIntervalMs: number;
5
+ batchSize: number;
6
+ maxRefreshPerScan: number;
7
+ }
8
+ export declare class StaleBalanceScanner {
9
+ private redis;
10
+ private balanceCache;
11
+ private config;
12
+ private isRunning;
13
+ private scanTimer?;
14
+ constructor(redis: any, balanceCache: BalanceCache, config?: Partial<StaleScannerConfig>);
15
+ /**
16
+ * Start the scanner
17
+ */
18
+ start(): Promise<void>;
19
+ /**
20
+ * Stop the scanner
21
+ */
22
+ stop(): Promise<void>;
23
+ /**
24
+ * Schedule next scan
25
+ */
26
+ private scheduleNextScan;
27
+ /**
28
+ * Scan for stale balances and queue refreshes
29
+ */
30
+ private scan;
31
+ /**
32
+ * Get scanner statistics
33
+ */
34
+ getStats(): any;
35
+ }
@@ -0,0 +1,180 @@
1
+ "use strict";
2
+ /*
3
+ Stale Balance Scanner
4
+
5
+ Proactively scans for stale balance cache entries and queues them for refresh.
6
+ This ensures balances that aren't actively accessed still get refreshed.
7
+
8
+ Prevents the issue where stale balances sit for days/weeks without being updated.
9
+ */
10
+ Object.defineProperty(exports, "__esModule", { value: true });
11
+ exports.StaleBalanceScanner = void 0;
12
+ const log = require('@pioneer-platform/loggerdog')();
13
+ const TAG = ' | StaleBalanceScanner | ';
14
+ class StaleBalanceScanner {
15
+ constructor(redis, balanceCache, config) {
16
+ this.isRunning = false;
17
+ this.redis = redis;
18
+ this.balanceCache = balanceCache;
19
+ this.config = {
20
+ staleThresholdMs: config?.staleThresholdMs || 60 * 60 * 1000, // 1 hour
21
+ scanIntervalMs: config?.scanIntervalMs || 5 * 60 * 1000, // 5 minutes
22
+ batchSize: config?.batchSize || 100,
23
+ maxRefreshPerScan: config?.maxRefreshPerScan || 50
24
+ };
25
+ log.info(TAG, 'Stale balance scanner initialized', {
26
+ staleThresholdHours: this.config.staleThresholdMs / (1000 * 60 * 60),
27
+ scanIntervalMinutes: this.config.scanIntervalMs / (1000 * 60)
28
+ });
29
+ }
30
+ /**
31
+ * Start the scanner
32
+ */
33
+ async start() {
34
+ const tag = TAG + 'start | ';
35
+ if (this.isRunning) {
36
+ log.warn(tag, 'Scanner already running');
37
+ return;
38
+ }
39
+ this.isRunning = true;
40
+ log.info(tag, '🔍 Starting stale balance scanner');
41
+ // Run first scan immediately, then schedule recurring
42
+ await this.scan();
43
+ this.scheduleNextScan();
44
+ }
45
+ /**
46
+ * Stop the scanner
47
+ */
48
+ async stop() {
49
+ const tag = TAG + 'stop | ';
50
+ log.info(tag, 'Stopping stale balance scanner');
51
+ this.isRunning = false;
52
+ if (this.scanTimer) {
53
+ clearTimeout(this.scanTimer);
54
+ this.scanTimer = undefined;
55
+ }
56
+ log.info(tag, '✅ Scanner stopped');
57
+ }
58
+ /**
59
+ * Schedule next scan
60
+ */
61
+ scheduleNextScan() {
62
+ if (!this.isRunning)
63
+ return;
64
+ this.scanTimer = setTimeout(async () => {
65
+ await this.scan();
66
+ this.scheduleNextScan();
67
+ }, this.config.scanIntervalMs);
68
+ }
69
+ /**
70
+ * Scan for stale balances and queue refreshes
71
+ */
72
+ async scan() {
73
+ const tag = TAG + 'scan | ';
74
+ const startTime = Date.now();
75
+ try {
76
+ log.info(tag, 'Starting stale balance scan...');
77
+ // Scan balance_v2:* keys in batches
78
+ const pattern = 'balance_v2:*';
79
+ const staleKeys = [];
80
+ const now = Date.now();
81
+ let cursor = '0';
82
+ let totalScanned = 0;
83
+ let refreshCount = 0;
84
+ do {
85
+ // SCAN with COUNT for batching
86
+ const [nextCursor, keys] = await this.redis.scan(cursor, 'MATCH', pattern, 'COUNT', this.config.batchSize);
87
+ cursor = nextCursor;
88
+ totalScanned += keys.length;
89
+ // Check each key for staleness
90
+ for (const key of keys) {
91
+ try {
92
+ const cached = await this.redis.get(key);
93
+ if (!cached)
94
+ continue;
95
+ const parsed = JSON.parse(cached);
96
+ const age = now - (parsed.timestamp || 0);
97
+ // Check if stale
98
+ if (age > this.config.staleThresholdMs) {
99
+ staleKeys.push(key);
100
+ // Stop if we've hit max refresh limit
101
+ if (staleKeys.length >= this.config.maxRefreshPerScan) {
102
+ log.info(tag, `Reached max refresh limit (${this.config.maxRefreshPerScan}), stopping scan`);
103
+ cursor = '0'; // Force exit
104
+ break;
105
+ }
106
+ }
107
+ }
108
+ catch (error) {
109
+ log.error(tag, `Error checking key ${key}:`, error);
110
+ }
111
+ }
112
+ } while (cursor !== '0' && this.isRunning);
113
+ // Queue refresh for stale balances
114
+ if (staleKeys.length > 0) {
115
+ log.info(tag, `Found ${staleKeys.length} stale balances out of ${totalScanned} scanned`);
116
+ for (const key of staleKeys) {
117
+ try {
118
+ // Extract CAIP and pubkey hash from key
119
+ // Key format: balance_v2:caip:hashedPubkey
120
+ // IMPORTANT: CAIP itself contains colons (e.g., ripple:4109c6f2045fc7eff4cde8f9905d19c2/slip44:144)
121
+ // So we must split from the right to get hashedPubkey, then everything else is the CAIP
122
+ const withoutPrefix = key.replace('balance_v2:', '');
123
+ const lastColonIndex = withoutPrefix.lastIndexOf(':');
124
+ if (lastColonIndex === -1)
125
+ continue;
126
+ const caip = withoutPrefix.substring(0, lastColonIndex);
127
+ const pubkeyHash = withoutPrefix.substring(lastColonIndex + 1);
128
+ // We need the original pubkey to refresh, but it's hashed in the key
129
+ // So we need to get it from the cached value
130
+ const cached = await this.redis.get(key);
131
+ if (!cached)
132
+ continue;
133
+ const parsed = JSON.parse(cached);
134
+ const pubkey = parsed.value?.pubkey;
135
+ if (!pubkey) {
136
+ log.warn(tag, `No pubkey found in cache for ${key}, skipping`);
137
+ continue;
138
+ }
139
+ // Trigger refresh by calling getBalance with waitForFresh=true
140
+ // This will queue a background refresh job
141
+ log.debug(tag, `Queueing refresh for ${caip} (age: ${Math.round((now - parsed.timestamp) / 1000 / 60)}min)`);
142
+ // Use setImmediate to avoid blocking the scan
143
+ setImmediate(async () => {
144
+ try {
145
+ await this.balanceCache.getBalance(caip, pubkey, true);
146
+ refreshCount++;
147
+ }
148
+ catch (error) {
149
+ log.error(tag, `Failed to refresh ${caip}/${pubkey}:`, error);
150
+ }
151
+ });
152
+ }
153
+ catch (error) {
154
+ log.error(tag, `Error queueing refresh for ${key}:`, error);
155
+ }
156
+ }
157
+ }
158
+ const scanTime = Date.now() - startTime;
159
+ log.info(tag, `✅ Scan complete: ${totalScanned} scanned, ${staleKeys.length} stale, ${refreshCount} queued in ${scanTime}ms`);
160
+ }
161
+ catch (error) {
162
+ log.error(tag, 'Error during scan:', error);
163
+ }
164
+ }
165
+ /**
166
+ * Get scanner statistics
167
+ */
168
+ getStats() {
169
+ return {
170
+ isRunning: this.isRunning,
171
+ config: {
172
+ staleThresholdHours: this.config.staleThresholdMs / (1000 * 60 * 60),
173
+ scanIntervalMinutes: this.config.scanIntervalMs / (1000 * 60),
174
+ batchSize: this.config.batchSize,
175
+ maxRefreshPerScan: this.config.maxRefreshPerScan
176
+ }
177
+ };
178
+ }
179
+ }
180
+ exports.StaleBalanceScanner = StaleBalanceScanner;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@pioneer-platform/pioneer-cache",
3
- "version": "1.19.3",
3
+ "version": "1.22.0",
4
4
  "description": "Unified caching system for Pioneer platform with Redis backend",
5
5
  "main": "./dist/index.js",
6
6
  "types": "./dist/index.d.ts",
@@ -15,8 +15,10 @@
15
15
  "license": "MIT",
16
16
  "dependencies": {
17
17
  "@pioneer-platform/loggerdog": "8.11.0",
18
- "@pioneer-platform/redis-queue": "8.12.17",
19
- "@pioneer-platform/default-redis": "8.11.7"
18
+ "@pioneer-platform/pioneer-discovery": "8.44.0",
19
+ "@pioneer-platform/pioneer-caip": "9.21.0",
20
+ "@pioneer-platform/default-redis": "8.11.7",
21
+ "@pioneer-platform/redis-queue": "8.12.17"
20
22
  },
21
23
  "devDependencies": {
22
24
  "@types/jest": "^29.5.0",
@@ -12,6 +12,7 @@ import { TransactionCache } from '../stores/transaction-cache';
12
12
  import { StakingCache } from '../stores/staking-cache';
13
13
  import { ZapperCache } from '../stores/zapper-cache';
14
14
  import { RefreshWorker, startUnifiedWorker } from '../workers/refresh-worker';
15
+ import { StaleBalanceScanner } from '../workers/stale-balance-scanner';
15
16
  import type { BaseCache } from './base-cache';
16
17
  import type { HealthCheckResult } from '../types';
17
18
 
@@ -53,6 +54,7 @@ export class CacheManager {
53
54
  private stakingCache?: StakingCache;
54
55
  private zapperCache?: ZapperCache;
55
56
  private workers: RefreshWorker[] = [];
57
+ private staleScanner?: StaleBalanceScanner;
56
58
 
57
59
  constructor(config: CacheManagerConfig) {
58
60
  this.redis = config.redis;
@@ -147,6 +149,79 @@ export class CacheManager {
147
149
  }
148
150
  }
149
151
 
152
+ /**
153
+ * Purge unsupported job types from the queue
154
+ * Scans the queue and removes jobs for cache types that aren't enabled
155
+ */
156
+ async purgeUnsupportedJobs(queueName: string = 'cache-refresh'): Promise<void> {
157
+ const tag = TAG + 'purgeUnsupportedJobs | ';
158
+
159
+ try {
160
+ const redisQueue = this.redisQueue;
161
+ let purgedCount = 0;
162
+ let scannedCount = 0;
163
+
164
+ log.info(tag, `Scanning queue for unsupported job types...`);
165
+
166
+ // Get enabled cache types
167
+ const enabledCaches = new Set<string>();
168
+ if (this.balanceCache) enabledCaches.add('balance');
169
+ if (this.priceCache) enabledCaches.add('price');
170
+ if (this.portfolioCache) enabledCaches.add('portfolio');
171
+ if (this.transactionCache) enabledCaches.add('transaction');
172
+ if (this.stakingCache) enabledCaches.add('staking');
173
+ if (this.zapperCache) enabledCaches.add('zapper');
174
+
175
+ log.info(tag, `Enabled caches: ${Array.from(enabledCaches).join(', ') || 'NONE'}`);
176
+
177
+ // Get queue length
178
+ const queueLength = await redisQueue.count(queueName);
179
+
180
+ if (queueLength === 0) {
181
+ log.info(tag, '✅ Queue is empty, no jobs to purge');
182
+ return;
183
+ }
184
+
185
+ log.info(tag, `Found ${queueLength} jobs in queue`);
186
+
187
+ // Scan all jobs
188
+ const tempJobs: any[] = [];
189
+
190
+ for (let i = 0; i < queueLength; i++) {
191
+ const job = await redisQueue.getWork(queueName, 1);
192
+
193
+ if (!job) break;
194
+
195
+ scannedCount++;
196
+ const jobType = job.type || '';
197
+ const cacheName = jobType.replace('REFRESH_', '').toLowerCase();
198
+
199
+ if (!enabledCaches.has(cacheName)) {
200
+ purgedCount++;
201
+ log.warn(tag, ` Purged unsupported job: ${jobType} (cache "${cacheName}" not enabled)`);
202
+ } else {
203
+ // Keep supported jobs - we'll re-queue them
204
+ tempJobs.push(job);
205
+ }
206
+ }
207
+
208
+ // Re-queue supported jobs
209
+ for (const job of tempJobs) {
210
+ await redisQueue.createWork(queueName, job);
211
+ }
212
+
213
+ if (purgedCount > 0) {
214
+ log.info(tag, `✅ Purged ${purgedCount} unsupported jobs (scanned ${scannedCount})`);
215
+ } else {
216
+ log.info(tag, `✅ No unsupported jobs found (scanned ${scannedCount})`);
217
+ }
218
+
219
+ } catch (error) {
220
+ log.error(tag, '⚠️ Error during unsupported job purge (non-fatal):', error);
221
+ // Don't throw - purge failure shouldn't prevent system from starting
222
+ }
223
+ }
224
+
150
225
  /**
151
226
  * Start background refresh workers for all caches
152
227
  */
@@ -156,6 +231,9 @@ export class CacheManager {
156
231
  try {
157
232
  log.info(tag, 'Starting refresh workers...');
158
233
 
234
+ // FIX: Purge unsupported job types before starting workers
235
+ await this.purgeUnsupportedJobs('cache-refresh');
236
+
159
237
  // Create cache registry for workers
160
238
  const cacheRegistry = new Map<string, BaseCache<any>>();
161
239
 
@@ -196,6 +274,19 @@ export class CacheManager {
196
274
  log.info(tag, '✅ Refresh worker started for all caches');
197
275
  }
198
276
 
277
+ // Start stale balance scanner if balance cache is enabled
278
+ if (this.balanceCache) {
279
+ this.staleScanner = new StaleBalanceScanner(this.redis, this.balanceCache, {
280
+ staleThresholdMs: 60 * 60 * 1000, // 1 hour (more aggressive than the 5min reactive threshold)
281
+ scanIntervalMs: 10 * 60 * 1000, // Scan every 10 minutes
282
+ batchSize: 100,
283
+ maxRefreshPerScan: 50
284
+ });
285
+
286
+ await this.staleScanner.start();
287
+ log.info(tag, '✅ Stale balance scanner started');
288
+ }
289
+
199
290
  } catch (error) {
200
291
  log.error(tag, '❌ Failed to start workers:', error);
201
292
  throw error;
@@ -215,6 +306,11 @@ export class CacheManager {
215
306
  await worker.stop();
216
307
  }
217
308
 
309
+ if (this.staleScanner) {
310
+ await this.staleScanner.stop();
311
+ this.staleScanner = undefined;
312
+ }
313
+
218
314
  this.workers = [];
219
315
  log.info(tag, '✅ All workers stopped');
220
316
 
@@ -8,6 +8,8 @@
8
8
  import { BaseCache } from '../core/base-cache';
9
9
  import type { CacheConfig } from '../types';
10
10
  import crypto from 'crypto';
11
+ import { assetData } from '@pioneer-platform/pioneer-discovery';
12
+ import { caipToNetworkId } from '@pioneer-platform/pioneer-caip';
11
13
 
12
14
  const log = require('@pioneer-platform/loggerdog')();
13
15
 
@@ -148,12 +150,22 @@ export class BalanceCache extends BaseCache<BalanceData> {
148
150
  const balanceInfo = await this.balanceModule.getBalance(asset, owner);
149
151
 
150
152
  if (!balanceInfo || !balanceInfo.balance) {
151
- log.warn(tag, `No balance returned for ${caip}/${sanitizePubkey(pubkey)}`);
153
+ // CRITICAL FIX: Distinguish between legitimate zero balance and failed API call
154
+ // Failed API calls should NOT be cached as valid "0" balance
155
+ const errorMsg = balanceInfo?.error || 'No balance data returned';
156
+
157
+ log.warn(tag, `No balance returned for ${caip}/${sanitizePubkey(pubkey)} - ${errorMsg}`);
158
+
159
+ // If balanceInfo explicitly indicates an error, throw it instead of caching "0"
160
+ if (balanceInfo?.error || balanceInfo === null || balanceInfo === undefined) {
161
+ throw new Error(`Failed to fetch balance for ${caip}: ${errorMsg}`);
162
+ }
163
+
164
+ // Otherwise, this is a legitimate zero balance (balanceInfo exists but balance is falsy)
152
165
  const now = Date.now();
153
166
 
154
167
  // Get asset metadata even for zero balances
155
- const assetData = require('@pioneer-platform/pioneer-discovery').assetData;
156
- const assetInfo = assetData[caip.toUpperCase()] || assetData[caip.toLowerCase()] || {};
168
+ const assetInfo = (assetData as any)[caip.toUpperCase()] || (assetData as any)[caip.toLowerCase()] || {};
157
169
 
158
170
  return {
159
171
  caip,
@@ -162,15 +174,13 @@ export class BalanceCache extends BaseCache<BalanceData> {
162
174
  decimals: assetInfo.decimals,
163
175
  precision: assetInfo.decimals, // Set precision to decimals for compatibility
164
176
  fetchedAt: now,
165
- fetchedAtISO: new Date(now).toISOString()
177
+ fetchedAtISO: new Date(now).toISOString(),
178
+ dataSource: 'Zero balance (legitimate)'
166
179
  };
167
180
  }
168
181
 
169
182
  // Get asset metadata
170
- const assetData = require('@pioneer-platform/pioneer-discovery').assetData;
171
- const { caipToNetworkId } = require('@pioneer-platform/pioneer-caip');
172
-
173
- const assetInfo = assetData[caip.toUpperCase()] || assetData[caip.toLowerCase()] || {};
183
+ const assetInfo = (assetData as any)[caip.toUpperCase()] || (assetData as any)[caip.toLowerCase()] || {};
174
184
  const networkId = caipToNetworkId(caip);
175
185
 
176
186
  // Use actual node URL if available, otherwise construct generic description
@@ -229,7 +239,6 @@ export class BalanceCache extends BaseCache<BalanceData> {
229
239
 
230
240
  try {
231
241
  const { caip, pubkey } = params;
232
- const { caipToNetworkId } = require('@pioneer-platform/pioneer-caip');
233
242
  const networkId = caipToNetworkId(caip);
234
243
 
235
244
  // Try legacy format: cache:balance:pubkey:asset
@@ -276,8 +285,15 @@ export class BalanceCache extends BaseCache<BalanceData> {
276
285
  *
277
286
  * CRITICAL FIX: When waitForFresh=true (forceRefresh), bypass cache entirely
278
287
  * and fetch fresh blockchain data for ALL items, not just cache misses
288
+ *
289
+ * PERFORMANCE FIX: When skipAsyncRefresh=true, disable background refresh jobs
290
+ * to prevent 50+ individual refresh jobs from being queued
279
291
  */
280
- async getBatchBalances(items: Array<{ caip: string; pubkey: string }>, waitForFresh?: boolean): Promise<BalanceData[]> {
292
+ async getBatchBalances(
293
+ items: Array<{ caip: string; pubkey: string }>,
294
+ waitForFresh?: boolean,
295
+ skipAsyncRefresh?: boolean
296
+ ): Promise<BalanceData[]> {
281
297
  const tag = this.TAG + 'getBatchBalances | ';
282
298
  const startTime = Date.now();
283
299
 
@@ -409,11 +425,15 @@ export class BalanceCache extends BaseCache<BalanceData> {
409
425
 
410
426
  await Promise.all(fetchPromises);
411
427
  log.info(tag, `Fetched ${missedItems.length} misses in ${Date.now() - fetchStart}ms`);
412
- } else {
413
- // Non-blocking: trigger background refresh for misses
428
+ } else if (!skipAsyncRefresh) {
429
+ // Non-blocking: trigger background refresh for misses (unless disabled)
430
+ log.info(tag, `📋 Queueing ${missedItems.length} background refresh jobs for cache misses`);
414
431
  missedItems.forEach(item => {
415
432
  this.triggerAsyncRefresh({ caip: item.caip, pubkey: item.pubkey }, 'high');
416
433
  });
434
+ } else {
435
+ // Background refresh disabled for performance
436
+ log.info(tag, `⚡ PERFORMANCE MODE: Skipping ${missedItems.length} background refresh jobs (skipAsyncRefresh=true)`);
417
437
  }
418
438
  }
419
439
 
@@ -69,12 +69,79 @@ export class RefreshWorker {
69
69
  log.info(tag, `🚀 Starting refresh worker for queue: ${this.config.queueName}`);
70
70
  log.info(tag, `Registered caches: ${Array.from(this.cacheRegistry.keys()).join(', ')}`);
71
71
 
72
+ // FIX #3: Purge stale jobs at startup
73
+ await this.purgeStaleJobsOnStartup();
74
+
72
75
  this.isRunning = true;
73
76
  this.poll();
74
77
 
75
78
  log.info(tag, '✅ Refresh worker started successfully');
76
79
  }
77
80
 
81
+ /**
82
+ * Purge stale jobs from queue on startup
83
+ * Scans queue and removes jobs older than 1 hour
84
+ */
85
+ private async purgeStaleJobsOnStartup(): Promise<void> {
86
+ const tag = TAG + 'purgeStaleJobsOnStartup | ';
87
+
88
+ try {
89
+ const MAX_JOB_AGE_MS = 60 * 60 * 1000; // 1 hour
90
+ const now = Date.now();
91
+ let purgedCount = 0;
92
+ let scannedCount = 0;
93
+
94
+ log.info(tag, `Scanning queue for stale jobs (age > ${MAX_JOB_AGE_MS / 1000}s)...`);
95
+
96
+ // Get queue length
97
+ const queueLength = await this.redisQueue.count(this.config.queueName);
98
+
99
+ if (queueLength === 0) {
100
+ log.info(tag, '✅ Queue is empty, no stale jobs to purge');
101
+ return;
102
+ }
103
+
104
+ log.info(tag, `Found ${queueLength} jobs in queue, checking for stale jobs...`);
105
+
106
+ // Scan up to 100 jobs to find stale ones
107
+ const MAX_SCAN = Math.min(queueLength, 100);
108
+ const tempJobs: RefreshJob[] = [];
109
+
110
+ for (let i = 0; i < MAX_SCAN; i++) {
111
+ const job = await this.redisQueue.getWork(this.config.queueName, 1);
112
+
113
+ if (!job) break;
114
+
115
+ scannedCount++;
116
+ const jobTimestamp = (job as any).timestamp || 0;
117
+ const jobAge = jobTimestamp > 0 ? now - jobTimestamp : 0;
118
+
119
+ if (jobAge > MAX_JOB_AGE_MS) {
120
+ purgedCount++;
121
+ log.warn(tag, ` Purged stale job: ${job.type} (age: ${Math.round(jobAge / 1000)}s)`);
122
+ } else {
123
+ // Keep fresh jobs - we'll re-queue them
124
+ tempJobs.push(job);
125
+ }
126
+ }
127
+
128
+ // Re-queue fresh jobs
129
+ for (const job of tempJobs) {
130
+ await this.redisQueue.createWork(this.config.queueName, job);
131
+ }
132
+
133
+ if (purgedCount > 0) {
134
+ log.info(tag, `✅ Purged ${purgedCount} stale jobs (scanned ${scannedCount})`);
135
+ } else {
136
+ log.info(tag, `✅ No stale jobs found (scanned ${scannedCount})`);
137
+ }
138
+
139
+ } catch (error) {
140
+ log.error(tag, '⚠️ Error during startup purge (non-fatal):', error);
141
+ // Don't throw - startup purge failure shouldn't prevent worker from starting
142
+ }
143
+ }
144
+
78
145
  /**
79
146
  * Stop the worker gracefully
80
147
  */
@@ -166,14 +233,34 @@ export class RefreshWorker {
166
233
 
167
234
  log.info(tag, `Processing ${type} for ${key} (retry: ${retryCount})`);
168
235
 
236
+ // FIX #1: Check job age and drop if too old (>1 hour)
237
+ const MAX_JOB_AGE_MS = 60 * 60 * 1000; // 1 hour
238
+ const jobAge = Date.now() - startTime;
239
+ const jobTimestamp = (job as any).timestamp || 0;
240
+
241
+ if (jobTimestamp > 0) {
242
+ const actualJobAge = Date.now() - jobTimestamp;
243
+ if (actualJobAge > MAX_JOB_AGE_MS) {
244
+ log.error(tag, `⚠️ DROPPING STALE JOB: ${type} (age: ${Math.round(actualJobAge / 1000)}s, max: ${MAX_JOB_AGE_MS / 1000}s)`);
245
+ log.error(tag, ` Job key: ${key}`);
246
+ log.error(tag, ` Job has been removed from queue and will not be retried`);
247
+ return;
248
+ }
249
+ }
250
+
169
251
  // Extract cache name from job type (e.g., "REFRESH_BALANCE" -> "balance")
170
252
  const cacheName = type.replace('REFRESH_', '').toLowerCase();
171
253
 
172
254
  // Get the appropriate cache instance
173
255
  const cache = this.cacheRegistry.get(cacheName);
174
256
 
257
+ // FIX #2: Drop unsupported job types with clear error message
175
258
  if (!cache) {
176
- log.error(tag, `No cache registered for type: ${type}`);
259
+ log.error(tag, `❌ DROPPING UNSUPPORTED JOB: ${type}`);
260
+ log.error(tag, ` Cache type "${cacheName}" is not registered in this worker`);
261
+ log.error(tag, ` Registered caches: ${Array.from(this.cacheRegistry.keys()).join(', ') || 'NONE'}`);
262
+ log.error(tag, ` Job key: ${key}`);
263
+ log.error(tag, ` Job has been removed from queue and will not be retried`);
177
264
  return;
178
265
  }
179
266
 
@@ -0,0 +1,223 @@
1
+ /*
2
+ Stale Balance Scanner
3
+
4
+ Proactively scans for stale balance cache entries and queues them for refresh.
5
+ This ensures balances that aren't actively accessed still get refreshed.
6
+
7
+ Prevents the issue where stale balances sit for days/weeks without being updated.
8
+ */
9
+
10
+ import type { BalanceCache } from '../stores/balance-cache';
11
+
12
+ const log = require('@pioneer-platform/loggerdog')();
13
+ const TAG = ' | StaleBalanceScanner | ';
14
+
15
+ export interface StaleScannerConfig {
16
+ staleThresholdMs: number; // Refresh balances older than this (default: 1 hour)
17
+ scanIntervalMs: number; // How often to scan (default: 5 minutes)
18
+ batchSize: number; // How many keys to scan per batch (default: 100)
19
+ maxRefreshPerScan: number; // Max balances to refresh per scan (default: 50)
20
+ }
21
+
22
+ export class StaleBalanceScanner {
23
+ private redis: any;
24
+ private balanceCache: BalanceCache;
25
+ private config: StaleScannerConfig;
26
+ private isRunning: boolean = false;
27
+ private scanTimer?: NodeJS.Timeout;
28
+
29
+ constructor(redis: any, balanceCache: BalanceCache, config?: Partial<StaleScannerConfig>) {
30
+ this.redis = redis;
31
+ this.balanceCache = balanceCache;
32
+
33
+ this.config = {
34
+ staleThresholdMs: config?.staleThresholdMs || 60 * 60 * 1000, // 1 hour
35
+ scanIntervalMs: config?.scanIntervalMs || 5 * 60 * 1000, // 5 minutes
36
+ batchSize: config?.batchSize || 100,
37
+ maxRefreshPerScan: config?.maxRefreshPerScan || 50
38
+ };
39
+
40
+ log.info(TAG, 'Stale balance scanner initialized', {
41
+ staleThresholdHours: this.config.staleThresholdMs / (1000 * 60 * 60),
42
+ scanIntervalMinutes: this.config.scanIntervalMs / (1000 * 60)
43
+ });
44
+ }
45
+
46
+ /**
47
+ * Start the scanner
48
+ */
49
+ async start(): Promise<void> {
50
+ const tag = TAG + 'start | ';
51
+
52
+ if (this.isRunning) {
53
+ log.warn(tag, 'Scanner already running');
54
+ return;
55
+ }
56
+
57
+ this.isRunning = true;
58
+ log.info(tag, '🔍 Starting stale balance scanner');
59
+
60
+ // Run first scan immediately, then schedule recurring
61
+ await this.scan();
62
+ this.scheduleNextScan();
63
+ }
64
+
65
+ /**
66
+ * Stop the scanner
67
+ */
68
+ async stop(): Promise<void> {
69
+ const tag = TAG + 'stop | ';
70
+
71
+ log.info(tag, 'Stopping stale balance scanner');
72
+ this.isRunning = false;
73
+
74
+ if (this.scanTimer) {
75
+ clearTimeout(this.scanTimer);
76
+ this.scanTimer = undefined;
77
+ }
78
+
79
+ log.info(tag, '✅ Scanner stopped');
80
+ }
81
+
82
+ /**
83
+ * Schedule next scan
84
+ */
85
+ private scheduleNextScan(): void {
86
+ if (!this.isRunning) return;
87
+
88
+ this.scanTimer = setTimeout(async () => {
89
+ await this.scan();
90
+ this.scheduleNextScan();
91
+ }, this.config.scanIntervalMs);
92
+ }
93
+
94
+ /**
95
+ * Scan for stale balances and queue refreshes
96
+ */
97
+ private async scan(): Promise<void> {
98
+ const tag = TAG + 'scan | ';
99
+ const startTime = Date.now();
100
+
101
+ try {
102
+ log.info(tag, 'Starting stale balance scan...');
103
+
104
+ // Scan balance_v2:* keys in batches
105
+ const pattern = 'balance_v2:*';
106
+ const staleKeys: string[] = [];
107
+ const now = Date.now();
108
+
109
+ let cursor = '0';
110
+ let totalScanned = 0;
111
+ let refreshCount = 0;
112
+
113
+ do {
114
+ // SCAN with COUNT for batching
115
+ const [nextCursor, keys] = await this.redis.scan(
116
+ cursor,
117
+ 'MATCH', pattern,
118
+ 'COUNT', this.config.batchSize
119
+ );
120
+
121
+ cursor = nextCursor;
122
+ totalScanned += keys.length;
123
+
124
+ // Check each key for staleness
125
+ for (const key of keys) {
126
+ try {
127
+ const cached = await this.redis.get(key);
128
+ if (!cached) continue;
129
+
130
+ const parsed = JSON.parse(cached);
131
+ const age = now - (parsed.timestamp || 0);
132
+
133
+ // Check if stale
134
+ if (age > this.config.staleThresholdMs) {
135
+ staleKeys.push(key);
136
+
137
+ // Stop if we've hit max refresh limit
138
+ if (staleKeys.length >= this.config.maxRefreshPerScan) {
139
+ log.info(tag, `Reached max refresh limit (${this.config.maxRefreshPerScan}), stopping scan`);
140
+ cursor = '0'; // Force exit
141
+ break;
142
+ }
143
+ }
144
+ } catch (error) {
145
+ log.error(tag, `Error checking key ${key}:`, error);
146
+ }
147
+ }
148
+
149
+ } while (cursor !== '0' && this.isRunning);
150
+
151
+ // Queue refresh for stale balances
152
+ if (staleKeys.length > 0) {
153
+ log.info(tag, `Found ${staleKeys.length} stale balances out of ${totalScanned} scanned`);
154
+
155
+ for (const key of staleKeys) {
156
+ try {
157
+ // Extract CAIP and pubkey hash from key
158
+ // Key format: balance_v2:caip:hashedPubkey
159
+ // IMPORTANT: CAIP itself contains colons (e.g., ripple:4109c6f2045fc7eff4cde8f9905d19c2/slip44:144)
160
+ // So we must split from the right to get hashedPubkey, then everything else is the CAIP
161
+ const withoutPrefix = key.replace('balance_v2:', '');
162
+ const lastColonIndex = withoutPrefix.lastIndexOf(':');
163
+ if (lastColonIndex === -1) continue;
164
+
165
+ const caip = withoutPrefix.substring(0, lastColonIndex);
166
+ const pubkeyHash = withoutPrefix.substring(lastColonIndex + 1);
167
+
168
+ // We need the original pubkey to refresh, but it's hashed in the key
169
+ // So we need to get it from the cached value
170
+ const cached = await this.redis.get(key);
171
+ if (!cached) continue;
172
+
173
+ const parsed = JSON.parse(cached);
174
+ const pubkey = parsed.value?.pubkey;
175
+
176
+ if (!pubkey) {
177
+ log.warn(tag, `No pubkey found in cache for ${key}, skipping`);
178
+ continue;
179
+ }
180
+
181
+ // Trigger refresh by calling getBalance with waitForFresh=true
182
+ // This will queue a background refresh job
183
+ log.debug(tag, `Queueing refresh for ${caip} (age: ${Math.round((now - parsed.timestamp) / 1000 / 60)}min)`);
184
+
185
+ // Use setImmediate to avoid blocking the scan
186
+ setImmediate(async () => {
187
+ try {
188
+ await this.balanceCache.getBalance(caip, pubkey, true);
189
+ refreshCount++;
190
+ } catch (error) {
191
+ log.error(tag, `Failed to refresh ${caip}/${pubkey}:`, error);
192
+ }
193
+ });
194
+
195
+ } catch (error) {
196
+ log.error(tag, `Error queueing refresh for ${key}:`, error);
197
+ }
198
+ }
199
+ }
200
+
201
+ const scanTime = Date.now() - startTime;
202
+ log.info(tag, `✅ Scan complete: ${totalScanned} scanned, ${staleKeys.length} stale, ${refreshCount} queued in ${scanTime}ms`);
203
+
204
+ } catch (error) {
205
+ log.error(tag, 'Error during scan:', error);
206
+ }
207
+ }
208
+
209
+ /**
210
+ * Get scanner statistics
211
+ */
212
+ getStats(): any {
213
+ return {
214
+ isRunning: this.isRunning,
215
+ config: {
216
+ staleThresholdHours: this.config.staleThresholdMs / (1000 * 60 * 60),
217
+ scanIntervalMinutes: this.config.scanIntervalMs / (1000 * 60),
218
+ batchSize: this.config.batchSize,
219
+ maxRefreshPerScan: this.config.maxRefreshPerScan
220
+ }
221
+ };
222
+ }
223
+ }
package/tsconfig.json CHANGED
@@ -2,17 +2,26 @@
2
2
  "compilerOptions": {
3
3
  "target": "ES2020",
4
4
  "module": "commonjs",
5
- "lib": ["ES2020"],
5
+ "lib": [
6
+ "ES2020",
7
+ "DOM"
8
+ ],
6
9
  "declaration": true,
7
10
  "outDir": "./dist",
8
11
  "rootDir": "./src",
9
12
  "strict": true,
10
13
  "esModuleInterop": true,
11
- "skipLibCheck": true,
12
14
  "forceConsistentCasingInFileNames": true,
13
15
  "resolveJsonModule": true,
14
- "moduleResolution": "node"
16
+ "moduleResolution": "node",
17
+ "skipLibCheck": true
15
18
  },
16
- "include": ["src/**/*"],
17
- "exclude": ["node_modules", "dist", "**/*.test.ts"]
19
+ "include": [
20
+ "src/**/*"
21
+ ],
22
+ "exclude": [
23
+ "node_modules",
24
+ "dist",
25
+ "**/*.test.ts"
26
+ ]
18
27
  }