@pioneer-platform/pioneer-cache 1.19.3 → 1.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +1 -1
- package/CHANGELOG.md +12 -0
- package/dist/core/cache-manager.d.ts +1 -0
- package/dist/core/cache-manager.js +16 -0
- package/dist/stores/balance-cache.js +11 -2
- package/dist/workers/stale-balance-scanner.d.ts +35 -0
- package/dist/workers/stale-balance-scanner.js +180 -0
- package/package.json +3 -3
- package/src/core/cache-manager.ts +20 -0
- package/src/stores/balance-cache.ts +14 -2
- package/src/workers/stale-balance-scanner.ts +223 -0
- package/tsconfig.json +1 -2
package/.turbo/turbo-build.log
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
|
|
2
2
|
|
|
3
|
-
> @pioneer-platform/pioneer-cache@1.
|
|
3
|
+
> @pioneer-platform/pioneer-cache@1.21.0 build /Users/highlander/WebstormProjects/keepkey-stack/projects/pioneer/modules/pioneer/pioneer-cache
|
|
4
4
|
> tsc
|
|
5
5
|
|
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,17 @@
|
|
|
1
1
|
# @pioneer-platform/pioneer-cache
|
|
2
2
|
|
|
3
|
+
## 1.21.0
|
|
4
|
+
|
|
5
|
+
### Minor Changes
|
|
6
|
+
|
|
7
|
+
- feat(monorepo): production release preparation with strict TypeScript and enhanced swap functionality
|
|
8
|
+
|
|
9
|
+
## 1.20.0
|
|
10
|
+
|
|
11
|
+
### Minor Changes
|
|
12
|
+
|
|
13
|
+
- feat(pioneer-client): add client-side host override for staging/blue testing
|
|
14
|
+
|
|
3
15
|
## 1.19.3
|
|
4
16
|
|
|
5
17
|
### Patch Changes
|
|
@@ -14,6 +14,7 @@ const transaction_cache_1 = require("../stores/transaction-cache");
|
|
|
14
14
|
const staking_cache_1 = require("../stores/staking-cache");
|
|
15
15
|
const zapper_cache_1 = require("../stores/zapper-cache");
|
|
16
16
|
const refresh_worker_1 = require("../workers/refresh-worker");
|
|
17
|
+
const stale_balance_scanner_1 = require("../workers/stale-balance-scanner");
|
|
17
18
|
const log = require('@pioneer-platform/loggerdog')();
|
|
18
19
|
const TAG = ' | CacheManager | ';
|
|
19
20
|
/**
|
|
@@ -139,6 +140,17 @@ class CacheManager {
|
|
|
139
140
|
this.workers.push(worker);
|
|
140
141
|
log.info(tag, '✅ Refresh worker started for all caches');
|
|
141
142
|
}
|
|
143
|
+
// Start stale balance scanner if balance cache is enabled
|
|
144
|
+
if (this.balanceCache) {
|
|
145
|
+
this.staleScanner = new stale_balance_scanner_1.StaleBalanceScanner(this.redis, this.balanceCache, {
|
|
146
|
+
staleThresholdMs: 60 * 60 * 1000, // 1 hour (more aggressive than the 5min reactive threshold)
|
|
147
|
+
scanIntervalMs: 10 * 60 * 1000, // Scan every 10 minutes
|
|
148
|
+
batchSize: 100,
|
|
149
|
+
maxRefreshPerScan: 50
|
|
150
|
+
});
|
|
151
|
+
await this.staleScanner.start();
|
|
152
|
+
log.info(tag, '✅ Stale balance scanner started');
|
|
153
|
+
}
|
|
142
154
|
}
|
|
143
155
|
catch (error) {
|
|
144
156
|
log.error(tag, '❌ Failed to start workers:', error);
|
|
@@ -155,6 +167,10 @@ class CacheManager {
|
|
|
155
167
|
for (const worker of this.workers) {
|
|
156
168
|
await worker.stop();
|
|
157
169
|
}
|
|
170
|
+
if (this.staleScanner) {
|
|
171
|
+
await this.staleScanner.stop();
|
|
172
|
+
this.staleScanner = undefined;
|
|
173
|
+
}
|
|
158
174
|
this.workers = [];
|
|
159
175
|
log.info(tag, '✅ All workers stopped');
|
|
160
176
|
}
|
|
@@ -110,7 +110,15 @@ class BalanceCache extends base_cache_1.BaseCache {
|
|
|
110
110
|
const owner = { pubkey };
|
|
111
111
|
const balanceInfo = await this.balanceModule.getBalance(asset, owner);
|
|
112
112
|
if (!balanceInfo || !balanceInfo.balance) {
|
|
113
|
-
|
|
113
|
+
// CRITICAL FIX: Distinguish between legitimate zero balance and failed API call
|
|
114
|
+
// Failed API calls should NOT be cached as valid "0" balance
|
|
115
|
+
const errorMsg = balanceInfo?.error || 'No balance data returned';
|
|
116
|
+
log.warn(tag, `No balance returned for ${caip}/${sanitizePubkey(pubkey)} - ${errorMsg}`);
|
|
117
|
+
// If balanceInfo explicitly indicates an error, throw it instead of caching "0"
|
|
118
|
+
if (balanceInfo?.error || balanceInfo === null || balanceInfo === undefined) {
|
|
119
|
+
throw new Error(`Failed to fetch balance for ${caip}: ${errorMsg}`);
|
|
120
|
+
}
|
|
121
|
+
// Otherwise, this is a legitimate zero balance (balanceInfo exists but balance is falsy)
|
|
114
122
|
const now = Date.now();
|
|
115
123
|
// Get asset metadata even for zero balances
|
|
116
124
|
const assetData = require('@pioneer-platform/pioneer-discovery').assetData;
|
|
@@ -122,7 +130,8 @@ class BalanceCache extends base_cache_1.BaseCache {
|
|
|
122
130
|
decimals: assetInfo.decimals,
|
|
123
131
|
precision: assetInfo.decimals, // Set precision to decimals for compatibility
|
|
124
132
|
fetchedAt: now,
|
|
125
|
-
fetchedAtISO: new Date(now).toISOString()
|
|
133
|
+
fetchedAtISO: new Date(now).toISOString(),
|
|
134
|
+
dataSource: 'Zero balance (legitimate)'
|
|
126
135
|
};
|
|
127
136
|
}
|
|
128
137
|
// Get asset metadata
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import type { BalanceCache } from '../stores/balance-cache';
|
|
2
|
+
export interface StaleScannerConfig {
|
|
3
|
+
staleThresholdMs: number;
|
|
4
|
+
scanIntervalMs: number;
|
|
5
|
+
batchSize: number;
|
|
6
|
+
maxRefreshPerScan: number;
|
|
7
|
+
}
|
|
8
|
+
export declare class StaleBalanceScanner {
|
|
9
|
+
private redis;
|
|
10
|
+
private balanceCache;
|
|
11
|
+
private config;
|
|
12
|
+
private isRunning;
|
|
13
|
+
private scanTimer?;
|
|
14
|
+
constructor(redis: any, balanceCache: BalanceCache, config?: Partial<StaleScannerConfig>);
|
|
15
|
+
/**
|
|
16
|
+
* Start the scanner
|
|
17
|
+
*/
|
|
18
|
+
start(): Promise<void>;
|
|
19
|
+
/**
|
|
20
|
+
* Stop the scanner
|
|
21
|
+
*/
|
|
22
|
+
stop(): Promise<void>;
|
|
23
|
+
/**
|
|
24
|
+
* Schedule next scan
|
|
25
|
+
*/
|
|
26
|
+
private scheduleNextScan;
|
|
27
|
+
/**
|
|
28
|
+
* Scan for stale balances and queue refreshes
|
|
29
|
+
*/
|
|
30
|
+
private scan;
|
|
31
|
+
/**
|
|
32
|
+
* Get scanner statistics
|
|
33
|
+
*/
|
|
34
|
+
getStats(): any;
|
|
35
|
+
}
|
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/*
|
|
3
|
+
Stale Balance Scanner
|
|
4
|
+
|
|
5
|
+
Proactively scans for stale balance cache entries and queues them for refresh.
|
|
6
|
+
This ensures balances that aren't actively accessed still get refreshed.
|
|
7
|
+
|
|
8
|
+
Prevents the issue where stale balances sit for days/weeks without being updated.
|
|
9
|
+
*/
|
|
10
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
11
|
+
exports.StaleBalanceScanner = void 0;
|
|
12
|
+
const log = require('@pioneer-platform/loggerdog')();
|
|
13
|
+
const TAG = ' | StaleBalanceScanner | ';
|
|
14
|
+
class StaleBalanceScanner {
|
|
15
|
+
constructor(redis, balanceCache, config) {
|
|
16
|
+
this.isRunning = false;
|
|
17
|
+
this.redis = redis;
|
|
18
|
+
this.balanceCache = balanceCache;
|
|
19
|
+
this.config = {
|
|
20
|
+
staleThresholdMs: config?.staleThresholdMs || 60 * 60 * 1000, // 1 hour
|
|
21
|
+
scanIntervalMs: config?.scanIntervalMs || 5 * 60 * 1000, // 5 minutes
|
|
22
|
+
batchSize: config?.batchSize || 100,
|
|
23
|
+
maxRefreshPerScan: config?.maxRefreshPerScan || 50
|
|
24
|
+
};
|
|
25
|
+
log.info(TAG, 'Stale balance scanner initialized', {
|
|
26
|
+
staleThresholdHours: this.config.staleThresholdMs / (1000 * 60 * 60),
|
|
27
|
+
scanIntervalMinutes: this.config.scanIntervalMs / (1000 * 60)
|
|
28
|
+
});
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Start the scanner
|
|
32
|
+
*/
|
|
33
|
+
async start() {
|
|
34
|
+
const tag = TAG + 'start | ';
|
|
35
|
+
if (this.isRunning) {
|
|
36
|
+
log.warn(tag, 'Scanner already running');
|
|
37
|
+
return;
|
|
38
|
+
}
|
|
39
|
+
this.isRunning = true;
|
|
40
|
+
log.info(tag, '🔍 Starting stale balance scanner');
|
|
41
|
+
// Run first scan immediately, then schedule recurring
|
|
42
|
+
await this.scan();
|
|
43
|
+
this.scheduleNextScan();
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Stop the scanner
|
|
47
|
+
*/
|
|
48
|
+
async stop() {
|
|
49
|
+
const tag = TAG + 'stop | ';
|
|
50
|
+
log.info(tag, 'Stopping stale balance scanner');
|
|
51
|
+
this.isRunning = false;
|
|
52
|
+
if (this.scanTimer) {
|
|
53
|
+
clearTimeout(this.scanTimer);
|
|
54
|
+
this.scanTimer = undefined;
|
|
55
|
+
}
|
|
56
|
+
log.info(tag, '✅ Scanner stopped');
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* Schedule next scan
|
|
60
|
+
*/
|
|
61
|
+
scheduleNextScan() {
|
|
62
|
+
if (!this.isRunning)
|
|
63
|
+
return;
|
|
64
|
+
this.scanTimer = setTimeout(async () => {
|
|
65
|
+
await this.scan();
|
|
66
|
+
this.scheduleNextScan();
|
|
67
|
+
}, this.config.scanIntervalMs);
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* Scan for stale balances and queue refreshes
|
|
71
|
+
*/
|
|
72
|
+
async scan() {
|
|
73
|
+
const tag = TAG + 'scan | ';
|
|
74
|
+
const startTime = Date.now();
|
|
75
|
+
try {
|
|
76
|
+
log.info(tag, 'Starting stale balance scan...');
|
|
77
|
+
// Scan balance_v2:* keys in batches
|
|
78
|
+
const pattern = 'balance_v2:*';
|
|
79
|
+
const staleKeys = [];
|
|
80
|
+
const now = Date.now();
|
|
81
|
+
let cursor = '0';
|
|
82
|
+
let totalScanned = 0;
|
|
83
|
+
let refreshCount = 0;
|
|
84
|
+
do {
|
|
85
|
+
// SCAN with COUNT for batching
|
|
86
|
+
const [nextCursor, keys] = await this.redis.scan(cursor, 'MATCH', pattern, 'COUNT', this.config.batchSize);
|
|
87
|
+
cursor = nextCursor;
|
|
88
|
+
totalScanned += keys.length;
|
|
89
|
+
// Check each key for staleness
|
|
90
|
+
for (const key of keys) {
|
|
91
|
+
try {
|
|
92
|
+
const cached = await this.redis.get(key);
|
|
93
|
+
if (!cached)
|
|
94
|
+
continue;
|
|
95
|
+
const parsed = JSON.parse(cached);
|
|
96
|
+
const age = now - (parsed.timestamp || 0);
|
|
97
|
+
// Check if stale
|
|
98
|
+
if (age > this.config.staleThresholdMs) {
|
|
99
|
+
staleKeys.push(key);
|
|
100
|
+
// Stop if we've hit max refresh limit
|
|
101
|
+
if (staleKeys.length >= this.config.maxRefreshPerScan) {
|
|
102
|
+
log.info(tag, `Reached max refresh limit (${this.config.maxRefreshPerScan}), stopping scan`);
|
|
103
|
+
cursor = '0'; // Force exit
|
|
104
|
+
break;
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
catch (error) {
|
|
109
|
+
log.error(tag, `Error checking key ${key}:`, error);
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
} while (cursor !== '0' && this.isRunning);
|
|
113
|
+
// Queue refresh for stale balances
|
|
114
|
+
if (staleKeys.length > 0) {
|
|
115
|
+
log.info(tag, `Found ${staleKeys.length} stale balances out of ${totalScanned} scanned`);
|
|
116
|
+
for (const key of staleKeys) {
|
|
117
|
+
try {
|
|
118
|
+
// Extract CAIP and pubkey hash from key
|
|
119
|
+
// Key format: balance_v2:caip:hashedPubkey
|
|
120
|
+
// IMPORTANT: CAIP itself contains colons (e.g., ripple:4109c6f2045fc7eff4cde8f9905d19c2/slip44:144)
|
|
121
|
+
// So we must split from the right to get hashedPubkey, then everything else is the CAIP
|
|
122
|
+
const withoutPrefix = key.replace('balance_v2:', '');
|
|
123
|
+
const lastColonIndex = withoutPrefix.lastIndexOf(':');
|
|
124
|
+
if (lastColonIndex === -1)
|
|
125
|
+
continue;
|
|
126
|
+
const caip = withoutPrefix.substring(0, lastColonIndex);
|
|
127
|
+
const pubkeyHash = withoutPrefix.substring(lastColonIndex + 1);
|
|
128
|
+
// We need the original pubkey to refresh, but it's hashed in the key
|
|
129
|
+
// So we need to get it from the cached value
|
|
130
|
+
const cached = await this.redis.get(key);
|
|
131
|
+
if (!cached)
|
|
132
|
+
continue;
|
|
133
|
+
const parsed = JSON.parse(cached);
|
|
134
|
+
const pubkey = parsed.value?.pubkey;
|
|
135
|
+
if (!pubkey) {
|
|
136
|
+
log.warn(tag, `No pubkey found in cache for ${key}, skipping`);
|
|
137
|
+
continue;
|
|
138
|
+
}
|
|
139
|
+
// Trigger refresh by calling getBalance with waitForFresh=true
|
|
140
|
+
// This will queue a background refresh job
|
|
141
|
+
log.debug(tag, `Queueing refresh for ${caip} (age: ${Math.round((now - parsed.timestamp) / 1000 / 60)}min)`);
|
|
142
|
+
// Use setImmediate to avoid blocking the scan
|
|
143
|
+
setImmediate(async () => {
|
|
144
|
+
try {
|
|
145
|
+
await this.balanceCache.getBalance(caip, pubkey, true);
|
|
146
|
+
refreshCount++;
|
|
147
|
+
}
|
|
148
|
+
catch (error) {
|
|
149
|
+
log.error(tag, `Failed to refresh ${caip}/${pubkey}:`, error);
|
|
150
|
+
}
|
|
151
|
+
});
|
|
152
|
+
}
|
|
153
|
+
catch (error) {
|
|
154
|
+
log.error(tag, `Error queueing refresh for ${key}:`, error);
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
const scanTime = Date.now() - startTime;
|
|
159
|
+
log.info(tag, `✅ Scan complete: ${totalScanned} scanned, ${staleKeys.length} stale, ${refreshCount} queued in ${scanTime}ms`);
|
|
160
|
+
}
|
|
161
|
+
catch (error) {
|
|
162
|
+
log.error(tag, 'Error during scan:', error);
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
/**
|
|
166
|
+
* Get scanner statistics
|
|
167
|
+
*/
|
|
168
|
+
getStats() {
|
|
169
|
+
return {
|
|
170
|
+
isRunning: this.isRunning,
|
|
171
|
+
config: {
|
|
172
|
+
staleThresholdHours: this.config.staleThresholdMs / (1000 * 60 * 60),
|
|
173
|
+
scanIntervalMinutes: this.config.scanIntervalMs / (1000 * 60),
|
|
174
|
+
batchSize: this.config.batchSize,
|
|
175
|
+
maxRefreshPerScan: this.config.maxRefreshPerScan
|
|
176
|
+
}
|
|
177
|
+
};
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
exports.StaleBalanceScanner = StaleBalanceScanner;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@pioneer-platform/pioneer-cache",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.21.0",
|
|
4
4
|
"description": "Unified caching system for Pioneer platform with Redis backend",
|
|
5
5
|
"main": "./dist/index.js",
|
|
6
6
|
"types": "./dist/index.d.ts",
|
|
@@ -15,8 +15,8 @@
|
|
|
15
15
|
"license": "MIT",
|
|
16
16
|
"dependencies": {
|
|
17
17
|
"@pioneer-platform/loggerdog": "8.11.0",
|
|
18
|
-
"@pioneer-platform/redis
|
|
19
|
-
"@pioneer-platform/
|
|
18
|
+
"@pioneer-platform/default-redis": "8.11.7",
|
|
19
|
+
"@pioneer-platform/redis-queue": "8.12.17"
|
|
20
20
|
},
|
|
21
21
|
"devDependencies": {
|
|
22
22
|
"@types/jest": "^29.5.0",
|
|
@@ -12,6 +12,7 @@ import { TransactionCache } from '../stores/transaction-cache';
|
|
|
12
12
|
import { StakingCache } from '../stores/staking-cache';
|
|
13
13
|
import { ZapperCache } from '../stores/zapper-cache';
|
|
14
14
|
import { RefreshWorker, startUnifiedWorker } from '../workers/refresh-worker';
|
|
15
|
+
import { StaleBalanceScanner } from '../workers/stale-balance-scanner';
|
|
15
16
|
import type { BaseCache } from './base-cache';
|
|
16
17
|
import type { HealthCheckResult } from '../types';
|
|
17
18
|
|
|
@@ -53,6 +54,7 @@ export class CacheManager {
|
|
|
53
54
|
private stakingCache?: StakingCache;
|
|
54
55
|
private zapperCache?: ZapperCache;
|
|
55
56
|
private workers: RefreshWorker[] = [];
|
|
57
|
+
private staleScanner?: StaleBalanceScanner;
|
|
56
58
|
|
|
57
59
|
constructor(config: CacheManagerConfig) {
|
|
58
60
|
this.redis = config.redis;
|
|
@@ -196,6 +198,19 @@ export class CacheManager {
|
|
|
196
198
|
log.info(tag, '✅ Refresh worker started for all caches');
|
|
197
199
|
}
|
|
198
200
|
|
|
201
|
+
// Start stale balance scanner if balance cache is enabled
|
|
202
|
+
if (this.balanceCache) {
|
|
203
|
+
this.staleScanner = new StaleBalanceScanner(this.redis, this.balanceCache, {
|
|
204
|
+
staleThresholdMs: 60 * 60 * 1000, // 1 hour (more aggressive than the 5min reactive threshold)
|
|
205
|
+
scanIntervalMs: 10 * 60 * 1000, // Scan every 10 minutes
|
|
206
|
+
batchSize: 100,
|
|
207
|
+
maxRefreshPerScan: 50
|
|
208
|
+
});
|
|
209
|
+
|
|
210
|
+
await this.staleScanner.start();
|
|
211
|
+
log.info(tag, '✅ Stale balance scanner started');
|
|
212
|
+
}
|
|
213
|
+
|
|
199
214
|
} catch (error) {
|
|
200
215
|
log.error(tag, '❌ Failed to start workers:', error);
|
|
201
216
|
throw error;
|
|
@@ -215,6 +230,11 @@ export class CacheManager {
|
|
|
215
230
|
await worker.stop();
|
|
216
231
|
}
|
|
217
232
|
|
|
233
|
+
if (this.staleScanner) {
|
|
234
|
+
await this.staleScanner.stop();
|
|
235
|
+
this.staleScanner = undefined;
|
|
236
|
+
}
|
|
237
|
+
|
|
218
238
|
this.workers = [];
|
|
219
239
|
log.info(tag, '✅ All workers stopped');
|
|
220
240
|
|
|
@@ -148,7 +148,18 @@ export class BalanceCache extends BaseCache<BalanceData> {
|
|
|
148
148
|
const balanceInfo = await this.balanceModule.getBalance(asset, owner);
|
|
149
149
|
|
|
150
150
|
if (!balanceInfo || !balanceInfo.balance) {
|
|
151
|
-
|
|
151
|
+
// CRITICAL FIX: Distinguish between legitimate zero balance and failed API call
|
|
152
|
+
// Failed API calls should NOT be cached as valid "0" balance
|
|
153
|
+
const errorMsg = balanceInfo?.error || 'No balance data returned';
|
|
154
|
+
|
|
155
|
+
log.warn(tag, `No balance returned for ${caip}/${sanitizePubkey(pubkey)} - ${errorMsg}`);
|
|
156
|
+
|
|
157
|
+
// If balanceInfo explicitly indicates an error, throw it instead of caching "0"
|
|
158
|
+
if (balanceInfo?.error || balanceInfo === null || balanceInfo === undefined) {
|
|
159
|
+
throw new Error(`Failed to fetch balance for ${caip}: ${errorMsg}`);
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
// Otherwise, this is a legitimate zero balance (balanceInfo exists but balance is falsy)
|
|
152
163
|
const now = Date.now();
|
|
153
164
|
|
|
154
165
|
// Get asset metadata even for zero balances
|
|
@@ -162,7 +173,8 @@ export class BalanceCache extends BaseCache<BalanceData> {
|
|
|
162
173
|
decimals: assetInfo.decimals,
|
|
163
174
|
precision: assetInfo.decimals, // Set precision to decimals for compatibility
|
|
164
175
|
fetchedAt: now,
|
|
165
|
-
fetchedAtISO: new Date(now).toISOString()
|
|
176
|
+
fetchedAtISO: new Date(now).toISOString(),
|
|
177
|
+
dataSource: 'Zero balance (legitimate)'
|
|
166
178
|
};
|
|
167
179
|
}
|
|
168
180
|
|
|
@@ -0,0 +1,223 @@
|
|
|
1
|
+
/*
|
|
2
|
+
Stale Balance Scanner
|
|
3
|
+
|
|
4
|
+
Proactively scans for stale balance cache entries and queues them for refresh.
|
|
5
|
+
This ensures balances that aren't actively accessed still get refreshed.
|
|
6
|
+
|
|
7
|
+
Prevents the issue where stale balances sit for days/weeks without being updated.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import type { BalanceCache } from '../stores/balance-cache';
|
|
11
|
+
|
|
12
|
+
const log = require('@pioneer-platform/loggerdog')();
|
|
13
|
+
const TAG = ' | StaleBalanceScanner | ';
|
|
14
|
+
|
|
15
|
+
export interface StaleScannerConfig {
|
|
16
|
+
staleThresholdMs: number; // Refresh balances older than this (default: 1 hour)
|
|
17
|
+
scanIntervalMs: number; // How often to scan (default: 5 minutes)
|
|
18
|
+
batchSize: number; // How many keys to scan per batch (default: 100)
|
|
19
|
+
maxRefreshPerScan: number; // Max balances to refresh per scan (default: 50)
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export class StaleBalanceScanner {
|
|
23
|
+
private redis: any;
|
|
24
|
+
private balanceCache: BalanceCache;
|
|
25
|
+
private config: StaleScannerConfig;
|
|
26
|
+
private isRunning: boolean = false;
|
|
27
|
+
private scanTimer?: NodeJS.Timeout;
|
|
28
|
+
|
|
29
|
+
constructor(redis: any, balanceCache: BalanceCache, config?: Partial<StaleScannerConfig>) {
|
|
30
|
+
this.redis = redis;
|
|
31
|
+
this.balanceCache = balanceCache;
|
|
32
|
+
|
|
33
|
+
this.config = {
|
|
34
|
+
staleThresholdMs: config?.staleThresholdMs || 60 * 60 * 1000, // 1 hour
|
|
35
|
+
scanIntervalMs: config?.scanIntervalMs || 5 * 60 * 1000, // 5 minutes
|
|
36
|
+
batchSize: config?.batchSize || 100,
|
|
37
|
+
maxRefreshPerScan: config?.maxRefreshPerScan || 50
|
|
38
|
+
};
|
|
39
|
+
|
|
40
|
+
log.info(TAG, 'Stale balance scanner initialized', {
|
|
41
|
+
staleThresholdHours: this.config.staleThresholdMs / (1000 * 60 * 60),
|
|
42
|
+
scanIntervalMinutes: this.config.scanIntervalMs / (1000 * 60)
|
|
43
|
+
});
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* Start the scanner
|
|
48
|
+
*/
|
|
49
|
+
async start(): Promise<void> {
|
|
50
|
+
const tag = TAG + 'start | ';
|
|
51
|
+
|
|
52
|
+
if (this.isRunning) {
|
|
53
|
+
log.warn(tag, 'Scanner already running');
|
|
54
|
+
return;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
this.isRunning = true;
|
|
58
|
+
log.info(tag, '🔍 Starting stale balance scanner');
|
|
59
|
+
|
|
60
|
+
// Run first scan immediately, then schedule recurring
|
|
61
|
+
await this.scan();
|
|
62
|
+
this.scheduleNextScan();
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* Stop the scanner
|
|
67
|
+
*/
|
|
68
|
+
async stop(): Promise<void> {
|
|
69
|
+
const tag = TAG + 'stop | ';
|
|
70
|
+
|
|
71
|
+
log.info(tag, 'Stopping stale balance scanner');
|
|
72
|
+
this.isRunning = false;
|
|
73
|
+
|
|
74
|
+
if (this.scanTimer) {
|
|
75
|
+
clearTimeout(this.scanTimer);
|
|
76
|
+
this.scanTimer = undefined;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
log.info(tag, '✅ Scanner stopped');
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* Schedule next scan
|
|
84
|
+
*/
|
|
85
|
+
private scheduleNextScan(): void {
|
|
86
|
+
if (!this.isRunning) return;
|
|
87
|
+
|
|
88
|
+
this.scanTimer = setTimeout(async () => {
|
|
89
|
+
await this.scan();
|
|
90
|
+
this.scheduleNextScan();
|
|
91
|
+
}, this.config.scanIntervalMs);
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/**
|
|
95
|
+
* Scan for stale balances and queue refreshes
|
|
96
|
+
*/
|
|
97
|
+
private async scan(): Promise<void> {
|
|
98
|
+
const tag = TAG + 'scan | ';
|
|
99
|
+
const startTime = Date.now();
|
|
100
|
+
|
|
101
|
+
try {
|
|
102
|
+
log.info(tag, 'Starting stale balance scan...');
|
|
103
|
+
|
|
104
|
+
// Scan balance_v2:* keys in batches
|
|
105
|
+
const pattern = 'balance_v2:*';
|
|
106
|
+
const staleKeys: string[] = [];
|
|
107
|
+
const now = Date.now();
|
|
108
|
+
|
|
109
|
+
let cursor = '0';
|
|
110
|
+
let totalScanned = 0;
|
|
111
|
+
let refreshCount = 0;
|
|
112
|
+
|
|
113
|
+
do {
|
|
114
|
+
// SCAN with COUNT for batching
|
|
115
|
+
const [nextCursor, keys] = await this.redis.scan(
|
|
116
|
+
cursor,
|
|
117
|
+
'MATCH', pattern,
|
|
118
|
+
'COUNT', this.config.batchSize
|
|
119
|
+
);
|
|
120
|
+
|
|
121
|
+
cursor = nextCursor;
|
|
122
|
+
totalScanned += keys.length;
|
|
123
|
+
|
|
124
|
+
// Check each key for staleness
|
|
125
|
+
for (const key of keys) {
|
|
126
|
+
try {
|
|
127
|
+
const cached = await this.redis.get(key);
|
|
128
|
+
if (!cached) continue;
|
|
129
|
+
|
|
130
|
+
const parsed = JSON.parse(cached);
|
|
131
|
+
const age = now - (parsed.timestamp || 0);
|
|
132
|
+
|
|
133
|
+
// Check if stale
|
|
134
|
+
if (age > this.config.staleThresholdMs) {
|
|
135
|
+
staleKeys.push(key);
|
|
136
|
+
|
|
137
|
+
// Stop if we've hit max refresh limit
|
|
138
|
+
if (staleKeys.length >= this.config.maxRefreshPerScan) {
|
|
139
|
+
log.info(tag, `Reached max refresh limit (${this.config.maxRefreshPerScan}), stopping scan`);
|
|
140
|
+
cursor = '0'; // Force exit
|
|
141
|
+
break;
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
} catch (error) {
|
|
145
|
+
log.error(tag, `Error checking key ${key}:`, error);
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
} while (cursor !== '0' && this.isRunning);
|
|
150
|
+
|
|
151
|
+
// Queue refresh for stale balances
|
|
152
|
+
if (staleKeys.length > 0) {
|
|
153
|
+
log.info(tag, `Found ${staleKeys.length} stale balances out of ${totalScanned} scanned`);
|
|
154
|
+
|
|
155
|
+
for (const key of staleKeys) {
|
|
156
|
+
try {
|
|
157
|
+
// Extract CAIP and pubkey hash from key
|
|
158
|
+
// Key format: balance_v2:caip:hashedPubkey
|
|
159
|
+
// IMPORTANT: CAIP itself contains colons (e.g., ripple:4109c6f2045fc7eff4cde8f9905d19c2/slip44:144)
|
|
160
|
+
// So we must split from the right to get hashedPubkey, then everything else is the CAIP
|
|
161
|
+
const withoutPrefix = key.replace('balance_v2:', '');
|
|
162
|
+
const lastColonIndex = withoutPrefix.lastIndexOf(':');
|
|
163
|
+
if (lastColonIndex === -1) continue;
|
|
164
|
+
|
|
165
|
+
const caip = withoutPrefix.substring(0, lastColonIndex);
|
|
166
|
+
const pubkeyHash = withoutPrefix.substring(lastColonIndex + 1);
|
|
167
|
+
|
|
168
|
+
// We need the original pubkey to refresh, but it's hashed in the key
|
|
169
|
+
// So we need to get it from the cached value
|
|
170
|
+
const cached = await this.redis.get(key);
|
|
171
|
+
if (!cached) continue;
|
|
172
|
+
|
|
173
|
+
const parsed = JSON.parse(cached);
|
|
174
|
+
const pubkey = parsed.value?.pubkey;
|
|
175
|
+
|
|
176
|
+
if (!pubkey) {
|
|
177
|
+
log.warn(tag, `No pubkey found in cache for ${key}, skipping`);
|
|
178
|
+
continue;
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
// Trigger refresh by calling getBalance with waitForFresh=true
|
|
182
|
+
// This will queue a background refresh job
|
|
183
|
+
log.debug(tag, `Queueing refresh for ${caip} (age: ${Math.round((now - parsed.timestamp) / 1000 / 60)}min)`);
|
|
184
|
+
|
|
185
|
+
// Use setImmediate to avoid blocking the scan
|
|
186
|
+
setImmediate(async () => {
|
|
187
|
+
try {
|
|
188
|
+
await this.balanceCache.getBalance(caip, pubkey, true);
|
|
189
|
+
refreshCount++;
|
|
190
|
+
} catch (error) {
|
|
191
|
+
log.error(tag, `Failed to refresh ${caip}/${pubkey}:`, error);
|
|
192
|
+
}
|
|
193
|
+
});
|
|
194
|
+
|
|
195
|
+
} catch (error) {
|
|
196
|
+
log.error(tag, `Error queueing refresh for ${key}:`, error);
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
const scanTime = Date.now() - startTime;
|
|
202
|
+
log.info(tag, `✅ Scan complete: ${totalScanned} scanned, ${staleKeys.length} stale, ${refreshCount} queued in ${scanTime}ms`);
|
|
203
|
+
|
|
204
|
+
} catch (error) {
|
|
205
|
+
log.error(tag, 'Error during scan:', error);
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
/**
|
|
210
|
+
* Get scanner statistics
|
|
211
|
+
*/
|
|
212
|
+
getStats(): any {
|
|
213
|
+
return {
|
|
214
|
+
isRunning: this.isRunning,
|
|
215
|
+
config: {
|
|
216
|
+
staleThresholdHours: this.config.staleThresholdMs / (1000 * 60 * 60),
|
|
217
|
+
scanIntervalMinutes: this.config.scanIntervalMs / (1000 * 60),
|
|
218
|
+
batchSize: this.config.batchSize,
|
|
219
|
+
maxRefreshPerScan: this.config.maxRefreshPerScan
|
|
220
|
+
}
|
|
221
|
+
};
|
|
222
|
+
}
|
|
223
|
+
}
|
package/tsconfig.json
CHANGED
|
@@ -2,13 +2,12 @@
|
|
|
2
2
|
"compilerOptions": {
|
|
3
3
|
"target": "ES2020",
|
|
4
4
|
"module": "commonjs",
|
|
5
|
-
"lib": ["ES2020"],
|
|
5
|
+
"lib": ["ES2020", "DOM"],
|
|
6
6
|
"declaration": true,
|
|
7
7
|
"outDir": "./dist",
|
|
8
8
|
"rootDir": "./src",
|
|
9
9
|
"strict": true,
|
|
10
10
|
"esModuleInterop": true,
|
|
11
|
-
"skipLibCheck": true,
|
|
12
11
|
"forceConsistentCasingInFileNames": true,
|
|
13
12
|
"resolveJsonModule": true,
|
|
14
13
|
"moduleResolution": "node"
|