@pioneer-platform/pioneer-cache 1.0.0

package/src/core/base-cache.ts

@@ -0,0 +1,595 @@
+/*
+    BaseCache - Abstract base class for all cache implementations
+
+    Contains all shared logic that was duplicated across balance/price caches.
+    Specific caches extend this and implement only their unique logic.
+*/
+
+import type {
+    CacheConfig,
+    CachedValue,
+    CacheResult,
+    HealthCheckResult,
+    CacheStats,
+    RefreshJob,
+    SourceFetcher,
+    KeyBuilder,
+    LegacyKeyPattern
+} from '../types';
+
+const log = require('@pioneer-platform/loggerdog')();
+
+export abstract class BaseCache<T> {
+    protected redis: any;
+    protected redisQueue: any;
+    protected config: CacheConfig;
+    protected queueInitialized: boolean = false;
+    protected TAG: string;
+
+    // Cache for stats (to avoid expensive SCAN operations on every health check)
+    private cachedStats: CacheStats | null = null;
+    private cachedStatsTimestamp: number = 0;
+    private readonly STATS_CACHE_TTL = 30000; // 30 seconds
+
+    constructor(redis: any, config: CacheConfig) {
+        this.redis = redis;
+        this.config = config;
+        this.TAG = ` | ${config.name}Cache | `;
+
+        // Initialize queue if enabled
+        if (config.enableQueue) {
+            this.initializeQueue();
+        } else {
+            log.info(this.TAG, `Queue disabled for ${config.name} cache`);
+        }
+    }
+
+    /**
+     * Initialize Redis queue for background refresh
+     */
+    private initializeQueue(): void {
+        try {
+            const redisQueueModule = require('@pioneer-platform/redis-queue');
+            redisQueueModule.init(this.config.queueName);
+            this.redisQueue = redisQueueModule;
+            this.queueInitialized = true;
+            log.info(this.TAG, `✅ Queue initialized: ${this.config.queueName}`);
+        } catch (error) {
+            // CRITICAL FIX #3: Make queue failures VERY visible
+            log.error(this.TAG, `❌ CRITICAL: Failed to initialize queue '${this.config.queueName}'!`);
+            log.error(this.TAG, 'Background refresh is DISABLED - cache will NOT update automatically!');
+            log.error(this.TAG, 'Error:', error);
+            this.redisQueue = null;
+            this.queueInitialized = false;
+        }
+    }
+
+    /**
+     * Main cache get method
+     * Implements stale-while-revalidate pattern with optional blocking
+     */
+    async get(params: Record<string, any>, waitForFresh?: boolean): Promise<CacheResult<T>> {
+        const tag = this.TAG + 'get | ';
+        const startTime = Date.now();
+
+        try {
+            const key = this.buildKey(params);
+
+            // Step 1: Try new cache format
+            const cachedValue = await this.getCached(key);
+
+            if (cachedValue) {
+                const age = Date.now() - cachedValue.timestamp;
+                const responseTime = Date.now() - startTime;
+
+                if (this.config.logCacheHits) {
+                    log.debug(tag, `Cache hit: ${key} (${responseTime}ms, age: ${age}ms)`);
+                }
+
+                // Check staleness async (don't wait)
+                if (this.config.staleThreshold && age > this.config.staleThreshold) {
+                    this.triggerAsyncRefresh(params, 'normal');
+                }
+
+                return {
+                    success: true,
+                    value: cachedValue.value,
+                    cached: true,
+                    fresh: this.config.staleThreshold ? age <= this.config.staleThreshold : true,
+                    age
+                };
+            }
+
+            // Step 2: Try legacy cache fallback
+            if (this.config.enableLegacyFallback) {
+                const legacyValue = await this.getLegacyCached(params);
+                if (legacyValue) {
+                    const responseTime = Date.now() - startTime;
+                    log.info(tag, `Legacy cache hit: ${key} (${responseTime}ms)`);
+
+                    // Migrate to new format (async)
+                    this.migrateLegacyValue(key, legacyValue);
+
+                    return {
+                        success: true,
+                        value: legacyValue,
+                        cached: true,
+                        fresh: false, // Legacy data is considered stale
+                        age: undefined
+                    };
+                }
+            }
+
+            // Step 3: Cache miss
+            const responseTime = Date.now() - startTime;
+            if (this.config.logCacheMisses) {
+                log.info(tag, `Cache miss: ${key} (${responseTime}ms)`);
+            }
+
+            // FIX #1: Optional blocking on cache miss
+            const shouldBlock = waitForFresh !== undefined ? waitForFresh : this.config.blockOnMiss;
+            if (shouldBlock) {
+                log.info(tag, `Blocking to fetch fresh data: ${key}`);
+                const freshValue = await this.fetchFresh(params);
+                return {
+                    success: true,
+                    value: freshValue,
+                    cached: false,
+                    fresh: true,
+                    age: 0
+                };
+            }
+
+            // Non-blocking: trigger async refresh and return default
+            this.triggerAsyncRefresh(params, 'high');
+
+            return {
+                success: true,
+                value: this.config.defaultValue,
+                cached: false,
+                fresh: false,
+                age: undefined
+            };
+
+        } catch (error) {
+            log.error(tag, 'Error getting cache value:', error);
+            return {
+                success: false,
+                value: this.config.defaultValue,
+                cached: false,
+                fresh: false,
+                error: error instanceof Error ? error.message : String(error)
+            };
+        }
+    }
+
+    /**
+     * Get value from cache
+     */
+    protected async getCached(key: string): Promise<CachedValue<T> | null> {
+        const tag = this.TAG + 'getCached | ';
+
+        try {
+            const cached = await this.redis.get(key);
+            if (!cached) {
+                return null;
+            }
+
+            const parsed: CachedValue<T> = JSON.parse(cached);
+
+            // Validate structure
+            if (!parsed.value || typeof parsed.timestamp !== 'number') {
+                log.warn(tag, `Invalid cache structure for ${key}, removing`);
+                await this.redis.del(key);
+                return null;
+            }
+
+            return parsed;
+
+        } catch (error) {
+            log.error(tag, `Error parsing cached value for ${key}:`, error);
+            return null;
+        }
+    }
+
+    /**
+     * Update cache with new value
+     * FIX #2: Always includes TTL
+     */
+    async updateCache(key: string, value: T, metadata?: Record<string, any>): Promise<void> {
+        const tag = this.TAG + 'updateCache | ';
+
+        try {
+            const cachedValue: CachedValue<T> = {
+                value,
+                timestamp: Date.now(),
+                source: 'network',
+                lastUpdated: new Date().toISOString(),
+                metadata
+            };
+
+            // FIX #2: Always set TTL (unless explicitly disabled)
+            if (this.config.enableTTL) {
+                const ttlSeconds = Math.floor(this.config.ttl / 1000);
+                await this.redis.set(key, JSON.stringify(cachedValue), 'EX', ttlSeconds);
+                log.info(tag, `Updated cache: ${key} [TTL: ${ttlSeconds}s]`);
+            } else {
+                // Permanent caching (for transactions)
+                await this.redis.set(key, JSON.stringify(cachedValue));
+                log.info(tag, `Updated cache: ${key} [PERMANENT]`);
+            }
+
+        } catch (error) {
+            log.error(tag, `Error updating cache for ${key}:`, error);
+            throw error;
+        }
+    }
+
+    /**
+     * Trigger background refresh job
+     * FIX #3: Loud error logging
+     * FIX #4: Synchronous fallback for high-priority
+     */
+    protected triggerAsyncRefresh(params: Record<string, any>, priority: 'high' | 'normal' | 'low' = 'normal'): void {
+        const tag = this.TAG + 'triggerAsyncRefresh | ';
+
+        try {
+            // Check if queue is enabled
+            if (!this.config.enableQueue) {
+                return; // Queue disabled, no background refresh
+            }
+
+            // FIX #3: Fail loudly if queue not initialized
+            if (!this.queueInitialized || !this.redisQueue) {
+                const key = this.buildKey(params);
+                log.error(tag, `❌ QUEUE NOT INITIALIZED! Cannot refresh ${key}`);
+                log.error(tag, `Background refresh is BROKEN - cache will NOT update!`);
+
+                // FIX #4: Synchronous fallback for high-priority
+                if (priority === 'high') {
+                    log.warn(tag, `Using synchronous fallback for high-priority refresh`);
+                    setImmediate(async () => {
+                        try {
+                            await this.fetchFresh(params);
+                        } catch (error) {
+                            log.error(tag, `Synchronous fallback failed:`, error);
+                        }
+                    });
+                }
+                return;
+            }
+
+            const job: RefreshJob = {
+                type: `REFRESH_${this.config.name.toUpperCase()}`,
+                key: this.buildKey(params),
+                params,
+                priority,
+                retryCount: 0,
+                timestamp: Date.now()
+            };
+
+            // Queue job async (don't wait)
+            setImmediate(async () => {
+                try {
+                    await this.redisQueue.createWork(this.config.queueName, job);
+
+                    if (this.config.logRefreshJobs) {
+                        log.debug(tag, `Queued refresh job: ${job.key} (priority: ${priority})`);
+                    }
+
+                } catch (error) {
+                    log.error(tag, `Error queuing refresh job:`, error);
+                }
+            });
+
+        } catch (error) {
+            log.error(tag, `Error triggering refresh:`, error);
+        }
+    }
+
+    /**
+     * Fetch fresh data and update cache
+     * FIX #1 & #4: Used for blocking requests and fallback
+     */
+    async fetchFresh(params: Record<string, any>): Promise<T> {
+        const tag = this.TAG + 'fetchFresh | ';
+        const startTime = Date.now();
+
+        try {
+            const key = this.buildKey(params);
+            log.info(tag, `Fetching fresh data: ${key}`);
+
+            // Call subclass-specific fetch implementation
+            const value = await this.fetchFromSource(params);
+
+            // Update cache
+            await this.updateCache(key, value);
+
+            const fetchTime = Date.now() - startTime;
+            log.info(tag, `✅ Fetched fresh data in ${fetchTime}ms: ${key}`);
+
+            return value;
+
+        } catch (error) {
+            const fetchTime = Date.now() - startTime;
+            log.error(tag, `Failed to fetch fresh data after ${fetchTime}ms:`, error);
+            return this.config.defaultValue;
+        }
+    }
+
+    /**
+     * Migrate legacy cache value to new format
+     * FIX #2: Includes TTL
+     */
+    protected migrateLegacyValue(key: string, value: T): void {
+        const tag = this.TAG + 'migrateLegacyValue | ';
+
+        setImmediate(async () => {
+            try {
+                const cachedValue: CachedValue<T> = {
+                    value,
+                    timestamp: Date.now() - (this.config.staleThreshold || this.config.ttl), // Mark as stale
+                    source: 'legacy',
+                    lastUpdated: new Date().toISOString()
+                };
+
+                // FIX #2: Set TTL on migrated data
+                if (this.config.enableTTL) {
+                    const ttlSeconds = Math.floor(this.config.ttl / 1000);
+                    await this.redis.set(key, JSON.stringify(cachedValue), 'EX', ttlSeconds);
+                    log.debug(tag, `Migrated legacy value: ${key} [TTL: ${ttlSeconds}s]`);
+                } else {
+                    await this.redis.set(key, JSON.stringify(cachedValue));
+                    log.debug(tag, `Migrated legacy value: ${key} [PERMANENT]`);
+                }
+
+            } catch (error) {
+                log.error(tag, `Error migrating legacy value:`, error);
+            }
+        });
+    }
+
+    /**
+     * Get cache statistics
+     * @param forceRefresh - Skip cache and fetch fresh stats (for testing/debugging)
+     */
+    async getCacheStats(forceRefresh: boolean = false): Promise<CacheStats> {
+        const tag = this.TAG + 'getCacheStats | ';
+
+        // Return cached stats if fresh (unless forceRefresh requested)
+        if (!forceRefresh && this.cachedStats && this.cachedStatsTimestamp) {
+            const age = Date.now() - this.cachedStatsTimestamp;
+            if (age < this.STATS_CACHE_TTL) {
+                log.debug(tag, `Returning cached stats (age: ${age}ms)`);
+                return this.cachedStats;
+            }
+        }
+
+        try {
+            const pattern = this.config.keyPrefix + '*';
+            const keys: string[] = [];
+
+            // Use SCAN instead of KEYS for non-blocking iteration
+            // SCAN is production-safe and doesn't block Redis
+            let cursor = '0';
+            const maxKeys = 20; // Sample up to 20 keys for stats (performance)
+
+            do {
+                // SCAN with MATCH pattern, COUNT 50 per iteration for faster collection
+                const result = await this.redis.scan(
+                    cursor,
+                    'MATCH', pattern,
+                    'COUNT', 50
+                );
+
+                cursor = result[0]; // Next cursor position
+                const foundKeys = result[1]; // Array of matching keys
+
+                keys.push(...foundKeys);
+
+                // Stop if we have enough samples or completed scan
+                if (keys.length >= maxKeys || cursor === '0') {
+                    break;
+                }
+            } while (cursor !== '0');
+
+            let totalEntries = 0;
+            let staleEntries = 0;
+            let entriesWithoutTTL = 0;
+            const sources: Record<string, number> = {};
+
+            // Analyze sampled keys
+            const sampled = keys.slice(0, maxKeys);
+
+            for (const key of sampled) {
+                try {
+                    const cached = await this.redis.get(key);
+                    if (cached) {
+                        const parsed: CachedValue<T> = JSON.parse(cached);
+                        totalEntries++;
+
+                        // Check staleness
+                        if (this.config.staleThreshold) {
+                            const age = Date.now() - parsed.timestamp;
+                            if (age > this.config.staleThreshold) {
+                                staleEntries++;
+                            }
+                        }
+
+                        // Track sources
+                        if (parsed.source) {
+                            sources[parsed.source] = (sources[parsed.source] || 0) + 1;
+                        }
+
+                        // Check TTL
+                        if (this.config.enableTTL) {
+                            const ttl = await this.redis.ttl(key);
+                            if (ttl === -1) {
+                                entriesWithoutTTL++;
+                            }
+                        }
+                    }
+                } catch (parseError) {
+                    log.warn(tag, `Invalid cache entry: ${key}`);
+                }
+            }
+
+            const freshEntries = totalEntries - staleEntries;
+            const stalenessRate = totalEntries > 0
+                ? ((staleEntries / totalEntries) * 100).toFixed(1) + '%'
+                : '0%';
+
+            const stats: CacheStats = {
+                totalEntries: keys.length, // Sampled keys count (≤20)
+                staleEntries,
+                freshEntries,
+                stalenessRate,
+                sources,
+                entriesWithoutTTL: this.config.enableTTL ? entriesWithoutTTL : undefined,
+                ttl: this.config.ttl,
+                staleThreshold: this.config.staleThreshold
+            };
+
+            // Cache the stats for future requests
+            this.cachedStats = stats;
+            this.cachedStatsTimestamp = Date.now();
+            log.debug(tag, `Stats refreshed and cached`);
+
+            return stats;
+
+        } catch (error) {
+            log.error(tag, 'Error getting cache stats:', error);
+            return {
+                totalEntries: 0,
+                staleEntries: 0,
+                freshEntries: 0,
+                stalenessRate: '0%'
+            };
+        }
+    }
+
+    /**
+     * FIX #5: Health check
+     * @param forceRefresh - Force refresh stats (bypasses 30s cache)
+     */
+    async getHealth(forceRefresh: boolean = false): Promise<HealthCheckResult> {
+        const tag = this.TAG + 'getHealth | ';
+
+        try {
+            const issues: string[] = [];
+            const warnings: string[] = [];
+
+            // Check queue initialization
+            if (this.config.enableQueue && !this.queueInitialized) {
+                issues.push('Queue not initialized - background refresh disabled');
+            }
+
+            // Check Redis connection
+            try {
+                await this.redis.ping();
+            } catch (error) {
+                issues.push('Redis connection failed');
+            }
+
+            // Get cache stats (with optional force refresh)
+            const stats = await this.getCacheStats(forceRefresh);
+
+            // Check for entries without TTL
+            if (stats.entriesWithoutTTL && stats.entriesWithoutTTL > 0) {
+                issues.push(`${stats.entriesWithoutTTL} cache entries without TTL detected`);
+            }
+
+            // Check staleness rate
+            if (this.config.staleThreshold) {
+                const stalenessRate = parseFloat(String(stats.stalenessRate).replace('%', ''));
+                if (stalenessRate > 50) {
+                    issues.push(`High staleness rate: ${stalenessRate}%`);
+                } else if (stalenessRate > 30) {
+                    warnings.push(`Elevated staleness rate: ${stalenessRate}%`);
+                }
+            }
+
+            // Determine overall status
+            let status: 'healthy' | 'degraded' | 'unhealthy' = 'healthy';
+            if (issues.length > 0) {
+                status = 'unhealthy';
+            } else if (warnings.length > 0) {
+                status = 'degraded';
+            }
+
+            return {
+                status,
+                queueInitialized: this.queueInitialized,
+                redisConnected: true,
+                stats,
+                issues,
+                warnings,
+                timestamp: Date.now(),
+                timestampISO: new Date().toISOString()
+            };
+
+        } catch (error) {
+            log.error(tag, 'Health check failed:', error);
+            return {
+                status: 'unhealthy',
+                queueInitialized: this.queueInitialized,
+                redisConnected: false,
+                stats: {
+                    totalEntries: 0,
+                    staleEntries: 0,
+                    freshEntries: 0,
+                    stalenessRate: '0%'
+                },
+                issues: [`Health check error: ${error instanceof Error ? error.message : String(error)}`],
+                warnings: [],
+                timestamp: Date.now(),
+                timestampISO: new Date().toISOString()
+            };
+        }
+    }
+
+    /**
+     * Clear all cache entries (use with caution)
+     */
+    async clearAll(): Promise<number> {
+        const tag = this.TAG + 'clearAll | ';
+
+        try {
+            const pattern = this.config.keyPrefix + '*';
+            const keys = await this.redis.keys(pattern);
+
+            if (keys.length === 0) {
+                log.info(tag, 'No cache entries to clear');
+                return 0;
+            }
+
+            await this.redis.del(...keys);
+            log.info(tag, `Cleared ${keys.length} cache entries`);
+            return keys.length;
+
+        } catch (error) {
+            log.error(tag, 'Error clearing cache:', error);
+            return 0;
+        }
+    }
+
+    // ========== ABSTRACT METHODS (must be implemented by subclasses) ==========
+
+    /**
+     * Build Redis key from parameters
+     * Each cache type implements its own key structure
+     */
+    protected abstract buildKey(params: Record<string, any>): string;
+
+    /**
+     * Fetch data from source (blockchain, API, etc.)
+     * Each cache type implements its own data fetching
+     */
+    protected abstract fetchFromSource(params: Record<string, any>): Promise<T>;
+
+    /**
+     * Get legacy cached value (optional)
+     * Each cache type can implement legacy key migration
+     */
+    protected abstract getLegacyCached(params: Record<string, any>): Promise<T | null>;
+}