@pioneer-platform/pioneer-cache 1.0.0

package/dist/core/base-cache.js

@@ -0,0 +1,493 @@
+"use strict";
+/*
+    BaseCache - Abstract base class for all cache implementations
+
+    Contains all shared logic that was duplicated across balance/price caches.
+    Specific caches extend this and implement only their unique logic.
+*/
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BaseCache = void 0;
+const log = require('@pioneer-platform/loggerdog')();
+class BaseCache {
+    constructor(redis, config) {
+        this.queueInitialized = false;
+        // Cache for stats (to avoid expensive SCAN operations on every health check)
+        this.cachedStats = null;
+        this.cachedStatsTimestamp = 0;
+        this.STATS_CACHE_TTL = 30000; // 30 seconds
+        this.redis = redis;
+        this.config = config;
+        this.TAG = ` | ${config.name}Cache | `;
+        // Initialize queue if enabled
+        if (config.enableQueue) {
+            this.initializeQueue();
+        }
+        else {
+            log.info(this.TAG, `Queue disabled for ${config.name} cache`);
+        }
+    }
+    /**
+     * Initialize Redis queue for background refresh
+     */
+    initializeQueue() {
+        try {
+            const redisQueueModule = require('@pioneer-platform/redis-queue');
+            redisQueueModule.init(this.config.queueName);
+            this.redisQueue = redisQueueModule;
+            this.queueInitialized = true;
+            log.info(this.TAG, `✅ Queue initialized: ${this.config.queueName}`);
+        }
+        catch (error) {
+            // CRITICAL FIX #3: Make queue failures VERY visible
+            log.error(this.TAG, `❌ CRITICAL: Failed to initialize queue '${this.config.queueName}'!`);
+            log.error(this.TAG, 'Background refresh is DISABLED - cache will NOT update automatically!');
+            log.error(this.TAG, 'Error:', error);
+            this.redisQueue = null;
+            this.queueInitialized = false;
+        }
+    }
+    /**
+     * Main cache get method
+     * Implements stale-while-revalidate pattern with optional blocking
+     */
+    async get(params, waitForFresh) {
+        const tag = this.TAG + 'get | ';
+        const startTime = Date.now();
+        try {
+            const key = this.buildKey(params);
+            // Step 1: Try new cache format
+            const cachedValue = await this.getCached(key);
+            if (cachedValue) {
+                const age = Date.now() - cachedValue.timestamp;
+                const responseTime = Date.now() - startTime;
+                if (this.config.logCacheHits) {
+                    log.debug(tag, `Cache hit: ${key} (${responseTime}ms, age: ${age}ms)`);
+                }
+                // Check staleness async (don't wait)
+                if (this.config.staleThreshold && age > this.config.staleThreshold) {
+                    this.triggerAsyncRefresh(params, 'normal');
+                }
+                return {
+                    success: true,
+                    value: cachedValue.value,
+                    cached: true,
+                    fresh: this.config.staleThreshold ? age <= this.config.staleThreshold : true,
+                    age
+                };
+            }
+            // Step 2: Try legacy cache fallback
+            if (this.config.enableLegacyFallback) {
+                const legacyValue = await this.getLegacyCached(params);
+                if (legacyValue) {
+                    const responseTime = Date.now() - startTime;
+                    log.info(tag, `Legacy cache hit: ${key} (${responseTime}ms)`);
+                    // Migrate to new format (async)
+                    this.migrateLegacyValue(key, legacyValue);
+                    return {
+                        success: true,
+                        value: legacyValue,
+                        cached: true,
+                        fresh: false, // Legacy data is considered stale
+                        age: undefined
+                    };
+                }
+            }
+            // Step 3: Cache miss
+            const responseTime = Date.now() - startTime;
+            if (this.config.logCacheMisses) {
+                log.info(tag, `Cache miss: ${key} (${responseTime}ms)`);
+            }
+            // FIX #1: Optional blocking on cache miss
+            const shouldBlock = waitForFresh !== undefined ? waitForFresh : this.config.blockOnMiss;
+            if (shouldBlock) {
+                log.info(tag, `Blocking to fetch fresh data: ${key}`);
+                const freshValue = await this.fetchFresh(params);
+                return {
+                    success: true,
+                    value: freshValue,
+                    cached: false,
+                    fresh: true,
+                    age: 0
+                };
+            }
+            // Non-blocking: trigger async refresh and return default
+            this.triggerAsyncRefresh(params, 'high');
+            return {
+                success: true,
+                value: this.config.defaultValue,
+                cached: false,
+                fresh: false,
+                age: undefined
+            };
+        }
+        catch (error) {
+            log.error(tag, 'Error getting cache value:', error);
+            return {
+                success: false,
+                value: this.config.defaultValue,
+                cached: false,
+                fresh: false,
+                error: error instanceof Error ? error.message : String(error)
+            };
+        }
+    }
+    /**
+     * Get value from cache
+     */
+    async getCached(key) {
+        const tag = this.TAG + 'getCached | ';
+        try {
+            const cached = await this.redis.get(key);
+            if (!cached) {
+                return null;
+            }
+            const parsed = JSON.parse(cached);
+            // Validate structure
+            if (!parsed.value || typeof parsed.timestamp !== 'number') {
+                log.warn(tag, `Invalid cache structure for ${key}, removing`);
+                await this.redis.del(key);
+                return null;
+            }
+            return parsed;
+        }
+        catch (error) {
+            log.error(tag, `Error parsing cached value for ${key}:`, error);
+            return null;
+        }
+    }
+    /**
+     * Update cache with new value
+     * FIX #2: Always includes TTL
+     */
+    async updateCache(key, value, metadata) {
+        const tag = this.TAG + 'updateCache | ';
+        try {
+            const cachedValue = {
+                value,
+                timestamp: Date.now(),
+                source: 'network',
+                lastUpdated: new Date().toISOString(),
+                metadata
+            };
+            // FIX #2: Always set TTL (unless explicitly disabled)
+            if (this.config.enableTTL) {
+                const ttlSeconds = Math.floor(this.config.ttl / 1000);
+                await this.redis.set(key, JSON.stringify(cachedValue), 'EX', ttlSeconds);
+                log.info(tag, `Updated cache: ${key} [TTL: ${ttlSeconds}s]`);
+            }
+            else {
+                // Permanent caching (for transactions)
+                await this.redis.set(key, JSON.stringify(cachedValue));
+                log.info(tag, `Updated cache: ${key} [PERMANENT]`);
+            }
+        }
+        catch (error) {
+            log.error(tag, `Error updating cache for ${key}:`, error);
+            throw error;
+        }
+    }
+    /**
+     * Trigger background refresh job
+     * FIX #3: Loud error logging
+     * FIX #4: Synchronous fallback for high-priority
+     */
+    triggerAsyncRefresh(params, priority = 'normal') {
+        const tag = this.TAG + 'triggerAsyncRefresh | ';
+        try {
+            // Check if queue is enabled
+            if (!this.config.enableQueue) {
+                return; // Queue disabled, no background refresh
+            }
+            // FIX #3: Fail loudly if queue not initialized
+            if (!this.queueInitialized || !this.redisQueue) {
+                const key = this.buildKey(params);
+                log.error(tag, `❌ QUEUE NOT INITIALIZED! Cannot refresh ${key}`);
+                log.error(tag, `Background refresh is BROKEN - cache will NOT update!`);
+                // FIX #4: Synchronous fallback for high-priority
+                if (priority === 'high') {
+                    log.warn(tag, `Using synchronous fallback for high-priority refresh`);
+                    setImmediate(async () => {
+                        try {
+                            await this.fetchFresh(params);
+                        }
+                        catch (error) {
+                            log.error(tag, `Synchronous fallback failed:`, error);
+                        }
+                    });
+                }
+                return;
+            }
+            const job = {
+                type: `REFRESH_${this.config.name.toUpperCase()}`,
+                key: this.buildKey(params),
+                params,
+                priority,
+                retryCount: 0,
+                timestamp: Date.now()
+            };
+            // Queue job async (don't wait)
+            setImmediate(async () => {
+                try {
+                    await this.redisQueue.createWork(this.config.queueName, job);
+                    if (this.config.logRefreshJobs) {
+                        log.debug(tag, `Queued refresh job: ${job.key} (priority: ${priority})`);
+                    }
+                }
+                catch (error) {
+                    log.error(tag, `Error queuing refresh job:`, error);
+                }
+            });
+        }
+        catch (error) {
+            log.error(tag, `Error triggering refresh:`, error);
+        }
+    }
+    /**
+     * Fetch fresh data and update cache
+     * FIX #1 & #4: Used for blocking requests and fallback
+     */
+    async fetchFresh(params) {
+        const tag = this.TAG + 'fetchFresh | ';
+        const startTime = Date.now();
+        try {
+            const key = this.buildKey(params);
+            log.info(tag, `Fetching fresh data: ${key}`);
+            // Call subclass-specific fetch implementation
+            const value = await this.fetchFromSource(params);
+            // Update cache
+            await this.updateCache(key, value);
+            const fetchTime = Date.now() - startTime;
+            log.info(tag, `✅ Fetched fresh data in ${fetchTime}ms: ${key}`);
+            return value;
+        }
+        catch (error) {
+            const fetchTime = Date.now() - startTime;
+            log.error(tag, `Failed to fetch fresh data after ${fetchTime}ms:`, error);
+            return this.config.defaultValue;
+        }
+    }
+    /**
+     * Migrate legacy cache value to new format
+     * FIX #2: Includes TTL
+     */
+    migrateLegacyValue(key, value) {
+        const tag = this.TAG + 'migrateLegacyValue | ';
+        setImmediate(async () => {
+            try {
+                const cachedValue = {
+                    value,
+                    timestamp: Date.now() - (this.config.staleThreshold || this.config.ttl), // Mark as stale
+                    source: 'legacy',
+                    lastUpdated: new Date().toISOString()
+                };
+                // FIX #2: Set TTL on migrated data
+                if (this.config.enableTTL) {
+                    const ttlSeconds = Math.floor(this.config.ttl / 1000);
+                    await this.redis.set(key, JSON.stringify(cachedValue), 'EX', ttlSeconds);
+                    log.debug(tag, `Migrated legacy value: ${key} [TTL: ${ttlSeconds}s]`);
+                }
+                else {
+                    await this.redis.set(key, JSON.stringify(cachedValue));
+                    log.debug(tag, `Migrated legacy value: ${key} [PERMANENT]`);
+                }
+            }
+            catch (error) {
+                log.error(tag, `Error migrating legacy value:`, error);
+            }
+        });
+    }
+    /**
+     * Get cache statistics
+     * @param forceRefresh - Skip cache and fetch fresh stats (for testing/debugging)
+     */
+    async getCacheStats(forceRefresh = false) {
+        const tag = this.TAG + 'getCacheStats | ';
+        // Return cached stats if fresh (unless forceRefresh requested)
+        if (!forceRefresh && this.cachedStats && this.cachedStatsTimestamp) {
+            const age = Date.now() - this.cachedStatsTimestamp;
+            if (age < this.STATS_CACHE_TTL) {
+                log.debug(tag, `Returning cached stats (age: ${age}ms)`);
+                return this.cachedStats;
+            }
+        }
+        try {
+            const pattern = this.config.keyPrefix + '*';
+            const keys = [];
+            // Use SCAN instead of KEYS for non-blocking iteration
+            // SCAN is production-safe and doesn't block Redis
+            let cursor = '0';
+            const maxKeys = 20; // Sample up to 20 keys for stats (performance)
+            do {
+                // SCAN with MATCH pattern, COUNT 50 per iteration for faster collection
+                const result = await this.redis.scan(cursor, 'MATCH', pattern, 'COUNT', 50);
+                cursor = result[0]; // Next cursor position
+                const foundKeys = result[1]; // Array of matching keys
+                keys.push(...foundKeys);
+                // Stop if we have enough samples or completed scan
+                if (keys.length >= maxKeys || cursor === '0') {
+                    break;
+                }
+            } while (cursor !== '0');
+            let totalEntries = 0;
+            let staleEntries = 0;
+            let entriesWithoutTTL = 0;
+            const sources = {};
+            // Analyze sampled keys
+            const sampled = keys.slice(0, maxKeys);
+            for (const key of sampled) {
+                try {
+                    const cached = await this.redis.get(key);
+                    if (cached) {
+                        const parsed = JSON.parse(cached);
+                        totalEntries++;
+                        // Check staleness
+                        if (this.config.staleThreshold) {
+                            const age = Date.now() - parsed.timestamp;
+                            if (age > this.config.staleThreshold) {
+                                staleEntries++;
+                            }
+                        }
+                        // Track sources
+                        if (parsed.source) {
+                            sources[parsed.source] = (sources[parsed.source] || 0) + 1;
+                        }
+                        // Check TTL
+                        if (this.config.enableTTL) {
+                            const ttl = await this.redis.ttl(key);
+                            if (ttl === -1) {
+                                entriesWithoutTTL++;
+                            }
+                        }
+                    }
+                }
+                catch (parseError) {
+                    log.warn(tag, `Invalid cache entry: ${key}`);
+                }
+            }
+            const freshEntries = totalEntries - staleEntries;
+            const stalenessRate = totalEntries > 0
+                ? ((staleEntries / totalEntries) * 100).toFixed(1) + '%'
+                : '0%';
+            const stats = {
+                totalEntries: keys.length, // Sampled keys count (≤20)
+                staleEntries,
+                freshEntries,
+                stalenessRate,
+                sources,
+                entriesWithoutTTL: this.config.enableTTL ? entriesWithoutTTL : undefined,
+                ttl: this.config.ttl,
+                staleThreshold: this.config.staleThreshold
+            };
+            // Cache the stats for future requests
+            this.cachedStats = stats;
+            this.cachedStatsTimestamp = Date.now();
+            log.debug(tag, `Stats refreshed and cached`);
+            return stats;
+        }
+        catch (error) {
+            log.error(tag, 'Error getting cache stats:', error);
+            return {
+                totalEntries: 0,
+                staleEntries: 0,
+                freshEntries: 0,
+                stalenessRate: '0%'
+            };
+        }
+    }
+    /**
+     * FIX #5: Health check
+     * @param forceRefresh - Force refresh stats (bypasses 30s cache)
+     */
+    async getHealth(forceRefresh = false) {
+        const tag = this.TAG + 'getHealth | ';
+        try {
+            const issues = [];
+            const warnings = [];
+            // Check queue initialization
+            if (this.config.enableQueue && !this.queueInitialized) {
+                issues.push('Queue not initialized - background refresh disabled');
+            }
+            // Check Redis connection
+            try {
+                await this.redis.ping();
+            }
+            catch (error) {
+                issues.push('Redis connection failed');
+            }
+            // Get cache stats (with optional force refresh)
+            const stats = await this.getCacheStats(forceRefresh);
+            // Check for entries without TTL
+            if (stats.entriesWithoutTTL && stats.entriesWithoutTTL > 0) {
+                issues.push(`${stats.entriesWithoutTTL} cache entries without TTL detected`);
+            }
+            // Check staleness rate
+            if (this.config.staleThreshold) {
+                const stalenessRate = parseFloat(String(stats.stalenessRate).replace('%', ''));
+                if (stalenessRate > 50) {
+                    issues.push(`High staleness rate: ${stalenessRate}%`);
+                }
+                else if (stalenessRate > 30) {
+                    warnings.push(`Elevated staleness rate: ${stalenessRate}%`);
+                }
+            }
+            // Determine overall status
+            let status = 'healthy';
+            if (issues.length > 0) {
+                status = 'unhealthy';
+            }
+            else if (warnings.length > 0) {
+                status = 'degraded';
+            }
+            return {
+                status,
+                queueInitialized: this.queueInitialized,
+                redisConnected: true,
+                stats,
+                issues,
+                warnings,
+                timestamp: Date.now(),
+                timestampISO: new Date().toISOString()
+            };
+        }
+        catch (error) {
+            log.error(tag, 'Health check failed:', error);
+            return {
+                status: 'unhealthy',
+                queueInitialized: this.queueInitialized,
+                redisConnected: false,
+                stats: {
+                    totalEntries: 0,
+                    staleEntries: 0,
+                    freshEntries: 0,
+                    stalenessRate: '0%'
+                },
+                issues: [`Health check error: ${error instanceof Error ? error.message : String(error)}`],
+                warnings: [],
+                timestamp: Date.now(),
+                timestampISO: new Date().toISOString()
+            };
+        }
+    }
+    /**
+     * Clear all cache entries (use with caution)
+     */
+    async clearAll() {
+        const tag = this.TAG + 'clearAll | ';
+        try {
+            const pattern = this.config.keyPrefix + '*';
+            const keys = await this.redis.keys(pattern);
+            if (keys.length === 0) {
+                log.info(tag, 'No cache entries to clear');
+                return 0;
+            }
+            await this.redis.del(...keys);
+            log.info(tag, `Cleared ${keys.length} cache entries`);
+            return keys.length;
+        }
+        catch (error) {
+            log.error(tag, 'Error clearing cache:', error);
+            return 0;
+        }
+    }
+}
+exports.BaseCache = BaseCache;

package/dist/core/cache-manager.d.ts

@@ -0,0 +1,62 @@
+import { BalanceCache } from '../stores/balance-cache';
+import { PriceCache } from '../stores/price-cache';
+import { TransactionCache } from '../stores/transaction-cache';
+import type { HealthCheckResult } from '../types';
+/**
+ * Configuration for CacheManager
+ */
+export interface CacheManagerConfig {
+    redis: any;
+    balanceModule?: any;
+    markets?: any;
+    enableBalanceCache?: boolean;
+    enablePriceCache?: boolean;
+    enableTransactionCache?: boolean;
+    startWorkers?: boolean;
+}
+/**
+ * CacheManager - Central coordinator for all caches
+ */
+export declare class CacheManager {
+    private redis;
+    private balanceCache?;
+    private priceCache?;
+    private transactionCache?;
+    private workers;
+    constructor(config: CacheManagerConfig);
+    /**
+     * Start background refresh workers for all caches
+     */
+    startWorkers(): Promise<void>;
+    /**
+     * Stop all workers gracefully
+     */
+    stopWorkers(): Promise<void>;
+    /**
+     * Get aggregate health status for all caches
+     * @param forceRefresh - Force refresh stats (bypasses 30s cache)
+     */
+    getHealth(forceRefresh?: boolean): Promise<HealthCheckResult & {
+        checks?: any;
+    }>;
+    /**
+     * Get all cache instances
+     */
+    getCaches(): {
+        balance: BalanceCache | undefined;
+        price: PriceCache | undefined;
+        transaction: TransactionCache | undefined;
+    };
+    /**
+     * Get specific cache by name
+     */
+    getCache(name: 'balance' | 'price' | 'transaction'): BalanceCache | PriceCache | TransactionCache | undefined;
+    /**
+     * Clear all caches (use with caution!)
+     */
+    clearAll(): Promise<{
+        balance?: number;
+        price?: number;
+        transaction?: number;
+    }>;
+}