@pioneer-platform/pioneer-cache 1.0.0 → 1.0.2

package/dist/stores/portfolio-cache.js

@@ -0,0 +1,189 @@
+"use strict";
+/*
+    PortfolioCache - Portfolio/Charts cache implementation
+
+    Extends BaseCache with portfolio-specific logic.
+    Designed for NON-BLOCKING, instant returns.
+*/
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PortfolioCache = void 0;
+const base_cache_1 = require("../core/base-cache");
+const log = require('@pioneer-platform/loggerdog')();
+/**
+ * PortfolioCache - Caches portfolio/chart data
+ *
+ * CRITICAL: This cache is NON-BLOCKING by design
+ * - Returns empty arrays immediately on cache miss
+ * - Never blocks waiting for blockchain APIs
+ * - Background jobs populate cache for next request
+ */
+class PortfolioCache extends base_cache_1.BaseCache {
+    constructor(redis, balanceModule, marketsModule, config) {
+        const defaultConfig = {
+            name: 'portfolio',
+            keyPrefix: 'portfolio_v2:',
+            ttl: 0, // Ignored when enableTTL: false
+            staleThreshold: 5 * 60 * 1000, // 5 minutes - triggers background refresh
+            enableTTL: false, // NEVER EXPIRE - data persists forever, show stale data instantly
+            queueName: 'cache-refresh',
+            enableQueue: true,
+            maxRetries: 3,
+            retryDelay: 5000,
+            blockOnMiss: false, // CRITICAL: NEVER WAIT! Return empty arrays instantly
+            enableLegacyFallback: false, // No legacy portfolio cache format
+            defaultValue: {
+                pubkeys: [],
+                charts: [],
+                totalValueUsd: 0,
+                timestamp: Date.now()
+            },
+            useSyncFallback: false, // CRITICAL: NEVER use synchronous fallback - always return instantly
+            maxConcurrentJobs: 3, // Limit concurrent portfolio refreshes
+            apiTimeout: 30000, // 30s timeout for full portfolio fetch
+            logCacheHits: true,
+            logCacheMisses: true,
+            logRefreshJobs: true
+        };
+        super(redis, { ...defaultConfig, ...config });
+        this.balanceModule = balanceModule;
+        this.marketsModule = marketsModule;
+    }
+    /**
+     * Build Redis key for portfolio data
+     *
+     * Key strategy: Hash all pubkeys+caips to create a stable identifier
+     * Format: portfolio_v2:hash(pubkeys)
+     *
+     * This allows caching the same portfolio regardless of pubkey order
+     */
+    buildKey(params) {
+        const { pubkeys } = params;
+        if (!pubkeys || !Array.isArray(pubkeys) || pubkeys.length === 0) {
+            throw new Error('PortfolioCache.buildKey: pubkeys array required');
+        }
+        // Sort pubkeys to create stable hash regardless of order
+        const sorted = [...pubkeys].sort((a, b) => {
+            const aKey = `${a.caip}:${a.pubkey}`;
+            const bKey = `${b.caip}:${b.pubkey}`;
+            return aKey.localeCompare(bKey);
+        });
+        // Create a simple hash from sorted pubkeys
+        const keyString = sorted.map(p => `${p.caip}:${p.pubkey}`).join('|');
+        const hash = this.simpleHash(keyString);
+        return `${this.config.keyPrefix}${hash}`;
+    }
+    /**
+     * Simple hash function for cache keys
+     * Not cryptographic - just needs to be stable and collision-resistant
+     */
+    simpleHash(str) {
+        let hash = 0;
+        for (let i = 0; i < str.length; i++) {
+            const char = str.charCodeAt(i);
+            hash = ((hash << 5) - hash) + char;
+            hash = hash & hash; // Convert to 32-bit integer
+        }
+        return Math.abs(hash).toString(36);
+    }
+    /**
+     * Fetch portfolio from blockchain APIs
+     *
+     * This is the SLOW operation that happens in the background
+     * It fetches balances for all pubkeys and enriches with pricing
+     */
+    async fetchFromSource(params) {
+        const tag = this.TAG + 'fetchFromSource | ';
+        const startTime = Date.now();
+        try {
+            const { pubkeys } = params;
+            log.info(tag, `Fetching portfolio for ${pubkeys.length} pubkeys`);
+            const charts = [];
+            // Fetch balances for all pubkeys in parallel
+            const balancePromises = pubkeys.map(async (item) => {
+                try {
+                    // Extract networkId from CAIP
+                    const networkId = item.caip.split('/')[0];
+                    // Fetch balance
+                    const asset = { caip: item.caip };
+                    const owner = { pubkey: item.pubkey };
+                    const balanceInfo = await this.balanceModule.getBalance(asset, owner);
+                    if (!balanceInfo || !balanceInfo.balance) {
+                        log.debug(tag, `No balance for ${item.caip}/${item.pubkey.substring(0, 10)}...`);
+                        return null;
+                    }
+                    // Skip zero balances
+                    const balanceNum = parseFloat(balanceInfo.balance);
+                    if (isNaN(balanceNum) || balanceNum === 0) {
+                        return null;
+                    }
+                    // Get asset metadata
+                    const assetData = require('@pioneer-platform/pioneer-discovery').assetData;
+                    const assetInfo = assetData[item.caip.toUpperCase()] || assetData[item.caip.toLowerCase()] || {};
+                    // Get price
+                    let priceUsd = 0;
+                    try {
+                        priceUsd = await this.marketsModule.getAssetPriceByCaip(item.caip);
+                        if (isNaN(priceUsd) || priceUsd < 0) {
+                            priceUsd = 0;
+                        }
+                    }
+                    catch (priceError) {
+                        log.warn(tag, `Error fetching price for ${item.caip}:`, priceError);
+                    }
+                    const valueUsd = balanceNum * priceUsd;
+                    const chartData = {
+                        caip: item.caip,
+                        pubkey: item.pubkey,
+                        networkId,
+                        symbol: assetInfo.symbol || 'UNKNOWN',
+                        name: assetInfo.name || 'Unknown Asset',
+                        balance: balanceInfo.balance,
+                        priceUsd,
+                        valueUsd,
+                        icon: assetInfo.icon || '',
+                        type: assetInfo.type || 'native',
+                        decimal: assetInfo.decimal
+                    };
+                    return chartData;
+                }
+                catch (error) {
+                    log.error(tag, `Error fetching balance for ${item.caip}/${item.pubkey}:`, error);
+                    return null;
+                }
+            });
+            const results = await Promise.all(balancePromises);
+            // Filter out nulls and calculate total
+            const validCharts = results.filter((c) => c !== null);
+            const totalValueUsd = validCharts.reduce((sum, c) => sum + c.valueUsd, 0);
+            const fetchTime = Date.now() - startTime;
+            log.info(tag, `✅ Fetched portfolio: ${validCharts.length} assets, $${totalValueUsd.toFixed(2)} in ${fetchTime}ms`);
+            return {
+                pubkeys,
+                charts: validCharts,
+                totalValueUsd,
+                timestamp: Date.now()
+            };
+        }
+        catch (error) {
+            log.error(tag, 'Error fetching portfolio:', error);
+            throw error;
+        }
+    }
+    /**
+     * No legacy cache format for portfolios
+     */
+    async getLegacyCached(params) {
+        return null;
+    }
+    /**
+     * Get portfolio for a set of pubkeys
+     * Convenience method that wraps base get()
+     *
+     * RETURNS INSTANTLY - either cached data or empty arrays
+     */
+    async getPortfolio(pubkeys, waitForFresh) {
+        const result = await this.get({ pubkeys }, waitForFresh);
+        return result.value || this.config.defaultValue;
+    }
+}
+exports.PortfolioCache = PortfolioCache;

package/dist/stores/price-cache.d.ts

@@ -20,7 +20,8 @@ export declare class PriceCache extends BaseCache<PriceData> {
      */
     protected buildKey(params: Record<string, any>): string;
     /**
-     * Fetch price from markets API
+     * Fetch price from markets API using CAIP-first approach
+     * FIX #7: Graceful handling of zero prices to prevent cache disruption
      */
     protected fetchFromSource(params: Record<string, any>): Promise<PriceData>;
     /**

package/dist/stores/price-cache.js

@@ -17,21 +17,22 @@ class PriceCache extends base_cache_1.BaseCache {
         const defaultConfig = {
             name: 'price',
             keyPrefix: 'price_v2:',
-            ttl: 60 * 60 * 1000, // 1 hour
-            staleThreshold: 30 * 60 * 1000, // 30 minutes (refresh after 30min)
-            enableTTL: true,
-            queueName: 'price-refresh-v2',
+            ttl: 0, // Ignored when enableTTL: false
+            staleThreshold: 30 * 60 * 1000, // 30 minutes - triggers background refresh
+            enableTTL: false, // NEVER EXPIRE - data persists forever, show stale prices instantly
+            queueName: 'cache-refresh',
             enableQueue: true,
             maxRetries: 3,
             retryDelay: 5000,
-            blockOnMiss: false, // Return immediately with $0 (prices less critical)
+            blockOnMiss: false, // CRITICAL: NEVER WAIT! Return $0 instantly on cache miss, refresh async in background
             enableLegacyFallback: true,
             defaultValue: {
                 caip: '',
                 price: 0
             },
+            useSyncFallback: false, // CRITICAL: NEVER use synchronous fallback - always return instantly with $0
             maxConcurrentJobs: 5,
-            apiTimeout: 5000,
+            apiTimeout: 2000, // Reduced from 5000ms for faster failures
             logCacheHits: false,
             logCacheMisses: true,
             logRefreshJobs: true
@@ -52,49 +53,53 @@ class PriceCache extends base_cache_1.BaseCache {
         return `${this.config.keyPrefix}${normalizedCaip}`;
     }
     /**
-     * Fetch price from markets API
+     * Fetch price from markets API using CAIP-first approach
+     * FIX #7: Graceful handling of zero prices to prevent cache disruption
      */
     async fetchFromSource(params) {
         const tag = this.TAG + 'fetchFromSource | ';
         try {
             const { caip } = params;
-            // Map CAIP to symbol for markets API
-            const assetData = require('@pioneer-platform/pioneer-discovery').assetData;
-            const asset = assetData[caip.toUpperCase()] || assetData[caip.toLowerCase()];
-            if (!asset || !asset.symbol) {
-                log.warn(tag, `No asset mapping found for ${caip}`);
-                return {
-                    caip,
-                    price: 0
-                };
-            }
-            // Get price from markets module
-            const symbol = asset.symbol.toLowerCase();
-            const priceResult = await this.markets.getPrice(symbol);
-            // Handle different response formats
-            let price = 0;
-            if (typeof priceResult === 'object' && priceResult.price) {
-                price = parseFloat(priceResult.price);
-            }
-            else if (typeof priceResult === 'number') {
-                price = priceResult;
-            }
+            // Use CAIP-first API (no symbol conversion needed!)
+            // This directly queries the markets module with CAIP identifiers
+            const price = await this.markets.getAssetPriceByCaip(caip);
+            // FIX #7: Gracefully handle zero prices without throwing
+            // This prevents disrupting batch operations during API rate limits
             if (isNaN(price) || price <= 0) {
-                log.warn(tag, `Invalid price for ${caip} (${symbol}): ${priceResult}`);
-                return {
-                    caip,
-                    price: 0
-                };
+                log.warn(tag, `Price fetch returned $${price} for ${caip} (likely API timeout or rate limit) - returning stale cache if available`);
+                // Try to get stale cached value instead of failing
+                const key = this.buildKey(params);
+                const cachedValue = await this.getCached(key);
+                if (cachedValue && cachedValue.value.price > 0) {
+                    log.info(tag, `Returning stale cached price for ${caip}: $${cachedValue.value.price}`);
+                    return cachedValue.value;
+                }
+                // Try legacy cache as fallback
+                const legacyValue = await this.getLegacyCached(params);
+                if (legacyValue && legacyValue.price > 0) {
+                    log.info(tag, `Returning legacy cached price for ${caip}: $${legacyValue.price}`);
+                    return legacyValue;
+                }
+                // Last resort: return zero price but don't cache it
+                log.warn(tag, `No cached price available for ${caip}, returning zero`);
+                throw new Error(`No valid price available for ${caip}`);
             }
-            log.debug(tag, `Fetched price for ${caip} (${symbol}): $${price}`);
+            log.debug(tag, `Fetched price for ${caip}: $${price}`);
             return {
                 caip,
                 price,
-                source: 'markets'
+                source: 'markets-caip'
             };
         }
         catch (error) {
-            log.error(tag, 'Error fetching price:', error);
+            // Log as warning instead of error for expected API issues
+            const errorMsg = error instanceof Error ? error.message : String(error);
+            if (errorMsg.includes('rate limit') || errorMsg.includes('timeout') || errorMsg.includes('No valid price')) {
+                log.warn(tag, `Expected API issue: ${errorMsg}`);
+            }
+            else {
+                log.error(tag, `Unexpected error fetching price:`, error);
+            }
             throw error;
         }
     }

package/dist/types/index.d.ts

@@ -14,6 +14,7 @@ export interface CacheConfig {
     blockOnMiss: boolean;
     enableLegacyFallback: boolean;
     defaultValue: any;
+    useSyncFallback?: boolean;
     maxConcurrentJobs: number;
     apiTimeout: number;
     logCacheHits: boolean;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pioneer-platform/pioneer-cache",
-  "version": "1.0.0",
+  "version": "1.0.2",
   "description": "Unified caching system for Pioneer platform with Redis backend",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -20,9 +20,9 @@
   "author": "Pioneer Platform",
   "license": "MIT",
   "dependencies": {
-    "@pioneer-platform/loggerdog": "workspace:*",
-    "@pioneer-platform/redis-queue": "workspace:*",
-    "@pioneer-platform/default-redis": "workspace:*"
+    "@pioneer-platform/loggerdog": "^8.0.0",
+    "@pioneer-platform/redis-queue": "^8.0.0",
+    "@pioneer-platform/default-redis": "^8.0.0"
   },
   "devDependencies": {
     "@types/node": "^20.0.0",

package/src/core/base-cache.ts

@@ -31,6 +31,10 @@ export abstract class BaseCache<T> {
     private cachedStatsTimestamp: number = 0;
     private readonly STATS_CACHE_TTL = 30000; // 30 seconds
 
+    // FIX #6: Request deduplication to prevent thundering herd
+    // Tracks in-flight network requests to prevent duplicate API calls
+    private pendingFetches: Map<string, Promise<T>> = new Map();
+
     constructor(redis: any, config: CacheConfig) {
         this.redis = redis;
         this.config = config;
@@ -73,10 +77,14 @@ export abstract class BaseCache<T> {
         const startTime = Date.now();
 
         try {
+            const t1 = Date.now();
             const key = this.buildKey(params);
+            log.info(tag, `⏱️  buildKey took ${Date.now() - t1}ms`);
 
             // Step 1: Try new cache format
+            const t2 = Date.now();
             const cachedValue = await this.getCached(key);
+            log.info(tag, `⏱️  getCached took ${Date.now() - t2}ms`);
 
             if (cachedValue) {
                 const age = Date.now() - cachedValue.timestamp;
@@ -102,7 +110,9 @@ export abstract class BaseCache<T> {
 
             // Step 2: Try legacy cache fallback
             if (this.config.enableLegacyFallback) {
+                const t3 = Date.now();
                 const legacyValue = await this.getLegacyCached(params);
+                log.info(tag, `⏱️  getLegacyCached took ${Date.now() - t3}ms`);
                 if (legacyValue) {
                     const responseTime = Date.now() - startTime;
                     log.info(tag, `Legacy cache hit: ${key} (${responseTime}ms)`);
@@ -141,7 +151,10 @@ export abstract class BaseCache<T> {
             }
 
             // Non-blocking: trigger async refresh and return default
+            const t4 = Date.now();
             this.triggerAsyncRefresh(params, 'high');
+            log.info(tag, `⏱️  triggerAsyncRefresh took ${Date.now() - t4}ms`);
+            log.info(tag, `⏱️  Returning default value after ${Date.now() - startTime}ms TOTAL`);
 
             return {
                 success: true,
@@ -168,22 +181,37 @@ export abstract class BaseCache<T> {
      */
     protected async getCached(key: string): Promise<CachedValue<T> | null> {
         const tag = this.TAG + 'getCached | ';
+        const t0 = Date.now();
 
         try {
-            const cached = await this.redis.get(key);
+            // Redis timeout for cache reads
+            // Increased from 100ms to 2000ms - 100ms was too aggressive and caused false cache misses
+            // Increased from 2000ms to 10000ms - IPv4/IPv6 DNS resolution can cause delays
+            const timeoutMs = 10000;
+            const cached = await Promise.race([
+                this.redis.get(key),
+                new Promise<null>((resolve) => setTimeout(() => {
+                    log.warn(tag, `⏱️  Redis timeout after ${timeoutMs}ms, returning cache miss`);
+                    resolve(null);
+                }, timeoutMs))
+            ]);
+            
             if (!cached) {
+                log.debug(tag, `Cache miss: ${key}`);
                 return null;
             }
 
             const parsed: CachedValue<T> = JSON.parse(cached);
 
-            // Validate structure
-            if (!parsed.value || typeof parsed.timestamp !== 'number') {
+            // Validate structure - Check for undefined/null, NOT falsy values!
+            // CRITICAL: Balance "0", empty arrays [], and empty objects {} are VALID!
+            if (parsed.value === undefined || parsed.value === null || typeof parsed.timestamp !== 'number') {
                 log.warn(tag, `Invalid cache structure for ${key}, removing`);
                 await this.redis.del(key);
                 return null;
             }
 
+            log.debug(tag, `Cache hit: ${key}`);
             return parsed;
 
         } catch (error) {
@@ -208,19 +236,33 @@ export abstract class BaseCache<T> {
                 metadata
             };
 
+            // DIAGNOSTIC: Log Redis connection details
+            const redisConstructor = this.redis.constructor?.name || 'unknown';
+            const redisOptions = this.redis.options || {};
+
             // FIX #2: Always set TTL (unless explicitly disabled)
             if (this.config.enableTTL) {
                 const ttlSeconds = Math.floor(this.config.ttl / 1000);
+
+                // PERF: Reduced logging for production performance
+                log.debug(tag, `📝 Writing to Redis: ${key} [${JSON.stringify(cachedValue).length} bytes, TTL: ${ttlSeconds}s]`);
+
+                // Write
                 await this.redis.set(key, JSON.stringify(cachedValue), 'EX', ttlSeconds);
-                log.info(tag, `Updated cache: ${key} [TTL: ${ttlSeconds}s]`);
+                
+                // PERF: Verification disabled for production performance
+                // Only enable for debugging cache issues
+                log.debug(tag, `✅ Updated cache: ${key} [TTL: ${ttlSeconds}s]`);
             } else {
                 // Permanent caching (for transactions)
                 await this.redis.set(key, JSON.stringify(cachedValue));
-                log.info(tag, `Updated cache: ${key} [PERMANENT]`);
+                
+                // PERF: Verification disabled for production performance
+                log.debug(tag, `✅ Updated cache: ${key} [PERMANENT]`);
             }
 
         } catch (error) {
-            log.error(tag, `Error updating cache for ${key}:`, error);
+            log.error(tag, `❌ Error updating cache for ${key}:`, error);
             throw error;
         }
     }
@@ -245,8 +287,13 @@ export abstract class BaseCache<T> {
                 log.error(tag, `❌ QUEUE NOT INITIALIZED! Cannot refresh ${key}`);
                 log.error(tag, `Background refresh is BROKEN - cache will NOT update!`);
 
-                // FIX #4: Synchronous fallback for high-priority
-                if (priority === 'high') {
+                // FIX #4: Synchronous fallback for high-priority (only if useSyncFallback is enabled)
+                // Default: use sync fallback only for blocking caches (blockOnMiss=true)
+                const shouldUseSyncFallback = this.config.useSyncFallback !== undefined
+                    ? this.config.useSyncFallback
+                    : this.config.blockOnMiss;
+
+                if (priority === 'high' && shouldUseSyncFallback) {
                     log.warn(tag, `Using synchronous fallback for high-priority refresh`);
                     setImmediate(async () => {
                         try {
@@ -290,31 +337,82 @@ export abstract class BaseCache<T> {
     /**
      * Fetch fresh data and update cache
      * FIX #1 & #4: Used for blocking requests and fallback
+     * FIX #6: Request deduplication to prevent thundering herd
+     * FIX #8: Return stale cache on fetch failures
      */
     async fetchFresh(params: Record<string, any>): Promise<T> {
         const tag = this.TAG + 'fetchFresh | ';
         const startTime = Date.now();
+        const key = this.buildKey(params);
+
+        // FIX #6: Check if there's already a pending fetch for this key
+        const existingFetch = this.pendingFetches.get(key);
+        if (existingFetch) {
+            // For non-blocking caches, return default value immediately instead of waiting
+            if (!this.config.blockOnMiss) {
+                log.debug(tag, `Non-blocking cache: returning default value while fetch in progress: ${key}`);
+                return this.config.defaultValue;
+            }
 
-        try {
-            const key = this.buildKey(params);
-            log.info(tag, `Fetching fresh data: ${key}`);
+            // For blocking caches, coalesce to prevent thundering herd
+            log.debug(tag, `Coalescing request for: ${key} (fetch already in progress)`);
+            return existingFetch;
+        }
+
+        // Create the fetch promise
+        const fetchPromise = (async () => {
+            try {
+                log.info(tag, `Fetching fresh data: ${key}`);
 
-            // Call subclass-specific fetch implementation
-            const value = await this.fetchFromSource(params);
+                // Call subclass-specific fetch implementation
+                const value = await this.fetchFromSource(params);
 
-            // Update cache
-            await this.updateCache(key, value);
+                // Update cache
+                await this.updateCache(key, value);
 
-            const fetchTime = Date.now() - startTime;
-            log.info(tag, `✅ Fetched fresh data in ${fetchTime}ms: ${key}`);
+                const fetchTime = Date.now() - startTime;
+                log.info(tag, `✅ Fetched fresh data in ${fetchTime}ms: ${key}`);
 
-            return value;
+                return value;
 
-        } catch (error) {
-            const fetchTime = Date.now() - startTime;
-            log.error(tag, `Failed to fetch fresh data after ${fetchTime}ms:`, error);
-            return this.config.defaultValue;
-        }
+            } catch (error) {
+                const fetchTime = Date.now() - startTime;
+                const errorMsg = error instanceof Error ? error.message : String(error);
+
+                // FIX #8: Try to return stale cache on fetch failures
+                const cachedValue = await this.getCached(key);
+                if (cachedValue) {
+                    log.warn(tag, `Fetch failed after ${fetchTime}ms, returning stale cache: ${key}`);
+                    return cachedValue.value;
+                }
+
+                // Try legacy cache as last resort
+                if (this.config.enableLegacyFallback) {
+                    const legacyValue = await this.getLegacyCached(params);
+                    if (legacyValue) {
+                        log.warn(tag, `Fetch failed after ${fetchTime}ms, returning legacy cache: ${key}`);
+                        return legacyValue;
+                    }
+                }
+
+                // Log as warning for expected issues, error for unexpected
+                if (errorMsg.includes('rate limit') || errorMsg.includes('timeout') || errorMsg.includes('No valid')) {
+                    log.warn(tag, `Expected fetch failure after ${fetchTime}ms: ${errorMsg}`);
+                } else {
+                    log.error(tag, `Unexpected fetch failure after ${fetchTime}ms:`, error);
+                }
+
+                return this.config.defaultValue;
+            } finally {
+                // Clean up pending fetch
+                this.pendingFetches.delete(key);
+            }
+        })();
+
+        // Store the promise so concurrent requests can reuse it
+        this.pendingFetches.set(key, fetchPromise);
+
+        return fetchPromise;
     }
 
     /**