@pioneer-platform/pioneer-cache 1.0.0 → 1.0.2

package/src/core/cache-manager.ts

@@ -7,6 +7,7 @@
 
 import { BalanceCache } from '../stores/balance-cache';
 import { PriceCache } from '../stores/price-cache';
+import { PortfolioCache } from '../stores/portfolio-cache';
 import { TransactionCache } from '../stores/transaction-cache';
 import { RefreshWorker, startUnifiedWorker } from '../workers/refresh-worker';
 import type { BaseCache } from './base-cache';
@@ -24,6 +25,7 @@ export interface CacheManagerConfig {
     markets?: any;                         // Optional: if not provided, price cache won't be initialized
     enableBalanceCache?: boolean;
     enablePriceCache?: boolean;
+    enablePortfolioCache?: boolean;
     enableTransactionCache?: boolean;
     startWorkers?: boolean;                // Auto-start workers on initialization
 }
@@ -35,6 +37,7 @@ export class CacheManager {
     private redis: any;
     private balanceCache?: BalanceCache;
     private priceCache?: PriceCache;
+    private portfolioCache?: PortfolioCache;
     private transactionCache?: TransactionCache;
     private workers: RefreshWorker[] = [];
 
@@ -53,6 +56,12 @@ export class CacheManager {
             log.info(TAG, '✅ Price cache initialized');
         }
 
+        // Initialize Portfolio Cache
+        if (config.enablePortfolioCache !== false && config.balanceModule && config.markets) {
+            this.portfolioCache = new PortfolioCache(this.redis, config.balanceModule, config.markets);
+            log.info(TAG, '✅ Portfolio cache initialized');
+        }
+
         // Initialize Transaction Cache
         if (config.enableTransactionCache !== false) {
             this.transactionCache = new TransactionCache(this.redis);
@@ -89,6 +98,10 @@ export class CacheManager {
                 cacheRegistry.set('price', this.priceCache);
             }
 
+            if (this.portfolioCache) {
+                cacheRegistry.set('portfolio', this.portfolioCache);
+            }
+
             // Start unified worker if we have any caches with queues
             if (cacheRegistry.size > 0) {
                 const worker = await startUnifiedWorker(
@@ -170,6 +183,18 @@ export class CacheManager {
                 }
             }
 
+            // Check portfolio cache
+            if (this.portfolioCache) {
+                const portfolioHealth = await this.portfolioCache.getHealth(forceRefresh);
+                checks.portfolio = portfolioHealth;
+
+                if (portfolioHealth.status === 'unhealthy') {
+                    issues.push(...portfolioHealth.issues.map(i => `Portfolio: ${i}`));
+                } else if (portfolioHealth.status === 'degraded') {
+                    warnings.push(...portfolioHealth.warnings.map(w => `Portfolio: ${w}`));
+                }
+            }
+
             // Check transaction cache (simple stats check)
             if (this.transactionCache) {
                 const txStats = await this.transactionCache.getStats();
@@ -241,6 +266,7 @@ export class CacheManager {
         return {
             balance: this.balanceCache,
             price: this.priceCache,
+            portfolio: this.portfolioCache,
             transaction: this.transactionCache
         };
     }
@@ -248,12 +274,14 @@ export class CacheManager {
     /**
      * Get specific cache by name
      */
-    getCache(name: 'balance' | 'price' | 'transaction') {
+    getCache(name: 'balance' | 'price' | 'portfolio' | 'transaction') {
         switch (name) {
             case 'balance':
                 return this.balanceCache;
             case 'price':
                 return this.priceCache;
+            case 'portfolio':
+                return this.portfolioCache;
             case 'transaction':
                 return this.transactionCache;
             default:
@@ -264,7 +292,7 @@ export class CacheManager {
     /**
      * Clear all caches (use with caution!)
      */
-    async clearAll(): Promise<{ balance?: number; price?: number; transaction?: number }> {
+    async clearAll(): Promise<{ balance?: number; price?: number; portfolio?: number; transaction?: number }> {
         const tag = TAG + 'clearAll | ';
 
         try {
@@ -278,6 +306,10 @@ export class CacheManager {
                 result.price = await this.priceCache.clearAll();
             }
 
+            if (this.portfolioCache) {
+                result.portfolio = await this.portfolioCache.clearAll();
+            }
+
             if (this.transactionCache) {
                 result.transaction = await this.transactionCache.clearAll();
             }

package/src/index.ts

@@ -12,6 +12,7 @@ export { CacheManager } from './core/cache-manager';
 // Cache implementations
 export { BalanceCache } from './stores/balance-cache';
 export { PriceCache } from './stores/price-cache';
+export { PortfolioCache } from './stores/portfolio-cache';
 export { TransactionCache } from './stores/transaction-cache';
 
 // Worker exports
@@ -30,6 +31,7 @@ export type {
 // Data type exports
 export type { BalanceData } from './stores/balance-cache';
 export type { PriceData } from './stores/price-cache';
+export type { PortfolioData, ChartData } from './stores/portfolio-cache';
 
 // Config type export
 export type { CacheManagerConfig } from './core/cache-manager';

package/src/stores/balance-cache.ts

@@ -34,14 +34,14 @@ export class BalanceCache extends BaseCache<BalanceData> {
         const defaultConfig: CacheConfig = {
             name: 'balance',
             keyPrefix: 'balance_v2:',
-            ttl: 5 * 60 * 1000,                    // 5 minutes
-            staleThreshold: 2 * 60 * 1000,         // 2 minutes
-            enableTTL: true,
-            queueName: 'balance-refresh-v2',
+            ttl: 0,                                // Ignored when enableTTL: false
+            staleThreshold: 5 * 60 * 1000,         // 5 minutes - triggers background refresh
+            enableTTL: false,                      // NEVER EXPIRE - data persists forever
+            queueName: 'cache-refresh',
             enableQueue: true,
             maxRetries: 3,
             retryDelay: 10000,
-            blockOnMiss: true,                     // Wait for fresh data on first request
+            blockOnMiss: true,                     // Wait for fresh data on first request - users need real balances!
             enableLegacyFallback: true,
             defaultValue: {
                 caip: '',
@@ -166,20 +166,79 @@ export class BalanceCache extends BaseCache<BalanceData> {
 
     /**
      * Get balances for multiple assets (batch operation)
+     * OPTIMIZED: Uses Redis MGET for single round-trip instead of N individual GETs
      */
     async getBatchBalances(items: Array<{ caip: string; pubkey: string }>, waitForFresh?: boolean): Promise<BalanceData[]> {
         const tag = this.TAG + 'getBatchBalances | ';
         const startTime = Date.now();
 
         try {
-            log.info(tag, `Batch request for ${items.length} balances`);
+            log.info(tag, `Batch request for ${items.length} balances using Redis MGET`);
+
+            // Build all Redis keys
+            const keys = items.map(item => this.buildKey({ caip: item.caip, pubkey: item.pubkey }));
+
+            // PERF: Use MGET to fetch all keys in ONE Redis round-trip
+            const cachedValues = await this.redis.mget(...keys);
+
+            // Process results
+            const results: BalanceData[] = [];
+            const missedItems: Array<{ caip: string; pubkey: string; index: number }> = [];
+
+            for (let i = 0; i < items.length; i++) {
+                const item = items[i];
+                const cached = cachedValues[i];
+
+                if (cached) {
+                    try {
+                        const parsed = JSON.parse(cached);
+                        if (parsed.value && parsed.value.caip && parsed.value.pubkey) {
+                            results[i] = parsed.value;
+                            continue;
+                        }
+                    } catch (e) {
+                        log.warn(tag, `Failed to parse cached value for ${keys[i]}`);
+                    }
+                }
 
-            // Get all balances in parallel
-            const promises = items.map(item => this.getBalance(item.caip, item.pubkey, waitForFresh));
-            const results = await Promise.all(promises);
+                // Cache miss - record for fetching
+                missedItems.push({ ...item, index: i });
+                results[i] = this.config.defaultValue; // Placeholder
+            }
 
             const responseTime = Date.now() - startTime;
-            log.info(tag, `Batch completed: ${results.length} balances in ${responseTime}ms (${(responseTime / results.length).toFixed(1)}ms avg)`);
+            const hitRate = ((items.length - missedItems.length) / items.length * 100).toFixed(1);
+            log.info(tag, `MGET completed: ${items.length} keys in ${responseTime}ms (${hitRate}% hit rate)`);
+
+            // If we have cache misses and blocking is enabled, fetch them
+            if (missedItems.length > 0) {
+                const shouldBlock = waitForFresh !== undefined ? waitForFresh : this.config.blockOnMiss;
+                
+                if (shouldBlock) {
+                    log.info(tag, `Fetching ${missedItems.length} cache misses...`);
+                    const fetchStart = Date.now();
+                    
+                    // Fetch all misses in parallel
+                    const fetchPromises = missedItems.map(async (item) => {
+                        try {
+                            // Use fetchFresh to ensure Redis is updated and requests are deduplicated
+                            const freshData = await this.fetchFresh({ caip: item.caip, pubkey: item.pubkey });
+                            results[item.index] = freshData;
+                        } catch (error) {
+                            log.error(tag, `Failed to fetch ${item.caip}/${item.pubkey}:`, error);
+                            results[item.index] = { caip: item.caip, pubkey: item.pubkey, balance: '0' };
+                        }
+                    });
+                    
+                    await Promise.all(fetchPromises);
+                    log.info(tag, `Fetched ${missedItems.length} misses in ${Date.now() - fetchStart}ms`);
+                } else {
+                    // Non-blocking: trigger background refresh for misses
+                    missedItems.forEach(item => {
+                        this.triggerAsyncRefresh({ caip: item.caip, pubkey: item.pubkey }, 'high');
+                    });
+                }
+            }
 
             return results;
 

package/src/stores/portfolio-cache.ts

@@ -0,0 +1,244 @@
+/*
+    PortfolioCache - Portfolio/Charts cache implementation
+
+    Extends BaseCache with portfolio-specific logic.
+    Designed for NON-BLOCKING, instant returns.
+*/
+
+import { BaseCache } from '../core/base-cache';
+import type { CacheConfig } from '../types';
+
+const log = require('@pioneer-platform/loggerdog')();
+
+/**
+ * Portfolio chart data structure
+ * Represents a single asset balance with pricing for charts
+ */
+export interface ChartData {
+    caip: string;
+    pubkey: string;
+    networkId: string;
+    symbol: string;
+    name: string;
+    balance: string;
+    priceUsd: number;
+    valueUsd: number;
+    icon?: string;
+    type?: string;          // 'native', 'token', etc.
+    decimal?: number;
+}
+
+/**
+ * Full portfolio data for a pubkey set
+ */
+export interface PortfolioData {
+    pubkeys: Array<{ pubkey: string; caip: string }>;
+    charts: ChartData[];
+    totalValueUsd: number;
+    timestamp: number;
+}
+
+/**
+ * PortfolioCache - Caches portfolio/chart data
+ * 
+ * CRITICAL: This cache is NON-BLOCKING by design
+ * - Returns empty arrays immediately on cache miss
+ * - Never blocks waiting for blockchain APIs
+ * - Background jobs populate cache for next request
+ */
+export class PortfolioCache extends BaseCache<PortfolioData> {
+    private balanceModule: any;
+    private marketsModule: any;
+
+    constructor(redis: any, balanceModule: any, marketsModule: any, config?: Partial<CacheConfig>) {
+        const defaultConfig: CacheConfig = {
+            name: 'portfolio',
+            keyPrefix: 'portfolio_v2:',
+            ttl: 0,                                // Ignored when enableTTL: false
+            staleThreshold: 5 * 60 * 1000,         // 5 minutes - triggers background refresh
+            enableTTL: false,                      // NEVER EXPIRE - data persists forever, show stale data instantly
+            queueName: 'cache-refresh',
+            enableQueue: true,
+            maxRetries: 3,
+            retryDelay: 5000,
+            blockOnMiss: false,                    // CRITICAL: NEVER WAIT! Return empty arrays instantly
+            enableLegacyFallback: false,           // No legacy portfolio cache format
+            defaultValue: {
+                pubkeys: [],
+                charts: [],
+                totalValueUsd: 0,
+                timestamp: Date.now()
+            },
+            useSyncFallback: false,                // CRITICAL: NEVER use synchronous fallback - always return instantly
+            maxConcurrentJobs: 3,                  // Limit concurrent portfolio refreshes
+            apiTimeout: 30000,                     // 30s timeout for full portfolio fetch
+            logCacheHits: true,
+            logCacheMisses: true,
+            logRefreshJobs: true
+        };
+
+        super(redis, { ...defaultConfig, ...config });
+        this.balanceModule = balanceModule;
+        this.marketsModule = marketsModule;
+    }
+
+    /**
+     * Build Redis key for portfolio data
+     * 
+     * Key strategy: Hash all pubkeys+caips to create a stable identifier
+     * Format: portfolio_v2:hash(pubkeys)
+     * 
+     * This allows caching the same portfolio regardless of pubkey order
+     */
+    protected buildKey(params: Record<string, any>): string {
+        const { pubkeys } = params;
+        if (!pubkeys || !Array.isArray(pubkeys) || pubkeys.length === 0) {
+            throw new Error('PortfolioCache.buildKey: pubkeys array required');
+        }
+
+        // Sort pubkeys to create stable hash regardless of order
+        const sorted = [...pubkeys].sort((a, b) => {
+            const aKey = `${a.caip}:${a.pubkey}`;
+            const bKey = `${b.caip}:${b.pubkey}`;
+            return aKey.localeCompare(bKey);
+        });
+
+        // Create a simple hash from sorted pubkeys
+        const keyString = sorted.map(p => `${p.caip}:${p.pubkey}`).join('|');
+        const hash = this.simpleHash(keyString);
+
+        return `${this.config.keyPrefix}${hash}`;
+    }
+
+    /**
+     * Simple hash function for cache keys
+     * Not cryptographic - just needs to be stable and collision-resistant
+     */
+    private simpleHash(str: string): string {
+        let hash = 0;
+        for (let i = 0; i < str.length; i++) {
+            const char = str.charCodeAt(i);
+            hash = ((hash << 5) - hash) + char;
+            hash = hash & hash; // Convert to 32-bit integer
+        }
+        return Math.abs(hash).toString(36);
+    }
+
+    /**
+     * Fetch portfolio from blockchain APIs
+     * 
+     * This is the SLOW operation that happens in the background
+     * It fetches balances for all pubkeys and enriches with pricing
+     */
+    protected async fetchFromSource(params: Record<string, any>): Promise<PortfolioData> {
+        const tag = this.TAG + 'fetchFromSource | ';
+        const startTime = Date.now();
+
+        try {
+            const { pubkeys } = params;
+            log.info(tag, `Fetching portfolio for ${pubkeys.length} pubkeys`);
+
+            const charts: ChartData[] = [];
+
+            // Fetch balances for all pubkeys in parallel
+            const balancePromises = pubkeys.map(async (item: { pubkey: string; caip: string }) => {
+                try {
+                    // Extract networkId from CAIP
+                    const networkId = item.caip.split('/')[0];
+
+                    // Fetch balance
+                    const asset = { caip: item.caip };
+                    const owner = { pubkey: item.pubkey };
+                    const balanceInfo = await this.balanceModule.getBalance(asset, owner);
+
+                    if (!balanceInfo || !balanceInfo.balance) {
+                        log.debug(tag, `No balance for ${item.caip}/${item.pubkey.substring(0, 10)}...`);
+                        return null;
+                    }
+
+                    // Skip zero balances
+                    const balanceNum = parseFloat(balanceInfo.balance);
+                    if (isNaN(balanceNum) || balanceNum === 0) {
+                        return null;
+                    }
+
+                    // Get asset metadata
+                    const assetData = require('@pioneer-platform/pioneer-discovery').assetData;
+                    const assetInfo = assetData[item.caip.toUpperCase()] || assetData[item.caip.toLowerCase()] || {};
+
+                    // Get price
+                    let priceUsd = 0;
+                    try {
+                        priceUsd = await this.marketsModule.getAssetPriceByCaip(item.caip);
+                        if (isNaN(priceUsd) || priceUsd < 0) {
+                            priceUsd = 0;
+                        }
+                    } catch (priceError) {
+                        log.warn(tag, `Error fetching price for ${item.caip}:`, priceError);
+                    }
+
+                    const valueUsd = balanceNum * priceUsd;
+
+                    const chartData: ChartData = {
+                        caip: item.caip,
+                        pubkey: item.pubkey,
+                        networkId,
+                        symbol: assetInfo.symbol || 'UNKNOWN',
+                        name: assetInfo.name || 'Unknown Asset',
+                        balance: balanceInfo.balance,
+                        priceUsd,
+                        valueUsd,
+                        icon: assetInfo.icon || '',
+                        type: assetInfo.type || 'native',
+                        decimal: assetInfo.decimal
+                    };
+
+                    return chartData;
+
+                } catch (error) {
+                    log.error(tag, `Error fetching balance for ${item.caip}/${item.pubkey}:`, error);
+                    return null;
+                }
+            });
+
+            const results = await Promise.all(balancePromises);
+            
+            // Filter out nulls and calculate total
+            const validCharts = results.filter((c): c is ChartData => c !== null);
+            const totalValueUsd = validCharts.reduce((sum, c) => sum + c.valueUsd, 0);
+
+            const fetchTime = Date.now() - startTime;
+            log.info(tag, `✅ Fetched portfolio: ${validCharts.length} assets, $${totalValueUsd.toFixed(2)} in ${fetchTime}ms`);
+
+            return {
+                pubkeys,
+                charts: validCharts,
+                totalValueUsd,
+                timestamp: Date.now()
+            };
+
+        } catch (error) {
+            log.error(tag, 'Error fetching portfolio:', error);
+            throw error;
+        }
+    }
+
+    /**
+     * No legacy cache format for portfolios
+     */
+    protected async getLegacyCached(params: Record<string, any>): Promise<PortfolioData | null> {
+        return null;
+    }
+
+    /**
+     * Get portfolio for a set of pubkeys
+     * Convenience method that wraps base get()
+     * 
+     * RETURNS INSTANTLY - either cached data or empty arrays
+     */
+    async getPortfolio(pubkeys: Array<{ pubkey: string; caip: string }>, waitForFresh?: boolean): Promise<PortfolioData> {
+        const result = await this.get({ pubkeys }, waitForFresh);
+        return result.value || this.config.defaultValue;
+    }
+}
+

package/src/stores/price-cache.ts

@@ -29,21 +29,22 @@ export class PriceCache extends BaseCache<PriceData> {
         const defaultConfig: CacheConfig = {
             name: 'price',
             keyPrefix: 'price_v2:',
-            ttl: 60 * 60 * 1000,                   // 1 hour
-            staleThreshold: 30 * 60 * 1000,        // 30 minutes (refresh after 30min)
-            enableTTL: true,
-            queueName: 'price-refresh-v2',
+            ttl: 0,                                // Ignored when enableTTL: false
+            staleThreshold: 30 * 60 * 1000,        // 30 minutes - triggers background refresh
+            enableTTL: false,                      // NEVER EXPIRE - data persists forever, show stale prices instantly
+            queueName: 'cache-refresh',
             enableQueue: true,
             maxRetries: 3,
             retryDelay: 5000,
-            blockOnMiss: false,                    // Return immediately with $0 (prices less critical)
+            blockOnMiss: false,                    // CRITICAL: NEVER WAIT! Return $0 instantly on cache miss, refresh async in background
             enableLegacyFallback: true,
             defaultValue: {
                 caip: '',
                 price: 0
             },
+            useSyncFallback: false,                // CRITICAL: NEVER use synchronous fallback - always return instantly with $0
             maxConcurrentJobs: 5,
-            apiTimeout: 5000,
+            apiTimeout: 2000,                      // Reduced from 5000ms for faster failures
             logCacheHits: false,
             logCacheMisses: true,
             logRefreshJobs: true
@@ -68,7 +69,8 @@ export class PriceCache extends BaseCache<PriceData> {
     }
 
     /**
-     * Fetch price from markets API
+     * Fetch price from markets API using CAIP-first approach
+     * FIX #7: Graceful handling of zero prices to prevent cache disruption
      */
     protected async fetchFromSource(params: Record<string, any>): Promise<PriceData> {
         const tag = this.TAG + 'fetchFromSource | ';
@@ -76,48 +78,52 @@ export class PriceCache extends BaseCache<PriceData> {
         try {
             const { caip } = params;
 
-            // Map CAIP to symbol for markets API
-            const assetData = require('@pioneer-platform/pioneer-discovery').assetData;
-            const asset = assetData[caip.toUpperCase()] || assetData[caip.toLowerCase()];
+            // Use CAIP-first API (no symbol conversion needed!)
+            // This directly queries the markets module with CAIP identifiers
+            const price = await this.markets.getAssetPriceByCaip(caip);
 
-            if (!asset || !asset.symbol) {
-                log.warn(tag, `No asset mapping found for ${caip}`);
-                return {
-                    caip,
-                    price: 0
-                };
-            }
+            // FIX #7: Gracefully handle zero prices without throwing
+            // This prevents disrupting batch operations during API rate limits
+            if (isNaN(price) || price <= 0) {
+                log.warn(tag, `Price fetch returned $${price} for ${caip} (likely API timeout or rate limit) - returning stale cache if available`);
 
-            // Get price from markets module
-            const symbol = asset.symbol.toLowerCase();
-            const priceResult = await this.markets.getPrice(symbol);
+                // Try to get stale cached value instead of failing
+                const key = this.buildKey(params);
+                const cachedValue = await this.getCached(key);
 
-            // Handle different response formats
-            let price = 0;
-            if (typeof priceResult === 'object' && priceResult.price) {
-                price = parseFloat(priceResult.price);
-            } else if (typeof priceResult === 'number') {
-                price = priceResult;
-            }
+                if (cachedValue && cachedValue.value.price > 0) {
+                    log.info(tag, `Returning stale cached price for ${caip}: $${cachedValue.value.price}`);
+                    return cachedValue.value;
+                }
 
-            if (isNaN(price) || price <= 0) {
-                log.warn(tag, `Invalid price for ${caip} (${symbol}): ${priceResult}`);
-                return {
-                    caip,
-                    price: 0
-                };
+                // Try legacy cache as fallback
+                const legacyValue = await this.getLegacyCached(params);
+                if (legacyValue && legacyValue.price > 0) {
+                    log.info(tag, `Returning legacy cached price for ${caip}: $${legacyValue.price}`);
+                    return legacyValue;
+                }
+
+                // Last resort: return zero price but don't cache it
+                log.warn(tag, `No cached price available for ${caip}, returning zero`);
+                throw new Error(`No valid price available for ${caip}`);
             }
 
-            log.debug(tag, `Fetched price for ${caip} (${symbol}): $${price}`);
+            log.debug(tag, `Fetched price for ${caip}: $${price}`);
 
             return {
                 caip,
                 price,
-                source: 'markets'
+                source: 'markets-caip'
             };
 
         } catch (error) {
-            log.error(tag, 'Error fetching price:', error);
+            // Log as warning instead of error for expected API issues
+            const errorMsg = error instanceof Error ? error.message : String(error);
+            if (errorMsg.includes('rate limit') || errorMsg.includes('timeout') || errorMsg.includes('No valid price')) {
+                log.warn(tag, `Expected API issue: ${errorMsg}`);
+            } else {
+                log.error(tag, `Unexpected error fetching price:`, error);
+            }
             throw error;
         }
     }

package/src/types/index.ts

@@ -25,6 +25,7 @@ export interface CacheConfig {
     blockOnMiss: boolean;                  // Wait for fresh data on cache miss
     enableLegacyFallback: boolean;         // Try legacy cache keys on miss
     defaultValue: any;                     // Default value to return on error
+    useSyncFallback?: boolean;             // Use synchronous fallback when queue fails (default: true for blockOnMiss, false otherwise)
 
     // Performance
     maxConcurrentJobs: number;             // Max jobs processed concurrently