@pioneer-platform/pioneer-cache 1.0.1 → 1.0.2

package/.turbo/turbo-build.log

@@ -1 +1,2 @@
-$ tsc
+
+$ tsc

package/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @pioneer-platform/pioneer-cache
 
+## 1.0.2
+
+### Patch Changes
+
+- feed4f1: Increase Redis cache timeout from 100ms to 2000ms to prevent false cache misses and reduce debug logging
+
 ## 1.0.1
 
 ### Patch Changes

package/dist/core/base-cache.d.ts

@@ -38,6 +38,7 @@ export declare abstract class BaseCache<T> {
      * Fetch fresh data and update cache
      * FIX #1 & #4: Used for blocking requests and fallback
      * FIX #6: Request deduplication to prevent thundering herd
+     * FIX #8: Return stale cache on fetch failures
      */
     fetchFresh(params: Record<string, any>): Promise<T>;
     /**

package/dist/core/base-cache.js

@@ -57,9 +57,13 @@ class BaseCache {
         const tag = this.TAG + 'get | ';
         const startTime = Date.now();
         try {
+            const t1 = Date.now();
             const key = this.buildKey(params);
+            log.info(tag, `⏱️  buildKey took ${Date.now() - t1}ms`);
             // Step 1: Try new cache format
+            const t2 = Date.now();
             const cachedValue = await this.getCached(key);
+            log.info(tag, `⏱️  getCached took ${Date.now() - t2}ms`);
             if (cachedValue) {
                 const age = Date.now() - cachedValue.timestamp;
                 const responseTime = Date.now() - startTime;
@@ -80,7 +84,9 @@ class BaseCache {
             }
             // Step 2: Try legacy cache fallback
             if (this.config.enableLegacyFallback) {
+                const t3 = Date.now();
                 const legacyValue = await this.getLegacyCached(params);
+                log.info(tag, `⏱️  getLegacyCached took ${Date.now() - t3}ms`);
                 if (legacyValue) {
                     const responseTime = Date.now() - startTime;
                     log.info(tag, `Legacy cache hit: ${key} (${responseTime}ms)`);
@@ -114,7 +120,10 @@ class BaseCache {
                 };
             }
             // Non-blocking: trigger async refresh and return default
+            const t4 = Date.now();
             this.triggerAsyncRefresh(params, 'high');
+            log.info(tag, `⏱️  triggerAsyncRefresh took ${Date.now() - t4}ms`);
+            log.info(tag, `⏱️  Returning default value after ${Date.now() - startTime}ms TOTAL`);
             return {
                 success: true,
                 value: this.config.defaultValue,
@@ -139,18 +148,32 @@ class BaseCache {
      */
     async getCached(key) {
         const tag = this.TAG + 'getCached | ';
+        const t0 = Date.now();
         try {
-            const cached = await this.redis.get(key);
+            // Redis timeout for cache reads
+            // Increased from 100ms to 2000ms - 100ms was too aggressive and caused false cache misses
+            // Increased from 2000ms to 10000ms - IPv4/IPv6 DNS resolution can cause delays
+            const timeoutMs = 10000;
+            const cached = await Promise.race([
+                this.redis.get(key),
+                new Promise((resolve) => setTimeout(() => {
+                    log.warn(tag, `⏱️  Redis timeout after ${timeoutMs}ms, returning cache miss`);
+                    resolve(null);
+                }, timeoutMs))
+            ]);
             if (!cached) {
+                log.debug(tag, `Cache miss: ${key}`);
                 return null;
             }
             const parsed = JSON.parse(cached);
-            // Validate structure
-            if (!parsed.value || typeof parsed.timestamp !== 'number') {
+            // Validate structure - Check for undefined/null, NOT falsy values!
+            // CRITICAL: Balance "0", empty arrays [], and empty objects {} are VALID!
+            if (parsed.value === undefined || parsed.value === null || typeof parsed.timestamp !== 'number') {
                 log.warn(tag, `Invalid cache structure for ${key}, removing`);
                 await this.redis.del(key);
                 return null;
             }
+            log.debug(tag, `Cache hit: ${key}`);
             return parsed;
         }
         catch (error) {
@@ -172,20 +195,29 @@ class BaseCache {
                 lastUpdated: new Date().toISOString(),
                 metadata
             };
+            // DIAGNOSTIC: Log Redis connection details
+            const redisConstructor = this.redis.constructor?.name || 'unknown';
+            const redisOptions = this.redis.options || {};
             // FIX #2: Always set TTL (unless explicitly disabled)
             if (this.config.enableTTL) {
                 const ttlSeconds = Math.floor(this.config.ttl / 1000);
+                // PERF: Reduced logging for production performance
+                log.debug(tag, `📝 Writing to Redis: ${key} [${JSON.stringify(cachedValue).length} bytes, TTL: ${ttlSeconds}s]`);
+                // Write
                 await this.redis.set(key, JSON.stringify(cachedValue), 'EX', ttlSeconds);
-                log.info(tag, `Updated cache: ${key} [TTL: ${ttlSeconds}s]`);
+                // PERF: Verification disabled for production performance
+                // Only enable for debugging cache issues
+                log.debug(tag, `✅ Updated cache: ${key} [TTL: ${ttlSeconds}s]`);
             }
             else {
                 // Permanent caching (for transactions)
                 await this.redis.set(key, JSON.stringify(cachedValue));
-                log.info(tag, `Updated cache: ${key} [PERMANENT]`);
+                // PERF: Verification disabled for production performance
+                log.debug(tag, `✅ Updated cache: ${key} [PERMANENT]`);
             }
         }
         catch (error) {
-            log.error(tag, `Error updating cache for ${key}:`, error);
+            log.error(tag, `❌ Error updating cache for ${key}:`, error);
             throw error;
         }
     }
@@ -206,8 +238,12 @@ class BaseCache {
                 const key = this.buildKey(params);
                 log.error(tag, `❌ QUEUE NOT INITIALIZED! Cannot refresh ${key}`);
                 log.error(tag, `Background refresh is BROKEN - cache will NOT update!`);
-                // FIX #4: Synchronous fallback for high-priority
-                if (priority === 'high') {
+                // FIX #4: Synchronous fallback for high-priority (only if useSyncFallback is enabled)
+                // Default: use sync fallback only for blocking caches (blockOnMiss=true)
+                const shouldUseSyncFallback = this.config.useSyncFallback !== undefined
+                    ? this.config.useSyncFallback
+                    : this.config.blockOnMiss;
+                if (priority === 'high' && shouldUseSyncFallback) {
                     log.warn(tag, `Using synchronous fallback for high-priority refresh`);
                     setImmediate(async () => {
                         try {
@@ -249,6 +285,7 @@ class BaseCache {
      * Fetch fresh data and update cache
      * FIX #1 & #4: Used for blocking requests and fallback
      * FIX #6: Request deduplication to prevent thundering herd
+     * FIX #8: Return stale cache on fetch failures
      */
     async fetchFresh(params) {
         const tag = this.TAG + 'fetchFresh | ';
@@ -257,6 +294,12 @@ class BaseCache {
         // FIX #6: Check if there's already a pending fetch for this key
         const existingFetch = this.pendingFetches.get(key);
         if (existingFetch) {
+            // For non-blocking caches, return default value immediately instead of waiting
+            if (!this.config.blockOnMiss) {
+                log.debug(tag, `Non-blocking cache: returning default value while fetch in progress: ${key}`);
+                return this.config.defaultValue;
+            }
+            // For blocking caches, coalesce to prevent thundering herd
             log.debug(tag, `Coalescing request for: ${key} (fetch already in progress)`);
             return existingFetch;
         }
@@ -274,7 +317,28 @@ class BaseCache {
             }
             catch (error) {
                 const fetchTime = Date.now() - startTime;
-                log.error(tag, `Failed to fetch fresh data after ${fetchTime}ms:`, error);
+                const errorMsg = error instanceof Error ? error.message : String(error);
+                // FIX #8: Try to return stale cache on fetch failures
+                const cachedValue = await this.getCached(key);
+                if (cachedValue) {
+                    log.warn(tag, `Fetch failed after ${fetchTime}ms, returning stale cache: ${key}`);
+                    return cachedValue.value;
+                }
+                // Try legacy cache as last resort
+                if (this.config.enableLegacyFallback) {
+                    const legacyValue = await this.getLegacyCached(params);
+                    if (legacyValue) {
+                        log.warn(tag, `Fetch failed after ${fetchTime}ms, returning legacy cache: ${key}`);
+                        return legacyValue;
+                    }
+                }
+                // Log as warning for expected issues, error for unexpected
+                if (errorMsg.includes('rate limit') || errorMsg.includes('timeout') || errorMsg.includes('No valid')) {
+                    log.warn(tag, `Expected fetch failure after ${fetchTime}ms: ${errorMsg}`);
+                }
+                else {
+                    log.error(tag, `Unexpected fetch failure after ${fetchTime}ms:`, error);
+                }
                 return this.config.defaultValue;
             }
             finally {

package/dist/core/cache-manager.d.ts

@@ -1,5 +1,6 @@
 import { BalanceCache } from '../stores/balance-cache';
 import { PriceCache } from '../stores/price-cache';
+import { PortfolioCache } from '../stores/portfolio-cache';
 import { TransactionCache } from '../stores/transaction-cache';
 import type { HealthCheckResult } from '../types';
 /**
@@ -11,6 +12,7 @@ export interface CacheManagerConfig {
     markets?: any;
     enableBalanceCache?: boolean;
     enablePriceCache?: boolean;
+    enablePortfolioCache?: boolean;
     enableTransactionCache?: boolean;
     startWorkers?: boolean;
 }
@@ -21,6 +23,7 @@ export declare class CacheManager {
     private redis;
     private balanceCache?;
     private priceCache?;
+    private portfolioCache?;
     private transactionCache?;
     private workers;
     constructor(config: CacheManagerConfig);
@@ -45,18 +48,20 @@ export declare class CacheManager {
     getCaches(): {
         balance: BalanceCache | undefined;
         price: PriceCache | undefined;
+        portfolio: PortfolioCache | undefined;
         transaction: TransactionCache | undefined;
     };
     /**
      * Get specific cache by name
      */
-    getCache(name: 'balance' | 'price' | 'transaction'): BalanceCache | PriceCache | TransactionCache | undefined;
+    getCache(name: 'balance' | 'price' | 'portfolio' | 'transaction'): BalanceCache | PriceCache | PortfolioCache | TransactionCache | undefined;
     /**
      * Clear all caches (use with caution!)
      */
     clearAll(): Promise<{
         balance?: number;
         price?: number;
+        portfolio?: number;
         transaction?: number;
     }>;
 }

package/dist/core/cache-manager.js

@@ -9,6 +9,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.CacheManager = void 0;
 const balance_cache_1 = require("../stores/balance-cache");
 const price_cache_1 = require("../stores/price-cache");
+const portfolio_cache_1 = require("../stores/portfolio-cache");
 const transaction_cache_1 = require("../stores/transaction-cache");
 const refresh_worker_1 = require("../workers/refresh-worker");
 const log = require('@pioneer-platform/loggerdog')();
@@ -30,6 +31,11 @@ class CacheManager {
             this.priceCache = new price_cache_1.PriceCache(this.redis, config.markets);
             log.info(TAG, '✅ Price cache initialized');
         }
+        // Initialize Portfolio Cache
+        if (config.enablePortfolioCache !== false && config.balanceModule && config.markets) {
+            this.portfolioCache = new portfolio_cache_1.PortfolioCache(this.redis, config.balanceModule, config.markets);
+            log.info(TAG, '✅ Portfolio cache initialized');
+        }
         // Initialize Transaction Cache
         if (config.enableTransactionCache !== false) {
             this.transactionCache = new transaction_cache_1.TransactionCache(this.redis);
@@ -58,6 +64,9 @@ class CacheManager {
             if (this.priceCache) {
                 cacheRegistry.set('price', this.priceCache);
             }
+            if (this.portfolioCache) {
+                cacheRegistry.set('portfolio', this.portfolioCache);
+            }
             // Start unified worker if we have any caches with queues
             if (cacheRegistry.size > 0) {
                 const worker = await (0, refresh_worker_1.startUnifiedWorker)(this.redis, cacheRegistry, 'cache-refresh', // Unified queue name
@@ -125,6 +134,17 @@ class CacheManager {
                     warnings.push(...priceHealth.warnings.map(w => `Price: ${w}`));
                 }
             }
+            // Check portfolio cache
+            if (this.portfolioCache) {
+                const portfolioHealth = await this.portfolioCache.getHealth(forceRefresh);
+                checks.portfolio = portfolioHealth;
+                if (portfolioHealth.status === 'unhealthy') {
+                    issues.push(...portfolioHealth.issues.map(i => `Portfolio: ${i}`));
+                }
+                else if (portfolioHealth.status === 'degraded') {
+                    warnings.push(...portfolioHealth.warnings.map(w => `Portfolio: ${w}`));
+                }
+            }
             // Check transaction cache (simple stats check)
             if (this.transactionCache) {
                 const txStats = await this.transactionCache.getStats();
@@ -192,6 +212,7 @@ class CacheManager {
         return {
             balance: this.balanceCache,
             price: this.priceCache,
+            portfolio: this.portfolioCache,
             transaction: this.transactionCache
         };
     }
@@ -204,6 +225,8 @@ class CacheManager {
                 return this.balanceCache;
             case 'price':
                 return this.priceCache;
+            case 'portfolio':
+                return this.portfolioCache;
             case 'transaction':
                 return this.transactionCache;
             default:
@@ -223,6 +246,9 @@ class CacheManager {
             if (this.priceCache) {
                 result.price = await this.priceCache.clearAll();
             }
+            if (this.portfolioCache) {
+                result.portfolio = await this.portfolioCache.clearAll();
+            }
             if (this.transactionCache) {
                 result.transaction = await this.transactionCache.clearAll();
             }

package/dist/index.d.ts

@@ -2,10 +2,12 @@ export { BaseCache } from './core/base-cache';
 export { CacheManager } from './core/cache-manager';
 export { BalanceCache } from './stores/balance-cache';
 export { PriceCache } from './stores/price-cache';
+export { PortfolioCache } from './stores/portfolio-cache';
 export { TransactionCache } from './stores/transaction-cache';
 export { RefreshWorker, startUnifiedWorker } from './workers/refresh-worker';
 export type { CacheConfig, CachedValue, CacheResult, RefreshJob, HealthCheckResult, CacheStats } from './types';
 export type { BalanceData } from './stores/balance-cache';
 export type { PriceData } from './stores/price-cache';
+export type { PortfolioData, ChartData } from './stores/portfolio-cache';
 export type { CacheManagerConfig } from './core/cache-manager';
 export type { WorkerConfig } from './workers/refresh-worker';

package/dist/index.js

@@ -6,7 +6,7 @@
     Provides stale-while-revalidate caching with TTL, background refresh, and health monitoring.
 */
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.startUnifiedWorker = exports.RefreshWorker = exports.TransactionCache = exports.PriceCache = exports.BalanceCache = exports.CacheManager = exports.BaseCache = void 0;
+exports.startUnifiedWorker = exports.RefreshWorker = exports.TransactionCache = exports.PortfolioCache = exports.PriceCache = exports.BalanceCache = exports.CacheManager = exports.BaseCache = void 0;
 // Core exports
 var base_cache_1 = require("./core/base-cache");
 Object.defineProperty(exports, "BaseCache", { enumerable: true, get: function () { return base_cache_1.BaseCache; } });
@@ -17,6 +17,8 @@ var balance_cache_1 = require("./stores/balance-cache");
 Object.defineProperty(exports, "BalanceCache", { enumerable: true, get: function () { return balance_cache_1.BalanceCache; } });
 var price_cache_1 = require("./stores/price-cache");
 Object.defineProperty(exports, "PriceCache", { enumerable: true, get: function () { return price_cache_1.PriceCache; } });
+var portfolio_cache_1 = require("./stores/portfolio-cache");
+Object.defineProperty(exports, "PortfolioCache", { enumerable: true, get: function () { return portfolio_cache_1.PortfolioCache; } });
 var transaction_cache_1 = require("./stores/transaction-cache");
 Object.defineProperty(exports, "TransactionCache", { enumerable: true, get: function () { return transaction_cache_1.TransactionCache; } });
 // Worker exports

package/dist/stores/balance-cache.d.ts

@@ -39,6 +39,7 @@ export declare class BalanceCache extends BaseCache<BalanceData> {
     getBalance(caip: string, pubkey: string, waitForFresh?: boolean): Promise<BalanceData>;
     /**
      * Get balances for multiple assets (batch operation)
+     * OPTIMIZED: Uses Redis MGET for single round-trip instead of N individual GETs
      */
     getBatchBalances(items: Array<{
         caip: string;

package/dist/stores/balance-cache.js

@@ -17,14 +17,14 @@ class BalanceCache extends base_cache_1.BaseCache {
         const defaultConfig = {
             name: 'balance',
             keyPrefix: 'balance_v2:',
-            ttl: 5 * 60 * 1000, // 5 minutes
-            staleThreshold: 2 * 60 * 1000, // 2 minutes
-            enableTTL: true,
-            queueName: 'balance-refresh-v2',
+            ttl: 0, // Ignored when enableTTL: false
+            staleThreshold: 5 * 60 * 1000, // 5 minutes - triggers background refresh
+            enableTTL: false, // NEVER EXPIRE - data persists forever
+            queueName: 'cache-refresh',
             enableQueue: true,
             maxRetries: 3,
             retryDelay: 10000,
-            blockOnMiss: true, // Wait for fresh data on first request
+            blockOnMiss: true, // Wait for fresh data on first request - users need real balances!
             enableLegacyFallback: true,
             defaultValue: {
                 caip: '',
@@ -131,17 +131,70 @@ class BalanceCache extends base_cache_1.BaseCache {
     }
     /**
      * Get balances for multiple assets (batch operation)
+     * OPTIMIZED: Uses Redis MGET for single round-trip instead of N individual GETs
      */
     async getBatchBalances(items, waitForFresh) {
         const tag = this.TAG + 'getBatchBalances | ';
         const startTime = Date.now();
         try {
-            log.info(tag, `Batch request for ${items.length} balances`);
-            // Get all balances in parallel
-            const promises = items.map(item => this.getBalance(item.caip, item.pubkey, waitForFresh));
-            const results = await Promise.all(promises);
+            log.info(tag, `Batch request for ${items.length} balances using Redis MGET`);
+            // Build all Redis keys
+            const keys = items.map(item => this.buildKey({ caip: item.caip, pubkey: item.pubkey }));
+            // PERF: Use MGET to fetch all keys in ONE Redis round-trip
+            const cachedValues = await this.redis.mget(...keys);
+            // Process results
+            const results = [];
+            const missedItems = [];
+            for (let i = 0; i < items.length; i++) {
+                const item = items[i];
+                const cached = cachedValues[i];
+                if (cached) {
+                    try {
+                        const parsed = JSON.parse(cached);
+                        if (parsed.value && parsed.value.caip && parsed.value.pubkey) {
+                            results[i] = parsed.value;
+                            continue;
+                        }
+                    }
+                    catch (e) {
+                        log.warn(tag, `Failed to parse cached value for ${keys[i]}`);
+                    }
+                }
+                // Cache miss - record for fetching
+                missedItems.push({ ...item, index: i });
+                results[i] = this.config.defaultValue; // Placeholder
+            }
             const responseTime = Date.now() - startTime;
-            log.info(tag, `Batch completed: ${results.length} balances in ${responseTime}ms (${(responseTime / results.length).toFixed(1)}ms avg)`);
+            const hitRate = ((items.length - missedItems.length) / items.length * 100).toFixed(1);
+            log.info(tag, `MGET completed: ${items.length} keys in ${responseTime}ms (${hitRate}% hit rate)`);
+            // If we have cache misses and blocking is enabled, fetch them
+            if (missedItems.length > 0) {
+                const shouldBlock = waitForFresh !== undefined ? waitForFresh : this.config.blockOnMiss;
+                if (shouldBlock) {
+                    log.info(tag, `Fetching ${missedItems.length} cache misses...`);
+                    const fetchStart = Date.now();
+                    // Fetch all misses in parallel
+                    const fetchPromises = missedItems.map(async (item) => {
+                        try {
+                            // Use fetchFresh to ensure Redis is updated and requests are deduplicated
+                            const freshData = await this.fetchFresh({ caip: item.caip, pubkey: item.pubkey });
+                            results[item.index] = freshData;
+                        }
+                        catch (error) {
+                            log.error(tag, `Failed to fetch ${item.caip}/${item.pubkey}:`, error);
+                            results[item.index] = { caip: item.caip, pubkey: item.pubkey, balance: '0' };
+                        }
+                    });
+                    await Promise.all(fetchPromises);
+                    log.info(tag, `Fetched ${missedItems.length} misses in ${Date.now() - fetchStart}ms`);
+                }
+                else {
+                    // Non-blocking: trigger background refresh for misses
+                    missedItems.forEach(item => {
+                        this.triggerAsyncRefresh({ caip: item.caip, pubkey: item.pubkey }, 'high');
+                    });
+                }
+            }
             return results;
         }
         catch (error) {

package/dist/stores/portfolio-cache.d.ts

@@ -0,0 +1,79 @@
+import { BaseCache } from '../core/base-cache';
+import type { CacheConfig } from '../types';
+/**
+ * Portfolio chart data structure
+ * Represents a single asset balance with pricing for charts
+ */
+export interface ChartData {
+    caip: string;
+    pubkey: string;
+    networkId: string;
+    symbol: string;
+    name: string;
+    balance: string;
+    priceUsd: number;
+    valueUsd: number;
+    icon?: string;
+    type?: string;
+    decimal?: number;
+}
+/**
+ * Full portfolio data for a pubkey set
+ */
+export interface PortfolioData {
+    pubkeys: Array<{
+        pubkey: string;
+        caip: string;
+    }>;
+    charts: ChartData[];
+    totalValueUsd: number;
+    timestamp: number;
+}
+/**
+ * PortfolioCache - Caches portfolio/chart data
+ *
+ * CRITICAL: This cache is NON-BLOCKING by design
+ * - Returns empty arrays immediately on cache miss
+ * - Never blocks waiting for blockchain APIs
+ * - Background jobs populate cache for next request
+ */
+export declare class PortfolioCache extends BaseCache<PortfolioData> {
+    private balanceModule;
+    private marketsModule;
+    constructor(redis: any, balanceModule: any, marketsModule: any, config?: Partial<CacheConfig>);
+    /**
+     * Build Redis key for portfolio data
+     *
+     * Key strategy: Hash all pubkeys+caips to create a stable identifier
+     * Format: portfolio_v2:hash(pubkeys)
+     *
+     * This allows caching the same portfolio regardless of pubkey order
+     */
+    protected buildKey(params: Record<string, any>): string;
+    /**
+     * Simple hash function for cache keys
+     * Not cryptographic - just needs to be stable and collision-resistant
+     */
+    private simpleHash;
+    /**
+     * Fetch portfolio from blockchain APIs
+     *
+     * This is the SLOW operation that happens in the background
+     * It fetches balances for all pubkeys and enriches with pricing
+     */
+    protected fetchFromSource(params: Record<string, any>): Promise<PortfolioData>;
+    /**
+     * No legacy cache format for portfolios
+     */
+    protected getLegacyCached(params: Record<string, any>): Promise<PortfolioData | null>;
+    /**
+     * Get portfolio for a set of pubkeys
+     * Convenience method that wraps base get()
+     *
+     * RETURNS INSTANTLY - either cached data or empty arrays
+     */
+    getPortfolio(pubkeys: Array<{
+        pubkey: string;
+        caip: string;
+    }>, waitForFresh?: boolean): Promise<PortfolioData>;
+}