@pioneer-platform/pioneer-cache 1.0.0

package/.turbo/turbo-build.log

@@ -0,0 +1,2 @@
+
+$ tsc

package/README.md

@@ -0,0 +1,451 @@
+# @pioneer-platform/pioneer-cache
+
+Unified caching system for Pioneer Platform with stale-while-revalidate pattern, TTL management, background refresh workers, and production health monitoring.
+
+## Features
+
+✅ **Stale-While-Revalidate** - Return cached data instantly, refresh in background
+✅ **Automatic TTL Management** - Redis keys auto-expire, preventing stale data
+✅ **Background Refresh Workers** - Queue-based async refresh with retry logic
+✅ **Health Monitoring** - Real-time cache health checks with staleness metrics
+✅ **Legacy Migration** - Automatic fallback to old cache formats
+✅ **Blocking Mode** - Wait for fresh data on first request (configurable)
+✅ **Batch Operations** - Efficient parallel requests for multiple items
+✅ **TypeScript** - Full type safety with generics
+
+## Installation
+
+```bash
+bun add @pioneer-platform/pioneer-cache
+```
+
+## Architecture
+
+### BaseCache Pattern
+
+All cache implementations extend `BaseCache<T>` which provides:
+- Stale-while-revalidate logic
+- TTL management
+- Background refresh coordination
+- Health monitoring
+- Legacy fallback
+
+Subclasses implement only 3 methods:
+1. `buildKey(params)` - Build Redis key from parameters
+2. `fetchFromSource(params)` - Fetch fresh data from source
+3. `getLegacyCached(params)` - Optional legacy cache migration
+
+### Cache Types
+
+**BalanceCache** - Blockchain balance data (5 minute TTL, 2 minute refresh)
+- Fetches from blockchain via balance module
+- Blocks on first miss for accurate balances
+- Supports batch balance requests
+
+**PriceCache** - USD price data (1 hour TTL, 30 minute refresh)
+- Fetches from markets API
+- Returns immediately with $0 on miss (non-blocking)
+- Supports batch price requests
+
+**TransactionCache** - Immutable transaction data (no TTL)
+- Classic cache-aside pattern
+- Permanent caching (blockchain data never changes)
+- No background workers needed
+
+## Quick Start
+
+### Using CacheManager (Recommended)
+
+```typescript
+import { CacheManager } from '@pioneer-platform/pioneer-cache';
+
+const cacheManager = new CacheManager({
+    redis,                              // Redis client
+    balanceModule,                      // Balance fetcher
+    markets,                            // Price fetcher
+    enableBalanceCache: true,
+    enablePriceCache: true,
+    enableTransactionCache: true,
+    startWorkers: true                  // Auto-start background workers
+});
+
+// Get caches
+const { balance, price, transaction } = cacheManager.getCaches();
+
+// Get balance
+const balanceData = await balance.getBalance('eip155:1/slip44:60', 'xpub123...');
+
+// Get price
+const btcPrice = await price.getPrice('eip155:1/slip44:0');
+
+// Get transaction
+const tx = await transaction.getOrFetch('txid123...', async () => {
+    return await fetchTransactionFromBlockchain(txid);
+});
+
+// Health check
+const health = await cacheManager.getHealth();
+console.log(health.status); // 'healthy' | 'degraded' | 'unhealthy'
+```
+
+### Using Individual Caches
+
+```typescript
+import { BalanceCache, PriceCache, TransactionCache } from '@pioneer-platform/pioneer-cache';
+
+// Balance Cache
+const balanceCache = new BalanceCache(redis, balanceModule, {
+    ttl: 5 * 60 * 1000,                // 5 minutes
+    staleThreshold: 2 * 60 * 1000,     // Refresh after 2 minutes
+    blockOnMiss: true                   // Wait for first request
+});
+
+const balance = await balanceCache.getBalance('eip155:1/slip44:60', 'xpub123...');
+console.log(balance.balance);          // "1234567890"
+
+// Price Cache
+const priceCache = new PriceCache(redis, markets, {
+    ttl: 60 * 60 * 1000,               // 1 hour
+    staleThreshold: 30 * 60 * 1000,    // Refresh after 30 minutes
+    blockOnMiss: false                  // Return $0 immediately
+});
+
+const price = await priceCache.getPrice('eip155:1/slip44:60');
+console.log(price);                    // 2500.00
+
+// Transaction Cache
+const txCache = new TransactionCache(redis);
+
+const tx = await txCache.getOrFetch('txid123...', async () => {
+    return await blockchain.getTransaction('txid123...');
+});
+```
+
+### Batch Operations
+
+```typescript
+// Batch balances
+const items = [
+    { caip: 'eip155:1/slip44:60', pubkey: 'xpub123...' },
+    { caip: 'eip155:1/slip44:0', pubkey: 'xpub456...' },
+];
+const balances = await balanceCache.getBatchBalances(items);
+
+// Batch prices
+const caips = ['eip155:1/slip44:60', 'eip155:1/slip44:0'];
+const prices = await priceCache.getBatchPrices(caips);
+console.log(prices.get('eip155:1/slip44:60')); // 2500.00
+```
+
+## Configuration
+
+### CacheConfig Interface
+
+```typescript
+interface CacheConfig {
+    name: string;                      // Cache name for logging
+    keyPrefix: string;                 // Redis key prefix (e.g., "balance_v2:")
+    ttl: number;                       // Time-to-live in milliseconds
+    staleThreshold: number;            // Refresh threshold in milliseconds
+    enableTTL: boolean;                // Enable automatic expiration
+    queueName: string;                 // Queue name for background jobs
+    enableQueue: boolean;              // Enable background refresh
+    maxRetries: number;                // Max retry attempts
+    retryDelay: number;                // Delay between retries (ms)
+    blockOnMiss: boolean;              // Block on first request vs return default
+    enableLegacyFallback: boolean;     // Try legacy cache formats
+    defaultValue: T;                   // Default value on miss
+    maxConcurrentJobs: number;         // Worker concurrency limit
+    apiTimeout: number;                // Source API timeout (ms)
+    logCacheHits: boolean;             // Log cache hits
+    logCacheMisses: boolean;           // Log cache misses
+    logRefreshJobs: boolean;           // Log refresh jobs
+}
+```
+
+### Default Configurations
+
+**Balance Cache** (Critical - block on miss)
+```typescript
+{
+    ttl: 5 * 60 * 1000,                // 5 minutes
+    staleThreshold: 2 * 60 * 1000,     // Refresh after 2 minutes
+    blockOnMiss: true,                  // Wait for fresh data
+    maxRetries: 3,
+    retryDelay: 10000
+}
+```
+
+**Price Cache** (Non-critical - return immediately)
+```typescript
+{
+    ttl: 60 * 60 * 1000,               // 1 hour
+    staleThreshold: 30 * 60 * 1000,    // Refresh after 30 minutes
+    blockOnMiss: false,                 // Return $0 immediately
+    maxRetries: 3,
+    retryDelay: 5000
+}
+```
+
+## Background Workers
+
+### Unified Worker
+
+A single worker processes refresh jobs for all cache types:
+
+```typescript
+import { startUnifiedWorker } from '@pioneer-platform/pioneer-cache';
+
+const cacheRegistry = new Map();
+cacheRegistry.set('balance', balanceCache);
+cacheRegistry.set('price', priceCache);
+
+const worker = await startUnifiedWorker(
+    redis,
+    cacheRegistry,
+    'cache-refresh',
+    {
+        maxRetries: 3,
+        retryDelay: 5000,
+        pollInterval: 100
+    }
+);
+
+// Worker stats
+const stats = await worker.getStats();
+console.log(stats.queueLength);        // Pending jobs
+console.log(stats.isRunning);          // Worker status
+
+// Stop worker
+await worker.stop();
+```
+
+## Health Monitoring
+
+### Cache Health
+
+```typescript
+const health = await balanceCache.getHealth();
+
+console.log(health.status);            // 'healthy' | 'degraded' | 'unhealthy'
+console.log(health.queueInitialized);  // true/false
+console.log(health.redisConnected);    // true/false
+console.log(health.stats);             // Cache statistics
+console.log(health.issues);            // Critical issues (unhealthy)
+console.log(health.warnings);          // Non-critical warnings (degraded)
+```
+
+**Status Levels:**
+- `healthy` - All systems operational
+- `degraded` - Non-critical issues (warnings)
+- `unhealthy` - Critical issues requiring immediate attention
+
+### Aggregate Health (CacheManager)
+
+```typescript
+const health = await cacheManager.getHealth();
+
+console.log(health.status);            // Overall status
+console.log(health.checks.balance);    // Balance cache health
+console.log(health.checks.price);      // Price cache health
+console.log(health.checks.transaction);// Transaction cache health
+console.log(health.checks.worker);     // Worker health
+```
+
+## Critical Fixes Included
+
+This module fixes 5 critical bugs from the legacy cache implementations:
+
+### ✅ Fix #1: Cache Miss Blocking
+**Problem:** Cache miss returned "0" immediately instead of waiting for fresh data
+**Solution:** Added `blockOnMiss` config and `fetchFresh()` method that waits for source
+
+### ✅ Fix #2: Redis TTL Management
+**Problem:** Cache values never expired, causing stale data to stick forever
+**Solution:** All cache writes include TTL: `redis.set(key, value, 'EX', ttlSeconds)`
+
+### ✅ Fix #3: Loud Queue Failures
+**Problem:** Queue initialization failures were silent (debug logs only)
+**Solution:** Changed to ERROR logs and added validation checks
+
+### ✅ Fix #4: Synchronous Fallback
+**Problem:** When queue failed, no fallback - refresh never happened
+**Solution:** Added immediate synchronous refresh when queue unavailable
+
+### ✅ Fix #5: Health Monitoring
+**Problem:** No way to detect cache issues in production
+**Solution:** Added `getHealth()` with staleness detection and issue reporting
+
+## Legacy Migration
+
+The cache automatically migrates from old formats:
+
+**Balance Cache Legacy:**
+- `cache:balance:pubkey:networkId` → `balance_v2:caip:pubkey`
+
+**Price Cache Legacy:**
+- `coingecko:caip` → `price_v2:caip`
+- `coincap:caip` → `price_v2:caip`
+
+Legacy keys are automatically migrated to new format on first access.
+
+## Advanced Usage
+
+### Custom Cache Implementation
+
+```typescript
+import { BaseCache, CacheConfig, CacheResult } from '@pioneer-platform/pioneer-cache';
+
+interface MyData {
+    id: string;
+    value: number;
+}
+
+class MyCache extends BaseCache<MyData> {
+    protected buildKey(params: Record<string, any>): string {
+        return `${this.config.keyPrefix}${params.id}`;
+    }
+
+    protected async fetchFromSource(params: Record<string, any>): Promise<MyData> {
+        // Fetch from your source
+        const response = await fetch(`https://api.example.com/data/${params.id}`);
+        return response.json();
+    }
+
+    protected async getLegacyCached(params: Record<string, any>): Promise<MyData | null> {
+        // Optional: migrate from old cache format
+        return null;
+    }
+}
+
+// Use it
+const myCache = new MyCache(redis, {
+    name: 'my-cache',
+    keyPrefix: 'my:',
+    ttl: 60 * 1000,
+    // ... other config
+});
+
+const data = await myCache.get({ id: '123' });
+```
+
+### Manual Refresh
+
+```typescript
+// Force refresh specific item
+await balanceCache.fetchFresh({ caip: 'eip155:1/slip44:60', pubkey: 'xpub123...' });
+
+// Clear all caches
+const cleared = await cacheManager.clearAll();
+console.log(cleared.balance);          // Number of keys deleted
+console.log(cleared.price);
+console.log(cleared.transaction);
+```
+
+## Testing
+
+```typescript
+import { BalanceCache } from '@pioneer-platform/pioneer-cache';
+
+describe('BalanceCache', () => {
+    let redis;
+    let balanceModule;
+    let cache;
+
+    beforeEach(() => {
+        redis = createMockRedis();
+        balanceModule = createMockBalanceModule();
+        cache = new BalanceCache(redis, balanceModule);
+    });
+
+    it('should block on first miss', async () => {
+        const result = await cache.getBalance('eip155:1/slip44:60', 'xpub123...');
+        expect(result.balance).not.toBe('0');
+        expect(result.balance).toBe('1234567890');
+    });
+
+    it('should return stale and refresh async', async () => {
+        // First request - cache miss, blocks
+        await cache.getBalance('eip155:1/slip44:60', 'xpub123...');
+
+        // Second request - cache hit, returns stale, refreshes async
+        const result = await cache.getBalance('eip155:1/slip44:60', 'xpub123...');
+        expect(result.source).toBe('cache_stale');
+    });
+
+    it('should include TTL on all writes', async () => {
+        await cache.getBalance('eip155:1/slip44:60', 'xpub123...');
+
+        // Verify TTL was set
+        expect(redis.set).toHaveBeenCalledWith(
+            expect.any(String),
+            expect.any(String),
+            'EX',
+            expect.any(Number)
+        );
+    });
+});
+```
+
+## Migration from Legacy Caches
+
+### Before (pioneer-server)
+
+```typescript
+// services/balance-cache.service.ts
+import { BalanceCacheService } from './services/balance-cache.service';
+
+const balanceCache = new BalanceCacheService(redis, balanceModule);
+const balance = await balanceCache.getBalance(caip, pubkey);
+```
+
+### After
+
+```typescript
+// Using CacheManager
+import { CacheManager } from '@pioneer-platform/pioneer-cache';
+
+const cacheManager = new CacheManager({
+    redis,
+    balanceModule,
+    markets,
+    enableBalanceCache: true,
+    enablePriceCache: true,
+    startWorkers: true
+});
+
+const balance = await cacheManager.getCache('balance')
+    .getBalance(caip, pubkey);
+
+// Or direct import
+import { BalanceCache } from '@pioneer-platform/pioneer-cache';
+
+const balanceCache = new BalanceCache(redis, balanceModule);
+const balance = await balanceCache.getBalance(caip, pubkey);
+```
+
+### Migration Checklist
+
+- [ ] Install `@pioneer-platform/pioneer-cache`
+- [ ] Replace `BalanceCacheService` imports with `BalanceCache`
+- [ ] Replace `PriceCacheService` imports with `PriceCache`
+- [ ] Update worker initialization to use `startUnifiedWorker()`
+- [ ] Add health check endpoints using `getHealth()`
+- [ ] Test thoroughly in staging environment
+- [ ] Monitor for issues in production
+- [ ] Remove old cache service files after validation
+
+## Performance
+
+**Token Reduction:** ~2,000 lines → ~1,800 lines (10% smaller)
+**Code Duplication:** 70-80% duplicate code eliminated
+**Maintenance:** Single BaseCache implementation for all fixes
+**Type Safety:** Full TypeScript generics support
+
+## License
+
+MIT
+
+## Contributing
+
+See main Pioneer Platform contributing guidelines.

package/dist/core/base-cache.d.ts

@@ -0,0 +1,75 @@
+import type { CacheConfig, CachedValue, CacheResult, HealthCheckResult, CacheStats } from '../types';
+export declare abstract class BaseCache<T> {
+    protected redis: any;
+    protected redisQueue: any;
+    protected config: CacheConfig;
+    protected queueInitialized: boolean;
+    protected TAG: string;
+    private cachedStats;
+    private cachedStatsTimestamp;
+    private readonly STATS_CACHE_TTL;
+    constructor(redis: any, config: CacheConfig);
+    /**
+     * Initialize Redis queue for background refresh
+     */
+    private initializeQueue;
+    /**
+     * Main cache get method
+     * Implements stale-while-revalidate pattern with optional blocking
+     */
+    get(params: Record<string, any>, waitForFresh?: boolean): Promise<CacheResult<T>>;
+    /**
+     * Get value from cache
+     */
+    protected getCached(key: string): Promise<CachedValue<T> | null>;
+    /**
+     * Update cache with new value
+     * FIX #2: Always includes TTL
+     */
+    updateCache(key: string, value: T, metadata?: Record<string, any>): Promise<void>;
+    /**
+     * Trigger background refresh job
+     * FIX #3: Loud error logging
+     * FIX #4: Synchronous fallback for high-priority
+     */
+    protected triggerAsyncRefresh(params: Record<string, any>, priority?: 'high' | 'normal' | 'low'): void;
+    /**
+     * Fetch fresh data and update cache
+     * FIX #1 & #4: Used for blocking requests and fallback
+     */
+    fetchFresh(params: Record<string, any>): Promise<T>;
+    /**
+     * Migrate legacy cache value to new format
+     * FIX #2: Includes TTL
+     */
+    protected migrateLegacyValue(key: string, value: T): void;
+    /**
+     * Get cache statistics
+     * @param forceRefresh - Skip cache and fetch fresh stats (for testing/debugging)
+     */
+    getCacheStats(forceRefresh?: boolean): Promise<CacheStats>;
+    /**
+     * FIX #5: Health check
+     * @param forceRefresh - Force refresh stats (bypasses 30s cache)
+     */
+    getHealth(forceRefresh?: boolean): Promise<HealthCheckResult>;
+    /**
+     * Clear all cache entries (use with caution)
+     */
+    clearAll(): Promise<number>;
+    /**
+     * Build Redis key from parameters
+     * Each cache type implements its own key structure
+     */
+    protected abstract buildKey(params: Record<string, any>): string;
+    /**
+     * Fetch data from source (blockchain, API, etc.)
+     * Each cache type implements its own data fetching
+     */
+    protected abstract fetchFromSource(params: Record<string, any>): Promise<T>;
+    /**
+     * Get legacy cached value (optional)
+     * Each cache type can implement legacy key migration
+     */
+    protected abstract getLegacyCached(params: Record<string, any>): Promise<T | null>;
+}