@pioneer-platform/pioneer-cache 1.0.6 → 1.0.8

package/__tests__/cache-concurrent-operations.test.ts

@@ -0,0 +1,393 @@
+/**
+ * Cache Concurrent Operations Stress Tests
+ *
+ * Tests to reproduce real-world cache usage patterns that cause timeouts:
+ * 1. Pioneer cache with 1000ms timeout showing warnings
+ * 2. Multiple cache types (price, balance) accessing Redis simultaneously
+ * 3. Queue workers + cache operations
+ */
+
+const { redis, redisQueue } = require('@pioneer-platform/default-redis');
+import { BaseCache } from '../src/core/base-cache';
+import type { CacheConfig, CachedValue, CacheResult } from '../src/types';
+
+// Mock cache implementation for testing
+class TestCache extends BaseCache<string> {
+    constructor(redis: any, config: Partial<CacheConfig>) {
+        const fullConfig: CacheConfig = {
+            name: 'test',
+            keyPrefix: 'test:',
+            ttl: 60000,
+            staleThreshold: 30000,
+            defaultValue: 'default',
+            enableQueue: false,
+            queueName: 'test-queue',
+            enableLegacyFallback: false,
+            blockOnMiss: false,
+            enableTTL: true,
+            logCacheHits: false,
+            logCacheMisses: false,
+            logRefreshJobs: false,
+            ...config
+        };
+        super(redis, fullConfig);
+    }
+
+    protected buildKey(params: Record<string, any>): string {
+        return `${this.config.keyPrefix}${params.id}`;
+    }
+
+    protected async fetchFromSource(params: Record<string, any>): Promise<string> {
+        // Simulate network delay
+        await new Promise(resolve => setTimeout(resolve, 100));
+        return `fetched_${params.id}`;
+    }
+
+    protected async getLegacyCached(params: Record<string, any>): Promise<string | null> {
+        return null;
+    }
+}
+
+describe('Cache Concurrent Operations Stress Tests', () => {
+    let priceCache: TestCache;
+    let balanceCache: TestCache;
+
+    beforeAll(async () => {
+        // Initialize multiple cache instances (simulating real usage)
+        priceCache = new TestCache(redis, {
+            name: 'price',
+            keyPrefix: 'test:price:',
+            staleThreshold: 10000
+        });
+
+        balanceCache = new TestCache(redis, {
+            name: 'balance',
+            keyPrefix: 'test:balance:',
+            staleThreshold: 5000
+        });
+
+        // Clear test data
+        const keys = await redis.keys('test:*');
+        if (keys.length > 0) {
+            await redis.del(...keys);
+        }
+    });
+
+    afterAll(async () => {
+        // Cleanup
+        const keys = await redis.keys('test:*');
+        if (keys.length > 0) {
+            await redis.del(...keys);
+        }
+    });
+
+    describe('Reproduce Real-World Timeout Scenarios', () => {
+        test('should reproduce price cache timeout warnings under load', async () => {
+            // Simulate the exact scenario from logs:
+            // Multiple concurrent cache.get() calls hitting 1000ms timeout
+
+            const assetIds = Array.from({ length: 50 }, (_, i) => i);
+            const startTime = Date.now();
+            let timeoutWarnings = 0;
+
+            // Hook into redis.get to detect timeouts
+            const originalGet = redis.get.bind(redis);
+            const monitoredGet = async (key: string) => {
+                const callStart = Date.now();
+                const result = await originalGet(key);
+                const duration = Date.now() - callStart;
+                if (duration > 1000) {
+                    timeoutWarnings++;
+                    console.warn(`⏱️  Redis timeout after ${duration}ms for ${key}`);
+                }
+                return result;
+            };
+            redis.get = monitoredGet;
+
+            try {
+                // Simulate portfolio loading: 50 concurrent price lookups
+                const promises = assetIds.map(id =>
+                    priceCache.get({ id: `asset_${id}` })
+                );
+
+                const results = await Promise.all(promises);
+                const totalDuration = Date.now() - startTime;
+
+                console.log(`Portfolio load: ${results.length} prices in ${totalDuration}ms`);
+                console.log(`Timeout warnings: ${timeoutWarnings}`);
+
+                expect(results).toHaveLength(50);
+                expect(timeoutWarnings).toBe(0); // Should be 0 with proper pooling
+            } finally {
+                redis.get = originalGet;
+            }
+        }, 15000);
+
+        test('should handle cache stampede without timeouts', async () => {
+            // Reproduce: Multiple users requesting same uncached asset simultaneously
+            const assetId = 'popular_asset';
+            const concurrentUsers = 100;
+
+            const startTime = Date.now();
+            const promises = Array.from({ length: concurrentUsers }, () =>
+                priceCache.get({ id: assetId }, false) // Don't block on miss
+            );
+
+            const results = await Promise.all(promises);
+            const duration = Date.now() - startTime;
+
+            console.log(`Cache stampede: ${concurrentUsers} concurrent requests in ${duration}ms`);
+
+            // All should complete without timeout
+            expect(results).toHaveLength(concurrentUsers);
+            expect(duration).toBeLessThan(5000);
+        }, 10000);
+
+        test('should handle mixed cache types concurrently', async () => {
+            // Simulate real usage: price cache + balance cache + queue operations
+            const operations = [];
+
+            // 20 price cache operations
+            for (let i = 0; i < 20; i++) {
+                operations.push(priceCache.get({ id: `asset_${i}` }));
+            }
+
+            // 20 balance cache operations
+            for (let i = 0; i < 20; i++) {
+                operations.push(balanceCache.get({ id: `wallet_${i}` }));
+            }
+
+            // 10 queue operations (non-blocking)
+            for (let i = 0; i < 10; i++) {
+                operations.push(
+                    redis.lpush(`test:queue:${i}`, `job_${i}`)
+                        .then(() => redis.rpop(`test:queue:${i}`))
+                );
+            }
+
+            const startTime = Date.now();
+            const results = await Promise.all(operations);
+            const duration = Date.now() - startTime;
+
+            console.log(`Mixed operations: ${results.length} ops in ${duration}ms`);
+            expect(duration).toBeLessThan(5000);
+        }, 15000);
+    });
+
+    describe('Cache + Queue Worker Interference', () => {
+        test('should not timeout cache operations while queue worker polls', async () => {
+            const queueName = 'test:worker:queue';
+
+            // Start simulated queue worker (polls every 5 seconds)
+            const workerPoll = async () => {
+                const result = await redisQueue.brpop(queueName, 10);
+                return result;
+            };
+
+            // Start worker poll (will block for 10 seconds)
+            const workerPromise = workerPoll();
+
+            // Meanwhile, do intensive cache operations
+            const cacheOps = [];
+            for (let i = 0; i < 100; i++) {
+                cacheOps.push(
+                    priceCache.get({ id: `asset_${i}` })
+                );
+            }
+
+            const startTime = Date.now();
+            const results = await Promise.all(cacheOps);
+            const duration = Date.now() - startTime;
+
+            console.log(`Cache ops while worker blocking: ${results.length} in ${duration}ms`);
+
+            // Cleanup: push to queue to unblock worker
+            await redis.lpush(queueName, 'cleanup');
+            await workerPromise;
+            await redis.del(queueName);
+
+            expect(results).toHaveLength(100);
+            expect(duration).toBeLessThan(5000); // Should not be blocked by BRPOP
+        }, 20000);
+
+        test('should handle cache writes while queue processing', async () => {
+            const queueName = 'test:process:queue';
+
+            // Fill queue with jobs
+            for (let i = 0; i < 10; i++) {
+                await redis.lpush(queueName, JSON.stringify({ id: i, type: 'job' }));
+            }
+
+            // Process queue in background
+            const processQueue = async () => {
+                const results = [];
+                for (let i = 0; i < 10; i++) {
+                    const job = await redisQueue.brpop(queueName, 5);
+                    if (job) {
+                        results.push(job);
+                        // Simulate job processing
+                        await new Promise(resolve => setTimeout(resolve, 50));
+                    }
+                }
+                return results;
+            };
+
+            const processingPromise = processQueue();
+
+            // While processing, do cache updates
+            const cacheUpdates = [];
+            for (let i = 0; i < 50; i++) {
+                cacheUpdates.push(
+                    priceCache.updateCache(`test:price:update_${i}`, `value_${i}`)
+                );
+            }
+
+            const [jobResults, cacheResults] = await Promise.all([
+                processingPromise,
+                Promise.all(cacheUpdates)
+            ]);
+
+            console.log(`Processed ${jobResults.length} jobs while updating ${cacheResults.length} cache entries`);
+
+            expect(jobResults).toHaveLength(10);
+            expect(cacheResults).toHaveLength(50);
+        }, 15000);
+    });
+
+    describe('High Concurrency Stress Tests', () => {
+        test('should handle 500 concurrent cache gets', async () => {
+            const operations = 500;
+            const startTime = Date.now();
+
+            const promises = Array.from({ length: operations }, (_, i) =>
+                priceCache.get({ id: `stress_${i % 50}` }) // 50 unique keys, 10x requests each
+            );
+
+            const results = await Promise.all(promises);
+            const duration = Date.now() - startTime;
+
+            const timeoutsOrErrors = results.filter(r => r.error).length;
+
+            console.log(`${operations} concurrent gets in ${duration}ms`);
+            console.log(`Errors/timeouts: ${timeoutsOrErrors}`);
+
+            expect(results).toHaveLength(operations);
+            expect(timeoutsOrErrors).toBe(0);
+            expect(duration).toBeLessThan(10000);
+        }, 20000);
+
+        test('should handle alternating read/write load', async () => {
+            const cycles = 100;
+            const startTime = Date.now();
+            let writeCount = 0;
+            let readCount = 0;
+
+            const operations = [];
+            for (let i = 0; i < cycles; i++) {
+                // Write
+                operations.push(
+                    priceCache.updateCache(`test:price:rw_${i}`, `value_${i}`)
+                        .then(() => writeCount++)
+                );
+
+                // Read
+                operations.push(
+                    priceCache.get({ id: `rw_${i}` })
+                        .then(() => readCount++)
+                );
+            }
+
+            await Promise.all(operations);
+            const duration = Date.now() - startTime;
+
+            console.log(`${cycles} read/write cycles: ${writeCount} writes, ${readCount} reads in ${duration}ms`);
+
+            expect(writeCount).toBe(cycles);
+            expect(readCount).toBe(cycles);
+            expect(duration).toBeLessThan(10000);
+        }, 20000);
+
+        test('should maintain performance under sustained concurrent load', async () => {
+            // Run for 10 seconds with constant load
+            const testDuration = 10000;
+            const startTime = Date.now();
+            let totalOps = 0;
+            let errors = 0;
+
+            const runConcurrentLoad = async (workerId: number) => {
+                while (Date.now() - startTime < testDuration) {
+                    try {
+                        // Random operation
+                        const op = Math.random();
+                        if (op < 0.5) {
+                            // Price cache read
+                            await priceCache.get({ id: `load_${workerId}_${Math.floor(Math.random() * 10)}` });
+                        } else if (op < 0.8) {
+                            // Balance cache read
+                            await balanceCache.get({ id: `balance_${workerId}` });
+                        } else {
+                            // Direct redis operation
+                            const key = `test:direct:${workerId}`;
+                            await redis.set(key, `value_${totalOps}`);
+                            await redis.get(key);
+                        }
+                        totalOps++;
+                    } catch (error) {
+                        errors++;
+                    }
+                }
+            };
+
+            // Run 10 concurrent workers
+            const workers = Array.from({ length: 10 }, (_, i) => runConcurrentLoad(i));
+            await Promise.all(workers);
+
+            const actualDuration = Date.now() - startTime;
+            const opsPerSecond = Math.floor((totalOps / actualDuration) * 1000);
+
+            console.log(`Sustained load test:`);
+            console.log(`  Duration: ${actualDuration}ms`);
+            console.log(`  Operations: ${totalOps}`);
+            console.log(`  Throughput: ${opsPerSecond} ops/sec`);
+            console.log(`  Errors: ${errors} (${((errors/totalOps)*100).toFixed(2)}%)`);
+
+            expect(errors).toBeLessThan(totalOps * 0.01); // <1% error rate
+            expect(opsPerSecond).toBeGreaterThan(100); // At least 100 ops/sec
+        }, 15000);
+    });
+
+    describe('Cache Timeout Reproduction', () => {
+        test('should not timeout with proper connection pooling', async () => {
+            // This test specifically reproduces the warning from logs:
+            // "Redis timeout after 1000ms, returning cache miss"
+
+            const iterations = 20;
+            let timeouts = 0;
+
+            for (let batch = 0; batch < iterations; batch++) {
+                const batchStart = Date.now();
+
+                // Simulate cache.get with 1000ms timeout (from base-cache.ts:162)
+                const promises = Array.from({ length: 50 }, (_, i) =>
+                    Promise.race([
+                        priceCache.get({ id: `timeout_test_${i}` }),
+                        new Promise((resolve) => setTimeout(() => {
+                            timeouts++;
+                            resolve({ timeout: true });
+                        }, 1000))
+                    ])
+                );
+
+                const results = await Promise.all(promises);
+                const batchDuration = Date.now() - batchStart;
+
+                if (batchDuration > 1000) {
+                    console.warn(`Batch ${batch} took ${batchDuration}ms`);
+                }
+            }
+
+            console.log(`Timeout test: ${timeouts} timeouts in ${iterations * 50} operations`);
+            expect(timeouts).toBe(0); // No timeouts expected with proper pool
+        }, 30000);
+    });
+});