@nina-protocol/nina-db 0.0.108 → 0.0.110

package/dist/index.js

@@ -3,6 +3,8 @@ import Knex from 'knex';
 import { Model } from 'objection';
 import Models from './models/index.js';
 import knexConfig from './knexfile.js';
+import RedisSubscriptions from './redis/subscriptions.js';
+import IndexerRedis from './redis/index.js';
 export const initDb = async (config) => {
     const db = Knex(config.development);
     await db.raw(`SELECT 'CREATE DATABASE ${process.env.POSTGRES_DATABASE}'
@@ -28,3 +30,5 @@ export const Tag = Models.Tag;
 export const Transaction = Models.Transaction;
 export const Verification = Models.Verification;
 export const config = knexConfig;
+export const redis = IndexerRedis;
+export const SubscriptionsWithCache = RedisSubscriptions;

package/dist/models/Release.js

@@ -133,7 +133,7 @@ export default class Release extends Model {
             return null;
         }
     };
-    static createRelease = async ({ publicKey, mint, metadata, datetime, publisherId, releaseAccount, programId }) => {
+    static createRelease = async ({ publicKey, mint, metadata, datetime, publisherId, releaseAccount, programId, solanaAddress }) => {
         const slug = await this.generateSlug(metadata);
         const price = releaseAccount.account?.price?.toNumber() ||
             releaseAccount?.price?.toNumber() ||
@@ -151,6 +151,7 @@ export default class Release extends Model {
             paymentMint,
             archived: false,
             programId,
+            solanaAddress,
         });
         if (metadata.properties.tags) {
             for await (let tag of metadata.properties.tags) {

package/dist/redis/index.js

@@ -0,0 +1,340 @@
+import Redis from 'ioredis';
+import dotenv from 'dotenv';
+dotenv.config();
+const REDIS_URL = process.env.REDIS_URL || 'redis://localhost:6379';
+const CACHE_TTL = 7200; // 2 hours in seconds
+const SEARCH_CACHE_TTL = 300; // 5 minutes in seconds for search queries
+const POOL_SIZE = 10;
+const CONNECTION_TIMEOUT = 5000; // 5 seconds
+const OPERATION_TIMEOUT = 10000; // 10 seconds
+// Track pool health
+let failedConnections = 0;
+let lastErrorTime = null;
+const MAX_FAILURES = 5;
+const FAILURE_WINDOW = 60000; // 1 minute
+// Alert thresholds
+const ALERT_THRESHOLDS = {
+    MIN_ACTIVE_CONNECTIONS: 5, // Alert if less than 5 active connections
+    MAX_FAILED_CONNECTIONS: 3, // Alert if more than 3 failed connections
+    ERROR_WINDOW: 5 * 60 * 1000 // 5 minutes
+};
+// Get a client from the pool using round-robin with timeout
+let currentClientIndex = 0;
+let totalRequests = 0;
+// Create Redis connection pool
+const createRedisPool = (size = POOL_SIZE) => {
+    const pool = [];
+    let activeConnections = 0;
+    for (let i = 0; i < size; i++) {
+        const client = new Redis(REDIS_URL, {
+            retryStrategy: (times) => {
+                const delay = Math.min(times * 1000, 5000);
+                return delay;
+            },
+            maxRetriesPerRequest: 3,
+            connectTimeout: CONNECTION_TIMEOUT,
+            commandTimeout: OPERATION_TIMEOUT,
+            enableOfflineQueue: true,
+            enableReadyCheck: true,
+            reconnectOnError: (err) => {
+                console.error('[Redis] Reconnect on error:', err);
+                return true;
+            },
+            lazyConnect: true,
+            keepAlive: 10000
+        });
+        client.on('error', (error) => {
+            console.error(`[Redis Pool Client ${i}] Error:`, error);
+            failedConnections++;
+            // Check if we're having too many failures
+            const now = Date.now();
+            if (lastErrorTime && (now - lastErrorTime) < FAILURE_WINDOW) {
+                if (failedConnections >= MAX_FAILURES) {
+                    console.error('[Redis] Too many failures in short time, removing client from pool');
+                    const index = pool.indexOf(client);
+                    if (index > -1) {
+                        pool.splice(index, 1);
+                        client.disconnect();
+                    }
+                }
+            }
+            else {
+                // Reset failure count if outside window
+                failedConnections = 1;
+                lastErrorTime = now;
+            }
+        });
+        client.on('connect', () => {
+            activeConnections++;
+            console.log(`[Redis] Client ${i} connected. Active connections: ${activeConnections}`);
+        });
+        client.on('close', () => {
+            activeConnections--;
+            console.log(`[Redis] Client ${i} disconnected. Active connections: ${activeConnections}`);
+        });
+        client.on('timeout', () => {
+            console.error(`[Redis] Client ${i} timed out`);
+            client.disconnect();
+        });
+        pool.push(client);
+    }
+    return pool;
+};
+// Create the pool
+const redisPool = createRedisPool();
+let isPoolInitialized = false;
+const getClient = () => {
+    if (!isPoolInitialized) {
+        throw new Error('Redis pool not initialized');
+    }
+    if (redisPool.length === 0) {
+        throw new Error('No available Redis connections in pool');
+    }
+    const client = redisPool[currentClientIndex];
+    currentClientIndex = (currentClientIndex + 1) % redisPool.length;
+    totalRequests++;
+    // Log every 1000 requests
+    if (totalRequests % 1000 === 0) {
+        console.log(`[Redis] Total requests processed: ${totalRequests}`);
+    }
+    return client;
+};
+const testRedisConnection = async (client) => {
+    try {
+        await client.set('test:connection', 'ok', 'EX', 10);
+        const result = await client.get('test:connection');
+        return result === 'ok';
+    }
+    catch (error) {
+        console.error('[Redis] Connection test failed:', error);
+        return false;
+    }
+};
+// Initialize all clients
+const initializePool = async () => {
+    try {
+        console.log('[Redis] Initializing pool...');
+        const results = await Promise.all(redisPool.map(client => testRedisConnection(client)));
+        const successCount = results.filter(Boolean).length;
+        if (successCount === 0) {
+            throw new Error('Failed to initialize any Redis connections');
+        }
+        if (successCount < redisPool.length) {
+            console.warn(`[Redis] Only ${successCount}/${redisPool.length} connections initialized successfully`);
+        }
+        isPoolInitialized = true;
+        console.log('[Redis] Pool initialization completed');
+    }
+    catch (error) {
+        console.error('[Redis] Pool initialization failed:', error);
+        throw error;
+    }
+};
+// Health check function
+export const checkPoolHealth = () => {
+    const health = {
+        totalConnections: redisPool.length,
+        activeConnections: redisPool.filter(client => client.status === 'ready').length,
+        failedConnections,
+        lastErrorTime,
+        timestamp: new Date().toISOString()
+    };
+    // Check for problems and log
+    if (health.activeConnections < ALERT_THRESHOLDS.MIN_ACTIVE_CONNECTIONS) {
+        console.error(`[Redis CRITICAL] Low active connections: ${health.activeConnections}/${POOL_SIZE}. Pool may be exhausted.`);
+    }
+    if (health.failedConnections > ALERT_THRESHOLDS.MAX_FAILED_CONNECTIONS) {
+        console.error(`[Redis CRITICAL] High number of failed connections: ${health.failedConnections}. Redis may be having issues.`);
+    }
+    if (health.lastErrorTime && (Date.now() - health.lastErrorTime) < ALERT_THRESHOLDS.ERROR_WINDOW) {
+        console.error(`[Redis WARNING] Recent Redis errors detected. Last error: ${new Date(health.lastErrorTime).toISOString()}`);
+    }
+    // Log health status
+    console.log('[Redis] Pool Health:', {
+        ...health,
+        lastErrorTime: health.lastErrorTime ? new Date(health.lastErrorTime).toISOString() : null
+    });
+    return health;
+};
+// Clear cache for a specific key
+export const clearCache = async (key) => {
+    const client = getClient();
+    try {
+        await client.del(key);
+    }
+    catch (error) {
+        console.error('[Redis] Clear cache error:', error);
+    }
+};
+// Clear cache by pattern
+export const clearCacheByPattern = async (pattern) => {
+    const client = getClient();
+    try {
+        const keys = await client.keys(pattern);
+        if (keys.length > 0) {
+            const pipeline = client.pipeline();
+            keys.forEach(key => pipeline.del(key));
+            await pipeline.exec();
+        }
+    }
+    catch (error) {
+        console.error('[Redis] Clear cache by pattern error:', error);
+    }
+};
+// Cleanup function to properly close all connections
+export const cleanupPool = async () => {
+    console.log('[Redis] Starting pool cleanup...');
+    isPoolInitialized = false;
+    try {
+        const closePromises = redisPool.map(async (client, index) => {
+            try {
+                await client.quit();
+                console.log(`[Redis] Client ${index} closed successfully`);
+            }
+            catch (error) {
+                console.error(`[Redis] Error closing client ${index}:`, error);
+                // Force close if quit fails
+                try {
+                    await client.disconnect();
+                }
+                catch (disconnectError) {
+                    console.error(`[Redis] Error force disconnecting client ${index}:`, disconnectError);
+                }
+            }
+        });
+        await Promise.all(closePromises);
+        console.log('[Redis] Pool cleanup completed');
+    }
+    catch (error) {
+        console.error('[Redis] Error during pool cleanup:', error);
+        throw error;
+    }
+};
+// Cache wrapper function
+export const withCache = async (key, fn, ttl = CACHE_TTL, override = false) => {
+    console.log('withCache', key, override);
+    const client = getClient();
+    try {
+        // Try to get from cache first
+        const cachedResult = await client.get(key);
+        if (cachedResult && !override) {
+            console.log('found cached result');
+            try {
+                const parsed = JSON.parse(cachedResult);
+                if (Array.isArray(parsed)) {
+                    return parsed.map(id => {
+                        if (typeof id === 'object' && id !== null) {
+                            return id.id;
+                        }
+                        return typeof id === 'string' ? parseInt(id, 10) : id;
+                    }).filter(id => !isNaN(id));
+                }
+                return parsed;
+            }
+            catch (parseError) {
+                await client.del(key);
+            }
+        }
+        const result = await fn();
+        if (result != null) {
+            console.log('no result - adding to cache for: ', key);
+            try {
+                const toCache = Array.isArray(result)
+                    ? result.map(id => {
+                        if (typeof id === 'object' && id !== null) {
+                            return id.id;
+                        }
+                        return typeof id === 'string' ? parseInt(id, 10) : id;
+                    }).filter(id => !isNaN(id))
+                    : result;
+                await client.setex(key, ttl, JSON.stringify(toCache));
+            }
+            catch (cacheError) {
+                console.error('[Redis] Cache error:', cacheError);
+            }
+        }
+        return result;
+    }
+    catch (error) {
+        try {
+            return await fn();
+        }
+        catch (fnError) {
+            throw fnError;
+        }
+    }
+};
+const startRedis = async () => {
+    // Initialize the pool
+    initializePool().then(() => {
+        // Run health check after pool is initialized
+        checkPoolHealth();
+    }).catch(error => {
+        console.error('[Redis] Failed to initialize pool:', error);
+        process.exit(1);
+    });
+    // Handle process termination
+    process.on('SIGTERM', async () => {
+        console.log('[Redis] Received SIGTERM signal');
+        await cleanupPool();
+        process.exit(0);
+    });
+    process.on('SIGINT', async () => {
+        console.log('[Redis] Received SIGINT signal');
+        await cleanupPool();
+        process.exit(0);
+    });
+    // Handle PM2 restarts and crashes
+    process.on('uncaughtException', async (error) => {
+        console.error('[Redis] Uncaught Exception:', error);
+        await cleanupPool();
+        process.exit(1);
+    });
+    process.on('unhandledRejection', async (reason, promise) => {
+        console.error('[Redis] Unhandled Rejection at:', promise, 'reason:', reason);
+        await cleanupPool();
+        process.exit(1);
+    });
+    // Handle PM2 graceful shutdown
+    if (process.env.NODE_ENV === 'production') {
+        process.on('message', async (msg) => {
+            if (msg === 'shutdown') {
+                console.log('[Redis] Received PM2 shutdown message');
+                await cleanupPool();
+                process.exit(0);
+            }
+        });
+    }
+};
+async function deleteCacheMatchingPattern(pattern, { count = 1000, useUnlink = true } = {}) {
+    const client = getClient(); // <-- get a pooled client
+    let cursor = '0';
+    do {
+        // ioredis scan signature: scan(cursor, ...args) -> [nextCursor, keys[]]
+        const [nextCursor, keys] = await client.scan(cursor, 'MATCH', pattern, 'COUNT', count);
+        if (keys.length) {
+            console.log('deleteCacheMatchingPattern: deleting keys:', keys);
+            const pipeline = client.pipeline();
+            // Prefer UNLINK for async deletion (doesn't block Redis); fall back to DEL
+            if (useUnlink && typeof client.unlink === 'function') {
+                keys.forEach(k => pipeline.unlink(k));
+            }
+            else {
+                keys.forEach(k => pipeline.del(k));
+            }
+            await pipeline.exec();
+        }
+        cursor = nextCursor;
+    } while (cursor !== '0');
+}
+export default {
+    getClient,
+    testRedisConnection,
+    checkPoolHealth,
+    clearCache,
+    clearCacheByPattern,
+    cleanupPool,
+    withCache,
+    startRedis,
+    deleteCacheMatchingPattern,
+};

package/dist/redis/subscriptions.js

@@ -0,0 +1,110 @@
+import redis from './index.js';
+import Subscription from '../models/Subscription.js';
+import Account from '../models/Account.js';
+import { formatColumnForJsonFields, BIG_LIMIT } from '../utils/index.js';
+const SUBSCRIPTION_TO = 'subscription:to';
+const getFollowersForAccountWithCache = async (publicKeyOrHandle, query, override = false) => {
+    try {
+        let { offset = 0, limit = BIG_LIMIT, sort = 'desc', column = 'datetime' } = query;
+        return redis.withCache(`${SUBSCRIPTION_TO}:${publicKeyOrHandle}:${offset}:${limit}:${sort}:${column}`, async () => {
+            let account = await Account.query().findOne({ publicKey: publicKeyOrHandle });
+            if (!account) {
+                account = await Account.query().findOne({ handle: publicKeyOrHandle });
+                if (!account) {
+                    throw new Error(`Account not found for publicKeyOrHandle ${publicKeyOrHandle}`);
+                }
+            }
+            const publicKey = account.publicKey;
+            column = formatColumnForJsonFields(column);
+            const subscriptions = await Subscription.query()
+                .where('to', publicKey)
+                .orderBy(column, sort)
+                .range(Number(offset), Number(offset) + Number(limit) - 1);
+            const followers = [];
+            for await (let subscription of subscriptions.results) {
+                if (subscription.subscriptionType === 'account') {
+                    const account = await Account.findOrCreate(subscription.from);
+                    await account.format();
+                    delete subscription.id;
+                    followers.push({
+                        account,
+                        subscription,
+                    });
+                }
+            }
+            return {
+                followers,
+                total: subscriptions.total
+            };
+        }, undefined, override);
+    }
+    catch (error) {
+        console.log('getFollowersForAccountWithCache error', error);
+    }
+};
+const getUserFollowingAccountWithCache = async (publicKeyOrHandle, followingPublicKeyOrHandle, override = false) => {
+    try {
+        return redis.withCache(`following:${publicKeyOrHandle}:${followingPublicKeyOrHandle}`, async () => {
+            let account = await Account.query().findOne({ publicKey: publicKeyOrHandle });
+            if (!account) {
+                account = await Account.query().findOne({ handle: publicKeyOrHandle });
+                if (!account) {
+                    ctx.status = 404;
+                    ctx.body = {
+                        success: false,
+                        following: false,
+                        message: `Account not found with publicKey: ${publicKeyOrHandle}`
+                    };
+                    return;
+                }
+            }
+            let followingAccount = await Account.query().findOne({ publicKey: followingPublicKeyOrHandle });
+            if (!followingAccount) {
+                followingAccount = await Account.query().findOne({ handle: followingPublicKeyOrHandle });
+                if (!followingAccount) {
+                    followingAccount = await Hub.query().findOne({ publicKey: followingPublicKeyOrHandle });
+                    if (!followingAccount) {
+                        followingAccount = await Hub.query().findOne({ handle: followingPublicKeyOrHandle });
+                    }
+                    if (!followingAccount) {
+                        ctx.status = 404;
+                        ctx.body = {
+                            success: false,
+                            following: false,
+                            message: `Account not found with publicKey: ${followingPublicKeyOrHandle}`
+                        };
+                        return;
+                    }
+                }
+            }
+            const publicKey = account.publicKey;
+            const followingPublicKey = followingAccount.publicKey;
+            const subscriptions = await Subscription.query()
+                .where('from', publicKey)
+                .andWhere('to', followingPublicKey);
+            return subscriptions.length > 0;
+        }, undefined, override);
+    }
+    catch (error) {
+        console.log('getUserFollowingAccountWithCache error', error);
+    }
+};
+const deleteCacheAfterAccountFollow = async (toPublicKey, toHandle, fromPublicKey, fromHandle) => {
+    try {
+        console.log('deleteCacheAfterAccountFollow', toHandle, toPublicKey);
+        await redis.deleteCacheMatchingPattern(`${SUBSCRIPTION_TO}:${toPublicKey}*`);
+        await redis.deleteCacheMatchingPattern(`${SUBSCRIPTION_TO}:${toHandle}*`);
+        await redis.deleteCacheMatchingPattern(`following:${fromPublicKey}:${toPublicKey}`);
+        await redis.deleteCacheMatchingPattern(`following:${fromPublicKey}:${toHandle}`);
+        await redis.deleteCacheMatchingPattern(`following:${fromHandle}:${toPublicKey}`);
+        await redis.deleteCacheMatchingPattern(`following:${fromHandle}:${toHandle}`);
+    }
+    catch (error) {
+        console.log('deleteCacheAfterAccountFollow error: ', error);
+    }
+};
+export default {
+    getFollowersForAccountWithCache,
+    getUserFollowingAccountWithCache,
+    deleteCacheAfterAccountFollow,
+};

package/dist/utils/index.js

@@ -43,3 +43,11 @@ export const tweetNewRelease = async (metadata, publisherId, slug) => {
         }
     }
 };
+export const BIG_LIMIT = 5000;
+export const formatColumnForJsonFields = (column, fieldName = 'metadata') => {
+    if (column.includes(':')) {
+        column = fieldName + ':' + column.split(':')[1];
+        column = ref(column).castText();
+    }
+    return column;
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nina-protocol/nina-db",
-  "version": "0.0.108",
+  "version": "0.0.110",
   "description": "",
   "source": "src/index.js",
   "main": "dist/index.js",
@@ -22,10 +22,12 @@
   "dependencies": {
     "@babel/plugin-proposal-object-rest-spread": "^7.20.7",
     "@babel/preset-es2015": "^7.0.0-beta.53",
-    "@metaplex-foundation/js": "^0.18.1",
     "@coral-xyz/anchor": "^0.31.0",
+    "@metaplex-foundation/js": "^0.18.1",
+    "@project-serum/anchor": "^0.26.0",
     "@solana/spl-token": "^0.4.9",
     "axios": "^0.27.2",
+    "ioredis": "^5.7.0",
     "knex": "^2.2.0",
     "nanoid": "^4.0.2",
     "objection": "2.2.15",

package/src/index.js

@@ -3,6 +3,8 @@ import Knex from 'knex'
 import { Model } from 'objection'
 import Models from './models/index.js'
 import knexConfig from './knexfile.js'
+import RedisSubscriptions from './redis/subscriptions.js'
+import IndexerRedis from './redis/index.js'
 
 export const initDb = async (config) => {
   const db = Knex(config.development)
@@ -34,4 +36,6 @@ export const Subscription = Models.Subscription
 export const Tag = Models.Tag
 export const Transaction = Models.Transaction
 export const Verification = Models.Verification
-export const config = knexConfig
+export const config = knexConfig
+export const redis = IndexerRedis
+export const SubscriptionsWithCache = RedisSubscriptions

package/src/models/Release.js

@@ -161,7 +161,7 @@ export default class Release extends Model {
     }
   };
 
-  static createRelease = async ({publicKey, mint, metadata, datetime, publisherId, releaseAccount, programId}) => {
+  static createRelease = async ({publicKey, mint, metadata, datetime, publisherId, releaseAccount, programId, solanaAddress}) => {
     const slug = await this.generateSlug(metadata);
     const price =
       releaseAccount.account?.price?.toNumber() ||
@@ -181,6 +181,7 @@ export default class Release extends Model {
       paymentMint,
       archived: false,
       programId,
+      solanaAddress,
     })
     if (metadata.properties.tags) {
       for await (let tag of metadata.properties.tags) {

package/src/redis/index.js

@@ -0,0 +1,382 @@
+import Redis from 'ioredis';
+import dotenv from 'dotenv';
+
+dotenv.config();
+
+const REDIS_URL = process.env.REDIS_URL || 'redis://localhost:6379';
+const CACHE_TTL = 7200; // 2 hours in seconds
+const SEARCH_CACHE_TTL = 300; // 5 minutes in seconds for search queries
+const POOL_SIZE = 10;
+const CONNECTION_TIMEOUT = 5000; // 5 seconds
+const OPERATION_TIMEOUT = 10000; // 10 seconds
+
+// Track pool health
+let failedConnections = 0;
+let lastErrorTime = null;
+const MAX_FAILURES = 5;
+const FAILURE_WINDOW = 60000; // 1 minute
+
+// Alert thresholds
+const ALERT_THRESHOLDS = {
+  MIN_ACTIVE_CONNECTIONS: 5,  // Alert if less than 5 active connections
+  MAX_FAILED_CONNECTIONS: 3,  // Alert if more than 3 failed connections
+  ERROR_WINDOW: 5 * 60 * 1000 // 5 minutes
+};
+
+// Get a client from the pool using round-robin with timeout
+let currentClientIndex = 0;
+let totalRequests = 0;
+
+// Create Redis connection pool
+const createRedisPool = (size = POOL_SIZE) => {
+  const pool = [];
+  let activeConnections = 0;
+  
+  for (let i = 0; i < size; i++) {
+    const client = new Redis(REDIS_URL, {
+      retryStrategy: (times) => {
+        const delay = Math.min(times * 1000, 5000);
+        return delay;
+      },
+      maxRetriesPerRequest: 3,
+      connectTimeout: CONNECTION_TIMEOUT,
+      commandTimeout: OPERATION_TIMEOUT,
+      enableOfflineQueue: true,
+      enableReadyCheck: true,
+      reconnectOnError: (err) => {
+        console.error('[Redis] Reconnect on error:', err);
+        return true;
+      },
+      lazyConnect: true,
+      keepAlive: 10000
+    });
+
+    client.on('error', (error) => {
+      console.error(`[Redis Pool Client ${i}] Error:`, error);
+      failedConnections++;
+      
+      // Check if we're having too many failures
+      const now = Date.now();
+      if (lastErrorTime && (now - lastErrorTime) < FAILURE_WINDOW) {
+        if (failedConnections >= MAX_FAILURES) {
+          console.error('[Redis] Too many failures in short time, removing client from pool');
+          const index = pool.indexOf(client);
+          if (index > -1) {
+            pool.splice(index, 1);
+            client.disconnect();
+          }
+        }
+      } else {
+        // Reset failure count if outside window
+        failedConnections = 1;
+        lastErrorTime = now;
+      }
+    });
+
+    client.on('connect', () => {
+      activeConnections++;
+      console.log(`[Redis] Client ${i} connected. Active connections: ${activeConnections}`);
+    });
+
+    client.on('close', () => {
+      activeConnections--;
+      console.log(`[Redis] Client ${i} disconnected. Active connections: ${activeConnections}`);
+    });
+
+    client.on('timeout', () => {
+      console.error(`[Redis] Client ${i} timed out`);
+      client.disconnect();
+    });
+
+    pool.push(client);
+  }
+  return pool;
+};
+
+// Create the pool
+const redisPool = createRedisPool();
+let isPoolInitialized = false;
+
+const getClient = () => {
+  if (!isPoolInitialized) {
+    throw new Error('Redis pool not initialized');
+  }
+
+  if (redisPool.length === 0) {
+    throw new Error('No available Redis connections in pool');
+  }
+
+  const client = redisPool[currentClientIndex];
+  currentClientIndex = (currentClientIndex + 1) % redisPool.length;
+  totalRequests++;
+  
+  // Log every 1000 requests
+  if (totalRequests % 1000 === 0) {
+    console.log(`[Redis] Total requests processed: ${totalRequests}`);
+  }
+  
+  return client;
+};
+
+const testRedisConnection = async (client) => {
+  try {
+    await client.set('test:connection', 'ok', 'EX', 10);
+    const result = await client.get('test:connection');
+    return result === 'ok';
+  } catch (error) {
+    console.error('[Redis] Connection test failed:', error);
+    return false;
+  }
+};
+
+// Initialize all clients
+const initializePool = async () => {
+  try {
+    console.log('[Redis] Initializing pool...');
+    const results = await Promise.all(redisPool.map(client => testRedisConnection(client)));
+    const successCount = results.filter(Boolean).length;
+    
+    if (successCount === 0) {
+      throw new Error('Failed to initialize any Redis connections');
+    }
+    
+    if (successCount < redisPool.length) {
+      console.warn(`[Redis] Only ${successCount}/${redisPool.length} connections initialized successfully`);
+    }
+    
+    isPoolInitialized = true;
+    console.log('[Redis] Pool initialization completed');
+  } catch (error) {
+    console.error('[Redis] Pool initialization failed:', error);
+    throw error;
+  }
+};
+
+
+// Health check function
+export const checkPoolHealth = () => {
+  const health = {
+    totalConnections: redisPool.length,
+    activeConnections: redisPool.filter(client => client.status === 'ready').length,
+    failedConnections,
+    lastErrorTime,
+    timestamp: new Date().toISOString()
+  };
+
+  // Check for problems and log
+  if (health.activeConnections < ALERT_THRESHOLDS.MIN_ACTIVE_CONNECTIONS) {
+    console.error(`[Redis CRITICAL] Low active connections: ${health.activeConnections}/${POOL_SIZE}. Pool may be exhausted.`);
+  }
+
+  if (health.failedConnections > ALERT_THRESHOLDS.MAX_FAILED_CONNECTIONS) {
+    console.error(`[Redis CRITICAL] High number of failed connections: ${health.failedConnections}. Redis may be having issues.`);
+  }
+
+  if (health.lastErrorTime && (Date.now() - health.lastErrorTime) < ALERT_THRESHOLDS.ERROR_WINDOW) {
+    console.error(`[Redis WARNING] Recent Redis errors detected. Last error: ${new Date(health.lastErrorTime).toISOString()}`);
+  }
+
+  // Log health status
+  console.log('[Redis] Pool Health:', {
+    ...health,
+    lastErrorTime: health.lastErrorTime ? new Date(health.lastErrorTime).toISOString() : null
+  });
+
+  return health;
+};
+
+
+// Clear cache for a specific key
+export const clearCache = async (key) => {
+  const client = getClient();
+  try {
+    await client.del(key);
+  } catch (error) {
+    console.error('[Redis] Clear cache error:', error);
+  }
+};
+
+// Clear cache by pattern
+export const clearCacheByPattern = async (pattern) => {
+  const client = getClient();
+  try {
+    const keys = await client.keys(pattern);
+    if (keys.length > 0) {
+      const pipeline = client.pipeline();
+      keys.forEach(key => pipeline.del(key));
+      await pipeline.exec();
+    }
+  } catch (error) {
+    console.error('[Redis] Clear cache by pattern error:', error);
+  }
+};
+
+// Cleanup function to properly close all connections
+export const cleanupPool = async () => {
+  console.log('[Redis] Starting pool cleanup...');
+  isPoolInitialized = false;
+  
+  try {
+    const closePromises = redisPool.map(async (client, index) => {
+      try {
+        await client.quit();
+        console.log(`[Redis] Client ${index} closed successfully`);
+      } catch (error) {
+        console.error(`[Redis] Error closing client ${index}:`, error);
+        // Force close if quit fails
+        try {
+          await client.disconnect();
+        } catch (disconnectError) {
+          console.error(`[Redis] Error force disconnecting client ${index}:`, disconnectError);
+        }
+      }
+    });
+
+    await Promise.all(closePromises);
+    console.log('[Redis] Pool cleanup completed');
+  } catch (error) {
+    console.error('[Redis] Error during pool cleanup:', error);
+    throw error;
+  }
+};
+
+// Cache wrapper function
+export const withCache = async (key, fn, ttl = CACHE_TTL, override = false) => {
+  console.log('withCache', key, override)
+  const client = getClient();
+  try {
+    // Try to get from cache first
+    const cachedResult = await client.get(key);
+    
+    if (cachedResult && !override) {
+      console.log('found cached result')
+      try {
+        const parsed = JSON.parse(cachedResult);
+        if (Array.isArray(parsed)) {
+          return parsed.map(id => {
+            if (typeof id === 'object' && id !== null) {
+              return id.id;
+            }
+            return typeof id === 'string' ? parseInt(id, 10) : id;
+          }).filter(id => !isNaN(id));
+        }
+        return parsed;
+      } catch (parseError) {
+        await client.del(key);
+      }
+    }
+
+    const result = await fn();
+    
+    if (result != null) {
+      console.log('no result - adding to cache for: ', key)
+      try {
+        const toCache = Array.isArray(result) 
+          ? result.map(id => {
+              if (typeof id === 'object' && id !== null) {
+                return id.id;
+              }
+              return typeof id === 'string' ? parseInt(id, 10) : id;
+            }).filter(id => !isNaN(id))
+          : result;
+
+        await client.setex(key, ttl, JSON.stringify(toCache));
+      } catch (cacheError) {
+        console.error('[Redis] Cache error:', cacheError);
+      }
+    }
+    
+    return result;
+  } catch (error) {
+    try {
+      return await fn();
+    } catch (fnError) {
+      throw fnError;
+    }
+  }
+};
+
+const startRedis = async() => {
+  // Initialize the pool
+  initializePool().then(() => {
+    // Run health check after pool is initialized
+    checkPoolHealth();
+  }).catch(error => {
+    console.error('[Redis] Failed to initialize pool:', error);
+    process.exit(1);
+  });
+
+  // Handle process termination
+  process.on('SIGTERM', async () => {
+    console.log('[Redis] Received SIGTERM signal');
+    await cleanupPool();
+    process.exit(0);
+  });
+
+  process.on('SIGINT', async () => {
+    console.log('[Redis] Received SIGINT signal');
+    await cleanupPool();
+    process.exit(0);
+  });
+
+  // Handle PM2 restarts and crashes
+  process.on('uncaughtException', async (error) => {
+    console.error('[Redis] Uncaught Exception:', error);
+    await cleanupPool();
+    process.exit(1);
+  });
+
+  process.on('unhandledRejection', async (reason, promise) => {
+    console.error('[Redis] Unhandled Rejection at:', promise, 'reason:', reason);
+    await cleanupPool();
+    process.exit(1);
+  });
+
+  // Handle PM2 graceful shutdown
+  if (process.env.NODE_ENV === 'production') {
+    process.on('message', async (msg) => {
+      if (msg === 'shutdown') {
+        console.log('[Redis] Received PM2 shutdown message');
+        await cleanupPool();
+        process.exit(0);
+      }
+    });
+  }
+}
+
+async function deleteCacheMatchingPattern(pattern, { count = 1000, useUnlink = true } = {}) {
+  const client = getClient(); // <-- get a pooled client
+  let cursor = '0';
+
+  do {
+    // ioredis scan signature: scan(cursor, ...args) -> [nextCursor, keys[]]
+    const [nextCursor, keys] = await client.scan(cursor, 'MATCH', pattern, 'COUNT', count);
+
+    if (keys.length) {
+      console.log('deleteCacheMatchingPattern: deleting keys:', keys)
+      const pipeline = client.pipeline();
+      // Prefer UNLINK for async deletion (doesn't block Redis); fall back to DEL
+      if (useUnlink && typeof client.unlink === 'function') {
+        keys.forEach(k => pipeline.unlink(k));
+      } else {
+        keys.forEach(k => pipeline.del(k));
+      }
+      await pipeline.exec();
+    }
+
+    cursor = nextCursor;
+  } while (cursor !== '0');
+}
+
+
+export default {
+  getClient,
+  testRedisConnection,
+  checkPoolHealth,
+  clearCache,
+  clearCacheByPattern,
+  cleanupPool,
+  withCache,
+  startRedis,
+  deleteCacheMatchingPattern,
+};

package/src/redis/subscriptions.js

@@ -0,0 +1,132 @@
+import redis from './index.js';
+import Subscription from '../models/Subscription.js';
+import Account from '../models/Account.js';
+import { formatColumnForJsonFields, BIG_LIMIT } from '../utils/index.js';
+
+const SUBSCRIPTION_TO = 'subscription:to'
+const getFollowersForAccountWithCache = async (publicKeyOrHandle, query, override=false) => {
+  try {
+    let { offset=0, limit=BIG_LIMIT, sort='desc', column='datetime' } = query;
+    return redis.withCache(`${SUBSCRIPTION_TO}:${publicKeyOrHandle}:${offset}:${limit}:${sort}:${column}`, async () => {
+      let account = await Account.query().findOne({publicKey: publicKeyOrHandle});
+      if (!account) {
+        account = await Account.query().findOne({handle: publicKeyOrHandle});
+        if (!account) {
+          throw new Error(`Account not found for publicKeyOrHandle ${publicKeyOrHandle}`)
+        }
+      }
+        const publicKey = account.publicKey
+        column = formatColumnForJsonFields(column);
+        const subscriptions = await Subscription.query()
+          .where('to', publicKey)
+          .orderBy(column, sort)
+          .range(Number(offset), Number(offset) + Number(limit) - 1);
+        
+        const followers = []
+        for await (let subscription of subscriptions.results) {
+          if (subscription.subscriptionType === 'account') {
+            const account = await Account.findOrCreate(subscription.from);
+            await account.format();
+            delete subscription.id
+    
+            followers.push({
+              account,
+              subscription,
+            })
+          }
+        }
+        return {
+          followers,
+          total: subscriptions.total
+        }
+      }, 
+      undefined,
+      override
+    );  
+  } catch (error) {
+    console.log('getFollowersForAccountWithCache error', error)
+  }
+};
+
+const getUserFollowingAccountWithCache = async (
+  publicKeyOrHandle,
+  followingPublicKeyOrHandle,
+  override = false,
+) => {
+  try {
+    return redis.withCache(
+      `following:${publicKeyOrHandle}:${followingPublicKeyOrHandle}`, async () => {
+        let account = await Account.query().findOne({publicKey: publicKeyOrHandle});
+        if (!account) {
+          account = await Account.query().findOne({handle: publicKeyOrHandle});
+          if (!account) {
+            ctx.status = 404
+            ctx.body = {
+              success: false,
+              following:false,
+              message: `Account not found with publicKey: ${publicKeyOrHandle}`
+            }
+            return;
+          }
+        }
+        let followingAccount = await Account.query().findOne({publicKey: followingPublicKeyOrHandle});
+        if (!followingAccount) {
+          followingAccount = await Account.query().findOne({handle: followingPublicKeyOrHandle});
+          if (!followingAccount) {
+            followingAccount = await Hub.query().findOne({publicKey: followingPublicKeyOrHandle});
+            if (!followingAccount) {
+              followingAccount = await Hub.query().findOne({handle: followingPublicKeyOrHandle});
+            }
+            if (!followingAccount) {
+              ctx.status = 404
+              ctx.body = {
+                success: false,
+                following:false,
+                message: `Account not found with publicKey: ${followingPublicKeyOrHandle}`
+              }
+              return;
+            }
+          }
+        }
+        const publicKey = account.publicKey
+        const followingPublicKey = followingAccount.publicKey
+        const subscriptions = await Subscription.query()
+          .where('from', publicKey)
+          .andWhere('to', followingPublicKey)
+
+        return subscriptions.length > 0
+    
+      },
+      undefined,
+      override
+    )
+  } catch (error) {
+    console.log('getUserFollowingAccountWithCache error', error)
+  }
+}
+
+const deleteCacheAfterAccountFollow = async(
+  toPublicKey,
+  toHandle,
+  fromPublicKey,
+  fromHandle,
+) => {
+  try {
+    console.log('deleteCacheAfterAccountFollow', toHandle, toPublicKey)
+    await redis.deleteCacheMatchingPattern(`${SUBSCRIPTION_TO}:${toPublicKey}*`)
+    await redis.deleteCacheMatchingPattern(`${SUBSCRIPTION_TO}:${toHandle}*`)
+    await redis.deleteCacheMatchingPattern(`following:${fromPublicKey}:${toPublicKey}`)
+    await redis.deleteCacheMatchingPattern(`following:${fromPublicKey}:${toHandle}`)
+    await redis.deleteCacheMatchingPattern(`following:${fromHandle}:${toPublicKey}`)
+    await redis.deleteCacheMatchingPattern(`following:${fromHandle}:${toHandle}`)
+    
+  } catch (error) {
+    console.log('deleteCacheAfterAccountFollow error: ', error)
+  }
+}
+
+export default {
+  getFollowersForAccountWithCache,
+  getUserFollowingAccountWithCache,
+  deleteCacheAfterAccountFollow,
+}

package/src/utils/index.js

@@ -45,4 +45,14 @@ export const tweetNewRelease = async (metadata, publisherId, slug) => {
       console.warn('error sending new release tweet: ', error, metadata)
     }
   }
+}
+
+export const BIG_LIMIT = 5000;
+
+export const formatColumnForJsonFields = (column, fieldName='metadata') => {
+  if (column.includes(':')) {
+    column = fieldName + ':' + column.split(':')[1]
+    column = ref(column).castText()
+  }
+  return column
 }

package/tsconfig.json

@@ -15,7 +15,8 @@
     "target": "esnext",
     "rootDir": "./src",
     "outDir": "dist",
-    "skipLibCheck": true  
+    "skipLibCheck": true,
+    "types": ["node"]
   },
   "include": ["src/**/*"],
   "exclude": ["node_modules"]