lumisjs 1.0.0

package/src/cache/MultiLevelCache.js

@@ -0,0 +1,362 @@
+'use strict';
+
+const { EventEmitter } = require('node:events');
+const Logger = require('../utils/Logger');
+
+/**
+ * Multi-Level Cache (MLC)
+ * Implements a tiered caching strategy with L1 (memory), L2 (Redis), and L3 (persistent storage).
+ * Automatically promotes/demotes data between levels based on access patterns.
+ */
+
+class MultiLevelCache extends EventEmitter {
+  /**
+   * @param {object} options
+   * @param {object} options.l1 - Memory cache options
+   * @param {object} options.l2 - Redis cache options
+   * @param {object} options.l3 - Persistent storage options
+   * @param {number} [options.promoteThreshold=3] - Access count to promote to L1
+   * @param {number} [options.demoteThreshold=10] - Time without access to demote from L1 (seconds)
+   */
+  constructor(options = {}) {
+    super();
+    
+    this.options = {
+      promoteThreshold: options.promoteThreshold ?? 3,
+      demoteThreshold: options.demoteThreshold ?? 10,
+      ...options,
+    };
+
+    this.logger = new Logger({ prefix: 'MLCache', level: 'info' });
+
+    // Access tracking
+    this._accessCounts = new Map();
+    this._lastAccess = new Map();
+
+    // Initialize cache levels
+    this._initL1(options.l1);
+    this._initL2(options.l2);
+    this._initL3(options.l3);
+
+    // Start cleanup interval
+    this._cleanupInterval = setInterval(() => this._cleanup(), 60000);
+    if (this._cleanupInterval.unref) this._cleanupInterval.unref();
+  }
+
+  _initL1(options) {
+    const MemoryAdapter = require('./MemoryAdapter');
+    this.l1 = new MemoryAdapter({
+      maxSize: options?.maxSize ?? 1000,
+      ttl: options?.ttl ?? 300,
+      cleanupIntervalSec: 60,
+    });
+  }
+
+  _initL2(options) {
+    if (!options) {
+      this.l2 = null;
+      return;
+    }
+
+    try {
+      const RedisAdapter = require('./RedisAdapter');
+      this.l2 = new RedisAdapter({
+        ...options,
+        enablePubSub: true,
+      });
+      this.logger.info('L2 cache (Redis) initialized');
+    } catch (error) {
+      this.logger.warn('L2 cache initialization failed:', error.message);
+      this.l2 = null;
+    }
+  }
+
+  _initL3(options) {
+    if (!options) {
+      this.l3 = null;
+      return;
+    }
+
+    try {
+      const SQLiteAdapter = require('./SQLiteAdapter');
+      this.l3 = new SQLiteAdapter({
+        ...options,
+      });
+      this.logger.info('L3 cache (SQLite) initialized');
+    } catch (error) {
+      this.logger.warn('L3 cache initialization failed:', error.message);
+      this.l3 = null;
+    }
+  }
+
+  /**
+   * Get a value from the cache, checking all levels.
+   * @param {string} key
+   * @returns {Promise<any>}
+   */
+  async get(key) {
+    this._trackAccess(key);
+
+    // Check L1 (fastest)
+    const l1Value = await this.l1.get(key);
+    if (l1Value !== undefined) {
+      this.emit('hit', { level: 'L1', key });
+      return l1Value;
+    }
+
+    // Check L2
+    if (this.l2) {
+      const l2Value = await this.l2.get(key);
+      if (l2Value !== undefined) {
+        this.emit('hit', { level: 'L2', key });
+        // Promote to L1 if accessed frequently
+        await this._maybePromote(key, l2Value);
+        return l2Value;
+      }
+    }
+
+    // Check L3
+    if (this.l3) {
+      const l3Value = await this.l3.get(key);
+      if (l3Value !== undefined) {
+        this.emit('hit', { level: 'L3', key });
+        // Promote to L2 and L1
+        if (this.l2) await this.l2.set(key, l3Value, this.options.l2?.ttl);
+        await this._maybePromote(key, l3Value);
+        return l3Value;
+      }
+    }
+
+    this.emit('miss', { key });
+    return undefined;
+  }
+
+  /**
+   * Set a value in all cache levels.
+   * @param {string} key
+   * @param {any} value
+   * @param {number} [ttl]
+   * @returns {Promise<void>}
+   */
+  async set(key, value, ttl) {
+    this._trackAccess(key);
+
+    // Set in L1
+    await this.l1.set(key, value, ttl ?? this.options.l1?.ttl);
+
+    // Set in L2 if available
+    if (this.l2) {
+      await this.l2.set(key, value, ttl ?? this.options.l2?.ttl);
+    }
+
+    // Set in L3 if available
+    if (this.l3) {
+      await this.l3.set(key, value, ttl ?? this.options.l3?.ttl);
+    }
+
+    this.emit('set', { key, ttl });
+  }
+
+  /**
+   * Delete a value from all cache levels.
+   * @param {string} key
+   * @returns {Promise<boolean>}
+   */
+  async delete(key) {
+    this._accessCounts.delete(key);
+    this._lastAccess.delete(key);
+
+    const results = await Promise.all([
+      this.l1.delete(key),
+      this.l2 ? this.l2.delete(key) : Promise.resolve(false),
+      this.l3 ? this.l3.delete(key) : Promise.resolve(false),
+    ]);
+
+    this.emit('delete', { key, existed: results.some(r => r) });
+    return results.some(r => r);
+  }
+
+  /**
+   * Clear all cache levels.
+   * @returns {Promise<void>}
+   */
+  async clear() {
+    this._accessCounts.clear();
+    this._lastAccess.clear();
+
+    await Promise.all([
+      this.l1.clear(),
+      this.l2 ? this.l2.clear() : Promise.resolve(),
+      this.l3 ? this.l3.clear() : Promise.resolve(),
+    ]);
+
+    this.emit('clear');
+  }
+
+  /**
+   * Get cache statistics for all levels.
+   * @returns {Promise<object>}
+   */
+  async getStats() {
+    const stats = {
+      levels: {},
+      accessTracking: {
+        totalKeys: this._accessCounts.size,
+        totalAccesses: Array.from(this._accessCounts.values()).reduce((a, b) => a + b, 0),
+      },
+    };
+
+    stats.levels.L1 = {
+      size: await this.l1.size(),
+      adapter: 'Memory',
+      stats: this.l1.getStats?.() || null,
+    };
+
+    if (this.l2) {
+      stats.levels.L2 = {
+        size: await this.l2.size(),
+        adapter: 'Redis',
+      };
+    }
+
+    if (this.l3) {
+      stats.levels.L3 = {
+        size: await this.l3.size(),
+        adapter: 'SQLite',
+        stats: await this.l3.getStats?.() || null,
+      };
+    }
+
+    return stats;
+  }
+
+  /**
+   * Warm up the cache with data.
+   * @param {Map<string, any>} data
+   * @param {number} [ttl]
+   * @returns {Promise<void>}
+   */
+  async warmup(data, ttl) {
+    const entries = Array.from(data.entries()).map(([key, value]) => [key, value, ttl]);
+    
+    // Load into L3 first (persistent)
+    if (this.l3) {
+      await this.l3.setMany(entries);
+    }
+    
+    // Then L2
+    if (this.l2) {
+      await this.l2.setMany(entries);
+    }
+    
+    // Finally L1 (memory)
+    await this.l1.setMany(entries);
+
+    this.emit('warmup', { count: entries.length });
+  }
+
+  /**
+   * Invalidate keys matching a pattern across all levels.
+   * @param {string} pattern
+   * @returns {Promise<number>}
+   */
+  async invalidatePattern(pattern) {
+    let totalDeleted = 0;
+
+    // Invalidate in L1
+    const l1Keys = await this.l1.keys();
+    const regex = new RegExp('^' + pattern.replace(/\*/g, '.*').replace(/\?/g, '.') + '$');
+    const l1ToDelete = l1Keys.filter(key => regex.test(key));
+    totalDeleted += l1ToDelete.length;
+    for (const key of l1ToDelete) {
+      await this.l1.delete(key);
+    }
+
+    // Invalidate in L2
+    if (this.l2) {
+      const l2Keys = await this.l2.scan(pattern);
+      totalDeleted += l2Keys.length;
+      for (const key of l2Keys) {
+        await this.l2.delete(key);
+      }
+    }
+
+    // Invalidate in L3
+    if (this.l3) {
+      const l3Keys = await this.l3.keys();
+      const l3ToDelete = l3Keys.filter(key => regex.test(key));
+      totalDeleted += l3ToDelete.length;
+      for (const key of l3ToDelete) {
+        await this.l3.delete(key);
+      }
+    }
+
+    this.emit('invalidatePattern', { pattern, count: totalDeleted });
+    return totalDeleted;
+  }
+
+  /** @private */
+  _trackAccess(key) {
+    const count = (this._accessCounts.get(key) || 0) + 1;
+    this._accessCounts.set(key, count);
+    this._lastAccess.set(key, Date.now());
+  }
+
+  /** @private */
+  async _maybePromote(key, value) {
+    const count = this._accessCounts.get(key) || 0;
+    
+    if (count >= this.options.promoteThreshold) {
+      // Promote to L1
+      await this.l1.set(key, value, this.options.l1?.ttl);
+      this.emit('promote', { key, from: 'L2/L3', to: 'L1' });
+    }
+  }
+
+  /** @private */
+  async _cleanup() {
+    const now = Date.now();
+    const demoteThresholdMs = this.options.demoteThreshold * 1000;
+
+    for (const [key, lastAccess] of this._lastAccess) {
+      if (now - lastAccess > demoteThresholdMs) {
+        // Demote from L1
+        const value = await this.l1.peek(key);
+        if (value !== undefined) {
+          if (this.l2) {
+            await this.l2.set(key, value, this.options.l2?.ttl);
+          }
+          await this.l1.delete(key);
+          this.emit('demote', { key, from: 'L1', to: 'L2/L3' });
+        }
+        
+        this._lastAccess.delete(key);
+        this._accessCounts.delete(key);
+      }
+    }
+  }
+
+  /**
+   * Destroy all cache levels.
+   * @returns {Promise<void>}
+   */
+  async destroy() {
+    if (this._cleanupInterval) {
+      clearInterval(this._cleanupInterval);
+      this._cleanupInterval = null;
+    }
+
+    this._accessCounts.clear();
+    this._lastAccess.clear();
+
+    await Promise.all([
+      this.l1.destroy(),
+      this.l2 ? this.l2.destroy() : Promise.resolve(),
+      this.l3 ? this.l3.destroy() : Promise.resolve(),
+    ]);
+
+    this.removeAllListeners();
+  }
+}
+
+module.exports = MultiLevelCache;

package/src/cache/RedisAdapter.js

@@ -0,0 +1,329 @@
+'use strict';
+
+const { EventEmitter } = require('node:events');
+const Logger = require('../utils/Logger');
+
+class RedisAdapter extends EventEmitter {
+  
+  constructor(options = {}) {
+    super();
+    this.prefix = options.prefix || 'lumis:';
+    this.defaultTTL = options.ttl || 0;
+    this.logger = new Logger({ prefix: 'RedisCache', level: 'info' });
+    this.enablePubSub = options.enablePubSub !== false;
+    this.pubSubChannel = options.pubSubChannel || 'lumis:cache:invalidations';
+
+    try {
+      const Redis = require('ioredis');
+      this.client = new Redis(options.redis || {});
+      this.logger.info('Redis connection established.');
+
+      // Setup pub/sub for distributed cache invalidation
+      if (this.enablePubSub) {
+        this.subscriber = new Redis(options.redis || {});
+        this._setupPubSub();
+      }
+    } catch (error) {
+      throw new Error(
+        'Redis adapter requires "ioredis" package. Installation: npm install ioredis'
+      );
+    }
+  }
+
+  _key(key) {
+    return `${this.prefix}${key}`;
+  }
+
+  async get(key) {
+    const data = await this.client.get(this._key(key));
+    if (data === null) return undefined;
+    try {
+      return JSON.parse(data);
+    } catch {
+      return data;
+    }
+  }
+
+  async set(key, value, ttl) {
+    const effectiveTTL = ttl ?? this.defaultTTL;
+    const serialized = JSON.stringify(value);
+
+    if (effectiveTTL > 0) {
+      await this.client.setex(this._key(key), effectiveTTL, serialized);
+    } else {
+      await this.client.set(this._key(key), serialized);
+    }
+  }
+
+  async delete(key) {
+    const result = await this.client.del(this._key(key));
+    
+    // Publish invalidation event
+    if (this.enablePubSub && result > 0) {
+      await this._publishInvalidation(key);
+    }
+    
+    return result > 0;
+  }
+
+  async has(key) {
+    const exists = await this.client.exists(this._key(key));
+    return exists === 1;
+  }
+
+  async clear() {
+    const keys = await this.client.keys(`${this.prefix}*`);
+    if (keys.length > 0) {
+      await this.client.del(...keys);
+      
+      // Publish clear event
+      if (this.enablePubSub) {
+        await this._publishInvalidation('*');
+      }
+    }
+  }
+
+  async size() {
+    const keys = await this.client.keys(`${this.prefix}*`);
+    return keys.length;
+  }
+
+  async values() {
+    const keys = await this.client.keys(`${this.prefix}*`);
+    if (keys.length === 0) return [];
+    const values = await this.client.mget(...keys);
+    return values.map((v) => {
+      try { return JSON.parse(v); } catch { return v; }
+    }).filter((v) => v !== null);
+  }
+
+  async keys() {
+    const keys = await this.client.keys(`${this.prefix}*`);
+    return keys.map((k) => k.slice(this.prefix.length));
+  }
+
+  async filter(fn) {
+    const allKeys = await this.keys();
+    const results = [];
+    for (const key of allKeys) {
+      const value = await this.get(key);
+      if (value !== undefined && fn(value, key)) {
+        results.push(value);
+      }
+    }
+    return results;
+  }
+
+  async destroy() {
+    if (this.subscriber) {
+      await this.subscriber.quit();
+    }
+    if (this.client) {
+      await this.client.quit();
+      this.logger.info('Redis connection closed.');
+    }
+  }
+
+  // ── Pub/Sub for Distributed Cache ─────────────────────────────────────────────
+
+  _setupPubSub() {
+    this.subscriber.subscribe(this.pubSubChannel, (err) => {
+      if (err) {
+        this.logger.error('Failed to subscribe to pub/sub channel:', err);
+        return;
+      }
+      this.logger.info('Subscribed to cache invalidation channel');
+    });
+
+    this.subscriber.on('message', (channel, message) => {
+      if (channel === this.pubSubChannel) {
+        try {
+          const data = JSON.parse(message);
+          this.emit('invalidation', data);
+        } catch (err) {
+          this.logger.error('Failed to parse invalidation message:', err);
+        }
+      }
+    });
+  }
+
+  async _publishInvalidation(key) {
+    try {
+      const message = JSON.stringify({
+        key,
+        timestamp: Date.now(),
+        type: key === '*' ? 'clear' : 'delete',
+      });
+      await this.client.publish(this.pubSubChannel, message);
+    } catch (err) {
+      this.logger.error('Failed to publish invalidation:', err);
+    }
+  }
+
+  // ── Advanced Redis Features ────────────────────────────────────────────────────
+
+  /**
+   * Get multiple keys using MGET for better performance.
+   * @param {string[]} keys
+   * @returns {Promise<Map<string, any>>}
+   */
+  async getMany(keys) {
+    if (keys.length === 0) return new Map();
+    
+    const redisKeys = keys.map(k => this._key(k));
+    const values = await this.client.mget(...redisKeys);
+    
+    const result = new Map();
+    keys.forEach((key, i) => {
+      const value = values[i];
+      if (value !== null) {
+        try {
+          result.set(key, JSON.parse(value));
+        } catch {
+          result.set(key, value);
+        }
+      }
+    });
+    
+    return result;
+  }
+
+  /**
+   * Set multiple keys using MSET for better performance.
+   * @param {Array<[string, any, number?]>} entries
+   * @returns {Promise<void>}
+   */
+  async setMany(entries) {
+    if (entries.length === 0) return;
+    
+    const pipeline = this.client.pipeline();
+    
+    for (const [key, value, ttl] of entries) {
+      const serialized = JSON.stringify(value);
+      const effectiveTTL = ttl ?? this.defaultTTL;
+      
+      if (effectiveTTL > 0) {
+        pipeline.setex(this._key(key), effectiveTTL, serialized);
+      } else {
+        pipeline.set(this._key(key), serialized);
+      }
+    }
+    
+    await pipeline.exec();
+  }
+
+  /**
+   * Increment a numeric value.
+   * @param {string} key
+   * @param {number} [amount=1]
+   * @returns {Promise<number>}
+   */
+  async increment(key, amount = 1) {
+    const redisKey = this._key(key);
+    if (amount === 1) {
+      return this.client.incr(redisKey);
+    }
+    return this.client.incrby(redisKey, amount);
+  }
+
+  /**
+   * Decrement a numeric value.
+   * @param {string} key
+   * @param {number} [amount=1]
+   * @returns {Promise<number>}
+   */
+  async decrement(key, amount = 1) {
+    const redisKey = this._key(key);
+    if (amount === 1) {
+      return this.client.decr(redisKey);
+    }
+    return this.client.decrby(redisKey, amount);
+  }
+
+  /**
+   * Get TTL of a key.
+   * @param {string} key
+   * @returns {Promise<number>} TTL in seconds, -1 if no expiry, -2 if key doesn't exist
+   */
+  async getTTL(key) {
+    return this.client.ttl(this._key(key));
+  }
+
+  /**
+   * Set TTL on existing key.
+   * @param {string} key
+   * @param {number} ttl TTL in seconds
+   * @returns {Promise<boolean>}
+   */
+  async setTTL(key, ttl) {
+    const result = await this.client.expire(this._key(key), ttl);
+    return result === 1;
+  }
+
+  /**
+   * Check if key exists and set if not (SETNX).
+   * @param {string} key
+   * @param {any} value
+   * @param {number} [ttl]
+   * @returns {Promise<boolean>} True if key was set, false if it already existed
+   */
+  async setIfNotExists(key, value, ttl) {
+    const redisKey = this._key(key);
+    const serialized = JSON.stringify(value);
+    
+    const result = await this.client.setnx(redisKey, serialized);
+    
+    if (result === 1 && ttl && ttl > 0) {
+      await this.client.expire(redisKey, ttl);
+    }
+    
+    return result === 1;
+  }
+
+  /**
+   * Get and set atomically.
+   * @param {string} key
+   * @param {any} value
+   * @param {number} [ttl]
+   * @returns {Promise<any>} Old value
+   */
+  async getAndSet(key, value, ttl) {
+    const redisKey = this._key(key);
+    const serialized = JSON.stringify(value);
+    
+    const oldValue = await this.client.getset(redisKey, serialized);
+    
+    if (ttl && ttl > 0) {
+      await this.client.expire(redisKey, ttl);
+    }
+    
+    if (oldValue === null) return undefined;
+    
+    try {
+      return JSON.parse(oldValue);
+    } catch {
+      return oldValue;
+    }
+  }
+
+  /**
+   * Scan keys with pattern (more efficient than KEYS for large datasets).
+   * @param {string} [pattern='*']
+   * @param {number} [count=100]
+   * @returns {Promise<string[]>}
+   */
+  async scan(pattern = '*', count = 100) {
+    const keys = [];
+    let cursor = '0';
+    
+    do {
+      const result = await this.client.scan(cursor, 'MATCH', `${this.prefix}${pattern}`, 'COUNT', count);
+      cursor = result[0];
+      keys.push(...result[1].map(k => k.slice(this.prefix.length)));
+    } while (cursor !== '0');
+    
+    return keys;
+  }
+}
+
+module.exports = RedisAdapter;