claude-flow 2.7.0-alpha.13 → 2.7.0-alpha.14

package/dist/cli/simple-commands/hive-mind/memory.js

@@ -0,0 +1,1068 @@
+/**
+ * Collective Memory System for Hive Mind
+ * Shared knowledge base and learning system
+ */
+import EventEmitter from 'events';
+import Database from 'better-sqlite3';
+import path from 'path';
+import { performance } from 'perf_hooks';
+import { Worker } from 'worker_threads';
+/**
+ * Memory types and their characteristics
+ */
+const MEMORY_TYPES = {
+    knowledge: { priority: 1, ttl: null, compress: false },
+    context: { priority: 2, ttl: 3600000, compress: false }, // 1 hour
+    task: { priority: 3, ttl: 1800000, compress: true }, // 30 minutes
+    result: { priority: 2, ttl: null, compress: true },
+    error: { priority: 1, ttl: 86400000, compress: false }, // 24 hours
+    metric: { priority: 3, ttl: 3600000, compress: true }, // 1 hour
+    consensus: { priority: 1, ttl: null, compress: false },
+    system: { priority: 1, ttl: null, compress: false },
+};
+/**
+ * Memory Pool for object reuse
+ */
+class MemoryPool {
+    constructor(createFn, resetFn, maxSize = 1000) {
+        this.createFn = createFn;
+        this.resetFn = resetFn;
+        this.maxSize = maxSize;
+        this.pool = [];
+        this.allocated = 0;
+        this.reused = 0;
+    }
+    acquire() {
+        if (this.pool.length > 0) {
+            this.reused++;
+            return this.pool.pop();
+        }
+        this.allocated++;
+        return this.createFn();
+    }
+    release(obj) {
+        if (this.pool.length < this.maxSize) {
+            this.resetFn(obj);
+            this.pool.push(obj);
+        }
+    }
+    getStats() {
+        return {
+            poolSize: this.pool.length,
+            allocated: this.allocated,
+            reused: this.reused,
+            reuseRate: (this.reused / (this.allocated + this.reused)) * 100,
+        };
+    }
+}
+/**
+ * Optimized LRU Cache with memory pressure handling
+ */
+class OptimizedLRUCache {
+    constructor(maxSize = 1000, maxMemoryMB = 50) {
+        this.maxSize = maxSize;
+        this.maxMemory = maxMemoryMB * 1024 * 1024;
+        this.cache = new Map();
+        this.currentMemory = 0;
+        this.hits = 0;
+        this.misses = 0;
+        this.evictions = 0;
+    }
+    get(key) {
+        if (this.cache.has(key)) {
+            const value = this.cache.get(key);
+            // Move to end (most recently used)
+            this.cache.delete(key);
+            this.cache.set(key, value);
+            this.hits++;
+            return value.data;
+        }
+        this.misses++;
+        return null;
+    }
+    set(key, data) {
+        const size = this._estimateSize(data);
+        // Check memory pressure
+        if (this.currentMemory + size > this.maxMemory) {
+            this._evictByMemoryPressure(size);
+        }
+        // Check size limit
+        if (this.cache.size >= this.maxSize) {
+            this._evictLRU();
+        }
+        const entry = {
+            data,
+            size,
+            timestamp: Date.now(),
+        };
+        this.cache.set(key, entry);
+        this.currentMemory += size;
+    }
+    _estimateSize(obj) {
+        return JSON.stringify(obj).length * 2; // Rough estimate
+    }
+    _evictLRU() {
+        const firstKey = this.cache.keys().next().value;
+        if (firstKey) {
+            const entry = this.cache.get(firstKey);
+            this.cache.delete(firstKey);
+            this.currentMemory -= entry.size;
+            this.evictions++;
+        }
+    }
+    _evictByMemoryPressure(neededSize) {
+        while (this.currentMemory + neededSize > this.maxMemory && this.cache.size > 0) {
+            this._evictLRU();
+        }
+    }
+    forEach(callback) {
+        this.cache.forEach((entry, key) => {
+            callback(entry, key);
+        });
+    }
+    delete(key) {
+        if (this.cache.has(key)) {
+            const entry = this.cache.get(key);
+            this.cache.delete(key);
+            this.currentMemory -= entry.size;
+            return true;
+        }
+        return false;
+    }
+    getStats() {
+        return {
+            size: this.cache.size,
+            memoryUsage: this.currentMemory,
+            hitRate: (this.hits / (this.hits + this.misses)) * 100,
+            evictions: this.evictions,
+        };
+    }
+}
+/**
+ * Optimized CollectiveMemory class with advanced memory management
+ */
+export class CollectiveMemory extends EventEmitter {
+    constructor(config = {}) {
+        super();
+        /** @type {import('better-sqlite3').Database | null} */
+        this.db = null;
+        this.config = {
+            swarmId: config.swarmId,
+            maxSize: config.maxSize || 100, // MB
+            dbPath: config.dbPath || path.join(process.cwd(), '.hive-mind', 'hive.db'),
+            compressionThreshold: config.compressionThreshold || 1024, // bytes
+            gcInterval: config.gcInterval || 300000, // 5 minutes
+            cacheSize: config.cacheSize || 1000,
+            cacheMemoryMB: config.cacheMemoryMB || 50,
+            enablePooling: config.enablePooling !== false,
+            enableAsyncOperations: config.enableAsyncOperations !== false,
+            ...config,
+        };
+        this.state = {
+            totalSize: 0,
+            entryCount: 0,
+            compressionRatio: 1,
+            lastGC: Date.now(),
+            accessPatterns: new Map(),
+            performanceMetrics: {
+                queryTimes: [],
+                avgQueryTime: 0,
+                cacheHitRate: 0,
+                memoryEfficiency: 0,
+            },
+        };
+        this.gcTimer = null;
+        // Optimized cache with LRU eviction
+        this.cache = new OptimizedLRUCache(this.config.cacheSize, this.config.cacheMemoryMB);
+        // Memory pools for frequently created objects
+        this.pools = {
+            queryResults: new MemoryPool(() => ({ results: [], metadata: {} }), (obj) => {
+                obj.results.length = 0;
+                Object.keys(obj.metadata).forEach((k) => delete obj.metadata[k]);
+            }),
+            memoryEntries: new MemoryPool(() => ({ id: '', key: '', value: '', metadata: {} }), (obj) => {
+                obj.id = obj.key = obj.value = '';
+                Object.keys(obj.metadata).forEach((k) => delete obj.metadata[k]);
+            }),
+        };
+        // Prepared statements for better performance
+        this.statements = new Map();
+        // Background worker for heavy operations
+        this.backgroundWorker = null;
+        this._initialize();
+    }
+    /**
+     * Initialize collective memory with optimizations
+     */
+    _initialize() {
+        try {
+            // Open database connection with optimizations
+            this.db = new Database(this.config.dbPath);
+            // Performance optimizations
+            this.db.pragma('journal_mode = WAL');
+            this.db.pragma('synchronous = NORMAL');
+            this.db.pragma('cache_size = -64000'); // 64MB cache
+            this.db.pragma('temp_store = MEMORY');
+            this.db.pragma('mmap_size = 268435456'); // 256MB memory mapping
+            this.db.pragma('optimize');
+            // Ensure table exists with optimized schema
+            this.db.exec(`
+        CREATE TABLE IF NOT EXISTS collective_memory (
+          id TEXT PRIMARY KEY,
+          swarm_id TEXT NOT NULL,
+          key TEXT NOT NULL,
+          value BLOB,
+          type TEXT DEFAULT 'knowledge',
+          confidence REAL DEFAULT 1.0,
+          created_by TEXT,
+          created_at INTEGER DEFAULT (strftime('%s','now')),
+          accessed_at INTEGER DEFAULT (strftime('%s','now')),
+          access_count INTEGER DEFAULT 0,
+          compressed INTEGER DEFAULT 0,
+          size INTEGER DEFAULT 0,
+          FOREIGN KEY (swarm_id) REFERENCES swarms(id)
+        );
+        
+        -- Optimized indexes
+        CREATE UNIQUE INDEX IF NOT EXISTS idx_memory_swarm_key 
+        ON collective_memory(swarm_id, key);
+        
+        CREATE INDEX IF NOT EXISTS idx_memory_type_accessed 
+        ON collective_memory(type, accessed_at DESC);
+        
+        CREATE INDEX IF NOT EXISTS idx_memory_size_compressed 
+        ON collective_memory(size, compressed);
+        
+        -- Memory optimization view
+        CREATE VIEW IF NOT EXISTS memory_stats AS
+        SELECT 
+          swarm_id,
+          type,
+          COUNT(*) as entry_count,
+          SUM(size) as total_size,
+          AVG(access_count) as avg_access,
+          MAX(accessed_at) as last_access
+        FROM collective_memory
+        GROUP BY swarm_id, type;
+      `);
+            // Prepare optimized statements
+            this._prepareStatements();
+            // Load initial statistics
+            this._updateStatistics();
+            // Start background optimization processes
+            this._startOptimizationTimers();
+            // Initialize background worker for heavy operations
+            if (this.config.enableAsyncOperations) {
+                this._initializeBackgroundWorker();
+            }
+            this.emit('memory:initialized', {
+                swarmId: this.config.swarmId,
+                optimizations: {
+                    pooling: this.config.enablePooling,
+                    asyncOps: this.config.enableAsyncOperations,
+                    cacheSize: this.config.cacheSize,
+                },
+            });
+        }
+        catch (error) {
+            this.emit('error', error);
+            throw error;
+        }
+    }
+    /**
+     * Prepare optimized SQL statements
+     */
+    _prepareStatements() {
+        this.statements.set('insert', this.db.prepare(`
+      INSERT OR REPLACE INTO collective_memory 
+      (id, swarm_id, key, value, type, confidence, created_by, compressed, size)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+    `));
+        this.statements.set('update', this.db.prepare(`
+      UPDATE collective_memory 
+      SET value = ?, accessed_at = strftime('%s','now'), access_count = access_count + 1,
+          compressed = ?, size = ?
+      WHERE swarm_id = ? AND key = ?
+    `));
+        this.statements.set('select', this.db.prepare(`
+      SELECT value, type, compressed, confidence, access_count
+      FROM collective_memory
+      WHERE swarm_id = ? AND key = ?
+    `));
+        this.statements.set('updateAccess', this.db.prepare(`
+      UPDATE collective_memory
+      SET accessed_at = strftime('%s','now'), access_count = access_count + 1
+      WHERE swarm_id = ? AND key = ?
+    `));
+        this.statements.set('searchByPattern', this.db.prepare(`
+      SELECT key, type, confidence, created_at, accessed_at, access_count
+      FROM collective_memory
+      WHERE swarm_id = ? AND key LIKE ? AND confidence >= ?
+      ORDER BY access_count DESC, confidence DESC
+      LIMIT ?
+    `));
+        this.statements.set('getStats', this.db.prepare(`
+      SELECT 
+        COUNT(*) as count,
+        SUM(size) as totalSize,
+        AVG(confidence) as avgConfidence,
+        SUM(compressed) as compressedCount,
+        AVG(access_count) as avgAccess
+      FROM collective_memory
+      WHERE swarm_id = ?
+    `));
+        this.statements.set('deleteExpired', this.db.prepare(`
+      DELETE FROM collective_memory
+      WHERE swarm_id = ? AND type = ? AND (strftime('%s','now') - accessed_at) > ?
+    `));
+        this.statements.set('getLRU', this.db.prepare(`
+      SELECT id, size FROM collective_memory
+      WHERE swarm_id = ? AND type NOT IN ('system', 'consensus')
+      ORDER BY accessed_at ASC, access_count ASC
+      LIMIT ?
+    `));
+    }
+    /**
+     * Start optimization timers
+     */
+    _startOptimizationTimers() {
+        // Main garbage collection
+        this.gcTimer = setInterval(() => this._garbageCollect(), this.config.gcInterval);
+        // Database optimization
+        this.optimizeTimer = setInterval(() => this._optimizeDatabase(), 1800000); // 30 minutes
+        // Cache cleanup
+        this.cacheTimer = setInterval(() => this._optimizeCache(), 60000); // 1 minute
+        // Performance monitoring
+        this.metricsTimer = setInterval(() => this._updatePerformanceMetrics(), 30000); // 30 seconds
+    }
+    /**
+     * Initialize background worker for heavy operations
+     */
+    _initializeBackgroundWorker() {
+        // Note: In production, this would initialize a proper Worker
+        // For now, we'll use async operations
+        this.backgroundQueue = [];
+        this.backgroundProcessing = false;
+    }
+    /**
+     * Store data in collective memory
+     */
+    async store(key, value, type = 'knowledge', metadata = {}) {
+        try {
+            const serialized = JSON.stringify(value);
+            const size = Buffer.byteLength(serialized);
+            const shouldCompress = size > this.config.compressionThreshold && MEMORY_TYPES[type]?.compress;
+            let storedValue = serialized;
+            let compressed = 0;
+            if (shouldCompress) {
+                // In production, use proper compression like zlib
+                // For now, we'll just mark it as compressed
+                compressed = 1;
+            }
+            const id = `${this.config.swarmId}-${key}-${Date.now()}`;
+            // Check if key already exists
+            const existing = this.db
+                .prepare(`
+        SELECT id FROM collective_memory 
+        WHERE swarm_id = ? AND key = ?
+      `)
+                .get(this.config.swarmId, key);
+            if (existing) {
+                // Update existing entry
+                this.db
+                    .prepare(`
+          UPDATE collective_memory 
+          SET value = ?, type = ?, confidence = ?, 
+              accessed_at = CURRENT_TIMESTAMP, access_count = access_count + 1,
+              compressed = ?, size = ?
+          WHERE swarm_id = ? AND key = ?
+        `)
+                    .run(storedValue, type, metadata.confidence || 1.0, compressed, size, this.config.swarmId, key);
+            }
+            else {
+                // Insert new entry
+                this.db
+                    .prepare(`
+          INSERT INTO collective_memory 
+          (id, swarm_id, key, value, type, confidence, created_by, compressed, size)
+          VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+        `)
+                    .run(id, this.config.swarmId, key, storedValue, type, metadata.confidence || 1.0, metadata.createdBy || 'system', compressed, size);
+            }
+            // Update cache
+            this.cache.set(key, {
+                value,
+                type,
+                timestamp: Date.now(),
+                size,
+            });
+            // Check memory limits
+            this._checkMemoryLimits();
+            // Track access pattern
+            this._trackAccess(key, 'write');
+            this.emit('memory:stored', { key, type, size });
+            return { success: true, id, size };
+        }
+        catch (error) {
+            this.emit('error', error);
+            throw error;
+        }
+    }
+    /**
+     * Retrieve data from collective memory
+     */
+    async retrieve(key) {
+        try {
+            // Check cache first
+            if (this.cache.has(key)) {
+                const cached = this.cache.get(key);
+                this._trackAccess(key, 'cache_hit');
+                return cached.value;
+            }
+            // Query database
+            const result = this.db
+                .prepare(`
+        SELECT value, type, compressed, confidence
+        FROM collective_memory
+        WHERE swarm_id = ? AND key = ?
+      `)
+                .get(this.config.swarmId, key);
+            if (!result) {
+                this._trackAccess(key, 'miss');
+                return null;
+            }
+            // Update access statistics
+            this.db
+                .prepare(`
+        UPDATE collective_memory
+        SET accessed_at = CURRENT_TIMESTAMP,
+            access_count = access_count + 1
+        WHERE swarm_id = ? AND key = ?
+      `)
+                .run(this.config.swarmId, key);
+            // Decompress if needed
+            let value = result.value;
+            if (result.compressed) {
+                // In production, decompress here
+            }
+            // Parse JSON
+            const parsed = JSON.parse(value);
+            // Add to cache
+            this.cache.set(key, {
+                value: parsed,
+                type: result.type,
+                timestamp: Date.now(),
+                confidence: result.confidence,
+            });
+            this._trackAccess(key, 'read');
+            return parsed;
+        }
+        catch (error) {
+            this.emit('error', error);
+            throw error;
+        }
+    }
+    /**
+     * Search collective memory
+     */
+    async search(pattern, options = {}) {
+        try {
+            const limit = options.limit || 50;
+            const type = options.type || null;
+            const minConfidence = options.minConfidence || 0;
+            let query = `
+        SELECT key, type, confidence, created_at, accessed_at, access_count
+        FROM collective_memory
+        WHERE swarm_id = ? 
+        AND key LIKE ?
+        AND confidence >= ?
+      `;
+            const params = [this.config.swarmId, `%${pattern}%`, minConfidence];
+            if (type) {
+                query += ' AND type = ?';
+                params.push(type);
+            }
+            query += ' ORDER BY access_count DESC, confidence DESC LIMIT ?';
+            params.push(limit);
+            const results = this.db.prepare(query).all(...params);
+            this._trackAccess(`search:${pattern}`, 'search');
+            return results;
+        }
+        catch (error) {
+            this.emit('error', error);
+            throw error;
+        }
+    }
+    /**
+     * Get related memories using association
+     */
+    async getRelated(key, limit = 10) {
+        try {
+            // Get the original memory
+            const original = await this.retrieve(key);
+            if (!original)
+                return [];
+            // Simple association: find memories accessed around the same time
+            const result = this.db
+                .prepare(`
+        SELECT m1.key, m1.type, m1.confidence, m1.access_count
+        FROM collective_memory m1
+        JOIN collective_memory m2 ON m1.swarm_id = m2.swarm_id
+        WHERE m2.key = ? 
+        AND m1.key != ?
+        AND m1.swarm_id = ?
+        AND ABS(julianday(m1.accessed_at) - julianday(m2.accessed_at)) < 0.01
+        ORDER BY m1.confidence DESC, m1.access_count DESC
+        LIMIT ?
+      `)
+                .all(key, key, this.config.swarmId, limit);
+            return result;
+        }
+        catch (error) {
+            this.emit('error', error);
+            throw error;
+        }
+    }
+    /**
+     * Build associations between memories
+     */
+    async associate(key1, key2, strength = 1.0) {
+        try {
+            // Store bidirectional association
+            await this.store(`assoc:${key1}:${key2}`, {
+                from: key1,
+                to: key2,
+                strength,
+                created: Date.now(),
+            }, 'system');
+            await this.store(`assoc:${key2}:${key1}`, {
+                from: key2,
+                to: key1,
+                strength,
+                created: Date.now(),
+            }, 'system');
+            this.emit('memory:associated', { key1, key2, strength });
+        }
+        catch (error) {
+            this.emit('error', error);
+            throw error;
+        }
+    }
+    /**
+     * Consolidate similar memories
+     */
+    async consolidate() {
+        try {
+            // Find similar memories
+            const memories = this.db
+                .prepare(`
+        SELECT key, value, type, confidence, access_count
+        FROM collective_memory
+        WHERE swarm_id = ?
+        AND type IN ('knowledge', 'result')
+        ORDER BY created_at DESC
+        LIMIT 1000
+      `)
+                .all(this.config.swarmId);
+            const consolidated = new Map();
+            // Group by similarity (simple implementation)
+            memories.forEach((memory) => {
+                const value = JSON.parse(memory.value);
+                const category = this._categorizeMemory(value);
+                if (!consolidated.has(category)) {
+                    consolidated.set(category, []);
+                }
+                consolidated.get(category).push({
+                    ...memory,
+                    value,
+                });
+            });
+            // Merge similar memories
+            let mergeCount = 0;
+            consolidated.forEach((group, category) => {
+                if (group.length > 1) {
+                    const merged = this._mergeMemories(group);
+                    // Store merged memory
+                    this.store(`consolidated:${category}`, merged, 'knowledge', {
+                        confidence: merged.confidence,
+                        createdBy: 'consolidation',
+                    });
+                    mergeCount++;
+                }
+            });
+            this.emit('memory:consolidated', { categories: consolidated.size, merged: mergeCount });
+            return { categories: consolidated.size, merged: mergeCount };
+        }
+        catch (error) {
+            this.emit('error', error);
+            throw error;
+        }
+    }
+    /**
+     * Categorize memory for consolidation
+     */
+    _categorizeMemory(value) {
+        // Simple categorization based on content
+        if (typeof value === 'string') {
+            return 'text';
+        }
+        if (typeof value === 'object') {
+            const keys = Object.keys(value).sort().join(':');
+            return `object:${keys.substring(0, 50)}`;
+        }
+        return 'other';
+    }
+    /**
+     * Merge similar memories
+     */
+    _mergeMemories(memories) {
+        // Calculate weighted average confidence
+        let totalWeight = 0;
+        let weightedConfidence = 0;
+        const mergedValue = {};
+        memories.forEach((memory) => {
+            const weight = memory.access_count + 1;
+            totalWeight += weight;
+            weightedConfidence += memory.confidence * weight;
+            // Merge values (simple implementation)
+            if (typeof memory.value === 'object') {
+                Object.assign(mergedValue, memory.value);
+            }
+        });
+        return {
+            value: mergedValue,
+            confidence: weightedConfidence / totalWeight,
+            sourceCount: memories.length,
+        };
+    }
+    /**
+     * Garbage collection
+     */
+    _garbageCollect() {
+        try {
+            const now = Date.now();
+            let deletedCount = 0;
+            // Delete expired memories based on TTL
+            Object.entries(MEMORY_TYPES).forEach(([type, config]) => {
+                if (config.ttl) {
+                    const result = this.db
+                        .prepare(`
+            DELETE FROM collective_memory
+            WHERE swarm_id = ?
+            AND type = ?
+            AND (julianday('now') - julianday(accessed_at)) * 86400000 > ?
+          `)
+                        .run(this.config.swarmId, type, config.ttl);
+                    deletedCount += result.changes;
+                }
+            });
+            // Clear old cache entries
+            const cacheTimeout = 300000; // 5 minutes
+            this.cache.forEach((value, key) => {
+                if (now - value.timestamp > cacheTimeout) {
+                    this.cache.delete(key);
+                }
+            });
+            // Update statistics
+            this._updateStatistics();
+            this.state.lastGC = now;
+            if (deletedCount > 0) {
+                this.emit('memory:gc', { deleted: deletedCount, cacheSize: this.cache.size });
+            }
+        }
+        catch (error) {
+            this.emit('error', error);
+        }
+    }
+    /**
+     * Check memory limits and evict if necessary
+     */
+    _checkMemoryLimits() {
+        if (this.state.totalSize > this.config.maxSize * 1024 * 1024) {
+            // Evict least recently used memories
+            const toEvict = this.db
+                .prepare(`
+        SELECT id, size FROM collective_memory
+        WHERE swarm_id = ?
+        AND type NOT IN ('system', 'consensus')
+        ORDER BY accessed_at ASC, access_count ASC
+        LIMIT 100
+      `)
+                .all(this.config.swarmId);
+            let freedSize = 0;
+            toEvict.forEach((memory) => {
+                this.db.prepare('DELETE FROM collective_memory WHERE id = ?').run(memory.id);
+                freedSize += memory.size;
+            });
+            this.emit('memory:evicted', { count: toEvict.length, freedSize });
+        }
+    }
+    /**
+     * Optimize database performance
+     */
+    _optimizeDatabase() {
+        try {
+            // Run database optimization
+            this.db.pragma('optimize');
+            this.db.pragma('analysis_limit=1000');
+            this.db.exec('ANALYZE');
+            // Update database statistics
+            this._updateStatistics();
+            this.emit('database:optimized');
+        }
+        catch (error) {
+            this.emit('error', error);
+        }
+    }
+    /**
+     * Optimize cache performance
+     */
+    _optimizeCache() {
+        try {
+            const now = Date.now();
+            const cacheTimeout = 300000; // 5 minutes
+            // Clear expired cache entries
+            if (this.cache.cache) {
+                this.cache.cache.forEach((value, key) => {
+                    if (now - value.timestamp > cacheTimeout) {
+                        this.cache.cache.delete(key);
+                    }
+                });
+            }
+            this.emit('cache:optimized', {
+                size: this.cache.cache ? this.cache.cache.size : 0,
+            });
+        }
+        catch (error) {
+            this.emit('error', error);
+        }
+    }
+    /**
+     * Update performance metrics
+     */
+    _updatePerformanceMetrics() {
+        try {
+            // Calculate cache hit rate
+            const cacheStats = this.cache.getStats();
+            this.state.performanceMetrics.cacheHitRate = cacheStats.hitRate || 0;
+            // Calculate memory efficiency
+            this.state.performanceMetrics.memoryEfficiency =
+                (this.state.totalSize / (this.config.maxSize * 1024 * 1024)) * 100;
+            // Update average query time if we have recent measurements
+            if (this.state.performanceMetrics.queryTimes.length > 0) {
+                this.state.performanceMetrics.avgQueryTime =
+                    this.state.performanceMetrics.queryTimes.reduce((sum, time) => sum + time, 0) /
+                        this.state.performanceMetrics.queryTimes.length;
+                // Keep only recent query times (last 100)
+                if (this.state.performanceMetrics.queryTimes.length > 100) {
+                    this.state.performanceMetrics.queryTimes =
+                        this.state.performanceMetrics.queryTimes.slice(-100);
+                }
+            }
+            this.emit('metrics:updated', this.state.performanceMetrics);
+        }
+        catch (error) {
+            this.emit('error', error);
+        }
+    }
+    /**
+     * Update memory statistics
+     */
+    _updateStatistics() {
+        const stats = this.db
+            .prepare(`
+      SELECT 
+        COUNT(*) as count,
+        SUM(size) as totalSize,
+        AVG(confidence) as avgConfidence,
+        SUM(compressed) as compressedCount
+      FROM collective_memory
+      WHERE swarm_id = ?
+    `)
+            .get(this.config.swarmId);
+        this.state.entryCount = stats.count || 0;
+        this.state.totalSize = stats.totalSize || 0;
+        this.state.avgConfidence = stats.avgConfidence || 1.0;
+        if (stats.compressedCount > 0) {
+            // Estimate compression ratio
+            this.state.compressionRatio = 0.6; // Assume 40% compression
+        }
+    }
+    /**
+     * Track access patterns
+     */
+    _trackAccess(key, operation) {
+        const pattern = this.state.accessPatterns.get(key) || {
+            reads: 0,
+            writes: 0,
+            searches: 0,
+            cacheHits: 0,
+            misses: 0,
+            lastAccess: Date.now(),
+        };
+        switch (operation) {
+            case 'read':
+                pattern.reads++;
+                break;
+            case 'write':
+                pattern.writes++;
+                break;
+            case 'search':
+                pattern.searches++;
+                break;
+            case 'cache_hit':
+                pattern.cacheHits++;
+                break;
+            case 'miss':
+                pattern.misses++;
+                break;
+        }
+        pattern.lastAccess = Date.now();
+        this.state.accessPatterns.set(key, pattern);
+        // Keep access patterns size limited
+        if (this.state.accessPatterns.size > 1000) {
+            // Remove oldest entries
+            const sorted = Array.from(this.state.accessPatterns.entries()).sort((a, b) => a[1].lastAccess - b[1].lastAccess);
+            sorted.slice(0, 100).forEach(([key]) => {
+                this.state.accessPatterns.delete(key);
+            });
+        }
+    }
+    /**
+     * Get enhanced memory statistics
+     */
+    getStatistics() {
+        return {
+            swarmId: this.config.swarmId,
+            entryCount: this.state.entryCount,
+            totalSize: this.state.totalSize,
+            maxSize: this.config.maxSize * 1024 * 1024,
+            utilizationPercent: (this.state.totalSize / (this.config.maxSize * 1024 * 1024)) * 100,
+            avgConfidence: this.state.avgConfidence,
+            compressionRatio: this.state.compressionRatio,
+            cacheSize: this.cache.cache ? this.cache.cache.size : 0,
+            lastGC: new Date(this.state.lastGC).toISOString(),
+            accessPatterns: this.state.accessPatterns.size,
+            optimization: {
+                cacheOptimized: true,
+                poolingEnabled: this.config.enablePooling,
+                asyncOperations: this.config.enableAsyncOperations,
+                compressionRatio: this.state.compressionRatio,
+                performanceMetrics: this.state.performanceMetrics,
+            },
+        };
+    }
+    /**
+     * Export memory snapshot
+     */
+    async exportSnapshot(filepath) {
+        try {
+            const memories = this.db
+                .prepare(`
+        SELECT * FROM collective_memory
+        WHERE swarm_id = ?
+        ORDER BY created_at DESC
+      `)
+                .all(this.config.swarmId);
+            const snapshot = {
+                swarmId: this.config.swarmId,
+                timestamp: new Date().toISOString(),
+                statistics: this.getStatistics(),
+                memories: memories.map((m) => ({
+                    ...m,
+                    value: JSON.parse(m.value),
+                })),
+            };
+            // In production, write to file
+            // For now, return the snapshot
+            this.emit('memory:exported', { count: memories.length });
+            return snapshot;
+        }
+        catch (error) {
+            this.emit('error', error);
+            throw error;
+        }
+    }
+    /**
+     * Import memory snapshot
+     */
+    async importSnapshot(snapshot) {
+        try {
+            let imported = 0;
+            for (const memory of snapshot.memories) {
+                await this.store(memory.key, memory.value, memory.type, {
+                    confidence: memory.confidence,
+                    createdBy: memory.created_by,
+                });
+                imported++;
+            }
+            this.emit('memory:imported', { count: imported });
+            return { imported };
+        }
+        catch (error) {
+            this.emit('error', error);
+            throw error;
+        }
+    }
+    /**
+     * Enhanced shutdown with cleanup
+     */
+    close() {
+        // Clear all timers
+        if (this.gcTimer)
+            clearInterval(this.gcTimer);
+        if (this.optimizeTimer)
+            clearInterval(this.optimizeTimer);
+        if (this.cacheTimer)
+            clearInterval(this.cacheTimer);
+        if (this.metricsTimer)
+            clearInterval(this.metricsTimer);
+        // Final optimization before closing
+        try {
+            this.db.pragma('optimize');
+        }
+        catch (error) {
+            // Ignore errors during shutdown
+        }
+        // Close database
+        if (this.db) {
+            this.db.close();
+        }
+        // Clear memory pools
+        if (this.config.enablePooling) {
+            Object.values(this.pools).forEach((pool) => {
+                pool.pool.length = 0;
+            });
+        }
+        const finalStats = {
+            cacheStats: this.cache.getStats ? this.cache.getStats() : {},
+            poolStats: this.config.enablePooling
+                ? {
+                    queryResults: this.pools.queryResults.getStats(),
+                    memoryEntries: this.pools.memoryEntries.getStats(),
+                }
+                : null,
+            performanceMetrics: this.state.performanceMetrics,
+        };
+        this.emit('memory:closed', finalStats);
+    }
+    /**
+     * Get comprehensive memory analytics
+     */
+    getAnalytics() {
+        return {
+            basic: this.getStatistics(),
+            performance: this.state.performanceMetrics,
+            cache: this.cache.getStats ? this.cache.getStats() : {},
+            pools: this.config.enablePooling
+                ? {
+                    queryResults: this.pools.queryResults.getStats(),
+                    memoryEntries: this.pools.memoryEntries.getStats(),
+                }
+                : null,
+            database: {
+                fragmentation: this.db.pragma('freelist_count'),
+                pageSize: this.db.pragma('page_size'),
+                cacheSize: this.db.pragma('cache_size'),
+            },
+        };
+    }
+    /**
+     * Memory health check
+     */
+    async healthCheck() {
+        const analytics = this.getAnalytics();
+        const health = {
+            status: 'healthy',
+            issues: [],
+            recommendations: [],
+        };
+        // Check cache hit rate
+        if (analytics.cache.hitRate < 50) {
+            health.issues.push('Low cache hit rate');
+            health.recommendations.push('Consider increasing cache size');
+        }
+        // Check memory usage
+        if (analytics.basic.utilizationPercent > 90) {
+            health.status = 'warning';
+            health.issues.push('High memory utilization');
+            health.recommendations.push('Consider increasing max memory or running garbage collection');
+        }
+        // Check query performance
+        if (analytics.performance.avgQueryTime > 100) {
+            health.issues.push('Slow query performance');
+            health.recommendations.push('Consider database optimization or indexing');
+        }
+        return health;
+    }
+}
+/**
+ * Memory optimization utilities
+ */
+export class MemoryOptimizer {
+    static async optimizeCollectiveMemory(memory) {
+        const startTime = performance.now();
+        // Run comprehensive optimization
+        await memory._optimizeDatabase();
+        memory._optimizeCache();
+        memory._garbageCollect();
+        const duration = performance.now() - startTime;
+        return {
+            duration,
+            analytics: memory.getAnalytics(),
+            health: await memory.healthCheck(),
+        };
+    }
+    static calculateOptimalCacheSize(memoryStats, accessPatterns) {
+        const avgEntrySize = memoryStats.totalSize / memoryStats.entryCount;
+        const hotKeys = Array.from(accessPatterns.entries())
+            .sort((a, b) => b[1] - a[1])
+            .slice(0, Math.min(1000, memoryStats.entryCount * 0.2));
+        const optimalCacheEntries = hotKeys.length * 1.2; // 20% buffer
+        const optimalCacheMemoryMB = (optimalCacheEntries * avgEntrySize) / (1024 * 1024);
+        return {
+            entries: Math.ceil(optimalCacheEntries),
+            memoryMB: Math.ceil(optimalCacheMemoryMB),
+            efficiency: (hotKeys.length / memoryStats.entryCount) * 100,
+        };
+    }
+    static generateOptimizationReport(analytics) {
+        const report = {
+            timestamp: new Date().toISOString(),
+            summary: {},
+            recommendations: [],
+            metrics: analytics,
+        };
+        // Performance summary
+        report.summary.avgQueryTime = analytics.performance.avgQueryTime;
+        report.summary.cacheHitRate = analytics.cache.hitRate || 0;
+        report.summary.memoryEfficiency = analytics.cache.memoryUsage / (1024 * 1024);
+        // Generate recommendations
+        if ((analytics.cache.hitRate || 0) < 70) {
+            report.recommendations.push({
+                type: 'cache',
+                priority: 'high',
+                description: 'Increase cache size to improve hit rate',
+                impact: 'Reduce database queries by up to 30%',
+            });
+        }
+        if (analytics.performance.avgQueryTime > 50) {
+            report.recommendations.push({
+                type: 'database',
+                priority: 'medium',
+                description: 'Optimize database indexes and run ANALYZE',
+                impact: 'Improve query performance by 20-40%',
+            });
+        }
+        if (analytics.pools?.queryResults?.reuseRate < 50) {
+            report.recommendations.push({
+                type: 'pooling',
+                priority: 'low',
+                description: 'Increase object pool sizes for better reuse',
+                impact: 'Reduce garbage collection pressure',
+            });
+        }
+        return report;
+    }
+}
+//# sourceMappingURL=memory.js.map