agentic-flow 3.0.0-alpha.1 → 3.0.0

package/dist/packages/agentdb/src/controllers/LearningSystem.js

@@ -1,1046 +0,0 @@
-/**
- * Learning System - Reinforcement Learning Session Management
- *
- * Manages RL training sessions with:
- * - Session lifecycle (start/end)
- * - Action prediction with confidence scores
- * - Feedback loop for policy learning
- * - Policy training with configurable parameters
- *
- * Supports 9 RL algorithms:
- * - Q-Learning
- * - SARSA
- * - Deep Q-Network (DQN)
- * - Policy Gradient
- * - Actor-Critic
- * - Proximal Policy Optimization (PPO)
- * - Decision Transformer
- * - Monte Carlo Tree Search (MCTS)
- * - Model-Based RL
- */
-import { cosineSimilarity } from '../utils/vector-math.js';
-import { RuVectorLearning } from '../backends/ruvector/RuVectorLearning.js';
-import { SonaTrajectoryService } from '../services/SonaTrajectoryService.js';
-import { GNNService } from '../services/GNNService.js';
-export class LearningSystem {
-    db;
-    embedder;
-    activeSessions = new Map();
-    // Phase 2: RuVector GNN and Sona integration
-    gnnLearning = null;
-    sonaService = null;
-    gnnService = null;
-    gnnEnabled = false;
-    sonaEnabled = false;
-    constructor(db, embedder) {
-        this.db = db;
-        this.embedder = embedder;
-        this.initializeSchema();
-        this.initializeRuVectorEnhancements().catch(err => {
-            console.warn('[LearningSystem] RuVector enhancements unavailable:', err.message);
-        });
-    }
-    /**
-     * Initialize RuVector GNN and Sona enhancements
-     */
-    async initializeRuVectorEnhancements() {
-        // Try to initialize GNN-enhanced learning
-        try {
-            const gnnConfig = {
-                inputDim: 384, // Default embedding size
-                hiddenDim: 256,
-                heads: 4,
-                dropout: 0.1
-            };
-            this.gnnLearning = new RuVectorLearning(gnnConfig);
-            await this.gnnLearning.initialize();
-            this.gnnEnabled = true;
-            console.log('✅ [LearningSystem] GNN-enhanced learning enabled (@ruvector/gnn)');
-        }
-        catch (error) {
-            console.warn('[LearningSystem] GNN unavailable, using standard learning');
-            this.gnnEnabled = false;
-        }
-        // Try to initialize Sona trajectory service
-        try {
-            this.sonaService = new SonaTrajectoryService();
-            const initialized = await this.sonaService.initialize();
-            this.sonaEnabled = initialized;
-            if (initialized) {
-                console.log(`[LearningSystem] Sona RL trajectory learning enabled (engine: ${this.sonaService.getEngineType()})`);
-            }
-            else {
-                console.warn('[LearningSystem] Sona unavailable, using in-memory trajectories');
-            }
-        }
-        catch (error) {
-            console.warn('[LearningSystem] Sona service initialization failed');
-            this.sonaEnabled = false;
-        }
-        // Try to initialize GNNService for intent classification
-        try {
-            this.gnnService = new GNNService({ inputDim: 384, hiddenDim: 128, outputDim: 64, layers: 3 });
-            await this.gnnService.initialize();
-            console.log(`[LearningSystem] GNNService initialized (engine: ${this.gnnService.getEngineType()})`);
-        }
-        catch (error) {
-            console.warn('[LearningSystem] GNNService initialization failed');
-            this.gnnService = null;
-        }
-    }
-    /**
-     * Get engine types for all RuVector integrations
-     */
-    getEngineTypes() {
-        return {
-            gnn: this.gnnEnabled ? (this.gnnLearning?.isInitialized() ? 'native' : 'js') : 'disabled',
-            sona: this.sonaEnabled ? (this.sonaService?.getEngineType() || 'js') : 'disabled',
-            gnnService: this.gnnService?.getEngineType() || 'disabled',
-        };
-    }
-    /**
-     * Initialize database schema for learning system
-     */
-    initializeSchema() {
-        this.db.exec(`
-      CREATE TABLE IF NOT EXISTS learning_sessions (
-        id TEXT PRIMARY KEY,
-        user_id TEXT NOT NULL,
-        session_type TEXT NOT NULL,
-        config TEXT NOT NULL,
-        start_time INTEGER NOT NULL,
-        end_time INTEGER,
-        status TEXT NOT NULL,
-        metadata TEXT
-      );
-
-      CREATE INDEX IF NOT EXISTS idx_learning_sessions_user ON learning_sessions(user_id);
-      CREATE INDEX IF NOT EXISTS idx_learning_sessions_status ON learning_sessions(status);
-
-      CREATE TABLE IF NOT EXISTS learning_experiences (
-        id INTEGER PRIMARY KEY AUTOINCREMENT,
-        session_id TEXT NOT NULL,
-        state TEXT NOT NULL,
-        action TEXT NOT NULL,
-        reward REAL NOT NULL,
-        next_state TEXT,
-        success INTEGER NOT NULL,
-        timestamp INTEGER NOT NULL,
-        metadata TEXT,
-        FOREIGN KEY (session_id) REFERENCES learning_sessions(id) ON DELETE CASCADE
-      );
-
-      CREATE INDEX IF NOT EXISTS idx_learning_experiences_session ON learning_experiences(session_id);
-      CREATE INDEX IF NOT EXISTS idx_learning_experiences_reward ON learning_experiences(reward);
-
-      CREATE TABLE IF NOT EXISTS learning_policies (
-        id INTEGER PRIMARY KEY AUTOINCREMENT,
-        session_id TEXT NOT NULL,
-        state_action_pairs TEXT NOT NULL,
-        q_values TEXT NOT NULL,
-        visit_counts TEXT NOT NULL,
-        avg_rewards TEXT NOT NULL,
-        version INTEGER NOT NULL,
-        created_at INTEGER DEFAULT (strftime('%s', 'now')),
-        FOREIGN KEY (session_id) REFERENCES learning_sessions(id) ON DELETE CASCADE
-      );
-
-      CREATE INDEX IF NOT EXISTS idx_learning_policies_session ON learning_policies(session_id);
-
-      CREATE TABLE IF NOT EXISTS learning_state_embeddings (
-        id INTEGER PRIMARY KEY AUTOINCREMENT,
-        session_id TEXT NOT NULL,
-        state TEXT NOT NULL,
-        embedding BLOB NOT NULL,
-        FOREIGN KEY (session_id) REFERENCES learning_sessions(id) ON DELETE CASCADE
-      );
-
-      CREATE INDEX IF NOT EXISTS idx_learning_state_embeddings_session ON learning_state_embeddings(session_id);
-    `);
-    }
-    /**
-     * Start a new learning session
-     */
-    async startSession(userId, sessionType, config) {
-        const sessionId = `session-${Date.now()}-${Math.random().toString(36).substring(7)}`;
-        const session = {
-            id: sessionId,
-            userId,
-            sessionType,
-            config,
-            startTime: Date.now(),
-            status: 'active',
-        };
-        // Store session in database
-        this.db.prepare(`
-      INSERT INTO learning_sessions (id, user_id, session_type, config, start_time, status)
-      VALUES (?, ?, ?, ?, ?, ?)
-    `).run(session.id, session.userId, session.sessionType, JSON.stringify(session.config), session.startTime, session.status);
-        // Cache in memory
-        this.activeSessions.set(sessionId, session);
-        console.log(`✅ Learning session started: ${sessionId} (${sessionType})`);
-        return sessionId;
-    }
-    /**
-     * End a learning session and save final policy
-     */
-    async endSession(sessionId) {
-        const session = this.activeSessions.get(sessionId) || this.getSession(sessionId);
-        if (!session) {
-            throw new Error(`Session not found: ${sessionId}`);
-        }
-        if (session.status === 'completed') {
-            throw new Error(`Session already completed: ${sessionId}`);
-        }
-        const endTime = Date.now();
-        // Save final policy
-        await this.savePolicy(sessionId);
-        // Update session status
-        this.db.prepare(`
-      UPDATE learning_sessions
-      SET status = 'completed', end_time = ?
-      WHERE id = ?
-    `).run(endTime, sessionId);
-        // Update memory cache
-        session.endTime = endTime;
-        session.status = 'completed';
-        // Remove from active sessions
-        this.activeSessions.delete(sessionId);
-        console.log(`✅ Learning session ended: ${sessionId} (duration: ${endTime - session.startTime}ms)`);
-    }
-    /**
-     * Predict next action with confidence scores
-     */
-    async predict(sessionId, state) {
-        const session = this.activeSessions.get(sessionId) || this.getSession(sessionId);
-        if (!session) {
-            throw new Error(`Session not found: ${sessionId}`);
-        }
-        if (session.status !== 'active') {
-            throw new Error(`Session not active: ${sessionId}`);
-        }
-        // Get or create state embedding
-        const stateEmbedding = await this.getStateEmbedding(sessionId, state);
-        // Get policy for this session
-        const policy = this.getLatestPolicy(sessionId);
-        // Calculate Q-values for all actions
-        const actionScores = await this.calculateActionScores(session, state, stateEmbedding, policy);
-        // Sort by score (highest first)
-        const sortedActions = actionScores.sort((a, b) => b.score - a.score);
-        // Epsilon-greedy exploration
-        const explorationRate = session.config.explorationRate || 0.1;
-        let selectedAction = sortedActions[0];
-        if (Math.random() < explorationRate) {
-            // Explore: random action
-            selectedAction = sortedActions[Math.floor(Math.random() * sortedActions.length)];
-        }
-        // Normalize confidence scores to [0, 1]
-        const maxScore = sortedActions[0].score;
-        const minScore = sortedActions[sortedActions.length - 1].score;
-        const scoreRange = maxScore - minScore || 1;
-        return {
-            action: selectedAction.action,
-            confidence: (selectedAction.score - minScore) / scoreRange,
-            qValue: selectedAction.score,
-            alternatives: sortedActions.slice(1, 4).map(a => ({
-                action: a.action,
-                confidence: (a.score - minScore) / scoreRange,
-                qValue: a.score,
-            })),
-        };
-    }
-    /**
-     * Submit feedback for learning
-     *
-     * Phase 2 Enhancement: Records to Sona for RL trajectory learning
-     */
-    async submitFeedback(feedback) {
-        const session = this.activeSessions.get(feedback.sessionId) || this.getSession(feedback.sessionId);
-        if (!session) {
-            throw new Error(`Session not found: ${feedback.sessionId}`);
-        }
-        // Store experience in database
-        this.db.prepare(`
-      INSERT INTO learning_experiences (
-        session_id, state, action, reward, next_state, success, timestamp
-      ) VALUES (?, ?, ?, ?, ?, ?, ?)
-    `).run(feedback.sessionId, feedback.state, feedback.action, feedback.reward, feedback.nextState || null, feedback.success ? 1 : 0, feedback.timestamp);
-        // Phase 2: Record to Sona for RL trajectory learning
-        if (this.sonaEnabled && this.sonaService) {
-            try {
-                const trajectoryStep = {
-                    state: { description: feedback.state },
-                    action: feedback.action,
-                    reward: feedback.reward
-                };
-                await this.sonaService.recordTrajectory(session.sessionType, [trajectoryStep]);
-            }
-            catch (error) {
-                console.warn('[LearningSystem] Failed to record Sona trajectory:', error);
-            }
-        }
-        // Update policy incrementally based on algorithm
-        await this.updatePolicyIncremental(session, feedback);
-        console.log(`✅ Feedback recorded: session=${feedback.sessionId}, action=${feedback.action}, reward=${feedback.reward}`);
-    }
-    /**
-     * Train policy with batch learning
-     */
-    async train(sessionId, epochs, batchSize, learningRate) {
-        const session = this.activeSessions.get(sessionId) || this.getSession(sessionId);
-        if (!session) {
-            throw new Error(`Session not found: ${sessionId}`);
-        }
-        const startTime = Date.now();
-        // Get all experiences for this session
-        const experiences = this.db.prepare(`
-      SELECT * FROM learning_experiences
-      WHERE session_id = ?
-      ORDER BY timestamp ASC
-    `).all(sessionId);
-        if (experiences.length === 0) {
-            throw new Error(`No training data available for session: ${sessionId}`);
-        }
-        let totalLoss = 0;
-        let totalReward = 0;
-        let batchCount = 0;
-        // Training loop
-        for (let epoch = 0; epoch < epochs; epoch++) {
-            // Shuffle experiences
-            const shuffled = this.shuffleArray([...experiences]);
-            // Process in batches
-            for (let i = 0; i < shuffled.length; i += batchSize) {
-                const batch = shuffled.slice(i, i + batchSize);
-                // Calculate loss and update policy
-                const batchLoss = await this.trainBatch(session, batch, learningRate);
-                totalLoss += batchLoss;
-                batchCount++;
-                // Accumulate rewards
-                totalReward += batch.reduce((sum, exp) => sum + exp.reward, 0);
-            }
-            // Log progress
-            if ((epoch + 1) % 10 === 0) {
-                console.log(`  Epoch ${epoch + 1}/${epochs} - Loss: ${(totalLoss / batchCount).toFixed(4)}`);
-            }
-        }
-        const trainingTimeMs = Date.now() - startTime;
-        const avgReward = totalReward / (experiences.length * epochs);
-        const finalLoss = totalLoss / batchCount;
-        // Save trained policy
-        await this.savePolicy(sessionId);
-        // Calculate convergence rate
-        const convergenceRate = this.calculateConvergenceRate(sessionId);
-        console.log(`✅ Training completed: ${epochs} epochs, ${trainingTimeMs}ms`);
-        return {
-            epochsCompleted: epochs,
-            finalLoss,
-            avgReward,
-            convergenceRate,
-            trainingTimeMs,
-        };
-    }
-    // ============================================================================
-    // Private Helper Methods
-    // ============================================================================
-    /**
-     * Get session from database
-     */
-    getSession(sessionId) {
-        const row = this.db.prepare(`
-      SELECT * FROM learning_sessions WHERE id = ?
-    `).get(sessionId);
-        if (!row)
-            return null;
-        return {
-            id: row.id,
-            userId: row.user_id,
-            sessionType: row.session_type,
-            config: JSON.parse(row.config),
-            startTime: row.start_time,
-            endTime: row.end_time,
-            status: row.status,
-            metadata: row.metadata ? JSON.parse(row.metadata) : undefined,
-        };
-    }
-    /**
-     * Get or create state embedding
-     */
-    async getStateEmbedding(sessionId, state) {
-        // Check if embedding exists
-        const existing = this.db.prepare(`
-      SELECT embedding FROM learning_state_embeddings
-      WHERE session_id = ? AND state = ?
-    `).get(sessionId, state);
-        if (existing) {
-            return new Float32Array(existing.embedding.buffer);
-        }
-        // Generate new embedding
-        const embedding = await this.embedder.embed(state);
-        // Store embedding
-        this.db.prepare(`
-      INSERT INTO learning_state_embeddings (session_id, state, embedding)
-      VALUES (?, ?, ?)
-    `).run(sessionId, state, Buffer.from(embedding.buffer));
-        return embedding;
-    }
-    /**
-     * Get latest policy for session
-     */
-    getLatestPolicy(sessionId) {
-        const policy = this.db.prepare(`
-      SELECT * FROM learning_policies
-      WHERE session_id = ?
-      ORDER BY version DESC
-      LIMIT 1
-    `).get(sessionId);
-        if (!policy) {
-            // Return empty policy
-            return {
-                stateActionPairs: {},
-                qValues: {},
-                visitCounts: {},
-                avgRewards: {},
-            };
-        }
-        return {
-            stateActionPairs: JSON.parse(policy.state_action_pairs),
-            qValues: JSON.parse(policy.q_values),
-            visitCounts: JSON.parse(policy.visit_counts),
-            avgRewards: JSON.parse(policy.avg_rewards),
-        };
-    }
-    /**
-     * Calculate action scores based on algorithm
-     *
-     * Phase 2 Enhancement: Uses GNN for embedding refinement and Sona for predictions
-     */
-    async calculateActionScores(session, state, stateEmbedding, policy) {
-        // Phase 2: Try GNN-enhanced prediction first
-        if (this.gnnEnabled && this.gnnLearning) {
-            try {
-                // Get similar states from policy for GNN neighbor aggregation
-                const similarStates = [];
-                const weights = [];
-                for (const [key, qValue] of Object.entries(policy.qValues)) {
-                    if (key.startsWith(state)) {
-                        similarStates.push(stateEmbedding); // Use same embedding for now
-                        weights.push(qValue);
-                    }
-                    if (similarStates.length >= 5)
-                        break; // Limit to top 5 neighbors
-                }
-                if (similarStates.length > 0) {
-                    // Enhance state embedding with GNN
-                    const enhancedEmbedding = this.gnnLearning.enhance(stateEmbedding, similarStates, weights);
-                    // Use enhanced embedding for better similarity matching
-                    // (This would improve action selection in production)
-                    console.log(`[LearningSystem] GNN enhanced embedding for state: ${state.substring(0, 50)}...`);
-                }
-            }
-            catch (error) {
-                console.warn('[LearningSystem] GNN enhancement failed, using standard scoring');
-            }
-        }
-        // Phase 2: Try Sona prediction if available
-        if (this.sonaEnabled && this.sonaService) {
-            try {
-                const prediction = await this.sonaService.predict({ state });
-                if (prediction.confidence > 0.7) {
-                    // High-confidence Sona prediction takes priority
-                    console.log(`[LearningSystem] Sona predicted action: ${prediction.action} (confidence: ${prediction.confidence.toFixed(2)})`);
-                    return [{
-                            action: prediction.action,
-                            score: prediction.confidence
-                        }];
-                }
-            }
-            catch (error) {
-                console.warn('[LearningSystem] Sona prediction failed, using policy');
-            }
-        }
-        // Get possible actions from past experiences
-        const actions = this.db.prepare(`
-      SELECT DISTINCT action FROM learning_experiences
-      WHERE session_id = ?
-    `).all(session.id).map((row) => row.action);
-        if (actions.length === 0) {
-            // Default actions if none exist
-            return [
-                { action: 'action_1', score: 0.5 },
-                { action: 'action_2', score: 0.4 },
-                { action: 'action_3', score: 0.3 },
-            ];
-        }
-        // Calculate scores based on algorithm type
-        const scores = [];
-        for (const action of actions) {
-            const key = `${state}|${action}`;
-            let score = 0;
-            switch (session.sessionType) {
-                case 'q-learning':
-                case 'sarsa':
-                case 'dqn':
-                    // Use Q-value from policy
-                    score = policy.qValues[key] || 0;
-                    break;
-                case 'policy-gradient':
-                case 'actor-critic':
-                case 'ppo':
-                    // Use average reward
-                    score = policy.avgRewards[key] || 0;
-                    break;
-                case 'decision-transformer':
-                    // Use reward-conditioned probability
-                    score = this.calculateTransformerScore(state, action, policy);
-                    break;
-                case 'mcts':
-                    // Use UCB1 formula
-                    score = this.calculateUCB1(state, action, policy);
-                    break;
-                case 'model-based':
-                    // Use model prediction
-                    score = this.calculateModelScore(state, action, policy);
-                    break;
-                default:
-                    score = Math.random();
-            }
-            scores.push({ action, score });
-        }
-        return scores;
-    }
-    /**
-     * Update policy incrementally after feedback
-     */
-    async updatePolicyIncremental(session, feedback) {
-        const policy = this.getLatestPolicy(feedback.sessionId);
-        const key = `${feedback.state}|${feedback.action}`;
-        // Initialize if not exists
-        if (!policy.qValues[key]) {
-            policy.qValues[key] = 0;
-            policy.visitCounts[key] = 0;
-            policy.avgRewards[key] = 0;
-        }
-        const alpha = session.config.learningRate;
-        const gamma = session.config.discountFactor;
-        switch (session.sessionType) {
-            case 'q-learning': {
-                // Q(s,a) ← Q(s,a) + α[r + γ max Q(s',a') - Q(s,a)]
-                let maxNextQ = 0;
-                if (feedback.nextState) {
-                    const nextActions = Object.keys(policy.qValues).filter(k => k.startsWith(feedback.nextState + '|'));
-                    maxNextQ = Math.max(...nextActions.map(k => policy.qValues[k]), 0);
-                }
-                const target = feedback.reward + gamma * maxNextQ;
-                policy.qValues[key] += alpha * (target - policy.qValues[key]);
-                break;
-            }
-            case 'sarsa': {
-                // SARSA: Q(s,a) ← Q(s,a) + α[r + γ Q(s',a') - Q(s,a)]
-                // For incremental update, we approximate with current Q-value
-                const target = feedback.reward + gamma * (policy.qValues[key] || 0);
-                policy.qValues[key] += alpha * (target - policy.qValues[key]);
-                break;
-            }
-            case 'policy-gradient':
-            case 'actor-critic':
-            case 'ppo': {
-                // Update average reward
-                policy.visitCounts[key]++;
-                const n = policy.visitCounts[key];
-                policy.avgRewards[key] += (feedback.reward - policy.avgRewards[key]) / n;
-                break;
-            }
-            default:
-                // Default: simple average
-                policy.visitCounts[key]++;
-                const n = policy.visitCounts[key];
-                policy.avgRewards[key] += (feedback.reward - policy.avgRewards[key]) / n;
-        }
-    }
-    /**
-     * Train batch of experiences
-     */
-    async trainBatch(session, batch, learningRate) {
-        let totalLoss = 0;
-        const policy = this.getLatestPolicy(session.id);
-        for (const exp of batch) {
-            const key = `${exp.state}|${exp.action}`;
-            // Initialize if needed
-            if (!policy.qValues[key]) {
-                policy.qValues[key] = 0;
-            }
-            // Calculate target based on algorithm
-            let target = exp.reward;
-            if (exp.next_state && session.sessionType !== 'policy-gradient') {
-                const nextActions = Object.keys(policy.qValues).filter(k => k.startsWith(exp.next_state + '|'));
-                const maxNextQ = Math.max(...nextActions.map(k => policy.qValues[k]), 0);
-                target += session.config.discountFactor * maxNextQ;
-            }
-            // Calculate loss (TD error)
-            const prediction = policy.qValues[key];
-            const loss = Math.pow(target - prediction, 2);
-            totalLoss += loss;
-            // Update Q-value
-            policy.qValues[key] += learningRate * (target - prediction);
-            // Update counts
-            policy.visitCounts[key] = (policy.visitCounts[key] || 0) + 1;
-        }
-        return totalLoss / batch.length;
-    }
-    /**
-     * Save policy to database
-     */
-    async savePolicy(sessionId) {
-        const policy = this.getLatestPolicy(sessionId);
-        const currentVersion = this.db.prepare(`
-      SELECT MAX(version) as max_version FROM learning_policies
-      WHERE session_id = ?
-    `).get(sessionId);
-        const version = (currentVersion?.max_version || 0) + 1;
-        this.db.prepare(`
-      INSERT INTO learning_policies (
-        session_id, state_action_pairs, q_values, visit_counts, avg_rewards, version
-      ) VALUES (?, ?, ?, ?, ?, ?)
-    `).run(sessionId, JSON.stringify(policy.stateActionPairs || {}), JSON.stringify(policy.qValues || {}), JSON.stringify(policy.visitCounts || {}), JSON.stringify(policy.avgRewards || {}), version);
-    }
-    /**
-     * Calculate convergence rate
-     */
-    calculateConvergenceRate(sessionId) {
-        // Get policy versions
-        const versions = this.db.prepare(`
-      SELECT version, q_values FROM learning_policies
-      WHERE session_id = ?
-      ORDER BY version DESC
-      LIMIT 10
-    `).all(sessionId);
-        if (versions.length < 2)
-            return 0;
-        // Calculate rate of change between versions
-        let totalChange = 0;
-        for (let i = 0; i < versions.length - 1; i++) {
-            const qValues1 = JSON.parse(versions[i].q_values);
-            const qValues2 = JSON.parse(versions[i + 1].q_values);
-            // Calculate mean absolute difference
-            const keys = new Set([...Object.keys(qValues1), ...Object.keys(qValues2)]);
-            let diff = 0;
-            keys.forEach(key => {
-                diff += Math.abs((qValues1[key] || 0) - (qValues2[key] || 0));
-            });
-            totalChange += diff / keys.size;
-        }
-        // Lower change = higher convergence
-        const avgChange = totalChange / (versions.length - 1);
-        return Math.max(0, 1 - avgChange);
-    }
-    // Algorithm-specific scoring methods
-    calculateTransformerScore(state, action, policy) {
-        const key = `${state}|${action}`;
-        return policy.avgRewards[key] || 0;
-    }
-    calculateUCB1(state, action, policy) {
-        const key = `${state}|${action}`;
-        const q = policy.avgRewards[key] || 0;
-        const n = policy.visitCounts[key] || 1;
-        const N = Object.values(policy.visitCounts).reduce((sum, val) => sum + val, 0) || 1;
-        const exploration = Math.sqrt(2 * Math.log(N) / n);
-        return q + exploration;
-    }
-    calculateModelScore(state, action, policy) {
-        const key = `${state}|${action}`;
-        return policy.avgRewards[key] || 0;
-    }
-    shuffleArray(array) {
-        const result = [...array];
-        for (let i = result.length - 1; i > 0; i--) {
-            const j = Math.floor(Math.random() * (i + 1));
-            [result[i], result[j]] = [result[j], result[i]];
-        }
-        return result;
-    }
-    // ============================================================================
-    // Extended Learning System Methods (Tools 6-10)
-    // ============================================================================
-    /**
-     * Get learning performance metrics with time windows and trends
-     */
-    async getMetrics(options) {
-        const { sessionId, timeWindowDays = 7, includeTrends = true, groupBy = 'task' } = options;
-        const cutoffTimestamp = Date.now() - (timeWindowDays * 24 * 60 * 60 * 1000);
-        // Base query filters
-        let whereClause = 'WHERE timestamp >= ?';
-        const params = [cutoffTimestamp];
-        if (sessionId) {
-            whereClause += ' AND session_id = ?';
-            params.push(sessionId);
-        }
-        // Overall metrics
-        const overallStats = this.db.prepare(`
-      SELECT
-        COUNT(*) as total_episodes,
-        AVG(reward) as avg_reward,
-        AVG(CASE WHEN success = 1 THEN 1.0 ELSE 0.0 END) as success_rate,
-        MIN(reward) as min_reward,
-        MAX(reward) as max_reward,
-        AVG(CASE WHEN metadata IS NOT NULL THEN json_extract(metadata, '$.latency_ms') END) as avg_latency_ms
-      FROM learning_experiences
-      ${whereClause}
-    `).get(...params);
-        // Group by metrics
-        let groupedMetrics = [];
-        if (groupBy === 'task') {
-            groupedMetrics = this.db.prepare(`
-        SELECT
-          state as group_key,
-          COUNT(*) as count,
-          AVG(reward) as avg_reward,
-          AVG(CASE WHEN success = 1 THEN 1.0 ELSE 0.0 END) as success_rate
-        FROM learning_experiences
-        ${whereClause}
-        GROUP BY state
-        ORDER BY count DESC
-        LIMIT 20
-      `).all(...params);
-        }
-        else if (groupBy === 'session') {
-            groupedMetrics = this.db.prepare(`
-        SELECT
-          session_id as group_key,
-          COUNT(*) as count,
-          AVG(reward) as avg_reward,
-          AVG(CASE WHEN success = 1 THEN 1.0 ELSE 0.0 END) as success_rate
-        FROM learning_experiences
-        ${whereClause}
-        GROUP BY session_id
-        ORDER BY count DESC
-        LIMIT 20
-      `).all(...params);
-        }
-        // Trend analysis
-        let trends = [];
-        if (includeTrends) {
-            trends = this.db.prepare(`
-        SELECT
-          DATE(timestamp / 1000, 'unixepoch') as date,
-          COUNT(*) as count,
-          AVG(reward) as avg_reward,
-          AVG(CASE WHEN success = 1 THEN 1.0 ELSE 0.0 END) as success_rate
-        FROM learning_experiences
-        ${whereClause}
-        GROUP BY date
-        ORDER BY date ASC
-      `).all(...params);
-        }
-        // Policy improvement metrics
-        const policyVersions = sessionId ? this.db.prepare(`
-      SELECT
-        version,
-        created_at,
-        q_values
-      FROM learning_policies
-      WHERE session_id = ?
-      ORDER BY version ASC
-    `).all(sessionId) : [];
-        let policyImprovement = 0;
-        if (policyVersions.length >= 2) {
-            const firstPolicy = JSON.parse(policyVersions[0].q_values);
-            const latestPolicy = JSON.parse(policyVersions[policyVersions.length - 1].q_values);
-            const commonKeys = Object.keys(firstPolicy).filter(k => latestPolicy[k] !== undefined);
-            if (commonKeys.length > 0) {
-                const avgFirst = commonKeys.reduce((sum, k) => sum + firstPolicy[k], 0) / commonKeys.length;
-                const avgLatest = commonKeys.reduce((sum, k) => sum + latestPolicy[k], 0) / commonKeys.length;
-                policyImprovement = avgLatest - avgFirst;
-            }
-        }
-        return {
-            timeWindow: {
-                days: timeWindowDays,
-                startTimestamp: cutoffTimestamp,
-                endTimestamp: Date.now(),
-            },
-            overall: {
-                totalEpisodes: overallStats.total_episodes || 0,
-                avgReward: overallStats.avg_reward || 0,
-                successRate: overallStats.success_rate || 0,
-                minReward: overallStats.min_reward || 0,
-                maxReward: overallStats.max_reward || 0,
-                avgLatencyMs: overallStats.avg_latency_ms || 0,
-            },
-            groupedMetrics: groupedMetrics.map(g => ({
-                key: g.group_key,
-                count: g.count,
-                avgReward: g.avg_reward,
-                successRate: g.success_rate,
-            })),
-            trends: trends.map(t => ({
-                date: t.date,
-                count: t.count,
-                avgReward: t.avg_reward,
-                successRate: t.success_rate,
-            })),
-            policyImprovement: {
-                versions: policyVersions.length,
-                qValueImprovement: policyImprovement,
-            },
-        };
-    }
-    /**
-     * Transfer learning between sessions or tasks
-     */
-    async transferLearning(options) {
-        const { sourceSession, targetSession, sourceTask, targetTask, minSimilarity = 0.7, transferType = 'all', maxTransfers = 10, } = options;
-        if (!sourceSession && !sourceTask) {
-            throw new Error('Must specify either sourceSession or sourceTask');
-        }
-        if (!targetSession && !targetTask) {
-            throw new Error('Must specify either targetSession or targetTask');
-        }
-        const transferred = {
-            episodes: 0,
-            skills: 0,
-            causalEdges: 0,
-            details: [],
-        };
-        // Transfer episodes
-        if (transferType === 'episodes' || transferType === 'all') {
-            const sourceEpisodes = this.db.prepare(`
-        SELECT * FROM learning_experiences
-        WHERE ${sourceSession ? 'session_id = ?' : 'state LIKE ?'}
-        ORDER BY reward DESC
-        LIMIT ?
-      `).all(sourceSession || `%${sourceTask}%`, maxTransfers);
-            for (const episode of sourceEpisodes) {
-                // Check similarity if transferring between tasks
-                if (sourceTask && targetTask) {
-                    const sourceEmbed = await this.embedder.embed(episode.state);
-                    const targetEmbed = await this.embedder.embed(targetTask);
-                    const similarity = cosineSimilarity(sourceEmbed, targetEmbed);
-                    if (similarity < minSimilarity) {
-                        continue;
-                    }
-                    transferred.details.push({
-                        type: 'episode',
-                        id: episode.id,
-                        similarity,
-                    });
-                }
-                // Insert transferred episode
-                this.db.prepare(`
-          INSERT INTO learning_experiences (
-            session_id, state, action, reward, next_state, success, timestamp, metadata
-          ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
-        `).run(targetSession || episode.session_id, targetTask || episode.state, episode.action, episode.reward, episode.next_state, episode.success, Date.now(), JSON.stringify({ transferred_from: episode.id }));
-                transferred.episodes++;
-            }
-        }
-        // Transfer policy/Q-values
-        if (sourceSession && targetSession && (transferType === 'all' || transferType === 'skills')) {
-            const sourcePolicy = this.getLatestPolicy(sourceSession);
-            const targetPolicy = this.getLatestPolicy(targetSession);
-            // Transfer Q-values with similarity weighting
-            let transferredQValues = 0;
-            for (const [key, qValue] of Object.entries(sourcePolicy.qValues)) {
-                const [state, action] = key.split('|');
-                // Check if target has similar state
-                if (targetTask) {
-                    const stateEmbed = await this.embedder.embed(state);
-                    const targetEmbed = await this.embedder.embed(targetTask);
-                    const similarity = cosineSimilarity(stateEmbed, targetEmbed);
-                    if (similarity >= minSimilarity) {
-                        const targetKey = `${targetTask}|${action}`;
-                        targetPolicy.qValues[targetKey] = qValue;
-                        transferredQValues++;
-                    }
-                }
-            }
-            if (transferredQValues > 0) {
-                // Save updated target policy
-                const version = this.db.prepare(`
-          SELECT MAX(version) as max_version FROM learning_policies WHERE session_id = ?
-        `).get(targetSession)?.max_version || 0;
-                this.db.prepare(`
-          INSERT INTO learning_policies (
-            session_id, state_action_pairs, q_values, visit_counts, avg_rewards, version
-          ) VALUES (?, ?, ?, ?, ?, ?)
-        `).run(targetSession, JSON.stringify(targetPolicy.stateActionPairs || {}), JSON.stringify(targetPolicy.qValues || {}), JSON.stringify(targetPolicy.visitCounts || {}), JSON.stringify(targetPolicy.avgRewards || {}), version + 1);
-                transferred.skills = transferredQValues;
-            }
-        }
-        return {
-            success: true,
-            transferred,
-            source: { session: sourceSession, task: sourceTask },
-            target: { session: targetSession, task: targetTask },
-            minSimilarity,
-            transferType,
-        };
-    }
-    /**
-     * Explain action recommendations with XAI (Explainable AI)
-     */
-    async explainAction(options) {
-        const { query, k = 5, explainDepth = 'detailed', includeConfidence = true, includeEvidence = true, includeCausal = true, } = options;
-        // Get query embedding
-        const queryEmbed = await this.embedder.embed(query);
-        // Find similar past experiences
-        const allExperiences = this.db.prepare(`
-      SELECT * FROM learning_experiences
-      ORDER BY timestamp DESC
-      LIMIT 100
-    `).all();
-        const rankedExperiences = [];
-        for (const exp of allExperiences) {
-            const stateEmbed = await this.getStateEmbedding(exp.session_id, exp.state);
-            const similarity = cosineSimilarity(queryEmbed, stateEmbed);
-            rankedExperiences.push({
-                ...exp,
-                similarity,
-            });
-        }
-        rankedExperiences.sort((a, b) => b.similarity - a.similarity);
-        const topExperiences = rankedExperiences.slice(0, k);
-        // Aggregate recommendations
-        const actionScores = {};
-        for (const exp of topExperiences) {
-            if (!actionScores[exp.action]) {
-                actionScores[exp.action] = {
-                    count: 0,
-                    avgReward: 0,
-                    successRate: 0,
-                    evidence: [],
-                };
-            }
-            const score = actionScores[exp.action];
-            score.count++;
-            score.avgReward += exp.reward;
-            score.successRate += exp.success ? 1 : 0;
-            if (includeEvidence) {
-                score.evidence.push({
-                    episodeId: exp.id,
-                    state: exp.state,
-                    reward: exp.reward,
-                    success: exp.success,
-                    similarity: exp.similarity,
-                    timestamp: exp.timestamp,
-                });
-            }
-        }
-        // Calculate final scores
-        const recommendations = Object.entries(actionScores).map(([action, data]) => ({
-            action,
-            confidence: data.count / topExperiences.length,
-            avgReward: data.avgReward / data.count,
-            successRate: data.successRate / data.count,
-            supportingExamples: data.count,
-            evidence: includeEvidence ? data.evidence.slice(0, 3) : undefined,
-        }));
-        recommendations.sort((a, b) => b.confidence - a.confidence);
-        // Causal reasoning chains (if enabled)
-        let causalChains = [];
-        if (includeCausal) {
-            causalChains = this.db.prepare(`
-        SELECT * FROM causal_edges
-        ORDER BY uplift DESC
-        LIMIT 5
-      `).all();
-        }
-        const response = {
-            query,
-            recommendations: recommendations.slice(0, k),
-            explainDepth,
-        };
-        if (explainDepth === 'detailed' || explainDepth === 'full') {
-            response.reasoning = {
-                similarExperiencesFound: topExperiences.length,
-                avgSimilarity: topExperiences.reduce((sum, e) => sum + e.similarity, 0) / topExperiences.length,
-                uniqueActions: recommendations.length,
-            };
-        }
-        if (explainDepth === 'full') {
-            response.causalChains = causalChains;
-            response.allEvidence = topExperiences;
-        }
-        return response;
-    }
-    /**
-     * Record tool execution as experience for offline learning
-     */
-    async recordExperience(options) {
-        const { sessionId, toolName, action, stateBefore, stateAfter, outcome, reward, success, latencyMs, metadata, } = options;
-        // Construct state representation
-        const state = `tool:${toolName}|${action}`;
-        const nextState = stateAfter ? JSON.stringify(stateAfter) : undefined;
-        // Store as learning experience
-        const result = this.db.prepare(`
-      INSERT INTO learning_experiences (
-        session_id, state, action, reward, next_state, success, timestamp, metadata
-      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
-    `).run(sessionId, state, outcome, reward, nextState, success ? 1 : 0, Date.now(), JSON.stringify({
-            toolName,
-            action,
-            stateBefore,
-            stateAfter,
-            latencyMs,
-            ...metadata,
-        }));
-        console.log(`✅ Experience recorded: tool=${toolName}, reward=${reward}, success=${success}`);
-        return result.lastInsertRowid;
-    }
-    /**
-     * Calculate reward signal with shaping based on multiple factors
-     */
-    calculateReward(options) {
-        const { episodeId, success, targetAchieved = true, efficiencyScore = 0.5, qualityScore = 0.5, timeTakenMs, expectedTimeMs, includeCausal = true, rewardFunction = 'standard', } = options;
-        let reward = 0;
-        switch (rewardFunction) {
-            case 'sparse':
-                // Sparse: Only reward on success
-                reward = success && targetAchieved ? 1.0 : 0.0;
-                break;
-            case 'dense':
-                // Dense: Partial rewards for progress
-                reward = success ? 1.0 : 0.0;
-                reward += targetAchieved ? 0.5 : 0.0;
-                reward += qualityScore * 0.3;
-                reward += efficiencyScore * 0.2;
-                break;
-            case 'shaped':
-                // Shaped: Reward shaping with time efficiency
-                reward = success ? 1.0 : -0.5;
-                if (targetAchieved)
-                    reward += 0.3;
-                // Time efficiency bonus
-                if (timeTakenMs && expectedTimeMs) {
-                    const timeRatio = timeTakenMs / expectedTimeMs;
-                    const timeBonus = Math.max(0, 1 - timeRatio) * 0.2;
-                    reward += timeBonus;
-                }
-                // Quality and efficiency
-                reward += (qualityScore - 0.5) * 0.3;
-                reward += (efficiencyScore - 0.5) * 0.2;
-                break;
-            case 'standard':
-            default:
-                // Standard: Weighted combination
-                reward = success ? 0.6 : 0.0;
-                reward += targetAchieved ? 0.2 : 0.0;
-                reward += qualityScore * 0.1;
-                reward += efficiencyScore * 0.1;
-                break;
-        }
-        // Causal impact adjustment
-        if (includeCausal && episodeId) {
-            const causalEdges = this.db.prepare(`
-        SELECT AVG(uplift) as avg_uplift
-        FROM causal_edges
-        WHERE from_memory_id = ? OR to_memory_id = ?
-      `).get(episodeId, episodeId);
-            if (causalEdges?.avg_uplift) {
-                reward += causalEdges.avg_uplift * 0.1; // 10% weight for causal impact
-            }
-        }
-        // Normalize to [0, 1] range
-        return Math.max(0, Math.min(1, reward));
-    }
-}
-//# sourceMappingURL=LearningSystem.js.map