agentic-flow 3.0.0-alpha.2 → 3.0.0

package/dist/services/rl-training-service.d.ts

@@ -1,161 +0,0 @@
-/**
- * RL Training Service - Reinforcement Learning Training Orchestration
- *
- * ADR-065 Phase P1-2: SONA RL Loop Implementation
- *
- * Provides high-level RL training coordination using SonaTrajectoryService
- * for policy gradients, value functions, experience replay, multi-agent RL,
- * and transfer learning.
- *
- * Features:
- * - PPO (Proximal Policy Optimization)
- * - A3C (Asynchronous Advantage Actor-Critic)
- * - Q-Learning with experience replay
- * - Multi-agent coordination
- * - Transfer learning between tasks
- * - Continuous learning loop
- *
- * Usage:
- *   const trainer = new RLTrainingService();
- *   await trainer.initialize();
- *
- *   // Train on episodes
- *   await trainer.trainEpisodes(episodes, { algorithm: 'ppo' });
- *
- *   // Continuous learning
- *   await trainer.learnFromExecution(state, action, reward, nextState);
- */
-import { SonaTrajectoryService, type StoredTrajectory } from 'agentdb';
-export interface PolicyGradientConfig {
-    learningRate?: number;
-    gamma?: number;
-    baseline?: 'none' | 'avg' | 'learned';
-}
-export interface ValueFunctionConfig {
-    learningRate?: number;
-    gamma?: number;
-    lambda?: number;
-}
-export interface ExperienceReplayConfig {
-    bufferSize?: number;
-    batchSize?: number;
-    minSize?: number;
-}
-export interface RLMetrics {
-    avgReward?: number;
-    avgLoss?: number;
-    episodeCount?: number;
-}
-export type RLAlgorithm = 'reinforce' | 'ppo' | 'a3c' | 'q-learning';
-export interface TrainingConfig {
-    algorithm: RLAlgorithm;
-    epochs: number;
-    batchSize: number;
-    learningRate: number;
-    gamma: number;
-    epsilon: number;
-    ppoClipRatio?: number;
-    entropyCoeff?: number;
-    valueCoeff?: number;
-}
-export interface TrainingResult {
-    algorithm: RLAlgorithm;
-    epochs: number;
-    finalLoss: number;
-    avgReward: number;
-    improvementRate: number;
-    convergenceIteration: number;
-    learningCurve: Array<{
-        iteration: number;
-        reward: number;
-        loss: number;
-    }>;
-}
-export interface MultiAgentConfig {
-    agentTypes: string[];
-    coordinationStrategy: 'centralized' | 'decentralized' | 'hybrid';
-    rewardSharing: 'equal' | 'contribution' | 'competitive';
-}
-export interface TransferConfig {
-    sourceTask: string;
-    targetTask: string;
-    transferRatio: number;
-    finetuneEpochs: number;
-}
-export declare class RLTrainingService {
-    private sona;
-    private initialized;
-    private trainingHistory;
-    constructor(sonaService?: SonaTrajectoryService);
-    /**
-     * Initialize the RL training service
-     */
-    initialize(): Promise<boolean>;
-    /**
-     * Train on episodes using specified algorithm
-     *
-     * @param episodes - Training episodes (trajectories)
-     * @param config - Training configuration
-     * @returns Training results with learning curve
-     */
-    trainEpisodes(episodes: StoredTrajectory[], config?: Partial<TrainingConfig>): Promise<TrainingResult>;
-    /**
-     * PPO (Proximal Policy Optimization) training
-     */
-    private trainPPO;
-    /**
-     * A3C (Asynchronous Advantage Actor-Critic) training
-     */
-    private trainA3C;
-    /**
-     * Q-Learning with experience replay
-     */
-    private trainQLearning;
-    /**
-     * Learn from a single execution step (continuous learning)
-     *
-     * @param state - Current state
-     * @param action - Action taken
-     * @param reward - Reward received
-     * @param nextState - Resulting state
-     * @returns Updated value estimate
-     */
-    learnFromExecution(state: any, action: string, reward: number, nextState: any): Promise<number>;
-    /**
-     * Multi-agent reinforcement learning
-     *
-     * @param agentStates - Map of agent IDs to states
-     * @param agentActions - Map of agent IDs to actions
-     * @param jointReward - Shared reward
-     * @param config - Multi-agent configuration
-     * @returns Individual rewards for each agent
-     */
-    multiAgentTrain(agentStates: Map<string, any>, agentActions: Map<string, string>, jointReward: number, config?: Partial<MultiAgentConfig>): Promise<Map<string, number>>;
-    /**
-     * Transfer learning from source task to target task
-     *
-     * @param config - Transfer learning configuration
-     * @returns Success indicator and training result
-     */
-    transferLearn(config: TransferConfig): Promise<{
-        success: boolean;
-        result?: TrainingResult;
-    }>;
-    /**
-     * Get training history
-     */
-    getTrainingHistory(): TrainingResult[];
-    /**
-     * Get current RL metrics
-     */
-    getMetrics(): RLMetrics;
-    /**
-     * Reset training state
-     */
-    reset(): void;
-    /**
-     * Get the underlying SONA service
-     */
-    getSonaService(): SonaTrajectoryService;
-}
-//# sourceMappingURL=rl-training-service.d.ts.map

package/dist/services/rl-training-service.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"rl-training-service.d.ts","sourceRoot":"","sources":["../../src/services/rl-training-service.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAEH,OAAO,EAAE,qBAAqB,EAAE,KAAK,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAGvE,MAAM,WAAW,oBAAoB;IACnC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,GAAG,KAAK,GAAG,SAAS,CAAC;CACvC;AAED,MAAM,WAAW,mBAAmB;IAClC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,sBAAsB;IACrC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,SAAS;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,MAAM,WAAW,GAAG,WAAW,GAAG,KAAK,GAAG,KAAK,GAAG,YAAY,CAAC;AAErE,MAAM,WAAW,cAAc;IAC7B,SAAS,EAAE,WAAW,CAAC;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,cAAc;IAC7B,SAAS,EAAE,WAAW,CAAC;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,eAAe,EAAE,MAAM,CAAC;IACxB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,aAAa,EAAE,KAAK,CAAC;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CAC3E;AAED,MAAM,WAAW,gBAAgB;IAC/B,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,oBAAoB,EAAE,aAAa,GAAG,eAAe,GAAG,QAAQ,CAAC;IACjE,aAAa,EAAE,OAAO,GAAG,cAAc,GAAG,aAAa,CAAC;CACzD;AAED,MAAM,WAAW,cAAc;IAC7B,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;CACxB;AAED,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,IAAI,CAAwB;IACpC,OAAO,CAAC,WAAW,CAAkB;IACrC,OAAO,CAAC,eAAe,CAAwB;gBAEnC,WAAW,CAAC,EAAE,qBAAqB;IAI/C;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC;IAUpC;;;;;;OAMG;IACG,aAAa,CAAC,QAAQ,EAAE,gBAAgB,EAAE,EAAE,MAAM,GAAE,OAAO,CAAC,cAAc,CAAM,GAAG,OAAO,CAAC,cAAc,CAAC;IAsHhH;;OAEG;YACW,QAAQ;IAmBtB;;OAEG;YACW,QAAQ;IAkCtB;;OAEG;YACW,cAAc;IAgC5B;;;;;;;;OAQG;IACG,kBAAkB,CAAC,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,GAAG,OAAO,CAAC,MAAM,CAAC;IAQrG;;;;;;;;OAQG;IACG,eAAe,CACnB,WAAW,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,EAC7B,YAAY,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,EACjC,WAAW,EAAE,MAAM,EACnB,MAAM,CAAC,EAAE,OAAO,CAAC,gBAAgB,CAAC,GACjC,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAmB/B;;;;;OAKG;IACG,aAAa,CAAC,MAAM,EAAE,cAAc,GAAG,OAAO,CAAC;QAAE,OAAO,EAAE,OAAO,CAAC;QAAC,MAAM,CAAC,EAAE,cAAc,CAAA;KAAE,CAAC;IAiCnG;;OAEG;IACH,kBAAkB,IAAI,cAAc,EAAE;IAItC;;OAEG;IACH,UAAU,IAAI,SAAS;IAIvB;;OAEG;IACH,KAAK,IAAI,IAAI;IAKb;;OAEG;IACH,cAAc,IAAI,qBAAqB;CAGxC"}

package/dist/services/rl-training-service.js

@@ -1,325 +0,0 @@
-/**
- * RL Training Service - Reinforcement Learning Training Orchestration
- *
- * ADR-065 Phase P1-2: SONA RL Loop Implementation
- *
- * Provides high-level RL training coordination using SonaTrajectoryService
- * for policy gradients, value functions, experience replay, multi-agent RL,
- * and transfer learning.
- *
- * Features:
- * - PPO (Proximal Policy Optimization)
- * - A3C (Asynchronous Advantage Actor-Critic)
- * - Q-Learning with experience replay
- * - Multi-agent coordination
- * - Transfer learning between tasks
- * - Continuous learning loop
- *
- * Usage:
- *   const trainer = new RLTrainingService();
- *   await trainer.initialize();
- *
- *   // Train on episodes
- *   await trainer.trainEpisodes(episodes, { algorithm: 'ppo' });
- *
- *   // Continuous learning
- *   await trainer.learnFromExecution(state, action, reward, nextState);
- */
-import { SonaTrajectoryService } from 'agentdb';
-export class RLTrainingService {
-    sona;
-    initialized = false;
-    trainingHistory = [];
-    constructor(sonaService) {
-        this.sona = sonaService || new SonaTrajectoryService();
-    }
-    /**
-     * Initialize the RL training service
-     */
-    async initialize() {
-        if (this.initialized)
-            return true;
-        await this.sona.initialize();
-        this.initialized = true;
-        console.log('[RLTrainingService] Initialized with engine:', this.sona.getEngineType());
-        return true;
-    }
-    /**
-     * Train on episodes using specified algorithm
-     *
-     * @param episodes - Training episodes (trajectories)
-     * @param config - Training configuration
-     * @returns Training results with learning curve
-     */
-    async trainEpisodes(episodes, config = {}) {
-        if (!this.initialized) {
-            await this.initialize();
-        }
-        const fullConfig = {
-            algorithm: config.algorithm || 'reinforce',
-            epochs: config.epochs || 100,
-            batchSize: config.batchSize || 32,
-            learningRate: config.learningRate || 0.001,
-            gamma: config.gamma || 0.99,
-            epsilon: config.epsilon || 0.1,
-            ppoClipRatio: config.ppoClipRatio || 0.2,
-            entropyCoeff: config.entropyCoeff || 0.01,
-            valueCoeff: config.valueCoeff || 0.5
-        };
-        // Configure SONA with training parameters
-        this.sona.configureRL({
-            policy: {
-                learningRate: fullConfig.learningRate,
-                gamma: fullConfig.gamma,
-                epsilon: fullConfig.epsilon,
-                entropyCoeff: fullConfig.entropyCoeff
-            },
-            value: {
-                learningRate: fullConfig.learningRate,
-                gamma: fullConfig.gamma,
-                lambda: 0.95
-            },
-            replay: {
-                bufferSize: 10000,
-                batchSize: fullConfig.batchSize,
-                priorityAlpha: 0.6,
-                priorityBeta: 0.4
-            }
-        });
-        const learningCurve = [];
-        let bestReward = -Infinity;
-        let convergenceIteration = fullConfig.epochs;
-        let improvementCount = 0;
-        console.log(`[RLTrainingService] Training ${episodes.length} episodes with ${fullConfig.algorithm}`);
-        for (let epoch = 0; epoch < fullConfig.epochs; epoch++) {
-            let loss = 0;
-            // Algorithm-specific training
-            switch (fullConfig.algorithm) {
-                case 'ppo':
-                    loss = await this.trainPPO(episodes, fullConfig);
-                    break;
-                case 'a3c':
-                    loss = await this.trainA3C(episodes, fullConfig);
-                    break;
-                case 'q-learning':
-                    loss = await this.trainQLearning(episodes, fullConfig);
-                    break;
-                case 'reinforce':
-                default:
-                    loss = await this.sona.trainPolicy(episodes, {
-                        learningRate: fullConfig.learningRate,
-                        gamma: fullConfig.gamma,
-                        epsilon: fullConfig.epsilon,
-                        entropyCoeff: fullConfig.entropyCoeff
-                    });
-                    break;
-            }
-            const metrics = this.sona.getRLMetrics();
-            // Track learning curve
-            learningCurve.push({
-                iteration: epoch,
-                reward: metrics.avgReward,
-                loss
-            });
-            // Check for improvement
-            if (metrics.avgReward > bestReward) {
-                bestReward = metrics.avgReward;
-                improvementCount++;
-                // Check for convergence (no improvement for 20 epochs)
-                if (epoch - convergenceIteration > 20) {
-                    convergenceIteration = epoch;
-                }
-            }
-            // Log progress every 10 epochs
-            if (epoch % 10 === 0) {
-                console.log(`[RLTrainingService] Epoch ${epoch}: Loss=${loss.toFixed(4)}, AvgReward=${metrics.avgReward.toFixed(4)}, Epsilon=${metrics.epsilon.toFixed(4)}`);
-            }
-        }
-        const finalMetrics = this.sona.getRLMetrics();
-        const initialReward = learningCurve[0]?.reward || 0;
-        const finalReward = finalMetrics.avgReward;
-        const improvementRate = initialReward === 0 ? 0 : ((finalReward - initialReward) / Math.abs(initialReward)) * 100;
-        const result = {
-            algorithm: fullConfig.algorithm,
-            epochs: fullConfig.epochs,
-            finalLoss: finalMetrics.loss,
-            avgReward: finalReward,
-            improvementRate,
-            convergenceIteration,
-            learningCurve
-        };
-        this.trainingHistory.push(result);
-        console.log(`[RLTrainingService] Training complete: ${improvementRate.toFixed(2)}% improvement`);
-        return result;
-    }
-    /**
-     * PPO (Proximal Policy Optimization) training
-     */
-    async trainPPO(episodes, config) {
-        let totalLoss = 0;
-        // PPO uses multiple epochs on the same data
-        for (let ppoEpoch = 0; ppoEpoch < 4; ppoEpoch++) {
-            // Standard policy gradient with clipping
-            const loss = await this.sona.trainPolicy(episodes, {
-                learningRate: config.learningRate * (1 - ppoEpoch * 0.1), // decay learning rate
-                gamma: config.gamma,
-                epsilon: config.epsilon,
-                entropyCoeff: config.entropyCoeff
-            });
-            totalLoss += loss;
-        }
-        return totalLoss / 4;
-    }
-    /**
-     * A3C (Asynchronous Advantage Actor-Critic) training
-     */
-    async trainA3C(episodes, config) {
-        let totalLoss = 0;
-        let count = 0;
-        // Train actor (policy) and critic (value) separately
-        for (const episode of episodes) {
-            // Update value function
-            for (let i = 0; i < episode.steps.length - 1; i++) {
-                const step = episode.steps[i];
-                const nextStep = episode.steps[i + 1];
-                await this.sona.estimateValue(step.state, step.reward, nextStep.state, {
-                    learningRate: config.learningRate * (config.valueCoeff || 0.5),
-                    gamma: config.gamma,
-                    lambda: 0.95
-                });
-            }
-            count++;
-        }
-        // Update policy
-        const policyLoss = await this.sona.trainPolicy(episodes, {
-            learningRate: config.learningRate,
-            gamma: config.gamma,
-            epsilon: config.epsilon,
-            entropyCoeff: config.entropyCoeff
-        });
-        totalLoss += policyLoss;
-        return totalLoss / Math.max(count, 1);
-    }
-    /**
-     * Q-Learning with experience replay
-     */
-    async trainQLearning(episodes, config) {
-        // Add all experiences to replay buffer
-        for (const episode of episodes) {
-            for (let i = 0; i < episode.steps.length - 1; i++) {
-                const step = episode.steps[i];
-                const nextStep = episode.steps[i + 1];
-                this.sona.addExperience(step.state, step.action, step.reward, nextStep.state);
-            }
-        }
-        // Sample and train
-        let totalLoss = 0;
-        const numBatches = Math.floor(episodes.length / config.batchSize);
-        for (let i = 0; i < numBatches; i++) {
-            const batch = this.sona.sampleExperience(config.batchSize);
-            for (const exp of batch) {
-                const value = await this.sona.estimateValue(exp.state, exp.reward, exp.nextState, {
-                    learningRate: config.learningRate,
-                    gamma: config.gamma,
-                    lambda: 0.95
-                });
-                totalLoss += Math.abs(value - exp.reward);
-            }
-        }
-        return totalLoss / Math.max(numBatches, 1);
-    }
-    /**
-     * Learn from a single execution step (continuous learning)
-     *
-     * @param state - Current state
-     * @param action - Action taken
-     * @param reward - Reward received
-     * @param nextState - Resulting state
-     * @returns Updated value estimate
-     */
-    async learnFromExecution(state, action, reward, nextState) {
-        if (!this.initialized) {
-            await this.initialize();
-        }
-        return await this.sona.continuousLearn(state, action, reward, nextState);
-    }
-    /**
-     * Multi-agent reinforcement learning
-     *
-     * @param agentStates - Map of agent IDs to states
-     * @param agentActions - Map of agent IDs to actions
-     * @param jointReward - Shared reward
-     * @param config - Multi-agent configuration
-     * @returns Individual rewards for each agent
-     */
-    async multiAgentTrain(agentStates, agentActions, jointReward, config) {
-        if (!this.initialized) {
-            await this.initialize();
-        }
-        const fullConfig = {
-            agentTypes: config?.agentTypes || Array.from(agentStates.keys()),
-            coordinationStrategy: config?.coordinationStrategy || 'hybrid',
-            rewardSharing: config?.rewardSharing || 'contribution'
-        };
-        console.log(`[RLTrainingService] Multi-agent training: ${fullConfig.agentTypes.length} agents, ${fullConfig.rewardSharing} reward sharing`);
-        // Use SONA's multi-agent learning
-        const individualRewards = await this.sona.multiAgentLearn(agentStates, agentActions, jointReward);
-        return individualRewards;
-    }
-    /**
-     * Transfer learning from source task to target task
-     *
-     * @param config - Transfer learning configuration
-     * @returns Success indicator and training result
-     */
-    async transferLearn(config) {
-        if (!this.initialized) {
-            await this.initialize();
-        }
-        console.log(`[RLTrainingService] Transfer learning: ${config.sourceTask} → ${config.targetTask} (ratio: ${config.transferRatio})`);
-        // Perform transfer
-        const success = await this.sona.transferLearning(config.sourceTask, config.targetTask, config.transferRatio);
-        if (!success) {
-            console.log('[RLTrainingService] Transfer failed: no source patterns available');
-            return { success: false };
-        }
-        // Fine-tune on target task if we have data
-        const targetPatterns = await this.sona.getPatterns(config.targetTask);
-        if (targetPatterns.length > 0 && config.finetuneEpochs > 0) {
-            console.log(`[RLTrainingService] Fine-tuning on ${targetPatterns.length} target patterns`);
-            const result = await this.trainEpisodes(targetPatterns, {
-                algorithm: 'reinforce',
-                epochs: config.finetuneEpochs,
-                learningRate: 0.0005 // lower learning rate for fine-tuning
-            });
-            return { success: true, result };
-        }
-        return { success: true };
-    }
-    /**
-     * Get training history
-     */
-    getTrainingHistory() {
-        return [...this.trainingHistory];
-    }
-    /**
-     * Get current RL metrics
-     */
-    getMetrics() {
-        return this.sona.getRLMetrics();
-    }
-    /**
-     * Reset training state
-     */
-    reset() {
-        this.sona.resetRL();
-        this.trainingHistory = [];
-    }
-    /**
-     * Get the underlying SONA service
-     */
-    getSonaService() {
-        return this.sona;
-    }
-}
-//# sourceMappingURL=rl-training-service.js.map

package/dist/services/rl-training-service.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"rl-training-service.js","sourceRoot":"","sources":["../../src/services/rl-training-service.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAEH,OAAO,EAAE,qBAAqB,EAAyB,MAAM,SAAS,CAAC;AAgEvE,MAAM,OAAO,iBAAiB;IACpB,IAAI,CAAwB;IAC5B,WAAW,GAAY,KAAK,CAAC;IAC7B,eAAe,GAAqB,EAAE,CAAC;IAE/C,YAAY,WAAmC;QAC7C,IAAI,CAAC,IAAI,GAAG,WAAW,IAAI,IAAI,qBAAqB,EAAE,CAAC;IACzD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU;QACd,IAAI,IAAI,CAAC,WAAW;YAAE,OAAO,IAAI,CAAC;QAElC,MAAM,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;QAC7B,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QAExB,OAAO,CAAC,GAAG,CAAC,8CAA8C,EAAE,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC,CAAC;QACvF,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,aAAa,CAAC,QAA4B,EAAE,SAAkC,EAAE;QACpF,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACtB,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QAC1B,CAAC;QAED,MAAM,UAAU,GAAmB;YACjC,SAAS,EAAE,MAAM,CAAC,SAAS,IAAI,WAAW;YAC1C,MAAM,EAAE,MAAM,CAAC,MAAM,IAAI,GAAG;YAC5B,SAAS,EAAE,MAAM,CAAC,SAAS,IAAI,EAAE;YACjC,YAAY,EAAE,MAAM,CAAC,YAAY,IAAI,KAAK;YAC1C,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,IAAI;YAC3B,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,GAAG;YAC9B,YAAY,EAAE,MAAM,CAAC,YAAY,IAAI,GAAG;YACxC,YAAY,EAAE,MAAM,CAAC,YAAY,IAAI,IAAI;YACzC,UAAU,EAAE,MAAM,CAAC,UAAU,IAAI,GAAG;SACrC,CAAC;QAEF,0CAA0C;QAC1C,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC;YACpB,MAAM,EAAE;gBACN,YAAY,EAAE,UAAU,CAAC,YAAY;gBACrC,KAAK,EAAE,UAAU,CAAC,KAAK;gBACvB,OAAO,EAAE,UAAU,CAAC,OAAO;gBAC3B,YAAY,EAAE,UAAU,CAAC,YAAY;aACtC;YACD,KAAK,EAAE;gBACL,YAAY,EAAE,UAAU,CAAC,YAAY;gBACrC,KAAK,EAAE,UAAU,CAAC,KAAK;gBACvB,MAAM,EAAE,IAAI;aACb;YACD,MAAM,EAAE;gBACN,UAAU,EAAE,KAAK;gBACjB,SAAS,EAAE,UAAU,CAAC,SAAS;gBAC/B,aAAa,EAAE,GAAG;gBAClB,YAAY,EAAE,GAAG;aAClB;SACF,CAAC,CAAC;QAEH,MAAM,aAAa,GAA+D,EAAE,CAAC;QACrF,IAAI,UAAU,GAAG,CAAC,QAAQ,CAAC;QAC3B,IAAI,oBAAoB,GAAG,UAAU,CAAC,MAAM,CAAC;QAC7C,IAAI,gBAAgB,GAAG,CAAC,CAAC;QAEzB,OAAO,CAAC,GAAG,CAAC,gCAAgC,QAAQ,CAAC,MAAM,kBAAkB,UAAU,CAAC,SAAS,EAAE,CAAC,CAAC;QAErG,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,UAAU,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;YACvD,IAAI,IAAI,GAAG,CAAC,CAAC;YAEb,8BAA8B;YAC9B,QAAQ,UAAU,CAAC,SAAS,EAAE,CAAC;gBAC7B,KAAK,KAAK;oBACR,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;oBACjD,MAAM;gBACR,KAAK,KAAK;oBACR,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;oBACjD,MAAM;gBACR,KAAK,YAAY;oBACf,IAAI,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;oBACvD,MAAM;gBACR,KAAK,WAAW,CAAC;gBACjB;oBACE,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE;wBAC3C,YAAY,EAAE,UAAU,CAAC,YAAY;wBACrC,KAAK,EAAE,UAAU,CAAC,KAAK;wBACvB,OAAO,EAAE,UAAU,CAAC,OAAO;wBAC3B,YAAY,EAAE,UAAU,CAAC,YAAY;qBACtC,CAAC,CAAC;oBACH,MAAM;YACV,CAAC;YAED,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YAEzC,uBAAuB;YACvB,aAAa,CAAC,IAAI,CAAC;gBACjB,SAAS,EAAE,KAAK;gBAChB,MAAM,EAAE,OAAO,CAAC,SAAS;gBACzB,IAAI;aACL,CAAC,CAAC;YAEH,wBAAwB;YACxB,IAAI,OAAO,CAAC,SAAS,GAAG,UAAU,EAAE,CAAC;gBACnC,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC;gBAC/B,gBAAgB,EAAE,CAAC;gBAEnB,uDAAuD;gBACvD,IAAI,KAAK,GAAG,oBAAoB,GAAG,EAAE,EAAE,CAAC;oBACtC,oBAAoB,GAAG,KAAK,CAAC;gBAC/B,CAAC;YACH,CAAC;YAED,+BAA+B;YAC/B,IAAI,KAAK,GAAG,EAAE,KAAK,CAAC,EAAE,CAAC;gBACrB,OAAO,CAAC,GAAG,CAAC,6BAA6B,KAAK,UAAU,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,aAAa,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAC/J,CAAC;QACH,CAAC;QAED,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;QAC9C,MAAM,aAAa,GAAG,aAAa,CAAC,CAAC,CAAC,EAAE,MAAM,IAAI,CAAC,CAAC;QACpD,MAAM,WAAW,GAAG,YAAY,CAAC,SAAS,CAAC;QAC3C,MAAM,eAAe,GAAG,aAAa,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,GAAG,aAAa,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,GAAG,GAAG,CAAC;QAElH,MAAM,MAAM,GAAmB;YAC7B,SAAS,EAAE,UAAU,CAAC,SAAS;YAC/B,MAAM,EAAE,UAAU,CAAC,MAAM;YACzB,SAAS,EAAE,YAAY,CAAC,IAAI;YAC5B,SAAS,EAAE,WAAW;YACtB,eAAe;YACf,oBAAoB;YACpB,aAAa;SACd,CAAC;QAEF,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAElC,OAAO,CAAC,GAAG,CAAC,0CAA0C,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC;QAEjG,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,QAAQ,CAAC,QAA4B,EAAE,MAAsB;QACzE,IAAI,SAAS,GAAG,CAAC,CAAC;QAElB,4CAA4C;QAC5C,KAAK,IAAI,QAAQ,GAAG,CAAC,EAAE,QAAQ,GAAG,CAAC,EAAE,QAAQ,EAAE,EAAE,CAAC;YAChD,yCAAyC;YACzC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE;gBACjD,YAAY,EAAE,MAAM,CAAC,YAAY,GAAG,CAAC,CAAC,GAAG,QAAQ,GAAG,GAAG,CAAC,EAAE,sBAAsB;gBAChF,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,YAAY,EAAE,MAAM,CAAC,YAAY;aAClC,CAAC,CAAC;YAEH,SAAS,IAAI,IAAI,CAAC;QACpB,CAAC;QAED,OAAO,SAAS,GAAG,CAAC,CAAC;IACvB,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,QAAQ,CAAC,QAA4B,EAAE,MAAsB;QACzE,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,IAAI,KAAK,GAAG,CAAC,CAAC;QAEd,qDAAqD;QACrD,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,wBAAwB;YACxB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBAClD,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBAC9B,MAAM,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;gBAEtC,MAAM,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,KAAK,EAAE;oBACrE,YAAY,EAAE,MAAM,CAAC,YAAY,GAAG,CAAC,MAAM,CAAC,UAAU,IAAI,GAAG,CAAC;oBAC9D,KAAK,EAAE,MAAM,CAAC,KAAK;oBACnB,MAAM,EAAE,IAAI;iBACb,CAAC,CAAC;YACL,CAAC;YAED,KAAK,EAAE,CAAC;QACV,CAAC;QAED,gBAAgB;QAChB,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE;YACvD,YAAY,EAAE,MAAM,CAAC,YAAY;YACjC,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,YAAY,EAAE,MAAM,CAAC,YAAY;SAClC,CAAC,CAAC;QAEH,SAAS,IAAI,UAAU,CAAC;QAExB,OAAO,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;IACxC,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,cAAc,CAAC,QAA4B,EAAE,MAAsB;QAC/E,uCAAuC;QACvC,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBAClD,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBAC9B,MAAM,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;gBAEtC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC;YAChF,CAAC;QACH,CAAC;QAED,mBAAmB;QACnB,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC;QAElE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;YACpC,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YAE3D,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;gBACxB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,SAAS,EAAE;oBAChF,YAAY,EAAE,MAAM,CAAC,YAAY;oBACjC,KAAK,EAAE,MAAM,CAAC,KAAK;oBACnB,MAAM,EAAE,IAAI;iBACb,CAAC,CAAC;gBAEH,SAAS,IAAI,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC;YAC5C,CAAC;QACH,CAAC;QAED,OAAO,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;IAC7C,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,kBAAkB,CAAC,KAAU,EAAE,MAAc,EAAE,MAAc,EAAE,SAAc;QACjF,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACtB,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QAC1B,CAAC;QAED,OAAO,MAAM,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;IAC3E,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,eAAe,CACnB,WAA6B,EAC7B,YAAiC,EACjC,WAAmB,EACnB,MAAkC;QAElC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACtB,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QAC1B,CAAC;QAED,MAAM,UAAU,GAAqB;YACnC,UAAU,EAAE,MAAM,EAAE,UAAU,IAAI,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC;YAChE,oBAAoB,EAAE,MAAM,EAAE,oBAAoB,IAAI,QAAQ;YAC9D,aAAa,EAAE,MAAM,EAAE,aAAa,IAAI,cAAc;SACvD,CAAC;QAEF,OAAO,CAAC,GAAG,CAAC,6CAA6C,UAAU,CAAC,UAAU,CAAC,MAAM,YAAY,UAAU,CAAC,aAAa,iBAAiB,CAAC,CAAC;QAE5I,kCAAkC;QAClC,MAAM,iBAAiB,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,WAAW,EAAE,YAAY,EAAE,WAAW,CAAC,CAAC;QAElG,OAAO,iBAAiB,CAAC;IAC3B,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,aAAa,CAAC,MAAsB;QACxC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACtB,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QAC1B,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,0CAA0C,MAAM,CAAC,UAAU,MAAM,MAAM,CAAC,UAAU,YAAY,MAAM,CAAC,aAAa,GAAG,CAAC,CAAC;QAEnI,mBAAmB;QACnB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC,aAAa,CAAC,CAAC;QAE7G,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,OAAO,CAAC,GAAG,CAAC,mEAAmE,CAAC,CAAC;YACjF,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;QAC5B,CAAC;QAED,2CAA2C;QAC3C,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QAEtE,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,CAAC,cAAc,GAAG,CAAC,EAAE,CAAC;YAC3D,OAAO,CAAC,GAAG,CAAC,sCAAsC,cAAc,CAAC,MAAM,kBAAkB,CAAC,CAAC;YAE3F,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,cAAc,EAAE;gBACtD,SAAS,EAAE,WAAW;gBACtB,MAAM,EAAE,MAAM,CAAC,cAAc;gBAC7B,YAAY,EAAE,MAAM,CAAC,sCAAsC;aAC5D,CAAC,CAAC;YAEH,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;QACnC,CAAC;QAED,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;IAC3B,CAAC;IAED;;OAEG;IACH,kBAAkB;QAChB,OAAO,CAAC,GAAG,IAAI,CAAC,eAAe,CAAC,CAAC;IACnC,CAAC;IAED;;OAEG;IACH,UAAU;QACR,OAAO,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;IAClC,CAAC;IAED;;OAEG;IACH,KAAK;QACH,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;QACpB,IAAI,CAAC,eAAe,GAAG,EAAE,CAAC;IAC5B,CAAC;IAED;;OAEG;IACH,cAAc;QACZ,OAAO,IAAI,CAAC,IAAI,CAAC;IACnB,CAAC;CACF","sourcesContent":["/**\n * RL Training Service - Reinforcement Learning Training Orchestration\n *\n * ADR-065 Phase P1-2: SONA RL Loop Implementation\n *\n * Provides high-level RL training coordination using SonaTrajectoryService\n * for policy gradients, value functions, experience replay, multi-agent RL,\n * and transfer learning.\n *\n * Features:\n * - PPO (Proximal Policy Optimization)\n * - A3C (Asynchronous Advantage Actor-Critic)\n * - Q-Learning with experience replay\n * - Multi-agent coordination\n * - Transfer learning between tasks\n * - Continuous learning loop\n *\n * Usage:\n *   const trainer = new RLTrainingService();\n *   await trainer.initialize();\n *\n *   // Train on episodes\n *   await trainer.trainEpisodes(episodes, { algorithm: 'ppo' });\n *\n *   // Continuous learning\n *   await trainer.learnFromExecution(state, action, reward, nextState);\n */\n\nimport { SonaTrajectoryService, type StoredTrajectory } from 'agentdb';\n\n// Types that may not be exported yet - define locally if needed\nexport interface PolicyGradientConfig {\n  learningRate?: number;\n  gamma?: number;\n  baseline?: 'none' | 'avg' | 'learned';\n}\n\nexport interface ValueFunctionConfig {\n  learningRate?: number;\n  gamma?: number;\n  lambda?: number;\n}\n\nexport interface ExperienceReplayConfig {\n  bufferSize?: number;\n  batchSize?: number;\n  minSize?: number;\n}\n\nexport interface RLMetrics {\n  avgReward?: number;\n  avgLoss?: number;\n  episodeCount?: number;\n}\n\nexport type RLAlgorithm = 'reinforce' | 'ppo' | 'a3c' | 'q-learning';\n\nexport interface TrainingConfig {\n  algorithm: RLAlgorithm;\n  epochs: number;\n  batchSize: number;\n  learningRate: number;\n  gamma: number;\n  epsilon: number;\n  ppoClipRatio?: number; // PPO-specific\n  entropyCoeff?: number; // PPO/A3C\n  valueCoeff?: number; // A3C-specific\n}\n\nexport interface TrainingResult {\n  algorithm: RLAlgorithm;\n  epochs: number;\n  finalLoss: number;\n  avgReward: number;\n  improvementRate: number; // percentage\n  convergenceIteration: number;\n  learningCurve: Array<{ iteration: number; reward: number; loss: number }>;\n}\n\nexport interface MultiAgentConfig {\n  agentTypes: string[];\n  coordinationStrategy: 'centralized' | 'decentralized' | 'hybrid';\n  rewardSharing: 'equal' | 'contribution' | 'competitive';\n}\n\nexport interface TransferConfig {\n  sourceTask: string;\n  targetTask: string;\n  transferRatio: number;\n  finetuneEpochs: number;\n}\n\nexport class RLTrainingService {\n  private sona: SonaTrajectoryService;\n  private initialized: boolean = false;\n  private trainingHistory: TrainingResult[] = [];\n\n  constructor(sonaService?: SonaTrajectoryService) {\n    this.sona = sonaService || new SonaTrajectoryService();\n  }\n\n  /**\n   * Initialize the RL training service\n   */\n  async initialize(): Promise<boolean> {\n    if (this.initialized) return true;\n\n    await this.sona.initialize();\n    this.initialized = true;\n\n    console.log('[RLTrainingService] Initialized with engine:', this.sona.getEngineType());\n    return true;\n  }\n\n  /**\n   * Train on episodes using specified algorithm\n   *\n   * @param episodes - Training episodes (trajectories)\n   * @param config - Training configuration\n   * @returns Training results with learning curve\n   */\n  async trainEpisodes(episodes: StoredTrajectory[], config: Partial<TrainingConfig> = {}): Promise<TrainingResult> {\n    if (!this.initialized) {\n      await this.initialize();\n    }\n\n    const fullConfig: TrainingConfig = {\n      algorithm: config.algorithm || 'reinforce',\n      epochs: config.epochs || 100,\n      batchSize: config.batchSize || 32,\n      learningRate: config.learningRate || 0.001,\n      gamma: config.gamma || 0.99,\n      epsilon: config.epsilon || 0.1,\n      ppoClipRatio: config.ppoClipRatio || 0.2,\n      entropyCoeff: config.entropyCoeff || 0.01,\n      valueCoeff: config.valueCoeff || 0.5\n    };\n\n    // Configure SONA with training parameters\n    this.sona.configureRL({\n      policy: {\n        learningRate: fullConfig.learningRate,\n        gamma: fullConfig.gamma,\n        epsilon: fullConfig.epsilon,\n        entropyCoeff: fullConfig.entropyCoeff\n      },\n      value: {\n        learningRate: fullConfig.learningRate,\n        gamma: fullConfig.gamma,\n        lambda: 0.95\n      },\n      replay: {\n        bufferSize: 10000,\n        batchSize: fullConfig.batchSize,\n        priorityAlpha: 0.6,\n        priorityBeta: 0.4\n      }\n    });\n\n    const learningCurve: Array<{ iteration: number; reward: number; loss: number }> = [];\n    let bestReward = -Infinity;\n    let convergenceIteration = fullConfig.epochs;\n    let improvementCount = 0;\n\n    console.log(`[RLTrainingService] Training ${episodes.length} episodes with ${fullConfig.algorithm}`);\n\n    for (let epoch = 0; epoch < fullConfig.epochs; epoch++) {\n      let loss = 0;\n\n      // Algorithm-specific training\n      switch (fullConfig.algorithm) {\n        case 'ppo':\n          loss = await this.trainPPO(episodes, fullConfig);\n          break;\n        case 'a3c':\n          loss = await this.trainA3C(episodes, fullConfig);\n          break;\n        case 'q-learning':\n          loss = await this.trainQLearning(episodes, fullConfig);\n          break;\n        case 'reinforce':\n        default:\n          loss = await this.sona.trainPolicy(episodes, {\n            learningRate: fullConfig.learningRate,\n            gamma: fullConfig.gamma,\n            epsilon: fullConfig.epsilon,\n            entropyCoeff: fullConfig.entropyCoeff\n          });\n          break;\n      }\n\n      const metrics = this.sona.getRLMetrics();\n\n      // Track learning curve\n      learningCurve.push({\n        iteration: epoch,\n        reward: metrics.avgReward,\n        loss\n      });\n\n      // Check for improvement\n      if (metrics.avgReward > bestReward) {\n        bestReward = metrics.avgReward;\n        improvementCount++;\n\n        // Check for convergence (no improvement for 20 epochs)\n        if (epoch - convergenceIteration > 20) {\n          convergenceIteration = epoch;\n        }\n      }\n\n      // Log progress every 10 epochs\n      if (epoch % 10 === 0) {\n        console.log(`[RLTrainingService] Epoch ${epoch}: Loss=${loss.toFixed(4)}, AvgReward=${metrics.avgReward.toFixed(4)}, Epsilon=${metrics.epsilon.toFixed(4)}`);\n      }\n    }\n\n    const finalMetrics = this.sona.getRLMetrics();\n    const initialReward = learningCurve[0]?.reward || 0;\n    const finalReward = finalMetrics.avgReward;\n    const improvementRate = initialReward === 0 ? 0 : ((finalReward - initialReward) / Math.abs(initialReward)) * 100;\n\n    const result: TrainingResult = {\n      algorithm: fullConfig.algorithm,\n      epochs: fullConfig.epochs,\n      finalLoss: finalMetrics.loss,\n      avgReward: finalReward,\n      improvementRate,\n      convergenceIteration,\n      learningCurve\n    };\n\n    this.trainingHistory.push(result);\n\n    console.log(`[RLTrainingService] Training complete: ${improvementRate.toFixed(2)}% improvement`);\n\n    return result;\n  }\n\n  /**\n   * PPO (Proximal Policy Optimization) training\n   */\n  private async trainPPO(episodes: StoredTrajectory[], config: TrainingConfig): Promise<number> {\n    let totalLoss = 0;\n\n    // PPO uses multiple epochs on the same data\n    for (let ppoEpoch = 0; ppoEpoch < 4; ppoEpoch++) {\n      // Standard policy gradient with clipping\n      const loss = await this.sona.trainPolicy(episodes, {\n        learningRate: config.learningRate * (1 - ppoEpoch * 0.1), // decay learning rate\n        gamma: config.gamma,\n        epsilon: config.epsilon,\n        entropyCoeff: config.entropyCoeff\n      });\n\n      totalLoss += loss;\n    }\n\n    return totalLoss / 4;\n  }\n\n  /**\n   * A3C (Asynchronous Advantage Actor-Critic) training\n   */\n  private async trainA3C(episodes: StoredTrajectory[], config: TrainingConfig): Promise<number> {\n    let totalLoss = 0;\n    let count = 0;\n\n    // Train actor (policy) and critic (value) separately\n    for (const episode of episodes) {\n      // Update value function\n      for (let i = 0; i < episode.steps.length - 1; i++) {\n        const step = episode.steps[i];\n        const nextStep = episode.steps[i + 1];\n\n        await this.sona.estimateValue(step.state, step.reward, nextStep.state, {\n          learningRate: config.learningRate * (config.valueCoeff || 0.5),\n          gamma: config.gamma,\n          lambda: 0.95\n        });\n      }\n\n      count++;\n    }\n\n    // Update policy\n    const policyLoss = await this.sona.trainPolicy(episodes, {\n      learningRate: config.learningRate,\n      gamma: config.gamma,\n      epsilon: config.epsilon,\n      entropyCoeff: config.entropyCoeff\n    });\n\n    totalLoss += policyLoss;\n\n    return totalLoss / Math.max(count, 1);\n  }\n\n  /**\n   * Q-Learning with experience replay\n   */\n  private async trainQLearning(episodes: StoredTrajectory[], config: TrainingConfig): Promise<number> {\n    // Add all experiences to replay buffer\n    for (const episode of episodes) {\n      for (let i = 0; i < episode.steps.length - 1; i++) {\n        const step = episode.steps[i];\n        const nextStep = episode.steps[i + 1];\n\n        this.sona.addExperience(step.state, step.action, step.reward, nextStep.state);\n      }\n    }\n\n    // Sample and train\n    let totalLoss = 0;\n    const numBatches = Math.floor(episodes.length / config.batchSize);\n\n    for (let i = 0; i < numBatches; i++) {\n      const batch = this.sona.sampleExperience(config.batchSize);\n\n      for (const exp of batch) {\n        const value = await this.sona.estimateValue(exp.state, exp.reward, exp.nextState, {\n          learningRate: config.learningRate,\n          gamma: config.gamma,\n          lambda: 0.95\n        });\n\n        totalLoss += Math.abs(value - exp.reward);\n      }\n    }\n\n    return totalLoss / Math.max(numBatches, 1);\n  }\n\n  /**\n   * Learn from a single execution step (continuous learning)\n   *\n   * @param state - Current state\n   * @param action - Action taken\n   * @param reward - Reward received\n   * @param nextState - Resulting state\n   * @returns Updated value estimate\n   */\n  async learnFromExecution(state: any, action: string, reward: number, nextState: any): Promise<number> {\n    if (!this.initialized) {\n      await this.initialize();\n    }\n\n    return await this.sona.continuousLearn(state, action, reward, nextState);\n  }\n\n  /**\n   * Multi-agent reinforcement learning\n   *\n   * @param agentStates - Map of agent IDs to states\n   * @param agentActions - Map of agent IDs to actions\n   * @param jointReward - Shared reward\n   * @param config - Multi-agent configuration\n   * @returns Individual rewards for each agent\n   */\n  async multiAgentTrain(\n    agentStates: Map<string, any>,\n    agentActions: Map<string, string>,\n    jointReward: number,\n    config?: Partial<MultiAgentConfig>\n  ): Promise<Map<string, number>> {\n    if (!this.initialized) {\n      await this.initialize();\n    }\n\n    const fullConfig: MultiAgentConfig = {\n      agentTypes: config?.agentTypes || Array.from(agentStates.keys()),\n      coordinationStrategy: config?.coordinationStrategy || 'hybrid',\n      rewardSharing: config?.rewardSharing || 'contribution'\n    };\n\n    console.log(`[RLTrainingService] Multi-agent training: ${fullConfig.agentTypes.length} agents, ${fullConfig.rewardSharing} reward sharing`);\n\n    // Use SONA's multi-agent learning\n    const individualRewards = await this.sona.multiAgentLearn(agentStates, agentActions, jointReward);\n\n    return individualRewards;\n  }\n\n  /**\n   * Transfer learning from source task to target task\n   *\n   * @param config - Transfer learning configuration\n   * @returns Success indicator and training result\n   */\n  async transferLearn(config: TransferConfig): Promise<{ success: boolean; result?: TrainingResult }> {\n    if (!this.initialized) {\n      await this.initialize();\n    }\n\n    console.log(`[RLTrainingService] Transfer learning: ${config.sourceTask} → ${config.targetTask} (ratio: ${config.transferRatio})`);\n\n    // Perform transfer\n    const success = await this.sona.transferLearning(config.sourceTask, config.targetTask, config.transferRatio);\n\n    if (!success) {\n      console.log('[RLTrainingService] Transfer failed: no source patterns available');\n      return { success: false };\n    }\n\n    // Fine-tune on target task if we have data\n    const targetPatterns = await this.sona.getPatterns(config.targetTask);\n\n    if (targetPatterns.length > 0 && config.finetuneEpochs > 0) {\n      console.log(`[RLTrainingService] Fine-tuning on ${targetPatterns.length} target patterns`);\n\n      const result = await this.trainEpisodes(targetPatterns, {\n        algorithm: 'reinforce',\n        epochs: config.finetuneEpochs,\n        learningRate: 0.0005 // lower learning rate for fine-tuning\n      });\n\n      return { success: true, result };\n    }\n\n    return { success: true };\n  }\n\n  /**\n   * Get training history\n   */\n  getTrainingHistory(): TrainingResult[] {\n    return [...this.trainingHistory];\n  }\n\n  /**\n   * Get current RL metrics\n   */\n  getMetrics(): RLMetrics {\n    return this.sona.getRLMetrics();\n  }\n\n  /**\n   * Reset training state\n   */\n  reset(): void {\n    this.sona.resetRL();\n    this.trainingHistory = [];\n  }\n\n  /**\n   * Get the underlying SONA service\n   */\n  getSonaService(): SonaTrajectoryService {\n    return this.sona;\n  }\n}\n"]}

package/dist/services/ruvector-service.d.ts

@@ -1,95 +0,0 @@
-export interface RuVectorAvailability {
-    core: boolean;
-    attention: boolean;
-    graphNode: boolean;
-    router: boolean;
-    gnn: boolean;
-}
-export interface VectorSearchResult {
-    id: string;
-    score: number;
-    data: Record<string, unknown>;
-}
-export interface AttentionResult {
-    output: number[];
-    weights: number[];
-    mechanism: string;
-}
-export interface GraphQueryResult {
-    nodes: any[];
-    edges: any[];
-    count: number;
-}
-export interface BenchmarkResult {
-    indexSize: number;
-    queryTimeMs: number;
-    recallAt10: number;
-    throughput: number;
-}
-export declare class RuVectorService {
-    private static instance;
-    private availability;
-    private modules;
-    private initialized;
-    private constructor();
-    static getInstance(): Promise<RuVectorService>;
-    static resetInstance(): void;
-    private initialize;
-    getAvailability(): RuVectorAvailability;
-    search(query: number[], k?: number): Promise<{
-        available: boolean;
-        results?: VectorSearchResult[];
-        reason?: string;
-    }>;
-    runAttention(params: {
-        queries: number[][];
-        keys: number[][];
-        values: number[][];
-        mechanism?: string;
-    }): Promise<{
-        available: boolean;
-        result?: AttentionResult;
-        reason?: string;
-    }>;
-    graphQuery(cypher: string): Promise<{
-        available: boolean;
-        result?: GraphQueryResult;
-        reason?: string;
-    }>;
-    graphCreate(params: {
-        nodes: any[];
-        edges: any[];
-    }): Promise<{
-        available: boolean;
-        result?: {
-            nodesCreated: number;
-            edgesCreated: number;
-        };
-        reason?: string;
-    }>;
-    route(query: number[], candidates: Array<{
-        id: string;
-        vector: number[];
-    }>): Promise<{
-        available: boolean;
-        result?: {
-            bestMatch: string;
-            score: number;
-            rankings: Array<{
-                id: string;
-                score: number;
-            }>;
-        };
-        reason?: string;
-    }>;
-    benchmark(params: {
-        dimension?: number;
-        numVectors?: number;
-        numQueries?: number;
-    }): Promise<{
-        available: boolean;
-        result?: BenchmarkResult;
-        reason?: string;
-    }>;
-}
-//# sourceMappingURL=ruvector-service.d.ts.map

package/dist/services/ruvector-service.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"ruvector-service.d.ts","sourceRoot":"","sources":["../../src/services/ruvector-service.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,oBAAoB;IACnC,IAAI,EAAE,OAAO,CAAC;IACd,SAAS,EAAE,OAAO,CAAC;IACnB,SAAS,EAAE,OAAO,CAAC;IACnB,MAAM,EAAE,OAAO,CAAC;IAChB,GAAG,EAAE,OAAO,CAAC;CACd;AAED,MAAM,WAAW,kBAAkB;IACjC,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAC/B;AAED,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,GAAG,EAAE,CAAC;IACb,KAAK,EAAE,GAAG,EAAE,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,qBAAa,eAAe;IAC1B,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAgC;IACvD,OAAO,CAAC,YAAY,CAElB;IACF,OAAO,CAAC,OAAO,CAA2B;IAC1C,OAAO,CAAC,WAAW,CAAS;IAE5B,OAAO;WAEM,WAAW,IAAI,OAAO,CAAC,eAAe,CAAC;IAQpD,MAAM,CAAC,aAAa,IAAI,IAAI;YAId,UAAU;IAuBxB,eAAe,IAAI,oBAAoB;IAIjC,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC,GAAE,MAAW,GAAG,OAAO,CAAC;QAAE,SAAS,EAAE,OAAO,CAAC;QAAC,OAAO,CAAC,EAAE,kBAAkB,EAAE,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAuBzH,YAAY,CAAC,MAAM,EAAE;QAAE,OAAO,EAAE,MAAM,EAAE,EAAE,CAAC;QAAC,IAAI,EAAE,MAAM,EAAE,EAAE,CAAC;QAAC,MAAM,EAAE,MAAM,EAAE,EAAE,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC;QAAE,SAAS,EAAE,OAAO,CAAC;QAAC,MAAM,CAAC,EAAE,eAAe,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAmCnL,UAAU,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,SAAS,EAAE,OAAO,CAAC;QAAC,MAAM,CAAC,EAAE,gBAAgB,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAuBvG,WAAW,CAAC,MAAM,EAAE;QAAE,KAAK,EAAE,GAAG,EAAE,CAAC;QAAC,KAAK,EAAE,GAAG,EAAE,CAAA;KAAE,GAAG,OAAO,CAAC;QAAE,SAAS,EAAE,OAAO,CAAC;QAAC,MAAM,CAAC,EAAE;YAAE,YAAY,EAAE,MAAM,CAAC;YAAC,YAAY,EAAE,MAAM,CAAA;SAAE,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAqB9J,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,UAAU,EAAE,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC,GAAG,OAAO,CAAC;QAAE,SAAS,EAAE,OAAO,CAAC;QAAC,MAAM,CAAC,EAAE;YAAE,SAAS,EAAE,MAAM,CAAC;YAAC,KAAK,EAAE,MAAM,CAAC;YAAC,QAAQ,EAAE,KAAK,CAAC;gBAAE,EAAE,EAAE,MAAM,CAAC;gBAAC,KAAK,EAAE,MAAM,CAAA;aAAE,CAAC,CAAA;SAAE,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAuB5N,SAAS,CAAC,MAAM,EAAE;QAAE,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC;QAAE,SAAS,EAAE,OAAO,CAAC;QAAC,MAAM,CAAC,EAAE,eAAe,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CAwCtK"}