agentic-flow 2.0.7 → 3.0.0-alpha.2

package/dist/services/rl-training-service.d.ts

@@ -0,0 +1,161 @@
+/**
+ * RL Training Service - Reinforcement Learning Training Orchestration
+ *
+ * ADR-065 Phase P1-2: SONA RL Loop Implementation
+ *
+ * Provides high-level RL training coordination using SonaTrajectoryService
+ * for policy gradients, value functions, experience replay, multi-agent RL,
+ * and transfer learning.
+ *
+ * Features:
+ * - PPO (Proximal Policy Optimization)
+ * - A3C (Asynchronous Advantage Actor-Critic)
+ * - Q-Learning with experience replay
+ * - Multi-agent coordination
+ * - Transfer learning between tasks
+ * - Continuous learning loop
+ *
+ * Usage:
+ *   const trainer = new RLTrainingService();
+ *   await trainer.initialize();
+ *
+ *   // Train on episodes
+ *   await trainer.trainEpisodes(episodes, { algorithm: 'ppo' });
+ *
+ *   // Continuous learning
+ *   await trainer.learnFromExecution(state, action, reward, nextState);
+ */
+import { SonaTrajectoryService, type StoredTrajectory } from 'agentdb';
+export interface PolicyGradientConfig {
+    learningRate?: number;
+    gamma?: number;
+    baseline?: 'none' | 'avg' | 'learned';
+}
+export interface ValueFunctionConfig {
+    learningRate?: number;
+    gamma?: number;
+    lambda?: number;
+}
+export interface ExperienceReplayConfig {
+    bufferSize?: number;
+    batchSize?: number;
+    minSize?: number;
+}
+export interface RLMetrics {
+    avgReward?: number;
+    avgLoss?: number;
+    episodeCount?: number;
+}
+export type RLAlgorithm = 'reinforce' | 'ppo' | 'a3c' | 'q-learning';
+export interface TrainingConfig {
+    algorithm: RLAlgorithm;
+    epochs: number;
+    batchSize: number;
+    learningRate: number;
+    gamma: number;
+    epsilon: number;
+    ppoClipRatio?: number;
+    entropyCoeff?: number;
+    valueCoeff?: number;
+}
+export interface TrainingResult {
+    algorithm: RLAlgorithm;
+    epochs: number;
+    finalLoss: number;
+    avgReward: number;
+    improvementRate: number;
+    convergenceIteration: number;
+    learningCurve: Array<{
+        iteration: number;
+        reward: number;
+        loss: number;
+    }>;
+}
+export interface MultiAgentConfig {
+    agentTypes: string[];
+    coordinationStrategy: 'centralized' | 'decentralized' | 'hybrid';
+    rewardSharing: 'equal' | 'contribution' | 'competitive';
+}
+export interface TransferConfig {
+    sourceTask: string;
+    targetTask: string;
+    transferRatio: number;
+    finetuneEpochs: number;
+}
+export declare class RLTrainingService {
+    private sona;
+    private initialized;
+    private trainingHistory;
+    constructor(sonaService?: SonaTrajectoryService);
+    /**
+     * Initialize the RL training service
+     */
+    initialize(): Promise<boolean>;
+    /**
+     * Train on episodes using specified algorithm
+     *
+     * @param episodes - Training episodes (trajectories)
+     * @param config - Training configuration
+     * @returns Training results with learning curve
+     */
+    trainEpisodes(episodes: StoredTrajectory[], config?: Partial<TrainingConfig>): Promise<TrainingResult>;
+    /**
+     * PPO (Proximal Policy Optimization) training
+     */
+    private trainPPO;
+    /**
+     * A3C (Asynchronous Advantage Actor-Critic) training
+     */
+    private trainA3C;
+    /**
+     * Q-Learning with experience replay
+     */
+    private trainQLearning;
+    /**
+     * Learn from a single execution step (continuous learning)
+     *
+     * @param state - Current state
+     * @param action - Action taken
+     * @param reward - Reward received
+     * @param nextState - Resulting state
+     * @returns Updated value estimate
+     */
+    learnFromExecution(state: any, action: string, reward: number, nextState: any): Promise<number>;
+    /**
+     * Multi-agent reinforcement learning
+     *
+     * @param agentStates - Map of agent IDs to states
+     * @param agentActions - Map of agent IDs to actions
+     * @param jointReward - Shared reward
+     * @param config - Multi-agent configuration
+     * @returns Individual rewards for each agent
+     */
+    multiAgentTrain(agentStates: Map<string, any>, agentActions: Map<string, string>, jointReward: number, config?: Partial<MultiAgentConfig>): Promise<Map<string, number>>;
+    /**
+     * Transfer learning from source task to target task
+     *
+     * @param config - Transfer learning configuration
+     * @returns Success indicator and training result
+     */
+    transferLearn(config: TransferConfig): Promise<{
+        success: boolean;
+        result?: TrainingResult;
+    }>;
+    /**
+     * Get training history
+     */
+    getTrainingHistory(): TrainingResult[];
+    /**
+     * Get current RL metrics
+     */
+    getMetrics(): RLMetrics;
+    /**
+     * Reset training state
+     */
+    reset(): void;
+    /**
+     * Get the underlying SONA service
+     */
+    getSonaService(): SonaTrajectoryService;
+}
+//# sourceMappingURL=rl-training-service.d.ts.map

package/dist/services/rl-training-service.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"rl-training-service.d.ts","sourceRoot":"","sources":["../../src/services/rl-training-service.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAEH,OAAO,EAAE,qBAAqB,EAAE,KAAK,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAGvE,MAAM,WAAW,oBAAoB;IACnC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,GAAG,KAAK,GAAG,SAAS,CAAC;CACvC;AAED,MAAM,WAAW,mBAAmB;IAClC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,sBAAsB;IACrC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,SAAS;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,MAAM,WAAW,GAAG,WAAW,GAAG,KAAK,GAAG,KAAK,GAAG,YAAY,CAAC;AAErE,MAAM,WAAW,cAAc;IAC7B,SAAS,EAAE,WAAW,CAAC;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,cAAc;IAC7B,SAAS,EAAE,WAAW,CAAC;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,eAAe,EAAE,MAAM,CAAC;IACxB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,aAAa,EAAE,KAAK,CAAC;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CAC3E;AAED,MAAM,WAAW,gBAAgB;IAC/B,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,oBAAoB,EAAE,aAAa,GAAG,eAAe,GAAG,QAAQ,CAAC;IACjE,aAAa,EAAE,OAAO,GAAG,cAAc,GAAG,aAAa,CAAC;CACzD;AAED,MAAM,WAAW,cAAc;IAC7B,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;CACxB;AAED,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,IAAI,CAAwB;IACpC,OAAO,CAAC,WAAW,CAAkB;IACrC,OAAO,CAAC,eAAe,CAAwB;gBAEnC,WAAW,CAAC,EAAE,qBAAqB;IAI/C;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC;IAUpC;;;;;;OAMG;IACG,aAAa,CAAC,QAAQ,EAAE,gBAAgB,EAAE,EAAE,MAAM,GAAE,OAAO,CAAC,cAAc,CAAM,GAAG,OAAO,CAAC,cAAc,CAAC;IAsHhH;;OAEG;YACW,QAAQ;IAmBtB;;OAEG;YACW,QAAQ;IAkCtB;;OAEG;YACW,cAAc;IAgC5B;;;;;;;;OAQG;IACG,kBAAkB,CAAC,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,GAAG,OAAO,CAAC,MAAM,CAAC;IAQrG;;;;;;;;OAQG;IACG,eAAe,CACnB,WAAW,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,EAC7B,YAAY,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,EACjC,WAAW,EAAE,MAAM,EACnB,MAAM,CAAC,EAAE,OAAO,CAAC,gBAAgB,CAAC,GACjC,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAmB/B;;;;;OAKG;IACG,aAAa,CAAC,MAAM,EAAE,cAAc,GAAG,OAAO,CAAC;QAAE,OAAO,EAAE,OAAO,CAAC;QAAC,MAAM,CAAC,EAAE,cAAc,CAAA;KAAE,CAAC;IAiCnG;;OAEG;IACH,kBAAkB,IAAI,cAAc,EAAE;IAItC;;OAEG;IACH,UAAU,IAAI,SAAS;IAIvB;;OAEG;IACH,KAAK,IAAI,IAAI;IAKb;;OAEG;IACH,cAAc,IAAI,qBAAqB;CAGxC"}

package/dist/services/rl-training-service.js

@@ -0,0 +1,325 @@
+/**
+ * RL Training Service - Reinforcement Learning Training Orchestration
+ *
+ * ADR-065 Phase P1-2: SONA RL Loop Implementation
+ *
+ * Provides high-level RL training coordination using SonaTrajectoryService
+ * for policy gradients, value functions, experience replay, multi-agent RL,
+ * and transfer learning.
+ *
+ * Features:
+ * - PPO (Proximal Policy Optimization)
+ * - A3C (Asynchronous Advantage Actor-Critic)
+ * - Q-Learning with experience replay
+ * - Multi-agent coordination
+ * - Transfer learning between tasks
+ * - Continuous learning loop
+ *
+ * Usage:
+ *   const trainer = new RLTrainingService();
+ *   await trainer.initialize();
+ *
+ *   // Train on episodes
+ *   await trainer.trainEpisodes(episodes, { algorithm: 'ppo' });
+ *
+ *   // Continuous learning
+ *   await trainer.learnFromExecution(state, action, reward, nextState);
+ */
+import { SonaTrajectoryService } from 'agentdb';
+export class RLTrainingService {
+    sona;
+    initialized = false;
+    trainingHistory = [];
+    constructor(sonaService) {
+        this.sona = sonaService || new SonaTrajectoryService();
+    }
+    /**
+     * Initialize the RL training service
+     */
+    async initialize() {
+        if (this.initialized)
+            return true;
+        await this.sona.initialize();
+        this.initialized = true;
+        console.log('[RLTrainingService] Initialized with engine:', this.sona.getEngineType());
+        return true;
+    }
+    /**
+     * Train on episodes using specified algorithm
+     *
+     * @param episodes - Training episodes (trajectories)
+     * @param config - Training configuration
+     * @returns Training results with learning curve
+     */
+    async trainEpisodes(episodes, config = {}) {
+        if (!this.initialized) {
+            await this.initialize();
+        }
+        const fullConfig = {
+            algorithm: config.algorithm || 'reinforce',
+            epochs: config.epochs || 100,
+            batchSize: config.batchSize || 32,
+            learningRate: config.learningRate || 0.001,
+            gamma: config.gamma || 0.99,
+            epsilon: config.epsilon || 0.1,
+            ppoClipRatio: config.ppoClipRatio || 0.2,
+            entropyCoeff: config.entropyCoeff || 0.01,
+            valueCoeff: config.valueCoeff || 0.5
+        };
+        // Configure SONA with training parameters
+        this.sona.configureRL({
+            policy: {
+                learningRate: fullConfig.learningRate,
+                gamma: fullConfig.gamma,
+                epsilon: fullConfig.epsilon,
+                entropyCoeff: fullConfig.entropyCoeff
+            },
+            value: {
+                learningRate: fullConfig.learningRate,
+                gamma: fullConfig.gamma,
+                lambda: 0.95
+            },
+            replay: {
+                bufferSize: 10000,
+                batchSize: fullConfig.batchSize,
+                priorityAlpha: 0.6,
+                priorityBeta: 0.4
+            }
+        });
+        const learningCurve = [];
+        let bestReward = -Infinity;
+        let convergenceIteration = fullConfig.epochs;
+        let improvementCount = 0;
+        console.log(`[RLTrainingService] Training ${episodes.length} episodes with ${fullConfig.algorithm}`);
+        for (let epoch = 0; epoch < fullConfig.epochs; epoch++) {
+            let loss = 0;
+            // Algorithm-specific training
+            switch (fullConfig.algorithm) {
+                case 'ppo':
+                    loss = await this.trainPPO(episodes, fullConfig);
+                    break;
+                case 'a3c':
+                    loss = await this.trainA3C(episodes, fullConfig);
+                    break;
+                case 'q-learning':
+                    loss = await this.trainQLearning(episodes, fullConfig);
+                    break;
+                case 'reinforce':
+                default:
+                    loss = await this.sona.trainPolicy(episodes, {
+                        learningRate: fullConfig.learningRate,
+                        gamma: fullConfig.gamma,
+                        epsilon: fullConfig.epsilon,
+                        entropyCoeff: fullConfig.entropyCoeff
+                    });
+                    break;
+            }
+            const metrics = this.sona.getRLMetrics();
+            // Track learning curve
+            learningCurve.push({
+                iteration: epoch,
+                reward: metrics.avgReward,
+                loss
+            });
+            // Check for improvement
+            if (metrics.avgReward > bestReward) {
+                bestReward = metrics.avgReward;
+                improvementCount++;
+                // Check for convergence (no improvement for 20 epochs)
+                if (epoch - convergenceIteration > 20) {
+                    convergenceIteration = epoch;
+                }
+            }
+            // Log progress every 10 epochs
+            if (epoch % 10 === 0) {
+                console.log(`[RLTrainingService] Epoch ${epoch}: Loss=${loss.toFixed(4)}, AvgReward=${metrics.avgReward.toFixed(4)}, Epsilon=${metrics.epsilon.toFixed(4)}`);
+            }
+        }
+        const finalMetrics = this.sona.getRLMetrics();
+        const initialReward = learningCurve[0]?.reward || 0;
+        const finalReward = finalMetrics.avgReward;
+        const improvementRate = initialReward === 0 ? 0 : ((finalReward - initialReward) / Math.abs(initialReward)) * 100;
+        const result = {
+            algorithm: fullConfig.algorithm,
+            epochs: fullConfig.epochs,
+            finalLoss: finalMetrics.loss,
+            avgReward: finalReward,
+            improvementRate,
+            convergenceIteration,
+            learningCurve
+        };
+        this.trainingHistory.push(result);
+        console.log(`[RLTrainingService] Training complete: ${improvementRate.toFixed(2)}% improvement`);
+        return result;
+    }
+    /**
+     * PPO (Proximal Policy Optimization) training
+     */
+    async trainPPO(episodes, config) {
+        let totalLoss = 0;
+        // PPO uses multiple epochs on the same data
+        for (let ppoEpoch = 0; ppoEpoch < 4; ppoEpoch++) {
+            // Standard policy gradient with clipping
+            const loss = await this.sona.trainPolicy(episodes, {
+                learningRate: config.learningRate * (1 - ppoEpoch * 0.1), // decay learning rate
+                gamma: config.gamma,
+                epsilon: config.epsilon,
+                entropyCoeff: config.entropyCoeff
+            });
+            totalLoss += loss;
+        }
+        return totalLoss / 4;
+    }
+    /**
+     * A3C (Asynchronous Advantage Actor-Critic) training
+     */
+    async trainA3C(episodes, config) {
+        let totalLoss = 0;
+        let count = 0;
+        // Train actor (policy) and critic (value) separately
+        for (const episode of episodes) {
+            // Update value function
+            for (let i = 0; i < episode.steps.length - 1; i++) {
+                const step = episode.steps[i];
+                const nextStep = episode.steps[i + 1];
+                await this.sona.estimateValue(step.state, step.reward, nextStep.state, {
+                    learningRate: config.learningRate * (config.valueCoeff || 0.5),
+                    gamma: config.gamma,
+                    lambda: 0.95
+                });
+            }
+            count++;
+        }
+        // Update policy
+        const policyLoss = await this.sona.trainPolicy(episodes, {
+            learningRate: config.learningRate,
+            gamma: config.gamma,
+            epsilon: config.epsilon,
+            entropyCoeff: config.entropyCoeff
+        });
+        totalLoss += policyLoss;
+        return totalLoss / Math.max(count, 1);
+    }
+    /**
+     * Q-Learning with experience replay
+     */
+    async trainQLearning(episodes, config) {
+        // Add all experiences to replay buffer
+        for (const episode of episodes) {
+            for (let i = 0; i < episode.steps.length - 1; i++) {
+                const step = episode.steps[i];
+                const nextStep = episode.steps[i + 1];
+                this.sona.addExperience(step.state, step.action, step.reward, nextStep.state);
+            }
+        }
+        // Sample and train
+        let totalLoss = 0;
+        const numBatches = Math.floor(episodes.length / config.batchSize);
+        for (let i = 0; i < numBatches; i++) {
+            const batch = this.sona.sampleExperience(config.batchSize);
+            for (const exp of batch) {
+                const value = await this.sona.estimateValue(exp.state, exp.reward, exp.nextState, {
+                    learningRate: config.learningRate,
+                    gamma: config.gamma,
+                    lambda: 0.95
+                });
+                totalLoss += Math.abs(value - exp.reward);
+            }
+        }
+        return totalLoss / Math.max(numBatches, 1);
+    }
+    /**
+     * Learn from a single execution step (continuous learning)
+     *
+     * @param state - Current state
+     * @param action - Action taken
+     * @param reward - Reward received
+     * @param nextState - Resulting state
+     * @returns Updated value estimate
+     */
+    async learnFromExecution(state, action, reward, nextState) {
+        if (!this.initialized) {
+            await this.initialize();
+        }
+        return await this.sona.continuousLearn(state, action, reward, nextState);
+    }
+    /**
+     * Multi-agent reinforcement learning
+     *
+     * @param agentStates - Map of agent IDs to states
+     * @param agentActions - Map of agent IDs to actions
+     * @param jointReward - Shared reward
+     * @param config - Multi-agent configuration
+     * @returns Individual rewards for each agent
+     */
+    async multiAgentTrain(agentStates, agentActions, jointReward, config) {
+        if (!this.initialized) {
+            await this.initialize();
+        }
+        const fullConfig = {
+            agentTypes: config?.agentTypes || Array.from(agentStates.keys()),
+            coordinationStrategy: config?.coordinationStrategy || 'hybrid',
+            rewardSharing: config?.rewardSharing || 'contribution'
+        };
+        console.log(`[RLTrainingService] Multi-agent training: ${fullConfig.agentTypes.length} agents, ${fullConfig.rewardSharing} reward sharing`);
+        // Use SONA's multi-agent learning
+        const individualRewards = await this.sona.multiAgentLearn(agentStates, agentActions, jointReward);
+        return individualRewards;
+    }
+    /**
+     * Transfer learning from source task to target task
+     *
+     * @param config - Transfer learning configuration
+     * @returns Success indicator and training result
+     */
+    async transferLearn(config) {
+        if (!this.initialized) {
+            await this.initialize();
+        }
+        console.log(`[RLTrainingService] Transfer learning: ${config.sourceTask} → ${config.targetTask} (ratio: ${config.transferRatio})`);
+        // Perform transfer
+        const success = await this.sona.transferLearning(config.sourceTask, config.targetTask, config.transferRatio);
+        if (!success) {
+            console.log('[RLTrainingService] Transfer failed: no source patterns available');
+            return { success: false };
+        }
+        // Fine-tune on target task if we have data
+        const targetPatterns = await this.sona.getPatterns(config.targetTask);
+        if (targetPatterns.length > 0 && config.finetuneEpochs > 0) {
+            console.log(`[RLTrainingService] Fine-tuning on ${targetPatterns.length} target patterns`);
+            const result = await this.trainEpisodes(targetPatterns, {
+                algorithm: 'reinforce',
+                epochs: config.finetuneEpochs,
+                learningRate: 0.0005 // lower learning rate for fine-tuning
+            });
+            return { success: true, result };
+        }
+        return { success: true };
+    }
+    /**
+     * Get training history
+     */
+    getTrainingHistory() {
+        return [...this.trainingHistory];
+    }
+    /**
+     * Get current RL metrics
+     */
+    getMetrics() {
+        return this.sona.getRLMetrics();
+    }
+    /**
+     * Reset training state
+     */
+    reset() {
+        this.sona.resetRL();
+        this.trainingHistory = [];
+    }
+    /**
+     * Get the underlying SONA service
+     */
+    getSonaService() {
+        return this.sona;
+    }
+}
+//# sourceMappingURL=rl-training-service.js.map

package/dist/services/rl-training-service.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"rl-training-service.js","sourceRoot":"","sources":["../../src/services/rl-training-service.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAEH,OAAO,EAAE,qBAAqB,EAAyB,MAAM,SAAS,CAAC;AAgEvE,MAAM,OAAO,iBAAiB;IACpB,IAAI,CAAwB;IAC5B,WAAW,GAAY,KAAK,CAAC;IAC7B,eAAe,GAAqB,EAAE,CAAC;IAE/C,YAAY,WAAmC;QAC7C,IAAI,CAAC,IAAI,GAAG,WAAW,IAAI,IAAI,qBAAqB,EAAE,CAAC;IACzD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU;QACd,IAAI,IAAI,CAAC,WAAW;YAAE,OAAO,IAAI,CAAC;QAElC,MAAM,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;QAC7B,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QAExB,OAAO,CAAC,GAAG,CAAC,8CAA8C,EAAE,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC,CAAC;QACvF,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,aAAa,CAAC,QAA4B,EAAE,SAAkC,EAAE;QACpF,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACtB,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QAC1B,CAAC;QAED,MAAM,UAAU,GAAmB;YACjC,SAAS,EAAE,MAAM,CAAC,SAAS,IAAI,WAAW;YAC1C,MAAM,EAAE,MAAM,CAAC,MAAM,IAAI,GAAG;YAC5B,SAAS,EAAE,MAAM,CAAC,SAAS,IAAI,EAAE;YACjC,YAAY,EAAE,MAAM,CAAC,YAAY,IAAI,KAAK;YAC1C,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,IAAI;YAC3B,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,GAAG;YAC9B,YAAY,EAAE,MAAM,CAAC,YAAY,IAAI,GAAG;YACxC,YAAY,EAAE,MAAM,CAAC,YAAY,IAAI,IAAI;YACzC,UAAU,EAAE,MAAM,CAAC,UAAU,IAAI,GAAG;SACrC,CAAC;QAEF,0CAA0C;QAC1C,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC;YACpB,MAAM,EAAE;gBACN,YAAY,EAAE,UAAU,CAAC,YAAY;gBACrC,KAAK,EAAE,UAAU,CAAC,KAAK;gBACvB,OAAO,EAAE,UAAU,CAAC,OAAO;gBAC3B,YAAY,EAAE,UAAU,CAAC,YAAY;aACtC;YACD,KAAK,EAAE;gBACL,YAAY,EAAE,UAAU,CAAC,YAAY;gBACrC,KAAK,EAAE,UAAU,CAAC,KAAK;gBACvB,MAAM,EAAE,IAAI;aACb;YACD,MAAM,EAAE;gBACN,UAAU,EAAE,KAAK;gBACjB,SAAS,EAAE,UAAU,CAAC,SAAS;gBAC/B,aAAa,EAAE,GAAG;gBAClB,YAAY,EAAE,GAAG;aAClB;SACF,CAAC,CAAC;QAEH,MAAM,aAAa,GAA+D,EAAE,CAAC;QACrF,IAAI,UAAU,GAAG,CAAC,QAAQ,CAAC;QAC3B,IAAI,oBAAoB,GAAG,UAAU,CAAC,MAAM,CAAC;QAC7C,IAAI,gBAAgB,GAAG,CAAC,CAAC;QAEzB,OAAO,CAAC,GAAG,CAAC,gCAAgC,QAAQ,CAAC,MAAM,kBAAkB,UAAU,CAAC,SAAS,EAAE,CAAC,CAAC;QAErG,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,UAAU,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;YACvD,IAAI,IAAI,GAAG,CAAC,CAAC;YAEb,8BAA8B;YAC9B,QAAQ,UAAU,CAAC,SAAS,EAAE,CAAC;gBAC7B,KAAK,KAAK;oBACR,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;oBACjD,MAAM;gBACR,KAAK,KAAK;oBACR,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;oBACjD,MAAM;gBACR,KAAK,YAAY;oBACf,IAAI,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;oBACvD,MAAM;gBACR,KAAK,WAAW,CAAC;gBACjB;oBACE,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE;wBAC3C,YAAY,EAAE,UAAU,CAAC,YAAY;wBACrC,KAAK,EAAE,UAAU,CAAC,KAAK;wBACvB,OAAO,EAAE,UAAU,CAAC,OAAO;wBAC3B,YAAY,EAAE,UAAU,CAAC,YAAY;qBACtC,CAAC,CAAC;oBACH,MAAM;YACV,CAAC;YAED,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YAEzC,uBAAuB;YACvB,aAAa,CAAC,IAAI,CAAC;gBACjB,SAAS,EAAE,KAAK;gBAChB,MAAM,EAAE,OAAO,CAAC,SAAS;gBACzB,IAAI;aACL,CAAC,CAAC;YAEH,wBAAwB;YACxB,IAAI,OAAO,CAAC,SAAS,GAAG,UAAU,EAAE,CAAC;gBACnC,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC;gBAC/B,gBAAgB,EAAE,CAAC;gBAEnB,uDAAuD;gBACvD,IAAI,KAAK,GAAG,oBAAoB,GAAG,EAAE,EAAE,CAAC;oBACtC,oBAAoB,GAAG,KAAK,CAAC;gBAC/B,CAAC;YACH,CAAC;YAED,+BAA+B;YAC/B,IAAI,KAAK,GAAG,EAAE,KAAK,CAAC,EAAE,CAAC;gBACrB,OAAO,CAAC,GAAG,CAAC,6BAA6B,KAAK,UAAU,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,aAAa,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAC/J,CAAC;QACH,CAAC;QAED,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;QAC9C,MAAM,aAAa,GAAG,aAAa,CAAC,CAAC,CAAC,EAAE,MAAM,IAAI,CAAC,CAAC;QACpD,MAAM,WAAW,GAAG,YAAY,CAAC,SAAS,CAAC;QAC3C,MAAM,eAAe,GAAG,aAAa,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,GAAG,aAAa,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,GAAG,GAAG,CAAC;QAElH,MAAM,MAAM,GAAmB;YAC7B,SAAS,EAAE,UAAU,CAAC,SAAS;YAC/B,MAAM,EAAE,UAAU,CAAC,MAAM;YACzB,SAAS,EAAE,YAAY,CAAC,IAAI;YAC5B,SAAS,EAAE,WAAW;YACtB,eAAe;YACf,oBAAoB;YACpB,aAAa;SACd,CAAC;QAEF,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAElC,OAAO,CAAC,GAAG,CAAC,0CAA0C,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC;QAEjG,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,QAAQ,CAAC,QAA4B,EAAE,MAAsB;QACzE,IAAI,SAAS,GAAG,CAAC,CAAC;QAElB,4CAA4C;QAC5C,KAAK,IAAI,QAAQ,GAAG,CAAC,EAAE,QAAQ,GAAG,CAAC,EAAE,QAAQ,EAAE,EAAE,CAAC;YAChD,yCAAyC;YACzC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE;gBACjD,YAAY,EAAE,MAAM,CAAC,YAAY,GAAG,CAAC,CAAC,GAAG,QAAQ,GAAG,GAAG,CAAC,EAAE,sBAAsB;gBAChF,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,YAAY,EAAE,MAAM,CAAC,YAAY;aAClC,CAAC,CAAC;YAEH,SAAS,IAAI,IAAI,CAAC;QACpB,CAAC;QAED,OAAO,SAAS,GAAG,CAAC,CAAC;IACvB,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,QAAQ,CAAC,QAA4B,EAAE,MAAsB;QACzE,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,IAAI,KAAK,GAAG,CAAC,CAAC;QAEd,qDAAqD;QACrD,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,wBAAwB;YACxB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBAClD,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBAC9B,MAAM,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;gBAEtC,MAAM,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,KAAK,EAAE;oBACrE,YAAY,EAAE,MAAM,CAAC,YAAY,GAAG,CAAC,MAAM,CAAC,UAAU,IAAI,GAAG,CAAC;oBAC9D,KAAK,EAAE,MAAM,CAAC,KAAK;oBACnB,MAAM,EAAE,IAAI;iBACb,CAAC,CAAC;YACL,CAAC;YAED,KAAK,EAAE,CAAC;QACV,CAAC;QAED,gBAAgB;QAChB,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE;YACvD,YAAY,EAAE,MAAM,CAAC,YAAY;YACjC,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,YAAY,EAAE,MAAM,CAAC,YAAY;SAClC,CAAC,CAAC;QAEH,SAAS,IAAI,UAAU,CAAC;QAExB,OAAO,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;IACxC,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,cAAc,CAAC,QAA4B,EAAE,MAAsB;QAC/E,uCAAuC;QACvC,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBAClD,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBAC9B,MAAM,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;gBAEtC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC;YAChF,CAAC;QACH,CAAC;QAED,mBAAmB;QACnB,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC;QAElE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;YACpC,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YAE3D,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;gBACxB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,SAAS,EAAE;oBAChF,YAAY,EAAE,MAAM,CAAC,YAAY;oBACjC,KAAK,EAAE,MAAM,CAAC,KAAK;oBACnB,MAAM,EAAE,IAAI;iBACb,CAAC,CAAC;gBAEH,SAAS,IAAI,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC;YAC5C,CAAC;QACH,CAAC;QAED,OAAO,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;IAC7C,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,kBAAkB,CAAC,KAAU,EAAE,MAAc,EAAE,MAAc,EAAE,SAAc;QACjF,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACtB,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QAC1B,CAAC;QAED,OAAO,MAAM,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;IAC3E,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,eAAe,CACnB,WAA6B,EAC7B,YAAiC,EACjC,WAAmB,EACnB,MAAkC;QAElC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACtB,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QAC1B,CAAC;QAED,MAAM,UAAU,GAAqB;YACnC,UAAU,EAAE,MAAM,EAAE,UAAU,IAAI,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC;YAChE,oBAAoB,EAAE,MAAM,EAAE,oBAAoB,IAAI,QAAQ;YAC9D,aAAa,EAAE,MAAM,EAAE,aAAa,IAAI,cAAc;SACvD,CAAC;QAEF,OAAO,CAAC,GAAG,CAAC,6CAA6C,UAAU,CAAC,UAAU,CAAC,MAAM,YAAY,UAAU,CAAC,aAAa,iBAAiB,CAAC,CAAC;QAE5I,kCAAkC;QAClC,MAAM,iBAAiB,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,WAAW,EAAE,YAAY,EAAE,WAAW,CAAC,CAAC;QAElG,OAAO,iBAAiB,CAAC;IAC3B,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,aAAa,CAAC,MAAsB;QACxC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACtB,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QAC1B,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,0CAA0C,MAAM,CAAC,UAAU,MAAM,MAAM,CAAC,UAAU,YAAY,MAAM,CAAC,aAAa,GAAG,CAAC,CAAC;QAEnI,mBAAmB;QACnB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC,aAAa,CAAC,CAAC;QAE7G,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,OAAO,CAAC,GAAG,CAAC,mEAAmE,CAAC,CAAC;YACjF,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;QAC5B,CAAC;QAED,2CAA2C;QAC3C,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QAEtE,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,CAAC,cAAc,GAAG,CAAC,EAAE,CAAC;YAC3D,OAAO,CAAC,GAAG,CAAC,sCAAsC,cAAc,CAAC,MAAM,kBAAkB,CAAC,CAAC;YAE3F,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,cAAc,EAAE;gBACtD,SAAS,EAAE,WAAW;gBACtB,MAAM,EAAE,MAAM,CAAC,cAAc;gBAC7B,YAAY,EAAE,MAAM,CAAC,sCAAsC;aAC5D,CAAC,CAAC;YAEH,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;QACnC,CAAC;QAED,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;IAC3B,CAAC;IAED;;OAEG;IACH,kBAAkB;QAChB,OAAO,CAAC,GAAG,IAAI,CAAC,eAAe,CAAC,CAAC;IACnC,CAAC;IAED;;OAEG;IACH,UAAU;QACR,OAAO,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;IAClC,CAAC;IAED;;OAEG;IACH,KAAK;QACH,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;QACpB,IAAI,CAAC,eAAe,GAAG,EAAE,CAAC;IAC5B,CAAC;IAED;;OAEG;IACH,cAAc;QACZ,OAAO,IAAI,CAAC,IAAI,CAAC;IACnB,CAAC;CACF","sourcesContent":["/**\n * RL Training Service - Reinforcement Learning Training Orchestration\n *\n * ADR-065 Phase P1-2: SONA RL Loop Implementation\n *\n * Provides high-level RL training coordination using SonaTrajectoryService\n * for policy gradients, value functions, experience replay, multi-agent RL,\n * and transfer learning.\n *\n * Features:\n * - PPO (Proximal Policy Optimization)\n * - A3C (Asynchronous Advantage Actor-Critic)\n * - Q-Learning with experience replay\n * - Multi-agent coordination\n * - Transfer learning between tasks\n * - Continuous learning loop\n *\n * Usage:\n *   const trainer = new RLTrainingService();\n *   await trainer.initialize();\n *\n *   // Train on episodes\n *   await trainer.trainEpisodes(episodes, { algorithm: 'ppo' });\n *\n *   // Continuous learning\n *   await trainer.learnFromExecution(state, action, reward, nextState);\n */\n\nimport { SonaTrajectoryService, type StoredTrajectory } from 'agentdb';\n\n// Types that may not be exported yet - define locally if needed\nexport interface PolicyGradientConfig {\n  learningRate?: number;\n  gamma?: number;\n  baseline?: 'none' | 'avg' | 'learned';\n}\n\nexport interface ValueFunctionConfig {\n  learningRate?: number;\n  gamma?: number;\n  lambda?: number;\n}\n\nexport interface ExperienceReplayConfig {\n  bufferSize?: number;\n  batchSize?: number;\n  minSize?: number;\n}\n\nexport interface RLMetrics {\n  avgReward?: number;\n  avgLoss?: number;\n  episodeCount?: number;\n}\n\nexport type RLAlgorithm = 'reinforce' | 'ppo' | 'a3c' | 'q-learning';\n\nexport interface TrainingConfig {\n  algorithm: RLAlgorithm;\n  epochs: number;\n  batchSize: number;\n  learningRate: number;\n  gamma: number;\n  epsilon: number;\n  ppoClipRatio?: number; // PPO-specific\n  entropyCoeff?: number; // PPO/A3C\n  valueCoeff?: number; // A3C-specific\n}\n\nexport interface TrainingResult {\n  algorithm: RLAlgorithm;\n  epochs: number;\n  finalLoss: number;\n  avgReward: number;\n  improvementRate: number; // percentage\n  convergenceIteration: number;\n  learningCurve: Array<{ iteration: number; reward: number; loss: number }>;\n}\n\nexport interface MultiAgentConfig {\n  agentTypes: string[];\n  coordinationStrategy: 'centralized' | 'decentralized' | 'hybrid';\n  rewardSharing: 'equal' | 'contribution' | 'competitive';\n}\n\nexport interface TransferConfig {\n  sourceTask: string;\n  targetTask: string;\n  transferRatio: number;\n  finetuneEpochs: number;\n}\n\nexport class RLTrainingService {\n  private sona: SonaTrajectoryService;\n  private initialized: boolean = false;\n  private trainingHistory: TrainingResult[] = [];\n\n  constructor(sonaService?: SonaTrajectoryService) {\n    this.sona = sonaService || new SonaTrajectoryService();\n  }\n\n  /**\n   * Initialize the RL training service\n   */\n  async initialize(): Promise<boolean> {\n    if (this.initialized) return true;\n\n    await this.sona.initialize();\n    this.initialized = true;\n\n    console.log('[RLTrainingService] Initialized with engine:', this.sona.getEngineType());\n    return true;\n  }\n\n  /**\n   * Train on episodes using specified algorithm\n   *\n   * @param episodes - Training episodes (trajectories)\n   * @param config - Training configuration\n   * @returns Training results with learning curve\n   */\n  async trainEpisodes(episodes: StoredTrajectory[], config: Partial<TrainingConfig> = {}): Promise<TrainingResult> {\n    if (!this.initialized) {\n      await this.initialize();\n    }\n\n    const fullConfig: TrainingConfig = {\n      algorithm: config.algorithm || 'reinforce',\n      epochs: config.epochs || 100,\n      batchSize: config.batchSize || 32,\n      learningRate: config.learningRate || 0.001,\n      gamma: config.gamma || 0.99,\n      epsilon: config.epsilon || 0.1,\n      ppoClipRatio: config.ppoClipRatio || 0.2,\n      entropyCoeff: config.entropyCoeff || 0.01,\n      valueCoeff: config.valueCoeff || 0.5\n    };\n\n    // Configure SONA with training parameters\n    this.sona.configureRL({\n      policy: {\n        learningRate: fullConfig.learningRate,\n        gamma: fullConfig.gamma,\n        epsilon: fullConfig.epsilon,\n        entropyCoeff: fullConfig.entropyCoeff\n      },\n      value: {\n        learningRate: fullConfig.learningRate,\n        gamma: fullConfig.gamma,\n        lambda: 0.95\n      },\n      replay: {\n        bufferSize: 10000,\n        batchSize: fullConfig.batchSize,\n        priorityAlpha: 0.6,\n        priorityBeta: 0.4\n      }\n    });\n\n    const learningCurve: Array<{ iteration: number; reward: number; loss: number }> = [];\n    let bestReward = -Infinity;\n    let convergenceIteration = fullConfig.epochs;\n    let improvementCount = 0;\n\n    console.log(`[RLTrainingService] Training ${episodes.length} episodes with ${fullConfig.algorithm}`);\n\n    for (let epoch = 0; epoch < fullConfig.epochs; epoch++) {\n      let loss = 0;\n\n      // Algorithm-specific training\n      switch (fullConfig.algorithm) {\n        case 'ppo':\n          loss = await this.trainPPO(episodes, fullConfig);\n          break;\n        case 'a3c':\n          loss = await this.trainA3C(episodes, fullConfig);\n          break;\n        case 'q-learning':\n          loss = await this.trainQLearning(episodes, fullConfig);\n          break;\n        case 'reinforce':\n        default:\n          loss = await this.sona.trainPolicy(episodes, {\n            learningRate: fullConfig.learningRate,\n            gamma: fullConfig.gamma,\n            epsilon: fullConfig.epsilon,\n            entropyCoeff: fullConfig.entropyCoeff\n          });\n          break;\n      }\n\n      const metrics = this.sona.getRLMetrics();\n\n      // Track learning curve\n      learningCurve.push({\n        iteration: epoch,\n        reward: metrics.avgReward,\n        loss\n      });\n\n      // Check for improvement\n      if (metrics.avgReward > bestReward) {\n        bestReward = metrics.avgReward;\n        improvementCount++;\n\n        // Check for convergence (no improvement for 20 epochs)\n        if (epoch - convergenceIteration > 20) {\n          convergenceIteration = epoch;\n        }\n      }\n\n      // Log progress every 10 epochs\n      if (epoch % 10 === 0) {\n        console.log(`[RLTrainingService] Epoch ${epoch}: Loss=${loss.toFixed(4)}, AvgReward=${metrics.avgReward.toFixed(4)}, Epsilon=${metrics.epsilon.toFixed(4)}`);\n      }\n    }\n\n    const finalMetrics = this.sona.getRLMetrics();\n    const initialReward = learningCurve[0]?.reward || 0;\n    const finalReward = finalMetrics.avgReward;\n    const improvementRate = initialReward === 0 ? 0 : ((finalReward - initialReward) / Math.abs(initialReward)) * 100;\n\n    const result: TrainingResult = {\n      algorithm: fullConfig.algorithm,\n      epochs: fullConfig.epochs,\n      finalLoss: finalMetrics.loss,\n      avgReward: finalReward,\n      improvementRate,\n      convergenceIteration,\n      learningCurve\n    };\n\n    this.trainingHistory.push(result);\n\n    console.log(`[RLTrainingService] Training complete: ${improvementRate.toFixed(2)}% improvement`);\n\n    return result;\n  }\n\n  /**\n   * PPO (Proximal Policy Optimization) training\n   */\n  private async trainPPO(episodes: StoredTrajectory[], config: TrainingConfig): Promise<number> {\n    let totalLoss = 0;\n\n    // PPO uses multiple epochs on the same data\n    for (let ppoEpoch = 0; ppoEpoch < 4; ppoEpoch++) {\n      // Standard policy gradient with clipping\n      const loss = await this.sona.trainPolicy(episodes, {\n        learningRate: config.learningRate * (1 - ppoEpoch * 0.1), // decay learning rate\n        gamma: config.gamma,\n        epsilon: config.epsilon,\n        entropyCoeff: config.entropyCoeff\n      });\n\n      totalLoss += loss;\n    }\n\n    return totalLoss / 4;\n  }\n\n  /**\n   * A3C (Asynchronous Advantage Actor-Critic) training\n   */\n  private async trainA3C(episodes: StoredTrajectory[], config: TrainingConfig): Promise<number> {\n    let totalLoss = 0;\n    let count = 0;\n\n    // Train actor (policy) and critic (value) separately\n    for (const episode of episodes) {\n      // Update value function\n      for (let i = 0; i < episode.steps.length - 1; i++) {\n        const step = episode.steps[i];\n        const nextStep = episode.steps[i + 1];\n\n        await this.sona.estimateValue(step.state, step.reward, nextStep.state, {\n          learningRate: config.learningRate * (config.valueCoeff || 0.5),\n          gamma: config.gamma,\n          lambda: 0.95\n        });\n      }\n\n      count++;\n    }\n\n    // Update policy\n    const policyLoss = await this.sona.trainPolicy(episodes, {\n      learningRate: config.learningRate,\n      gamma: config.gamma,\n      epsilon: config.epsilon,\n      entropyCoeff: config.entropyCoeff\n    });\n\n    totalLoss += policyLoss;\n\n    return totalLoss / Math.max(count, 1);\n  }\n\n  /**\n   * Q-Learning with experience replay\n   */\n  private async trainQLearning(episodes: StoredTrajectory[], config: TrainingConfig): Promise<number> {\n    // Add all experiences to replay buffer\n    for (const episode of episodes) {\n      for (let i = 0; i < episode.steps.length - 1; i++) {\n        const step = episode.steps[i];\n        const nextStep = episode.steps[i + 1];\n\n        this.sona.addExperience(step.state, step.action, step.reward, nextStep.state);\n      }\n    }\n\n    // Sample and train\n    let totalLoss = 0;\n    const numBatches = Math.floor(episodes.length / config.batchSize);\n\n    for (let i = 0; i < numBatches; i++) {\n      const batch = this.sona.sampleExperience(config.batchSize);\n\n      for (const exp of batch) {\n        const value = await this.sona.estimateValue(exp.state, exp.reward, exp.nextState, {\n          learningRate: config.learningRate,\n          gamma: config.gamma,\n          lambda: 0.95\n        });\n\n        totalLoss += Math.abs(value - exp.reward);\n      }\n    }\n\n    return totalLoss / Math.max(numBatches, 1);\n  }\n\n  /**\n   * Learn from a single execution step (continuous learning)\n   *\n   * @param state - Current state\n   * @param action - Action taken\n   * @param reward - Reward received\n   * @param nextState - Resulting state\n   * @returns Updated value estimate\n   */\n  async learnFromExecution(state: any, action: string, reward: number, nextState: any): Promise<number> {\n    if (!this.initialized) {\n      await this.initialize();\n    }\n\n    return await this.sona.continuousLearn(state, action, reward, nextState);\n  }\n\n  /**\n   * Multi-agent reinforcement learning\n   *\n   * @param agentStates - Map of agent IDs to states\n   * @param agentActions - Map of agent IDs to actions\n   * @param jointReward - Shared reward\n   * @param config - Multi-agent configuration\n   * @returns Individual rewards for each agent\n   */\n  async multiAgentTrain(\n    agentStates: Map<string, any>,\n    agentActions: Map<string, string>,\n    jointReward: number,\n    config?: Partial<MultiAgentConfig>\n  ): Promise<Map<string, number>> {\n    if (!this.initialized) {\n      await this.initialize();\n    }\n\n    const fullConfig: MultiAgentConfig = {\n      agentTypes: config?.agentTypes || Array.from(agentStates.keys()),\n      coordinationStrategy: config?.coordinationStrategy || 'hybrid',\n      rewardSharing: config?.rewardSharing || 'contribution'\n    };\n\n    console.log(`[RLTrainingService] Multi-agent training: ${fullConfig.agentTypes.length} agents, ${fullConfig.rewardSharing} reward sharing`);\n\n    // Use SONA's multi-agent learning\n    const individualRewards = await this.sona.multiAgentLearn(agentStates, agentActions, jointReward);\n\n    return individualRewards;\n  }\n\n  /**\n   * Transfer learning from source task to target task\n   *\n   * @param config - Transfer learning configuration\n   * @returns Success indicator and training result\n   */\n  async transferLearn(config: TransferConfig): Promise<{ success: boolean; result?: TrainingResult }> {\n    if (!this.initialized) {\n      await this.initialize();\n    }\n\n    console.log(`[RLTrainingService] Transfer learning: ${config.sourceTask} → ${config.targetTask} (ratio: ${config.transferRatio})`);\n\n    // Perform transfer\n    const success = await this.sona.transferLearning(config.sourceTask, config.targetTask, config.transferRatio);\n\n    if (!success) {\n      console.log('[RLTrainingService] Transfer failed: no source patterns available');\n      return { success: false };\n    }\n\n    // Fine-tune on target task if we have data\n    const targetPatterns = await this.sona.getPatterns(config.targetTask);\n\n    if (targetPatterns.length > 0 && config.finetuneEpochs > 0) {\n      console.log(`[RLTrainingService] Fine-tuning on ${targetPatterns.length} target patterns`);\n\n      const result = await this.trainEpisodes(targetPatterns, {\n        algorithm: 'reinforce',\n        epochs: config.finetuneEpochs,\n        learningRate: 0.0005 // lower learning rate for fine-tuning\n      });\n\n      return { success: true, result };\n    }\n\n    return { success: true };\n  }\n\n  /**\n   * Get training history\n   */\n  getTrainingHistory(): TrainingResult[] {\n    return [...this.trainingHistory];\n  }\n\n  /**\n   * Get current RL metrics\n   */\n  getMetrics(): RLMetrics {\n    return this.sona.getRLMetrics();\n  }\n\n  /**\n   * Reset training state\n   */\n  reset(): void {\n    this.sona.resetRL();\n    this.trainingHistory = [];\n  }\n\n  /**\n   * Get the underlying SONA service\n   */\n  getSonaService(): SonaTrajectoryService {\n    return this.sona;\n  }\n}\n"]}

package/dist/services/ruvector-service.d.ts

@@ -0,0 +1,95 @@
+export interface RuVectorAvailability {
+    core: boolean;
+    attention: boolean;
+    graphNode: boolean;
+    router: boolean;
+    gnn: boolean;
+}
+export interface VectorSearchResult {
+    id: string;
+    score: number;
+    data: Record<string, unknown>;
+}
+export interface AttentionResult {
+    output: number[];
+    weights: number[];
+    mechanism: string;
+}
+export interface GraphQueryResult {
+    nodes: any[];
+    edges: any[];
+    count: number;
+}
+export interface BenchmarkResult {
+    indexSize: number;
+    queryTimeMs: number;
+    recallAt10: number;
+    throughput: number;
+}
+export declare class RuVectorService {
+    private static instance;
+    private availability;
+    private modules;
+    private initialized;
+    private constructor();
+    static getInstance(): Promise<RuVectorService>;
+    static resetInstance(): void;
+    private initialize;
+    getAvailability(): RuVectorAvailability;
+    search(query: number[], k?: number): Promise<{
+        available: boolean;
+        results?: VectorSearchResult[];
+        reason?: string;
+    }>;
+    runAttention(params: {
+        queries: number[][];
+        keys: number[][];
+        values: number[][];
+        mechanism?: string;
+    }): Promise<{
+        available: boolean;
+        result?: AttentionResult;
+        reason?: string;
+    }>;
+    graphQuery(cypher: string): Promise<{
+        available: boolean;
+        result?: GraphQueryResult;
+        reason?: string;
+    }>;
+    graphCreate(params: {
+        nodes: any[];
+        edges: any[];
+    }): Promise<{
+        available: boolean;
+        result?: {
+            nodesCreated: number;
+            edgesCreated: number;
+        };
+        reason?: string;
+    }>;
+    route(query: number[], candidates: Array<{
+        id: string;
+        vector: number[];
+    }>): Promise<{
+        available: boolean;
+        result?: {
+            bestMatch: string;
+            score: number;
+            rankings: Array<{
+                id: string;
+                score: number;
+            }>;
+        };
+        reason?: string;
+    }>;
+    benchmark(params: {
+        dimension?: number;
+        numVectors?: number;
+        numQueries?: number;
+    }): Promise<{
+        available: boolean;
+        result?: BenchmarkResult;
+        reason?: string;
+    }>;
+}
+//# sourceMappingURL=ruvector-service.d.ts.map

package/dist/services/ruvector-service.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ruvector-service.d.ts","sourceRoot":"","sources":["../../src/services/ruvector-service.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,oBAAoB;IACnC,IAAI,EAAE,OAAO,CAAC;IACd,SAAS,EAAE,OAAO,CAAC;IACnB,SAAS,EAAE,OAAO,CAAC;IACnB,MAAM,EAAE,OAAO,CAAC;IAChB,GAAG,EAAE,OAAO,CAAC;CACd;AAED,MAAM,WAAW,kBAAkB;IACjC,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAC/B;AAED,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,GAAG,EAAE,CAAC;IACb,KAAK,EAAE,GAAG,EAAE,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,qBAAa,eAAe;IAC1B,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAgC;IACvD,OAAO,CAAC,YAAY,CAElB;IACF,OAAO,CAAC,OAAO,CAA2B;IAC1C,OAAO,CAAC,WAAW,CAAS;IAE5B,OAAO;WAEM,WAAW,IAAI,OAAO,CAAC,eAAe,CAAC;IAQpD,MAAM,CAAC,aAAa,IAAI,IAAI;YAId,UAAU;IAuBxB,eAAe,IAAI,oBAAoB;IAIjC,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC,GAAE,MAAW,GAAG,OAAO,CAAC;QAAE,SAAS,EAAE,OAAO,CAAC;QAAC,OAAO,CAAC,EAAE,kBAAkB,EAAE,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAuBzH,YAAY,CAAC,MAAM,EAAE;QAAE,OAAO,EAAE,MAAM,EAAE,EAAE,CAAC;QAAC,IAAI,EAAE,MAAM,EAAE,EAAE,CAAC;QAAC,MAAM,EAAE,MAAM,EAAE,EAAE,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC;QAAE,SAAS,EAAE,OAAO,CAAC;QAAC,MAAM,CAAC,EAAE,eAAe,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAmCnL,UAAU,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,SAAS,EAAE,OAAO,CAAC;QAAC,MAAM,CAAC,EAAE,gBAAgB,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAuBvG,WAAW,CAAC,MAAM,EAAE;QAAE,KAAK,EAAE,GAAG,EAAE,CAAC;QAAC,KAAK,EAAE,GAAG,EAAE,CAAA;KAAE,GAAG,OAAO,CAAC;QAAE,SAAS,EAAE,OAAO,CAAC;QAAC,MAAM,CAAC,EAAE;YAAE,YAAY,EAAE,MAAM,CAAC;YAAC,YAAY,EAAE,MAAM,CAAA;SAAE,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAqB9J,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,UAAU,EAAE,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC,GAAG,OAAO,CAAC;QAAE,SAAS,EAAE,OAAO,CAAC;QAAC,MAAM,CAAC,EAAE;YAAE,SAAS,EAAE,MAAM,CAAC;YAAC,KAAK,EAAE,MAAM,CAAC;YAAC,QAAQ,EAAE,KAAK,CAAC;gBAAE,EAAE,EAAE,MAAM,CAAC;gBAAC,KAAK,EAAE,MAAM,CAAA;aAAE,CAAC,CAAA;SAAE,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAuB5N,SAAS,CAAC,MAAM,EAAE;QAAE,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC;QAAE,SAAS,EAAE,OAAO,CAAC;QAAC,MAAM,CAAC,EAAE,eAAe,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CAwCtK"}