agentic-flow 2.0.6 → 3.0.0-alpha.1

package/dist/agentic-flow/src/services/rl-training-service.d.ts

@@ -0,0 +1,233 @@
+/**
+ * RL Training Service - Reinforcement Learning Training Orchestration
+ *
+ * ADR-065 Phase P1-2: SONA RL Loop Implementation
+ *
+ * Provides high-level RL training coordination using SonaTrajectoryService
+ * for policy gradients, value functions, experience replay, multi-agent RL,
+ * and transfer learning.
+ *
+ * Features:
+ * - PPO (Proximal Policy Optimization)
+ * - A3C (Asynchronous Advantage Actor-Critic)
+ * - Q-Learning with experience replay
+ * - Multi-agent coordination
+ * - Transfer learning between tasks
+ * - Continuous learning loop
+ *
+ * Usage:
+ *   const trainer = new RLTrainingService();
+ *   await trainer.initialize();
+ *
+ *   // Train on episodes
+ *   await trainer.trainEpisodes(episodes, { algorithm: 'ppo' });
+ *
+ *   // Continuous learning
+ *   await trainer.learnFromExecution(state, action, reward, nextState);
+ */
+export interface TrajectoryStep {
+    state: any;
+    action: any;
+    reward: number;
+    nextState?: any;
+    done?: boolean;
+}
+export interface StoredTrajectory {
+    trajectoryId: string;
+    state: any;
+    action: any;
+    reward: number;
+    nextState: any;
+    done: boolean;
+    timestamp: number;
+    steps: TrajectoryStep[];
+}
+export interface RLMetrics {
+    avgReward?: number;
+    avgLoss?: number;
+    episodeCount?: number;
+    episodeReward?: number;
+    loss?: number;
+    epsilon?: number;
+    iterationCount?: number;
+}
+interface ISonaTrajectoryService {
+    initialize(): Promise<void>;
+    storePolicyGradient(config: any): Promise<any>;
+    storeValueFunction(config: any): Promise<any>;
+    storeExperienceReplay(config: any): Promise<any>;
+    getRLMetrics(): Promise<RLMetrics>;
+    getEngineType(): string;
+    estimateValue(state: any, reward: number, nextState: any, config: any): Promise<number>;
+    trainPolicy(episodes: StoredTrajectory[], config: any): Promise<number>;
+    addExperience(state: any, action: any, reward: number, nextState: any): void;
+    sampleExperience(batchSize: number): any[];
+    continuousLearn(state: any, action: any, reward: number, nextState: any): Promise<RLMetrics>;
+    multiAgentLearn(agentEpisodes: Map<string, StoredTrajectory[]>, config: any): Promise<RLMetrics>;
+    transferLearning(sourceTask: string, targetTask: string, config: any): Promise<any>;
+    getPatterns(minSupport?: number): Promise<any[]>;
+    resetRL(): Promise<void>;
+    configureRL(config: any): void;
+    getStats(): any;
+    clear(): Promise<void>;
+}
+declare class SonaTrajectoryServiceStub implements ISonaTrajectoryService {
+    initialize(): Promise<void>;
+    storePolicyGradient(config: any): Promise<any>;
+    storeValueFunction(config: any): Promise<any>;
+    storeExperienceReplay(config: any): Promise<any>;
+    getRLMetrics(): Promise<RLMetrics>;
+    getEngineType(): string;
+    estimateValue(state: any, reward: number, nextState: any, config: any): Promise<number>;
+    trainPolicy(episodes: StoredTrajectory[], config: any): Promise<number>;
+    addExperience(state: any, action: any, reward: number, nextState: any): void;
+    sampleExperience(batchSize: number): any[];
+    continuousLearn(state: any, action: any, reward: number, nextState: any): Promise<RLMetrics>;
+    multiAgentLearn(agentEpisodes: Map<string, StoredTrajectory[]>, config: any): Promise<RLMetrics>;
+    transferLearning(sourceTask: string, targetTask: string, config: any): Promise<any>;
+    getPatterns(minSupport?: number): Promise<any[]>;
+    resetRL(): Promise<void>;
+    configureRL(config: any): void;
+    getStats(): any;
+    clear(): Promise<void>;
+}
+type SonaTrajectoryService = ISonaTrajectoryService;
+declare const SonaTrajectoryService: typeof SonaTrajectoryServiceStub;
+export interface PolicyGradientConfig {
+    learningRate?: number;
+    gamma?: number;
+    baseline?: 'none' | 'avg' | 'learned';
+}
+export interface ValueFunctionConfig {
+    learningRate?: number;
+    gamma?: number;
+    lambda?: number;
+}
+export interface ExperienceReplayConfig {
+    bufferSize?: number;
+    batchSize?: number;
+    minSize?: number;
+}
+export interface RLMetrics {
+    avgReward?: number;
+    avgLoss?: number;
+    episodeCount?: number;
+    episodeReward?: number;
+    loss?: number;
+    epsilon?: number;
+    iterationCount?: number;
+}
+export type RLAlgorithm = 'reinforce' | 'ppo' | 'a3c' | 'q-learning';
+export interface TrainingConfig {
+    algorithm: RLAlgorithm;
+    epochs: number;
+    batchSize: number;
+    learningRate: number;
+    gamma: number;
+    epsilon: number;
+    ppoClipRatio?: number;
+    entropyCoeff?: number;
+    valueCoeff?: number;
+}
+export interface TrainingResult {
+    algorithm: RLAlgorithm;
+    epochs: number;
+    finalLoss: number;
+    avgReward: number;
+    improvementRate: number;
+    convergenceIteration: number;
+    learningCurve: Array<{
+        iteration: number;
+        reward: number;
+        loss: number;
+    }>;
+}
+export interface MultiAgentConfig {
+    agentTypes: string[];
+    coordinationStrategy: 'centralized' | 'decentralized' | 'hybrid';
+    rewardSharing: 'equal' | 'contribution' | 'competitive';
+}
+export interface TransferConfig {
+    sourceTask: string;
+    targetTask: string;
+    transferRatio: number;
+    finetuneEpochs: number;
+}
+export declare class RLTrainingService {
+    private sona;
+    private initialized;
+    private trainingHistory;
+    constructor(sonaService?: SonaTrajectoryService);
+    /**
+     * Initialize the RL training service
+     */
+    initialize(): Promise<boolean>;
+    /**
+     * Train on episodes using specified algorithm
+     *
+     * @param episodes - Training episodes (trajectories)
+     * @param config - Training configuration
+     * @returns Training results with learning curve
+     */
+    trainEpisodes(episodes: StoredTrajectory[], config?: Partial<TrainingConfig>): Promise<TrainingResult>;
+    /**
+     * PPO (Proximal Policy Optimization) training
+     */
+    private trainPPO;
+    /**
+     * A3C (Asynchronous Advantage Actor-Critic) training
+     */
+    private trainA3C;
+    /**
+     * Q-Learning with experience replay
+     */
+    private trainQLearning;
+    /**
+     * Learn from a single execution step (continuous learning)
+     *
+     * @param state - Current state
+     * @param action - Action taken
+     * @param reward - Reward received
+     * @param nextState - Resulting state
+     * @returns Updated value estimate
+     */
+    learnFromExecution(state: any, action: string, reward: number, nextState: any): Promise<number>;
+    /**
+     * Multi-agent reinforcement learning
+     *
+     * @param agentStates - Map of agent IDs to states
+     * @param agentActions - Map of agent IDs to actions
+     * @param jointReward - Shared reward
+     * @param config - Multi-agent configuration
+     * @returns Individual rewards for each agent
+     */
+    multiAgentTrain(agentStates: Map<string, any>, agentActions: Map<string, string>, jointReward: number, config?: Partial<MultiAgentConfig>): Promise<Map<string, number>>;
+    /**
+     * Transfer learning from source task to target task
+     *
+     * @param config - Transfer learning configuration
+     * @returns Success indicator and training result
+     */
+    transferLearn(config: TransferConfig): Promise<{
+        success: boolean;
+        result?: TrainingResult;
+    }>;
+    /**
+     * Get training history
+     */
+    getTrainingHistory(): TrainingResult[];
+    /**
+     * Get current RL metrics
+     */
+    getMetrics(): Promise<RLMetrics>;
+    /**
+     * Reset training state
+     */
+    reset(): Promise<void>;
+    /**
+     * Get the underlying SONA service
+     */
+    getSonaService(): SonaTrajectoryService;
+}
+export {};
+//# sourceMappingURL=rl-training-service.d.ts.map

package/dist/agentic-flow/src/services/rl-training-service.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"rl-training-service.d.ts","sourceRoot":"","sources":["../../../../src/services/rl-training-service.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAGH,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,GAAG,CAAC;IACX,MAAM,EAAE,GAAG,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,GAAG,CAAC;IAChB,IAAI,CAAC,EAAE,OAAO,CAAC;CAChB;AAED,MAAM,WAAW,gBAAgB;IAC/B,YAAY,EAAE,MAAM,CAAC;IACrB,KAAK,EAAE,GAAG,CAAC;IACX,MAAM,EAAE,GAAG,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,GAAG,CAAC;IACf,IAAI,EAAE,OAAO,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,cAAc,EAAE,CAAC;CACzB;AAGD,MAAM,WAAW,SAAS;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAGD,UAAU,sBAAsB;IAC9B,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5B,mBAAmB,CAAC,MAAM,EAAE,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;IAC/C,kBAAkB,CAAC,MAAM,EAAE,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;IAC9C,qBAAqB,CAAC,MAAM,EAAE,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;IACjD,YAAY,IAAI,OAAO,CAAC,SAAS,CAAC,CAAC;IACnC,aAAa,IAAI,MAAM,CAAC;IACxB,aAAa,CAAC,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACxF,WAAW,CAAC,QAAQ,EAAE,gBAAgB,EAAE,EAAE,MAAM,EAAE,GAAG,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACxE,aAAa,CAAC,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,GAAG,IAAI,CAAC;IAC7E,gBAAgB,CAAC,SAAS,EAAE,MAAM,GAAG,GAAG,EAAE,CAAC;IAC3C,eAAe,CAAC,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC;IAC7F,eAAe,CAAC,aAAa,EAAE,GAAG,CAAC,MAAM,EAAE,gBAAgB,EAAE,CAAC,EAAE,MAAM,EAAE,GAAG,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC;IACjG,gBAAgB,CAAC,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;IACpF,WAAW,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;IACjD,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACzB,WAAW,CAAC,MAAM,EAAE,GAAG,GAAG,IAAI,CAAC;IAC/B,QAAQ,IAAI,GAAG,CAAC;IAChB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACxB;AAGD,cAAM,yBAA0B,YAAW,sBAAsB;IACzD,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAC3B,mBAAmB,CAAC,MAAM,EAAE,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC;IAG9C,kBAAkB,CAAC,MAAM,EAAE,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC;IAG7C,qBAAqB,CAAC,MAAM,EAAE,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC;IAGhD,YAAY,IAAI,OAAO,CAAC,SAAS,CAAC;IAWxC,aAAa,IAAI,MAAM;IAGjB,aAAa,CAAC,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,GAAG,OAAO,CAAC,MAAM,CAAC;IAGvF,WAAW,CAAC,QAAQ,EAAE,gBAAgB,EAAE,EAAE,MAAM,EAAE,GAAG,GAAG,OAAO,CAAC,MAAM,CAAC;IAG7E,aAAa,CAAC,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,GAAG,IAAI;IAC5E,gBAAgB,CAAC,SAAS,EAAE,MAAM,GAAG,GAAG,EAAE;IAGpC,eAAe,CAAC,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,GAAG,OAAO,CAAC,SAAS,CAAC;IAG5F,eAAe,CAAC,aAAa,EAAE,GAAG,CAAC,MAAM,EAAE,gBAAgB,EAAE,CAAC,EAAE,MAAM,EAAE,GAAG,GAAG,OAAO,CAAC,SAAS,CAAC;IAGhG,gBAAgB,CAAC,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC;IAGnF,WAAW,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;IAGhD,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAC9B,WAAW,CAAC,MAAM,EAAE,GAAG,GAAG,IAAI;IAC9B,QAAQ,IAAI,GAAG;IAGT,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAC7B;AAED,KAAK,qBAAqB,GAAG,sBAAsB,CAAC;AACpD,QAAA,MAAM,qBAAqB,kCAA4B,CAAC;AAGxD,MAAM,WAAW,oBAAoB;IACnC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,GAAG,KAAK,GAAG,SAAS,CAAC;CACvC;AAED,MAAM,WAAW,mBAAmB;IAClC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,sBAAsB;IACrC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,SAAS;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,MAAM,WAAW,GAAG,WAAW,GAAG,KAAK,GAAG,KAAK,GAAG,YAAY,CAAC;AAErE,MAAM,WAAW,cAAc;IAC7B,SAAS,EAAE,WAAW,CAAC;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,cAAc;IAC7B,SAAS,EAAE,WAAW,CAAC;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,eAAe,EAAE,MAAM,CAAC;IACxB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,aAAa,EAAE,KAAK,CAAC;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CAC3E;AAED,MAAM,WAAW,gBAAgB;IAC/B,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,oBAAoB,EAAE,aAAa,GAAG,eAAe,GAAG,QAAQ,CAAC;IACjE,aAAa,EAAE,OAAO,GAAG,cAAc,GAAG,aAAa,CAAC;CACzD;AAED,MAAM,WAAW,cAAc;IAC7B,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;CACxB;AAED,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,IAAI,CAAwB;IACpC,OAAO,CAAC,WAAW,CAAkB;IACrC,OAAO,CAAC,eAAe,CAAwB;gBAEnC,WAAW,CAAC,EAAE,qBAAqB;IAI/C;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC;IAUpC;;;;;;OAMG;IACG,aAAa,CAAC,QAAQ,EAAE,gBAAgB,EAAE,EAAE,MAAM,GAAE,OAAO,CAAC,cAAc,CAAM,GAAG,OAAO,CAAC,cAAc,CAAC;IAsHhH;;OAEG;YACW,QAAQ;IAmBtB;;OAEG;YACW,QAAQ;IAkCtB;;OAEG;YACW,cAAc;IAgC5B;;;;;;;;OAQG;IACG,kBAAkB,CAAC,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,GAAG,OAAO,CAAC,MAAM,CAAC;IAQrG;;;;;;;;OAQG;IACG,eAAe,CACnB,WAAW,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,EAC7B,YAAY,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,EACjC,WAAW,EAAE,MAAM,EACnB,MAAM,CAAC,EAAE,OAAO,CAAC,gBAAgB,CAAC,GACjC,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAwC/B;;;;;OAKG;IACG,aAAa,CAAC,MAAM,EAAE,cAAc,GAAG,OAAO,CAAC;QAAE,OAAO,EAAE,OAAO,CAAC;QAAC,MAAM,CAAC,EAAE,cAAc,CAAA;KAAE,CAAC;IAkCnG;;OAEG;IACH,kBAAkB,IAAI,cAAc,EAAE;IAItC;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,SAAS,CAAC;IAItC;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAK5B;;OAEG;IACH,cAAc,IAAI,qBAAqB;CAGxC"}

package/dist/agentic-flow/src/services/rl-training-service.js

@@ -0,0 +1,401 @@
+/**
+ * RL Training Service - Reinforcement Learning Training Orchestration
+ *
+ * ADR-065 Phase P1-2: SONA RL Loop Implementation
+ *
+ * Provides high-level RL training coordination using SonaTrajectoryService
+ * for policy gradients, value functions, experience replay, multi-agent RL,
+ * and transfer learning.
+ *
+ * Features:
+ * - PPO (Proximal Policy Optimization)
+ * - A3C (Asynchronous Advantage Actor-Critic)
+ * - Q-Learning with experience replay
+ * - Multi-agent coordination
+ * - Transfer learning between tasks
+ * - Continuous learning loop
+ *
+ * Usage:
+ *   const trainer = new RLTrainingService();
+ *   await trainer.initialize();
+ *
+ *   // Train on episodes
+ *   await trainer.trainEpisodes(episodes, { algorithm: 'ppo' });
+ *
+ *   // Continuous learning
+ *   await trainer.learnFromExecution(state, action, reward, nextState);
+ */
+// Stub implementation - will be replaced when agentdb@3.x is available
+class SonaTrajectoryServiceStub {
+    async initialize() { }
+    async storePolicyGradient(config) {
+        return { success: true };
+    }
+    async storeValueFunction(config) {
+        return { success: true };
+    }
+    async storeExperienceReplay(config) {
+        return { success: true };
+    }
+    async getRLMetrics() {
+        return {
+            avgReward: 0,
+            avgLoss: 0,
+            episodeCount: 0,
+            episodeReward: 0,
+            loss: 0,
+            epsilon: 0.1,
+            iterationCount: 0
+        };
+    }
+    getEngineType() {
+        return 'stub';
+    }
+    async estimateValue(state, reward, nextState, config) {
+        return 0;
+    }
+    async trainPolicy(episodes, config) {
+        return 0;
+    }
+    addExperience(state, action, reward, nextState) { }
+    sampleExperience(batchSize) {
+        return [];
+    }
+    async continuousLearn(state, action, reward, nextState) {
+        return await this.getRLMetrics();
+    }
+    async multiAgentLearn(agentEpisodes, config) {
+        return await this.getRLMetrics();
+    }
+    async transferLearning(sourceTask, targetTask, config) {
+        return { success: true };
+    }
+    async getPatterns(minSupport) {
+        return [];
+    }
+    async resetRL() { }
+    configureRL(config) { }
+    getStats() {
+        return { totalExperiences: 0, avgReward: 0 };
+    }
+    async clear() { }
+}
+const SonaTrajectoryService = SonaTrajectoryServiceStub;
+export class RLTrainingService {
+    sona;
+    initialized = false;
+    trainingHistory = [];
+    constructor(sonaService) {
+        this.sona = sonaService || new SonaTrajectoryService();
+    }
+    /**
+     * Initialize the RL training service
+     */
+    async initialize() {
+        if (this.initialized)
+            return true;
+        await this.sona.initialize();
+        this.initialized = true;
+        console.log('[RLTrainingService] Initialized with engine:', this.sona.getEngineType());
+        return true;
+    }
+    /**
+     * Train on episodes using specified algorithm
+     *
+     * @param episodes - Training episodes (trajectories)
+     * @param config - Training configuration
+     * @returns Training results with learning curve
+     */
+    async trainEpisodes(episodes, config = {}) {
+        if (!this.initialized) {
+            await this.initialize();
+        }
+        const fullConfig = {
+            algorithm: config.algorithm || 'reinforce',
+            epochs: config.epochs || 100,
+            batchSize: config.batchSize || 32,
+            learningRate: config.learningRate || 0.001,
+            gamma: config.gamma || 0.99,
+            epsilon: config.epsilon || 0.1,
+            ppoClipRatio: config.ppoClipRatio || 0.2,
+            entropyCoeff: config.entropyCoeff || 0.01,
+            valueCoeff: config.valueCoeff || 0.5
+        };
+        // Configure SONA with training parameters
+        this.sona.configureRL({
+            policy: {
+                learningRate: fullConfig.learningRate,
+                gamma: fullConfig.gamma,
+                epsilon: fullConfig.epsilon,
+                entropyCoeff: fullConfig.entropyCoeff
+            },
+            value: {
+                learningRate: fullConfig.learningRate,
+                gamma: fullConfig.gamma,
+                lambda: 0.95
+            },
+            replay: {
+                bufferSize: 10000,
+                batchSize: fullConfig.batchSize,
+                priorityAlpha: 0.6,
+                priorityBeta: 0.4
+            }
+        });
+        const learningCurve = [];
+        let bestReward = -Infinity;
+        let convergenceIteration = fullConfig.epochs;
+        let improvementCount = 0;
+        console.log(`[RLTrainingService] Training ${episodes.length} episodes with ${fullConfig.algorithm}`);
+        for (let epoch = 0; epoch < fullConfig.epochs; epoch++) {
+            let loss = 0;
+            // Algorithm-specific training
+            switch (fullConfig.algorithm) {
+                case 'ppo':
+                    loss = await this.trainPPO(episodes, fullConfig);
+                    break;
+                case 'a3c':
+                    loss = await this.trainA3C(episodes, fullConfig);
+                    break;
+                case 'q-learning':
+                    loss = await this.trainQLearning(episodes, fullConfig);
+                    break;
+                case 'reinforce':
+                default:
+                    loss = await this.sona.trainPolicy(episodes, {
+                        learningRate: fullConfig.learningRate,
+                        gamma: fullConfig.gamma,
+                        epsilon: fullConfig.epsilon,
+                        entropyCoeff: fullConfig.entropyCoeff
+                    });
+                    break;
+            }
+            const metrics = await this.sona.getRLMetrics();
+            // Track learning curve
+            learningCurve.push({
+                iteration: epoch,
+                reward: metrics.avgReward || 0,
+                loss
+            });
+            // Check for improvement
+            if ((metrics.avgReward || 0) > bestReward) {
+                bestReward = metrics.avgReward || 0;
+                improvementCount++;
+                // Check for convergence (no improvement for 20 epochs)
+                if (epoch - convergenceIteration > 20) {
+                    convergenceIteration = epoch;
+                }
+            }
+            // Log progress every 10 epochs
+            if (epoch % 10 === 0) {
+                console.log(`[RLTrainingService] Epoch ${epoch}: Loss=${loss.toFixed(4)}, AvgReward=${(metrics.avgReward || 0).toFixed(4)}, Epsilon=${(metrics.epsilon || 0).toFixed(4)}`);
+            }
+        }
+        const finalMetrics = await this.sona.getRLMetrics();
+        const initialReward = learningCurve[0]?.reward || 0;
+        const finalReward = finalMetrics.avgReward || 0;
+        const improvementRate = initialReward === 0 ? 0 : ((finalReward - initialReward) / Math.abs(initialReward)) * 100;
+        const result = {
+            algorithm: fullConfig.algorithm,
+            epochs: fullConfig.epochs,
+            finalLoss: finalMetrics.loss || 0,
+            avgReward: finalReward,
+            improvementRate,
+            convergenceIteration,
+            learningCurve
+        };
+        this.trainingHistory.push(result);
+        console.log(`[RLTrainingService] Training complete: ${improvementRate.toFixed(2)}% improvement`);
+        return result;
+    }
+    /**
+     * PPO (Proximal Policy Optimization) training
+     */
+    async trainPPO(episodes, config) {
+        let totalLoss = 0;
+        // PPO uses multiple epochs on the same data
+        for (let ppoEpoch = 0; ppoEpoch < 4; ppoEpoch++) {
+            // Standard policy gradient with clipping
+            const loss = await this.sona.trainPolicy(episodes, {
+                learningRate: config.learningRate * (1 - ppoEpoch * 0.1), // decay learning rate
+                gamma: config.gamma,
+                epsilon: config.epsilon,
+                entropyCoeff: config.entropyCoeff
+            });
+            totalLoss += loss;
+        }
+        return totalLoss / 4;
+    }
+    /**
+     * A3C (Asynchronous Advantage Actor-Critic) training
+     */
+    async trainA3C(episodes, config) {
+        let totalLoss = 0;
+        let count = 0;
+        // Train actor (policy) and critic (value) separately
+        for (const episode of episodes) {
+            // Update value function
+            for (let i = 0; i < episode.steps.length - 1; i++) {
+                const step = episode.steps[i];
+                const nextStep = episode.steps[i + 1];
+                await this.sona.estimateValue(step.state, step.reward, nextStep.state, {
+                    learningRate: config.learningRate * (config.valueCoeff || 0.5),
+                    gamma: config.gamma,
+                    lambda: 0.95
+                });
+            }
+            count++;
+        }
+        // Update policy
+        const policyLoss = await this.sona.trainPolicy(episodes, {
+            learningRate: config.learningRate,
+            gamma: config.gamma,
+            epsilon: config.epsilon,
+            entropyCoeff: config.entropyCoeff
+        });
+        totalLoss += policyLoss;
+        return totalLoss / Math.max(count, 1);
+    }
+    /**
+     * Q-Learning with experience replay
+     */
+    async trainQLearning(episodes, config) {
+        // Add all experiences to replay buffer
+        for (const episode of episodes) {
+            for (let i = 0; i < episode.steps.length - 1; i++) {
+                const step = episode.steps[i];
+                const nextStep = episode.steps[i + 1];
+                this.sona.addExperience(step.state, step.action, step.reward, nextStep.state);
+            }
+        }
+        // Sample and train
+        let totalLoss = 0;
+        const numBatches = Math.floor(episodes.length / config.batchSize);
+        for (let i = 0; i < numBatches; i++) {
+            const batch = this.sona.sampleExperience(config.batchSize);
+            for (const exp of batch) {
+                const value = await this.sona.estimateValue(exp.state, exp.reward, exp.nextState, {
+                    learningRate: config.learningRate,
+                    gamma: config.gamma,
+                    lambda: 0.95
+                });
+                totalLoss += Math.abs(value - exp.reward);
+            }
+        }
+        return totalLoss / Math.max(numBatches, 1);
+    }
+    /**
+     * Learn from a single execution step (continuous learning)
+     *
+     * @param state - Current state
+     * @param action - Action taken
+     * @param reward - Reward received
+     * @param nextState - Resulting state
+     * @returns Updated value estimate
+     */
+    async learnFromExecution(state, action, reward, nextState) {
+        if (!this.initialized) {
+            await this.initialize();
+        }
+        return await this.sona.continuousLearn(state, action, reward, nextState);
+    }
+    /**
+     * Multi-agent reinforcement learning
+     *
+     * @param agentStates - Map of agent IDs to states
+     * @param agentActions - Map of agent IDs to actions
+     * @param jointReward - Shared reward
+     * @param config - Multi-agent configuration
+     * @returns Individual rewards for each agent
+     */
+    async multiAgentTrain(agentStates, agentActions, jointReward, config) {
+        if (!this.initialized) {
+            await this.initialize();
+        }
+        const fullConfig = {
+            agentTypes: config?.agentTypes || Array.from(agentStates.keys()),
+            coordinationStrategy: config?.coordinationStrategy || 'hybrid',
+            rewardSharing: config?.rewardSharing || 'contribution'
+        };
+        console.log(`[RLTrainingService] Multi-agent training: ${fullConfig.agentTypes.length} agents, ${fullConfig.rewardSharing} reward sharing`);
+        // Convert to episodes format for multiAgentLearn
+        const agentEpisodes = new Map();
+        for (const [agentId, state] of agentStates) {
+            const action = agentActions.get(agentId);
+            agentEpisodes.set(agentId, [{
+                    trajectoryId: agentId,
+                    state,
+                    action: action || '',
+                    reward: jointReward / agentStates.size,
+                    nextState: null,
+                    done: false,
+                    timestamp: Date.now(),
+                    steps: []
+                }]);
+        }
+        // Use SONA's multi-agent learning
+        await this.sona.multiAgentLearn(agentEpisodes, fullConfig);
+        // Return individual rewards (stub implementation)
+        const individualRewards = new Map();
+        for (const agentId of agentStates.keys()) {
+            individualRewards.set(agentId, jointReward / agentStates.size);
+        }
+        return individualRewards;
+    }
+    /**
+     * Transfer learning from source task to target task
+     *
+     * @param config - Transfer learning configuration
+     * @returns Success indicator and training result
+     */
+    async transferLearn(config) {
+        if (!this.initialized) {
+            await this.initialize();
+        }
+        console.log(`[RLTrainingService] Transfer learning: ${config.sourceTask} → ${config.targetTask} (ratio: ${config.transferRatio})`);
+        // Perform transfer
+        const success = await this.sona.transferLearning(config.sourceTask, config.targetTask, config.transferRatio);
+        if (!success) {
+            console.log('[RLTrainingService] Transfer failed: no source patterns available');
+            return { success: false };
+        }
+        // Fine-tune on target task if we have data
+        // TODO: Filter patterns by task when available
+        const targetPatterns = await this.sona.getPatterns();
+        if (targetPatterns.length > 0 && config.finetuneEpochs > 0) {
+            console.log(`[RLTrainingService] Fine-tuning on ${targetPatterns.length} target patterns`);
+            const result = await this.trainEpisodes(targetPatterns, {
+                algorithm: 'reinforce',
+                epochs: config.finetuneEpochs,
+                learningRate: 0.0005 // lower learning rate for fine-tuning
+            });
+            return { success: true, result };
+        }
+        return { success: true };
+    }
+    /**
+     * Get training history
+     */
+    getTrainingHistory() {
+        return [...this.trainingHistory];
+    }
+    /**
+     * Get current RL metrics
+     */
+    async getMetrics() {
+        return await this.sona.getRLMetrics();
+    }
+    /**
+     * Reset training state
+     */
+    async reset() {
+        await this.sona.resetRL();
+        this.trainingHistory = [];
+    }
+    /**
+     * Get the underlying SONA service
+     */
+    getSonaService() {
+        return this.sona;
+    }
+}
+//# sourceMappingURL=rl-training-service.js.map