agentic-flow 3.0.0-alpha.1 → 3.0.0

package/dist/agentic-flow/src/services/rl-training-service.d.ts

@@ -1,233 +0,0 @@
-/**
- * RL Training Service - Reinforcement Learning Training Orchestration
- *
- * ADR-065 Phase P1-2: SONA RL Loop Implementation
- *
- * Provides high-level RL training coordination using SonaTrajectoryService
- * for policy gradients, value functions, experience replay, multi-agent RL,
- * and transfer learning.
- *
- * Features:
- * - PPO (Proximal Policy Optimization)
- * - A3C (Asynchronous Advantage Actor-Critic)
- * - Q-Learning with experience replay
- * - Multi-agent coordination
- * - Transfer learning between tasks
- * - Continuous learning loop
- *
- * Usage:
- *   const trainer = new RLTrainingService();
- *   await trainer.initialize();
- *
- *   // Train on episodes
- *   await trainer.trainEpisodes(episodes, { algorithm: 'ppo' });
- *
- *   // Continuous learning
- *   await trainer.learnFromExecution(state, action, reward, nextState);
- */
-export interface TrajectoryStep {
-    state: any;
-    action: any;
-    reward: number;
-    nextState?: any;
-    done?: boolean;
-}
-export interface StoredTrajectory {
-    trajectoryId: string;
-    state: any;
-    action: any;
-    reward: number;
-    nextState: any;
-    done: boolean;
-    timestamp: number;
-    steps: TrajectoryStep[];
-}
-export interface RLMetrics {
-    avgReward?: number;
-    avgLoss?: number;
-    episodeCount?: number;
-    episodeReward?: number;
-    loss?: number;
-    epsilon?: number;
-    iterationCount?: number;
-}
-interface ISonaTrajectoryService {
-    initialize(): Promise<void>;
-    storePolicyGradient(config: any): Promise<any>;
-    storeValueFunction(config: any): Promise<any>;
-    storeExperienceReplay(config: any): Promise<any>;
-    getRLMetrics(): Promise<RLMetrics>;
-    getEngineType(): string;
-    estimateValue(state: any, reward: number, nextState: any, config: any): Promise<number>;
-    trainPolicy(episodes: StoredTrajectory[], config: any): Promise<number>;
-    addExperience(state: any, action: any, reward: number, nextState: any): void;
-    sampleExperience(batchSize: number): any[];
-    continuousLearn(state: any, action: any, reward: number, nextState: any): Promise<RLMetrics>;
-    multiAgentLearn(agentEpisodes: Map<string, StoredTrajectory[]>, config: any): Promise<RLMetrics>;
-    transferLearning(sourceTask: string, targetTask: string, config: any): Promise<any>;
-    getPatterns(minSupport?: number): Promise<any[]>;
-    resetRL(): Promise<void>;
-    configureRL(config: any): void;
-    getStats(): any;
-    clear(): Promise<void>;
-}
-declare class SonaTrajectoryServiceStub implements ISonaTrajectoryService {
-    initialize(): Promise<void>;
-    storePolicyGradient(config: any): Promise<any>;
-    storeValueFunction(config: any): Promise<any>;
-    storeExperienceReplay(config: any): Promise<any>;
-    getRLMetrics(): Promise<RLMetrics>;
-    getEngineType(): string;
-    estimateValue(state: any, reward: number, nextState: any, config: any): Promise<number>;
-    trainPolicy(episodes: StoredTrajectory[], config: any): Promise<number>;
-    addExperience(state: any, action: any, reward: number, nextState: any): void;
-    sampleExperience(batchSize: number): any[];
-    continuousLearn(state: any, action: any, reward: number, nextState: any): Promise<RLMetrics>;
-    multiAgentLearn(agentEpisodes: Map<string, StoredTrajectory[]>, config: any): Promise<RLMetrics>;
-    transferLearning(sourceTask: string, targetTask: string, config: any): Promise<any>;
-    getPatterns(minSupport?: number): Promise<any[]>;
-    resetRL(): Promise<void>;
-    configureRL(config: any): void;
-    getStats(): any;
-    clear(): Promise<void>;
-}
-type SonaTrajectoryService = ISonaTrajectoryService;
-declare const SonaTrajectoryService: typeof SonaTrajectoryServiceStub;
-export interface PolicyGradientConfig {
-    learningRate?: number;
-    gamma?: number;
-    baseline?: 'none' | 'avg' | 'learned';
-}
-export interface ValueFunctionConfig {
-    learningRate?: number;
-    gamma?: number;
-    lambda?: number;
-}
-export interface ExperienceReplayConfig {
-    bufferSize?: number;
-    batchSize?: number;
-    minSize?: number;
-}
-export interface RLMetrics {
-    avgReward?: number;
-    avgLoss?: number;
-    episodeCount?: number;
-    episodeReward?: number;
-    loss?: number;
-    epsilon?: number;
-    iterationCount?: number;
-}
-export type RLAlgorithm = 'reinforce' | 'ppo' | 'a3c' | 'q-learning';
-export interface TrainingConfig {
-    algorithm: RLAlgorithm;
-    epochs: number;
-    batchSize: number;
-    learningRate: number;
-    gamma: number;
-    epsilon: number;
-    ppoClipRatio?: number;
-    entropyCoeff?: number;
-    valueCoeff?: number;
-}
-export interface TrainingResult {
-    algorithm: RLAlgorithm;
-    epochs: number;
-    finalLoss: number;
-    avgReward: number;
-    improvementRate: number;
-    convergenceIteration: number;
-    learningCurve: Array<{
-        iteration: number;
-        reward: number;
-        loss: number;
-    }>;
-}
-export interface MultiAgentConfig {
-    agentTypes: string[];
-    coordinationStrategy: 'centralized' | 'decentralized' | 'hybrid';
-    rewardSharing: 'equal' | 'contribution' | 'competitive';
-}
-export interface TransferConfig {
-    sourceTask: string;
-    targetTask: string;
-    transferRatio: number;
-    finetuneEpochs: number;
-}
-export declare class RLTrainingService {
-    private sona;
-    private initialized;
-    private trainingHistory;
-    constructor(sonaService?: SonaTrajectoryService);
-    /**
-     * Initialize the RL training service
-     */
-    initialize(): Promise<boolean>;
-    /**
-     * Train on episodes using specified algorithm
-     *
-     * @param episodes - Training episodes (trajectories)
-     * @param config - Training configuration
-     * @returns Training results with learning curve
-     */
-    trainEpisodes(episodes: StoredTrajectory[], config?: Partial<TrainingConfig>): Promise<TrainingResult>;
-    /**
-     * PPO (Proximal Policy Optimization) training
-     */
-    private trainPPO;
-    /**
-     * A3C (Asynchronous Advantage Actor-Critic) training
-     */
-    private trainA3C;
-    /**
-     * Q-Learning with experience replay
-     */
-    private trainQLearning;
-    /**
-     * Learn from a single execution step (continuous learning)
-     *
-     * @param state - Current state
-     * @param action - Action taken
-     * @param reward - Reward received
-     * @param nextState - Resulting state
-     * @returns Updated value estimate
-     */
-    learnFromExecution(state: any, action: string, reward: number, nextState: any): Promise<number>;
-    /**
-     * Multi-agent reinforcement learning
-     *
-     * @param agentStates - Map of agent IDs to states
-     * @param agentActions - Map of agent IDs to actions
-     * @param jointReward - Shared reward
-     * @param config - Multi-agent configuration
-     * @returns Individual rewards for each agent
-     */
-    multiAgentTrain(agentStates: Map<string, any>, agentActions: Map<string, string>, jointReward: number, config?: Partial<MultiAgentConfig>): Promise<Map<string, number>>;
-    /**
-     * Transfer learning from source task to target task
-     *
-     * @param config - Transfer learning configuration
-     * @returns Success indicator and training result
-     */
-    transferLearn(config: TransferConfig): Promise<{
-        success: boolean;
-        result?: TrainingResult;
-    }>;
-    /**
-     * Get training history
-     */
-    getTrainingHistory(): TrainingResult[];
-    /**
-     * Get current RL metrics
-     */
-    getMetrics(): Promise<RLMetrics>;
-    /**
-     * Reset training state
-     */
-    reset(): Promise<void>;
-    /**
-     * Get the underlying SONA service
-     */
-    getSonaService(): SonaTrajectoryService;
-}
-export {};
-//# sourceMappingURL=rl-training-service.d.ts.map

package/dist/agentic-flow/src/services/rl-training-service.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"rl-training-service.d.ts","sourceRoot":"","sources":["../../../../src/services/rl-training-service.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAGH,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,GAAG,CAAC;IACX,MAAM,EAAE,GAAG,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,GAAG,CAAC;IAChB,IAAI,CAAC,EAAE,OAAO,CAAC;CAChB;AAED,MAAM,WAAW,gBAAgB;IAC/B,YAAY,EAAE,MAAM,CAAC;IACrB,KAAK,EAAE,GAAG,CAAC;IACX,MAAM,EAAE,GAAG,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,GAAG,CAAC;IACf,IAAI,EAAE,OAAO,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,cAAc,EAAE,CAAC;CACzB;AAGD,MAAM,WAAW,SAAS;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAGD,UAAU,sBAAsB;IAC9B,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5B,mBAAmB,CAAC,MAAM,EAAE,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;IAC/C,kBAAkB,CAAC,MAAM,EAAE,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;IAC9C,qBAAqB,CAAC,MAAM,EAAE,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;IACjD,YAAY,IAAI,OAAO,CAAC,SAAS,CAAC,CAAC;IACnC,aAAa,IAAI,MAAM,CAAC;IACxB,aAAa,CAAC,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACxF,WAAW,CAAC,QAAQ,EAAE,gBAAgB,EAAE,EAAE,MAAM,EAAE,GAAG,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACxE,aAAa,CAAC,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,GAAG,IAAI,CAAC;IAC7E,gBAAgB,CAAC,SAAS,EAAE,MAAM,GAAG,GAAG,EAAE,CAAC;IAC3C,eAAe,CAAC,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC;IAC7F,eAAe,CAAC,aAAa,EAAE,GAAG,CAAC,MAAM,EAAE,gBAAgB,EAAE,CAAC,EAAE,MAAM,EAAE,GAAG,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC;IACjG,gBAAgB,CAAC,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;IACpF,WAAW,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;IACjD,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACzB,WAAW,CAAC,MAAM,EAAE,GAAG,GAAG,IAAI,CAAC;IAC/B,QAAQ,IAAI,GAAG,CAAC;IAChB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACxB;AAGD,cAAM,yBAA0B,YAAW,sBAAsB;IACzD,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAC3B,mBAAmB,CAAC,MAAM,EAAE,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC;IAG9C,kBAAkB,CAAC,MAAM,EAAE,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC;IAG7C,qBAAqB,CAAC,MAAM,EAAE,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC;IAGhD,YAAY,IAAI,OAAO,CAAC,SAAS,CAAC;IAWxC,aAAa,IAAI,MAAM;IAGjB,aAAa,CAAC,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,GAAG,OAAO,CAAC,MAAM,CAAC;IAGvF,WAAW,CAAC,QAAQ,EAAE,gBAAgB,EAAE,EAAE,MAAM,EAAE,GAAG,GAAG,OAAO,CAAC,MAAM,CAAC;IAG7E,aAAa,CAAC,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,GAAG,IAAI;IAC5E,gBAAgB,CAAC,SAAS,EAAE,MAAM,GAAG,GAAG,EAAE;IAGpC,eAAe,CAAC,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,GAAG,OAAO,CAAC,SAAS,CAAC;IAG5F,eAAe,CAAC,aAAa,EAAE,GAAG,CAAC,MAAM,EAAE,gBAAgB,EAAE,CAAC,EAAE,MAAM,EAAE,GAAG,GAAG,OAAO,CAAC,SAAS,CAAC;IAGhG,gBAAgB,CAAC,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC;IAGnF,WAAW,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;IAGhD,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAC9B,WAAW,CAAC,MAAM,EAAE,GAAG,GAAG,IAAI;IAC9B,QAAQ,IAAI,GAAG;IAGT,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAC7B;AAED,KAAK,qBAAqB,GAAG,sBAAsB,CAAC;AACpD,QAAA,MAAM,qBAAqB,kCAA4B,CAAC;AAGxD,MAAM,WAAW,oBAAoB;IACnC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,GAAG,KAAK,GAAG,SAAS,CAAC;CACvC;AAED,MAAM,WAAW,mBAAmB;IAClC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,sBAAsB;IACrC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,SAAS;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,MAAM,WAAW,GAAG,WAAW,GAAG,KAAK,GAAG,KAAK,GAAG,YAAY,CAAC;AAErE,MAAM,WAAW,cAAc;IAC7B,SAAS,EAAE,WAAW,CAAC;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,cAAc;IAC7B,SAAS,EAAE,WAAW,CAAC;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,eAAe,EAAE,MAAM,CAAC;IACxB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,aAAa,EAAE,KAAK,CAAC;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CAC3E;AAED,MAAM,WAAW,gBAAgB;IAC/B,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,oBAAoB,EAAE,aAAa,GAAG,eAAe,GAAG,QAAQ,CAAC;IACjE,aAAa,EAAE,OAAO,GAAG,cAAc,GAAG,aAAa,CAAC;CACzD;AAED,MAAM,WAAW,cAAc;IAC7B,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;CACxB;AAED,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,IAAI,CAAwB;IACpC,OAAO,CAAC,WAAW,CAAkB;IACrC,OAAO,CAAC,eAAe,CAAwB;gBAEnC,WAAW,CAAC,EAAE,qBAAqB;IAI/C;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC;IAUpC;;;;;;OAMG;IACG,aAAa,CAAC,QAAQ,EAAE,gBAAgB,EAAE,EAAE,MAAM,GAAE,OAAO,CAAC,cAAc,CAAM,GAAG,OAAO,CAAC,cAAc,CAAC;IAsHhH;;OAEG;YACW,QAAQ;IAmBtB;;OAEG;YACW,QAAQ;IAkCtB;;OAEG;YACW,cAAc;IAgC5B;;;;;;;;OAQG;IACG,kBAAkB,CAAC,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,GAAG,OAAO,CAAC,MAAM,CAAC;IAQrG;;;;;;;;OAQG;IACG,eAAe,CACnB,WAAW,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,EAC7B,YAAY,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,EACjC,WAAW,EAAE,MAAM,EACnB,MAAM,CAAC,EAAE,OAAO,CAAC,gBAAgB,CAAC,GACjC,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAwC/B;;;;;OAKG;IACG,aAAa,CAAC,MAAM,EAAE,cAAc,GAAG,OAAO,CAAC;QAAE,OAAO,EAAE,OAAO,CAAC;QAAC,MAAM,CAAC,EAAE,cAAc,CAAA;KAAE,CAAC;IAkCnG;;OAEG;IACH,kBAAkB,IAAI,cAAc,EAAE;IAItC;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,SAAS,CAAC;IAItC;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAK5B;;OAEG;IACH,cAAc,IAAI,qBAAqB;CAGxC"}

package/dist/agentic-flow/src/services/rl-training-service.js

@@ -1,401 +0,0 @@
-/**
- * RL Training Service - Reinforcement Learning Training Orchestration
- *
- * ADR-065 Phase P1-2: SONA RL Loop Implementation
- *
- * Provides high-level RL training coordination using SonaTrajectoryService
- * for policy gradients, value functions, experience replay, multi-agent RL,
- * and transfer learning.
- *
- * Features:
- * - PPO (Proximal Policy Optimization)
- * - A3C (Asynchronous Advantage Actor-Critic)
- * - Q-Learning with experience replay
- * - Multi-agent coordination
- * - Transfer learning between tasks
- * - Continuous learning loop
- *
- * Usage:
- *   const trainer = new RLTrainingService();
- *   await trainer.initialize();
- *
- *   // Train on episodes
- *   await trainer.trainEpisodes(episodes, { algorithm: 'ppo' });
- *
- *   // Continuous learning
- *   await trainer.learnFromExecution(state, action, reward, nextState);
- */
-// Stub implementation - will be replaced when agentdb@3.x is available
-class SonaTrajectoryServiceStub {
-    async initialize() { }
-    async storePolicyGradient(config) {
-        return { success: true };
-    }
-    async storeValueFunction(config) {
-        return { success: true };
-    }
-    async storeExperienceReplay(config) {
-        return { success: true };
-    }
-    async getRLMetrics() {
-        return {
-            avgReward: 0,
-            avgLoss: 0,
-            episodeCount: 0,
-            episodeReward: 0,
-            loss: 0,
-            epsilon: 0.1,
-            iterationCount: 0
-        };
-    }
-    getEngineType() {
-        return 'stub';
-    }
-    async estimateValue(state, reward, nextState, config) {
-        return 0;
-    }
-    async trainPolicy(episodes, config) {
-        return 0;
-    }
-    addExperience(state, action, reward, nextState) { }
-    sampleExperience(batchSize) {
-        return [];
-    }
-    async continuousLearn(state, action, reward, nextState) {
-        return await this.getRLMetrics();
-    }
-    async multiAgentLearn(agentEpisodes, config) {
-        return await this.getRLMetrics();
-    }
-    async transferLearning(sourceTask, targetTask, config) {
-        return { success: true };
-    }
-    async getPatterns(minSupport) {
-        return [];
-    }
-    async resetRL() { }
-    configureRL(config) { }
-    getStats() {
-        return { totalExperiences: 0, avgReward: 0 };
-    }
-    async clear() { }
-}
-const SonaTrajectoryService = SonaTrajectoryServiceStub;
-export class RLTrainingService {
-    sona;
-    initialized = false;
-    trainingHistory = [];
-    constructor(sonaService) {
-        this.sona = sonaService || new SonaTrajectoryService();
-    }
-    /**
-     * Initialize the RL training service
-     */
-    async initialize() {
-        if (this.initialized)
-            return true;
-        await this.sona.initialize();
-        this.initialized = true;
-        console.log('[RLTrainingService] Initialized with engine:', this.sona.getEngineType());
-        return true;
-    }
-    /**
-     * Train on episodes using specified algorithm
-     *
-     * @param episodes - Training episodes (trajectories)
-     * @param config - Training configuration
-     * @returns Training results with learning curve
-     */
-    async trainEpisodes(episodes, config = {}) {
-        if (!this.initialized) {
-            await this.initialize();
-        }
-        const fullConfig = {
-            algorithm: config.algorithm || 'reinforce',
-            epochs: config.epochs || 100,
-            batchSize: config.batchSize || 32,
-            learningRate: config.learningRate || 0.001,
-            gamma: config.gamma || 0.99,
-            epsilon: config.epsilon || 0.1,
-            ppoClipRatio: config.ppoClipRatio || 0.2,
-            entropyCoeff: config.entropyCoeff || 0.01,
-            valueCoeff: config.valueCoeff || 0.5
-        };
-        // Configure SONA with training parameters
-        this.sona.configureRL({
-            policy: {
-                learningRate: fullConfig.learningRate,
-                gamma: fullConfig.gamma,
-                epsilon: fullConfig.epsilon,
-                entropyCoeff: fullConfig.entropyCoeff
-            },
-            value: {
-                learningRate: fullConfig.learningRate,
-                gamma: fullConfig.gamma,
-                lambda: 0.95
-            },
-            replay: {
-                bufferSize: 10000,
-                batchSize: fullConfig.batchSize,
-                priorityAlpha: 0.6,
-                priorityBeta: 0.4
-            }
-        });
-        const learningCurve = [];
-        let bestReward = -Infinity;
-        let convergenceIteration = fullConfig.epochs;
-        let improvementCount = 0;
-        console.log(`[RLTrainingService] Training ${episodes.length} episodes with ${fullConfig.algorithm}`);
-        for (let epoch = 0; epoch < fullConfig.epochs; epoch++) {
-            let loss = 0;
-            // Algorithm-specific training
-            switch (fullConfig.algorithm) {
-                case 'ppo':
-                    loss = await this.trainPPO(episodes, fullConfig);
-                    break;
-                case 'a3c':
-                    loss = await this.trainA3C(episodes, fullConfig);
-                    break;
-                case 'q-learning':
-                    loss = await this.trainQLearning(episodes, fullConfig);
-                    break;
-                case 'reinforce':
-                default:
-                    loss = await this.sona.trainPolicy(episodes, {
-                        learningRate: fullConfig.learningRate,
-                        gamma: fullConfig.gamma,
-                        epsilon: fullConfig.epsilon,
-                        entropyCoeff: fullConfig.entropyCoeff
-                    });
-                    break;
-            }
-            const metrics = await this.sona.getRLMetrics();
-            // Track learning curve
-            learningCurve.push({
-                iteration: epoch,
-                reward: metrics.avgReward || 0,
-                loss
-            });
-            // Check for improvement
-            if ((metrics.avgReward || 0) > bestReward) {
-                bestReward = metrics.avgReward || 0;
-                improvementCount++;
-                // Check for convergence (no improvement for 20 epochs)
-                if (epoch - convergenceIteration > 20) {
-                    convergenceIteration = epoch;
-                }
-            }
-            // Log progress every 10 epochs
-            if (epoch % 10 === 0) {
-                console.log(`[RLTrainingService] Epoch ${epoch}: Loss=${loss.toFixed(4)}, AvgReward=${(metrics.avgReward || 0).toFixed(4)}, Epsilon=${(metrics.epsilon || 0).toFixed(4)}`);
-            }
-        }
-        const finalMetrics = await this.sona.getRLMetrics();
-        const initialReward = learningCurve[0]?.reward || 0;
-        const finalReward = finalMetrics.avgReward || 0;
-        const improvementRate = initialReward === 0 ? 0 : ((finalReward - initialReward) / Math.abs(initialReward)) * 100;
-        const result = {
-            algorithm: fullConfig.algorithm,
-            epochs: fullConfig.epochs,
-            finalLoss: finalMetrics.loss || 0,
-            avgReward: finalReward,
-            improvementRate,
-            convergenceIteration,
-            learningCurve
-        };
-        this.trainingHistory.push(result);
-        console.log(`[RLTrainingService] Training complete: ${improvementRate.toFixed(2)}% improvement`);
-        return result;
-    }
-    /**
-     * PPO (Proximal Policy Optimization) training
-     */
-    async trainPPO(episodes, config) {
-        let totalLoss = 0;
-        // PPO uses multiple epochs on the same data
-        for (let ppoEpoch = 0; ppoEpoch < 4; ppoEpoch++) {
-            // Standard policy gradient with clipping
-            const loss = await this.sona.trainPolicy(episodes, {
-                learningRate: config.learningRate * (1 - ppoEpoch * 0.1), // decay learning rate
-                gamma: config.gamma,
-                epsilon: config.epsilon,
-                entropyCoeff: config.entropyCoeff
-            });
-            totalLoss += loss;
-        }
-        return totalLoss / 4;
-    }
-    /**
-     * A3C (Asynchronous Advantage Actor-Critic) training
-     */
-    async trainA3C(episodes, config) {
-        let totalLoss = 0;
-        let count = 0;
-        // Train actor (policy) and critic (value) separately
-        for (const episode of episodes) {
-            // Update value function
-            for (let i = 0; i < episode.steps.length - 1; i++) {
-                const step = episode.steps[i];
-                const nextStep = episode.steps[i + 1];
-                await this.sona.estimateValue(step.state, step.reward, nextStep.state, {
-                    learningRate: config.learningRate * (config.valueCoeff || 0.5),
-                    gamma: config.gamma,
-                    lambda: 0.95
-                });
-            }
-            count++;
-        }
-        // Update policy
-        const policyLoss = await this.sona.trainPolicy(episodes, {
-            learningRate: config.learningRate,
-            gamma: config.gamma,
-            epsilon: config.epsilon,
-            entropyCoeff: config.entropyCoeff
-        });
-        totalLoss += policyLoss;
-        return totalLoss / Math.max(count, 1);
-    }
-    /**
-     * Q-Learning with experience replay
-     */
-    async trainQLearning(episodes, config) {
-        // Add all experiences to replay buffer
-        for (const episode of episodes) {
-            for (let i = 0; i < episode.steps.length - 1; i++) {
-                const step = episode.steps[i];
-                const nextStep = episode.steps[i + 1];
-                this.sona.addExperience(step.state, step.action, step.reward, nextStep.state);
-            }
-        }
-        // Sample and train
-        let totalLoss = 0;
-        const numBatches = Math.floor(episodes.length / config.batchSize);
-        for (let i = 0; i < numBatches; i++) {
-            const batch = this.sona.sampleExperience(config.batchSize);
-            for (const exp of batch) {
-                const value = await this.sona.estimateValue(exp.state, exp.reward, exp.nextState, {
-                    learningRate: config.learningRate,
-                    gamma: config.gamma,
-                    lambda: 0.95
-                });
-                totalLoss += Math.abs(value - exp.reward);
-            }
-        }
-        return totalLoss / Math.max(numBatches, 1);
-    }
-    /**
-     * Learn from a single execution step (continuous learning)
-     *
-     * @param state - Current state
-     * @param action - Action taken
-     * @param reward - Reward received
-     * @param nextState - Resulting state
-     * @returns Updated value estimate
-     */
-    async learnFromExecution(state, action, reward, nextState) {
-        if (!this.initialized) {
-            await this.initialize();
-        }
-        return await this.sona.continuousLearn(state, action, reward, nextState);
-    }
-    /**
-     * Multi-agent reinforcement learning
-     *
-     * @param agentStates - Map of agent IDs to states
-     * @param agentActions - Map of agent IDs to actions
-     * @param jointReward - Shared reward
-     * @param config - Multi-agent configuration
-     * @returns Individual rewards for each agent
-     */
-    async multiAgentTrain(agentStates, agentActions, jointReward, config) {
-        if (!this.initialized) {
-            await this.initialize();
-        }
-        const fullConfig = {
-            agentTypes: config?.agentTypes || Array.from(agentStates.keys()),
-            coordinationStrategy: config?.coordinationStrategy || 'hybrid',
-            rewardSharing: config?.rewardSharing || 'contribution'
-        };
-        console.log(`[RLTrainingService] Multi-agent training: ${fullConfig.agentTypes.length} agents, ${fullConfig.rewardSharing} reward sharing`);
-        // Convert to episodes format for multiAgentLearn
-        const agentEpisodes = new Map();
-        for (const [agentId, state] of agentStates) {
-            const action = agentActions.get(agentId);
-            agentEpisodes.set(agentId, [{
-                    trajectoryId: agentId,
-                    state,
-                    action: action || '',
-                    reward: jointReward / agentStates.size,
-                    nextState: null,
-                    done: false,
-                    timestamp: Date.now(),
-                    steps: []
-                }]);
-        }
-        // Use SONA's multi-agent learning
-        await this.sona.multiAgentLearn(agentEpisodes, fullConfig);
-        // Return individual rewards (stub implementation)
-        const individualRewards = new Map();
-        for (const agentId of agentStates.keys()) {
-            individualRewards.set(agentId, jointReward / agentStates.size);
-        }
-        return individualRewards;
-    }
-    /**
-     * Transfer learning from source task to target task
-     *
-     * @param config - Transfer learning configuration
-     * @returns Success indicator and training result
-     */
-    async transferLearn(config) {
-        if (!this.initialized) {
-            await this.initialize();
-        }
-        console.log(`[RLTrainingService] Transfer learning: ${config.sourceTask} → ${config.targetTask} (ratio: ${config.transferRatio})`);
-        // Perform transfer
-        const success = await this.sona.transferLearning(config.sourceTask, config.targetTask, config.transferRatio);
-        if (!success) {
-            console.log('[RLTrainingService] Transfer failed: no source patterns available');
-            return { success: false };
-        }
-        // Fine-tune on target task if we have data
-        // TODO: Filter patterns by task when available
-        const targetPatterns = await this.sona.getPatterns();
-        if (targetPatterns.length > 0 && config.finetuneEpochs > 0) {
-            console.log(`[RLTrainingService] Fine-tuning on ${targetPatterns.length} target patterns`);
-            const result = await this.trainEpisodes(targetPatterns, {
-                algorithm: 'reinforce',
-                epochs: config.finetuneEpochs,
-                learningRate: 0.0005 // lower learning rate for fine-tuning
-            });
-            return { success: true, result };
-        }
-        return { success: true };
-    }
-    /**
-     * Get training history
-     */
-    getTrainingHistory() {
-        return [...this.trainingHistory];
-    }
-    /**
-     * Get current RL metrics
-     */
-    async getMetrics() {
-        return await this.sona.getRLMetrics();
-    }
-    /**
-     * Reset training state
-     */
-    async reset() {
-        await this.sona.resetRL();
-        this.trainingHistory = [];
-    }
-    /**
-     * Get the underlying SONA service
-     */
-    getSonaService() {
-        return this.sona;
-    }
-}
-//# sourceMappingURL=rl-training-service.js.map