agentic-flow 2.0.7 → 3.0.0-alpha.2

package/dist/mcp/fastmcp/tools/sona-tools.js

@@ -0,0 +1,615 @@
+/**
+ * SONA MCP Tools - Reinforcement Learning via Model Context Protocol
+ *
+ * ADR-065 Phase P1-2: SONA RL Loop MCP Integration
+ *
+ * Provides 8 MCP tools for reinforcement learning:
+ * 1. sona_train_policy - Train policy using RL algorithms
+ * 2. sona_estimate_value - Estimate state value
+ * 3. sona_replay_experience - Add/sample from experience replay
+ * 4. sona_transfer_learning - Transfer knowledge between tasks
+ * 5. sona_multi_agent - Multi-agent RL coordination
+ * 6. sona_continuous_learn - Continuous learning from execution
+ * 7. sona_get_metrics - Get RL training metrics
+ * 8. sona_reset - Reset RL state
+ */
+import { RLTrainingService } from '../../../services/rl-training-service.js';
+const rlService = new RLTrainingService();
+let initialized = false;
+async function ensureInitialized() {
+    if (!initialized) {
+        await rlService.initialize();
+        initialized = true;
+    }
+}
+/**
+ * Tool 1: Train policy using RL algorithms
+ */
+export const sonaTrainPolicyTool = {
+    name: 'sona_train_policy',
+    description: 'Train reinforcement learning policy on episode trajectories. Supports REINFORCE, PPO, A3C, Q-Learning. Returns learning curve and improvement metrics.',
+    inputSchema: {
+        type: 'object',
+        properties: {
+            episodes: {
+                type: 'array',
+                description: 'Training episodes (trajectories)',
+                items: {
+                    type: 'object',
+                    properties: {
+                        steps: {
+                            type: 'array',
+                            items: {
+                                type: 'object',
+                                properties: {
+                                    state: { type: 'object', description: 'State representation' },
+                                    action: { type: 'string', description: 'Action taken' },
+                                    reward: { type: 'number', description: 'Reward received' }
+                                },
+                                required: ['state', 'action', 'reward']
+                            }
+                        },
+                        reward: { type: 'number', description: 'Total episode reward' }
+                    },
+                    required: ['steps', 'reward']
+                }
+            },
+            algorithm: {
+                type: 'string',
+                enum: ['reinforce', 'ppo', 'a3c', 'q-learning'],
+                description: 'RL algorithm to use',
+                default: 'reinforce'
+            },
+            epochs: {
+                type: 'number',
+                description: 'Number of training epochs',
+                default: 100
+            },
+            learningRate: {
+                type: 'number',
+                description: 'Learning rate',
+                default: 0.001
+            },
+            gamma: {
+                type: 'number',
+                description: 'Discount factor',
+                default: 0.99
+            }
+        },
+        required: ['episodes']
+    }
+};
+export async function handleSonaTrainPolicy(args) {
+    await ensureInitialized();
+    const episodes = args.episodes || [];
+    if (episodes.length === 0) {
+        return JSON.stringify({
+            success: false,
+            error: 'No episodes provided for training'
+        }, null, 2);
+    }
+    const config = {
+        algorithm: args.algorithm || 'reinforce',
+        epochs: args.epochs || 100,
+        learningRate: args.learningRate || 0.001,
+        gamma: args.gamma || 0.99,
+        epsilon: args.epsilon || 0.1
+    };
+    try {
+        const result = await rlService.trainEpisodes(episodes, config);
+        return JSON.stringify({
+            success: true,
+            result: {
+                algorithm: result.algorithm,
+                epochs: result.epochs,
+                finalLoss: result.finalLoss,
+                avgReward: result.avgReward,
+                improvementRate: result.improvementRate,
+                convergenceIteration: result.convergenceIteration,
+                learningCurve: result.learningCurve
+            }
+        }, null, 2);
+    }
+    catch (error) {
+        return JSON.stringify({
+            success: false,
+            error: error instanceof Error ? error.message : String(error)
+        }, null, 2);
+    }
+}
+/**
+ * Tool 2: Estimate state value
+ */
+export const sonaEstimateValueTool = {
+    name: 'sona_estimate_value',
+    description: 'Estimate value function for a given state using TD learning. Returns estimated value.',
+    inputSchema: {
+        type: 'object',
+        properties: {
+            state: {
+                type: 'object',
+                description: 'Current state to estimate value for'
+            },
+            reward: {
+                type: 'number',
+                description: 'Observed reward'
+            },
+            nextState: {
+                type: 'object',
+                description: 'Next state'
+            },
+            learningRate: {
+                type: 'number',
+                description: 'Learning rate for value update',
+                default: 0.001
+            },
+            gamma: {
+                type: 'number',
+                description: 'Discount factor',
+                default: 0.99
+            }
+        },
+        required: ['state', 'reward', 'nextState']
+    }
+};
+export async function handleSonaEstimateValue(args) {
+    await ensureInitialized();
+    try {
+        const value = await rlService.getSonaService().estimateValue(args.state, args.reward, args.nextState, {
+            learningRate: args.learningRate || 0.001,
+            gamma: args.gamma || 0.99,
+            lambda: 0.95
+        });
+        return JSON.stringify({
+            success: true,
+            value,
+            state: args.state
+        }, null, 2);
+    }
+    catch (error) {
+        return JSON.stringify({
+            success: false,
+            error: error instanceof Error ? error.message : String(error)
+        }, null, 2);
+    }
+}
+/**
+ * Tool 3: Experience replay operations
+ */
+export const sonaReplayExperienceTool = {
+    name: 'sona_replay_experience',
+    description: 'Add experience to replay buffer or sample batch for training. Supports priority sampling.',
+    inputSchema: {
+        type: 'object',
+        properties: {
+            operation: {
+                type: 'string',
+                enum: ['add', 'sample'],
+                description: 'Operation: add experience or sample batch'
+            },
+            state: {
+                type: 'object',
+                description: 'Current state (for add operation)'
+            },
+            action: {
+                type: 'string',
+                description: 'Action taken (for add operation)'
+            },
+            reward: {
+                type: 'number',
+                description: 'Reward received (for add operation)'
+            },
+            nextState: {
+                type: 'object',
+                description: 'Next state (for add operation)'
+            },
+            priority: {
+                type: 'number',
+                description: 'Experience priority (for add operation)',
+                default: 1.0
+            },
+            batchSize: {
+                type: 'number',
+                description: 'Batch size (for sample operation)',
+                default: 32
+            }
+        },
+        required: ['operation']
+    }
+};
+export async function handleSonaReplayExperience(args) {
+    await ensureInitialized();
+    try {
+        if (args.operation === 'add') {
+            if (!args.state || !args.action || args.reward === undefined || !args.nextState) {
+                return JSON.stringify({
+                    success: false,
+                    error: 'Missing required fields for add operation: state, action, reward, nextState'
+                }, null, 2);
+            }
+            rlService.getSonaService().addExperience(args.state, args.action, args.reward, args.nextState, args.priority || 1.0);
+            return JSON.stringify({
+                success: true,
+                operation: 'add',
+                message: 'Experience added to replay buffer'
+            }, null, 2);
+        }
+        else if (args.operation === 'sample') {
+            const batch = rlService.getSonaService().sampleExperience(args.batchSize || 32);
+            return JSON.stringify({
+                success: true,
+                operation: 'sample',
+                batchSize: batch.length,
+                batch
+            }, null, 2);
+        }
+        else {
+            return JSON.stringify({
+                success: false,
+                error: 'Invalid operation. Must be "add" or "sample"'
+            }, null, 2);
+        }
+    }
+    catch (error) {
+        return JSON.stringify({
+            success: false,
+            error: error instanceof Error ? error.message : String(error)
+        }, null, 2);
+    }
+}
+/**
+ * Tool 4: Transfer learning
+ */
+export const sonaTransferLearningTool = {
+    name: 'sona_transfer_learning',
+    description: 'Transfer learned knowledge from source task to target task. Supports fine-tuning.',
+    inputSchema: {
+        type: 'object',
+        properties: {
+            sourceTask: {
+                type: 'string',
+                description: 'Source task/agent type to transfer from'
+            },
+            targetTask: {
+                type: 'string',
+                description: 'Target task/agent type to transfer to'
+            },
+            transferRatio: {
+                type: 'number',
+                description: 'How much knowledge to transfer (0-1)',
+                default: 0.7
+            },
+            finetuneEpochs: {
+                type: 'number',
+                description: 'Number of fine-tuning epochs',
+                default: 20
+            }
+        },
+        required: ['sourceTask', 'targetTask']
+    }
+};
+export async function handleSonaTransferLearning(args) {
+    await ensureInitialized();
+    const config = {
+        sourceTask: args.sourceTask,
+        targetTask: args.targetTask,
+        transferRatio: args.transferRatio || 0.7,
+        finetuneEpochs: args.finetuneEpochs || 20
+    };
+    try {
+        const result = await rlService.transferLearn(config);
+        return JSON.stringify({
+            success: result.success,
+            sourceTask: config.sourceTask,
+            targetTask: config.targetTask,
+            transferRatio: config.transferRatio,
+            finetuneResult: result.result
+        }, null, 2);
+    }
+    catch (error) {
+        return JSON.stringify({
+            success: false,
+            error: error instanceof Error ? error.message : String(error)
+        }, null, 2);
+    }
+}
+/**
+ * Tool 5: Multi-agent RL
+ */
+export const sonaMultiAgentTool = {
+    name: 'sona_multi_agent',
+    description: 'Multi-agent reinforcement learning coordination. Returns individual rewards for each agent.',
+    inputSchema: {
+        type: 'object',
+        properties: {
+            agentStates: {
+                type: 'object',
+                description: 'Map of agent IDs to their states',
+                additionalProperties: {
+                    type: 'object'
+                }
+            },
+            agentActions: {
+                type: 'object',
+                description: 'Map of agent IDs to their actions',
+                additionalProperties: {
+                    type: 'string'
+                }
+            },
+            jointReward: {
+                type: 'number',
+                description: 'Shared reward for all agents'
+            },
+            coordinationStrategy: {
+                type: 'string',
+                enum: ['centralized', 'decentralized', 'hybrid'],
+                description: 'Coordination strategy',
+                default: 'hybrid'
+            },
+            rewardSharing: {
+                type: 'string',
+                enum: ['equal', 'contribution', 'competitive'],
+                description: 'Reward sharing method',
+                default: 'contribution'
+            }
+        },
+        required: ['agentStates', 'agentActions', 'jointReward']
+    }
+};
+export async function handleSonaMultiAgent(args) {
+    await ensureInitialized();
+    try {
+        // Convert objects to Maps
+        const agentStates = new Map(Object.entries(args.agentStates || {}));
+        const agentActions = new Map(Object.entries(args.agentActions || {}));
+        if (agentStates.size === 0 || agentActions.size === 0) {
+            return JSON.stringify({
+                success: false,
+                error: 'No agents provided'
+            }, null, 2);
+        }
+        const config = {
+            agentTypes: Array.from(agentStates.keys()),
+            coordinationStrategy: args.coordinationStrategy || 'hybrid',
+            rewardSharing: args.rewardSharing || 'contribution'
+        };
+        const individualRewards = await rlService.multiAgentTrain(agentStates, agentActions, args.jointReward, config);
+        // Convert Map to object for JSON
+        const rewardsObj = Object.fromEntries(individualRewards);
+        return JSON.stringify({
+            success: true,
+            jointReward: args.jointReward,
+            individualRewards: rewardsObj,
+            numAgents: individualRewards.size
+        }, null, 2);
+    }
+    catch (error) {
+        return JSON.stringify({
+            success: false,
+            error: error instanceof Error ? error.message : String(error)
+        }, null, 2);
+    }
+}
+/**
+ * Tool 6: Continuous learning
+ */
+export const sonaContinuousLearnTool = {
+    name: 'sona_continuous_learn',
+    description: 'Continuous learning from single execution step. Updates policy and value function incrementally.',
+    inputSchema: {
+        type: 'object',
+        properties: {
+            state: {
+                type: 'object',
+                description: 'Current state'
+            },
+            action: {
+                type: 'string',
+                description: 'Action taken'
+            },
+            reward: {
+                type: 'number',
+                description: 'Reward received'
+            },
+            nextState: {
+                type: 'object',
+                description: 'Resulting state'
+            }
+        },
+        required: ['state', 'action', 'reward', 'nextState']
+    }
+};
+export async function handleSonaContinuousLearn(args) {
+    await ensureInitialized();
+    try {
+        const value = await rlService.learnFromExecution(args.state, args.action, args.reward, args.nextState);
+        const metrics = rlService.getMetrics();
+        return JSON.stringify({
+            success: true,
+            value,
+            metrics: {
+                episodeReward: metrics.episodeReward,
+                avgReward: metrics.avgReward,
+                loss: metrics.loss,
+                iterationCount: metrics.iterationCount
+            }
+        }, null, 2);
+    }
+    catch (error) {
+        return JSON.stringify({
+            success: false,
+            error: error instanceof Error ? error.message : String(error)
+        }, null, 2);
+    }
+}
+/**
+ * Tool 7: Get RL metrics
+ */
+export const sonaGetMetricsTool = {
+    name: 'sona_get_metrics',
+    description: 'Get current RL training metrics including rewards, loss, and training history.',
+    inputSchema: {
+        type: 'object',
+        properties: {
+            includeHistory: {
+                type: 'boolean',
+                description: 'Include full training history',
+                default: false
+            }
+        }
+    }
+};
+export async function handleSonaGetMetrics(args) {
+    await ensureInitialized();
+    try {
+        const metrics = rlService.getMetrics();
+        const sonaStats = rlService.getSonaService().getStats();
+        const result = {
+            success: true,
+            metrics: {
+                episodeReward: metrics.episodeReward,
+                avgReward: metrics.avgReward,
+                loss: metrics.loss,
+                epsilon: metrics.epsilon,
+                iterationCount: metrics.iterationCount
+            },
+            sonaStats: {
+                available: sonaStats.available,
+                engineType: rlService.getSonaService().getEngineType(),
+                trajectoryCount: sonaStats.trajectoryCount,
+                agentTypes: sonaStats.agentTypes
+            }
+        };
+        if (args.includeHistory) {
+            result.trainingHistory = rlService.getTrainingHistory();
+        }
+        return JSON.stringify(result, null, 2);
+    }
+    catch (error) {
+        return JSON.stringify({
+            success: false,
+            error: error instanceof Error ? error.message : String(error)
+        }, null, 2);
+    }
+}
+/**
+ * Tool 8: Reset RL state
+ */
+export const sonaResetTool = {
+    name: 'sona_reset',
+    description: 'Reset RL training state (policy, value function, experience replay). Use for new training session.',
+    inputSchema: {
+        type: 'object',
+        properties: {
+            clearTrajectories: {
+                type: 'boolean',
+                description: 'Also clear stored trajectories',
+                default: false
+            }
+        }
+    }
+};
+export async function handleSonaReset(args) {
+    await ensureInitialized();
+    try {
+        rlService.reset();
+        if (args.clearTrajectories) {
+            rlService.getSonaService().clear();
+        }
+        return JSON.stringify({
+            success: true,
+            message: 'RL state reset successfully',
+            trajectoriesCleared: args.clearTrajectories || false
+        }, null, 2);
+    }
+    catch (error) {
+        return JSON.stringify({
+            success: false,
+            error: error instanceof Error ? error.message : String(error)
+        }, null, 2);
+    }
+}
+// Export all tools and handlers
+export const sonaTools = [
+    sonaTrainPolicyTool,
+    sonaEstimateValueTool,
+    sonaReplayExperienceTool,
+    sonaTransferLearningTool,
+    sonaMultiAgentTool,
+    sonaContinuousLearnTool,
+    sonaGetMetricsTool,
+    sonaResetTool
+];
+export const sonaHandlers = {
+    sona_train_policy: handleSonaTrainPolicy,
+    sona_estimate_value: handleSonaEstimateValue,
+    sona_replay_experience: handleSonaReplayExperience,
+    sona_transfer_learning: handleSonaTransferLearning,
+    sona_multi_agent: handleSonaMultiAgent,
+    sona_continuous_learn: handleSonaContinuousLearn,
+    sona_get_metrics: handleSonaGetMetrics,
+    sona_reset: handleSonaReset
+};
+/**
+ * Register all SONA RL tools with FastMCP server
+ */
+export function registerSonaTools(server) {
+    // Tool 1: Train Policy
+    server.addTool({
+        name: 'sona_train_policy',
+        description: sonaTrainPolicyTool.description,
+        parameters: sonaTrainPolicyTool.inputSchema,
+        execute: handleSonaTrainPolicy
+    });
+    // Tool 2: Estimate Value
+    server.addTool({
+        name: 'sona_estimate_value',
+        description: sonaEstimateValueTool.description,
+        parameters: sonaEstimateValueTool.inputSchema,
+        execute: handleSonaEstimateValue
+    });
+    // Tool 3: Replay Experience
+    server.addTool({
+        name: 'sona_replay_experience',
+        description: sonaReplayExperienceTool.description,
+        parameters: sonaReplayExperienceTool.inputSchema,
+        execute: handleSonaReplayExperience
+    });
+    // Tool 4: Transfer Learning
+    server.addTool({
+        name: 'sona_transfer_learning',
+        description: sonaTransferLearningTool.description,
+        parameters: sonaTransferLearningTool.inputSchema,
+        execute: handleSonaTransferLearning
+    });
+    // Tool 5: Multi-Agent
+    server.addTool({
+        name: 'sona_multi_agent',
+        description: sonaMultiAgentTool.description,
+        parameters: sonaMultiAgentTool.inputSchema,
+        execute: handleSonaMultiAgent
+    });
+    // Tool 6: Continuous Learn
+    server.addTool({
+        name: 'sona_continuous_learn',
+        description: sonaContinuousLearnTool.description,
+        parameters: sonaContinuousLearnTool.inputSchema,
+        execute: handleSonaContinuousLearn
+    });
+    // Tool 7: Get Metrics
+    server.addTool({
+        name: 'sona_get_metrics',
+        description: sonaGetMetricsTool.description,
+        parameters: sonaGetMetricsTool.inputSchema,
+        execute: handleSonaGetMetrics
+    });
+    // Tool 8: Reset
+    server.addTool({
+        name: 'sona_reset',
+        description: sonaResetTool.description,
+        parameters: sonaResetTool.inputSchema,
+        execute: handleSonaReset
+    });
+    console.error('[SONA] Registered 8 RL tools: train_policy, estimate_value, replay_experience, transfer_learning, multi_agent, continuous_learn, get_metrics, reset');
+}
+//# sourceMappingURL=sona-tools.js.map