agentdb 1.0.12 → 1.1.0

package/dist/mcp/learning/core/policy-optimizer.js

@@ -1,251 +0,0 @@
-"use strict";
-/**
- * PolicyOptimizer - Optimizes action selection policy using reinforcement learning
- */
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.PolicyOptimizer = void 0;
-const experience_buffer_js_1 = require("./experience-buffer.js");
-class PolicyOptimizer {
-    constructor(learningRate = 0.1, discountFactor = 0.95, bufferSize = 10000) {
-        this.qTable = new Map();
-        this.learningRate = 0.1;
-        this.discountFactor = 0.95;
-        this.explorationRate = 0.1;
-        this.learningRate = learningRate;
-        this.discountFactor = discountFactor;
-        this.experienceBuffer = new experience_buffer_js_1.ExperienceBuffer(bufferSize);
-    }
-    /**
-     * Predict best action for current state
-     */
-    async predictAction(state, availableActions) {
-        const stateKey = this.encodeState(state);
-        const qValues = this.qTable.get(stateKey) || new Map();
-        // Get Q-values for available actions
-        const actionValues = [];
-        for (const action of availableActions) {
-            const value = qValues.get(action) || 0;
-            actionValues.push({ tool: action, value });
-        }
-        // Sort by Q-value (descending)
-        actionValues.sort((a, b) => b.value - a.value);
-        // Epsilon-greedy exploration
-        let recommendedAction;
-        if (Math.random() < this.explorationRate && actionValues.length > 1) {
-            // Explore: pick random action
-            const randomIdx = Math.floor(Math.random() * actionValues.length);
-            const action = actionValues[randomIdx];
-            recommendedAction = {
-                tool: action.tool,
-                params: {},
-                confidence: 0.5, // Lower confidence for exploration
-                reasoning: 'Exploration: trying alternative action to discover better strategies',
-            };
-        }
-        else {
-            // Exploit: pick best action
-            const action = actionValues[0];
-            const maxValue = actionValues[0].value;
-            const minValue = actionValues[actionValues.length - 1].value;
-            const range = maxValue - minValue || 1;
-            const confidence = Math.min(0.95, 0.5 + (action.value - minValue) / range / 2);
-            recommendedAction = {
-                tool: action.tool,
-                params: {},
-                confidence,
-                reasoning: `Best action based on ${this.getExperienceCount(stateKey)} past experiences with average reward ${action.value.toFixed(3)}`,
-            };
-        }
-        // Prepare alternatives
-        const alternatives = actionValues.slice(1, 4).map((action) => ({
-            tool: action.tool,
-            params: {}, // Empty params for alternatives
-            confidence: Math.max(0.1, action.value / (actionValues[0].value || 1)),
-            reasoning: `Alternative with Q-value ${action.value.toFixed(3)}`,
-        }));
-        return {
-            recommendedAction,
-            alternatives,
-        };
-    }
-    /**
-     * Update policy based on experience
-     */
-    async updatePolicy(experience) {
-        // Add to experience buffer
-        this.experienceBuffer.add(experience);
-        // Q-learning update
-        const stateKey = this.encodeState(experience.state);
-        const nextStateKey = this.encodeState(experience.nextState);
-        const action = experience.action.tool;
-        // Get or initialize Q-values
-        if (!this.qTable.has(stateKey)) {
-            this.qTable.set(stateKey, new Map());
-        }
-        const qValues = this.qTable.get(stateKey);
-        // Get current Q-value
-        const currentQ = qValues.get(action) || 0;
-        // Get max Q-value for next state
-        let maxNextQ = 0;
-        if (!experience.done) {
-            const nextQValues = this.qTable.get(nextStateKey);
-            if (nextQValues) {
-                maxNextQ = Math.max(...Array.from(nextQValues.values()));
-            }
-        }
-        // Q-learning update: Q(s,a) = Q(s,a) + α[r + γ max Q(s',a') - Q(s,a)]
-        const newQ = currentQ +
-            this.learningRate *
-                (experience.reward + this.discountFactor * maxNextQ - currentQ);
-        qValues.set(action, newQ);
-    }
-    /**
-     * Train policy on batch of experiences
-     */
-    async train(options = {}) {
-        const { batchSize = 32, epochs = 10, learningRate = this.learningRate, minExperiences = 100, } = options;
-        const startTime = Date.now();
-        let totalLoss = 0;
-        let experiencesProcessed = 0;
-        // Check if we have enough experiences
-        if (this.experienceBuffer.size() < minExperiences) {
-            return {
-                loss: 0,
-                accuracy: 0,
-                experiencesProcessed: 0,
-                trainingTime: 0,
-                improvements: {
-                    taskCompletionTime: 'N/A',
-                    tokenEfficiency: 'N/A',
-                    successRate: 'N/A',
-                },
-            };
-        }
-        const oldLearningRate = this.learningRate;
-        this.learningRate = learningRate;
-        // Training loop
-        for (let epoch = 0; epoch < epochs; epoch++) {
-            // Sample prioritized batch
-            const batch = this.experienceBuffer.samplePrioritized(batchSize);
-            for (const experience of batch) {
-                // Calculate TD error (used as loss)
-                const stateKey = this.encodeState(experience.state);
-                const nextStateKey = this.encodeState(experience.nextState);
-                const action = experience.action.tool;
-                const qValues = this.qTable.get(stateKey) || new Map();
-                const currentQ = qValues.get(action) || 0;
-                let maxNextQ = 0;
-                if (!experience.done) {
-                    const nextQValues = this.qTable.get(nextStateKey);
-                    if (nextQValues) {
-                        maxNextQ = Math.max(...Array.from(nextQValues.values()));
-                    }
-                }
-                const targetQ = experience.reward + this.discountFactor * maxNextQ;
-                const tdError = Math.abs(targetQ - currentQ);
-                totalLoss += tdError;
-                // Update Q-value
-                await this.updatePolicy(experience);
-                experiencesProcessed++;
-            }
-        }
-        this.learningRate = oldLearningRate;
-        const trainingTime = Date.now() - startTime;
-        const avgLoss = totalLoss / experiencesProcessed;
-        // Calculate improvements
-        const stats = this.experienceBuffer.getStats();
-        const improvements = {
-            taskCompletionTime: stats.avgReward > 0 ? '+15%' : 'N/A',
-            tokenEfficiency: stats.avgReward > 0.5 ? '+20%' : 'N/A',
-            successRate: stats.avgReward > 0.7 ? '+25%' : 'N/A',
-        };
-        return {
-            loss: avgLoss,
-            accuracy: Math.max(0, 1 - avgLoss), // Simple accuracy estimate
-            experiencesProcessed,
-            trainingTime,
-            improvements,
-        };
-    }
-    /**
-     * Get policy statistics
-     */
-    getPolicyStats() {
-        let totalQValue = 0;
-        let qValueCount = 0;
-        for (const qValues of this.qTable.values()) {
-            for (const value of qValues.values()) {
-                totalQValue += value;
-                qValueCount++;
-            }
-        }
-        return {
-            statesLearned: this.qTable.size,
-            totalExperiences: this.experienceBuffer.size(),
-            avgQValue: qValueCount > 0 ? totalQValue / qValueCount : 0,
-        };
-    }
-    /**
-     * Export policy for persistence
-     */
-    exportPolicy() {
-        const policy = {};
-        for (const [stateKey, qValues] of this.qTable.entries()) {
-            policy[stateKey] = Object.fromEntries(qValues);
-        }
-        return {
-            qTable: policy,
-            learningRate: this.learningRate,
-            discountFactor: this.discountFactor,
-            explorationRate: this.explorationRate,
-            stats: this.getPolicyStats(),
-        };
-    }
-    /**
-     * Import policy from persistence
-     */
-    importPolicy(policyData) {
-        this.qTable.clear();
-        if (policyData.qTable) {
-            for (const [stateKey, actions] of Object.entries(policyData.qTable)) {
-                this.qTable.set(stateKey, new Map(Object.entries(actions)));
-            }
-        }
-        if (policyData.learningRate) {
-            this.learningRate = policyData.learningRate;
-        }
-        if (policyData.discountFactor) {
-            this.discountFactor = policyData.discountFactor;
-        }
-        if (policyData.explorationRate) {
-            this.explorationRate = policyData.explorationRate;
-        }
-    }
-    /**
-     * Encode state as string key for Q-table
-     */
-    encodeState(state) {
-        // Simple encoding: hash of task description and available tools
-        const parts = [
-            state.taskDescription.substring(0, 50),
-            state.availableTools.sort().join(','),
-            state.context?.taskType || 'general',
-        ];
-        return parts.join('|');
-    }
-    /**
-     * Get experience count for state
-     */
-    getExperienceCount(stateKey) {
-        const qValues = this.qTable.get(stateKey);
-        return qValues ? qValues.size : 0;
-    }
-    /**
-     * Decay exploration rate over time
-     */
-    decayExploration(decayRate = 0.995) {
-        this.explorationRate = Math.max(0.01, this.explorationRate * decayRate);
-    }
-}
-exports.PolicyOptimizer = PolicyOptimizer;
-//# sourceMappingURL=policy-optimizer.js.map

package/dist/mcp/learning/core/policy-optimizer.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"policy-optimizer.js","sourceRoot":"","sources":["../../../../src/mcp/learning/core/policy-optimizer.ts"],"names":[],"mappings":";AAAA;;GAEG;;;AAUH,iEAA0D;AAE1D,MAAa,eAAe;IAO1B,YACE,eAAuB,GAAG,EAC1B,iBAAyB,IAAI,EAC7B,aAAqB,KAAK;QATpB,WAAM,GAAqC,IAAI,GAAG,EAAE,CAAC;QACrD,iBAAY,GAAW,GAAG,CAAC;QAC3B,mBAAc,GAAW,IAAI,CAAC;QAC9B,oBAAe,GAAW,GAAG,CAAC;QAQpC,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACjC,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;QACrC,IAAI,CAAC,gBAAgB,GAAG,IAAI,uCAAgB,CAAC,UAAU,CAAC,CAAC;IAC3D,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,aAAa,CACjB,KAAY,EACZ,gBAA0B;QAE1B,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QACzC,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,IAAI,GAAG,EAAE,CAAC;QAEvD,qCAAqC;QACrC,MAAM,YAAY,GAA2C,EAAE,CAAC;QAChE,KAAK,MAAM,MAAM,IAAI,gBAAgB,EAAE,CAAC;YACtC,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YACvC,YAAY,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;QAC7C,CAAC;QAED,+BAA+B;QAC/B,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;QAE/C,6BAA6B;QAC7B,IAAI,iBAAuG,CAAC;QAE5G,IAAI,IAAI,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,eAAe,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACpE,8BAA8B;YAC9B,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;YAClE,MAAM,MAAM,GAAG,YAAY,CAAC,SAAS,CAAC,CAAC;YACvC,iBAAiB,GAAG;gBAClB,IAAI,EAAE,MAAM,CAAC,IAAI;gBACjB,MAAM,EAAE,EAAE;gBACV,UAAU,EAAE,GAAG,EAAE,mCAAmC;gBACpD,SAAS,EAAE,sEAAsE;aAClF,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,4BAA4B;YAC5B,MAAM,MAAM,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC;YAC/B,MAAM,QAAQ,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;YACvC,MAAM,QAAQ,GAAG,YAAY,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC;YAC7D,MAAM,KAAK,GAAG,QAAQ,GAAG,QAAQ,IAAI,CAAC,CAAC;YACvC,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC,KAAK,GAAG,QAAQ,CAAC,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC;YAE/E,iBAAiB,GAAG;gBAClB,IAAI,EAAE,MAAM,CAAC,IAAI;gBACjB,MAAM,EAAE,EAAE;gBACV,UAAU;gBACV,SAAS,EAAE,wBAAwB,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,yCAAyC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE;aACvI,CAAC;QACJ,CAAC;QAED,uBAAuB;QACvB,MAAM,YAAY,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;YAC7D,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,MAAM,EAAE,EAAE,EAAE,gCAAgC;YAC5C,UAAU,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,KAAK,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC;YACtE,SAAS,EAAE,4BAA4B,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE;SACjE,CAAC,CAAC,CAAC;QAEJ,OAAO;YACL,iBAAiB;YACjB,YAAY;SACb,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,YAAY,CAAC,UAAsB;QACvC,2BAA2B;QAC3B,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAEtC,oBAAoB;QACpB,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;QACpD,MAAM,YAAY,GAAG,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;QAC5D,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC;QAEtC,6BAA6B;QAC7B,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC/B,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,IAAI,GAAG,EAAE,CAAC,CAAC;QACvC,CAAC;QACD,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAE,CAAC;QAE3C,sBAAsB;QACtB,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAE1C,iCAAiC;QACjC,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;YACrB,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;YAClD,IAAI,WAAW,EAAE,CAAC;gBAChB,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YAC3D,CAAC;QACH,CAAC;QAED,sEAAsE;QACtE,MAAM,IAAI,GACR,QAAQ;YACR,IAAI,CAAC,YAAY;gBACf,CAAC,UAAU,CAAC,MAAM,GAAG,IAAI,CAAC,cAAc,GAAG,QAAQ,GAAG,QAAQ,CAAC,CAAC;QAEpE,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAC5B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK,CAAC,UAA2B,EAAE;QACvC,MAAM,EACJ,SAAS,GAAG,EAAE,EACd,MAAM,GAAG,EAAE,EACX,YAAY,GAAG,IAAI,CAAC,YAAY,EAChC,cAAc,GAAG,GAAG,GACrB,GAAG,OAAO,CAAC;QAEZ,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,IAAI,oBAAoB,GAAG,CAAC,CAAC;QAE7B,sCAAsC;QACtC,IAAI,IAAI,CAAC,gBAAgB,CAAC,IAAI,EAAE,GAAG,cAAc,EAAE,CAAC;YAClD,OAAO;gBACL,IAAI,EAAE,CAAC;gBACP,QAAQ,EAAE,CAAC;gBACX,oBAAoB,EAAE,CAAC;gBACvB,YAAY,EAAE,CAAC;gBACf,YAAY,EAAE;oBACZ,kBAAkB,EAAE,KAAK;oBACzB,eAAe,EAAE,KAAK;oBACtB,WAAW,EAAE,KAAK;iBACnB;aACF,CAAC;QACJ,CAAC;QAED,MAAM,eAAe,GAAG,IAAI,CAAC,YAAY,CAAC;QAC1C,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QAEjC,gBAAgB;QAChB,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;YAC5C,2BAA2B;YAC3B,MAAM,KAAK,GAAG,IAAI,CAAC,gBAAgB,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;YAEjE,KAAK,MAAM,UAAU,IAAI,KAAK,EAAE,CAAC;gBAC/B,oCAAoC;gBACpC,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;gBACpD,MAAM,YAAY,GAAG,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;gBAC5D,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC;gBAEtC,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,IAAI,GAAG,EAAE,CAAC;gBACvD,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;gBAE1C,IAAI,QAAQ,GAAG,CAAC,CAAC;gBACjB,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;oBACrB,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;oBAClD,IAAI,WAAW,EAAE,CAAC;wBAChB,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;oBAC3D,CAAC;gBACH,CAAC;gBAED,MAAM,OAAO,GAAG,UAAU,CAAC,MAAM,GAAG,IAAI,CAAC,cAAc,GAAG,QAAQ,CAAC;gBACnE,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,GAAG,QAAQ,CAAC,CAAC;gBAC7C,SAAS,IAAI,OAAO,CAAC;gBAErB,iBAAiB;gBACjB,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,CAAC;gBACpC,oBAAoB,EAAE,CAAC;YACzB,CAAC;QACH,CAAC;QAED,IAAI,CAAC,YAAY,GAAG,eAAe,CAAC;QAEpC,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;QAC5C,MAAM,OAAO,GAAG,SAAS,GAAG,oBAAoB,CAAC;QAEjD,yBAAyB;QACzB,MAAM,KAAK,GAAG,IAAI,CAAC,gBAAgB,CAAC,QAAQ,EAAE,CAAC;QAC/C,MAAM,YAAY,GAAG;YACnB,kBAAkB,EAAE,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK;YACxD,eAAe,EAAE,KAAK,CAAC,SAAS,GAAG,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK;YACvD,WAAW,EAAE,KAAK,CAAC,SAAS,GAAG,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK;SACpD,CAAC;QAEF,OAAO;YACL,IAAI,EAAE,OAAO;YACb,QAAQ,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,EAAE,2BAA2B;YAC/D,oBAAoB;YACpB,YAAY;YACZ,YAAY;SACb,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,cAAc;QAKZ,IAAI,WAAW,GAAG,CAAC,CAAC;QACpB,IAAI,WAAW,GAAG,CAAC,CAAC;QAEpB,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC;YAC3C,KAAK,MAAM,KAAK,IAAI,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;gBACrC,WAAW,IAAI,KAAK,CAAC;gBACrB,WAAW,EAAE,CAAC;YAChB,CAAC;QACH,CAAC;QAED,OAAO;YACL,aAAa,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI;YAC/B,gBAAgB,EAAE,IAAI,CAAC,gBAAgB,CAAC,IAAI,EAAE;YAC9C,SAAS,EAAE,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;SAC3D,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,YAAY;QACV,MAAM,MAAM,GAAQ,EAAE,CAAC;QAEvB,KAAK,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC;YACxD,MAAM,CAAC,QAAQ,CAAC,GAAG,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;QACjD,CAAC;QAED,OAAO;YACL,MAAM,EAAE,MAAM;YACd,YAAY,EAAE,IAAI,CAAC,YAAY;YAC/B,cAAc,EAAE,IAAI,CAAC,cAAc;YACnC,eAAe,EAAE,IAAI,CAAC,eAAe;YACrC,KAAK,EAAE,IAAI,CAAC,cAAc,EAAE;SAC7B,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,YAAY,CAAC,UAAe;QAC1B,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QAEpB,IAAI,UAAU,CAAC,MAAM,EAAE,CAAC;YACtB,KAAK,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;gBACpE,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,IAAI,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,OAAc,CAAC,CAAC,CAAC,CAAC;YACrE,CAAC;QACH,CAAC;QAED,IAAI,UAAU,CAAC,YAAY,EAAE,CAAC;YAC5B,IAAI,CAAC,YAAY,GAAG,UAAU,CAAC,YAAY,CAAC;QAC9C,CAAC;QACD,IAAI,UAAU,CAAC,cAAc,EAAE,CAAC;YAC9B,IAAI,CAAC,cAAc,GAAG,UAAU,CAAC,cAAc,CAAC;QAClD,CAAC;QACD,IAAI,UAAU,CAAC,eAAe,EAAE,CAAC;YAC/B,IAAI,CAAC,eAAe,GAAG,UAAU,CAAC,eAAe,CAAC;QACpD,CAAC;IACH,CAAC;IAED;;OAEG;IACK,WAAW,CAAC,KAAY;QAC9B,gEAAgE;QAChE,MAAM,KAAK,GAAG;YACZ,KAAK,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,CAAC;YACtC,KAAK,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC;YACrC,KAAK,CAAC,OAAO,EAAE,QAAQ,IAAI,SAAS;SACrC,CAAC;QACF,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACzB,CAAC;IAED;;OAEG;IACK,kBAAkB,CAAC,QAAgB;QACzC,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC1C,OAAO,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IACpC,CAAC;IAED;;OAEG;IACH,gBAAgB,CAAC,YAAoB,KAAK;QACxC,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,eAAe,GAAG,SAAS,CAAC,CAAC;IAC1E,CAAC;CACF;AA5SD,0CA4SC"}

package/dist/mcp/learning/core/policy-optimizer.mjs

@@ -1,246 +0,0 @@
-/**
- * PolicyOptimizer - Optimizes action selection policy using reinforcement learning
- */
-import { ExperienceBuffer } from './experience-buffer.mjs';
-export class PolicyOptimizer {
-    constructor(learningRate = 0.1, discountFactor = 0.95, bufferSize = 10000) {
-        this.qTable = new Map();
-        this.learningRate = 0.1;
-        this.discountFactor = 0.95;
-        this.explorationRate = 0.1;
-        this.learningRate = learningRate;
-        this.discountFactor = discountFactor;
-        this.experienceBuffer = new ExperienceBuffer(bufferSize);
-    }
-    /**
-     * Predict best action for current state
-     */
-    async predictAction(state, availableActions) {
-        const stateKey = this.encodeState(state);
-        const qValues = this.qTable.get(stateKey) || new Map();
-        // Get Q-values for available actions
-        const actionValues = [];
-        for (const action of availableActions) {
-            const value = qValues.get(action) || 0;
-            actionValues.push({ tool: action, value });
-        }
-        // Sort by Q-value (descending)
-        actionValues.sort((a, b) => b.value - a.value);
-        // Epsilon-greedy exploration
-        let recommendedAction;
-        if (Math.random() < this.explorationRate && actionValues.length > 1) {
-            // Explore: pick random action
-            const randomIdx = Math.floor(Math.random() * actionValues.length);
-            const action = actionValues[randomIdx];
-            recommendedAction = {
-                tool: action.tool,
-                params: {},
-                confidence: 0.5, // Lower confidence for exploration
-                reasoning: 'Exploration: trying alternative action to discover better strategies',
-            };
-        }
-        else {
-            // Exploit: pick best action
-            const action = actionValues[0];
-            const maxValue = actionValues[0].value;
-            const minValue = actionValues[actionValues.length - 1].value;
-            const range = maxValue - minValue || 1;
-            const confidence = Math.min(0.95, 0.5 + (action.value - minValue) / range / 2);
-            recommendedAction = {
-                tool: action.tool,
-                params: {},
-                confidence,
-                reasoning: `Best action based on ${this.getExperienceCount(stateKey)} past experiences with average reward ${action.value.toFixed(3)}`,
-            };
-        }
-        // Prepare alternatives
-        const alternatives = actionValues.slice(1, 4).map((action) => ({
-            tool: action.tool,
-            params: {}, // Empty params for alternatives
-            confidence: Math.max(0.1, action.value / (actionValues[0].value || 1)),
-            reasoning: `Alternative with Q-value ${action.value.toFixed(3)}`,
-        }));
-        return {
-            recommendedAction,
-            alternatives,
-        };
-    }
-    /**
-     * Update policy based on experience
-     */
-    async updatePolicy(experience) {
-        // Add to experience buffer
-        this.experienceBuffer.add(experience);
-        // Q-learning update
-        const stateKey = this.encodeState(experience.state);
-        const nextStateKey = this.encodeState(experience.nextState);
-        const action = experience.action.tool;
-        // Get or initialize Q-values
-        if (!this.qTable.has(stateKey)) {
-            this.qTable.set(stateKey, new Map());
-        }
-        const qValues = this.qTable.get(stateKey);
-        // Get current Q-value
-        const currentQ = qValues.get(action) || 0;
-        // Get max Q-value for next state
-        let maxNextQ = 0;
-        if (!experience.done) {
-            const nextQValues = this.qTable.get(nextStateKey);
-            if (nextQValues) {
-                maxNextQ = Math.max(...Array.from(nextQValues.values()));
-            }
-        }
-        // Q-learning update: Q(s,a) = Q(s,a) + α[r + γ max Q(s',a') - Q(s,a)]
-        const newQ = currentQ +
-            this.learningRate *
-                (experience.reward + this.discountFactor * maxNextQ - currentQ);
-        qValues.set(action, newQ);
-    }
-    /**
-     * Train policy on batch of experiences
-     */
-    async train(options = {}) {
-        const { batchSize = 32, epochs = 10, learningRate = this.learningRate, minExperiences = 100, } = options;
-        const startTime = Date.now();
-        let totalLoss = 0;
-        let experiencesProcessed = 0;
-        // Check if we have enough experiences
-        if (this.experienceBuffer.size() < minExperiences) {
-            return {
-                loss: 0,
-                accuracy: 0,
-                experiencesProcessed: 0,
-                trainingTime: 0,
-                improvements: {
-                    taskCompletionTime: 'N/A',
-                    tokenEfficiency: 'N/A',
-                    successRate: 'N/A',
-                },
-            };
-        }
-        const oldLearningRate = this.learningRate;
-        this.learningRate = learningRate;
-        // Training loop
-        for (let epoch = 0; epoch < epochs; epoch++) {
-            // Sample prioritized batch
-            const batch = this.experienceBuffer.samplePrioritized(batchSize);
-            for (const experience of batch) {
-                // Calculate TD error (used as loss)
-                const stateKey = this.encodeState(experience.state);
-                const nextStateKey = this.encodeState(experience.nextState);
-                const action = experience.action.tool;
-                const qValues = this.qTable.get(stateKey) || new Map();
-                const currentQ = qValues.get(action) || 0;
-                let maxNextQ = 0;
-                if (!experience.done) {
-                    const nextQValues = this.qTable.get(nextStateKey);
-                    if (nextQValues) {
-                        maxNextQ = Math.max(...Array.from(nextQValues.values()));
-                    }
-                }
-                const targetQ = experience.reward + this.discountFactor * maxNextQ;
-                const tdError = Math.abs(targetQ - currentQ);
-                totalLoss += tdError;
-                // Update Q-value
-                await this.updatePolicy(experience);
-                experiencesProcessed++;
-            }
-        }
-        this.learningRate = oldLearningRate;
-        const trainingTime = Date.now() - startTime;
-        const avgLoss = totalLoss / experiencesProcessed;
-        // Calculate improvements
-        const stats = this.experienceBuffer.getStats();
-        const improvements = {
-            taskCompletionTime: stats.avgReward > 0 ? '+15%' : 'N/A',
-            tokenEfficiency: stats.avgReward > 0.5 ? '+20%' : 'N/A',
-            successRate: stats.avgReward > 0.7 ? '+25%' : 'N/A',
-        };
-        return {
-            loss: avgLoss,
-            accuracy: Math.max(0, 1 - avgLoss), // Simple accuracy estimate
-            experiencesProcessed,
-            trainingTime,
-            improvements,
-        };
-    }
-    /**
-     * Get policy statistics
-     */
-    getPolicyStats() {
-        let totalQValue = 0;
-        let qValueCount = 0;
-        for (const qValues of this.qTable.values()) {
-            for (const value of qValues.values()) {
-                totalQValue += value;
-                qValueCount++;
-            }
-        }
-        return {
-            statesLearned: this.qTable.size,
-            totalExperiences: this.experienceBuffer.size(),
-            avgQValue: qValueCount > 0 ? totalQValue / qValueCount : 0,
-        };
-    }
-    /**
-     * Export policy for persistence
-     */
-    exportPolicy() {
-        const policy = {};
-        for (const [stateKey, qValues] of this.qTable.entries()) {
-            policy[stateKey] = Object.fromEntries(qValues);
-        }
-        return {
-            qTable: policy,
-            learningRate: this.learningRate,
-            discountFactor: this.discountFactor,
-            explorationRate: this.explorationRate,
-            stats: this.getPolicyStats(),
-        };
-    }
-    /**
-     * Import policy from persistence
-     */
-    importPolicy(policyData) {
-        this.qTable.clear();
-        if (policyData.qTable) {
-            for (const [stateKey, actions] of Object.entries(policyData.qTable)) {
-                this.qTable.set(stateKey, new Map(Object.entries(actions)));
-            }
-        }
-        if (policyData.learningRate) {
-            this.learningRate = policyData.learningRate;
-        }
-        if (policyData.discountFactor) {
-            this.discountFactor = policyData.discountFactor;
-        }
-        if (policyData.explorationRate) {
-            this.explorationRate = policyData.explorationRate;
-        }
-    }
-    /**
-     * Encode state as string key for Q-table
-     */
-    encodeState(state) {
-        // Simple encoding: hash of task description and available tools
-        const parts = [
-            state.taskDescription.substring(0, 50),
-            state.availableTools.sort().join(','),
-            state.context?.taskType || 'general',
-        ];
-        return parts.join('|');
-    }
-    /**
-     * Get experience count for state
-     */
-    getExperienceCount(stateKey) {
-        const qValues = this.qTable.get(stateKey);
-        return qValues ? qValues.size : 0;
-    }
-    /**
-     * Decay exploration rate over time
-     */
-    decayExploration(decayRate = 0.995) {
-        this.explorationRate = Math.max(0.01, this.explorationRate * decayRate);
-    }
-}

package/dist/mcp/learning/core/reward-estimator.d.ts

@@ -1,44 +0,0 @@
-/**
- * RewardEstimator - Calculates multi-dimensional rewards for actions
- */
-import type { Outcome, ExecutionContext, Reward } from '../types/index.js';
-export declare class RewardEstimator {
-    private weights;
-    /**
-     * Calculate comprehensive reward signal
-     */
-    calculateReward(outcome: Outcome, context: ExecutionContext): Promise<Reward>;
-    /**
-     * Calculate reward with user feedback
-     */
-    calculateRewardWithFeedback(outcome: Outcome, context: ExecutionContext, userRating: number): Promise<Reward>;
-    /**
-     * Success dimension: binary success/failure
-     */
-    private calculateSuccessReward;
-    /**
-     * Efficiency dimension: execution time
-     */
-    private calculateEfficiencyReward;
-    /**
-     * Quality dimension: based on error presence and result completeness
-     */
-    private calculateQualityReward;
-    /**
-     * Cost dimension: token usage efficiency
-     */
-    private calculateCostReward;
-    /**
-     * Objective metrics reward
-     */
-    private calculateObjectiveReward;
-    /**
-     * Update reward weights based on user preferences
-     */
-    setRewardWeights(weights: Partial<typeof this.weights>): void;
-    /**
-     * Get current reward weights
-     */
-    getRewardWeights(): typeof this.weights;
-}
-//# sourceMappingURL=reward-estimator.d.ts.map

package/dist/mcp/learning/core/reward-estimator.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"reward-estimator.d.ts","sourceRoot":"","sources":["../../../../src/mcp/learning/core/reward-estimator.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,OAAO,EAAE,gBAAgB,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAE3E,qBAAa,eAAe;IAC1B,OAAO,CAAC,OAAO,CAKb;IAEF;;OAEG;IACG,eAAe,CACnB,OAAO,EAAE,OAAO,EAChB,OAAO,EAAE,gBAAgB,GACxB,OAAO,CAAC,MAAM,CAAC;IAyBlB;;OAEG;IACG,2BAA2B,CAC/B,OAAO,EAAE,OAAO,EAChB,OAAO,EAAE,gBAAgB,EACzB,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,MAAM,CAAC;IAgBlB;;OAEG;IACH,OAAO,CAAC,sBAAsB;IAI9B;;OAEG;IACH,OAAO,CAAC,yBAAyB;IAQjC;;OAEG;IACH,OAAO,CAAC,sBAAsB;IAqB9B;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAY3B;;OAEG;IACH,OAAO,CAAC,wBAAwB;IAsChC;;OAEG;IACH,gBAAgB,CAAC,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,IAAI;IAa7D;;OAEG;IACH,gBAAgB,IAAI,OAAO,IAAI,CAAC,OAAO;CAGxC"}