agentic-qe 2.1.2 → 2.2.1

package/dist/learning/algorithms/AbstractRLLearner.js

@@ -0,0 +1,300 @@
+"use strict";
+/**
+ * AbstractRLLearner - Base class for Reinforcement Learning algorithms
+ *
+ * Provides common functionality for all RL algorithms including:
+ * - Epsilon-greedy exploration policy
+ * - State/action encoding
+ * - Q-table management
+ * - Experience replay integration
+ * - Statistics tracking
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.AbstractRLLearner = void 0;
+const Logger_1 = require("../../utils/Logger");
+const ExperienceReplayBuffer_1 = require("../ExperienceReplayBuffer");
+/**
+ * Abstract base class for RL algorithms
+ */
+class AbstractRLLearner {
+    constructor(config) {
+        this.logger = Logger_1.Logger.getInstance();
+        this.config = config;
+        this.qTable = new Map();
+        this.stepCount = 0;
+        this.episodeCount = 0;
+        // Initialize experience replay buffer if enabled
+        if (this.config.useExperienceReplay) {
+            this.replayBuffer = new ExperienceReplayBuffer_1.ExperienceReplayBuffer({
+                maxSize: this.config.replayBufferSize,
+                minSize: this.config.batchSize,
+                prioritized: false
+            });
+        }
+        this.logger.info(`${this.constructor.name} initialized`, { config });
+    }
+    /**
+     * Select action using epsilon-greedy policy
+     * With probability ε, select random action (exploration)
+     * Otherwise, select action with highest Q-value (exploitation)
+     */
+    selectAction(state, availableActions) {
+        if (availableActions.length === 0) {
+            throw new Error('No available actions to select from');
+        }
+        // Exploration: random action
+        if (Math.random() < this.config.explorationRate) {
+            const randomIndex = Math.floor(Math.random() * availableActions.length);
+            return availableActions[randomIndex];
+        }
+        // Exploitation: best action based on Q-values
+        return this.getBestAction(state, availableActions);
+    }
+    /**
+     * Get best action based on current Q-values (greedy policy)
+     */
+    getBestAction(state, availableActions) {
+        const stateKey = this.encodeState(state);
+        const stateActions = this.qTable.get(stateKey);
+        if (!stateActions || stateActions.size === 0) {
+            // No Q-values yet, return random action
+            const randomIndex = Math.floor(Math.random() * availableActions.length);
+            return availableActions[randomIndex];
+        }
+        // Find action with highest Q-value
+        let bestAction = availableActions[0];
+        let bestValue = -Infinity;
+        for (const action of availableActions) {
+            const actionKey = this.encodeAction(action);
+            const qValue = stateActions.get(actionKey);
+            if (qValue && qValue.value > bestValue) {
+                bestValue = qValue.value;
+                bestAction = action;
+            }
+        }
+        return bestAction;
+    }
+    /**
+     * Get Q-value for a state-action pair
+     */
+    getQValue(state, action) {
+        const stateKey = this.encodeState(state);
+        const actionKey = this.encodeAction(action);
+        const stateActions = this.qTable.get(stateKey);
+        if (!stateActions) {
+            return 0;
+        }
+        const qValue = stateActions.get(actionKey);
+        return qValue?.value ?? 0;
+    }
+    /**
+     * Set Q-value for a state-action pair (protected for subclass use)
+     */
+    setQValue(stateKey, actionKey, value) {
+        if (!this.qTable.has(stateKey)) {
+            this.qTable.set(stateKey, new Map());
+        }
+        const stateActions = this.qTable.get(stateKey);
+        const currentQValue = stateActions.get(actionKey);
+        stateActions.set(actionKey, {
+            state: stateKey,
+            action: actionKey,
+            value,
+            updateCount: (currentQValue?.updateCount ?? 0) + 1,
+            lastUpdated: Date.now()
+        });
+    }
+    /**
+     * Get all Q-values for a state
+     */
+    getStateValues(state) {
+        const stateKey = this.encodeState(state);
+        const stateActions = this.qTable.get(stateKey);
+        if (!stateActions) {
+            return new Map();
+        }
+        const values = new Map();
+        for (const [actionKey, qValue] of stateActions.entries()) {
+            values.set(actionKey, qValue.value);
+        }
+        return values;
+    }
+    /**
+     * Get value of a state (max Q-value over all actions)
+     * V(s) = max_a Q(s,a)
+     */
+    getStateValue(state) {
+        const stateKey = this.encodeState(state);
+        const stateActions = this.qTable.get(stateKey);
+        if (!stateActions || stateActions.size === 0) {
+            return 0;
+        }
+        return Math.max(...Array.from(stateActions.values()).map(qv => qv.value));
+    }
+    /**
+     * Perform batch update using experience replay
+     * Samples random batch from replay buffer and updates Q-values
+     */
+    batchUpdate() {
+        if (!this.replayBuffer || !this.replayBuffer.canSample(this.config.batchSize)) {
+            return;
+        }
+        const batch = this.replayBuffer.sample(this.config.batchSize);
+        for (const experience of batch) {
+            this.update(experience);
+        }
+        this.logger.debug(`Performed batch update with ${batch.length} experiences`);
+    }
+    /**
+     * Decay exploration rate (epsilon)
+     * Called after each episode to gradually reduce exploration
+     */
+    decayExploration() {
+        this.config.explorationRate = Math.max(this.config.minExplorationRate, this.config.explorationRate * this.config.explorationDecay);
+    }
+    /**
+     * Mark end of episode
+     */
+    endEpisode() {
+        this.episodeCount++;
+        this.decayExploration();
+        // Perform batch update if using experience replay
+        if (this.config.useExperienceReplay) {
+            this.batchUpdate();
+        }
+    }
+    /**
+     * Encode state to string key for Q-table
+     * Creates normalized feature vector and discretizes for generalization
+     */
+    encodeState(state) {
+        // Create normalized feature vector
+        const features = [
+            state.taskComplexity,
+            state.requiredCapabilities.length / 10, // normalize
+            state.previousAttempts / 5, // normalize
+            state.availableResources,
+            state.timeConstraint ? Math.min(state.timeConstraint / 300000, 1) : 1 // normalize to 5 min
+        ];
+        // Round to reduce state space (discretization)
+        return features.map(f => Math.round(f * 10) / 10).join(',');
+    }
+    /**
+     * Encode action to string key for Q-table
+     */
+    encodeAction(action) {
+        return `${action.strategy}:${action.parallelization.toFixed(1)}:${action.retryPolicy}`;
+    }
+    /**
+     * Get current exploration rate (epsilon)
+     */
+    getExplorationRate() {
+        return this.config.explorationRate;
+    }
+    /**
+     * Get total number of learning steps
+     */
+    getStepCount() {
+        return this.stepCount;
+    }
+    /**
+     * Get total number of episodes
+     */
+    getEpisodeCount() {
+        return this.episodeCount;
+    }
+    /**
+     * Get Q-table size (number of state-action pairs)
+     */
+    getTableSize() {
+        let size = 0;
+        for (const stateActions of this.qTable.values()) {
+            size += stateActions.size;
+        }
+        return size;
+    }
+    /**
+     * Get statistics about learning progress
+     */
+    getStatistics() {
+        let totalQValue = 0;
+        let count = 0;
+        let maxQ = -Infinity;
+        let minQ = Infinity;
+        for (const stateActions of this.qTable.values()) {
+            for (const qValue of stateActions.values()) {
+                totalQValue += qValue.value;
+                maxQ = Math.max(maxQ, qValue.value);
+                minQ = Math.min(minQ, qValue.value);
+                count++;
+            }
+        }
+        return {
+            steps: this.stepCount,
+            episodes: this.episodeCount,
+            tableSize: count,
+            explorationRate: this.config.explorationRate,
+            avgQValue: count > 0 ? totalQValue / count : 0,
+            maxQValue: count > 0 ? maxQ : 0,
+            minQValue: count > 0 ? minQ : 0
+        };
+    }
+    /**
+     * Reset Q-table and learning state
+     */
+    reset() {
+        this.qTable.clear();
+        this.stepCount = 0;
+        this.episodeCount = 0;
+        this.config.explorationRate = this.getDefaultExplorationRate();
+        if (this.replayBuffer) {
+            this.replayBuffer.clear();
+        }
+        this.logger.info(`${this.constructor.name} reset to initial state`);
+    }
+    /**
+     * Export Q-table and state for persistence
+     */
+    export() {
+        const serializedQTable = {};
+        for (const [state, actions] of this.qTable.entries()) {
+            serializedQTable[state] = {};
+            for (const [action, qValue] of actions.entries()) {
+                serializedQTable[state][action] = qValue;
+            }
+        }
+        return {
+            qTable: serializedQTable,
+            config: { ...this.config },
+            stepCount: this.stepCount,
+            episodeCount: this.episodeCount
+        };
+    }
+    /**
+     * Import Q-table and state from persistence
+     */
+    import(state) {
+        this.qTable.clear();
+        for (const [stateKey, actions] of Object.entries(state.qTable)) {
+            const actionMap = new Map();
+            for (const [actionKey, qValue] of Object.entries(actions)) {
+                actionMap.set(actionKey, qValue);
+            }
+            this.qTable.set(stateKey, actionMap);
+        }
+        this.config = { ...state.config };
+        this.stepCount = state.stepCount;
+        this.episodeCount = state.episodeCount;
+        this.logger.info(`Imported Q-table with ${this.getTableSize()} state-action pairs`);
+    }
+    /**
+     * Get memory usage estimate in bytes
+     */
+    getMemoryUsage() {
+        const qTableSize = JSON.stringify(this.export().qTable).length;
+        const bufferSize = this.replayBuffer?.getMemoryUsage() ?? 0;
+        return qTableSize + bufferSize;
+    }
+}
+exports.AbstractRLLearner = AbstractRLLearner;
+//# sourceMappingURL=AbstractRLLearner.js.map

package/dist/learning/algorithms/AbstractRLLearner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"AbstractRLLearner.js","sourceRoot":"","sources":["../../../src/learning/algorithms/AbstractRLLearner.ts"],"names":[],"mappings":";AAAA;;;;;;;;;GASG;;;AAEH,+CAA4C;AAE5C,sEAAmE;AA2BnE;;GAEG;AACH,MAAsB,iBAAiB;IAQrC,YAAY,MAAgB;QAC1B,IAAI,CAAC,MAAM,GAAG,eAAM,CAAC,WAAW,EAAE,CAAC;QACnC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,IAAI,GAAG,EAAE,CAAC;QACxB,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC;QACnB,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC;QAEtB,iDAAiD;QACjD,IAAI,IAAI,CAAC,MAAM,CAAC,mBAAmB,EAAE,CAAC;YACpC,IAAI,CAAC,YAAY,GAAG,IAAI,+CAAsB,CAAC;gBAC7C,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,gBAAgB;gBACrC,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS;gBAC9B,WAAW,EAAE,KAAK;aACnB,CAAC,CAAC;QACL,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,cAAc,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;IACvE,CAAC;IAED;;;;OAIG;IACH,YAAY,CAAC,KAAgB,EAAE,gBAA+B;QAC5D,IAAI,gBAAgB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAClC,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;QACzD,CAAC;QAED,6BAA6B;QAC7B,IAAI,IAAI,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,eAAe,EAAE,CAAC;YAChD,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC;YACxE,OAAO,gBAAgB,CAAC,WAAW,CAAC,CAAC;QACvC,CAAC;QAED,8CAA8C;QAC9C,OAAO,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,gBAAgB,CAAC,CAAC;IACrD,CAAC;IAED;;OAEG;IACH,aAAa,CAAC,KAAgB,EAAE,gBAA+B;QAC7D,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QACzC,MAAM,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAE/C,IAAI,CAAC,YAAY,IAAI,YAAY,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;YAC7C,wCAAwC;YACxC,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC;YACxE,OAAO,gBAAgB,CAAC,WAAW,CAAC,CAAC;QACvC,CAAC;QAED,mCAAmC;QACnC,IAAI,UAAU,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC;QACrC,IAAI,SAAS,GAAG,CAAC,QAAQ,CAAC;QAE1B,KAAK,MAAM,MAAM,IAAI,gBAAgB,EAAE,CAAC;YACtC,MAAM,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;YAC5C,MAAM,MAAM,GAAG,YAAY,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YAE3C,IAAI,MAAM,IAAI,MAAM,CAAC,KAAK,GAAG,SAAS,EAAE,CAAC;gBACvC,SAAS,GAAG,MAAM,CAAC,KAAK,CAAC;gBACzB,UAAU,GAAG,MAAM,CAAC;YACtB,CAAC;QACH,CAAC;QAED,OAAO,UAAU,CAAC;IACpB,CAAC;IAQD;;OAEG;IACH,SAAS,CAAC,KAAgB,EAAE,MAAmB;QAC7C,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QACzC,MAAM,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;QAE5C,MAAM,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC/C,IAAI,CAAC,YAAY,EAAE,CAAC;YAClB,OAAO,CAAC,CAAC;QACX,CAAC;QAED,MAAM,MAAM,GAAG,YAAY,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAC3C,OAAO,MAAM,EAAE,KAAK,IAAI,CAAC,CAAC;IAC5B,CAAC;IAED;;OAEG;IACO,SAAS,CAAC,QAAgB,EAAE,SAAiB,EAAE,KAAa;QACpE,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC/B,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,IAAI,GAAG,EAAE,CAAC,CAAC;QACvC,CAAC;QACD,MAAM,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAE,CAAC;QAEhD,MAAM,aAAa,GAAG,YAAY,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAClD,YAAY,CAAC,GAAG,CAAC,SAAS,EAAE;YAC1B,KAAK,EAAE,QAAQ;YACf,MAAM,EAAE,SAAS;YACjB,KAAK;YACL,WAAW,EAAE,CAAC,aAAa,EAAE,WAAW,IAAI,CAAC,CAAC,GAAG,CAAC;YAClD,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE;SACxB,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,cAAc,CAAC,KAAgB;QAC7B,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QACzC,MAAM,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAE/C,IAAI,CAAC,YAAY,EAAE,CAAC;YAClB,OAAO,IAAI,GAAG,EAAE,CAAC;QACnB,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;QACzC,KAAK,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,IAAI,YAAY,CAAC,OAAO,EAAE,EAAE,CAAC;YACzD,MAAM,CAAC,GAAG,CAAC,SAAS,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;QACtC,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;;OAGG;IACH,aAAa,CAAC,KAAgB;QAC5B,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QACzC,MAAM,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAE/C,IAAI,CAAC,YAAY,IAAI,YAAY,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;YAC7C,OAAO,CAAC,CAAC;QACX,CAAC;QAED,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC;IAC5E,CAAC;IAED;;;OAGG;IACH,WAAW;QACT,IAAI,CAAC,IAAI,CAAC,YAAY,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;YAC9E,OAAO;QACT,CAAC;QAED,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAE9D,KAAK,MAAM,UAAU,IAAI,KAAK,EAAE,CAAC;YAC/B,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QAC1B,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,+BAA+B,KAAK,CAAC,MAAM,cAAc,CAAC,CAAC;IAC/E,CAAC;IAED;;;OAGG;IACH,gBAAgB;QACd,IAAI,CAAC,MAAM,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,CACpC,IAAI,CAAC,MAAM,CAAC,kBAAkB,EAC9B,IAAI,CAAC,MAAM,CAAC,eAAe,GAAG,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAC3D,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,UAAU;QACR,IAAI,CAAC,YAAY,EAAE,CAAC;QACpB,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAExB,kDAAkD;QAClD,IAAI,IAAI,CAAC,MAAM,CAAC,mBAAmB,EAAE,CAAC;YACpC,IAAI,CAAC,WAAW,EAAE,CAAC;QACrB,CAAC;IACH,CAAC;IAED;;;OAGG;IACO,WAAW,CAAC,KAAgB;QACpC,mCAAmC;QACnC,MAAM,QAAQ,GAAG;YACf,KAAK,CAAC,cAAc;YACpB,KAAK,CAAC,oBAAoB,CAAC,MAAM,GAAG,EAAE,EAAE,YAAY;YACpD,KAAK,CAAC,gBAAgB,GAAG,CAAC,EAAE,YAAY;YACxC,KAAK,CAAC,kBAAkB;YACxB,KAAK,CAAC,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,cAAc,GAAG,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,qBAAqB;SAC5F,CAAC;QAEF,+CAA+C;QAC/C,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC9D,CAAC;IAED;;OAEG;IACO,YAAY,CAAC,MAAmB;QACxC,OAAO,GAAG,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;IACzF,CAAC;IAED;;OAEG;IACH,kBAAkB;QAChB,OAAO,IAAI,CAAC,MAAM,CAAC,eAAe,CAAC;IACrC,CAAC;IAED;;OAEG;IACH,YAAY;QACV,OAAO,IAAI,CAAC,SAAS,CAAC;IACxB,CAAC;IAED;;OAEG;IACH,eAAe;QACb,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC;IAED;;OAEG;IACH,YAAY;QACV,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,KAAK,MAAM,YAAY,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC;YAChD,IAAI,IAAI,YAAY,CAAC,IAAI,CAAC;QAC5B,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;OAEG;IACH,aAAa;QASX,IAAI,WAAW,GAAG,CAAC,CAAC;QACpB,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,IAAI,IAAI,GAAG,CAAC,QAAQ,CAAC;QACrB,IAAI,IAAI,GAAG,QAAQ,CAAC;QAEpB,KAAK,MAAM,YAAY,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC;YAChD,KAAK,MAAM,MAAM,IAAI,YAAY,CAAC,MAAM,EAAE,EAAE,CAAC;gBAC3C,WAAW,IAAI,MAAM,CAAC,KAAK,CAAC;gBAC5B,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;gBACpC,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;gBACpC,KAAK,EAAE,CAAC;YACV,CAAC;QACH,CAAC;QAED,OAAO;YACL,KAAK,EAAE,IAAI,CAAC,SAAS;YACrB,QAAQ,EAAE,IAAI,CAAC,YAAY;YAC3B,SAAS,EAAE,KAAK;YAChB,eAAe,EAAE,IAAI,CAAC,MAAM,CAAC,eAAe;YAC5C,SAAS,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YAC9C,SAAS,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC/B,SAAS,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SAChC,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,KAAK;QACH,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QACpB,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC;QACnB,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC;QACtB,IAAI,CAAC,MAAM,CAAC,eAAe,GAAG,IAAI,CAAC,yBAAyB,EAAE,CAAC;QAE/D,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACtB,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC;QAC5B,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,yBAAyB,CAAC,CAAC;IACtE,CAAC;IAOD;;OAEG;IACH,MAAM;QAMJ,MAAM,gBAAgB,GAA2C,EAAE,CAAC;QAEpE,KAAK,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC;YACrD,gBAAgB,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;YAC7B,KAAK,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;gBACjD,gBAAgB,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,GAAG,MAAM,CAAC;YAC3C,CAAC;QACH,CAAC;QAED,OAAO;YACL,MAAM,EAAE,gBAAgB;YACxB,MAAM,EAAE,EAAE,GAAG,IAAI,CAAC,MAAM,EAAE;YAC1B,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,YAAY,EAAE,IAAI,CAAC,YAAY;SAChC,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,KAKN;QACC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QAEpB,KAAK,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;YAC/D,MAAM,SAAS,GAAG,IAAI,GAAG,EAAkB,CAAC;YAC5C,KAAK,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC1D,SAAS,CAAC,GAAG,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;YACnC,CAAC;YACD,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;QACvC,CAAC;QAED,IAAI,CAAC,MAAM,GAAG,EAAE,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;QAClC,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC,SAAS,CAAC;QACjC,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC,YAAY,CAAC;QAEvC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,yBAAyB,IAAI,CAAC,YAAY,EAAE,qBAAqB,CAAC,CAAC;IACtF,CAAC;IAED;;OAEG;IACH,cAAc;QACZ,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;QAC/D,MAAM,UAAU,GAAG,IAAI,CAAC,YAAY,EAAE,cAAc,EAAE,IAAI,CAAC,CAAC;QAC5D,OAAO,UAAU,GAAG,UAAU,CAAC;IACjC,CAAC;CACF;AAjXD,8CAiXC"}

package/dist/learning/algorithms/ActorCriticLearner.d.ts

@@ -0,0 +1,201 @@
+/**
+ * ActorCriticLearner - Actor-Critic Reinforcement Learning Algorithm
+ *
+ * Implements Advantage Actor-Critic (A2C) combining:
+ * - Actor: Policy network that selects actions using softmax policy
+ * - Critic: Value network that estimates state values for advantage calculation
+ *
+ * Key features:
+ * - Continuous action probabilities via softmax
+ * - Advantage-based updates to reduce variance
+ * - Entropy bonus for exploration
+ * - Policy gradient with baseline
+ *
+ * Update rules:
+ * - Critic (Value): V(s) += α_c * δ where δ = r + γV(s') - V(s)
+ * - Actor (Policy): π(a|s) += α_a * δ * ∇log(π(a|s)) + β * H(π)
+ *
+ * @module learning/algorithms/ActorCriticLearner
+ * @version 1.0.0
+ */
+import { AbstractRLLearner, RLConfig } from './AbstractRLLearner';
+import { TaskState, AgentAction, TaskExperience } from '../types';
+/**
+ * Configuration specific to Actor-Critic algorithm
+ */
+export interface ActorCriticConfig extends RLConfig {
+    /** Actor learning rate (α_a) - typically smaller than critic */
+    actorLearningRate: number;
+    /** Critic learning rate (α_c) */
+    criticLearningRate: number;
+    /** Entropy coefficient (β) for exploration bonus */
+    entropyCoefficient: number;
+    /** Temperature for softmax action selection */
+    temperature: number;
+    /** Whether to use advantage normalization */
+    normalizeAdvantage: boolean;
+    /** Target network update frequency (for stability) */
+    targetUpdateFrequency: number;
+}
+/**
+ * Policy entry storing action probabilities
+ */
+interface PolicyEntry {
+    action: string;
+    probability: number;
+    logProbability: number;
+    updateCount: number;
+    lastUpdated: number;
+}
+/**
+ * State value entry for critic
+ */
+interface StateValueEntry {
+    state: string;
+    value: number;
+    updateCount: number;
+    lastUpdated: number;
+}
+/**
+ * ActorCriticLearner - Advantage Actor-Critic implementation
+ *
+ * Combines policy gradient (actor) with value function approximation (critic)
+ * for more stable and efficient learning than pure Q-learning.
+ *
+ * Usage:
+ * ```typescript
+ * const ac = new ActorCriticLearner({
+ *   learningRate: 0.1,
+ *   actorLearningRate: 0.01,
+ *   criticLearningRate: 0.1,
+ *   discountFactor: 0.95,
+ *   explorationRate: 0.3,
+ *   explorationDecay: 0.995,
+ *   minExplorationRate: 0.01,
+ *   entropyCoefficient: 0.01,
+ *   temperature: 1.0,
+ *   normalizeAdvantage: true,
+ *   targetUpdateFrequency: 100,
+ *   useExperienceReplay: true,
+ *   replayBufferSize: 10000,
+ *   batchSize: 32
+ * });
+ *
+ * const action = ac.selectAction(state, availableActions);
+ * ac.update(experience);
+ * ```
+ */
+export declare class ActorCriticLearner extends AbstractRLLearner {
+    private actorConfig;
+    private policyTable;
+    private valueTable;
+    private targetValueTable;
+    private updatesSinceTargetSync;
+    private advantageHistory;
+    private readonly defaultExploration;
+    constructor(config: ActorCriticConfig);
+    /**
+     * Select action using softmax policy with exploration
+     * π(a|s) = exp(Q(s,a)/τ) / Σ_a' exp(Q(s,a')/τ)
+     */
+    selectAction(state: TaskState, availableActions: AgentAction[]): AgentAction;
+    /**
+     * Sample action from softmax policy distribution
+     */
+    private sampleFromPolicy;
+    /**
+     * Get softmax action probabilities
+     * π(a|s) = exp(preference(s,a)/τ) / Σ_a' exp(preference(s,a')/τ)
+     */
+    private getActionProbabilities;
+    /**
+     * Get preference for state-action pair from policy table
+     */
+    private getPreference;
+    /**
+     * Update actor and critic using temporal difference
+     *
+     * TD Error (advantage): δ = r + γV(s') - V(s)
+     * Critic update: V(s) += α_c * δ
+     * Actor update: preference(s,a) += α_a * δ * (1 - π(a|s))
+     */
+    update(experience: TaskExperience, nextAction?: AgentAction): void;
+    /**
+     * Update critic (value function)
+     * V(s) += α_c * δ
+     */
+    private updateCritic;
+    /**
+     * Update actor (policy)
+     * For softmax policy: preference(s,a) += α_a * δ * (1 - π(a|s))
+     * This increases preference for actions with positive advantage
+     */
+    private updateActor;
+    /**
+     * Calculate entropy bonus for a state
+     * H(π(·|s)) = -Σ_a π(a|s) log(π(a|s))
+     */
+    private calculateEntropyBonus;
+    /**
+     * Get softmax probability for a specific action
+     */
+    private softmaxProb;
+    /**
+     * Normalize advantage using running statistics
+     */
+    private normalizeAdvantage;
+    /**
+     * Get state value from value table
+     */
+    getStateValue(state: TaskState): number;
+    /**
+     * Get state value from target network (for stability)
+     */
+    private getTargetStateValue;
+    /**
+     * Sync target network with main network
+     */
+    private syncTargetNetwork;
+    /**
+     * Extract experience components
+     */
+    private extractExperience;
+    /**
+     * Get default exploration rate for reset
+     */
+    protected getDefaultExplorationRate(): number;
+    /**
+     * Get actor-critic specific statistics
+     */
+    getActorCriticStatistics(): {
+        valueTableSize: number;
+        policyTableSize: number;
+        avgStateValue: number;
+        avgEntropy: number;
+        advantageMean: number;
+        advantageStd: number;
+    };
+    /**
+     * Reset actor-critic specific state
+     */
+    reset(): void;
+    /**
+     * Export complete actor-critic state
+     */
+    exportActorCritic(): {
+        base: ReturnType<AbstractRLLearner['export']>;
+        valueTable: Record<string, StateValueEntry>;
+        policyTable: Record<string, Record<string, PolicyEntry>>;
+        actorConfig: ActorCriticConfig;
+    };
+    /**
+     * Import complete actor-critic state
+     */
+    importActorCritic(state: ReturnType<typeof this.exportActorCritic>): void;
+}
+/**
+ * Create default Actor-Critic configuration
+ */
+export declare function createDefaultActorCriticConfig(): ActorCriticConfig;
+export {};
+//# sourceMappingURL=ActorCriticLearner.d.ts.map

package/dist/learning/algorithms/ActorCriticLearner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ActorCriticLearner.d.ts","sourceRoot":"","sources":["../../../src/learning/algorithms/ActorCriticLearner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,EAAE,iBAAiB,EAAE,QAAQ,EAAU,MAAM,qBAAqB,CAAC;AAC1E,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,UAAU,CAAC;AAElE;;GAEG;AACH,MAAM,WAAW,iBAAkB,SAAQ,QAAQ;IACjD,gEAAgE;IAChE,iBAAiB,EAAE,MAAM,CAAC;IAC1B,iCAAiC;IACjC,kBAAkB,EAAE,MAAM,CAAC;IAC3B,oDAAoD;IACpD,kBAAkB,EAAE,MAAM,CAAC;IAC3B,+CAA+C;IAC/C,WAAW,EAAE,MAAM,CAAC;IACpB,6CAA6C;IAC7C,kBAAkB,EAAE,OAAO,CAAC;IAC5B,sDAAsD;IACtD,qBAAqB,EAAE,MAAM,CAAC;CAC/B;AAED;;GAEG;AACH,UAAU,WAAW;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;IACvB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,UAAU,eAAe;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;CACrB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,qBAAa,kBAAmB,SAAQ,iBAAiB;IACvD,OAAO,CAAC,WAAW,CAAoB;IACvC,OAAO,CAAC,WAAW,CAAwC;IAC3D,OAAO,CAAC,UAAU,CAA+B;IACjD,OAAO,CAAC,gBAAgB,CAA+B;IACvD,OAAO,CAAC,sBAAsB,CAAS;IACvC,OAAO,CAAC,gBAAgB,CAAW;IACnC,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAS;gBAEhC,MAAM,EAAE,iBAAiB;IAkBrC;;;OAGG;IACM,YAAY,CAAC,KAAK,EAAE,SAAS,EAAE,gBAAgB,EAAE,WAAW,EAAE,GAAG,WAAW;IAerF;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAmBxB;;;OAGG;IACH,OAAO,CAAC,sBAAsB;IAmB9B;;OAEG;IACH,OAAO,CAAC,aAAa;IAUrB;;;;;;OAMG;IACM,MAAM,CAAC,UAAU,EAAE,cAAc,EAAE,UAAU,CAAC,EAAE,WAAW,GAAG,IAAI;IA8C3E;;;OAGG;IACH,OAAO,CAAC,YAAY;IAYpB;;;;OAIG;IACH,OAAO,CAAC,WAAW;IA2BnB;;;OAGG;IACH,OAAO,CAAC,qBAAqB;IAuB7B;;OAEG;IACH,OAAO,CAAC,WAAW;IAwBnB;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAmB1B;;OAEG;IACM,aAAa,CAAC,KAAK,EAAE,SAAS,GAAG,MAAM;IAMhD;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAM3B;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAQzB;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAgBzB;;OAEG;IACH,SAAS,CAAC,yBAAyB,IAAI,MAAM;IAI7C;;OAEG;IACH,wBAAwB,IAAI;QAC1B,cAAc,EAAE,MAAM,CAAC;QACvB,eAAe,EAAE,MAAM,CAAC;QACxB,aAAa,EAAE,MAAM,CAAC;QACtB,UAAU,EAAE,MAAM,CAAC;QACnB,aAAa,EAAE,MAAM,CAAC;QACtB,YAAY,EAAE,MAAM,CAAC;KACtB;IA0CD;;OAEG;IACM,KAAK,IAAI,IAAI;IAUtB;;OAEG;IACH,iBAAiB,IAAI;QACnB,IAAI,EAAE,UAAU,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC,CAAC;QAC9C,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;QAC5C,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC,CAAC;QACzD,WAAW,EAAE,iBAAiB,CAAC;KAChC;IAsBD;;OAEG;IACH,iBAAiB,CAAC,KAAK,EAAE,UAAU,CAAC,OAAO,IAAI,CAAC,iBAAiB,CAAC,GAAG,IAAI;CAyB1E;AAED;;GAEG;AACH,wBAAgB,8BAA8B,IAAI,iBAAiB,CAiBlE"}