agentic-qe 2.1.2 → 2.2.0

package/dist/learning/algorithms/ActorCriticLearner.js

@@ -0,0 +1,447 @@
+"use strict";
+/**
+ * ActorCriticLearner - Actor-Critic Reinforcement Learning Algorithm
+ *
+ * Implements Advantage Actor-Critic (A2C) combining:
+ * - Actor: Policy network that selects actions using softmax policy
+ * - Critic: Value network that estimates state values for advantage calculation
+ *
+ * Key features:
+ * - Continuous action probabilities via softmax
+ * - Advantage-based updates to reduce variance
+ * - Entropy bonus for exploration
+ * - Policy gradient with baseline
+ *
+ * Update rules:
+ * - Critic (Value): V(s) += α_c * δ where δ = r + γV(s') - V(s)
+ * - Actor (Policy): π(a|s) += α_a * δ * ∇log(π(a|s)) + β * H(π)
+ *
+ * @module learning/algorithms/ActorCriticLearner
+ * @version 1.0.0
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ActorCriticLearner = void 0;
+exports.createDefaultActorCriticConfig = createDefaultActorCriticConfig;
+const AbstractRLLearner_1 = require("./AbstractRLLearner");
+/**
+ * ActorCriticLearner - Advantage Actor-Critic implementation
+ *
+ * Combines policy gradient (actor) with value function approximation (critic)
+ * for more stable and efficient learning than pure Q-learning.
+ *
+ * Usage:
+ * ```typescript
+ * const ac = new ActorCriticLearner({
+ *   learningRate: 0.1,
+ *   actorLearningRate: 0.01,
+ *   criticLearningRate: 0.1,
+ *   discountFactor: 0.95,
+ *   explorationRate: 0.3,
+ *   explorationDecay: 0.995,
+ *   minExplorationRate: 0.01,
+ *   entropyCoefficient: 0.01,
+ *   temperature: 1.0,
+ *   normalizeAdvantage: true,
+ *   targetUpdateFrequency: 100,
+ *   useExperienceReplay: true,
+ *   replayBufferSize: 10000,
+ *   batchSize: 32
+ * });
+ *
+ * const action = ac.selectAction(state, availableActions);
+ * ac.update(experience);
+ * ```
+ */
+class ActorCriticLearner extends AbstractRLLearner_1.AbstractRLLearner {
+    constructor(config) {
+        super(config);
+        this.actorConfig = config;
+        this.policyTable = new Map();
+        this.valueTable = new Map();
+        this.targetValueTable = new Map();
+        this.updatesSinceTargetSync = 0;
+        this.advantageHistory = [];
+        this.defaultExploration = config.explorationRate;
+        this.logger.info('ActorCriticLearner initialized', {
+            actorLR: config.actorLearningRate,
+            criticLR: config.criticLearningRate,
+            entropy: config.entropyCoefficient,
+            temperature: config.temperature
+        });
+    }
+    /**
+     * Select action using softmax policy with exploration
+     * π(a|s) = exp(Q(s,a)/τ) / Σ_a' exp(Q(s,a')/τ)
+     */
+    selectAction(state, availableActions) {
+        if (availableActions.length === 0) {
+            throw new Error('No available actions to select from');
+        }
+        // With probability ε, use random action (exploration fallback)
+        if (Math.random() < this.config.explorationRate) {
+            const randomIndex = Math.floor(Math.random() * availableActions.length);
+            return availableActions[randomIndex];
+        }
+        // Use softmax policy
+        return this.sampleFromPolicy(state, availableActions);
+    }
+    /**
+     * Sample action from softmax policy distribution
+     */
+    sampleFromPolicy(state, availableActions) {
+        const stateKey = this.encodeState(state);
+        const probabilities = this.getActionProbabilities(stateKey, availableActions);
+        // Sample from categorical distribution
+        const random = Math.random();
+        let cumulative = 0;
+        for (let i = 0; i < availableActions.length; i++) {
+            cumulative += probabilities[i];
+            if (random <= cumulative) {
+                return availableActions[i];
+            }
+        }
+        // Fallback (shouldn't reach here due to normalization)
+        return availableActions[availableActions.length - 1];
+    }
+    /**
+     * Get softmax action probabilities
+     * π(a|s) = exp(preference(s,a)/τ) / Σ_a' exp(preference(s,a')/τ)
+     */
+    getActionProbabilities(stateKey, availableActions) {
+        const temperature = this.actorConfig.temperature;
+        const preferences = [];
+        // Get preferences (Q-values or policy table values)
+        for (const action of availableActions) {
+            const actionKey = this.encodeAction(action);
+            const preference = this.getPreference(stateKey, actionKey);
+            preferences.push(preference / temperature);
+        }
+        // Softmax with numerical stability
+        const maxPref = Math.max(...preferences);
+        const expPrefs = preferences.map(p => Math.exp(p - maxPref));
+        const sumExp = expPrefs.reduce((sum, e) => sum + e, 0);
+        return expPrefs.map(e => e / sumExp);
+    }
+    /**
+     * Get preference for state-action pair from policy table
+     */
+    getPreference(stateKey, actionKey) {
+        const statePolicy = this.policyTable.get(stateKey);
+        if (!statePolicy) {
+            return 0; // uniform preference initially
+        }
+        const entry = statePolicy.get(actionKey);
+        return entry ? entry.probability : 0;
+    }
+    /**
+     * Update actor and critic using temporal difference
+     *
+     * TD Error (advantage): δ = r + γV(s') - V(s)
+     * Critic update: V(s) += α_c * δ
+     * Actor update: preference(s,a) += α_a * δ * (1 - π(a|s))
+     */
+    update(experience, nextAction) {
+        this.stepCount++;
+        const { state, action, reward, nextState, done } = this.extractExperience(experience);
+        const stateKey = this.encodeState(state);
+        const actionKey = this.encodeAction(action);
+        // Get current and next state values from critic
+        const currentV = this.getStateValue(state);
+        const nextV = done ? 0 : this.getTargetStateValue(nextState);
+        // Calculate TD error (advantage)
+        let advantage = reward + this.config.discountFactor * nextV - currentV;
+        // Normalize advantage if enabled
+        if (this.actorConfig.normalizeAdvantage) {
+            advantage = this.normalizeAdvantage(advantage);
+        }
+        // Update critic (value function)
+        this.updateCritic(stateKey, currentV, advantage);
+        // Update actor (policy)
+        this.updateActor(stateKey, actionKey, advantage);
+        // Store in replay buffer if enabled
+        if (this.replayBuffer) {
+            this.replayBuffer.add(experience);
+        }
+        // Sync target network periodically
+        this.updatesSinceTargetSync++;
+        if (this.updatesSinceTargetSync >= this.actorConfig.targetUpdateFrequency) {
+            this.syncTargetNetwork();
+            this.updatesSinceTargetSync = 0;
+        }
+        this.logger.debug('Actor-Critic update', {
+            state: stateKey,
+            action: actionKey,
+            reward,
+            advantage,
+            valueUpdate: currentV + this.actorConfig.criticLearningRate * advantage
+        });
+    }
+    /**
+     * Update critic (value function)
+     * V(s) += α_c * δ
+     */
+    updateCritic(stateKey, currentV, advantage) {
+        const newValue = currentV + this.actorConfig.criticLearningRate * advantage;
+        const existingEntry = this.valueTable.get(stateKey);
+        this.valueTable.set(stateKey, {
+            state: stateKey,
+            value: newValue,
+            updateCount: (existingEntry?.updateCount ?? 0) + 1,
+            lastUpdated: Date.now()
+        });
+    }
+    /**
+     * Update actor (policy)
+     * For softmax policy: preference(s,a) += α_a * δ * (1 - π(a|s))
+     * This increases preference for actions with positive advantage
+     */
+    updateActor(stateKey, actionKey, advantage) {
+        if (!this.policyTable.has(stateKey)) {
+            this.policyTable.set(stateKey, new Map());
+        }
+        const statePolicy = this.policyTable.get(stateKey);
+        // Get current preference and probability
+        const currentEntry = statePolicy.get(actionKey);
+        const currentPref = currentEntry?.probability ?? 0;
+        // Approximate gradient: increase preference proportional to advantage
+        // Also add entropy bonus to encourage exploration
+        const entropyBonus = this.calculateEntropyBonus(stateKey);
+        const newPref = currentPref + this.actorConfig.actorLearningRate * (advantage + entropyBonus);
+        statePolicy.set(actionKey, {
+            action: actionKey,
+            probability: newPref,
+            logProbability: Math.log(Math.max(0.001, this.softmaxProb(stateKey, actionKey))),
+            updateCount: (currentEntry?.updateCount ?? 0) + 1,
+            lastUpdated: Date.now()
+        });
+        // Also update Q-table for getBestAction compatibility
+        this.setQValue(stateKey, actionKey, newPref);
+    }
+    /**
+     * Calculate entropy bonus for a state
+     * H(π(·|s)) = -Σ_a π(a|s) log(π(a|s))
+     */
+    calculateEntropyBonus(stateKey) {
+        const statePolicy = this.policyTable.get(stateKey);
+        if (!statePolicy || statePolicy.size === 0) {
+            return 0;
+        }
+        // Calculate entropy over stored actions
+        const prefs = Array.from(statePolicy.values()).map(e => e.probability);
+        const maxPref = Math.max(...prefs);
+        const expPrefs = prefs.map(p => Math.exp((p - maxPref) / this.actorConfig.temperature));
+        const sumExp = expPrefs.reduce((sum, e) => sum + e, 0);
+        const probs = expPrefs.map(e => e / sumExp);
+        let entropy = 0;
+        for (const p of probs) {
+            if (p > 0) {
+                entropy -= p * Math.log(p);
+            }
+        }
+        return this.actorConfig.entropyCoefficient * entropy;
+    }
+    /**
+     * Get softmax probability for a specific action
+     */
+    softmaxProb(stateKey, actionKey) {
+        const statePolicy = this.policyTable.get(stateKey);
+        if (!statePolicy || statePolicy.size === 0) {
+            return 1.0 / Math.max(1, statePolicy?.size ?? 1);
+        }
+        const prefs = Array.from(statePolicy.entries());
+        const temp = this.actorConfig.temperature;
+        const maxPref = Math.max(...prefs.map(([, e]) => e.probability));
+        let sumExp = 0;
+        let targetExp = 0;
+        for (const [key, entry] of prefs) {
+            const exp = Math.exp((entry.probability - maxPref) / temp);
+            sumExp += exp;
+            if (key === actionKey) {
+                targetExp = exp;
+            }
+        }
+        return targetExp / sumExp;
+    }
+    /**
+     * Normalize advantage using running statistics
+     */
+    normalizeAdvantage(advantage) {
+        this.advantageHistory.push(advantage);
+        // Keep limited history
+        if (this.advantageHistory.length > 1000) {
+            this.advantageHistory.shift();
+        }
+        if (this.advantageHistory.length < 10) {
+            return advantage;
+        }
+        const mean = this.advantageHistory.reduce((s, a) => s + a, 0) / this.advantageHistory.length;
+        const variance = this.advantageHistory.reduce((s, a) => s + (a - mean) ** 2, 0) / this.advantageHistory.length;
+        const std = Math.sqrt(variance) + 1e-8;
+        return (advantage - mean) / std;
+    }
+    /**
+     * Get state value from value table
+     */
+    getStateValue(state) {
+        const stateKey = this.encodeState(state);
+        const entry = this.valueTable.get(stateKey);
+        return entry?.value ?? 0;
+    }
+    /**
+     * Get state value from target network (for stability)
+     */
+    getTargetStateValue(state) {
+        const stateKey = this.encodeState(state);
+        const entry = this.targetValueTable.get(stateKey);
+        return entry?.value ?? this.getStateValue(state);
+    }
+    /**
+     * Sync target network with main network
+     */
+    syncTargetNetwork() {
+        this.targetValueTable.clear();
+        for (const [key, value] of this.valueTable.entries()) {
+            this.targetValueTable.set(key, { ...value });
+        }
+        this.logger.debug('Target network synchronized');
+    }
+    /**
+     * Extract experience components
+     */
+    extractExperience(experience) {
+        return {
+            state: experience.state,
+            action: experience.action,
+            reward: experience.reward,
+            nextState: experience.nextState,
+            done: experience.done ?? false
+        };
+    }
+    /**
+     * Get default exploration rate for reset
+     */
+    getDefaultExplorationRate() {
+        return this.defaultExploration;
+    }
+    /**
+     * Get actor-critic specific statistics
+     */
+    getActorCriticStatistics() {
+        // Calculate average state value
+        let totalValue = 0;
+        for (const entry of this.valueTable.values()) {
+            totalValue += entry.value;
+        }
+        const avgStateValue = this.valueTable.size > 0 ? totalValue / this.valueTable.size : 0;
+        // Calculate policy table size
+        let policySize = 0;
+        for (const statePolicy of this.policyTable.values()) {
+            policySize += statePolicy.size;
+        }
+        // Calculate average entropy
+        let totalEntropy = 0;
+        let entropyCount = 0;
+        for (const stateKey of this.policyTable.keys()) {
+            const entropy = this.calculateEntropyBonus(stateKey) / this.actorConfig.entropyCoefficient;
+            totalEntropy += entropy;
+            entropyCount++;
+        }
+        const avgEntropy = entropyCount > 0 ? totalEntropy / entropyCount : 0;
+        // Calculate advantage statistics
+        const advMean = this.advantageHistory.length > 0
+            ? this.advantageHistory.reduce((s, a) => s + a, 0) / this.advantageHistory.length
+            : 0;
+        const advVariance = this.advantageHistory.length > 0
+            ? this.advantageHistory.reduce((s, a) => s + (a - advMean) ** 2, 0) / this.advantageHistory.length
+            : 0;
+        return {
+            valueTableSize: this.valueTable.size,
+            policyTableSize: policySize,
+            avgStateValue,
+            avgEntropy,
+            advantageMean: advMean,
+            advantageStd: Math.sqrt(advVariance)
+        };
+    }
+    /**
+     * Reset actor-critic specific state
+     */
+    reset() {
+        super.reset();
+        this.policyTable.clear();
+        this.valueTable.clear();
+        this.targetValueTable.clear();
+        this.advantageHistory = [];
+        this.updatesSinceTargetSync = 0;
+        this.logger.info('ActorCriticLearner reset');
+    }
+    /**
+     * Export complete actor-critic state
+     */
+    exportActorCritic() {
+        const serializedPolicy = {};
+        for (const [state, actions] of this.policyTable.entries()) {
+            serializedPolicy[state] = {};
+            for (const [action, entry] of actions.entries()) {
+                serializedPolicy[state][action] = entry;
+            }
+        }
+        const serializedValue = {};
+        for (const [state, entry] of this.valueTable.entries()) {
+            serializedValue[state] = entry;
+        }
+        return {
+            base: this.export(),
+            valueTable: serializedValue,
+            policyTable: serializedPolicy,
+            actorConfig: { ...this.actorConfig }
+        };
+    }
+    /**
+     * Import complete actor-critic state
+     */
+    importActorCritic(state) {
+        this.import(state.base);
+        this.valueTable.clear();
+        for (const [stateKey, entry] of Object.entries(state.valueTable)) {
+            this.valueTable.set(stateKey, entry);
+        }
+        this.policyTable.clear();
+        for (const [stateKey, actions] of Object.entries(state.policyTable)) {
+            const actionMap = new Map();
+            for (const [actionKey, entry] of Object.entries(actions)) {
+                actionMap.set(actionKey, entry);
+            }
+            this.policyTable.set(stateKey, actionMap);
+        }
+        this.actorConfig = { ...state.actorConfig };
+        this.syncTargetNetwork();
+        this.logger.info('Imported Actor-Critic state', {
+            valueTableSize: this.valueTable.size,
+            policyTableSize: this.policyTable.size
+        });
+    }
+}
+exports.ActorCriticLearner = ActorCriticLearner;
+/**
+ * Create default Actor-Critic configuration
+ */
+function createDefaultActorCriticConfig() {
+    return {
+        learningRate: 0.1,
+        actorLearningRate: 0.01,
+        criticLearningRate: 0.1,
+        discountFactor: 0.95,
+        explorationRate: 0.3,
+        explorationDecay: 0.995,
+        minExplorationRate: 0.01,
+        entropyCoefficient: 0.01,
+        temperature: 1.0,
+        normalizeAdvantage: true,
+        targetUpdateFrequency: 100,
+        useExperienceReplay: true,
+        replayBufferSize: 10000,
+        batchSize: 32
+    };
+}
+//# sourceMappingURL=ActorCriticLearner.js.map

package/dist/learning/algorithms/ActorCriticLearner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ActorCriticLearner.js","sourceRoot":"","sources":["../../../src/learning/algorithms/ActorCriticLearner.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;;;;;GAmBG;;;AAyhBH,wEAiBC;AAxiBD,2DAA0E;AA0C1E;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,MAAa,kBAAmB,SAAQ,qCAAiB;IASvD,YAAY,MAAyB;QACnC,KAAK,CAAC,MAAM,CAAC,CAAC;QACd,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC;QAC1B,IAAI,CAAC,WAAW,GAAG,IAAI,GAAG,EAAE,CAAC;QAC7B,IAAI,CAAC,UAAU,GAAG,IAAI,GAAG,EAAE,CAAC;QAC5B,IAAI,CAAC,gBAAgB,GAAG,IAAI,GAAG,EAAE,CAAC;QAClC,IAAI,CAAC,sBAAsB,GAAG,CAAC,CAAC;QAChC,IAAI,CAAC,gBAAgB,GAAG,EAAE,CAAC;QAC3B,IAAI,CAAC,kBAAkB,GAAG,MAAM,CAAC,eAAe,CAAC;QAEjD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,gCAAgC,EAAE;YACjD,OAAO,EAAE,MAAM,CAAC,iBAAiB;YACjC,QAAQ,EAAE,MAAM,CAAC,kBAAkB;YACnC,OAAO,EAAE,MAAM,CAAC,kBAAkB;YAClC,WAAW,EAAE,MAAM,CAAC,WAAW;SAChC,CAAC,CAAC;IACL,CAAC;IAED;;;OAGG;IACM,YAAY,CAAC,KAAgB,EAAE,gBAA+B;QACrE,IAAI,gBAAgB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAClC,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;QACzD,CAAC;QAED,+DAA+D;QAC/D,IAAI,IAAI,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,eAAe,EAAE,CAAC;YAChD,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC;YACxE,OAAO,gBAAgB,CAAC,WAAW,CAAC,CAAC;QACvC,CAAC;QAED,qBAAqB;QACrB,OAAO,IAAI,CAAC,gBAAgB,CAAC,KAAK,EAAE,gBAAgB,CAAC,CAAC;IACxD,CAAC;IAED;;OAEG;IACK,gBAAgB,CAAC,KAAgB,EAAE,gBAA+B;QACxE,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QACzC,MAAM,aAAa,GAAG,IAAI,CAAC,sBAAsB,CAAC,QAAQ,EAAE,gBAAgB,CAAC,CAAC;QAE9E,uCAAuC;QACvC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;QAC7B,IAAI,UAAU,GAAG,CAAC,CAAC;QAEnB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,gBAAgB,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACjD,UAAU,IAAI,aAAa,CAAC,CAAC,CAAC,CAAC;YAC/B,IAAI,MAAM,IAAI,UAAU,EAAE,CAAC;gBACzB,OAAO,gBAAgB,CAAC,CAAC,CAAC,CAAC;YAC7B,CAAC;QACH,CAAC;QAED,uDAAuD;QACvD,OAAO,gBAAgB,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACvD,CAAC;IAED;;;OAGG;IACK,sBAAsB,CAAC,QAAgB,EAAE,gBAA+B;QAC9E,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,WAAW,CAAC;QACjD,MAAM,WAAW,GAAa,EAAE,CAAC;QAEjC,oDAAoD;QACpD,KAAK,MAAM,MAAM,IAAI,gBAAgB,EAAE,CAAC;YACtC,MAAM,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;YAC5C,MAAM,UAAU,GAAG,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;YAC3D,WAAW,CAAC,IAAI,CAAC,UAAU,GAAG,WAAW,CAAC,CAAC;QAC7C,CAAC;QAED,mCAAmC;QACnC,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,WAAW,CAAC,CAAC;QACzC,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;QAC7D,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;QAEvD,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC;IACvC,CAAC;IAED;;OAEG;IACK,aAAa,CAAC,QAAgB,EAAE,SAAiB;QACvD,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACnD,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,OAAO,CAAC,CAAC,CAAC,+BAA+B;QAC3C,CAAC;QAED,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QACzC,OAAO,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC;IACvC,CAAC;IAED;;;;;;OAMG;IACM,MAAM,CAAC,UAA0B,EAAE,UAAwB;QAClE,IAAI,CAAC,SAAS,EAAE,CAAC;QAEjB,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC,iBAAiB,CAAC,UAAU,CAAC,CAAC;QACtF,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QACzC,MAAM,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;QAE5C,gDAAgD;QAChD,MAAM,QAAQ,GAAG,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;QAC3C,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,mBAAmB,CAAC,SAAS,CAAC,CAAC;QAE7D,iCAAiC;QACjC,IAAI,SAAS,GAAG,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,cAAc,GAAG,KAAK,GAAG,QAAQ,CAAC;QAEvE,iCAAiC;QACjC,IAAI,IAAI,CAAC,WAAW,CAAC,kBAAkB,EAAE,CAAC;YACxC,SAAS,GAAG,IAAI,CAAC,kBAAkB,CAAC,SAAS,CAAC,CAAC;QACjD,CAAC;QAED,iCAAiC;QACjC,IAAI,CAAC,YAAY,CAAC,QAAQ,EAAE,QAAQ,EAAE,SAAS,CAAC,CAAC;QAEjD,wBAAwB;QACxB,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;QAEjD,oCAAoC;QACpC,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACtB,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QACpC,CAAC;QAED,mCAAmC;QACnC,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAC9B,IAAI,IAAI,CAAC,sBAAsB,IAAI,IAAI,CAAC,WAAW,CAAC,qBAAqB,EAAE,CAAC;YAC1E,IAAI,CAAC,iBAAiB,EAAE,CAAC;YACzB,IAAI,CAAC,sBAAsB,GAAG,CAAC,CAAC;QAClC,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,qBAAqB,EAAE;YACvC,KAAK,EAAE,QAAQ;YACf,MAAM,EAAE,SAAS;YACjB,MAAM;YACN,SAAS;YACT,WAAW,EAAE,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,kBAAkB,GAAG,SAAS;SACxE,CAAC,CAAC;IACL,CAAC;IAED;;;OAGG;IACK,YAAY,CAAC,QAAgB,EAAE,QAAgB,EAAE,SAAiB;QACxE,MAAM,QAAQ,GAAG,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,kBAAkB,GAAG,SAAS,CAAC;QAE5E,MAAM,aAAa,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACpD,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,EAAE;YAC5B,KAAK,EAAE,QAAQ;YACf,KAAK,EAAE,QAAQ;YACf,WAAW,EAAE,CAAC,aAAa,EAAE,WAAW,IAAI,CAAC,CAAC,GAAG,CAAC;YAClD,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE;SACxB,CAAC,CAAC;IACL,CAAC;IAED;;;;OAIG;IACK,WAAW,CAAC,QAAgB,EAAE,SAAiB,EAAE,SAAiB;QACxE,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YACpC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,QAAQ,EAAE,IAAI,GAAG,EAAE,CAAC,CAAC;QAC5C,CAAC;QACD,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAE,CAAC;QAEpD,yCAAyC;QACzC,MAAM,YAAY,GAAG,WAAW,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAChD,MAAM,WAAW,GAAG,YAAY,EAAE,WAAW,IAAI,CAAC,CAAC;QAEnD,sEAAsE;QACtE,kDAAkD;QAClD,MAAM,YAAY,GAAG,IAAI,CAAC,qBAAqB,CAAC,QAAQ,CAAC,CAAC;QAC1D,MAAM,OAAO,GAAG,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,iBAAiB,GAAG,CAAC,SAAS,GAAG,YAAY,CAAC,CAAC;QAE9F,WAAW,CAAC,GAAG,CAAC,SAAS,EAAE;YACzB,MAAM,EAAE,SAAS;YACjB,WAAW,EAAE,OAAO;YACpB,cAAc,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC,CAAC;YAChF,WAAW,EAAE,CAAC,YAAY,EAAE,WAAW,IAAI,CAAC,CAAC,GAAG,CAAC;YACjD,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE;SACxB,CAAC,CAAC;QAEH,sDAAsD;QACtD,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;IAC/C,CAAC;IAED;;;OAGG;IACK,qBAAqB,CAAC,QAAgB;QAC5C,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACnD,IAAI,CAAC,WAAW,IAAI,WAAW,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;YAC3C,OAAO,CAAC,CAAC;QACX,CAAC;QAED,wCAAwC;QACxC,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC;QACvE,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC;QACnC,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC,CAAC;QACxF,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;QACvD,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC;QAE5C,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;YACtB,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;gBACV,OAAO,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YAC7B,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC,WAAW,CAAC,kBAAkB,GAAG,OAAO,CAAC;IACvD,CAAC;IAED;;OAEG;IACK,WAAW,CAAC,QAAgB,EAAE,SAAiB;QACrD,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACnD,IAAI,CAAC,WAAW,IAAI,WAAW,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;YAC3C,OAAO,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,EAAE,IAAI,IAAI,CAAC,CAAC,CAAC;QACnD,CAAC;QAED,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC,CAAC;QAChD,MAAM,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,WAAW,CAAC;QAE1C,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC;QACjE,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,IAAI,SAAS,GAAG,CAAC,CAAC;QAElB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,KAAK,EAAE,CAAC;YACjC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,WAAW,GAAG,OAAO,CAAC,GAAG,IAAI,CAAC,CAAC;YAC3D,MAAM,IAAI,GAAG,CAAC;YACd,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;gBACtB,SAAS,GAAG,GAAG,CAAC;YAClB,CAAC;QACH,CAAC;QAED,OAAO,SAAS,GAAG,MAAM,CAAC;IAC5B,CAAC;IAED;;OAEG;IACK,kBAAkB,CAAC,SAAiB;QAC1C,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAEtC,uBAAuB;QACvB,IAAI,IAAI,CAAC,gBAAgB,CAAC,MAAM,GAAG,IAAI,EAAE,CAAC;YACxC,IAAI,CAAC,gBAAgB,CAAC,KAAK,EAAE,CAAC;QAChC,CAAC;QAED,IAAI,IAAI,CAAC,gBAAgB,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;YACtC,OAAO,SAAS,CAAC;QACnB,CAAC;QAED,MAAM,IAAI,GAAG,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC;QAC7F,MAAM,QAAQ,GAAG,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC;QAC/G,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,IAAI,CAAC;QAEvC,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,GAAG,CAAC;IAClC,CAAC;IAED;;OAEG;IACM,aAAa,CAAC,KAAgB;QACrC,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QACzC,MAAM,KAAK,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC5C,OAAO,KAAK,EAAE,KAAK,IAAI,CAAC,CAAC;IAC3B,CAAC;IAED;;OAEG;IACK,mBAAmB,CAAC,KAAgB;QAC1C,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QACzC,MAAM,KAAK,GAAG,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAClD,OAAO,KAAK,EAAE,KAAK,IAAI,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;IACnD,CAAC;IAED;;OAEG;IACK,iBAAiB;QACvB,IAAI,CAAC,gBAAgB,CAAC,KAAK,EAAE,CAAC;QAC9B,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC;YACrD,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,GAAG,KAAK,EAAE,CAAC,CAAC;QAC/C,CAAC;QACD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAC;IACnD,CAAC;IAED;;OAEG;IACK,iBAAiB,CAAC,UAA0B;QAOlD,OAAO;YACL,KAAK,EAAE,UAAU,CAAC,KAAK;YACvB,MAAM,EAAE,UAAU,CAAC,MAAM;YACzB,MAAM,EAAE,UAAU,CAAC,MAAM;YACzB,SAAS,EAAE,UAAU,CAAC,SAAS;YAC/B,IAAI,EAAE,UAAU,CAAC,IAAI,IAAI,KAAK;SAC/B,CAAC;IACJ,CAAC;IAED;;OAEG;IACO,yBAAyB;QACjC,OAAO,IAAI,CAAC,kBAAkB,CAAC;IACjC,CAAC;IAED;;OAEG;IACH,wBAAwB;QAQtB,gCAAgC;QAChC,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,EAAE,CAAC;YAC7C,UAAU,IAAI,KAAK,CAAC,KAAK,CAAC;QAC5B,CAAC;QACD,MAAM,aAAa,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QAEvF,8BAA8B;QAC9B,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,KAAK,MAAM,WAAW,IAAI,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,EAAE,CAAC;YACpD,UAAU,IAAI,WAAW,CAAC,IAAI,CAAC;QACjC,CAAC;QAED,4BAA4B;QAC5B,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,EAAE,CAAC;YAC/C,MAAM,OAAO,GAAG,IAAI,CAAC,qBAAqB,CAAC,QAAQ,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC,kBAAkB,CAAC;YAC3F,YAAY,IAAI,OAAO,CAAC;YACxB,YAAY,EAAE,CAAC;QACjB,CAAC;QACD,MAAM,UAAU,GAAG,YAAY,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC;QAEtE,iCAAiC;QACjC,MAAM,OAAO,GAAG,IAAI,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC;YAC9C,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,gBAAgB,CAAC,MAAM;YACjF,CAAC,CAAC,CAAC,CAAC;QACN,MAAM,WAAW,GAAG,IAAI,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC;YAClD,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,gBAAgB,CAAC,MAAM;YAClG,CAAC,CAAC,CAAC,CAAC;QAEN,OAAO;YACL,cAAc,EAAE,IAAI,CAAC,UAAU,CAAC,IAAI;YACpC,eAAe,EAAE,UAAU;YAC3B,aAAa;YACb,UAAU;YACV,aAAa,EAAE,OAAO;YACtB,YAAY,EAAE,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC;SACrC,CAAC;IACJ,CAAC;IAED;;OAEG;IACM,KAAK;QACZ,KAAK,CAAC,KAAK,EAAE,CAAC;QACd,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;QACxB,IAAI,CAAC,gBAAgB,CAAC,KAAK,EAAE,CAAC;QAC9B,IAAI,CAAC,gBAAgB,GAAG,EAAE,CAAC;QAC3B,IAAI,CAAC,sBAAsB,GAAG,CAAC,CAAC;QAChC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;IAC/C,CAAC;IAED;;OAEG;IACH,iBAAiB;QAMf,MAAM,gBAAgB,GAAgD,EAAE,CAAC;QACzE,KAAK,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,EAAE,CAAC;YAC1D,gBAAgB,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;YAC7B,KAAK,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;gBAChD,gBAAgB,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC;YAC1C,CAAC;QACH,CAAC;QAED,MAAM,eAAe,GAAoC,EAAE,CAAC;QAC5D,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC;YACvD,eAAe,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC;QACjC,CAAC;QAED,OAAO;YACL,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE;YACnB,UAAU,EAAE,eAAe;YAC3B,WAAW,EAAE,gBAAgB;YAC7B,WAAW,EAAE,EAAE,GAAG,IAAI,CAAC,WAAW,EAAE;SACrC,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,iBAAiB,CAAC,KAAgD;QAChE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAExB,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;QACxB,KAAK,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,EAAE,CAAC;YACjE,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QACvC,CAAC;QAED,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;QACzB,KAAK,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,EAAE,CAAC;YACpE,MAAM,SAAS,GAAG,IAAI,GAAG,EAAuB,CAAC;YACjD,KAAK,MAAM,CAAC,SAAS,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;gBACzD,SAAS,CAAC,GAAG,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;YAClC,CAAC;YACD,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;QAC5C,CAAC;QAED,IAAI,CAAC,WAAW,GAAG,EAAE,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC;QAC5C,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAEzB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,6BAA6B,EAAE;YAC9C,cAAc,EAAE,IAAI,CAAC,UAAU,CAAC,IAAI;YACpC,eAAe,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI;SACvC,CAAC,CAAC;IACL,CAAC;CACF;AA3cD,gDA2cC;AAED;;GAEG;AACH,SAAgB,8BAA8B;IAC5C,OAAO;QACL,YAAY,EAAE,GAAG;QACjB,iBAAiB,EAAE,IAAI;QACvB,kBAAkB,EAAE,GAAG;QACvB,cAAc,EAAE,IAAI;QACpB,eAAe,EAAE,GAAG;QACpB,gBAAgB,EAAE,KAAK;QACvB,kBAAkB,EAAE,IAAI;QACxB,kBAAkB,EAAE,IAAI;QACxB,WAAW,EAAE,GAAG;QAChB,kBAAkB,EAAE,IAAI;QACxB,qBAAqB,EAAE,GAAG;QAC1B,mBAAmB,EAAE,IAAI;QACzB,gBAAgB,EAAE,KAAK;QACvB,SAAS,EAAE,EAAE;KACd,CAAC;AACJ,CAAC"}

package/dist/learning/algorithms/PPOLearner.d.ts

@@ -0,0 +1,207 @@
+/**
+ * PPOLearner - Proximal Policy Optimization Algorithm
+ *
+ * Implements PPO-Clip, the most widely used variant of PPO:
+ * - Clipped surrogate objective to prevent large policy updates
+ * - Generalized Advantage Estimation (GAE) for variance reduction
+ * - Value function clipping for stability
+ * - Multiple epochs over collected trajectories
+ *
+ * Key features:
+ * - Trust region optimization without KL constraint
+ * - Sample efficient with mini-batch updates
+ * - Robust to hyperparameter choices
+ * - Suitable for continuous and discrete action spaces
+ *
+ * PPO-Clip objective:
+ * L^CLIP(θ) = E[min(r(θ)Â, clip(r(θ), 1-ε, 1+ε)Â)]
+ * where r(θ) = π_θ(a|s) / π_θ_old(a|s)
+ *
+ * @module learning/algorithms/PPOLearner
+ * @version 1.0.0
+ */
+import { AbstractRLLearner, RLConfig } from './AbstractRLLearner';
+import { TaskState, AgentAction, TaskExperience } from '../types';
+/**
+ * Configuration specific to PPO algorithm
+ */
+export interface PPOConfig extends RLConfig {
+    /** Clipping parameter (ε) - typically 0.1-0.3 */
+    clipEpsilon: number;
+    /** Number of epochs to train on collected data */
+    ppoEpochs: number;
+    /** Mini-batch size for training */
+    miniBatchSize: number;
+    /** Value function loss coefficient */
+    valueLossCoefficient: number;
+    /** Entropy loss coefficient for exploration */
+    entropyCoefficient: number;
+    /** GAE lambda for advantage estimation */
+    gaeLambda: number;
+    /** Maximum gradient norm for clipping */
+    maxGradNorm: number;
+    /** Whether to clip value function updates */
+    clipValueLoss: boolean;
+    /** Learning rate for policy network */
+    policyLearningRate: number;
+    /** Learning rate for value network */
+    valueLearningRate: number;
+}
+/**
+ * Policy parameters for a state-action pair
+ */
+interface PolicyParams {
+    preference: number;
+    logProb: number;
+    updateCount: number;
+}
+/**
+ * PPOLearner - Proximal Policy Optimization implementation
+ *
+ * PPO is a state-of-the-art policy gradient method that achieves
+ * strong performance while being simpler than TRPO.
+ *
+ * Usage:
+ * ```typescript
+ * const ppo = new PPOLearner({
+ *   learningRate: 0.0003,
+ *   discountFactor: 0.99,
+ *   explorationRate: 0.0,
+ *   explorationDecay: 1.0,
+ *   minExplorationRate: 0.0,
+ *   clipEpsilon: 0.2,
+ *   ppoEpochs: 4,
+ *   miniBatchSize: 64,
+ *   valueLossCoefficient: 0.5,
+ *   entropyCoefficient: 0.01,
+ *   gaeLambda: 0.95,
+ *   maxGradNorm: 0.5,
+ *   clipValueLoss: true,
+ *   policyLearningRate: 0.0003,
+ *   valueLearningRate: 0.001,
+ *   useExperienceReplay: false,
+ *   replayBufferSize: 2048,
+ *   batchSize: 64
+ * });
+ *
+ * // Collect trajectory
+ * ppo.collectStep(state, action, reward, nextState, done);
+ *
+ * // Train on collected trajectory
+ * ppo.trainOnTrajectory();
+ * ```
+ */
+export declare class PPOLearner extends AbstractRLLearner {
+    private ppoConfig;
+    private policyTable;
+    private valueTable;
+    private oldPolicyTable;
+    private trajectory;
+    private readonly defaultExploration;
+    constructor(config: PPOConfig);
+    /**
+     * Select action using current policy (softmax)
+     */
+    selectAction(state: TaskState, availableActions: AgentAction[]): AgentAction;
+    /**
+     * Get action probabilities using softmax policy
+     */
+    private getActionProbabilities;
+    /**
+     * Get policy parameters for state-action pair
+     */
+    private getPolicyParams;
+    /**
+     * Get log probability of action under current policy
+     */
+    private getLogProb;
+    /**
+     * Get state value from value network
+     */
+    getStateValue(state: TaskState): number;
+    /**
+     * Collect a step in the trajectory
+     */
+    collectStep(state: TaskState, action: AgentAction, reward: number, nextState: TaskState, done: boolean): void;
+    /**
+     * Standard update interface - collects experience and trains when ready
+     */
+    update(experience: TaskExperience, nextAction?: AgentAction): void;
+    /**
+     * Train on collected trajectory using PPO
+     */
+    trainOnTrajectory(): void;
+    /**
+     * Compute Generalized Advantage Estimation (GAE)
+     *
+     * GAE: Â_t = Σ_{l=0}^∞ (γλ)^l δ_{t+l}
+     * where δ_t = r_t + γV(s_{t+1}) - V(s_t)
+     */
+    private computeGAE;
+    /**
+     * Save current policy as old policy for ratio computation
+     */
+    private saveOldPolicy;
+    /**
+     * Get old log probability for ratio computation
+     */
+    private getOldLogProb;
+    /**
+     * Train one epoch on the trajectory
+     */
+    private trainEpoch;
+    /**
+     * Train on a mini-batch
+     */
+    private trainMiniBatch;
+    /**
+     * Update policy parameters
+     */
+    private updatePolicy;
+    /**
+     * Update value function
+     */
+    private updateValue;
+    /**
+     * Compute entropy of policy at state
+     */
+    private computeEntropy;
+    /**
+     * Get default exploration rate for reset
+     */
+    protected getDefaultExplorationRate(): number;
+    /**
+     * Get PPO-specific statistics
+     */
+    getPPOStatistics(): {
+        trajectoryLength: number;
+        valueTableSize: number;
+        policyTableSize: number;
+        avgValue: number;
+        avgAdvantage: number;
+        clipFraction: number;
+    };
+    /**
+     * Reset PPO-specific state
+     */
+    reset(): void;
+    /**
+     * Export PPO state
+     */
+    exportPPO(): {
+        base: ReturnType<AbstractRLLearner['export']>;
+        policyTable: Record<string, Record<string, PolicyParams>>;
+        valueTable: Record<string, number>;
+        ppoConfig: PPOConfig;
+    };
+    /**
+     * Import PPO state
+     */
+    importPPO(state: ReturnType<typeof this.exportPPO>): void;
+}
+/**
+ * Create default PPO configuration
+ */
+export declare function createDefaultPPOConfig(): PPOConfig;
+export {};
+//# sourceMappingURL=PPOLearner.d.ts.map

package/dist/learning/algorithms/PPOLearner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"PPOLearner.d.ts","sourceRoot":"","sources":["../../../src/learning/algorithms/PPOLearner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,EAAE,iBAAiB,EAAE,QAAQ,EAAU,MAAM,qBAAqB,CAAC;AAC1E,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,UAAU,CAAC;AAElE;;GAEG;AACH,MAAM,WAAW,SAAU,SAAQ,QAAQ;IACzC,iDAAiD;IACjD,WAAW,EAAE,MAAM,CAAC;IACpB,kDAAkD;IAClD,SAAS,EAAE,MAAM,CAAC;IAClB,mCAAmC;IACnC,aAAa,EAAE,MAAM,CAAC;IACtB,sCAAsC;IACtC,oBAAoB,EAAE,MAAM,CAAC;IAC7B,+CAA+C;IAC/C,kBAAkB,EAAE,MAAM,CAAC;IAC3B,0CAA0C;IAC1C,SAAS,EAAE,MAAM,CAAC;IAClB,yCAAyC;IACzC,WAAW,EAAE,MAAM,CAAC;IACpB,6CAA6C;IAC7C,aAAa,EAAE,OAAO,CAAC;IACvB,uCAAuC;IACvC,kBAAkB,EAAE,MAAM,CAAC;IAC3B,sCAAsC;IACtC,iBAAiB,EAAE,MAAM,CAAC;CAC3B;AAiBD;;GAEG;AACH,UAAU,YAAY;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;CACrB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AACH,qBAAa,UAAW,SAAQ,iBAAiB;IAC/C,OAAO,CAAC,SAAS,CAAY;IAC7B,OAAO,CAAC,WAAW,CAAyC;IAC5D,OAAO,CAAC,UAAU,CAAsB;IACxC,OAAO,CAAC,cAAc,CAAyC;IAC/D,OAAO,CAAC,UAAU,CAAmB;IACrC,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAS;gBAEhC,MAAM,EAAE,SAAS;IAiB7B;;OAEG;IACM,YAAY,CAAC,KAAK,EAAE,SAAS,EAAE,gBAAgB,EAAE,WAAW,EAAE,GAAG,WAAW;IAsBrF;;OAEG;IACH,OAAO,CAAC,sBAAsB;IAiB9B;;OAEG;IACH,OAAO,CAAC,eAAe;IAQvB;;OAEG;IACH,OAAO,CAAC,UAAU;IA6BlB;;OAEG;IACM,aAAa,CAAC,KAAK,EAAE,SAAS,GAAG,MAAM;IAKhD;;OAEG;IACH,WAAW,CACT,KAAK,EAAE,SAAS,EAChB,MAAM,EAAE,WAAW,EACnB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,SAAS,EACpB,IAAI,EAAE,OAAO,GACZ,IAAI;IAqBP;;OAEG;IACM,MAAM,CAAC,UAAU,EAAE,cAAc,EAAE,UAAU,CAAC,EAAE,WAAW,GAAG,IAAI;IAe3E;;OAEG;IACH,iBAAiB,IAAI,IAAI;IAyBzB;;;;;OAKG;IACH,OAAO,CAAC,UAAU;IAsClB;;OAEG;IACH,OAAO,CAAC,aAAa;IAWrB;;OAEG;IACH,OAAO,CAAC,aAAa;IAQrB;;OAEG;IACH,OAAO,CAAC,UAAU;IAWlB;;OAEG;IACH,OAAO,CAAC,cAAc;IA0CtB;;OAEG;IACH,OAAO,CAAC,YAAY;IAkCpB;;OAEG;IACH,OAAO,CAAC,WAAW;IAMnB;;OAEG;IACH,OAAO,CAAC,cAAc;IAsBtB;;OAEG;IACH,SAAS,CAAC,yBAAyB,IAAI,MAAM;IAI7C;;OAEG;IACH,gBAAgB,IAAI;QAClB,gBAAgB,EAAE,MAAM,CAAC;QACzB,cAAc,EAAE,MAAM,CAAC;QACvB,eAAe,EAAE,MAAM,CAAC;QACxB,QAAQ,EAAE,MAAM,CAAC;QACjB,YAAY,EAAE,MAAM,CAAC;QACrB,YAAY,EAAE,MAAM,CAAC;KACtB;IAyBD;;OAEG;IACM,KAAK,IAAI,IAAI;IAStB;;OAEG;IACH,SAAS,IAAI;QACX,IAAI,EAAE,UAAU,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC,CAAC;QAC9C,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC,CAAC;QAC1D,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACnC,SAAS,EAAE,SAAS,CAAC;KACtB;IAsBD;;OAEG;IACH,SAAS,CAAC,KAAK,EAAE,UAAU,CAAC,OAAO,IAAI,CAAC,SAAS,CAAC,GAAG,IAAI;CAwB1D;AAED;;GAEG;AACH,wBAAgB,sBAAsB,IAAI,SAAS,CAqBlD"}