agentic-qe 2.1.2 → 2.2.1

package/dist/learning/algorithms/PPOLearner.d.ts

@@ -0,0 +1,207 @@
+/**
+ * PPOLearner - Proximal Policy Optimization Algorithm
+ *
+ * Implements PPO-Clip, the most widely used variant of PPO:
+ * - Clipped surrogate objective to prevent large policy updates
+ * - Generalized Advantage Estimation (GAE) for variance reduction
+ * - Value function clipping for stability
+ * - Multiple epochs over collected trajectories
+ *
+ * Key features:
+ * - Trust region optimization without KL constraint
+ * - Sample efficient with mini-batch updates
+ * - Robust to hyperparameter choices
+ * - Suitable for continuous and discrete action spaces
+ *
+ * PPO-Clip objective:
+ * L^CLIP(θ) = E[min(r(θ)Â, clip(r(θ), 1-ε, 1+ε)Â)]
+ * where r(θ) = π_θ(a|s) / π_θ_old(a|s)
+ *
+ * @module learning/algorithms/PPOLearner
+ * @version 1.0.0
+ */
+import { AbstractRLLearner, RLConfig } from './AbstractRLLearner';
+import { TaskState, AgentAction, TaskExperience } from '../types';
+/**
+ * Configuration specific to PPO algorithm
+ */
+export interface PPOConfig extends RLConfig {
+    /** Clipping parameter (ε) - typically 0.1-0.3 */
+    clipEpsilon: number;
+    /** Number of epochs to train on collected data */
+    ppoEpochs: number;
+    /** Mini-batch size for training */
+    miniBatchSize: number;
+    /** Value function loss coefficient */
+    valueLossCoefficient: number;
+    /** Entropy loss coefficient for exploration */
+    entropyCoefficient: number;
+    /** GAE lambda for advantage estimation */
+    gaeLambda: number;
+    /** Maximum gradient norm for clipping */
+    maxGradNorm: number;
+    /** Whether to clip value function updates */
+    clipValueLoss: boolean;
+    /** Learning rate for policy network */
+    policyLearningRate: number;
+    /** Learning rate for value network */
+    valueLearningRate: number;
+}
+/**
+ * Policy parameters for a state-action pair
+ */
+interface PolicyParams {
+    preference: number;
+    logProb: number;
+    updateCount: number;
+}
+/**
+ * PPOLearner - Proximal Policy Optimization implementation
+ *
+ * PPO is a state-of-the-art policy gradient method that achieves
+ * strong performance while being simpler than TRPO.
+ *
+ * Usage:
+ * ```typescript
+ * const ppo = new PPOLearner({
+ *   learningRate: 0.0003,
+ *   discountFactor: 0.99,
+ *   explorationRate: 0.0,
+ *   explorationDecay: 1.0,
+ *   minExplorationRate: 0.0,
+ *   clipEpsilon: 0.2,
+ *   ppoEpochs: 4,
+ *   miniBatchSize: 64,
+ *   valueLossCoefficient: 0.5,
+ *   entropyCoefficient: 0.01,
+ *   gaeLambda: 0.95,
+ *   maxGradNorm: 0.5,
+ *   clipValueLoss: true,
+ *   policyLearningRate: 0.0003,
+ *   valueLearningRate: 0.001,
+ *   useExperienceReplay: false,
+ *   replayBufferSize: 2048,
+ *   batchSize: 64
+ * });
+ *
+ * // Collect trajectory
+ * ppo.collectStep(state, action, reward, nextState, done);
+ *
+ * // Train on collected trajectory
+ * ppo.trainOnTrajectory();
+ * ```
+ */
+export declare class PPOLearner extends AbstractRLLearner {
+    private ppoConfig;
+    private policyTable;
+    private valueTable;
+    private oldPolicyTable;
+    private trajectory;
+    private readonly defaultExploration;
+    constructor(config: PPOConfig);
+    /**
+     * Select action using current policy (softmax)
+     */
+    selectAction(state: TaskState, availableActions: AgentAction[]): AgentAction;
+    /**
+     * Get action probabilities using softmax policy
+     */
+    private getActionProbabilities;
+    /**
+     * Get policy parameters for state-action pair
+     */
+    private getPolicyParams;
+    /**
+     * Get log probability of action under current policy
+     */
+    private getLogProb;
+    /**
+     * Get state value from value network
+     */
+    getStateValue(state: TaskState): number;
+    /**
+     * Collect a step in the trajectory
+     */
+    collectStep(state: TaskState, action: AgentAction, reward: number, nextState: TaskState, done: boolean): void;
+    /**
+     * Standard update interface - collects experience and trains when ready
+     */
+    update(experience: TaskExperience, nextAction?: AgentAction): void;
+    /**
+     * Train on collected trajectory using PPO
+     */
+    trainOnTrajectory(): void;
+    /**
+     * Compute Generalized Advantage Estimation (GAE)
+     *
+     * GAE: Â_t = Σ_{l=0}^∞ (γλ)^l δ_{t+l}
+     * where δ_t = r_t + γV(s_{t+1}) - V(s_t)
+     */
+    private computeGAE;
+    /**
+     * Save current policy as old policy for ratio computation
+     */
+    private saveOldPolicy;
+    /**
+     * Get old log probability for ratio computation
+     */
+    private getOldLogProb;
+    /**
+     * Train one epoch on the trajectory
+     */
+    private trainEpoch;
+    /**
+     * Train on a mini-batch
+     */
+    private trainMiniBatch;
+    /**
+     * Update policy parameters
+     */
+    private updatePolicy;
+    /**
+     * Update value function
+     */
+    private updateValue;
+    /**
+     * Compute entropy of policy at state
+     */
+    private computeEntropy;
+    /**
+     * Get default exploration rate for reset
+     */
+    protected getDefaultExplorationRate(): number;
+    /**
+     * Get PPO-specific statistics
+     */
+    getPPOStatistics(): {
+        trajectoryLength: number;
+        valueTableSize: number;
+        policyTableSize: number;
+        avgValue: number;
+        avgAdvantage: number;
+        clipFraction: number;
+    };
+    /**
+     * Reset PPO-specific state
+     */
+    reset(): void;
+    /**
+     * Export PPO state
+     */
+    exportPPO(): {
+        base: ReturnType<AbstractRLLearner['export']>;
+        policyTable: Record<string, Record<string, PolicyParams>>;
+        valueTable: Record<string, number>;
+        ppoConfig: PPOConfig;
+    };
+    /**
+     * Import PPO state
+     */
+    importPPO(state: ReturnType<typeof this.exportPPO>): void;
+}
+/**
+ * Create default PPO configuration
+ */
+export declare function createDefaultPPOConfig(): PPOConfig;
+export {};
+//# sourceMappingURL=PPOLearner.d.ts.map

package/dist/learning/algorithms/PPOLearner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"PPOLearner.d.ts","sourceRoot":"","sources":["../../../src/learning/algorithms/PPOLearner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,EAAE,iBAAiB,EAAE,QAAQ,EAAU,MAAM,qBAAqB,CAAC;AAC1E,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,UAAU,CAAC;AAElE;;GAEG;AACH,MAAM,WAAW,SAAU,SAAQ,QAAQ;IACzC,iDAAiD;IACjD,WAAW,EAAE,MAAM,CAAC;IACpB,kDAAkD;IAClD,SAAS,EAAE,MAAM,CAAC;IAClB,mCAAmC;IACnC,aAAa,EAAE,MAAM,CAAC;IACtB,sCAAsC;IACtC,oBAAoB,EAAE,MAAM,CAAC;IAC7B,+CAA+C;IAC/C,kBAAkB,EAAE,MAAM,CAAC;IAC3B,0CAA0C;IAC1C,SAAS,EAAE,MAAM,CAAC;IAClB,yCAAyC;IACzC,WAAW,EAAE,MAAM,CAAC;IACpB,6CAA6C;IAC7C,aAAa,EAAE,OAAO,CAAC;IACvB,uCAAuC;IACvC,kBAAkB,EAAE,MAAM,CAAC;IAC3B,sCAAsC;IACtC,iBAAiB,EAAE,MAAM,CAAC;CAC3B;AAiBD;;GAEG;AACH,UAAU,YAAY;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;CACrB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AACH,qBAAa,UAAW,SAAQ,iBAAiB;IAC/C,OAAO,CAAC,SAAS,CAAY;IAC7B,OAAO,CAAC,WAAW,CAAyC;IAC5D,OAAO,CAAC,UAAU,CAAsB;IACxC,OAAO,CAAC,cAAc,CAAyC;IAC/D,OAAO,CAAC,UAAU,CAAmB;IACrC,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAS;gBAEhC,MAAM,EAAE,SAAS;IAiB7B;;OAEG;IACM,YAAY,CAAC,KAAK,EAAE,SAAS,EAAE,gBAAgB,EAAE,WAAW,EAAE,GAAG,WAAW;IAsBrF;;OAEG;IACH,OAAO,CAAC,sBAAsB;IAiB9B;;OAEG;IACH,OAAO,CAAC,eAAe;IAQvB;;OAEG;IACH,OAAO,CAAC,UAAU;IA6BlB;;OAEG;IACM,aAAa,CAAC,KAAK,EAAE,SAAS,GAAG,MAAM;IAKhD;;OAEG;IACH,WAAW,CACT,KAAK,EAAE,SAAS,EAChB,MAAM,EAAE,WAAW,EACnB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,SAAS,EACpB,IAAI,EAAE,OAAO,GACZ,IAAI;IAqBP;;OAEG;IACM,MAAM,CAAC,UAAU,EAAE,cAAc,EAAE,UAAU,CAAC,EAAE,WAAW,GAAG,IAAI;IAe3E;;OAEG;IACH,iBAAiB,IAAI,IAAI;IAyBzB;;;;;OAKG;IACH,OAAO,CAAC,UAAU;IAsClB;;OAEG;IACH,OAAO,CAAC,aAAa;IAWrB;;OAEG;IACH,OAAO,CAAC,aAAa;IAQrB;;OAEG;IACH,OAAO,CAAC,UAAU;IAWlB;;OAEG;IACH,OAAO,CAAC,cAAc;IA0CtB;;OAEG;IACH,OAAO,CAAC,YAAY;IAkCpB;;OAEG;IACH,OAAO,CAAC,WAAW;IAMnB;;OAEG;IACH,OAAO,CAAC,cAAc;IAsBtB;;OAEG;IACH,SAAS,CAAC,yBAAyB,IAAI,MAAM;IAI7C;;OAEG;IACH,gBAAgB,IAAI;QAClB,gBAAgB,EAAE,MAAM,CAAC;QACzB,cAAc,EAAE,MAAM,CAAC;QACvB,eAAe,EAAE,MAAM,CAAC;QACxB,QAAQ,EAAE,MAAM,CAAC;QACjB,YAAY,EAAE,MAAM,CAAC;QACrB,YAAY,EAAE,MAAM,CAAC;KACtB;IAyBD;;OAEG;IACM,KAAK,IAAI,IAAI;IAStB;;OAEG;IACH,SAAS,IAAI;QACX,IAAI,EAAE,UAAU,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC,CAAC;QAC9C,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC,CAAC;QAC1D,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACnC,SAAS,EAAE,SAAS,CAAC;KACtB;IAsBD;;OAEG;IACH,SAAS,CAAC,KAAK,EAAE,UAAU,CAAC,OAAO,IAAI,CAAC,SAAS,CAAC,GAAG,IAAI;CAwB1D;AAED;;GAEG;AACH,wBAAgB,sBAAsB,IAAI,SAAS,CAqBlD"}

package/dist/learning/algorithms/PPOLearner.js

@@ -0,0 +1,490 @@
+"use strict";
+/**
+ * PPOLearner - Proximal Policy Optimization Algorithm
+ *
+ * Implements PPO-Clip, the most widely used variant of PPO:
+ * - Clipped surrogate objective to prevent large policy updates
+ * - Generalized Advantage Estimation (GAE) for variance reduction
+ * - Value function clipping for stability
+ * - Multiple epochs over collected trajectories
+ *
+ * Key features:
+ * - Trust region optimization without KL constraint
+ * - Sample efficient with mini-batch updates
+ * - Robust to hyperparameter choices
+ * - Suitable for continuous and discrete action spaces
+ *
+ * PPO-Clip objective:
+ * L^CLIP(θ) = E[min(r(θ)Â, clip(r(θ), 1-ε, 1+ε)Â)]
+ * where r(θ) = π_θ(a|s) / π_θ_old(a|s)
+ *
+ * @module learning/algorithms/PPOLearner
+ * @version 1.0.0
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PPOLearner = void 0;
+exports.createDefaultPPOConfig = createDefaultPPOConfig;
+const AbstractRLLearner_1 = require("./AbstractRLLearner");
+/**
+ * PPOLearner - Proximal Policy Optimization implementation
+ *
+ * PPO is a state-of-the-art policy gradient method that achieves
+ * strong performance while being simpler than TRPO.
+ *
+ * Usage:
+ * ```typescript
+ * const ppo = new PPOLearner({
+ *   learningRate: 0.0003,
+ *   discountFactor: 0.99,
+ *   explorationRate: 0.0,
+ *   explorationDecay: 1.0,
+ *   minExplorationRate: 0.0,
+ *   clipEpsilon: 0.2,
+ *   ppoEpochs: 4,
+ *   miniBatchSize: 64,
+ *   valueLossCoefficient: 0.5,
+ *   entropyCoefficient: 0.01,
+ *   gaeLambda: 0.95,
+ *   maxGradNorm: 0.5,
+ *   clipValueLoss: true,
+ *   policyLearningRate: 0.0003,
+ *   valueLearningRate: 0.001,
+ *   useExperienceReplay: false,
+ *   replayBufferSize: 2048,
+ *   batchSize: 64
+ * });
+ *
+ * // Collect trajectory
+ * ppo.collectStep(state, action, reward, nextState, done);
+ *
+ * // Train on collected trajectory
+ * ppo.trainOnTrajectory();
+ * ```
+ */
+class PPOLearner extends AbstractRLLearner_1.AbstractRLLearner {
+    constructor(config) {
+        super(config);
+        this.ppoConfig = config;
+        this.policyTable = new Map();
+        this.valueTable = new Map();
+        this.oldPolicyTable = new Map();
+        this.trajectory = [];
+        this.defaultExploration = config.explorationRate;
+        this.logger.info('PPOLearner initialized', {
+            clipEpsilon: config.clipEpsilon,
+            epochs: config.ppoEpochs,
+            gaeLambda: config.gaeLambda,
+            entropyCoeff: config.entropyCoefficient
+        });
+    }
+    /**
+     * Select action using current policy (softmax)
+     */
+    selectAction(state, availableActions) {
+        if (availableActions.length === 0) {
+            throw new Error('No available actions to select from');
+        }
+        const stateKey = this.encodeState(state);
+        const probs = this.getActionProbabilities(stateKey, availableActions);
+        // Sample from distribution
+        const random = Math.random();
+        let cumulative = 0;
+        for (let i = 0; i < availableActions.length; i++) {
+            cumulative += probs[i];
+            if (random <= cumulative) {
+                return availableActions[i];
+            }
+        }
+        return availableActions[availableActions.length - 1];
+    }
+    /**
+     * Get action probabilities using softmax policy
+     */
+    getActionProbabilities(stateKey, availableActions) {
+        const preferences = [];
+        for (const action of availableActions) {
+            const actionKey = this.encodeAction(action);
+            const params = this.getPolicyParams(stateKey, actionKey);
+            preferences.push(params.preference);
+        }
+        // Softmax with numerical stability
+        const maxPref = Math.max(...preferences);
+        const expPrefs = preferences.map(p => Math.exp(p - maxPref));
+        const sumExp = expPrefs.reduce((sum, e) => sum + e, 0);
+        return expPrefs.map(e => e / sumExp);
+    }
+    /**
+     * Get policy parameters for state-action pair
+     */
+    getPolicyParams(stateKey, actionKey) {
+        const statePolicy = this.policyTable.get(stateKey);
+        if (!statePolicy) {
+            return { preference: 0, logProb: 0, updateCount: 0 };
+        }
+        return statePolicy.get(actionKey) ?? { preference: 0, logProb: 0, updateCount: 0 };
+    }
+    /**
+     * Get log probability of action under current policy
+     */
+    getLogProb(stateKey, actionKey, availableActions) {
+        // Get preference for target action
+        const params = this.getPolicyParams(stateKey, actionKey);
+        // If we don't know the action space, return stored log prob
+        if (!availableActions) {
+            return params.logProb;
+        }
+        // Calculate actual log probability
+        const prefs = [];
+        let targetPref = params.preference;
+        for (const action of availableActions) {
+            const ak = this.encodeAction(action);
+            const p = this.getPolicyParams(stateKey, ak);
+            prefs.push(p.preference);
+            if (ak === actionKey) {
+                targetPref = p.preference;
+            }
+        }
+        const maxPref = Math.max(...prefs, targetPref);
+        const expTarget = Math.exp(targetPref - maxPref);
+        const sumExp = prefs.reduce((sum, p) => sum + Math.exp(p - maxPref), 0);
+        return Math.log(expTarget / sumExp);
+    }
+    /**
+     * Get state value from value network
+     */
+    getStateValue(state) {
+        const stateKey = this.encodeState(state);
+        return this.valueTable.get(stateKey) ?? 0;
+    }
+    /**
+     * Collect a step in the trajectory
+     */
+    collectStep(state, action, reward, nextState, done) {
+        const stateKey = this.encodeState(state);
+        const actionKey = this.encodeAction(action);
+        const nextStateKey = this.encodeState(nextState);
+        const value = this.valueTable.get(stateKey) ?? 0;
+        const logProb = this.getLogProb(stateKey, actionKey);
+        this.trajectory.push({
+            state: stateKey,
+            action: actionKey,
+            reward,
+            nextState: nextStateKey,
+            done,
+            value,
+            logProb,
+            advantage: 0, // Computed later
+            returns: 0 // Computed later
+        });
+    }
+    /**
+     * Standard update interface - collects experience and trains when ready
+     */
+    update(experience, nextAction) {
+        this.stepCount++;
+        const { state, action, reward, nextState } = experience;
+        const done = experience.done ?? false;
+        // Collect step
+        this.collectStep(state, action, reward, nextState, done);
+        // Train when trajectory is large enough
+        if (this.trajectory.length >= this.ppoConfig.replayBufferSize) {
+            this.trainOnTrajectory();
+        }
+    }
+    /**
+     * Train on collected trajectory using PPO
+     */
+    trainOnTrajectory() {
+        if (this.trajectory.length === 0) {
+            return;
+        }
+        // Compute advantages using GAE
+        this.computeGAE();
+        // Save old policy for ratio computation
+        this.saveOldPolicy();
+        // Multiple epochs of training
+        for (let epoch = 0; epoch < this.ppoConfig.ppoEpochs; epoch++) {
+            this.trainEpoch();
+        }
+        // Clear trajectory
+        this.trajectory = [];
+        this.logger.info('PPO training complete', {
+            epochs: this.ppoConfig.ppoEpochs,
+            steps: this.stepCount
+        });
+    }
+    /**
+     * Compute Generalized Advantage Estimation (GAE)
+     *
+     * GAE: Â_t = Σ_{l=0}^∞ (γλ)^l δ_{t+l}
+     * where δ_t = r_t + γV(s_{t+1}) - V(s_t)
+     */
+    computeGAE() {
+        const gamma = this.config.discountFactor;
+        const lambda = this.ppoConfig.gaeLambda;
+        let lastGaeLam = 0;
+        const n = this.trajectory.length;
+        // Compute returns and advantages backwards
+        for (let t = n - 1; t >= 0; t--) {
+            const step = this.trajectory[t];
+            const nextValue = step.done
+                ? 0
+                : (t < n - 1 ? this.trajectory[t + 1].value : this.valueTable.get(step.nextState) ?? 0);
+            // TD error
+            const delta = step.reward + gamma * nextValue - step.value;
+            // GAE advantage
+            lastGaeLam = step.done
+                ? delta
+                : delta + gamma * lambda * lastGaeLam;
+            step.advantage = lastGaeLam;
+            step.returns = step.advantage + step.value;
+        }
+        // Normalize advantages
+        const advantages = this.trajectory.map(s => s.advantage);
+        const mean = advantages.reduce((s, a) => s + a, 0) / advantages.length;
+        const variance = advantages.reduce((s, a) => s + (a - mean) ** 2, 0) / advantages.length;
+        const std = Math.sqrt(variance) + 1e-8;
+        for (const step of this.trajectory) {
+            step.advantage = (step.advantage - mean) / std;
+        }
+    }
+    /**
+     * Save current policy as old policy for ratio computation
+     */
+    saveOldPolicy() {
+        this.oldPolicyTable.clear();
+        for (const [state, actions] of this.policyTable.entries()) {
+            const actionMap = new Map();
+            for (const [action, params] of actions.entries()) {
+                actionMap.set(action, { ...params });
+            }
+            this.oldPolicyTable.set(state, actionMap);
+        }
+    }
+    /**
+     * Get old log probability for ratio computation
+     */
+    getOldLogProb(stateKey, actionKey) {
+        const statePolicy = this.oldPolicyTable.get(stateKey);
+        if (!statePolicy) {
+            return 0;
+        }
+        return statePolicy.get(actionKey)?.logProb ?? 0;
+    }
+    /**
+     * Train one epoch on the trajectory
+     */
+    trainEpoch() {
+        // Shuffle trajectory
+        const shuffled = [...this.trajectory].sort(() => Math.random() - 0.5);
+        // Mini-batch updates
+        for (let i = 0; i < shuffled.length; i += this.ppoConfig.miniBatchSize) {
+            const batch = shuffled.slice(i, i + this.ppoConfig.miniBatchSize);
+            this.trainMiniBatch(batch);
+        }
+    }
+    /**
+     * Train on a mini-batch
+     */
+    trainMiniBatch(batch) {
+        for (const step of batch) {
+            // Compute probability ratio
+            const newLogProb = this.getLogProb(step.state, step.action);
+            const oldLogProb = step.logProb; // Use stored log prob
+            const ratio = Math.exp(newLogProb - oldLogProb);
+            // Compute clipped and unclipped objectives
+            const eps = this.ppoConfig.clipEpsilon;
+            const surr1 = ratio * step.advantage;
+            const surr2 = Math.max(Math.min(ratio, 1 + eps), 1 - eps) * step.advantage;
+            // Policy loss (negative because we want to maximize)
+            const policyLoss = -Math.min(surr1, surr2);
+            // Value loss
+            const valueTarget = step.returns;
+            const currentValue = this.valueTable.get(step.state) ?? 0;
+            let valueLoss = (currentValue - valueTarget) ** 2;
+            // Clip value loss if enabled
+            if (this.ppoConfig.clipValueLoss) {
+                const clippedValue = step.value + Math.max(Math.min(currentValue - step.value, eps), -eps);
+                const clippedValueLoss = (clippedValue - valueTarget) ** 2;
+                valueLoss = Math.max(valueLoss, clippedValueLoss);
+            }
+            // Entropy bonus
+            const entropy = this.computeEntropy(step.state);
+            const entropyLoss = -this.ppoConfig.entropyCoefficient * entropy;
+            // Total loss
+            const totalLoss = policyLoss + this.ppoConfig.valueLossCoefficient * valueLoss + entropyLoss;
+            // Update policy (gradient ascent direction)
+            this.updatePolicy(step.state, step.action, step.advantage, ratio);
+            // Update value function
+            this.updateValue(step.state, valueTarget);
+        }
+    }
+    /**
+     * Update policy parameters
+     */
+    updatePolicy(stateKey, actionKey, advantage, ratio) {
+        if (!this.policyTable.has(stateKey)) {
+            this.policyTable.set(stateKey, new Map());
+        }
+        const statePolicy = this.policyTable.get(stateKey);
+        const current = statePolicy.get(actionKey) ?? { preference: 0, logProb: 0, updateCount: 0 };
+        // Clipped gradient
+        const eps = this.ppoConfig.clipEpsilon;
+        let gradient = advantage;
+        if ((ratio > 1 + eps && advantage > 0) || (ratio < 1 - eps && advantage < 0)) {
+            gradient = 0; // Clipped - no update
+        }
+        // Update preference
+        const newPreference = current.preference + this.ppoConfig.policyLearningRate * gradient;
+        const newLogProb = this.getLogProb(stateKey, actionKey);
+        statePolicy.set(actionKey, {
+            preference: newPreference,
+            logProb: newLogProb,
+            updateCount: current.updateCount + 1
+        });
+        // Update Q-table for compatibility
+        this.setQValue(stateKey, actionKey, newPreference);
+    }
+    /**
+     * Update value function
+     */
+    updateValue(stateKey, target) {
+        const current = this.valueTable.get(stateKey) ?? 0;
+        const newValue = current + this.ppoConfig.valueLearningRate * (target - current);
+        this.valueTable.set(stateKey, newValue);
+    }
+    /**
+     * Compute entropy of policy at state
+     */
+    computeEntropy(stateKey) {
+        const statePolicy = this.policyTable.get(stateKey);
+        if (!statePolicy || statePolicy.size === 0) {
+            return 0;
+        }
+        const prefs = Array.from(statePolicy.values()).map(p => p.preference);
+        const maxPref = Math.max(...prefs);
+        const expPrefs = prefs.map(p => Math.exp(p - maxPref));
+        const sumExp = expPrefs.reduce((s, e) => s + e, 0);
+        const probs = expPrefs.map(e => e / sumExp);
+        let entropy = 0;
+        for (const p of probs) {
+            if (p > 0) {
+                entropy -= p * Math.log(p);
+            }
+        }
+        return entropy;
+    }
+    /**
+     * Get default exploration rate for reset
+     */
+    getDefaultExplorationRate() {
+        return this.defaultExploration;
+    }
+    /**
+     * Get PPO-specific statistics
+     */
+    getPPOStatistics() {
+        let totalValue = 0;
+        for (const v of this.valueTable.values()) {
+            totalValue += v;
+        }
+        let policySize = 0;
+        for (const statePolicy of this.policyTable.values()) {
+            policySize += statePolicy.size;
+        }
+        const avgAdvantage = this.trajectory.length > 0
+            ? this.trajectory.reduce((s, t) => s + t.advantage, 0) / this.trajectory.length
+            : 0;
+        return {
+            trajectoryLength: this.trajectory.length,
+            valueTableSize: this.valueTable.size,
+            policyTableSize: policySize,
+            avgValue: this.valueTable.size > 0 ? totalValue / this.valueTable.size : 0,
+            avgAdvantage,
+            clipFraction: 0 // Would need tracking during training
+        };
+    }
+    /**
+     * Reset PPO-specific state
+     */
+    reset() {
+        super.reset();
+        this.policyTable.clear();
+        this.valueTable.clear();
+        this.oldPolicyTable.clear();
+        this.trajectory = [];
+        this.logger.info('PPOLearner reset');
+    }
+    /**
+     * Export PPO state
+     */
+    exportPPO() {
+        const serializedPolicy = {};
+        for (const [state, actions] of this.policyTable.entries()) {
+            serializedPolicy[state] = {};
+            for (const [action, params] of actions.entries()) {
+                serializedPolicy[state][action] = params;
+            }
+        }
+        const serializedValue = {};
+        for (const [state, value] of this.valueTable.entries()) {
+            serializedValue[state] = value;
+        }
+        return {
+            base: this.export(),
+            policyTable: serializedPolicy,
+            valueTable: serializedValue,
+            ppoConfig: { ...this.ppoConfig }
+        };
+    }
+    /**
+     * Import PPO state
+     */
+    importPPO(state) {
+        this.import(state.base);
+        this.policyTable.clear();
+        for (const [stateKey, actions] of Object.entries(state.policyTable)) {
+            const actionMap = new Map();
+            for (const [actionKey, params] of Object.entries(actions)) {
+                actionMap.set(actionKey, params);
+            }
+            this.policyTable.set(stateKey, actionMap);
+        }
+        this.valueTable.clear();
+        for (const [stateKey, value] of Object.entries(state.valueTable)) {
+            this.valueTable.set(stateKey, value);
+        }
+        this.ppoConfig = { ...state.ppoConfig };
+        this.logger.info('Imported PPO state', {
+            policySize: this.policyTable.size,
+            valueSize: this.valueTable.size
+        });
+    }
+}
+exports.PPOLearner = PPOLearner;
+/**
+ * Create default PPO configuration
+ */
+function createDefaultPPOConfig() {
+    return {
+        learningRate: 0.0003,
+        discountFactor: 0.99,
+        explorationRate: 0.0, // PPO uses entropy for exploration
+        explorationDecay: 1.0,
+        minExplorationRate: 0.0,
+        clipEpsilon: 0.2,
+        ppoEpochs: 4,
+        miniBatchSize: 64,
+        valueLossCoefficient: 0.5,
+        entropyCoefficient: 0.01,
+        gaeLambda: 0.95,
+        maxGradNorm: 0.5,
+        clipValueLoss: true,
+        policyLearningRate: 0.0003,
+        valueLearningRate: 0.001,
+        useExperienceReplay: false, // PPO doesn't use replay buffer
+        replayBufferSize: 2048, // Used as trajectory buffer size
+        batchSize: 64
+    };
+}
+//# sourceMappingURL=PPOLearner.js.map