agentic-qe 2.1.2 → 2.2.0

package/dist/learning/algorithms/PPOLearner.js

@@ -0,0 +1,490 @@
+"use strict";
+/**
+ * PPOLearner - Proximal Policy Optimization Algorithm
+ *
+ * Implements PPO-Clip, the most widely used variant of PPO:
+ * - Clipped surrogate objective to prevent large policy updates
+ * - Generalized Advantage Estimation (GAE) for variance reduction
+ * - Value function clipping for stability
+ * - Multiple epochs over collected trajectories
+ *
+ * Key features:
+ * - Trust region optimization without KL constraint
+ * - Sample efficient with mini-batch updates
+ * - Robust to hyperparameter choices
+ * - Suitable for continuous and discrete action spaces
+ *
+ * PPO-Clip objective:
+ * L^CLIP(θ) = E[min(r(θ)Â, clip(r(θ), 1-ε, 1+ε)Â)]
+ * where r(θ) = π_θ(a|s) / π_θ_old(a|s)
+ *
+ * @module learning/algorithms/PPOLearner
+ * @version 1.0.0
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PPOLearner = void 0;
+exports.createDefaultPPOConfig = createDefaultPPOConfig;
+const AbstractRLLearner_1 = require("./AbstractRLLearner");
+/**
+ * PPOLearner - Proximal Policy Optimization implementation
+ *
+ * PPO is a state-of-the-art policy gradient method that achieves
+ * strong performance while being simpler than TRPO.
+ *
+ * Usage:
+ * ```typescript
+ * const ppo = new PPOLearner({
+ *   learningRate: 0.0003,
+ *   discountFactor: 0.99,
+ *   explorationRate: 0.0,
+ *   explorationDecay: 1.0,
+ *   minExplorationRate: 0.0,
+ *   clipEpsilon: 0.2,
+ *   ppoEpochs: 4,
+ *   miniBatchSize: 64,
+ *   valueLossCoefficient: 0.5,
+ *   entropyCoefficient: 0.01,
+ *   gaeLambda: 0.95,
+ *   maxGradNorm: 0.5,
+ *   clipValueLoss: true,
+ *   policyLearningRate: 0.0003,
+ *   valueLearningRate: 0.001,
+ *   useExperienceReplay: false,
+ *   replayBufferSize: 2048,
+ *   batchSize: 64
+ * });
+ *
+ * // Collect trajectory
+ * ppo.collectStep(state, action, reward, nextState, done);
+ *
+ * // Train on collected trajectory
+ * ppo.trainOnTrajectory();
+ * ```
+ */
+class PPOLearner extends AbstractRLLearner_1.AbstractRLLearner {
+    constructor(config) {
+        super(config);
+        this.ppoConfig = config;
+        this.policyTable = new Map();
+        this.valueTable = new Map();
+        this.oldPolicyTable = new Map();
+        this.trajectory = [];
+        this.defaultExploration = config.explorationRate;
+        this.logger.info('PPOLearner initialized', {
+            clipEpsilon: config.clipEpsilon,
+            epochs: config.ppoEpochs,
+            gaeLambda: config.gaeLambda,
+            entropyCoeff: config.entropyCoefficient
+        });
+    }
+    /**
+     * Select action using current policy (softmax)
+     */
+    selectAction(state, availableActions) {
+        if (availableActions.length === 0) {
+            throw new Error('No available actions to select from');
+        }
+        const stateKey = this.encodeState(state);
+        const probs = this.getActionProbabilities(stateKey, availableActions);
+        // Sample from distribution
+        const random = Math.random();
+        let cumulative = 0;
+        for (let i = 0; i < availableActions.length; i++) {
+            cumulative += probs[i];
+            if (random <= cumulative) {
+                return availableActions[i];
+            }
+        }
+        return availableActions[availableActions.length - 1];
+    }
+    /**
+     * Get action probabilities using softmax policy
+     */
+    getActionProbabilities(stateKey, availableActions) {
+        const preferences = [];
+        for (const action of availableActions) {
+            const actionKey = this.encodeAction(action);
+            const params = this.getPolicyParams(stateKey, actionKey);
+            preferences.push(params.preference);
+        }
+        // Softmax with numerical stability
+        const maxPref = Math.max(...preferences);
+        const expPrefs = preferences.map(p => Math.exp(p - maxPref));
+        const sumExp = expPrefs.reduce((sum, e) => sum + e, 0);
+        return expPrefs.map(e => e / sumExp);
+    }
+    /**
+     * Get policy parameters for state-action pair
+     */
+    getPolicyParams(stateKey, actionKey) {
+        const statePolicy = this.policyTable.get(stateKey);
+        if (!statePolicy) {
+            return { preference: 0, logProb: 0, updateCount: 0 };
+        }
+        return statePolicy.get(actionKey) ?? { preference: 0, logProb: 0, updateCount: 0 };
+    }
+    /**
+     * Get log probability of action under current policy
+     */
+    getLogProb(stateKey, actionKey, availableActions) {
+        // Get preference for target action
+        const params = this.getPolicyParams(stateKey, actionKey);
+        // If we don't know the action space, return stored log prob
+        if (!availableActions) {
+            return params.logProb;
+        }
+        // Calculate actual log probability
+        const prefs = [];
+        let targetPref = params.preference;
+        for (const action of availableActions) {
+            const ak = this.encodeAction(action);
+            const p = this.getPolicyParams(stateKey, ak);
+            prefs.push(p.preference);
+            if (ak === actionKey) {
+                targetPref = p.preference;
+            }
+        }
+        const maxPref = Math.max(...prefs, targetPref);
+        const expTarget = Math.exp(targetPref - maxPref);
+        const sumExp = prefs.reduce((sum, p) => sum + Math.exp(p - maxPref), 0);
+        return Math.log(expTarget / sumExp);
+    }
+    /**
+     * Get state value from value network
+     */
+    getStateValue(state) {
+        const stateKey = this.encodeState(state);
+        return this.valueTable.get(stateKey) ?? 0;
+    }
+    /**
+     * Collect a step in the trajectory
+     */
+    collectStep(state, action, reward, nextState, done) {
+        const stateKey = this.encodeState(state);
+        const actionKey = this.encodeAction(action);
+        const nextStateKey = this.encodeState(nextState);
+        const value = this.valueTable.get(stateKey) ?? 0;
+        const logProb = this.getLogProb(stateKey, actionKey);
+        this.trajectory.push({
+            state: stateKey,
+            action: actionKey,
+            reward,
+            nextState: nextStateKey,
+            done,
+            value,
+            logProb,
+            advantage: 0, // Computed later
+            returns: 0 // Computed later
+        });
+    }
+    /**
+     * Standard update interface - collects experience and trains when ready
+     */
+    update(experience, nextAction) {
+        this.stepCount++;
+        const { state, action, reward, nextState } = experience;
+        const done = experience.done ?? false;
+        // Collect step
+        this.collectStep(state, action, reward, nextState, done);
+        // Train when trajectory is large enough
+        if (this.trajectory.length >= this.ppoConfig.replayBufferSize) {
+            this.trainOnTrajectory();
+        }
+    }
+    /**
+     * Train on collected trajectory using PPO
+     */
+    trainOnTrajectory() {
+        if (this.trajectory.length === 0) {
+            return;
+        }
+        // Compute advantages using GAE
+        this.computeGAE();
+        // Save old policy for ratio computation
+        this.saveOldPolicy();
+        // Multiple epochs of training
+        for (let epoch = 0; epoch < this.ppoConfig.ppoEpochs; epoch++) {
+            this.trainEpoch();
+        }
+        // Clear trajectory
+        this.trajectory = [];
+        this.logger.info('PPO training complete', {
+            epochs: this.ppoConfig.ppoEpochs,
+            steps: this.stepCount
+        });
+    }
+    /**
+     * Compute Generalized Advantage Estimation (GAE)
+     *
+     * GAE: Â_t = Σ_{l=0}^∞ (γλ)^l δ_{t+l}
+     * where δ_t = r_t + γV(s_{t+1}) - V(s_t)
+     */
+    computeGAE() {
+        const gamma = this.config.discountFactor;
+        const lambda = this.ppoConfig.gaeLambda;
+        let lastGaeLam = 0;
+        const n = this.trajectory.length;
+        // Compute returns and advantages backwards
+        for (let t = n - 1; t >= 0; t--) {
+            const step = this.trajectory[t];
+            const nextValue = step.done
+                ? 0
+                : (t < n - 1 ? this.trajectory[t + 1].value : this.valueTable.get(step.nextState) ?? 0);
+            // TD error
+            const delta = step.reward + gamma * nextValue - step.value;
+            // GAE advantage
+            lastGaeLam = step.done
+                ? delta
+                : delta + gamma * lambda * lastGaeLam;
+            step.advantage = lastGaeLam;
+            step.returns = step.advantage + step.value;
+        }
+        // Normalize advantages
+        const advantages = this.trajectory.map(s => s.advantage);
+        const mean = advantages.reduce((s, a) => s + a, 0) / advantages.length;
+        const variance = advantages.reduce((s, a) => s + (a - mean) ** 2, 0) / advantages.length;
+        const std = Math.sqrt(variance) + 1e-8;
+        for (const step of this.trajectory) {
+            step.advantage = (step.advantage - mean) / std;
+        }
+    }
+    /**
+     * Save current policy as old policy for ratio computation
+     */
+    saveOldPolicy() {
+        this.oldPolicyTable.clear();
+        for (const [state, actions] of this.policyTable.entries()) {
+            const actionMap = new Map();
+            for (const [action, params] of actions.entries()) {
+                actionMap.set(action, { ...params });
+            }
+            this.oldPolicyTable.set(state, actionMap);
+        }
+    }
+    /**
+     * Get old log probability for ratio computation
+     */
+    getOldLogProb(stateKey, actionKey) {
+        const statePolicy = this.oldPolicyTable.get(stateKey);
+        if (!statePolicy) {
+            return 0;
+        }
+        return statePolicy.get(actionKey)?.logProb ?? 0;
+    }
+    /**
+     * Train one epoch on the trajectory
+     */
+    trainEpoch() {
+        // Shuffle trajectory
+        const shuffled = [...this.trajectory].sort(() => Math.random() - 0.5);
+        // Mini-batch updates
+        for (let i = 0; i < shuffled.length; i += this.ppoConfig.miniBatchSize) {
+            const batch = shuffled.slice(i, i + this.ppoConfig.miniBatchSize);
+            this.trainMiniBatch(batch);
+        }
+    }
+    /**
+     * Train on a mini-batch
+     */
+    trainMiniBatch(batch) {
+        for (const step of batch) {
+            // Compute probability ratio
+            const newLogProb = this.getLogProb(step.state, step.action);
+            const oldLogProb = step.logProb; // Use stored log prob
+            const ratio = Math.exp(newLogProb - oldLogProb);
+            // Compute clipped and unclipped objectives
+            const eps = this.ppoConfig.clipEpsilon;
+            const surr1 = ratio * step.advantage;
+            const surr2 = Math.max(Math.min(ratio, 1 + eps), 1 - eps) * step.advantage;
+            // Policy loss (negative because we want to maximize)
+            const policyLoss = -Math.min(surr1, surr2);
+            // Value loss
+            const valueTarget = step.returns;
+            const currentValue = this.valueTable.get(step.state) ?? 0;
+            let valueLoss = (currentValue - valueTarget) ** 2;
+            // Clip value loss if enabled
+            if (this.ppoConfig.clipValueLoss) {
+                const clippedValue = step.value + Math.max(Math.min(currentValue - step.value, eps), -eps);
+                const clippedValueLoss = (clippedValue - valueTarget) ** 2;
+                valueLoss = Math.max(valueLoss, clippedValueLoss);
+            }
+            // Entropy bonus
+            const entropy = this.computeEntropy(step.state);
+            const entropyLoss = -this.ppoConfig.entropyCoefficient * entropy;
+            // Total loss
+            const totalLoss = policyLoss + this.ppoConfig.valueLossCoefficient * valueLoss + entropyLoss;
+            // Update policy (gradient ascent direction)
+            this.updatePolicy(step.state, step.action, step.advantage, ratio);
+            // Update value function
+            this.updateValue(step.state, valueTarget);
+        }
+    }
+    /**
+     * Update policy parameters
+     */
+    updatePolicy(stateKey, actionKey, advantage, ratio) {
+        if (!this.policyTable.has(stateKey)) {
+            this.policyTable.set(stateKey, new Map());
+        }
+        const statePolicy = this.policyTable.get(stateKey);
+        const current = statePolicy.get(actionKey) ?? { preference: 0, logProb: 0, updateCount: 0 };
+        // Clipped gradient
+        const eps = this.ppoConfig.clipEpsilon;
+        let gradient = advantage;
+        if ((ratio > 1 + eps && advantage > 0) || (ratio < 1 - eps && advantage < 0)) {
+            gradient = 0; // Clipped - no update
+        }
+        // Update preference
+        const newPreference = current.preference + this.ppoConfig.policyLearningRate * gradient;
+        const newLogProb = this.getLogProb(stateKey, actionKey);
+        statePolicy.set(actionKey, {
+            preference: newPreference,
+            logProb: newLogProb,
+            updateCount: current.updateCount + 1
+        });
+        // Update Q-table for compatibility
+        this.setQValue(stateKey, actionKey, newPreference);
+    }
+    /**
+     * Update value function
+     */
+    updateValue(stateKey, target) {
+        const current = this.valueTable.get(stateKey) ?? 0;
+        const newValue = current + this.ppoConfig.valueLearningRate * (target - current);
+        this.valueTable.set(stateKey, newValue);
+    }
+    /**
+     * Compute entropy of policy at state
+     */
+    computeEntropy(stateKey) {
+        const statePolicy = this.policyTable.get(stateKey);
+        if (!statePolicy || statePolicy.size === 0) {
+            return 0;
+        }
+        const prefs = Array.from(statePolicy.values()).map(p => p.preference);
+        const maxPref = Math.max(...prefs);
+        const expPrefs = prefs.map(p => Math.exp(p - maxPref));
+        const sumExp = expPrefs.reduce((s, e) => s + e, 0);
+        const probs = expPrefs.map(e => e / sumExp);
+        let entropy = 0;
+        for (const p of probs) {
+            if (p > 0) {
+                entropy -= p * Math.log(p);
+            }
+        }
+        return entropy;
+    }
+    /**
+     * Get default exploration rate for reset
+     */
+    getDefaultExplorationRate() {
+        return this.defaultExploration;
+    }
+    /**
+     * Get PPO-specific statistics
+     */
+    getPPOStatistics() {
+        let totalValue = 0;
+        for (const v of this.valueTable.values()) {
+            totalValue += v;
+        }
+        let policySize = 0;
+        for (const statePolicy of this.policyTable.values()) {
+            policySize += statePolicy.size;
+        }
+        const avgAdvantage = this.trajectory.length > 0
+            ? this.trajectory.reduce((s, t) => s + t.advantage, 0) / this.trajectory.length
+            : 0;
+        return {
+            trajectoryLength: this.trajectory.length,
+            valueTableSize: this.valueTable.size,
+            policyTableSize: policySize,
+            avgValue: this.valueTable.size > 0 ? totalValue / this.valueTable.size : 0,
+            avgAdvantage,
+            clipFraction: 0 // Would need tracking during training
+        };
+    }
+    /**
+     * Reset PPO-specific state
+     */
+    reset() {
+        super.reset();
+        this.policyTable.clear();
+        this.valueTable.clear();
+        this.oldPolicyTable.clear();
+        this.trajectory = [];
+        this.logger.info('PPOLearner reset');
+    }
+    /**
+     * Export PPO state
+     */
+    exportPPO() {
+        const serializedPolicy = {};
+        for (const [state, actions] of this.policyTable.entries()) {
+            serializedPolicy[state] = {};
+            for (const [action, params] of actions.entries()) {
+                serializedPolicy[state][action] = params;
+            }
+        }
+        const serializedValue = {};
+        for (const [state, value] of this.valueTable.entries()) {
+            serializedValue[state] = value;
+        }
+        return {
+            base: this.export(),
+            policyTable: serializedPolicy,
+            valueTable: serializedValue,
+            ppoConfig: { ...this.ppoConfig }
+        };
+    }
+    /**
+     * Import PPO state
+     */
+    importPPO(state) {
+        this.import(state.base);
+        this.policyTable.clear();
+        for (const [stateKey, actions] of Object.entries(state.policyTable)) {
+            const actionMap = new Map();
+            for (const [actionKey, params] of Object.entries(actions)) {
+                actionMap.set(actionKey, params);
+            }
+            this.policyTable.set(stateKey, actionMap);
+        }
+        this.valueTable.clear();
+        for (const [stateKey, value] of Object.entries(state.valueTable)) {
+            this.valueTable.set(stateKey, value);
+        }
+        this.ppoConfig = { ...state.ppoConfig };
+        this.logger.info('Imported PPO state', {
+            policySize: this.policyTable.size,
+            valueSize: this.valueTable.size
+        });
+    }
+}
+exports.PPOLearner = PPOLearner;
+/**
+ * Create default PPO configuration
+ */
+function createDefaultPPOConfig() {
+    return {
+        learningRate: 0.0003,
+        discountFactor: 0.99,
+        explorationRate: 0.0, // PPO uses entropy for exploration
+        explorationDecay: 1.0,
+        minExplorationRate: 0.0,
+        clipEpsilon: 0.2,
+        ppoEpochs: 4,
+        miniBatchSize: 64,
+        valueLossCoefficient: 0.5,
+        entropyCoefficient: 0.01,
+        gaeLambda: 0.95,
+        maxGradNorm: 0.5,
+        clipValueLoss: true,
+        policyLearningRate: 0.0003,
+        valueLearningRate: 0.001,
+        useExperienceReplay: false, // PPO doesn't use replay buffer
+        replayBufferSize: 2048, // Used as trajectory buffer size
+        batchSize: 64
+    };
+}
+//# sourceMappingURL=PPOLearner.js.map

package/dist/learning/algorithms/PPOLearner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"PPOLearner.js","sourceRoot":"","sources":["../../../src/learning/algorithms/PPOLearner.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;;;AA2lBH,wDAqBC;AA9mBD,2DAA0E;AAqD1E;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AACH,MAAa,UAAW,SAAQ,qCAAiB;IAQ/C,YAAY,MAAiB;QAC3B,KAAK,CAAC,MAAM,CAAC,CAAC;QACd,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC;QACxB,IAAI,CAAC,WAAW,GAAG,IAAI,GAAG,EAAE,CAAC;QAC7B,IAAI,CAAC,UAAU,GAAG,IAAI,GAAG,EAAE,CAAC;QAC5B,IAAI,CAAC,cAAc,GAAG,IAAI,GAAG,EAAE,CAAC;QAChC,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC;QACrB,IAAI,CAAC,kBAAkB,GAAG,MAAM,CAAC,eAAe,CAAC;QAEjD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,wBAAwB,EAAE;YACzC,WAAW,EAAE,MAAM,CAAC,WAAW;YAC/B,MAAM,EAAE,MAAM,CAAC,SAAS;YACxB,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,YAAY,EAAE,MAAM,CAAC,kBAAkB;SACxC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACM,YAAY,CAAC,KAAgB,EAAE,gBAA+B;QACrE,IAAI,gBAAgB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAClC,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;QACzD,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QACzC,MAAM,KAAK,GAAG,IAAI,CAAC,sBAAsB,CAAC,QAAQ,EAAE,gBAAgB,CAAC,CAAC;QAEtE,2BAA2B;QAC3B,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;QAC7B,IAAI,UAAU,GAAG,CAAC,CAAC;QAEnB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,gBAAgB,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACjD,UAAU,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC;YACvB,IAAI,MAAM,IAAI,UAAU,EAAE,CAAC;gBACzB,OAAO,gBAAgB,CAAC,CAAC,CAAC,CAAC;YAC7B,CAAC;QACH,CAAC;QAED,OAAO,gBAAgB,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACvD,CAAC;IAED;;OAEG;IACK,sBAAsB,CAAC,QAAgB,EAAE,gBAA+B;QAC9E,MAAM,WAAW,GAAa,EAAE,CAAC;QAEjC,KAAK,MAAM,MAAM,IAAI,gBAAgB,EAAE,CAAC;YACtC,MAAM,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;YAC5C,MAAM,MAAM,GAAG,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;YACzD,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QACtC,CAAC;QAED,mCAAmC;QACnC,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,WAAW,CAAC,CAAC;QACzC,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;QAC7D,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;QAEvD,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC;IACvC,CAAC;IAED;;OAEG;IACK,eAAe,CAAC,QAAgB,EAAE,SAAiB;QACzD,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACnD,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,OAAO,EAAE,UAAU,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,CAAC;QACvD,CAAC;QACD,OAAO,WAAW,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,EAAE,UAAU,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,CAAC;IACrF,CAAC;IAED;;OAEG;IACK,UAAU,CAAC,QAAgB,EAAE,SAAiB,EAAE,gBAAgC;QACtF,mCAAmC;QACnC,MAAM,MAAM,GAAG,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;QAEzD,4DAA4D;QAC5D,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACtB,OAAO,MAAM,CAAC,OAAO,CAAC;QACxB,CAAC;QAED,mCAAmC;QACnC,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,IAAI,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC;QAEnC,KAAK,MAAM,MAAM,IAAI,gBAAgB,EAAE,CAAC;YACtC,MAAM,EAAE,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;YACrC,MAAM,CAAC,GAAG,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;YAC7C,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;YACzB,IAAI,EAAE,KAAK,SAAS,EAAE,CAAC;gBACrB,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC;YAC5B,CAAC;QACH,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,KAAK,EAAE,UAAU,CAAC,CAAC;QAC/C,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,GAAG,OAAO,CAAC,CAAC;QACjD,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC;QAExE,OAAO,IAAI,CAAC,GAAG,CAAC,SAAS,GAAG,MAAM,CAAC,CAAC;IACtC,CAAC;IAED;;OAEG;IACM,aAAa,CAAC,KAAgB;QACrC,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QACzC,OAAO,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IAC5C,CAAC;IAED;;OAEG;IACH,WAAW,CACT,KAAgB,EAChB,MAAmB,EACnB,MAAc,EACd,SAAoB,EACpB,IAAa;QAEb,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QACzC,MAAM,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;QAC5C,MAAM,YAAY,GAAG,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC;QAEjD,MAAM,KAAK,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACjD,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;QAErD,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;YACnB,KAAK,EAAE,QAAQ;YACf,MAAM,EAAE,SAAS;YACjB,MAAM;YACN,SAAS,EAAE,YAAY;YACvB,IAAI;YACJ,KAAK;YACL,OAAO;YACP,SAAS,EAAE,CAAC,EAAE,iBAAiB;YAC/B,OAAO,EAAE,CAAC,CAAI,iBAAiB;SAChC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACM,MAAM,CAAC,UAA0B,EAAE,UAAwB;QAClE,IAAI,CAAC,SAAS,EAAE,CAAC;QAEjB,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,UAAU,CAAC;QACxD,MAAM,IAAI,GAAG,UAAU,CAAC,IAAI,IAAI,KAAK,CAAC;QAEtC,eAAe;QACf,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC;QAEzD,wCAAwC;QACxC,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,CAAC,gBAAgB,EAAE,CAAC;YAC9D,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAC3B,CAAC;IACH,CAAC;IAED;;OAEG;IACH,iBAAiB;QACf,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACjC,OAAO;QACT,CAAC;QAED,+BAA+B;QAC/B,IAAI,CAAC,UAAU,EAAE,CAAC;QAElB,wCAAwC;QACxC,IAAI,CAAC,aAAa,EAAE,CAAC;QAErB,8BAA8B;QAC9B,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,KAAK,EAAE,EAAE,CAAC;YAC9D,IAAI,CAAC,UAAU,EAAE,CAAC;QACpB,CAAC;QAED,mBAAmB;QACnB,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC;QAErB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,uBAAuB,EAAE;YACxC,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS;YAChC,KAAK,EAAE,IAAI,CAAC,SAAS;SACtB,CAAC,CAAC;IACL,CAAC;IAED;;;;;OAKG;IACK,UAAU;QAChB,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC;QACzC,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC;QAExC,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,MAAM,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;QAEjC,2CAA2C;QAC3C,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAChC,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;YAEhC,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI;gBACzB,CAAC,CAAC,CAAC;gBACH,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;YAE1F,WAAW;YACX,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,GAAG,KAAK,GAAG,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC;YAE3D,gBAAgB;YAChB,UAAU,GAAG,IAAI,CAAC,IAAI;gBACpB,CAAC,CAAC,KAAK;gBACP,CAAC,CAAC,KAAK,GAAG,KAAK,GAAG,MAAM,GAAG,UAAU,CAAC;YAExC,IAAI,CAAC,SAAS,GAAG,UAAU,CAAC;YAC5B,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC;QAC7C,CAAC;QAED,uBAAuB;QACvB,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QACzD,MAAM,IAAI,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,UAAU,CAAC,MAAM,CAAC;QACvE,MAAM,QAAQ,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,GAAG,UAAU,CAAC,MAAM,CAAC;QACzF,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,IAAI,CAAC;QAEvC,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACnC,IAAI,CAAC,SAAS,GAAG,CAAC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,GAAG,CAAC;QACjD,CAAC;IACH,CAAC;IAED;;OAEG;IACK,aAAa;QACnB,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,CAAC;QAC5B,KAAK,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,EAAE,CAAC;YAC1D,MAAM,SAAS,GAAG,IAAI,GAAG,EAAwB,CAAC;YAClD,KAAK,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;gBACjD,SAAS,CAAC,GAAG,CAAC,MAAM,EAAE,EAAE,GAAG,MAAM,EAAE,CAAC,CAAC;YACvC,CAAC;YACD,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QAC5C,CAAC;IACH,CAAC;IAED;;OAEG;IACK,aAAa,CAAC,QAAgB,EAAE,SAAiB;QACvD,MAAM,WAAW,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACtD,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,OAAO,CAAC,CAAC;QACX,CAAC;QACD,OAAO,WAAW,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,OAAO,IAAI,CAAC,CAAC;IAClD,CAAC;IAED;;OAEG;IACK,UAAU;QAChB,qBAAqB;QACrB,MAAM,QAAQ,GAAG,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG,CAAC,CAAC;QAEtE,qBAAqB;QACrB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,aAAa,EAAE,CAAC;YACvE,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC;YAClE,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;QAC7B,CAAC;IACH,CAAC;IAED;;OAEG;IACK,cAAc,CAAC,KAAuB;QAC5C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,4BAA4B;YAC5B,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;YAC5D,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,sBAAsB;YACvD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,GAAG,UAAU,CAAC,CAAC;YAEhD,2CAA2C;YAC3C,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC;YACvC,MAAM,KAAK,GAAG,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC;YACrC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,GAAG,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC;YAE3E,qDAAqD;YACrD,MAAM,UAAU,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;YAE3C,aAAa;YACb,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC;YACjC,MAAM,YAAY,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAC1D,IAAI,SAAS,GAAG,CAAC,YAAY,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;YAElD,6BAA6B;YAC7B,IAAI,IAAI,CAAC,SAAS,CAAC,aAAa,EAAE,CAAC;gBACjC,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,YAAY,GAAG,IAAI,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC;gBAC3F,MAAM,gBAAgB,GAAG,CAAC,YAAY,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;gBAC3D,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,gBAAgB,CAAC,CAAC;YACpD,CAAC;YAED,gBAAgB;YAChB,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAChD,MAAM,WAAW,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,kBAAkB,GAAG,OAAO,CAAC;YAEjE,aAAa;YACb,MAAM,SAAS,GAAG,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,oBAAoB,GAAG,SAAS,GAAG,WAAW,CAAC;YAE7F,4CAA4C;YAC5C,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;YAElE,wBAAwB;YACxB,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;QAC5C,CAAC;IACH,CAAC;IAED;;OAEG;IACK,YAAY,CAClB,QAAgB,EAChB,SAAiB,EACjB,SAAiB,EACjB,KAAa;QAEb,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YACpC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,QAAQ,EAAE,IAAI,GAAG,EAAE,CAAC,CAAC;QAC5C,CAAC;QACD,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAE,CAAC;QAEpD,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,EAAE,UAAU,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,CAAC;QAE5F,mBAAmB;QACnB,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC;QACvC,IAAI,QAAQ,GAAG,SAAS,CAAC;QACzB,IAAI,CAAC,KAAK,GAAG,CAAC,GAAG,GAAG,IAAI,SAAS,GAAG,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,GAAG,GAAG,IAAI,SAAS,GAAG,CAAC,CAAC,EAAE,CAAC;YAC7E,QAAQ,GAAG,CAAC,CAAC,CAAC,sBAAsB;QACtC,CAAC;QAED,oBAAoB;QACpB,MAAM,aAAa,GAAG,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,kBAAkB,GAAG,QAAQ,CAAC;QACxF,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;QAExD,WAAW,CAAC,GAAG,CAAC,SAAS,EAAE;YACzB,UAAU,EAAE,aAAa;YACzB,OAAO,EAAE,UAAU;YACnB,WAAW,EAAE,OAAO,CAAC,WAAW,GAAG,CAAC;SACrC,CAAC,CAAC;QAEH,mCAAmC;QACnC,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,SAAS,EAAE,aAAa,CAAC,CAAC;IACrD,CAAC;IAED;;OAEG;IACK,WAAW,CAAC,QAAgB,EAAE,MAAc;QAClD,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACnD,MAAM,QAAQ,GAAG,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,iBAAiB,GAAG,CAAC,MAAM,GAAG,OAAO,CAAC,CAAC;QACjF,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IAC1C,CAAC;IAED;;OAEG;IACK,cAAc,CAAC,QAAgB;QACrC,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACnD,IAAI,CAAC,WAAW,IAAI,WAAW,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;YAC3C,OAAO,CAAC,CAAC;QACX,CAAC;QAED,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;QACtE,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC;QACnC,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;QACvD,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;QACnD,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC;QAE5C,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;YACtB,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;gBACV,OAAO,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YAC7B,CAAC;QACH,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;OAEG;IACO,yBAAyB;QACjC,OAAO,IAAI,CAAC,kBAAkB,CAAC;IACjC,CAAC;IAED;;OAEG;IACH,gBAAgB;QAQd,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,EAAE,CAAC;YACzC,UAAU,IAAI,CAAC,CAAC;QAClB,CAAC;QAED,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,KAAK,MAAM,WAAW,IAAI,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,EAAE,CAAC;YACpD,UAAU,IAAI,WAAW,CAAC,IAAI,CAAC;QACjC,CAAC;QAED,MAAM,YAAY,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC;YAC7C,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM;YAC/E,CAAC,CAAC,CAAC,CAAC;QAEN,OAAO;YACL,gBAAgB,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM;YACxC,cAAc,EAAE,IAAI,CAAC,UAAU,CAAC,IAAI;YACpC,eAAe,EAAE,UAAU;YAC3B,QAAQ,EAAE,IAAI,CAAC,UAAU,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC1E,YAAY;YACZ,YAAY,EAAE,CAAC,CAAC,sCAAsC;SACvD,CAAC;IACJ,CAAC;IAED;;OAEG;IACM,KAAK;QACZ,KAAK,CAAC,KAAK,EAAE,CAAC;QACd,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;QACxB,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,CAAC;QAC5B,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC;QACrB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;IACvC,CAAC;IAED;;OAEG;IACH,SAAS;QAMP,MAAM,gBAAgB,GAAiD,EAAE,CAAC;QAC1E,KAAK,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,EAAE,CAAC;YAC1D,gBAAgB,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;YAC7B,KAAK,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;gBACjD,gBAAgB,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,GAAG,MAAM,CAAC;YAC3C,CAAC;QACH,CAAC;QAED,MAAM,eAAe,GAA2B,EAAE,CAAC;QACnD,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC;YACvD,eAAe,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC;QACjC,CAAC;QAED,OAAO;YACL,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE;YACnB,WAAW,EAAE,gBAAgB;YAC7B,UAAU,EAAE,eAAe;YAC3B,SAAS,EAAE,EAAE,GAAG,IAAI,CAAC,SAAS,EAAE;SACjC,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,SAAS,CAAC,KAAwC;QAChD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAExB,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;QACzB,KAAK,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,EAAE,CAAC;YACpE,MAAM,SAAS,GAAG,IAAI,GAAG,EAAwB,CAAC;YAClD,KAAK,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC1D,SAAS,CAAC,GAAG,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;YACnC,CAAC;YACD,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;QAC5C,CAAC;QAED,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;QACxB,KAAK,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,EAAE,CAAC;YACjE,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QACvC,CAAC;QAED,IAAI,CAAC,SAAS,GAAG,EAAE,GAAG,KAAK,CAAC,SAAS,EAAE,CAAC;QAExC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,oBAAoB,EAAE;YACrC,UAAU,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI;YACjC,SAAS,EAAE,IAAI,CAAC,UAAU,CAAC,IAAI;SAChC,CAAC,CAAC;IACL,CAAC;CACF;AA3fD,gCA2fC;AAED;;GAEG;AACH,SAAgB,sBAAsB;IACpC,OAAO;QACL,YAAY,EAAE,MAAM;QACpB,cAAc,EAAE,IAAI;QACpB,eAAe,EAAE,GAAG,EAAE,mCAAmC;QACzD,gBAAgB,EAAE,GAAG;QACrB,kBAAkB,EAAE,GAAG;QACvB,WAAW,EAAE,GAAG;QAChB,SAAS,EAAE,CAAC;QACZ,aAAa,EAAE,EAAE;QACjB,oBAAoB,EAAE,GAAG;QACzB,kBAAkB,EAAE,IAAI;QACxB,SAAS,EAAE,IAAI;QACf,WAAW,EAAE,GAAG;QAChB,aAAa,EAAE,IAAI;QACnB,kBAAkB,EAAE,MAAM;QAC1B,iBAAiB,EAAE,KAAK;QACxB,mBAAmB,EAAE,KAAK,EAAE,gCAAgC;QAC5D,gBAAgB,EAAE,IAAI,EAAM,iCAAiC;QAC7D,SAAS,EAAE,EAAE;KACd,CAAC;AACJ,CAAC"}

package/dist/learning/algorithms/QLearning.d.ts

@@ -0,0 +1,68 @@
+/**
+ * QLearning - Off-policy TD(0) Reinforcement Learning
+ *
+ * Implements standard Q-learning algorithm for reinforcement learning.
+ * Key differences from SARSA:
+ * - Off-policy: learns optimal Q-values regardless of policy being followed
+ * - Uses max Q-value for next state, not actual next action
+ * - Update rule: Q(s,a) ← Q(s,a) + α[r + γ·max(Q(s',a')) - Q(s,a)]
+ * - More aggressive than SARSA, finds optimal policy faster
+ */
+import { AbstractRLLearner, RLConfig } from './AbstractRLLearner';
+import { TaskExperience, AgentAction } from '../types';
+/**
+ * Q-learning configuration (extends base RL config)
+ */
+export interface QLearningConfig extends RLConfig {
+}
+/**
+ * QLearning - Standard Q-learning implementation
+ *
+ * Implements the classic Q-learning algorithm with:
+ * - Epsilon-greedy exploration policy
+ * - Off-policy temporal difference (TD) learning
+ * - Q-table for state-action values
+ * - Optional experience replay for stability
+ *
+ * Update Rule:
+ * Q(s,a) ← Q(s,a) + α[r + γ·max_a'(Q(s',a')) - Q(s,a)]
+ *
+ * Key characteristics:
+ * - Off-policy: learns about optimal policy while following exploration policy
+ * - Uses max Q-value (greedy) for bootstrapping
+ * - Converges to optimal Q* under certain conditions
+ * - More sample-efficient than on-policy methods
+ */
+export declare class QLearning extends AbstractRLLearner {
+    private readonly defaultConfig;
+    constructor(config?: Partial<QLearningConfig>);
+    /**
+     * Update Q-value using Q-learning update rule
+     * Q(s,a) ← Q(s,a) + α[r + γ·max(Q(s',a')) - Q(s,a)]
+     *
+     * @param experience The transition experience (s, a, r, s')
+     * @param nextAction Ignored in Q-learning (uses max Q-value instead)
+     */
+    update(experience: TaskExperience, nextAction?: AgentAction): void;
+    /**
+     * Get the default exploration rate for this algorithm
+     */
+    protected getDefaultExplorationRate(): number;
+    /**
+     * Get algorithm name
+     */
+    getAlgorithmName(): string;
+    /**
+     * Get algorithm type (off-policy)
+     */
+    getAlgorithmType(): 'on-policy' | 'off-policy';
+    /**
+     * Get detailed statistics including Q-learning-specific metrics
+     */
+    getDetailedStatistics(): {
+        algorithm: string;
+        type: 'on-policy' | 'off-policy';
+        stats: ReturnType<AbstractRLLearner['getStatistics']>;
+    };
+}
+//# sourceMappingURL=QLearning.d.ts.map

package/dist/learning/algorithms/QLearning.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"QLearning.d.ts","sourceRoot":"","sources":["../../../src/learning/algorithms/QLearning.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EAAE,iBAAiB,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAClE,OAAO,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AAEvD;;GAEG;AACH,MAAM,WAAW,eAAgB,SAAQ,QAAQ;CAGhD;AAgBD;;;;;;;;;;;;;;;;;GAiBG;AACH,qBAAa,SAAU,SAAQ,iBAAiB;IAC9C,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAkB;gBAEpC,MAAM,GAAE,OAAO,CAAC,eAAe,CAAM;IAOjD;;;;;;OAMG;IACH,MAAM,CAAC,UAAU,EAAE,cAAc,EAAE,UAAU,CAAC,EAAE,WAAW,GAAG,IAAI;IAiClE;;OAEG;IACH,SAAS,CAAC,yBAAyB,IAAI,MAAM;IAI7C;;OAEG;IACH,gBAAgB,IAAI,MAAM;IAI1B;;OAEG;IACH,gBAAgB,IAAI,WAAW,GAAG,YAAY;IAI9C;;OAEG;IACH,qBAAqB,IAAI;QACvB,SAAS,EAAE,MAAM,CAAC;QAClB,IAAI,EAAE,WAAW,GAAG,YAAY,CAAC;QACjC,KAAK,EAAE,UAAU,CAAC,iBAAiB,CAAC,eAAe,CAAC,CAAC,CAAC;KACvD;CAOF"}