agentic-qe 2.1.2 → 2.2.1

package/dist/learning/algorithms/ActorCriticLearner.js

@@ -0,0 +1,447 @@
+"use strict";
+/**
+ * ActorCriticLearner - Actor-Critic Reinforcement Learning Algorithm
+ *
+ * Implements Advantage Actor-Critic (A2C) combining:
+ * - Actor: Policy network that selects actions using softmax policy
+ * - Critic: Value network that estimates state values for advantage calculation
+ *
+ * Key features:
+ * - Continuous action probabilities via softmax
+ * - Advantage-based updates to reduce variance
+ * - Entropy bonus for exploration
+ * - Policy gradient with baseline
+ *
+ * Update rules:
+ * - Critic (Value): V(s) += α_c * δ where δ = r + γV(s') - V(s)
+ * - Actor (Policy): π(a|s) += α_a * δ * ∇log(π(a|s)) + β * H(π)
+ *
+ * @module learning/algorithms/ActorCriticLearner
+ * @version 1.0.0
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ActorCriticLearner = void 0;
+exports.createDefaultActorCriticConfig = createDefaultActorCriticConfig;
+const AbstractRLLearner_1 = require("./AbstractRLLearner");
+/**
+ * ActorCriticLearner - Advantage Actor-Critic implementation
+ *
+ * Combines policy gradient (actor) with value function approximation (critic)
+ * for more stable and efficient learning than pure Q-learning.
+ *
+ * Usage:
+ * ```typescript
+ * const ac = new ActorCriticLearner({
+ *   learningRate: 0.1,
+ *   actorLearningRate: 0.01,
+ *   criticLearningRate: 0.1,
+ *   discountFactor: 0.95,
+ *   explorationRate: 0.3,
+ *   explorationDecay: 0.995,
+ *   minExplorationRate: 0.01,
+ *   entropyCoefficient: 0.01,
+ *   temperature: 1.0,
+ *   normalizeAdvantage: true,
+ *   targetUpdateFrequency: 100,
+ *   useExperienceReplay: true,
+ *   replayBufferSize: 10000,
+ *   batchSize: 32
+ * });
+ *
+ * const action = ac.selectAction(state, availableActions);
+ * ac.update(experience);
+ * ```
+ */
+class ActorCriticLearner extends AbstractRLLearner_1.AbstractRLLearner {
+    constructor(config) {
+        super(config);
+        this.actorConfig = config;
+        this.policyTable = new Map();
+        this.valueTable = new Map();
+        this.targetValueTable = new Map();
+        this.updatesSinceTargetSync = 0;
+        this.advantageHistory = [];
+        this.defaultExploration = config.explorationRate;
+        this.logger.info('ActorCriticLearner initialized', {
+            actorLR: config.actorLearningRate,
+            criticLR: config.criticLearningRate,
+            entropy: config.entropyCoefficient,
+            temperature: config.temperature
+        });
+    }
+    /**
+     * Select action using softmax policy with exploration
+     * π(a|s) = exp(Q(s,a)/τ) / Σ_a' exp(Q(s,a')/τ)
+     */
+    selectAction(state, availableActions) {
+        if (availableActions.length === 0) {
+            throw new Error('No available actions to select from');
+        }
+        // With probability ε, use random action (exploration fallback)
+        if (Math.random() < this.config.explorationRate) {
+            const randomIndex = Math.floor(Math.random() * availableActions.length);
+            return availableActions[randomIndex];
+        }
+        // Use softmax policy
+        return this.sampleFromPolicy(state, availableActions);
+    }
+    /**
+     * Sample action from softmax policy distribution
+     */
+    sampleFromPolicy(state, availableActions) {
+        const stateKey = this.encodeState(state);
+        const probabilities = this.getActionProbabilities(stateKey, availableActions);
+        // Sample from categorical distribution
+        const random = Math.random();
+        let cumulative = 0;
+        for (let i = 0; i < availableActions.length; i++) {
+            cumulative += probabilities[i];
+            if (random <= cumulative) {
+                return availableActions[i];
+            }
+        }
+        // Fallback (shouldn't reach here due to normalization)
+        return availableActions[availableActions.length - 1];
+    }
+    /**
+     * Get softmax action probabilities
+     * π(a|s) = exp(preference(s,a)/τ) / Σ_a' exp(preference(s,a')/τ)
+     */
+    getActionProbabilities(stateKey, availableActions) {
+        const temperature = this.actorConfig.temperature;
+        const preferences = [];
+        // Get preferences (Q-values or policy table values)
+        for (const action of availableActions) {
+            const actionKey = this.encodeAction(action);
+            const preference = this.getPreference(stateKey, actionKey);
+            preferences.push(preference / temperature);
+        }
+        // Softmax with numerical stability
+        const maxPref = Math.max(...preferences);
+        const expPrefs = preferences.map(p => Math.exp(p - maxPref));
+        const sumExp = expPrefs.reduce((sum, e) => sum + e, 0);
+        return expPrefs.map(e => e / sumExp);
+    }
+    /**
+     * Get preference for state-action pair from policy table
+     */
+    getPreference(stateKey, actionKey) {
+        const statePolicy = this.policyTable.get(stateKey);
+        if (!statePolicy) {
+            return 0; // uniform preference initially
+        }
+        const entry = statePolicy.get(actionKey);
+        return entry ? entry.probability : 0;
+    }
+    /**
+     * Update actor and critic using temporal difference
+     *
+     * TD Error (advantage): δ = r + γV(s') - V(s)
+     * Critic update: V(s) += α_c * δ
+     * Actor update: preference(s,a) += α_a * δ * (1 - π(a|s))
+     */
+    update(experience, nextAction) {
+        this.stepCount++;
+        const { state, action, reward, nextState, done } = this.extractExperience(experience);
+        const stateKey = this.encodeState(state);
+        const actionKey = this.encodeAction(action);
+        // Get current and next state values from critic
+        const currentV = this.getStateValue(state);
+        const nextV = done ? 0 : this.getTargetStateValue(nextState);
+        // Calculate TD error (advantage)
+        let advantage = reward + this.config.discountFactor * nextV - currentV;
+        // Normalize advantage if enabled
+        if (this.actorConfig.normalizeAdvantage) {
+            advantage = this.normalizeAdvantage(advantage);
+        }
+        // Update critic (value function)
+        this.updateCritic(stateKey, currentV, advantage);
+        // Update actor (policy)
+        this.updateActor(stateKey, actionKey, advantage);
+        // Store in replay buffer if enabled
+        if (this.replayBuffer) {
+            this.replayBuffer.add(experience);
+        }
+        // Sync target network periodically
+        this.updatesSinceTargetSync++;
+        if (this.updatesSinceTargetSync >= this.actorConfig.targetUpdateFrequency) {
+            this.syncTargetNetwork();
+            this.updatesSinceTargetSync = 0;
+        }
+        this.logger.debug('Actor-Critic update', {
+            state: stateKey,
+            action: actionKey,
+            reward,
+            advantage,
+            valueUpdate: currentV + this.actorConfig.criticLearningRate * advantage
+        });
+    }
+    /**
+     * Update critic (value function)
+     * V(s) += α_c * δ
+     */
+    updateCritic(stateKey, currentV, advantage) {
+        const newValue = currentV + this.actorConfig.criticLearningRate * advantage;
+        const existingEntry = this.valueTable.get(stateKey);
+        this.valueTable.set(stateKey, {
+            state: stateKey,
+            value: newValue,
+            updateCount: (existingEntry?.updateCount ?? 0) + 1,
+            lastUpdated: Date.now()
+        });
+    }
+    /**
+     * Update actor (policy)
+     * For softmax policy: preference(s,a) += α_a * δ * (1 - π(a|s))
+     * This increases preference for actions with positive advantage
+     */
+    updateActor(stateKey, actionKey, advantage) {
+        if (!this.policyTable.has(stateKey)) {
+            this.policyTable.set(stateKey, new Map());
+        }
+        const statePolicy = this.policyTable.get(stateKey);
+        // Get current preference and probability
+        const currentEntry = statePolicy.get(actionKey);
+        const currentPref = currentEntry?.probability ?? 0;
+        // Approximate gradient: increase preference proportional to advantage
+        // Also add entropy bonus to encourage exploration
+        const entropyBonus = this.calculateEntropyBonus(stateKey);
+        const newPref = currentPref + this.actorConfig.actorLearningRate * (advantage + entropyBonus);
+        statePolicy.set(actionKey, {
+            action: actionKey,
+            probability: newPref,
+            logProbability: Math.log(Math.max(0.001, this.softmaxProb(stateKey, actionKey))),
+            updateCount: (currentEntry?.updateCount ?? 0) + 1,
+            lastUpdated: Date.now()
+        });
+        // Also update Q-table for getBestAction compatibility
+        this.setQValue(stateKey, actionKey, newPref);
+    }
+    /**
+     * Calculate entropy bonus for a state
+     * H(π(·|s)) = -Σ_a π(a|s) log(π(a|s))
+     */
+    calculateEntropyBonus(stateKey) {
+        const statePolicy = this.policyTable.get(stateKey);
+        if (!statePolicy || statePolicy.size === 0) {
+            return 0;
+        }
+        // Calculate entropy over stored actions
+        const prefs = Array.from(statePolicy.values()).map(e => e.probability);
+        const maxPref = Math.max(...prefs);
+        const expPrefs = prefs.map(p => Math.exp((p - maxPref) / this.actorConfig.temperature));
+        const sumExp = expPrefs.reduce((sum, e) => sum + e, 0);
+        const probs = expPrefs.map(e => e / sumExp);
+        let entropy = 0;
+        for (const p of probs) {
+            if (p > 0) {
+                entropy -= p * Math.log(p);
+            }
+        }
+        return this.actorConfig.entropyCoefficient * entropy;
+    }
+    /**
+     * Get softmax probability for a specific action
+     */
+    softmaxProb(stateKey, actionKey) {
+        const statePolicy = this.policyTable.get(stateKey);
+        if (!statePolicy || statePolicy.size === 0) {
+            return 1.0 / Math.max(1, statePolicy?.size ?? 1);
+        }
+        const prefs = Array.from(statePolicy.entries());
+        const temp = this.actorConfig.temperature;
+        const maxPref = Math.max(...prefs.map(([, e]) => e.probability));
+        let sumExp = 0;
+        let targetExp = 0;
+        for (const [key, entry] of prefs) {
+            const exp = Math.exp((entry.probability - maxPref) / temp);
+            sumExp += exp;
+            if (key === actionKey) {
+                targetExp = exp;
+            }
+        }
+        return targetExp / sumExp;
+    }
+    /**
+     * Normalize advantage using running statistics
+     */
+    normalizeAdvantage(advantage) {
+        this.advantageHistory.push(advantage);
+        // Keep limited history
+        if (this.advantageHistory.length > 1000) {
+            this.advantageHistory.shift();
+        }
+        if (this.advantageHistory.length < 10) {
+            return advantage;
+        }
+        const mean = this.advantageHistory.reduce((s, a) => s + a, 0) / this.advantageHistory.length;
+        const variance = this.advantageHistory.reduce((s, a) => s + (a - mean) ** 2, 0) / this.advantageHistory.length;
+        const std = Math.sqrt(variance) + 1e-8;
+        return (advantage - mean) / std;
+    }
+    /**
+     * Get state value from value table
+     */
+    getStateValue(state) {
+        const stateKey = this.encodeState(state);
+        const entry = this.valueTable.get(stateKey);
+        return entry?.value ?? 0;
+    }
+    /**
+     * Get state value from target network (for stability)
+     */
+    getTargetStateValue(state) {
+        const stateKey = this.encodeState(state);
+        const entry = this.targetValueTable.get(stateKey);
+        return entry?.value ?? this.getStateValue(state);
+    }
+    /**
+     * Sync target network with main network
+     */
+    syncTargetNetwork() {
+        this.targetValueTable.clear();
+        for (const [key, value] of this.valueTable.entries()) {
+            this.targetValueTable.set(key, { ...value });
+        }
+        this.logger.debug('Target network synchronized');
+    }
+    /**
+     * Extract experience components
+     */
+    extractExperience(experience) {
+        return {
+            state: experience.state,
+            action: experience.action,
+            reward: experience.reward,
+            nextState: experience.nextState,
+            done: experience.done ?? false
+        };
+    }
+    /**
+     * Get default exploration rate for reset
+     */
+    getDefaultExplorationRate() {
+        return this.defaultExploration;
+    }
+    /**
+     * Get actor-critic specific statistics
+     */
+    getActorCriticStatistics() {
+        // Calculate average state value
+        let totalValue = 0;
+        for (const entry of this.valueTable.values()) {
+            totalValue += entry.value;
+        }
+        const avgStateValue = this.valueTable.size > 0 ? totalValue / this.valueTable.size : 0;
+        // Calculate policy table size
+        let policySize = 0;
+        for (const statePolicy of this.policyTable.values()) {
+            policySize += statePolicy.size;
+        }
+        // Calculate average entropy
+        let totalEntropy = 0;
+        let entropyCount = 0;
+        for (const stateKey of this.policyTable.keys()) {
+            const entropy = this.calculateEntropyBonus(stateKey) / this.actorConfig.entropyCoefficient;
+            totalEntropy += entropy;
+            entropyCount++;
+        }
+        const avgEntropy = entropyCount > 0 ? totalEntropy / entropyCount : 0;
+        // Calculate advantage statistics
+        const advMean = this.advantageHistory.length > 0
+            ? this.advantageHistory.reduce((s, a) => s + a, 0) / this.advantageHistory.length
+            : 0;
+        const advVariance = this.advantageHistory.length > 0
+            ? this.advantageHistory.reduce((s, a) => s + (a - advMean) ** 2, 0) / this.advantageHistory.length
+            : 0;
+        return {
+            valueTableSize: this.valueTable.size,
+            policyTableSize: policySize,
+            avgStateValue,
+            avgEntropy,
+            advantageMean: advMean,
+            advantageStd: Math.sqrt(advVariance)
+        };
+    }
+    /**
+     * Reset actor-critic specific state
+     */
+    reset() {
+        super.reset();
+        this.policyTable.clear();
+        this.valueTable.clear();
+        this.targetValueTable.clear();
+        this.advantageHistory = [];
+        this.updatesSinceTargetSync = 0;
+        this.logger.info('ActorCriticLearner reset');
+    }
+    /**
+     * Export complete actor-critic state
+     */
+    exportActorCritic() {
+        const serializedPolicy = {};
+        for (const [state, actions] of this.policyTable.entries()) {
+            serializedPolicy[state] = {};
+            for (const [action, entry] of actions.entries()) {
+                serializedPolicy[state][action] = entry;
+            }
+        }
+        const serializedValue = {};
+        for (const [state, entry] of this.valueTable.entries()) {
+            serializedValue[state] = entry;
+        }
+        return {
+            base: this.export(),
+            valueTable: serializedValue,
+            policyTable: serializedPolicy,
+            actorConfig: { ...this.actorConfig }
+        };
+    }
+    /**
+     * Import complete actor-critic state
+     */
+    importActorCritic(state) {
+        this.import(state.base);
+        this.valueTable.clear();
+        for (const [stateKey, entry] of Object.entries(state.valueTable)) {
+            this.valueTable.set(stateKey, entry);
+        }
+        this.policyTable.clear();
+        for (const [stateKey, actions] of Object.entries(state.policyTable)) {
+            const actionMap = new Map();
+            for (const [actionKey, entry] of Object.entries(actions)) {
+                actionMap.set(actionKey, entry);
+            }
+            this.policyTable.set(stateKey, actionMap);
+        }
+        this.actorConfig = { ...state.actorConfig };
+        this.syncTargetNetwork();
+        this.logger.info('Imported Actor-Critic state', {
+            valueTableSize: this.valueTable.size,
+            policyTableSize: this.policyTable.size
+        });
+    }
+}
+exports.ActorCriticLearner = ActorCriticLearner;
+/**
+ * Create default Actor-Critic configuration
+ */
+function createDefaultActorCriticConfig() {
+    return {
+        learningRate: 0.1,
+        actorLearningRate: 0.01,
+        criticLearningRate: 0.1,
+        discountFactor: 0.95,
+        explorationRate: 0.3,
+        explorationDecay: 0.995,
+        minExplorationRate: 0.01,
+        entropyCoefficient: 0.01,
+        temperature: 1.0,
+        normalizeAdvantage: true,
+        targetUpdateFrequency: 100,
+        useExperienceReplay: true,
+        replayBufferSize: 10000,
+        batchSize: 32
+    };
+}
+//# sourceMappingURL=ActorCriticLearner.js.map

package/dist/learning/algorithms/ActorCriticLearner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ActorCriticLearner.js","sourceRoot":"","sources":["../../../src/learning/algorithms/ActorCriticLearner.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;;;;;GAmBG;;;AAyhBH,wEAiBC;AAxiBD,2DAA0E;AA0C1E;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,MAAa,kBAAmB,SAAQ,qCAAiB;IASvD,YAAY,MAAyB;QACnC,KAAK,CAAC,MAAM,CAAC,CAAC;QACd,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC;QAC1B,IAAI,CAAC,WAAW,GAAG,IAAI,GAAG,EAAE,CAAC;QAC7B,IAAI,CAAC,UAAU,GAAG,IAAI,GAAG,EAAE,CAAC;QAC5B,IAAI,CAAC,gBAAgB,GAAG,IAAI,GAAG,EAAE,CAAC;QAClC,IAAI,CAAC,sBAAsB,GAAG,CAAC,CAAC;QAChC,IAAI,CAAC,gBAAgB,GAAG,EAAE,CAAC;QAC3B,IAAI,CAAC,kBAAkB,GAAG,MAAM,CAAC,eAAe,CAAC;QAEjD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,gCAAgC,EAAE;YACjD,OAAO,EAAE,MAAM,CAAC,iBAAiB;YACjC,QAAQ,EAAE,MAAM,CAAC,kBAAkB;YACnC,OAAO,EAAE,MAAM,CAAC,kBAAkB;YAClC,WAAW,EAAE,MAAM,CAAC,WAAW;SAChC,CAAC,CAAC;IACL,CAAC;IAED;;;OAGG;IACM,YAAY,CAAC,KAAgB,EAAE,gBAA+B;QACrE,IAAI,gBAAgB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAClC,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;QACzD,CAAC;QAED,+DAA+D;QAC/D,IAAI,IAAI,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,eAAe,EAAE,CAAC;YAChD,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC;YACxE,OAAO,gBAAgB,CAAC,WAAW,CAAC,CAAC;QACvC,CAAC;QAED,qBAAqB;QACrB,OAAO,IAAI,CAAC,gBAAgB,CAAC,KAAK,EAAE,gBAAgB,CAAC,CAAC;IACxD,CAAC;IAED;;OAEG;IACK,gBAAgB,CAAC,KAAgB,EAAE,gBAA+B;QACxE,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QACzC,MAAM,aAAa,GAAG,IAAI,CAAC,sBAAsB,CAAC,QAAQ,EAAE,gBAAgB,CAAC,CAAC;QAE9E,uCAAuC;QACvC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;QAC7B,IAAI,UAAU,GAAG,CAAC,CAAC;QAEnB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,gBAAgB,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACjD,UAAU,IAAI,aAAa,CAAC,CAAC,CAAC,CAAC;YAC/B,IAAI,MAAM,IAAI,UAAU,EAAE,CAAC;gBACzB,OAAO,gBAAgB,CAAC,CAAC,CAAC,CAAC;YAC7B,CAAC;QACH,CAAC;QAED,uDAAuD;QACvD,OAAO,gBAAgB,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACvD,CAAC;IAED;;;OAGG;IACK,sBAAsB,CAAC,QAAgB,EAAE,gBAA+B;QAC9E,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,WAAW,CAAC;QACjD,MAAM,WAAW,GAAa,EAAE,CAAC;QAEjC,oDAAoD;QACpD,KAAK,MAAM,MAAM,IAAI,gBAAgB,EAAE,CAAC;YACtC,MAAM,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;YAC5C,MAAM,UAAU,GAAG,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;YAC3D,WAAW,CAAC,IAAI,CAAC,UAAU,GAAG,WAAW,CAAC,CAAC;QAC7C,CAAC;QAED,mCAAmC;QACnC,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,WAAW,CAAC,CAAC;QACzC,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;QAC7D,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;QAEvD,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC;IACvC,CAAC;IAED;;OAEG;IACK,aAAa,CAAC,QAAgB,EAAE,SAAiB;QACvD,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACnD,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,OAAO,CAAC,CAAC,CAAC,+BAA+B;QAC3C,CAAC;QAED,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QACzC,OAAO,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC;IACvC,CAAC;IAED;;;;;;OAMG;IACM,MAAM,CAAC,UAA0B,EAAE,UAAwB;QAClE,IAAI,CAAC,SAAS,EAAE,CAAC;QAEjB,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC,iBAAiB,CAAC,UAAU,CAAC,CAAC;QACtF,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QACzC,MAAM,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;QAE5C,gDAAgD;QAChD,MAAM,QAAQ,GAAG,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;QAC3C,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,mBAAmB,CAAC,SAAS,CAAC,CAAC;QAE7D,iCAAiC;QACjC,IAAI,SAAS,GAAG,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,cAAc,GAAG,KAAK,GAAG,QAAQ,CAAC;QAEvE,iCAAiC;QACjC,IAAI,IAAI,CAAC,WAAW,CAAC,kBAAkB,EAAE,CAAC;YACxC,SAAS,GAAG,IAAI,CAAC,kBAAkB,CAAC,SAAS,CAAC,CAAC;QACjD,CAAC;QAED,iCAAiC;QACjC,IAAI,CAAC,YAAY,CAAC,QAAQ,EAAE,QAAQ,EAAE,SAAS,CAAC,CAAC;QAEjD,wBAAwB;QACxB,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;QAEjD,oCAAoC;QACpC,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACtB,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QACpC,CAAC;QAED,mCAAmC;QACnC,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAC9B,IAAI,IAAI,CAAC,sBAAsB,IAAI,IAAI,CAAC,WAAW,CAAC,qBAAqB,EAAE,CAAC;YAC1E,IAAI,CAAC,iBAAiB,EAAE,CAAC;YACzB,IAAI,CAAC,sBAAsB,GAAG,CAAC,CAAC;QAClC,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,qBAAqB,EAAE;YACvC,KAAK,EAAE,QAAQ;YACf,MAAM,EAAE,SAAS;YACjB,MAAM;YACN,SAAS;YACT,WAAW,EAAE,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,kBAAkB,GAAG,SAAS;SACxE,CAAC,CAAC;IACL,CAAC;IAED;;;OAGG;IACK,YAAY,CAAC,QAAgB,EAAE,QAAgB,EAAE,SAAiB;QACxE,MAAM,QAAQ,GAAG,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,kBAAkB,GAAG,SAAS,CAAC;QAE5E,MAAM,aAAa,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACpD,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,EAAE;YAC5B,KAAK,EAAE,QAAQ;YACf,KAAK,EAAE,QAAQ;YACf,WAAW,EAAE,CAAC,aAAa,EAAE,WAAW,IAAI,CAAC,CAAC,GAAG,CAAC;YAClD,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE;SACxB,CAAC,CAAC;IACL,CAAC;IAED;;;;OAIG;IACK,WAAW,CAAC,QAAgB,EAAE,SAAiB,EAAE,SAAiB;QACxE,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YACpC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,QAAQ,EAAE,IAAI,GAAG,EAAE,CAAC,CAAC;QAC5C,CAAC;QACD,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAE,CAAC;QAEpD,yCAAyC;QACzC,MAAM,YAAY,GAAG,WAAW,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAChD,MAAM,WAAW,GAAG,YAAY,EAAE,WAAW,IAAI,CAAC,CAAC;QAEnD,sEAAsE;QACtE,kDAAkD;QAClD,MAAM,YAAY,GAAG,IAAI,CAAC,qBAAqB,CAAC,QAAQ,CAAC,CAAC;QAC1D,MAAM,OAAO,GAAG,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,iBAAiB,GAAG,CAAC,SAAS,GAAG,YAAY,CAAC,CAAC;QAE9F,WAAW,CAAC,GAAG,CAAC,SAAS,EAAE;YACzB,MAAM,EAAE,SAAS;YACjB,WAAW,EAAE,OAAO;YACpB,cAAc,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC,CAAC;YAChF,WAAW,EAAE,CAAC,YAAY,EAAE,WAAW,IAAI,CAAC,CAAC,GAAG,CAAC;YACjD,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE;SACxB,CAAC,CAAC;QAEH,sDAAsD;QACtD,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;IAC/C,CAAC;IAED;;;OAGG;IACK,qBAAqB,CAAC,QAAgB;QAC5C,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACnD,IAAI,CAAC,WAAW,IAAI,WAAW,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;YAC3C,OAAO,CAAC,CAAC;QACX,CAAC;QAED,wCAAwC;QACxC,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC;QACvE,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC;QACnC,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC,CAAC;QACxF,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;QACvD,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC;QAE5C,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;YACtB,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;gBACV,OAAO,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YAC7B,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC,WAAW,CAAC,kBAAkB,GAAG,OAAO,CAAC;IACvD,CAAC;IAED;;OAEG;IACK,WAAW,CAAC,QAAgB,EAAE,SAAiB;QACrD,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACnD,IAAI,CAAC,WAAW,IAAI,WAAW,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;YAC3C,OAAO,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,EAAE,IAAI,IAAI,CAAC,CAAC,CAAC;QACnD,CAAC;QAED,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC,CAAC;QAChD,MAAM,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,WAAW,CAAC;QAE1C,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC;QACjE,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,IAAI,SAAS,GAAG,CAAC,CAAC;QAElB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,KAAK,EAAE,CAAC;YACjC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,WAAW,GAAG,OAAO,CAAC,GAAG,IAAI,CAAC,CAAC;YAC3D,MAAM,IAAI,GAAG,CAAC;YACd,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;gBACtB,SAAS,GAAG,GAAG,CAAC;YAClB,CAAC;QACH,CAAC;QAED,OAAO,SAAS,GAAG,MAAM,CAAC;IAC5B,CAAC;IAED;;OAEG;IACK,kBAAkB,CAAC,SAAiB;QAC1C,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAEtC,uBAAuB;QACvB,IAAI,IAAI,CAAC,gBAAgB,CAAC,MAAM,GAAG,IAAI,EAAE,CAAC;YACxC,IAAI,CAAC,gBAAgB,CAAC,KAAK,EAAE,CAAC;QAChC,CAAC;QAED,IAAI,IAAI,CAAC,gBAAgB,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;YACtC,OAAO,SAAS,CAAC;QACnB,CAAC;QAED,MAAM,IAAI,GAAG,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC;QAC7F,MAAM,QAAQ,GAAG,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC;QAC/G,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,IAAI,CAAC;QAEvC,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,GAAG,CAAC;IAClC,CAAC;IAED;;OAEG;IACM,aAAa,CAAC,KAAgB;QACrC,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QACzC,MAAM,KAAK,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC5C,OAAO,KAAK,EAAE,KAAK,IAAI,CAAC,CAAC;IAC3B,CAAC;IAED;;OAEG;IACK,mBAAmB,CAAC,KAAgB;QAC1C,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QACzC,MAAM,KAAK,GAAG,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAClD,OAAO,KAAK,EAAE,KAAK,IAAI,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;IACnD,CAAC;IAED;;OAEG;IACK,iBAAiB;QACvB,IAAI,CAAC,gBAAgB,CAAC,KAAK,EAAE,CAAC;QAC9B,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC;YACrD,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,GAAG,KAAK,EAAE,CAAC,CAAC;QAC/C,CAAC;QACD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAC;IACnD,CAAC;IAED;;OAEG;IACK,iBAAiB,CAAC,UAA0B;QAOlD,OAAO;YACL,KAAK,EAAE,UAAU,CAAC,KAAK;YACvB,MAAM,EAAE,UAAU,CAAC,MAAM;YACzB,MAAM,EAAE,UAAU,CAAC,MAAM;YACzB,SAAS,EAAE,UAAU,CAAC,SAAS;YAC/B,IAAI,EAAE,UAAU,CAAC,IAAI,IAAI,KAAK;SAC/B,CAAC;IACJ,CAAC;IAED;;OAEG;IACO,yBAAyB;QACjC,OAAO,IAAI,CAAC,kBAAkB,CAAC;IACjC,CAAC;IAED;;OAEG;IACH,wBAAwB;QAQtB,gCAAgC;QAChC,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,EAAE,CAAC;YAC7C,UAAU,IAAI,KAAK,CAAC,KAAK,CAAC;QAC5B,CAAC;QACD,MAAM,aAAa,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QAEvF,8BAA8B;QAC9B,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,KAAK,MAAM,WAAW,IAAI,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,EAAE,CAAC;YACpD,UAAU,IAAI,WAAW,CAAC,IAAI,CAAC;QACjC,CAAC;QAED,4BAA4B;QAC5B,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,EAAE,CAAC;YAC/C,MAAM,OAAO,GAAG,IAAI,CAAC,qBAAqB,CAAC,QAAQ,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC,kBAAkB,CAAC;YAC3F,YAAY,IAAI,OAAO,CAAC;YACxB,YAAY,EAAE,CAAC;QACjB,CAAC;QACD,MAAM,UAAU,GAAG,YAAY,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC;QAEtE,iCAAiC;QACjC,MAAM,OAAO,GAAG,IAAI,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC;YAC9C,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,gBAAgB,CAAC,MAAM;YACjF,CAAC,CAAC,CAAC,CAAC;QACN,MAAM,WAAW,GAAG,IAAI,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC;YAClD,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,gBAAgB,CAAC,MAAM;YAClG,CAAC,CAAC,CAAC,CAAC;QAEN,OAAO;YACL,cAAc,EAAE,IAAI,CAAC,UAAU,CAAC,IAAI;YACpC,eAAe,EAAE,UAAU;YAC3B,aAAa;YACb,UAAU;YACV,aAAa,EAAE,OAAO;YACtB,YAAY,EAAE,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC;SACrC,CAAC;IACJ,CAAC;IAED;;OAEG;IACM,KAAK;QACZ,KAAK,CAAC,KAAK,EAAE,CAAC;QACd,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;QACxB,IAAI,CAAC,gBAAgB,CAAC,KAAK,EAAE,CAAC;QAC9B,IAAI,CAAC,gBAAgB,GAAG,EAAE,CAAC;QAC3B,IAAI,CAAC,sBAAsB,GAAG,CAAC,CAAC;QAChC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;IAC/C,CAAC;IAED;;OAEG;IACH,iBAAiB;QAMf,MAAM,gBAAgB,GAAgD,EAAE,CAAC;QACzE,KAAK,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,EAAE,CAAC;YAC1D,gBAAgB,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;YAC7B,KAAK,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;gBAChD,gBAAgB,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC;YAC1C,CAAC;QACH,CAAC;QAED,MAAM,eAAe,GAAoC,EAAE,CAAC;QAC5D,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC;YACvD,eAAe,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC;QACjC,CAAC;QAED,OAAO;YACL,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE;YACnB,UAAU,EAAE,eAAe;YAC3B,WAAW,EAAE,gBAAgB;YAC7B,WAAW,EAAE,EAAE,GAAG,IAAI,CAAC,WAAW,EAAE;SACrC,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,iBAAiB,CAAC,KAAgD;QAChE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAExB,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;QACxB,KAAK,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,EAAE,CAAC;YACjE,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QACvC,CAAC;QAED,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;QACzB,KAAK,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,EAAE,CAAC;YACpE,MAAM,SAAS,GAAG,IAAI,GAAG,EAAuB,CAAC;YACjD,KAAK,MAAM,CAAC,SAAS,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;gBACzD,SAAS,CAAC,GAAG,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;YAClC,CAAC;YACD,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;QAC5C,CAAC;QAED,IAAI,CAAC,WAAW,GAAG,EAAE,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC;QAC5C,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAEzB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,6BAA6B,EAAE;YAC9C,cAAc,EAAE,IAAI,CAAC,UAAU,CAAC,IAAI;YACpC,eAAe,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI;SACvC,CAAC,CAAC;IACL,CAAC;CACF;AA3cD,gDA2cC;AAED;;GAEG;AACH,SAAgB,8BAA8B;IAC5C,OAAO;QACL,YAAY,EAAE,GAAG;QACjB,iBAAiB,EAAE,IAAI;QACvB,kBAAkB,EAAE,GAAG;QACvB,cAAc,EAAE,IAAI;QACpB,eAAe,EAAE,GAAG;QACpB,gBAAgB,EAAE,KAAK;QACvB,kBAAkB,EAAE,IAAI;QACxB,kBAAkB,EAAE,IAAI;QACxB,WAAW,EAAE,GAAG;QAChB,kBAAkB,EAAE,IAAI;QACxB,qBAAqB,EAAE,GAAG;QAC1B,mBAAmB,EAAE,IAAI;QACzB,gBAAgB,EAAE,KAAK;QACvB,SAAS,EAAE,EAAE;KACd,CAAC;AACJ,CAAC"}

package/dist/learning/algorithms/MAMLMetaLearner.d.ts

@@ -0,0 +1,218 @@
+/**
+ * MAMLMetaLearner - Model-Agnostic Meta-Learning for QE Agents
+ *
+ * Implements MAML-style meta-learning that enables agents to "learn how to learn".
+ * Agents can quickly adapt to new testing domains with just 5-10 examples.
+ *
+ * Key Concepts:
+ * - Inner Loop: Fast adaptation to new task with few gradient steps (5-10 examples)
+ * - Outer Loop: Learn initialization parameters that enable fast adaptation
+ * - Meta-Learning: After seeing few examples of new test pattern, agent performs well
+ *
+ * Algorithm:
+ * 1. Initialize meta-parameters θ (Q-table initialization)
+ * 2. For each task Ti in task distribution:
+ *    a. Sample K examples from Ti (support set)
+ *    b. Adapt: θ'i = θ - α∇Loss(θ, support) [inner loop]
+ *    c. Evaluate on query set from Ti
+ * 3. Update meta-parameters: θ = θ - β∇Loss(θ', query) [outer loop]
+ * 4. Result: θ is optimized for fast adaptation to new tasks
+ *
+ * Use Cases:
+ * - New testing framework adoption (5-10 examples → proficient)
+ * - New project domain (few examples → effective testing strategy)
+ * - API testing → UI testing transfer learning
+ */
+import { AbstractRLLearner, RLConfig, QValue } from './AbstractRLLearner';
+import { TaskExperience, AgentAction } from '../types';
+/**
+ * MAML configuration extends base RL config with meta-learning parameters
+ */
+export interface MAMLConfig extends RLConfig {
+    innerLearningRate: number;
+    innerSteps: number;
+    metaLearningRate: number;
+    minTaskExamples: number;
+    maxTaskExamples: number;
+    taskBatchSize: number;
+    firstOrderApproximation: boolean;
+    baseAlgorithm?: AbstractRLLearner;
+}
+/**
+ * Task for meta-learning (contains support and query sets)
+ */
+interface MetaTask {
+    id: string;
+    taskType: string;
+    supportSet: TaskExperience[];
+    querySet: TaskExperience[];
+    metadata?: Record<string, any>;
+}
+/**
+ * Meta-learning episode tracking
+ */
+interface MetaEpisode {
+    episodeId: string;
+    tasks: MetaTask[];
+    preAdaptationLoss: number;
+    postAdaptationLoss: number;
+    metaLoss: number;
+    improvement: number;
+    timestamp: Date;
+}
+/**
+ * MAMLMetaLearner - Model-Agnostic Meta-Learning for Fast Adaptation
+ *
+ * Learns an initialization of Q-values that enables rapid adaptation to new
+ * testing tasks with minimal examples (5-10 shots).
+ *
+ * Example:
+ * - Agent trained on Jest, Mocha, Jasmine unit testing
+ * - Sees 5-10 examples of Vitest tests
+ * - Immediately generates high-quality Vitest tests
+ *
+ * Meta-Learning Process:
+ * 1. Sample batch of tasks (different testing scenarios)
+ * 2. For each task:
+ *    - Adapt Q-table with support set (inner loop)
+ *    - Evaluate adapted Q-table on query set
+ * 3. Compute meta-gradient from all tasks
+ * 4. Update meta-parameters (Q-table initialization)
+ */
+export declare class MAMLMetaLearner extends AbstractRLLearner {
+    private readonly mamlConfig;
+    private metaQTable;
+    private taskBuffer;
+    private metaEpisodes;
+    private metaStepCount;
+    private baseAlgorithm?;
+    constructor(config?: Partial<MAMLConfig>);
+    /**
+     * Update Q-value using base algorithm (delegates to wrapped algorithm if available)
+     * For MAML, this is called during inner loop adaptation
+     */
+    update(experience: TaskExperience, nextAction?: AgentAction): void;
+    /**
+     * Q-Learning update rule (default inner loop algorithm)
+     */
+    private qLearningUpdate;
+    /**
+     * Buffer experience by task type for meta-learning
+     */
+    private bufferExperience;
+    /**
+     * Perform meta-learning update (outer loop)
+     * Learns Q-table initialization that enables fast adaptation
+     *
+     * This should be called periodically after collecting enough task examples
+     */
+    performMetaUpdate(): Promise<MetaEpisode | null>;
+    /**
+     * Sample batch of meta-tasks from task buffer
+     * Each task contains support set (for adaptation) and query set (for evaluation)
+     */
+    private sampleMetaTasks;
+    /**
+     * Adapt Q-table to a specific task using support set (inner loop)
+     */
+    private adaptToTask;
+    /**
+     * Evaluate loss (TD error) on a set of experiences
+     */
+    private evaluateLoss;
+    /**
+     * Compute first-order meta-gradients (FOMAML)
+     * Faster approximation that ignores second-order derivatives
+     */
+    private computeFirstOrderGradients;
+    /**
+     * Compute second-order meta-gradients (Full MAML)
+     * More accurate but computationally expensive
+     */
+    private computeSecondOrderGradients;
+    /**
+     * Update meta-parameters using accumulated gradients
+     */
+    private updateMetaParameters;
+    /**
+     * Fast adaptation to new task (few-shot learning)
+     * Given 5-10 examples, quickly adapt Q-table for new testing domain
+     *
+     * @param examples Few examples of new task (5-10)
+     * @returns Adapted Q-table
+     */
+    fastAdapt(examples: TaskExperience[]): Promise<Map<string, Map<string, QValue>>>;
+    /**
+     * Get Q-value from specific Q-table (helper)
+     */
+    private getQValueFromTable;
+    /**
+     * Set Q-value in specific Q-table (helper)
+     */
+    private setQValueInTable;
+    /**
+     * Get Q-value (raw, without creating entry)
+     */
+    private getQValueRaw;
+    /**
+     * Clone Q-table
+     */
+    private cloneQTable;
+    /**
+     * Copy Q-table from source to destination
+     */
+    private copyQTable;
+    /**
+     * Get meta-learning statistics
+     */
+    getMetaStatistics(): {
+        metaSteps: number;
+        metaEpisodes: number;
+        avgPreAdaptLoss: number;
+        avgPostAdaptLoss: number;
+        avgImprovement: number;
+        taskTypes: number;
+        bufferedExperiences: number;
+    };
+    /**
+     * Get meta-episodes history
+     */
+    getMetaEpisodes(): MetaEpisode[];
+    /**
+     * Clear task buffer
+     */
+    clearTaskBuffer(): void;
+    /**
+     * Get default exploration rate
+     */
+    protected getDefaultExplorationRate(): number;
+    /**
+     * Get algorithm name
+     */
+    getAlgorithmName(): string;
+    /**
+     * Override getStatistics to include meta-learning metrics
+     */
+    getStatistics(): ReturnType<AbstractRLLearner['getStatistics']> & {
+        maml: ReturnType<MAMLMetaLearner['getMetaStatistics']>;
+    };
+    /**
+     * Export meta-learner state
+     */
+    export(): ReturnType<AbstractRLLearner['export']> & {
+        metaQTable: Record<string, Record<string, QValue>>;
+        taskBuffer: Record<string, TaskExperience[]>;
+        metaEpisodes: MetaEpisode[];
+        metaStepCount: number;
+    };
+    /**
+     * Import meta-learner state
+     */
+    import(state: ReturnType<MAMLMetaLearner['export']>): void;
+}
+/**
+ * Create default MAML configuration
+ */
+export declare function createDefaultMAMLConfig(): MAMLConfig;
+export {};
+//# sourceMappingURL=MAMLMetaLearner.d.ts.map