agentic-qe 1.3.4 → 1.3.5

package/dist/learning/ExperienceReplayBuffer.d.ts

@@ -0,0 +1,143 @@
+/**
+ * ExperienceReplayBuffer - Phase 2 (Milestone 2.2)
+ *
+ * Implements experience replay buffer for reinforcement learning.
+ * Stores transitions and enables batch sampling for training.
+ */
+import { TaskExperience } from './types';
+/**
+ * Configuration for experience replay buffer
+ */
+export interface ReplayBufferConfig {
+    maxSize: number;
+    minSize: number;
+    prioritized: boolean;
+}
+/**
+ * Prioritized experience with importance weight
+ */
+interface PrioritizedExperience {
+    experience: TaskExperience;
+    priority: number;
+    timestamp: number;
+}
+/**
+ * ExperienceReplayBuffer - FIFO buffer with optional prioritization
+ *
+ * Implements experience replay for more stable and efficient learning.
+ * Supports both uniform random sampling and prioritized experience replay.
+ */
+export declare class ExperienceReplayBuffer {
+    private readonly logger;
+    private readonly config;
+    private buffer;
+    private totalExperiences;
+    constructor(config?: Partial<ReplayBufferConfig>);
+    /**
+     * Add a new experience to the buffer
+     * Uses FIFO eviction when buffer is full
+     */
+    add(experience: TaskExperience, priority?: number): void;
+    /**
+     * Add multiple experiences in batch
+     */
+    addBatch(experiences: TaskExperience[]): void;
+    /**
+     * Sample a random batch of experiences
+     * Uses uniform random sampling or prioritized sampling based on config
+     */
+    sample(batchSize: number): TaskExperience[];
+    /**
+     * Uniform random sampling (default)
+     */
+    private uniformSample;
+    /**
+     * Prioritized experience replay sampling
+     * Samples based on priority with probability proportional to priority
+     */
+    private prioritizedSample;
+    /**
+     * Update priority for a specific experience
+     * Used in prioritized experience replay to adjust importance weights
+     */
+    updatePriority(experienceId: string, newPriority: number): boolean;
+    /**
+     * Calculate default priority based on TD-error magnitude
+     * Higher absolute rewards get higher priority
+     */
+    private calculateDefaultPriority;
+    /**
+     * Check if buffer has enough experiences to sample
+     */
+    canSample(batchSize: number): boolean;
+    /**
+     * Get recent experiences (for temporal coherence)
+     */
+    getRecent(count: number): TaskExperience[];
+    /**
+     * Get oldest experiences
+     */
+    getOldest(count: number): TaskExperience[];
+    /**
+     * Get all experiences matching a filter
+     */
+    filter(predicate: (exp: TaskExperience) => boolean): TaskExperience[];
+    /**
+     * Clear the buffer
+     */
+    clear(): void;
+    /**
+     * Get current buffer size
+     */
+    size(): number;
+    /**
+     * Check if buffer is empty
+     */
+    isEmpty(): boolean;
+    /**
+     * Check if buffer is full
+     */
+    isFull(): boolean;
+    /**
+     * Get total number of experiences ever added (including evicted)
+     */
+    getTotalExperiences(): number;
+    /**
+     * Get buffer statistics
+     */
+    getStatistics(): {
+        size: number;
+        maxSize: number;
+        utilization: number;
+        totalAdded: number;
+        avgPriority: number;
+        avgReward: number;
+    };
+    /**
+     * Export buffer state for persistence
+     */
+    export(): {
+        buffer: PrioritizedExperience[];
+        config: ReplayBufferConfig;
+        totalExperiences: number;
+    };
+    /**
+     * Import buffer state from persistence
+     */
+    import(state: {
+        buffer: PrioritizedExperience[];
+        config: ReplayBufferConfig;
+        totalExperiences: number;
+    }): void;
+    /**
+     * Prune old experiences beyond retention limit
+     * Keeps most recent experiences
+     */
+    prune(retentionCount: number): number;
+    /**
+     * Get memory usage estimate in bytes
+     */
+    getMemoryUsage(): number;
+}
+export {};
+//# sourceMappingURL=ExperienceReplayBuffer.d.ts.map

package/dist/learning/ExperienceReplayBuffer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ExperienceReplayBuffer.d.ts","sourceRoot":"","sources":["../../src/learning/ExperienceReplayBuffer.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAGzC;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,OAAO,CAAC;CACtB;AAWD;;GAEG;AACH,UAAU,qBAAqB;IAC7B,UAAU,EAAE,cAAc,CAAC;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;;;;GAKG;AACH,qBAAa,sBAAsB;IACjC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAqB;IAC5C,OAAO,CAAC,MAAM,CAA0B;IACxC,OAAO,CAAC,gBAAgB,CAAS;gBAErB,MAAM,GAAE,OAAO,CAAC,kBAAkB,CAAM;IAOpD;;;OAGG;IACH,GAAG,CAAC,UAAU,EAAE,cAAc,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI;IAgBxD;;OAEG;IACH,QAAQ,CAAC,WAAW,EAAE,cAAc,EAAE,GAAG,IAAI;IAO7C;;;OAGG;IACH,MAAM,CAAC,SAAS,EAAE,MAAM,GAAG,cAAc,EAAE;IAe3C;;OAEG;IACH,OAAO,CAAC,aAAa;IAgBrB;;;OAGG;IACH,OAAO,CAAC,iBAAiB;IAqBzB;;;OAGG;IACH,cAAc,CAAC,YAAY,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,GAAG,OAAO;IAalE;;;OAGG;IACH,OAAO,CAAC,wBAAwB;IAUhC;;OAEG;IACH,SAAS,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO;IAIrC;;OAEG;IACH,SAAS,CAAC,KAAK,EAAE,MAAM,GAAG,cAAc,EAAE;IAO1C;;OAEG;IACH,SAAS,CAAC,KAAK,EAAE,MAAM,GAAG,cAAc,EAAE;IAO1C;;OAEG;IACH,MAAM,CAAC,SAAS,EAAE,CAAC,GAAG,EAAE,cAAc,KAAK,OAAO,GAAG,cAAc,EAAE;IAMrE;;OAEG;IACH,KAAK,IAAI,IAAI;IAKb;;OAEG;IACH,IAAI,IAAI,MAAM;IAId;;OAEG;IACH,OAAO,IAAI,OAAO;IAIlB;;OAEG;IACH,MAAM,IAAI,OAAO;IAIjB;;OAEG;IACH,mBAAmB,IAAI,MAAM;IAI7B;;OAEG;IACH,aAAa,IAAI;QACf,IAAI,EAAE,MAAM,CAAC;QACb,OAAO,EAAE,MAAM,CAAC;QAChB,WAAW,EAAE,MAAM,CAAC;QACpB,UAAU,EAAE,MAAM,CAAC;QACnB,WAAW,EAAE,MAAM,CAAC;QACpB,SAAS,EAAE,MAAM,CAAC;KACnB;IAmBD;;OAEG;IACH,MAAM,IAAI;QACR,MAAM,EAAE,qBAAqB,EAAE,CAAC;QAChC,MAAM,EAAE,kBAAkB,CAAC;QAC3B,gBAAgB,EAAE,MAAM,CAAC;KAC1B;IAQD;;OAEG;IACH,MAAM,CAAC,KAAK,EAAE;QACZ,MAAM,EAAE,qBAAqB,EAAE,CAAC;QAChC,MAAM,EAAE,kBAAkB,CAAC;QAC3B,gBAAgB,EAAE,MAAM,CAAC;KAC1B,GAAG,IAAI;IAMR;;;OAGG;IACH,KAAK,CAAC,cAAc,EAAE,MAAM,GAAG,MAAM;IAYrC;;OAEG;IACH,cAAc,IAAI,MAAM;CAGzB"}

package/dist/learning/ExperienceReplayBuffer.js

@@ -0,0 +1,255 @@
+"use strict";
+/**
+ * ExperienceReplayBuffer - Phase 2 (Milestone 2.2)
+ *
+ * Implements experience replay buffer for reinforcement learning.
+ * Stores transitions and enables batch sampling for training.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ExperienceReplayBuffer = void 0;
+const Logger_1 = require("../utils/Logger");
+/**
+ * Default replay buffer configuration
+ */
+const DEFAULT_CONFIG = {
+    maxSize: 10000,
+    minSize: 100,
+    prioritized: false
+};
+/**
+ * ExperienceReplayBuffer - FIFO buffer with optional prioritization
+ *
+ * Implements experience replay for more stable and efficient learning.
+ * Supports both uniform random sampling and prioritized experience replay.
+ */
+class ExperienceReplayBuffer {
+    constructor(config = {}) {
+        this.logger = Logger_1.Logger.getInstance();
+        this.config = { ...DEFAULT_CONFIG, ...config };
+        this.buffer = [];
+        this.totalExperiences = 0;
+    }
+    /**
+     * Add a new experience to the buffer
+     * Uses FIFO eviction when buffer is full
+     */
+    add(experience, priority) {
+        const prioritizedExp = {
+            experience,
+            priority: priority ?? this.calculateDefaultPriority(experience),
+            timestamp: Date.now()
+        };
+        // FIFO eviction: remove oldest when full
+        if (this.buffer.length >= this.config.maxSize) {
+            this.buffer.shift(); // Remove oldest (first element)
+        }
+        this.buffer.push(prioritizedExp);
+        this.totalExperiences++;
+    }
+    /**
+     * Add multiple experiences in batch
+     */
+    addBatch(experiences) {
+        for (const experience of experiences) {
+            this.add(experience);
+        }
+        this.logger.debug(`Added batch of ${experiences.length} experiences`);
+    }
+    /**
+     * Sample a random batch of experiences
+     * Uses uniform random sampling or prioritized sampling based on config
+     */
+    sample(batchSize) {
+        if (!this.canSample(batchSize)) {
+            throw new Error(`Cannot sample: buffer has ${this.buffer.length} experiences, ` +
+                `need at least ${Math.max(batchSize, this.config.minSize)}`);
+        }
+        if (this.config.prioritized) {
+            return this.prioritizedSample(batchSize);
+        }
+        else {
+            return this.uniformSample(batchSize);
+        }
+    }
+    /**
+     * Uniform random sampling (default)
+     */
+    uniformSample(batchSize) {
+        const sampled = [];
+        const indices = new Set();
+        // Sample without replacement
+        while (indices.size < batchSize) {
+            const randomIndex = Math.floor(Math.random() * this.buffer.length);
+            if (!indices.has(randomIndex)) {
+                indices.add(randomIndex);
+                sampled.push(this.buffer[randomIndex].experience);
+            }
+        }
+        return sampled;
+    }
+    /**
+     * Prioritized experience replay sampling
+     * Samples based on priority with probability proportional to priority
+     */
+    prioritizedSample(batchSize) {
+        const sampled = [];
+        const totalPriority = this.buffer.reduce((sum, exp) => sum + exp.priority, 0);
+        // Sample with replacement based on priorities
+        for (let i = 0; i < batchSize; i++) {
+            let random = Math.random() * totalPriority;
+            let cumulativePriority = 0;
+            for (const exp of this.buffer) {
+                cumulativePriority += exp.priority;
+                if (random <= cumulativePriority) {
+                    sampled.push(exp.experience);
+                    break;
+                }
+            }
+        }
+        return sampled;
+    }
+    /**
+     * Update priority for a specific experience
+     * Used in prioritized experience replay to adjust importance weights
+     */
+    updatePriority(experienceId, newPriority) {
+        const index = this.buffer.findIndex(exp => exp.experience.taskId === experienceId);
+        if (index === -1) {
+            return false;
+        }
+        this.buffer[index].priority = newPriority;
+        return true;
+    }
+    /**
+     * Calculate default priority based on TD-error magnitude
+     * Higher absolute rewards get higher priority
+     */
+    calculateDefaultPriority(experience) {
+        // Priority based on absolute reward (experiences with higher impact are prioritized)
+        const basePriority = Math.abs(experience.reward) + 0.01; // Add small constant to avoid zero priority
+        // Recent experiences get slight boost
+        const recencyBoost = 1.0;
+        return basePriority * recencyBoost;
+    }
+    /**
+     * Check if buffer has enough experiences to sample
+     */
+    canSample(batchSize) {
+        return this.buffer.length >= Math.max(batchSize, this.config.minSize);
+    }
+    /**
+     * Get recent experiences (for temporal coherence)
+     */
+    getRecent(count) {
+        const actualCount = Math.min(count, this.buffer.length);
+        return this.buffer
+            .slice(-actualCount)
+            .map(exp => exp.experience);
+    }
+    /**
+     * Get oldest experiences
+     */
+    getOldest(count) {
+        const actualCount = Math.min(count, this.buffer.length);
+        return this.buffer
+            .slice(0, actualCount)
+            .map(exp => exp.experience);
+    }
+    /**
+     * Get all experiences matching a filter
+     */
+    filter(predicate) {
+        return this.buffer
+            .filter(exp => predicate(exp.experience))
+            .map(exp => exp.experience);
+    }
+    /**
+     * Clear the buffer
+     */
+    clear() {
+        this.buffer = [];
+        this.logger.info('Experience replay buffer cleared');
+    }
+    /**
+     * Get current buffer size
+     */
+    size() {
+        return this.buffer.length;
+    }
+    /**
+     * Check if buffer is empty
+     */
+    isEmpty() {
+        return this.buffer.length === 0;
+    }
+    /**
+     * Check if buffer is full
+     */
+    isFull() {
+        return this.buffer.length >= this.config.maxSize;
+    }
+    /**
+     * Get total number of experiences ever added (including evicted)
+     */
+    getTotalExperiences() {
+        return this.totalExperiences;
+    }
+    /**
+     * Get buffer statistics
+     */
+    getStatistics() {
+        const avgPriority = this.buffer.length > 0
+            ? this.buffer.reduce((sum, exp) => sum + exp.priority, 0) / this.buffer.length
+            : 0;
+        const avgReward = this.buffer.length > 0
+            ? this.buffer.reduce((sum, exp) => sum + exp.experience.reward, 0) / this.buffer.length
+            : 0;
+        return {
+            size: this.buffer.length,
+            maxSize: this.config.maxSize,
+            utilization: this.buffer.length / this.config.maxSize,
+            totalAdded: this.totalExperiences,
+            avgPriority,
+            avgReward
+        };
+    }
+    /**
+     * Export buffer state for persistence
+     */
+    export() {
+        return {
+            buffer: [...this.buffer],
+            config: { ...this.config },
+            totalExperiences: this.totalExperiences
+        };
+    }
+    /**
+     * Import buffer state from persistence
+     */
+    import(state) {
+        this.buffer = [...state.buffer];
+        this.totalExperiences = state.totalExperiences;
+        this.logger.info(`Imported replay buffer with ${this.buffer.length} experiences`);
+    }
+    /**
+     * Prune old experiences beyond retention limit
+     * Keeps most recent experiences
+     */
+    prune(retentionCount) {
+        if (this.buffer.length <= retentionCount) {
+            return 0;
+        }
+        const removeCount = this.buffer.length - retentionCount;
+        this.buffer = this.buffer.slice(-retentionCount);
+        this.logger.info(`Pruned ${removeCount} old experiences from buffer`);
+        return removeCount;
+    }
+    /**
+     * Get memory usage estimate in bytes
+     */
+    getMemoryUsage() {
+        return JSON.stringify(this.buffer).length;
+    }
+}
+exports.ExperienceReplayBuffer = ExperienceReplayBuffer;
+//# sourceMappingURL=ExperienceReplayBuffer.js.map

package/dist/learning/ExperienceReplayBuffer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ExperienceReplayBuffer.js","sourceRoot":"","sources":["../../src/learning/ExperienceReplayBuffer.ts"],"names":[],"mappings":";AAAA;;;;;GAKG;;;AAGH,4CAAyC;AAWzC;;GAEG;AACH,MAAM,cAAc,GAAuB;IACzC,OAAO,EAAE,KAAK;IACd,OAAO,EAAE,GAAG;IACZ,WAAW,EAAE,KAAK;CACnB,CAAC;AAWF;;;;;GAKG;AACH,MAAa,sBAAsB;IAMjC,YAAY,SAAsC,EAAE;QAClD,IAAI,CAAC,MAAM,GAAG,eAAM,CAAC,WAAW,EAAE,CAAC;QACnC,IAAI,CAAC,MAAM,GAAG,EAAE,GAAG,cAAc,EAAE,GAAG,MAAM,EAAE,CAAC;QAC/C,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC;QACjB,IAAI,CAAC,gBAAgB,GAAG,CAAC,CAAC;IAC5B,CAAC;IAED;;;OAGG;IACH,GAAG,CAAC,UAA0B,EAAE,QAAiB;QAC/C,MAAM,cAAc,GAA0B;YAC5C,UAAU;YACV,QAAQ,EAAE,QAAQ,IAAI,IAAI,CAAC,wBAAwB,CAAC,UAAU,CAAC;YAC/D,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;SACtB,CAAC;QAEF,yCAAyC;QACzC,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YAC9C,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,gCAAgC;QACvD,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QACjC,IAAI,CAAC,gBAAgB,EAAE,CAAC;IAC1B,CAAC;IAED;;OAEG;IACH,QAAQ,CAAC,WAA6B;QACpC,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;YACrC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QACvB,CAAC;QACD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,kBAAkB,WAAW,CAAC,MAAM,cAAc,CAAC,CAAC;IACxE,CAAC;IAED;;;OAGG;IACH,MAAM,CAAC,SAAiB;QACtB,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,EAAE,CAAC;YAC/B,MAAM,IAAI,KAAK,CACb,6BAA6B,IAAI,CAAC,MAAM,CAAC,MAAM,gBAAgB;gBAC/D,iBAAiB,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAC5D,CAAC;QACJ,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;YAC5B,OAAO,IAAI,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;QAC3C,CAAC;aAAM,CAAC;YACN,OAAO,IAAI,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC;QACvC,CAAC;IACH,CAAC;IAED;;OAEG;IACK,aAAa,CAAC,SAAiB;QACrC,MAAM,OAAO,GAAqB,EAAE,CAAC;QACrC,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;QAElC,6BAA6B;QAC7B,OAAO,OAAO,CAAC,IAAI,GAAG,SAAS,EAAE,CAAC;YAChC,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YACnE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,CAAC;gBAC9B,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;gBACzB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,UAAU,CAAC,CAAC;YACpD,CAAC;QACH,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;;OAGG;IACK,iBAAiB,CAAC,SAAiB;QACzC,MAAM,OAAO,GAAqB,EAAE,CAAC;QACrC,MAAM,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;QAE9E,8CAA8C;QAC9C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,EAAE,CAAC,EAAE,EAAE,CAAC;YACnC,IAAI,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,aAAa,CAAC;YAC3C,IAAI,kBAAkB,GAAG,CAAC,CAAC;YAE3B,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBAC9B,kBAAkB,IAAI,GAAG,CAAC,QAAQ,CAAC;gBACnC,IAAI,MAAM,IAAI,kBAAkB,EAAE,CAAC;oBACjC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;oBAC7B,MAAM;gBACR,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;;OAGG;IACH,cAAc,CAAC,YAAoB,EAAE,WAAmB;QACtD,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,CACjC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,MAAM,KAAK,YAAY,CAC9C,CAAC;QAEF,IAAI,KAAK,KAAK,CAAC,CAAC,EAAE,CAAC;YACjB,OAAO,KAAK,CAAC;QACf,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,QAAQ,GAAG,WAAW,CAAC;QAC1C,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;;OAGG;IACK,wBAAwB,CAAC,UAA0B;QACzD,qFAAqF;QACrF,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC,4CAA4C;QAErG,sCAAsC;QACtC,MAAM,YAAY,GAAG,GAAG,CAAC;QAEzB,OAAO,YAAY,GAAG,YAAY,CAAC;IACrC,CAAC;IAED;;OAEG;IACH,SAAS,CAAC,SAAiB;QACzB,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACxE,CAAC;IAED;;OAEG;IACH,SAAS,CAAC,KAAa;QACrB,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACxD,OAAO,IAAI,CAAC,MAAM;aACf,KAAK,CAAC,CAAC,WAAW,CAAC;aACnB,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;IAChC,CAAC;IAED;;OAEG;IACH,SAAS,CAAC,KAAa;QACrB,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACxD,OAAO,IAAI,CAAC,MAAM;aACf,KAAK,CAAC,CAAC,EAAE,WAAW,CAAC;aACrB,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;IAChC,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,SAA2C;QAChD,OAAO,IAAI,CAAC,MAAM;aACf,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;aACxC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;IAChC,CAAC;IAED;;OAEG;IACH,KAAK;QACH,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC;QACjB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAC;IACvD,CAAC;IAED;;OAEG;IACH,IAAI;QACF,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;IAC5B,CAAC;IAED;;OAEG;IACH,OAAO;QACL,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC;IAClC,CAAC;IAED;;OAEG;IACH,MAAM;QACJ,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC;IACnD,CAAC;IAED;;OAEG;IACH,mBAAmB;QACjB,OAAO,IAAI,CAAC,gBAAgB,CAAC;IAC/B,CAAC;IAED;;OAEG;IACH,aAAa;QAQX,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC;YACxC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM;YAC9E,CAAC,CAAC,CAAC,CAAC;QAEN,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC;YACtC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,GAAG,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM;YACvF,CAAC,CAAC,CAAC,CAAC;QAEN,OAAO;YACL,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM;YACxB,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO;YAC5B,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO;YACrD,UAAU,EAAE,IAAI,CAAC,gBAAgB;YACjC,WAAW;YACX,SAAS;SACV,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,MAAM;QAKJ,OAAO;YACL,MAAM,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC;YACxB,MAAM,EAAE,EAAE,GAAG,IAAI,CAAC,MAAM,EAAE;YAC1B,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;SACxC,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,KAIN;QACC,IAAI,CAAC,MAAM,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;QAChC,IAAI,CAAC,gBAAgB,GAAG,KAAK,CAAC,gBAAgB,CAAC;QAC/C,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,+BAA+B,IAAI,CAAC,MAAM,CAAC,MAAM,cAAc,CAAC,CAAC;IACpF,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,cAAsB;QAC1B,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,cAAc,EAAE,CAAC;YACzC,OAAO,CAAC,CAAC;QACX,CAAC;QAED,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,cAAc,CAAC;QACxD,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,cAAc,CAAC,CAAC;QAEjD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,WAAW,8BAA8B,CAAC,CAAC;QACtE,OAAO,WAAW,CAAC;IACrB,CAAC;IAED;;OAEG;IACH,cAAc;QACZ,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAC5C,CAAC;CACF;AAhSD,wDAgSC"}

package/dist/learning/LearningEngine.d.ts

@@ -5,7 +5,8 @@
  * Uses Q-learning algorithm to optimize task execution strategies.
  */
 import { SwarmMemoryManager } from '../core/memory/SwarmMemoryManager';
-import { LearningConfig, TaskState, LearningFeedback, LearningOutcome, LearnedPattern, FailurePattern, StrategyRecommendation } from './types';
+import { QLearningConfig } from './QLearning';
+import { LearningConfig, TaskExperience, TaskState, AgentAction, LearningFeedback, LearningOutcome, LearnedPattern, FailurePattern, StrategyRecommendation } from './types';
 /**
  * LearningEngine - Reinforcement learning for agents
  */
@@ -15,6 +16,8 @@ export declare class LearningEngine {
     private readonly agentId;
     private config;
     private qTable;
+    private qLearning?;
+    private useQLearning;
     private experiences;
     private patterns;
     private failurePatterns;
@@ -112,6 +115,14 @@ export declare class LearningEngine {
      * Deserialize Q-table from storage
      */
     private deserializeQTable;
+    /**
+     * Serialize Q-table for QLearning import (converts to QValue format)
+     */
+    private serializeQTableForQLearning;
+    /**
+     * Deserialize Q-table from QLearning export (extracts values from QValue format)
+     */
+    private deserializeQTableFromQLearning;
     /**
      * Calculate state size in bytes
      */
@@ -140,5 +151,43 @@ export declare class LearningEngine {
      * Check if learning is enabled
      */
     isEnabled(): boolean;
+    /**
+     * Enable Q-learning mode (Phase 2 Integration)
+     * Switches from basic Q-table to full QLearning algorithm with experience replay
+     */
+    enableQLearning(config?: Partial<QLearningConfig>): void;
+    /**
+     * Disable Q-learning mode (revert to basic implementation)
+     */
+    disableQLearning(): void;
+    /**
+     * Learn from experience using Q-learning (when enabled)
+     * This method integrates with the QLearning algorithm
+     */
+    learnFromExperience(experience: TaskExperience): Promise<void>;
+    /**
+     * Select action with policy (Q-learning integration)
+     * Uses epsilon-greedy policy when Q-learning is enabled
+     */
+    selectActionWithPolicy(state: TaskState, availableActions: AgentAction[]): Promise<AgentAction>;
+    /**
+     * Get Q-learning statistics (when enabled)
+     */
+    getQLearningStats(): {
+        enabled: boolean;
+        stats?: {
+            steps: number;
+            episodes: number;
+            tableSize: number;
+            explorationRate: number;
+            avgQValue: number;
+            maxQValue: number;
+            minQValue: number;
+        };
+    };
+    /**
+     * Check if Q-learning mode is enabled
+     */
+    isQLearningEnabled(): boolean;
 }
 //# sourceMappingURL=LearningEngine.d.ts.map

package/dist/learning/LearningEngine.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"LearningEngine.d.ts","sourceRoot":"","sources":["../../src/learning/LearningEngine.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAIH,OAAO,EAAE,kBAAkB,EAAE,MAAM,mCAAmC,CAAC;AAKvE,OAAO,EACL,cAAc,EAEd,SAAS,EAET,gBAAgB,EAChB,eAAe,EACf,cAAc,EAEd,cAAc,EACd,sBAAsB,EAEvB,MAAM,SAAS,CAAC;AAiBjB;;GAEG;AACH,qBAAa,cAAc;IACzB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAqB;IACjD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,MAAM,CAAiB;IAC/B,OAAO,CAAC,MAAM,CAAmC;IACjD,OAAO,CAAC,WAAW,CAAmB;IACtC,OAAO,CAAC,QAAQ,CAA8B;IAC9C,OAAO,CAAC,eAAe,CAA8B;IACrD,OAAO,CAAC,SAAS,CAAS;gBAGxB,OAAO,EAAE,MAAM,EACf,WAAW,EAAE,kBAAkB,EAC/B,MAAM,GAAE,OAAO,CAAC,cAAc,CAAM;IAatC;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAgBjC;;OAEG;IACG,kBAAkB,CACtB,IAAI,EAAE,GAAG,EACT,MAAM,EAAE,GAAG,EACX,QAAQ,CAAC,EAAE,gBAAgB,GAC1B,OAAO,CAAC,eAAe,CAAC;IAqD3B;;OAEG;IACG,iBAAiB,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAAC,sBAAsB,CAAC;IAoD1E;;OAEG;IACH,WAAW,IAAI,cAAc,EAAE;IAK/B;;OAEG;IACH,kBAAkB,IAAI,cAAc,EAAE;IAKtC;;OAEG;IACH,OAAO,CAAC,eAAe;IAUvB;;OAEG;IACH,OAAO,CAAC,WAAW;IAKnB;;OAEG;IACH,OAAO,CAAC,YAAY;IAIpB;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAyCzB;;OAEG;IACH,OAAO,CAAC,eAAe;IA+BvB;;OAEG;YACW,YAAY;IA4B1B;;OAEG;YACW,kBAAkB;IAgBhC;;OAEG;YACW,cAAc;IAoC5B;;OAEG;YACW,oBAAoB;IAoBlC;;OAEG;YACW,oBAAoB;IAwBlC;;OAEG;IACH,OAAO,CAAC,aAAa;IAiBrB;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAc1B;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAIzB;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAOxB;;OAEG;YACW,SAAS;IA6BvB;;OAEG;YACW,SAAS;IAmBvB;;OAEG;IACH,OAAO,CAAC,eAAe;IAQvB;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAOzB;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAQ1B;;OAEG;YACW,qBAAqB;IAiBnC;;OAEG;YACW,iBAAiB;IAiB/B;;OAEG;IACH,kBAAkB,IAAI,MAAM;IAI5B;;OAEG;IACH,mBAAmB,IAAI,MAAM;IAI7B;;OAEG;IACH,UAAU,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI;IAIlC;;OAEG;IACH,SAAS,IAAI,OAAO;CAGrB"}
+{"version":3,"file":"LearningEngine.d.ts","sourceRoot":"","sources":["../../src/learning/LearningEngine.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAIH,OAAO,EAAE,kBAAkB,EAAE,MAAM,mCAAmC,CAAC;AACvE,OAAO,EAAa,eAAe,EAAE,MAAM,aAAa,CAAC;AAKzD,OAAO,EACL,cAAc,EACd,cAAc,EACd,SAAS,EACT,WAAW,EACX,gBAAgB,EAChB,eAAe,EACf,cAAc,EAEd,cAAc,EACd,sBAAsB,EAEvB,MAAM,SAAS,CAAC;AAiBjB;;GAEG;AACH,qBAAa,cAAc;IACzB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAqB;IACjD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,MAAM,CAAiB;IAC/B,OAAO,CAAC,MAAM,CAAmC;IACjD,OAAO,CAAC,SAAS,CAAC,CAAY;IAC9B,OAAO,CAAC,YAAY,CAAU;IAC9B,OAAO,CAAC,WAAW,CAAmB;IACtC,OAAO,CAAC,QAAQ,CAA8B;IAC9C,OAAO,CAAC,eAAe,CAA8B;IACrD,OAAO,CAAC,SAAS,CAAS;gBAGxB,OAAO,EAAE,MAAM,EACf,WAAW,EAAE,kBAAkB,EAC/B,MAAM,GAAE,OAAO,CAAC,cAAc,CAAM;IActC;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAgBjC;;OAEG;IACG,kBAAkB,CACtB,IAAI,EAAE,GAAG,EACT,MAAM,EAAE,GAAG,EACX,QAAQ,CAAC,EAAE,gBAAgB,GAC1B,OAAO,CAAC,eAAe,CAAC;IAqD3B;;OAEG;IACG,iBAAiB,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAAC,sBAAsB,CAAC;IAoD1E;;OAEG;IACH,WAAW,IAAI,cAAc,EAAE;IAK/B;;OAEG;IACH,kBAAkB,IAAI,cAAc,EAAE;IAKtC;;OAEG;IACH,OAAO,CAAC,eAAe;IAUvB;;OAEG;IACH,OAAO,CAAC,WAAW;IAKnB;;OAEG;IACH,OAAO,CAAC,YAAY;IAIpB;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAyCzB;;OAEG;IACH,OAAO,CAAC,eAAe;IA+BvB;;OAEG;YACW,YAAY;IA4B1B;;OAEG;YACW,kBAAkB;IAgBhC;;OAEG;YACW,cAAc;IAoC5B;;OAEG;YACW,oBAAoB;IAoBlC;;OAEG;YACW,oBAAoB;IAwBlC;;OAEG;IACH,OAAO,CAAC,aAAa;IAiBrB;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAc1B;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAIzB;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAOxB;;OAEG;YACW,SAAS;IA6BvB;;OAEG;YACW,SAAS;IAmBvB;;OAEG;IACH,OAAO,CAAC,eAAe;IAQvB;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAOzB;;OAEG;IACH,OAAO,CAAC,2BAA2B;IAiBnC;;OAEG;IACH,OAAO,CAAC,8BAA8B;IAWtC;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAQ1B;;OAEG;YACW,qBAAqB;IAiBnC;;OAEG;YACW,iBAAiB;IAiB/B;;OAEG;IACH,kBAAkB,IAAI,MAAM;IAI5B;;OAEG;IACH,mBAAmB,IAAI,MAAM;IAI7B;;OAEG;IACH,UAAU,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI;IAIlC;;OAEG;IACH,SAAS,IAAI,OAAO;IAIpB;;;OAGG;IACH,eAAe,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC,eAAe,CAAC,GAAG,IAAI;IAgCxD;;OAEG;IACH,gBAAgB,IAAI,IAAI;IAaxB;;;OAGG;IACG,mBAAmB,CAAC,UAAU,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC;IA4BpE;;;OAGG;IACG,sBAAsB,CAAC,KAAK,EAAE,SAAS,EAAE,gBAAgB,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC,WAAW,CAAC;IAiBrG;;OAEG;IACH,iBAAiB,IAAI;QACnB,OAAO,EAAE,OAAO,CAAC;QACjB,KAAK,CAAC,EAAE;YACN,KAAK,EAAE,MAAM,CAAC;YACd,QAAQ,EAAE,MAAM,CAAC;YACjB,SAAS,EAAE,MAAM,CAAC;YAClB,eAAe,EAAE,MAAM,CAAC;YACxB,SAAS,EAAE,MAAM,CAAC;YAClB,SAAS,EAAE,MAAM,CAAC;YAClB,SAAS,EAAE,MAAM,CAAC;SACnB,CAAC;KACH;IAWD;;OAEG;IACH,kBAAkB,IAAI,OAAO;CAG9B"}

package/dist/learning/LearningEngine.js

@@ -9,6 +9,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.LearningEngine = void 0;
 const uuid_1 = require("uuid");
 const Logger_1 = require("../utils/Logger");
+const QLearning_1 = require("./QLearning");
 // Import version from package.json to maintain consistency
 const packageJson = require('../../package.json');
 const PACKAGE_VERSION = packageJson.version;
@@ -36,6 +37,7 @@ class LearningEngine {
         this.memoryStore = memoryStore;
         this.config = { ...DEFAULT_CONFIG, ...config };
         this.qTable = new Map();
+        this.useQLearning = false; // Default to legacy implementation
         this.experiences = [];
         this.patterns = new Map();
         this.failurePatterns = new Map();
@@ -459,6 +461,38 @@ class LearningEngine {
             this.qTable.set(state, new Map(Object.entries(actions)));
         }
     }
+    /**
+     * Serialize Q-table for QLearning import (converts to QValue format)
+     */
+    serializeQTableForQLearning() {
+        const serialized = {};
+        for (const [state, actions] of this.qTable.entries()) {
+            serialized[state] = {};
+            for (const [action, value] of actions.entries()) {
+                serialized[state][action] = {
+                    state,
+                    action,
+                    value,
+                    updateCount: 1,
+                    lastUpdated: Date.now()
+                };
+            }
+        }
+        return serialized;
+    }
+    /**
+     * Deserialize Q-table from QLearning export (extracts values from QValue format)
+     */
+    deserializeQTableFromQLearning(data) {
+        this.qTable.clear();
+        for (const [state, actions] of Object.entries(data)) {
+            const actionMap = new Map();
+            for (const [action, qValue] of Object.entries(actions)) {
+                actionMap.set(action, qValue.value);
+            }
+            this.qTable.set(state, actionMap);
+        }
+    }
     /**
      * Calculate state size in bytes
      */
@@ -529,6 +563,112 @@ class LearningEngine {
     isEnabled() {
         return this.config.enabled;
     }
+    /**
+     * Enable Q-learning mode (Phase 2 Integration)
+     * Switches from basic Q-table to full QLearning algorithm with experience replay
+     */
+    enableQLearning(config) {
+        const qLearningConfig = {
+            learningRate: this.config.learningRate,
+            discountFactor: this.config.discountFactor,
+            explorationRate: this.config.explorationRate,
+            explorationDecay: this.config.explorationDecay,
+            minExplorationRate: this.config.minExplorationRate,
+            useExperienceReplay: true,
+            replayBufferSize: 10000,
+            batchSize: this.config.batchSize,
+            ...config
+        };
+        this.qLearning = new QLearning_1.QLearning(qLearningConfig);
+        this.useQLearning = true;
+        // Import existing Q-table into QLearning if we have data
+        if (this.qTable.size > 0) {
+            const serialized = this.serializeQTableForQLearning();
+            this.qLearning.import({
+                qTable: serialized,
+                config: qLearningConfig,
+                stepCount: this.taskCount,
+                episodeCount: Math.floor(this.taskCount / 10)
+            });
+        }
+        this.logger.info(`Q-learning mode enabled for agent ${this.agentId}`, {
+            config: qLearningConfig
+        });
+    }
+    /**
+     * Disable Q-learning mode (revert to basic implementation)
+     */
+    disableQLearning() {
+        if (this.qLearning && this.useQLearning) {
+            // Export Q-learning state to basic Q-table
+            const exported = this.qLearning.export();
+            this.deserializeQTableFromQLearning(exported.qTable);
+        }
+        this.qLearning = undefined;
+        this.useQLearning = false;
+        this.logger.info(`Q-learning mode disabled for agent ${this.agentId}`);
+    }
+    /**
+     * Learn from experience using Q-learning (when enabled)
+     * This method integrates with the QLearning algorithm
+     */
+    async learnFromExperience(experience) {
+        if (!this.config.enabled) {
+            return;
+        }
+        if (this.useQLearning && this.qLearning) {
+            // Use QLearning algorithm
+            this.qLearning.update(experience);
+            this.experiences.push(experience);
+            // Perform batch update periodically
+            if (this.taskCount % this.config.updateFrequency === 0) {
+                this.qLearning.batchUpdate();
+            }
+            // End episode periodically to trigger exploration decay
+            if (this.taskCount % 10 === 0) {
+                this.qLearning.endEpisode();
+            }
+        }
+        else {
+            // Use legacy Q-table implementation
+            await this.updateQTable(experience);
+            this.experiences.push(experience);
+        }
+        this.taskCount++;
+    }
+    /**
+     * Select action with policy (Q-learning integration)
+     * Uses epsilon-greedy policy when Q-learning is enabled
+     */
+    async selectActionWithPolicy(state, availableActions) {
+        if (this.useQLearning && this.qLearning) {
+            // Use Q-learning's epsilon-greedy policy
+            return this.qLearning.selectAction(state, availableActions);
+        }
+        // Fallback to recommendation-based selection
+        const recommendation = await this.recommendStrategy(state);
+        // Find the action matching the recommended strategy
+        const matchingAction = availableActions.find(action => action.strategy === recommendation.strategy);
+        return matchingAction || availableActions[0];
+    }
+    /**
+     * Get Q-learning statistics (when enabled)
+     */
+    getQLearningStats() {
+        if (!this.useQLearning || !this.qLearning) {
+            return { enabled: false };
+        }
+        return {
+            enabled: true,
+            stats: this.qLearning.getStatistics()
+        };
+    }
+    /**
+     * Check if Q-learning mode is enabled
+     */
+    isQLearningEnabled() {
+        return this.useQLearning;
+    }
 }
 exports.LearningEngine = LearningEngine;
 //# sourceMappingURL=LearningEngine.js.map