agentdb 1.0.0

package/dist/plugins/implementations/q-learning.mjs

@@ -0,0 +1,430 @@
+/**
+ * Q-Learning Plugin
+ *
+ * Implements the Q-Learning algorithm with experience replay.
+ * Q-Learning is a model-free, off-policy RL algorithm that learns
+ * the optimal action-value function (Q-function).
+ *
+ * Key features:
+ * - Epsilon-greedy exploration
+ * - Experience replay buffer
+ * - Optional prioritized experience replay
+ * - Temporal difference learning
+ */
+import { BasePlugin } from '../base-plugin.mjs';
+/**
+ * Experience replay buffer
+ */
+class ReplayBuffer {
+    constructor(capacity) {
+        this.buffer = [];
+        this.position = 0;
+        this.capacity = capacity;
+    }
+    /**
+     * Add experience to buffer
+     */
+    add(experience) {
+        if (this.buffer.length < this.capacity) {
+            this.buffer.push(experience);
+        }
+        else {
+            this.buffer[this.position] = experience;
+        }
+        this.position = (this.position + 1) % this.capacity;
+    }
+    /**
+     * Sample random batch
+     */
+    sample(batchSize) {
+        const batch = [];
+        for (let i = 0; i < Math.min(batchSize, this.buffer.length); i++) {
+            const index = Math.floor(Math.random() * this.buffer.length);
+            batch.push(this.buffer[index]);
+        }
+        return batch;
+    }
+    /**
+     * Get buffer size
+     */
+    size() {
+        return this.buffer.length;
+    }
+    /**
+     * Check if buffer has enough samples
+     */
+    hasEnough(minSize) {
+        return this.buffer.length >= minSize;
+    }
+}
+/**
+ * Prioritized Experience Replay Buffer
+ */
+class PrioritizedReplayBuffer extends ReplayBuffer {
+    constructor(capacity, alpha = 0.6, beta = 0.4, betaIncrement = 0.001) {
+        super(capacity);
+        this.priorities = [];
+        this.alpha = alpha;
+        this.beta = beta;
+        this.betaIncrement = betaIncrement;
+    }
+    /**
+     * Add experience with priority
+     */
+    addWithPriority(experience, priority) {
+        this.add(experience);
+        if (this.priorities.length < this.size()) {
+            this.priorities.push(priority);
+        }
+        else {
+            const pos = this.position - 1;
+            this.priorities[pos >= 0 ? pos : this.priorities.length - 1] = priority;
+        }
+    }
+    /**
+     * Sample batch using prioritized sampling
+     */
+    samplePrioritized(batchSize) {
+        const n = this.size();
+        const batch = [];
+        const indices = [];
+        const weights = [];
+        // Compute sampling probabilities
+        const probs = this.priorities.map(p => Math.pow(p, this.alpha));
+        const totalProb = probs.reduce((sum, p) => sum + p, 0);
+        // Sample
+        for (let i = 0; i < Math.min(batchSize, n); i++) {
+            const rand = Math.random() * totalProb;
+            let cumProb = 0;
+            let index = 0;
+            for (let j = 0; j < probs.length; j++) {
+                cumProb += probs[j];
+                if (rand <= cumProb) {
+                    index = j;
+                    break;
+                }
+            }
+            batch.push(this.buffer[index]);
+            indices.push(index);
+            // Compute importance sampling weight
+            const prob = probs[index] / totalProb;
+            const weight = Math.pow(n * prob, -this.beta);
+            weights.push(weight);
+        }
+        // Normalize weights
+        const maxWeight = Math.max(...weights);
+        const normalizedWeights = weights.map(w => w / maxWeight);
+        // Anneal beta
+        this.beta = Math.min(1.0, this.beta + this.betaIncrement);
+        return { experiences: batch, indices, weights: normalizedWeights };
+    }
+    /**
+     * Update priority for specific experience
+     */
+    updatePriority(index, priority) {
+        if (index >= 0 && index < this.priorities.length) {
+            this.priorities[index] = priority;
+        }
+    }
+}
+/**
+ * Q-Learning Plugin Implementation
+ */
+export class QLearningPlugin extends BasePlugin {
+    constructor() {
+        super(...arguments);
+        this.name = 'q-learning';
+        this.version = '1.0.0';
+        this.qTable = new Map();
+        this.epsilon = 1.0;
+        this.epsilonMin = 0.01;
+        this.epsilonDecay = 0.995;
+        this.usePrioritized = false;
+        this.trainCounter = 0;
+    }
+    /**
+     * Initialize Q-Learning plugin
+     */
+    async onInitialize() {
+        // Initialize epsilon
+        this.epsilon = this.config.algorithm.epsilonStart || 1.0;
+        this.epsilonMin = this.config.algorithm.epsilonEnd || 0.01;
+        this.epsilonDecay = this.config.algorithm.epsilonDecay || 0.995;
+        // Initialize replay buffer
+        const replayConfig = this.config.experienceReplay;
+        if (replayConfig && replayConfig.type === 'prioritized') {
+            this.usePrioritized = true;
+            this.replayBuffer = new PrioritizedReplayBuffer(replayConfig.capacity, replayConfig.alpha, replayConfig.beta, replayConfig.betaIncrement);
+        }
+        else if (replayConfig && replayConfig.type === 'uniform') {
+            this.replayBuffer = new ReplayBuffer(replayConfig.capacity);
+        }
+        else {
+            // Default replay buffer
+            this.replayBuffer = new ReplayBuffer(10000);
+        }
+    }
+    /**
+     * Select action using epsilon-greedy policy
+     *
+     * @param state - Current state vector
+     * @param context - Optional context
+     * @returns Selected action
+     */
+    async selectAction(state, context) {
+        this.checkInitialized();
+        // Epsilon-greedy exploration
+        if (Math.random() < this.epsilon) {
+            return this.randomAction(state);
+        }
+        // Exploit: Select action with highest Q-value
+        return this.greedyAction(state);
+    }
+    /**
+     * Select random action (exploration)
+     */
+    async randomAction(state) {
+        // Find similar states to get action space
+        const similar = await this.retrieveSimilar(state, 10);
+        if (similar.length === 0) {
+            // No similar states, return random embedding
+            return {
+                id: 'random',
+                embedding: Array.from({ length: 768 }, () => Math.random() * 2 - 1),
+                source: 'policy',
+                confidence: 0,
+                metadata: { exploration: true },
+            };
+        }
+        // Random action from similar states
+        const randomIdx = Math.floor(Math.random() * similar.length);
+        const randomExp = similar[randomIdx];
+        if (!randomExp.metadata) {
+            // Fallback to random embedding
+            return {
+                id: 'random',
+                embedding: Array.from({ length: 768 }, () => Math.random() * 2 - 1),
+                source: 'policy',
+                confidence: 0,
+                metadata: { exploration: true },
+            };
+        }
+        return {
+            id: randomExp.id,
+            embedding: randomExp.metadata.action,
+            source: 'policy',
+            confidence: 0,
+            metadata: { exploration: true },
+        };
+    }
+    /**
+     * Select greedy action (exploitation)
+     */
+    async greedyAction(state) {
+        const stateKey = this.hashState(state);
+        // Get Q-values for this state
+        const qValues = this.qTable.get(stateKey);
+        if (!qValues || qValues.size === 0) {
+            // No Q-values yet, explore similar states
+            const similar = await this.retrieveSimilar(state, 1);
+            if (similar.length > 0 && similar[0].metadata) {
+                return {
+                    id: similar[0].id,
+                    embedding: similar[0].metadata.action,
+                    source: 'policy',
+                    confidence: similar[0].score,
+                    metadata: { exploration: false },
+                };
+            }
+            // Fallback to random
+            return this.randomAction(state);
+        }
+        // Find action with maximum Q-value
+        let maxQ = -Infinity;
+        let bestAction = '';
+        for (const [action, qValue] of qValues.entries()) {
+            if (qValue > maxQ) {
+                maxQ = qValue;
+                bestAction = action;
+            }
+        }
+        // Retrieve action embedding from storage
+        const actionData = await this.getActionEmbedding(bestAction);
+        return {
+            id: bestAction,
+            embedding: actionData,
+            source: 'policy',
+            confidence: this.normalizeQValue(maxQ),
+            metadata: { exploration: false, qValue: maxQ },
+        };
+    }
+    /**
+     * Store experience and update Q-table
+     */
+    async onStoreExperience(experience) {
+        // Add to replay buffer
+        if (this.usePrioritized) {
+            const tdError = this.computeTDError(experience);
+            this.replayBuffer.addWithPriority(experience, Math.abs(tdError) + 1e-6);
+        }
+        else {
+            this.replayBuffer.add(experience);
+        }
+        // Train periodically
+        this.trainCounter++;
+        const trainEvery = this.config.training.trainEvery || 100;
+        if (this.trainCounter >= trainEvery && this.replayBuffer.hasEnough(this.config.training.minExperiences)) {
+            await this.train({ epochs: 1, verbose: false });
+            this.trainCounter = 0;
+        }
+    }
+    /**
+     * Train Q-Learning on replay buffer
+     */
+    async train(options) {
+        this.checkInitialized();
+        const epochs = options?.epochs || 1;
+        const batchSize = options?.batchSize || this.config.training.batchSize;
+        const learningRate = options?.learningRate || this.config.algorithm.learningRate;
+        const gamma = this.config.algorithm.discountFactor;
+        let totalLoss = 0;
+        let avgQValue = 0;
+        for (let epoch = 0; epoch < epochs; epoch++) {
+            let epochLoss = 0;
+            let epochQSum = 0;
+            // Sample batch
+            let batch;
+            let weights = [];
+            if (this.usePrioritized) {
+                const sampled = this.replayBuffer.samplePrioritized(batchSize);
+                batch = sampled.experiences;
+                weights = sampled.weights;
+            }
+            else {
+                batch = this.replayBuffer.sample(batchSize);
+                weights = new Array(batch.length).fill(1.0);
+            }
+            // Update Q-values
+            for (let i = 0; i < batch.length; i++) {
+                const exp = batch[i];
+                const weight = weights[i];
+                const stateKey = this.hashState(exp.state);
+                const actionKey = this.hashAction(exp.action);
+                // Get current Q-value
+                const currentQ = this.getQValue(stateKey, actionKey);
+                // Compute target Q-value
+                const nextStateKey = this.hashState(exp.nextState);
+                const maxNextQ = exp.done ? 0 : this.getMaxQValue(nextStateKey);
+                const targetQ = exp.reward + gamma * maxNextQ;
+                // TD error
+                const tdError = targetQ - currentQ;
+                // Update Q-value
+                const newQ = currentQ + learningRate * weight * tdError;
+                this.setQValue(stateKey, actionKey, newQ);
+                // Track metrics
+                epochLoss += tdError * tdError * weight;
+                epochQSum += newQ;
+                // Update priority if using prioritized replay
+                if (this.usePrioritized) {
+                    const sampled = this.replayBuffer.samplePrioritized(batchSize);
+                    this.replayBuffer.updatePriority(sampled.indices[i], Math.abs(tdError) + 1e-6);
+                }
+            }
+            totalLoss = epochLoss / batch.length;
+            avgQValue = epochQSum / batch.length;
+        }
+        // Decay epsilon
+        this.epsilon = Math.max(this.epsilonMin, this.epsilon * this.epsilonDecay);
+        return {
+            loss: totalLoss,
+            avgQValue,
+            epsilon: this.epsilon,
+        };
+    }
+    /**
+     * Compute TD error for prioritized replay
+     */
+    computeTDError(experience) {
+        const stateKey = this.hashState(experience.state);
+        const actionKey = this.hashAction(experience.action);
+        const gamma = this.config.algorithm.discountFactor;
+        const currentQ = this.getQValue(stateKey, actionKey);
+        const nextStateKey = this.hashState(experience.nextState);
+        const maxNextQ = experience.done ? 0 : this.getMaxQValue(nextStateKey);
+        const targetQ = experience.reward + gamma * maxNextQ;
+        return targetQ - currentQ;
+    }
+    /**
+     * Get Q-value for state-action pair
+     */
+    getQValue(stateKey, actionKey) {
+        const qValues = this.qTable.get(stateKey);
+        return qValues?.get(actionKey) || 0;
+    }
+    /**
+     * Set Q-value for state-action pair
+     */
+    setQValue(stateKey, actionKey, value) {
+        if (!this.qTable.has(stateKey)) {
+            this.qTable.set(stateKey, new Map());
+        }
+        this.qTable.get(stateKey).set(actionKey, value);
+    }
+    /**
+     * Get maximum Q-value for a state
+     */
+    getMaxQValue(stateKey) {
+        const qValues = this.qTable.get(stateKey);
+        if (!qValues || qValues.size === 0) {
+            return 0;
+        }
+        return Math.max(...Array.from(qValues.values()));
+    }
+    /**
+     * Hash state vector to string key
+     */
+    hashState(state) {
+        // Simple hash - in production, use better hashing or clustering
+        return state.slice(0, 10).map((x) => x.toFixed(2)).join(',');
+    }
+    /**
+     * Hash action to string key
+     */
+    hashAction(action) {
+        if (typeof action === 'string') {
+            return action;
+        }
+        if (Array.isArray(action)) {
+            return action.slice(0, 10).map((x) => x.toFixed(2)).join(',');
+        }
+        return String(action);
+    }
+    /**
+     * Get action embedding from ID
+     */
+    async getActionEmbedding(actionId) {
+        // In production, retrieve from database
+        // For now, return random embedding
+        return Array.from({ length: 768 }, () => Math.random() * 2 - 1);
+    }
+    /**
+     * Normalize Q-value to 0-1 confidence
+     */
+    normalizeQValue(qValue) {
+        return 1 / (1 + Math.exp(-qValue)); // Sigmoid
+    }
+    /**
+     * Save Q-table
+     */
+    async onSave(path) {
+        console.log(`Saving Q-Learning model to ${path}`);
+        // In production, serialize Q-table to file
+    }
+    /**
+     * Load Q-table
+     */
+    async onLoad(path) {
+        console.log(`Loading Q-Learning model from ${path}`);
+        // In production, deserialize Q-table from file
+    }
+}

package/dist/plugins/implementations/sarsa.d.ts

@@ -0,0 +1,103 @@
+/**
+ * SARSA Plugin
+ *
+ * Implements the SARSA (State-Action-Reward-State-Action) algorithm
+ * with eligibility traces (SARSA(λ)).
+ *
+ * SARSA is an on-policy TD control algorithm that learns from the
+ * actual actions taken by the current policy, unlike Q-Learning which
+ * is off-policy.
+ *
+ * Key features:
+ * - On-policy learning
+ * - Eligibility traces for faster learning
+ * - Epsilon-greedy exploration
+ * - More conservative than Q-Learning
+ */
+import { BasePlugin } from '../base-plugin';
+import { Action, Context, Experience, TrainOptions, TrainingMetrics } from '../learning-plugin.interface';
+/**
+ * SARSA Plugin Implementation
+ */
+export declare class SARSAPlugin extends BasePlugin {
+    name: string;
+    version: string;
+    private qTable;
+    private epsilon;
+    private epsilonMin;
+    private epsilonDecay;
+    private eligibilityTraces;
+    private lambda;
+    private lastState;
+    private lastAction;
+    /**
+     * Initialize SARSA plugin
+     */
+    protected onInitialize(): Promise<void>;
+    /**
+     * Select action using epsilon-greedy policy
+     *
+     * @param state - Current state vector
+     * @param context - Optional context
+     * @returns Selected action
+     */
+    selectAction(state: number[], context?: Context): Promise<Action>;
+    /**
+     * Select random action (exploration)
+     */
+    private randomAction;
+    /**
+     * Select greedy action (exploitation)
+     */
+    private greedyAction;
+    /**
+     * Store experience and perform SARSA update
+     *
+     * SARSA uses the actual next action taken, not the max Q-value
+     */
+    protected onStoreExperience(experience: Experience): Promise<void>;
+    /**
+     * Perform SARSA(λ) update with eligibility traces
+     */
+    private sarsaUpdate;
+    /**
+     * Train SARSA on stored experiences
+     *
+     * For online learning, this is called after each experience.
+     * For offline learning, this processes batches of episodes.
+     */
+    train(options?: TrainOptions): Promise<TrainingMetrics>;
+    /**
+     * Get Q-value for state-action pair
+     */
+    private getQValue;
+    /**
+     * Set Q-value for state-action pair
+     */
+    private setQValue;
+    /**
+     * Hash state vector to string key
+     */
+    private hashState;
+    /**
+     * Hash action to string key
+     */
+    private hashAction;
+    /**
+     * Get action embedding from ID
+     */
+    private getActionEmbedding;
+    /**
+     * Normalize Q-value to 0-1 confidence
+     */
+    private normalizeQValue;
+    /**
+     * Save Q-table and eligibility traces
+     */
+    protected onSave(path: string): Promise<void>;
+    /**
+     * Load Q-table and eligibility traces
+     */
+    protected onLoad(path: string): Promise<void>;
+}
+//# sourceMappingURL=sarsa.d.ts.map

package/dist/plugins/implementations/sarsa.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sarsa.d.ts","sourceRoot":"","sources":["../../../src/plugins/implementations/sarsa.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAC5C,OAAO,EACL,MAAM,EACN,OAAO,EACP,UAAU,EACV,YAAY,EACZ,eAAe,EAChB,MAAM,8BAA8B,CAAC;AA8EtC;;GAEG;AACH,qBAAa,WAAY,SAAQ,UAAU;IAClC,IAAI,SAAW;IACf,OAAO,SAAW;IAEzB,OAAO,CAAC,MAAM,CAA+C;IAC7D,OAAO,CAAC,OAAO,CAAe;IAC9B,OAAO,CAAC,UAAU,CAAgB;IAClC,OAAO,CAAC,YAAY,CAAiB;IACrC,OAAO,CAAC,iBAAiB,CAAoB;IAC7C,OAAO,CAAC,MAAM,CAAe;IAC7B,OAAO,CAAC,SAAS,CAAyB;IAC1C,OAAO,CAAC,UAAU,CAAa;IAE/B;;OAEG;cACa,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;IAc7C;;;;;;OAMG;IACG,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,OAAO,CAAC,EAAE,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC;IAYvE;;OAEG;YACW,YAAY;IAuC1B;;OAEG;YACW,YAAY;IA+C1B;;;;OAIG;cACa,iBAAiB,CAAC,UAAU,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAyBxE;;OAEG;YACW,WAAW;IAuCzB;;;;;OAKG;IACG,KAAK,CAAC,OAAO,CAAC,EAAE,YAAY,GAAG,OAAO,CAAC,eAAe,CAAC;IA4B7D;;OAEG;IACH,OAAO,CAAC,SAAS;IAKjB;;OAEG;IACH,OAAO,CAAC,SAAS;IAOjB;;OAEG;IACH,OAAO,CAAC,SAAS;IAKjB;;OAEG;IACH,OAAO,CAAC,UAAU;IAYlB;;OAEG;YACW,kBAAkB;IAMhC;;OAEG;IACH,OAAO,CAAC,eAAe;IAIvB;;OAEG;cACa,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAKnD;;OAEG;cACa,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;CAIpD"}