agentdb 1.0.0

package/dist/plugins/implementations/active-learning.mjs

@@ -0,0 +1,367 @@
+/**
+ * Active Learning Plugin
+ *
+ * Implements query-based learning where the agent actively selects
+ * the most informative samples to learn from, maximizing data efficiency.
+ *
+ * Key features:
+ * - Uncertainty sampling
+ * - Query-by-committee
+ * - Expected model change
+ * - Diversity-based sampling
+ * - Budget-aware querying
+ */
+import { BasePlugin } from '../base-plugin.mjs';
+/**
+ * Active Learning Plugin Implementation
+ */
+export class ActiveLearningPlugin extends BasePlugin {
+    constructor(config) {
+        super();
+        this.name = 'active-learning';
+        this.version = '1.0.0';
+        this.experiences = [];
+        this.queryStrategy = 'uncertainty';
+        this.labelingBudget = 1000;
+        this.queriedSamples = new Set();
+        this.unlabeledPool = [];
+        // Committee for query-by-committee
+        this.committee = [];
+        this.committeeSize = 5;
+        // Diversity tracking
+        this.diversityThreshold = 0.3;
+        if (config) {
+            this.queryStrategy = config.queryStrategy || 'uncertainty';
+            this.labelingBudget = config.labelingBudget !== undefined ? config.labelingBudget : 1000;
+            this.committeeSize = config.committeeSize || 5;
+        }
+        if (this.queryStrategy === 'committee') {
+            this.initializeCommittee();
+        }
+        // Mark as initialized for in-memory operation
+        this.initialized = true;
+    }
+    /**
+     * Override to skip initialization check for in-memory operation
+     */
+    checkInitialized() {
+        // No-op for active learning - operates in-memory
+    }
+    /**
+     * Override selectAction to provide base implementation
+     */
+    async selectAction(state, context) {
+        // Simple default action selection for active learning
+        return {
+            id: '0',
+            embedding: Array.isArray(state) ? state : [],
+            confidence: 0.5,
+        };
+    }
+    /**
+     * Override to store experiences in-memory without vectorDB
+     */
+    async storeExperience(experience) {
+        this.experiences.push(experience);
+    }
+    /**
+     * Override to retrieve from local experiences
+     */
+    async retrieveSimilar(state, k) {
+        return this.experiences.slice(0, k).map((exp, idx) => ({
+            id: exp.id || `exp-${idx}`,
+            embedding: exp.state,
+            metadata: exp,
+            score: 1.0 - (idx * 0.1), // Mock similarity score
+        }));
+    }
+    /**
+     * Initialize committee for query-by-committee
+     */
+    initializeCommittee() {
+        for (let i = 0; i < this.committeeSize; i++) {
+            const weights = new Array(256).fill(0).map(() => Math.random() * 0.1 - 0.05);
+            this.committee.push({
+                id: `member_${i}`,
+                weights,
+                predictions: new Map(),
+            });
+        }
+    }
+    /**
+     * Add unlabeled sample to pool
+     */
+    async addUnlabeledSample(state, metadata) {
+        this.unlabeledPool.push({
+            state,
+            uncertainty: 0,
+            queryValue: 0,
+            metadata,
+        });
+    }
+    /**
+     * Calculate uncertainty for a sample
+     */
+    async calculateUncertainty(sample) {
+        switch (this.queryStrategy) {
+            case 'uncertainty':
+                return this.uncertaintySampling(sample);
+            case 'margin':
+                return this.marginSampling(sample);
+            case 'entropy':
+                return this.entropySampling(sample);
+            case 'committee':
+                return this.queryByCommittee(sample);
+            case 'expected_model_change':
+                return this.expectedModelChange(sample);
+            case 'diverse':
+                return this.diversitySampling(sample);
+            default:
+                return this.uncertaintySampling(sample);
+        }
+    }
+    /**
+     * Uncertainty sampling: Select samples with highest prediction uncertainty
+     */
+    async uncertaintySampling(sample) {
+        const action = await this.selectAction(sample.state);
+        const confidence = action.confidence || 0;
+        // Uncertainty is 1 - confidence
+        return 1 - confidence;
+    }
+    /**
+     * Margin sampling: Select samples where top two predictions are close
+     */
+    async marginSampling(sample) {
+        // Get top K predictions
+        const predictions = await this.getTopPredictions(sample.state, 2);
+        if (predictions.length < 2) {
+            return 1.0;
+        }
+        // Margin is the difference between top two
+        const margin = Math.abs(predictions[0] - predictions[1]);
+        return 1 - margin; // Smaller margin = higher query value
+    }
+    /**
+     * Entropy sampling: Select samples with highest prediction entropy
+     */
+    async entropySampling(sample) {
+        const predictions = await this.getTopPredictions(sample.state, 10);
+        // Calculate Shannon entropy
+        let entropy = 0;
+        const sum = predictions.reduce((a, b) => a + b, 0);
+        for (const pred of predictions) {
+            const prob = pred / sum;
+            if (prob > 0) {
+                entropy -= prob * Math.log2(prob);
+            }
+        }
+        return entropy;
+    }
+    /**
+     * Query-by-committee: Select samples where committee disagrees most
+     */
+    async queryByCommittee(sample) {
+        const predictions = [];
+        // Get prediction from each committee member
+        for (const member of this.committee) {
+            const pred = this.predictWithWeights(sample.state, member.weights);
+            predictions.push(pred);
+        }
+        // Calculate variance (disagreement)
+        const mean = predictions.reduce((a, b) => a + b, 0) / predictions.length;
+        const variance = predictions.reduce((sum, p) => sum + Math.pow(p - mean, 2), 0) / predictions.length;
+        return Math.sqrt(variance); // Standard deviation
+    }
+    /**
+     * Expected model change: Select samples that would change model most
+     */
+    async expectedModelChange(sample) {
+        // Approximate gradient magnitude
+        const epsilon = 0.01;
+        const baseline = await this.selectAction(sample.state);
+        // Perturb state slightly
+        const perturbedState = sample.state.map((x) => x + epsilon);
+        const perturbed = await this.selectAction(perturbedState);
+        // Gradient magnitude
+        const gradMag = Math.abs((perturbed.confidence - baseline.confidence) / epsilon);
+        return gradMag;
+    }
+    /**
+     * Diversity sampling: Select diverse batch of samples
+     */
+    async diversitySampling(sample) {
+        if (this.queriedSamples.size === 0) {
+            return 1.0;
+        }
+        // Calculate distance to already queried samples
+        const queriedStates = Array.from(this.queriedSamples).map(id => {
+            const s = this.unlabeledPool.find(u => this.hashState(u.state) === id);
+            return s?.state || sample.state;
+        });
+        let minDistance = Infinity;
+        for (const qState of queriedStates) {
+            const dist = this.euclideanDistance(sample.state, qState);
+            minDistance = Math.min(minDistance, dist);
+        }
+        // Higher diversity value for samples far from queried ones
+        return minDistance;
+    }
+    /**
+     * Get top K predictions for a state
+     */
+    async getTopPredictions(state, k) {
+        // Simplified: return random predictions
+        const predictions = [];
+        for (let i = 0; i < k; i++) {
+            predictions.push(Math.random());
+        }
+        return predictions.sort((a, b) => b - a);
+    }
+    /**
+     * Predict using specific weights
+     */
+    predictWithWeights(state, weights) {
+        let activation = 0;
+        for (let i = 0; i < Math.min(state.length, weights.length); i++) {
+            activation += state[i] * weights[i];
+        }
+        return Math.tanh(activation);
+    }
+    /**
+     * Hash state for tracking
+     */
+    hashState(state) {
+        return state.slice(0, 10).join(',');
+    }
+    /**
+     * Euclidean distance between states
+     */
+    euclideanDistance(a, b) {
+        let sum = 0;
+        for (let i = 0; i < Math.min(a.length, b.length); i++) {
+            sum += Math.pow(a[i] - b[i], 2);
+        }
+        return Math.sqrt(sum);
+    }
+    /**
+     * Select next batch of samples to query (label)
+     */
+    async selectQueryBatch(batchSize) {
+        if (this.unlabeledPool.length === 0 || this.labelingBudget <= 0) {
+            return [];
+        }
+        // Calculate query value for each sample
+        for (const sample of this.unlabeledPool) {
+            sample.uncertainty = await this.calculateUncertainty(sample);
+            sample.queryValue = sample.uncertainty;
+        }
+        // Sort by query value
+        const sorted = [...this.unlabeledPool].sort((a, b) => b.queryValue - a.queryValue);
+        // Select top samples
+        const selected = sorted.slice(0, Math.min(batchSize, this.labelingBudget));
+        // Mark as queried
+        selected.forEach(s => {
+            this.queriedSamples.add(this.hashState(s.state));
+        });
+        // Update budget
+        this.labelingBudget -= selected.length;
+        // Remove from unlabeled pool
+        this.unlabeledPool = this.unlabeledPool.filter(s => !selected.includes(s));
+        return selected;
+    }
+    /**
+     * Train with active learning
+     */
+    async train(options) {
+        const startTime = Date.now();
+        const epochs = options?.epochs || 10;
+        const batchSize = options?.batchSize || 16;
+        let totalLoss = 0;
+        let experiencesProcessed = 0;
+        for (let epoch = 0; epoch < epochs; epoch++) {
+            // Select informative batch
+            const batch = await this.selectQueryBatch(batchSize);
+            if (batch.length === 0) {
+                break;
+            }
+            // Simulate labeling and training on queried samples
+            for (const sample of batch) {
+                // In real scenario, would query oracle for label
+                const simulatedLabel = Math.random();
+                const exp = {
+                    state: sample.state,
+                    action: { id: '0', embedding: sample.state },
+                    reward: simulatedLabel,
+                    nextState: sample.state,
+                    done: false,
+                };
+                await this.storeExperience(exp);
+                // Compute loss
+                const prediction = await this.selectAction(sample.state);
+                const loss = Math.pow(prediction.confidence - simulatedLabel, 2);
+                totalLoss += loss;
+                experiencesProcessed++;
+            }
+            // Update committee if using query-by-committee
+            if (this.queryStrategy === 'committee') {
+                await this.updateCommittee(batch);
+            }
+        }
+        const duration = Date.now() - startTime;
+        return {
+            loss: totalLoss / Math.max(1, experiencesProcessed),
+            experiencesProcessed,
+            duration,
+            samplesQueried: this.queriedSamples.size,
+            remainingBudget: this.labelingBudget,
+            unlabeledPoolSize: this.unlabeledPool.length,
+        };
+    }
+    /**
+     * Update committee members
+     */
+    async updateCommittee(batch) {
+        for (const member of this.committee) {
+            // Random subset for diversity
+            const subset = batch
+                .filter(() => Math.random() > 0.5)
+                .map(s => ({
+                state: s.state,
+                action: { id: '0', embedding: s.state },
+                reward: Math.random(),
+                nextState: s.state,
+                done: false,
+            }));
+            // Update weights (simplified SGD)
+            for (const exp of subset) {
+                const pred = this.predictWithWeights(exp.state, member.weights);
+                const error = pred - exp.reward;
+                for (let i = 0; i < member.weights.length; i++) {
+                    member.weights[i] -= 0.01 * error * exp.state[i % exp.state.length];
+                }
+            }
+        }
+    }
+    /**
+     * Get active learning statistics
+     */
+    getStats() {
+        const avgUncertainty = this.unlabeledPool.length > 0
+            ? this.unlabeledPool.reduce((sum, s) => sum + s.uncertainty, 0) / this.unlabeledPool.length
+            : 0;
+        return {
+            queriedCount: this.queriedSamples.size,
+            remainingBudget: this.labelingBudget,
+            unlabeledCount: this.unlabeledPool.length,
+            averageUncertainty: avgUncertainty,
+        };
+    }
+    /**
+     * Reset querying state
+     */
+    resetQuery() {
+        this.queriedSamples.clear();
+        this.unlabeledPool = [];
+    }
+}

package/dist/plugins/implementations/actor-critic.d.ts

@@ -0,0 +1,64 @@
+/**
+ * Actor-Critic Plugin
+ *
+ * Implements the Actor-Critic algorithm with policy gradient learning.
+ * Combines value-based and policy-based methods:
+ * - Actor: Policy network that selects actions
+ * - Critic: Value network that evaluates actions
+ *
+ * Key features:
+ * - Policy gradient learning (REINFORCE with baseline)
+ * - Advantage estimation (GAE - Generalized Advantage Estimation)
+ * - Continuous or discrete action spaces
+ * - Natural policy gradients
+ */
+import { BasePlugin } from '../base-plugin';
+import { Action, Context, Experience, TrainOptions, TrainingMetrics } from '../learning-plugin.interface';
+/**
+ * Actor-Critic Plugin Implementation
+ */
+export declare class ActorCriticPlugin extends BasePlugin {
+    name: string;
+    version: string;
+    private actor;
+    private critic;
+    private experienceBuffer;
+    private gaeLambda;
+    /**
+     * Initialize Actor-Critic plugin
+     */
+    protected onInitialize(): Promise<void>;
+    /**
+     * Select action using actor network
+     *
+     * @param state - Current state vector
+     * @param context - Optional context
+     * @returns Selected action
+     */
+    selectAction(state: number[], context?: Context): Promise<Action>;
+    /**
+     * Store experience in buffer
+     */
+    protected onStoreExperience(experience: Experience): Promise<void>;
+    /**
+     * Train on completed episode
+     */
+    private trainOnEpisode;
+    /**
+     * Compute Generalized Advantage Estimation (GAE)
+     */
+    private computeGAE;
+    /**
+     * Train the actor-critic networks
+     */
+    train(options?: TrainOptions): Promise<TrainingMetrics>;
+    /**
+     * Save actor and critic networks
+     */
+    protected onSave(path: string): Promise<void>;
+    /**
+     * Load actor and critic networks
+     */
+    protected onLoad(path: string): Promise<void>;
+}
+//# sourceMappingURL=actor-critic.d.ts.map

package/dist/plugins/implementations/actor-critic.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"actor-critic.d.ts","sourceRoot":"","sources":["../../../src/plugins/implementations/actor-critic.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAC5C,OAAO,EACL,MAAM,EACN,OAAO,EACP,UAAU,EACV,YAAY,EACZ,eAAe,EAChB,MAAM,8BAA8B,CAAC;AAqUtC;;GAEG;AACH,qBAAa,iBAAkB,SAAQ,UAAU;IACxC,IAAI,SAAkB;IACtB,OAAO,SAAW;IAEzB,OAAO,CAAC,KAAK,CAAgB;IAC7B,OAAO,CAAC,MAAM,CAAiB;IAC/B,OAAO,CAAC,gBAAgB,CAAoB;IAC5C,OAAO,CAAC,SAAS,CAAgB;IAEjC;;OAEG;cACa,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;IAa7C;;;;;;OAMG;IACG,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,OAAO,CAAC,EAAE,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC;IAevE;;OAEG;cACa,iBAAiB,CAAC,UAAU,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAUxE;;OAEG;YACW,cAAc;IAyB5B;;OAEG;IACH,OAAO,CAAC,UAAU;IAmBlB;;OAEG;IACG,KAAK,CAAC,OAAO,CAAC,EAAE,YAAY,GAAG,OAAO,CAAC,eAAe,CAAC;IA0B7D;;OAEG;cACa,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAKnD;;OAEG;cACa,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;CAIpD"}