agentic-qe 3.7.5 → 3.7.6

package/dist/neural-optimizer/topology-optimizer.js

@@ -1,657 +0,0 @@
-/**
- * Agentic QE v3 - Neural Topology Optimizer
- * ADR-034: RL-based swarm topology optimization
- *
- * Implements reinforcement learning-based topology optimization:
- * - Q-learning with value network for state evaluation
- * - Experience replay for stable training
- * - Epsilon-greedy exploration
- * - Multi-objective reward (min-cut, efficiency, load balance)
- */
-import { DEFAULT_OPTIMIZER_CONFIG, actionToIndex, indexToActionType, ACTION_TYPES } from './types';
-import { ValueNetwork } from './value-network';
-import { PrioritizedReplayBuffer } from './replay-buffer';
-import { secureRandom, secureRandomInt } from '../shared/utils/crypto-random.js';
-// ============================================================================
-// Neural Topology Optimizer
-// ============================================================================
-/**
- * Neural Topology Optimizer using reinforcement learning
- *
- * Uses a value network to estimate state values and learns to select
- * topology modifications that improve overall swarm performance.
- */
-export class NeuralTopologyOptimizer {
-    /** Primary value network */
-    valueNetwork;
-    /** Target network for stable learning */
-    targetNetwork;
-    /** Experience replay buffer */
-    replayBuffer;
-    /** Optimizer configuration */
-    config;
-    /** Reference to topology being optimized */
-    topology;
-    /** Previous state for learning */
-    prevState = null;
-    /** Previous min-cut estimate */
-    prevMinCut = 0;
-    /** Current exploration rate */
-    epsilon;
-    /** Simulation time */
-    time = 0;
-    /** Total optimization steps */
-    totalSteps = 0;
-    /** Episode count */
-    episodes = 0;
-    /** Cumulative reward */
-    cumulativeReward = 0;
-    /** Action counts */
-    actionCounts;
-    /** Min-cut history */
-    minCutHistory = [];
-    /** Reward history */
-    rewardHistory = [];
-    /** Last action taken (for feedback) */
-    lastAction = null;
-    constructor(topology, config = {}) {
-        this.topology = topology;
-        this.config = { ...DEFAULT_OPTIMIZER_CONFIG, ...config };
-        this.epsilon = this.config.epsilon;
-        // Initialize value networks
-        this.valueNetwork = new ValueNetwork(this.config.inputSize, this.config.hiddenSize);
-        this.targetNetwork = new ValueNetwork(this.config.inputSize, this.config.hiddenSize);
-        this.targetNetwork.copyFrom(this.valueNetwork);
-        // Initialize replay buffer
-        this.replayBuffer = new PrioritizedReplayBuffer(this.config.replayBufferSize, { alpha: 0.6, beta: 0.4 });
-        // Initialize action counts
-        this.actionCounts = {};
-        for (const actionType of ACTION_TYPES) {
-            this.actionCounts[actionType] = 0;
-        }
-        // Initialize state
-        this.prevState = this.extractFeatures();
-        this.prevMinCut = this.estimateMinCut();
-    }
-    // ============================================================================
-    // Public API
-    // ============================================================================
-    /**
-     * Run one optimization step
-     */
-    optimizeStep() {
-        // 1. Encode current state
-        const state = this.extractFeatures();
-        const valueBefore = this.valueNetwork.estimate(state);
-        // 2. Select action using epsilon-greedy
-        const action = this.selectAction(state);
-        this.lastAction = action;
-        // 3. Execute action
-        const oldMinCut = this.estimateMinCut();
-        this.applyAction(action);
-        const newMinCut = this.estimateMinCut();
-        // 4. Calculate multi-objective reward
-        const reward = this.calculateReward(oldMinCut, newMinCut);
-        // 5. Get new state
-        const nextState = this.extractFeatures();
-        const valueAfter = this.valueNetwork.estimate(nextState);
-        // 6. Calculate TD error
-        const targetValue = reward + this.config.gamma * this.targetNetwork.estimate(nextState);
-        const tdError = targetValue - valueBefore;
-        // 7. Update value network
-        this.valueNetwork.update(state, tdError, this.config.learningRate);
-        // 8. Store experience
-        if (this.prevState !== null) {
-            const experience = {
-                state: this.prevState,
-                actionIdx: actionToIndex(action),
-                reward,
-                nextState,
-                done: false,
-                tdError: Math.abs(tdError),
-                timestamp: Date.now(),
-            };
-            this.replayBuffer.push(experience);
-        }
-        // 9. Train from replay buffer
-        if (this.replayBuffer.length >= this.config.minExperiencesForTraining) {
-            this.trainFromReplay();
-        }
-        // 10. Update target network periodically
-        this.totalSteps++;
-        if (this.totalSteps % this.config.targetUpdateFrequency === 0) {
-            this.targetNetwork.softUpdate(this.valueNetwork, 0.01);
-        }
-        // 11. Decay exploration rate
-        this.epsilon = Math.max(this.config.minEpsilon, this.epsilon * this.config.epsilonDecay);
-        // 12. Update tracking
-        this.prevState = nextState;
-        this.prevMinCut = newMinCut;
-        this.time += this.config.dt;
-        this.actionCounts[action.type]++;
-        this.cumulativeReward += reward;
-        this.minCutHistory.push(newMinCut);
-        this.rewardHistory.push(reward);
-        // Limit history size
-        if (this.minCutHistory.length > 1000) {
-            this.minCutHistory.shift();
-            this.rewardHistory.shift();
-        }
-        // Calculate metrics
-        const loadStats = this.getLoadStats();
-        return {
-            action,
-            reward,
-            newMinCut,
-            communicationEfficiency: this.measureCommunicationEfficiency(),
-            loadBalance: 1 - loadStats.variance,
-            tdError,
-            epsilon: this.epsilon,
-            valueBefore,
-            valueAfter,
-        };
-    }
-    /**
-     * Run multiple optimization steps
-     */
-    optimize(steps) {
-        const results = [];
-        for (let i = 0; i < steps; i++) {
-            results.push(this.optimizeStep());
-        }
-        this.episodes++;
-        return results;
-    }
-    /**
-     * Provide external feedback (e.g., from task completion)
-     */
-    provideFeedback(reward) {
-        if (this.prevState === null || this.lastAction === null)
-            return;
-        const state = this.extractFeatures();
-        const experience = {
-            state: this.prevState,
-            actionIdx: actionToIndex(this.lastAction),
-            reward,
-            nextState: state,
-            done: false,
-            tdError: Math.abs(reward),
-            timestamp: Date.now(),
-        };
-        this.replayBuffer.push(experience);
-        // Immediate learning from feedback
-        const currentValue = this.valueNetwork.estimate(this.prevState);
-        const nextValue = this.targetNetwork.estimate(state);
-        const tdError = reward + this.config.gamma * nextValue - currentValue;
-        this.valueNetwork.update(this.prevState, tdError, this.config.learningRate);
-    }
-    /**
-     * Get skip regions (low activity areas)
-     */
-    getSkipRegions() {
-        return this.topology.agents
-            .filter((agent) => {
-            const degree = this.topology.connections.filter((c) => c.from === agent.id || c.to === agent.id).length;
-            return degree < 2;
-        })
-            .map((agent) => agent.id);
-    }
-    /**
-     * Get optimization statistics
-     */
-    getStats() {
-        const avgReward = this.totalSteps > 0 ? this.cumulativeReward / this.totalSteps : 0;
-        const avgTdError = this.rewardHistory.length > 0
-            ? this.rewardHistory.reduce((sum, r) => sum + Math.abs(r), 0) /
-                this.rewardHistory.length
-            : 0;
-        return {
-            totalSteps: this.totalSteps,
-            episodes: this.episodes,
-            cumulativeReward: this.cumulativeReward,
-            avgReward,
-            avgTdError,
-            actionCounts: { ...this.actionCounts },
-            minCutHistory: [...this.minCutHistory],
-            rewardHistory: [...this.rewardHistory],
-            currentEpsilon: this.epsilon,
-        };
-    }
-    /**
-     * Reset optimizer state
-     */
-    reset() {
-        this.prevState = this.extractFeatures();
-        this.prevMinCut = this.estimateMinCut();
-        this.time = 0;
-        this.epsilon = this.config.epsilon;
-        this.lastAction = null;
-    }
-    /**
-     * Hard reset (clear learning)
-     */
-    hardReset() {
-        this.reset();
-        this.replayBuffer.clear();
-        this.totalSteps = 0;
-        this.episodes = 0;
-        this.cumulativeReward = 0;
-        this.minCutHistory = [];
-        this.rewardHistory = [];
-        for (const actionType of ACTION_TYPES) {
-            this.actionCounts[actionType] = 0;
-        }
-        // Reinitialize networks
-        this.valueNetwork = new ValueNetwork(this.config.inputSize, this.config.hiddenSize);
-        this.targetNetwork = new ValueNetwork(this.config.inputSize, this.config.hiddenSize);
-        this.targetNetwork.copyFrom(this.valueNetwork);
-    }
-    /**
-     * Export learned model
-     */
-    exportModel() {
-        return {
-            type: 'neural-topology-optimizer',
-            version: '1.0.0',
-            config: { ...this.config },
-            valueNetwork: this.valueNetwork.export(),
-            targetNetwork: this.targetNetwork.export(),
-            stats: this.getStats(),
-            exportedAt: new Date().toISOString(),
-        };
-    }
-    /**
-     * Import learned model
-     */
-    importModel(model) {
-        if (model.type !== 'neural-topology-optimizer') {
-            throw new Error(`Invalid model type: ${model.type}`);
-        }
-        // Import config
-        this.config = { ...DEFAULT_OPTIMIZER_CONFIG, ...model.config };
-        this.epsilon = model.stats.currentEpsilon;
-        // Import networks
-        this.valueNetwork = new ValueNetwork(this.config.inputSize, this.config.hiddenSize);
-        this.valueNetwork.import(model.valueNetwork);
-        if (model.targetNetwork) {
-            this.targetNetwork = new ValueNetwork(this.config.inputSize, this.config.hiddenSize);
-            this.targetNetwork.import(model.targetNetwork);
-        }
-        else {
-            this.targetNetwork.copyFrom(this.valueNetwork);
-        }
-        // Import stats
-        this.totalSteps = model.stats.totalSteps;
-        this.episodes = model.stats.episodes;
-        this.cumulativeReward = model.stats.cumulativeReward;
-        this.actionCounts = { ...model.stats.actionCounts };
-        this.minCutHistory = [...model.stats.minCutHistory];
-        this.rewardHistory = [...model.stats.rewardHistory];
-    }
-    // ============================================================================
-    // Private Methods
-    // ============================================================================
-    /**
-     * Extract features from topology for state representation
-     */
-    extractFeatures() {
-        const n = this.topology.agents.length;
-        const m = this.topology.connections.length;
-        const loadStats = this.getLoadStats();
-        const state = {
-            agentCount: n,
-            connectionCount: m,
-            density: this.getDensity(),
-            avgDegree: this.getAverageDegree(),
-            minDegree: this.getMinDegree(),
-            avgWeight: this.getAverageWeight(),
-            weightVariance: this.getWeightVariance(),
-            avgLoad: loadStats.avg,
-            loadVariance: loadStats.variance,
-            avgLatency: this.getAverageLatency(),
-            idleAgents: loadStats.idle,
-            overloadedAgents: loadStats.overloaded,
-            clusteringCoefficient: this.getClusteringCoefficient(),
-            time: this.time,
-            extra: [],
-        };
-        // Convert to feature vector (normalized)
-        const features = new Array(this.config.inputSize).fill(0);
-        if (this.config.inputSize > 0)
-            features[0] = n / 100;
-        if (this.config.inputSize > 1)
-            features[1] = m / 500;
-        if (this.config.inputSize > 2)
-            features[2] = state.density;
-        if (this.config.inputSize > 3)
-            features[3] = state.avgDegree / 10;
-        if (this.config.inputSize > 4)
-            features[4] = state.minDegree / 10;
-        if (this.config.inputSize > 5)
-            features[5] = state.avgWeight / 5;
-        if (this.config.inputSize > 6)
-            features[6] = Math.min(1, state.weightVariance);
-        if (this.config.inputSize > 7)
-            features[7] = state.avgLoad;
-        if (this.config.inputSize > 8)
-            features[8] = Math.min(1, state.loadVariance);
-        if (this.config.inputSize > 9)
-            features[9] = state.avgLatency / 100;
-        if (this.config.inputSize > 10)
-            features[10] = state.idleAgents / Math.max(1, n);
-        if (this.config.inputSize > 11)
-            features[11] = state.overloadedAgents / Math.max(1, n);
-        if (this.config.inputSize > 12)
-            features[12] = state.clusteringCoefficient;
-        if (this.config.inputSize > 13)
-            features[13] = Math.sin(state.time * 0.1);
-        if (this.config.inputSize > 14)
-            features[14] = Math.cos(state.time * 0.1);
-        if (this.config.inputSize > 15)
-            features[15] = this.prevMinCut / Math.max(1, m);
-        return features;
-    }
-    /**
-     * Select action using epsilon-greedy policy
-     */
-    selectAction(state) {
-        // Exploration: random action
-        if (secureRandom() < this.epsilon) {
-            return this.randomAction();
-        }
-        // Exploitation: select action with highest estimated value
-        let bestAction = { type: 'no_op' };
-        let bestValue = -Infinity;
-        for (let i = 0; i < this.config.numActions; i++) {
-            const action = this.indexToAction(i);
-            const hypotheticalState = this.simulateAction(state, action);
-            const value = this.valueNetwork.estimate(hypotheticalState);
-            if (value > bestValue) {
-                bestValue = value;
-                bestAction = action;
-            }
-        }
-        return bestAction;
-    }
-    /**
-     * Convert index to action
-     */
-    indexToAction(idx) {
-        const agents = this.topology.agents;
-        if (agents.length < 2)
-            return { type: 'no_op' };
-        const actionType = indexToActionType(idx);
-        const from = agents[idx % agents.length].id;
-        const to = agents[(idx + 1) % agents.length].id;
-        switch (actionType) {
-            case 'add_connection':
-                if (!this.hasConnection(from, to)) {
-                    return { type: 'add_connection', from, to, weight: 1.0 };
-                }
-                return { type: 'no_op' };
-            case 'remove_connection':
-                if (this.hasConnection(from, to)) {
-                    return { type: 'remove_connection', from, to };
-                }
-                return { type: 'no_op' };
-            case 'strengthen_connection':
-                return { type: 'strengthen_connection', from, to, delta: 0.1 };
-            case 'weaken_connection':
-                return { type: 'weaken_connection', from, to, delta: 0.1 };
-            default:
-                return { type: 'no_op' };
-        }
-    }
-    /**
-     * Generate random action
-     */
-    randomAction() {
-        const agents = this.topology.agents;
-        if (agents.length < 2)
-            return { type: 'no_op' };
-        const actionTypeIdx = secureRandomInt(0, ACTION_TYPES.length);
-        const actionType = ACTION_TYPES[actionTypeIdx];
-        const fromIdx = secureRandomInt(0, agents.length);
-        let toIdx = secureRandomInt(0, agents.length);
-        while (toIdx === fromIdx && agents.length > 1) {
-            toIdx = secureRandomInt(0, agents.length);
-        }
-        const from = agents[fromIdx].id;
-        const to = agents[toIdx].id;
-        switch (actionType) {
-            case 'add_connection':
-                if (!this.hasConnection(from, to)) {
-                    return { type: 'add_connection', from, to, weight: 1.0 };
-                }
-                return { type: 'no_op' };
-            case 'remove_connection':
-                if (this.hasConnection(from, to)) {
-                    return { type: 'remove_connection', from, to };
-                }
-                return { type: 'no_op' };
-            case 'strengthen_connection':
-                return { type: 'strengthen_connection', from, to, delta: 0.1 };
-            case 'weaken_connection':
-                return { type: 'weaken_connection', from, to, delta: 0.1 };
-            default:
-                return { type: 'no_op' };
-        }
-    }
-    /**
-     * Simulate action effect on state (for lookahead)
-     */
-    simulateAction(state, action) {
-        const newState = [...state];
-        switch (action.type) {
-            case 'add_connection':
-                if (this.config.inputSize > 1)
-                    newState[1] += 0.002; // connection count
-                if (this.config.inputSize > 2)
-                    newState[2] += 0.01; // density
-                if (this.config.inputSize > 3)
-                    newState[3] += 0.1; // avg degree
-                break;
-            case 'remove_connection':
-                if (this.config.inputSize > 1)
-                    newState[1] -= 0.002;
-                if (this.config.inputSize > 2)
-                    newState[2] -= 0.01;
-                if (this.config.inputSize > 3)
-                    newState[3] -= 0.1;
-                break;
-            case 'strengthen_connection':
-                if (this.config.inputSize > 5)
-                    newState[5] += 0.02; // avg weight
-                break;
-            case 'weaken_connection':
-                if (this.config.inputSize > 5)
-                    newState[5] -= 0.02;
-                break;
-        }
-        return newState;
-    }
-    /**
-     * Apply action to topology
-     */
-    applyAction(action) {
-        switch (action.type) {
-            case 'add_connection':
-                this.topology.addConnection(action.from, action.to, action.weight || 1.0);
-                break;
-            case 'remove_connection':
-                this.topology.removeConnection(action.from, action.to);
-                break;
-            case 'strengthen_connection':
-                this.topology.updateConnectionWeight(action.from, action.to, action.delta);
-                break;
-            case 'weaken_connection':
-                this.topology.updateConnectionWeight(action.from, action.to, -action.delta);
-                break;
-            case 'no_op':
-                // Do nothing
-                break;
-        }
-    }
-    /**
-     * Calculate multi-objective reward
-     */
-    calculateReward(oldMinCut, newMinCut) {
-        // 1. Min-cut improvement (primary objective)
-        const minCutReward = oldMinCut > 0 ? (newMinCut - oldMinCut) / oldMinCut : 0;
-        // 2. Communication efficiency
-        const efficiency = this.measureCommunicationEfficiency();
-        // 3. Load balance
-        const loadStats = this.getLoadStats();
-        const loadBalance = 1 - loadStats.variance;
-        // 4. Latency penalty
-        const avgLatency = this.getAverageLatency();
-        const latencyPenalty = avgLatency > 0 ? -avgLatency / 100 : 0;
-        // Combine with weights
-        const reward = minCutReward +
-            this.config.efficiencyWeight * efficiency +
-            this.config.loadBalanceWeight * loadBalance +
-            this.config.latencyWeight * latencyPenalty;
-        return Math.max(-1, Math.min(1, reward));
-    }
-    /**
-     * Train from replay buffer
-     */
-    trainFromReplay() {
-        const { experiences, weights, indices } = this.replayBuffer.sampleWithWeights(this.config.batchSize);
-        const newPriorities = [];
-        for (let i = 0; i < experiences.length; i++) {
-            const exp = experiences[i];
-            const weight = weights[i];
-            // Calculate TD error
-            const currentValue = this.valueNetwork.estimate(exp.state);
-            const nextValue = exp.done
-                ? 0
-                : this.targetNetwork.estimate(exp.nextState);
-            const targetValue = exp.reward + this.config.gamma * nextValue;
-            const tdError = targetValue - currentValue;
-            // Weighted update
-            this.valueNetwork.update(exp.state, tdError * weight, this.config.learningRate);
-            newPriorities.push(Math.abs(tdError));
-        }
-        // Update priorities
-        this.replayBuffer.updatePriorities(indices, newPriorities);
-    }
-    // ============================================================================
-    // Topology Metric Helpers
-    // ============================================================================
-    estimateMinCut() {
-        if (this.topology.agents.length === 0)
-            return 0;
-        return this.getMinDegree();
-    }
-    hasConnection(from, to) {
-        return this.topology.connections.some((c) => (c.from === from && c.to === to) || (c.from === to && c.to === from));
-    }
-    getDensity() {
-        const n = this.topology.agents.length;
-        if (n < 2)
-            return 0;
-        const maxConnections = (n * (n - 1)) / 2;
-        return this.topology.connections.length / maxConnections;
-    }
-    getAverageDegree() {
-        const n = this.topology.agents.length;
-        if (n === 0)
-            return 0;
-        const totalDegree = this.topology.agents.reduce((sum, agent) => {
-            return (sum +
-                this.topology.connections.filter((c) => c.from === agent.id || c.to === agent.id).length);
-        }, 0);
-        return totalDegree / n;
-    }
-    getMinDegree() {
-        if (this.topology.agents.length === 0)
-            return 0;
-        let minDegree = Infinity;
-        for (const agent of this.topology.agents) {
-            const degree = this.topology.connections.filter((c) => c.from === agent.id || c.to === agent.id).length;
-            minDegree = Math.min(minDegree, degree);
-        }
-        return minDegree === Infinity ? 0 : minDegree;
-    }
-    getAverageWeight() {
-        if (this.topology.connections.length === 0)
-            return 0;
-        const total = this.topology.connections.reduce((sum, c) => sum + c.weight, 0);
-        return total / this.topology.connections.length;
-    }
-    getWeightVariance() {
-        if (this.topology.connections.length === 0)
-            return 0;
-        const avg = this.getAverageWeight();
-        const squaredDiffs = this.topology.connections.map((c) => (c.weight - avg) ** 2);
-        return squaredDiffs.reduce((sum, d) => sum + d, 0) / this.topology.connections.length;
-    }
-    getAverageLatency() {
-        const connectionsWithLatency = this.topology.connections.filter((c) => c.latencyMs !== undefined);
-        if (connectionsWithLatency.length === 0)
-            return 0;
-        const total = connectionsWithLatency.reduce((sum, c) => sum + (c.latencyMs || 0), 0);
-        return total / connectionsWithLatency.length;
-    }
-    measureCommunicationEfficiency() {
-        const n = this.topology.agents.length;
-        if (n < 2)
-            return 0;
-        const maxConnections = (n * (n - 1)) / 2;
-        return this.topology.connections.length / maxConnections;
-    }
-    getClusteringCoefficient() {
-        if (this.topology.agents.length < 3)
-            return 0;
-        let totalCoeff = 0;
-        for (const agent of this.topology.agents) {
-            const neighbors = this.getNeighbors(agent.id);
-            const k = neighbors.length;
-            if (k < 2)
-                continue;
-            let triangles = 0;
-            for (let i = 0; i < neighbors.length; i++) {
-                for (let j = i + 1; j < neighbors.length; j++) {
-                    if (this.hasConnection(neighbors[i], neighbors[j])) {
-                        triangles++;
-                    }
-                }
-            }
-            const possibleTriangles = (k * (k - 1)) / 2;
-            totalCoeff += triangles / possibleTriangles;
-        }
-        return totalCoeff / this.topology.agents.length;
-    }
-    getNeighbors(agentId) {
-        const neighbors = [];
-        for (const conn of this.topology.connections) {
-            if (conn.from === agentId)
-                neighbors.push(conn.to);
-            else if (conn.to === agentId)
-                neighbors.push(conn.from);
-        }
-        return neighbors;
-    }
-    getLoadStats() {
-        if (this.topology.agents.length === 0) {
-            return { avg: 0, variance: 0, idle: 0, overloaded: 0 };
-        }
-        const loads = this.topology.agents.map((a) => a.metrics?.currentLoad ?? 0);
-        const avg = loads.reduce((sum, l) => sum + l, 0) / loads.length;
-        const variance = loads.reduce((sum, l) => sum + (l - avg) ** 2, 0) / loads.length;
-        const idle = this.topology.agents.filter((a) => a.status === 'idle').length;
-        const overloaded = this.topology.agents.filter((a) => (a.metrics?.currentLoad ?? 0) > 0.8).length;
-        return { avg, variance, idle, overloaded };
-    }
-}
-// ============================================================================
-// Factory Function
-// ============================================================================
-/**
- * Create a neural topology optimizer
- */
-export function createNeuralTopologyOptimizer(topology, config) {
-    return new NeuralTopologyOptimizer(topology, config);
-}
-//# sourceMappingURL=topology-optimizer.js.map