agent-relay 1.1.0 → 1.2.3

package/dist/cloud/services/auto-scaler.js

@@ -0,0 +1,439 @@
+/**
+ * Auto-Scaler Service
+ *
+ * Monitors workspace metrics and automatically scales instances based on
+ * defined policies. Uses Redis pub/sub for cross-server coordination to
+ * ensure only one scaling operation happens at a time.
+ *
+ * Key responsibilities:
+ * - Subscribe to metrics updates from monitoring service
+ * - Evaluate scaling policies periodically
+ * - Coordinate scaling decisions across multiple cloud servers
+ * - Execute scaling actions via workspace provisioner
+ * - Track scaling history and pending operations
+ */
+import { EventEmitter } from 'events';
+import { createClient } from 'redis';
+import { getScalingPolicyService, } from './scaling-policy.js';
+const DEFAULT_CONFIG = {
+    enabled: true,
+    evaluationIntervalMs: 30000, // 30 seconds
+    lockTimeoutMs: 60000, // 1 minute
+    maxConcurrentOperations: 5,
+    redisKeyPrefix: 'autoscaler:',
+};
+// Redis pub/sub channels
+const CHANNELS = {
+    METRICS_UPDATE: 'autoscaler:metrics',
+    SCALING_REQUEST: 'autoscaler:scale',
+    SCALING_COMPLETE: 'autoscaler:complete',
+    LOCK_ACQUIRED: 'autoscaler:lock',
+};
+export class AutoScaler extends EventEmitter {
+    config;
+    policyService;
+    redis = null;
+    subscriber = null;
+    evaluationTimer = null;
+    pendingOperations = new Map();
+    metricsCache = new Map();
+    isLeader = false;
+    serverId;
+    lastScalingActions = new Map(); // userId -> lastAction
+    constructor(config = {}) {
+        super();
+        this.config = { ...DEFAULT_CONFIG, ...config };
+        this.policyService = getScalingPolicyService();
+        this.serverId = `server-${process.pid}-${Date.now()}`;
+    }
+    /**
+     * Initialize with Redis connection for cross-server coordination
+     */
+    async initialize(redisUrl) {
+        if (!this.config.enabled) {
+            this.emit('disabled');
+            return;
+        }
+        try {
+            // Main Redis client for commands
+            this.redis = createClient({ url: redisUrl });
+            this.redis.on('error', (err) => this.emit('error', { context: 'redis', error: err }));
+            // Separate client for subscriptions
+            this.subscriber = createClient({ url: redisUrl });
+            this.subscriber.on('error', (err) => this.emit('error', { context: 'subscriber', error: err }));
+            await Promise.all([this.redis.connect(), this.subscriber.connect()]);
+            // Set up pub/sub subscriptions
+            await this.setupSubscriptions();
+            // Start evaluation loop
+            this.startEvaluationLoop();
+            // Attempt to become leader
+            await this.attemptLeadership();
+            this.emit('initialized', { serverId: this.serverId, isLeader: this.isLeader });
+        }
+        catch (error) {
+            this.emit('error', error);
+            throw error;
+        }
+    }
+    /**
+     * Set up Redis pub/sub subscriptions
+     */
+    async setupSubscriptions() {
+        if (!this.subscriber)
+            return;
+        // Subscribe to all channels
+        await this.subscriber.subscribe(CHANNELS.METRICS_UPDATE, (message) => {
+            this.handleChannelMessage(CHANNELS.METRICS_UPDATE, message);
+        });
+        await this.subscriber.subscribe(CHANNELS.SCALING_REQUEST, (message) => {
+            this.handleChannelMessage(CHANNELS.SCALING_REQUEST, message);
+        });
+        await this.subscriber.subscribe(CHANNELS.SCALING_COMPLETE, (message) => {
+            this.handleChannelMessage(CHANNELS.SCALING_COMPLETE, message);
+        });
+        await this.subscriber.subscribe(CHANNELS.LOCK_ACQUIRED, (message) => {
+            this.handleChannelMessage(CHANNELS.LOCK_ACQUIRED, message);
+        });
+    }
+    /**
+     * Handle channel message
+     */
+    handleChannelMessage(channel, message) {
+        try {
+            const data = JSON.parse(message);
+            this.handlePubSubMessage(channel, data).catch((err) => {
+                this.emit('error', { context: 'message_handler', error: err });
+            });
+        }
+        catch (error) {
+            this.emit('error', { context: 'pubsub_parse', error });
+        }
+    }
+    /**
+     * Handle incoming pub/sub messages
+     */
+    async handlePubSubMessage(channel, data) {
+        switch (channel) {
+            case CHANNELS.METRICS_UPDATE:
+                await this.handleMetricsUpdate(data);
+                break;
+            case CHANNELS.SCALING_REQUEST:
+                await this.handleScalingRequest(data);
+                break;
+            case CHANNELS.SCALING_COMPLETE:
+                await this.handleScalingComplete(data);
+                break;
+            case CHANNELS.LOCK_ACQUIRED:
+                this.handleLeadershipChange(data);
+                break;
+        }
+    }
+    /**
+     * Handle metrics update from monitoring service
+     */
+    async handleMetricsUpdate(snapshot) {
+        this.metricsCache.set(snapshot.userId, snapshot);
+        this.emit('metrics_received', snapshot);
+        // If we're the leader, evaluate immediately for this user
+        if (this.isLeader) {
+            await this.evaluateUserScaling(snapshot.userId);
+        }
+    }
+    /**
+     * Handle scaling request (from any server)
+     */
+    async handleScalingRequest(operation) {
+        // Track the operation
+        this.pendingOperations.set(operation.id, operation);
+        this.emit('scaling_started', operation);
+    }
+    /**
+     * Handle scaling completion
+     */
+    async handleScalingComplete(operation) {
+        const pending = this.pendingOperations.get(operation.id);
+        if (pending) {
+            this.pendingOperations.delete(operation.id);
+            this.lastScalingActions.set(operation.userId, new Date());
+        }
+        this.emit('scaling_completed', operation);
+    }
+    /**
+     * Handle leadership change
+     */
+    handleLeadershipChange(data) {
+        if (data.serverId !== this.serverId) {
+            this.isLeader = false;
+            this.emit('leadership_lost');
+        }
+    }
+    /**
+     * Attempt to become the leader (only leader evaluates scaling)
+     */
+    async attemptLeadership() {
+        if (!this.redis)
+            return false;
+        const lockKey = `${this.config.redisKeyPrefix}leader`;
+        const result = await this.redis.set(lockKey, this.serverId, {
+            PX: this.config.lockTimeoutMs,
+            NX: true,
+        });
+        if (result === 'OK') {
+            this.isLeader = true;
+            await this.redis.publish(CHANNELS.LOCK_ACQUIRED, JSON.stringify({ serverId: this.serverId }));
+            this.emit('became_leader');
+            // Renew leadership periodically
+            this.scheduleLeadershipRenewal();
+            return true;
+        }
+        return false;
+    }
+    /**
+     * Schedule leadership lock renewal
+     */
+    scheduleLeadershipRenewal() {
+        const renewInterval = this.config.lockTimeoutMs / 2;
+        setInterval(async () => {
+            if (this.isLeader && this.redis) {
+                const lockKey = `${this.config.redisKeyPrefix}leader`;
+                const currentHolder = await this.redis.get(lockKey);
+                if (currentHolder === this.serverId) {
+                    await this.redis.pExpire(lockKey, this.config.lockTimeoutMs);
+                }
+                else {
+                    this.isLeader = false;
+                    this.emit('leadership_lost');
+                }
+            }
+        }, renewInterval);
+    }
+    /**
+     * Start the periodic evaluation loop
+     */
+    startEvaluationLoop() {
+        if (this.evaluationTimer) {
+            clearInterval(this.evaluationTimer);
+        }
+        this.evaluationTimer = setInterval(async () => {
+            if (this.isLeader) {
+                await this.evaluateAllUsers();
+            }
+            else {
+                // Try to become leader if current leader is gone
+                await this.attemptLeadership();
+            }
+        }, this.config.evaluationIntervalMs);
+    }
+    /**
+     * Evaluate scaling for all cached users
+     */
+    async evaluateAllUsers() {
+        const evaluations = [];
+        for (const userId of this.metricsCache.keys()) {
+            evaluations.push(this.evaluateUserScaling(userId));
+        }
+        await Promise.allSettled(evaluations);
+    }
+    /**
+     * Evaluate scaling for a specific user
+     */
+    async evaluateUserScaling(userId) {
+        const snapshot = this.metricsCache.get(userId);
+        if (!snapshot)
+            return;
+        // Check if we have too many pending operations
+        const userPendingOps = Array.from(this.pendingOperations.values()).filter((op) => op.userId === userId && op.status === 'in_progress').length;
+        if (userPendingOps >= this.config.maxConcurrentOperations) {
+            return;
+        }
+        // Build context for policy evaluation
+        const context = await this.buildUserContext(userId, snapshot);
+        if (!context)
+            return;
+        // Evaluate policies
+        const decision = this.policyService.evaluate(context);
+        if (decision.shouldScale && decision.action) {
+            await this.requestScaling(userId, decision);
+        }
+        this.emit('evaluation_complete', { userId, decision });
+    }
+    /**
+     * Build user context for policy evaluation
+     */
+    async buildUserContext(userId, snapshot) {
+        if (!this.redis)
+            return null;
+        // Get user plan from Redis cache or database
+        const userPlanKey = `${this.config.redisKeyPrefix}user:${userId}:plan`;
+        let plan = (await this.redis.get(userPlanKey));
+        if (!plan) {
+            plan = 'free'; // Default, should be fetched from database
+        }
+        const maxWorkspaces = this.policyService.getMaxWorkspaces(plan);
+        const lastScalingAction = this.lastScalingActions.get(userId);
+        return {
+            userId,
+            plan,
+            currentWorkspaceCount: snapshot.workspaces.length,
+            maxWorkspaces,
+            workspaceMetrics: snapshot.workspaces,
+            lastScalingAction,
+        };
+    }
+    /**
+     * Request a scaling operation
+     */
+    async requestScaling(userId, decision) {
+        if (!this.redis || !decision.action)
+            return;
+        const operation = {
+            id: `scale-${userId}-${Date.now()}`,
+            userId,
+            action: decision.action.type,
+            targetWorkspaceId: decision.action.targetWorkspaceId,
+            targetResourceTier: decision.action.resourceTier,
+            targetAgentLimit: decision.action.newAgentLimit,
+            status: 'pending',
+            startedAt: new Date(),
+            triggeredBy: decision.triggeredPolicy || 'manual',
+            metrics: decision.metrics,
+        };
+        // Acquire distributed lock for this user's scaling
+        const lockKey = `${this.config.redisKeyPrefix}scaling:${userId}`;
+        const lockAcquired = await this.redis.set(lockKey, operation.id, {
+            PX: 60000,
+            NX: true,
+        });
+        if (lockAcquired !== 'OK') {
+            // Another scaling operation is in progress
+            this.emit('scaling_skipped', { reason: 'lock_held', userId });
+            return;
+        }
+        try {
+            // Publish scaling request
+            await this.redis.publish(CHANNELS.SCALING_REQUEST, JSON.stringify(operation));
+            // Execute the scaling operation
+            operation.status = 'in_progress';
+            await this.executeScaling(operation, decision);
+        }
+        catch (error) {
+            operation.status = 'failed';
+            operation.error = error instanceof Error ? error.message : 'Unknown error';
+            this.emit('scaling_error', { operation, error });
+        }
+        finally {
+            // Release lock
+            await this.redis.del(lockKey);
+            // Publish completion
+            operation.completedAt = new Date();
+            await this.redis.publish(CHANNELS.SCALING_COMPLETE, JSON.stringify(operation));
+        }
+    }
+    /**
+     * Execute the actual scaling operation
+     */
+    async executeScaling(operation, decision) {
+        // This will be integrated with the workspace provisioner
+        // For now, emit event for external handling
+        this.emit('execute_scaling', { operation, decision });
+        // The actual implementation would:
+        // 1. Call workspaceProvisioner.provisionWorkspace() for scale_up
+        // 2. Call workspaceProvisioner.terminateWorkspace() for scale_down
+        // 3. Call coordinator.rebalanceAgents() for rebalance
+        operation.status = 'completed';
+        this.emit('scaling_executed', operation);
+    }
+    /**
+     * Report metrics from monitoring service
+     */
+    async reportMetrics(userId, workspaces) {
+        if (!this.redis)
+            return;
+        const snapshot = {
+            userId,
+            workspaces,
+            timestamp: new Date(),
+        };
+        // Cache locally
+        this.metricsCache.set(userId, snapshot);
+        // Publish to all servers
+        await this.redis.publish(CHANNELS.METRICS_UPDATE, JSON.stringify(snapshot));
+    }
+    /**
+     * Manually trigger scaling evaluation for a user
+     */
+    async triggerEvaluation(userId) {
+        const snapshot = this.metricsCache.get(userId);
+        if (!snapshot)
+            return null;
+        const context = await this.buildUserContext(userId, snapshot);
+        if (!context)
+            return null;
+        return this.policyService.evaluate(context);
+    }
+    /**
+     * Get current scaling status
+     */
+    getStatus() {
+        return {
+            enabled: this.config.enabled,
+            isLeader: this.isLeader,
+            serverId: this.serverId,
+            pendingOperations: this.pendingOperations.size,
+            cachedUsers: this.metricsCache.size,
+        };
+    }
+    /**
+     * Get pending operations for a user
+     */
+    getPendingOperations(userId) {
+        const ops = Array.from(this.pendingOperations.values());
+        return userId ? ops.filter((op) => op.userId === userId) : ops;
+    }
+    /**
+     * Update user plan in cache
+     */
+    async setUserPlan(userId, plan) {
+        if (!this.redis)
+            return;
+        const key = `${this.config.redisKeyPrefix}user:${userId}:plan`;
+        await this.redis.set(key, plan, { EX: 3600 }); // 1 hour TTL
+    }
+    /**
+     * Clean shutdown
+     */
+    async shutdown() {
+        if (this.evaluationTimer) {
+            clearInterval(this.evaluationTimer);
+            this.evaluationTimer = null;
+        }
+        if (this.subscriber) {
+            await this.subscriber.unsubscribe();
+            await this.subscriber.quit();
+            this.subscriber = null;
+        }
+        if (this.redis) {
+            // Release leadership if we have it
+            if (this.isLeader) {
+                const lockKey = `${this.config.redisKeyPrefix}leader`;
+                await this.redis.del(lockKey);
+            }
+            await this.redis.quit();
+            this.redis = null;
+        }
+        this.emit('shutdown');
+    }
+}
+// Singleton instance
+let _autoScaler = null;
+export function getAutoScaler() {
+    if (!_autoScaler) {
+        _autoScaler = new AutoScaler();
+    }
+    return _autoScaler;
+}
+export function createAutoScaler(config = {}) {
+    _autoScaler = new AutoScaler(config);
+    return _autoScaler;
+}
+//# sourceMappingURL=auto-scaler.js.map

package/dist/cloud/services/capacity-manager.d.ts

@@ -0,0 +1,148 @@
+/**
+ * Capacity Manager
+ *
+ * Tracks workspace capacity across the fleet and provides:
+ * - Real-time capacity metrics
+ * - Optimal agent placement recommendations
+ * - Load balancing decisions
+ * - Capacity forecasting based on trends
+ *
+ * Works with AutoScaler to determine when to provision new instances
+ * and with Coordinator to place agents optimally.
+ */
+import { EventEmitter } from 'events';
+import { WorkspaceMetrics } from './scaling-policy.js';
+export interface WorkspaceCapacity {
+    workspaceId: string;
+    userId: string;
+    provider: string;
+    region: string;
+    currentAgents: number;
+    maxAgents: number;
+    memoryUsedBytes: number;
+    memoryLimitBytes: number;
+    cpuPercent: number;
+    agentCapacityPercent: number;
+    memoryCapacityPercent: number;
+    overallHealthScore: number;
+    lastHeartbeat: Date;
+    lastMetricsUpdate: Date;
+}
+export interface PlacementRecommendation {
+    workspaceId: string;
+    score: number;
+    reason: string;
+    estimatedCapacityAfter: number;
+}
+export interface CapacitySnapshot {
+    userId: string;
+    totalWorkspaces: number;
+    totalAgents: number;
+    totalMaxAgents: number;
+    totalMemoryBytes: number;
+    totalMemoryLimitBytes: number;
+    averageHealthScore: number;
+    workspaces: WorkspaceCapacity[];
+    timestamp: Date;
+}
+export interface CapacityForecast {
+    userId: string;
+    currentAgents: number;
+    projectedAgents15Min: number;
+    projectedAgents60Min: number;
+    memoryTrendPerMinute: number;
+    willExceedCapacity: boolean;
+    timeToCapacityExhaustion?: number;
+    recommendation: 'none' | 'scale_soon' | 'scale_now' | 'critical';
+}
+export interface CapacityManagerConfig {
+    healthCheckIntervalMs: number;
+    staleThresholdMs: number;
+    memoryWeightFactor: number;
+    agentWeightFactor: number;
+    cpuWeightFactor: number;
+    redisKeyPrefix: string;
+}
+export declare class CapacityManager extends EventEmitter {
+    private config;
+    private redis;
+    private subscriber;
+    private capacityMap;
+    private userWorkspaces;
+    private trendHistory;
+    private healthCheckTimer;
+    constructor(config?: Partial<CapacityManagerConfig>);
+    /**
+     * Initialize with Redis for cross-server sync
+     */
+    initialize(redisUrl: string): Promise<void>;
+    /**
+     * Load capacity data from Redis
+     */
+    private loadFromRedis;
+    /**
+     * Update local capacity map
+     */
+    private updateLocalCapacity;
+    /**
+     * Update trend history for forecasting
+     */
+    private updateTrendHistory;
+    /**
+     * Report capacity from a workspace
+     */
+    reportCapacity(workspaceId: string, userId: string, metrics: Partial<WorkspaceCapacity>): Promise<void>;
+    /**
+     * Calculate health score for a workspace (lower is healthier/better for placement)
+     */
+    private calculateHealthScore;
+    /**
+     * Get best workspace for placing a new agent
+     */
+    recommendPlacement(userId: string, agentCount?: number): PlacementRecommendation[];
+    /**
+     * Generate human-readable placement reason
+     */
+    private getPlacementReason;
+    /**
+     * Get capacity snapshot for a user
+     */
+    getCapacitySnapshot(userId: string): CapacitySnapshot | null;
+    /**
+     * Forecast capacity needs based on trends
+     */
+    getCapacityForecast(userId: string): CapacityForecast | null;
+    /**
+     * Convert workspace metrics to capacity format
+     */
+    fromWorkspaceMetrics(userId: string, metrics: WorkspaceMetrics): Partial<WorkspaceCapacity>;
+    /**
+     * Health check loop - detect stale workspaces
+     */
+    private startHealthCheckLoop;
+    /**
+     * Remove a workspace from tracking
+     */
+    removeWorkspace(workspaceId: string): Promise<void>;
+    /**
+     * Get all workspaces for a user
+     */
+    getUserWorkspaces(userId: string): WorkspaceCapacity[];
+    /**
+     * Get global capacity metrics
+     */
+    getGlobalMetrics(): {
+        totalWorkspaces: number;
+        totalAgents: number;
+        totalMaxAgents: number;
+        averageUtilization: number;
+        staleWorkspaces: number;
+    };
+    /**
+     * Clean shutdown
+     */
+    shutdown(): Promise<void>;
+}
+export declare function getCapacityManager(): CapacityManager;
+export declare function createCapacityManager(config?: Partial<CapacityManagerConfig>): CapacityManager;
+//# sourceMappingURL=capacity-manager.d.ts.map