claude-flow 1.0.0

package/src/coordination/resources.ts

@@ -0,0 +1,322 @@
+/**
+ * Resource manager for preventing conflicts and deadlocks
+ */
+
+import { Resource, CoordinationConfig, SystemEvents } from '../utils/types.ts';
+import { IEventBus } from '../core/event-bus.ts';
+import { ILogger } from '../core/logger.ts';
+import { ResourceLockError } from '../utils/errors.ts';
+import { delay, timeout } from '../utils/helpers.ts';
+
+interface LockRequest {
+  agentId: string;
+  resourceId: string;
+  timestamp: Date;
+  priority: number;
+}
+
+/**
+ * Resource manager implementation
+ */
+export class ResourceManager {
+  private resources = new Map<string, Resource>();
+  private locks = new Map<string, string>(); // resourceId -> agentId
+  private waitQueue = new Map<string, LockRequest[]>(); // resourceId -> queue
+  private agentResources = new Map<string, Set<string>>(); // agentId -> resourceIds
+
+  constructor(
+    private config: CoordinationConfig,
+    private eventBus: IEventBus,
+    private logger: ILogger,
+  ) {}
+
+  async initialize(): Promise<void> {
+    this.logger.info('Initializing resource manager');
+    
+    // Set up periodic cleanup
+    setInterval(() => this.cleanup(), 30000); // Every 30 seconds
+  }
+
+  async shutdown(): Promise<void> {
+    this.logger.info('Shutting down resource manager');
+    
+    // Release all locks
+    for (const [resourceId, agentId] of this.locks) {
+      await this.release(resourceId, agentId);
+    }
+    
+    this.resources.clear();
+    this.locks.clear();
+    this.waitQueue.clear();
+    this.agentResources.clear();
+  }
+
+  async acquire(resourceId: string, agentId: string, priority = 0): Promise<void> {
+    this.logger.debug('Resource acquisition requested', { resourceId, agentId });
+
+    // Check if resource exists
+    if (!this.resources.has(resourceId)) {
+      this.resources.set(resourceId, {
+        id: resourceId,
+        type: 'generic',
+        locked: false,
+      });
+    }
+
+    const resource = this.resources.get(resourceId)!;
+
+    // Check if already locked by this agent
+    if (this.locks.get(resourceId) === agentId) {
+      this.logger.debug('Resource already locked by agent', { resourceId, agentId });
+      return;
+    }
+
+    // Try to acquire lock
+    if (!resource.locked) {
+      await this.lockResource(resourceId, agentId);
+      return;
+    }
+
+    // Add to wait queue
+    const request: LockRequest = {
+      agentId,
+      resourceId,
+      timestamp: new Date(),
+      priority,
+    };
+
+    if (!this.waitQueue.has(resourceId)) {
+      this.waitQueue.set(resourceId, []);
+    }
+
+    const queue = this.waitQueue.get(resourceId)!;
+    queue.push(request);
+    
+    // Sort by priority and timestamp
+    queue.sort((a, b) => {
+      if (a.priority !== b.priority) {
+        return b.priority - a.priority; // Higher priority first
+      }
+      return a.timestamp.getTime() - b.timestamp.getTime(); // Earlier first
+    });
+
+    this.logger.info('Agent added to resource wait queue', { 
+      resourceId,
+      agentId,
+      queueLength: queue.length,
+    });
+
+    // Wait for resource with timeout
+    const startTime = Date.now();
+    while (Date.now() - startTime < this.config.resourceTimeout) {
+      // Check if we're next in queue and resource is available
+      const nextRequest = queue[0];
+      if (nextRequest?.agentId === agentId && !resource.locked) {
+        // Remove from queue and acquire
+        queue.shift();
+        await this.lockResource(resourceId, agentId);
+        return;
+      }
+
+      // Check if our request is still in queue
+      const ourRequest = queue.find(req => req.agentId === agentId);
+      if (!ourRequest) {
+        // Request was removed (possibly by cleanup)
+        throw new ResourceLockError('Resource request cancelled');
+      }
+
+      await delay(100);
+    }
+
+    // Timeout - remove from queue
+    const index = queue.findIndex(req => req.agentId === agentId);
+    if (index !== -1) {
+      queue.splice(index, 1);
+    }
+
+    throw new ResourceLockError(
+      `Resource acquisition timeout for ${resourceId}`,
+      { resourceId, agentId, timeout: this.config.resourceTimeout },
+    );
+  }
+
+  async release(resourceId: string, agentId: string): Promise<void> {
+    this.logger.debug('Resource release requested', { resourceId, agentId });
+
+    const currentLock = this.locks.get(resourceId);
+    if (currentLock !== agentId) {
+      this.logger.warn('Attempted to release unowned resource', { 
+        resourceId,
+        agentId,
+        currentLock,
+      });
+      return;
+    }
+
+    // Release the lock
+    this.unlockResource(resourceId, agentId);
+
+    // Process wait queue
+    const queue = this.waitQueue.get(resourceId);
+    if (queue && queue.length > 0) {
+      const nextRequest = queue.shift()!;
+      
+      // Grant lock to next in queue
+      await this.lockResource(resourceId, nextRequest.agentId);
+    }
+  }
+
+  async releaseAllForAgent(agentId: string): Promise<void> {
+    const resources = this.agentResources.get(agentId);
+    if (!resources) {
+      return;
+    }
+
+    this.logger.info('Releasing all resources for agent', { 
+      agentId,
+      resourceCount: resources.size,
+    });
+
+    const promises = Array.from(resources).map(
+      resourceId => this.release(resourceId, agentId),
+    );
+
+    await Promise.all(promises);
+    this.agentResources.delete(agentId);
+  }
+
+  getAllocations(): Map<string, string> {
+    return new Map(this.locks);
+  }
+
+  getWaitingRequests(): Map<string, string[]> {
+    const waiting = new Map<string, string[]>();
+    
+    for (const [resourceId, queue] of this.waitQueue) {
+      if (queue.length > 0) {
+        waiting.set(
+          queue[0].agentId,
+          [...(waiting.get(queue[0].agentId) || []), resourceId],
+        );
+      }
+    }
+    
+    return waiting;
+  }
+
+  async getHealthStatus(): Promise<{ 
+    healthy: boolean; 
+    error?: string; 
+    metrics?: Record<string, number>;
+  }> {
+    const totalResources = this.resources.size;
+    const lockedResources = this.locks.size;
+    const waitingAgents = new Set<string>();
+    let totalWaiting = 0;
+
+    for (const queue of this.waitQueue.values()) {
+      totalWaiting += queue.length;
+      queue.forEach(req => waitingAgents.add(req.agentId));
+    }
+
+    return {
+      healthy: true,
+      metrics: {
+        totalResources,
+        lockedResources,
+        freeResources: totalResources - lockedResources,
+        waitingAgents: waitingAgents.size,
+        totalWaitingRequests: totalWaiting,
+      },
+    };
+  }
+
+  private async lockResource(resourceId: string, agentId: string): Promise<void> {
+    const resource = this.resources.get(resourceId)!;
+    
+    resource.locked = true;
+    resource.lockedBy = agentId;
+    resource.lockedAt = new Date();
+    
+    this.locks.set(resourceId, agentId);
+    
+    // Track agent resources
+    if (!this.agentResources.has(agentId)) {
+      this.agentResources.set(agentId, new Set());
+    }
+    this.agentResources.get(agentId)!.add(resourceId);
+
+    this.logger.info('Resource locked', { resourceId, agentId });
+
+    // Emit event
+    this.eventBus.emit(SystemEvents.RESOURCE_ACQUIRED, { resourceId, agentId });
+  }
+
+  private unlockResource(resourceId: string, agentId: string): void {
+    const resource = this.resources.get(resourceId);
+    if (!resource) {
+      return;
+    }
+
+    resource.locked = false;
+    delete resource.lockedBy;
+    delete resource.lockedAt;
+    
+    this.locks.delete(resourceId);
+    
+    // Remove from agent resources
+    this.agentResources.get(agentId)?.delete(resourceId);
+
+    this.logger.info('Resource unlocked', { resourceId, agentId });
+
+    // Emit event
+    this.eventBus.emit(SystemEvents.RESOURCE_RELEASED, { resourceId, agentId });
+  }
+
+  async performMaintenance(): Promise<void> {
+    this.logger.debug('Performing resource manager maintenance');
+    this.cleanup();
+  }
+
+  private cleanup(): void {
+    const now = Date.now();
+
+    // Clean up stale wait requests
+    for (const [resourceId, queue] of this.waitQueue) {
+      const filtered = queue.filter(req => {
+        const age = now - req.timestamp.getTime();
+        if (age > this.config.resourceTimeout) {
+          this.logger.warn('Removing stale resource request', { 
+            resourceId,
+            agentId: req.agentId,
+            age,
+          });
+          return false;
+        }
+        return true;
+      });
+
+      if (filtered.length === 0) {
+        this.waitQueue.delete(resourceId);
+      } else {
+        this.waitQueue.set(resourceId, filtered);
+      }
+    }
+
+    // Clean up locks held too long
+    for (const [resourceId, agentId] of this.locks) {
+      const resource = this.resources.get(resourceId);
+      if (resource?.lockedAt) {
+        const lockAge = now - resource.lockedAt.getTime();
+        if (lockAge > this.config.resourceTimeout * 2) {
+          this.logger.warn('Force releasing stale lock', { 
+            resourceId,
+            agentId,
+            lockAge,
+          });
+          this.unlockResource(resourceId, agentId);
+        }
+      }
+    }
+  }
+}

package/src/coordination/scheduler.ts

@@ -0,0 +1,390 @@
+/**
+ * Task scheduler implementation
+ */
+
+import { Task, TaskStatus, CoordinationConfig, SystemEvents } from '../utils/types.ts';
+import { IEventBus } from '../core/event-bus.ts';
+import { ILogger } from '../core/logger.ts';
+import { TaskError, TaskTimeoutError, TaskDependencyError } from '../utils/errors.ts';
+import { delay } from '../utils/helpers.ts';
+
+interface ScheduledTask {
+  task: Task;
+  agentId: string;
+  attempts: number;
+  lastAttempt?: Date;
+  timeout?: number;
+}
+
+/**
+ * Task scheduler for managing task assignment and execution
+ */
+export class TaskScheduler {
+  protected tasks = new Map<string, ScheduledTask>();
+  protected agentTasks = new Map<string, Set<string>>(); // agentId -> taskIds
+  protected taskDependencies = new Map<string, Set<string>>(); // taskId -> dependent taskIds
+  protected completedTasks = new Set<string>();
+
+  constructor(
+    protected config: CoordinationConfig,
+    protected eventBus: IEventBus,
+    protected logger: ILogger,
+  ) {}
+
+  async initialize(): Promise<void> {
+    this.logger.info('Initializing task scheduler');
+    
+    // Set up periodic cleanup
+    setInterval(() => this.cleanup(), 60000); // Every minute
+  }
+
+  async shutdown(): Promise<void> {
+    this.logger.info('Shutting down task scheduler');
+    
+    // Cancel all active tasks
+    const taskIds = Array.from(this.tasks.keys());
+    await Promise.all(taskIds.map(id => this.cancelTask(id, 'Scheduler shutdown')));
+    
+    this.tasks.clear();
+    this.agentTasks.clear();
+    this.taskDependencies.clear();
+    this.completedTasks.clear();
+  }
+
+  async assignTask(task: Task, agentId: string): Promise<void> {
+    this.logger.info('Assigning task', { taskId: task.id, agentId });
+
+    // Check dependencies
+    if (task.dependencies.length > 0) {
+      const unmetDependencies = task.dependencies.filter(
+        depId => !this.completedTasks.has(depId),
+      );
+      
+      if (unmetDependencies.length > 0) {
+        throw new TaskDependencyError(task.id, unmetDependencies);
+      }
+    }
+
+    // Create scheduled task
+    const scheduledTask: ScheduledTask = {
+      task: { ...task, status: 'assigned', assignedAgent: agentId },
+      agentId,
+      attempts: 0,
+    };
+
+    // Store task
+    this.tasks.set(task.id, scheduledTask);
+
+    // Update agent tasks
+    if (!this.agentTasks.has(agentId)) {
+      this.agentTasks.set(agentId, new Set());
+    }
+    this.agentTasks.get(agentId)!.add(task.id);
+
+    // Update dependencies
+    for (const depId of task.dependencies) {
+      if (!this.taskDependencies.has(depId)) {
+        this.taskDependencies.set(depId, new Set());
+      }
+      this.taskDependencies.get(depId)!.add(task.id);
+    }
+
+    // Start task execution
+    this.startTask(task.id);
+  }
+
+  async completeTask(taskId: string, result: unknown): Promise<void> {
+    const scheduled = this.tasks.get(taskId);
+    if (!scheduled) {
+      throw new TaskError(`Task not found: ${taskId}`);
+    }
+
+    this.logger.info('Task completed', { taskId, agentId: scheduled.agentId });
+
+    // Update task status
+    scheduled.task.status = 'completed';
+    scheduled.task.output = result as Record<string, unknown>;
+    scheduled.task.completedAt = new Date();
+
+    // Clear timeout
+    if (scheduled.timeout) {
+      clearTimeout(scheduled.timeout);
+    }
+
+    // Remove from active tasks
+    this.tasks.delete(taskId);
+    this.agentTasks.get(scheduled.agentId)?.delete(taskId);
+    
+    // Add to completed tasks
+    this.completedTasks.add(taskId);
+
+    // Check and start dependent tasks
+    const dependents = this.taskDependencies.get(taskId);
+    if (dependents) {
+      for (const dependentId of dependents) {
+        const dependent = this.tasks.get(dependentId);
+        if (dependent && this.canStartTask(dependent.task)) {
+          this.startTask(dependentId);
+        }
+      }
+    }
+  }
+
+  async failTask(taskId: string, error: Error): Promise<void> {
+    const scheduled = this.tasks.get(taskId);
+    if (!scheduled) {
+      throw new TaskError(`Task not found: ${taskId}`);
+    }
+
+    this.logger.error('Task failed', { 
+      taskId, 
+      agentId: scheduled.agentId,
+      attempt: scheduled.attempts,
+      error,
+    });
+
+    // Clear timeout
+    if (scheduled.timeout) {
+      clearTimeout(scheduled.timeout);
+    }
+
+    scheduled.attempts++;
+    scheduled.lastAttempt = new Date();
+
+    // Check if we should retry
+    if (scheduled.attempts < this.config.maxRetries) {
+      this.logger.info('Retrying task', { 
+        taskId,
+        attempt: scheduled.attempts,
+        maxRetries: this.config.maxRetries,
+      });
+
+      // Schedule retry with exponential backoff
+      const retryDelay = this.config.retryDelay * Math.pow(2, scheduled.attempts - 1);
+      
+      setTimeout(() => {
+        this.startTask(taskId);
+      }, retryDelay);
+    } else {
+      // Max retries exceeded, mark as failed
+      scheduled.task.status = 'failed';
+      scheduled.task.error = error;
+      scheduled.task.completedAt = new Date();
+
+      // Remove from active tasks
+      this.tasks.delete(taskId);
+      this.agentTasks.get(scheduled.agentId)?.delete(taskId);
+
+      // Cancel dependent tasks
+      await this.cancelDependentTasks(taskId, 'Parent task failed');
+    }
+  }
+
+  async cancelTask(taskId: string, reason: string): Promise<void> {
+    const scheduled = this.tasks.get(taskId);
+    if (!scheduled) {
+      return; // Already cancelled or completed
+    }
+
+    this.logger.info('Cancelling task', { taskId, reason });
+
+    // Clear timeout
+    if (scheduled.timeout) {
+      clearTimeout(scheduled.timeout);
+    }
+
+    // Update task status
+    scheduled.task.status = 'cancelled';
+    scheduled.task.completedAt = new Date();
+
+    // Emit cancellation event
+    this.eventBus.emit(SystemEvents.TASK_CANCELLED, { taskId, reason });
+
+    // Remove from active tasks
+    this.tasks.delete(taskId);
+    this.agentTasks.get(scheduled.agentId)?.delete(taskId);
+
+    // Cancel dependent tasks
+    await this.cancelDependentTasks(taskId, 'Parent task cancelled');
+  }
+
+  async cancelAgentTasks(agentId: string): Promise<void> {
+    const taskIds = this.agentTasks.get(agentId);
+    if (!taskIds) {
+      return;
+    }
+
+    this.logger.info('Cancelling all tasks for agent', { 
+      agentId,
+      taskCount: taskIds.size,
+    });
+
+    const promises = Array.from(taskIds).map(
+      taskId => this.cancelTask(taskId, 'Agent terminated'),
+    );
+
+    await Promise.all(promises);
+    this.agentTasks.delete(agentId);
+  }
+
+  async rescheduleAgentTasks(agentId: string): Promise<void> {
+    const taskIds = this.agentTasks.get(agentId);
+    if (!taskIds || taskIds.size === 0) {
+      return;
+    }
+
+    this.logger.info('Rescheduling tasks for agent', { 
+      agentId,
+      taskCount: taskIds.size,
+    });
+
+    for (const taskId of taskIds) {
+      const scheduled = this.tasks.get(taskId);
+      if (scheduled && scheduled.task.status === 'running') {
+        // Reset task status
+        scheduled.task.status = 'queued';
+        scheduled.attempts = 0;
+        
+        // Re-emit task created event for reassignment
+        this.eventBus.emit(SystemEvents.TASK_CREATED, { 
+          task: scheduled.task,
+        });
+      }
+    }
+  }
+
+  getAgentTaskCount(agentId: string): number {
+    return this.agentTasks.get(agentId)?.size || 0;
+  }
+
+  async getHealthStatus(): Promise<{ 
+    healthy: boolean; 
+    error?: string; 
+    metrics?: Record<string, number>;
+  }> {
+    const activeTasks = this.tasks.size;
+    const completedTasks = this.completedTasks.size;
+    const agentsWithTasks = this.agentTasks.size;
+    
+    const tasksByStatus: Record<TaskStatus, number> = {
+      pending: 0,
+      queued: 0,
+      assigned: 0,
+      running: 0,
+      completed: completedTasks,
+      failed: 0,
+      cancelled: 0,
+    };
+
+    for (const scheduled of this.tasks.values()) {
+      tasksByStatus[scheduled.task.status]++;
+    }
+
+    return {
+      healthy: true,
+      metrics: {
+        activeTasks,
+        completedTasks,
+        agentsWithTasks,
+        ...tasksByStatus,
+      },
+    };
+  }
+
+  async getAgentTasks(agentId: string): Promise<Task[]> {
+    const taskIds = this.agentTasks.get(agentId);
+    if (!taskIds) {
+      return [];
+    }
+
+    const tasks: Task[] = [];
+    for (const taskId of taskIds) {
+      const scheduled = this.tasks.get(taskId);
+      if (scheduled) {
+        tasks.push(scheduled.task);
+      }
+    }
+
+    return tasks;
+  }
+
+  async performMaintenance(): Promise<void> {
+    this.logger.debug('Performing task scheduler maintenance');
+    
+    // Cleanup old completed tasks
+    this.cleanup();
+    
+    // Check for stuck tasks
+    const now = new Date();
+    for (const [taskId, scheduled] of this.tasks) {
+      if (scheduled.task.status === 'running' && scheduled.task.startedAt) {
+        const runtime = now.getTime() - scheduled.task.startedAt.getTime();
+        if (runtime > this.config.resourceTimeout * 2) {
+          this.logger.warn('Found stuck task', { 
+            taskId,
+            runtime,
+            agentId: scheduled.agentId,
+          });
+          
+          // Force fail the task
+          await this.failTask(taskId, new TaskTimeoutError(taskId, runtime));
+        }
+      }
+    }
+  }
+
+  private startTask(taskId: string): void {
+    const scheduled = this.tasks.get(taskId);
+    if (!scheduled) {
+      return;
+    }
+
+    // Update status
+    scheduled.task.status = 'running';
+    scheduled.task.startedAt = new Date();
+
+    // Emit task started event
+    this.eventBus.emit(SystemEvents.TASK_STARTED, { 
+      taskId,
+      agentId: scheduled.agentId,
+    });
+
+    // Set timeout for task execution
+    const timeoutMs = this.config.resourceTimeout;
+    scheduled.timeout = setTimeout(() => {
+      this.failTask(taskId, new TaskTimeoutError(taskId, timeoutMs));
+    }, timeoutMs);
+  }
+
+  private canStartTask(task: Task): boolean {
+    // Check if all dependencies are completed
+    return task.dependencies.every(depId => this.completedTasks.has(depId));
+  }
+
+  private async cancelDependentTasks(taskId: string, reason: string): Promise<void> {
+    const dependents = this.taskDependencies.get(taskId);
+    if (!dependents) {
+      return;
+    }
+
+    for (const dependentId of dependents) {
+      await this.cancelTask(dependentId, reason);
+    }
+  }
+
+  private cleanup(): void {
+    // Clean up old completed tasks (keep last 1000)
+    if (this.completedTasks.size > 1000) {
+      const toRemove = this.completedTasks.size - 1000;
+      const iterator = this.completedTasks.values();
+      
+      for (let i = 0; i < toRemove; i++) {
+        const result = iterator.next();
+        if (!result.done && result.value) {
+          this.completedTasks.delete(result.value);
+          this.taskDependencies.delete(result.value);
+        }
+      }
+    }
+  }
+}