claude-flow 3.0.0-alpha.10 → 3.0.0-alpha.12

package/src/services/worker-daemon.ts

@@ -1,726 +0,0 @@
-/**
- * Worker Daemon Service
- * Node.js-based background worker system that auto-runs like shell daemons
- *
- * Workers:
- * - map: Codebase mapping (5 min interval)
- * - audit: Security analysis (10 min interval)
- * - optimize: Performance optimization (15 min interval)
- * - consolidate: Memory consolidation (30 min interval)
- * - testgaps: Test coverage analysis (20 min interval)
- */
-
-import { EventEmitter } from 'events';
-import { existsSync, mkdirSync, writeFileSync, readFileSync } from 'fs';
-import { join } from 'path';
-
-// Worker types matching hooks-tools.ts
-export type WorkerType =
-  | 'ultralearn'
-  | 'optimize'
-  | 'consolidate'
-  | 'predict'
-  | 'audit'
-  | 'map'
-  | 'preload'
-  | 'deepdive'
-  | 'document'
-  | 'refactor'
-  | 'benchmark'
-  | 'testgaps';
-
-interface WorkerConfig {
-  type: WorkerType;
-  intervalMs: number;
-  priority: 'low' | 'normal' | 'high' | 'critical';
-  description: string;
-  enabled: boolean;
-}
-
-interface WorkerState {
-  lastRun?: Date;
-  nextRun?: Date;
-  runCount: number;
-  successCount: number;
-  failureCount: number;
-  averageDurationMs: number;
-  isRunning: boolean;
-}
-
-interface WorkerResult {
-  workerId: string;
-  type: WorkerType;
-  success: boolean;
-  durationMs: number;
-  output?: unknown;
-  error?: string;
-  timestamp: Date;
-}
-
-interface DaemonStatus {
-  running: boolean;
-  pid: number;
-  startedAt?: Date;
-  workers: Map<WorkerType, WorkerState>;
-  config: DaemonConfig;
-}
-
-interface DaemonConfig {
-  autoStart: boolean;
-  logDir: string;
-  stateFile: string;
-  maxConcurrent: number;
-  workerTimeoutMs: number;
-  resourceThresholds: {
-    maxCpuLoad: number;
-    minFreeMemoryPercent: number;
-  };
-  workers: WorkerConfig[];
-}
-
-// Worker configuration with staggered offsets to prevent overlap
-interface WorkerConfigInternal extends WorkerConfig {
-  offsetMs: number; // Stagger start time
-}
-
-// Default worker configurations with improved intervals (P0 fix: map 5min -> 15min)
-const DEFAULT_WORKERS: WorkerConfigInternal[] = [
-  { type: 'map', intervalMs: 15 * 60 * 1000, offsetMs: 0, priority: 'normal', description: 'Codebase mapping', enabled: true },
-  { type: 'audit', intervalMs: 10 * 60 * 1000, offsetMs: 2 * 60 * 1000, priority: 'critical', description: 'Security analysis', enabled: true },
-  { type: 'optimize', intervalMs: 15 * 60 * 1000, offsetMs: 4 * 60 * 1000, priority: 'high', description: 'Performance optimization', enabled: true },
-  { type: 'consolidate', intervalMs: 30 * 60 * 1000, offsetMs: 6 * 60 * 1000, priority: 'low', description: 'Memory consolidation', enabled: true },
-  { type: 'testgaps', intervalMs: 20 * 60 * 1000, offsetMs: 8 * 60 * 1000, priority: 'normal', description: 'Test coverage analysis', enabled: true },
-  { type: 'predict', intervalMs: 10 * 60 * 1000, offsetMs: 0, priority: 'low', description: 'Predictive preloading', enabled: false },
-  { type: 'document', intervalMs: 60 * 60 * 1000, offsetMs: 0, priority: 'low', description: 'Auto-documentation', enabled: false },
-];
-
-// Worker timeout (5 minutes max per worker)
-const DEFAULT_WORKER_TIMEOUT_MS = 5 * 60 * 1000;
-
-/**
- * Worker Daemon - Manages background workers with Node.js
- */
-export class WorkerDaemon extends EventEmitter {
-  private config: DaemonConfig;
-  private workers: Map<WorkerType, WorkerState> = new Map();
-  private timers: Map<WorkerType, NodeJS.Timeout> = new Map();
-  private running = false;
-  private startedAt?: Date;
-  private projectRoot: string;
-  private runningWorkers: Set<WorkerType> = new Set(); // Track concurrent workers
-  private pendingWorkers: WorkerType[] = []; // Queue for deferred workers
-
-  constructor(projectRoot: string, config?: Partial<DaemonConfig>) {
-    super();
-    this.projectRoot = projectRoot;
-
-    const claudeFlowDir = join(projectRoot, '.claude-flow');
-
-    this.config = {
-      autoStart: config?.autoStart ?? false, // P1 fix: Default to false for explicit consent
-      logDir: config?.logDir ?? join(claudeFlowDir, 'logs'),
-      stateFile: config?.stateFile ?? join(claudeFlowDir, 'daemon-state.json'),
-      maxConcurrent: config?.maxConcurrent ?? 2, // P0 fix: Limit concurrent workers
-      workerTimeoutMs: config?.workerTimeoutMs ?? DEFAULT_WORKER_TIMEOUT_MS,
-      resourceThresholds: config?.resourceThresholds ?? {
-        maxCpuLoad: 2.0,
-        minFreeMemoryPercent: 20,
-      },
-      workers: config?.workers ?? DEFAULT_WORKERS,
-    };
-
-    // Setup graceful shutdown handlers
-    this.setupShutdownHandlers();
-
-    // Ensure directories exist
-    if (!existsSync(claudeFlowDir)) {
-      mkdirSync(claudeFlowDir, { recursive: true });
-    }
-    if (!existsSync(this.config.logDir)) {
-      mkdirSync(this.config.logDir, { recursive: true });
-    }
-
-    // Initialize worker states
-    this.initializeWorkerStates();
-  }
-
-  /**
-   * Setup graceful shutdown handlers
-   */
-  private setupShutdownHandlers(): void {
-    const shutdown = async () => {
-      this.log('info', 'Received shutdown signal, stopping daemon...');
-      await this.stop();
-      process.exit(0);
-    };
-
-    process.on('SIGTERM', shutdown);
-    process.on('SIGINT', shutdown);
-    process.on('SIGHUP', shutdown);
-  }
-
-  /**
-   * Check if system resources allow worker execution
-   */
-  private async canRunWorker(): Promise<{ allowed: boolean; reason?: string }> {
-    const os = await import('os');
-    const cpuLoad = os.loadavg()[0];
-    const totalMem = os.totalmem();
-    const freeMem = os.freemem();
-    const freePercent = (freeMem / totalMem) * 100;
-
-    if (cpuLoad > this.config.resourceThresholds.maxCpuLoad) {
-      return { allowed: false, reason: `CPU load too high: ${cpuLoad.toFixed(2)}` };
-    }
-    if (freePercent < this.config.resourceThresholds.minFreeMemoryPercent) {
-      return { allowed: false, reason: `Memory too low: ${freePercent.toFixed(1)}% free` };
-    }
-    return { allowed: true };
-  }
-
-  /**
-   * Process pending workers queue
-   */
-  private async processPendingWorkers(): Promise<void> {
-    while (this.pendingWorkers.length > 0 && this.runningWorkers.size < this.config.maxConcurrent) {
-      const workerType = this.pendingWorkers.shift()!;
-      const workerConfig = this.config.workers.find(w => w.type === workerType);
-      if (workerConfig) {
-        await this.executeWorkerWithConcurrencyControl(workerConfig);
-      }
-    }
-  }
-
-  private initializeWorkerStates(): void {
-    // Try to restore state from file
-    if (existsSync(this.config.stateFile)) {
-      try {
-        const saved = JSON.parse(readFileSync(this.config.stateFile, 'utf-8'));
-        if (saved.workers) {
-          for (const [type, state] of Object.entries(saved.workers)) {
-            const savedState = state as Record<string, unknown>;
-            const lastRunValue = savedState.lastRun;
-            this.workers.set(type as WorkerType, {
-              runCount: (savedState.runCount as number) || 0,
-              successCount: (savedState.successCount as number) || 0,
-              failureCount: (savedState.failureCount as number) || 0,
-              averageDurationMs: (savedState.averageDurationMs as number) || 0,
-              lastRun: lastRunValue ? new Date(lastRunValue as string) : undefined,
-              nextRun: undefined,
-              isRunning: false,
-            });
-          }
-        }
-      } catch {
-        // Ignore parse errors, start fresh
-      }
-    }
-
-    // Initialize any missing workers
-    for (const workerConfig of this.config.workers) {
-      if (!this.workers.has(workerConfig.type)) {
-        this.workers.set(workerConfig.type, {
-          runCount: 0,
-          successCount: 0,
-          failureCount: 0,
-          averageDurationMs: 0,
-          isRunning: false,
-        });
-      }
-    }
-  }
-
-  /**
-   * Start the daemon and all enabled workers
-   */
-  async start(): Promise<void> {
-    if (this.running) {
-      this.emit('warning', 'Daemon already running');
-      return;
-    }
-
-    this.running = true;
-    this.startedAt = new Date();
-    this.emit('started', { pid: process.pid, startedAt: this.startedAt });
-
-    // Schedule all enabled workers
-    for (const workerConfig of this.config.workers) {
-      if (workerConfig.enabled) {
-        this.scheduleWorker(workerConfig);
-      }
-    }
-
-    // Save state
-    this.saveState();
-
-    this.log('info', `Daemon started with ${this.config.workers.filter(w => w.enabled).length} workers`);
-  }
-
-  /**
-   * Stop the daemon and all workers
-   */
-  async stop(): Promise<void> {
-    if (!this.running) {
-      this.emit('warning', 'Daemon not running');
-      return;
-    }
-
-    // Clear all timers
-    for (const [type, timer] of this.timers.entries()) {
-      clearTimeout(timer);
-      this.log('info', `Stopped worker: ${type}`);
-    }
-    this.timers.clear();
-
-    this.running = false;
-    this.saveState();
-    this.emit('stopped', { stoppedAt: new Date() });
-    this.log('info', 'Daemon stopped');
-  }
-
-  /**
-   * Get daemon status
-   */
-  getStatus(): DaemonStatus {
-    return {
-      running: this.running,
-      pid: process.pid,
-      startedAt: this.startedAt,
-      workers: new Map(this.workers),
-      config: this.config,
-    };
-  }
-
-  /**
-   * Schedule a worker to run at intervals with staggered start
-   */
-  private scheduleWorker(workerConfig: WorkerConfig): void {
-    const state = this.workers.get(workerConfig.type)!;
-    const internalConfig = workerConfig as WorkerConfigInternal;
-    const staggerOffset = internalConfig.offsetMs || 0;
-
-    // Calculate initial delay with stagger offset
-    let initialDelay = staggerOffset;
-    if (state.lastRun) {
-      const timeSinceLastRun = Date.now() - state.lastRun.getTime();
-      initialDelay = Math.max(staggerOffset, workerConfig.intervalMs - timeSinceLastRun);
-    }
-
-    state.nextRun = new Date(Date.now() + initialDelay);
-
-    const runAndReschedule = async () => {
-      if (!this.running) return;
-
-      // Use concurrency-controlled execution (P0 fix)
-      await this.executeWorkerWithConcurrencyControl(workerConfig);
-
-      // Reschedule
-      if (this.running) {
-        const timer = setTimeout(runAndReschedule, workerConfig.intervalMs);
-        this.timers.set(workerConfig.type, timer);
-        state.nextRun = new Date(Date.now() + workerConfig.intervalMs);
-      }
-    };
-
-    // Schedule first run with stagger offset
-    const timer = setTimeout(runAndReschedule, initialDelay);
-    this.timers.set(workerConfig.type, timer);
-
-    this.log('info', `Scheduled ${workerConfig.type} (interval: ${workerConfig.intervalMs / 1000}s, first run in ${initialDelay / 1000}s)`);
-  }
-
-  /**
-   * Execute a worker with concurrency control (P0 fix)
-   */
-  private async executeWorkerWithConcurrencyControl(workerConfig: WorkerConfig): Promise<WorkerResult | null> {
-    // Check concurrency limit
-    if (this.runningWorkers.size >= this.config.maxConcurrent) {
-      this.log('info', `Worker ${workerConfig.type} deferred: max concurrent (${this.config.maxConcurrent}) reached`);
-      this.pendingWorkers.push(workerConfig.type);
-      this.emit('worker:deferred', { type: workerConfig.type, reason: 'max_concurrent' });
-      return null;
-    }
-
-    // Check resource availability
-    const resourceCheck = await this.canRunWorker();
-    if (!resourceCheck.allowed) {
-      this.log('info', `Worker ${workerConfig.type} deferred: ${resourceCheck.reason}`);
-      this.pendingWorkers.push(workerConfig.type);
-      this.emit('worker:deferred', { type: workerConfig.type, reason: resourceCheck.reason });
-      return null;
-    }
-
-    return this.executeWorker(workerConfig);
-  }
-
-  /**
-   * Execute a worker with timeout protection
-   */
-  private async executeWorker(workerConfig: WorkerConfig): Promise<WorkerResult> {
-    const state = this.workers.get(workerConfig.type)!;
-    const workerId = `${workerConfig.type}_${Date.now()}`;
-    const startTime = Date.now();
-
-    // Track running worker
-    this.runningWorkers.add(workerConfig.type);
-    state.isRunning = true;
-    this.emit('worker:start', { workerId, type: workerConfig.type });
-    this.log('info', `Starting worker: ${workerConfig.type} (${this.runningWorkers.size}/${this.config.maxConcurrent} concurrent)`);
-
-    try {
-      // Execute worker logic with timeout (P1 fix)
-      const output = await this.runWithTimeout(
-        () => this.runWorkerLogic(workerConfig),
-        this.config.workerTimeoutMs,
-        `Worker ${workerConfig.type} timed out after ${this.config.workerTimeoutMs / 1000}s`
-      );
-      const durationMs = Date.now() - startTime;
-
-      // Update state
-      state.runCount++;
-      state.successCount++;
-      state.lastRun = new Date();
-      state.averageDurationMs = (state.averageDurationMs * (state.runCount - 1) + durationMs) / state.runCount;
-      state.isRunning = false;
-
-      const result: WorkerResult = {
-        workerId,
-        type: workerConfig.type,
-        success: true,
-        durationMs,
-        output,
-        timestamp: new Date(),
-      };
-
-      this.emit('worker:complete', result);
-      this.log('info', `Worker ${workerConfig.type} completed in ${durationMs}ms`);
-      this.saveState();
-
-      return result;
-    } catch (error) {
-      const durationMs = Date.now() - startTime;
-
-      state.runCount++;
-      state.failureCount++;
-      state.lastRun = new Date();
-      state.isRunning = false;
-
-      const result: WorkerResult = {
-        workerId,
-        type: workerConfig.type,
-        success: false,
-        durationMs,
-        error: error instanceof Error ? error.message : String(error),
-        timestamp: new Date(),
-      };
-
-      this.emit('worker:error', result);
-      this.log('error', `Worker ${workerConfig.type} failed: ${result.error}`);
-      this.saveState();
-
-      return result;
-    } finally {
-      // Remove from running set and process queue
-      this.runningWorkers.delete(workerConfig.type);
-      this.processPendingWorkers();
-    }
-  }
-
-  /**
-   * Run a function with timeout (P1 fix)
-   */
-  private async runWithTimeout<T>(
-    fn: () => Promise<T>,
-    timeoutMs: number,
-    timeoutMessage: string
-  ): Promise<T> {
-    return new Promise<T>((resolve, reject) => {
-      const timer = setTimeout(() => {
-        reject(new Error(timeoutMessage));
-      }, timeoutMs);
-
-      fn()
-        .then((result) => {
-          clearTimeout(timer);
-          resolve(result);
-        })
-        .catch((error) => {
-          clearTimeout(timer);
-          reject(error);
-        });
-    });
-  }
-
-  /**
-   * Run the actual worker logic
-   */
-  private async runWorkerLogic(workerConfig: WorkerConfig): Promise<unknown> {
-    switch (workerConfig.type) {
-      case 'map':
-        return this.runMapWorker();
-      case 'audit':
-        return this.runAuditWorker();
-      case 'optimize':
-        return this.runOptimizeWorker();
-      case 'consolidate':
-        return this.runConsolidateWorker();
-      case 'testgaps':
-        return this.runTestGapsWorker();
-      case 'predict':
-        return this.runPredictWorker();
-      case 'document':
-        return this.runDocumentWorker();
-      default:
-        return { status: 'unknown worker type' };
-    }
-  }
-
-  // Worker implementations
-
-  private async runMapWorker(): Promise<unknown> {
-    // Scan project structure and update metrics
-    const metricsFile = join(this.projectRoot, '.claude-flow', 'metrics', 'codebase-map.json');
-    const metricsDir = join(this.projectRoot, '.claude-flow', 'metrics');
-
-    if (!existsSync(metricsDir)) {
-      mkdirSync(metricsDir, { recursive: true });
-    }
-
-    const map = {
-      timestamp: new Date().toISOString(),
-      projectRoot: this.projectRoot,
-      structure: {
-        hasPackageJson: existsSync(join(this.projectRoot, 'package.json')),
-        hasTsConfig: existsSync(join(this.projectRoot, 'tsconfig.json')),
-        hasClaudeConfig: existsSync(join(this.projectRoot, '.claude')),
-        hasClaudeFlow: existsSync(join(this.projectRoot, '.claude-flow')),
-      },
-      scannedAt: Date.now(),
-    };
-
-    writeFileSync(metricsFile, JSON.stringify(map, null, 2));
-    return map;
-  }
-
-  private async runAuditWorker(): Promise<unknown> {
-    // Basic security checks
-    const auditFile = join(this.projectRoot, '.claude-flow', 'metrics', 'security-audit.json');
-    const metricsDir = join(this.projectRoot, '.claude-flow', 'metrics');
-
-    if (!existsSync(metricsDir)) {
-      mkdirSync(metricsDir, { recursive: true });
-    }
-
-    const audit = {
-      timestamp: new Date().toISOString(),
-      checks: {
-        envFilesProtected: !existsSync(join(this.projectRoot, '.env.local')),
-        gitIgnoreExists: existsSync(join(this.projectRoot, '.gitignore')),
-        noHardcodedSecrets: true, // Would need actual scanning
-      },
-      riskLevel: 'low',
-      recommendations: [],
-    };
-
-    writeFileSync(auditFile, JSON.stringify(audit, null, 2));
-    return audit;
-  }
-
-  private async runOptimizeWorker(): Promise<unknown> {
-    // Update performance metrics
-    const optimizeFile = join(this.projectRoot, '.claude-flow', 'metrics', 'performance.json');
-    const metricsDir = join(this.projectRoot, '.claude-flow', 'metrics');
-
-    if (!existsSync(metricsDir)) {
-      mkdirSync(metricsDir, { recursive: true });
-    }
-
-    const perf = {
-      timestamp: new Date().toISOString(),
-      memoryUsage: process.memoryUsage(),
-      uptime: process.uptime(),
-      optimizations: {
-        cacheHitRate: 0.78,
-        avgResponseTime: 45,
-      },
-    };
-
-    writeFileSync(optimizeFile, JSON.stringify(perf, null, 2));
-    return perf;
-  }
-
-  private async runConsolidateWorker(): Promise<unknown> {
-    // Memory consolidation - clean up old patterns
-    const consolidateFile = join(this.projectRoot, '.claude-flow', 'metrics', 'consolidation.json');
-    const metricsDir = join(this.projectRoot, '.claude-flow', 'metrics');
-
-    if (!existsSync(metricsDir)) {
-      mkdirSync(metricsDir, { recursive: true });
-    }
-
-    const result = {
-      timestamp: new Date().toISOString(),
-      patternsConsolidated: 0,
-      memoryCleaned: 0,
-      duplicatesRemoved: 0,
-    };
-
-    writeFileSync(consolidateFile, JSON.stringify(result, null, 2));
-    return result;
-  }
-
-  private async runTestGapsWorker(): Promise<unknown> {
-    // Check for test coverage gaps
-    const testGapsFile = join(this.projectRoot, '.claude-flow', 'metrics', 'test-gaps.json');
-    const metricsDir = join(this.projectRoot, '.claude-flow', 'metrics');
-
-    if (!existsSync(metricsDir)) {
-      mkdirSync(metricsDir, { recursive: true });
-    }
-
-    const result = {
-      timestamp: new Date().toISOString(),
-      hasTestDir: existsSync(join(this.projectRoot, 'tests')) || existsSync(join(this.projectRoot, '__tests__')),
-      estimatedCoverage: 'unknown',
-      gaps: [],
-    };
-
-    writeFileSync(testGapsFile, JSON.stringify(result, null, 2));
-    return result;
-  }
-
-  private async runPredictWorker(): Promise<unknown> {
-    return {
-      timestamp: new Date().toISOString(),
-      predictions: [],
-      preloaded: [],
-    };
-  }
-
-  private async runDocumentWorker(): Promise<unknown> {
-    return {
-      timestamp: new Date().toISOString(),
-      filesDocumented: 0,
-      suggestedDocs: [],
-    };
-  }
-
-  /**
-   * Manually trigger a worker
-   */
-  async triggerWorker(type: WorkerType): Promise<WorkerResult> {
-    const workerConfig = this.config.workers.find(w => w.type === type);
-    if (!workerConfig) {
-      throw new Error(`Unknown worker type: ${type}`);
-    }
-    return this.executeWorker(workerConfig);
-  }
-
-  /**
-   * Enable/disable a worker
-   */
-  setWorkerEnabled(type: WorkerType, enabled: boolean): void {
-    const workerConfig = this.config.workers.find(w => w.type === type);
-    if (workerConfig) {
-      workerConfig.enabled = enabled;
-
-      if (enabled && this.running) {
-        this.scheduleWorker(workerConfig);
-      } else if (!enabled) {
-        const timer = this.timers.get(type);
-        if (timer) {
-          clearTimeout(timer);
-          this.timers.delete(type);
-        }
-      }
-
-      this.saveState();
-    }
-  }
-
-  /**
-   * Save daemon state to file
-   */
-  private saveState(): void {
-    const state = {
-      running: this.running,
-      startedAt: this.startedAt?.toISOString(),
-      workers: Object.fromEntries(
-        Array.from(this.workers.entries()).map(([type, state]) => [
-          type,
-          {
-            ...state,
-            lastRun: state.lastRun?.toISOString(),
-            nextRun: state.nextRun?.toISOString(),
-          }
-        ])
-      ),
-      config: {
-        ...this.config,
-        workers: this.config.workers.map(w => ({ ...w })),
-      },
-      savedAt: new Date().toISOString(),
-    };
-
-    try {
-      writeFileSync(this.config.stateFile, JSON.stringify(state, null, 2));
-    } catch (error) {
-      this.log('error', `Failed to save state: ${error}`);
-    }
-  }
-
-  /**
-   * Log message
-   */
-  private log(level: 'info' | 'warn' | 'error', message: string): void {
-    const timestamp = new Date().toISOString();
-    const logMessage = `[${timestamp}] [${level.toUpperCase()}] ${message}`;
-
-    this.emit('log', { level, message, timestamp });
-
-    // Also write to log file
-    try {
-      const logFile = join(this.config.logDir, 'daemon.log');
-      const fs = require('fs');
-      fs.appendFileSync(logFile, logMessage + '\n');
-    } catch {
-      // Ignore log write errors
-    }
-  }
-}
-
-// Singleton instance for global access
-let daemonInstance: WorkerDaemon | null = null;
-
-/**
- * Get or create daemon instance
- */
-export function getDaemon(projectRoot?: string): WorkerDaemon {
-  if (!daemonInstance && projectRoot) {
-    daemonInstance = new WorkerDaemon(projectRoot);
-  }
-  if (!daemonInstance) {
-    throw new Error('Daemon not initialized. Provide projectRoot on first call.');
-  }
-  return daemonInstance;
-}
-
-/**
- * Start daemon (for use in session-start hook)
- */
-export async function startDaemon(projectRoot: string): Promise<WorkerDaemon> {
-  const daemon = getDaemon(projectRoot);
-  await daemon.start();
-  return daemon;
-}
-
-/**
- * Stop daemon
- */
-export async function stopDaemon(): Promise<void> {
-  if (daemonInstance) {
-    await daemonInstance.stop();
-  }
-}
-
-export default WorkerDaemon;