claude-mycelium 2.1.0 → 2.2.0

package/src/task/agent-coordinator.ts

@@ -0,0 +1,220 @@
+/**
+ * Task Agent Coordinator
+ *
+ * Coordinates multi-agent execution of task steps using the mycelium
+ * process spawning system (Phase 3) for true RALPH-style parallelization.
+ *
+ * This replaces the simple Promise.all() approach with real agent processes
+ * that use file locks, inhibitors, and the full executeAgent() cycle.
+ */
+
+import { fork, ChildProcess } from 'child_process';
+import { Task, TaskStep, Mode } from '../types/index.js';
+import { logDebug, logError, logInfo } from '../utils/logger.js';
+import { randomUUID } from 'crypto';
+import * as path from 'path';
+import { fileURLToPath } from 'url';
+import { dirname } from 'path';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+
+export interface StepAssignment {
+  stepOrder: number;
+  targetFile: string;
+  mode: Mode;
+  description: string;
+  taskId: string;
+}
+
+export interface AgentResult {
+  stepOrder: number;
+  success: boolean;
+  traceId?: string;
+  error?: string;
+  exitCode: number | null;
+}
+
+/**
+ * Spawn an agent for a specific task step
+ * Each agent runs as an independent process via child_process.fork()
+ */
+export function spawnAgentForStep(
+  task: Task,
+  step: TaskStep
+): Promise<AgentResult> {
+  return new Promise((resolve) => {
+    const agentId = `agent-${randomUUID().substring(0, 8)}`;
+
+    logDebug('Spawning agent for task step', {
+      agentId,
+      taskId: task.id,
+      stepOrder: step.order,
+      targetFile: step.target_file,
+      mode: step.mode,
+    });
+
+    // Path to the compiled task worker file
+    const workerPath = path.join(__dirname, '../agent/task-worker.js');
+
+    // Spawn agent process with task context
+    const child = fork(workerPath, [], {
+      env: {
+        ...process.env,
+        AGENT_ID: agentId,
+        TASK_ID: task.id,
+        STEP_ORDER: String(step.order),
+        TARGET_FILE: step.target_file,
+        MODE: step.mode,
+        STEP_DESCRIPTION: step.description,
+      },
+      stdio: ['pipe', 'pipe', 'pipe', 'ipc'],
+    });
+
+    let result: AgentResult = {
+      stepOrder: step.order,
+      success: false,
+      exitCode: null,
+    };
+
+    // Set timeout (5 minutes)
+    const timeout = setTimeout(() => {
+      logError('Agent timeout', new Error('Agent timed out'), {
+        agentId,
+        stepOrder: step.order,
+      });
+      child.kill('SIGTERM');
+      result.error = 'Agent timed out after 5 minutes';
+      result.exitCode = -1;
+      resolve(result);
+    }, 300_000);
+
+    // Handle IPC messages from agent
+    child.on('message', (message: any) => {
+      if (message.type === 'result') {
+        logDebug('Agent result received', {
+          agentId,
+          stepOrder: step.order,
+          success: message.success,
+        });
+
+        result = {
+          stepOrder: step.order,
+          success: message.success,
+          traceId: message.traceId,
+          error: message.error,
+          exitCode: 0,
+        };
+      } else if (message.type === 'progress') {
+        logInfo('Agent progress', {
+          agentId,
+          stepOrder: step.order,
+          message: message.message,
+        });
+      }
+    });
+
+    // Handle agent exit
+    child.on('exit', (code, signal) => {
+      clearTimeout(timeout);
+
+      result.exitCode = code;
+
+      logInfo('Agent exited', {
+        agentId,
+        stepOrder: step.order,
+        code,
+        signal,
+        success: result.success,
+      });
+
+      // If we didn't receive a result message, mark as failure
+      if (!result.success && !result.error) {
+        result.error = `Agent exited with code ${code}`;
+      }
+
+      resolve(result);
+    });
+
+    // Handle errors
+    child.on('error', (error) => {
+      clearTimeout(timeout);
+      logError('Agent error', error, {
+        agentId,
+        stepOrder: step.order,
+      });
+
+      result.error = error.message;
+      result.exitCode = -1;
+      resolve(result);
+    });
+
+    // Forward stderr for debugging
+    if (child.stderr) {
+      child.stderr.on('data', (data) => {
+        logDebug('Agent stderr', {
+          agentId,
+          stepOrder: step.order,
+          output: data.toString(),
+        });
+      });
+    }
+  });
+}
+
+/**
+ * Execute a wave of task steps in parallel using agent processes
+ * This is the true mycelium multi-agent coordination
+ */
+export async function executeWaveWithAgents(
+  task: Task,
+  wave: TaskStep[]
+): Promise<AgentResult[]> {
+  logInfo('Executing wave with agent processes', {
+    taskId: task.id,
+    waveSize: wave.length,
+    steps: wave.map(s => s.order),
+  });
+
+  // Spawn all agents in parallel
+  const agentPromises = wave.map(step => spawnAgentForStep(task, step));
+
+  // Wait for all agents to complete
+  const results = await Promise.all(agentPromises);
+
+  logInfo('Wave execution complete', {
+    taskId: task.id,
+    total: results.length,
+    successful: results.filter(r => r.success).length,
+    failed: results.filter(r => !r.success).length,
+  });
+
+  return results;
+}
+
+/**
+ * Get parallelizable steps grouped by dependency waves
+ * Same logic as before, but now we spawn agents for each wave
+ */
+export function getParallelizableSteps(steps: TaskStep[]): TaskStep[][] {
+  const waves: TaskStep[][] = [];
+  const completed = new Set<number>();
+
+  while (completed.size < steps.length) {
+    // Find steps whose dependencies are all completed
+    const ready = steps.filter(
+      step =>
+        !completed.has(step.order) &&
+        step.depends_on.every(dep => completed.has(dep))
+    );
+
+    if (ready.length === 0) {
+      throw new Error('Circular dependency detected in task steps');
+    }
+
+    waves.push(ready);
+    ready.forEach(step => completed.add(step.order));
+  }
+
+  return waves;
+}

package/src/task/executor.ts

@@ -2,23 +2,33 @@
  * Task Execution Implementation
  * Per second-spec §12.2: Task Step Execution
  *
- * Executes task steps in dependency order, tracking progress
- * and handling create mode appropriately.
+ * Executes task steps using the mycelium multi-agent system.
+ * Each step spawns an independent agent process that uses file locks,
+ * inhibitors, and the full RALPH-style executeAgent() cycle.
+ *
+ * This is TRUE multi-agent coordination as per ADR-004.
  */
 
-import { Task, TaskStep, Mode, ChangeSet, CostRecord } from '../types/index.js';
-import { callLLM, calculateCost } from '../llm/anthropic-client.js';
-import { logDebug, logError } from '../utils/logger.js';
-import { fileExists, readFile, writeFile } from '../utils/file-utils.js';
-import { ciProvider } from '../utils/ci-provider.js';
-import { createTraceEvent, recordTraceEvent } from '../trace/index.js';
-import { calculateGradient } from '../core/gradient.js';
+import { Task, TaskStep } from '../types/index.js';
+import { logDebug, logError, logInfo } from '../utils/logger.js';
+import {
+  executeWaveWithAgents,
+  getParallelizableSteps,
+  type AgentResult,
+} from './agent-coordinator.js';
 
 /**
  * Execute a complete task by running all steps in dependency order
+ *
+ * This spawns independent agent processes for each step, coordinated
+ * via file locks. Multiple agents can work in parallel on different
+ * files, implementing true RALPH-style multi-agent coordination.
  */
 export async function executeTask(task: Task): Promise<Task> {
-  logDebug('Executing task', { taskId: task.id, steps: task.plan?.steps.length });
+  logInfo('Executing task with multi-agent coordination', {
+    taskId: task.id,
+    steps: task.plan?.steps.length,
+  });
 
   if (!task.plan) {
     throw new Error('Cannot execute task without a plan');
@@ -26,83 +36,82 @@ export async function executeTask(task: Task): Promise<Task> {
 
   const waves = getParallelizableSteps(task.plan.steps);
 
+  logInfo('Task execution plan', {
+    taskId: task.id,
+    totalSteps: task.plan.steps.length,
+    waves: waves.length,
+    waveSizes: waves.map(w => w.length),
+  });
+
   for (let waveIndex = 0; waveIndex < waves.length; waveIndex++) {
     const wave = waves[waveIndex];
-    logDebug('Executing wave', { wave: waveIndex + 1, steps: wave.length });
+    logInfo('Executing wave', {
+      taskId: task.id,
+      wave: waveIndex + 1,
+      totalWaves: waves.length,
+      stepsInWave: wave.length,
+    });
 
-    // Execute all steps in this wave (they have no dependencies on each other)
-    const results = await Promise.all(
-      wave.map(step => executeStep(task, step))
-    );
+    // Execute all steps in this wave using agent processes
+    // Each agent runs in its own process with file locks
+    const results: AgentResult[] = await executeWaveWithAgents(task, wave);
 
     // Check for failures
     const failed = results.find(r => !r.success);
     if (failed) {
       task.status = 'failed';
-      task.error = failed.error;
-      logError('Task step failed', new Error(failed.error), { taskId: task.id });
+      task.error = failed.error || 'Agent execution failed';
+      logError('Task wave failed', new Error(task.error), {
+        taskId: task.id,
+        wave: waveIndex + 1,
+        failedStep: failed.stepOrder,
+      });
       return task;
     }
 
     // Update completed steps
     for (let i = 0; i < wave.length; i++) {
-      wave[i].completed = true;
-      wave[i].trace_id = results[i].trace_id;
-      task.steps_completed++;
-    }
+      const result = results.find(r => r.stepOrder === wave[i].order);
+      if (result && result.success) {
+        wave[i].completed = true;
+        wave[i].trace_id = result.traceId;
+        task.steps_completed++;
+
+        // Track file changes
+        if (wave[i].mode === 'create') {
+          task.files_created.push(wave[i].target_file);
+        } else {
+          if (!task.files_modified.includes(wave[i].target_file)) {
+            task.files_modified.push(wave[i].target_file);
+          }
+        }
 
-    // Track file changes
-    for (let i = 0; i < wave.length; i++) {
-      if (wave[i].mode === 'create') {
-        task.files_created.push(wave[i].target_file);
-      } else {
-        if (!task.files_modified.includes(wave[i].target_file)) {
-          task.files_modified.push(wave[i].target_file);
+        if (result.traceId) {
+          task.traces.push(result.traceId);
         }
       }
-      const traceId = results[i].trace_id;
-      if (traceId !== undefined) {
-        task.traces.push(traceId);
-      }
     }
+
+    logInfo('Wave complete', {
+      taskId: task.id,
+      wave: waveIndex + 1,
+      successCount: results.filter(r => r.success).length,
+      failureCount: results.filter(r => !r.success).length,
+    });
   }
 
   task.status = 'completed';
-  logDebug('Task execution completed', { taskId: task.id });
-  return task;
-}
-
-/**
- * Execute a single task step
- */
-export async function executeStep(
-  task: Task,
-  step: TaskStep
-): Promise<{ success: boolean; trace_id?: string; error?: string }> {
-  logDebug('Executing step', {
+  logInfo('Task execution completed', {
     taskId: task.id,
-    stepOrder: step.order,
-    mode: step.mode,
-    file: step.target_file,
+    stepsCompleted: task.steps_completed,
+    filesCreated: task.files_created.length,
+    filesModified: task.files_modified.length,
   });
 
-  try {
-    if (step.mode === 'create') {
-      return await executeCreateStep(task, step);
-    } else {
-      return await executeModifyStep(task, step);
-    }
-  } catch (error) {
-    const errorMsg = error instanceof Error ? error.message : 'Unknown error';
-    const errorObj = error instanceof Error ? error : new Error(String(error));
-    logError('Step execution failed', errorObj, {
-      taskId: task.id,
-      stepOrder: step.order,
-    });
-    return { success: false, error: errorMsg };
-  }
+  return task;
 }
 
+
 /**
  * Track progress of task execution
  */
@@ -124,10 +133,6 @@ export function trackProgress(task: Task): {
   };
 }
 
-/**
- * Execute a create mode step (new file)
- */
-async function executeCreateStep(
   task: Task,
   step: TaskStep
 ): Promise<{ success: boolean; trace_id?: string; error?: string }> {

package/tests/trace/trace-event.test.ts

@@ -53,11 +53,21 @@ describe('Trace Event System', () => {
         gradient_before: 0.8,
         gradient_after: 0.6,
         gradient_delta: 0.2,
-        changes_made: ['fix null check', 'add error handling'],
-        tokens_used: 1000,
-        cost_usd: 0.05,
-        duration_ms: 30000,
-        success: true,
+        metabolic_cost: 0.05,
+        efficiency: 4.0,
+        ci_passed: true,
+        changes: {
+          additions: 5,
+          deletions: 2,
+          files_touched: ['src/test.ts'],
+        },
+        notes: ['fix null check', 'add error handling'],
+        cost: {
+          tokens_in: 500,
+          tokens_out: 500,
+          model: 'claude-sonnet-4',
+          estimated_usd: 0.05,
+        },
       };
 
       await recordTrace(trace);
@@ -82,11 +92,21 @@ describe('Trace Event System', () => {
         gradient_before: 0.9,
         gradient_after: 0.7,
         gradient_delta: undefined as any,
-        changes_made: ['extract function'],
-        tokens_used: 800,
-        cost_usd: 0.03,
-        duration_ms: 25000,
-        success: true,
+        metabolic_cost: 0.03,
+        efficiency: 6.67,
+        ci_passed: true,
+        changes: {
+          additions: 3,
+          deletions: 1,
+          files_touched: ['src/auth.ts'],
+        },
+        notes: ['extract function'],
+        cost: {
+          tokens_in: 400,
+          tokens_out: 400,
+          model: 'claude-sonnet-4',
+          estimated_usd: 0.03,
+        },
       };
 
       await recordTrace(trace);
@@ -102,11 +122,22 @@ describe('Trace Event System', () => {
         mode: 'debt_payer',
         gradient_before: 0.5,
         gradient_after: 0.4,
-        changes_made: ['fix lint'],
-        tokens_used: 500,
-        cost_usd: 0.02,
-        duration_ms: 15000,
-        success: true,
+        gradient_delta: 0.1,
+        metabolic_cost: 0.02,
+        efficiency: 5.0,
+        ci_passed: true,
+        changes: {
+          additions: 2,
+          deletions: 1,
+          files_touched: ['src/test1.ts'],
+        },
+        notes: ['fix lint'],
+        cost: {
+          tokens_in: 250,
+          tokens_out: 250,
+          model: 'claude-sonnet-4',
+          estimated_usd: 0.02,
+        },
       });
 
       const trace2: TraceEvent = createTraceEvent({
@@ -115,11 +146,22 @@ describe('Trace Event System', () => {
         mode: 'stabilizer',
         gradient_before: 0.6,
         gradient_after: 0.5,
-        changes_made: ['add tests'],
-        tokens_used: 600,
-        cost_usd: 0.025,
-        duration_ms: 20000,
-        success: true,
+        gradient_delta: 0.1,
+        metabolic_cost: 0.025,
+        efficiency: 4.0,
+        ci_passed: true,
+        changes: {
+          additions: 5,
+          deletions: 0,
+          files_touched: ['src/test2.ts'],
+        },
+        notes: ['add tests'],
+        cost: {
+          tokens_in: 300,
+          tokens_out: 300,
+          model: 'claude-sonnet-4',
+          estimated_usd: 0.025,
+        },
       });
 
       await recordTrace(trace1);