@litmers/cursorflow-orchestrator 0.1.6 → 0.1.9

package/src/cli/run.ts

@@ -99,6 +99,7 @@ async function run(args: string[]): Promise<void> {
       executor: options.executor || config.executor,
       pollInterval: config.pollInterval * 1000,
       runDir: path.join(logsDir, 'runs', `run-${Date.now()}`),
+      maxConcurrentLanes: config.maxConcurrentLanes,
     });
   } catch (error: any) {
     // Re-throw to be handled by the main entry point

package/src/core/orchestrator.ts

@@ -10,11 +10,12 @@ import { spawn, ChildProcess } from 'child_process';
 
 import * as logger from '../utils/logger';
 import { loadState } from '../utils/state';
-import { LaneState } from '../utils/types';
+import { LaneState, RunnerConfig } from '../utils/types';
 
 export interface LaneInfo {
   name: string;
   path: string;
+  dependsOn: string[];
 }
 
 export interface SpawnLaneResult {
@@ -76,7 +77,7 @@ export function waitChild(proc: ChildProcess): Promise<number> {
 }
 
 /**
- * List lane task files in directory
+ * List lane task files in directory and load their configs for dependencies
  */
 export function listLaneFiles(tasksDir: string): LaneInfo[] {
   if (!fs.existsSync(tasksDir)) {
@@ -87,10 +88,24 @@ export function listLaneFiles(tasksDir: string): LaneInfo[] {
   return files
     .filter(f => f.endsWith('.json'))
     .sort()
-    .map(f => ({
-      name: path.basename(f, '.json'),
-      path: path.join(tasksDir, f),
-    }));
+    .map(f => {
+      const filePath = path.join(tasksDir, f);
+      const name = path.basename(f, '.json');
+      let dependsOn: string[] = [];
+      
+      try {
+        const config = JSON.parse(fs.readFileSync(filePath, 'utf8')) as RunnerConfig;
+        dependsOn = config.dependsOn || [];
+      } catch (e) {
+        logger.warn(`Failed to parse config for lane ${name}: ${e}`);
+      }
+      
+      return {
+        name,
+        path: filePath,
+        dependsOn,
+      };
+    });
 }
 
 /**
@@ -105,7 +120,8 @@ export function printLaneStatus(lanes: LaneInfo[], laneRunDirs: Record<string, s
     const state = loadState<LaneState>(statePath);
     
     if (!state) {
-      return { lane: lane.name, status: '(no state)', task: '-' };
+      const isWaiting = lane.dependsOn.length > 0;
+      return { lane: lane.name, status: isWaiting ? 'waiting' : 'pending', task: '-' };
     }
     
     const idx = (state.currentTaskIndex || 0) + 1;
@@ -123,12 +139,13 @@ export function printLaneStatus(lanes: LaneInfo[], laneRunDirs: Record<string, s
 }
 
 /**
- * Run orchestration
+ * Run orchestration with dependency management
  */
 export async function orchestrate(tasksDir: string, options: { 
   runDir?: string; 
   executor?: string; 
   pollInterval?: number; 
+  maxConcurrentLanes?: number;
 } = {}): Promise<{ lanes: LaneInfo[]; exitCodes: Record<string, number>; runRoot: string }> {
   const lanes = listLaneFiles(tasksDir);
   
@@ -142,6 +159,7 @@ export async function orchestrate(tasksDir: string, options: {
   const laneRunDirs: Record<string, string> = {};
   for (const lane of lanes) {
     laneRunDirs[lane.name] = path.join(runRoot, 'lanes', lane.name);
+    fs.mkdirSync(laneRunDirs[lane.name], { recursive: true });
   }
   
   logger.section('🧭 Starting Orchestration');
@@ -149,31 +167,88 @@ export async function orchestrate(tasksDir: string, options: {
   logger.info(`Run directory: ${runRoot}`);
   logger.info(`Lanes: ${lanes.length}`);
   
-  // Spawn all lanes
-  const running: { lane: string; child: ChildProcess; logPath: string }[] = [];
-  
-  for (const lane of lanes) {
-    const { child, logPath } = spawnLane({
-      laneName: lane.name,
-      tasksFile: lane.path,
-      laneRunDir: laneRunDirs[lane.name]!,
-      executor: options.executor || 'cursor-agent',
-    });
-    
-    running.push({ lane: lane.name, child, logPath });
-    logger.info(`Lane started: ${lane.name}`);
-  }
+  const maxConcurrent = options.maxConcurrentLanes || 10;
+  const running: Map<string, { child: ChildProcess; logPath: string }> = new Map();
+  const exitCodes: Record<string, number> = {};
+  const completedLanes = new Set<string>();
+  const failedLanes = new Set<string>();
   
   // Monitor lanes
   const monitorInterval = setInterval(() => {
     printLaneStatus(lanes, laneRunDirs);
   }, options.pollInterval || 60000);
   
-  // Wait for all lanes
-  const exitCodes: Record<string, number> = {};
-  
-  for (const r of running) {
-    exitCodes[r.lane] = await waitChild(r.child);
+  while (completedLanes.size + failedLanes.size < lanes.length) {
+    // 1. Identify lanes ready to start
+    const readyToStart = lanes.filter(lane => {
+      // Not already running or completed
+      if (running.has(lane.name) || completedLanes.has(lane.name) || failedLanes.has(lane.name)) {
+        return false;
+      }
+      
+      // Check dependencies
+      for (const dep of lane.dependsOn) {
+        if (failedLanes.has(dep)) {
+          // If a dependency failed, this lane fails too
+          logger.error(`Lane ${lane.name} failed because dependency ${dep} failed`);
+          failedLanes.add(lane.name);
+          exitCodes[lane.name] = 1;
+          return false;
+        }
+        if (!completedLanes.has(dep)) {
+          return false;
+        }
+      }
+      return true;
+    });
+    
+    // 2. Spawn ready lanes up to maxConcurrent
+    for (const lane of readyToStart) {
+      if (running.size >= maxConcurrent) break;
+      
+      logger.info(`Lane started: ${lane.name}`);
+      const spawnResult = spawnLane({
+        laneName: lane.name,
+        tasksFile: lane.path,
+        laneRunDir: laneRunDirs[lane.name]!,
+        executor: options.executor || 'cursor-agent',
+      });
+      
+      running.set(lane.name, spawnResult);
+    }
+    
+    // 3. Wait for any running lane to finish
+    if (running.size > 0) {
+      // We need to wait for at least one to finish
+      const promises = Array.from(running.entries()).map(async ([name, { child }]) => {
+        const code = await waitChild(child);
+        return { name, code };
+      });
+      
+      const finished = await Promise.race(promises);
+      
+      running.delete(finished.name);
+      exitCodes[finished.name] = finished.code;
+      
+      if (finished.code === 0 || finished.code === 2) {
+        completedLanes.add(finished.name);
+      } else {
+        failedLanes.add(finished.name);
+      }
+      
+      printLaneStatus(lanes, laneRunDirs);
+    } else {
+      // Nothing running and nothing ready (but not all finished)
+      // This could happen if there's a circular dependency or some logic error
+      if (readyToStart.length === 0 && completedLanes.size + failedLanes.size < lanes.length) {
+        const remaining = lanes.filter(l => !completedLanes.has(l.name) && !failedLanes.has(l.name));
+        logger.error(`Deadlock detected! Remaining lanes cannot start: ${remaining.map(l => l.name).join(', ')}`);
+        for (const l of remaining) {
+          failedLanes.add(l.name);
+          exitCodes[l.name] = 1;
+        }
+      }
+    }
   }
   
   clearInterval(monitorInterval);

package/src/core/runner.ts

@@ -6,7 +6,7 @@
 
 import * as fs from 'fs';
 import * as path from 'path';
-import { execSync, spawnSync } from 'child_process';
+import { execSync, spawn, spawnSync } from 'child_process';
 
 import * as git from '../utils/git';
 import * as logger from '../utils/logger';
@@ -107,12 +107,80 @@ function parseJsonFromStdout(stdout: string): any {
   return null;
 }
 
-export function cursorAgentSend({ workspaceDir, chatId, prompt, model }: { 
+/** Default timeout: 5 minutes */
+const DEFAULT_TIMEOUT_MS = 300000;
+
+/** Heartbeat interval: 30 seconds */
+const HEARTBEAT_INTERVAL_MS = 30000;
+
+/**
+ * Validate task configuration
+ * @throws Error if validation fails
+ */
+export function validateTaskConfig(config: RunnerConfig): void {
+  if (!config.tasks || !Array.isArray(config.tasks)) {
+    throw new Error('Invalid config: "tasks" must be an array');
+  }
+  
+  if (config.tasks.length === 0) {
+    throw new Error('Invalid config: "tasks" array is empty');
+  }
+  
+  for (let i = 0; i < config.tasks.length; i++) {
+    const task = config.tasks[i];
+    const taskNum = i + 1;
+    
+    if (!task) {
+      throw new Error(`Invalid config: Task ${taskNum} is null or undefined`);
+    }
+    
+    if (!task.name || typeof task.name !== 'string') {
+      throw new Error(
+        `Invalid config: Task ${taskNum} missing required "name" field.\n` +
+        `  Found: ${JSON.stringify(task, null, 2).substring(0, 200)}...\n` +
+        `  Expected: { "name": "task-name", "prompt": "..." }`
+      );
+    }
+    
+    if (!task.prompt || typeof task.prompt !== 'string') {
+      throw new Error(
+        `Invalid config: Task "${task.name}" (${taskNum}) missing required "prompt" field`
+      );
+    }
+    
+    // Validate task name format (no spaces, special chars that could break branch names)
+    if (!/^[a-zA-Z0-9_-]+$/.test(task.name)) {
+      throw new Error(
+        `Invalid config: Task name "${task.name}" contains invalid characters.\n` +
+        `  Task names must only contain: letters, numbers, underscore (_), hyphen (-)`
+      );
+    }
+  }
+  
+  // Validate timeout if provided
+  if (config.timeout !== undefined) {
+    if (typeof config.timeout !== 'number' || config.timeout <= 0) {
+      throw new Error(
+        `Invalid config: "timeout" must be a positive number (milliseconds).\n` +
+        `  Found: ${config.timeout}`
+      );
+    }
+  }
+}
+
+/**
+ * Execute cursor-agent command with streaming and better error handling
+ */
+export async function cursorAgentSend({ workspaceDir, chatId, prompt, model, signalDir, timeout, enableIntervention }: { 
   workspaceDir: string; 
   chatId: string; 
   prompt: string; 
   model?: string; 
-}): AgentSendResult {
+  signalDir?: string;
+  timeout?: number;
+  /** Enable stdin piping for intervention feature (may cause buffering issues on some systems) */
+  enableIntervention?: boolean;
+}): Promise<AgentSendResult> {
   const args = [
     '--print',
     '--output-format', 'json',
@@ -122,76 +190,166 @@ export function cursorAgentSend({ workspaceDir, chatId, prompt, model }: {
     prompt,
   ];
   
-  logger.info('Executing cursor-agent...');
+  const timeoutMs = timeout || DEFAULT_TIMEOUT_MS;
+  logger.info(`Executing cursor-agent... (timeout: ${Math.round(timeoutMs / 1000)}s)`);
   
-  const res = spawnSync('cursor-agent', args, {
-    encoding: 'utf8',
-    stdio: 'pipe',
-    timeout: 300000, // 5 minute timeout for LLM response
-  });
+  // Determine stdio mode based on intervention setting
+  // When intervention is enabled, we pipe stdin for message injection
+  // When disabled (default), we ignore stdin to avoid buffering issues
+  const stdinMode = enableIntervention ? 'pipe' : 'ignore';
   
-  // Check for timeout
-  if (res.error) {
-    if ((res.error as any).code === 'ETIMEDOUT') {
-      return {
-        ok: false,
-        exitCode: -1,
-        error: 'cursor-agent timed out after 5 minutes. The LLM request may be taking too long or there may be network issues.',
-      };
-    }
-    
-    return {
-      ok: false,
-      exitCode: -1,
-      error: `cursor-agent error: ${res.error.message}`,
-    };
+  if (enableIntervention) {
+    logger.info('Intervention mode enabled (stdin piped)');
   }
   
-  const json = parseJsonFromStdout(res.stdout);
-  
-  if (res.status !== 0 || !json || json.type !== 'result') {
-    let errorMsg = res.stderr?.trim() || res.stdout?.trim() || `exit=${res.status}`;
+  return new Promise((resolve) => {
+    // Build environment, preserving user's NODE_OPTIONS but disabling problematic flags
+    const childEnv = { ...process.env };
     
-    // Check for authentication errors
-    if (errorMsg.includes('not authenticated') || 
-        errorMsg.includes('login') || 
-        errorMsg.includes('auth')) {
-      errorMsg = 'Authentication error. Please:\n' +
-        '  1. Open Cursor IDE\n' +
-        '  2. Sign in to your account\n' +
-        '  3. Verify AI features are working\n' +
-        '  4. Try again\n\n' +
-        `Details: ${errorMsg}`;
+    // Only filter out specific problematic NODE_OPTIONS, don't clear entirely
+    if (childEnv.NODE_OPTIONS) {
+      // Remove flags that might interfere with cursor-agent
+      const filtered = childEnv.NODE_OPTIONS
+        .split(' ')
+        .filter(opt => !opt.includes('--inspect') && !opt.includes('--debug'))
+        .join(' ');
+      childEnv.NODE_OPTIONS = filtered;
     }
     
-    // Check for rate limit errors
-    if (errorMsg.includes('rate limit') || errorMsg.includes('quota')) {
-      errorMsg = 'API rate limit or quota exceeded. Please:\n' +
-        '  1. Check your Cursor subscription\n' +
-        '  2. Wait a few minutes and try again\n\n' +
-        `Details: ${errorMsg}`;
-    }
+    // Disable Python buffering in case cursor-agent uses Python
+    childEnv.PYTHONUNBUFFERED = '1';
     
-    // Check for model errors
-    if (errorMsg.includes('model')) {
-      errorMsg = `Model error (requested: ${model || 'default'}). ` +
-        'Please check if the model is available in your Cursor subscription.\n\n' +
-        `Details: ${errorMsg}`;
+    const child = spawn('cursor-agent', args, {
+      stdio: [stdinMode, 'pipe', 'pipe'],
+      env: childEnv,
+    });
+
+    // Save PID to state if possible
+    if (child.pid && signalDir) {
+      try {
+        const statePath = path.join(signalDir, 'state.json');
+        if (fs.existsSync(statePath)) {
+          const state = JSON.parse(fs.readFileSync(statePath, 'utf8'));
+          state.pid = child.pid;
+          fs.writeFileSync(statePath, JSON.stringify(state, null, 2));
+        }
+      } catch (e) {
+        // Best effort
+      }
     }
-    
-    return {
-      ok: false,
-      exitCode: res.status ?? -1,
-      error: errorMsg,
-    };
-  }
-  
-  return {
-    ok: !json.is_error,
-    exitCode: res.status ?? 0,
-    sessionId: json.session_id || chatId,
-    resultText: json.result || '',
-  };
+
+    let fullStdout = '';
+    let fullStderr = '';
+
+    // Heartbeat logging to show progress
+    let lastHeartbeat = Date.now();
+    let bytesReceived = 0;
+    const heartbeatInterval = setInterval(() => {
+      const elapsed = Math.round((Date.now() - lastHeartbeat) / 1000);
+      const totalElapsed = Math.round((Date.now() - startTime) / 1000);
+      logger.info(`⏱ Heartbeat: ${totalElapsed}s elapsed, ${bytesReceived} bytes received`);
+    }, HEARTBEAT_INTERVAL_MS);
+    const startTime = Date.now();
+
+    // Watch for "intervention.txt" signal file if any
+    const interventionPath = signalDir ? path.join(signalDir, 'intervention.txt') : null;
+    let interventionWatcher: fs.FSWatcher | null = null;
+
+    if (interventionPath && fs.existsSync(path.dirname(interventionPath))) {
+      interventionWatcher = fs.watch(path.dirname(interventionPath), (event, filename) => {
+        if (filename === 'intervention.txt' && fs.existsSync(interventionPath)) {
+          try {
+            const message = fs.readFileSync(interventionPath, 'utf8').trim();
+            if (message) {
+              if (enableIntervention && child.stdin) {
+                logger.info(`Injecting intervention: ${message}`);
+                child.stdin.write(message + '\n');
+              } else {
+                logger.warn(`Intervention requested but stdin not available: ${message}`);
+                logger.warn('To enable intervention, set enableIntervention: true in config');
+              }
+              fs.unlinkSync(interventionPath); // Clear it
+            }
+          } catch (e) {
+            logger.warn('Failed to read intervention file');
+          }
+        }
+      });
+    }
+
+    if (child.stdout) {
+      child.stdout.on('data', (data) => {
+        const str = data.toString();
+        fullStdout += str;
+        bytesReceived += data.length;
+        // Also pipe to our own stdout so it goes to terminal.log
+        process.stdout.write(data);
+      });
+    }
+
+    if (child.stderr) {
+      child.stderr.on('data', (data) => {
+        fullStderr += data.toString();
+        // Pipe to our own stderr so it goes to terminal.log
+        process.stderr.write(data);
+      });
+    }
+
+    const timeoutHandle = setTimeout(() => {
+      clearInterval(heartbeatInterval);
+      child.kill();
+      const timeoutSec = Math.round(timeoutMs / 1000);
+      resolve({
+        ok: false,
+        exitCode: -1,
+        error: `cursor-agent timed out after ${timeoutSec} seconds. The LLM request may be taking too long or there may be network issues.`,
+      });
+    }, timeoutMs);
+
+    child.on('close', (code) => {
+      clearTimeout(timeoutHandle);
+      clearInterval(heartbeatInterval);
+      if (interventionWatcher) interventionWatcher.close();
+      
+      const json = parseJsonFromStdout(fullStdout);
+      
+      if (code !== 0 || !json || json.type !== 'result') {
+        let errorMsg = fullStderr.trim() || fullStdout.trim() || `exit=${code}`;
+        
+        // Check for common errors
+        if (errorMsg.includes('not authenticated') || errorMsg.includes('login') || errorMsg.includes('auth')) {
+          errorMsg = 'Authentication error. Please sign in to Cursor IDE.';
+        } else if (errorMsg.includes('rate limit') || errorMsg.includes('quota')) {
+          errorMsg = 'API rate limit or quota exceeded.';
+        } else if (errorMsg.includes('model')) {
+          errorMsg = `Model error (requested: ${model || 'default'}). Check your subscription.`;
+        }
+        
+        resolve({
+          ok: false,
+          exitCode: code ?? -1,
+          error: errorMsg,
+        });
+      } else {
+        resolve({
+          ok: !json.is_error,
+          exitCode: code ?? 0,
+          sessionId: json.session_id || chatId,
+          resultText: json.result || '',
+        });
+      }
+    });
+
+    child.on('error', (err) => {
+      clearTimeout(timeoutHandle);
+      clearInterval(heartbeatInterval);
+      resolve({
+        ok: false,
+        exitCode: -1,
+        error: `Failed to start cursor-agent: ${err.message}`,
+      });
+    });
+  });
 }
 
 /**
@@ -326,11 +484,14 @@ export async function runTask({
   }));
   
   logger.info('Sending prompt to agent...');
-  const r1 = cursorAgentSend({
+  const r1 = await cursorAgentSend({
     workspaceDir: worktreeDir,
     chatId,
     prompt: prompt1,
     model,
+    signalDir: runDir,
+    timeout: config.timeout,
+    enableIntervention: config.enableIntervention,
   });
   
   appendLog(convoPath, createConversationEntry('assistant', r1.resultText || r1.error || 'No response', {
@@ -374,6 +535,17 @@ export async function runTask({
 export async function runTasks(tasksFile: string, config: RunnerConfig, runDir: string, options: { startIndex?: number } = {}): Promise<TaskExecutionResult[]> {
   const startIndex = options.startIndex || 0;
   
+  // Validate configuration before starting
+  logger.info('Validating task configuration...');
+  try {
+    validateTaskConfig(config);
+    logger.success('✓ Configuration valid');
+  } catch (validationError: any) {
+    logger.error('❌ Configuration validation failed');
+    logger.error(`   ${validationError.message}`);
+    throw validationError;
+  }
+  
   // Ensure cursor-agent is installed
   ensureCursorAgent();
   
@@ -426,7 +598,7 @@ export async function runTasks(tasksFile: string, config: RunnerConfig, runDir:
   
   // Create worktree only if starting fresh
   if (startIndex === 0 || !fs.existsSync(worktreeDir)) {
-    git.createWorktree(worktreeDir, pipelineBranch, {
+    git.createWorktree(worktreeDir, pipelineBranch, { 
       baseBranch: config.baseBranch || 'main',
       cwd: repoRoot,
     });
@@ -450,15 +622,61 @@ export async function runTasks(tasksFile: string, config: RunnerConfig, runDir:
       error: null,
       dependencyRequest: null,
       tasksFile, // Store tasks file for resume
+      dependsOn: config.dependsOn || [],
     };
   } else {
     state.status = 'running';
     state.error = null;
     state.dependencyRequest = null;
+    state.dependsOn = config.dependsOn || [];
   }
   
   saveState(statePath, state);
   
+  // Merge dependencies if any
+  if (startIndex === 0 && config.dependsOn && config.dependsOn.length > 0) {
+    logger.section('🔗 Merging Dependencies');
+    
+    // The runDir for the lane is passed in. Dependencies are in ../<depName> relative to this runDir
+    const lanesRoot = path.dirname(runDir);
+    
+    for (const depName of config.dependsOn) {
+      const depRunDir = path.join(lanesRoot, depName);
+      const depStatePath = path.join(depRunDir, 'state.json');
+      
+      if (!fs.existsSync(depStatePath)) {
+        logger.warn(`Dependency state not found for ${depName} at ${depStatePath}`);
+        continue;
+      }
+      
+      try {
+        const depState = JSON.parse(fs.readFileSync(depStatePath, 'utf8')) as LaneState;
+        if (depState.status !== 'completed') {
+          logger.warn(`Dependency ${depName} is in status ${depState.status}, merge might be incomplete`);
+        }
+        
+        if (depState.pipelineBranch) {
+          logger.info(`Merging dependency branch: ${depState.pipelineBranch} (${depName})`);
+          
+          // Fetch first to ensure we have the branch
+          git.runGit(['fetch', 'origin', depState.pipelineBranch], { cwd: worktreeDir, silent: true });
+          
+          // Merge
+          git.merge(depState.pipelineBranch, { 
+            cwd: worktreeDir, 
+            noFf: true, 
+            message: `chore: merge dependency ${depName} (${depState.pipelineBranch})` 
+          });
+        }
+      } catch (e) {
+        logger.error(`Failed to merge dependency ${depName}: ${e}`);
+      }
+    }
+    
+    // Push the merged state
+    git.push(pipelineBranch, { cwd: worktreeDir });
+  }
+  
   // Run tasks
   const results: TaskExecutionResult[] = [];
   

package/src/utils/types.ts

@@ -40,6 +40,7 @@ export interface Task {
 
 export interface RunnerConfig {
   tasks: Task[];
+  dependsOn?: string[];
   pipelineBranch?: string;
   branchPrefix?: string;
   worktreeRoot?: string;
@@ -49,6 +50,14 @@ export interface RunnerConfig {
   reviewModel?: string;
   maxReviewIterations?: number;
   acceptanceCriteria?: string[];
+  /** Task execution timeout in milliseconds. Default: 300000 (5 minutes) */
+  timeout?: number;
+  /** 
+   * Enable intervention feature (stdin piping for message injection).
+   * Warning: May cause stdout buffering issues on some systems.
+   * Default: false
+   */
+  enableIntervention?: boolean;
 }
 
 export interface DependencyRequestPlan {
@@ -109,6 +118,8 @@ export interface LaneState {
   dependencyRequest: DependencyRequestPlan | null;
   updatedAt?: number;
   tasksFile?: string; // Original tasks file path
+  dependsOn?: string[];
+  pid?: number;
 }
 
 export interface ConversationEntry {